From 9dcac35c2d2651008071ab18a7437ea6b79adcb4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 May 2013 10:35:41 -0700 Subject: [PATCH 1/9] Use make instead of new Upstream-commit: c4ebf870c8a9cef909e66fb517e607896ea8a518 Component: engine --- components/engine/image.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/components/engine/image.go b/components/engine/image.go index bf86e2e7f7..5778931655 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -267,13 +267,13 @@ func (img *Image) Checksum() (string, error) { } checksumDictPth := path.Join(root, "..", "..", "checksums") - checksums := new(map[string]string) + checksums := make(map[string]string) if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { if err := json.Unmarshal(checksumDict, checksums); err != nil { return "", err } - if checksum, ok := (*checksums)[img.Id]; ok { + if checksum, ok := checksums[img.Id]; ok { return checksum, nil } } @@ -304,10 +304,8 @@ func (img *Image) Checksum() (string, error) { } hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) - if *checksums == nil { - *checksums = map[string]string{} - } - (*checksums)[img.Id] = hash + checksums[img.Id] = hash + checksumJson, err := json.Marshal(checksums) if err != nil { return hash, err From c6f97b4c7e3e6c7f8f6de591e1b79dee734649c0 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 May 2013 10:42:29 -0700 Subject: [PATCH 2/9] Make the checksum async within commit Upstream-commit: 8ff1765674c93a169fe29483503709a008f30618 Component: engine --- components/engine/graph.go | 25 +++++++++++++++++++------ components/engine/image.go | 27 +++++++++++++++++++++------ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/components/engine/graph.go b/components/engine/graph.go index 9a8fd1b3cb..4b504525e7 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -9,14 +9,18 @@ import ( "path" "path/filepath" "strings" + "sync" "time" ) // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *TruncIndex - httpClient *http.Client + Root string + idIndex *TruncIndex + httpClient *http.Client + checksumLock map[string]*sync.Mutex + lockSumFile *sync.Mutex + lockSumMap *sync.Mutex } // NewGraph instantiates a new graph at the given root path in the filesystem. @@ -31,8 +35,11 @@ func NewGraph(root string) (*Graph, error) { return nil, err } graph := &Graph{ - Root: abspath, - idIndex: NewTruncIndex(), + Root: abspath, + idIndex: NewTruncIndex(), + checksumLock: make(map[string]*sync.Mutex), + lockSumFile: &sync.Mutex{}, + lockSumMap: &sync.Mutex{}, } if err := graph.restore(); err != nil { return nil, err @@ -82,6 +89,11 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) } img.graph = graph + graph.lockSumMap.Lock() + defer graph.lockSumMap.Unlock() + if _, exists := graph.checksumLock[img.Id]; !exists { + graph.checksumLock[img.Id] = &sync.Mutex{} + } return img, nil } @@ -103,7 +115,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut if err := graph.Register(layerData, img); err != nil { return nil, err } - img.Checksum() + go img.Checksum() return img, nil } @@ -131,6 +143,7 @@ func (graph *Graph) Register(layerData Archive, img *Image) error { } img.graph = graph graph.idIndex.Add(img.Id) + graph.checksumLock[img.Id] = &sync.Mutex{} return nil } diff --git a/components/engine/image.go b/components/engine/image.go index 5778931655..3e8ac55d3d 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -35,8 +35,9 @@ func LoadImage(root string) (*Image, error) { if err != nil { return nil, err } - var img Image - if err := json.Unmarshal(jsonData, &img); err != nil { + img := &Image{} + + if err := json.Unmarshal(jsonData, img); err != nil { return nil, err } if err := ValidateId(img.Id); err != nil { @@ -52,8 +53,7 @@ func LoadImage(root string) (*Image, error) { } else if !stat.IsDir() { return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root)) } - - return &img, nil + return img, nil } func StoreImage(img *Image, layerData Archive, root string) error { @@ -261,6 +261,9 @@ func (img *Image) layer() (string, error) { } func (img *Image) Checksum() (string, error) { + img.graph.checksumLock[img.Id].Lock() + defer img.graph.checksumLock[img.Id].Unlock() + root, err := img.root() if err != nil { return "", err @@ -270,7 +273,7 @@ func (img *Image) Checksum() (string, error) { checksums := make(map[string]string) if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { - if err := json.Unmarshal(checksumDict, checksums); err != nil { + if err := json.Unmarshal(checksumDict, &checksums); err != nil { return "", err } if checksum, ok := checksums[img.Id]; ok { @@ -299,18 +302,30 @@ func (img *Image) Checksum() (string, error) { if _, err := h.Write([]byte("\n")); err != nil { return "", err } + + fmt.Printf("precopy %s: %s\n", img.ShortId(), time.Now().String()) + if _, err := io.Copy(h, layerData); err != nil { return "", err } + fmt.Printf("postcopy presum %s: %s\n", img.ShortId(), time.Now().String()) hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) checksums[img.Id] = hash + fmt.Printf("postsum %s: %s\n", img.ShortId(), time.Now().String()) + // Reload the json file to make sure not to overwrite faster sums + img.graph.lockSumFile.Lock() + defer img.graph.lockSumFile.Unlock() + if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { + if err := json.Unmarshal(checksumDict, &checksums); err != nil { + return "", err + } + } checksumJson, err := json.Marshal(checksums) if err != nil { return hash, err } - if err := ioutil.WriteFile(checksumDictPth, checksumJson, 0600); err != nil { return hash, err } From 6a329f5dc01f761e989b83721cb35f99e8306250 Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Wed, 8 May 2013 12:13:31 -0700 Subject: [PATCH 3/9] Added a line to css to make sure th woul align left. Upstream-commit: 070b1cd541cb8a60755a3c5244af00bca6ccaf51 Component: engine --- components/engine/docs/theme/docker/static/css/main.css | 4 ++++ components/engine/docs/theme/docker/static/css/main.less | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/components/engine/docs/theme/docker/static/css/main.css b/components/engine/docs/theme/docker/static/css/main.css index 3948acc55d..0629efeb48 100755 --- a/components/engine/docs/theme/docker/static/css/main.css +++ b/components/engine/docs/theme/docker/static/css/main.css @@ -330,3 +330,7 @@ section.header { @media (max-width: 480px) { } +/* Misc fixes */ +table th { + text-align: left; +} diff --git a/components/engine/docs/theme/docker/static/css/main.less b/components/engine/docs/theme/docker/static/css/main.less index 69f53f9e1b..50c8fe6b4b 100644 --- a/components/engine/docs/theme/docker/static/css/main.less +++ b/components/engine/docs/theme/docker/static/css/main.less @@ -449,4 +449,9 @@ section.header { @media (max-width: 480px) { +} + +/* Misc fixes */ +table th { + text-align: left; } \ No newline at end of file From 0b78bccf2888136d4089d97ee12336b687d2b0a4 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 8 May 2013 14:50:26 -0600 Subject: [PATCH 4/9] Swap "go get" for "go get -d", especially to compile on go1.1rc; fixes #561 Upstream-commit: c4ad6b077d87bb501ed651bfc87c3e332c7d38d7 Component: engine --- components/engine/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index bae7d64909..9527d3f750 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -39,7 +39,7 @@ $(DOCKER_BIN): $(DOCKER_DIR) $(DOCKER_DIR): @mkdir -p $(dir $@) @if [ -h $@ ]; then rm -f $@; fi; ln -sf $(CURDIR)/ $@ - @(cd $(DOCKER_MAIN); go get $(GO_OPTIONS)) + @(cd $(DOCKER_MAIN); go get -d $(GO_OPTIONS)) whichrelease: echo $(RELEASE_VERSION) From cc0ef6eaae840cd5bd6cf902cd6e69a92e1c4f0f Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 May 2013 14:19:38 -0700 Subject: [PATCH 5/9] Better error output upon push failure Upstream-commit: 6cafed45afb74d07b13e8c00798e559da47822e7 Component: engine --- components/engine/registry.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/components/engine/registry.go b/components/engine/registry.go index cc8ee68153..07268313a3 100644 --- a/components/engine/registry.go +++ b/components/engine/registry.go @@ -427,9 +427,15 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) } - res3.Body.Close() + defer res3.Body.Close() + if res3.StatusCode != 200 { - return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) + errBody, err := ioutil.ReadAll(res3.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ + " trying to parse response body: %v", res.StatusCode, err) + } + return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody) } return nil } @@ -612,8 +618,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re } func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]string, error) { - var result []map[string]string - checksums := map[string]string{} + checksums := make(map[string]string) for _, id := range repo { img, err := graph.Get(id) if err != nil { @@ -634,7 +639,7 @@ func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]s } } i := 0 - result = make([]map[string]string, len(checksums)) + result := make([]map[string]string, len(checksums)) for id, sum := range checksums { result[i] = map[string]string{ "id": id, From acc707fbcea52b0283303625b50b06e45d11bbed Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 May 2013 15:26:44 -0700 Subject: [PATCH 6/9] + Runtime: Add go version to debug infos Upstream-commit: d6c24092eb0e24f7aaa921110c62e99e81ae1f88 Component: engine --- components/engine/commands.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index dba8cd8572..1f2b13e6f0 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -11,6 +11,7 @@ import ( "log" "net/http" "net/url" + "os" "path/filepath" "runtime" "strconv" @@ -284,9 +285,10 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string VERSION, imgcount) - if !rcli.DEBUG_FLAG { + if os.Getenv("DEBUG") == "" { return nil } + fmt.Fprintf(stdout, "Go version: %s\n", runtime.Version()) fmt.Fprintln(stdout, "debug mode enabled") fmt.Fprintf(stdout, "fds: %d\ngoroutines: %d\n", getTotalUsedFds(), runtime.NumGoroutine()) return nil From dd5e4add4f0a1e9792222de1e8e070f9c73ce5eb Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 May 2013 16:22:12 -0700 Subject: [PATCH 7/9] Store the checksums when pulling a repository Upstream-commit: 0e23b4e10ec7f7c49c5715cd115f859319adf476 Component: engine --- components/engine/graph.go | 2 +- components/engine/image.go | 4 --- components/engine/registry.go | 52 +++++++++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/components/engine/graph.go b/components/engine/graph.go index 4b504525e7..84b8c9a1e9 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -31,7 +31,7 @@ func NewGraph(root string) (*Graph, error) { return nil, err } // Create the root directory if it doesn't exists - if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { return nil, err } graph := &Graph{ diff --git a/components/engine/image.go b/components/engine/image.go index 3e8ac55d3d..d208ed6ce8 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -303,16 +303,12 @@ func (img *Image) Checksum() (string, error) { return "", err } - fmt.Printf("precopy %s: %s\n", img.ShortId(), time.Now().String()) - if _, err := io.Copy(h, layerData); err != nil { return "", err } - fmt.Printf("postcopy presum %s: %s\n", img.ShortId(), time.Now().String()) hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) checksums[img.Id] = hash - fmt.Printf("postsum %s: %s\n", img.ShortId(), time.Now().String()) // Reload the json file to make sure not to overwrite faster sums img.graph.lockSumFile.Lock() diff --git a/components/engine/registry.go b/components/engine/registry.go index 07268313a3..bf6556f6d5 100644 --- a/components/engine/registry.go +++ b/components/engine/registry.go @@ -194,18 +194,16 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit return nil, fmt.Errorf("Repository not found") } - result := new(map[string]string) + result := make(map[string]string) rawJson, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err } - if err = json.Unmarshal(rawJson, result); err != nil { + if err = json.Unmarshal(rawJson, &result); err != nil { return nil, err } - - return *result, nil - + return result, nil } return nil, fmt.Errorf("Could not reach any registry endpoint") } @@ -308,6 +306,50 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return fmt.Errorf("Index response didn't contain any endpoints") } + checksumsJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + + // Reload the json file to make sure not to overwrite faster sums + err = func() error { + localChecksums := make(map[string]string) + remoteChecksums := []struct { + Id string `json: "id"` + Checksum string `json: "checksum"` + }{} + checksumDictPth := path.Join(graph.Root, "..", "checksums") + + if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil { + return err + } + + graph.lockSumFile.Lock() + defer graph.lockSumFile.Unlock() + + if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { + if err := json.Unmarshal(checksumDict, &localChecksums); err != nil { + return err + } + } + + for _, elem := range remoteChecksums { + localChecksums[elem.Id] = elem.Checksum + } + + checksumsJson, err = json.Marshal(localChecksums) + if err != nil { + return err + } + if err := ioutil.WriteFile(checksumDictPth, checksumsJson, 0600); err != nil { + return err + } + return nil + }() + if err != nil { + return err + } + var tagsList map[string]string if askedTag == "" { tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) From f0b0638c967df88491b81834576fe5b6caa18d1a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 May 2013 16:40:48 -0700 Subject: [PATCH 8/9] Display the go version inf CmdInfo in non-debug mode Upstream-commit: 5098c4fc006c15788142a669a0d169b03de7be8b Component: engine --- components/engine/commands.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 1f2b13e6f0..34de4f60bd 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -284,11 +284,12 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string len(srv.runtime.List()), VERSION, imgcount) + fmt.Fprintf(stdout, "Go version: %s\n", runtime.Version()) if os.Getenv("DEBUG") == "" { return nil } - fmt.Fprintf(stdout, "Go version: %s\n", runtime.Version()) + fmt.Fprintln(stdout, "debug mode enabled") fmt.Fprintf(stdout, "fds: %d\ngoroutines: %d\n", getTotalUsedFds(), runtime.NumGoroutine()) return nil From 28ba5b0027e3bd2ec51550b35c0b960e95ab932b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 8 May 2013 16:52:47 -0700 Subject: [PATCH 9/9] Bumped version to 0.3.1 Upstream-commit: 06767fb99d997fe939febb086e8fa12414419053 Component: engine --- components/engine/CHANGELOG.md | 25 ++++++++++++++++++ components/engine/commands.go | 2 +- components/engine/packaging/ubuntu/changelog | 27 ++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index 5036d87c15..d28e843468 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## 0.3.1 (2013-05-08) + + Builder: Implement the autorun capability within docker builder + + Builder: Add caching to docker builder + + Builder: Add support for docker builder with native API as top level command + + Runtime: Add go version to debug infos + + Builder: Implement ENV within docker builder + + Registry: Add docker search top level command in order to search a repository + + Images: output graph of images to dot (graphviz) + + Documentation: new introduction and high-level overview + + Documentation: Add the documentation for docker builder + + Website: new high-level overview + - Makefile: Swap "go get" for "go get -d", especially to compile on go1.1rc + - Images: fix ByParent function + - Builder: Check the command existance prior create and add Unit tests for the case + - Registry: Fix pull for official images with specific tag + - Registry: Fix issue when login in with a different user and trying to push + - Documentation: CSS fix for docker documentation to make REST API docs look better. + - Documentation: Fixed CouchDB example page header mistake + - Documentation: fixed README formatting + * Registry: Improve checksum - async calculation + * Runtime: kernel version - don't show the dash if flavor is empty + * Documentation: updated www.docker.io website. + * Builder: use any whitespaces instead of tabs + * Packaging: packaging ubuntu; issue #510: Use goland-stable PPA package to build docker + ## 0.3.0 (2013-05-06) + Registry: Implement the new registry + Documentation: new example: sharing data between 2 couchdb databases diff --git a/components/engine/commands.go b/components/engine/commands.go index 34de4f60bd..9fb47819b5 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -21,7 +21,7 @@ import ( "unicode" ) -const VERSION = "0.3.0" +const VERSION = "0.3.1" var ( GIT_COMMIT string diff --git a/components/engine/packaging/ubuntu/changelog b/components/engine/packaging/ubuntu/changelog index b3e68558ff..2a5575d8c3 100644 --- a/components/engine/packaging/ubuntu/changelog +++ b/components/engine/packaging/ubuntu/changelog @@ -1,3 +1,30 @@ +lxc-docker (0.3.1-1) precise; urgency=low + - Builder: Implement the autorun capability within docker builder + - Builder: Add caching to docker builder + - Builder: Add support for docker builder with native API as top level command + - Runtime: Add go version to debug infos + - Builder: Implement ENV within docker builder + - Registry: Add docker search top level command in order to search a repository + - Images: output graph of images to dot (graphviz) + - Documentation: new introduction and high-level overview + - Documentation: Add the documentation for docker builder + - Website: new high-level overview + - Makefile: Swap "go get" for "go get -d", especially to compile on go1.1rc + - Images: fix ByParent function + - Builder: Check the command existance prior create and add Unit tests for the case + - Registry: Fix pull for official images with specific tag + - Registry: Fix issue when login in with a different user and trying to push + - Documentation: CSS fix for docker documentation to make REST API docs look better. + - Documentation: Fixed CouchDB example page header mistake + - Documentation: fixed README formatting + - Registry: Improve checksum - async calculation + - Runtime: kernel version - don't show the dash if flavor is empty + - Documentation: updated www.docker.io website. + - Builder: use any whitespaces instead of tabs + - Packaging: packaging ubuntu; issue #510: Use goland-stable PPA package to build docker + + -- dotCloud Fri, 8 May 2013 00:00:00 -0700 + lxc-docker (0.3.0-1) precise; urgency=low - Registry: Implement the new registry - Documentation: new example: sharing data between 2 couchdb databases