diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index 5036d87c15..d28e843468 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## 0.3.1 (2013-05-08) + + Builder: Implement the autorun capability within docker builder + + Builder: Add caching to docker builder + + Builder: Add support for docker builder with native API as top level command + + Runtime: Add go version to debug infos + + Builder: Implement ENV within docker builder + + Registry: Add docker search top level command in order to search a repository + + Images: output graph of images to dot (graphviz) + + Documentation: new introduction and high-level overview + + Documentation: Add the documentation for docker builder + + Website: new high-level overview + - Makefile: Swap "go get" for "go get -d", especially to compile on go1.1rc + - Images: fix ByParent function + - Builder: Check the command existance prior create and add Unit tests for the case + - Registry: Fix pull for official images with specific tag + - Registry: Fix issue when login in with a different user and trying to push + - Documentation: CSS fix for docker documentation to make REST API docs look better. + - Documentation: Fixed CouchDB example page header mistake + - Documentation: fixed README formatting + * Registry: Improve checksum - async calculation + * Runtime: kernel version - don't show the dash if flavor is empty + * Documentation: updated www.docker.io website. + * Builder: use any whitespaces instead of tabs + * Packaging: packaging ubuntu; issue #510: Use goland-stable PPA package to build docker + ## 0.3.0 (2013-05-06) + Registry: Implement the new registry + Documentation: new example: sharing data between 2 couchdb databases diff --git a/components/engine/Makefile b/components/engine/Makefile index bae7d64909..9527d3f750 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -39,7 +39,7 @@ $(DOCKER_BIN): $(DOCKER_DIR) $(DOCKER_DIR): @mkdir -p $(dir $@) @if [ -h $@ ]; then rm -f $@; fi; ln -sf $(CURDIR)/ $@ - @(cd $(DOCKER_MAIN); go get $(GO_OPTIONS)) + @(cd $(DOCKER_MAIN); go get -d $(GO_OPTIONS)) whichrelease: echo $(RELEASE_VERSION) diff --git a/components/engine/api_params.go b/components/engine/api_params.go index c727b2b292..c7c15585f9 100644 --- a/components/engine/api_params.go +++ b/components/engine/api_params.go @@ -18,6 +18,7 @@ type ApiInfo struct { Version string Images int Debug bool + GoVersion string NFd int `json:",omitempty"` NGoroutines int `json:",omitempty"` } diff --git a/components/engine/commands.go b/components/engine/commands.go index a6aa1c46cc..b2db3b32d1 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -22,7 +22,7 @@ import ( "unicode" ) -const VERSION = "0.3.0" +const VERSION = "0.3.1" var ( GIT_COMMIT string @@ -362,7 +362,7 @@ func CmdInfo(args ...string) error { if err != nil { return err } - fmt.Printf("containers: %d\nversion: %s\nimages: %d\n", out.Containers, out.Version, out.Images) + fmt.Printf("containers: %d\nversion: %s\nimages: %d\nGo version: %s\n", out.Containers, out.Version, out.Images, out.GoVersion) if out.Debug { fmt.Println("debug mode enabled") fmt.Printf("fds: %d\ngoroutines: %d\n", out.NFd, out.NGoroutines) diff --git a/components/engine/docs/sources/remote-api/api.rst b/components/engine/docs/sources/remote-api/api.rst index aa8c90bf99..124e576162 100644 --- a/components/engine/docs/sources/remote-api/api.rst +++ b/components/engine/docs/sources/remote-api/api.rst @@ -874,6 +874,7 @@ Display system-wide information "Containers":11, "Version":"0.2.2", "Images":16, + "GoVersion":"go1.0.3", "Debug":false } diff --git a/components/engine/docs/theme/docker/static/css/main.css b/components/engine/docs/theme/docker/static/css/main.css index 3948acc55d..0629efeb48 100755 --- a/components/engine/docs/theme/docker/static/css/main.css +++ b/components/engine/docs/theme/docker/static/css/main.css @@ -330,3 +330,7 @@ section.header { @media (max-width: 480px) { } +/* Misc fixes */ +table th { + text-align: left; +} diff --git a/components/engine/docs/theme/docker/static/css/main.less b/components/engine/docs/theme/docker/static/css/main.less index 69f53f9e1b..50c8fe6b4b 100644 --- a/components/engine/docs/theme/docker/static/css/main.less +++ b/components/engine/docs/theme/docker/static/css/main.less @@ -449,4 +449,9 @@ section.header { @media (max-width: 480px) { +} + +/* Misc fixes */ +table th { + text-align: left; } \ No newline at end of file diff --git a/components/engine/graph.go b/components/engine/graph.go index 9a8fd1b3cb..84b8c9a1e9 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -9,14 +9,18 @@ import ( "path" "path/filepath" "strings" + "sync" "time" ) // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *TruncIndex - httpClient *http.Client + Root string + idIndex *TruncIndex + httpClient *http.Client + checksumLock map[string]*sync.Mutex + lockSumFile *sync.Mutex + lockSumMap *sync.Mutex } // NewGraph instantiates a new graph at the given root path in the filesystem. @@ -27,12 +31,15 @@ func NewGraph(root string) (*Graph, error) { return nil, err } // Create the root directory if it doesn't exists - if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { return nil, err } graph := &Graph{ - Root: abspath, - idIndex: NewTruncIndex(), + Root: abspath, + idIndex: NewTruncIndex(), + checksumLock: make(map[string]*sync.Mutex), + lockSumFile: &sync.Mutex{}, + lockSumMap: &sync.Mutex{}, } if err := graph.restore(); err != nil { return nil, err @@ -82,6 +89,11 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) } img.graph = graph + graph.lockSumMap.Lock() + defer graph.lockSumMap.Unlock() + if _, exists := graph.checksumLock[img.Id]; !exists { + graph.checksumLock[img.Id] = &sync.Mutex{} + } return img, nil } @@ -103,7 +115,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut if err := graph.Register(layerData, img); err != nil { return nil, err } - img.Checksum() + go img.Checksum() return img, nil } @@ -131,6 +143,7 @@ func (graph *Graph) Register(layerData Archive, img *Image) error { } img.graph = graph graph.idIndex.Add(img.Id) + graph.checksumLock[img.Id] = &sync.Mutex{} return nil } diff --git a/components/engine/image.go b/components/engine/image.go index bf86e2e7f7..d208ed6ce8 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -35,8 +35,9 @@ func LoadImage(root string) (*Image, error) { if err != nil { return nil, err } - var img Image - if err := json.Unmarshal(jsonData, &img); err != nil { + img := &Image{} + + if err := json.Unmarshal(jsonData, img); err != nil { return nil, err } if err := ValidateId(img.Id); err != nil { @@ -52,8 +53,7 @@ func LoadImage(root string) (*Image, error) { } else if !stat.IsDir() { return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root)) } - - return &img, nil + return img, nil } func StoreImage(img *Image, layerData Archive, root string) error { @@ -261,19 +261,22 @@ func (img *Image) layer() (string, error) { } func (img *Image) Checksum() (string, error) { + img.graph.checksumLock[img.Id].Lock() + defer img.graph.checksumLock[img.Id].Unlock() + root, err := img.root() if err != nil { return "", err } checksumDictPth := path.Join(root, "..", "..", "checksums") - checksums := new(map[string]string) + checksums := make(map[string]string) if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { - if err := json.Unmarshal(checksumDict, checksums); err != nil { + if err := json.Unmarshal(checksumDict, &checksums); err != nil { return "", err } - if checksum, ok := (*checksums)[img.Id]; ok { + if checksum, ok := checksums[img.Id]; ok { return checksum, nil } } @@ -299,20 +302,26 @@ func (img *Image) Checksum() (string, error) { if _, err := h.Write([]byte("\n")); err != nil { return "", err } + if _, err := io.Copy(h, layerData); err != nil { return "", err } hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) - if *checksums == nil { - *checksums = map[string]string{} + checksums[img.Id] = hash + + // Reload the json file to make sure not to overwrite faster sums + img.graph.lockSumFile.Lock() + defer img.graph.lockSumFile.Unlock() + if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { + if err := json.Unmarshal(checksumDict, &checksums); err != nil { + return "", err + } } - (*checksums)[img.Id] = hash checksumJson, err := json.Marshal(checksums) if err != nil { return hash, err } - if err := ioutil.WriteFile(checksumDictPth, checksumJson, 0600); err != nil { return hash, err } diff --git a/components/engine/packaging/ubuntu/changelog b/components/engine/packaging/ubuntu/changelog index b3e68558ff..2a5575d8c3 100644 --- a/components/engine/packaging/ubuntu/changelog +++ b/components/engine/packaging/ubuntu/changelog @@ -1,3 +1,30 @@ +lxc-docker (0.3.1-1) precise; urgency=low + - Builder: Implement the autorun capability within docker builder + - Builder: Add caching to docker builder + - Builder: Add support for docker builder with native API as top level command + - Runtime: Add go version to debug infos + - Builder: Implement ENV within docker builder + - Registry: Add docker search top level command in order to search a repository + - Images: output graph of images to dot (graphviz) + - Documentation: new introduction and high-level overview + - Documentation: Add the documentation for docker builder + - Website: new high-level overview + - Makefile: Swap "go get" for "go get -d", especially to compile on go1.1rc + - Images: fix ByParent function + - Builder: Check the command existance prior create and add Unit tests for the case + - Registry: Fix pull for official images with specific tag + - Registry: Fix issue when login in with a different user and trying to push + - Documentation: CSS fix for docker documentation to make REST API docs look better. + - Documentation: Fixed CouchDB example page header mistake + - Documentation: fixed README formatting + - Registry: Improve checksum - async calculation + - Runtime: kernel version - don't show the dash if flavor is empty + - Documentation: updated www.docker.io website. + - Builder: use any whitespaces instead of tabs + - Packaging: packaging ubuntu; issue #510: Use goland-stable PPA package to build docker + + -- dotCloud Fri, 8 May 2013 00:00:00 -0700 + lxc-docker (0.3.0-1) precise; urgency=low - Registry: Implement the new registry - Documentation: new example: sharing data between 2 couchdb databases diff --git a/components/engine/registry.go b/components/engine/registry.go index cc8ee68153..bf6556f6d5 100644 --- a/components/engine/registry.go +++ b/components/engine/registry.go @@ -194,18 +194,16 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit return nil, fmt.Errorf("Repository not found") } - result := new(map[string]string) + result := make(map[string]string) rawJson, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err } - if err = json.Unmarshal(rawJson, result); err != nil { + if err = json.Unmarshal(rawJson, &result); err != nil { return nil, err } - - return *result, nil - + return result, nil } return nil, fmt.Errorf("Could not reach any registry endpoint") } @@ -308,6 +306,50 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return fmt.Errorf("Index response didn't contain any endpoints") } + checksumsJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + + // Reload the json file to make sure not to overwrite faster sums + err = func() error { + localChecksums := make(map[string]string) + remoteChecksums := []struct { + Id string `json: "id"` + Checksum string `json: "checksum"` + }{} + checksumDictPth := path.Join(graph.Root, "..", "checksums") + + if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil { + return err + } + + graph.lockSumFile.Lock() + defer graph.lockSumFile.Unlock() + + if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { + if err := json.Unmarshal(checksumDict, &localChecksums); err != nil { + return err + } + } + + for _, elem := range remoteChecksums { + localChecksums[elem.Id] = elem.Checksum + } + + checksumsJson, err = json.Marshal(localChecksums) + if err != nil { + return err + } + if err := ioutil.WriteFile(checksumDictPth, checksumsJson, 0600); err != nil { + return err + } + return nil + }() + if err != nil { + return err + } + var tagsList map[string]string if askedTag == "" { tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) @@ -427,9 +469,15 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) } - res3.Body.Close() + defer res3.Body.Close() + if res3.StatusCode != 200 { - return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) + errBody, err := ioutil.ReadAll(res3.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ + " trying to parse response body: %v", res.StatusCode, err) + } + return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody) } return nil } @@ -612,8 +660,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re } func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]string, error) { - var result []map[string]string - checksums := map[string]string{} + checksums := make(map[string]string) for _, id := range repo { img, err := graph.Get(id) if err != nil { @@ -634,7 +681,7 @@ func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]s } } i := 0 - result = make([]map[string]string, len(checksums)) + result := make([]map[string]string, len(checksums)) for id, sum := range checksums { result[i] = map[string]string{ "id": id, diff --git a/components/engine/server.go b/components/engine/server.go index ffc66d622a..bf312b3474 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -201,6 +201,7 @@ func (srv *Server) DockerInfo() ApiInfo { out.Containers = len(srv.runtime.List()) out.Version = VERSION out.Images = imgcount + out.GoVersion = runtime.Version() if os.Getenv("DEBUG") != "" { out.Debug = true out.NFd = getTotalUsedFds()