From c7d63374580bd505b56c3473ed0bc41a333e2b29 Mon Sep 17 00:00:00 2001 From: unclejack Date: Fri, 27 Jun 2014 15:04:28 +0300 Subject: [PATCH 1/8] resumablerequestreader: allow initial response Make it possible to inspect an initial response and pass it to ResumableRequestReader. This makes it possible to inspect an initial response and passing it to ResumableRequestReader to avoid making an extra request. Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: f033ce3ee9ee055612acacde537687e41865e14f Component: engine --- components/engine/utils/resumablerequestreader.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/engine/utils/resumablerequestreader.go b/components/engine/utils/resumablerequestreader.go index e01f4e6d71..bed202ec0c 100644 --- a/components/engine/utils/resumablerequestreader.go +++ b/components/engine/utils/resumablerequestreader.go @@ -24,6 +24,10 @@ func ResumableRequestReader(c *http.Client, r *http.Request, maxfail uint32, tot return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize} } +func ResumableRequestReaderWithInitialResponse(c *http.Client, r *http.Request, maxfail uint32, totalsize int64, initialResponse *http.Response) io.ReadCloser { + return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize, currentResponse: initialResponse} +} + func (r *resumableRequestReader) Read(p []byte) (n int, err error) { if r.client == nil || r.request == nil { return 0, fmt.Errorf("client and request can't be nil\n") From 43007a03a714aa20e4e2033d0a85e138d3e86460 Mon Sep 17 00:00:00 2001 From: unclejack Date: Fri, 27 Jun 2014 15:10:30 +0300 Subject: [PATCH 2/8] get layer: remove HEAD req & pass down response Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: c47ebe7a351bc639028cd48aed9d2fa2310a2a65 Component: engine --- components/engine/registry/registry.go | 56 +++++++++++--------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 748636dca8..57795f1c34 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -390,52 +390,42 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { var ( - retries = 5 - headRes *http.Response - client *http.Client - hasResume bool = false - imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) + retries = 5 + client *http.Client + res *http.Response + imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) - headReq, err := r.reqFactory.NewRequest("HEAD", imageURL, nil) - if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - - setTokenAuth(headReq, token) - for i := 1; i <= retries; i++ { - headRes, client, err = r.doRequest(headReq) - if err != nil && i == retries { - return nil, fmt.Errorf("Eror while making head request: %s\n", err) - } else if err != nil { - time.Sleep(time.Duration(i) * 5 * time.Second) - continue - } - break - } - - if headRes.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - hasResume = true - } req, err := r.reqFactory.NewRequest("GET", imageURL, nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } setTokenAuth(req, token) - if hasResume { - utils.Debugf("server supports resume") - return utils.ResumableRequestReader(client, req, 5, imgSize), nil - } - utils.Debugf("server doesn't support resume") - res, _, err := r.doRequest(req) - if err != nil { - return nil, err + for i := 1; i <= retries; i++ { + res, client, err = r.doRequest(req) + if err != nil { + res.Body.Close() + if i == retries { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } + time.Sleep(time.Duration(i) * 5 * time.Second) + continue + } + break } + if res.StatusCode != 200 { res.Body.Close() return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", res.StatusCode, imgID) } + + if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { + utils.Debugf("server supports resume") + return utils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil + } + utils.Debugf("server doesn't support resume") return res.Body, nil } From c36c61ebfeda636678689b45649a175ebe16efc7 Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 14 Jul 2014 19:31:19 +0300 Subject: [PATCH 3/8] archive: add buffers to operations with tarballs Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: a377844998f6a866d59bf356790f76d0d25bf01f Component: engine --- components/engine/archive/archive.go | 23 +++++++++++++++++------ components/engine/archive/changes.go | 4 +++- components/engine/archive/common.go | 4 ++++ components/engine/archive/diff.go | 5 ++++- 4 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 components/engine/archive/common.go diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index bf6e0b7797..550ef3f040 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -131,7 +131,7 @@ func (compression *Compression) Extension() string { return "" } -func addTarFile(path, name string, tw *tar.Writer) error { +func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error { fi, err := os.Lstat(path) if err != nil { return err @@ -181,11 +181,18 @@ func addTarFile(path, name string, tw *tar.Writer) error { if err != nil { return err } - if _, err := io.Copy(tw, file); err != nil { - file.Close() + + twBuf.Reset(tw) + _, err = io.Copy(twBuf, file) + file.Close() + if err != nil { return err } - file.Close() + err = twBuf.Flush() + if err != nil { + return err + } + twBuf.Reset(nil) } return nil @@ -328,6 +335,8 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) options.Includes = []string{"."} } + twBuf := bufio.NewWriterSize(nil, twBufSize) + for _, include := range options.Includes { filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error { if err != nil { @@ -355,7 +364,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) } } - if err := addTarFile(filePath, relFilePath, tw); err != nil { + if err := addTarFile(filePath, relFilePath, tw, twBuf); err != nil { utils.Debugf("Can't add file %s to tar: %s\n", srcPath, err) } return nil @@ -394,6 +403,7 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error { defer decompressedArchive.Close() tr := tar.NewReader(decompressedArchive) + trBuf := bufio.NewReaderSize(nil, trBufSize) var dirs []*tar.Header @@ -439,7 +449,8 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error { } } } - if err := createTarFile(path, dest, hdr, tr, options == nil || !options.NoLchown); err != nil { + trBuf.Reset(tr) + if err := createTarFile(path, dest, hdr, trBuf, options == nil || !options.NoLchown); err != nil { return err } diff --git a/components/engine/archive/changes.go b/components/engine/archive/changes.go index 1e588b8eb5..154a527aec 100644 --- a/components/engine/archive/changes.go +++ b/components/engine/archive/changes.go @@ -1,6 +1,7 @@ package archive import ( + "bufio" "bytes" "fmt" "io" @@ -343,6 +344,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) { tw := tar.NewWriter(writer) go func() { + twBuf := bufio.NewWriterSize(nil, twBufSize) // In general we log errors here but ignore them because // during e.g. a diff operation the container can continue // mutating the filesystem and we can see transient errors @@ -365,7 +367,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) { } } else { path := filepath.Join(dir, change.Path) - if err := addTarFile(path, change.Path[1:], tw); err != nil { + if err := addTarFile(path, change.Path[1:], tw, twBuf); err != nil { utils.Debugf("Can't add file %s to tar: %s\n", path, err) } } diff --git a/components/engine/archive/common.go b/components/engine/archive/common.go new file mode 100644 index 0000000000..2aac34e840 --- /dev/null +++ b/components/engine/archive/common.go @@ -0,0 +1,4 @@ +package archive + +const twBufSize = 32 * 1024 +const trBufSize = 32 * 1024 diff --git a/components/engine/archive/diff.go b/components/engine/archive/diff.go index d169669126..34a887663d 100644 --- a/components/engine/archive/diff.go +++ b/components/engine/archive/diff.go @@ -1,6 +1,7 @@ package archive import ( + "bufio" "fmt" "io" "io/ioutil" @@ -32,6 +33,7 @@ func ApplyLayer(dest string, layer ArchiveReader) error { } tr := tar.NewReader(layer) + trBuf := bufio.NewReaderSize(nil, trBufSize) var dirs []*tar.Header @@ -108,7 +110,8 @@ func ApplyLayer(dest string, layer ArchiveReader) error { } } - srcData := io.Reader(tr) + trBuf.Reset(tr) + srcData := io.Reader(trBuf) srcHdr := hdr // Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so From 597e28c8ddf76f8908b936932c73c87e7bf3492f Mon Sep 17 00:00:00 2001 From: unclejack Date: Wed, 16 Jul 2014 21:50:02 +0300 Subject: [PATCH 4/8] archive: add a benchmark for TarUntar Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: 76429cc11f6e2bd731e9ee77a2a538f15ef80eea Component: engine --- components/engine/archive/archive_test.go | 39 +++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/components/engine/archive/archive_test.go b/components/engine/archive/archive_test.go index 61ee0af8e7..1a68234129 100644 --- a/components/engine/archive/archive_test.go +++ b/components/engine/archive/archive_test.go @@ -199,3 +199,42 @@ func TestUntarUstarGnuConflict(t *testing.T) { t.Fatalf("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm") } } + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} + +func BenchmarkTarUntar(b *testing.B) { + origin, err := ioutil.TempDir("", "docker-test-untar-origin") + if err != nil { + b.Fatal(err) + } + tempDir, err := ioutil.TempDir("", "docker-test-untar-destination") + if err != nil { + b.Fatal(err) + } + target := path.Join(tempDir, "dest") + n, err := prepareUntarSourceDirectory(100, origin) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + b.SetBytes(int64(n)) + defer os.RemoveAll(origin) + defer os.RemoveAll(tempDir) + for n := 0; n < b.N; n++ { + err := TarUntar(origin, target) + if err != nil { + b.Fatal(err) + } + os.RemoveAll(target) + } +} From 71d1cc48f004593497a75b9efb023540d9baf651 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 21 Jul 2014 22:00:26 -0600 Subject: [PATCH 5/8] Reorganize and clarify the DOCKER_BUILDTAGS docs Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 1b31a80eb78789834b8a7d4daff583f0b9a938ed Component: engine --- components/engine/hack/PACKAGERS.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/components/engine/hack/PACKAGERS.md b/components/engine/hack/PACKAGERS.md index 82d959c9e2..4312d5d3c0 100644 --- a/components/engine/hack/PACKAGERS.md +++ b/components/engine/hack/PACKAGERS.md @@ -152,11 +152,16 @@ directory, and the local "./vendor" directory as necessary. If you're building a binary that may need to be used on platforms that include AppArmor, you will need to set `DOCKER_BUILDTAGS` as follows: - ```bash export DOCKER_BUILDTAGS='apparmor' ``` +If you're building a binary that may need to be used on platforms that include +SELinux, you will need to use the `selinux` build tag: +```bash +export DOCKER_BUILDTAGS='selinux' +``` + There are build tags for disabling graphdrivers as well. By default, support for all graphdrivers are built in. @@ -175,13 +180,9 @@ To disable aufs: export DOCKER_BUILDTAGS='exclude_graphdriver_aufs' ``` -NOTE: if you need to set more than one build tag, space separate them. - -If you're building a binary that may need to be used on platforms that include -SELinux, you will need to set `DOCKER_BUILDTAGS` as follows: - +NOTE: if you need to set more than one build tag, space separate them: ```bash -export DOCKER_BUILDTAGS='selinux' +export DOCKER_BUILDTAGS='apparmor selinux exclude_graphdriver_aufs' ``` ### Static Daemon From da9850e0f6db619f2d3a8f31e91c71a5fa1ad040 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Wed, 23 Jul 2014 09:57:41 +0400 Subject: [PATCH 6/8] Add AUDIT_WRITE cap Fixes #6345 Thanks @larsks for outstanding investigation Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) Upstream-commit: 29ecc95c31ecfe15e3b3d8db94cea1c555e526a3 Component: engine --- .../engine/daemon/execdriver/native/template/default_template.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/daemon/execdriver/native/template/default_template.go b/components/engine/daemon/execdriver/native/template/default_template.go index cc5cc4f428..be3dd5a5c1 100644 --- a/components/engine/daemon/execdriver/native/template/default_template.go +++ b/components/engine/daemon/execdriver/native/template/default_template.go @@ -23,6 +23,7 @@ func New() *libcontainer.Config { "NET_BIND_SERVICE", "SYS_CHROOT", "KILL", + "AUDIT_WRITE", }, Namespaces: map[string]bool{ "NEWNS": true, From d72a08903066fc3a7f1c634aadc6e8c5f3e08aa3 Mon Sep 17 00:00:00 2001 From: Tim Ruffles Date: Wed, 23 Jul 2014 12:11:14 +0100 Subject: [PATCH 7/8] [DOCS] replace foo/bar with concrete names namespaces are not well documented, and I had to jump around to other docs. replacing `foo/bar` hopefully makes what's going on here a bit more obvious. Docker-DCO-1.1-Signed-off-by: Tim Ruffles (github: timruffles) Upstream-commit: 455e837e207a01153947fc81e3a22d6bb37d2c89 Component: engine --- .../docs/sources/reference/api/registry_api.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/components/engine/docs/sources/reference/api/registry_api.md b/components/engine/docs/sources/reference/api/registry_api.md index a3d4f23d66..49776b9b18 100644 --- a/components/engine/docs/sources/reference/api/registry_api.md +++ b/components/engine/docs/sources/reference/api/registry_api.md @@ -67,6 +67,8 @@ The latter would only require two new commands in docker, e.g., (and optionally doing consistency checks). Authentication and authorization are then delegated to SSH (e.g., with public keys). +The default namespace for a private repository is `library`. + # Endpoints ## Images @@ -305,7 +307,7 @@ Get all of the tags for the given repo. **Example Request**: - GET /v1/repositories/foo/bar/tags HTTP/1.1 + GET /v1/repositories/reynholm/help-system-server/tags HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json @@ -341,7 +343,7 @@ Get a tag for the given repo. **Example Request**: - GET /v1/repositories/foo/bar/tags/latest HTTP/1.1 + GET /v1/repositories/reynholm/help-system-server/tags/latest HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json @@ -375,7 +377,7 @@ Delete the tag for the repo **Example Request**: - DELETE /v1/repositories/foo/bar/tags/latest HTTP/1.1 + DELETE /v1/repositories/reynholm/help-system-server/tags/latest HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json @@ -408,7 +410,7 @@ Put a tag for the given repo. **Example Request**: - PUT /v1/repositories/foo/bar/tags/latest HTTP/1.1 + PUT /v1/repositories/reynholm/help-system-server/tags/latest HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json @@ -446,7 +448,7 @@ Delete a repository **Example Request**: - DELETE /v1/repositories/foo/bar/ HTTP/1.1 + DELETE /v1/repositories/reynholm/help-system-server/ HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json From c26325b9c4b32a93341672d4833cf5a51552c5e3 Mon Sep 17 00:00:00 2001 From: William Henry Date: Wed, 23 Jul 2014 16:49:07 -0400 Subject: [PATCH 8/8] Typo changes to docker-run-1.md - changing '-' to '--' where approp. Docker-DCO-1.1-Signed-off-by: William Henry (github: ipbabble) Upstream-commit: 140337296353b2dd37678b0e30cbf0ebe2cfd9e7 Component: engine --- components/engine/docs/man/docker-run.1.md | 24 ++++++++++------------ 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/components/engine/docs/man/docker-run.1.md b/components/engine/docs/man/docker-run.1.md index 4bf0ce0247..4dee97f195 100644 --- a/components/engine/docs/man/docker-run.1.md +++ b/components/engine/docs/man/docker-run.1.md @@ -82,7 +82,7 @@ run**. **--cpuset**="" CPUs in which to allow execution (0-3, 0,1) -**-d**, **-detach**=*true*|*false* +**-d**, **--detach**=*true*|*false* Detached mode. This runs the container in the background. It outputs the new container's ID and any error messages. At any time you can run **docker ps** in the other shell to view a list of the running containers. You can reattach to a @@ -101,12 +101,9 @@ stopping the process by pressing the keys CTRL-P CTRL-Q. Set custom DNS servers. This option can be used to override the DNS configuration passed to the container. Typically this is necessary when the host DNS configuration is invalid for the container (e.g., 127.0.0.1). When this -is the case the **-dns** flags is necessary for every run. +is the case the **--dns** flags is necessary for every run. -**-e**, **--env**=[] - Set environment variables - -**-e**, **-env**=*environment* +**-e**, **--env**=*environment* Set environment variables. This option allows you to specify arbitrary environment variables that are available for the process that will be launched inside of the container. @@ -123,6 +120,7 @@ pass in more options via the COMMAND. But, sometimes an operator may want to run something else inside the container, so you can override the default ENTRYPOINT at runtime by using a **--entrypoint** and a string to specify the new ENTRYPOINT. + **--env-file**=[] Read in a line delimited file of environment variables @@ -133,10 +131,10 @@ developer can expose the port using the EXPOSE parameter of the Dockerfile, 2) the operator can use the **--expose** option with **docker run**, or 3) the container can be started with the **--link**. -**-h**, **-hostname**=*hostname* +**-h**, **--hostname**=*hostname* Sets the container host name that is available inside the container. -**-i**, **-interactive**=*true*|*false* +**-i**, **--interactive**=*true*|*false* When set to true, keep stdin open even if not attached. The default is false. **--link**=*name*:*alias* @@ -149,7 +147,7 @@ which interface and port to use. **--lxc-conf**=[] (lxc exec-driver only) Add custom lxc options --lxc-conf="lxc.cgroup.cpuset.cpus = 0,1" -**-m**, **-memory**=*memory-limit* +**-m**, **--memory**=*memory-limit* Allows you to constrain the memory available to a container. If the host supports swap memory, then the -m memory setting can be larger than physical RAM. If a limit of 0 is specified, the container's memory is not limited. The @@ -178,14 +176,14 @@ and foreground Docker containers. 'container:': reuses another container network stack 'host': use the host network stack inside the container. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure. -**-P**, **-publish-all**=*true*|*false* +**-P**, **--publish-all**=*true*|*false* When set to true publish all exposed ports to the host interfaces. The default is false. If the operator uses -P (or -p) then Docker will make the exposed port accessible on the host and the ports will be available to any client that can reach the host. To find the map between the host ports and the exposed ports, use **docker port**. -**-p**, **-publish**=[] +**-p**, **--publish**=[] Publish a container's port to the host (format: ip:hostPort:containerPort | ip::containerPort | hostPort:containerPort) (use **docker port** to see the actual mapping) @@ -217,7 +215,7 @@ interactive shell. The default is value is false. Username or UID -**-v**, **-volume**=*volume*[:ro|:rw] +**-v**, **--volume**=*volume*[:ro|:rw] Bind mount a volume to the container. The **-v** option can be used one or @@ -241,7 +239,7 @@ default, the volumes are mounted in the same mode (read write or read only) as the reference container. -**-w**, **-workdir**=*directory* +**-w**, **--workdir**=*directory* Working directory inside the container. The default working directory for running binaries within a container is the root directory (/). The developer can set a different default with the Dockerfile WORKDIR instruction. The operator