diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index bf6e0b7797..550ef3f040 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -131,7 +131,7 @@ func (compression *Compression) Extension() string { return "" } -func addTarFile(path, name string, tw *tar.Writer) error { +func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error { fi, err := os.Lstat(path) if err != nil { return err @@ -181,11 +181,18 @@ func addTarFile(path, name string, tw *tar.Writer) error { if err != nil { return err } - if _, err := io.Copy(tw, file); err != nil { - file.Close() + + twBuf.Reset(tw) + _, err = io.Copy(twBuf, file) + file.Close() + if err != nil { return err } - file.Close() + err = twBuf.Flush() + if err != nil { + return err + } + twBuf.Reset(nil) } return nil @@ -328,6 +335,8 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) options.Includes = []string{"."} } + twBuf := bufio.NewWriterSize(nil, twBufSize) + for _, include := range options.Includes { filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error { if err != nil { @@ -355,7 +364,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) } } - if err := addTarFile(filePath, relFilePath, tw); err != nil { + if err := addTarFile(filePath, relFilePath, tw, twBuf); err != nil { utils.Debugf("Can't add file %s to tar: %s\n", srcPath, err) } return nil @@ -394,6 +403,7 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error { defer decompressedArchive.Close() tr := tar.NewReader(decompressedArchive) + trBuf := bufio.NewReaderSize(nil, trBufSize) var dirs []*tar.Header @@ -439,7 +449,8 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error { } } } - if err := createTarFile(path, dest, hdr, tr, options == nil || !options.NoLchown); err != nil { + trBuf.Reset(tr) + if err := createTarFile(path, dest, hdr, trBuf, options == nil || !options.NoLchown); err != nil { return err } diff --git a/components/engine/archive/archive_test.go b/components/engine/archive/archive_test.go index 61ee0af8e7..1a68234129 100644 --- a/components/engine/archive/archive_test.go +++ b/components/engine/archive/archive_test.go @@ -199,3 +199,42 @@ func TestUntarUstarGnuConflict(t *testing.T) { t.Fatalf("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm") } } + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} + +func BenchmarkTarUntar(b *testing.B) { + origin, err := ioutil.TempDir("", "docker-test-untar-origin") + if err != nil { + b.Fatal(err) + } + tempDir, err := ioutil.TempDir("", "docker-test-untar-destination") + if err != nil { + b.Fatal(err) + } + target := path.Join(tempDir, "dest") + n, err := prepareUntarSourceDirectory(100, origin) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + b.SetBytes(int64(n)) + defer os.RemoveAll(origin) + defer os.RemoveAll(tempDir) + for n := 0; n < b.N; n++ { + err := TarUntar(origin, target) + if err != nil { + b.Fatal(err) + } + os.RemoveAll(target) + } +} diff --git a/components/engine/archive/changes.go b/components/engine/archive/changes.go index 1e588b8eb5..154a527aec 100644 --- a/components/engine/archive/changes.go +++ b/components/engine/archive/changes.go @@ -1,6 +1,7 @@ package archive import ( + "bufio" "bytes" "fmt" "io" @@ -343,6 +344,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) { tw := tar.NewWriter(writer) go func() { + twBuf := bufio.NewWriterSize(nil, twBufSize) // In general we log errors here but ignore them because // during e.g. a diff operation the container can continue // mutating the filesystem and we can see transient errors @@ -365,7 +367,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) { } } else { path := filepath.Join(dir, change.Path) - if err := addTarFile(path, change.Path[1:], tw); err != nil { + if err := addTarFile(path, change.Path[1:], tw, twBuf); err != nil { utils.Debugf("Can't add file %s to tar: %s\n", path, err) } } diff --git a/components/engine/archive/common.go b/components/engine/archive/common.go new file mode 100644 index 0000000000..2aac34e840 --- /dev/null +++ b/components/engine/archive/common.go @@ -0,0 +1,4 @@ +package archive + +const twBufSize = 32 * 1024 +const trBufSize = 32 * 1024 diff --git a/components/engine/archive/diff.go b/components/engine/archive/diff.go index d169669126..34a887663d 100644 --- a/components/engine/archive/diff.go +++ b/components/engine/archive/diff.go @@ -1,6 +1,7 @@ package archive import ( + "bufio" "fmt" "io" "io/ioutil" @@ -32,6 +33,7 @@ func ApplyLayer(dest string, layer ArchiveReader) error { } tr := tar.NewReader(layer) + trBuf := bufio.NewReaderSize(nil, trBufSize) var dirs []*tar.Header @@ -108,7 +110,8 @@ func ApplyLayer(dest string, layer ArchiveReader) error { } } - srcData := io.Reader(tr) + trBuf.Reset(tr) + srcData := io.Reader(trBuf) srcHdr := hdr // Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so diff --git a/components/engine/daemon/execdriver/native/template/default_template.go b/components/engine/daemon/execdriver/native/template/default_template.go index cc5cc4f428..be3dd5a5c1 100644 --- a/components/engine/daemon/execdriver/native/template/default_template.go +++ b/components/engine/daemon/execdriver/native/template/default_template.go @@ -23,6 +23,7 @@ func New() *libcontainer.Config { "NET_BIND_SERVICE", "SYS_CHROOT", "KILL", + "AUDIT_WRITE", }, Namespaces: map[string]bool{ "NEWNS": true, diff --git a/components/engine/docs/man/docker-run.1.md b/components/engine/docs/man/docker-run.1.md index 4bf0ce0247..4dee97f195 100644 --- a/components/engine/docs/man/docker-run.1.md +++ b/components/engine/docs/man/docker-run.1.md @@ -82,7 +82,7 @@ run**. **--cpuset**="" CPUs in which to allow execution (0-3, 0,1) -**-d**, **-detach**=*true*|*false* +**-d**, **--detach**=*true*|*false* Detached mode. This runs the container in the background. It outputs the new container's ID and any error messages. At any time you can run **docker ps** in the other shell to view a list of the running containers. You can reattach to a @@ -101,12 +101,9 @@ stopping the process by pressing the keys CTRL-P CTRL-Q. Set custom DNS servers. This option can be used to override the DNS configuration passed to the container. Typically this is necessary when the host DNS configuration is invalid for the container (e.g., 127.0.0.1). When this -is the case the **-dns** flags is necessary for every run. +is the case the **--dns** flags is necessary for every run. -**-e**, **--env**=[] - Set environment variables - -**-e**, **-env**=*environment* +**-e**, **--env**=*environment* Set environment variables. This option allows you to specify arbitrary environment variables that are available for the process that will be launched inside of the container. @@ -123,6 +120,7 @@ pass in more options via the COMMAND. But, sometimes an operator may want to run something else inside the container, so you can override the default ENTRYPOINT at runtime by using a **--entrypoint** and a string to specify the new ENTRYPOINT. + **--env-file**=[] Read in a line delimited file of environment variables @@ -133,10 +131,10 @@ developer can expose the port using the EXPOSE parameter of the Dockerfile, 2) the operator can use the **--expose** option with **docker run**, or 3) the container can be started with the **--link**. -**-h**, **-hostname**=*hostname* +**-h**, **--hostname**=*hostname* Sets the container host name that is available inside the container. -**-i**, **-interactive**=*true*|*false* +**-i**, **--interactive**=*true*|*false* When set to true, keep stdin open even if not attached. The default is false. **--link**=*name*:*alias* @@ -149,7 +147,7 @@ which interface and port to use. **--lxc-conf**=[] (lxc exec-driver only) Add custom lxc options --lxc-conf="lxc.cgroup.cpuset.cpus = 0,1" -**-m**, **-memory**=*memory-limit* +**-m**, **--memory**=*memory-limit* Allows you to constrain the memory available to a container. If the host supports swap memory, then the -m memory setting can be larger than physical RAM. If a limit of 0 is specified, the container's memory is not limited. The @@ -178,14 +176,14 @@ and foreground Docker containers. 'container:': reuses another container network stack 'host': use the host network stack inside the container. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure. -**-P**, **-publish-all**=*true*|*false* +**-P**, **--publish-all**=*true*|*false* When set to true publish all exposed ports to the host interfaces. The default is false. If the operator uses -P (or -p) then Docker will make the exposed port accessible on the host and the ports will be available to any client that can reach the host. To find the map between the host ports and the exposed ports, use **docker port**. -**-p**, **-publish**=[] +**-p**, **--publish**=[] Publish a container's port to the host (format: ip:hostPort:containerPort | ip::containerPort | hostPort:containerPort) (use **docker port** to see the actual mapping) @@ -217,7 +215,7 @@ interactive shell. The default is value is false. Username or UID -**-v**, **-volume**=*volume*[:ro|:rw] +**-v**, **--volume**=*volume*[:ro|:rw] Bind mount a volume to the container. The **-v** option can be used one or @@ -241,7 +239,7 @@ default, the volumes are mounted in the same mode (read write or read only) as the reference container. -**-w**, **-workdir**=*directory* +**-w**, **--workdir**=*directory* Working directory inside the container. The default working directory for running binaries within a container is the root directory (/). The developer can set a different default with the Dockerfile WORKDIR instruction. The operator diff --git a/components/engine/docs/sources/reference/api/registry_api.md b/components/engine/docs/sources/reference/api/registry_api.md index a3d4f23d66..49776b9b18 100644 --- a/components/engine/docs/sources/reference/api/registry_api.md +++ b/components/engine/docs/sources/reference/api/registry_api.md @@ -67,6 +67,8 @@ The latter would only require two new commands in docker, e.g., (and optionally doing consistency checks). Authentication and authorization are then delegated to SSH (e.g., with public keys). +The default namespace for a private repository is `library`. + # Endpoints ## Images @@ -305,7 +307,7 @@ Get all of the tags for the given repo. **Example Request**: - GET /v1/repositories/foo/bar/tags HTTP/1.1 + GET /v1/repositories/reynholm/help-system-server/tags HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json @@ -341,7 +343,7 @@ Get a tag for the given repo. **Example Request**: - GET /v1/repositories/foo/bar/tags/latest HTTP/1.1 + GET /v1/repositories/reynholm/help-system-server/tags/latest HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json @@ -375,7 +377,7 @@ Delete the tag for the repo **Example Request**: - DELETE /v1/repositories/foo/bar/tags/latest HTTP/1.1 + DELETE /v1/repositories/reynholm/help-system-server/tags/latest HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json @@ -408,7 +410,7 @@ Put a tag for the given repo. **Example Request**: - PUT /v1/repositories/foo/bar/tags/latest HTTP/1.1 + PUT /v1/repositories/reynholm/help-system-server/tags/latest HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json @@ -446,7 +448,7 @@ Delete a repository **Example Request**: - DELETE /v1/repositories/foo/bar/ HTTP/1.1 + DELETE /v1/repositories/reynholm/help-system-server/ HTTP/1.1 Host: registry-1.docker.io Accept: application/json Content-Type: application/json diff --git a/components/engine/hack/PACKAGERS.md b/components/engine/hack/PACKAGERS.md index 82d959c9e2..4312d5d3c0 100644 --- a/components/engine/hack/PACKAGERS.md +++ b/components/engine/hack/PACKAGERS.md @@ -152,11 +152,16 @@ directory, and the local "./vendor" directory as necessary. If you're building a binary that may need to be used on platforms that include AppArmor, you will need to set `DOCKER_BUILDTAGS` as follows: - ```bash export DOCKER_BUILDTAGS='apparmor' ``` +If you're building a binary that may need to be used on platforms that include +SELinux, you will need to use the `selinux` build tag: +```bash +export DOCKER_BUILDTAGS='selinux' +``` + There are build tags for disabling graphdrivers as well. By default, support for all graphdrivers are built in. @@ -175,13 +180,9 @@ To disable aufs: export DOCKER_BUILDTAGS='exclude_graphdriver_aufs' ``` -NOTE: if you need to set more than one build tag, space separate them. - -If you're building a binary that may need to be used on platforms that include -SELinux, you will need to set `DOCKER_BUILDTAGS` as follows: - +NOTE: if you need to set more than one build tag, space separate them: ```bash -export DOCKER_BUILDTAGS='selinux' +export DOCKER_BUILDTAGS='apparmor selinux exclude_graphdriver_aufs' ``` ### Static Daemon diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index e10516f9af..974e7fb9f8 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -390,52 +390,42 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { var ( - retries = 5 - headRes *http.Response - client *http.Client - hasResume bool = false - imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) + retries = 5 + client *http.Client + res *http.Response + imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) - headReq, err := r.reqFactory.NewRequest("HEAD", imageURL, nil) - if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - - setTokenAuth(headReq, token) - for i := 1; i <= retries; i++ { - headRes, client, err = r.doRequest(headReq) - if err != nil && i == retries { - return nil, fmt.Errorf("Eror while making head request: %s\n", err) - } else if err != nil { - time.Sleep(time.Duration(i) * 5 * time.Second) - continue - } - break - } - - if headRes.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - hasResume = true - } req, err := r.reqFactory.NewRequest("GET", imageURL, nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } setTokenAuth(req, token) - if hasResume { - utils.Debugf("server supports resume") - return utils.ResumableRequestReader(client, req, 5, imgSize), nil - } - utils.Debugf("server doesn't support resume") - res, _, err := r.doRequest(req) - if err != nil { - return nil, err + for i := 1; i <= retries; i++ { + res, client, err = r.doRequest(req) + if err != nil { + res.Body.Close() + if i == retries { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } + time.Sleep(time.Duration(i) * 5 * time.Second) + continue + } + break } + if res.StatusCode != 200 { res.Body.Close() return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", res.StatusCode, imgID) } + + if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { + utils.Debugf("server supports resume") + return utils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil + } + utils.Debugf("server doesn't support resume") return res.Body, nil } diff --git a/components/engine/utils/resumablerequestreader.go b/components/engine/utils/resumablerequestreader.go index e01f4e6d71..bed202ec0c 100644 --- a/components/engine/utils/resumablerequestreader.go +++ b/components/engine/utils/resumablerequestreader.go @@ -24,6 +24,10 @@ func ResumableRequestReader(c *http.Client, r *http.Request, maxfail uint32, tot return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize} } +func ResumableRequestReaderWithInitialResponse(c *http.Client, r *http.Request, maxfail uint32, totalsize int64, initialResponse *http.Response) io.ReadCloser { + return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize, currentResponse: initialResponse} +} + func (r *resumableRequestReader) Read(p []byte) (n int, err error) { if r.client == nil || r.request == nil { return 0, fmt.Errorf("client and request can't be nil\n")