Merge remote-tracking branch 'upstream/master' into doc_remove

Docker-DCO-1.1-Signed-off-by: hollietealok <hollie@docker.com> (github: hollietealok)
Upstream-commit: 97283fa985623ca59183026a13951e7e9e6ce144
Component: engine

components/engine/archive/archive.go

@@ -131,7 +131,7 @@ func (compression *Compression) Extension() string {
 	return ""
 }
 
-func addTarFile(path, name string, tw *tar.Writer) error {
+func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error {
 	fi, err := os.Lstat(path)
 	if err != nil {
 		return err
@@ -181,11 +181,18 @@ func addTarFile(path, name string, tw *tar.Writer) error {
 		if err != nil {
 			return err
 		}
-		if _, err := io.Copy(tw, file); err != nil {
-			file.Close()
+
+		twBuf.Reset(tw)
+		_, err = io.Copy(twBuf, file)
+		file.Close()
+		if err != nil {
 			return err
 		}
-		file.Close()
+		err = twBuf.Flush()
+		if err != nil {
+			return err
+		}
+		twBuf.Reset(nil)
 	}
 
 	return nil
@@ -328,6 +335,8 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
 			options.Includes = []string{"."}
 		}
 
+		twBuf := bufio.NewWriterSize(nil, twBufSize)
+
 		for _, include := range options.Includes {
 			filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error {
 				if err != nil {
@@ -355,7 +364,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
 					}
 				}
 
-				if err := addTarFile(filePath, relFilePath, tw); err != nil {
+				if err := addTarFile(filePath, relFilePath, tw, twBuf); err != nil {
 					utils.Debugf("Can't add file %s to tar: %s\n", srcPath, err)
 				}
 				return nil
@@ -394,6 +403,7 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error {
 	defer decompressedArchive.Close()
 
 	tr := tar.NewReader(decompressedArchive)
+	trBuf := bufio.NewReaderSize(nil, trBufSize)
 
 	var dirs []*tar.Header
 
@@ -439,7 +449,8 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error {
 				}
 			}
 		}
-		if err := createTarFile(path, dest, hdr, tr, options == nil || !options.NoLchown); err != nil {
+		trBuf.Reset(tr)
+		if err := createTarFile(path, dest, hdr, trBuf, options == nil || !options.NoLchown); err != nil {
 			return err
 		}
 

components/engine/archive/archive_test.go

@@ -199,3 +199,42 @@ func TestUntarUstarGnuConflict(t *testing.T) {
 		t.Fatalf("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm")
 	}
 }
+
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}
+
+func BenchmarkTarUntar(b *testing.B) {
+	origin, err := ioutil.TempDir("", "docker-test-untar-origin")
+	if err != nil {
+		b.Fatal(err)
+	}
+	tempDir, err := ioutil.TempDir("", "docker-test-untar-destination")
+	if err != nil {
+		b.Fatal(err)
+	}
+	target := path.Join(tempDir, "dest")
+	n, err := prepareUntarSourceDirectory(100, origin)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	b.SetBytes(int64(n))
+	defer os.RemoveAll(origin)
+	defer os.RemoveAll(tempDir)
+	for n := 0; n < b.N; n++ {
+		err := TarUntar(origin, target)
+		if err != nil {
+			b.Fatal(err)
+		}
+		os.RemoveAll(target)
+	}
+}

components/engine/archive/changes.go

@@ -1,6 +1,7 @@
 package archive
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"io"
@@ -343,6 +344,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) {
 	tw := tar.NewWriter(writer)
 
 	go func() {
+		twBuf := bufio.NewWriterSize(nil, twBufSize)
 		// In general we log errors here but ignore them because
 		// during e.g. a diff operation the container can continue
 		// mutating the filesystem and we can see transient errors
@@ -365,7 +367,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) {
 				}
 			} else {
 				path := filepath.Join(dir, change.Path)
-				if err := addTarFile(path, change.Path[1:], tw); err != nil {
+				if err := addTarFile(path, change.Path[1:], tw, twBuf); err != nil {
 					utils.Debugf("Can't add file %s to tar: %s\n", path, err)
 				}
 			}

components/engine/archive/common.go

@@ -0,0 +1,4 @@
+package archive
+
+const twBufSize = 32 * 1024
+const trBufSize = 32 * 1024

components/engine/archive/diff.go

@@ -1,6 +1,7 @@
 package archive
 
 import (
+	"bufio"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -32,6 +33,7 @@ func ApplyLayer(dest string, layer ArchiveReader) error {
 	}
 
 	tr := tar.NewReader(layer)
+	trBuf := bufio.NewReaderSize(nil, trBufSize)
 
 	var dirs []*tar.Header
 
@@ -108,7 +110,8 @@ func ApplyLayer(dest string, layer ArchiveReader) error {
 				}
 			}
 
-			srcData := io.Reader(tr)
+			trBuf.Reset(tr)
+			srcData := io.Reader(trBuf)
 			srcHdr := hdr
 
 			// Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so

components/engine/daemon/execdriver/native/template/default_template.go

@@ -23,6 +23,7 @@ func New() *libcontainer.Config {
 			"NET_BIND_SERVICE",
 			"SYS_CHROOT",
 			"KILL",
+			"AUDIT_WRITE",
 		},
 		Namespaces: map[string]bool{
 			"NEWNS":  true,

components/engine/docs/man/docker-run.1.md

@@ -82,7 +82,7 @@ run**.
 **--cpuset**=""
    CPUs in which to allow execution (0-3, 0,1)
 
-**-d**, **-detach**=*true*|*false*
+**-d**, **--detach**=*true*|*false*
    Detached mode. This runs the container in the background. It outputs the new
 container's ID and any error messages. At any time you can run **docker ps** in
 the other shell to view a list of the running containers. You can reattach to a
@@ -101,12 +101,9 @@ stopping the process by pressing the keys CTRL-P CTRL-Q.
    Set custom DNS servers. This option can be used to override the DNS
 configuration passed to the container. Typically this is necessary when the
 host DNS configuration is invalid for the container (e.g., 127.0.0.1). When this
-is the case the **-dns** flags is necessary for every run.
+is the case the **--dns** flags is necessary for every run.
 
-**-e**, **--env**=[]
-   Set environment variables
-
-**-e**, **-env**=*environment*
+**-e**, **--env**=*environment*
    Set environment variables. This option allows you to specify arbitrary
 environment variables that are available for the process that will be launched
 inside of the container.
@@ -123,6 +120,7 @@ pass in more options via the COMMAND. But, sometimes an operator may want to run
 something else inside the container, so you can override the default ENTRYPOINT
 at runtime by using a **--entrypoint** and a string to specify the new
 ENTRYPOINT.
+
 **--env-file**=[]
    Read in a line delimited file of environment variables
 
@@ -133,10 +131,10 @@ developer can expose the port using the EXPOSE parameter of the Dockerfile, 2)
 the operator can use the **--expose** option with **docker run**, or 3) the
 container can be started with the **--link**.
 
-**-h**, **-hostname**=*hostname*
+**-h**, **--hostname**=*hostname*
    Sets the container host name that is available inside the container.
 
-**-i**, **-interactive**=*true*|*false*
+**-i**, **--interactive**=*true*|*false*
    When set to true, keep stdin open even if not attached. The default is false.
 
 **--link**=*name*:*alias*
@@ -149,7 +147,7 @@ which interface and port to use.
 **--lxc-conf**=[]
    (lxc exec-driver only) Add custom lxc options --lxc-conf="lxc.cgroup.cpuset.cpus = 0,1"
 
-**-m**, **-memory**=*memory-limit*
+**-m**, **--memory**=*memory-limit*
    Allows you to constrain the memory available to a container. If the host
 supports swap memory, then the -m memory setting can be larger than physical
 RAM. If a limit of 0 is specified, the container's memory is not limited. The
@@ -178,14 +176,14 @@ and foreground Docker containers.
                                'container:<name|id>': reuses another container network stack
                                'host': use the host network stack inside the container.  Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure.
 
-**-P**, **-publish-all**=*true*|*false*
+**-P**, **--publish-all**=*true*|*false*
    When set to true publish all exposed ports to the host interfaces. The
 default is false. If the operator uses -P (or -p) then Docker will make the
 exposed port accessible on the host and the ports will be available to any
 client that can reach the host. To find the map between the host ports and the
 exposed ports, use **docker port**.
 
-**-p**, **-publish**=[]
+**-p**, **--publish**=[]
    Publish a container's port to the host (format: ip:hostPort:containerPort |
 ip::containerPort | hostPort:containerPort) (use **docker port** to see the
 actual mapping)
@@ -217,7 +215,7 @@ interactive shell. The default is value is false.
    Username or UID
 
 
-**-v**, **-volume**=*volume*[:ro|:rw]
+**-v**, **--volume**=*volume*[:ro|:rw]
    Bind mount a volume to the container. 
 
 The **-v** option can be used one or
@@ -241,7 +239,7 @@ default, the volumes are mounted in the same mode (read write or read only) as
 the reference container.
 
 
-**-w**, **-workdir**=*directory*
+**-w**, **--workdir**=*directory*
    Working directory inside the container. The default working directory for
 running binaries within a container is the root directory (/). The developer can
 set a different default with the Dockerfile WORKDIR instruction. The operator

components/engine/docs/sources/reference/api/registry_api.md

@@ -67,6 +67,8 @@ The latter would only require two new commands in docker, e.g.,
 (and optionally doing consistency checks). Authentication and authorization
 are then delegated to SSH (e.g., with public keys).
 
+The default namespace for a private repository is `library`.
+
 # Endpoints
 
 ## Images
@@ -305,7 +307,7 @@ Get all of the tags for the given repo.
 
     **Example Request**:
 
-        GET /v1/repositories/foo/bar/tags HTTP/1.1
+        GET /v1/repositories/reynholm/help-system-server/tags HTTP/1.1
         Host: registry-1.docker.io
         Accept: application/json
         Content-Type: application/json
@@ -341,7 +343,7 @@ Get a tag for the given repo.
 
     **Example Request**:
 
-        GET /v1/repositories/foo/bar/tags/latest HTTP/1.1
+        GET /v1/repositories/reynholm/help-system-server/tags/latest HTTP/1.1
         Host: registry-1.docker.io
         Accept: application/json
         Content-Type: application/json
@@ -375,7 +377,7 @@ Delete the tag for the repo
 
     **Example Request**:
 
-        DELETE /v1/repositories/foo/bar/tags/latest HTTP/1.1
+        DELETE /v1/repositories/reynholm/help-system-server/tags/latest HTTP/1.1
         Host: registry-1.docker.io
         Accept: application/json
         Content-Type: application/json
@@ -408,7 +410,7 @@ Put a tag for the given repo.
 
     **Example Request**:
 
-        PUT /v1/repositories/foo/bar/tags/latest HTTP/1.1
+        PUT /v1/repositories/reynholm/help-system-server/tags/latest HTTP/1.1
         Host: registry-1.docker.io
         Accept: application/json
         Content-Type: application/json
@@ -446,7 +448,7 @@ Delete a repository
 
     **Example Request**:
 
-        DELETE /v1/repositories/foo/bar/ HTTP/1.1
+        DELETE /v1/repositories/reynholm/help-system-server/ HTTP/1.1
         Host: registry-1.docker.io
         Accept: application/json
         Content-Type: application/json

components/engine/hack/PACKAGERS.md

@@ -152,11 +152,16 @@ directory, and the local "./vendor" directory as necessary.
 
 If you're building a binary that may need to be used on platforms that include
 AppArmor, you will need to set `DOCKER_BUILDTAGS` as follows:
-
 ```bash
 export DOCKER_BUILDTAGS='apparmor'
 ```
 
+If you're building a binary that may need to be used on platforms that include
+SELinux, you will need to use the `selinux` build tag:
+```bash
+export DOCKER_BUILDTAGS='selinux'
+```
+
 There are build tags for disabling graphdrivers as well. By default, support
 for all graphdrivers are built in.
 
@@ -175,13 +180,9 @@ To disable aufs:
 export DOCKER_BUILDTAGS='exclude_graphdriver_aufs'
 ```
 
-NOTE: if you need to set more than one build tag, space separate them.
-
-If you're building a binary that may need to be used on platforms that include
-SELinux, you will need to set `DOCKER_BUILDTAGS` as follows:
-
+NOTE: if you need to set more than one build tag, space separate them:
 ```bash
-export DOCKER_BUILDTAGS='selinux'
+export DOCKER_BUILDTAGS='apparmor selinux exclude_graphdriver_aufs'
 ```
 
 ### Static Daemon

components/engine/registry/registry.go

@@ -390,52 +390,42 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([
 
 func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) {
 	var (
-		retries   = 5
-		headRes   *http.Response
-		client    *http.Client
-		hasResume bool = false
-		imageURL       = fmt.Sprintf("%simages/%s/layer", registry, imgID)
+		retries  = 5
+		client   *http.Client
+		res      *http.Response
+		imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID)
 	)
-	headReq, err := r.reqFactory.NewRequest("HEAD", imageURL, nil)
-	if err != nil {
-		return nil, fmt.Errorf("Error while getting from the server: %s\n", err)
-	}
-
-	setTokenAuth(headReq, token)
-	for i := 1; i <= retries; i++ {
-		headRes, client, err = r.doRequest(headReq)
-		if err != nil && i == retries {
-			return nil, fmt.Errorf("Eror while making head request: %s\n", err)
-		} else if err != nil {
-			time.Sleep(time.Duration(i) * 5 * time.Second)
-			continue
-		}
-		break
-	}
-
-	if headRes.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 {
-		hasResume = true
-	}
 
 	req, err := r.reqFactory.NewRequest("GET", imageURL, nil)
 	if err != nil {
 		return nil, fmt.Errorf("Error while getting from the server: %s\n", err)
 	}
 	setTokenAuth(req, token)
-	if hasResume {
-		utils.Debugf("server supports resume")
-		return utils.ResumableRequestReader(client, req, 5, imgSize), nil
-	}
-	utils.Debugf("server doesn't support resume")
-	res, _, err := r.doRequest(req)
-	if err != nil {
-		return nil, err
+	for i := 1; i <= retries; i++ {
+		res, client, err = r.doRequest(req)
+		if err != nil {
+			res.Body.Close()
+			if i == retries {
+				return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)",
+					res.StatusCode, imgID)
+			}
+			time.Sleep(time.Duration(i) * 5 * time.Second)
+			continue
+		}
+		break
 	}
+
 	if res.StatusCode != 200 {
 		res.Body.Close()
 		return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)",
 			res.StatusCode, imgID)
 	}
+
+	if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 {
+		utils.Debugf("server supports resume")
+		return utils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil
+	}
+	utils.Debugf("server doesn't support resume")
 	return res.Body, nil
 }
 

components/engine/utils/resumablerequestreader.go

@@ -24,6 +24,10 @@ func ResumableRequestReader(c *http.Client, r *http.Request, maxfail uint32, tot
 	return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize}
 }
 
+func ResumableRequestReaderWithInitialResponse(c *http.Client, r *http.Request, maxfail uint32, totalsize int64, initialResponse *http.Response) io.ReadCloser {
+	return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize, currentResponse: initialResponse}
+}
+
 func (r *resumableRequestReader) Read(p []byte) (n int, err error) {
 	if r.client == nil || r.request == nil {
 		return 0, fmt.Errorf("client and request can't be nil\n")