'docker pull' and 'docker put' automatically detect tar compression (gzip, bzip2 or uncompressed). -j and -z flags are no longer required.

Upstream-commit: f437f5b8b4a706d56f9a7cfba1dbc6b21e9a33f3
Component: engine

components/engine/image/archive.go

@@ -0,0 +1,71 @@
+package image
+
+import (
+	"io"
+	"io/ioutil"
+	"os/exec"
+	"errors"
+)
+
+type Compression uint32
+
+const (
+	Uncompressed	Compression = iota
+	Bzip2
+	Gzip
+)
+
+func (compression *Compression) Flag() string {
+	switch *compression {
+		case Bzip2: return "j"
+		case Gzip: return "z"
+	}
+	return ""
+}
+
+func Tar(path string, compression Compression) (io.Reader, error) {
+	cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-c" + compression.Flag(), ".")
+	return CmdStream(cmd)
+}
+
+func Untar(archive io.Reader, path string) error {
+	cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-x")
+	cmd.Stdin = archive
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.New(err.Error() + ": " + string(output))
+	}
+	return nil
+}
+
+func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, err
+	}
+	pipeR, pipeW := io.Pipe()
+	go func() {
+		_, err := io.Copy(pipeW, stdout)
+		if err != nil {
+			pipeW.CloseWithError(err)
+		}
+		errText, e := ioutil.ReadAll(stderr)
+		if e != nil {
+			errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")")
+		}
+		if err := cmd.Wait(); err != nil {
+			// FIXME: can this block if stderr outputs more than the size of StderrPipe()'s buffer?
+			pipeW.CloseWithError(errors.New(err.Error() + ": " + string(errText)))
+		} else {
+			pipeW.Close()
+		}
+	}()
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+	return pipeR, nil
+}

components/engine/image/archive_test.go

@@ -0,0 +1,54 @@
+package image
+
+import (
+	"testing"
+	"os"
+	"os/exec"
+	"io/ioutil"
+)
+
+func TestCmdStreamBad(t *testing.T) {
+	badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
+	out, err := CmdStream(badCmd)
+	if err != nil {
+		t.Fatalf("Failed to start command: " + err.Error())
+	}
+	if output, err := ioutil.ReadAll(out); err == nil {
+		t.Fatalf("Command should have failed")
+	} else if err.Error() != "exit status 1: error couldn't reverse the phase pulser\n" {
+		t.Fatalf("Wrong error value (%s)", err.Error())
+	} else if s := string(output); s != "hello\n" {
+		t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
+	}
+}
+
+func TestCmdStreamGood(t *testing.T) {
+	cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0")
+	out, err := CmdStream(cmd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if output, err := ioutil.ReadAll(out); err != nil {
+		t.Fatalf("Command should not have failed (err=%s)", err)
+	} else if s := string(output); s != "hello\n" {
+		t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
+	}
+}
+
+func TestTarUntar(t *testing.T) {
+	archive, err := Tar(".", Uncompressed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmp, err := ioutil.TempDir("", "docker-test-untar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	if err := Untar(archive, tmp); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(tmp); err != nil {
+		t.Fatalf("Error stating %s: %s", tmp, err.Error())
+	}
+}

components/engine/image/image.go

@@ -44,16 +44,10 @@ func New(root string) (*Store, error) {
 	}, nil
 }
 
-type Compression uint32
-
-const (
-	Uncompressed	Compression = iota
-	Bzip2
-	Gzip
-)
-
-func (store *Store) Import(name string, archive io.Reader, stderr io.Writer, parent *Image, compression Compression) (*Image, error) {
-	layer, err := store.Layers.AddLayer(archive, stderr, compression)
+// Import creates a new image from the contents of `archive` and registers it in the store as `name`.
+// If `parent` is not nil, it will registered as the parent of the new image.
+func (store *Store) Import(name string, archive io.Reader, parent *Image) (*Image, error) {
+	layer, err := store.Layers.AddLayer(archive)
 	if err != nil {
 		return nil, err
 	}

components/engine/image/layers.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"github.com/dotcloud/docker/future"
 )
 
@@ -82,50 +81,42 @@ func (store *LayerStore) layerPath(id string) string {
 }
 
 
-func (store *LayerStore) AddLayer(archive io.Reader, stderr io.Writer, compression Compression) (string, error) {
+func (store *LayerStore) AddLayer(archive io.Reader) (string, error) {
+	errors := make(chan error)
+	// Untar
 	tmp, err := store.Mktemp()
 	defer os.RemoveAll(tmp)
 	if err != nil {
 		return "", err
 	}
-	extractFlags := "-x"
-	if compression == Bzip2 {
-		extractFlags += "j"
-	} else if compression == Gzip {
-		extractFlags += "z"
-	}
-	untarCmd := exec.Command("tar", "-C", tmp, extractFlags)
-	untarW, err := untarCmd.StdinPipe()
-	if err != nil {
-		return "", err
-	}
-	untarStderr, err := untarCmd.StderrPipe()
-	if err != nil {
-		return "", err
-	}
-	go io.Copy(stderr, untarStderr)
-	untarStdout, err := untarCmd.StdoutPipe()
-	if err != nil {
-		return "", err
-	}
-	go io.Copy(stderr, untarStdout)
-	untarCmd.Start()
+	untarR, untarW := io.Pipe()
+	go func() {
+		errors <- Untar(untarR, tmp)
+	}()
+	// Compute ID
+	var id string
 	hashR, hashW := io.Pipe()
-	job_copy := future.Go(func() error {
-		_, err := io.Copy(io.MultiWriter(hashW, untarW), archive)
-		hashW.Close()
-		untarW.Close()
-		return err
-	})
-	id, err := future.ComputeId(hashR)
+	go func() {
+		_id, err := future.ComputeId(hashR)
+		id = _id
+		errors <- err
+	}()
+	// Duplicate archive to each stream
+	_, err = io.Copy(io.MultiWriter(hashW, untarW), archive)
+	hashW.Close()
+	untarW.Close()
 	if err != nil {
 		return "", err
 	}
-	if err := untarCmd.Wait(); err != nil {
-		return "", err
-	}
-	if err := <-job_copy; err != nil {
-		return "", err
+	// Wait for goroutines
+	for i:=0; i<2; i+=1 {
+		select {
+			case err := <-errors: {
+				if err != nil {
+					return "", err
+				}
+			}
+		}
 	}
 	layer := store.layerPath(id)
 	if !store.Exists(id) {