From 667532e025d128f628aa26e0e44efa58903906e6 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 15 Jun 2013 09:30:52 -0700 Subject: [PATCH 01/25] Removed deprecated file builder_client.go Upstream-commit: cc7de8df755d9e4a845a8aa5800424b1546492d6 Component: engine --- components/engine/builder_client.go | 314 ---------------------------- 1 file changed, 314 deletions(-) delete mode 100644 components/engine/builder_client.go diff --git a/components/engine/builder_client.go b/components/engine/builder_client.go deleted file mode 100644 index dc9528ff41..0000000000 --- a/components/engine/builder_client.go +++ /dev/null @@ -1,314 +0,0 @@ -package docker - -import ( - "bufio" - "encoding/json" - "fmt" - "github.com/dotcloud/docker/utils" - "io" - "net/url" - "os" - "reflect" - "strings" -) - -type builderClient struct { - cli *DockerCli - - image string - maintainer string - config *Config - - tmpContainers map[string]struct{} - tmpImages map[string]struct{} - - needCommit bool -} - -func (b *builderClient) clearTmp(containers, images map[string]struct{}) { - for i := range images { - if _, _, err := b.cli.call("DELETE", "/images/"+i, nil); err != nil { - utils.Debugf("%s", err) - } - utils.Debugf("Removing image %s", i) - } -} - -func (b *builderClient) CmdFrom(name string) error { - obj, statusCode, err := b.cli.call("GET", "/images/"+name+"/json", nil) - if statusCode == 404 { - - remote := name - var tag string - if strings.Contains(remote, ":") { - remoteParts := strings.Split(remote, ":") - tag = remoteParts[1] - remote = remoteParts[0] - } - var out io.Writer - if os.Getenv("DEBUG") != "" { - out = os.Stdout - } else { - out = &utils.NopWriter{} - } - if err := b.cli.stream("POST", "/images/create?fromImage="+remote+"&tag="+tag, nil, out); err != nil { - return err - } - obj, _, err = b.cli.call("GET", "/images/"+name+"/json", nil) - if err != nil { - return err - } - } - if err != nil { - return err - } - - img := &APIID{} - if err := json.Unmarshal(obj, img); err != nil { - return err - } - b.image = img.ID - utils.Debugf("Using image %s", b.image) - return nil -} - -func (b *builderClient) CmdMaintainer(name string) error { - b.needCommit = true - b.maintainer = name - return nil -} - -func (b *builderClient) CmdRun(args string) error { - if b.image == "" { - return fmt.Errorf("Please provide a source image with `from` prior to run") - } - config, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, nil) - if err != nil { - return err - } - - cmd, env := b.config.Cmd, b.config.Env - b.config.Cmd = nil - MergeConfig(b.config, config) - - body, statusCode, err := b.cli.call("POST", "/images/getCache", &APIImageConfig{ID: b.image, Config: b.config}) - if err != nil { - if statusCode != 404 { - return err - } - } - if statusCode != 404 { - apiID := &APIID{} - if err := json.Unmarshal(body, apiID); err != nil { - return err - } - utils.Debugf("Use cached version") - b.image = apiID.ID - return nil - } - cid, err := b.run() - if err != nil { - return err - } - b.config.Cmd, b.config.Env = cmd, env - return b.commit(cid) -} - -func (b *builderClient) CmdEnv(args string) error { - b.needCommit = true - tmp := strings.SplitN(args, " ", 2) - if len(tmp) != 2 { - return fmt.Errorf("Invalid ENV format") - } - key := strings.Trim(tmp[0], " ") - value := strings.Trim(tmp[1], " ") - - for i, elem := range b.config.Env { - if strings.HasPrefix(elem, key+"=") { - b.config.Env[i] = key + "=" + value - return nil - } - } - b.config.Env = append(b.config.Env, key+"="+value) - return nil -} - -func (b *builderClient) CmdCmd(args string) error { - b.needCommit = true - var cmd []string - if err := json.Unmarshal([]byte(args), &cmd); err != nil { - utils.Debugf("Error unmarshalling: %s, using /bin/sh -c", err) - b.config.Cmd = []string{"/bin/sh", "-c", args} - } else { - b.config.Cmd = cmd - } - return nil -} - -func (b *builderClient) CmdExpose(args string) error { - ports := strings.Split(args, " ") - b.config.PortSpecs = append(ports, b.config.PortSpecs...) - return nil -} - -func (b *builderClient) CmdInsert(args string) error { - // tmp := strings.SplitN(args, "\t ", 2) - // sourceUrl, destPath := tmp[0], tmp[1] - - // v := url.Values{} - // v.Set("url", sourceUrl) - // v.Set("path", destPath) - // body, _, err := b.cli.call("POST", "/images/insert?"+v.Encode(), nil) - // if err != nil { - // return err - // } - - // apiId := &APIId{} - // if err := json.Unmarshal(body, apiId); err != nil { - // return err - // } - - // FIXME: Reimplement this, we need to retrieve the resulting Id - return fmt.Errorf("INSERT not implemented") -} - -func (b *builderClient) run() (string, error) { - if b.image == "" { - return "", fmt.Errorf("Please provide a source image with `from` prior to run") - } - b.config.Image = b.image - body, _, err := b.cli.call("POST", "/containers/create", b.config) - if err != nil { - return "", err - } - - apiRun := &APIRun{} - if err := json.Unmarshal(body, apiRun); err != nil { - return "", err - } - for _, warning := range apiRun.Warnings { - fmt.Fprintln(os.Stderr, "WARNING: ", warning) - } - - //start the container - _, _, err = b.cli.call("POST", "/containers/"+apiRun.ID+"/start", nil) - if err != nil { - return "", err - } - b.tmpContainers[apiRun.ID] = struct{}{} - - // Wait for it to finish - body, _, err = b.cli.call("POST", "/containers/"+apiRun.ID+"/wait", nil) - if err != nil { - return "", err - } - apiWait := &APIWait{} - if err := json.Unmarshal(body, apiWait); err != nil { - return "", err - } - if apiWait.StatusCode != 0 { - return "", fmt.Errorf("The command %v returned a non-zero code: %d", b.config.Cmd, apiWait.StatusCode) - } - - return apiRun.ID, nil -} - -func (b *builderClient) commit(id string) error { - if b.image == "" { - return fmt.Errorf("Please provide a source image with `from` prior to run") - } - b.config.Image = b.image - - if id == "" { - cmd := b.config.Cmd - b.config.Cmd = []string{"true"} - cid, err := b.run() - if err != nil { - return err - } - id = cid - b.config.Cmd = cmd - } - - // Commit the container - v := url.Values{} - v.Set("container", id) - v.Set("author", b.maintainer) - - body, _, err := b.cli.call("POST", "/commit?"+v.Encode(), b.config) - if err != nil { - return err - } - apiID := &APIID{} - if err := json.Unmarshal(body, apiID); err != nil { - return err - } - b.tmpImages[apiID.ID] = struct{}{} - b.image = apiID.ID - b.needCommit = false - return nil -} - -func (b *builderClient) Build(dockerfile, context io.Reader) (string, error) { - defer b.clearTmp(b.tmpContainers, b.tmpImages) - file := bufio.NewReader(dockerfile) - for { - line, err := file.ReadString('\n') - if err != nil { - if err == io.EOF { - break - } - return "", err - } - line = strings.Replace(strings.TrimSpace(line), " ", " ", 1) - // Skip comments and empty line - if len(line) == 0 || line[0] == '#' { - continue - } - tmp := strings.SplitN(line, " ", 2) - if len(tmp) != 2 { - return "", fmt.Errorf("Invalid Dockerfile format") - } - instruction := strings.ToLower(strings.Trim(tmp[0], " ")) - arguments := strings.Trim(tmp[1], " ") - - fmt.Fprintf(os.Stderr, "%s %s (%s)\n", strings.ToUpper(instruction), arguments, b.image) - - method, exists := reflect.TypeOf(b).MethodByName("Cmd" + strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:])) - if !exists { - fmt.Fprintf(os.Stderr, "Skipping unknown instruction %s\n", strings.ToUpper(instruction)) - } - ret := method.Func.Call([]reflect.Value{reflect.ValueOf(b), reflect.ValueOf(arguments)})[0].Interface() - if ret != nil { - return "", ret.(error) - } - - fmt.Fprintf(os.Stderr, "===> %v\n", b.image) - } - if b.needCommit { - if err := b.commit(""); err != nil { - return "", err - } - } - if b.image != "" { - // The build is successful, keep the temporary containers and images - for i := range b.tmpImages { - delete(b.tmpImages, i) - } - for i := range b.tmpContainers { - delete(b.tmpContainers, i) - } - fmt.Fprintf(os.Stderr, "Build finished. image id: %s\n", b.image) - return b.image, nil - } - return "", fmt.Errorf("An error occured during the build\n") -} - -func NewBuilderClient(addr string, port int) BuildFile { - return &builderClient{ - cli: NewDockerCli(addr, port), - config: &Config{}, - tmpContainers: make(map[string]struct{}), - tmpImages: make(map[string]struct{}), - } -} From 4bba5db0acc470e650267b7a356773bfbac836c7 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 15 Jun 2013 09:38:18 -0700 Subject: [PATCH 02/25] * Builder: simplify the upload of the build context. Simply stream a tarball instead of multipart upload with 4 intermediary buffers. Simpler, less memory usage, less disk usage, and faster. Upstream-commit: 38554fc2a722d1a77514f477a189b0dbd149691b Component: engine --- components/engine/api.go | 21 +------ components/engine/buildfile.go | 29 ++++++---- components/engine/buildfile_test.go | 13 ++++- components/engine/commands.go | 88 ++++++++++++----------------- 4 files changed, 66 insertions(+), 85 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index e870ca7723..563888c6b1 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -13,7 +13,7 @@ import ( "strings" ) -const APIVERSION = 1.2 +const APIVERSION = 1.3 func hijackServer(w http.ResponseWriter) (io.ReadCloser, io.Writer, error) { conn, _, err := w.(http.Hijacker).Hijack() @@ -715,9 +715,7 @@ func postImagesGetCache(srv *Server, version float64, w http.ResponseWriter, r * } func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if err := r.ParseMultipartForm(4096); err != nil { - return err - } + // FIXME: "remote" is not a clear variable name. remote := r.FormValue("t") tag := "" if strings.Contains(remote, ":") { @@ -725,21 +723,8 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ tag = remoteParts[1] remote = remoteParts[0] } - - dockerfile, _, err := r.FormFile("Dockerfile") - if err != nil { - return err - } - - context, _, err := r.FormFile("Context") - if err != nil { - if err != http.ErrMissingFile { - return err - } - } - b := NewBuildFile(srv, utils.NewWriteFlusher(w)) - if id, err := b.Build(dockerfile, context); err != nil { + if id, err := b.Build(r.Body); err != nil { fmt.Fprintf(w, "Error build: %s\n", err) } else if remote != "" { srv.runtime.repositories.Set(remote, tag, id, false) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index aaa6f6bc7c..a4e6c7f725 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -14,7 +14,7 @@ import ( ) type BuildFile interface { - Build(io.Reader, io.Reader) (string, error) + Build(io.Reader) (string, error) CmdFrom(string) error CmdRun(string) error } @@ -305,18 +305,23 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { return nil } -func (b *buildFile) Build(dockerfile, context io.Reader) (string, error) { - if context != nil { - name, err := ioutil.TempDir("/tmp", "docker-build") - if err != nil { - return "", err - } - if err := Untar(context, name); err != nil { - return "", err - } - defer os.RemoveAll(name) - b.context = name +func (b *buildFile) Build(context io.Reader) (string, error) { + // FIXME: @creack any reason for using /tmp instead of ""? + // FIXME: @creack "name" is a terrible variable name + name, err := ioutil.TempDir("/tmp", "docker-build") + if err != nil { + return "", err } + if err := Untar(context, name); err != nil { + return "", err + } + defer os.RemoveAll(name) + b.context = name + dockerfile, err := os.Open(path.Join(name, "Dockerfile")) + if err != nil { + return "", fmt.Errorf("Can't build a directory with no Dockerfile") + } + // FIXME: "file" is also a terrible variable name ;) file := bufio.NewReader(dockerfile) for { line, err := file.ReadString('\n') diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index d9c60a70d5..0432fda3e6 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -2,7 +2,6 @@ package docker import ( "github.com/dotcloud/docker/utils" - "strings" "testing" ) @@ -23,6 +22,16 @@ from ` + unitTestImageName + ` run sh -c 'echo root:testpass > /tmp/passwd' run mkdir -p /var/run/sshd` +// mkTestContext generates a build context from the contents of the provided dockerfile. +// This context is suitable for use as an argument to BuildFile.Build() +func mkTestContext(dockerfile string, t *testing.T) Archive { + context, err := mkBuildContext(dockerfile) + if err != nil { + t.Fatal(err) + } + return context +} + func TestBuild(t *testing.T) { dockerfiles := []string{Dockerfile, DockerfileNoNewLine} for _, Dockerfile := range dockerfiles { @@ -36,7 +45,7 @@ func TestBuild(t *testing.T) { buildfile := NewBuildFile(srv, &utils.NopWriter{}) - imgID, err := buildfile.Build(strings.NewReader(Dockerfile), nil) + imgID, err := buildfile.Build(mkTestContext(Dockerfile, t)) if err != nil { t.Fatal(err) } diff --git a/components/engine/commands.go b/components/engine/commands.go index 04eb994f01..4441536a95 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1,6 +1,7 @@ package docker import ( + "archive/tar" "bytes" "encoding/json" "flag" @@ -10,14 +11,12 @@ import ( "github.com/dotcloud/docker/utils" "io" "io/ioutil" - "mime/multipart" "net" "net/http" "net/http/httputil" "net/url" "os" "os/signal" - "path" "path/filepath" "reflect" "regexp" @@ -131,6 +130,27 @@ func (cli *DockerCli) CmdInsert(args ...string) error { return nil } +// mkBuildContext returns an archive of an empty context with the contents +// of `dockerfile` at the path ./Dockerfile +func mkBuildContext(content string) (Archive, error) { + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + hdr := &tar.Header{ + Name: "Dockerfile", + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + if _, err := tw.Write([]byte(content)); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, err + } + return buf, nil +} + func (cli *DockerCli) CmdBuild(args ...string) error { cmd := Subcmd("build", "[OPTIONS] PATH | -", "Build a new container image from the source code at PATH") tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success") @@ -143,70 +163,32 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } var ( - multipartBody io.Reader - file io.ReadCloser - contextPath string + context Archive + err error ) - // Init the needed component for the Multipart - buff := bytes.NewBuffer([]byte{}) - multipartBody = buff - w := multipart.NewWriter(buff) - boundary := strings.NewReader("\r\n--" + w.Boundary() + "--\r\n") - - compression := Bzip2 - if cmd.Arg(0) == "-" { - file = os.Stdin + // As a special case, 'docker build -' will build from an empty context with the + // contents of stdin as a Dockerfile + dockerfile, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return err + } + context, err = mkBuildContext(string(dockerfile)) } else { - // Send Dockerfile from arg/Dockerfile (deprecate later) - f, err := os.Open(path.Join(cmd.Arg(0), "Dockerfile")) - if err != nil { - return err - } - file = f - // Send context from arg - // Create a FormFile multipart for the context if needed - // FIXME: Use NewTempArchive in order to have the size and avoid too much memory usage? - context, err := Tar(cmd.Arg(0), compression) - if err != nil { - return err - } - // NOTE: Do this in case '.' or '..' is input - absPath, err := filepath.Abs(cmd.Arg(0)) - if err != nil { - return err - } - wField, err := w.CreateFormFile("Context", filepath.Base(absPath)+"."+compression.Extension()) - if err != nil { - return err - } - // FIXME: Find a way to have a progressbar for the upload too - sf := utils.NewStreamFormatter(false) - io.Copy(wField, utils.ProgressReader(ioutil.NopCloser(context), -1, os.Stdout, sf.FormatProgress("Caching Context", "%v/%v (%v)"), sf)) - multipartBody = io.MultiReader(multipartBody, boundary) + context, err = Tar(cmd.Arg(0), Uncompressed) } - // Create a FormFile multipart for the Dockerfile - wField, err := w.CreateFormFile("Dockerfile", "Dockerfile") if err != nil { return err } - io.Copy(wField, file) - multipartBody = io.MultiReader(multipartBody, boundary) - + // Upload the build context v := &url.Values{} v.Set("t", *tag) - // Send the multipart request with correct content-type - req, err := http.NewRequest("POST", fmt.Sprintf("http://%s:%d%s?%s", cli.host, cli.port, "/build", v.Encode()), multipartBody) + req, err := http.NewRequest("POST", fmt.Sprintf("http://%s:%d%s?%s", cli.host, cli.port, "/build", v.Encode()), context) if err != nil { return err } - req.Header.Set("Content-Type", w.FormDataContentType()) - if contextPath != "" { - req.Header.Set("X-Docker-Context-Compression", compression.Flag()) - fmt.Println("Uploading Context...") - } - + req.Header.Set("Content-Type", "application/tar") resp, err := http.DefaultClient.Do(req) if err != nil { return err From 89a21cc6a641cf3041f883420b744e5a28e24618 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 15 Jun 2013 11:07:49 -0700 Subject: [PATCH 03/25] * Builder: reorganized unit tests for better code reuse, and to test non-empty contexts Upstream-commit: 061f8d12e04e494fdcbee949facaf598d201c2ec Component: engine --- components/engine/buildfile_test.go | 136 ++++++++++++++++------------ components/engine/commands.go | 26 +++--- 2 files changed, 92 insertions(+), 70 deletions(-) diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 0432fda3e6..383463ce24 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -1,40 +1,75 @@ package docker import ( - "github.com/dotcloud/docker/utils" + "io/ioutil" "testing" ) -const Dockerfile = ` -# VERSION 0.1 -# DOCKER-VERSION 0.2 - -from ` + unitTestImageName + ` -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd -` - -const DockerfileNoNewLine = ` -# VERSION 0.1 -# DOCKER-VERSION 0.2 - -from ` + unitTestImageName + ` -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd` - // mkTestContext generates a build context from the contents of the provided dockerfile. // This context is suitable for use as an argument to BuildFile.Build() -func mkTestContext(dockerfile string, t *testing.T) Archive { - context, err := mkBuildContext(dockerfile) +func mkTestContext(dockerfile string, files [][2]string, t *testing.T) Archive { + context, err := mkBuildContext(dockerfile, files) if err != nil { t.Fatal(err) } return context } +// A testContextTemplate describes a build context and how to test it +type testContextTemplate struct { + // Contents of the Dockerfile + dockerfile string + // Additional files in the context, eg [][2]string{"./passwd", "gordon"} + files [][2]string + // Test commands to run in the resulting image + tests []testCommand +} + +// A testCommand describes a command to run in a container, and the exact output required to pass the test +type testCommand struct { + // The command to run, eg. []string{"echo", "hello", "world"} + cmd []string + // The exact output expected, eg. "hello world\n" + output string +} + +// A table of all the contexts to build and test. +// A new docker runtime will be created and torn down for each context. +var testContexts []testContextTemplate = []testContextTemplate{ + { + ` +# VERSION 0.1 +# DOCKER-VERSION 0.2 + +from docker-ut +run sh -c 'echo root:testpass > /tmp/passwd' +run mkdir -p /var/run/sshd +`, + nil, + []testCommand{ + {[]string{"cat", "/tmp/passwd"}, "root:testpass\n"}, + {[]string{"ls", "-d", "/var/run/sshd"}, "/var/run/sshd\n"}, + }, + }, + + { + ` +# VERSION 0.1 +# DOCKER-VERSION 0.2 + +from docker-ut +run sh -c 'echo root:testpass > /tmp/passwd' +run mkdir -p /var/run/sshd`, + nil, + []testCommand{ + {[]string{"cat", "/tmp/passwd"}, "root:testpass\n"}, + {[]string{"ls", "-d", "/var/run/sshd"}, "/var/run/sshd\n"}, + }, + }, +} + func TestBuild(t *testing.T) { - dockerfiles := []string{Dockerfile, DockerfileNoNewLine} - for _, Dockerfile := range dockerfiles { + for _, ctx := range testContexts { runtime, err := newTestRuntime() if err != nil { t.Fatal(err) @@ -43,50 +78,33 @@ func TestBuild(t *testing.T) { srv := &Server{runtime: runtime} - buildfile := NewBuildFile(srv, &utils.NopWriter{}) + buildfile := NewBuildFile(srv, ioutil.Discard) - imgID, err := buildfile.Build(mkTestContext(Dockerfile, t)) + imgID, err := buildfile.Build(mkTestContext(ctx.dockerfile, ctx.files, t)) if err != nil { t.Fatal(err) } builder := NewBuilder(runtime) - container, err := builder.Create( - &Config{ - Image: imgID, - Cmd: []string{"cat", "/tmp/passwd"}, - }, - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container) + for _, testCmd := range ctx.tests { + container, err := builder.Create( + &Config{ + Image: imgID, + Cmd: testCmd.cmd, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) - output, err := container.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "root:testpass\n" { - t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, "root:testpass\n") - } - - container2, err := builder.Create( - &Config{ - Image: imgID, - Cmd: []string{"ls", "-d", "/var/run/sshd"}, - }, - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container2) - - output, err = container2.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "/var/run/sshd\n" { - t.Fatal("/var/run/sshd has not been created") + output, err := container.Output() + if err != nil { + t.Fatal(err) + } + if string(output) != testCmd.output { + t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, testCmd.output) + } } } } diff --git a/components/engine/commands.go b/components/engine/commands.go index 4441536a95..274ea6fae5 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -132,18 +132,22 @@ func (cli *DockerCli) CmdInsert(args ...string) error { // mkBuildContext returns an archive of an empty context with the contents // of `dockerfile` at the path ./Dockerfile -func mkBuildContext(content string) (Archive, error) { +func mkBuildContext(dockerfile string, files [][2]string) (Archive, error) { buf := new(bytes.Buffer) tw := tar.NewWriter(buf) - hdr := &tar.Header{ - Name: "Dockerfile", - Size: int64(len(content)), - } - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - if _, err := tw.Write([]byte(content)); err != nil { - return nil, err + files = append(files, [2]string{"Dockerfile", dockerfile}) + for _, file := range files { + name, content := file[0], file[1] + hdr := &tar.Header{ + Name: name, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + if _, err := tw.Write([]byte(content)); err != nil { + return nil, err + } } if err := tw.Close(); err != nil { return nil, err @@ -174,7 +178,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if err != nil { return err } - context, err = mkBuildContext(string(dockerfile)) + context, err = mkBuildContext(string(dockerfile), nil) } else { context, err = Tar(cmd.Arg(0), Uncompressed) } From 39f9189f0289220c3fd5955d2cfb822db42bcd85 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 15 Jun 2013 11:35:56 -0700 Subject: [PATCH 04/25] * Builder: added a regression test for #895 Upstream-commit: f50e40008f39c77cab2b516ac2102f875bdef215 Component: engine --- components/engine/buildfile_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 383463ce24..3b03b8dd04 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -66,6 +66,16 @@ run mkdir -p /var/run/sshd`, {[]string{"ls", "-d", "/var/run/sshd"}, "/var/run/sshd\n"}, }, }, + + { + ` +from docker-ut +add foo /usr/lib/bla/bar`, + [][2]string{{"foo", "hello world!"}}, + []testCommand{ + {[]string{"cat", "/usr/lib/bla/bar"}, "hello world!"}, + }, + }, } func TestBuild(t *testing.T) { From 5daee8b0f613a675ca2f8a45cd5f6d40b5636e33 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 17 Jun 2013 18:13:40 +0000 Subject: [PATCH 05/25] use go 1.1 cookiejar and revome ResetClient Upstream-commit: fde82f448f6b1508e03f419a3ce4c9b80311d466 Component: engine --- components/engine/registry/registry.go | 15 +++++---------- components/engine/server.go | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 131b02708e..21979fad8b 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -7,10 +7,10 @@ import ( "fmt" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/utils" - "github.com/shin-/cookiejar" "io" "io/ioutil" "net/http" + "net/http/cookiejar" "net/url" "strings" ) @@ -438,11 +438,6 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { return result, err } -func (r *Registry) ResetClient(authConfig *auth.AuthConfig) { - r.authConfig = authConfig - r.client.Jar = cookiejar.NewCookieJar() -} - func (r *Registry) GetAuthConfig(withPasswd bool) *auth.AuthConfig { password := "" if withPasswd { @@ -478,18 +473,18 @@ type Registry struct { authConfig *auth.AuthConfig } -func NewRegistry(root string, authConfig *auth.AuthConfig) *Registry { +func NewRegistry(root string, authConfig *auth.AuthConfig) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, } - r := &Registry{ + r = &Registry{ authConfig: authConfig, client: &http.Client{ Transport: httpTransport, }, } - r.client.Jar = cookiejar.NewCookieJar() - return r + r.client.Jar, err = cookiejar.New(nil) + return r, err } diff --git a/components/engine/server.go b/components/engine/server.go index 30e3ec6b3a..7b7eabe321 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -54,8 +54,11 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { } func (srv *Server) ImagesSearch(term string) ([]APISearch, error) { - - results, err := registry.NewRegistry(srv.runtime.root, nil).SearchRepositories(term) + r, err := registry.NewRegistry(srv.runtime.root, nil) + if err != nil { + return nil, err + } + results, err := r.SearchRepositories(term) if err != nil { return nil, err } @@ -402,7 +405,10 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re } func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { - r := registry.NewRegistry(srv.runtime.root, authConfig) + r, err := registry.NewRegistry(srv.runtime.root, authConfig) + if err != nil { + return err + } out = utils.NewWriteFlusher(out) if endpoint != "" { if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { @@ -596,8 +602,10 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId, func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { out = utils.NewWriteFlusher(out) img, err := srv.runtime.graph.Get(name) - r := registry.NewRegistry(srv.runtime.root, authConfig) - + r, err2 := registry.NewRegistry(srv.runtime.root, authConfig) + if err2 != nil { + return err2 + } if err != nil { out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name]))) // If it fails, try to get the repository From fb46f44c43548dfcf39c6be246b8d22d1f3eaf1c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 17 Jun 2013 18:26:41 -0700 Subject: [PATCH 06/25] * Builder: upload progress bar Fix progress bar Upstream-commit: 0809f649d356b6b8d34a03bee8447049161ea09c Component: engine --- components/engine/FIXME | 1 + components/engine/commands.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/components/engine/FIXME b/components/engine/FIXME index b88757e229..c0bbd7e482 100644 --- a/components/engine/FIXME +++ b/components/engine/FIXME @@ -34,3 +34,4 @@ to put them - so we put them here :) * Caching after an ADD * entry point config * bring back git revision info, looks like it was lost +* Clean up the ProgressReader api, it's a PITA to use diff --git a/components/engine/commands.go b/components/engine/commands.go index 274ea6fae5..39cf749ef8 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -185,10 +185,14 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if err != nil { return err } + // Setup an upload progress bar + // FIXME: ProgressReader shouldn't be this annoyning to use + sf := utils.NewStreamFormatter(false) + body := utils.ProgressReader(ioutil.NopCloser(context), 0, os.Stderr, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s"), sf) // Upload the build context v := &url.Values{} v.Set("t", *tag) - req, err := http.NewRequest("POST", fmt.Sprintf("http://%s:%d%s?%s", cli.host, cli.port, "/build", v.Encode()), context) + req, err := http.NewRequest("POST", fmt.Sprintf("http://%s:%d%s?%s", cli.host, cli.port, "/build", v.Encode()), body) if err != nil { return err } From 11334d506a9794a5fc454c385df130e4ff38085e Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 18 Jun 2013 18:59:56 +0000 Subject: [PATCH 07/25] add basic support for unix sockets Upstream-commit: 3adf9ce04ef1632f82f7bd1585c135bb141aa450 Component: engine --- components/engine/api.go | 19 +++++-- components/engine/builder_client.go | 4 +- components/engine/commands.go | 47 +++++++++++------ components/engine/docker/docker.go | 82 ++++++++++++++++++++--------- components/engine/utils/utils.go | 2 + 5 files changed, 109 insertions(+), 45 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index e870ca7723..0b89fcf323 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -8,12 +8,16 @@ import ( "github.com/gorilla/mux" "io" "log" + "net" "net/http" + "os" "strconv" "strings" ) const APIVERSION = 1.2 +const DEFAULTHTTPHOST string = "127.0.0.1" +const DEFAULTHTTPPORT int = 4243 func hijackServer(w http.ResponseWriter) (io.ReadCloser, io.Writer, error) { conn, _, err := w.(http.Hijacker).Hijack() @@ -848,12 +852,21 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) { return r, nil } -func ListenAndServe(addr string, srv *Server, logging bool) error { - log.Printf("Listening for HTTP on %s\n", addr) +func ListenAndServe(proto, addr string, srv *Server, logging bool) error { + log.Printf("Listening for HTTP on %s (%s)\n", addr, proto) r, err := createRouter(srv, logging) if err != nil { return err } - return http.ListenAndServe(addr, r) + l, e := net.Listen(proto, addr) + if e != nil { + return e + } + //as the daemon is launched as root, change to permission of the socket to allow non-root to connect + if proto == "unix" { + os.Chmod(addr, 0777) + } + httpSrv := http.Server{Addr: addr, Handler: r} + return httpSrv.Serve(l) } diff --git a/components/engine/builder_client.go b/components/engine/builder_client.go index dc9528ff41..d11e7fc995 100644 --- a/components/engine/builder_client.go +++ b/components/engine/builder_client.go @@ -304,9 +304,9 @@ func (b *builderClient) Build(dockerfile, context io.Reader) (string, error) { return "", fmt.Errorf("An error occured during the build\n") } -func NewBuilderClient(addr string, port int) BuildFile { +func NewBuilderClient(proto, addr string) BuildFile { return &builderClient{ - cli: NewDockerCli(addr, port), + cli: NewDockerCli(proto, addr), config: &Config{}, tmpContainers: make(map[string]struct{}), tmpImages: make(map[string]struct{}), diff --git a/components/engine/commands.go b/components/engine/commands.go index cf23a8d65e..2c33485ecb 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -40,8 +40,8 @@ func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) { return reflect.TypeOf(cli).MethodByName(methodName) } -func ParseCommands(addr string, port int, args ...string) error { - cli := NewDockerCli(addr, port) +func ParseCommands(proto, addr string, args ...string) error { + cli := NewDockerCli(proto, addr) if len(args) > 0 { method, exists := cli.getMethod(args[0]) @@ -74,7 +74,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error { return nil } } - help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=\"%s:%d\": Host:port to bind/connect to\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", cli.host, cli.port) + help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=\"%s:%d\": Host:port to bind/connect to\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTHTTPHOST, DEFAULTHTTPPORT) for _, command := range [][2]string{ {"attach", "Attach to a running container"}, {"build", "Build a container from a Dockerfile"}, @@ -197,7 +197,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { v := &url.Values{} v.Set("t", *tag) // Send the multipart request with correct content-type - req, err := http.NewRequest("POST", fmt.Sprintf("http://%s:%d%s?%s", cli.host, cli.port, "/build", v.Encode()), multipartBody) + req, err := http.NewRequest("POST", fmt.Sprintf("/v%s/build?%s", APIVERSION, v.Encode()), multipartBody) if err != nil { return err } @@ -206,8 +206,13 @@ func (cli *DockerCli) CmdBuild(args ...string) error { req.Header.Set("X-Docker-Context-Compression", compression.Flag()) fmt.Println("Uploading Context...") } - - resp, err := http.DefaultClient.Do(req) + dial, err := net.Dial(cli.proto, cli.addr) + if err != nil { + return err + } + clientconn := httputil.NewClientConn(dial, nil) + resp, err := clientconn.Do(req) + defer clientconn.Close() if err != nil { return err } @@ -1339,7 +1344,7 @@ func (cli *DockerCli) call(method, path string, data interface{}) ([]byte, int, params = bytes.NewBuffer(buf) } - req, err := http.NewRequest(method, fmt.Sprintf("http://%s:%d/v%g%s", cli.host, cli.port, APIVERSION, path), params) + req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), params) if err != nil { return nil, -1, err } @@ -1349,7 +1354,13 @@ func (cli *DockerCli) call(method, path string, data interface{}) ([]byte, int, } else if method == "POST" { req.Header.Set("Content-Type", "plain/text") } - resp, err := http.DefaultClient.Do(req) + dial, err := net.Dial(cli.proto, cli.addr) + if err != nil { + return nil, -1, err + } + clientconn := httputil.NewClientConn(dial, nil) + resp, err := clientconn.Do(req) + defer clientconn.Close() if err != nil { if strings.Contains(err.Error(), "connection refused") { return nil, -1, fmt.Errorf("Can't connect to docker daemon. Is 'docker -d' running on this host?") @@ -1374,7 +1385,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e if (method == "POST" || method == "PUT") && in == nil { in = bytes.NewReader([]byte{}) } - req, err := http.NewRequest(method, fmt.Sprintf("http://%s:%d/v%g%s", cli.host, cli.port, APIVERSION, path), in) + req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), in) if err != nil { return err } @@ -1382,7 +1393,13 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e if method == "POST" { req.Header.Set("Content-Type", "plain/text") } - resp, err := http.DefaultClient.Do(req) + dial, err := net.Dial(cli.proto, cli.addr) + if err != nil { + return err + } + clientconn := httputil.NewClientConn(dial, nil) + resp, err := clientconn.Do(req) + defer clientconn.Close() if err != nil { if strings.Contains(err.Error(), "connection refused") { return fmt.Errorf("Can't connect to docker daemon. Is 'docker -d' running on this host?") @@ -1432,7 +1449,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.Fi return err } req.Header.Set("Content-Type", "plain/text") - dial, err := net.Dial("tcp", fmt.Sprintf("%s:%d", cli.host, cli.port)) + dial, err := net.Dial(cli.proto, cli.addr) if err != nil { return err } @@ -1515,13 +1532,13 @@ func Subcmd(name, signature, description string) *flag.FlagSet { return flags } -func NewDockerCli(addr string, port int) *DockerCli { +func NewDockerCli(proto, addr string) *DockerCli { authConfig, _ := auth.LoadConfig(os.Getenv("HOME")) - return &DockerCli{addr, port, authConfig} + return &DockerCli{proto, addr, authConfig} } type DockerCli struct { - host string - port int + proto string + addr string authConfig *auth.AuthConfig } diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 2e23999ad8..85c78c89da 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -24,15 +24,13 @@ func main() { docker.SysInit() return } - host := "127.0.0.1" - port := 4243 // FIXME: Switch d and D ? (to be more sshd like) flDaemon := flag.Bool("d", false, "Daemon mode") flDebug := flag.Bool("D", false, "Debug mode") flAutoRestart := flag.Bool("r", false, "Restart previously running containers") bridgeName := flag.String("b", "", "Attach containers to a pre-existing network bridge") pidfile := flag.String("p", "/var/run/docker.pid", "File containing process PID") - flHost := flag.String("H", fmt.Sprintf("%s:%d", host, port), "Host:port to bind/connect to") + flHost := flag.String("H", fmt.Sprintf("%s:%d", docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT), "Host:port to bind/connect to") flEnableCors := flag.Bool("api-enable-cors", false, "Enable CORS requests in the remote api.") flDns := flag.String("dns", "", "Set custom dns servers") flag.Parse() @@ -42,21 +40,8 @@ func main() { docker.NetworkBridgeIface = docker.DefaultNetworkBridge } - if strings.Contains(*flHost, ":") { - hostParts := strings.Split(*flHost, ":") - if len(hostParts) != 2 { - log.Fatal("Invalid bind address format.") - os.Exit(-1) - } - if hostParts[0] != "" { - host = hostParts[0] - } - if p, err := strconv.Atoi(hostParts[1]); err == nil { - port = p - } - } else { - host = *flHost - } + protoAddr := parseHost(*flHost) + protoAddrs := []string{protoAddr} if *flDebug { os.Setenv("DEBUG", "1") @@ -67,12 +52,13 @@ func main() { flag.Usage() return } - if err := daemon(*pidfile, host, port, *flAutoRestart, *flEnableCors, *flDns); err != nil { + if err := daemon(*pidfile, protoAddrs, *flAutoRestart, *flEnableCors, *flDns); err != nil { log.Fatal(err) os.Exit(-1) } } else { - if err := docker.ParseCommands(host, port, flag.Args()...); err != nil { + protoAddrParts := strings.SplitN(protoAddrs[0], "://", 2) + if err := docker.ParseCommands(protoAddrParts[0], protoAddrParts[1], flag.Args()...); err != nil { log.Fatal(err) os.Exit(-1) } @@ -106,10 +92,7 @@ func removePidFile(pidfile string) { } } -func daemon(pidfile, addr string, port int, autoRestart, enableCors bool, flDns string) error { - if addr != "127.0.0.1" { - log.Println("/!\\ DON'T BIND ON ANOTHER IP ADDRESS THAN 127.0.0.1 IF YOU DON'T KNOW WHAT YOU'RE DOING /!\\") - } +func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, flDns string) error { if err := createPidFile(pidfile); err != nil { log.Fatal(err) } @@ -131,6 +114,55 @@ func daemon(pidfile, addr string, port int, autoRestart, enableCors bool, flDns if err != nil { return err } + chErrors := make(chan error, len(protoAddrs)) + for _, protoAddr := range protoAddrs { + protoAddrParts := strings.SplitN(protoAddr, "://", 2) + if protoAddrParts[0] == "unix" { + syscall.Unlink(protoAddrParts[1]); + } else if protoAddrParts[0] == "tcp" { + if !strings.HasPrefix(protoAddrParts[1], "127.0.0.1") { + log.Println("/!\\ DON'T BIND ON ANOTHER IP ADDRESS THAN 127.0.0.1 IF YOU DON'T KNOW WHAT YOU'RE DOING /!\\") + } + } else { + log.Fatal("Invalid protocol format.") + os.Exit(-1) + } + go func() { + chErrors <- docker.ListenAndServe(protoAddrParts[0], protoAddrParts[1], server, true) + }() + } + for i :=0 ; i < len(protoAddrs); i+=1 { + err := <-chErrors + if err != nil { + return err + } + } + return nil +} - return docker.ListenAndServe(fmt.Sprintf("%s:%d", addr, port), server, true) +func parseHost(addr string) string { + if strings.HasPrefix(addr, "unix://") { + return addr + } + host := docker.DEFAULTHTTPHOST + port := docker.DEFAULTHTTPPORT + if strings.HasPrefix(addr, "tcp://") { + addr = strings.TrimPrefix(addr, "tcp://") + } + if strings.Contains(addr, ":") { + hostParts := strings.Split(addr, ":") + if len(hostParts) != 2 { + log.Fatal("Invalid bind address format.") + os.Exit(-1) + } + if hostParts[0] != "" { + host = hostParts[0] + } + if p, err := strconv.Atoi(hostParts[1]); err == nil { + port = p + } + } else { + host = addr + } + return fmt.Sprintf("tcp://%s:%d", host, port) } diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index b77c1ea053..da848c45bc 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -652,3 +652,5 @@ func CheckLocalDns() bool { } return false } + + From cbe318e16599c87cdb3eeab8c750e068aefc5f48 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 19 Jun 2013 12:31:54 +0000 Subject: [PATCH 08/25] add the possibility to use multiple -H Upstream-commit: dede1585ee00f957e153691c464aab293c2dc469 Component: engine --- components/engine/commands.go | 2 +- components/engine/docker/docker.go | 48 ++++++++++-------------------- components/engine/utils/utils.go | 26 ++++++++++++++++ 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 2c33485ecb..84af648f13 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -74,7 +74,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error { return nil } } - help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=\"%s:%d\": Host:port to bind/connect to\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTHTTPHOST, DEFAULTHTTPPORT) + help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=[tcp://%s:%d]: tcp://host:port to bind/connect to or unix://path/to/socker to use\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTHTTPHOST, DEFAULTHTTPPORT) for _, command := range [][2]string{ {"attach", "Attach to a running container"}, {"build", "Build a container from a Dockerfile"}, diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 85c78c89da..6d79972bd6 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -30,19 +30,23 @@ func main() { flAutoRestart := flag.Bool("r", false, "Restart previously running containers") bridgeName := flag.String("b", "", "Attach containers to a pre-existing network bridge") pidfile := flag.String("p", "/var/run/docker.pid", "File containing process PID") - flHost := flag.String("H", fmt.Sprintf("%s:%d", docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT), "Host:port to bind/connect to") flEnableCors := flag.Bool("api-enable-cors", false, "Enable CORS requests in the remote api.") flDns := flag.String("dns", "", "Set custom dns servers") + flHosts := docker.ListOpts{fmt.Sprintf("tcp://%s:%d", docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT)} + flag.Var(&flHosts, "H", "tcp://host:port to bind/connect to or unix://path/to/socket to use") flag.Parse() + if len(flHosts) > 1 { + flHosts = flHosts[1:len(flHosts)] //trick to display a nice defaul value in the usage + } + for i, flHost := range flHosts { + flHosts[i] = utils.ParseHost(docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT, flHost) + } + if *bridgeName != "" { docker.NetworkBridgeIface = *bridgeName } else { docker.NetworkBridgeIface = docker.DefaultNetworkBridge } - - protoAddr := parseHost(*flHost) - protoAddrs := []string{protoAddr} - if *flDebug { os.Setenv("DEBUG", "1") } @@ -52,12 +56,16 @@ func main() { flag.Usage() return } - if err := daemon(*pidfile, protoAddrs, *flAutoRestart, *flEnableCors, *flDns); err != nil { + if err := daemon(*pidfile, flHosts, *flAutoRestart, *flEnableCors, *flDns); err != nil { log.Fatal(err) os.Exit(-1) } } else { - protoAddrParts := strings.SplitN(protoAddrs[0], "://", 2) + if len(flHosts) > 1 { + log.Fatal("Please specify only one -H") + return + } + protoAddrParts := strings.SplitN(flHosts[0], "://", 2) if err := docker.ParseCommands(protoAddrParts[0], protoAddrParts[1], flag.Args()...); err != nil { log.Fatal(err) os.Exit(-1) @@ -140,29 +148,3 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f return nil } -func parseHost(addr string) string { - if strings.HasPrefix(addr, "unix://") { - return addr - } - host := docker.DEFAULTHTTPHOST - port := docker.DEFAULTHTTPPORT - if strings.HasPrefix(addr, "tcp://") { - addr = strings.TrimPrefix(addr, "tcp://") - } - if strings.Contains(addr, ":") { - hostParts := strings.Split(addr, ":") - if len(hostParts) != 2 { - log.Fatal("Invalid bind address format.") - os.Exit(-1) - } - if hostParts[0] != "" { - host = hostParts[0] - } - if p, err := strconv.Atoi(hostParts[1]); err == nil { - port = p - } - } else { - host = addr - } - return fmt.Sprintf("tcp://%s:%d", host, port) -} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index da848c45bc..37fda5c1c8 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -10,6 +10,7 @@ import ( "index/suffixarray" "io" "io/ioutil" + "log" "net/http" "os" "os/exec" @@ -653,4 +654,29 @@ func CheckLocalDns() bool { return false } +func ParseHost(host string, port int, addr string) string { + if strings.HasPrefix(addr, "unix://") { + return addr + } + if strings.HasPrefix(addr, "tcp://") { + addr = strings.TrimPrefix(addr, "tcp://") + } + if strings.Contains(addr, ":") { + hostParts := strings.Split(addr, ":") + if len(hostParts) != 2 { + log.Fatal("Invalid bind address format.") + os.Exit(-1) + } + if hostParts[0] != "" { + host = hostParts[0] + } + if p, err := strconv.Atoi(hostParts[1]); err == nil { + port = p + } + } else { + host = addr + } + return fmt.Sprintf("tcp://%s:%d", host, port) +} + From 8ec498cc12fab03231787ce0b662bef23d8e083b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 19 Jun 2013 12:40:01 +0000 Subject: [PATCH 09/25] add tests Upstream-commit: 9632bf228744dddfae2c303a0f2e8b962b6e0673 Component: engine --- components/engine/utils/utils_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index eec06d5134..623f08e383 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -274,3 +274,21 @@ func TestHumanSize(t *testing.T) { t.Errorf("1024 -> expected 1.024 kB, got %s", size1024) } } + +func TestParseHost(t *testing.T) { + if addr := ParseHost("127.0.0.1", 4243, "0.0.0.0"); addr != "tcp://0.0.0.0:4243" { + t.Errorf("0.0.0.0 -> expected tcp://0.0.0.0:4243, got %s", addr) + } + if addr := ParseHost("127.0.0.1", 4243, "0.0.0.1:5555"); addr != "tcp://0.0.0.1:5555" { + t.Errorf("0.0.0.1:5555 -> expected tcp://0.0.0.1:5555, got %s", addr) + } + if addr := ParseHost("127.0.0.1", 4243, ":6666"); addr != "tcp://127.0.0.1:6666" { + t.Errorf(":6666 -> expected tcp://127.0.0.1:6666, got %s", addr) + } + if addr := ParseHost("127.0.0.1", 4243, "tcp://:7777"); addr != "tcp://127.0.0.1:7777" { + t.Errorf("tcp://:7777 -> expected tcp://127.0.0.1:7777, got %s", addr) + } + if addr := ParseHost("127.0.0.1", 4243, "unix:///var/run/docker.sock"); addr != "unix:///var/run/docker.sock" { + t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) + } +} From bdd6ad8a7a28c998b87e26e6c6b0beb32857e297 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 19 Jun 2013 12:48:50 +0000 Subject: [PATCH 10/25] update docs Upstream-commit: 063c838c927615e9507eca2a5c2f38a5fc4c35b2 Component: engine --- .../sources/api/docker_remote_api_v1.2.rst | 2 +- .../engine/docs/sources/commandline/cli.rst | 2 +- components/engine/docs/sources/use/basics.rst | 22 ++++++++++++++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.2.rst b/components/engine/docs/sources/api/docker_remote_api_v1.2.rst index fb69168120..a34ee01e0e 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.2.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.2.rst @@ -1026,5 +1026,5 @@ In this version of the API, /attach, uses hijacking to transport stdin, stdout a To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. - docker -d -H="192.168.1.9:4243" -api-enable-cors + docker -d -H="tcp://192.168.1.9:4243" -api-enable-cors diff --git a/components/engine/docs/sources/commandline/cli.rst b/components/engine/docs/sources/commandline/cli.rst index 02691b4f56..118f42f6e8 100644 --- a/components/engine/docs/sources/commandline/cli.rst +++ b/components/engine/docs/sources/commandline/cli.rst @@ -15,7 +15,7 @@ To list available commands, either run ``docker`` with no parameters or execute $ docker Usage: docker [OPTIONS] COMMAND [arg...] - -H="127.0.0.1:4243": Host:port to bind/connect to + -H=[tcp://127.0.0.1:4243]: tcp://host:port to bind/connect to or unix://path/to/socket to use A self-sufficient runtime for linux containers. diff --git a/components/engine/docs/sources/use/basics.rst b/components/engine/docs/sources/use/basics.rst index 444b74db51..0f64ec4cf8 100644 --- a/components/engine/docs/sources/use/basics.rst +++ b/components/engine/docs/sources/use/basics.rst @@ -33,11 +33,16 @@ Running an interactive shell # allocate a tty, attach stdin and stdout docker run -i -t base /bin/bash -Bind Docker to another host/port --------------------------------- +Bind Docker to another host/port or a unix socket +------------------------------------------------- If you want Docker to listen to another port and bind to another ip -use -host and -port on both deamon and client +use -H on both deamon and client. +-H could be (if no sheme, tcp is assumed): +* tcp://host -> tcp connection on host:4243 +* tcp://host:port -> tcp connection on host:port +* tcp://:port -> tcp connection on 127.0.0.1:port +* unix://path/to/socket -> unix socket located at path/to/socket .. code-block:: bash @@ -46,6 +51,17 @@ use -host and -port on both deamon and client # Download a base image docker -H :5555 pull base +You can use multiple -H, for exemple, if you want to listen +on both tcp and a unix socket + +.. code-block:: bash + + # Run docker in daemon mode + sudo /docker -H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock + # Download a base image + docker pull base + # OR + docker -H unix:///var/run/docker.sock pull base Starting a long-running worker process -------------------------------------- From e55048dc4dec9dda59be78f48e5f829cef4a1222 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 19 Jun 2013 14:50:58 +0000 Subject: [PATCH 11/25] gofmt and test sub directories in makefile Upstream-commit: 5dcab2d361956e75d00d5bee18371395f17a663f Component: engine --- components/engine/Makefile | 2 +- components/engine/auth/auth.go | 2 +- components/engine/registry/registry.go | 2 +- components/engine/term/termios_darwin.go | 20 ++++++++++---------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index 8676014ad4..ee85221175 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -72,7 +72,7 @@ else ifneq ($(DOCKER_DIR), $(realpath $(DOCKER_DIR))) endif test: all - @(cd $(DOCKER_DIR); sudo -E go test $(GO_OPTIONS)) + @(cd $(DOCKER_DIR); sudo -E go test ./... $(GO_OPTIONS)) fmt: @gofmt -s -l -w . diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index b7ad73b9ed..159c80f73a 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -82,7 +82,7 @@ func decodeAuth(authStr string) (*AuthConfig, error) { func LoadConfig(rootPath string) (*AuthConfig, error) { confFile := path.Join(rootPath, CONFIGFILE) if _, err := os.Stat(confFile); err != nil { - return &AuthConfig{rootPath:rootPath}, ErrConfigFileMissing + return &AuthConfig{rootPath: rootPath}, ErrConfigFileMissing } b, err := ioutil.ReadFile(confFile) if err != nil { diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 131b02708e..23aef432c7 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -481,7 +481,7 @@ type Registry struct { func NewRegistry(root string, authConfig *auth.AuthConfig) *Registry { httpTransport := &http.Transport{ DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, + Proxy: http.ProxyFromEnvironment, } r := &Registry{ diff --git a/components/engine/term/termios_darwin.go b/components/engine/term/termios_darwin.go index ac18aab692..24e79de4b2 100644 --- a/components/engine/term/termios_darwin.go +++ b/components/engine/term/termios_darwin.go @@ -9,16 +9,16 @@ const ( getTermios = syscall.TIOCGETA setTermios = syscall.TIOCSETA - ECHO = 0x00000008 - ONLCR = 0x2 - ISTRIP = 0x20 - INLCR = 0x40 - ISIG = 0x80 - IGNCR = 0x80 - ICANON = 0x100 - ICRNL = 0x100 - IXOFF = 0x400 - IXON = 0x200 + ECHO = 0x00000008 + ONLCR = 0x2 + ISTRIP = 0x20 + INLCR = 0x40 + ISIG = 0x80 + IGNCR = 0x80 + ICANON = 0x100 + ICRNL = 0x100 + IXOFF = 0x400 + IXON = 0x200 ) type Termios struct { From df7ed98febadf5d85cc49e73979dfb476a30246d Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 19 Jun 2013 18:21:46 +0000 Subject: [PATCH 12/25] add testall rule Upstream-commit: 2d6a49215c15e7649f8b041e5a04e75b4d360c94 Component: engine --- components/engine/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/Makefile b/components/engine/Makefile index ee85221175..44497d7d32 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -72,6 +72,9 @@ else ifneq ($(DOCKER_DIR), $(realpath $(DOCKER_DIR))) endif test: all + @(cd $(DOCKER_DIR); sudo -E go test $(GO_OPTIONS)) + +testall: all @(cd $(DOCKER_DIR); sudo -E go test ./... $(GO_OPTIONS)) fmt: From a1b0b51b91b5441eabbbd95390e6deef09398aa4 Mon Sep 17 00:00:00 2001 From: unclejack Date: Wed, 19 Jun 2013 22:07:56 +0300 Subject: [PATCH 13/25] batch apt-get install operations for speed The dockerbuilder Dockerfile was installing one package per apt-get install operation. This changes it so that consecutive run apt-get install operations are batched into a single operation. Upstream-commit: 88279439af6b5196aaeb9bcb3d7a7154be04d4bc Component: engine --- components/engine/hack/dockerbuilder/Dockerfile | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/components/engine/hack/dockerbuilder/Dockerfile b/components/engine/hack/dockerbuilder/Dockerfile index 5ccd293ca2..f377f0be76 100644 --- a/components/engine/hack/dockerbuilder/Dockerfile +++ b/components/engine/hack/dockerbuilder/Dockerfile @@ -19,19 +19,14 @@ run add-apt-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc) run add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu run apt-get update # Packages required to checkout, build and upload docker -run DEBIAN_FRONTEND=noninteractive apt-get install -y -q s3cmd -run DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl +run DEBIAN_FRONTEND=noninteractive apt-get install -y -q s3cmd curl run curl -s -o /go.tar.gz https://go.googlecode.com/files/go1.1.1.linux-amd64.tar.gz run tar -C /usr/local -xzf /go.tar.gz run echo "export PATH=/usr/local/go/bin:$PATH" > /.bashrc run echo "export PATH=/usr/local/go/bin:$PATH" > /.bash_profile -run DEBIAN_FRONTEND=noninteractive apt-get install -y -q git -run DEBIAN_FRONTEND=noninteractive apt-get install -y -q build-essential +run DEBIAN_FRONTEND=noninteractive apt-get install -y -q git build-essential # Packages required to build an ubuntu package -run DEBIAN_FRONTEND=noninteractive apt-get install -y -q golang-stable -run DEBIAN_FRONTEND=noninteractive apt-get install -y -q debhelper -run DEBIAN_FRONTEND=noninteractive apt-get install -y -q autotools-dev -run apt-get install -y -q devscripts +run DEBIAN_FRONTEND=noninteractive apt-get install -y -q golang-stable debhelper autotools-dev devscripts # Copy dockerbuilder files into the container add . /src run cp /src/dockerbuilder /usr/local/bin/ && chmod +x /usr/local/bin/dockerbuilder From f1b51af0188349b5783ab260ff395692da19f0e8 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 19 Jun 2013 14:59:28 -0700 Subject: [PATCH 14/25] *Builder: warn pre-1.3 clients that they need to upgrade. This breaks semver, but our API should still be in 0.X versioning, in which case semver allows breaking changes. Upstream-commit: 90dde9beabcac3874a274f9f7952f25ba8c1c7a3 Component: engine --- components/engine/api.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/api.go b/components/engine/api.go index 563888c6b1..1aa46e6b3d 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -715,6 +715,9 @@ func postImagesGetCache(srv *Server, version float64, w http.ResponseWriter, r * } func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if version < 1.3 { + return fmt.Errorf("Multipart upload for build is no longer supported. Please upgrade your docker client.") + } // FIXME: "remote" is not a clear variable name. remote := r.FormValue("t") tag := "" From a327fa094d877a1566ca38f6e8c8af213b4a2194 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 19 Jun 2013 14:59:42 -0700 Subject: [PATCH 15/25] * Builder: remove duplicate unit test Upstream-commit: 55edbcd02fe5d519069677c3970ea90e9973a4eb Component: engine --- components/engine/buildfile_test.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 3b03b8dd04..2a5cc6682e 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -52,21 +52,6 @@ run mkdir -p /var/run/sshd }, }, - { - ` -# VERSION 0.1 -# DOCKER-VERSION 0.2 - -from docker-ut -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd`, - nil, - []testCommand{ - {[]string{"cat", "/tmp/passwd"}, "root:testpass\n"}, - {[]string{"ls", "-d", "/var/run/sshd"}, "/var/run/sshd\n"}, - }, - }, - { ` from docker-ut From f576f996f6c539f51fa1e88563532229027be037 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 19 Jun 2013 15:03:33 -0700 Subject: [PATCH 16/25] * Remote API: updated docs for 1.3 Upstream-commit: d6ab71f450ac3673ffd4bc806de23e42a6d32c07 Component: engine --- .../sources/api/docker_remote_api_v1.3.rst | 1038 +++++++++++++++++ 1 file changed, 1038 insertions(+) create mode 100644 components/engine/docs/sources/api/docker_remote_api_v1.3.rst diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst new file mode 100644 index 0000000000..c6b5856c42 --- /dev/null +++ b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst @@ -0,0 +1,1038 @@ +:title: Remote API v1.2 +:description: API Documentation for Docker +:keywords: API, Docker, rcli, REST, documentation + +====================== +Docker Remote API v1.2 +====================== + +.. contents:: Table of Contents + +1. Brief introduction +===================== + +- The Remote API is replacing rcli +- Default port in the docker deamon is 4243 +- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr + +2. Endpoints +============ + +2.1 Containers +-------------- + +List containers +*************** + +.. http:get:: /containers/json + + List containers + + **Example request**: + + .. sourcecode:: http + + GET /containers/json?all=1&before=8dfafdbc3a40 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id": "8dfafdbc3a40", + "Image": "base:latest", + "Command": "echo 1", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "9cd87474be90", + "Image": "base:latest", + "Command": "echo 222222", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "3176a2479c92", + "Image": "base:latest", + "Command": "echo 3333333333333333", + "Created": 1367854154, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "4cb07b47f9fb", + "Image": "base:latest", + "Command": "echo 444444444444444444444444444444444", + "Created": 1367854152, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + } + ] + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :query limit: Show ``limit`` last created containers, include non-running ones. + :query since: Show only containers created since Id, include non-running ones. + :query before: Show only containers created before Id, include non-running ones. + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create a container +****************** + +.. http:post:: /containers/create + + Create a container + + **Example request**: + + .. sourcecode:: http + + POST /containers/create HTTP/1.1 + Content-Type: application/json + + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":true, + "AttachStderr":true, + "PortSpecs":null, + "Tty":false, + "OpenStdin":false, + "StdinOnce":false, + "Env":null, + "Cmd":[ + "date" + ], + "Dns":null, + "Image":"base", + "Volumes":{}, + "VolumesFrom":"" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/json + + { + "Id":"e90e34656806" + "Warnings":[] + } + + :jsonparam config: the container's configuration + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 406: impossible to attach (container not running) + :statuscode 500: server error + + +Inspect a container +******************* + +.. http:get:: /containers/(id)/json + + Return low-level information on the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Id": "4fa6e0f0c6786287e131c3852c58a2e01cc697a68231826813597e4994f1d6e2", + "Created": "2013-05-07T14:51:42.041847+02:00", + "Path": "date", + "Args": [], + "Config": { + "Hostname": "4fa6e0f0c678", + "User": "", + "Memory": 0, + "MemorySwap": 0, + "AttachStdin": false, + "AttachStdout": true, + "AttachStderr": true, + "PortSpecs": null, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": [ + "date" + ], + "Dns": null, + "Image": "base", + "Volumes": {}, + "VolumesFrom": "" + }, + "State": { + "Running": false, + "Pid": 0, + "ExitCode": 0, + "StartedAt": "2013-05-07T14:51:42.087658+02:01360", + "Ghost": false + }, + "Image": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "NetworkSettings": { + "IpAddress": "", + "IpPrefixLen": 0, + "Gateway": "", + "Bridge": "", + "PortMapping": null + }, + "SysInitPath": "/home/kitty/go/src/github.com/dotcloud/docker/bin/docker", + "ResolvConfPath": "/etc/resolv.conf", + "Volumes": {} + } + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Inspect changes on a container's filesystem +******************************************* + +.. http:get:: /containers/(id)/changes + + Inspect changes on container ``id`` 's filesystem + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/changes HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Path":"/dev", + "Kind":0 + }, + { + "Path":"/dev/kmsg", + "Kind":1 + }, + { + "Path":"/test", + "Kind":1 + } + ] + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Export a container +****************** + +.. http:get:: /containers/(id)/export + + Export the contents of container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/export HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Start a container +***************** + +.. http:post:: /containers/(id)/start + + Start the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/start HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Stop a contaier +*************** + +.. http:post:: /containers/(id)/stop + + Stop the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/stop?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Restart a container +******************* + +.. http:post:: /containers/(id)/restart + + Restart the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/restart?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Kill a container +**************** + +.. http:post:: /containers/(id)/kill + + Kill the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/kill HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Attach to a container +********************* + +.. http:post:: /containers/(id)/attach + + Attach to the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/attach?logs=1&stream=0&stdout=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/vnd.docker.raw-stream + + {{ STREAM }} + + :query logs: 1/True/true or 0/False/false, return logs. Default false + :query stream: 1/True/true or 0/False/false, return stream. Default false + :query stdin: 1/True/true or 0/False/false, if stream=true, attach to stdin. Default false + :query stdout: 1/True/true or 0/False/false, if logs=true, return stdout log, if stream=true, attach to stdout. Default false + :query stderr: 1/True/true or 0/False/false, if logs=true, return stderr log, if stream=true, attach to stderr. Default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Wait a container +**************** + +.. http:post:: /containers/(id)/wait + + Block until container ``id`` stops, then returns the exit code + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/wait HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"StatusCode":0} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Remove a container +******************* + +.. http:delete:: /containers/(id) + + Remove the container ``id`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /containers/16253994b7c4?v=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query v: 1/True/true or 0/False/false, Remove the volumes associated to the container. Default false + :statuscode 204: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +2.2 Images +---------- + +List Images +*********** + +.. http:get:: /images/(format) + + List images ``format`` could be json or viz (json default) + + **Example request**: + + .. sourcecode:: http + + GET /images/json?all=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Repository":"base", + "Tag":"ubuntu-12.10", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + }, + { + "Repository":"base", + "Tag":"ubuntu-quantal", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + } + ] + + + **Example request**: + + .. sourcecode:: http + + GET /images/viz HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: text/plain + + digraph docker { + "d82cbacda43a" -> "074be284591f" + "1496068ca813" -> "08306dc45919" + "08306dc45919" -> "0e7893146ac2" + "b750fe79269d" -> "1496068ca813" + base -> "27cf78414709" [style=invis] + "f71189fff3de" -> "9a33b36209ed" + "27cf78414709" -> "b750fe79269d" + "0e7893146ac2" -> "d6434d954665" + "d6434d954665" -> "d82cbacda43a" + base -> "e9aa60c60128" [style=invis] + "074be284591f" -> "f71189fff3de" + "b750fe79269d" [label="b750fe79269d\nbase",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "e9aa60c60128" [label="e9aa60c60128\nbase2",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "9a33b36209ed" [label="9a33b36209ed\ntest",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + base [style=invisible] + } + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create an image +*************** + +.. http:post:: /images/create + + Create an image, either by pull it from the registry or by importing it + + **Example request**: + + .. sourcecode:: http + + POST /images/create?fromImage=base HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pulling..."} + {"status":"Pulling", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query fromImage: name of the image to pull + :query fromSrc: source to import, - means stdin + :query repo: repository + :query tag: tag + :query registry: the registry to pull from + :statuscode 200: no error + :statuscode 500: server error + + +Insert a file in a image +************************ + +.. http:post:: /images/(name)/insert + + Insert a file from ``url`` in the image ``name`` at ``path`` + + **Example request**: + + .. sourcecode:: http + + POST /images/test/insert?path=/usr&url=myurl HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Inserting..."} + {"status":"Inserting", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :statuscode 200: no error + :statuscode 500: server error + + +Inspect an image +**************** + +.. http:get:: /images/(name)/json + + Return low-level information on the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "id":"b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "parent":"27cf784147099545", + "created":"2013-03-23T22:24:18.818426-07:00", + "container":"3d67245a8d72ecf13f33dffac9f79dcdf70f75acb84d308770391510e0c23ad0", + "container_config": + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":false, + "AttachStderr":false, + "PortSpecs":null, + "Tty":true, + "OpenStdin":true, + "StdinOnce":false, + "Env":null, + "Cmd": ["/bin/bash"] + ,"Dns":null, + "Image":"base", + "Volumes":null, + "VolumesFrom":"" + }, + "Size": 6824592 + } + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Get the history of an image +*************************** + +.. http:get:: /images/(name)/history + + Return the history of the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/history HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id":"b750fe79269d", + "Created":1364102658, + "CreatedBy":"/bin/bash" + }, + { + "Id":"27cf78414709", + "Created":1364068391, + "CreatedBy":"" + } + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Push an image on the registry +***************************** + +.. http:post:: /images/(name)/push + + Push the image ``name`` on the registry + + **Example request**: + + .. sourcecode:: http + + POST /images/test/push HTTP/1.1 + {{ authConfig }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pushing..."} + {"status":"Pushing", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query registry: the registry you wan to push, optional + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Tag an image into a repository +****************************** + +.. http:post:: /images/(name)/tag + + Tag the image ``name`` into a repository + + **Example request**: + + .. sourcecode:: http + + POST /images/test/tag?repo=myrepo&force=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :query repo: The repository to tag in + :query force: 1/True/true or 0/False/false, default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Remove an image +*************** + +.. http:delete:: /images/(name) + + Remove the image ``name`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /images/test HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + {"Untagged":"3e2f21a89f"}, + {"Deleted":"3e2f21a89f"}, + {"Deleted":"53b4f83ac9"} + ] + + :statuscode 204: no error + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Search images +************* + +.. http:get:: /images/search + + Search for an image in the docker index + + **Example request**: + + .. sourcecode:: http + + GET /images/search?term=sshd HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Name":"cespare/sshd", + "Description":"" + }, + { + "Name":"johnfuller/sshd", + "Description":"" + }, + { + "Name":"dhrp/mongodb-sshd", + "Description":"" + } + ] + + :query term: term to search + :statuscode 200: no error + :statuscode 500: server error + + +2.3 Misc +-------- + +Build an image from Dockerfile via stdin +**************************************** + +.. http:post:: /build + + Build an image from Dockerfile via stdin + + **Example request**: + + .. sourcecode:: http + + POST /build HTTP/1.1 + + {{ STREAM }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + {{ STREAM }} + + + The stream must be a tar archive compressed with one of the following algorithms: + identity (no compression), gzip, bzip2, xz. The archive must include a file called + `Dockerfile` at its root. It may include any number of other files, which will be + accessible in the build context (See the ADD build command). + + The Content-type header should be set to "application/tar". + + :query t: tag to be applied to the resulting image in case of success + :statuscode 200: no error + :statuscode 500: server error + + +Check auth configuration +************************ + +.. http:post:: /auth + + Get the default username and email + + **Example request**: + + .. sourcecode:: http + + POST /auth HTTP/1.1 + Content-Type: application/json + + { + "username":"hannibal", + "password:"xxxx", + "email":"hannibal@a-team.com" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 204: no error + :statuscode 500: server error + + +Display system-wide information +******************************* + +.. http:get:: /info + + Display system-wide information + + **Example request**: + + .. sourcecode:: http + + GET /info HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Containers":11, + "Images":16, + "Debug":false, + "NFd": 11, + "NGoroutines":21, + "MemoryLimit":true, + "SwapLimit":false + } + + :statuscode 200: no error + :statuscode 500: server error + + +Show the docker version information +*********************************** + +.. http:get:: /version + + Show the docker version information + + **Example request**: + + .. sourcecode:: http + + GET /version HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Version":"0.2.2", + "GitCommit":"5a2a5cc+CHANGES", + "GoVersion":"go1.0.3" + } + + :statuscode 200: no error + :statuscode 500: server error + + +Create a new image from a container's changes +********************************************* + +.. http:post:: /commit + + Create a new image from a container's changes + + **Example request**: + + .. sourcecode:: http + + POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/vnd.docker.raw-stream + + {"Id":"596069db4bf5"} + + :query container: source container + :query repo: repository + :query tag: tag + :query m: commit message + :query author: author (eg. "John Hannibal Smith ") + :query run: config automatically applied when the image is run. (ex: {"Cmd": ["cat", "/world"], "PortSpecs":["22"]}) + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 500: server error + + +3. Going further +================ + +3.1 Inside 'docker run' +----------------------- + +Here are the steps of 'docker run' : + +* Create the container +* If the status code is 404, it means the image doesn't exists: + * Try to pull it + * Then retry to create the container +* Start the container +* If you are not in detached mode: + * Attach to the container, using logs=1 (to have stdout and stderr from the container's start) and stream=1 +* If in detached mode or only stdin is attached: + * Display the container's id + + +3.2 Hijacking +------------- + +In this version of the API, /attach, uses hijacking to transport stdin, stdout and stderr on the same socket. This might change in the future. + +3.3 CORS Requests +----------------- + +To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. + + docker -d -H="192.168.1.9:4243" -api-enable-cors + From e9a955e303e985117387f30840e1b7465f975a4e Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 20 Jun 2013 12:31:48 +0000 Subject: [PATCH 17/25] update docs Upstream-commit: 05796bed57d4e1de352e02fcd6b28aed14f1b670 Component: engine --- components/engine/docs/sources/use/basics.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/components/engine/docs/sources/use/basics.rst b/components/engine/docs/sources/use/basics.rst index 0f64ec4cf8..11d55a5bea 100644 --- a/components/engine/docs/sources/use/basics.rst +++ b/components/engine/docs/sources/use/basics.rst @@ -36,9 +36,13 @@ Running an interactive shell Bind Docker to another host/port or a unix socket ------------------------------------------------- -If you want Docker to listen to another port and bind to another ip -use -H on both deamon and client. --H could be (if no sheme, tcp is assumed): +With -H it is possible to make the Docker daemon to listen on a specific ip and port. By default, it will listen on 127.0.0.1:4243 to allow only local connections but you can set it to 0.0.0.0:4243 or a specific host ip to give access to everybody. + +Similarly, the Docker client can use -H to connect to a custom port. + +-H accepts host and port assignment in the following format: tcp://[host][:port] or unix://path +For example: + * tcp://host -> tcp connection on host:4243 * tcp://host:port -> tcp connection on host:port * tcp://:port -> tcp connection on 127.0.0.1:port @@ -51,7 +55,7 @@ use -H on both deamon and client. # Download a base image docker -H :5555 pull base -You can use multiple -H, for exemple, if you want to listen +You can use multiple -H, for example, if you want to listen on both tcp and a unix socket .. code-block:: bash From 9322ac2e409d780c6282e9df975967c47a4f158b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 19 Jun 2013 14:27:54 -0700 Subject: [PATCH 18/25] Bumped version to 0.4.3 Upstream-commit: da5bb4db96b369365c4b913d9009d8bb1d0f51f4 Component: engine --- components/engine/CHANGELOG.md | 20 ++++++++++++++++++++ components/engine/commands.go | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index 1e2112218b..fa255e7c40 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 0.4.3 (2013-06-19) + + Builder: ADD of a local file will detect tar archives and unpack them + * Runtime: Remove bsdtar dependency + * Runtime: Add unix socket and multiple -H support + * Runtime: Prevent rm of running containers + * Runtime: Use go1.1 cookiejar + * Builder: ADD improvements: use tar for copy + automatically unpack local archives + * Builder: ADD uses tar/untar for copies instead of calling 'cp -ar' + * Builder: nicer output for 'docker build' + * Builder: fixed the behavior of ADD to be (mostly) reverse-compatible, predictable and well-documented. + * Client: HumanReadable ProgressBar sizes in pull + * Client: Fix docker version's git commit output + * API: Send all tags on History API call + * API: Add tag lookup to history command. Fixes #882 + - Runtime: Fix issue detaching from running TTY container + - Runtime: Forbid parralel push/pull for a single image/repo. Fixes #311 + - Runtime: Fix race condition within Run command when attaching. + - Builder: fix a bug which caused builds to fail if ADD was the first command + - Documentation: fix missing command in irc bouncer example + ## 0.4.2 (2013-06-17) - Packaging: Bumped version to work around an Ubuntu bug diff --git a/components/engine/commands.go b/components/engine/commands.go index cce2ec8557..6ff9ac4425 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -29,7 +29,7 @@ import ( "unicode" ) -const VERSION = "0.4.2" +const VERSION = "0.4.3" var ( GITCOMMIT string From f26e4fdaf0872e6f18765b0af9dfd2f17887aa37 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 20 Jun 2013 12:30:02 -0700 Subject: [PATCH 19/25] Fixed API version numbers in api docs Upstream-commit: cff2187a4ccf90a5be821c316b564e934d80998c Component: engine --- components/engine/docs/sources/api/docker_remote_api_v1.3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst index c6b5856c42..2237b52398 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst @@ -1,9 +1,9 @@ -:title: Remote API v1.2 +:title: Remote API v1.3 :description: API Documentation for Docker :keywords: API, Docker, rcli, REST, documentation ====================== -Docker Remote API v1.2 +Docker Remote API v1.3 ====================== .. contents:: Table of Contents From 931d03178dc6a339cdedd4e782982a8ec5924833 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 20 Jun 2013 14:29:34 -0700 Subject: [PATCH 20/25] - Builder: hotfix for bug introduced in 3adf9ce04ef1632f82f7bd1585c135bb141aa450 Upstream-commit: dbfb3eb92391d90741be3238551a0fffcf2dbcdc Component: engine --- components/engine/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 6ff9ac4425..2a3979ba5e 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -197,7 +197,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { v := &url.Values{} v.Set("t", *tag) // Send the multipart request with correct content-type - req, err := http.NewRequest("POST", fmt.Sprintf("/v%s/build?%s", APIVERSION, v.Encode()), multipartBody) + req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), multipartBody) if err != nil { return err } From 37052c0a2181c160542cea413f90783dbb74ae1e Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 20 Jun 2013 14:33:59 -0700 Subject: [PATCH 21/25] Bump version to 0.4.4 Upstream-commit: 02f0c1e46d1d73677aed899c1f0deaa69ab11ffc Component: engine --- components/engine/CHANGELOG.md | 3 +++ components/engine/commands.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index fa255e7c40..1144800150 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.4.4 (2013-06-19) + - Builder: fix a regression introduced in 0.4.3 which caused builds to fail on new clients. + ## 0.4.3 (2013-06-19) + Builder: ADD of a local file will detect tar archives and unpack them * Runtime: Remove bsdtar dependency diff --git a/components/engine/commands.go b/components/engine/commands.go index 2a3979ba5e..7cd981571b 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -29,7 +29,7 @@ import ( "unicode" ) -const VERSION = "0.4.3" +const VERSION = "0.4.4" var ( GITCOMMIT string From 1c06271442a139e1a9f2ab494ba35221b24616fc Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 20 Jun 2013 16:31:11 -0700 Subject: [PATCH 22/25] remove unused files Upstream-commit: 08825fa611cf5067b99bed452ffe6320ed00003a Component: engine --- components/engine/getKernelVersion_darwin.go | 10 --- components/engine/getKernelVersion_linux.go | 71 -------------------- 2 files changed, 81 deletions(-) delete mode 100644 components/engine/getKernelVersion_darwin.go delete mode 100644 components/engine/getKernelVersion_linux.go diff --git a/components/engine/getKernelVersion_darwin.go b/components/engine/getKernelVersion_darwin.go deleted file mode 100644 index 2fce282716..0000000000 --- a/components/engine/getKernelVersion_darwin.go +++ /dev/null @@ -1,10 +0,0 @@ -package docker - -import ( - "fmt" - "github.com/dotcloud/docker/utils" -) - -func getKernelVersion() (*utils.KernelVersionInfo, error) { - return nil, fmt.Errorf("Kernel version detection is not available on darwin") -} diff --git a/components/engine/getKernelVersion_linux.go b/components/engine/getKernelVersion_linux.go deleted file mode 100644 index 4f9c7db70c..0000000000 --- a/components/engine/getKernelVersion_linux.go +++ /dev/null @@ -1,71 +0,0 @@ -package docker - -import ( - "bytes" - "github.com/dotcloud/docker/utils" - "strconv" - "strings" - "syscall" -) - -// FIXME: Move this to utils package -func getKernelVersion() (*utils.KernelVersionInfo, error) { - var ( - uts syscall.Utsname - flavor string - kernel, major, minor int - err error - ) - - if err := syscall.Uname(&uts); err != nil { - return nil, err - } - - release := make([]byte, len(uts.Release)) - - i := 0 - for _, c := range uts.Release { - release[i] = byte(c) - i++ - } - - // Remove the \x00 from the release for Atoi to parse correctly - release = release[:bytes.IndexByte(release, 0)] - - tmp := strings.SplitN(string(release), "-", 2) - tmp2 := strings.SplitN(tmp[0], ".", 3) - - if len(tmp2) > 0 { - kernel, err = strconv.Atoi(tmp2[0]) - if err != nil { - return nil, err - } - } - - if len(tmp2) > 1 { - major, err = strconv.Atoi(tmp2[1]) - if err != nil { - return nil, err - } - } - - if len(tmp2) > 2 { - minor, err = strconv.Atoi(tmp2[2]) - if err != nil { - return nil, err - } - } - - if len(tmp) == 2 { - flavor = tmp[1] - } else { - flavor = "" - } - - return &utils.KernelVersionInfo{ - Kernel: kernel, - Major: major, - Minor: minor, - Flavor: flavor, - }, nil -} From 5f45c865b5581ebad5926ac52f178ccb0599eb7b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 20 Jun 2013 18:18:36 -0700 Subject: [PATCH 23/25] Use hijack for logs instead of stream Upstream-commit: b419699ab8f79f4826ec94583c6f7a46f74eeaa2 Component: engine --- components/engine/api.go | 2 ++ components/engine/commands.go | 12 +++++++++--- components/engine/server.go | 8 ++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index 0b89fcf323..403254649b 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -816,6 +816,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) { localFct := fct f := func(w http.ResponseWriter, r *http.Request) { utils.Debugf("Calling %s %s", localMethod, localRoute) + if logging { log.Println(r.Method, r.RequestURI) } @@ -836,6 +837,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) { w.WriteHeader(http.StatusNotFound) return } + if err := localFct(srv, version, w, r, mux.Vars(r)); err != nil { httpError(w, err) } diff --git a/components/engine/commands.go b/components/engine/commands.go index 7cd981571b..1f3426f900 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1036,10 +1036,10 @@ func (cli *DockerCli) CmdLogs(args ...string) error { return nil } - if err := cli.stream("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stdout=1", nil, os.Stdout); err != nil { + if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stdout=1", false, nil, os.Stdout); err != nil { return err } - if err := cli.stream("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stderr=1", nil, os.Stderr); err != nil { + if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stderr=1", false, nil, os.Stderr); err != nil { return err } return nil @@ -1370,6 +1370,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e return err } defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 400 { body, err := ioutil.ReadAll(resp.Body) if err != nil { @@ -1407,19 +1408,24 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e } func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.File, out io.Writer) error { + req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), nil) if err != nil { return err } + req.Header.Set("User-Agent", "Docker-Client/"+VERSION) req.Header.Set("Content-Type", "plain/text") + dial, err := net.Dial(cli.proto, cli.addr) if err != nil { return err } clientconn := httputil.NewClientConn(dial, nil) - clientconn.Do(req) defer clientconn.Close() + // Server hijacks the connection, error 'connection closed' expected + clientconn.Do(req) + rwc, br := clientconn.Hijack() defer rwc.Close() diff --git a/components/engine/server.go b/components/engine/server.go index 35388d8605..ad4f092a7f 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -979,17 +979,17 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std if stdout { cLog, err := container.ReadLog("stdout") if err != nil { - utils.Debugf(err.Error()) + utils.Debugf("Error reading logs (stdout): %s", err) } else if _, err := io.Copy(out, cLog); err != nil { - utils.Debugf(err.Error()) + utils.Debugf("Error streaming logs (stdout): %s", err) } } if stderr { cLog, err := container.ReadLog("stderr") if err != nil { - utils.Debugf(err.Error()) + utils.Debugf("Error reading logs (stderr): %s", err) } else if _, err := io.Copy(out, cLog); err != nil { - utils.Debugf(err.Error()) + utils.Debugf("Error streaming logs (stderr): %s", err) } } } From 99b7da757be3b513f48ec0b782c032a53f69c545 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 20 Jun 2013 20:16:39 -0700 Subject: [PATCH 24/25] * Builder: simplified unit tests. The tests are now embedded in the build itself. Yeah baby. Upstream-commit: cc0f59742f192ccbc178c59bdc7eb98d0f6dd943 Component: engine --- components/engine/buildfile_test.go | 48 ++++------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 33e6a3146b..773470b809 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -13,6 +13,8 @@ const Dockerfile = ` from ` + unitTestImageName + ` run sh -c 'echo root:testpass > /tmp/passwd' run mkdir -p /var/run/sshd +run [ "$(cat /tmp/passwd)" = "root:testpass" ] +run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] ` const DockerfileNoNewLine = ` @@ -21,7 +23,9 @@ const DockerfileNoNewLine = ` from ` + unitTestImageName + ` run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd` +run mkdir -p /var/run/sshd +run [ "$(cat /tmp/passwd)" = "root:testpass" ] +run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ]` // FIXME: test building with a context @@ -42,48 +46,8 @@ func TestBuild(t *testing.T) { buildfile := NewBuildFile(srv, &utils.NopWriter{}) - imgID, err := buildfile.Build(strings.NewReader(Dockerfile), nil) - if err != nil { + if _, err := buildfile.Build(strings.NewReader(Dockerfile), nil); err != nil { t.Fatal(err) } - - builder := NewBuilder(runtime) - container, err := builder.Create( - &Config{ - Image: imgID, - Cmd: []string{"cat", "/tmp/passwd"}, - }, - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container) - - output, err := container.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "root:testpass\n" { - t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, "root:testpass\n") - } - - container2, err := builder.Create( - &Config{ - Image: imgID, - Cmd: []string{"ls", "-d", "/var/run/sshd"}, - }, - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container2) - - output, err = container2.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "/var/run/sshd\n" { - t.Fatal("/var/run/sshd has not been created") - } } } From 7a7ea6028990284e724fdeca5b1a454fbfdb603e Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 20 Jun 2013 20:20:16 -0700 Subject: [PATCH 25/25] - Builder: fixed a regression in ADD. Improved regression tests for build behavior. Upstream-commit: 36d610a38806feb4cb5d09c4705a6f1fff42ef04 Component: engine --- components/engine/archive.go | 98 +++++++++++--------- components/engine/hack/test-build/Dockerfile | 17 ++++ components/engine/hack/test-build/README.md | 9 ++ components/engine/hack/test-build/d/ga | 1 + components/engine/hack/test-build/f | 1 + 5 files changed, 83 insertions(+), 43 deletions(-) create mode 100644 components/engine/hack/test-build/Dockerfile create mode 100644 components/engine/hack/test-build/README.md create mode 100644 components/engine/hack/test-build/d/ga create mode 100644 components/engine/hack/test-build/f diff --git a/components/engine/archive.go b/components/engine/archive.go index 16401e29fb..5756490bff 100644 --- a/components/engine/archive.go +++ b/components/engine/archive.go @@ -1,7 +1,9 @@ package docker import ( + "archive/tar" "bufio" + "bytes" "errors" "fmt" "github.com/dotcloud/docker/utils" @@ -10,6 +12,7 @@ import ( "os" "os/exec" "path" + "path/filepath" ) type Archive io.Reader @@ -160,51 +163,60 @@ func CopyWithTar(src, dst string) error { if err != nil { return err } - var dstExists bool - dstSt, err := os.Stat(dst) - if err != nil { - if !os.IsNotExist(err) { - return err - } - } else { - dstExists = true - } - // Things that can go wrong if the source is a directory - if srcSt.IsDir() { - // The destination exists and is a regular file - if dstExists && !dstSt.IsDir() { - return fmt.Errorf("Can't copy a directory over a regular file") - } - // Things that can go wrong if the source is a regular file - } else { - utils.Debugf("The destination exists, it's a directory, and doesn't end in /") - // The destination exists, it's a directory, and doesn't end in / - if dstExists && dstSt.IsDir() && dst[len(dst)-1] != '/' { - return fmt.Errorf("Can't copy a regular file over a directory %s |%s|", dst, dst[len(dst)-1]) - } - } - // Create the destination - var dstDir string - if srcSt.IsDir() || dst[len(dst)-1] == '/' { - // The destination ends in /, or the source is a directory - // --> dst is the holding directory and needs to be created for -C - dstDir = dst - } else { - // The destination doesn't end in / - // --> dst is the file - dstDir = path.Dir(dst) - } - if !dstExists { - // Create the holding directory if necessary - utils.Debugf("Creating the holding directory %s", dstDir) - if err := os.MkdirAll(dstDir, 0700); err != nil && !os.IsExist(err) { - return err - } - } if !srcSt.IsDir() { - return TarUntar(path.Dir(src), []string{path.Base(src)}, dstDir) + return CopyFileWithTar(src, dst) } - return TarUntar(src, nil, dstDir) + // Create dst, copy src's content into it + utils.Debugf("Creating dest directory: %s", dst) + if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) { + return err + } + utils.Debugf("Calling TarUntar(%s, %s)", src, dst) + return TarUntar(src, nil, dst) +} + +// CopyFileWithTar emulates the behavior of the 'cp' command-line +// for a single file. It copies a regular file from path `src` to +// path `dst`, and preserves all its metadata. +// +// If `dst` ends with a trailing slash '/', the final destination path +// will be `dst/base(src)`. +func CopyFileWithTar(src, dst string) error { + utils.Debugf("CopyFileWithTar(%s, %s)", src, dst) + srcSt, err := os.Stat(src) + if err != nil { + return err + } + if srcSt.IsDir() { + return fmt.Errorf("Can't copy a directory") + } + // Clean up the trailing / + if dst[len(dst)-1] == '/' { + dst = path.Join(dst, filepath.Base(src)) + } + // Create the holding directory if necessary + if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) { + return err + } + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + hdr, err := tar.FileInfoHeader(srcSt, "") + if err != nil { + return err + } + hdr.Name = filepath.Base(dst) + if err := tw.WriteHeader(hdr); err != nil { + return err + } + srcF, err := os.Open(src) + if err != nil { + return err + } + if _, err := io.Copy(tw, srcF); err != nil { + return err + } + tw.Close() + return Untar(buf, filepath.Dir(dst)) } // CmdStream executes a command, and returns its stdout as a stream. diff --git a/components/engine/hack/test-build/Dockerfile b/components/engine/hack/test-build/Dockerfile new file mode 100644 index 0000000000..a232253c9c --- /dev/null +++ b/components/engine/hack/test-build/Dockerfile @@ -0,0 +1,17 @@ +from busybox +add f / +run [ "$(cat /f)" = "hello" ] +add f /abc +run [ "$(cat /abc)" = "hello" ] +add f /x/y/z +run [ "$(cat /x/y/z)" = "hello" ] +add f /x/y/d/ +run [ "$(cat /x/y/d/f)" = "hello" ] +add d / +run [ "$(cat /ga)" = "bu" ] +add d /somewhere +run [ "$(cat /somewhere/ga)" = "bu" ] +add d /anotherplace/ +run [ "$(cat /anotherplace/ga)" = "bu" ] +add d /somewheeeere/over/the/rainbooow +run [ "$(cat /somewheeeere/over/the/rainbooow/ga)" = "bu" ] diff --git a/components/engine/hack/test-build/README.md b/components/engine/hack/test-build/README.md new file mode 100644 index 0000000000..77636dcda6 --- /dev/null +++ b/components/engine/hack/test-build/README.md @@ -0,0 +1,9 @@ +This directory is meant to test the behavior of 'docker build' in general, and its ADD command in particular. +Ideally it would be automatically called as part of the unit test suite - but that requires slightly more glue, +so for now we're just calling it manually, because it's better than nothing. + +To test, simply build this directory with the following command: + + docker build . + +The tests are embedded in the Dockerfile: if the build succeeds, the tests have passed. diff --git a/components/engine/hack/test-build/d/ga b/components/engine/hack/test-build/d/ga new file mode 100644 index 0000000000..7e6edb8302 --- /dev/null +++ b/components/engine/hack/test-build/d/ga @@ -0,0 +1 @@ +bu diff --git a/components/engine/hack/test-build/f b/components/engine/hack/test-build/f new file mode 100644 index 0000000000..ce01362503 --- /dev/null +++ b/components/engine/hack/test-build/f @@ -0,0 +1 @@ +hello