From 2394c9c8dca18831eea5f2b3d67da9ea3c5d18f2 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 3 Jul 2013 17:11:00 +0000 Subject: [PATCH 001/134] add last version Upstream-commit: 64450ae3f89b8f9b5288224c5a7d109a166cf22a Component: engine --- components/engine/commands.go | 9 +++++++++ components/engine/utils/utils.go | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/components/engine/commands.go b/components/engine/commands.go index 6e1e5e88c2..e275bdd671 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -407,6 +407,15 @@ func (cli *DockerCli) CmdVersion(args ...string) error { if out.GoVersion != "" { fmt.Fprintf(cli.out, "Go version: %s\n", out.GoVersion) } + + release := utils.GetReleaseVersion() + if release != "" { + fmt.Fprintf(cli.out, "Last stable version: %s", release) + if VERSION != release || out.Version != release { + fmt.Fprintf(cli.out, ", please update docker") + } + fmt.Fprintf(cli.out, "\n") + } return nil } diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 2f2a52867e..b4a41ea420 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -687,3 +687,25 @@ func ParseHost(host string, port int, addr string) string { } return fmt.Sprintf("tcp://%s:%d", host, port) } + +func GetReleaseVersion() string { + type githubTag struct { + Name string `json:"name"` + } + + resp, err := http.Get("https://api.github.com/repos/dotcloud/docker/tags") + if err != nil { + return "" + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "" + } + var tags []githubTag + err = json.Unmarshal(body, &tags) + if err != nil || len(tags) == 0 { + return "" + } + return strings.TrimPrefix(tags[0].Name, "v") +} From dedc7724c83e96157eda2d2ad5b388d7b792703c Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 10 Jul 2013 22:44:31 +0000 Subject: [PATCH 002/134] change tag -> repo name in build usage Upstream-commit: fc3a8e409d182ade45c697c244562e20beee9f9a Component: engine --- components/engine/commands.go | 2 +- components/engine/docs/sources/api/docker_remote_api_v1.0.rst | 2 +- components/engine/docs/sources/api/docker_remote_api_v1.2.rst | 2 +- components/engine/docs/sources/api/docker_remote_api_v1.3.rst | 2 +- components/engine/docs/sources/commandline/command/build.rst | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index feab558259..d12a5755e4 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -156,7 +156,7 @@ func mkBuildContext(dockerfile string, files [][2]string) (Archive, error) { func (cli *DockerCli) CmdBuild(args ...string) error { cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH") - tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success") + tag := cmd.String("t", "", "Repository name to be applied to the resulting image in case of success") if err := cmd.Parse(args); err != nil { return nil } diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.0.rst b/components/engine/docs/sources/api/docker_remote_api_v1.0.rst index a789337093..f2e9a99939 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.0.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.0.rst @@ -827,7 +827,7 @@ Build an image from Dockerfile via stdin {{ STREAM }} - :query t: tag to be applied to the resulting image in case of success + :query t: repository name to be applied to the resulting image in case of success :statuscode 200: no error :statuscode 500: server error diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.2.rst b/components/engine/docs/sources/api/docker_remote_api_v1.2.rst index a6c2c31920..bec5677425 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.2.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.2.rst @@ -865,7 +865,7 @@ Build an image from Dockerfile via stdin {{ STREAM }} - :query t: tag to be applied to the resulting image in case of success + :query t: repository name to be applied to the resulting image in case of success :query remote: resource to fetch, as URI :statuscode 200: no error :statuscode 500: server error diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst index b0955ce496..efd695b82c 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst @@ -880,7 +880,7 @@ Build an image from Dockerfile via stdin The Content-type header should be set to "application/tar". - :query t: tag to be applied to the resulting image in case of success + :query t: repository name to be applied to the resulting image in case of success :statuscode 200: no error :statuscode 500: server error diff --git a/components/engine/docs/sources/commandline/command/build.rst b/components/engine/docs/sources/commandline/command/build.rst index 1645002ba2..aeba8cc899 100644 --- a/components/engine/docs/sources/commandline/command/build.rst +++ b/components/engine/docs/sources/commandline/command/build.rst @@ -10,7 +10,7 @@ Usage: docker build [OPTIONS] PATH | URL | - Build a new container image from the source code at PATH - -t="": Tag to be applied to the resulting image in case of success. + -t="": Repository name to be applied to the resulting image in case of success. When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context From 4b7018962aac5e057cc6a5c674b6871dca26e954 Mon Sep 17 00:00:00 2001 From: Jonas Pfenniger Date: Mon, 15 Jul 2013 11:36:05 +0100 Subject: [PATCH 003/134] docker.upstart: avoid spawning a `sh` process start script / end script create an intermediate sh process. Upstream-commit: 24dd50490a027f01ea086eb90663d53348fa770e Component: engine --- components/engine/packaging/ubuntu/docker.upstart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/engine/packaging/ubuntu/docker.upstart b/components/engine/packaging/ubuntu/docker.upstart index 2bd5565ee7..1d35d7a493 100644 --- a/components/engine/packaging/ubuntu/docker.upstart +++ b/components/engine/packaging/ubuntu/docker.upstart @@ -4,6 +4,4 @@ start on runlevel [2345] stop on starting rc RUNLEVEL=[016] respawn -script - /usr/bin/docker -d -end script +exec /usr/bin/docker -d From 632014a7bf21f6db9ed19062cb00f454e5359ac9 Mon Sep 17 00:00:00 2001 From: Jonas Pfenniger Date: Mon, 15 Jul 2013 11:41:19 +0100 Subject: [PATCH 004/134] docker.upstart: use the same start/stop events as sshd Is probably more solid Upstream-commit: 0900d3b7a6e94bfa42e3d4ac6dc6f5542f65a9b0 Component: engine --- components/engine/packaging/ubuntu/docker.upstart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/packaging/ubuntu/docker.upstart b/components/engine/packaging/ubuntu/docker.upstart index 1d35d7a493..f4d2fbe922 100644 --- a/components/engine/packaging/ubuntu/docker.upstart +++ b/components/engine/packaging/ubuntu/docker.upstart @@ -1,7 +1,8 @@ description "Run docker" -start on runlevel [2345] -stop on starting rc RUNLEVEL=[016] +start on filesystem or runlevel [2345] +stop on runlevel [!2345] + respawn exec /usr/bin/docker -d From 7c7e6abf06a30ca4ce1319a7e3556c3064862f03 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 14 Jun 2013 16:56:08 -0700 Subject: [PATCH 005/134] + Runtime: inject dockerinit at /.dockerinit instead of overwriting /sbin/init. This makes it possible to run /sbin/init inside a container. Upstream-commit: 5d8efc107d2c7b7da61a6d22657190c6f13713d2 Component: engine --- components/engine/container.go | 2 +- components/engine/docker/docker.go | 2 +- components/engine/graph.go | 33 +++++++++++++++++++++++++++++- components/engine/image.go | 14 +++++++++++++ components/engine/lxc_template.go | 2 +- components/engine/runtime_test.go | 2 +- 6 files changed, 50 insertions(+), 5 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index 3772cf29d2..e8159aae48 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -615,7 +615,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { "-n", container.ID, "-f", container.lxcConfigPath(), "--", - "/sbin/init", + "/.dockerinit", } // Networking diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index fb7c465369..d609a64a51 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -19,7 +19,7 @@ var ( ) func main() { - if utils.SelfPath() == "/sbin/init" { + if selfPath := utils.SelfPath(); selfPath == "/sbin/init" || selfPath == "/.dockerinit" { // Running in init mode docker.SysInit() return diff --git a/components/engine/graph.go b/components/engine/graph.go index 42d1bdbd4c..1e60d6dbe8 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -193,8 +193,39 @@ func (graph *Graph) Mktemp(id string) (string, error) { return tmp.imageRoot(id), nil } +// getDockerInitLayer returns the path of a layer containing a mountpoint suitable +// for bind-mounting dockerinit into the container. The mountpoint is simply an +// empty file at /.dockerinit +// +// This extra layer is used by all containers as the top-most ro layer. It protects +// the container from unwanted side-effects on the rw layer. +func (graph *Graph) getDockerInitLayer() (string, error) { + tmp, err := graph.tmp() + if err != nil { + return "", err + } + initLayer := tmp.imageRoot("_dockerinit") + if err := os.Mkdir(initLayer, 0700); err != nil && !os.IsExist(err) { + // If directory already existed, keep going. + // For all other errors, abort. + return "", err + } + // FIXME: how the hell do I break down this line in a way + // that is idiomatic and not ugly as hell? + if f, err := os.OpenFile(path.Join(initLayer, ".dockerinit"), os.O_CREATE|os.O_TRUNC, 0700); err != nil && !os.IsExist(err) { + // If file already existed, keep going. + // For all other errors, abort. + return "", err + } else { + f.Close() + } + // Layer is ready to use, if it wasn't before. + return initLayer, nil +} + func (graph *Graph) tmp() (*Graph, error) { - return NewGraph(path.Join(graph.Root, ":tmp:")) + // Changed to _tmp from :tmp:, because it messed with ":" separators in aufs branch syntax... + return NewGraph(path.Join(graph.Root, "_tmp")) } // Check if given error is "not empty". diff --git a/components/engine/image.go b/components/engine/image.go index e1b1ac0418..af7b5030be 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -263,6 +263,13 @@ func (img *Image) layers() ([]string, error) { if len(list) == 0 { return nil, fmt.Errorf("No layer found for image %s\n", img.ID) } + + // Inject the dockerinit layer (empty place-holder for mount-binding dockerinit) + if dockerinitLayer, err := img.getDockerInitLayer(); err != nil { + return nil, err + } else { + list = append([]string{dockerinitLayer}, list...) + } return list, nil } @@ -292,6 +299,13 @@ func (img *Image) GetParent() (*Image, error) { return img.graph.Get(img.Parent) } +func (img *Image) getDockerInitLayer() (string, error) { + if img.graph == nil { + return "", fmt.Errorf("Can't lookup dockerinit layer of unregistered image") + } + return img.graph.getDockerInitLayer() +} + func (img *Image) root() (string, error) { if img.graph == nil { return "", fmt.Errorf("Can't lookup root of unregistered image") diff --git a/components/engine/lxc_template.go b/components/engine/lxc_template.go index 93b795e901..e00b2f88c5 100644 --- a/components/engine/lxc_template.go +++ b/components/engine/lxc_template.go @@ -79,7 +79,7 @@ lxc.mount.entry = devpts {{$ROOTFS}}/dev/pts devpts newinstance,ptmxmode=0666,no #lxc.mount.entry = shm {{$ROOTFS}}/dev/shm tmpfs size=65536k,nosuid,nodev,noexec 0 0 # Inject docker-init -lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/sbin/init none bind,ro 0 0 +lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/.dockerinit none bind,ro 0 0 # In order to get a working DNS environment, mount bind (ro) the host's /etc/resolv.conf into the container lxc.mount.entry = {{.ResolvConfPath}} {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index 9d43bd46e5..7c30322fc5 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -69,7 +69,7 @@ func layerArchive(tarfile string) (io.Reader, error) { func init() { // Hack to run sys init during unit testing - if utils.SelfPath() == "/sbin/init" { + if selfPath := utils.SelfPath(); selfPath == "/sbin/init" || selfPath == "/.dockerinit" { SysInit() return } From 9c8c35f4bd8616581c8840eb4b72b08630d89ab5 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 18 Jul 2013 20:56:41 +0000 Subject: [PATCH 006/134] handle -dev Upstream-commit: 0afed3eded0950b5899729714fb11d6a7322301e Component: engine --- components/engine/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 5f3913f9c2..7b70faa198 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -436,7 +436,7 @@ func (cli *DockerCli) CmdVersion(args ...string) error { release := utils.GetReleaseVersion() if release != "" { fmt.Fprintf(cli.out, "Last stable version: %s", release) - if VERSION != release || out.Version != release { + if strings.Trim(VERSION, "-dev") != release || strings.Trim(out.Version, "-dev") != release { fmt.Fprintf(cli.out, ", please update docker") } fmt.Fprintf(cli.out, "\n") From 37e3d486fa87ae31293b385a6f957865d61b13bb Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Jul 2013 10:06:32 +0000 Subject: [PATCH 007/134] add ps args to docker top Upstream-commit: cfec1c3e1b88ceeca73144198df7a210ed3dc421 Component: engine --- components/engine/api.go | 6 +++++- components/engine/api_params.go | 6 ++---- components/engine/api_test.go | 27 ++++++++++++++++++--------- components/engine/commands.go | 17 +++++++++++------ components/engine/server.go | 29 ++++++++++++++--------------- 5 files changed, 50 insertions(+), 35 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index b6ab7badfa..d3b84df5f9 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -255,8 +255,12 @@ func getContainersTop(srv *Server, version float64, w http.ResponseWriter, r *ht if vars == nil { return fmt.Errorf("Missing parameter") } + if err := parseForm(r); err != nil { + return err + } name := vars["name"] - procsStr, err := srv.ContainerTop(name) + ps_args := r.Form.Get("ps_args") + procsStr, err := srv.ContainerTop(name, ps_args) if err != nil { return err } diff --git a/components/engine/api_params.go b/components/engine/api_params.go index b371ca314f..2296ee792e 100644 --- a/components/engine/api_params.go +++ b/components/engine/api_params.go @@ -27,10 +27,8 @@ type APIInfo struct { } type APITop struct { - PID string - Tty string - Time string - Cmd string + Titles []string + Processes [][]string } type APIRmi struct { diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 17ada96eab..9b7f08d1db 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -444,24 +444,33 @@ func TestGetContainersTop(t *testing.T) { } r := httptest.NewRecorder() - if err := getContainersTop(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err != nil { + req, err := http.NewRequest("GET", "/"+container.ID+"/top?ps_args=u", bytes.NewReader([]byte{})) + if err != nil { t.Fatal(err) } - procs := []APITop{} + if err := getContainersTop(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { + t.Fatal(err) + } + procs := APITop{} if err := json.Unmarshal(r.Body.Bytes(), &procs); err != nil { t.Fatal(err) } - if len(procs) != 2 { - t.Fatalf("Expected 2 processes, found %d.", len(procs)) + if len(procs.Titles) != 11 { + t.Fatalf("Expected 11 titles, found %d.", len(procs.Titles)) + } + if procs.Titles[0] != "USER" || procs.Titles[10] != "COMMAND" { + t.Fatalf("Expected Titles[0] to be USER and Titles[10] to be COMMAND, found %s and %s.", procs.Titles[0], procs.Titles[10]) } - if procs[0].Cmd != "sh" && procs[0].Cmd != "busybox" { - t.Fatalf("Expected `busybox` or `sh`, found %s.", procs[0].Cmd) + if len(procs.Processes) != 2 { + t.Fatalf("Expected 2 processes, found %d.", len(procs.Processes)) } - - if procs[1].Cmd != "sh" && procs[1].Cmd != "busybox" { - t.Fatalf("Expected `busybox` or `sh`, found %s.", procs[1].Cmd) + if procs.Processes[0][10] != "/bin/sh" && procs.Processes[0][10] != "sleep" { + t.Fatalf("Expected `sleep` or `/bin/sh`, found %s.", procs.Processes[0][10]) + } + if procs.Processes[1][10] != "/bin/sh" && procs.Processes[1][10] != "sleep" { + t.Fatalf("Expected `sleep` or `/bin/sh`, found %s.", procs.Processes[1][10]) } } diff --git a/components/engine/commands.go b/components/engine/commands.go index 936b23fea2..b25e928efa 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -585,23 +585,28 @@ func (cli *DockerCli) CmdTop(args ...string) error { if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() != 1 { + if cmd.NArg() == 0 { cmd.Usage() return nil } - body, _, err := cli.call("GET", "/containers/"+cmd.Arg(0)+"/top", nil) + val := url.Values{} + if cmd.NArg() > 1 { + val.Set("ps_args", strings.Join(cmd.Args()[1:], " ")) + } + + body, _, err := cli.call("GET", "/containers/"+cmd.Arg(0)+"/top?"+val.Encode(), nil) if err != nil { return err } - var procs []APITop + procs := APITop{} err = json.Unmarshal(body, &procs) if err != nil { return err } w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) - fmt.Fprintln(w, "PID\tTTY\tTIME\tCMD") - for _, proc := range procs { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", proc.PID, proc.Tty, proc.Time, proc.Cmd) + fmt.Fprintln(w, strings.Join(procs.Titles, "\t")) + for _, proc := range procs.Processes { + fmt.Fprintln(w, strings.Join(proc, "\t")) } w.Flush() return nil diff --git a/components/engine/server.go b/components/engine/server.go index 954bbb208f..ae5f605267 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -249,35 +249,34 @@ func (srv *Server) ImageHistory(name string) ([]APIHistory, error) { } -func (srv *Server) ContainerTop(name string) ([]APITop, error) { +func (srv *Server) ContainerTop(name, ps_args string) (*APITop, error) { if container := srv.runtime.Get(name); container != nil { - output, err := exec.Command("lxc-ps", "--name", container.ID).CombinedOutput() + output, err := exec.Command("lxc-ps", "--name", container.ID, "--", ps_args).CombinedOutput() if err != nil { return nil, fmt.Errorf("Error trying to use lxc-ps: %s (%s)", err, output) } - var procs []APITop + procs := APITop{} for i, line := range strings.Split(string(output), "\n") { - if i == 0 || len(line) == 0 { + if len(line) == 0 { continue } - proc := APITop{} + words := []string{} scanner := bufio.NewScanner(strings.NewReader(line)) scanner.Split(bufio.ScanWords) if !scanner.Scan() { return nil, fmt.Errorf("Error trying to use lxc-ps") } // no scanner.Text because we skip container id - scanner.Scan() - proc.PID = scanner.Text() - scanner.Scan() - proc.Tty = scanner.Text() - scanner.Scan() - proc.Time = scanner.Text() - scanner.Scan() - proc.Cmd = scanner.Text() - procs = append(procs, proc) + for scanner.Scan() { + words = append(words, scanner.Text()) + } + if i == 0 { + procs.Titles = words + } else { + procs.Processes = append(procs.Processes, words) + } } - return procs, nil + return &procs, nil } return nil, fmt.Errorf("No such container: %s", name) From 797b7b513bdcdbda74626e391699d652d7106f8c Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Jul 2013 10:34:55 +0000 Subject: [PATCH 008/134] bump api version to 1.4 Upstream-commit: eb4a0271fbb7318a7d66911ccd388c048db0c2fa Component: engine --- components/engine/api.go | 5 +- .../docs/sources/api/docker_remote_api.rst | 17 +- .../sources/api/docker_remote_api_v1.4.rst | 1093 +++++++++++++++++ 3 files changed, 1112 insertions(+), 3 deletions(-) create mode 100644 components/engine/docs/sources/api/docker_remote_api_v1.4.rst diff --git a/components/engine/api.go b/components/engine/api.go index d3b84df5f9..a3dd52108e 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -17,7 +17,7 @@ import ( "strings" ) -const APIVERSION = 1.3 +const APIVERSION = 1.4 const DEFAULTHTTPHOST string = "127.0.0.1" const DEFAULTHTTPPORT int = 4243 @@ -252,6 +252,9 @@ func getContainersChanges(srv *Server, version float64, w http.ResponseWriter, r } func getContainersTop(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if version < 1.4 { + return fmt.Errorf("top was improved a lot since 1.3, Please upgrade your docker client.") + } if vars == nil { return fmt.Errorf("Missing parameter") } diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index 183347c23b..a08fb46940 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -19,8 +19,8 @@ Docker Remote API 2. Versions =========== -The current verson of the API is 1.3 -Calling /images//insert is the same as calling /v1.3/images//insert +The current verson of the API is 1.4 +Calling /images//insert is the same as calling /v1.4/images//insert You can still call an old version of the api using /v1.0/images//insert :doc:`docker_remote_api_v1.3` @@ -31,6 +31,18 @@ What's new Listing processes (/top): +- You can now use ps args with docker top, like `docker top aux` + +:doc:`docker_remote_api_v1.3` +***************************** + +docker v0.5.0 51f6c4a_ + +What's new +---------- + +Listing processes (/top): + - List the processes inside a container @@ -109,6 +121,7 @@ Initial version .. _a8ae398: https://github.com/dotcloud/docker/commit/a8ae398bf52e97148ee7bd0d5868de2e15bd297f .. _8d73740: https://github.com/dotcloud/docker/commit/8d73740343778651c09160cde9661f5f387b36f4 .. _2e7649b: https://github.com/dotcloud/docker/commit/2e7649beda7c820793bd46766cbc2cfeace7b168 +.. _51f6c4a: https://github.com/dotcloud/docker/commit/51f6c4a7372450d164c61e0054daf0223ddbd909 ================================== Docker Remote API Client Libraries diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst new file mode 100644 index 0000000000..c42adb286f --- /dev/null +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -0,0 +1,1093 @@ +:title: Remote API v1.3 +:description: API Documentation for Docker +:keywords: API, Docker, rcli, REST, documentation + +====================== +Docker Remote API v1.3 +====================== + +.. contents:: Table of Contents + +1. Brief introduction +===================== + +- The Remote API is replacing rcli +- Default port in the docker deamon is 4243 +- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr + +2. Endpoints +============ + +2.1 Containers +-------------- + +List containers +*************** + +.. http:get:: /containers/json + + List containers + + **Example request**: + + .. sourcecode:: http + + GET /containers/json?all=1&before=8dfafdbc3a40&size=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id": "8dfafdbc3a40", + "Image": "base:latest", + "Command": "echo 1", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "9cd87474be90", + "Image": "base:latest", + "Command": "echo 222222", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "3176a2479c92", + "Image": "base:latest", + "Command": "echo 3333333333333333", + "Created": 1367854154, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "4cb07b47f9fb", + "Image": "base:latest", + "Command": "echo 444444444444444444444444444444444", + "Created": 1367854152, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + } + ] + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :query limit: Show ``limit`` last created containers, include non-running ones. + :query since: Show only containers created since Id, include non-running ones. + :query before: Show only containers created before Id, include non-running ones. + :query size: 1/True/true or 0/False/false, Show the containers sizes + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create a container +****************** + +.. http:post:: /containers/create + + Create a container + + **Example request**: + + .. sourcecode:: http + + POST /containers/create HTTP/1.1 + Content-Type: application/json + + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":true, + "AttachStderr":true, + "PortSpecs":null, + "Tty":false, + "OpenStdin":false, + "StdinOnce":false, + "Env":null, + "Cmd":[ + "date" + ], + "Dns":null, + "Image":"base", + "Volumes":{}, + "VolumesFrom":"" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/json + + { + "Id":"e90e34656806" + "Warnings":[] + } + + :jsonparam config: the container's configuration + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 406: impossible to attach (container not running) + :statuscode 500: server error + + +Inspect a container +******************* + +.. http:get:: /containers/(id)/json + + Return low-level information on the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Id": "4fa6e0f0c6786287e131c3852c58a2e01cc697a68231826813597e4994f1d6e2", + "Created": "2013-05-07T14:51:42.041847+02:00", + "Path": "date", + "Args": [], + "Config": { + "Hostname": "4fa6e0f0c678", + "User": "", + "Memory": 0, + "MemorySwap": 0, + "AttachStdin": false, + "AttachStdout": true, + "AttachStderr": true, + "PortSpecs": null, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": [ + "date" + ], + "Dns": null, + "Image": "base", + "Volumes": {}, + "VolumesFrom": "" + }, + "State": { + "Running": false, + "Pid": 0, + "ExitCode": 0, + "StartedAt": "2013-05-07T14:51:42.087658+02:01360", + "Ghost": false + }, + "Image": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "NetworkSettings": { + "IpAddress": "", + "IpPrefixLen": 0, + "Gateway": "", + "Bridge": "", + "PortMapping": null + }, + "SysInitPath": "/home/kitty/go/src/github.com/dotcloud/docker/bin/docker", + "ResolvConfPath": "/etc/resolv.conf", + "Volumes": {} + } + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +List processes running inside a container +***************************************** + +.. http:get:: /containers/(id)/top + + List processes running inside the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/top HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Titles":[ + "USER", + "PID", + "%CPU", + "%MEM", + "VSZ", + "RSS", + "TTY", + "STAT", + "START", + "TIME", + "COMMAND" + ], + "Processes":[ + ["root","20147","0.0","0.1","18060","1864","pts/4","S","10:06","0:00","bash"], + ["root","20271","0.0","0.0","4312","352","pts/4","S+","10:07","0:00","sleep","10"] + ] + } + + :query ps_args: ps arguments to use (eg. aux) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Inspect changes on a container's filesystem +******************************************* + +.. http:get:: /containers/(id)/changes + + Inspect changes on container ``id`` 's filesystem + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/changes HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Path":"/dev", + "Kind":0 + }, + { + "Path":"/dev/kmsg", + "Kind":1 + }, + { + "Path":"/test", + "Kind":1 + } + ] + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Export a container +****************** + +.. http:get:: /containers/(id)/export + + Export the contents of container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/export HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Start a container +***************** + +.. http:post:: /containers/(id)/start + + Start the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/(id)/start HTTP/1.1 + Content-Type: application/json + + { + "Binds":["/tmp:/tmp"] + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 No Content + Content-Type: text/plain + + :jsonparam hostConfig: the container's host configuration (optional) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Stop a contaier +*************** + +.. http:post:: /containers/(id)/stop + + Stop the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/stop?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Restart a container +******************* + +.. http:post:: /containers/(id)/restart + + Restart the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/restart?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Kill a container +**************** + +.. http:post:: /containers/(id)/kill + + Kill the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/kill HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Attach to a container +********************* + +.. http:post:: /containers/(id)/attach + + Attach to the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/attach?logs=1&stream=0&stdout=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/vnd.docker.raw-stream + + {{ STREAM }} + + :query logs: 1/True/true or 0/False/false, return logs. Default false + :query stream: 1/True/true or 0/False/false, return stream. Default false + :query stdin: 1/True/true or 0/False/false, if stream=true, attach to stdin. Default false + :query stdout: 1/True/true or 0/False/false, if logs=true, return stdout log, if stream=true, attach to stdout. Default false + :query stderr: 1/True/true or 0/False/false, if logs=true, return stderr log, if stream=true, attach to stderr. Default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Wait a container +**************** + +.. http:post:: /containers/(id)/wait + + Block until container ``id`` stops, then returns the exit code + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/wait HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"StatusCode":0} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Remove a container +******************* + +.. http:delete:: /containers/(id) + + Remove the container ``id`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /containers/16253994b7c4?v=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query v: 1/True/true or 0/False/false, Remove the volumes associated to the container. Default false + :statuscode 204: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +2.2 Images +---------- + +List Images +*********** + +.. http:get:: /images/(format) + + List images ``format`` could be json or viz (json default) + + **Example request**: + + .. sourcecode:: http + + GET /images/json?all=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Repository":"base", + "Tag":"ubuntu-12.10", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + }, + { + "Repository":"base", + "Tag":"ubuntu-quantal", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + } + ] + + + **Example request**: + + .. sourcecode:: http + + GET /images/viz HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: text/plain + + digraph docker { + "d82cbacda43a" -> "074be284591f" + "1496068ca813" -> "08306dc45919" + "08306dc45919" -> "0e7893146ac2" + "b750fe79269d" -> "1496068ca813" + base -> "27cf78414709" [style=invis] + "f71189fff3de" -> "9a33b36209ed" + "27cf78414709" -> "b750fe79269d" + "0e7893146ac2" -> "d6434d954665" + "d6434d954665" -> "d82cbacda43a" + base -> "e9aa60c60128" [style=invis] + "074be284591f" -> "f71189fff3de" + "b750fe79269d" [label="b750fe79269d\nbase",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "e9aa60c60128" [label="e9aa60c60128\nbase2",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "9a33b36209ed" [label="9a33b36209ed\ntest",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + base [style=invisible] + } + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create an image +*************** + +.. http:post:: /images/create + + Create an image, either by pull it from the registry or by importing it + + **Example request**: + + .. sourcecode:: http + + POST /images/create?fromImage=base HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pulling..."} + {"status":"Pulling", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query fromImage: name of the image to pull + :query fromSrc: source to import, - means stdin + :query repo: repository + :query tag: tag + :query registry: the registry to pull from + :statuscode 200: no error + :statuscode 500: server error + + +Insert a file in a image +************************ + +.. http:post:: /images/(name)/insert + + Insert a file from ``url`` in the image ``name`` at ``path`` + + **Example request**: + + .. sourcecode:: http + + POST /images/test/insert?path=/usr&url=myurl HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Inserting..."} + {"status":"Inserting", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :statuscode 200: no error + :statuscode 500: server error + + +Inspect an image +**************** + +.. http:get:: /images/(name)/json + + Return low-level information on the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "id":"b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "parent":"27cf784147099545", + "created":"2013-03-23T22:24:18.818426-07:00", + "container":"3d67245a8d72ecf13f33dffac9f79dcdf70f75acb84d308770391510e0c23ad0", + "container_config": + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":false, + "AttachStderr":false, + "PortSpecs":null, + "Tty":true, + "OpenStdin":true, + "StdinOnce":false, + "Env":null, + "Cmd": ["/bin/bash"] + ,"Dns":null, + "Image":"base", + "Volumes":null, + "VolumesFrom":"" + }, + "Size": 6824592 + } + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Get the history of an image +*************************** + +.. http:get:: /images/(name)/history + + Return the history of the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/history HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id":"b750fe79269d", + "Created":1364102658, + "CreatedBy":"/bin/bash" + }, + { + "Id":"27cf78414709", + "Created":1364068391, + "CreatedBy":"" + } + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Push an image on the registry +***************************** + +.. http:post:: /images/(name)/push + + Push the image ``name`` on the registry + + **Example request**: + + .. sourcecode:: http + + POST /images/test/push HTTP/1.1 + {{ authConfig }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pushing..."} + {"status":"Pushing", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query registry: the registry you wan to push, optional + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Tag an image into a repository +****************************** + +.. http:post:: /images/(name)/tag + + Tag the image ``name`` into a repository + + **Example request**: + + .. sourcecode:: http + + POST /images/test/tag?repo=myrepo&force=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :query repo: The repository to tag in + :query force: 1/True/true or 0/False/false, default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Remove an image +*************** + +.. http:delete:: /images/(name) + + Remove the image ``name`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /images/test HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + {"Untagged":"3e2f21a89f"}, + {"Deleted":"3e2f21a89f"}, + {"Deleted":"53b4f83ac9"} + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Search images +************* + +.. http:get:: /images/search + + Search for an image in the docker index + + **Example request**: + + .. sourcecode:: http + + GET /images/search?term=sshd HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Name":"cespare/sshd", + "Description":"" + }, + { + "Name":"johnfuller/sshd", + "Description":"" + }, + { + "Name":"dhrp/mongodb-sshd", + "Description":"" + } + ] + + :query term: term to search + :statuscode 200: no error + :statuscode 500: server error + + +2.3 Misc +-------- + +Build an image from Dockerfile via stdin +**************************************** + +.. http:post:: /build + + Build an image from Dockerfile via stdin + + **Example request**: + + .. sourcecode:: http + + POST /build HTTP/1.1 + + {{ STREAM }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + {{ STREAM }} + + + The stream must be a tar archive compressed with one of the following algorithms: + identity (no compression), gzip, bzip2, xz. The archive must include a file called + `Dockerfile` at its root. It may include any number of other files, which will be + accessible in the build context (See the ADD build command). + + The Content-type header should be set to "application/tar". + + :query t: tag to be applied to the resulting image in case of success + :query q: suppress verbose build output + :statuscode 200: no error + :statuscode 500: server error + + +Check auth configuration +************************ + +.. http:post:: /auth + + Get the default username and email + + **Example request**: + + .. sourcecode:: http + + POST /auth HTTP/1.1 + Content-Type: application/json + + { + "username":"hannibal", + "password:"xxxx", + "email":"hannibal@a-team.com" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 204: no error + :statuscode 500: server error + + +Display system-wide information +******************************* + +.. http:get:: /info + + Display system-wide information + + **Example request**: + + .. sourcecode:: http + + GET /info HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Containers":11, + "Images":16, + "Debug":false, + "NFd": 11, + "NGoroutines":21, + "MemoryLimit":true, + "SwapLimit":false + } + + :statuscode 200: no error + :statuscode 500: server error + + +Show the docker version information +*********************************** + +.. http:get:: /version + + Show the docker version information + + **Example request**: + + .. sourcecode:: http + + GET /version HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Version":"0.2.2", + "GitCommit":"5a2a5cc+CHANGES", + "GoVersion":"go1.0.3" + } + + :statuscode 200: no error + :statuscode 500: server error + + +Create a new image from a container's changes +********************************************* + +.. http:post:: /commit + + Create a new image from a container's changes + + **Example request**: + + .. sourcecode:: http + + POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/vnd.docker.raw-stream + + {"Id":"596069db4bf5"} + + :query container: source container + :query repo: repository + :query tag: tag + :query m: commit message + :query author: author (eg. "John Hannibal Smith ") + :query run: config automatically applied when the image is run. (ex: {"Cmd": ["cat", "/world"], "PortSpecs":["22"]}) + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 500: server error + + +3. Going further +================ + +3.1 Inside 'docker run' +----------------------- + +Here are the steps of 'docker run' : + +* Create the container +* If the status code is 404, it means the image doesn't exists: + * Try to pull it + * Then retry to create the container +* Start the container +* If you are not in detached mode: + * Attach to the container, using logs=1 (to have stdout and stderr from the container's start) and stream=1 +* If in detached mode or only stdin is attached: + * Display the container's id + + +3.2 Hijacking +------------- + +In this version of the API, /attach, uses hijacking to transport stdin, stdout and stderr on the same socket. This might change in the future. + +3.3 CORS Requests +----------------- + +To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. + + docker -d -H="192.168.1.9:4243" -api-enable-cors + From 744c5e76581069d10de0cecac5565f9359cc098b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Jul 2013 16:36:23 +0000 Subject: [PATCH 009/134] add LXC version to docker info in debug mode Upstream-commit: 921c6994b1ad41c940bdb08732225b8db74b68f2 Component: engine --- components/engine/api_params.go | 9 +++++---- components/engine/commands.go | 1 + components/engine/server.go | 9 +++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/components/engine/api_params.go b/components/engine/api_params.go index b371ca314f..967b736bdc 100644 --- a/components/engine/api_params.go +++ b/components/engine/api_params.go @@ -20,10 +20,11 @@ type APIInfo struct { Debug bool Containers int Images int - NFd int `json:",omitempty"` - NGoroutines int `json:",omitempty"` - MemoryLimit bool `json:",omitempty"` - SwapLimit bool `json:",omitempty"` + NFd int `json:",omitempty"` + NGoroutines int `json:",omitempty"` + MemoryLimit bool `json:",omitempty"` + SwapLimit bool `json:",omitempty"` + LXCVersion string `json:",omitempty"` } type APITop struct { diff --git a/components/engine/commands.go b/components/engine/commands.go index f0e1695b3f..e2b26321f5 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -463,6 +463,7 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "Debug mode (client): %v\n", os.Getenv("DEBUG") != "") fmt.Fprintf(cli.out, "Fds: %d\n", out.NFd) fmt.Fprintf(cli.out, "Goroutines: %d\n", out.NGoroutines) + fmt.Fprintf(cli.out, "LXC Version: %s\n", out.LXCVersion) } if !out.MemoryLimit { fmt.Fprintf(cli.err, "WARNING: No memory limit support\n") diff --git a/components/engine/server.go b/components/engine/server.go index b92ed8fd73..90d56f74a0 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -208,6 +208,14 @@ func (srv *Server) DockerInfo() *APIInfo { } else { imgcount = len(images) } + lxcVersion := "" + if output, err := exec.Command("lxc-version").CombinedOutput(); err == nil { + outputStr := string(output) + if len(strings.SplitN(outputStr, ":", 2)) == 2 { + lxcVersion = strings.TrimSpace(strings.SplitN(string(output), ":", 2)[1]) + } + } + return &APIInfo{ Containers: len(srv.runtime.List()), Images: imgcount, @@ -216,6 +224,7 @@ func (srv *Server) DockerInfo() *APIInfo { Debug: os.Getenv("DEBUG") != "", NFd: utils.GetTotalUsedFds(), NGoroutines: runtime.NumGoroutine(), + LXCVersion: lxcVersion, } } From a6dc7d327c375791dffbd30a7c8abac1da8acbe8 Mon Sep 17 00:00:00 2001 From: Sridatta Thatipamala Date: Mon, 22 Jul 2013 14:55:07 -0700 Subject: [PATCH 010/134] change permissions of initLayer to be readable by non-root users Upstream-commit: 945033f1ccbc4896e1a784d00f1f2b465ed9e04b Component: engine --- components/engine/graph.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/graph.go b/components/engine/graph.go index 1e60d6dbe8..b75ddb7ec0 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -205,7 +205,7 @@ func (graph *Graph) getDockerInitLayer() (string, error) { return "", err } initLayer := tmp.imageRoot("_dockerinit") - if err := os.Mkdir(initLayer, 0700); err != nil && !os.IsExist(err) { + if err := os.Mkdir(initLayer, 0755); err != nil && !os.IsExist(err) { // If directory already existed, keep going. // For all other errors, abort. return "", err From ff1a26c4e5d8f0fb8115a6f03e3b9176f45a0cfd Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Tue, 23 Jul 2013 13:05:06 -0700 Subject: [PATCH 011/134] Docs: Fixed navigaton links to about page and community page Website: Removed the website sources from the repo. The website sources are now hosted on github.com/dotcloud/www.docker.io/ Upstream-commit: e701dce33978a0206627a02b258da6a0269c4b60 Component: engine --- components/engine/docs/theme/MAINTAINERS | 2 +- .../engine/docs/theme/docker/layout.html | 8 +- components/engine/docs/website/MAINTAINERS | 1 - components/engine/docs/website/dotcloud.yml | 2 - .../docs/website/gettingstarted/index.html | 220 ----------- components/engine/docs/website/index.html | 359 ------------------ components/engine/docs/website/nginx.conf | 6 - components/engine/docs/website/static | 1 - 8 files changed, 5 insertions(+), 594 deletions(-) delete mode 100644 components/engine/docs/website/MAINTAINERS delete mode 100644 components/engine/docs/website/dotcloud.yml delete mode 100644 components/engine/docs/website/gettingstarted/index.html delete mode 100644 components/engine/docs/website/index.html delete mode 100644 components/engine/docs/website/nginx.conf delete mode 120000 components/engine/docs/website/static diff --git a/components/engine/docs/theme/MAINTAINERS b/components/engine/docs/theme/MAINTAINERS index 6df367c073..606a1dd746 100644 --- a/components/engine/docs/theme/MAINTAINERS +++ b/components/engine/docs/theme/MAINTAINERS @@ -1 +1 @@ -Thatcher Penskens +Thatcher Peskens diff --git a/components/engine/docs/theme/docker/layout.html b/components/engine/docs/theme/docker/layout.html index 198cd5d7d8..0b22f22fab 100755 --- a/components/engine/docs/theme/docker/layout.html +++ b/components/engine/docs/theme/docker/layout.html @@ -68,12 +68,12 @@
diff --git a/components/engine/docs/website/MAINTAINERS b/components/engine/docs/website/MAINTAINERS deleted file mode 100644 index 6df367c073..0000000000 --- a/components/engine/docs/website/MAINTAINERS +++ /dev/null @@ -1 +0,0 @@ -Thatcher Penskens diff --git a/components/engine/docs/website/dotcloud.yml b/components/engine/docs/website/dotcloud.yml deleted file mode 100644 index 5a8f50f9e9..0000000000 --- a/components/engine/docs/website/dotcloud.yml +++ /dev/null @@ -1,2 +0,0 @@ -www: - type: static \ No newline at end of file diff --git a/components/engine/docs/website/gettingstarted/index.html b/components/engine/docs/website/gettingstarted/index.html deleted file mode 100644 index de0cc3512d..0000000000 --- a/components/engine/docs/website/gettingstarted/index.html +++ /dev/null @@ -1,220 +0,0 @@ - - - - - - - - - - Docker - the Linux container runtime - - - - - - - - - - - - - - - - - - - - -
-
-
- -
- -
- -
- -
-
-
-
- - -
-
- -
- - - -

GETTING STARTED

-
- -
- -
- -
-
- Docker is still under heavy development. It should not yet be used in production. Check the repo for recent progress. -
-
-
-
-

- - Installing on Ubuntu

- -

Requirements

-
    -
  • Ubuntu 12.04 (LTS) (64-bit)
    • -
  • or Ubuntu 12.10 (quantal) (64-bit)
    • -
  • The 3.8 Linux Kernel
    • -
-
    -
  1. -

    Install dependencies

    - The linux-image-extra package is only needed on standard Ubuntu EC2 AMIs in order to install the aufs kernel module. - 
    sudo apt-get install linux-image-extra-`uname -r`
    - - -
    2. -
  2. -

    Install Docker

    -

    Add the Ubuntu PPA (Personal Package Archive) sources to your apt sources list, update and install.

    -

    This may import a new GPG key (key 63561DC6: public key "Launchpad PPA for dotcloud team" imported).

    -
    - 
    sudo apt-get install software-properties-common
    - 
    sudo add-apt-repository ppa:dotcloud/lxc-docker
    - 
    sudo apt-get update
    - 
    sudo apt-get install lxc-docker
    -
    - - -
    3. - -
  3. -

    Run!

    - -
    - 
    docker run -i -t ubuntu /bin/bash
    -
    -
    4. - Continue with the Hello world example.
    - Or check more detailed installation instructions -
-
- -
-

Contributing to Docker

- -

Want to hack on Docker? Awesome! We have some instructions to get you started. They are probably not perfect, please let us know if anything feels wrong or incomplete.

-
- -
-
-
-

Quick install on other operating systems

-

For other operating systems we recommend and provide a streamlined install with virtualbox, - vagrant and an Ubuntu virtual machine.

- - - -
- -
-

Questions? Want to get in touch?

-

There are several ways to get in touch:

-

Join the discussion on IRC. We can be found in the #docker channel on chat.freenode.net

-

Discussions happen on our google group: docker-club at googlegroups.com

-

All our development and decisions are made out in the open on Github github.com/dotcloud/docker

-

Get help on using Docker by asking on Stackoverflow

-

And of course, tweet your tweets to twitter.com/getdocker

-
- - -
-
- Fill out my online form. -
- -
- -
-
-
- - -
- -
- - - - - - - - - - - diff --git a/components/engine/docs/website/index.html b/components/engine/docs/website/index.html deleted file mode 100644 index f6f4efbccb..0000000000 --- a/components/engine/docs/website/index.html +++ /dev/null @@ -1,359 +0,0 @@ - - - - - - - - - - - Docker - the Linux container engine - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- - - - -
- -
-
-
-
- - -
-
- -
-
- -
-
- docker letters - -

The Linux container engine

-
- -
- -
- Docker is an open-source engine which automates the deployment of applications as highly portable, self-sufficient containers which are independent of hardware, language, framework, packaging system and hosting provider. -
- -
- - - - -
- -
-
- -
-
- -
-
-
-
-
- -
-
- -
-
-

Heterogeneous payloads

-

Any combination of binaries, libraries, configuration files, scripts, virtualenvs, jars, gems, tarballs, you name it. No more juggling between domain-specific tools. Docker can deploy and run them all.

-

Any server

-

Docker can run on any x64 machine with a modern linux kernel - whether it's a laptop, a bare metal server or a VM. This makes it perfect for multi-cloud deployments.

-

Isolation

-

Docker isolates processes from each other and from the underlying host, using lightweight containers.

-

Repeatability

-

Because each container is isolated in its own filesystem, they behave the same regardless of where, when, and alongside what they run.

-
-
-
-
- we're hiring -
-
-

Do you think it is cool to hack on docker? Join us!

-
    -
  • Work on open source
    • -
  • Program in Go
    • -
- read more -
-
- -
-
-
-
-

New! Docker Index

- On the Docker Index you can find and explore pre-made container images. It allows you to share your images and download them. - -

- -
- DOCKER index -
- -   - - -
-
-
- Fill out my online form. -
- -
-
-
- -
- -
- -
-
-
- - Mitchell Hashimoto ‏@mitchellh: Docker launched today. It is incredible. They’re also working RIGHT NOW on a Vagrant provider. LXC is COMING!! -
-
-
-
- - Adam Jacob ‏@adamhjk: Docker is clearly the right idea. @solomonstre absolutely killed it. Containerized app deployment is the future, I think. -
-
-
-
-
-
- - Matt Townsend ‏@mtownsend: I have a serious code crush on docker.io - it's Lego for PaaS. Motherfucking awesome Lego. -
-
-
-
- - Rob Harrop ‏@robertharrop: Impressed by @getdocker - it's all kinds of magic. Serious rethink of AWS architecture happening @skillsmatter. -
-
-
-
-
-
- - John Willis @botchagalupe: IMHO docker is to paas what chef was to Iaas 4 years ago -
-
-
-
- - John Feminella ‏@superninjarobot: So, @getdocker is pure excellence. If you've ever wished for arbitrary, PaaS-agnostic, lxc/aufs Linux containers, this is your jam! -
-
-
-
-
-
- - David Romulan ‏@destructuring: I haven't had this much fun since AWS -
-
-
-
- - Ricardo Gladwell ‏@rgladwell: wow @getdocker is either amazing or totally stupid -
-
- -
-
- -
-
-
- -
- -

Notable features

- -
    -
  • Filesystem isolation: each process container runs in a completely separate root filesystem.
    • -
  • Resource isolation: system resources like cpu and memory can be allocated differently to each process container, using cgroups.
    • -
  • Network isolation: each process container runs in its own network namespace, with a virtual interface and IP address of its own.
    • -
  • Copy-on-write: root filesystems are created using copy-on-write, which makes deployment extremely fast, memory-cheap and disk-cheap.
    • -
  • Logging: the standard streams (stdout/stderr/stdin) of each process container is collected and logged for real-time or batch retrieval.
    • -
  • Change management: changes to a container's filesystem can be committed into a new image and re-used to create more containers. No templating or manual configuration required.
    • -
  • Interactive shell: docker can allocate a pseudo-tty and attach to the standard input of any container, for example to run a throwaway interactive shell.
    • -
- -

Under the hood

- -

Under the hood, Docker is built on the following components:

- -
    -
  • The cgroup and namespacing capabilities of the Linux kernel;
    • -
  • AUFS, a powerful union filesystem with copy-on-write capabilities;
    • -
  • The Go programming language;
    • -
  • lxc, a set of convenience scripts to simplify the creation of linux containers.
    • -
- -

Who started it

-

- Docker is an open-source implementation of the deployment engine which powers dotCloud, a popular Platform-as-a-Service.

- -

It benefits directly from the experience accumulated over several years of large-scale operation and support of hundreds of thousands - of applications and databases. -

- -
-
- -
- - -
-

Twitter

- - -
- -
-
- -
- - -
- -
- - - - - - - - - - - - diff --git a/components/engine/docs/website/nginx.conf b/components/engine/docs/website/nginx.conf deleted file mode 100644 index 97ffd2c0e5..0000000000 --- a/components/engine/docs/website/nginx.conf +++ /dev/null @@ -1,6 +0,0 @@ - -# rule to redirect original links created when hosted on github pages -rewrite ^/documentation/(.*).html http://docs.docker.io/en/latest/$1/ permanent; - -# rewrite the stuff which was on the current page -rewrite ^/gettingstarted.html$ /gettingstarted/ permanent; diff --git a/components/engine/docs/website/static b/components/engine/docs/website/static deleted file mode 120000 index 95bc97aa10..0000000000 --- a/components/engine/docs/website/static +++ /dev/null @@ -1 +0,0 @@ -../theme/docker/static \ No newline at end of file From 600442316a12c03b8e232ad3720a559e575d841e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 23 Jul 2013 17:27:49 -0700 Subject: [PATCH 012/134] Reimplement old Commands unit tests in order to insure behavior Upstream-commit: bc823acc255a786b00a7b2759b67fd7b3256fce3 Component: engine --- components/engine/commands.go | 4 +- components/engine/commands_test.go | 223 ++++++++++++++++++++++++++++- 2 files changed, 225 insertions(+), 2 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index b0e32162e6..44498b04da 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1378,7 +1378,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { if config.AttachStdin || config.AttachStdout || config.AttachStderr { if config.Tty { if err := cli.monitorTtySize(runResult.ID); err != nil { - return err + utils.Debugf("Error monitoring TTY size: %s\n", err) } } @@ -1555,6 +1555,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea receiveStdout := utils.Go(func() error { _, err := io.Copy(out, br) + utils.Debugf("[hijack] End of stdout") return err }) @@ -1569,6 +1570,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea sendStdin := utils.Go(func() error { if in != nil { io.Copy(rwc, in) + utils.Debugf("[hijack] End of stdin") } if tcpc, ok := rwc.(*net.TCPConn); ok { if err := tcpc.CloseWrite(); err != nil { diff --git a/components/engine/commands_test.go b/components/engine/commands_test.go index 233c6337d4..88902b8503 100644 --- a/components/engine/commands_test.go +++ b/components/engine/commands_test.go @@ -73,7 +73,7 @@ func TestRunHostname(t *testing.T) { t.Fatal(err) } }() - utils.Debugf("--") + setTimeout(t, "Reading command output time out", 2*time.Second, func() { cmdOutput, err := bufio.NewReader(stdout).ReadString('\n') if err != nil { @@ -90,6 +90,157 @@ func TestRunHostname(t *testing.T) { } +func TestRunExit(t *testing.T) { + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + c1 := make(chan struct{}) + go func() { + cli.CmdRun("-i", unitTestImageID, "/bin/cat") + close(c1) + }() + + setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + t.Fatal(err) + } + }) + + container := globalRuntime.List()[0] + + // Closing /bin/cat stdin, expect it to exit + if err := stdin.Close(); err != nil { + t.Fatal(err) + } + + // as the process exited, CmdRun must finish and unblock. Wait for it + setTimeout(t, "Waiting for CmdRun timed out", 10*time.Second, func() { + <-c1 + + go func() { + cli.CmdWait(container.ID) + }() + + if _, err := bufio.NewReader(stdout).ReadString('\n'); err != nil { + t.Fatal(err) + } + }) + + // Make sure that the client has been disconnected + setTimeout(t, "The client should have been disconnected once the remote process exited.", 2*time.Second, func() { + // Expecting pipe i/o error, just check that read does not block + stdin.Read([]byte{}) + }) + + // Cleanup pipes + if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil { + t.Fatal(err) + } +} + +// Expected behaviour: the process dies when the client disconnects +func TestRunDisconnect(t *testing.T) { + + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + c1 := make(chan struct{}) + go func() { + // We're simulating a disconnect so the return value doesn't matter. What matters is the + // fact that CmdRun returns. + cli.CmdRun("-i", unitTestImageID, "/bin/cat") + close(c1) + }() + + setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + t.Fatal(err) + } + }) + + // Close pipes (simulate disconnect) + if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil { + t.Fatal(err) + } + + // as the pipes are close, we expect the process to die, + // therefore CmdRun to unblock. Wait for CmdRun + setTimeout(t, "Waiting for CmdRun timed out", 2*time.Second, func() { + <-c1 + }) + + // Client disconnect after run -i should cause stdin to be closed, which should + // cause /bin/cat to exit. + setTimeout(t, "Waiting for /bin/cat to exit timed out", 2*time.Second, func() { + container := globalRuntime.List()[0] + container.Wait() + if container.State.Running { + t.Fatalf("/bin/cat is still running after closing stdin") + } + }) +} + +// Expected behaviour: the process dies when the client disconnects +func TestRunDisconnectTty(t *testing.T) { + + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + c1 := make(chan struct{}) + go func() { + // We're simulating a disconnect so the return value doesn't matter. What matters is the + // fact that CmdRun returns. + if err := cli.CmdRun("-i", "-t", unitTestImageID, "/bin/cat"); err != nil { + utils.Debugf("Error CmdRun: %s\n", err) + } + + close(c1) + }() + + setTimeout(t, "Waiting for the container to be started timed out", 10*time.Second, func() { + for { + // Client disconnect after run -i should keep stdin out in TTY mode + l := globalRuntime.List() + if len(l) == 1 && l[0].State.Running { + break + } + time.Sleep(10 * time.Millisecond) + } + }) + + // Client disconnect after run -i should keep stdin out in TTY mode + container := globalRuntime.List()[0] + + setTimeout(t, "Read/Write assertion timed out", 2000*time.Second, func() { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + t.Fatal(err) + } + }) + + // Close pipes (simulate disconnect) + if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil { + t.Fatal(err) + } + + // In tty mode, we expect the process to stay alive even after client's stdin closes. + // Do not wait for run to finish + + // Give some time to monitor to do his thing + container.WaitTimeout(500 * time.Millisecond) + if !container.State.Running { + t.Fatalf("/bin/cat should still be running after closing stdin (tty mode)") + } +} + // TestAttachStdin checks attaching to stdin without stdout and stderr. // 'docker run -i -a stdin' should sends the client's stdin to the command, // then detach from it and print the container id. @@ -157,3 +308,73 @@ func TestRunAttachStdin(t *testing.T) { } } } + +// Expected behaviour, the process stays alive when the client disconnects +func TestAttachDisconnect(t *testing.T) { + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + go func() { + // Start a process in daemon mode + if err := cli.CmdRun("-d", "-i", unitTestImageID, "/bin/cat"); err != nil { + utils.Debugf("Error CmdRun: %s\n", err) + } + }() + + setTimeout(t, "Waiting for CmdRun timed out", 10*time.Second, func() { + if _, err := bufio.NewReader(stdout).ReadString('\n'); err != nil { + t.Fatal(err) + } + }) + + setTimeout(t, "Waiting for the container to be started timed out", 10*time.Second, func() { + for { + l := globalRuntime.List() + if len(l) == 1 && l[0].State.Running { + break + } + time.Sleep(10 * time.Millisecond) + } + }) + + container := globalRuntime.List()[0] + + // Attach to it + c1 := make(chan struct{}) + go func() { + // We're simulating a disconnect so the return value doesn't matter. What matters is the + // fact that CmdAttach returns. + cli.CmdAttach(container.ID) + close(c1) + }() + + setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + t.Fatal(err) + } + }) + // Close pipes (client disconnects) + if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil { + t.Fatal(err) + } + + // Wait for attach to finish, the client disconnected, therefore, Attach finished his job + setTimeout(t, "Waiting for CmdAttach timed out", 2*time.Second, func() { + <-c1 + }) + + // We closed stdin, expect /bin/cat to still be running + // Wait a little bit to make sure container.monitor() did his thing + err := container.WaitTimeout(500 * time.Millisecond) + if err == nil || !container.State.Running { + t.Fatalf("/bin/cat is not running after closing stdin") + } + + // Try to avoid the timeoout in destroy. Best effort, don't check error + cStdin, _ := container.StdinPipe() + cStdin.Close() + container.Wait() +} From 5190220000386db2d54955aad25f7e37abdf97f0 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Tue, 23 Jul 2013 18:13:53 -0700 Subject: [PATCH 013/134] Cleaned up long lines, switched graphic to Docker logo. General cleanup. Upstream-commit: 78c02d038f90b111f7f8ad306f9573c0f64b370c Component: engine --- components/engine/README.md | 289 +++++++++++++++++++++++++----------- 1 file changed, 204 insertions(+), 85 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 823e48496a..96da13feaf 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -1,80 +1,129 @@ Docker: the Linux container engine ================================== -Docker is an open-source engine which automates the deployment of applications as highly portable, self-sufficient containers. +Docker is an open-source engine which automates the deployment of +applications as highly portable, self-sufficient containers. -Docker containers are both *hardware-agnostic* and *platform-agnostic*. This means that they can run anywhere, from your -laptop to the largest EC2 compute instance and everything in between - and they don't require that you use a particular -language, framework or packaging system. That makes them great building blocks for deploying and scaling web apps, databases -and backend services without depending on a particular stack or provider. +Docker containers are both *hardware-agnostic* and +*platform-agnostic*. This means that they can run anywhere, from your +laptop to the largest EC2 compute instance and everything in between - +and they don't require that you use a particular language, framework +or packaging system. That makes them great building blocks for +deploying and scaling web apps, databases and backend services without +depending on a particular stack or provider. -Docker is an open-source implementation of the deployment engine which powers [dotCloud](http://dotcloud.com), a popular Platform-as-a-Service. -It benefits directly from the experience accumulated over several years of large-scale operation and support of hundreds of thousands -of applications and databases. +Docker is an open-source implementation of the deployment engine which +powers [dotCloud](http://dotcloud.com), a popular +Platform-as-a-Service. It benefits directly from the experience +accumulated over several years of large-scale operation and support of +hundreds of thousands of applications and databases. -![Docker L](docs/sources/concepts/images/lego_docker.jpg "Docker") +![Docker L](docs/sources/concepts/images/dockerlogo-h.png "Docker") ## Better than VMs -A common method for distributing applications and sandbox their execution is to use virtual machines, or VMs. Typical VM formats -are VMWare's vmdk, Oracle Virtualbox's vdi, and Amazon EC2's ami. In theory these formats should allow every developer to -automatically package their application into a "machine" for easy distribution and deployment. In practice, that almost never -happens, for a few reasons: +A common method for distributing applications and sandbox their +execution is to use virtual machines, or VMs. Typical VM formats are +VMWare's vmdk, Oracle Virtualbox's vdi, and Amazon EC2's ami. In +theory these formats should allow every developer to automatically +package their application into a "machine" for easy distribution and +deployment. In practice, that almost never happens, for a few reasons: - * *Size*: VMs are very large which makes them impractical to store and transfer. - * *Performance*: running VMs consumes significant CPU and memory, which makes them impractical in many scenarios, for example local development of multi-tier applications, and - large-scale deployment of cpu and memory-intensive applications on large numbers of machines. - * *Portability*: competing VM environments don't play well with each other. Although conversion tools do exist, they are limited and add even more overhead. - * *Hardware-centric*: VMs were designed with machine operators in mind, not software developers. As a result, they offer very limited tooling for what developers need most: - building, testing and running their software. For example, VMs offer no facilities for application versioning, monitoring, configuration, logging or service discovery. + * *Size*: VMs are very large which makes them impractical to store + and transfer. + * *Performance*: running VMs consumes significant CPU and memory, + which makes them impractical in many scenarios, for example local + development of multi-tier applications, and large-scale deployment + of cpu and memory-intensive applications on large numbers of + machines. + * *Portability*: competing VM environments don't play well with each + other. Although conversion tools do exist, they are limited and + add even more overhead. + * *Hardware-centric*: VMs were designed with machine operators in + mind, not software developers. As a result, they offer very + limited tooling for what developers need most: building, testing + and running their software. For example, VMs offer no facilities + for application versioning, monitoring, configuration, logging or + service discovery. -By contrast, Docker relies on a different sandboxing method known as *containerization*. Unlike traditional virtualization, -containerization takes place at the kernel level. Most modern operating system kernels now support the primitives necessary -for containerization, including Linux with [openvz](http://openvz.org), [vserver](http://linux-vserver.org) and more recently [lxc](http://lxc.sourceforge.net), - Solaris with [zones](http://docs.oracle.com/cd/E26502_01/html/E29024/preface-1.html#scrolltoc) and FreeBSD with [Jails](http://www.freebsd.org/doc/handbook/jails.html). +By contrast, Docker relies on a different sandboxing method known as +*containerization*. Unlike traditional virtualization, +containerization takes place at the kernel level. Most modern +operating system kernels now support the primitives necessary for +containerization, including Linux with [openvz](http://openvz.org), +[vserver](http://linux-vserver.org) and more recently +[lxc](http://lxc.sourceforge.net), Solaris with +[zones](http://docs.oracle.com/cd/E26502_01/html/E29024/preface-1.html#scrolltoc) +and FreeBSD with +[Jails](http://www.freebsd.org/doc/handbook/jails.html). -Docker builds on top of these low-level primitives to offer developers a portable format and runtime environment that solves -all 4 problems. Docker containers are small (and their transfer can be optimized with layers), they have basically zero memory and cpu overhead, -they are completely portable and are designed from the ground up with an application-centric design. +Docker builds on top of these low-level primitives to offer developers +a portable format and runtime environment that solves all 4 +problems. Docker containers are small (and their transfer can be +optimized with layers), they have basically zero memory and cpu +overhead, they are completely portable and are designed from the +ground up with an application-centric design. -The best part: because docker operates at the OS level, it can still be run inside a VM! +The best part: because ``docker`` operates at the OS level, it can +still be run inside a VM! ## Plays well with others -Docker does not require that you buy into a particular programming language, framework, packaging system or configuration language. +Docker does not require that you buy into a particular programming +language, framework, packaging system or configuration language. -Is your application a unix process? Does it use files, tcp connections, environment variables, standard unix streams and command-line -arguments as inputs and outputs? Then docker can run it. +Is your application a Unix process? Does it use files, tcp +connections, environment variables, standard Unix streams and +command-line arguments as inputs and outputs? Then ``docker`` can run +it. -Can your application's build be expressed as a sequence of such commands? Then docker can build it. +Can your application's build be expressed as a sequence of such +commands? Then ``docker`` can build it. ## Escape dependency hell -A common problem for developers is the difficulty of managing all their application's dependencies in a simple and automated way. +A common problem for developers is the difficulty of managing all +their application's dependencies in a simple and automated way. This is usually difficult for several reasons: - * *Cross-platform dependencies*. Modern applications often depend on a combination of system libraries and binaries, language-specific packages, framework-specific modules, - internal components developed for another project, etc. These dependencies live in different "worlds" and require different tools - these tools typically don't work - well with each other, requiring awkward custom integrations. + * *Cross-platform dependencies*. Modern applications often depend on + a combination of system libraries and binaries, language-specific + packages, framework-specific modules, internal components + developed for another project, etc. These dependencies live in + different "worlds" and require different tools - these tools + typically don't work well with each other, requiring awkward + custom integrations. - * Conflicting dependencies. Different applications may depend on different versions of the same dependency. Packaging tools handle these situations with various degrees of ease - - but they all handle them in different and incompatible ways, which again forces the developer to do extra work. + * Conflicting dependencies. Different applications may depend on + different versions of the same dependency. Packaging tools handle + these situations with various degrees of ease - but they all + handle them in different and incompatible ways, which again forces + the developer to do extra work. - * Custom dependencies. A developer may need to prepare a custom version of their application's dependency. Some packaging systems can handle custom versions of a dependency, - others can't - and all of them handle it differently. + * Custom dependencies. A developer may need to prepare a custom + version of their application's dependency. Some packaging systems + can handle custom versions of a dependency, others can't - and all + of them handle it differently. -Docker solves dependency hell by giving the developer a simple way to express *all* their application's dependencies in one place, -and streamline the process of assembling them. If this makes you think of [XKCD 927](http://xkcd.com/927/), don't worry. Docker doesn't -*replace* your favorite packaging systems. It simply orchestrates their use in a simple and repeatable way. How does it do that? With layers. +Docker solves dependency hell by giving the developer a simple way to +express *all* their application's dependencies in one place, and +streamline the process of assembling them. If this makes you think of +[XKCD 927](http://xkcd.com/927/), don't worry. Docker doesn't +*replace* your favorite packaging systems. It simply orchestrates +their use in a simple and repeatable way. How does it do that? With +layers. -Docker defines a build as running a sequence of unix commands, one after the other, in the same container. Build commands modify the contents of the container -(usually by installing new files on the filesystem), the next command modifies it some more, etc. Since each build command inherits the result of the previous -commands, the *order* in which the commands are executed expresses *dependencies*. +Docker defines a build as running a sequence of Unix commands, one +after the other, in the same container. Build commands modify the +contents of the container (usually by installing new files on the +filesystem), the next command modifies it some more, etc. Since each +build command inherits the result of the previous commands, the +*order* in which the commands are executed expresses *dependencies*. -Here's a typical docker build process: +Here's a typical Docker build process: ```bash from ubuntu:12.10 @@ -87,7 +136,8 @@ run curl -L https://github.com/shykes/helloflask/archive/master.tar.gz | tar -xz run cd helloflask-master && pip install -r requirements.txt ``` -Note that Docker doesn't care *how* dependencies are built - as long as they can be built by running a unix command in a container. +Note that Docker doesn't care *how* dependencies are built - as long +as they can be built by running a Unix command in a container. Install instructions @@ -103,8 +153,9 @@ curl get.docker.io | sudo sh -x Binary installs ---------------- -Docker supports the following binary installation methods. -Note that some methods are community contributions and not yet officially supported. +Docker supports the following binary installation methods. Note that +some methods are community contributions and not yet officially +supported. * [Ubuntu 12.04 and 12.10 (officially supported)](http://docs.docker.io/en/latest/installation/ubuntulinux/) * [Arch Linux](http://docs.docker.io/en/latest/installation/archlinux/) @@ -115,15 +166,15 @@ Note that some methods are community contributions and not yet officially suppor Installing from source ---------------------- -1. Make sure you have a [Go language](http://golang.org/doc/install) compiler and [git](http://git-scm.com) installed. - +1. Make sure you have a [Go language](http://golang.org/doc/install) +compiler and [git](http://git-scm.com) installed. 2. Checkout the source code ```bash git clone http://github.com/dotcloud/docker ``` -3. Build the docker binary +3. Build the ``docker`` binary ```bash cd docker @@ -134,17 +185,20 @@ Installing from source Usage examples ============== -First run the docker daemon ---------------------------- +First run the ``docker`` daemon +------------------------------- -All the examples assume your machine is running the docker daemon. To run the docker daemon in the background, simply type: +All the examples assume your machine is running the ``docker`` +daemon. To run the ``docker`` daemon in the background, simply type: ```bash # On a production system you want this running in an init script sudo docker -d & ``` -Now you can run docker in client mode: all commands will be forwarded to the docker daemon, so the client can run from any account. +Now you can run ``docker`` in client mode: all commands will be +forwarded to the ``docker`` daemon, so the client can run from any +account. ```bash # Now you can run docker commands from any account. @@ -152,7 +206,7 @@ docker help ``` -Throwaway shell in a base ubuntu image +Throwaway shell in a base Ubuntu image -------------------------------------- ```bash @@ -202,7 +256,8 @@ docker commit -m "Installed curl" $CONTAINER $USER/betterbase docker push $USER/betterbase ``` -A list of publicly available images is [available here](https://github.com/dotcloud/docker/wiki/Public-docker-images). +A list of publicly available images is [available +here](https://github.com/dotcloud/docker/wiki/Public-docker-images). Expose a service on a TCP port ------------------------------ @@ -229,32 +284,40 @@ Under the hood Under the hood, Docker is built on the following components: - -* The [cgroup](http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-24-c) and [namespacing](http://blog.dotcloud.com/under-the-hood-linux-kernels-on-dotcloud-part) capabilities of the Linux kernel; - -* [AUFS](http://aufs.sourceforge.net/aufs.html), a powerful union filesystem with copy-on-write capabilities; - +* The + [cgroup](http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-24-c) + and + [namespacing](http://blog.dotcloud.com/under-the-hood-linux-kernels-on-dotcloud-part) + capabilities of the Linux kernel; +* [AUFS](http://aufs.sourceforge.net/aufs.html), a powerful union + filesystem with copy-on-write capabilities; * The [Go](http://golang.org) programming language; - -* [lxc](http://lxc.sourceforge.net/), a set of convenience scripts to simplify the creation of linux containers. +* [lxc](http://lxc.sourceforge.net/), a set of convenience scripts to + simplify the creation of Linux containers. Contributing to Docker ====================== -Want to hack on Docker? Awesome! There are instructions to get you started on the website: http://docs.docker.io/en/latest/contributing/contributing/ +Want to hack on Docker? Awesome! There are instructions to get you +started on the website: +http://docs.docker.io/en/latest/contributing/contributing/ -They are probably not perfect, please let us know if anything feels wrong or incomplete. +They are probably not perfect, please let us know if anything feels +wrong or incomplete. Note ---- -We also keep the documentation in this repository. The website documentation is generated using sphinx using these sources. -Please find it under docs/sources/ and read more about it https://github.com/dotcloud/docker/tree/master/docs/README.md +We also keep the documentation in this repository. The website +documentation is generated using Sphinx using these sources. Please +find it under docs/sources/ and read more about it +https://github.com/dotcloud/docker/tree/master/docs/README.md -Please feel free to fix / update the documentation and send us pull requests. More tutorials are also welcome. +Please feel free to fix / update the documentation and send us pull +requests. More tutorials are also welcome. Setting up a dev environment @@ -289,42 +352,96 @@ Run the `go install` command (above) to recompile docker. What is a Standard Container? ============================= -Docker defines a unit of software delivery called a Standard Container. The goal of a Standard Container is to encapsulate a software component and all its dependencies in -a format that is self-describing and portable, so that any compliant runtime can run it without extra dependencies, regardless of the underlying machine and the contents of the container. +Docker defines a unit of software delivery called a Standard +Container. The goal of a Standard Container is to encapsulate a +software component and all its dependencies in a format that is +self-describing and portable, so that any compliant runtime can run it +without extra dependencies, regardless of the underlying machine and +the contents of the container. -The spec for Standard Containers is currently a work in progress, but it is very straightforward. It mostly defines 1) an image format, 2) a set of standard operations, and 3) an execution environment. +The spec for Standard Containers is currently a work in progress, but +it is very straightforward. It mostly defines 1) an image format, 2) a +set of standard operations, and 3) an execution environment. -A great analogy for this is the shipping container. Just like how Standard Containers are a fundamental unit of software delivery, shipping containers (http://bricks.argz.com/ins/7823-1/12) are a fundamental unit of physical delivery. +A great analogy for this is the shipping container. Just like how +Standard Containers are a fundamental unit of software delivery, +shipping containers are a fundamental unit of physical delivery. ### 1. STANDARD OPERATIONS -Just like shipping containers, Standard Containers define a set of STANDARD OPERATIONS. Shipping containers can be lifted, stacked, locked, loaded, unloaded and labelled. Similarly, standard containers can be started, stopped, copied, snapshotted, downloaded, uploaded and tagged. +Just like shipping containers, Standard Containers define a set of +STANDARD OPERATIONS. Shipping containers can be lifted, stacked, +locked, loaded, unloaded and labelled. Similarly, Standard Containers +can be started, stopped, copied, snapshotted, downloaded, uploaded and +tagged. ### 2. CONTENT-AGNOSTIC -Just like shipping containers, Standard Containers are CONTENT-AGNOSTIC: all standard operations have the same effect regardless of the contents. A shipping container will be stacked in exactly the same way whether it contains Vietnamese powder coffee or spare Maserati parts. Similarly, Standard Containers are started or uploaded in the same way whether they contain a postgres database, a php application with its dependencies and application server, or Java build artifacts. +Just like shipping containers, Standard Containers are +CONTENT-AGNOSTIC: all standard operations have the same effect +regardless of the contents. A shipping container will be stacked in +exactly the same way whether it contains Vietnamese powder coffee or +spare Maserati parts. Similarly, Standard Containers are started or +uploaded in the same way whether they contain a postgres database, a +php application with its dependencies and application server, or Java +build artifacts. ### 3. INFRASTRUCTURE-AGNOSTIC -Both types of containers are INFRASTRUCTURE-AGNOSTIC: they can be transported to thousands of facilities around the world, and manipulated by a wide variety of equipment. A shipping container can be packed in a factory in Ukraine, transported by truck to the nearest routing center, stacked onto a train, loaded into a German boat by an Australian-built crane, stored in a warehouse at a US facility, etc. Similarly, a standard container can be bundled on my laptop, uploaded to S3, downloaded, run and snapshotted by a build server at Equinix in Virginia, uploaded to 10 staging servers in a home-made Openstack cluster, then sent to 30 production instances across 3 EC2 regions. +Both types of containers are INFRASTRUCTURE-AGNOSTIC: they can be +transported to thousands of facilities around the world, and +manipulated by a wide variety of equipment. A shipping container can +be packed in a factory in Ukraine, transported by truck to the nearest +routing center, stacked onto a train, loaded into a German boat by an +Australian-built crane, stored in a warehouse at a US facility, +etc. Similarly, a standard container can be bundled on my laptop, +uploaded to S3, downloaded, run and snapshotted by a build server at +Equinix in Virginia, uploaded to 10 staging servers in a home-made +Openstack cluster, then sent to 30 production instances across 3 EC2 +regions. ### 4. DESIGNED FOR AUTOMATION -Because they offer the same standard operations regardless of content and infrastructure, Standard Containers, just like their physical counterpart, are extremely well-suited for automation. In fact, you could say automation is their secret weapon. +Because they offer the same standard operations regardless of content +and infrastructure, Standard Containers, just like their physical +counterparts, are extremely well-suited for automation. In fact, you +could say automation is their secret weapon. -Many things that once required time-consuming and error-prone human effort can now be programmed. Before shipping containers, a bag of powder coffee was hauled, dragged, dropped, rolled and stacked by 10 different people in 10 different locations by the time it reached its destination. 1 out of 50 disappeared. 1 out of 20 was damaged. The process was slow, inefficient and cost a fortune - and was entirely different depending on the facility and the type of goods. +Many things that once required time-consuming and error-prone human +effort can now be programmed. Before shipping containers, a bag of +powder coffee was hauled, dragged, dropped, rolled and stacked by 10 +different people in 10 different locations by the time it reached its +destination. 1 out of 50 disappeared. 1 out of 20 was damaged. The +process was slow, inefficient and cost a fortune - and was entirely +different depending on the facility and the type of goods. -Similarly, before Standard Containers, by the time a software component ran in production, it had been individually built, configured, bundled, documented, patched, vendored, templated, tweaked and instrumented by 10 different people on 10 different computers. Builds failed, libraries conflicted, mirrors crashed, post-it notes were lost, logs were misplaced, cluster updates were half-broken. The process was slow, inefficient and cost a fortune - and was entirely different depending on the language and infrastructure provider. +Similarly, before Standard Containers, by the time a software +component ran in production, it had been individually built, +configured, bundled, documented, patched, vendored, templated, tweaked +and instrumented by 10 different people on 10 different +computers. Builds failed, libraries conflicted, mirrors crashed, +post-it notes were lost, logs were misplaced, cluster updates were +half-broken. The process was slow, inefficient and cost a fortune - +and was entirely different depending on the language and +infrastructure provider. ### 5. INDUSTRIAL-GRADE DELIVERY -There are 17 million shipping containers in existence, packed with every physical good imaginable. Every single one of them can be loaded onto the same boats, by the same cranes, in the same facilities, and sent anywhere in the World with incredible efficiency. It is embarrassing to think that a 30 ton shipment of coffee can safely travel half-way across the World in *less time* than it takes a software team to deliver its code from one datacenter to another sitting 10 miles away. +There are 17 million shipping containers in existence, packed with +every physical good imaginable. Every single one of them can be loaded +onto the same boats, by the same cranes, in the same facilities, and +sent anywhere in the World with incredible efficiency. It is +embarrassing to think that a 30 ton shipment of coffee can safely +travel half-way across the World in *less time* than it takes a +software team to deliver its code from one datacenter to another +sitting 10 miles away. -With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. +With Standard Containers we can put an end to that embarrassment, by +making INDUSTRIAL-GRADE DELIVERY of software a reality. @@ -374,8 +491,10 @@ Standard Container Specification ### Legal -Transfers of Docker shall be in accordance with applicable export controls of any country and all other applicable -legal requirements. Docker shall not be distributed or downloaded to or in Cuba, Iran, North Korea, Sudan or Syria -and shall not be distributed or downloaded to any person on the Denied Persons List administered by the U.S. +Transfers of Docker shall be in accordance with applicable export +controls of any country and all other applicable legal requirements. +Docker shall not be distributed or downloaded to or in Cuba, Iran, +North Korea, Sudan or Syria and shall not be distributed or downloaded +to any person on the Denied Persons List administered by the U.S. Department of Commerce. From 743adaf705fb847ff7dd4837752df2e65bfbc87a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 24 Jul 2013 13:35:38 +0000 Subject: [PATCH 014/134] add kernel version Upstream-commit: 6057e6ad70ba4550df358e89b9a7e1da61666604 Component: engine --- components/engine/api_params.go | 1 + components/engine/commands.go | 1 + .../engine/docs/sources/api/docker_remote_api_v1.3.rst | 5 ++++- components/engine/server.go | 5 +++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/components/engine/api_params.go b/components/engine/api_params.go index fcffb04346..0214b89fdb 100644 --- a/components/engine/api_params.go +++ b/components/engine/api_params.go @@ -26,6 +26,7 @@ type APIInfo struct { SwapLimit bool `json:",omitempty"` LXCVersion string `json:",omitempty"` NEventsListener int `json:",omitempty"` + KernelVersion string `json:",omitempty"` } type APITop struct { diff --git a/components/engine/commands.go b/components/engine/commands.go index c0e349c8a3..a868b5e5b2 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -473,6 +473,7 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "Goroutines: %d\n", out.NGoroutines) fmt.Fprintf(cli.out, "LXC Version: %s\n", out.LXCVersion) fmt.Fprintf(cli.out, "EventsListeners: %d\n", out.NEventsListener) + fmt.Fprintf(cli.out, "Kernel Version: %s\n", out.KernelVersion) } if !out.MemoryLimit { fmt.Fprintf(cli.err, "WARNING: No memory limit support\n") diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst index 69f480e453..401855365e 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst @@ -989,7 +989,10 @@ Display system-wide information "NFd": 11, "NGoroutines":21, "MemoryLimit":true, - "SwapLimit":false + "SwapLimit":false, + "EventsListeners":"0", + "LXCVersion":"0.7.5", + "KernelVersion":"3.8.0-19-generic" } :statuscode 200: no error diff --git a/components/engine/server.go b/components/engine/server.go index 1395007911..147c5fa436 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -218,6 +218,10 @@ func (srv *Server) DockerInfo() *APIInfo { lxcVersion = strings.TrimSpace(strings.SplitN(string(output), ":", 2)[1]) } } + kernelVersion := "" + if kv, err := utils.GetKernelVersion(); err == nil { + kernelVersion = kv.String() + } return &APIInfo{ Containers: len(srv.runtime.List()), @@ -229,6 +233,7 @@ func (srv *Server) DockerInfo() *APIInfo { NGoroutines: runtime.NumGoroutine(), LXCVersion: lxcVersion, NEventsListener: len(srv.events), + KernelVersion: kernelVersion, } } From 53749f98cec05feb9a97643af5bf2c1bd592e32b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 24 Jul 2013 15:41:34 +0000 Subject: [PATCH 015/134] Add ID to JSONMessage in pull Use goroutines to pull in parallel If multiple images pulled at the same time, each progress is displayed on a new line Upstream-commit: 0e71e368a8a781f593b25fdd1318d3882e6d28e5 Component: engine --- components/engine/commands.go | 2 +- components/engine/graph.go | 2 +- components/engine/server.go | 63 +++++++++++++++++++------------- components/engine/utils/utils.go | 13 +++++-- 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 2d8ea4efb5..946c83dd11 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -196,7 +196,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { // FIXME: ProgressReader shouldn't be this annoyning to use if context != nil { sf := utils.NewStreamFormatter(false) - body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s"), sf) + body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s", ""), sf) } // Upload the build context v := &url.Values{} diff --git a/components/engine/graph.go b/components/engine/graph.go index 42d1bdbd4c..3ae342e7c5 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -175,7 +175,7 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *uti if err != nil { return nil, err } - return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("Buffering to disk", "%v/%v (%v)"), sf), tmp.Root) + return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("Buffering to disk", "%v/%v (%v)", ""), sf), tmp.Root) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. diff --git a/components/engine/server.go b/components/engine/server.go index 4179a1e160..1a221d3f00 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -145,7 +145,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. return "", err } - if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), path); err != nil { + if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("Downloading", "%8v/%v (%v)", ""), sf), path); err != nil { return "", err } // FIXME: Handle custom repo, tag comment, author @@ -425,7 +425,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return err } defer layer.Close() - if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)", id), sf), false, img); err != nil { return err } } @@ -477,30 +477,43 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName repoData.ImgList[id].Tag = askedTag } - for _, img := range repoData.ImgList { - if askedTag != "" && img.Tag != askedTag { - utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) - continue - } - - if img.Tag == "" { - utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) - continue - } - out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName)) - success := false - for _, ep := range repoData.Endpoints { - if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { - out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) - continue + errors := make(chan error) + for _, image := range repoData.ImgList { + go func(img *registry.ImgData) { + if askedTag != "" && img.Tag != askedTag { + utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) + errors <- nil + return } - success = true - break - } - if !success { - return fmt.Errorf("Could not find repository on any of the indexed registries.") + + if img.Tag == "" { + utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) + errors <- nil + return + } + out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName)) + success := false + for _, ep := range repoData.Endpoints { + if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { + out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) + continue + } + success = true + break + } + if !success { + errors <- fmt.Errorf("Could not find repository on any of the indexed registries.") + } + errors <- nil + }(image) + } + + for i := 0; i < len(repoData.ImgList); i++ { + if err := <-errors; err != nil { + return err } } + for tag, id := range tagsList { if askedTag != "" && tag != askedTag { continue @@ -748,7 +761,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } // Send the layer - if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)", ""), sf), ep, token); err != nil { return err } return nil @@ -818,7 +831,7 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write if err != nil { return err } - archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("Importing", "%8v/%v (%v)"), sf) + archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("Importing", "%8v/%v (%v)", ""), sf) } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil) if err != nil { diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index acb015becd..ffba2352a7 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -107,7 +107,7 @@ func (r *progressReader) Close() error { func ProgressReader(r io.ReadCloser, size int, output io.Writer, template []byte, sf *StreamFormatter) *progressReader { tpl := string(template) if tpl == "" { - tpl = string(sf.FormatProgress("", "%8v/%v (%v)")) + tpl = string(sf.FormatProgress("", "%8v/%v (%v)", "")) } return &progressReader{r, NewWriteFlusher(output), size, 0, 0, tpl, sf} } @@ -587,11 +587,14 @@ type NopFlusher struct{} func (f *NopFlusher) Flush() {} type WriteFlusher struct { + sync.Mutex w io.Writer flusher http.Flusher } func (wf *WriteFlusher) Write(b []byte) (n int, err error) { + wf.Lock() + defer wf.Unlock() n, err = wf.w.Write(b) wf.flusher.Flush() return n, err @@ -619,7 +622,9 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } - if jm.Progress != "" { + if jm.Progress != "" && jm.ID != ""{ + fmt.Fprintf(out, "\n%s %s %s\r", jm.Status, jm.ID, jm.Progress) + } else if jm.Progress != "" { fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress) } else if jm.Error != "" { return fmt.Errorf(jm.Error) @@ -665,10 +670,10 @@ func (sf *StreamFormatter) FormatError(err error) []byte { return []byte("Error: " + err.Error() + "\r\n") } -func (sf *StreamFormatter) FormatProgress(action, str string) []byte { +func (sf *StreamFormatter) FormatProgress(action, str, id string) []byte { sf.used = true if sf.json { - b, err := json.Marshal(&JSONMessage{Status: action, Progress: str}) + b, err := json.Marshal(&JSONMessage{Status: action, Progress: str, ID:id}) if err != nil { return nil } From a761fb8db0311ab8c9ff65d4b18d3d3c500ba799 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 24 Jul 2013 17:10:59 +0000 Subject: [PATCH 016/134] improve client output Upstream-commit: 8742649aa7f3524bbfa99b68c8d87ffc5aba0af9 Component: engine --- components/engine/commands.go | 7 +++-- components/engine/graph.go | 2 +- components/engine/server.go | 44 ++++++++++++++++---------------- components/engine/utils/utils.go | 30 +++++++++------------- 4 files changed, 40 insertions(+), 43 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 946c83dd11..2a672d49e3 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -196,7 +196,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { // FIXME: ProgressReader shouldn't be this annoyning to use if context != nil { sf := utils.NewStreamFormatter(false) - body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s", ""), sf) + body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("", "Uploading context", "%v bytes%0.0s%0.0s"), sf) } // Upload the build context v := &url.Values{} @@ -1537,8 +1537,8 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e if resp.Header.Get("Content-Type") == "application/json" { dec := json.NewDecoder(resp.Body) + jm := utils.JSONMessage{} for { - var jm utils.JSONMessage if err := dec.Decode(&jm); err == io.EOF { break } else if err != nil { @@ -1546,6 +1546,9 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e } jm.Display(out) } + if jm.Progress != "" { + fmt.Fprintf(out, "\n") + } } else { if _, err := io.Copy(out, resp.Body); err != nil { return err diff --git a/components/engine/graph.go b/components/engine/graph.go index 3ae342e7c5..5b9162b871 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -175,7 +175,7 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *uti if err != nil { return nil, err } - return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("Buffering to disk", "%v/%v (%v)", ""), sf), tmp.Root) + return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf), tmp.Root) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. diff --git a/components/engine/server.go b/components/engine/server.go index 1a221d3f00..a1d22d3c44 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -145,7 +145,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. return "", err } - if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("Downloading", "%8v/%v (%v)", ""), sf), path); err != nil { + if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("", "Downloading", "%8v/%v (%v)"), sf), path); err != nil { return "", err } // FIXME: Handle custom repo, tag comment, author @@ -153,7 +153,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. if err != nil { return "", err } - out.Write(sf.FormatStatus(img.ID)) + out.Write(sf.FormatStatus("", img.ID)) return img.ShortID(), nil } @@ -407,7 +407,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin // FIXME: Launch the getRemoteImage() in goroutines for _, id := range history { if !srv.runtime.graph.Exists(id) { - out.Write(sf.FormatStatus("Pulling %s metadata", id)) + out.Write(sf.FormatStatus(utils.TruncateID(id), "Pulling metadata")) imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token) if err != nil { // FIXME: Keep goging in case of error? @@ -419,13 +419,13 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin } // Get the layer - out.Write(sf.FormatStatus("Pulling %s fs layer", id)) + out.Write(sf.FormatStatus(utils.TruncateID(id), "Pulling fs layer")) layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token) if err != nil { return err } defer layer.Close() - if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)", id), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { return err } } @@ -434,7 +434,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin } func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error { - out.Write(sf.FormatStatus("Pulling repository %s", localName)) + out.Write(sf.FormatStatus("", "Pulling repository %s", localName)) repoData, err := r.GetRepositoryData(indexEp, remoteName) if err != nil { @@ -491,11 +491,11 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName errors <- nil return } - out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName)) + out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Pulling image (%s) from %s", img.Tag, localName)) success := false for _, ep := range repoData.Endpoints { if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { - out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) + out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) continue } success = true @@ -665,12 +665,12 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) - out.Write(sf.FormatStatus("Processing checksums")) + out.Write(sf.FormatStatus("", "Processing checksums")) imgList, err := srv.getImageList(localRepo) if err != nil { return err } - out.Write(sf.FormatStatus("Sending image list")) + out.Write(sf.FormatStatus("", "Sending image list")) var repoData *registry.RepositoryData repoData, err = r.PushImageJSONIndex(indexEp, remoteName, imgList, false, nil) @@ -679,21 +679,21 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName } for _, ep := range repoData.Endpoints { - out.Write(sf.FormatStatus("Pushing repository %s (%d tags)", localName, len(localRepo))) + out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", localName, len(localRepo))) // For each image within the repo, push them for _, elem := range imgList { if _, exists := repoData.ImgList[elem.ID]; exists { - out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) + out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID)) continue } else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) { - out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) + out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID)) continue } if err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? return err } - out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) + out.Write(sf.FormatStatus("", "Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { return err } @@ -713,7 +713,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, if err != nil { return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) } - out.Write(sf.FormatStatus("Pushing %s", imgID)) + out.Write(sf.FormatStatus("", "Pushing %s", imgID)) // Make sure we have the image's checksum checksum, err := srv.getChecksum(imgID) @@ -728,7 +728,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, // Send the json if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { - out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID)) + out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", imgData.ID)) return nil } return err @@ -761,7 +761,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } // Send the layer - if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)", ""), sf), ep, token); err != nil { + if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { return err } return nil @@ -789,7 +789,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo if err != nil { reposLen := len(srv.runtime.repositories.Repositories[localName]) - out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", localName, reposLen)) + out.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", localName, reposLen)) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[localName]; exists { if err := srv.pushRepository(r, out, localName, remoteName, localRepo, endpoint, sf); err != nil { @@ -801,7 +801,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo } var token []string - out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName)) + out.Write(sf.FormatStatus("", "The push refers to an image: [%s]", localName)) if err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { return err } @@ -824,14 +824,14 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write u.Host = src u.Path = "" } - out.Write(sf.FormatStatus("Downloading from %s", u)) + out.Write(sf.FormatStatus("", "Downloading from %s", u)) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() resp, err = utils.Download(u.String(), out) if err != nil { return err } - archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("Importing", "%8v/%v (%v)", ""), sf) + archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("", "Importing", "%8v/%v (%v)"), sf) } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil) if err != nil { @@ -843,7 +843,7 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write return err } } - out.Write(sf.FormatStatus(img.ShortID())) + out.Write(sf.FormatStatus("", img.ShortID())) return nil } diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index ffba2352a7..6361945eef 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -94,11 +94,6 @@ func (r *progressReader) Read(p []byte) (n int, err error) { } r.lastUpdate = r.readProgress } - // Send newline when complete - if err != nil { - r.output.Write(r.sf.FormatStatus("")) - } - return read, err } func (r *progressReader) Close() error { @@ -619,24 +614,23 @@ type JSONMessage struct { } func (jm *JSONMessage) Display(out io.Writer) (error) { + if jm.Error != "" { + return fmt.Errorf(jm.Error) + } if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } - if jm.Progress != "" && jm.ID != ""{ - fmt.Fprintf(out, "\n%s %s %s\r", jm.Status, jm.ID, jm.Progress) - } else if jm.Progress != "" { + if jm.ID != "" { + fmt.Fprintf(out, "%s: ", jm.ID) + } + if jm.Progress != "" { fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress) - } else if jm.Error != "" { - return fmt.Errorf(jm.Error) - } else if jm.ID != "" { - fmt.Fprintf(out, "%s: %s\n", jm.ID, jm.Status) } else { fmt.Fprintf(out, "%s\n", jm.Status) } return nil } - type StreamFormatter struct { json bool used bool @@ -646,11 +640,11 @@ func NewStreamFormatter(json bool) *StreamFormatter { return &StreamFormatter{json, false} } -func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte { +func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []byte { sf.used = true str := fmt.Sprintf(format, a...) if sf.json { - b, err := json.Marshal(&JSONMessage{Status: str}) + b, err := json.Marshal(&JSONMessage{ID: id, Status: str}) if err != nil { return sf.FormatError(err) } @@ -670,16 +664,16 @@ func (sf *StreamFormatter) FormatError(err error) []byte { return []byte("Error: " + err.Error() + "\r\n") } -func (sf *StreamFormatter) FormatProgress(action, str, id string) []byte { +func (sf *StreamFormatter) FormatProgress(id, action, progress string) []byte { sf.used = true if sf.json { - b, err := json.Marshal(&JSONMessage{Status: action, Progress: str, ID:id}) + b, err := json.Marshal(&JSONMessage{Status: action, Progress: progress, ID:id}) if err != nil { return nil } return b } - return []byte(action + " " + str + "\r") + return []byte(action + " " + progress + "\r") } func (sf *StreamFormatter) Used() bool { From 4350e3145d3ebd2877707937d6d0aa6725e8d37f Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Jul 2013 15:48:18 -0700 Subject: [PATCH 017/134] Fixes #505 - Make sure all output is send on the network before closing Upstream-commit: fd9ad1a19469d07944ca9b417861d63ecec2ef42 Component: engine --- components/engine/api_test.go | 8 +++++++- components/engine/container.go | 13 +++++++------ components/engine/runtime_test.go | 19 +++++++++++++------ components/engine/z_final_test.go | 9 +++++++-- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 13731dbf9e..a0724b0ba3 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -895,6 +895,12 @@ func TestPostContainersAttach(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() + // Try to avoid the timeoout in destroy. Best effort, don't check error + defer func() { + closeWrap(stdin, stdinPipe, stdout, stdoutPipe) + container.Kill() + }() + // Attach to it c1 := make(chan struct{}) go func() { @@ -934,7 +940,7 @@ func TestPostContainersAttach(t *testing.T) { } // Wait for attach to finish, the client disconnected, therefore, Attach finished his job - setTimeout(t, "Waiting for CmdAttach timed out", 2*time.Second, func() { + setTimeout(t, "Waiting for CmdAttach timed out", 10*time.Second, func() { <-c1 }) diff --git a/components/engine/container.go b/components/engine/container.go index dea81b6def..ec4abffc1b 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -379,14 +379,15 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s utils.Debugf("[start] attach stdin\n") defer utils.Debugf("[end] attach stdin\n") // No matter what, when stdin is closed (io.Copy unblock), close stdout and stderr - if cStdout != nil { - defer cStdout.Close() - } - if cStderr != nil { - defer cStderr.Close() - } if container.Config.StdinOnce && !container.Config.Tty { defer cStdin.Close() + } else { + if cStdout != nil { + defer cStdout.Close() + } + if cStderr != nil { + defer cStderr.Close() + } } if container.Config.Tty { _, err = utils.CopyEscapable(cStdin, stdin) diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index 807097404d..0b0f62f199 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -8,6 +8,7 @@ import ( "log" "net" "os" + "runtime" "strconv" "strings" "sync" @@ -25,7 +26,11 @@ const ( testDaemonProto = "tcp" ) -var globalRuntime *Runtime +var ( + globalRuntime *Runtime + startFds int + startGoroutines int +) func nuke(runtime *Runtime) error { var wg sync.WaitGroup @@ -80,21 +85,21 @@ func init() { NetworkBridgeIface = unitTestNetworkBridge // Make it our Store root - runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false) - if err != nil { + if runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false); err != nil { panic(err) + } else { + globalRuntime = runtime } - globalRuntime = runtime // Create the "Server" srv := &Server{ - runtime: runtime, + runtime: globalRuntime, enableCors: false, pullingPool: make(map[string]struct{}), pushingPool: make(map[string]struct{}), } // If the unit test is not found, try to download it. - if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID { + if img, err := globalRuntime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID { // Retrieve the Image if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil { panic(err) @@ -109,6 +114,8 @@ func init() { // Give some time to ListenAndServer to actually start time.Sleep(time.Second) + + startFds, startGoroutines = utils.GetTotalUsedFds(), runtime.NumGoroutine() } // FIXME: test that ImagePull(json=true) send correct json output diff --git a/components/engine/z_final_test.go b/components/engine/z_final_test.go index 78a7acf6e7..08a180baaf 100644 --- a/components/engine/z_final_test.go +++ b/components/engine/z_final_test.go @@ -6,7 +6,12 @@ import ( "testing" ) -func TestFinal(t *testing.T) { - cleanup(globalRuntime) +func displayFdGoroutines(t *testing.T) { t.Logf("Fds: %d, Goroutines: %d", utils.GetTotalUsedFds(), runtime.NumGoroutine()) } + +func TestFinal(t *testing.T) { + cleanup(globalRuntime) + t.Logf("Start Fds: %d, Start Goroutines: %d", startFds, startGoroutines) + displayFdGoroutines(t) +} From 164f76762a891b2a371d329a5efb2445ada505f1 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 25 Jul 2013 00:35:52 +0000 Subject: [PATCH 018/134] Copy authConfigs on save so data is not modified SaveConfig sets the Username and Password to an empty string on save. A copy of the authConfigs need to be made so that the in memory data is not modified. Upstream-commit: 9332c00ca562e97045490d3d45d8f805fae30330 Component: engine --- components/engine/auth/auth.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index bffed49807..39de876875 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -116,14 +116,19 @@ func SaveConfig(configFile *ConfigFile) error { os.Remove(confFile) return nil } + + configs := make(map[string]AuthConfig, len(configFile.Configs)) for k, authConfig := range configFile.Configs { - authConfig.Auth = encodeAuth(&authConfig) - authConfig.Username = "" - authConfig.Password = "" - configFile.Configs[k] = authConfig + authCopy := authConfig + + authCopy.Auth = encodeAuth(&authCopy) + authCopy.Username = "" + authCopy.Password = "" + + configs[k] = authCopy } - b, err := json.Marshal(configFile.Configs) + b, err := json.Marshal(configs) if err != nil { return err } From e6f304a5c604455cad0d51c8ad27367ee0c243e8 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 25 Jul 2013 03:25:16 +0000 Subject: [PATCH 019/134] Add regression test for authConfig overwrite Upstream-commit: 0fc11699ab26121e4f89808ffacb2becf536ff5d Component: engine --- components/engine/auth/auth_test.go | 39 ++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/components/engine/auth/auth_test.go b/components/engine/auth/auth_test.go index 458a505ea2..d94d429da1 100644 --- a/components/engine/auth/auth_test.go +++ b/components/engine/auth/auth_test.go @@ -3,6 +3,7 @@ package auth import ( "crypto/rand" "encoding/hex" + "io/ioutil" "os" "strings" "testing" @@ -51,7 +52,7 @@ func TestCreateAccount(t *testing.T) { } token := hex.EncodeToString(tokenBuffer)[:12] username := "ut" + token - authConfig := &AuthConfig{Username: username, Password: "test42", Email: "docker-ut+"+token+"@example.com"} + authConfig := &AuthConfig{Username: username, Password: "test42", Email: "docker-ut+" + token + "@example.com"} status, err := Login(authConfig) if err != nil { t.Fatal(err) @@ -73,3 +74,39 @@ func TestCreateAccount(t *testing.T) { t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err) } } + +func TestSameAuthDataPostSave(t *testing.T) { + root, err := ioutil.TempDir("", "docker-test") + if err != nil { + t.Fatal(err) + } + configFile := &ConfigFile{ + rootPath: root, + Configs: make(map[string]AuthConfig, 1), + } + + configFile.Configs["testIndex"] = AuthConfig{ + Username: "docker-user", + Password: "docker-pass", + Email: "docker@docker.io", + } + + err = SaveConfig(configFile) + if err != nil { + t.Fatal(err) + } + + authConfig := configFile.Configs["testIndex"] + if authConfig.Username != "docker-user" { + t.Fail() + } + if authConfig.Password != "docker-pass" { + t.Fail() + } + if authConfig.Email != "docker@docker.io" { + t.Fail() + } + if authConfig.Auth != "" { + t.Fail() + } +} From c0c3fbc7b6cf7beefd4ddf99351940fb6db525ca Mon Sep 17 00:00:00 2001 From: Daniel YC Lin Date: Thu, 25 Jul 2013 15:36:32 +0800 Subject: [PATCH 020/134] Fixes #1286 Upstream-commit: 8f6b6d57840410d1491321d7681ef2a946e90bc9 Component: engine --- components/engine/commands.go | 2 +- .../engine/docs/sources/commandline/command/import.rst | 7 ++++--- components/engine/packaging/debian/lxc-docker.1 | 6 ++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index f0e1695b3f..2e346e2034 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -756,7 +756,7 @@ func (cli *DockerCli) CmdKill(args ...string) error { } func (cli *DockerCli) CmdImport(args ...string) error { - cmd := Subcmd("import", "URL|- [REPOSITORY [TAG]]", "Create a new filesystem image from the contents of a tarball") + cmd := Subcmd("import", "URL|- [REPOSITORY [TAG]]", "Create a new filesystem image from the contents of a tarball(.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz).") if err := cmd.Parse(args); err != nil { return nil diff --git a/components/engine/docs/sources/commandline/command/import.rst b/components/engine/docs/sources/commandline/command/import.rst index 66bcf5de52..0083068e10 100644 --- a/components/engine/docs/sources/commandline/command/import.rst +++ b/components/engine/docs/sources/commandline/command/import.rst @@ -12,8 +12,9 @@ Create a new filesystem image from the contents of a tarball -At this time, the URL must start with ``http`` and point to a single file archive (.tar, .tar.gz, .bzip) -containing a root filesystem. If you would like to import from a local directory or archive, +At this time, the URL must start with ``http`` and point to a single file archive +(.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) +containing a root filesystem. If you would like to import from a local directory or archive, you can use the ``-`` parameter to take the data from standard in. Examples @@ -30,7 +31,7 @@ Import from a local file Import to docker via pipe and standard in ``$ cat exampleimage.tgz | docker import - exampleimagelocal`` - + Import from a local directory ............................. diff --git a/components/engine/packaging/debian/lxc-docker.1 b/components/engine/packaging/debian/lxc-docker.1 index 8f98785285..cc20299fad 100644 --- a/components/engine/packaging/debian/lxc-docker.1 +++ b/components/engine/packaging/debian/lxc-docker.1 @@ -923,6 +923,12 @@ List images Usage: docker import [OPTIONS] URL|\- [REPOSITORY [TAG]] .sp Create a new filesystem image from the contents of a tarball + +At this time, the URL must start with ``http`` and point to a single file archive +(.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) +containing a root filesystem. If you would like to import from a local directory or archive, +you can use the ``-`` parameter to take the data from standard in. + .SS info .sp .nf From ce008bc61285c2071975ea427e72bd8678fccebc Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 24 Jul 2013 20:16:02 +0000 Subject: [PATCH 021/134] Use VT100 escape codes : Upstream-commit: f1dd299227b15696872822f40a4ab9f1a54098a7 Component: engine --- components/engine/commands.go | 14 +----------- components/engine/utils/utils.go | 38 +++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 2a672d49e3..99d91ac771 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1536,19 +1536,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e } if resp.Header.Get("Content-Type") == "application/json" { - dec := json.NewDecoder(resp.Body) - jm := utils.JSONMessage{} - for { - if err := dec.Decode(&jm); err == io.EOF { - break - } else if err != nil { - return err - } - jm.Display(out) - } - if jm.Progress != "" { - fmt.Fprintf(out, "\n") - } + utils.DisplayJSONMessagesStream(resp.Body, out) } else { if _, err := io.Copy(out, resp.Body); err != nil { return err diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 6361945eef..6081935b41 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -617,6 +617,7 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { if jm.Error != "" { return fmt.Errorf(jm.Error) } + fmt.Fprintf(out, "%c[2K", 27) if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -626,8 +627,43 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { if jm.Progress != "" { fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress) } else { - fmt.Fprintf(out, "%s\n", jm.Status) + fmt.Fprintf(out, "%s\r", jm.Status) } + if jm.ID == "" { + fmt.Fprintf(out, "\n") + } + return nil +} + +func DisplayJSONMessagesStream(in io.Reader, out io.Writer) error { + dec := json.NewDecoder(in) + jm := JSONMessage{} + ids := make(map[string]int) + diff := 0 + for { + if err := dec.Decode(&jm); err == io.EOF { + break + } else if err != nil { + return err + } + if jm.ID != "" { + line, ok := ids[jm.ID] + if !ok { + line = len(ids) + ids[jm.ID] = line + fmt.Fprintf(out, "\n") + diff = 0 + } else { + diff = len(ids) - line + } + fmt.Fprintf(out, "%c[%dA", 27, diff) + } + jm.Display(out) + if jm.ID != "" { + fmt.Fprintf(out, "%c[%dB", 27, diff) + } + } +// fmt.Fprintf(out, "\n") return nil } From 1a3ff090dc05e3a53036838f04a85a128ff788d4 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 25 Jul 2013 14:32:46 +0000 Subject: [PATCH 022/134] fix errors Upstream-commit: 01e98bf0dd26a1de7fd280fd4b0f0a79aedd0cdd Component: engine --- components/engine/commands.go | 2 +- components/engine/utils/utils.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 99d91ac771..f0d8f61999 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1536,7 +1536,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e } if resp.Header.Get("Content-Type") == "application/json" { - utils.DisplayJSONMessagesStream(resp.Body, out) + return utils.DisplayJSONMessagesStream(resp.Body, out) } else { if _, err := io.Copy(out, resp.Body); err != nil { return err diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 6081935b41..a66319a1ba 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -658,12 +658,14 @@ func DisplayJSONMessagesStream(in io.Reader, out io.Writer) error { } fmt.Fprintf(out, "%c[%dA", 27, diff) } - jm.Display(out) + err := jm.Display(out) if jm.ID != "" { fmt.Fprintf(out, "%c[%dB", 27, diff) } + if err != nil { + return err + } } -// fmt.Fprintf(out, "\n") return nil } From 2cfc902d2966e236fe2c32855b6bbbe49de2b343 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 25 Jul 2013 15:18:34 +0000 Subject: [PATCH 023/134] ensure mount in commit Upstream-commit: f385f1860bdebe2434bb122bd5ac8fec85687970 Component: engine --- components/engine/builder.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/engine/builder.go b/components/engine/builder.go index ab07233cb8..420370b1e6 100644 --- a/components/engine/builder.go +++ b/components/engine/builder.go @@ -124,6 +124,10 @@ func (builder *Builder) Create(config *Config) (*Container, error) { func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { // FIXME: freeze the container before copying it to avoid data corruption? // FIXME: this shouldn't be in commands. + if err := container.EnsureMounted(); err != nil { + return nil, err + } + rwTar, err := container.ExportRw() if err != nil { return nil, err From 85f64e1df75a5e81b45ae149e44e30dc6d015b65 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 25 Jul 2013 15:20:56 +0000 Subject: [PATCH 024/134] add regression test + go fmt Upstream-commit: 48833c7b0784bc7055f29dd3448df3af93c462ec Component: engine --- components/engine/server_test.go | 21 +++++++++++++++++++++ components/engine/utils/utils.go | 7 +++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/engine/server_test.go b/components/engine/server_test.go index 8612b3fcea..f67b8fb57c 100644 --- a/components/engine/server_test.go +++ b/components/engine/server_test.go @@ -90,6 +90,27 @@ func TestCreateRm(t *testing.T) { } +func TestCommit(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + + srv := &Server{runtime: runtime} + + config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil) + if err != nil { + t.Fatal(err) + } + + id, err := srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + if _, err := srv.ContainerCommit(id, "testrepo", "testtag", "", "", config); err != nil { + t.Fatal(err) + } +} + func TestCreateStartRestartStopStartKillRm(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index acb015becd..c70e80b72e 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -611,11 +611,11 @@ type JSONMessage struct { Status string `json:"status,omitempty"` Progress string `json:"progress,omitempty"` Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -631,7 +631,6 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { return nil } - type StreamFormatter struct { json bool used bool From 15664f41616d48853dd8fd425b707f61160446d9 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 25 Jul 2013 15:45:15 +0000 Subject: [PATCH 025/134] use 0755 instead of 0700 Upstream-commit: 1c509f4350d943c6aa8b9bff8dcbed28ee803735 Component: engine --- components/engine/image.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/image.go b/components/engine/image.go index e1b1ac0418..5240ec776f 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -68,7 +68,7 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { } // Store the layer layer := layerPath(root) - if err := os.MkdirAll(layer, 0700); err != nil { + if err := os.MkdirAll(layer, 0755); err != nil { return err } From 795426f1f2b90b3ddabcbe8be4af37ee44737831 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 23 Jul 2013 11:37:13 -0700 Subject: [PATCH 026/134] Make sure the cookie is used in all registry queries Upstream-commit: 3425c1b84c3f58ac5bb2feb91c4901b61561c58c Component: engine --- components/engine/registry/registry.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index e6f4f592e2..adef1c7baa 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -109,7 +109,14 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) } - return c.Do(req) + res, err := c.Do(req) + if err != nil { + return nil, err + } + if len(res.Cookies()) > 0 { + c.Jar.SetCookies(req.URL, res.Cookies()) + } + return res, err } // Set the user agent field in the header based on the versions provided @@ -135,7 +142,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) @@ -182,7 +189,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -210,7 +217,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } @@ -231,7 +238,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } @@ -326,7 +333,7 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { // FIXME: try json with UTF8 - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } @@ -341,9 +348,6 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() - if len(res.Cookies()) > 0 { - r.client.Jar.SetCookies(req.URL, res.Cookies()) - } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { From 7c51899dd52c23c8ae59b5660314c2d0df5c5e7a Mon Sep 17 00:00:00 2001 From: Fareed Dudhia Date: Tue, 9 Jul 2013 07:01:45 +0000 Subject: [PATCH 027/134] Fixes 1136; Reopened from 1175 with latest changes. Upstream-commit: d86898b0142f2f9a834aa0c727b10d62ef647262 Component: engine --- components/engine/buildfile.go | 69 +++++++++++++++++++++++++---- components/engine/buildfile_test.go | 42 +++++++++++++++++- components/engine/utils/utils.go | 7 ++- 3 files changed, 103 insertions(+), 15 deletions(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 75ebdd7a7c..c5171aaa91 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -11,6 +11,7 @@ import ( "os" "path" "reflect" + "regexp" "strings" ) @@ -67,6 +68,9 @@ func (b *buildFile) CmdFrom(name string) error { } b.image = image.ID b.config = &Config{} + if b.config.Env == nil || len(b.config.Env) == 0 { + b.config.Env = append(b.config.Env, "HOME=/", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") + } return nil } @@ -112,6 +116,40 @@ func (b *buildFile) CmdRun(args string) error { return nil } +func (b *buildFile) FindEnvKey(key string) int { + for k, envVar := range b.config.Env { + envParts := strings.SplitN(envVar, "=", 2) + if key == envParts[0] { + return k + } + } + return -1 +} + +func (b *buildFile) ReplaceEnvMatches(value string) (string, error) { + exp, err := regexp.Compile("(\\\\\\\\+|[^\\\\]|\\b|\\A)\\$({?)([[:alnum:]_]+)(}?)") + if err != nil { + return value, err + } + matches := exp.FindAllString(value, -1) + for _, match := range matches { + match = match[strings.Index(match, "$"):] + matchKey := strings.Trim(match, "${}") + + for _, envVar := range b.config.Env { + envParts := strings.SplitN(envVar, "=", 2) + envKey := envParts[0] + envValue := envParts[1] + + if envKey == matchKey { + value = strings.Replace(value, match, envValue, -1) + break + } + } + } + return value, nil +} + func (b *buildFile) CmdEnv(args string) error { tmp := strings.SplitN(args, " ", 2) if len(tmp) != 2 { @@ -120,14 +158,19 @@ func (b *buildFile) CmdEnv(args string) error { key := strings.Trim(tmp[0], " \t") value := strings.Trim(tmp[1], " \t") - for i, elem := range b.config.Env { - if strings.HasPrefix(elem, key+"=") { - b.config.Env[i] = key + "=" + value - return nil - } + envKey := b.FindEnvKey(key) + replacedValue, err := b.ReplaceEnvMatches(value) + if err != nil { + return err } - b.config.Env = append(b.config.Env, key+"="+value) - return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s=%s", key, value)) + replacedVar := fmt.Sprintf("%s=%s", key, replacedValue) + + if envKey >= 0 { + b.config.Env[envKey] = replacedVar + return nil + } + b.config.Env = append(b.config.Env, replacedVar) + return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s", replacedVar)) } func (b *buildFile) CmdCmd(args string) error { @@ -260,8 +303,16 @@ func (b *buildFile) CmdAdd(args string) error { if len(tmp) != 2 { return fmt.Errorf("Invalid ADD format") } - orig := strings.Trim(tmp[0], " \t") - dest := strings.Trim(tmp[1], " \t") + + orig, err := b.ReplaceEnvMatches(strings.Trim(tmp[0], " \t")) + if err != nil { + return err + } + + dest, err := b.ReplaceEnvMatches(strings.Trim(tmp[1], " \t")) + if err != nil { + return err + } cmd := b.config.Cmd b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)} diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index b7eca52336..78e53b8419 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -129,6 +129,38 @@ CMD Hello world nil, nil, }, + + { + ` +from {IMAGE} +env FOO /foo/baz +env BAR /bar +env BAZ $BAR +env FOOPATH $PATH:$FOO +run [ "$BAR" = "$BAZ" ] +run [ "$FOOPATH" = "$PATH:/foo/baz" ] +`, + nil, + nil, + }, + + { + ` +from {IMAGE} +env FOO /bar +env TEST testdir +env BAZ /foobar +add testfile $BAZ/ +add $TEST $FOO +run [ "$(cat /foobar/testfile)" = "test1" ] +run [ "$(cat /bar/withfile)" = "test2" ] +`, + [][2]string{ + {"testfile", "test1"}, + {"testdir/withfile", "test2"}, + }, + nil, + }, } // FIXME: test building with 2 successive overlapping ADD commands @@ -242,8 +274,14 @@ func TestBuildEnv(t *testing.T) { env port 4243 `, nil, nil}, t) - - if img.Config.Env[0] != "port=4243" { + hasEnv := false + for _, envVar := range img.Config.Env { + if envVar == "port=4243" { + hasEnv = true + break + } + } + if !hasEnv { t.Fail() } } diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index acb015becd..c70e80b72e 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -611,11 +611,11 @@ type JSONMessage struct { Status string `json:"status,omitempty"` Progress string `json:"progress,omitempty"` Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -631,7 +631,6 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { return nil } - type StreamFormatter struct { json bool used bool From 2ad5267228f5e29b224f78b7ede02495d7475acc Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Thu, 25 Jul 2013 17:19:58 -0700 Subject: [PATCH 028/134] Enabled the docs to generate manpages. * changed conf.py to reference toctree.rst instead of index * Added note to README to upgrade your sphinx to the latest version to prevent a bug with .. note:: blocks. Upstream-commit: f4b63d9eeaddae6cb15775bea13ec4bca6e115da Component: engine --- components/engine/docs/README.md | 5 +++-- components/engine/docs/sources/conf.py | 6 +++--- components/engine/docs/sources/use/builder.rst | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/components/engine/docs/README.md b/components/engine/docs/README.md index 74ab2bd0cd..32be549ef5 100644 --- a/components/engine/docs/README.md +++ b/components/engine/docs/README.md @@ -39,8 +39,6 @@ When you need to add images, try to make them as small as possible (e.g. as gif) Notes ----- -* The index.html and gettingstarted.html files are copied from the source dir to the output dir without modification. -So changes to those pages should be made directly in html * For the template the css is compiled from less. When changes are needed they can be compiled using lessc ``lessc main.less`` or watched using watch-lessc ``watch-lessc -i main.less -o main.css`` @@ -75,3 +73,6 @@ Guides on using sphinx * Code examples Start without $, so it's easy to copy and paste. + +* To make the manpages, simply run 'make man'. Pleae note there is a bug in spinx 1.1.3 which makes this fail. +Upgrade to the latest version of sphinx. \ No newline at end of file diff --git a/components/engine/docs/sources/conf.py b/components/engine/docs/sources/conf.py index ea8e1f43f0..b4c23f0c58 100644 --- a/components/engine/docs/sources/conf.py +++ b/components/engine/docs/sources/conf.py @@ -203,7 +203,7 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'Docker.tex', u'Docker Documentation', + ('toctree', 'Docker.tex', u'Docker Documentation', u'Team Docker', 'manual'), ] @@ -233,7 +233,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'docker', u'Docker Documentation', + ('toctree', 'docker', u'Docker Documentation', [u'Team Docker'], 1) ] @@ -247,7 +247,7 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Docker', u'Docker Documentation', + ('toctree', 'Docker', u'Docker Documentation', u'Team Docker', 'Docker', 'One line description of project.', 'Miscellaneous'), ] diff --git a/components/engine/docs/sources/use/builder.rst b/components/engine/docs/sources/use/builder.rst index aa2fd6b92a..4a9786ee47 100644 --- a/components/engine/docs/sources/use/builder.rst +++ b/components/engine/docs/sources/use/builder.rst @@ -107,8 +107,8 @@ the image. This is functionally equivalent to running ``docker commit -run '{"Cmd": }'`` outside the builder. .. note:: - Don't confuse `RUN` with `CMD`. `RUN` actually runs a - command and commits the result; `CMD` does not execute anything at + Don't confuse ``RUN`` with ``CMD``. ``RUN`` actually runs a + command and commits the result; ``CMD`` does not execute anything at build time, but specifies the intended command for the image. 3.5 EXPOSE From 4118c3f05998e5880d5d5d1e9a878e3400e96760 Mon Sep 17 00:00:00 2001 From: Mike Gaffney Date: Fri, 26 Jul 2013 01:10:42 -0700 Subject: [PATCH 029/134] Change reserve-compatibility to reverse-compatibility Upstream-commit: 4ebe2cf348915415c34503aca7a5663177e0002f Component: engine --- components/engine/container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/container.go b/components/engine/container.go index ec4abffc1b..d0b6ca4ce2 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -52,7 +52,7 @@ type Container struct { waitLock chan struct{} Volumes map[string]string - // Store rw/ro in a separate structure to preserve reserve-compatibility on-disk. + // Store rw/ro in a separate structure to preserve reverse-compatibility on-disk. // Easier than migrating older container configs :) VolumesRW map[string]bool } From 8f7401af36bf513edf440d779a1be4e7f7177b44 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 26 Jul 2013 09:19:26 +0000 Subject: [PATCH 030/134] fix wrong untag when using rmi via id Upstream-commit: e608296bc62ceeaf41ebf2bc80b21c0a1883d4f0 Component: engine --- components/engine/server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/server.go b/components/engine/server.go index ce1fc8eaf8..fa8d8a0262 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -995,6 +995,9 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro parsedRepo := strings.Split(repoAndTag, ":")[0] if strings.Contains(img.ID, repoName) { repoName = parsedRepo + if len(strings.Split(repoAndTag, ":")) > 1 { + tag = strings.Split(repoAndTag, ":")[1] + } } else if repoName != parsedRepo { // the id belongs to multiple repos, like base:latest and user:test, // in that case return conflict From 71ca6b437812a8ce19b134015e1e9781ab62e16d Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 26 Jul 2013 10:04:46 +0000 Subject: [PATCH 031/134] fix docs Upstream-commit: 513a5674831fd952fc8e7b8fbdbc4939393ecb5b Component: engine --- components/engine/docs/sources/api/docker_remote_api.rst | 6 +++--- .../engine/docs/sources/api/docker_remote_api_v1.4.rst | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index d07c7634b9..0a1ff21cf1 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -26,15 +26,15 @@ Docker Remote API 2. Versions =========== -The current verson of the API is 1.3 +The current verson of the API is 1.4 Calling /images//insert is the same as calling -/v1.3/images//insert +/v1.4/images//insert You can still call an old version of the api using /v1.0/images//insert -:doc:`docker_remote_api_v1.3` +:doc:`docker_remote_api_v1.4` ***************************** What's new diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst index c42adb286f..6ee0b35fa2 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -1,9 +1,9 @@ -:title: Remote API v1.3 +:title: Remote API v1.4 :description: API Documentation for Docker :keywords: API, Docker, rcli, REST, documentation ====================== -Docker Remote API v1.3 +Docker Remote API v1.4 ====================== .. contents:: Table of Contents From cdbe046a767d82cc86c585545ff89f52fd3f59f0 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 26 Jul 2013 10:01:41 +0000 Subject: [PATCH 032/134] add regression test Upstream-commit: e592f1b298c778d0b9adfd6751f5fe1843a7001d Component: engine --- components/engine/server.go | 2 +- components/engine/server_test.go | 86 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/components/engine/server.go b/components/engine/server.go index fa8d8a0262..56f738a5cd 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -995,7 +995,7 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro parsedRepo := strings.Split(repoAndTag, ":")[0] if strings.Contains(img.ID, repoName) { repoName = parsedRepo - if len(strings.Split(repoAndTag, ":")) > 1 { + if len(srv.runtime.repositories.ByID()[img.ID]) == 1 && len(strings.Split(repoAndTag, ":")) > 1 { tag = strings.Split(repoAndTag, ":")[1] } } else if repoName != parsedRepo { diff --git a/components/engine/server_test.go b/components/engine/server_test.go index 8612b3fcea..b6b21cc75f 100644 --- a/components/engine/server_test.go +++ b/components/engine/server_test.go @@ -2,6 +2,7 @@ package docker import ( "github.com/dotcloud/docker/utils" + "strings" "testing" "time" ) @@ -203,3 +204,88 @@ func TestLogEvent(t *testing.T) { }) } + +func TestRmi(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + srv := &Server{runtime: runtime} + + initialImages, err := srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + config, hostConfig, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) + if err != nil { + t.Fatal(err) + } + + containerID, err := srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + //To remove + err = srv.ContainerStart(containerID, hostConfig) + if err != nil { + t.Fatal(err) + } + + imageID, err := srv.ContainerCommit(containerID, "test", "", "", "", nil) + if err != nil { + t.Fatal(err) + } + + err = srv.ContainerTag(imageID, "test", "0.1", false) + if err != nil { + t.Fatal(err) + } + + containerID, err = srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + //To remove + err = srv.ContainerStart(containerID, hostConfig) + if err != nil { + t.Fatal(err) + } + + _, err = srv.ContainerCommit(containerID, "test", "", "", "", nil) + if err != nil { + t.Fatal(err) + } + + images, err := srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + if len(images)-len(initialImages) != 2 { + t.Fatalf("Expected 2 new images, found %d.", len(images)-len(initialImages)) + } + + _, err = srv.ImageDelete(imageID, true) + if err != nil { + t.Fatal(err) + } + + images, err = srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + if len(images)-len(initialImages) != 1 { + t.Fatalf("Expected 1 new image, found %d.", len(images)-len(initialImages)) + } + + for _, image := range images { + if strings.Contains(unitTestImageID, image.ID) { + continue + } + if image.Repository == "" { + t.Fatalf("Expected tagged image, got untagged one.") + } + } +} From d391806121d62d6538dab5ad1305a6169a11ed36 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 26 Jul 2013 10:21:17 -0700 Subject: [PATCH 033/134] Clean up 'manifesto' in docs Upstream-commit: a97d858b2a0d6bc4d044a241036b97fd78f93022 Component: engine --- .../docs/sources/concepts/manifesto.rst | 61 ------------------- 1 file changed, 61 deletions(-) diff --git a/components/engine/docs/sources/concepts/manifesto.rst b/components/engine/docs/sources/concepts/manifesto.rst index ae09647094..7dd4b4bdda 100644 --- a/components/engine/docs/sources/concepts/manifesto.rst +++ b/components/engine/docs/sources/concepts/manifesto.rst @@ -4,10 +4,6 @@ .. _dockermanifesto: -*(This was our original Welcome page, but it is a bit forward-looking -for docs, and maybe not enough vision for a true manifesto. We'll -reveal more vision in the future to make it more Manifesto-y.)* - Docker Manifesto ---------------- @@ -131,60 +127,3 @@ sitting 10 miles away. With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. - -Standard Container Specification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -(TODO) - -Image format -~~~~~~~~~~~~ - -Standard operations -~~~~~~~~~~~~~~~~~~~ - -- Copy -- Run -- Stop -- Wait -- Commit -- Attach standard streams -- List filesystem changes -- ... - -Execution environment -~~~~~~~~~~~~~~~~~~~~~ - -Root filesystem -^^^^^^^^^^^^^^^ - -Environment variables -^^^^^^^^^^^^^^^^^^^^^ - -Process arguments -^^^^^^^^^^^^^^^^^ - -Networking -^^^^^^^^^^ - -Process namespacing -^^^^^^^^^^^^^^^^^^^ - -Resource limits -^^^^^^^^^^^^^^^ - -Process monitoring -^^^^^^^^^^^^^^^^^^ - -Logging -^^^^^^^ - -Signals -^^^^^^^ - -Pseudo-terminal allocation -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Security -^^^^^^^^ - From b0bd5c4e73bce08fe90dcb32da06427da5ac82b5 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 26 Jul 2013 15:40:55 -0400 Subject: [PATCH 034/134] Bind daemon to 0.0.0.0 in Vagrant. Fixes #1304 Upstream-commit: bdc79ac8b2dfad302f9e144711067a566726cfa2 Component: engine --- components/engine/Vagrantfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/Vagrantfile b/components/engine/Vagrantfile index aadabb8711..7258af5bf7 100644 --- a/components/engine/Vagrantfile +++ b/components/engine/Vagrantfile @@ -20,6 +20,8 @@ Vagrant::Config.run do |config| pkg_cmd = "apt-get update -qq; apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/lxc-docker; apt-get update -qq; " \ "apt-get install -q -y lxc-docker; " + # Listen on all interfaces so that the daemon is accessible from the host + pkg_cmd << "sed -i -E 's| /usr/bin/docker -d| /usr/bin/docker -d -H 0.0.0.0|' /etc/init/docker.conf;" # Add X.org Ubuntu backported 3.8 kernel pkg_cmd << "add-apt-repository -y ppa:ubuntu-x-swat/r-lts-backport; " \ "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " From cd3a1b485585f13de208d23b5e546adec9603ab3 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 26 Jul 2013 15:44:06 -0400 Subject: [PATCH 035/134] Update AUTHORS Upstream-commit: 5eb590e79d081ecfa82ba940ea856c4a2c0411ca Component: engine --- components/engine/.mailmap | 2 ++ components/engine/AUTHORS | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/components/engine/.mailmap b/components/engine/.mailmap index 452ac41d8f..11ff5357d8 100644 --- a/components/engine/.mailmap +++ b/components/engine/.mailmap @@ -23,3 +23,5 @@ Thatcher Peskens Walter Stanish Roberto Hashioka +Konstantin Pelykh +David Sissitka diff --git a/components/engine/AUTHORS b/components/engine/AUTHORS index 86d03f6e12..c811de6526 100644 --- a/components/engine/AUTHORS +++ b/components/engine/AUTHORS @@ -4,12 +4,15 @@ # For a list of active project maintainers, see the MAINTAINERS file. # Al Tobey +Alex Gaynor Alexey Shamrin Andrea Luzzardi Andreas Tiefenthaler Andrew Munsell +Andrews Medina Andy Rothfusz Andy Smith +Anthony Bishopric Antony Messerli Barry Allard Brandon Liu @@ -23,17 +26,22 @@ Daniel Gasienica Daniel Mizyrycki Daniel Robinson Daniel Von Fange +Daniel YC Lin +David Sissitka Dominik Honnef Don Spaulding Dr Nic Williams Elias Probst Eric Hanchrow -Evan Wies Eric Myhre +Erno Hopearuoho +Evan Wies ezbercih +Fabrizio Regini Flavio Castelli Francisco Souza Frederick F. Kautz IV +Gabriel Monroy Gareth Rushgrove Guillaume J. Charmes Harley Laue @@ -41,6 +49,7 @@ Hunter Blanks Jeff Lindsay Jeremy Grosser Joffrey F +Johan Euphrosine John Costa Jon Wedaman Jonas Pfenniger @@ -48,28 +57,39 @@ Jonathan Rudenberg Joseph Anthony Pasquale Holsten Julien Barbier Jérôme Petazzoni +Karan Lyons +Keli Hu Ken Cochrane Kevin J. Lynagh kim0 +Kimbro Staken Kiran Gangadharan +Konstantin Pelykh Louis Opter +Marco Hennings Marcus Farkas Mark McGranaghan Maxim Treskin meejah Michael Crosby +Mike Gaffney Mikhail Sobolev +Nan Monnand Deng Nate Jones Nelson Chen Niall O'Higgins +Nick Stenning +Nick Stinemates odk- Paul Bowsher Paul Hammond Phil Spitler Piotr Bogdan Renato Riccieri Santos Zannon +Rhys Hiltner Robert Obryk Roberto Hashioka +Ryan Fowler Sam Alba Sam J Sharpe Shawn Siefkas @@ -83,6 +103,8 @@ Thomas Hansen Tianon Gravi Tim Terhorst Tobias Bieniek +Tobias Schwab +Tom Hulihan unclejack Victor Vieux Vivek Agarwal From bdd8af79196724d7ddfbef579bd46ec292c6dd3e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 26 Jul 2013 14:57:16 -0700 Subject: [PATCH 036/134] - Builder: Create directories with 755 instead of 700 within ADD instruction Upstream-commit: b15cfd3530cc228dc065746c9323758c8abb0481 Component: engine --- components/engine/buildfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 75ebdd7a7c..a25b86d794 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -242,7 +242,7 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { } else if err := UntarPath(origPath, destPath); err != nil { utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) // If that fails, just copy it as a regular file - if err := os.MkdirAll(path.Dir(destPath), 0700); err != nil { + if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { return err } if err := CopyWithTar(origPath, destPath); err != nil { From 3fa2b3c80a598c17233759282718060e16740f84 Mon Sep 17 00:00:00 2001 From: Mike Gaffney Date: Fri, 26 Jul 2013 18:29:27 -0700 Subject: [PATCH 037/134] Add required go version for compilation Upstream-commit: 2d85a20c71c9418301c3161c6853a07fca612676 Component: engine --- components/engine/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/README.md b/components/engine/README.md index 96da13feaf..5e03b2993c 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -167,7 +167,7 @@ Installing from source ---------------------- 1. Make sure you have a [Go language](http://golang.org/doc/install) -compiler and [git](http://git-scm.com) installed. +compiler >= 1.1 and [git](http://git-scm.com) installed. 2. Checkout the source code ```bash From fba18c65a3185bea2bb32ef461b57e3aff48c25c Mon Sep 17 00:00:00 2001 From: David Calavera Date: Sat, 27 Jul 2013 10:00:36 -0700 Subject: [PATCH 038/134] Do not show empty parenthesis if the default configuration is missing. Upstream-commit: d4f70397930b15c8860a71df1abb9bc5ac9ab4ff Component: engine --- components/engine/commands.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 0fabaa385f..a7df369e5f 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -314,13 +314,21 @@ func (cli *DockerCli) CmdLogin(args ...string) error { email string ) + var promptDefault = func(stdout io.Writer, prompt string, configDefault string) { + if configDefault == "" { + fmt.Fprintf(cli.out, "%s: ", prompt) + } else { + fmt.Fprintf(cli.out, "%s (%s): ", prompt, configDefault) + } + } + authconfig, ok := cli.configFile.Configs[auth.IndexServerAddress()] if !ok { authconfig = auth.AuthConfig{} } if *flUsername == "" { - fmt.Fprintf(cli.out, "Username (%s): ", authconfig.Username) + promptDefault(cli.out, "Username", authconfig.Username) username = readAndEchoString(cli.in, cli.out) if username == "" { username = authconfig.Username @@ -340,7 +348,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flEmail == "" { - fmt.Fprintf(cli.out, "Email (%s): ", authconfig.Email) + promptDefault(cli.out, "Email", authconfig.Email) email = readAndEchoString(cli.in, cli.out) if email == "" { email = authconfig.Email From 86b74909af3248f1cb0bcf103445df71baed118c Mon Sep 17 00:00:00 2001 From: David Calavera Date: Sat, 27 Jul 2013 10:17:57 -0700 Subject: [PATCH 039/134] Remove unused argument. Upstream-commit: 88b6ea993d76fb8891ee7a7fa8828b5c5753c7f5 Component: engine --- components/engine/commands.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index a7df369e5f..7288b58229 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -314,7 +314,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { email string ) - var promptDefault = func(stdout io.Writer, prompt string, configDefault string) { + var promptDefault = func(prompt string, configDefault string) { if configDefault == "" { fmt.Fprintf(cli.out, "%s: ", prompt) } else { @@ -328,7 +328,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flUsername == "" { - promptDefault(cli.out, "Username", authconfig.Username) + promptDefault("Username", authconfig.Username) username = readAndEchoString(cli.in, cli.out) if username == "" { username = authconfig.Username @@ -348,7 +348,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flEmail == "" { - promptDefault(cli.out, "Email", authconfig.Email) + promptDefault("Email", authconfig.Email) email = readAndEchoString(cli.in, cli.out) if email == "" { email = authconfig.Email From 730b8810f90e162ce6969e977be10fcee945aae5 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 28 Jul 2013 12:57:09 -0700 Subject: [PATCH 040/134] Remove deprecated copy from README Upstream-commit: 97a2dc96f23b25bc7980d2a4514b7065ff4edb9e Component: engine --- components/engine/README.md | 45 ------------------------------------- 1 file changed, 45 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 5e03b2993c..5eee1a3db6 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -444,51 +444,6 @@ With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. - - -Standard Container Specification --------------------------------- - -(TODO) - -### Image format - - -### Standard operations - -* Copy -* Run -* Stop -* Wait -* Commit -* Attach standard streams -* List filesystem changes -* ... - -### Execution environment - -#### Root filesystem - -#### Environment variables - -#### Process arguments - -#### Networking - -#### Process namespacing - -#### Resource limits - -#### Process monitoring - -#### Logging - -#### Signals - -#### Pseudo-terminal allocation - -#### Security - ### Legal Transfers of Docker shall be in accordance with applicable export From 183aa3c2d9a6e42e18f19f9c4219fb7775b6a957 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 12:15:27 +0000 Subject: [PATCH 041/134] use ParseRepositoryTag instead on split on : in imagedelete Upstream-commit: 63876e7dbdd372745855345632af62b8cd976dfa Component: engine --- components/engine/server.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/components/engine/server.go b/components/engine/server.go index ce1fc8eaf8..f416d8f06d 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1037,13 +1037,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { return nil, nil } - var tag string - if strings.Contains(name, ":") { - nameParts := strings.Split(name, ":") - name = nameParts[0] - tag = nameParts[1] - } - + name, tag := utils.ParseRepositoryTag(name) return srv.deleteImage(img, name, tag) } From 77e59c214867e93be8aacf505d35acc9aa0be941 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 12:16:01 +0000 Subject: [PATCH 042/134] add ParseRepositoryTag tests Upstream-commit: 3852d0599097581d5dc0bcfcb7aa010d564beb9a Component: engine --- components/engine/utils/utils.go | 7 +++---- components/engine/utils/utils_test.go | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index acb015becd..c70e80b72e 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -611,11 +611,11 @@ type JSONMessage struct { Status string `json:"status,omitempty"` Progress string `json:"progress,omitempty"` Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -631,7 +631,6 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { return nil } - type StreamFormatter struct { json bool used bool diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index 5caa809f67..5c480b9438 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -282,3 +282,24 @@ func TestParseHost(t *testing.T) { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } } + +func TestParseRepositoryTag(t *testing.T) { + if repo, tag := ParseRepositoryTag("root"); repo != "root" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("root:tag"); repo != "root" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo"); repo != "user/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo:tag"); repo != "user/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo"); repo != "url:5000/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo:tag"); repo != "url:5000/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "tag", repo, tag) + } +} From c1c0668f041a479eff88225c7926a0c3b14a9250 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 12:16:14 +0000 Subject: [PATCH 043/134] add regression tests Upstream-commit: bb241c10e2f124406f00c8e40e00fca67934638a Component: engine --- components/engine/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/server_test.go b/components/engine/server_test.go index 8612b3fcea..0caf8a5f24 100644 --- a/components/engine/server_test.go +++ b/components/engine/server_test.go @@ -20,7 +20,7 @@ func TestContainerTagImageDelete(t *testing.T) { if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil { t.Fatal(err) } - if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil { + if err := srv.runtime.repositories.Set("utest:5000/docker", "tag2", unitTestImageName, false); err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func TestContainerTagImageDelete(t *testing.T) { t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) } - if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil { + if _, err := srv.ImageDelete("utest:5000/docker:tag2", true); err != nil { t.Fatal(err) } From 224e45118c5680834731d0624d6877543f3273d5 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 16:40:35 +0000 Subject: [PATCH 044/134] fix #1314 discard error when loading old container format Upstream-commit: b2aa877bf0a3f140516d1c17117369b2194f2b91 Component: engine --- components/engine/container.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/container.go b/components/engine/container.go index d0b6ca4ce2..cbcd3f17fa 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -266,7 +266,8 @@ func (container *Container) FromDisk() error { return err } // Load container settings - if err := json.Unmarshal(data, container); err != nil { + // udp broke compat of docker.PortMapping, but it's not used when loading a container, we can skip it + if err := json.Unmarshal(data, container); err != nil && !strings.Contains(err.Error(), "docker.PortMapping") { return err } return nil From 1ccfbf8698584ad97e67a104b38d9bba7a89155f Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Mon, 29 Jul 2013 09:45:19 -0700 Subject: [PATCH 045/134] testing, issue #1331: Add registry functional test to docker-ci Upstream-commit: 17ffb0ac84874656a61a477fa29048cb82b28748 Component: engine --- components/engine/testing/README.rst | 4 ++++ components/engine/testing/Vagrantfile | 4 +++- .../engine/testing/buildbot/credentials.cfg | 5 +++++ components/engine/testing/buildbot/master.cfg | 20 +++++++++++++++++-- .../engine/testing/buildbot/requirements.txt | 1 + .../testing/buildbot/setup_credentials.sh | 17 ++++++++++++++++ .../testing/functionaltests/test_registry.sh | 11 ++++++++++ 7 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 components/engine/testing/buildbot/credentials.cfg create mode 100755 components/engine/testing/buildbot/setup_credentials.sh create mode 100755 components/engine/testing/functionaltests/test_registry.sh diff --git a/components/engine/testing/README.rst b/components/engine/testing/README.rst index 3b11092f9f..ce5aa837a4 100644 --- a/components/engine/testing/README.rst +++ b/components/engine/testing/README.rst @@ -40,6 +40,10 @@ Deployment export SMTP_USER=xxxxxxxxxxxx export SMTP_PWD=xxxxxxxxxxxx + # Define docker registry functional test credentials + export REGISTRY_USER=xxxxxxxxxxxx + export REGISTRY_PWD=xxxxxxxxxxxx + # Checkout docker git clone git://github.com/dotcloud/docker.git diff --git a/components/engine/testing/Vagrantfile b/components/engine/testing/Vagrantfile index 47257201dc..e76a951508 100644 --- a/components/engine/testing/Vagrantfile +++ b/components/engine/testing/Vagrantfile @@ -29,7 +29,9 @@ Vagrant::Config.run do |config| "chown #{USER}.#{USER} /data; cd /data; " \ "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \ "#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \ - "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " + "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " \ + "#{CFG_PATH}/setup_credentials.sh #{USER} " \ + "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ diff --git a/components/engine/testing/buildbot/credentials.cfg b/components/engine/testing/buildbot/credentials.cfg new file mode 100644 index 0000000000..fbdd35d578 --- /dev/null +++ b/components/engine/testing/buildbot/credentials.cfg @@ -0,0 +1,5 @@ +# Credentials for tests. Buildbot source this file on tests +# when needed. + +# Docker registry credentials. Format: 'username:password' +export DOCKER_CREDS='' diff --git a/components/engine/testing/buildbot/master.cfg b/components/engine/testing/buildbot/master.cfg index 61912808ec..29926dbe5f 100644 --- a/components/engine/testing/buildbot/master.cfg +++ b/components/engine/testing/buildbot/master.cfg @@ -19,6 +19,7 @@ TEST_USER = 'buildbot' # Credential to authenticate build triggers TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' GITHUB_DOCKER = 'github.com/dotcloud/docker' +BUILDBOT_PATH = '/data/buildbot' DOCKER_PATH = '/data/docker' BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME) DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker' @@ -41,16 +42,19 @@ c['db'] = {'db_url':"sqlite:///state.sqlite"} c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)] c['slavePortnum'] = PORT_MASTER + # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME, - 'coverage'])] + 'registry','coverage'])] c['schedulers'] += [SingleBranchScheduler(name="all", change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=[BUILDER_NAME])] -c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage'], +c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage','registry'], hour=0, minute=30)] + # Builders +# Docker commit test factory = BuildFactory() factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; " @@ -58,6 +62,7 @@ factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, "go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))])) c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], factory=factory)] + # Docker coverage test coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n' 'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n' @@ -69,6 +74,17 @@ factory.addStep(ShellCommand(description='Coverage',logEnviron=False,usePTY=True c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], factory=factory)] +# Registry Functionaltest builder +factory = BuildFactory() +factory.addStep(ShellCommand(description='registry', logEnviron=False, + command='. {0}/master/credentials.cfg; ' + '{1}/testing/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, + DOCKER_PATH), usePTY=True)) + +c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], + factory=factory)] + + # Status authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]), forceBuild='auth') diff --git a/components/engine/testing/buildbot/requirements.txt b/components/engine/testing/buildbot/requirements.txt index 0e451b017d..4e183ba062 100644 --- a/components/engine/testing/buildbot/requirements.txt +++ b/components/engine/testing/buildbot/requirements.txt @@ -4,3 +4,4 @@ buildbot==0.8.7p1 buildbot_slave==0.8.7p1 nose==1.2.1 requests==1.1.0 +flask==0.10.1 diff --git a/components/engine/testing/buildbot/setup_credentials.sh b/components/engine/testing/buildbot/setup_credentials.sh new file mode 100755 index 0000000000..f093815d60 --- /dev/null +++ b/components/engine/testing/buildbot/setup_credentials.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Setup of test credentials. Called by Vagrantfile +export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" + +USER=$1 +REGISTRY_USER=$2 +REGISTRY_PWD=$3 + +BUILDBOT_PATH="/data/buildbot" +DOCKER_PATH="/data/docker" + +function run { su $USER -c "$1"; } + +run "cp $DOCKER_PATH/testing/buildbot/credentials.cfg $BUILDBOT_PATH/master" +cd $BUILDBOT_PATH/master +run "sed -i -E 's#(export DOCKER_CREDS=).+#\1\"$REGISTRY_USER:$REGISTRY_PWD\"#' credentials.cfg" diff --git a/components/engine/testing/functionaltests/test_registry.sh b/components/engine/testing/functionaltests/test_registry.sh new file mode 100755 index 0000000000..095a731631 --- /dev/null +++ b/components/engine/testing/functionaltests/test_registry.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Cleanup +rm -rf docker-registry + +# Get latest docker registry +git clone https://github.com/dotcloud/docker-registry.git + +# Configure and run registry tests +cd docker-registry; cp config_sample.yml config.yml +cd test; python -m unittest workflow From 4b37d9ec34f816bd7fb8a1ad22ea0a2703016091 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 29 Jul 2013 10:28:41 -0700 Subject: [PATCH 046/134] Remove unnecessary signal conditional. Upstream-commit: c8ec36d1b9bfbe1e22acd0124409ecb5a109d406 Component: engine --- components/engine/commands.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 0fabaa385f..b3e7976995 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1658,14 +1658,11 @@ func (cli *DockerCli) monitorTtySize(id string) error { } cli.resizeTty(id) - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGWINCH) + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGWINCH) go func() { - for sig := range c { - if sig == syscall.SIGWINCH { - cli.resizeTty(id) - } - } + <-sigchan + cli.resizeTty(id) }() return nil } From dc883510210f5a92f0a21ad468df47e382116799 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 29 Jul 2013 11:03:09 -0700 Subject: [PATCH 047/134] Make sure ADD will create everything in 0755 Upstream-commit: f7542664e3efb35b6b153dc2b2cc9cdac181989a Component: engine --- components/engine/archive.go | 2 +- components/engine/docs/sources/use/builder.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/archive.go b/components/engine/archive.go index 01af86006f..bb79cd34d4 100644 --- a/components/engine/archive.go +++ b/components/engine/archive.go @@ -173,7 +173,7 @@ func CopyWithTar(src, dst string) error { } // Create dst, copy src's content into it utils.Debugf("Creating dest directory: %s", dst) - if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { return err } utils.Debugf("Calling TarUntar(%s, %s)", src, dst) diff --git a/components/engine/docs/sources/use/builder.rst b/components/engine/docs/sources/use/builder.rst index aa2fd6b92a..9f5ee8b795 100644 --- a/components/engine/docs/sources/use/builder.rst +++ b/components/engine/docs/sources/use/builder.rst @@ -182,7 +182,7 @@ The copy obeys the following rules: written at ````. * If ```` doesn't exist, it is created along with all missing directories in its path. All new files and directories are created - with mode 0700, uid and gid 0. + with mode 0755, uid and gid 0. 3.8 ENTRYPOINT -------------- From 4dd9531b175f79d5bfaa9cbb9582027850ec9b7e Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 29 Jul 2013 11:13:59 -0700 Subject: [PATCH 048/134] Keep the loop to allow resizing more than once. Upstream-commit: 10e37198aa14cb3192fd0bf29572a5dce58c348f Component: engine --- components/engine/commands.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index b3e7976995..b4853e967d 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1661,8 +1661,10 @@ func (cli *DockerCli) monitorTtySize(id string) error { sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGWINCH) go func() { - <-sigchan - cli.resizeTty(id) + for { + <-sigchan + cli.resizeTty(id) + } }() return nil } From e9895c9839e36227492d8394557c3c7cd4572d8a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 17 Jul 2013 12:13:22 -0700 Subject: [PATCH 049/134] Refactor checksum Upstream-commit: 8ca7b0646e6c4346075656f46847f53c2e868a3d Component: engine --- components/engine/auth/auth.go | 4 +- components/engine/graph.go | 61 +--------- components/engine/image.go | 99 ---------------- components/engine/registry/registry.go | 58 +++++++-- components/engine/server.go | 96 +++------------ components/engine/utils/tarsum.go | 155 +++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 247 deletions(-) create mode 100644 components/engine/utils/tarsum.go diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index 39de876875..e402031ca2 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -16,9 +16,9 @@ import ( const CONFIGFILE = ".dockercfg" // Only used for user auth + account creation -const INDEXSERVER = "https://index.docker.io/v1/" +//const INDEXSERVER = "https://index.docker.io/v1/" -//const INDEXSERVER = "http://indexstaging-docker.dotcloud.com/" +const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/components/engine/graph.go b/components/engine/graph.go index 42d1bdbd4c..eea2cbec8a 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -1,9 +1,7 @@ package docker import ( - "encoding/json" "fmt" - "github.com/dotcloud/docker/registry" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -11,17 +9,13 @@ import ( "path" "path/filepath" "strings" - "sync" "time" ) // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *utils.TruncIndex - checksumLock map[string]*sync.Mutex - lockSumFile *sync.Mutex - lockSumMap *sync.Mutex + Root string + idIndex *utils.TruncIndex } // NewGraph instantiates a new graph at the given root path in the filesystem. @@ -36,11 +30,8 @@ func NewGraph(root string) (*Graph, error) { return nil, err } graph := &Graph{ - Root: abspath, - idIndex: utils.NewTruncIndex(), - checksumLock: make(map[string]*sync.Mutex), - lockSumFile: &sync.Mutex{}, - lockSumMap: &sync.Mutex{}, + Root: abspath, + idIndex: utils.NewTruncIndex(), } if err := graph.restore(); err != nil { return nil, err @@ -99,11 +90,6 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, err } } - graph.lockSumMap.Lock() - defer graph.lockSumMap.Unlock() - if _, exists := graph.checksumLock[img.ID]; !exists { - graph.checksumLock[img.ID] = &sync.Mutex{} - } return img, nil } @@ -126,7 +112,6 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut if err := graph.Register(layerData, layerData != nil, img); err != nil { return nil, err } - go img.Checksum() return img, nil } @@ -154,7 +139,6 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { } img.graph = graph graph.idIndex.Add(img.ID) - graph.checksumLock[img.ID] = &sync.Mutex{} return nil } @@ -311,40 +295,3 @@ func (graph *Graph) Heads() (map[string]*Image, error) { func (graph *Graph) imageRoot(id string) string { return path.Join(graph.Root, id) } - -func (graph *Graph) getStoredChecksums() (map[string]string, error) { - checksums := make(map[string]string) - // FIXME: Store the checksum in memory - - if checksumDict, err := ioutil.ReadFile(path.Join(graph.Root, "checksums")); err == nil { - if err := json.Unmarshal(checksumDict, &checksums); err != nil { - return nil, err - } - } - return checksums, nil -} - -func (graph *Graph) storeChecksums(checksums map[string]string) error { - checksumJSON, err := json.Marshal(checksums) - if err != nil { - return err - } - if err := ioutil.WriteFile(path.Join(graph.Root, "checksums"), checksumJSON, 0600); err != nil { - return err - } - return nil -} - -func (graph *Graph) UpdateChecksums(newChecksums map[string]*registry.ImgData) error { - graph.lockSumFile.Lock() - defer graph.lockSumFile.Unlock() - - localChecksums, err := graph.getStoredChecksums() - if err != nil { - return err - } - for id, elem := range newChecksums { - localChecksums[id] = elem.Checksum - } - return graph.storeChecksums(localChecksums) -} diff --git a/components/engine/image.go b/components/engine/image.go index 5240ec776f..dd066f88cd 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -2,7 +2,6 @@ package docker import ( "crypto/rand" - "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -72,26 +71,6 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { return err } - if store { - layerArchive := layerArchivePath(root) - file, err := os.OpenFile(layerArchive, os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return err - } - // FIXME: Retrieve the image layer size from here? - if _, err := io.Copy(file, layerData); err != nil { - return err - } - // FIXME: Don't close/open, read/write instead of Copy - file.Close() - - file, err = os.Open(layerArchive) - if err != nil { - return err - } - defer file.Close() - layerData = file - } // If layerData is not nil, unpack it into the new layer if layerData != nil { start := time.Now() @@ -128,10 +107,6 @@ func layerPath(root string) string { return path.Join(root, "layer") } -func layerArchivePath(root string) string { - return path.Join(root, "layer.tar.xz") -} - func jsonPath(root string) string { return path.Join(root, "json") } @@ -308,80 +283,6 @@ func (img *Image) layer() (string, error) { return layerPath(root), nil } -func (img *Image) Checksum() (string, error) { - img.graph.checksumLock[img.ID].Lock() - defer img.graph.checksumLock[img.ID].Unlock() - - root, err := img.root() - if err != nil { - return "", err - } - - checksums, err := img.graph.getStoredChecksums() - if err != nil { - return "", err - } - if checksum, ok := checksums[img.ID]; ok { - return checksum, nil - } - - layer, err := img.layer() - if err != nil { - return "", err - } - jsonData, err := ioutil.ReadFile(jsonPath(root)) - if err != nil { - return "", err - } - - var layerData io.Reader - - if file, err := os.Open(layerArchivePath(root)); err != nil { - if os.IsNotExist(err) { - layerData, err = Tar(layer, Xz) - if err != nil { - return "", err - } - } else { - return "", err - } - } else { - defer file.Close() - layerData = file - } - - h := sha256.New() - if _, err := h.Write(jsonData); err != nil { - return "", err - } - if _, err := h.Write([]byte("\n")); err != nil { - return "", err - } - - if _, err := io.Copy(h, layerData); err != nil { - return "", err - } - hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) - - // Reload the json file to make sure not to overwrite faster sums - img.graph.lockSumFile.Lock() - defer img.graph.lockSumFile.Unlock() - - checksums, err = img.graph.getStoredChecksums() - if err != nil { - return "", err - } - - checksums[img.ID] = hash - - // Dump the checksums to disc - if err := img.graph.storeChecksums(checksums); err != nil { - return hash, err - } - - return hash, nil -} - func (img *Image) getParentsSize(size int64) int64 { parentImage, err := img.GetParent() if err != nil || parentImage == nil { diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index adef1c7baa..cac77ba049 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -330,16 +330,52 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e }, nil } +func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + req.Header.Set("X-Docker-Checksum", imgData.Checksum) + + res, err := doWithCookies(r.client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + r.client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + return nil +} + // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - // FIXME: try json with UTF8 + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - req.Header.Set("X-Docker-Checksum", imgData.Checksum) r.setUserAgent(req) utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) @@ -364,10 +400,14 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error { - req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer) +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) (checksum string, err error) { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + + tarsumLayer := &utils.TarSum{Reader: layer} + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { - return err + return "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} @@ -375,18 +415,18 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) + return "", fmt.Errorf("Failed to upload layer: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) } - return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) } - return nil + return tarsumLayer.Sum(), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { diff --git a/components/engine/server.go b/components/engine/server.go index ce1fc8eaf8..6944df315f 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -455,12 +455,6 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName return err } - utils.Debugf("Updating checksums") - // Reload the json file to make sure not to overwrite faster sums - if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { - return err - } - utils.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens) if err != nil { @@ -598,41 +592,6 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut return nil } -// Retrieve the checksum of an image -// Priority: -// - Check on the stored checksums -// - Check if the archive exists, if it does not, ask the registry -// - If the archive does exists, process the checksum from it -// - If the archive does not exists and not found on registry, process checksum from layer -func (srv *Server) getChecksum(imageID string) (string, error) { - // FIXME: Use in-memory map instead of reading the file each time - if sums, err := srv.runtime.graph.getStoredChecksums(); err != nil { - return "", err - } else if checksum, exists := sums[imageID]; exists { - return checksum, nil - } - - img, err := srv.runtime.graph.Get(imageID) - if err != nil { - return "", err - } - - if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageID))); err != nil { - if os.IsNotExist(err) { - // TODO: Ask the registry for the checksum - // As the archive is not there, it is supposed to come from a pull. - } else { - return "", err - } - } - - checksum, err := img.Checksum() - if err != nil { - return "", err - } - return checksum, nil -} - // Retrieve the all the images to be uploaded in the correct order // Note: we can't use a map as it is not ordered func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgData, error) { @@ -649,14 +608,10 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return nil } imageSet[img.ID] = struct{}{} - checksum, err := srv.getChecksum(img.ID) - if err != nil { - return err - } + imgList = append([]*registry.ImgData{{ - ID: img.ID, - Checksum: checksum, - Tag: tag, + ID: img.ID, + Tag: tag, }}, imgList...) return nil }) @@ -666,7 +621,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) - out.Write(sf.FormatStatus("Processing checksums")) + imgList, err := srv.getImageList(localRepo) if err != nil { return err @@ -716,14 +671,8 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } out.Write(sf.FormatStatus("Pushing %s", imgID)) - // Make sure we have the image's checksum - checksum, err := srv.getChecksum(imgID) - if err != nil { - return err - } imgData := ®istry.ImgData{ - ID: imgID, - Checksum: checksum, + ID: imgID, } // Send the json @@ -735,36 +684,23 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, return err } - // Retrieve the tarball to be sent - var layerData *TempArchive - // If the archive exists, use it - file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgID))) + layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out) if err != nil { - if os.IsNotExist(err) { - // If the archive does not exist, create one from the layer - layerData, err = srv.runtime.graph.TempLayerArchive(imgID, Xz, sf, out) - if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) - } - } else { - return err - } - } else { - defer file.Close() - st, err := file.Stat() - if err != nil { - return err - } - layerData = &TempArchive{ - File: file, - Size: st.Size(), - } + return fmt.Errorf("Failed to generate layer archive: %s", err) } // Send the layer - if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + return err + } else { + imgData.Checksum = checksum + } + + // Send the checksum + if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { return err } + return nil } diff --git a/components/engine/utils/tarsum.go b/components/engine/utils/tarsum.go new file mode 100644 index 0000000000..0fd5ac106b --- /dev/null +++ b/components/engine/utils/tarsum.go @@ -0,0 +1,155 @@ +package utils + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "hash" + "io" + "sort" + "strconv" +) + +type verboseHash struct { + hash.Hash +} + +func (h verboseHash) Write(buf []byte) (int, error) { + Debugf("--->%s<---", buf) + return h.Hash.Write(buf) +} + +type TarSum struct { + io.Reader + tarR *tar.Reader + tarW *tar.Writer + gz *gzip.Writer + bufTar *bytes.Buffer + bufGz *bytes.Buffer + h hash.Hash + h2 verboseHash + sums []string + finished bool + first bool +} + +func (ts *TarSum) encodeHeader(h *tar.Header) error { + for _, elem := range [][2]string{ + {"name", h.Name}, + {"mode", strconv.Itoa(int(h.Mode))}, + {"uid", strconv.Itoa(h.Uid)}, + {"gid", strconv.Itoa(h.Gid)}, + {"size", strconv.Itoa(int(h.Size))}, + {"mtime", strconv.Itoa(int(h.ModTime.UTC().Unix()))}, + {"typeflag", string([]byte{h.Typeflag})}, + {"linkname", h.Linkname}, + {"uname", h.Uname}, + {"gname", h.Gname}, + {"devmajor", strconv.Itoa(int(h.Devmajor))}, + {"devminor", strconv.Itoa(int(h.Devminor))}, + // {"atime", strconv.Itoa(int(h.AccessTime.UTC().Unix()))}, + // {"ctime", strconv.Itoa(int(h.ChangeTime.UTC().Unix()))}, + } { + // Debugf("-->%s<-- -->%s<--", elem[0], elem[1]) + if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil { + return err + } + } + return nil +} + +func (ts *TarSum) Read(buf []byte) (int, error) { + if ts.gz == nil { + ts.bufTar = bytes.NewBuffer([]byte{}) + ts.bufGz = bytes.NewBuffer([]byte{}) + ts.tarR = tar.NewReader(ts.Reader) + ts.tarW = tar.NewWriter(ts.bufTar) + ts.gz = gzip.NewWriter(ts.bufGz) + ts.h = sha256.New() + // ts.h = verboseHash{sha256.New()} + ts.h.Reset() + ts.first = true + } + + if ts.finished { + return ts.bufGz.Read(buf) + } + buf2 := make([]byte, len(buf), cap(buf)) + + n, err := ts.tarR.Read(buf2) + if err != nil { + if err == io.EOF { + if _, err := ts.h.Write(buf2[:n]); err != nil { + return 0, err + } + if !ts.first { + ts.sums = append(ts.sums, hex.EncodeToString(ts.h.Sum(nil))) + ts.h.Reset() + } else { + ts.first = false + } + + currentHeader, err := ts.tarR.Next() + if err != nil { + if err == io.EOF { + if err := ts.gz.Close(); err != nil { + return 0, err + } + ts.finished = true + return n, nil + } + return n, err + } + if err := ts.encodeHeader(currentHeader); err != nil { + return 0, err + } + if err := ts.tarW.WriteHeader(currentHeader); err != nil { + return 0, err + } + if _, err := ts.tarW.Write(buf2[:n]); err != nil { + return 0, err + } + ts.tarW.Flush() + if _, err := io.Copy(ts.gz, ts.bufTar); err != nil { + return 0, err + } + ts.gz.Flush() + + return ts.bufGz.Read(buf) + } + return n, err + } + + // Filling the hash buffer + if _, err = ts.h.Write(buf2[:n]); err != nil { + return 0, err + } + + // Filling the tar writter + if _, err = ts.tarW.Write(buf2[:n]); err != nil { + return 0, err + } + ts.tarW.Flush() + + // Filling the gz writter + if _, err = io.Copy(ts.gz, ts.bufTar); err != nil { + return 0, err + } + ts.gz.Flush() + + return ts.bufGz.Read(buf) +} + +func (ts *TarSum) Sum() string { + sort.Strings(ts.sums) + h := sha256.New() + for _, sum := range ts.sums { + Debugf("-->%s<--", sum) + h.Write([]byte(sum)) + } + checksum := "tarsum+sha256:" + hex.EncodeToString(ts.h.Sum(nil)) + Debugf("checksum processed: %s", checksum) + return checksum +} From 0d6f4c9ed448770f52cc0be7785237520b4eaca1 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 14:50:32 -0700 Subject: [PATCH 050/134] Handle extra-paremeter within checksum calculations Upstream-commit: e3f68b22d8f0635a8c08ab56721e56dbe570a49a Component: engine --- components/engine/registry/registry.go | 17 +++++++++++------ components/engine/server.go | 2 +- components/engine/utils/tarsum.go | 7 +++++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index cac77ba049..40b9872a4a 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -17,8 +17,10 @@ import ( "strings" ) -var ErrAlreadyExists = errors.New("Image already exists") -var ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +var ( + ErrAlreadyExists = errors.New("Image already exists") + ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +) func pingRegistryEndpoint(endpoint string) error { if endpoint == auth.IndexServerAddress() { @@ -266,8 +268,11 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + utils.Debugf("[registry] Calling GET %s", repositoryTarget) + req, err := r.opaqueRequest("GET", repositoryTarget, nil) if err != nil { return nil, err @@ -378,7 +383,6 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) r.setUserAgent(req) - utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -400,11 +404,12 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) (checksum string, err error) { +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, err error) { utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") tarsumLayer := &utils.TarSum{Reader: layer} + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { return "", err @@ -426,7 +431,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) } - return tarsumLayer.Sum(), nil + return tarsumLayer.Sum(jsonRaw), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { @@ -474,7 +479,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData } u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) - utils.Debugf("PUT %s", u) + utils.Debugf("[registry] PUT %s", u) utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { diff --git a/components/engine/server.go b/components/engine/server.go index 6944df315f..7309279805 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -690,7 +690,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } // Send the layer - if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil { return err } else { imgData.Checksum = checksum diff --git a/components/engine/utils/tarsum.go b/components/engine/utils/tarsum.go index 0fd5ac106b..015b9b3076 100644 --- a/components/engine/utils/tarsum.go +++ b/components/engine/utils/tarsum.go @@ -142,14 +142,17 @@ func (ts *TarSum) Read(buf []byte) (int, error) { return ts.bufGz.Read(buf) } -func (ts *TarSum) Sum() string { +func (ts *TarSum) Sum(extra []byte) string { sort.Strings(ts.sums) h := sha256.New() for _, sum := range ts.sums { Debugf("-->%s<--", sum) h.Write([]byte(sum)) } - checksum := "tarsum+sha256:" + hex.EncodeToString(ts.h.Sum(nil)) + if extra != nil { + h.Write(extra) + } + checksum := "tarsum+sha256:" + hex.EncodeToString(h.Sum(nil)) Debugf("checksum processed: %s", checksum) return checksum } From 78c57a64f00c90de82b3a766eb89c502151f7872 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 15:40:33 -0700 Subject: [PATCH 051/134] Refactor the image size storage Upstream-commit: 0badda9f1587c11a13dca17c68b30addd757237c Component: engine --- components/engine/graph.go | 6 +++--- components/engine/image.go | 43 +++++++++++++++++++++++++++++-------- components/engine/server.go | 2 +- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/components/engine/graph.go b/components/engine/graph.go index eea2cbec8a..1fe14f6458 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -109,7 +109,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Container = container.ID img.ContainerConfig = *container.Config } - if err := graph.Register(layerData, layerData != nil, img); err != nil { + if err := graph.Register(nil, layerData, layerData != nil, img); err != nil { return nil, err } return img, nil @@ -117,7 +117,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut // Register imports a pre-existing image into the graph. // FIXME: pass img as first argument -func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { +func (graph *Graph) Register(jsonData []byte, layerData Archive, store bool, img *Image) error { if err := ValidateID(img.ID); err != nil { return err } @@ -130,7 +130,7 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { if err != nil { return fmt.Errorf("Mktemp failed: %s", err) } - if err := StoreImage(img, layerData, tmp, store); err != nil { + if err := StoreImage(img, jsonData, layerData, tmp, store); err != nil { return err } // Commit diff --git a/components/engine/image.go b/components/engine/image.go index dd066f88cd..7f03cc4bf8 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -13,6 +13,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "time" ) @@ -46,6 +47,19 @@ func LoadImage(root string) (*Image, error) { if err := ValidateID(img.ID); err != nil { return nil, err } + + if buf, err := ioutil.ReadFile(path.Join(root, "layersize")); err != nil { + if !os.IsNotExist(err) { + return nil, err + } + } else { + if size, err := strconv.Atoi(string(buf)); err != nil { + return nil, err + } else { + img.Size = int64(size) + } + } + // Check that the filesystem layer exists if stat, err := os.Stat(layerPath(root)); err != nil { if os.IsNotExist(err) { @@ -58,7 +72,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, layerData Archive, root string, store bool) error { +func StoreImage(img *Image, jsonData []byte, layerData Archive, root string, store bool) error { // Check that root doesn't already exist if _, err := os.Stat(root); err == nil { return fmt.Errorf("Image %s already exists", img.ID) @@ -81,25 +95,36 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { utils.Debugf("Untar time: %vs\n", time.Now().Sub(start).Seconds()) } + // If raw json is provided, then use it + if jsonData != nil { + return ioutil.WriteFile(jsonPath(root), jsonData, 0600) + } else { // Otherwise, unmarshal the image + jsonData, err := json.Marshal(img) + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + } + return StoreSize(img, root) } func StoreSize(img *Image, root string) error { layer := layerPath(root) + var totalSize int64 = 0 filepath.Walk(layer, func(path string, fileInfo os.FileInfo, err error) error { - img.Size += fileInfo.Size() + totalSize += fileInfo.Size() return nil }) + img.Size = totalSize - // Store the json ball - jsonData, err := json.Marshal(img) - if err != nil { - return err - } - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err + if err := ioutil.WriteFile(path.Join(root, "layersize"), []byte(strconv.Itoa(int(totalSize))), 0600); err != nil { + return nil } + return nil } diff --git a/components/engine/server.go b/components/engine/server.go index 7309279805..de73d6c815 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -439,7 +439,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return err } defer layer.Close() - if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { return err } } From c364da0b5142fa2e10773de9e3e651112fd662e4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 15:44:55 -0700 Subject: [PATCH 052/134] Remove unused parameter Upstream-commit: 0f134b4bf81a4d0160932852854b190b7ee7e3b9 Component: engine --- components/engine/graph.go | 6 +++--- components/engine/image.go | 2 +- components/engine/server.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/graph.go b/components/engine/graph.go index 1fe14f6458..bf27f93ed4 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -109,7 +109,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Container = container.ID img.ContainerConfig = *container.Config } - if err := graph.Register(nil, layerData, layerData != nil, img); err != nil { + if err := graph.Register(nil, layerData, img); err != nil { return nil, err } return img, nil @@ -117,7 +117,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut // Register imports a pre-existing image into the graph. // FIXME: pass img as first argument -func (graph *Graph) Register(jsonData []byte, layerData Archive, store bool, img *Image) error { +func (graph *Graph) Register(jsonData []byte, layerData Archive, img *Image) error { if err := ValidateID(img.ID); err != nil { return err } @@ -130,7 +130,7 @@ func (graph *Graph) Register(jsonData []byte, layerData Archive, store bool, img if err != nil { return fmt.Errorf("Mktemp failed: %s", err) } - if err := StoreImage(img, jsonData, layerData, tmp, store); err != nil { + if err := StoreImage(img, jsonData, layerData, tmp); err != nil { return err } // Commit diff --git a/components/engine/image.go b/components/engine/image.go index 7f03cc4bf8..220f1c70a5 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -72,7 +72,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, jsonData []byte, layerData Archive, root string, store bool) error { +func StoreImage(img *Image, jsonData []byte, layerData Archive, root string) error { // Check that root doesn't already exist if _, err := os.Stat(root); err == nil { return fmt.Errorf("Image %s already exists", img.ID) diff --git a/components/engine/server.go b/components/engine/server.go index de73d6c815..aecc730794 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -439,7 +439,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return err } defer layer.Close() - if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), img); err != nil { return err } } From 0e6bb26f7715b300bd7a152b7da15092df6336d8 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 16:16:31 -0700 Subject: [PATCH 053/134] Switch json/payload order Upstream-commit: 394941b6b0a30fecf8ae7b6de5880fa553141f93 Component: engine --- components/engine/auth/auth.go | 4 ++-- components/engine/utils/tarsum.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index e402031ca2..6dd6ceb620 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -16,9 +16,9 @@ import ( const CONFIGFILE = ".dockercfg" // Only used for user auth + account creation -//const INDEXSERVER = "https://index.docker.io/v1/" +const INDEXSERVER = "https://index.docker.io/v1/" -const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" +//const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/components/engine/utils/tarsum.go b/components/engine/utils/tarsum.go index 015b9b3076..d3e1db61f1 100644 --- a/components/engine/utils/tarsum.go +++ b/components/engine/utils/tarsum.go @@ -145,13 +145,13 @@ func (ts *TarSum) Read(buf []byte) (int, error) { func (ts *TarSum) Sum(extra []byte) string { sort.Strings(ts.sums) h := sha256.New() + if extra != nil { + h.Write(extra) + } for _, sum := range ts.sums { Debugf("-->%s<--", sum) h.Write([]byte(sum)) } - if extra != nil { - h.Write(extra) - } checksum := "tarsum+sha256:" + hex.EncodeToString(h.Sum(nil)) Debugf("checksum processed: %s", checksum) return checksum From 753746a54d821b03cdaa9b1bcf6d04904e2f7427 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 16:44:34 -0700 Subject: [PATCH 054/134] Make sure the index also receives the checksums Upstream-commit: 5b27652ac6eaf1bc4c2a16e51919ec4272a58fd6 Component: engine --- components/engine/registry/registry.go | 14 +++++++++++++- components/engine/server.go | 22 ++++++++++++---------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 40b9872a4a..4e9dd8895f 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -469,7 +469,19 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { - imgListJSON, err := json.Marshal(imgList) + cleanImgList := []*ImgData{} + + if validate { + for _, elem := range imgList { + if elem.Checksum != "" { + cleanImgList = append(cleanImgList, elem) + } + } + } else { + cleanImgList = imgList + } + + imgListJSON, err := json.Marshal(cleanImgList) if err != nil { return nil, err } diff --git a/components/engine/server.go b/components/engine/server.go index aecc730794..88ff2ac3a6 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -645,9 +645,11 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) continue } - if err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { + if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? return err + } else { + elem.Checksum = checksum } out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { @@ -663,11 +665,11 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName return nil } -func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) error { +func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) { out = utils.NewWriteFlusher(out) jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgID, "json")) if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) + return "", fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) } out.Write(sf.FormatStatus("Pushing %s", imgID)) @@ -679,29 +681,29 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID)) - return nil + return "", nil } - return err + return "", err } layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out) if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) + return "", fmt.Errorf("Failed to generate layer archive: %s", err) } // Send the layer if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil { - return err + return "", err } else { imgData.Checksum = checksum } // Send the checksum if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { - return err + return "", err } - return nil + return imgData.Checksum, nil } // FIXME: Allow to interupt current push when new push of same image is done. @@ -739,7 +741,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo var token []string out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName)) - if err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { + if _, err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { return err } return nil From 7e99f4ac78242ffb92e6778514ff7d85a83832c3 Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Mon, 29 Jul 2013 14:17:15 -0700 Subject: [PATCH 055/134] Updated the description of run -d The goal is to make it more clear this will give you the container id after run completes. Since stdout is now standard on run, "docker run -d" is the best (or only) way to get the container ID returned from docker after a plain run, but the description (help) does not hint any such thing. Upstream-commit: 5dc86d7bca17c2996264a18cc26f06d30e532588 Component: engine --- components/engine/container.go | 2 +- components/engine/docs/sources/commandline/command/run.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index d0b6ca4ce2..d610c3c7d4 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -100,7 +100,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, flHostname := cmd.String("h", "", "Container host name") flUser := cmd.String("u", "", "Username or UID") - flDetach := cmd.Bool("d", false, "Detached mode: leave the container running in the background") + flDetach := cmd.Bool("d", false, "Detached mode: Run container in the background, print new container id") flAttach := NewAttachOpts() cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.") flStdin := cmd.Bool("i", false, "Keep stdin open even if not attached") diff --git a/components/engine/docs/sources/commandline/command/run.rst b/components/engine/docs/sources/commandline/command/run.rst index db67ef0705..db043c3b3f 100644 --- a/components/engine/docs/sources/commandline/command/run.rst +++ b/components/engine/docs/sources/commandline/command/run.rst @@ -15,7 +15,7 @@ -a=map[]: Attach to stdin, stdout or stderr. -c=0: CPU shares (relative weight) -cidfile="": Write the container ID to the file - -d=false: Detached mode: leave the container running in the background + -d=false: Detached mode: Run container in the background, print new container id -e=[]: Set environment variables -h="": Container host name -i=false: Keep stdin open even if not attached From 86d430ea320a9d5c165d430544ba05f90b2305bb Mon Sep 17 00:00:00 2001 From: dsissitka Date: Tue, 30 Jul 2013 01:39:29 -0400 Subject: [PATCH 056/134] Fixed a couple of minor syntax errors. Upstream-commit: 9ba998312de5826f24f693c8518ecda700135b4b Component: engine --- .../engine/docs/sources/examples/couchdb_data_volumes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/sources/examples/couchdb_data_volumes.rst b/components/engine/docs/sources/examples/couchdb_data_volumes.rst index d6babe557f..97af733a82 100644 --- a/components/engine/docs/sources/examples/couchdb_data_volumes.rst +++ b/components/engine/docs/sources/examples/couchdb_data_volumes.rst @@ -39,7 +39,7 @@ This time, we're requesting shared access to $COUCH1's volumes. .. code-block:: bash - COUCH2=$(docker run -d -volumes-from $COUCH1) shykes/couchdb:2013-05-03) + COUCH2=$(docker run -d -volumes-from $COUCH1 shykes/couchdb:2013-05-03) Browse data on the second database ---------------------------------- @@ -48,6 +48,6 @@ Browse data on the second database HOST=localhost URL="http://$HOST:$(docker port $COUCH2 5984)/_utils/" - echo "Navigate to $URL in your browser. You should see the same data as in the first database!" + echo "Navigate to $URL in your browser. You should see the same data as in the first database"'!' Congratulations, you are running 2 Couchdb containers, completely isolated from each other *except* for their data. From fb6e560276b5e8f6bd32f112d4e3bbdd39302c53 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 13:47:29 +0200 Subject: [PATCH 057/134] add ufw doc Upstream-commit: 7d0b8c726c0d178291063751ac735d6caebfb45a Component: engine --- .../docs/sources/installation/ubuntulinux.rst | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index ed592d3a9d..299b7a29e9 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -19,6 +19,8 @@ Docker has the following dependencies * Linux kernel 3.8 (read more about :ref:`kernel`) * AUFS file system support (we are working on BTRFS support as an alternative) +Please read :ref:`ufw`, if you plan to use `UFW (Uncomplicated Firewall) `_ + .. _ubuntu_precise: Ubuntu Precise 12.04 (LTS) (64-bit) @@ -135,3 +137,35 @@ Verify it worked **Done!**, now continue with the :ref:`hello_world` example. + + +.. _ufw: + +Docker and UFW +^^^^^^^^^^^^^^ + +Docker uses a bridge to manage containers networking, by default UFW drop all `forwarding`, a first step is to enable forwarding: + +.. code-block:: bash + + sudo nano /etc/default/ufw + ---- + # Change: + # DEFAULT_FORWARD_POLICY="DROP" + # to + DEFAULT_FORWARD_POLICY="ACCEPT" + +Then reload UFW: + +.. code-block:: bash + + sudo ufw reload + + +UFW's default set of rules denied all `incoming`, so if you want to be able to reach your containers from another host, +you should allow incoming connexions on the docker port (default 4243): + +.. code-block:: bash + + sudo ufw allow 4243/tcp + From d2c440fbc78c4da8c0590a369f0b48a7518b5c96 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 12:09:07 +0000 Subject: [PATCH 058/134] add parallel pull to 1.4 Upstream-commit: 46f59dd9333baa578b184eb25b386ac2f41caf04 Component: engine --- components/engine/api.go | 2 +- components/engine/buildfile.go | 2 +- .../docs/sources/api/docker_remote_api.rst | 4 ++++ components/engine/runtime_test.go | 2 +- components/engine/server.go | 24 ++++++++++++------- components/engine/utils/utils.go | 5 +--- 6 files changed, 24 insertions(+), 15 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index 4ad2ba461a..cc7482be71 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -386,7 +386,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } sf := utils.NewStreamFormatter(version > 1.0) if image != "" { //pull - if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { + if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}, version > 1.3); err != nil { if sf.Used() { w.Write(sf.FormatError(err)) return nil diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 736725e915..fda6e5bf41 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -55,7 +55,7 @@ func (b *buildFile) CmdFrom(name string) error { if err != nil { if b.runtime.graph.IsNotExist(err) { remote, tag := utils.ParseRepositoryTag(name) - if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil); err != nil { + if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil, true); err != nil { return err } image, err = b.runtime.repositories.LookupImage(name) diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index 193be501d0..dd017cde8d 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -40,6 +40,10 @@ You can still call an old version of the api using What's new ---------- +.. http:post:: /images/create + + **New!** When pull a repo, all images are now downloaded in parallel. + .. http:get:: /containers/(id)/top **New!** You can now use ps args with docker top, like `docker top aux` diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index 0b0f62f199..7ecf199e09 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -101,7 +101,7 @@ func init() { // If the unit test is not found, try to download it. if img, err := globalRuntime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID { // Retrieve the Image - if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil { + if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil, true); err != nil { panic(err) } } diff --git a/components/engine/server.go b/components/engine/server.go index dc08ee5fbd..7417264e50 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -446,7 +446,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return nil } -func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error { +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error { out.Write(sf.FormatStatus("", "Pulling repository %s", localName)) repoData, err := r.GetRepositoryData(indexEp, remoteName) @@ -492,7 +492,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName errors := make(chan error) for _, image := range repoData.ImgList { - go func(img *registry.ImgData) { + downloadImage := func(img *registry.ImgData) { if askedTag != "" && img.Tag != askedTag { utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) errors <- nil @@ -518,12 +518,20 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName errors <- fmt.Errorf("Could not find repository on any of the indexed registries.") } errors <- nil - }(image) + } + + if parallel { + go downloadImage(image) + } else { + downloadImage(image) + } } - for i := 0; i < len(repoData.ImgList); i++ { - if err := <-errors; err != nil { - return err + if parallel { + for i := 0; i < len(repoData.ImgList); i++ { + if err := <-errors; err != nil { + return err + } } } @@ -577,7 +585,7 @@ func (srv *Server) poolRemove(kind, key string) error { return nil } -func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { +func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig, parallel bool) error { r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.versionInfos()...) if err != nil { return err @@ -599,7 +607,7 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut } out = utils.NewWriteFlusher(out) - err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf) + err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf, parallel) if err != nil { if err := srv.pullImage(r, out, remoteName, endpoint, nil, sf); err != nil { return err diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 77f701c248..f91fb0663c 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -635,7 +635,6 @@ func (jm *JSONMessage) Display(out io.Writer) error { return nil } - func DisplayJSONMessagesStream(in io.Reader, out io.Writer) error { dec := json.NewDecoder(in) jm := JSONMessage{} @@ -670,8 +669,6 @@ func DisplayJSONMessagesStream(in io.Reader, out io.Writer) error { return nil } -======= ->>>>>>> master type StreamFormatter struct { json bool used bool @@ -708,7 +705,7 @@ func (sf *StreamFormatter) FormatError(err error) []byte { func (sf *StreamFormatter) FormatProgress(id, action, progress string) []byte { sf.used = true if sf.json { - b, err := json.Marshal(&JSONMessage{Status: action, Progress: progress, ID:id}) + b, err := json.Marshal(&JSONMessage{Status: action, Progress: progress, ID: id}) if err != nil { return nil } From 6c28901fdbf8d8547cd76a21e0a38849f346fa46 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 13:13:18 +0000 Subject: [PATCH 059/134] fix tests about refactor checksums Upstream-commit: b14c251862021b2bc82aa8e0146e5e5e80f1c713 Component: engine --- components/engine/graph_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/engine/graph_test.go b/components/engine/graph_test.go index 18682338d9..2898fccf99 100644 --- a/components/engine/graph_test.go +++ b/components/engine/graph_test.go @@ -38,7 +38,7 @@ func TestInterruptedRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - go graph.Register(badArchive, false, image) + go graph.Register(nil, badArchive, image) time.Sleep(200 * time.Millisecond) w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling) if _, err := graph.Get(image.ID); err == nil { @@ -49,7 +49,7 @@ func TestInterruptedRegister(t *testing.T) { if err != nil { t.Fatal(err) } - if err := graph.Register(goodArchive, false, image); err != nil { + if err := graph.Register(nil, goodArchive, image); err != nil { t.Fatal(err) } } @@ -95,7 +95,7 @@ func TestRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - err = graph.Register(archive, false, image) + err = graph.Register(nil, archive, image) if err != nil { t.Fatal(err) } @@ -225,7 +225,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } // Test delete twice (pull -> rm -> pull -> rm) - if err := graph.Register(archive, false, img1); err != nil { + if err := graph.Register(nil, archive, img1); err != nil { t.Fatal(err) } if err := graph.Delete(img1.ID); err != nil { From c7d245ec6d1824f5980ac9918eb086ed02cc4e1b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 16:39:35 +0000 Subject: [PATCH 060/134] Add check that the request is good Upstream-commit: e4752c8c1a09fc3cc96dbb9be7183b271db3d6b7 Component: engine --- components/engine/utils/utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 190dc175cd..def5ae5a50 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -736,6 +736,9 @@ func GetReleaseVersion() string { return "" } defer resp.Body.Close() + if resp.ContentLength > 24 || resp.StatusCode != 200 { + return "" + } body, err := ioutil.ReadAll(resp.Body) if err != nil { return "" From 554c357a3ee4b84a513a3e7458c062c12714fb33 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 17:18:19 +0000 Subject: [PATCH 061/134] update http://get.docker.io/latest Upstream-commit: e66e0289abc213164dca1e1eadfb0380b6e81904 Component: engine --- components/engine/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index 6050000582..dd365dc30e 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -50,6 +50,7 @@ release: $(BINRELEASE) s3cmd -P put $(BINRELEASE) s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-$(RELEASE_VERSION).tgz s3cmd -P put docker-latest.tgz s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-latest.tgz s3cmd -P put $(SRCRELEASE)/bin/docker s3://get.docker.io/builds/`uname -s`/`uname -m`/docker + echo $(RELEASE_VERSION) > latest ; s3cmd -P put latest s3://get.docker.io/latest ; rm latest srcrelease: $(SRCRELEASE) deps: $(DOCKER_DIR) @@ -65,7 +66,6 @@ $(BINRELEASE): $(SRCRELEASE) rm -f $(BINRELEASE) cd $(SRCRELEASE); make; cp -R bin docker-$(RELEASE_VERSION); tar -f ../$(BINRELEASE) -zv -c docker-$(RELEASE_VERSION) cd $(SRCRELEASE); cp -R bin docker-latest; tar -f ../docker-latest.tgz -zv -c docker-latest - clean: @rm -rf $(dir $(DOCKER_BIN)) ifeq ($(GOPATH), $(BUILD_DIR)) From 7aa4663949e8f79b0adb7e8f8d71126631c7596a Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 24 Jul 2013 03:01:24 +0000 Subject: [PATCH 062/134] Return registy status code in error Added Details map to the JSONMessage Upstream-commit: 3043c2641990d94298c6377b7ef14709263a4709 Component: engine --- components/engine/api.go | 12 +++++++---- components/engine/commands.go | 29 +++++++++++++++++--------- components/engine/registry/registry.go | 24 ++++++++++----------- components/engine/utils/error.go | 18 ++++++++++++++++ components/engine/utils/utils.go | 26 ++++++++++++++++------- 5 files changed, 75 insertions(+), 34 deletions(-) create mode 100644 components/engine/utils/error.go diff --git a/components/engine/api.go b/components/engine/api.go index 4ad2ba461a..5869669df0 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -388,7 +388,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht if image != "" { //pull if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + w.Write(sf.FormatError(err, 0)) return nil } return err @@ -396,7 +396,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } else { //import if err := srv.ImageImport(src, repo, tag, r.Body, w, sf); err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + w.Write(sf.FormatError(err, 0)) return nil } return err @@ -441,7 +441,7 @@ func postImagesInsert(srv *Server, version float64, w http.ResponseWriter, r *ht imgID, err := srv.ImageInsert(name, url, path, w, sf) if err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + w.Write(sf.FormatError(err, 0)) return nil } } @@ -472,7 +472,11 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http sf := utils.NewStreamFormatter(version > 1.0) if err := srv.ImagePush(name, w, sf, authConfig); err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + var code int + if httpErr, ok := err.(*utils.HTTPRequestError); ok { + code = httpErr.StatusCode + } + w.Write(sf.FormatError(err, code)) return nil } return err diff --git a/components/engine/commands.go b/components/engine/commands.go index 95ddca1f1d..9ad2c367ae 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -30,7 +30,8 @@ import ( const VERSION = "0.5.0-dev" var ( - GITCOMMIT string + GITCOMMIT string + AuthRequiredError error = fmt.Errorf("Authentication is required.") ) func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) { @@ -814,10 +815,6 @@ func (cli *DockerCli) CmdPush(args ...string) error { return nil } - if err := cli.checkIfLogged("push"); err != nil { - return err - } - // If we're not using a custom registry, we know the restrictions // applied to repository names and can warn the user in advance. // Custom repositories can have different rules, and we must also @@ -826,13 +823,22 @@ func (cli *DockerCli) CmdPush(args ...string) error { return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.configFile.Configs[auth.IndexServerAddress()].Username, name) } - buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) - if err != nil { - return err + v := url.Values{} + push := func() error { + buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) + if err != nil { + return err + } + + return cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out) } - v := url.Values{} - if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out); err != nil { + if err := push(); err != nil { + if err == AuthRequiredError { + if err = cli.checkIfLogged("push"); err == nil { + return push() + } + } return err } return nil @@ -1559,6 +1565,9 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e } else if err != nil { return err } + if jm.Error != nil && jm.Error.Code == 401 { + return AuthRequiredError + } jm.Display(out) } } else { diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 4e9dd8895f..ed6f4c7df8 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -147,7 +147,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { - return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } return nil, err } @@ -197,7 +197,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } defer res.Body.Close() if res.StatusCode != 200 { - return nil, -1, fmt.Errorf("HTTP code %d", res.StatusCode) + return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size")) @@ -289,12 +289,12 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e } defer res.Body.Close() if res.StatusCode == 401 { - return nil, fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Please login first (HTTP code %d)", res.StatusCode), res) } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode != 200 { - return nil, fmt.Errorf("HTTP code: %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) } var tokens []string @@ -391,7 +391,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return utils.NewHTTPRequestError(fmt.Sprint("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { @@ -399,7 +399,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) } return nil } @@ -427,9 +427,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil } @@ -463,7 +463,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) + return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) } return nil } @@ -540,7 +540,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] @@ -564,7 +564,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) } } @@ -586,7 +586,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { } defer res.Body.Close() if res.StatusCode != 200 { - return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } rawData, err := ioutil.ReadAll(res.Body) if err != nil { diff --git a/components/engine/utils/error.go b/components/engine/utils/error.go new file mode 100644 index 0000000000..7e3c846ebc --- /dev/null +++ b/components/engine/utils/error.go @@ -0,0 +1,18 @@ +package utils + +import ( + "net/http" +) + +type HTTPRequestError struct { + Message string + StatusCode int +} + +func (e *HTTPRequestError) Error() string { + return e.Message +} + +func NewHTTPRequestError(msg string, resp *http.Response) error { + return &HTTPRequestError{Message: msg, StatusCode: resp.StatusCode} +} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index c70e80b72e..659d960d0a 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -607,12 +607,22 @@ func NewWriteFlusher(w io.Writer) *WriteFlusher { return &WriteFlusher{w: w, flusher: flusher} } +type JSONError struct { + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + type JSONMessage struct { - Status string `json:"status,omitempty"` - Progress string `json:"progress,omitempty"` - Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + Status string `json:"status,omitempty"` + Progress string `json:"progress,omitempty"` + ErrorMessage string `json:"error,omitempty"` //deprecated + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` + Error *JSONError `json:"errorDetail,omitempty"` +} + +func (e *JSONError) Error() string { + return e.Message } func (jm *JSONMessage) Display(out io.Writer) error { @@ -621,8 +631,8 @@ func (jm *JSONMessage) Display(out io.Writer) error { } if jm.Progress != "" { fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress) - } else if jm.Error != "" { - return fmt.Errorf(jm.Error) + } else if jm.Error != nil { + return jm.Error } else if jm.ID != "" { fmt.Fprintf(out, "%s: %s\n", jm.ID, jm.Status) } else { @@ -656,7 +666,7 @@ func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte func (sf *StreamFormatter) FormatError(err error) []byte { sf.used = true if sf.json { - if b, err := json.Marshal(&JSONMessage{Error: err.Error()}); err == nil { + if b, err := json.Marshal(&JSONMessage{Error: &JSONError{Code: code, Message: err.Error()}, ErrorMessage: err.Error()}); err == nil { return b } return []byte("{\"error\":\"format error\"}") From e5429eecead9274240eaf141d373665a1111cf16 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 31 Jul 2013 07:56:53 +0000 Subject: [PATCH 063/134] improve tests Upstream-commit: 73c6d9f135493220f034d440d26fedc0242e133a Component: engine --- components/engine/server_test.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/components/engine/server_test.go b/components/engine/server_test.go index 0caf8a5f24..de95d743ba 100644 --- a/components/engine/server_test.go +++ b/components/engine/server_test.go @@ -20,7 +20,11 @@ func TestContainerTagImageDelete(t *testing.T) { if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil { t.Fatal(err) } - if err := srv.runtime.repositories.Set("utest:5000/docker", "tag2", unitTestImageName, false); err != nil { + + if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil { + t.Fatal(err) + } + if err := srv.runtime.repositories.Set("utest:5000/docker", "tag3", unitTestImageName, false); err != nil { t.Fatal(err) } @@ -29,11 +33,24 @@ func TestContainerTagImageDelete(t *testing.T) { t.Fatal(err) } + if len(images) != len(initialImages)+3 { + t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) + } + + if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil { + t.Fatal(err) + } + + images, err = srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + if len(images) != len(initialImages)+2 { t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) } - if _, err := srv.ImageDelete("utest:5000/docker:tag2", true); err != nil { + if _, err := srv.ImageDelete("utest:5000/docker:tag3", true); err != nil { t.Fatal(err) } From 496b0008be2bd1ac2abbe63c82e2cc95c978ef61 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 31 Jul 2013 08:01:20 +0000 Subject: [PATCH 064/134] fix same issue in api.go Upstream-commit: a7068510a5ee4af6776221ba00bc332266f97088 Component: engine --- components/engine/api.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index 834c41a68c..d0a8ada249 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -786,12 +786,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ remoteURL := r.FormValue("remote") repoName := r.FormValue("t") rawSuppressOutput := r.FormValue("q") - tag := "" - if strings.Contains(repoName, ":") { - remoteParts := strings.Split(repoName, ":") - tag = remoteParts[1] - repoName = remoteParts[0] - } + repoName, tag := utils.ParseRepositoryTag(repoName) var context io.Reader From 30e6d541d0b27da785ccbb43ffb848adc62488b9 Mon Sep 17 00:00:00 2001 From: Nolan Date: Tue, 30 Jul 2013 13:23:34 -0500 Subject: [PATCH 065/134] Add hostname to the container environment. Upstream-commit: 9a604acc23d30d00ae907acdf756cbcdf0e4cf83 Component: engine --- components/engine/container.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/container.go b/components/engine/container.go index d610c3c7d4..ccc7ab3e9f 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -652,6 +652,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { "-e", "HOME=/", "-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "-e", "container=lxc", + "-e", "HOSTNAME="+container.Config.Hostname, ) for _, elem := range container.Config.Env { From d69f94df65007e156adc9bc9fed2c65e1dd958aa Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Wed, 31 Jul 2013 12:17:42 -0700 Subject: [PATCH 066/134] Solved the logo being squished in Safari Upstream-commit: e0c24ccfc37b64eb919c5675d8a8f383201fa7cb Component: engine --- components/engine/docs/theme/docker/layout.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/theme/docker/layout.html b/components/engine/docs/theme/docker/layout.html index 0b22f22fab..ca26f44dc0 100755 --- a/components/engine/docs/theme/docker/layout.html +++ b/components/engine/docs/theme/docker/layout.html @@ -79,7 +79,7 @@
- +
From 0a6a10bae5bc5e904abe7cd452dacaf862cb4da7 Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Wed, 31 Jul 2013 13:44:10 -0700 Subject: [PATCH 067/134] Updated docs README with instructions to preview the generated manfile, and where to get the one generated by sphinx. Upstream-commit: 26229d78f2b77592ab6761a4ffce38ab7d859f3b Component: engine --- components/engine/docs/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/components/engine/docs/README.md b/components/engine/docs/README.md index 32be549ef5..7d7b67ca04 100644 --- a/components/engine/docs/README.md +++ b/components/engine/docs/README.md @@ -74,5 +74,12 @@ Guides on using sphinx Start without $, so it's easy to copy and paste. +Manpages +-------- + * To make the manpages, simply run 'make man'. Pleae note there is a bug in spinx 1.1.3 which makes this fail. -Upgrade to the latest version of sphinx. \ No newline at end of file +Upgrade to the latest version of sphinx. +* Then preview the manpage by running `man _build/man/docker.1`, where _build/man/docker.1 is the path to the generated +manfile + +The manpages are also autogenerated by our hosted readthedocs here: http://docs-docker.dotcloud.com/projects/docker/downloads/ \ No newline at end of file From 9fb9438668ed6dd123f3053be1e14dff64f5c56d Mon Sep 17 00:00:00 2001 From: Thatcher Date: Wed, 31 Jul 2013 13:59:56 -0700 Subject: [PATCH 068/134] Some more improvements on the docs readme. Removed references to website. Upstream-commit: ad3b091d535cde5544ec3c11da4cf8d9b278a325 Component: engine --- components/engine/docs/README.md | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/components/engine/docs/README.md b/components/engine/docs/README.md index 7d7b67ca04..1a40b5bb5c 100644 --- a/components/engine/docs/README.md +++ b/components/engine/docs/README.md @@ -1,14 +1,12 @@ -Docker documentation and website -================================ +Docker Documentation +==================== Documentation ------------- -This is your definite place to contribute to the docker documentation. The documentation is generated from the -.rst files under sources. - -The folder also contains the other files to create the http://docker.io website, but you can generally ignore -most of those. +This is your definite place to contribute to the docker documentation. After each push to master the documentation +is automatically generated and made available on [docs.docker.io](http://docs.docker.io) +Each of the .rst files under sources reflects a page on the documentation. Installation ------------ @@ -36,13 +34,11 @@ Images ------ When you need to add images, try to make them as small as possible (e.g. as gif). - Notes ----- * For the template the css is compiled from less. When changes are needed they can be compiled using lessc ``lessc main.less`` or watched using watch-lessc ``watch-lessc -i main.less -o main.css`` - Guides on using sphinx ---------------------- * To make links to certain pages create a link target like so: @@ -77,9 +73,8 @@ Guides on using sphinx Manpages -------- -* To make the manpages, simply run 'make man'. Pleae note there is a bug in spinx 1.1.3 which makes this fail. +* To make the manpages, simply run 'make man'. Please note there is a bug in spinx 1.1.3 which makes this fail. Upgrade to the latest version of sphinx. * Then preview the manpage by running `man _build/man/docker.1`, where _build/man/docker.1 is the path to the generated manfile - -The manpages are also autogenerated by our hosted readthedocs here: http://docs-docker.dotcloud.com/projects/docker/downloads/ \ No newline at end of file +* The manpages are also autogenerated by our hosted readthedocs here: http://docs-docker.dotcloud.com/projects/docker/downloads/ From 2eba08c4b7ed08233d3c090950e2ede4b059ecfe Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 1 Aug 2013 02:42:22 +0200 Subject: [PATCH 069/134] Handle ip route showing mask-less IP addresses Sometimes `ip route` will show mask-less IPs, so net.ParseCIDR will fail. If it does we check if we can net.ParseIP, and fail only if we can't. Fixes #1214 Fixes #362 Upstream-commit: 2e72882216ce13169a578614202830a5b084bfb4 Component: engine --- components/engine/network.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/engine/network.go b/components/engine/network.go index 2e2dc7785c..daffbfcde4 100644 --- a/components/engine/network.go +++ b/components/engine/network.go @@ -104,7 +104,11 @@ func checkRouteOverlaps(dockerNetwork *net.IPNet) error { continue } if _, network, err := net.ParseCIDR(strings.Split(line, " ")[0]); err != nil { - return fmt.Errorf("Unexpected ip route output: %s (%s)", err, line) + // is this a mask-less IP address? + if ip := net.ParseIP(strings.Split(line, " ")[0]); ip == nil { + // fail only if it's neither a network nor a mask-less IP address + return fmt.Errorf("Unexpected ip route output: %s (%s)", err, line) + } } else if networkOverlaps(dockerNetwork, network) { return fmt.Errorf("Network %s is already routed: '%s'", dockerNetwork.String(), line) } From 55962b1db1a3c6b3b8beaa51424e8e27f414096d Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Mon, 29 Jul 2013 09:45:19 -0700 Subject: [PATCH 070/134] testing, issue #1331: Add registry functional test to docker-ci Upstream-commit: 2e7df5182cc94e3699ebc1031e89b1605f9d42f9 Component: engine --- components/engine/testing/README.rst | 4 ++++ components/engine/testing/Vagrantfile | 4 +++- .../engine/testing/buildbot/credentials.cfg | 5 +++++ components/engine/testing/buildbot/master.cfg | 20 +++++++++++++++++-- .../engine/testing/buildbot/requirements.txt | 1 + .../testing/buildbot/setup_credentials.sh | 17 ++++++++++++++++ .../testing/functionaltests/test_registry.sh | 11 ++++++++++ 7 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 components/engine/testing/buildbot/credentials.cfg create mode 100755 components/engine/testing/buildbot/setup_credentials.sh create mode 100755 components/engine/testing/functionaltests/test_registry.sh diff --git a/components/engine/testing/README.rst b/components/engine/testing/README.rst index 3b11092f9f..ce5aa837a4 100644 --- a/components/engine/testing/README.rst +++ b/components/engine/testing/README.rst @@ -40,6 +40,10 @@ Deployment export SMTP_USER=xxxxxxxxxxxx export SMTP_PWD=xxxxxxxxxxxx + # Define docker registry functional test credentials + export REGISTRY_USER=xxxxxxxxxxxx + export REGISTRY_PWD=xxxxxxxxxxxx + # Checkout docker git clone git://github.com/dotcloud/docker.git diff --git a/components/engine/testing/Vagrantfile b/components/engine/testing/Vagrantfile index 47257201dc..e76a951508 100644 --- a/components/engine/testing/Vagrantfile +++ b/components/engine/testing/Vagrantfile @@ -29,7 +29,9 @@ Vagrant::Config.run do |config| "chown #{USER}.#{USER} /data; cd /data; " \ "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \ "#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \ - "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " + "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " \ + "#{CFG_PATH}/setup_credentials.sh #{USER} " \ + "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ diff --git a/components/engine/testing/buildbot/credentials.cfg b/components/engine/testing/buildbot/credentials.cfg new file mode 100644 index 0000000000..fbdd35d578 --- /dev/null +++ b/components/engine/testing/buildbot/credentials.cfg @@ -0,0 +1,5 @@ +# Credentials for tests. Buildbot source this file on tests +# when needed. + +# Docker registry credentials. Format: 'username:password' +export DOCKER_CREDS='' diff --git a/components/engine/testing/buildbot/master.cfg b/components/engine/testing/buildbot/master.cfg index 61912808ec..29926dbe5f 100644 --- a/components/engine/testing/buildbot/master.cfg +++ b/components/engine/testing/buildbot/master.cfg @@ -19,6 +19,7 @@ TEST_USER = 'buildbot' # Credential to authenticate build triggers TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' GITHUB_DOCKER = 'github.com/dotcloud/docker' +BUILDBOT_PATH = '/data/buildbot' DOCKER_PATH = '/data/docker' BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME) DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker' @@ -41,16 +42,19 @@ c['db'] = {'db_url':"sqlite:///state.sqlite"} c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)] c['slavePortnum'] = PORT_MASTER + # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME, - 'coverage'])] + 'registry','coverage'])] c['schedulers'] += [SingleBranchScheduler(name="all", change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=[BUILDER_NAME])] -c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage'], +c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage','registry'], hour=0, minute=30)] + # Builders +# Docker commit test factory = BuildFactory() factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; " @@ -58,6 +62,7 @@ factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, "go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))])) c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], factory=factory)] + # Docker coverage test coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n' 'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n' @@ -69,6 +74,17 @@ factory.addStep(ShellCommand(description='Coverage',logEnviron=False,usePTY=True c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], factory=factory)] +# Registry Functionaltest builder +factory = BuildFactory() +factory.addStep(ShellCommand(description='registry', logEnviron=False, + command='. {0}/master/credentials.cfg; ' + '{1}/testing/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, + DOCKER_PATH), usePTY=True)) + +c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], + factory=factory)] + + # Status authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]), forceBuild='auth') diff --git a/components/engine/testing/buildbot/requirements.txt b/components/engine/testing/buildbot/requirements.txt index 0e451b017d..4e183ba062 100644 --- a/components/engine/testing/buildbot/requirements.txt +++ b/components/engine/testing/buildbot/requirements.txt @@ -4,3 +4,4 @@ buildbot==0.8.7p1 buildbot_slave==0.8.7p1 nose==1.2.1 requests==1.1.0 +flask==0.10.1 diff --git a/components/engine/testing/buildbot/setup_credentials.sh b/components/engine/testing/buildbot/setup_credentials.sh new file mode 100755 index 0000000000..f093815d60 --- /dev/null +++ b/components/engine/testing/buildbot/setup_credentials.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Setup of test credentials. Called by Vagrantfile +export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" + +USER=$1 +REGISTRY_USER=$2 +REGISTRY_PWD=$3 + +BUILDBOT_PATH="/data/buildbot" +DOCKER_PATH="/data/docker" + +function run { su $USER -c "$1"; } + +run "cp $DOCKER_PATH/testing/buildbot/credentials.cfg $BUILDBOT_PATH/master" +cd $BUILDBOT_PATH/master +run "sed -i -E 's#(export DOCKER_CREDS=).+#\1\"$REGISTRY_USER:$REGISTRY_PWD\"#' credentials.cfg" diff --git a/components/engine/testing/functionaltests/test_registry.sh b/components/engine/testing/functionaltests/test_registry.sh new file mode 100755 index 0000000000..095a731631 --- /dev/null +++ b/components/engine/testing/functionaltests/test_registry.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Cleanup +rm -rf docker-registry + +# Get latest docker registry +git clone https://github.com/dotcloud/docker-registry.git + +# Configure and run registry tests +cd docker-registry; cp config_sample.yml config.yml +cd test; python -m unittest workflow From ce7a658e04b8465e50d5d463b69b30a2deb2fe6e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 1 Aug 2013 18:12:39 -0700 Subject: [PATCH 071/134] Make sure the routes IP are taken into consideration + add unit test for network overlap detection Upstream-commit: f5a8e90d101cd2dbb4ce19543ed15fff48579877 Component: engine --- components/engine/network.go | 31 ++++++++++++++++++++----------- components/engine/network_test.go | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/components/engine/network.go b/components/engine/network.go index daffbfcde4..eefd36df3b 100644 --- a/components/engine/network.go +++ b/components/engine/network.go @@ -93,24 +93,29 @@ func iptables(args ...string) error { return nil } -func checkRouteOverlaps(dockerNetwork *net.IPNet) error { - output, err := ip("route") - if err != nil { - return err - } - utils.Debugf("Routes:\n\n%s", output) - for _, line := range strings.Split(output, "\n") { +func checkRouteOverlaps(routes string, dockerNetwork *net.IPNet) error { + utils.Debugf("Routes:\n\n%s", routes) + for _, line := range strings.Split(routes, "\n") { if strings.Trim(line, "\r\n\t ") == "" || strings.Contains(line, "default") { continue } - if _, network, err := net.ParseCIDR(strings.Split(line, " ")[0]); err != nil { + _, network, err := net.ParseCIDR(strings.Split(line, " ")[0]) + if err != nil { // is this a mask-less IP address? if ip := net.ParseIP(strings.Split(line, " ")[0]); ip == nil { // fail only if it's neither a network nor a mask-less IP address return fmt.Errorf("Unexpected ip route output: %s (%s)", err, line) + } else { + _, network, err = net.ParseCIDR(ip.String() + "/32") + if err != nil { + return err + } + } + } + if err == nil && network != nil { + if networkOverlaps(dockerNetwork, network) { + return fmt.Errorf("Network %s is already routed: '%s'", dockerNetwork, line) } - } else if networkOverlaps(dockerNetwork, network) { - return fmt.Errorf("Network %s is already routed: '%s'", dockerNetwork.String(), line) } } return nil @@ -146,7 +151,11 @@ func CreateBridgeIface(ifaceName string) error { if err != nil { return err } - if err := checkRouteOverlaps(dockerNetwork); err == nil { + routes, err := ip("route") + if err != nil { + return err + } + if err := checkRouteOverlaps(routes, dockerNetwork); err == nil { ifaceAddr = addr break } else { diff --git a/components/engine/network_test.go b/components/engine/network_test.go index 8e6eaad773..bd3a16a1be 100644 --- a/components/engine/network_test.go +++ b/components/engine/network_test.go @@ -383,3 +383,22 @@ func TestNetworkOverlaps(t *testing.T) { //netX starts and ends before netY AssertNoOverlap("172.16.1.1/25", "172.16.2.1/24", t) } + +func TestCheckRouteOverlaps(t *testing.T) { + routes := `default via 10.0.2.2 dev eth0 +10.0.2.0 dev eth0 proto kernel scope link src 10.0.2.15 +10.0.3.0/24 dev lxcbr0 proto kernel scope link src 10.0.3.1 +10.0.42.0/24 dev testdockbr0 proto kernel scope link src 10.0.42.1 +172.16.42.0/24 dev docker0 proto kernel scope link src 172.16.42.1 +192.168.142.0/24 dev eth1 proto kernel scope link src 192.168.142.142` + + _, netX, _ := net.ParseCIDR("172.16.0.1/24") + if err := checkRouteOverlaps(routes, netX); err != nil { + t.Fatal(err) + } + + _, netX, _ = net.ParseCIDR("10.0.2.0/24") + if err := checkRouteOverlaps(routes, netX); err == nil { + t.Fatalf("10.0.2.0/24 and 10.0.2.0 should overlap but it doesn't") + } +} From da0dfa9ebd66eff0cbd7708bcd77e21dca48936d Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Fri, 2 Aug 2013 12:24:38 +0700 Subject: [PATCH 072/134] Move note about officially supported kernel It seems this a general note about kernel issues and not specific to Cgroups or namespaces. Upstream-commit: 2424480e2ce14d848c041c53cc041aab91308081 Component: engine --- .../engine/docs/sources/installation/kernel.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/components/engine/docs/sources/installation/kernel.rst b/components/engine/docs/sources/installation/kernel.rst index 58730f8191..7c5715a62d 100644 --- a/components/engine/docs/sources/installation/kernel.rst +++ b/components/engine/docs/sources/installation/kernel.rst @@ -15,12 +15,11 @@ In short, Docker has the following kernel requirements: - Cgroups and namespaces must be enabled. - - The officially supported kernel is the one recommended by the - :ref:`ubuntu_linux` installation path. It is the one that most developers - will use, and the one that receives the most attention from the core - contributors. If you decide to go with a different kernel and hit a bug, - please try to reproduce it with the official kernels first. +The officially supported kernel is the one recommended by the +:ref:`ubuntu_linux` installation path. It is the one that most developers +will use, and the one that receives the most attention from the core +contributors. If you decide to go with a different kernel and hit a bug, +please try to reproduce it with the official kernels first. If you cannot or do not want to use the "official" kernels, here is some technical background about the features (both optional and From f137da22a475029e5cfaebe026c762b2fe7bb737 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 02:47:58 -0400 Subject: [PATCH 073/134] http utils Upstream-commit: 793fd983ef937d2bea1edf5d8d855e2a452a4aa7 Component: engine --- components/engine/utils/http.go | 129 ++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 components/engine/utils/http.go diff --git a/components/engine/utils/http.go b/components/engine/utils/http.go new file mode 100644 index 0000000000..61fdbff04e --- /dev/null +++ b/components/engine/utils/http.go @@ -0,0 +1,129 @@ +package utils + +import ( + "bytes" + "io" + "net/http" + "strings" +) + +// VersionInfo is used to model entities which has a version. +// It is basically a tupple with name and version. +type VersionInfo interface { + Name() string + Version() string +} + +func validVersion(version VersionInfo) bool { + stopChars := " \t\r\n/" + if strings.ContainsAny(version.Name(), stopChars) { + return false + } + if strings.ContainsAny(version.Version(), stopChars) { + return false + } + return true +} + +// Convert versions to a string and append the string to the string base. +// +// Each VersionInfo will be converted to a string in the format of +// "product/version", where the "product" is get from the Name() method, while +// version is get from the Version() method. Several pieces of verson information +// will be concatinated and separated by space. +func appendVersions(base string, versions ...VersionInfo) string { + if len(versions) == 0 { + return base + } + + var buf bytes.Buffer + if len(base) > 0 { + buf.Write([]byte(base)) + } + + for _, v := range versions { + name := []byte(v.Name()) + version := []byte(v.Version()) + + if len(name) == 0 || len(version) == 0 { + continue + } + if !validVersion(v) { + continue + } + buf.Write([]byte(v.Name())) + buf.Write([]byte("/")) + buf.Write([]byte(v.Version())) + buf.Write([]byte(" ")) + } + return buf.String() +} + +// HTTPRequestDecorator is used to change an instance of +// http.Request. It could be used to add more header fields, +// change body, etc. +type HTTPRequestDecorator interface { + // ChangeRequest() changes the request accordingly. + // The changed request will be returned or err will be non-nil + // if an error occur. + ChangeRequest(req *http.Request) (newReq *http.Request, err error) +} + +// HTTPUserAgentDecorator appends the product/version to the user agent field +// of a request. +type HTTPUserAgentDecorator struct { + versions []VersionInfo +} + +func NewHTTPUserAgentDecorator(versions ...VersionInfo) HTTPRequestDecorator { + ret := new(HTTPUserAgentDecorator) + ret.versions = versions + return ret +} + +func (self *HTTPUserAgentDecorator) ChangeRequest(req *http.Request) (newReq *http.Request, err error) { + if req == nil { + return req, nil + } + + userAgent := appendVersions(req.UserAgent(), self.versions...) + if len(userAgent) > 0 { + req.Header.Set("User-Agent", userAgent) + } + return req, nil +} + +// HTTPRequestFactory creates an HTTP request +// and applies a list of decorators on the request. +type HTTPRequestFactory struct { + decorators []HTTPRequestDecorator +} + +func NewHTTPRequestFactory(d ...HTTPRequestDecorator) *HTTPRequestFactory { + ret := new(HTTPRequestFactory) + ret.decorators = d + return ret +} + +// NewRequest() creates a new *http.Request, +// applies all decorators in the HTTPRequestFactory on the request, +// then applies decorators provided by d on the request. +func (self *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader, d ...HTTPRequestDecorator) (*http.Request, error) { + req, err := http.NewRequest(method, urlStr, body) + if err != nil { + return nil, err + } + for _, dec := range self.decorators { + req, err = dec.ChangeRequest(req) + if err != nil { + return nil, err + } + } + for _, dec := range d { + req, err = dec.ChangeRequest(req) + if err != nil { + return nil, err + } + } + return req, err +} From 63124b5509fcd754b6529cce54d9a3e409915d39 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 03:08:08 -0400 Subject: [PATCH 074/134] reqFactory in Registry Upstream-commit: 7dac26ce69b442d55122caa2897572d3ac8255fa Component: engine --- components/engine/registry/registry.go | 101 ++++--------------------- 1 file changed, 15 insertions(+), 86 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 4e9dd8895f..da5c83bff1 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -100,13 +100,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return endpoint, reposName, err } -// VersionInfo is used to model entities which has a version. -// It is basically a tupple with name and version. -type VersionInfo interface { - Name() string - Version() string -} - func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -121,29 +114,14 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } -// Set the user agent field in the header based on the versions provided -// in NewRegistry() and extra. -func (r *Registry) setUserAgent(req *http.Request, extra ...VersionInfo) { - if len(r.baseVersions)+len(extra) == 0 { - return - } - if len(extra) == 0 { - req.Header.Set("User-Agent", r.baseVersionsStr) - } else { - req.Header.Set("User-Agent", appendVersions(r.baseVersionsStr, extra...)) - } - return -} - // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -170,7 +148,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } @@ -185,12 +163,11 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo // Retrieve an image from the Registry. func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -213,12 +190,11 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -239,7 +215,6 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -281,7 +256,6 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) res, err := r.client.Do(req) if err != nil { @@ -339,7 +313,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { return err } @@ -375,13 +349,12 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { @@ -410,14 +383,13 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr tarsumLayer := &utils.TarSum{Reader: layer} - req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { return "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -435,7 +407,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { - req, err := http.NewRequest(method, urlStr, body) + req, err := r.reqFactory.NewRequest(method, urlStr, body) if err != nil { return nil, err } @@ -455,7 +427,6 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { @@ -500,7 +471,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -521,7 +491,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -576,7 +545,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) - req, err := http.NewRequest("GET", u, nil) + req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err } @@ -628,52 +597,12 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - baseVersions []VersionInfo - baseVersionsStr string + client *http.Client + authConfig *auth.AuthConfig + reqFactory *utils.HTTPRequestFactory } -func validVersion(version VersionInfo) bool { - stopChars := " \t\r\n/" - if strings.ContainsAny(version.Name(), stopChars) { - return false - } - if strings.ContainsAny(version.Version(), stopChars) { - return false - } - return true -} - -// Convert versions to a string and append the string to the string base. -// -// Each VersionInfo will be converted to a string in the format of -// "product/version", where the "product" is get from the Name() method, while -// version is get from the Version() method. Several pieces of verson information -// will be concatinated and separated by space. -func appendVersions(base string, versions ...VersionInfo) string { - if len(versions) == 0 { - return base - } - - var buf bytes.Buffer - if len(base) > 0 { - buf.Write([]byte(base)) - } - - for _, v := range versions { - if !validVersion(v) { - continue - } - buf.Write([]byte(v.Name())) - buf.Write([]byte("/")) - buf.Write([]byte(v.Version())) - buf.Write([]byte(" ")) - } - return buf.String() -} - -func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...VersionInfo) (r *Registry, err error) { +func NewRegistry(root string, authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -689,7 +618,7 @@ func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...Versi if err != nil { return nil, err } - r.baseVersions = baseVersions - r.baseVersionsStr = appendVersions("", baseVersions...) + + r.reqFactory = factory return r, nil } From 4e4b48bb117af1e547de0b179f8904c7ded9d046 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 03:23:46 -0400 Subject: [PATCH 075/134] Server now use request factory Upstream-commit: 6a56b7b391ac967540915c2ee8f82b23714ad84c Component: engine --- components/engine/server.go | 15 ++++++++++----- components/engine/utils/http.go | 5 +++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/components/engine/server.go b/components/engine/server.go index cb7b2cf1be..b015e459f3 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -52,9 +52,9 @@ func (v *simpleVersionInfo) Version() string { // docker, go, git-commit (of the docker) and the host's kernel. // // Such information will be used on call to NewRegistry(). -func (srv *Server) versionInfos() []registry.VersionInfo { +func (srv *Server) versionInfos() []utils.VersionInfo { v := srv.DockerVersion() - ret := make([]registry.VersionInfo, 0, 4) + ret := make([]utils.VersionInfo, 0, 4) ret = append(ret, &simpleVersionInfo{"docker", v.Version}) if len(v.GoVersion) > 0 { @@ -102,7 +102,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { } func (srv *Server) ImagesSearch(term string) ([]APISearch, error) { - r, err := registry.NewRegistry(srv.runtime.root, nil, srv.versionInfos()...) + r, err := registry.NewRegistry(srv.runtime.root, nil, srv.reqFactory) if err != nil { return nil, err } @@ -559,7 +559,7 @@ func (srv *Server) poolRemove(kind, key string) error { } func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { - r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.versionInfos()...) + r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.reqFactory) if err != nil { return err } @@ -720,7 +720,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo out = utils.NewWriteFlusher(out) img, err := srv.runtime.graph.Get(localName) - r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.versionInfos()...) + r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.reqFactory) if err2 != nil { return err2 } @@ -1164,7 +1164,11 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( pushingPool: make(map[string]struct{}), events: make([]utils.JSONMessage, 0, 64), //only keeps the 64 last events listeners: make(map[string]chan utils.JSONMessage), + reqFactory: nil, } + ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) + factory := utils.NewHTTPRequestFactory(ud) + srv.reqFactory = factory runtime.srv = srv return srv, nil } @@ -1189,4 +1193,5 @@ type Server struct { pushingPool map[string]struct{} events []utils.JSONMessage listeners map[string]chan utils.JSONMessage + reqFactory *utils.HTTPRequestFactory } diff --git a/components/engine/utils/http.go b/components/engine/utils/http.go index 61fdbff04e..8c1e4b7a79 100644 --- a/components/engine/utils/http.go +++ b/components/engine/utils/http.go @@ -113,6 +113,11 @@ func (self *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader if err != nil { return nil, err } + + // By default, a nil factory should work. + if self == nil { + return req, nil + } for _, dec := range self.decorators { req, err = dec.ChangeRequest(req) if err != nil { From 1c96da0a5386a498585a5c86d2c1be5a4c9105db Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 03:30:45 -0400 Subject: [PATCH 076/134] auth with user agent Upstream-commit: 4bd287e107eab1623a0e77aeaecda77fc26e7536 Component: engine --- components/engine/api.go | 2 +- components/engine/auth/auth.go | 5 +++-- components/engine/server.go | 9 +++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index b8b7897c32..77faaaf23c 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -87,7 +87,7 @@ func postAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Reque if err != nil { return err } - status, err := auth.Login(authConfig) + status, err := auth.Login(authConfig, srv.HTTPRequestFactory()) if err != nil { return err } diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index 6dd6ceb620..b9e1ee153b 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/dotcloud/docker/utils" "io/ioutil" "net/http" "os" @@ -140,7 +141,7 @@ func SaveConfig(configFile *ConfigFile) error { } // try to register/login to the registry server -func Login(authConfig *AuthConfig) (string, error) { +func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { client := &http.Client{} reqStatusCode := 0 var status string @@ -171,7 +172,7 @@ func Login(authConfig *AuthConfig) (string, error) { "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { - req, err := http.NewRequest("GET", IndexServerAddress()+"users/", nil) + req, err := factory.NewRequest("GET", IndexServerAddress()+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { diff --git a/components/engine/server.go b/components/engine/server.go index b015e459f3..fa6f19ad4b 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1173,6 +1173,15 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( return srv, nil } +func (srv *Server) HTTPRequestFactory() *utils.HTTPRequestFactory { + if srv.reqFactory == nil { + ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) + factory := utils.NewHTTPRequestFactory(ud) + srv.reqFactory = factory + } + return srv.reqFactory +} + func (srv *Server) LogEvent(action, id string) { now := time.Now().Unix() jm := utils.JSONMessage{Status: action, ID: id, Time: now} From 9660113205be311ad8ebee13847c9e1e1ce15216 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 04:10:26 -0400 Subject: [PATCH 077/134] factory generated from one place. Upstream-commit: 5bc344ab73f73a1fb5bea0933a031ad0418cb8f8 Component: engine --- components/engine/server.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/components/engine/server.go b/components/engine/server.go index fa6f19ad4b..d77bcdb59e 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -102,7 +102,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { } func (srv *Server) ImagesSearch(term string) ([]APISearch, error) { - r, err := registry.NewRegistry(srv.runtime.root, nil, srv.reqFactory) + r, err := registry.NewRegistry(srv.runtime.root, nil, srv.HTTPRequestFactory()) if err != nil { return nil, err } @@ -559,7 +559,7 @@ func (srv *Server) poolRemove(kind, key string) error { } func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { - r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.reqFactory) + r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory()) if err != nil { return err } @@ -720,7 +720,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo out = utils.NewWriteFlusher(out) img, err := srv.runtime.graph.Get(localName) - r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.reqFactory) + r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory()) if err2 != nil { return err2 } @@ -1166,9 +1166,6 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( listeners: make(map[string]chan utils.JSONMessage), reqFactory: nil, } - ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) - factory := utils.NewHTTPRequestFactory(ud) - srv.reqFactory = factory runtime.srv = srv return srv, nil } From ac12d9b409aec16a0cfbade7ed576e6bf8832b82 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 2 Aug 2013 16:18:54 +0000 Subject: [PATCH 078/134] Add no cache for docker build Add a new flag to disable the image cache when building images. Upstream-commit: 3a123bc479457c4dfa14e39b7c42d9a9dccf8c32 Component: engine --- components/engine/api.go | 7 ++- components/engine/buildfile.go | 55 +++++++++++-------- components/engine/buildfile_test.go | 2 +- components/engine/commands.go | 4 ++ .../sources/api/docker_remote_api_v1.4.rst | 1 + .../sources/commandline/command/build.rst | 1 + 6 files changed, 44 insertions(+), 26 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index b8b7897c32..95b9c98de8 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -793,6 +793,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ remoteURL := r.FormValue("remote") repoName := r.FormValue("t") rawSuppressOutput := r.FormValue("q") + rawNoCache := r.FormValue("nocache") repoName, tag := utils.ParseRepositoryTag(repoName) var context io.Reader @@ -839,8 +840,12 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ if err != nil { return err } + noCache, err := getBoolParam(rawNoCache) + if err != nil { + return err + } - b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput) + b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput, !noCache) id, err := b.Build(context) if err != nil { fmt.Fprintf(w, "Error build: %s\n", err) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 736725e915..159d7ba704 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -26,11 +26,12 @@ type buildFile struct { builder *Builder srv *Server - image string - maintainer string - config *Config - context string - verbose bool + image string + maintainer string + config *Config + context string + verbose bool + utilizeCache bool tmpContainers map[string]struct{} tmpImages map[string]struct{} @@ -94,15 +95,17 @@ func (b *buildFile) CmdRun(args string) error { utils.Debugf("Command to be executed: %v", b.config.Cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } cid, err := b.run() @@ -397,16 +400,19 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment} defer func(cmd []string) { b.config.Cmd = cmd }(cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } + container, err := b.builder.Create(b.config) if err != nil { return err @@ -500,7 +506,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) { return "", fmt.Errorf("An error occured during the build\n") } -func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { +func NewBuildFile(srv *Server, out io.Writer, verbose, utilizeCache bool) BuildFile { return &buildFile{ builder: NewBuilder(srv.runtime), runtime: srv.runtime, @@ -510,5 +516,6 @@ func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { tmpContainers: make(map[string]struct{}), tmpImages: make(map[string]struct{}), verbose: verbose, + utilizeCache: utilizeCache, } } diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 78e53b8419..eda047d8ea 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -227,7 +227,7 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { ip := runtime.networkManager.bridgeNetwork.IP dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := NewBuildFile(srv, ioutil.Discard, false) + buildfile := NewBuildFile(srv, ioutil.Discard, false, true) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { t.Fatal(err) diff --git a/components/engine/commands.go b/components/engine/commands.go index 95ddca1f1d..7a1935ee61 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -160,6 +160,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH") tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success") suppressOutput := cmd.Bool("q", false, "Suppress verbose build output") + noCache := cmd.Bool("no-cache", false, "Do not use cache when building the image") if err := cmd.Parse(args); err != nil { return nil @@ -208,6 +209,9 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if isRemote { v.Set("remote", cmd.Arg(0)) } + if *noCache { + v.Set("nocache", "1") + } req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), body) if err != nil { return err diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst index 6ee0b35fa2..6830bacde0 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -928,6 +928,7 @@ Build an image from Dockerfile via stdin :query t: tag to be applied to the resulting image in case of success :query q: suppress verbose build output + :query nocache: do not use the cache when building the image :statuscode 200: no error :statuscode 500: server error diff --git a/components/engine/docs/sources/commandline/command/build.rst b/components/engine/docs/sources/commandline/command/build.rst index 45b6d2ec8e..fcd78dd1ab 100644 --- a/components/engine/docs/sources/commandline/command/build.rst +++ b/components/engine/docs/sources/commandline/command/build.rst @@ -12,6 +12,7 @@ Build a new container image from the source code at PATH -t="": Tag to be applied to the resulting image in case of success. -q=false: Suppress verbose build output. + -no-cache: Do not use the cache when building the image. When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context From a320ff8945d73d52d4fe33e5a2e0726e3392b72d Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 14:08:16 -0400 Subject: [PATCH 079/134] update auth_test.go Upstream-commit: 7bade49d4c661c5037de586e6f69291999038ef9 Component: engine --- components/engine/auth/auth_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/auth/auth_test.go b/components/engine/auth/auth_test.go index d94d429da1..24a0666cf7 100644 --- a/components/engine/auth/auth_test.go +++ b/components/engine/auth/auth_test.go @@ -33,7 +33,7 @@ func TestLogin(t *testing.T) { os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") defer os.Setenv("DOCKER_INDEX_URL", "") authConfig := &AuthConfig{Username: "unittester", Password: "surlautrerivejetattendrai", Email: "noise+unittester@dotcloud.com"} - status, err := Login(authConfig) + status, err := Login(authConfig, nil) if err != nil { t.Fatal(err) } @@ -53,7 +53,7 @@ func TestCreateAccount(t *testing.T) { token := hex.EncodeToString(tokenBuffer)[:12] username := "ut" + token authConfig := &AuthConfig{Username: username, Password: "test42", Email: "docker-ut+" + token + "@example.com"} - status, err := Login(authConfig) + status, err := Login(authConfig, nil) if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestCreateAccount(t *testing.T) { t.Fatalf("Expected status: \"%s\", found \"%s\" instead.", expectedStatus, status) } - status, err = Login(authConfig) + status, err = Login(authConfig, nil) if err == nil { t.Fatalf("Expected error but found nil instead") } From b7d9cc36c6037696ffb6d7f0a495f9150123bec6 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 2 Aug 2013 19:12:38 +0000 Subject: [PATCH 080/134] Add unit tests for build no cache Upstream-commit: b9f06959244e3f77eb212c7c234b06eb7b750999 Component: engine --- components/engine/buildfile_test.go | 102 ++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 19 deletions(-) diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index eda047d8ea..0e2d9ecefc 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -195,21 +195,23 @@ func mkTestingFileServer(files [][2]string) (*httptest.Server, error) { func TestBuild(t *testing.T) { for _, ctx := range testContexts { - buildImage(ctx, t) + buildImage(ctx, t, nil, true) } } -func buildImage(context testContextTemplate, t *testing.T) *Image { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) +func buildImage(context testContextTemplate, t *testing.T, srv *Server, useCache bool) *Image { + if srv == nil { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) - srv := &Server{ - runtime: runtime, - pullingPool: make(map[string]struct{}), - pushingPool: make(map[string]struct{}), + srv = &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } } httpServer, err := mkTestingFileServer(context.remoteFiles) @@ -224,10 +226,10 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { } port := httpServer.URL[idx+1:] - ip := runtime.networkManager.bridgeNetwork.IP + ip := srv.runtime.networkManager.bridgeNetwork.IP dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := NewBuildFile(srv, ioutil.Discard, false, true) + buildfile := NewBuildFile(srv, ioutil.Discard, false, useCache) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { t.Fatal(err) @@ -245,7 +247,7 @@ func TestVolume(t *testing.T) { from {IMAGE} volume /test cmd Hello world - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if len(img.Config.Volumes) == 0 { t.Fail() @@ -261,7 +263,7 @@ func TestBuildMaintainer(t *testing.T) { img := buildImage(testContextTemplate{` from {IMAGE} maintainer dockerio - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if img.Author != "dockerio" { t.Fail() @@ -273,7 +275,7 @@ func TestBuildEnv(t *testing.T) { from {IMAGE} env port 4243 `, - nil, nil}, t) + nil, nil}, t, nil, true) hasEnv := false for _, envVar := range img.Config.Env { if envVar == "port=4243" { @@ -291,7 +293,7 @@ func TestBuildCmd(t *testing.T) { from {IMAGE} cmd ["/bin/echo", "Hello World"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Cmd[0] != "/bin/echo" { t.Log(img.Config.Cmd[0]) @@ -308,7 +310,7 @@ func TestBuildExpose(t *testing.T) { from {IMAGE} expose 4243 `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.PortSpecs[0] != "4243" { t.Fail() @@ -320,8 +322,70 @@ func TestBuildEntrypoint(t *testing.T) { from {IMAGE} entrypoint ["/bin/echo"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Entrypoint[0] != "/bin/echo" { } } + +func TestBuildImageWithCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, true) + + if imageId != img.ID { + t.Logf("Image ids should match: %s != %s", imageId, img.ID) + t.Fail() + } +} + +func TestBuildImageWithoutCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, false) + + if imageId == img.ID { + t.Logf("Image ids should not match: %s == %s", imageId, img.ID) + t.Fail() + } +} From 294e6e658233ec6ae76abf2da38831e00b6ab26e Mon Sep 17 00:00:00 2001 From: Joe Van Dyk Date: Fri, 2 Aug 2013 15:10:57 -0700 Subject: [PATCH 081/134] Update amazon.rst to explain that Vagrant is not necessary for running Docker on ec2 Upstream-commit: b6bff0cbb15535b3d1bc991b975c3df35061aa83 Component: engine --- components/engine/docs/sources/installation/amazon.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/installation/amazon.rst b/components/engine/docs/sources/installation/amazon.rst index 59896bb63c..2b5d8423ce 100644 --- a/components/engine/docs/sources/installation/amazon.rst +++ b/components/engine/docs/sources/installation/amazon.rst @@ -5,10 +5,14 @@ Amazon EC2 ========== +This page explains how to setup and run an Amazon EC2 instance from your local machine. +Vagrant is not necessary to run Docker on EC2. You can follow the :ref:`ubuntu_linux` instructions +installing Docker on any EC2 instance running Ubuntu + Please note this is a community contributed installation path. The only 'official' installation is using the :ref:`ubuntu_linux` installation path. This version may sometimes be out of date. - - + + Installation ------------ @@ -89,4 +93,4 @@ Docker can now be installed on Amazon EC2 with a single vagrant command. Vagrant docker -Continue with the :ref:`hello_world` example. \ No newline at end of file +Continue with the :ref:`hello_world` example. From 48643d0043c3ed12e37db9aa4f4ea315edf889b8 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 2 Aug 2013 19:18:02 -0300 Subject: [PATCH 082/134] Revert "Bind daemon to 0.0.0.0 in Vagrant. Fixes #1304" This reverts commit bdc79ac8b2dfad302f9e144711067a566726cfa2. Upstream-commit: 07fee445593982ef9063a184c05ba93ecacd2e65 Component: engine --- components/engine/Vagrantfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/engine/Vagrantfile b/components/engine/Vagrantfile index 7258af5bf7..aadabb8711 100644 --- a/components/engine/Vagrantfile +++ b/components/engine/Vagrantfile @@ -20,8 +20,6 @@ Vagrant::Config.run do |config| pkg_cmd = "apt-get update -qq; apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/lxc-docker; apt-get update -qq; " \ "apt-get install -q -y lxc-docker; " - # Listen on all interfaces so that the daemon is accessible from the host - pkg_cmd << "sed -i -E 's| /usr/bin/docker -d| /usr/bin/docker -d -H 0.0.0.0|' /etc/init/docker.conf;" # Add X.org Ubuntu backported 3.8 kernel pkg_cmd << "add-apt-repository -y ppa:ubuntu-x-swat/r-lts-backport; " \ "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " From 71b5dff29912802026a8b6fb58d4150c4c9ef224 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 2 Aug 2013 15:23:36 -0700 Subject: [PATCH 083/134] Consider empty /etc/resolv.conf as local dns + add unit test Upstream-commit: 3e9575e275c40acb04c505fa14c1ac63ba490b75 Component: engine --- components/engine/api.go | 7 +++++- components/engine/builder.go | 7 +++++- components/engine/utils/utils.go | 24 ++++++++++++++----- components/engine/utils/utils_test.go | 34 +++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index 95b9c98de8..a48b11c771 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -488,7 +488,12 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r return err } - if len(config.Dns) == 0 && len(srv.runtime.Dns) == 0 && utils.CheckLocalDns() { + resolvConf, err := utils.GetResolvConf() + if err != nil { + return err + } + + if len(config.Dns) == 0 && len(srv.runtime.Dns) == 0 && utils.CheckLocalDns(resolvConf) { out.Warnings = append(out.Warnings, fmt.Sprintf("Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns)) config.Dns = defaultDns } diff --git a/components/engine/builder.go b/components/engine/builder.go index 420370b1e6..82ad1c1271 100644 --- a/components/engine/builder.go +++ b/components/engine/builder.go @@ -80,7 +80,12 @@ func (builder *Builder) Create(config *Config) (*Container, error) { return nil, err } - if len(config.Dns) == 0 && len(builder.runtime.Dns) == 0 && utils.CheckLocalDns() { + resolvConf, err := utils.GetResolvConf() + if err != nil { + return nil, err + } + + if len(config.Dns) == 0 && len(builder.runtime.Dns) == 0 && utils.CheckLocalDns(resolvConf) { //"WARNING: Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns builder.runtime.Dns = defaultDns } diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index def5ae5a50..74b0e12c36 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -688,17 +688,29 @@ func IsGIT(str string) bool { return strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "github.com/") } -func CheckLocalDns() bool { +// GetResolvConf opens and read the content of /etc/resolv.conf. +// It returns it as byte slice. +func GetResolvConf() ([]byte, error) { resolv, err := ioutil.ReadFile("/etc/resolv.conf") if err != nil { Debugf("Error openning resolv.conf: %s", err) - return false + return nil, err } - for _, ip := range []string{ - "127.0.0.1", - "127.0.1.1", + return resolv, nil +} + +// CheckLocalDns looks into the /etc/resolv.conf, +// it returns true if there is a local nameserver or if there is no nameserver. +func CheckLocalDns(resolvConf []byte) bool { + if !bytes.Contains(resolvConf, []byte("nameserver")) { + return true + } + + for _, ip := range [][]byte{ + []byte("127.0.0.1"), + []byte("127.0.1.1"), } { - if strings.Contains(string(resolv), ip) { + if bytes.Contains(resolvConf, ip) { return true } } diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index 5c480b9438..882165ac54 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -303,3 +303,37 @@ func TestParseRepositoryTag(t *testing.T) { t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "tag", repo, tag) } } + +func TestGetResolvConf(t *testing.T) { + resolvConfUtils, err := GetResolvConf() + if err != nil { + t.Fatal(err) + } + resolvConfSystem, err := ioutil.ReadFile("/etc/resolv.conf") + if err != nil { + t.Fatal(err) + } + if string(resolvConfUtils) != string(resolvConfSystem) { + t.Fatalf("/etc/resolv.conf and GetResolvConf have different content.") + } +} + +func TestCheclLocalDns(t *testing.T) { + for resolv, result := range map[string]bool{`# Dynamic +nameserver 10.0.2.3 +search dotcloud.net`: false, + `# Dynamic +nameserver 127.0.0.1 +search dotcloud.net`: true, + `# Dynamic +nameserver 127.0.1.1 +search dotcloud.net`: true, + `# Dynamic +`: true, + ``: true, + } { + if CheckLocalDns([]byte(resolv)) != result { + t.Fatalf("Wrong local dns detection: {%s} should be %v", resolv, result) + } + } +} From 46b9c958878fd716ab26af38c3192dffe1c5cc1b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 2 Aug 2013 15:58:10 -0700 Subject: [PATCH 084/134] Fix TestEnv Upstream-commit: dde8f74ceae83f26386ec29e42f615fdc7945e80 Component: engine --- components/engine/container_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/container_test.go b/components/engine/container_test.go index a1ac0bd33a..f29ae9e4ea 100644 --- a/components/engine/container_test.go +++ b/components/engine/container_test.go @@ -960,6 +960,7 @@ func TestEnv(t *testing.T) { "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "HOME=/", "container=lxc", + "HOSTNAME=" + container.ShortID(), } sort.Strings(goodEnv) if len(goodEnv) != len(actualEnv) { From 3029f070e629c91386d2bb05829f925f1749aff3 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 30 Jul 2013 22:48:20 +0000 Subject: [PATCH 085/134] Return JSONError for HTTPResponse error Upstream-commit: dae585c6e4c19817b2dbd106171728a0bb564ccc Component: engine --- components/engine/api.go | 12 ++++-------- components/engine/commands.go | 2 +- components/engine/registry/registry.go | 4 ++-- components/engine/utils/error.go | 18 ------------------ components/engine/utils/utils.go | 13 ++++++++++++- components/engine/utils_test.go | 2 +- 6 files changed, 20 insertions(+), 31 deletions(-) delete mode 100644 components/engine/utils/error.go diff --git a/components/engine/api.go b/components/engine/api.go index 5869669df0..4ad2ba461a 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -388,7 +388,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht if image != "" { //pull if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { if sf.Used() { - w.Write(sf.FormatError(err, 0)) + w.Write(sf.FormatError(err)) return nil } return err @@ -396,7 +396,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } else { //import if err := srv.ImageImport(src, repo, tag, r.Body, w, sf); err != nil { if sf.Used() { - w.Write(sf.FormatError(err, 0)) + w.Write(sf.FormatError(err)) return nil } return err @@ -441,7 +441,7 @@ func postImagesInsert(srv *Server, version float64, w http.ResponseWriter, r *ht imgID, err := srv.ImageInsert(name, url, path, w, sf) if err != nil { if sf.Used() { - w.Write(sf.FormatError(err, 0)) + w.Write(sf.FormatError(err)) return nil } } @@ -472,11 +472,7 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http sf := utils.NewStreamFormatter(version > 1.0) if err := srv.ImagePush(name, w, sf, authConfig); err != nil { if sf.Used() { - var code int - if httpErr, ok := err.(*utils.HTTPRequestError); ok { - code = httpErr.StatusCode - } - w.Write(sf.FormatError(err, code)) + w.Write(sf.FormatError(err)) return nil } return err diff --git a/components/engine/commands.go b/components/engine/commands.go index 9ad2c367ae..355e91f4bf 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -31,7 +31,7 @@ const VERSION = "0.5.0-dev" var ( GITCOMMIT string - AuthRequiredError error = fmt.Errorf("Authentication is required.") + AuthRequiredError = fmt.Errorf("Authentication is required.") ) func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) { diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index ed6f4c7df8..5b8480d183 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -427,9 +427,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil } diff --git a/components/engine/utils/error.go b/components/engine/utils/error.go deleted file mode 100644 index 7e3c846ebc..0000000000 --- a/components/engine/utils/error.go +++ /dev/null @@ -1,18 +0,0 @@ -package utils - -import ( - "net/http" -) - -type HTTPRequestError struct { - Message string - StatusCode int -} - -func (e *HTTPRequestError) Error() string { - return e.Message -} - -func NewHTTPRequestError(msg string, resp *http.Response) error { - return &HTTPRequestError{Message: msg, StatusCode: resp.StatusCode} -} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 659d960d0a..2323829f65 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -625,6 +625,13 @@ func (e *JSONError) Error() string { return e.Message } +func NewHTTPRequestError(msg string, res *http.Response) error { + return &JSONError{ + Message: msg, + Code: res.StatusCode, + } +} + func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) @@ -666,7 +673,11 @@ func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte func (sf *StreamFormatter) FormatError(err error) []byte { sf.used = true if sf.json { - if b, err := json.Marshal(&JSONMessage{Error: &JSONError{Code: code, Message: err.Error()}, ErrorMessage: err.Error()}); err == nil { + jsonError, ok := err.(*JSONError) + if !ok { + jsonError = &JSONError{Message: err.Error()} + } + if b, err := json.Marshal(&JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { return b } return []byte("{\"error\":\"format error\"}") diff --git a/components/engine/utils_test.go b/components/engine/utils_test.go index c4adeb4a74..91df6183fc 100644 --- a/components/engine/utils_test.go +++ b/components/engine/utils_test.go @@ -191,7 +191,7 @@ func TestMergeConfig(t *testing.T) { if len(configUser.Volumes) != 3 { t.Fatalf("Expected 3 volumes, /test1, /test2 and /test3, found %d", len(configUser.Volumes)) } - for v, _ := range configUser.Volumes { + for v := range configUser.Volumes { if v != "/test1" && v != "/test2" && v != "/test3" { t.Fatalf("Expected /test1 or /test2 or /test3, found %s", v) } From b034acd005bba101cf97e23d0604234ad71183e9 Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Sat, 3 Aug 2013 17:31:20 +0700 Subject: [PATCH 086/134] Clarify Amazon EC2 installation type Make clear this documentation is about installing docker on EC2 with the help of vagrant, which doesn't make it easy to start multiple instances, instead of plain vanilla EC2 installation. Upstream-commit: 708cd3458696101a4aa78d4e8086a29158e133cf Component: engine --- components/engine/docs/sources/installation/amazon.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/installation/amazon.rst b/components/engine/docs/sources/installation/amazon.rst index 59896bb63c..e32dfd40f8 100644 --- a/components/engine/docs/sources/installation/amazon.rst +++ b/components/engine/docs/sources/installation/amazon.rst @@ -1,9 +1,9 @@ -:title: Installation on Amazon EC2 +:title: Installation on Amazon EC2 :description: Docker installation on Amazon EC2 with a single vagrant command. Vagrant 1.1 or higher is required. :keywords: amazon ec2, virtualization, cloud, docker, documentation, installation -Amazon EC2 -========== +Using Vagrant (Amazon EC2) +========================== Please note this is a community contributed installation path. The only 'official' installation is using the :ref:`ubuntu_linux` installation path. This version may sometimes be out of date. From e591df1e0871248b68f56d9717fc693e19595780 Mon Sep 17 00:00:00 2001 From: Michael Gorsuch Date: Sat, 3 Aug 2013 22:11:59 -0500 Subject: [PATCH 087/134] typo: s/connexions/connections Upstream-commit: db0ccaac9b4e9e12d4e33e5d48e437771aaf8dcd Component: engine --- components/engine/docs/sources/installation/ubuntulinux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index 299b7a29e9..cf41b79266 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -163,7 +163,7 @@ Then reload UFW: UFW's default set of rules denied all `incoming`, so if you want to be able to reach your containers from another host, -you should allow incoming connexions on the docker port (default 4243): +you should allow incoming connections on the docker port (default 4243): .. code-block:: bash From 7c9fe26485113d27e5a13dbc8a0c5e7f8b3aa6a8 Mon Sep 17 00:00:00 2001 From: Faiz K Date: Sun, 4 Aug 2013 08:26:56 -0500 Subject: [PATCH 088/134] bash commands while in the container aren't in the transcript! Added. Upstream-commit: 22df1249b582a80f0a1c57d647549d0b91f1cb30 Component: engine --- .../docs/sources/examples/running_ssh_service.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/components/engine/docs/sources/examples/running_ssh_service.rst b/components/engine/docs/sources/examples/running_ssh_service.rst index c2f8b86aca..59ad1b1cab 100644 --- a/components/engine/docs/sources/examples/running_ssh_service.rst +++ b/components/engine/docs/sources/examples/running_ssh_service.rst @@ -40,7 +40,17 @@ The password is 'screencast' # now let's connect using -i for interactive and with -t for terminal # we execute /bin/bash to get a prompt. $ docker run -i -t base /bin/bash - # now let's commit it + # yes! we are in! + # now lets install openssh + $ apt-get update + $ apt-get install openssh-server + # ok. lets see if we can run it. + $ which sshd + # we need to create priviledge separation directory + $ mkdir /var/run/sshd + $ /usr/sbin/sshd + $ exit + # now let's commit it # which container was it? $ docker ps -a |more $ docker commit a30a3a2f2b130749995f5902f079dc6ad31ea0621fac595128ec59c6da07feea dhrp/sshd From 868133628728b02e275ab2eefb1dff2b4f69cf94 Mon Sep 17 00:00:00 2001 From: Daniel Nordberg Date: Mon, 5 Aug 2013 01:16:00 +0300 Subject: [PATCH 089/134] 'Base' is depreciated and should no longer be referenced in the docs. https://groups.google.com/forum/\#!topic/docker-club/pEjqMgcrnqA Upstream-commit: 51d0c9238bb01088c08048e022b948a292f815ab Component: engine --- .../docs/sources/examples/hello_world.rst | 6 ++-- .../sources/examples/hello_world_daemon.rst | 10 +++--- .../sources/examples/running_ssh_service.rst | 6 ++-- components/engine/docs/sources/use/basics.rst | 22 ++++++------ components/engine/docs/sources/use/puppet.rst | 12 +++---- .../engine/packaging/debian/lxc-docker.1 | 36 +++++++++---------- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/components/engine/docs/sources/examples/hello_world.rst b/components/engine/docs/sources/examples/hello_world.rst index 1d391f5fb1..448247e572 100644 --- a/components/engine/docs/sources/examples/hello_world.rst +++ b/components/engine/docs/sources/examples/hello_world.rst @@ -15,8 +15,8 @@ Download the base container .. code-block:: bash - # Download a base image - docker pull base + # Download an ubuntu image + docker pull ubuntu The *base* image is a minimal *ubuntu* based container, alternatively you can select *busybox*, a bare minimal linux system. The images are retrieved from the docker repository. @@ -47,4 +47,4 @@ See the example in action -Continue to the :ref:`hello_world_daemon` example. \ No newline at end of file +Continue to the :ref:`hello_world_daemon` example. diff --git a/components/engine/docs/sources/examples/hello_world_daemon.rst b/components/engine/docs/sources/examples/hello_world_daemon.rst index 7ca251aec8..7d876329dd 100644 --- a/components/engine/docs/sources/examples/hello_world_daemon.rst +++ b/components/engine/docs/sources/examples/hello_world_daemon.rst @@ -11,20 +11,20 @@ Hello World Daemon The most boring daemon ever written. -This example assumes you have Docker installed and with the base image already imported ``docker pull base``. -We will use the base image to run a simple hello world daemon that will just print hello world to standard +This example assumes you have Docker installed and with the ubuntu image already imported ``docker pull ubuntu``. +We will use the ubuntu image to run a simple hello world daemon that will just print hello world to standard out every second. It will continue to do this until we stop it. **Steps:** .. code-block:: bash - CONTAINER_ID=$(docker run -d base /bin/sh -c "while true; do echo hello world; sleep 1; done") + CONTAINER_ID=$(docker run -d ubuntu /bin/sh -c "while true; do echo hello world; sleep 1; done") -We are going to run a simple hello world daemon in a new container made from the base image. +We are going to run a simple hello world daemon in a new container made from the ubuntu image. - **"docker run -d "** run a command in a new container. We pass "-d" so it runs as a daemon. -- **"base"** is the image we want to run the command inside of. +- **"ubuntu"** is the image we want to run the command inside of. - **"/bin/sh -c"** is the command we want to run in the container - **"while true; do echo hello world; sleep 1; done"** is the mini script we want to run, that will just print hello world once a second until we stop it. - **$CONTAINER_ID** the output of the run command will return a container id, we can use in future commands to see what is going on with this process. diff --git a/components/engine/docs/sources/examples/running_ssh_service.rst b/components/engine/docs/sources/examples/running_ssh_service.rst index c2f8b86aca..fa3a2a7ad5 100644 --- a/components/engine/docs/sources/examples/running_ssh_service.rst +++ b/components/engine/docs/sources/examples/running_ssh_service.rst @@ -34,12 +34,12 @@ The password is 'screencast' .. code-block:: bash # Hello! We are going to try and install openssh on a container and run it as a servic - # let's pull base to get a base ubuntu image. - $ docker pull base + # let's pull ubuntu to get a base ubuntu image. + $ docker pull ubuntu # I had it so it was quick # now let's connect using -i for interactive and with -t for terminal # we execute /bin/bash to get a prompt. - $ docker run -i -t base /bin/bash + $ docker run -i -t ubuntu /bin/bash # now let's commit it # which container was it? $ docker ps -a |more diff --git a/components/engine/docs/sources/use/basics.rst b/components/engine/docs/sources/use/basics.rst index 7ce86416dd..a716ab7fa5 100644 --- a/components/engine/docs/sources/use/basics.rst +++ b/components/engine/docs/sources/use/basics.rst @@ -26,12 +26,12 @@ Running an interactive shell .. code-block:: bash - # Download a base image - docker pull base + # Download an ubuntu image + docker pull ubuntu - # Run an interactive shell in the base image, + # Run an interactive shell in the ubuntu image, # allocate a tty, attach stdin and stdout - docker run -i -t base /bin/bash + docker run -i -t ubuntu /bin/bash Bind Docker to another host/port or a unix socket ------------------------------------------------- @@ -52,8 +52,8 @@ For example: # Run docker in daemon mode sudo /docker -H 0.0.0.0:5555 -d & - # Download a base image - docker -H :5555 pull base + # Download an ubuntu image + docker -H :5555 pull ubuntu You can use multiple -H, for example, if you want to listen on both tcp and a unix socket @@ -62,10 +62,10 @@ on both tcp and a unix socket # Run docker in daemon mode sudo /docker -H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock -d & - # Download a base image - docker pull base + # Download an ubuntu image + docker pull ubuntu # OR - docker -H unix:///var/run/docker.sock pull base + docker -H unix:///var/run/docker.sock pull ubuntu Starting a long-running worker process -------------------------------------- @@ -73,7 +73,7 @@ Starting a long-running worker process .. code-block:: bash # Start a very useful long-running process - JOB=$(docker run -d base /bin/sh -c "while true; do echo Hello world; sleep 1; done") + JOB=$(docker run -d ubuntu /bin/sh -c "while true; do echo Hello world; sleep 1; done") # Collect the output of the job so far docker logs $JOB @@ -95,7 +95,7 @@ Expose a service on a TCP port .. code-block:: bash # Expose port 4444 of this container, and tell netcat to listen on it - JOB=$(docker run -d -p 4444 base /bin/nc -l -p 4444) + JOB=$(docker run -d -p 4444 ubuntu /bin/nc -l -p 4444) # Which public port is NATed to my container? PORT=$(docker port $JOB 4444) diff --git a/components/engine/docs/sources/use/puppet.rst b/components/engine/docs/sources/use/puppet.rst index 5606f2a863..b89d95d8fe 100644 --- a/components/engine/docs/sources/use/puppet.rst +++ b/components/engine/docs/sources/use/puppet.rst @@ -53,13 +53,13 @@ defined type which can be used like so: .. code-block:: ruby - docker::image { 'base': } + docker::image { 'ubuntu': } This is equivalent to running: .. code-block:: bash - docker pull base + docker pull ubuntu Note that it will only if the image of that name does not already exist. This is downloading a large binary so on first run can take a while. @@ -68,7 +68,7 @@ for exec. Note that you can also remove images you no longer need with: .. code-block:: ruby - docker::image { 'base': + docker::image { 'ubuntu': ensure => 'absent', } @@ -81,7 +81,7 @@ docker. .. code-block:: ruby docker::run { 'helloworld': - image => 'base', + image => 'ubuntu', command => '/bin/sh -c "while true; do echo hello world; sleep 1; done"', } @@ -89,14 +89,14 @@ This is equivalent to running the following command, but under upstart: .. code-block:: bash - docker run -d base /bin/sh -c "while true; do echo hello world; sleep 1; done" + docker run -d ubuntu /bin/sh -c "while true; do echo hello world; sleep 1; done" Run also contains a number of optional parameters: .. code-block:: ruby docker::run { 'helloworld': - image => 'base', + image => 'ubuntu', command => '/bin/sh -c "while true; do echo hello world; sleep 1; done"', ports => ['4444', '4555'], volumes => ['/var/lib/counchdb', '/var/log'], diff --git a/components/engine/packaging/debian/lxc-docker.1 b/components/engine/packaging/debian/lxc-docker.1 index cc20299fad..b92853f95b 100644 --- a/components/engine/packaging/debian/lxc-docker.1 +++ b/components/engine/packaging/debian/lxc-docker.1 @@ -146,7 +146,7 @@ cd docker\-master .sp .nf .ft C -sudo ./docker run \-i \-t base /bin/bash +sudo ./docker run \-i \-t ubuntu /bin/bash .ft P .fi .sp @@ -496,22 +496,22 @@ This is the most basic example available for using docker .sp This example assumes you have Docker installed. .sp -Download the base container +Download the ubuntu container .sp .nf .ft C -# Download a base image -docker pull base +# Download an ubuntu image +docker pull ubuntu .ft P .fi .sp -The \fIbase\fP image is a minimal \fIubuntu\fP based container, alternatively you can select \fIbusybox\fP, a bare -minimal linux system. The images are retrieved from the docker repository. +Alternatively you can select \fIbusybox\fP, a bare minimal linux system. The +images are retrieved from the docker repository. .sp .nf .ft C #run a simple echo command, that will echo hello world back to the console over standard out. -docker run base /bin/echo hello world +docker run ubuntu /bin/echo hello world .ft P .fi .sp @@ -520,7 +520,7 @@ docker run base /bin/echo hello world .IP \(bu 2 \fB"docker run"\fP run a command in a new container .IP \(bu 2 -\fB"base"\fP is the image we want to run the command inside of. +\fB"ubuntu"\fP is the image we want to run the command inside of. .IP \(bu 2 \fB"/bin/echo"\fP is the command we want to run in the container .IP \(bu 2 @@ -536,15 +536,15 @@ Continue to the \fIhello_world_daemon\fP example. .sp The most boring daemon ever written. .sp -This example assumes you have Docker installed and with the base image already imported \fBdocker pull base\fP. -We will use the base image to run a simple hello world daemon that will just print hello world to standard +This example assumes you have Docker installed and with the ubuntu image already imported \fBdocker pull ubuntu\fP. +We will use the ubuntu image to run a simple hello world daemon that will just print hello world to standard out every second. It will continue to do this until we stop it. .sp \fBSteps:\fP .sp .nf .ft C -$ CONTAINER_ID=$(docker run \-d base /bin/sh \-c "while true; do echo hello world; sleep 1; done") +$ CONTAINER_ID=$(docker run \-d ubuntu /bin/sh \-c "while true; do echo hello world; sleep 1; done") .ft P .fi .sp @@ -553,7 +553,7 @@ We are going to run a simple hello world daemon in a new container made from the .IP \(bu 2 \fB"docker run \-d "\fP run a command in a new container. We pass "\-d" so it runs as a daemon. .IP \(bu 2 -\fB"base"\fP is the image we want to run the command inside of. +\fB"ubuntu"\fP is the image we want to run the command inside of. .IP \(bu 2 \fB"/bin/sh \-c"\fP is the command we want to run in the container .IP \(bu 2 @@ -766,12 +766,12 @@ Contents: .sp .nf .ft C -# Download a base image -docker import base +# Download an ubuntu image +docker import ubuntu -# Run an interactive shell in the base image, +# Run an interactive shell in the ubuntu image, # allocate a tty, attach stdin and stdout -docker run \-a \-i \-t base /bin/bash +docker run \-a \-i \-t ubuntu /bin/bash .ft P .fi .SS Starting a long\-running worker process @@ -782,7 +782,7 @@ docker run \-a \-i \-t base /bin/bash (docker \-d || echo "Docker daemon already running") & # Start a very useful long\-running process -JOB=$(docker run base /bin/sh \-c "while true; do echo Hello world!; sleep 1; done") +JOB=$(docker run ubuntu /bin/sh \-c "while true; do echo Hello world!; sleep 1; done") # Collect the output of the job so far docker logs $JOB @@ -803,7 +803,7 @@ docker ps .nf .ft C # Expose port 4444 of this container, and tell netcat to listen on it -JOB=$(docker run \-p 4444 base /bin/nc \-l \-p 4444) +JOB=$(docker run \-p 4444 ubuntu /bin/nc \-l \-p 4444) # Which public port is NATed to my container? PORT=$(docker port $JOB 4444) From 026e661083159a368e10f6ab6c1106bf4046b0ba Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Sun, 4 Aug 2013 17:42:24 -0700 Subject: [PATCH 090/134] Reduce connect and read timeout when pinging the registry (fixes issue #1363) Upstream-commit: c860945be25c7768ee456b5a71524631ec0dddbd Component: engine --- components/engine/registry/registry.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 5b8480d183..aa7e524292 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -9,12 +9,14 @@ import ( "github.com/dotcloud/docker/utils" "io" "io/ioutil" + "net" "net/http" "net/http/cookiejar" "net/url" "regexp" "strconv" "strings" + "time" ) var ( @@ -28,7 +30,19 @@ func pingRegistryEndpoint(endpoint string) error { // (and we never want to fallback to http in case of error) return nil } - resp, err := http.Get(endpoint + "_ping") + httpDial := func(proto string, addr string) (net.Conn, error) { + // Set the connect timeout to 5 seconds + conn, err := net.DialTimeout(proto, addr, time.Duration(5)*time.Second) + if err != nil { + return nil, err + } + // Set the recv timeout to 10 seconds + conn.SetDeadline(time.Now().Add(time.Duration(10) * time.Second)) + return conn, nil + } + httpTransport := &http.Transport{Dial: httpDial} + client := &http.Client{Transport: httpTransport} + resp, err := client.Get(endpoint + "_ping") if err != nil { return err } From 193eaad4136d5ba41828ea69ea811c9b3de1f2c6 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Sun, 4 Aug 2013 17:59:12 -0700 Subject: [PATCH 091/134] Always consider localhost as a domain name when parsing the FQN repos name Upstream-commit: c22f2617ad8ed2450d4d9dbb6e6ec39da4e51f2f Component: engine --- components/engine/registry/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 5b8480d183..f23ef6c09b 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -69,7 +69,8 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return "", "", ErrInvalidRepositoryName } nameParts := strings.SplitN(reposName, "/", 2) - if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") { + if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") && + nameParts[0] != "localhost" { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) return auth.IndexServerAddress(), reposName, err From fe0fe929451fd555c650366d436133c4af46ad52 Mon Sep 17 00:00:00 2001 From: Isao Jonas Date: Sun, 4 Aug 2013 19:11:23 -0500 Subject: [PATCH 092/134] fix entrypoint without cmd Upstream-commit: 0f249c85ea9acc7fba33994aa5d20a897463db2c Component: engine --- components/engine/builder.go | 4 +++- components/engine/buildfile.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/components/engine/builder.go b/components/engine/builder.go index 82ad1c1271..9124f76ac1 100644 --- a/components/engine/builder.go +++ b/components/engine/builder.go @@ -38,7 +38,9 @@ func (builder *Builder) Create(config *Config) (*Container, error) { MergeConfig(config, img.Config) } - if config.Cmd == nil || len(config.Cmd) == 0 { + if len(config.Entrypoint) != 0 && config.Cmd == nil { + config.Cmd = []string{} + } else if config.Cmd == nil || len(config.Cmd) == 0 { return nil, fmt.Errorf("No command specified") } diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 159d7ba704..ba4427a49f 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -93,6 +93,8 @@ func (b *buildFile) CmdRun(args string) error { b.config.Cmd = nil MergeConfig(b.config, config) + defer func(cmd []string) { b.config.Cmd = cmd }(cmd) + utils.Debugf("Command to be executed: %v", b.config.Cmd) if b.utilizeCache { @@ -115,7 +117,7 @@ func (b *buildFile) CmdRun(args string) error { if err := b.commit(cid, cmd, "run"); err != nil { return err } - b.config.Cmd = cmd + return nil } From d445832a9e40816b1b408c614f78a3cb56736c2d Mon Sep 17 00:00:00 2001 From: Isao Jonas Date: Mon, 5 Aug 2013 09:03:42 -0500 Subject: [PATCH 093/134] added tests for 1405 Upstream-commit: d00fb4096700cd8feed06ca32c93f080fb6446b1 Component: engine --- components/engine/buildfile_test.go | 34 +++++++++++++++++++++++++++++ components/engine/container_test.go | 22 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 0e2d9ecefc..9d002f0665 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -328,6 +328,40 @@ func TestBuildEntrypoint(t *testing.T) { } } +// testing #1405 - config.Cmd does not get cleaned up if +// utilizing cache +func TestBuildEntrypointRunCleanup(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + img := buildImage(testContextTemplate{` + from {IMAGE} + run echo "hello" + `, + nil, nil}, t, srv, true) + + img = buildImage(testContextTemplate{` + from {IMAGE} + run echo "hello" + add foo /foo + entrypoint ["/bin/echo"] + `, + [][2]string{{"foo", "HEYO"}}, nil}, t, srv, true) + + if len(img.Config.Cmd) != 0 { + t.Fail() + } +} + func TestBuildImageWithCache(t *testing.T) { runtime, err := newTestRuntime() if err != nil { diff --git a/components/engine/container_test.go b/components/engine/container_test.go index f29ae9e4ea..2f9da15f6e 100644 --- a/components/engine/container_test.go +++ b/components/engine/container_test.go @@ -996,6 +996,28 @@ func TestEntrypoint(t *testing.T) { } } +func TestEntrypointNoCmd(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + container, err := NewBuilder(runtime).Create( + &Config{ + Image: GetTestImage(runtime).ID, + Entrypoint: []string{"/bin/echo", "foobar"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + output, err := container.Output() + if err != nil { + t.Fatal(err) + } + if strings.Trim(string(output), "\r\n") != "foobar" { + t.Error(string(output)) + } +} + func grepFile(t *testing.T, path string, pattern string) { f, err := os.Open(path) if err != nil { From 9ddc15ebdaeef18b8f0ba3e09915f282073418b4 Mon Sep 17 00:00:00 2001 From: Andrew Macgregor Date: Mon, 5 Aug 2013 22:47:16 +0800 Subject: [PATCH 094/134] Minor typos found while reading docs Upstream-commit: ce97a71adf879a228f352ee10b7fb62ff86bc4e6 Component: engine --- .../engine/docs/sources/examples/python_web_app.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/engine/docs/sources/examples/python_web_app.rst b/components/engine/docs/sources/examples/python_web_app.rst index 8a4ca70750..ed4a490c72 100644 --- a/components/engine/docs/sources/examples/python_web_app.rst +++ b/components/engine/docs/sources/examples/python_web_app.rst @@ -42,7 +42,7 @@ We attach to the new container to see what is going on. Ctrl-C to disconnect BUILD_IMG=$(docker commit $BUILD_JOB _/builds/github.com/shykes/helloflask/master) -Save the changed we just made in the container to a new image called "_/builds/github.com/hykes/helloflask/master" and save the image id in the BUILD_IMG variable name. +Save the changes we just made in the container to a new image called "_/builds/github.com/hykes/helloflask/master" and save the image id in the BUILD_IMG variable name. .. code-block:: bash @@ -60,13 +60,13 @@ Use the new image we just created and create a new container with network port 5 docker logs $WEB_WORKER * Running on http://0.0.0.0:5000/ -view the logs for the new container using the WEB_WORKER variable, and if everything worked as planned you should see the line "Running on http://0.0.0.0:5000/" in the log output. +View the logs for the new container using the WEB_WORKER variable, and if everything worked as planned you should see the line "Running on http://0.0.0.0:5000/" in the log output. .. code-block:: bash WEB_PORT=$(docker port $WEB_WORKER 5000) -lookup the public-facing port which is NAT-ed store the private port used by the container and store it inside of the WEB_PORT variable. +Look up the public-facing port which is NAT-ed. Find the private port used by the container and store it inside of the WEB_PORT variable. .. code-block:: bash @@ -74,7 +74,7 @@ lookup the public-facing port which is NAT-ed store the private port used by the curl http://127.0.0.1:$WEB_PORT Hello world! -access the web app using curl. If everything worked as planned you should see the line "Hello world!" inside of your console. +Access the web app using curl. If everything worked as planned you should see the line "Hello world!" inside of your console. **Video:** From 31aa6c57876472e28ddbf482b8ef91ff22b9c0c8 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 24 Jul 2013 17:44:55 -0700 Subject: [PATCH 095/134] Implemented a Mocked version of the Registry server Upstream-commit: 5f7abd5347fc28ff36b59a03a90dceee22e16606 Component: engine --- .../engine/registry/registry_mock_test.go | 321 ++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 components/engine/registry/registry_mock_test.go diff --git a/components/engine/registry/registry_mock_test.go b/components/engine/registry/registry_mock_test.go new file mode 100644 index 0000000000..f1e65cad32 --- /dev/null +++ b/components/engine/registry/registry_mock_test.go @@ -0,0 +1,321 @@ +package registry + +import ( + "encoding/json" + "fmt" + "github.com/gorilla/mux" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +var ( + testHttpServer *httptest.Server + testLayers = map[string]map[string]string{ + "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { + "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + "comment":"test base image","created":"2013-03-23T12:53:11.10432-07:00", + "container_config":{"Hostname":"","User":"","Memory":0,"MemorySwap":0, + "CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false, + "PortSpecs":null,"Tty":false,"OpenStdin":false,"StdinOnce":false, + "Env":null,"Cmd":null,"Dns":null,"Image":"","Volumes":null, + "VolumesFrom":"","Entrypoint":null},"Size":424242}`, + "checksum_simple": "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", + "checksum_tarsum": "tarsum+sha256:4409a0685741ca86d38df878ed6f8cbba4c99de5dc73cd71aef04be3bb70be7c", + "ancestry": `["77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20"]`, + "layer": string([]byte{ + 0x1f, 0x8b, 0x08, 0x08, 0x0e, 0xb0, 0xee, 0x51, 0x02, 0x03, 0x6c, 0x61, 0x79, 0x65, + 0x72, 0x2e, 0x74, 0x61, 0x72, 0x00, 0xed, 0xd2, 0x31, 0x0e, 0xc2, 0x30, 0x0c, 0x05, + 0x50, 0xcf, 0x9c, 0xc2, 0x27, 0x48, 0xed, 0x38, 0x4e, 0xce, 0x13, 0x44, 0x2b, 0x66, + 0x62, 0x24, 0x8e, 0x4f, 0xa0, 0x15, 0x63, 0xb6, 0x20, 0x21, 0xfc, 0x96, 0xbf, 0x78, + 0xb0, 0xf5, 0x1d, 0x16, 0x98, 0x8e, 0x88, 0x8a, 0x2a, 0xbe, 0x33, 0xef, 0x49, 0x31, + 0xed, 0x79, 0x40, 0x8e, 0x5c, 0x44, 0x85, 0x88, 0x33, 0x12, 0x73, 0x2c, 0x02, 0xa8, + 0xf0, 0x05, 0xf7, 0x66, 0xf5, 0xd6, 0x57, 0x69, 0xd7, 0x7a, 0x19, 0xcd, 0xf5, 0xb1, + 0x6d, 0x1b, 0x1f, 0xf9, 0xba, 0xe3, 0x93, 0x3f, 0x22, 0x2c, 0xb6, 0x36, 0x0b, 0xf6, + 0xb0, 0xa9, 0xfd, 0xe7, 0x94, 0x46, 0xfd, 0xeb, 0xd1, 0x7f, 0x2c, 0xc4, 0xd2, 0xfb, + 0x97, 0xfe, 0x02, 0x80, 0xe4, 0xfd, 0x4f, 0x77, 0xae, 0x6d, 0x3d, 0x81, 0x73, 0xce, + 0xb9, 0x7f, 0xf3, 0x04, 0x41, 0xc1, 0xab, 0xc6, 0x00, 0x0a, 0x00, 0x00, + }), + }, + "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d": { + "json": `{"id":"42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + "parent":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + "comment":"test base image","created":"2013-03-23T12:55:11.10432-07:00", + "container_config":{"Hostname":"","User":"","Memory":0,"MemorySwap":0, + "CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false, + "PortSpecs":null,"Tty":false,"OpenStdin":false,"StdinOnce":false, + "Env":null,"Cmd":null,"Dns":null,"Image":"","Volumes":null, + "VolumesFrom":"","Entrypoint":null},"Size":424242}`, + "checksum_simple": "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", + "checksum_tarsum": "tarsum+sha256:68fdb56fb364f074eec2c9b3f85ca175329c4dcabc4a6a452b7272aa613a07a2", + "ancestry": `["42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20"]`, + "layer": string([]byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xbd, 0xb3, 0xee, 0x51, 0x02, 0x03, 0x6c, 0x61, 0x79, 0x65, + 0x72, 0x2e, 0x74, 0x61, 0x72, 0x00, 0xed, 0xd1, 0x31, 0x0e, 0xc2, 0x30, 0x0c, 0x05, + 0x50, 0xcf, 0x9c, 0xc2, 0x27, 0x48, 0x9d, 0x38, 0x8e, 0xcf, 0x53, 0x51, 0xaa, 0x56, + 0xea, 0x44, 0x82, 0xc4, 0xf1, 0x09, 0xb4, 0xea, 0x98, 0x2d, 0x48, 0x08, 0xbf, 0xe5, + 0x2f, 0x1e, 0xfc, 0xf5, 0xdd, 0x00, 0xdd, 0x11, 0x91, 0x8a, 0xe0, 0x27, 0xd3, 0x9e, + 0x14, 0xe2, 0x9e, 0x07, 0xf4, 0xc1, 0x2b, 0x0b, 0xfb, 0xa4, 0x82, 0xe4, 0x3d, 0x93, + 0x02, 0x0a, 0x7c, 0xc1, 0x23, 0x97, 0xf1, 0x5e, 0x5f, 0xc9, 0xcb, 0x38, 0xb5, 0xee, + 0xea, 0xd9, 0x3c, 0xb7, 0x4b, 0xbe, 0x7b, 0x9c, 0xf9, 0x23, 0xdc, 0x50, 0x6e, 0xb9, + 0xb8, 0xf2, 0x2c, 0x5d, 0xf7, 0x4f, 0x31, 0xb6, 0xf6, 0x4f, 0xc7, 0xfe, 0x41, 0x55, + 0x63, 0xdd, 0x9f, 0x89, 0x09, 0x90, 0x6c, 0xff, 0xee, 0xae, 0xcb, 0xba, 0x4d, 0x17, + 0x30, 0xc6, 0x18, 0xf3, 0x67, 0x5e, 0xc1, 0xed, 0x21, 0x5d, 0x00, 0x0a, 0x00, 0x00, + }), + }, + } + testRepositories = map[string]map[string]string{ + "foo/bar": { + "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + }, + } +) + +func init() { + r := mux.NewRouter() + r.HandleFunc("/v1/_ping", handlerGetPing).Methods("GET") + r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|ancestry}", handlerGetImage).Methods("GET") + r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|checksum}", handlerPutImage).Methods("PUT") + r.HandleFunc("/v1/repositories/{repository:.+}/tags", handlerGetDeleteTags).Methods("GET", "DELETE") + r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerGetTag).Methods("GET") + r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerPutTag).Methods("PUT") + r.HandleFunc("/v1/users{null:.*}", handlerUsers).Methods("GET", "POST", "PUT") + r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Method("GET", "PUT", "DELETE") + r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") + r.HandleFunc("/v1/search", handlerSearch).Methods("GET") + testHttpServer = httptest.NewServer(r) +} + +func makeURL(req string) string { + return testHttpServer.URL + req +} + +func writeHeaders(w http.ResponseWriter) { + h := w.Header() + h.Add("Server", "docker-tests/mock") + h.Add("Expires", "-1") + h.Add("Content-Type", "application/json") + h.Add("Pragma", "no-cache") + h.Add("Cache-Control", "no-cache") + h.Add("X-Docker-Registry-Version", "0.0.0") + h.Add("X-Docker-Registry-Config", "mock") +} + +func writeResponse(w http.ResponseWriter, message interface{}, code int) { + writeHeaders(w) + w.WriteHeader(code) + body, err := json.Marshal(message) + if err != nil { + io.WriteString(w, err.Error()) + return + } + w.Write(body) +} + +func readJSON(r *http.Request, dest interface{}) error { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return err + } + return json.Unmarshal(body, dest) +} + +func apiError(w http.ResponseWriter, message string, code int) { + body := map[string]string{ + "error": message, + } + writeResponse(w, body, code) +} + +func assertEqual(t *testing.T, a interface{}, b interface{}, message string) { + if a == b { + return + } + if len(message) == 0 { + message = fmt.Sprintf("%v != %v", a, b) + } + t.Fatal(message) +} + +func requiresAuth(w http.ResponseWriter, r *http.Request) bool { + writeCookie := func() { + value := fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano()) + cookie := &http.Cookie{Name: "session", Value: value, MaxAge: 3600} + http.SetCookie(w, cookie) + } + if len(r.Cookies()) > 0 { + writeCookie() + return true + } + if len(r.Header.Get("Authorization")) > 0 { + writeCookie() + return true + } + w.Header().Add("WWW-Authenticate", "token") + apiError(w, "Wrong auth", 401) + return false +} + +func handlerGetPing(w http.ResponseWriter, r *http.Request) { + writeResponse(w, true, 200) +} + +func handlerGetImage(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + layer, exists := testLayers[vars["image_id"]] + if !exists { + http.NotFound(w, r) + return + } + writeHeaders(w) + io.WriteString(w, layer[vars["action"]]) +} + +func handlerPutImage(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + image_id := vars["image_id"] + action := vars["action"] + layer, exists := testLayers[image_id] + if !exists { + if action != "json" { + http.NotFound(w, r) + return + } + layer = make(map[string]string) + testLayers[image_id] = layer + } + if checksum := r.Header.Get("X-Docker-Checksum"); checksum != "" { + if checksum != layer["checksum_simple"] && checksum != layer["checksum_tarsum"] { + apiError(w, "Wrong checksum", 400) + return + } + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + apiError(w, fmt.Sprintf("Error: %s", err), 500) + return + } + layer[action] = string(body) + writeResponse(w, true, 200) +} + +func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + repositoryName := mux.Vars(r)["repository"] + tags, exists := testRepositories[repositoryName] + if !exists { + apiError(w, "Repository not found", 404) + } + if r.Method == "DELETE" { + delete(testRepositories, repositoryName) + writeResponse(w, true, 200) + return + } + writeResponse(w, tags, 200) +} + +func handlerGetTag(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + repositoryName := vars["repository"] + tagName := vars["tag"] + tags, exists := testRepositories[repositoryName] + if !exists { + apiError(w, "Repository not found", 404) + } + tag, exists := tags[tagName] + if !exists { + apiError(w, "Tag not found", 404) + } + writeResponse(w, tag, 200) +} + +func handlerPutTag(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + repositoryName := vars["repository"] + tagName := vars["tag"] + tags, exists := testRepositories[repositoryName] + if !exists { + tags := make(map[string]string) + testRepositories[repositoryName] = tags + } + tagValue := "" + readJSON(r, tagValue) + tags[tagName] = tagValue + writeResponse(w, true, 200) +} + +func handlerUsers(w http.ResponseWriter, r *http.Request) { + code := 200 + if r.Method == "POST" { + code = 201 + } else if r.Method == "PUT" { + code = 204 + } + writeResponse(w, "", code) +} + +func handlerImages(w http.ResponseWriter, r *http.Request) { + if r.Method == "PUT" { + writeResponse(w, "", 200) + return + } + if r.Method == "DELETE" { + writeResponse(w, "", 204) + return + } + images := make([]map[string]string) + for image_id, layer := range testLayers { + image := make(map[string]string) + image["id"] = image_id + image["checksum"] = layer["checksum_tarsum"] + append(images, image) + } + writeResponse(w, images, 200) +} + +func handlerAuth(w http.ResponseWriter, r *http.Request) { + writeResponse(w, "OK", 200) +} + +func handlerSearch(w http.ResponseWriter, r *http.Request) { + writeResponse(w, "{}", 200) +} + +func TestPing(t *testing.T) { + res, err := http.Get(makeURL("/v1/_ping")) + if err != nil { + t.Fatal(err) + } + assertEqual(t, res.StatusCode, 200, "") + assertEqual(t, res.Header.Get("X-Docker-Registry-Config"), "mock", + "This is not a Mocked Registry") +} + +/* Uncomment this to test Mocked Registry locally with curl + * WARNING: Don't push on the repos uncommented, it'll block the tests + * +func TestWait(t *testing.T) { + fmt.Println("Test HTTP server ready and waiting...") + fmt.Println(testHttpServer.URL) + c := make(chan int) + <-c +} +//*/ From 90a0202ee859ddc6c4722ad8efcfbfba250b8220 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 24 Jul 2013 19:22:36 -0700 Subject: [PATCH 096/134] Fixed mocked registry Upstream-commit: 97d1d6f5d20c1475b7cdfcecdf4eeba08de888bd Component: engine --- components/engine/registry/registry_mock_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/components/engine/registry/registry_mock_test.go b/components/engine/registry/registry_mock_test.go index f1e65cad32..3bbef25df7 100644 --- a/components/engine/registry/registry_mock_test.go +++ b/components/engine/registry/registry_mock_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "net/url" "testing" "time" ) @@ -69,7 +70,7 @@ var ( }, } testRepositories = map[string]map[string]string{ - "foo/bar": { + "foo42/bar": { "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", }, } @@ -84,7 +85,7 @@ func init() { r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerGetTag).Methods("GET") r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerPutTag).Methods("PUT") r.HandleFunc("/v1/users{null:.*}", handlerUsers).Methods("GET", "POST", "PUT") - r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Method("GET", "PUT", "DELETE") + r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE") r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") r.HandleFunc("/v1/search", handlerSearch).Methods("GET") testHttpServer = httptest.NewServer(r) @@ -103,6 +104,8 @@ func writeHeaders(w http.ResponseWriter) { h.Add("Cache-Control", "no-cache") h.Add("X-Docker-Registry-Version", "0.0.0") h.Add("X-Docker-Registry-Config", "mock") + u, _ := url.Parse(testHttpServer.URL) + h.Add("X-Docker-Endpoints", u.Host) } func writeResponse(w http.ResponseWriter, message interface{}, code int) { @@ -146,6 +149,9 @@ func requiresAuth(w http.ResponseWriter, r *http.Request) bool { value := fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano()) cookie := &http.Cookie{Name: "session", Value: value, MaxAge: 3600} http.SetCookie(w, cookie) + //FIXME(sam): this should be sent only on Index routes + value = fmt.Sprintf("FAKE-TOKEN-%d", time.Now().UnixNano()) + w.Header().Add("X-Docker-Token", value) } if len(r.Cookies()) > 0 { writeCookie() @@ -281,12 +287,12 @@ func handlerImages(w http.ResponseWriter, r *http.Request) { writeResponse(w, "", 204) return } - images := make([]map[string]string) + images := []map[string]string{} for image_id, layer := range testLayers { image := make(map[string]string) image["id"] = image_id image["checksum"] = layer["checksum_tarsum"] - append(images, image) + images = append(images, image) } writeResponse(w, images, 200) } From 8bf1856b5cc496e8fc71af0f943dca384209b765 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 25 Jul 2013 12:57:09 -0700 Subject: [PATCH 097/134] Mocked registry: Added X-Docker-Size when fetching the layer Upstream-commit: 6926ba558f14da4e48ff5c409a5e78f1e9e0e991 Component: engine --- .../engine/registry/registry_mock_test.go | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/components/engine/registry/registry_mock_test.go b/components/engine/registry/registry_mock_test.go index 3bbef25df7..5783067266 100644 --- a/components/engine/registry/registry_mock_test.go +++ b/components/engine/registry/registry_mock_test.go @@ -6,9 +6,11 @@ import ( "github.com/gorilla/mux" "io" "io/ioutil" + "log" "net/http" "net/http/httptest" "net/url" + "strconv" "testing" "time" ) @@ -88,7 +90,15 @@ func init() { r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE") r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") r.HandleFunc("/v1/search", handlerSearch).Methods("GET") - testHttpServer = httptest.NewServer(r) + testHttpServer = httptest.NewServer(handlerAccessLog(r)) +} + +func handlerAccessLog(handler http.Handler) http.Handler { + logHandler := func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) + handler.ServeHTTP(w, r) + } + return http.HandlerFunc(logHandler) } func makeURL(req string) string { @@ -104,8 +114,6 @@ func writeHeaders(w http.ResponseWriter) { h.Add("Cache-Control", "no-cache") h.Add("X-Docker-Registry-Version", "0.0.0") h.Add("X-Docker-Registry-Config", "mock") - u, _ := url.Parse(testHttpServer.URL) - h.Add("X-Docker-Endpoints", u.Host) } func writeResponse(w http.ResponseWriter, message interface{}, code int) { @@ -181,6 +189,8 @@ func handlerGetImage(w http.ResponseWriter, r *http.Request) { return } writeHeaders(w) + layer_size := len(layer["layer"]) + w.Header().Add("X-Docker-Size", strconv.Itoa(layer_size)) io.WriteString(w, layer[vars["action"]]) } @@ -279,6 +289,8 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) { } func handlerImages(w http.ResponseWriter, r *http.Request) { + u, _ := url.Parse(testHttpServer.URL) + w.Header().Add("X-Docker-Endpoints", u.Host) if r.Method == "PUT" { writeResponse(w, "", 200) return @@ -292,6 +304,7 @@ func handlerImages(w http.ResponseWriter, r *http.Request) { image := make(map[string]string) image["id"] = image_id image["checksum"] = layer["checksum_tarsum"] + image["Tag"] = "latest" images = append(images, image) } writeResponse(w, images, 200) @@ -317,11 +330,11 @@ func TestPing(t *testing.T) { /* Uncomment this to test Mocked Registry locally with curl * WARNING: Don't push on the repos uncommented, it'll block the tests - * + */ func TestWait(t *testing.T) { - fmt.Println("Test HTTP server ready and waiting...") - fmt.Println(testHttpServer.URL) + log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) c := make(chan int) <-c } + //*/ From 4732ba2be6a8f813f66c3b742b2525bc6d64281b Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 26 Jul 2013 17:28:17 -0700 Subject: [PATCH 098/134] Disabled test server in the tests Upstream-commit: 310ddec823cbeadf694c396b27b5610474f05bcc Component: engine --- components/engine/registry/registry_mock_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/registry/registry_mock_test.go b/components/engine/registry/registry_mock_test.go index 5783067266..f634877b63 100644 --- a/components/engine/registry/registry_mock_test.go +++ b/components/engine/registry/registry_mock_test.go @@ -330,7 +330,7 @@ func TestPing(t *testing.T) { /* Uncomment this to test Mocked Registry locally with curl * WARNING: Don't push on the repos uncommented, it'll block the tests - */ + * func TestWait(t *testing.T) { log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) c := make(chan int) From 64f23d891db0a700c3f942c65f0ae8fdaefe576b Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 31 Jul 2013 19:03:14 +0200 Subject: [PATCH 099/134] registry: Fixed a bug where token and cookie info wouldn't be sent when using LookupRemoteImage(). Fixed a bug where no error would be reported when getting a non-200 status code in GetRemoteImageLayer() Upstream-commit: 553ce165c1235542d3a5dd526c063c7a4b9904f4 Component: engine --- components/engine/registry/registry.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 0d2a64e053..579b34e8a3 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -147,13 +147,14 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s // Check if an image exists in the Registry func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { - rt := &http.Transport{Proxy: http.ProxyFromEnvironment} + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } - res, err := rt.RoundTrip(req) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := doWithCookies(r.client, req) if err != nil { return false } @@ -200,6 +201,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( if err != nil { return nil, err } + if res.StatusCode != 200 { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } return res.Body, nil } From 4d1189a20db635f258aefa820754027e4d07c6d9 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 31 Jul 2013 19:05:06 +0200 Subject: [PATCH 100/134] Mock registry: Fixed a bug where the index validation path would return a 200 status code instead of the expected 204 Upstream-commit: 29f69211c957a62f13b10036a579191a920f62a7 Component: engine --- components/engine/registry/registry_mock_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/components/engine/registry/registry_mock_test.go b/components/engine/registry/registry_mock_test.go index f634877b63..236dc00dac 100644 --- a/components/engine/registry/registry_mock_test.go +++ b/components/engine/registry/registry_mock_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "net/url" "strconv" + "strings" "testing" "time" ) @@ -291,7 +292,12 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) { func handlerImages(w http.ResponseWriter, r *http.Request) { u, _ := url.Parse(testHttpServer.URL) w.Header().Add("X-Docker-Endpoints", u.Host) + w.Header().Add("X-Docker-Token", fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano())) if r.Method == "PUT" { + if strings.HasSuffix(r.URL.Path, "images") { + writeResponse(w, "", 204) + return + } writeResponse(w, "", 200) return } @@ -330,6 +336,7 @@ func TestPing(t *testing.T) { /* Uncomment this to test Mocked Registry locally with curl * WARNING: Don't push on the repos uncommented, it'll block the tests +<<<<<<< HEAD * func TestWait(t *testing.T) { log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) @@ -338,3 +345,11 @@ func TestWait(t *testing.T) { } //*/ +======= + */ +// func TestWait(t *testing.T) { +// log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) +// c := make(chan int) +// <-c +// } +>>>>>>> Mock registry: Fixed a bug where the index validation path would return a 200 status code instead of the expected 204 From d6b8f8b07ec8cd58d869a52a52438417b32ad1de Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 31 Jul 2013 19:07:31 +0200 Subject: [PATCH 101/134] New registry unit tests remade from scratch, using the mock registry Upstream-commit: 97b7b173b999c7422a3b045c3ceffa331a7239ca Component: engine --- components/engine/registry/registry_test.go | 331 +++++++++++--------- 1 file changed, 187 insertions(+), 144 deletions(-) diff --git a/components/engine/registry/registry_test.go b/components/engine/registry/registry_test.go index fd955b7b73..68a5f75f1c 100644 --- a/components/engine/registry/registry_test.go +++ b/components/engine/registry/registry_test.go @@ -1,168 +1,211 @@ package registry -// import ( -// "crypto/rand" -// "encoding/hex" -// "github.com/dotcloud/docker" -// "github.com/dotcloud/docker/auth" -// "io/ioutil" -// "os" -// "path" -// "testing" -// ) +import ( + "github.com/dotcloud/docker/auth" + "strings" + "testing" +) +var ( + IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d" + TOKEN = []string{"fake-token"} + REPO = "foo42/bar" +) -// func newTestRuntime() (*Runtime, error) { -// root, err := ioutil.TempDir("", "docker-test") -// if err != nil { -// return nil, err -// } -// if err := os.Remove(root); err != nil { -// return nil, err -// } +type simpleVersionInfo struct { + name string + version string +} -// if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { -// return nil, err -// } +func (v *simpleVersionInfo) Name() string { + return v.name +} -// return runtime, nil -// } +func (v *simpleVersionInfo) Version() string { + return v.version +} -// func TestPull(t *testing.T) { -// os.Setenv("DOCKER_INDEX_URL", "") -// runtime, err := newTestRuntime() -// if err != nil { -// t.Fatal(err) -// } -// defer nuke(runtime) +func spawnTestRegistry(t *testing.T) *Registry { + versionInfo := make([]VersionInfo, 0, 4) + versionInfo = append(versionInfo, &simpleVersionInfo{"docker", "0.0.0test"}) + versionInfo = append(versionInfo, &simpleVersionInfo{"go", "test"}) + versionInfo = append(versionInfo, &simpleVersionInfo{"git-commit", "test"}) + versionInfo = append(versionInfo, &simpleVersionInfo{"kernel", "test"}) + authConfig := &auth.AuthConfig{} + r, err := NewRegistry("", authConfig, versionInfo...) + if err != nil { + t.Fatal(err) + } + return r +} -// err = runtime.graph.PullRepository(ioutil.Discard, "busybox", "", runtime.repositories, nil) -// if err != nil { -// t.Fatal(err) -// } -// img, err := runtime.repositories.LookupImage("busybox") -// if err != nil { -// t.Fatal(err) -// } +func TestPingRegistryEndpoint(t *testing.T) { + err := pingRegistryEndpoint(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } +} -// // Try to run something on this image to make sure the layer's been downloaded properly. -// config, _, err := docker.ParseRun([]string{img.Id, "echo", "Hello World"}, runtime.capabilities) -// if err != nil { -// t.Fatal(err) -// } +func TestGetRemoteHistory(t *testing.T) { + r := spawnTestRegistry(t) + hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } + assertEqual(t, len(hist), 2, "Expected 2 images in history") + assertEqual(t, hist[0], IMAGE_ID, "Expected " + IMAGE_ID + "as first ancestry") + assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + "Unexpected second ancestry") +} -// b := NewBuilder(runtime) -// container, err := b.Create(config) -// if err != nil { -// t.Fatal(err) -// } -// if err := container.Start(); err != nil { -// t.Fatal(err) -// } +func TestLookupRemoteImage(t *testing.T) { + r := spawnTestRegistry(t) + found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN) + assertEqual(t, found, true, "Expected remote lookup to succeed") + found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN) + assertEqual(t, found, false, "Expected remote lookup to fail") +} -// if status := container.Wait(); status != 0 { -// t.Fatalf("Expected status code 0, found %d instead", status) -// } -// } +func TestGetRemoteImageJSON(t *testing.T) { + r := spawnTestRegistry(t) + json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } + assertEqual(t, size, 154, "Expected size 154") + if len(json) <= 0 { + t.Fatal("Expected non-empty json") + } -// func TestPullTag(t *testing.T) { -// os.Setenv("DOCKER_INDEX_URL", "") -// runtime, err := newTestRuntime() -// if err != nil { -// t.Fatal(err) -// } -// defer nuke(runtime) + _, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), TOKEN) + if err == nil { + t.Fatal("Expected image not found error") + } +} -// err = runtime.graph.PullRepository(ioutil.Discard, "ubuntu", "12.04", runtime.repositories, nil) -// if err != nil { -// t.Fatal(err) -// } -// _, err = runtime.repositories.LookupImage("ubuntu:12.04") -// if err != nil { -// t.Fatal(err) -// } +func TestGetRemoteImageLayer(t *testing.T) { + r := spawnTestRegistry(t) + data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } + if data == nil { + t.Fatal("Expected non-nil data result") + } -// img2, err := runtime.repositories.LookupImage("ubuntu:12.10") -// if img2 != nil { -// t.Fatalf("Expected nil image but found %v instead", img2.Id) -// } -// } + _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN) + if err == nil { + t.Fatal("Expected image not found error") + } +} -// func login(runtime *Runtime) error { -// authConfig := auth.NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", runtime.root) -// runtime.authConfig = authConfig -// _, err := auth.Login(authConfig) -// return err -// } +func TestGetRemoteTags(t *testing.T) { + r := spawnTestRegistry(t) + tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN) + if err != nil { + t.Fatal(err) + } + assertEqual(t, len(tags), 1, "Expected one tag") + assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to " + IMAGE_ID) -// func TestPush(t *testing.T) { -// os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") -// defer os.Setenv("DOCKER_INDEX_URL", "") -// runtime, err := newTestRuntime() -// if err != nil { -// t.Fatal(err) -// } -// defer nuke(runtime) + _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN) + if err == nil { + t.Fatal("Expected error when fetching tags for bogus repo") + } +} -// err = login(runtime) -// if err != nil { -// t.Fatal(err) -// } +func TestGetRepositoryData(t *testing.T) { + r := spawnTestRegistry(t) + data, err := r.GetRepositoryData(makeURL("/v1/"), "foo42/bar") + if err != nil { + t.Fatal(err) + } + assertEqual(t, len(data.ImgList), 2, "Expected 2 images in ImgList") + assertEqual(t, len(data.Endpoints), 1, "Expected one endpoint in Endpoints") +} -// err = runtime.graph.PullRepository(ioutil.Discard, "joffrey/busybox", "", runtime.repositories, nil) -// if err != nil { -// t.Fatal(err) -// } -// tokenBuffer := make([]byte, 16) -// _, err = rand.Read(tokenBuffer) -// if err != nil { -// t.Fatal(err) -// } -// token := hex.EncodeToString(tokenBuffer)[:29] -// config, _, err := ParseRun([]string{"joffrey/busybox", "touch", "/" + token}, runtime.capabilities) -// if err != nil { -// t.Fatal(err) -// } +func TestPushImageJSONRegistry(t *testing.T) { + r := spawnTestRegistry(t) + imgData := &ImgData{ + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", + } -// b := NewBuilder(runtime) -// container, err := b.Create(config) -// if err != nil { -// t.Fatal(err) -// } -// if err := container.Start(); err != nil { -// t.Fatal(err) -// } + err := r.PushImageJSONRegistry(imgData, []byte{ 0x42, 0xdf, 0x0 }, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } +} -// if status := container.Wait(); status != 0 { -// t.Fatalf("Expected status code 0, found %d instead", status) -// } +func TestPushImageLayerRegistry(t *testing.T) { + r := spawnTestRegistry(t) + layer := strings.NewReader("FAKELAYER") + r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN) +} -// img, err := b.Commit(container, "unittester/"+token, "", "", "", nil) -// if err != nil { -// t.Fatal(err) -// } +func TestResolveRepositoryName(t *testing.T) { + _, _, err := ResolveRepositoryName("https://github.com/dotcloud/docker") + assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name") + ep, repo, err := ResolveRepositoryName("fooo/bar") + if err != nil { + t.Fatal(err) + } + assertEqual(t, ep, auth.IndexServerAddress(), "Expected endpoint to be index server address") + assertEqual(t, repo, "fooo/bar", "Expected resolved repo to be foo/bar") -// repo := runtime.repositories.Repositories["unittester/"+token] -// err = runtime.graph.PushRepository(ioutil.Discard, "unittester/"+token, repo, runtime.authConfig) -// if err != nil { -// t.Fatal(err) -// } + u := makeURL("")[7:] + ep, repo, err = ResolveRepositoryName(u + "/private/moonbase") + if err != nil { + t.Fatal(err) + } + assertEqual(t, ep, "http://" + u + "/v1/", "Expected endpoint to be " + u) + assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase") +} -// // Remove image so we can pull it again -// if err := runtime.graph.Delete(img.Id); err != nil { -// t.Fatal(err) -// } +func TestPushRegistryTag(t *testing.T) { + r := spawnTestRegistry(t) + err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } +} -// err = runtime.graph.PullRepository(ioutil.Discard, "unittester/"+token, "", runtime.repositories, runtime.authConfig) -// if err != nil { -// t.Fatal(err) -// } +func TestPushImageJSONIndex(t *testing.T) { + r := spawnTestRegistry(t) + imgData := []*ImgData{ + &ImgData{ + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", + }, + &ImgData{ + ID: "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", + }, + } + ep := makeURL("/v1/") + repoData, err := r.PushImageJSONIndex(ep, "foo42/bar", imgData, false, nil) + if err != nil { + t.Fatal(err) + } + if repoData == nil { + t.Fatal("Expected RepositoryData object") + } + repoData, err = r.PushImageJSONIndex(ep, "foo42/bar", imgData, true, []string{ep}) + if err != nil { + t.Fatal(err) + } + if repoData == nil { + t.Fatal("Expected RepositoryData object") + } +} -// layerPath, err := img.layer() -// if err != nil { -// t.Fatal(err) -// } - -// if _, err := os.Stat(path.Join(layerPath, token)); err != nil { -// t.Fatalf("Error while trying to retrieve token file: %v", err) -// } -// } +func TestSearchRepositories(t *testing.T) { + r := spawnTestRegistry(t) + results, err := r.SearchRepositories("supercalifragilisticepsialidocious") + if err != nil { + t.Fatal(err) + } + if results == nil { + t.Fatal("Expected non-nil SearchResults object") + } + assertEqual(t, results.NumResults, 0, "Expected 0 search results") +} \ No newline at end of file From 779219a1f1e63c9a9d27eefb9d66218f536c7299 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 31 Jul 2013 19:12:40 +0200 Subject: [PATCH 102/134] gofmt Upstream-commit: 484ba4a8c5cd4c06330bdd5fb58f98f07619e2aa Component: engine --- components/engine/registry/registry_test.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/components/engine/registry/registry_test.go b/components/engine/registry/registry_test.go index 68a5f75f1c..642fe1c02d 100644 --- a/components/engine/registry/registry_test.go +++ b/components/engine/registry/registry_test.go @@ -5,10 +5,11 @@ import ( "strings" "testing" ) + var ( IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d" - TOKEN = []string{"fake-token"} - REPO = "foo42/bar" + TOKEN = []string{"fake-token"} + REPO = "foo42/bar" ) type simpleVersionInfo struct { @@ -52,7 +53,7 @@ func TestGetRemoteHistory(t *testing.T) { t.Fatal(err) } assertEqual(t, len(hist), 2, "Expected 2 images in history") - assertEqual(t, hist[0], IMAGE_ID, "Expected " + IMAGE_ID + "as first ancestry") + assertEqual(t, hist[0], IMAGE_ID, "Expected "+IMAGE_ID+"as first ancestry") assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", "Unexpected second ancestry") } @@ -105,7 +106,7 @@ func TestGetRemoteTags(t *testing.T) { t.Fatal(err) } assertEqual(t, len(tags), 1, "Expected one tag") - assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to " + IMAGE_ID) + assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to "+IMAGE_ID) _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN) if err == nil { @@ -126,11 +127,11 @@ func TestGetRepositoryData(t *testing.T) { func TestPushImageJSONRegistry(t *testing.T) { r := spawnTestRegistry(t) imgData := &ImgData{ - ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", } - err := r.PushImageJSONRegistry(imgData, []byte{ 0x42, 0xdf, 0x0 }, makeURL("/v1/"), TOKEN) + err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), TOKEN) if err != nil { t.Fatal(err) } @@ -157,7 +158,7 @@ func TestResolveRepositoryName(t *testing.T) { if err != nil { t.Fatal(err) } - assertEqual(t, ep, "http://" + u + "/v1/", "Expected endpoint to be " + u) + assertEqual(t, ep, "http://"+u+"/v1/", "Expected endpoint to be "+u) assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase") } @@ -173,11 +174,11 @@ func TestPushImageJSONIndex(t *testing.T) { r := spawnTestRegistry(t) imgData := []*ImgData{ &ImgData{ - ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", }, &ImgData{ - ID: "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + ID: "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", }, } @@ -208,4 +209,4 @@ func TestSearchRepositories(t *testing.T) { t.Fatal("Expected non-nil SearchResults object") } assertEqual(t, results.NumResults, 0, "Expected 0 search results") -} \ No newline at end of file +} From 4c19fdff3976f1bd339121c4567775cbe60290a8 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 5 Aug 2013 16:22:46 +0200 Subject: [PATCH 103/134] Mock access logs don't show up in non-debug mode Upstream-commit: 9159c819c3baf33844f0cc38a4baa0c6674e8b9a Component: engine --- components/engine/registry/registry_mock_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/registry/registry_mock_test.go b/components/engine/registry/registry_mock_test.go index 236dc00dac..e39637c16c 100644 --- a/components/engine/registry/registry_mock_test.go +++ b/components/engine/registry/registry_mock_test.go @@ -3,10 +3,10 @@ package registry import ( "encoding/json" "fmt" + "github.com/dotcloud/docker/utils" "github.com/gorilla/mux" "io" "io/ioutil" - "log" "net/http" "net/http/httptest" "net/url" @@ -96,7 +96,7 @@ func init() { func handlerAccessLog(handler http.Handler) http.Handler { logHandler := func(w http.ResponseWriter, r *http.Request) { - log.Printf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) + utils.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) handler.ServeHTTP(w, r) } return http.HandlerFunc(logHandler) From 67eddf7cb1d224e2aebb0247b1ab8b2408c9a55b Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 5 Aug 2013 19:07:23 +0200 Subject: [PATCH 104/134] Cleanup Upstream-commit: 2c85b964e3be0f7ca3ff513b855f7bd26b936cc9 Component: engine --- components/engine/registry/registry_mock_test.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/components/engine/registry/registry_mock_test.go b/components/engine/registry/registry_mock_test.go index e39637c16c..e752315510 100644 --- a/components/engine/registry/registry_mock_test.go +++ b/components/engine/registry/registry_mock_test.go @@ -336,7 +336,6 @@ func TestPing(t *testing.T) { /* Uncomment this to test Mocked Registry locally with curl * WARNING: Don't push on the repos uncommented, it'll block the tests -<<<<<<< HEAD * func TestWait(t *testing.T) { log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) @@ -344,12 +343,4 @@ func TestWait(t *testing.T) { <-c } -//*/ -======= - */ -// func TestWait(t *testing.T) { -// log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) -// c := make(chan int) -// <-c -// } ->>>>>>> Mock registry: Fixed a bug where the index validation path would return a 200 status code instead of the expected 204 +//*/ \ No newline at end of file From 54865cc8e51051e462c030ce2f63d36d1fc89ec9 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 5 Aug 2013 20:28:05 +0200 Subject: [PATCH 105/134] Adapted tests to latest registry changes Upstream-commit: 8aa9985ad07f71fa9bd3b31dde81c0da6d0dc655 Component: engine --- components/engine/registry/registry_test.go | 28 ++++++--------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/components/engine/registry/registry_test.go b/components/engine/registry/registry_test.go index 642fe1c02d..a8543f186d 100644 --- a/components/engine/registry/registry_test.go +++ b/components/engine/registry/registry_test.go @@ -2,6 +2,7 @@ package registry import ( "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/utils" "strings" "testing" ) @@ -12,27 +13,9 @@ var ( REPO = "foo42/bar" ) -type simpleVersionInfo struct { - name string - version string -} - -func (v *simpleVersionInfo) Name() string { - return v.name -} - -func (v *simpleVersionInfo) Version() string { - return v.version -} - func spawnTestRegistry(t *testing.T) *Registry { - versionInfo := make([]VersionInfo, 0, 4) - versionInfo = append(versionInfo, &simpleVersionInfo{"docker", "0.0.0test"}) - versionInfo = append(versionInfo, &simpleVersionInfo{"go", "test"}) - versionInfo = append(versionInfo, &simpleVersionInfo{"git-commit", "test"}) - versionInfo = append(versionInfo, &simpleVersionInfo{"kernel", "test"}) authConfig := &auth.AuthConfig{} - r, err := NewRegistry("", authConfig, versionInfo...) + r, err := NewRegistry("", authConfig, utils.NewHTTPRequestFactory()) if err != nil { t.Fatal(err) } @@ -139,8 +122,11 @@ func TestPushImageJSONRegistry(t *testing.T) { func TestPushImageLayerRegistry(t *testing.T) { r := spawnTestRegistry(t) - layer := strings.NewReader("FAKELAYER") - r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN) + layer := strings.NewReader("") + _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{}) + if err != nil { + t.Fatal(err) + } } func TestResolveRepositoryName(t *testing.T) { From 3eb7f19faeffe41624a077f5f22823c06c95ed03 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Mon, 1 Jul 2013 19:05:05 -0400 Subject: [PATCH 106/134] fixed #910. print user name to docker info output Upstream-commit: 4179f25286dde9f9e83bffa07c303373844b5ed2 Component: engine --- components/engine/commands.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/engine/commands.go b/components/engine/commands.go index aa991811ca..9c59857ae1 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -495,6 +495,11 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "EventsListeners: %d\n", out.NEventsListener) fmt.Fprintf(cli.out, "Kernel Version: %s\n", out.KernelVersion) } + if cli.authConfig != nil { + fmt.Fprintf(cli.out, "Username: %v\n", cli.authConfig.Username) + // XXX Should we print registry address even if the user was not logged in? + fmt.Fprintf(cli.out, "Registry: %v\n", auth.IndexServerAddress()) + } if !out.MemoryLimit { fmt.Fprintf(cli.err, "WARNING: No memory limit support\n") } From 4b74bc802f28622aee9260369568fbf55319d619 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 5 Aug 2013 14:32:54 -0700 Subject: [PATCH 107/134] Update CHANGELOG.md Upstream-commit: c03561eea84f88d0eeddd84a7021876852c0a7c0 Component: engine --- components/engine/CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index 00e358c136..843f0212d0 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 0.5.1 (2013-07-30) + + API: Docker client now sets useragent (RFC 2616) + + Runtime: Add `ps` args to `docker top` + + Runtime: Add support for container ID files (pidfile like) + + Runtime: Add container=lxc in default env + + Runtime: Support networkless containers with `docker run -n` and `docker -d -b=none` + + API: Add /events endpoint + + Builder: ADD command now understands URLs + + Builder: CmdAdd and CmdEnv now respect Dockerfile-set ENV variables + * Hack: Simplify unit tests with helpers + * Hack: Improve docker.upstart event + * Hack: Add coverage testing into docker-ci + * Runtime: Stdout/stderr logs are now stored in the same file as JSON + * Runtime: Allocate a /16 IP range by default, with fallback to /24. Try 12 ranges instead of 3. + * Runtime: Change .dockercfg format to json and support multiple auth remote + - Runtime: Do not override volumes from config + - Runtime: Fix issue with EXPOSE override + - Builder: Create directories with 755 instead of 700 within ADD instruction + ## 0.5.0 (2013-07-17) + Runtime: List all processes running inside a container with 'docker top' + Runtime: Host directories can be mounted as volumes with 'docker run -v' From 5e3a4de02f7cf379f677140fad3d1a608cb31cb3 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 5 Aug 2013 14:49:37 -0700 Subject: [PATCH 108/134] Update version. Upstream-commit: 590fc58de70bb281812371be6e44ccb7d73d3ec6 Component: engine --- components/engine/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index aa991811ca..78916f3a7c 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -27,7 +27,7 @@ import ( "unicode" ) -const VERSION = "0.5.0-dev" +const VERSION = "0.5.1-dev" var ( GITCOMMIT string From 46b7e044d5045518d857914b42ea2a7bd7ed50a0 Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Mon, 5 Aug 2013 15:55:40 -0700 Subject: [PATCH 109/134] Fixed some typo's and formatting issues in remote api documentation. Upstream-commit: ff6b6f2ce1f51c38b00d1a70376182645b4a0c85 Component: engine --- components/engine/docs/README.md | 2 +- .../docs/sources/api/docker_remote_api.rst | 2 +- .../sources/api/docker_remote_api_v1.0.rst | 2 +- .../sources/api/docker_remote_api_v1.1.rst | 2 +- .../sources/api/docker_remote_api_v1.2.rst | 2 +- .../sources/api/docker_remote_api_v1.3.rst | 2 +- .../sources/api/docker_remote_api_v1.4.rst | 91 ++++++++++--------- 7 files changed, 53 insertions(+), 50 deletions(-) diff --git a/components/engine/docs/README.md b/components/engine/docs/README.md index 1a40b5bb5c..366b7ed8f2 100644 --- a/components/engine/docs/README.md +++ b/components/engine/docs/README.md @@ -23,7 +23,7 @@ Usage * Change the `.rst` files with your favorite editor to your liking. * Run `make docs` to clean up old files and generate new ones. * Your static website can now be found in the `_build` directory. -* To preview what you have generated run `make server` and open in your favorite browser. +* To preview what you have generated run `make server` and open http://localhost:8000/ in your favorite browser. Working using GitHub's file editor ---------------------------------- diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index dd017cde8d..3be02141c0 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -15,7 +15,7 @@ Docker Remote API ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.0.rst b/components/engine/docs/sources/api/docker_remote_api_v1.0.rst index f499a5f53a..3ee3245137 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.0.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.0.rst @@ -17,7 +17,7 @@ Docker Remote API v1.0 ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr 2. Endpoints diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.1.rst b/components/engine/docs/sources/api/docker_remote_api_v1.1.rst index e0159ddb65..3e906ed50f 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.1.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.1.rst @@ -17,7 +17,7 @@ Docker Remote API v1.1 ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr 2. Endpoints diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.2.rst b/components/engine/docs/sources/api/docker_remote_api_v1.2.rst index bd869ca824..b956d1dfe6 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.2.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.2.rst @@ -17,7 +17,7 @@ Docker Remote API v1.2 ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr 2. Endpoints diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst index f2b4062fef..8e5c7b2a3b 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst @@ -17,7 +17,7 @@ Docker Remote API v1.3 ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr 2. Endpoints diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst index 7406a42b9d..31e5b5f766 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -2,6 +2,8 @@ :description: API Documentation for Docker :keywords: API, Docker, rcli, REST, documentation +:orphan: + ====================== Docker Remote API v1.4 ====================== @@ -12,7 +14,7 @@ Docker Remote API v1.4 ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr 2. Endpoints @@ -765,29 +767,29 @@ Push an image on the registry .. http:post:: /images/(name)/push - Push the image ``name`` on the registry + Push the image ``name`` on the registry - **Example request**: + **Example request**: - .. sourcecode:: http + .. sourcecode:: http - POST /images/test/push HTTP/1.1 - {{ authConfig }} + POST /images/test/push HTTP/1.1 + {{ authConfig }} - **Example response**: + **Example response**: - .. sourcecode:: http + .. sourcecode:: http - HTTP/1.1 200 OK - Content-Type: application/json + HTTP/1.1 200 OK + Content-Type: application/json - {"status":"Pushing..."} - {"status":"Pushing", "progress":"1/? (n/a)"} - {"error":"Invalid..."} - ... + {"status":"Pushing..."} + {"status":"Pushing", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... - :query registry: the registry you wan to push, optional - :statuscode 200: no error + :query registry: the registry you wan to push, optional + :statuscode 200: no error :statuscode 404: no such image :statuscode 500: server error @@ -900,37 +902,37 @@ Build an image from Dockerfile via stdin .. http:post:: /build - Build an image from Dockerfile via stdin + Build an image from Dockerfile via stdin - **Example request**: + **Example request**: - .. sourcecode:: http + .. sourcecode:: http - POST /build HTTP/1.1 - - {{ STREAM }} + POST /build HTTP/1.1 - **Example response**: + {{ STREAM }} - .. sourcecode:: http + **Example response**: - HTTP/1.1 200 OK - - {{ STREAM }} + .. sourcecode:: http + + HTTP/1.1 200 OK + + {{ STREAM }} - The stream must be a tar archive compressed with one of the following algorithms: - identity (no compression), gzip, bzip2, xz. The archive must include a file called - `Dockerfile` at its root. It may include any number of other files, which will be - accessible in the build context (See the ADD build command). + The stream must be a tar archive compressed with one of the following algorithms: + identity (no compression), gzip, bzip2, xz. The archive must include a file called + `Dockerfile` at its root. It may include any number of other files, which will be + accessible in the build context (See the ADD build command). - The Content-type header should be set to "application/tar". + The Content-type header should be set to "application/tar". :query t: repository name (and optionally a tag) to be applied to the resulting image in case of success :query q: suppress verbose build output :query nocache: do not use the cache when building the image :statuscode 200: no error - :statuscode 500: server error + :statuscode 500: server error Check auth configuration @@ -1033,22 +1035,22 @@ Create a new image from a container's changes .. http:post:: /commit - Create a new image from a container's changes + Create a new image from a container's changes - **Example request**: + **Example request**: - .. sourcecode:: http + .. sourcecode:: http - POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 + POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 **Example response**: - .. sourcecode:: http + .. sourcecode:: http - HTTP/1.1 201 OK - Content-Type: application/vnd.docker.raw-stream + HTTP/1.1 201 OK + Content-Type: application/vnd.docker.raw-stream - {"Id":"596069db4bf5"} + {"Id":"596069db4bf5"} :query container: source container :query repo: repository @@ -1060,7 +1062,6 @@ Create a new image from a container's changes :statuscode 404: no such container :statuscode 500: server error - 3. Going further ================ @@ -1089,6 +1090,8 @@ In this version of the API, /attach, uses hijacking to transport stdin, stdout a ----------------- To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. - - docker -d -H="192.168.1.9:4243" -api-enable-cors + +.. code-block:: bash + + docker -d -H="192.168.1.9:4243" -api-enable-cors From 74373c4d59f0a7bde2e99cd94e65509ac1f495ca Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 5 Aug 2013 16:32:25 -0700 Subject: [PATCH 110/134] Update utils_test.go Upstream-commit: dcf9dfb12961fc8101624af0b925d026fde4cde1 Component: engine --- components/engine/utils/utils_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index 882165ac54..d8ba7f1c31 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -318,7 +318,7 @@ func TestGetResolvConf(t *testing.T) { } } -func TestCheclLocalDns(t *testing.T) { +func TestCheckLocalDns(t *testing.T) { for resolv, result := range map[string]bool{`# Dynamic nameserver 10.0.2.3 search dotcloud.net`: false, From cb2611957a237f8d6f381947d4c67055ea13c61c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 5 Aug 2013 18:11:13 -0700 Subject: [PATCH 111/134] fix the upstart script generated by get.docker.io (it was not starting dockerd on boot) Upstream-commit: 049d28868e87df084748a91db5d1cfecb0de6d00 Component: engine --- components/engine/contrib/install.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/components/engine/contrib/install.sh b/components/engine/contrib/install.sh index cf097da670..e58f054685 100755 --- a/components/engine/contrib/install.sh +++ b/components/engine/contrib/install.sh @@ -45,7 +45,13 @@ then echo "Upstart script already exists." else echo "Creating /etc/init/dockerd.conf..." - echo "exec env LANG=\"en_US.UTF-8\" /usr/local/bin/docker -d" > /etc/init/dockerd.conf + cat >/etc/init/dockerd.conf < Date: Mon, 5 Aug 2013 19:58:48 -0700 Subject: [PATCH 112/134] Make sure all sources have the wanted revision Upstream-commit: 120a520a225725475348b718a81562c6d6c1da86 Component: engine --- components/engine/packaging/ubuntu/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/packaging/ubuntu/Makefile b/components/engine/packaging/ubuntu/Makefile index 7bddf2376f..a1e53b1d5c 100644 --- a/components/engine/packaging/ubuntu/Makefile +++ b/components/engine/packaging/ubuntu/Makefile @@ -32,7 +32,7 @@ ubuntu: # Retrieve docker project and its go structure from internet rm -rf ${BUILD_SRC} git clone $(shell git rev-parse --show-toplevel) ${BUILD_SRC}/${GITHUB_PATH} - cd ${BUILD_SRC}/${GITHUB_PATH}; git checkout v${VERSION} && GOPATH=${BUILD_SRC} go get -d + cd ${BUILD_SRC}/${GITHUB_PATH}; git checkout v${VERSION} && GOPATH=${BUILD_SRC} go get -d && cd ${BUILD_SRC}/src/${GITHUB_PATH} && git checkout v${VERSION} # Add debianization mkdir ${BUILD_SRC}/debian cp Makefile ${BUILD_SRC} From 2eadef5023dec5d7e30fc84f112118c6b44d80dd Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Mon, 22 Jul 2013 14:42:31 -0400 Subject: [PATCH 113/134] Added index address into APIInfo. Upstream-commit: 303490168fb53af2528375ca9ec818c7a179a044 Component: engine --- components/engine/api_params.go | 21 +++++++++++---------- components/engine/commands.go | 10 ++++++---- components/engine/server.go | 21 +++++++++++---------- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/components/engine/api_params.go b/components/engine/api_params.go index 1ae26072e7..ca4c5503ec 100644 --- a/components/engine/api_params.go +++ b/components/engine/api_params.go @@ -17,16 +17,17 @@ type APIImages struct { } type APIInfo struct { - Debug bool - Containers int - Images int - NFd int `json:",omitempty"` - NGoroutines int `json:",omitempty"` - MemoryLimit bool `json:",omitempty"` - SwapLimit bool `json:",omitempty"` - LXCVersion string `json:",omitempty"` - NEventsListener int `json:",omitempty"` - KernelVersion string `json:",omitempty"` + Debug bool + Containers int + Images int + NFd int `json:",omitempty"` + NGoroutines int `json:",omitempty"` + MemoryLimit bool `json:",omitempty"` + SwapLimit bool `json:",omitempty"` + LXCVersion string `json:",omitempty"` + NEventsListener int `json:",omitempty"` + KernelVersion string `json:",omitempty"` + IndexServerAddress string `json:",omitempty"` } type APITop struct { diff --git a/components/engine/commands.go b/components/engine/commands.go index 9c59857ae1..167d33ce1d 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -495,10 +495,12 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "EventsListeners: %d\n", out.NEventsListener) fmt.Fprintf(cli.out, "Kernel Version: %s\n", out.KernelVersion) } - if cli.authConfig != nil { - fmt.Fprintf(cli.out, "Username: %v\n", cli.authConfig.Username) - // XXX Should we print registry address even if the user was not logged in? - fmt.Fprintf(cli.out, "Registry: %v\n", auth.IndexServerAddress()) + if len(out.IndexServerAddress) != 0 { + u := cli.configFile.Configs[out.IndexServerAddress].Username + if len(u) > 0 { + fmt.Fprintf(cli.out, "Username: %v\n", u) + fmt.Fprintf(cli.out, "Registry: %v\n", out.IndexServerAddress) + } } if !out.MemoryLimit { fmt.Fprintf(cli.err, "WARNING: No memory limit support\n") diff --git a/components/engine/server.go b/components/engine/server.go index 7607851f49..85e0c255cd 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -265,16 +265,17 @@ func (srv *Server) DockerInfo() *APIInfo { } return &APIInfo{ - Containers: len(srv.runtime.List()), - Images: imgcount, - MemoryLimit: srv.runtime.capabilities.MemoryLimit, - SwapLimit: srv.runtime.capabilities.SwapLimit, - Debug: os.Getenv("DEBUG") != "", - NFd: utils.GetTotalUsedFds(), - NGoroutines: runtime.NumGoroutine(), - LXCVersion: lxcVersion, - NEventsListener: len(srv.events), - KernelVersion: kernelVersion, + Containers: len(srv.runtime.List()), + Images: imgcount, + MemoryLimit: srv.runtime.capabilities.MemoryLimit, + SwapLimit: srv.runtime.capabilities.SwapLimit, + Debug: os.Getenv("DEBUG") != "", + NFd: utils.GetTotalUsedFds(), + NGoroutines: runtime.NumGoroutine(), + LXCVersion: lxcVersion, + NEventsListener: len(srv.events), + KernelVersion: kernelVersion, + IndexServerAddress: auth.IndexServerAddress(), } } From 4260a5cc23e7859a8f1277c31cab80c03fcc3179 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Mon, 22 Jul 2013 16:09:11 -0400 Subject: [PATCH 114/134] fixing #910 Upstream-commit: 965de6ef50ceb479168c9e4f7156fe2b855e4245 Component: engine --- components/engine/commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/commands.go b/components/engine/commands.go index 167d33ce1d..387717f360 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -495,6 +495,7 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "EventsListeners: %d\n", out.NEventsListener) fmt.Fprintf(cli.out, "Kernel Version: %s\n", out.KernelVersion) } + if len(out.IndexServerAddress) != 0 { u := cli.configFile.Configs[out.IndexServerAddress].Username if len(u) > 0 { From f5e3a270806c30b1d2aed7d2e555d2cd2fbff1b5 Mon Sep 17 00:00:00 2001 From: Manuel Meurer Date: Tue, 6 Aug 2013 15:21:26 +0200 Subject: [PATCH 115/134] Fix indentation in Vagrantfile Upstream-commit: b9149f45bf6bdef3b7025fe7c94a930696a4b4ab Component: engine --- components/engine/Vagrantfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/engine/Vagrantfile b/components/engine/Vagrantfile index aadabb8711..18aa5b5afb 100644 --- a/components/engine/Vagrantfile +++ b/components/engine/Vagrantfile @@ -82,15 +82,15 @@ Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| end if !FORWARD_DOCKER_PORTS.nil? - Vagrant::VERSION < "1.1.0" and Vagrant::Config.run do |config| - (49000..49900).each do |port| - config.vm.forward_port port, port - end + Vagrant::VERSION < "1.1.0" and Vagrant::Config.run do |config| + (49000..49900).each do |port| + config.vm.forward_port port, port end + end - Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| - (49000..49900).each do |port| - config.vm.network :forwarded_port, :host => port, :guest => port - end + Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| + (49000..49900).each do |port| + config.vm.network :forwarded_port, :host => port, :guest => port end + end end From 82d1681ff07be55fcab7919f4c49ab64cd9e110d Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 6 Aug 2013 14:31:51 +0000 Subject: [PATCH 116/134] fix small \n error un docker build Upstream-commit: ba17f4a06a75a66e11b6cf2ca2cdb5bee4f7bfa8 Component: engine --- components/engine/commands.go | 2 +- components/engine/graph.go | 2 +- components/engine/server.go | 8 ++++---- components/engine/utils/utils.go | 18 +++++++++++++----- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 78916f3a7c..d562426a1a 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -196,7 +196,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { // FIXME: ProgressReader shouldn't be this annoyning to use if context != nil { sf := utils.NewStreamFormatter(false) - body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("", "Uploading context", "%v bytes%0.0s%0.0s"), sf) + body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("", "Uploading context", "%v bytes%0.0s%0.0s"), sf, true) } // Upload the build context v := &url.Values{} diff --git a/components/engine/graph.go b/components/engine/graph.go index d02017bf8f..1d2a9f807f 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -159,7 +159,7 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *uti if err != nil { return nil, err } - return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf), tmp.Root) + return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp.Root) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. diff --git a/components/engine/server.go b/components/engine/server.go index 7607851f49..62fd2d55da 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -145,7 +145,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. return "", err } - if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("", "Downloading", "%8v/%v (%v)"), sf), path); err != nil { + if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("", "Downloading", "%8v/%v (%v)"), sf, true), path); err != nil { return "", err } // FIXME: Handle custom repo, tag comment, author @@ -438,7 +438,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return err } defer layer.Close() - if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf), img); err != nil { + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf, false), img); err != nil { return err } } @@ -711,7 +711,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } // Send the layer - if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil { + if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf, true), ep, token, jsonRaw); err != nil { return "", err } else { imgData.Checksum = checksum @@ -789,7 +789,7 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write if err != nil { return err } - archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("", "Importing", "%8v/%v (%v)"), sf) + archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("", "Importing", "%8v/%v (%v)"), sf, true) } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil) if err != nil { diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index e0083d02d8..d52dcafe17 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -73,6 +73,7 @@ type progressReader struct { lastUpdate int // How many bytes read at least update template string // Template to print. Default "%v/%v (%v)" sf *StreamFormatter + newLine bool } func (r *progressReader) Read(p []byte) (n int, err error) { @@ -94,17 +95,24 @@ func (r *progressReader) Read(p []byte) (n int, err error) { } r.lastUpdate = r.readProgress } + // Send newline when complete + if r.newLine && err != nil { + r.output.Write(r.sf.FormatStatus("", "")) + } return read, err } func (r *progressReader) Close() error { return io.ReadCloser(r.reader).Close() } -func ProgressReader(r io.ReadCloser, size int, output io.Writer, template []byte, sf *StreamFormatter) *progressReader { - tpl := string(template) - if tpl == "" { - tpl = string(sf.FormatProgress("", "%8v/%v (%v)", "")) +func ProgressReader(r io.ReadCloser, size int, output io.Writer, tpl []byte, sf *StreamFormatter, newline bool) *progressReader { + return &progressReader{ + reader: r, + output: NewWriteFlusher(output), + readTotal: size, + template: string(tpl), + sf: sf, + newLine: newline, } - return &progressReader{r, NewWriteFlusher(output), size, 0, 0, tpl, sf} } // HumanDuration returns a human-readable approximation of a duration From 6ad1290069a4767df7bcb9d2d58967b14262d1d7 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sat, 3 Aug 2013 20:55:54 +0000 Subject: [PATCH 117/134] Use mime types to parse Content-Type Upstream-commit: 754ed9043d6e1fed98160c7fb443bed0a9464acf Component: engine --- components/engine/api.go | 11 ++++++++++- components/engine/api_test.go | 14 ++++++++++++++ components/engine/commands.go | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index f520d7a73b..f5f0df1b98 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -9,6 +9,7 @@ import ( "io" "io/ioutil" "log" + "mime" "net" "net/http" "os" @@ -81,6 +82,14 @@ func getBoolParam(value string) (bool, error) { return ret, nil } +func matchesContentType(contentType, expectedType string) bool { + mimetype, _, err := mime.ParseMediaType(contentType) + if err != nil { + utils.Debugf("Error parsing media type: %s error: %s", contentType, err.Error()) + } + return err == nil && mimetype == expectedType +} + func postAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { authConfig := &auth.AuthConfig{} err := json.NewDecoder(r.Body).Decode(authConfig) @@ -594,7 +603,7 @@ func postContainersStart(srv *Server, version float64, w http.ResponseWriter, r // allow a nil body for backwards compatibility if r.Body != nil { - if r.Header.Get("Content-Type") == "application/json" { + if matchesContentType(r.Header.Get("Content-Type"), "application/json") { if err := json.NewDecoder(r.Body).Decode(hostConfig); err != nil { return err } diff --git a/components/engine/api_test.go b/components/engine/api_test.go index be535924b7..5316b4fc5e 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -1142,6 +1142,20 @@ func TestDeleteImages(t *testing.T) { } */ } +func TestJsonContentType(t *testing.T) { + if !matchesContentType("application/json", "application/json") { + t.Fail() + } + + if !matchesContentType("application/json; charset=utf-8", "application/json") { + t.Fail() + } + + if matchesContentType("dockerapplication/json", "application/json") { + t.Fail() + } +} + // Mocked types for tests type NopConn struct { io.ReadCloser diff --git a/components/engine/commands.go b/components/engine/commands.go index 78916f3a7c..c52126519a 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1567,7 +1567,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e return fmt.Errorf("Error: %s", body) } - if resp.Header.Get("Content-Type") == "application/json" { + if matchesContentType(resp.Header.Get("Content-Type"), "application/json") { return utils.DisplayJSONMessagesStream(resp.Body, out) } else { if _, err := io.Copy(out, resp.Body); err != nil { From 3ea6442a29071b3764074a5f659e2f77f4514c51 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 16 Jul 2013 19:07:41 -0900 Subject: [PATCH 118/134] Add cp command and copy api endpoint The cp command and copy api endpoint allows users to copy files and or folders from a containers filesystem. Closes #382 Upstream-commit: 5b8cfbe15c29efd3e72ea97ba87867590aeeba25 Component: engine --- components/engine/api.go | 30 +++++++++ components/engine/api_params.go | 5 ++ components/engine/api_test.go | 64 +++++++++++++++++++ components/engine/commands.go | 32 ++++++++++ components/engine/container.go | 7 ++ .../sources/api/docker_remote_api_v1.3.rst | 33 +++++++++- .../engine/docs/sources/commandline/cli.rst | 1 + .../docs/sources/commandline/command/cp.rst | 13 ++++ .../engine/docs/sources/commandline/index.rst | 1 + components/engine/server.go | 17 +++++ 10 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 components/engine/docs/sources/commandline/command/cp.rst diff --git a/components/engine/api.go b/components/engine/api.go index f5f0df1b98..f9257b1ccd 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -871,6 +871,35 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ return nil } +func postContainersCopy(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + name := vars["name"] + + copyData := &APICopy{} + if r.Header.Get("Content-Type") == "application/json" { + if err := json.NewDecoder(r.Body).Decode(copyData); err != nil { + return err + } + } else { + return fmt.Errorf("Content-Type not supported: %s", r.Header.Get("Content-Type")) + } + + if copyData.Resource == "" { + return fmt.Errorf("Resource cannot be empty") + } + if copyData.Resource[0] == '/' { + return fmt.Errorf("Resource cannot contain a leading /") + } + + if err := srv.ContainerCopy(name, copyData.Resource, w); err != nil { + utils.Debugf("%s", err) + return err + } + return nil +} + func optionsHandler(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { w.WriteHeader(http.StatusOK) return nil @@ -918,6 +947,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) { "/containers/{name:.*}/wait": postContainersWait, "/containers/{name:.*}/resize": postContainersResize, "/containers/{name:.*}/attach": postContainersAttach, + "/containers/{name:.*}/copy": postContainersCopy, }, "DELETE": { "/containers/{name:.*}": deleteContainers, diff --git a/components/engine/api_params.go b/components/engine/api_params.go index 1ae26072e7..b75e54c116 100644 --- a/components/engine/api_params.go +++ b/components/engine/api_params.go @@ -86,3 +86,8 @@ type APIImageConfig struct { ID string `json:"Id"` *Config } + +type APICopy struct { + Resource string + HostPath string +} diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 5316b4fc5e..52baf35f38 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -1156,6 +1156,70 @@ func TestJsonContentType(t *testing.T) { } } +func TestPostContainersCopy(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + + srv := &Server{runtime: runtime} + + builder := NewBuilder(runtime) + + // Create a container and remove a file + container, err := builder.Create( + &Config{ + Image: GetTestImage(runtime).ID, + Cmd: []string{"touch", "/test.txt"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + + if err := container.Run(); err != nil { + t.Fatal(err) + } + + r := httptest.NewRecorder() + copyData := APICopy{HostPath: ".", Resource: "test.txt"} + + jsonData, err := json.Marshal(copyData) + if err != nil { + t.Fatal(err) + } + + req, err := http.NewRequest("POST", "/containers/"+container.ID+"/copy", bytes.NewReader(jsonData)) + if err != nil { + t.Fatal(err) + } + req.Header.Add("Content-Type", "application/json") + if err = postContainersCopy(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { + t.Fatal(err) + } + + if r.Code != http.StatusOK { + t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code) + } + + found := false + for tarReader := tar.NewReader(r.Body); ; { + h, err := tarReader.Next() + if err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + if h.Name == "test.txt" { + found = true + break + } + } + if !found { + t.Fatalf("The created test file has not been found in the copied output") + } +} + // Mocked types for tests type NopConn struct { io.ReadCloser diff --git a/components/engine/commands.go b/components/engine/commands.go index c52126519a..eb05202ded 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -77,6 +77,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error { {"attach", "Attach to a running container"}, {"build", "Build a container from a Dockerfile"}, {"commit", "Create a new image from a container's changes"}, + {"cp", "Copy files/folders from the containers filesystem to the host path"}, {"diff", "Inspect changes on a container's filesystem"}, {"events", "Get real time events from the server"}, {"export", "Stream the contents of a container as a tar archive"}, @@ -1469,6 +1470,37 @@ func (cli *DockerCli) CmdRun(args ...string) error { return nil } +func (cli *DockerCli) CmdCp(args ...string) error { + cmd := Subcmd("cp", "CONTAINER:RESOURCE HOSTPATH", "Copy files/folders from the RESOURCE to the HOSTPATH") + if err := cmd.Parse(args); err != nil { + return nil + } + + if cmd.NArg() != 2 { + cmd.Usage() + return nil + } + + var copyData APICopy + info := strings.Split(cmd.Arg(0), ":") + + copyData.Resource = info[1] + copyData.HostPath = cmd.Arg(1) + + data, statusCode, err := cli.call("POST", "/containers/"+info[0]+"/copy", copyData) + if err != nil { + return err + } + + r := bytes.NewReader(data) + if statusCode == 200 { + if err := Untar(r, copyData.HostPath); err != nil { + return err + } + } + return nil +} + func (cli *DockerCli) checkIfLogged(action string) error { // If condition AND the login failed if cli.configFile.Configs[auth.IndexServerAddress()].Username == "" { diff --git a/components/engine/container.go b/components/engine/container.go index 6c92973920..7791d4b4af 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -1089,3 +1089,10 @@ func (container *Container) GetSize() (int64, int64) { } return sizeRw, sizeRootfs } + +func (container *Container) Copy(resource string) (Archive, error) { + if err := container.EnsureMounted(); err != nil { + return nil, err + } + return TarFilter(container.RootfsPath(), Uncompressed, []string{resource}) +} diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst index 8e5c7b2a3b..fe3d204d4d 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst @@ -525,6 +525,38 @@ Remove a container :statuscode 500: server error +Copy files or folders from a container +************************************** + +.. http:post:: /containers/(id)/copy + + Copy files or folders of container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/4fa6e0f0c678/copy HTTP/1.1 + Content-Type: application/json + + { + "Resource":"test.txt" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + 2.2 Images ---------- @@ -1091,7 +1123,6 @@ Monitor Docker's events :statuscode 200: no error :statuscode 500: server error - 3. Going further ================ diff --git a/components/engine/docs/sources/commandline/cli.rst b/components/engine/docs/sources/commandline/cli.rst index e499b1f096..bdc4827a60 100644 --- a/components/engine/docs/sources/commandline/cli.rst +++ b/components/engine/docs/sources/commandline/cli.rst @@ -30,6 +30,7 @@ Available Commands command/attach command/build command/commit + command/cp command/diff command/export command/history diff --git a/components/engine/docs/sources/commandline/command/cp.rst b/components/engine/docs/sources/commandline/command/cp.rst new file mode 100644 index 0000000000..14b5061ef7 --- /dev/null +++ b/components/engine/docs/sources/commandline/command/cp.rst @@ -0,0 +1,13 @@ +:title: Cp Command +:description: Copy files/folders from the containers filesystem to the host path +:keywords: cp, docker, container, documentation, copy + +=========================================================== +``cp`` -- Copy files/folders from the containers filesystem to the host path +=========================================================== + +:: + + Usage: docker cp CONTAINER:RESOURCE HOSTPATH + + Copy files/folders from the containers filesystem to the host path. Paths are relative to the root of the filesystem. diff --git a/components/engine/docs/sources/commandline/index.rst b/components/engine/docs/sources/commandline/index.rst index a7296b27da..5c2b373205 100644 --- a/components/engine/docs/sources/commandline/index.rst +++ b/components/engine/docs/sources/commandline/index.rst @@ -15,6 +15,7 @@ Contents: attach build commit + cp diff export history diff --git a/components/engine/server.go b/components/engine/server.go index 7607851f49..c7b2ad2dcb 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1169,6 +1169,23 @@ func (srv *Server) ImageInspect(name string) (*Image, error) { return nil, fmt.Errorf("No such image: %s", name) } +func (srv *Server) ContainerCopy(name string, resource string, out io.Writer) error { + if container := srv.runtime.Get(name); container != nil { + + data, err := container.Copy(resource) + if err != nil { + return err + } + + if _, err := io.Copy(out, data); err != nil { + return err + } + return nil + } + return fmt.Errorf("No such container: %s", name) + +} + func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) (*Server, error) { if runtime.GOARCH != "amd64" { log.Fatalf("The docker runtime currently only supports amd64 (not %s). This will change in the future. Aborting.", runtime.GOARCH) From 073f998d1de1f622c5d54f904ee2947e192d2a8b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 17 Jul 2013 05:25:49 -0900 Subject: [PATCH 119/134] Strip leading forward slash from resource Upstream-commit: d94b1860806e11038086e91c693d86a009dcd3b0 Component: engine --- components/engine/api.go | 9 +++++---- components/engine/api_test.go | 2 +- components/engine/commands.go | 2 +- components/engine/container.go | 16 +++++++++++++++- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index f9257b1ccd..46fe8251e0 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -878,23 +878,24 @@ func postContainersCopy(srv *Server, version float64, w http.ResponseWriter, r * name := vars["name"] copyData := &APICopy{} - if r.Header.Get("Content-Type") == "application/json" { + contentType := r.Header.Get("Content-Type") + if contentType == "application/json" { if err := json.NewDecoder(r.Body).Decode(copyData); err != nil { return err } } else { - return fmt.Errorf("Content-Type not supported: %s", r.Header.Get("Content-Type")) + return fmt.Errorf("Content-Type not supported: %s", contentType) } if copyData.Resource == "" { return fmt.Errorf("Resource cannot be empty") } if copyData.Resource[0] == '/' { - return fmt.Errorf("Resource cannot contain a leading /") + copyData.Resource = copyData.Resource[1:] } if err := srv.ContainerCopy(name, copyData.Resource, w); err != nil { - utils.Debugf("%s", err) + utils.Debugf("%s", err.Error()) return err } return nil diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 52baf35f38..928daf963c 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -1181,7 +1181,7 @@ func TestPostContainersCopy(t *testing.T) { } r := httptest.NewRecorder() - copyData := APICopy{HostPath: ".", Resource: "test.txt"} + copyData := APICopy{HostPath: ".", Resource: "/test.txt"} jsonData, err := json.Marshal(copyData) if err != nil { diff --git a/components/engine/commands.go b/components/engine/commands.go index eb05202ded..d4cf91ea10 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1492,8 +1492,8 @@ func (cli *DockerCli) CmdCp(args ...string) error { return err } - r := bytes.NewReader(data) if statusCode == 200 { + r := bytes.NewReader(data) if err := Untar(r, copyData.HostPath); err != nil { return err } diff --git a/components/engine/container.go b/components/engine/container.go index 7791d4b4af..6a60597edb 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -1094,5 +1094,19 @@ func (container *Container) Copy(resource string) (Archive, error) { if err := container.EnsureMounted(); err != nil { return nil, err } - return TarFilter(container.RootfsPath(), Uncompressed, []string{resource}) + var filter []string + basePath := path.Join(container.RootfsPath(), resource) + stat, err := os.Stat(basePath) + if err != nil { + return nil, err + } + if !stat.IsDir() { + d, f := path.Split(basePath) + basePath = d + filter = []string{f} + } else { + filter = []string{path.Base(basePath)} + basePath = path.Dir(basePath) + } + return TarFilter(basePath, Uncompressed, filter) } From 2de21e781e56029bb524964d69d3522f1278b80f Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 6 Aug 2013 16:01:09 +0000 Subject: [PATCH 120/134] Move copy command docs to api 1.4 document Upstream-commit: 583f5868c989b9da574638ffdc532826e5070347 Component: engine --- .../sources/api/docker_remote_api_v1.3.rst | 33 +------------------ .../sources/api/docker_remote_api_v1.4.rst | 32 ++++++++++++++++++ 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst index fe3d204d4d..8e5c7b2a3b 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst @@ -525,38 +525,6 @@ Remove a container :statuscode 500: server error -Copy files or folders from a container -************************************** - -.. http:post:: /containers/(id)/copy - - Copy files or folders of container ``id`` - - **Example request**: - - .. sourcecode:: http - - POST /containers/4fa6e0f0c678/copy HTTP/1.1 - Content-Type: application/json - - { - "Resource":"test.txt" - } - - **Example response**: - - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/octet-stream - - {{ STREAM }} - - :statuscode 200: no error - :statuscode 404: no such container - :statuscode 500: server error - - 2.2 Images ---------- @@ -1123,6 +1091,7 @@ Monitor Docker's events :statuscode 200: no error :statuscode 500: server error + 3. Going further ================ diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst index 31e5b5f766..fe73cb5405 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -528,6 +528,38 @@ Remove a container :statuscode 500: server error +Copy files or folders from a container +************************************** + +.. http:post:: /containers/(id)/copy + + Copy files or folders of container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/4fa6e0f0c678/copy HTTP/1.1 + Content-Type: application/json + + { + "Resource":"test.txt" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + 2.2 Images ---------- From 5081a0aa8b7284f953da0f9a1df9b443ac2c30af Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Tue, 6 Aug 2013 11:40:16 -0700 Subject: [PATCH 121/134] changed the twitter handle Upstream-commit: 1d654f615673aecb32011bd5a28c2ac43dbbc1fb Component: engine --- components/engine/docs/sources/faq.rst | 2 +- components/engine/docs/theme/docker/layout.html | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/components/engine/docs/sources/faq.rst b/components/engine/docs/sources/faq.rst index 12b9751338..3cc0086c5d 100644 --- a/components/engine/docs/sources/faq.rst +++ b/components/engine/docs/sources/faq.rst @@ -47,7 +47,7 @@ Most frequently asked questions. .. _IRC, docker on freenode: irc://chat.freenode.net#docker .. _Github: http://www.github.com/dotcloud/docker .. _Ask questions on Stackoverflow: http://stackoverflow.com/search?q=docker - .. _Join the conversation on Twitter: http://twitter.com/getdocker + .. _Join the conversation on Twitter: http://twitter.com/docker Looking for something else to read? Checkout the :ref:`hello_world` example. diff --git a/components/engine/docs/theme/docker/layout.html b/components/engine/docs/theme/docker/layout.html index ca26f44dc0..d6bfff79ba 100755 --- a/components/engine/docs/theme/docker/layout.html +++ b/components/engine/docs/theme/docker/layout.html @@ -130,7 +130,7 @@
@@ -156,11 +156,6 @@ {# {%- endif %}#} -{##} -{#
#} -{# #} -{# GitHub#} -{#
#}
From 2b2233fe775fa20d1fe47151ef7e6d2fe0d0aee5 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Tue, 6 Aug 2013 13:56:29 -0700 Subject: [PATCH 122/134] Suggest installing linux-headers by default. Address #1187 and #1031 Upstream-commit: e0a6f27d1bbf053baf8a94c87fddd3f9ed30964c Component: engine --- .../engine/docs/sources/installation/ubuntulinux.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index cf41b79266..587c8fe31b 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -34,13 +34,20 @@ Dependencies **Linux kernel 3.8** -Due to a bug in LXC docker works best on the 3.8 kernel. Precise comes with a 3.2 kernel, so we need to upgrade it. The kernel we install comes with AUFS built in. +Due to a bug in LXC, docker works best on the 3.8 kernel. Precise +comes with a 3.2 kernel, so we need to upgrade it. The kernel we +install comes with AUFS built in. We also include the generic headers +to enable packages that depend on them, like ZFS and the VirtualBox +guest additions. If you didn't install the headers for your "precise" +kernel, then you can skip these headers for the "raring" kernel. But +it is safer to include them if you're not sure. .. code-block:: bash # install the backported kernel - sudo apt-get update && sudo apt-get install linux-image-generic-lts-raring + sudo apt-get update + sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring # reboot sudo reboot From 217ae3229e56daba1f303d64323cc1008546729b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Tue, 6 Aug 2013 17:24:10 -0700 Subject: [PATCH 123/134] change network range to avoid conflict with EC2 DNS Upstream-commit: 9f1c9686e0ee47289adeeedd6cf8d5296a058463 Component: engine --- components/engine/network.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/network.go b/components/engine/network.go index eefd36df3b..156ca3b47d 100644 --- a/components/engine/network.go +++ b/components/engine/network.go @@ -131,8 +131,8 @@ func CreateBridgeIface(ifaceName string) error { // In theory this shouldn't matter - in practice there's bound to be a few scripts relying // on the internal addressing or other stupid things like that. // The shouldn't, but hey, let's not break them unless we really have to. - "172.16.42.1/16", - "10.0.42.1/16", // Don't even try using the entire /8, that's too intrusive + "172.17.42.1/16", // Don't use 172.16.0.0/16, it conflicts with EC2 DNS 172.16.0.23 + "10.0.42.1/16", // Don't even try using the entire /8, that's too intrusive "10.1.42.1/16", "10.42.42.1/16", "172.16.42.1/24", From 437d59d219909441e0a6024a30bfb5dab2ac78bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Tue, 6 Aug 2013 10:00:45 -0700 Subject: [PATCH 124/134] Let userland proxy handle container-bound traffic Upstream-commit: fea2d5f2fe81d983efbdd803cef2fd1c70f98ebc Component: engine --- components/engine/network.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/network.go b/components/engine/network.go index eefd36df3b..2916fc5070 100644 --- a/components/engine/network.go +++ b/components/engine/network.go @@ -253,6 +253,7 @@ func (mapper *PortMapper) setup() error { func (mapper *PortMapper) iptablesForward(rule string, port int, proto string, dest_addr string, dest_port int) error { return iptables("-t", "nat", rule, "DOCKER", "-p", proto, "--dport", strconv.Itoa(port), + "!", "-i", NetworkBridgeIface, "-j", "DNAT", "--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port))) } @@ -264,7 +265,7 @@ func (mapper *PortMapper) Map(port int, backendAddr net.Addr) error { return err } mapper.tcpMapping[port] = backendAddr.(*net.TCPAddr) - proxy, err := NewProxy(&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}, backendAddr) + proxy, err := NewProxy(&net.TCPAddr{IP: net.IPv4(0, 0, 0, 0), Port: port}, backendAddr) if err != nil { mapper.Unmap(port, "tcp") return err @@ -278,7 +279,7 @@ func (mapper *PortMapper) Map(port int, backendAddr net.Addr) error { return err } mapper.udpMapping[port] = backendAddr.(*net.UDPAddr) - proxy, err := NewProxy(&net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}, backendAddr) + proxy, err := NewProxy(&net.UDPAddr{IP: net.IPv4(0, 0, 0, 0), Port: port}, backendAddr) if err != nil { mapper.Unmap(port, "udp") return err From faa72ea812c6cc33b40757ff3868a9e53f8a939f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Tue, 6 Aug 2013 18:31:05 -0700 Subject: [PATCH 125/134] relax the lo interface test to allow iface index != 1 Upstream-commit: 84790aafd8b3050e502a16f03ee288c1bb9c21b6 Component: engine --- components/engine/container_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/container_test.go b/components/engine/container_test.go index 2f9da15f6e..1050bfb2ad 100644 --- a/components/engine/container_test.go +++ b/components/engine/container_test.go @@ -1306,10 +1306,10 @@ func TestOnlyLoopbackExistsWhenUsingDisableNetworkOption(t *testing.T) { interfaces := regexp.MustCompile(`(?m)^[0-9]+: [a-zA-Z0-9]+`).FindAllString(string(output), -1) if len(interfaces) != 1 { - t.Fatalf("Wrong interface count in test container: expected [1: lo], got [%s]", interfaces) + t.Fatalf("Wrong interface count in test container: expected [*: lo], got %s", interfaces) } - if interfaces[0] != "1: lo" { - t.Fatalf("Wrong interface in test container: expected [1: lo], got [%s]", interfaces) + if !strings.HasSuffix(interfaces[0], ": lo") { + t.Fatalf("Wrong interface in test container: expected [*: lo], got %s", interfaces) } } From 3b3499d0fa37386b8359dfb6208e1cb7e3ff806f Mon Sep 17 00:00:00 2001 From: David Sissitka Date: Wed, 7 Aug 2013 05:33:03 -0400 Subject: [PATCH 126/134] Updated the Docker CLI to specify a value for the "Host" header. Upstream-commit: 6bbe66d2e6414a127f0f543c0138f1709bd3b8b4 Component: engine --- components/engine/commands.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/commands.go b/components/engine/commands.go index f0e1695b3f..e0d33d7d36 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1433,6 +1433,7 @@ func (cli *DockerCli) call(method, path string, data interface{}) ([]byte, int, return nil, -1, err } req.Header.Set("User-Agent", "Docker-Client/"+VERSION) + req.Host = cli.addr if data != nil { req.Header.Set("Content-Type", "application/json") } else if method == "POST" { @@ -1474,6 +1475,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e return err } req.Header.Set("User-Agent", "Docker-Client/"+VERSION) + req.Host = cli.addr if method == "POST" { req.Header.Set("Content-Type", "plain/text") } @@ -1536,6 +1538,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea } req.Header.Set("User-Agent", "Docker-Client/"+VERSION) req.Header.Set("Content-Type", "plain/text") + req.Host = cli.addr dial, err := net.Dial(cli.proto, cli.addr) if err != nil { From b0ec69fb38bb939f1448533d5541b7d702728ff2 Mon Sep 17 00:00:00 2001 From: David Sissitka Date: Wed, 7 Aug 2013 05:35:38 -0400 Subject: [PATCH 127/134] Updated my last commit to use tabs instead of spaces. Upstream-commit: 416d0986886cd322ecae1114c13565bbc27b305c Component: engine --- components/engine/commands.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index e0d33d7d36..a47de9f2ba 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1433,7 +1433,7 @@ func (cli *DockerCli) call(method, path string, data interface{}) ([]byte, int, return nil, -1, err } req.Header.Set("User-Agent", "Docker-Client/"+VERSION) - req.Host = cli.addr + req.Host = cli.addr if data != nil { req.Header.Set("Content-Type", "application/json") } else if method == "POST" { @@ -1475,7 +1475,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e return err } req.Header.Set("User-Agent", "Docker-Client/"+VERSION) - req.Host = cli.addr + req.Host = cli.addr if method == "POST" { req.Header.Set("Content-Type", "plain/text") } @@ -1538,7 +1538,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea } req.Header.Set("User-Agent", "Docker-Client/"+VERSION) req.Header.Set("Content-Type", "plain/text") - req.Host = cli.addr + req.Host = cli.addr dial, err := net.Dial(cli.proto, cli.addr) if err != nil { From 16fcec2f1e9385830b00e515fba443c97275c66e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dra=C5=BEen=20Lu=C4=8Danin?= Date: Wed, 7 Aug 2013 13:57:31 +0200 Subject: [PATCH 128/134] doc: syntax to run a specific image tag Upstream-commit: 6115348dd9e302e8323b857ae8ab1061159e69cd Component: engine --- components/engine/docs/sources/commandline/command/run.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/commandline/command/run.rst b/components/engine/docs/sources/commandline/command/run.rst index db043c3b3f..c5b363d787 100644 --- a/components/engine/docs/sources/commandline/command/run.rst +++ b/components/engine/docs/sources/commandline/command/run.rst @@ -8,7 +8,7 @@ :: - Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...] + Usage: docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...] Run a command in a new container From fa6b9b59de91a78c76a951ebe96d9f4ebcd22340 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 6 Aug 2013 16:58:52 +0000 Subject: [PATCH 129/134] Forbid certain paths within docker build ADD Upstream-commit: 3104fc8d33ce4c26a446826639a7b2dba524264f Component: engine --- components/engine/buildfile.go | 3 ++ components/engine/buildfile_test.go | 49 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 00c747d6d0..3c4229d8bf 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -278,6 +278,9 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { if strings.HasSuffix(dest, "/") { destPath = destPath + "/" } + if !strings.HasPrefix(origPath, b.context) { + return fmt.Errorf("Forbidden path: %s", origPath) + } fi, err := os.Stat(origPath) if err != nil { return err diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 9d002f0665..d89c40d16c 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -423,3 +423,52 @@ func TestBuildImageWithoutCache(t *testing.T) { t.Fail() } } + +func TestForbiddenContextPath(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + context := testContextTemplate{` + from {IMAGE} + maintainer dockerio + add ../../ test/ + `, + [][2]string{{"test.txt", "test1"}, {"other.txt", "other"}}, nil} + + httpServer, err := mkTestingFileServer(context.remoteFiles) + if err != nil { + t.Fatal(err) + } + defer httpServer.Close() + + idx := strings.LastIndex(httpServer.URL, ":") + if idx < 0 { + t.Fatalf("could not get port from test http server address %s", httpServer.URL) + } + port := httpServer.URL[idx+1:] + + ip := srv.runtime.networkManager.bridgeNetwork.IP + dockerfile := constructDockerfile(context.dockerfile, ip, port) + + buildfile := NewBuildFile(srv, ioutil.Discard, false, true) + _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) + + if err == nil { + t.Log("Error should not be nil") + t.Fail() + } + + if err.Error() != "Forbidden path: /" { + t.Logf("Error message is not expected: %s", err.Error()) + t.Fail() + } +} From a692d94c4fc57fb17227ccadc80863d7df1e5995 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 7 Aug 2013 21:24:41 +0000 Subject: [PATCH 130/134] Add Michael Crosby to core maintainers Upstream-commit: 4860df1689f2f98d65aece0f4768b8ae98301d1e Component: engine --- components/engine/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/MAINTAINERS b/components/engine/MAINTAINERS index dc0d1a19f6..55cb07b72b 100644 --- a/components/engine/MAINTAINERS +++ b/components/engine/MAINTAINERS @@ -1,5 +1,6 @@ Solomon Hykes Guillaume Charmes Victor Vieux +Michael Crosby api.go: Victor Vieux Vagrantfile: Daniel Mizyrycki From c746f1a64f0cd6453ca1b24e65926ed093c5cb3e Mon Sep 17 00:00:00 2001 From: Colin Rice Date: Sat, 3 Aug 2013 18:06:58 -0400 Subject: [PATCH 131/134] Add warning when net.ipv4.ip_forwarding = 0 Added warnings to api.go, container.go, commands.go, and runtime.go Also updated APIInfo to return whether IPv4Forwarding is enabled Upstream-commit: 10190be5d74bc4b13a885f2ba27c2299c868fd19 Component: engine --- components/engine/api.go | 5 +++++ components/engine/api_params.go | 1 + components/engine/commands.go | 3 +++ components/engine/container.go | 4 ++++ .../docs/sources/api/docker_remote_api_v1.4.rst | 3 ++- components/engine/runtime.go | 11 +++++++++-- components/engine/server.go | 1 + 7 files changed, 25 insertions(+), 3 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index 46fe8251e0..0223cdd673 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -522,6 +522,11 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r out.Warnings = append(out.Warnings, "Your kernel does not support memory swap capabilities. Limitation discarded.") } + if srv.runtime.capabilities.IPv4Forwarding { + log.Println("Warning: IPv4 forwarding is disabled.") + out.Warnings = append(out.Warnings, "IPv4 forwarding is disabled.") + } + b, err := json.Marshal(out) if err != nil { return err diff --git a/components/engine/api_params.go b/components/engine/api_params.go index 2737943f41..df879f63ee 100644 --- a/components/engine/api_params.go +++ b/components/engine/api_params.go @@ -24,6 +24,7 @@ type APIInfo struct { NGoroutines int `json:",omitempty"` MemoryLimit bool `json:",omitempty"` SwapLimit bool `json:",omitempty"` + IPv4Forwarding bool `json:",omitempty"` LXCVersion string `json:",omitempty"` NEventsListener int `json:",omitempty"` KernelVersion string `json:",omitempty"` diff --git a/components/engine/commands.go b/components/engine/commands.go index 39b8eb69af..0ddcfad18d 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -510,6 +510,9 @@ func (cli *DockerCli) CmdInfo(args ...string) error { if !out.SwapLimit { fmt.Fprintf(cli.err, "WARNING: No swap limit support\n") } + if !out.IPv4Forwarding { + fmt.Fprintf(cli.err, "WARNING: IPv4 forwarding is disabled.\n") + } return nil } diff --git a/components/engine/container.go b/components/engine/container.go index 6a60597edb..8721d45a55 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -534,6 +534,10 @@ func (container *Container) Start(hostConfig *HostConfig) error { container.Config.MemorySwap = -1 } + if !container.runtime.capabilities.IPv4Forwarding { + log.Printf("WARNING: IPv4 forwarding is disabled. Networking will not work") + } + // Create the requested bind mounts binds := make(map[string]BindMap) // Define illegal container destinations diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst index fe73cb5405..06e8f46f99 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -1025,7 +1025,8 @@ Display system-wide information "NFd": 11, "NGoroutines":21, "MemoryLimit":true, - "SwapLimit":false + "SwapLimit":false, + "IPv4Forwarding":true } :statuscode 200: no error diff --git a/components/engine/runtime.go b/components/engine/runtime.go index f4c5b4d380..894028354e 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -15,8 +15,9 @@ import ( ) type Capabilities struct { - MemoryLimit bool - SwapLimit bool + MemoryLimit bool + SwapLimit bool + IPv4Forwarding bool } type Runtime struct { @@ -240,6 +241,12 @@ func (runtime *Runtime) UpdateCapabilities(quiet bool) { if !runtime.capabilities.SwapLimit && !quiet { log.Printf("WARNING: Your kernel does not support cgroup swap limit.") } + + content, err3 := ioutil.ReadFile("/proc/sys/net/ipv4/ip_forward") + runtime.capabilities.IPv4Forwarding = err3 == nil && len(content) > 0 && content[0] == '1' + if !runtime.capabilities.IPv4Forwarding && !quiet { + log.Printf("WARNING: IPv4 forwarding is disabled.") + } } } diff --git a/components/engine/server.go b/components/engine/server.go index 6d6ae5c934..8a4ce0ccf9 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -269,6 +269,7 @@ func (srv *Server) DockerInfo() *APIInfo { Images: imgcount, MemoryLimit: srv.runtime.capabilities.MemoryLimit, SwapLimit: srv.runtime.capabilities.SwapLimit, + IPv4Forwarding: srv.runtime.capabilities.IPv4Forwarding, Debug: os.Getenv("DEBUG") != "", NFd: utils.GetTotalUsedFds(), NGoroutines: runtime.NumGoroutine(), From 8dc1c40e5752af3c7d08b66eabc01ba2ae4fa9d5 Mon Sep 17 00:00:00 2001 From: Colin Rice Date: Sat, 3 Aug 2013 18:15:24 -0400 Subject: [PATCH 132/134] Add Colin Rice to AUTHORS file Upstream-commit: ccffa6976616ddf5fc8c83f84d17e53ca8e0bca9 Component: engine --- components/engine/AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/AUTHORS b/components/engine/AUTHORS index 773ad6e4d3..6f6c597722 100644 --- a/components/engine/AUTHORS +++ b/components/engine/AUTHORS @@ -22,6 +22,7 @@ Caleb Spare Calen Pennington Charles Hooper Christopher Currie +Colin Rice Daniel Gasienica Daniel Mizyrycki Daniel Robinson From c75bcf8bab1ba8857778fe6effbcbf2dbb0d6e59 Mon Sep 17 00:00:00 2001 From: Colin Rice Date: Mon, 5 Aug 2013 18:26:06 -0400 Subject: [PATCH 133/134] Fix reversed IPv4Forwarding check in api.go Upstream-commit: 3e491f8698860ed92b9f4cd6790eda82e083ae1e Component: engine --- components/engine/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/api.go b/components/engine/api.go index 0223cdd673..8eb892f259 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -522,7 +522,7 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r out.Warnings = append(out.Warnings, "Your kernel does not support memory swap capabilities. Limitation discarded.") } - if srv.runtime.capabilities.IPv4Forwarding { + if !srv.runtime.capabilities.IPv4Forwarding { log.Println("Warning: IPv4 forwarding is disabled.") out.Warnings = append(out.Warnings, "IPv4 forwarding is disabled.") } From c2560e21bfb81629b8a2565ae60f41803840ca12 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 7 Aug 2013 17:23:49 -0700 Subject: [PATCH 134/134] Make sure ENV instruction within build perform a commit each time Upstream-commit: 6a6a2ad8a4af85b543a24c5d22a2be5fdfc252d6 Component: engine --- components/engine/buildfile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 3c4229d8bf..b969f758ec 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -172,9 +172,9 @@ func (b *buildFile) CmdEnv(args string) error { if envKey >= 0 { b.config.Env[envKey] = replacedVar - return nil + } else { + b.config.Env = append(b.config.Env, replacedVar) } - b.config.Env = append(b.config.Env, replacedVar) return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s", replacedVar)) }