diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile new file mode 100644 index 0000000000..46f9b585c8 --- /dev/null +++ b/components/engine/Dockerfile @@ -0,0 +1,30 @@ +# This file describes the standard way to build Docker, using docker +docker-version 0.4.2 +from ubuntu:12.04 +maintainer Solomon Hykes +# Build dependencies +run apt-get install -y -q curl +run apt-get install -y -q git +# Install Go +run curl -s https://go.googlecode.com/files/go1.1.1.linux-amd64.tar.gz | tar -v -C /usr/local -xz +env PATH /usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin +env GOPATH /go +env CGO_ENABLED 0 +run cd /tmp && echo 'package main' > t.go && go test -a -i -v +# Download dependencies +run PKG=github.com/kr/pty REV=27435c699; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV +run PKG=github.com/gorilla/context/ REV=708054d61e5; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV +run PKG=github.com/gorilla/mux/ REV=9b36453141c; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV +# Run dependencies +run apt-get install -y iptables +# lxc requires updating ubuntu sources +run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list +run apt-get update +run apt-get install -y lxc +run apt-get install -y aufs-tools +# Upload docker source +add . /go/src/github.com/dotcloud/docker +# Build the binary +run cd /go/src/github.com/dotcloud/docker/docker && go install -ldflags "-X main.GITCOMMIT '??' -d -w" +env PATH /usr/local/go/bin:/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin +cmd ["docker"] diff --git a/components/engine/README.md b/components/engine/README.md index 376ecea703..823e48496a 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -23,15 +23,15 @@ happens, for a few reasons: * *Size*: VMs are very large which makes them impractical to store and transfer. * *Performance*: running VMs consumes significant CPU and memory, which makes them impractical in many scenarios, for example local development of multi-tier applications, and - large-scale deployment of cpu and memory-intensive applications on large numbers of machines. + large-scale deployment of cpu and memory-intensive applications on large numbers of machines. * *Portability*: competing VM environments don't play well with each other. Although conversion tools do exist, they are limited and add even more overhead. * *Hardware-centric*: VMs were designed with machine operators in mind, not software developers. As a result, they offer very limited tooling for what developers need most: - building, testing and running their software. For example, VMs offer no facilities for application versioning, monitoring, configuration, logging or service discovery. + building, testing and running their software. For example, VMs offer no facilities for application versioning, monitoring, configuration, logging or service discovery. By contrast, Docker relies on a different sandboxing method known as *containerization*. Unlike traditional virtualization, containerization takes place at the kernel level. Most modern operating system kernels now support the primitives necessary for containerization, including Linux with [openvz](http://openvz.org), [vserver](http://linux-vserver.org) and more recently [lxc](http://lxc.sourceforge.net), - Solaris with [zones](http://docs.oracle.com/cd/E26502_01/html/E29024/preface-1.html#scrolltoc) and FreeBSD with [Jails](http://www.freebsd.org/doc/handbook/jails.html). + Solaris with [zones](http://docs.oracle.com/cd/E26502_01/html/E29024/preface-1.html#scrolltoc) and FreeBSD with [Jails](http://www.freebsd.org/doc/handbook/jails.html). Docker builds on top of these low-level primitives to offer developers a portable format and runtime environment that solves all 4 problems. Docker containers are small (and their transfer can be optimized with layers), they have basically zero memory and cpu overhead, @@ -56,17 +56,17 @@ A common problem for developers is the difficulty of managing all their applicat This is usually difficult for several reasons: * *Cross-platform dependencies*. Modern applications often depend on a combination of system libraries and binaries, language-specific packages, framework-specific modules, - internal components developed for another project, etc. These dependencies live in different "worlds" and require different tools - these tools typically don't work - well with each other, requiring awkward custom integrations. + internal components developed for another project, etc. These dependencies live in different "worlds" and require different tools - these tools typically don't work + well with each other, requiring awkward custom integrations. * Conflicting dependencies. Different applications may depend on different versions of the same dependency. Packaging tools handle these situations with various degrees of ease - - but they all handle them in different and incompatible ways, which again forces the developer to do extra work. + but they all handle them in different and incompatible ways, which again forces the developer to do extra work. - * Custom dependencies. A developer may need to prepare a custom version of his application's dependency. Some packaging systems can handle custom versions of a dependency, - others can't - and all of them handle it differently. + * Custom dependencies. A developer may need to prepare a custom version of their application's dependency. Some packaging systems can handle custom versions of a dependency, + others can't - and all of them handle it differently. -Docker solves dependency hell by giving the developer a simple way to express *all* his application's dependencies in one place, +Docker solves dependency hell by giving the developer a simple way to express *all* their application's dependencies in one place, and streamline the process of assembling them. If this makes you think of [XKCD 927](http://xkcd.com/927/), don't worry. Docker doesn't *replace* your favorite packaging systems. It simply orchestrates their use in a simple and repeatable way. How does it do that? With layers. diff --git a/components/engine/api.go b/components/engine/api.go index 8772994e47..c4a5222dc6 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -170,7 +170,7 @@ func getContainersExport(srv *Server, version float64, w http.ResponseWriter, r name := vars["name"] if err := srv.ContainerExport(name, w); err != nil { - utils.Debugf("%s", err.Error()) + utils.Debugf("%s", err) return err } return nil @@ -306,7 +306,7 @@ func postCommit(srv *Server, version float64, w http.ResponseWriter, r *http.Req } config := &Config{} if err := json.NewDecoder(r.Body).Decode(config); err != nil { - utils.Debugf("%s", err.Error()) + utils.Debugf("%s", err) } repo := r.Form.Get("repo") tag := r.Form.Get("tag") @@ -342,8 +342,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } sf := utils.NewStreamFormatter(version > 1.0) if image != "" { //pull - registry := r.Form.Get("registry") - if err := srv.ImagePull(image, tag, registry, w, sf, &auth.AuthConfig{}); err != nil { + if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { if sf.Used() { w.Write(sf.FormatError(err)) return nil @@ -426,7 +425,6 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http if err := parseForm(r); err != nil { return err } - registry := r.Form.Get("registry") if vars == nil { return fmt.Errorf("Missing parameter") @@ -436,7 +434,7 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http w.Header().Set("Content-Type", "application/json") } sf := utils.NewStreamFormatter(version > 1.0) - if err := srv.ImagePush(name, registry, w, sf, authConfig); err != nil { + if err := srv.ImagePush(name, w, sf, authConfig); err != nil { if sf.Used() { w.Write(sf.FormatError(err)) return nil @@ -880,7 +878,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) { localMethod := method localFct := fct f := func(w http.ResponseWriter, r *http.Request) { - utils.Debugf("Calling %s %s", localMethod, localRoute) + utils.Debugf("Calling %s %s from %s", localMethod, localRoute, r.RemoteAddr) if logging { log.Println(r.Method, r.RequestURI) diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 487394a549..d111f258c3 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -5,7 +5,6 @@ import ( "bufio" "bytes" "encoding/json" - "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/utils" "io" "net" @@ -41,44 +40,6 @@ func TestGetBoolParam(t *testing.T) { } } -func TestPostAuth(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) - - srv := &Server{ - runtime: runtime, - } - - r := httptest.NewRecorder() - - authConfig := &auth.AuthConfig{ - Username: "utest", - Password: "utest", - Email: "utest@yopmail.com", - } - - authConfigJSON, err := json.Marshal(authConfig) - if err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("POST", "/auth", bytes.NewReader(authConfigJSON)) - if err != nil { - t.Fatal(err) - } - - if err := postAuth(srv, APIVERSION, r, req, nil); err != nil { - t.Fatal(err) - } - - if r.Code != http.StatusOK && r.Code != 0 { - t.Fatalf("%d OK or 0 expected, received %d\n", http.StatusOK, r.Code) - } -} - func TestGetVersion(t *testing.T) { runtime, err := newTestRuntime() if err != nil { @@ -99,7 +60,7 @@ func TestGetVersion(t *testing.T) { t.Fatal(err) } if v.Version != VERSION { - t.Errorf("Excepted version %s, %s found", VERSION, v.Version) + t.Errorf("Expected version %s, %s found", VERSION, v.Version) } } @@ -129,7 +90,7 @@ func TestGetInfo(t *testing.T) { t.Fatal(err) } if infos.Images != len(initialImages) { - t.Errorf("Excepted images: %d, %d found", len(initialImages), infos.Images) + t.Errorf("Expected images: %d, %d found", len(initialImages), infos.Images) } } @@ -166,7 +127,7 @@ func TestGetImagesJSON(t *testing.T) { } if len(images) != len(initialImages) { - t.Errorf("Excepted %d image, %d found", len(initialImages), len(images)) + t.Errorf("Expected %d image, %d found", len(initialImages), len(images)) } found := false @@ -177,7 +138,7 @@ func TestGetImagesJSON(t *testing.T) { } } if !found { - t.Errorf("Excepted image %s, %+v found", unitTestImageName, images) + t.Errorf("Expected image %s, %+v found", unitTestImageName, images) } r2 := httptest.NewRecorder() @@ -204,7 +165,7 @@ func TestGetImagesJSON(t *testing.T) { } if len(images2) != len(initialImages) { - t.Errorf("Excepted %d image, %d found", len(initialImages), len(images2)) + t.Errorf("Expected %d image, %d found", len(initialImages), len(images2)) } found = false @@ -236,7 +197,7 @@ func TestGetImagesJSON(t *testing.T) { } if len(images3) != 0 { - t.Errorf("Excepted 0 image, %d found", len(images3)) + t.Errorf("Expected 0 image, %d found", len(images3)) } r4 := httptest.NewRecorder() @@ -282,38 +243,7 @@ func TestGetImagesViz(t *testing.T) { t.Fatal(err) } if line != "digraph docker {\n" { - t.Errorf("Excepted digraph docker {\n, %s found", line) - } -} - -func TestGetImagesSearch(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) - - srv := &Server{ - runtime: runtime, - } - - r := httptest.NewRecorder() - - req, err := http.NewRequest("GET", "/images/search?term=redis", nil) - if err != nil { - t.Fatal(err) - } - - if err := getImagesSearch(srv, APIVERSION, r, req, nil); err != nil { - t.Fatal(err) - } - - results := []APISearch{} - if err := json.Unmarshal(r.Body.Bytes(), &results); err != nil { - t.Fatal(err) - } - if len(results) < 2 { - t.Errorf("Excepted at least 2 lines, %d found", len(results)) + t.Errorf("Expected digraph docker {\n, %s found", line) } } @@ -337,7 +267,7 @@ func TestGetImagesHistory(t *testing.T) { t.Fatal(err) } if len(history) != 1 { - t.Errorf("Excepted 1 line, %d found", len(history)) + t.Errorf("Expected 1 line, %d found", len(history)) } } @@ -359,7 +289,7 @@ func TestGetImagesByName(t *testing.T) { if err := json.Unmarshal(r.Body.Bytes(), img); err != nil { t.Fatal(err) } - if img.ID != unitTestImageId { + if img.ID != unitTestImageID { t.Errorf("Error inspecting image") } } @@ -396,7 +326,7 @@ func TestGetContainersJSON(t *testing.T) { t.Fatal(err) } if len(containers) != 1 { - t.Fatalf("Excepted %d container, %d found", 1, len(containers)) + t.Fatalf("Expected %d container, %d found", 1, len(containers)) } if containers[0].ID != container.ID { t.Fatalf("Container ID mismatch. Expected: %s, received: %s\n", container.ID, containers[0].ID) @@ -1356,24 +1286,34 @@ func TestDeleteImages(t *testing.T) { } if len(images) != len(initialImages)+1 { - t.Errorf("Excepted %d images, %d found", len(initialImages)+1, len(images)) + t.Errorf("Expected %d images, %d found", len(initialImages)+1, len(images)) } - req, err := http.NewRequest("DELETE", "/images/test:test", nil) + req, err := http.NewRequest("DELETE", "/images/"+unitTestImageID, nil) if err != nil { t.Fatal(err) } r := httptest.NewRecorder() - if err := deleteImages(srv, APIVERSION, r, req, map[string]string{"name": "test:test"}); err != nil { + if err := deleteImages(srv, APIVERSION, r, req, map[string]string{"name": unitTestImageID}); err == nil { + t.Fatalf("Expected conflict error, got none") + } + + req2, err := http.NewRequest("DELETE", "/images/test:test", nil) + if err != nil { t.Fatal(err) } - if r.Code != http.StatusOK { + + r2 := httptest.NewRecorder() + if err := deleteImages(srv, APIVERSION, r2, req2, map[string]string{"name": "test:test"}); err != nil { + t.Fatal(err) + } + if r2.Code != http.StatusOK { t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code) } var outs []APIRmi - if err := json.Unmarshal(r.Body.Bytes(), &outs); err != nil { + if err := json.Unmarshal(r2.Body.Bytes(), &outs); err != nil { t.Fatal(err) } if len(outs) != 1 { @@ -1385,7 +1325,7 @@ func TestDeleteImages(t *testing.T) { } if len(images) != len(initialImages) { - t.Errorf("Excepted %d image, %d found", len(initialImages), len(images)) + t.Errorf("Expected %d image, %d found", len(initialImages), len(images)) } /* if c := runtime.Get(container.Id); c != nil { diff --git a/components/engine/archive.go b/components/engine/archive.go index 357fdd9a71..01af86006f 100644 --- a/components/engine/archive.go +++ b/components/engine/archive.go @@ -2,9 +2,7 @@ package docker import ( "archive/tar" - "bufio" "bytes" - "errors" "fmt" "github.com/dotcloud/docker/utils" "io" @@ -27,10 +25,6 @@ const ( ) func DetectCompression(source []byte) Compression { - for _, c := range source[:10] { - utils.Debugf("%x", c) - } - sourceLen := len(source) for compression, m := range map[Compression][]byte{ Bzip2: {0x42, 0x5A, 0x68}, @@ -111,17 +105,26 @@ func Untar(archive io.Reader, path string) error { if archive == nil { return fmt.Errorf("Empty archive") } - bufferedArchive := bufio.NewReaderSize(archive, 10) - buf, err := bufferedArchive.Peek(10) - if err != nil { - return err + + buf := make([]byte, 10) + totalN := 0 + for totalN < 10 { + if n, err := archive.Read(buf[totalN:]); err != nil { + if err == io.EOF { + return fmt.Errorf("Tarball too short") + } + return err + } else { + totalN += n + utils.Debugf("[tar autodetect] n: %d", n) + } } compression := DetectCompression(buf) utils.Debugf("Archive compression detected: %s", compression.Extension()) cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag()) - cmd.Stdin = bufferedArchive + cmd.Stdin = io.MultiReader(bytes.NewReader(buf), archive) // Hardcode locale environment for predictable outcome regardless of host configuration. // (see https://github.com/dotcloud/docker/issues/355) cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"} @@ -251,7 +254,7 @@ func CmdStream(cmd *exec.Cmd) (io.Reader, error) { } errText := <-errChan if err := cmd.Wait(); err != nil { - pipeW.CloseWithError(errors.New(err.Error() + ": " + string(errText))) + pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errText)) } else { pipeW.Close() } diff --git a/components/engine/archive_test.go b/components/engine/archive_test.go index bb4235ad5b..9a0a8e1b9e 100644 --- a/components/engine/archive_test.go +++ b/components/engine/archive_test.go @@ -16,7 +16,7 @@ func TestCmdStreamLargeStderr(t *testing.T) { cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello") out, err := CmdStream(cmd) if err != nil { - t.Fatalf("Failed to start command: " + err.Error()) + t.Fatalf("Failed to start command: %s", err) } errCh := make(chan error) go func() { @@ -26,7 +26,7 @@ func TestCmdStreamLargeStderr(t *testing.T) { select { case err := <-errCh: if err != nil { - t.Fatalf("Command should not have failed (err=%s...)", err.Error()[:100]) + t.Fatalf("Command should not have failed (err=%.100s...)", err) } case <-time.After(5 * time.Second): t.Fatalf("Command did not complete in 5 seconds; probable deadlock") @@ -37,12 +37,12 @@ func TestCmdStreamBad(t *testing.T) { badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1") out, err := CmdStream(badCmd) if err != nil { - t.Fatalf("Failed to start command: " + err.Error()) + t.Fatalf("Failed to start command: %s", err) } if output, err := ioutil.ReadAll(out); err == nil { t.Fatalf("Command should have failed") } else if err.Error() != "exit status 1: error couldn't reverse the phase pulser\n" { - t.Fatalf("Wrong error value (%s)", err.Error()) + t.Fatalf("Wrong error value (%s)", err) } else if s := string(output); s != "hello\n" { t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output) } diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index 205b9479f5..97df928b6b 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -15,8 +15,8 @@ import ( // Where we store the config file const CONFIGFILE = ".dockercfg" -// the registry server we want to login against -const INDEXSERVER = "https://index.docker.io/v1" +// Only used for user auth + account creation +const INDEXSERVER = "https://index.docker.io/v1/" //const INDEXSERVER = "http://indexstaging-docker.dotcloud.com/" @@ -41,9 +41,6 @@ func NewAuthConfig(username, password, email, rootPath string) *AuthConfig { } func IndexServerAddress() string { - if os.Getenv("DOCKER_INDEX_URL") != "" { - return os.Getenv("DOCKER_INDEX_URL") + "/v1" - } return INDEXSERVER } @@ -132,7 +129,7 @@ func Login(authConfig *AuthConfig, store bool) (string, error) { // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := http.Post(IndexServerAddress()+"/users/", "application/json; charset=utf-8", b) + req1, err := http.Post(IndexServerAddress()+"users/", "application/json; charset=utf-8", b) if err != nil { return "", fmt.Errorf("Server Error: %s", err) } @@ -152,7 +149,7 @@ func Login(authConfig *AuthConfig, store bool) (string, error) { "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { - req, err := http.NewRequest("GET", IndexServerAddress()+"/users/", nil) + req, err := http.NewRequest("GET", IndexServerAddress()+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { diff --git a/components/engine/auth/auth_test.go b/components/engine/auth/auth_test.go index 8e7adaef8d..d036de7364 100644 --- a/components/engine/auth/auth_test.go +++ b/components/engine/auth/auth_test.go @@ -68,6 +68,6 @@ func TestCreateAccount(t *testing.T) { expectedError := "Login: Account is not Active" if !strings.Contains(err.Error(), expectedError) { - t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err.Error()) + t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err) } } diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index cdc2a6b041..38e6c330d6 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -29,6 +29,7 @@ type buildFile struct { config *Config context string + lastContainer *Container tmpContainers map[string]struct{} tmpImages map[string]struct{} @@ -51,20 +52,10 @@ func (b *buildFile) CmdFrom(name string) error { image, err := b.runtime.repositories.LookupImage(name) if err != nil { if b.runtime.graph.IsNotExist(err) { - - var tag, remote string - if strings.Contains(name, ":") { - remoteParts := strings.Split(name, ":") - tag = remoteParts[1] - remote = remoteParts[0] - } else { - remote = name - } - - if err := b.srv.ImagePull(remote, tag, "", b.out, utils.NewStreamFormatter(false), nil); err != nil { + remote, tag := utils.ParseRepositoryTag(name) + if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil); err != nil { return err } - image, err = b.runtime.repositories.LookupImage(name) if err != nil { return err @@ -182,6 +173,27 @@ func (b *buildFile) CmdEntrypoint(args string) error { return nil } +func (b *buildFile) CmdVolume(args string) error { + if args == "" { + return fmt.Errorf("Volume cannot be empty") + } + + var volume []string + if err := json.Unmarshal([]byte(args), &volume); err != nil { + volume = []string{args} + } + if b.config.Volumes == nil { + b.config.Volumes = NewPathOpts() + } + for _, v := range volume { + b.config.Volumes[v] = struct{}{} + } + if err := b.commit("", b.config.Cmd, fmt.Sprintf("VOLUME %s", args)); err != nil { + return err + } + return nil +} + func (b *buildFile) addRemote(container *Container, orig, dest string) error { file, err := utils.Download(orig, ioutil.Discard) if err != nil { @@ -242,6 +254,7 @@ func (b *buildFile) CmdAdd(args string) error { return err } b.tmpContainers[container.ID] = struct{}{} + b.lastContainer = container if err := container.EnsureMounted(); err != nil { return err @@ -277,8 +290,13 @@ func (b *buildFile) run() (string, error) { return "", err } b.tmpContainers[c.ID] = struct{}{} + b.lastContainer = c fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(c.ID)) + // override the entry point that may have been picked up from the base image + c.Path = b.config.Cmd[0] + c.Args = b.config.Cmd[1:] + //start the container hostConfig := &HostConfig{} if err := c.Start(hostConfig); err != nil { @@ -319,6 +337,7 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { return err } b.tmpContainers[container.ID] = struct{}{} + b.lastContainer = container fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID)) id = container.ID if err := container.EnsureMounted(); err != nil { @@ -346,6 +365,29 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { } func (b *buildFile) Build(context io.Reader) (string, error) { + defer func() { + // If we have an error and a container, the display the logs + if b.lastContainer != nil { + fmt.Fprintf(b.out, "******** Logs from last container (%s) *******\n", b.lastContainer.ShortID()) + + cLog, err := b.lastContainer.ReadLog("stdout") + if err != nil { + utils.Debugf("Error reading logs (stdout): %s", err) + } + if _, err := io.Copy(b.out, cLog); err != nil { + utils.Debugf("Error streaming logs (stdout): %s", err) + } + cLog, err = b.lastContainer.ReadLog("stderr") + if err != nil { + utils.Debugf("Error reading logs (stderr): %s", err) + } + if _, err := io.Copy(b.out, cLog); err != nil { + utils.Debugf("Error streaming logs (stderr): %s", err) + } + fmt.Fprintf(b.out, "************* End of logs for %s *************\n", b.lastContainer.ShortID()) + } + }() + // FIXME: @creack any reason for using /tmp instead of ""? // FIXME: @creack "name" is a terrible variable name name, err := ioutil.TempDir("/tmp", "docker-build") @@ -398,6 +440,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) { return "", ret.(error) } + b.lastContainer = nil fmt.Fprintf(b.out, " ---> %v\n", utils.TruncateID(b.image)) } if b.image != "" { diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 8dc0410623..9250f73765 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -3,14 +3,13 @@ package docker import ( "fmt" "io/ioutil" - "sync" "testing" ) // mkTestContext generates a build context from the contents of the provided dockerfile. // This context is suitable for use as an argument to BuildFile.Build() func mkTestContext(dockerfile string, files [][2]string, t *testing.T) Archive { - context, err := mkBuildContext(fmt.Sprintf(dockerfile, unitTestImageId), files) + context, err := mkBuildContext(fmt.Sprintf(dockerfile, unitTestImageID), files) if err != nil { t.Fatal(err) } @@ -27,7 +26,7 @@ type testContextTemplate struct { // A table of all the contexts to build and test. // A new docker runtime will be created and torn down for each context. -var testContexts []testContextTemplate = []testContextTemplate{ +var testContexts = []testContextTemplate{ { ` from %s @@ -85,9 +84,18 @@ run [ "$FOO" = "BAR" ] { ` -from docker-ut +from %s ENTRYPOINT /bin/echo CMD Hello world +`, + nil, + }, + + { + ` +from %s +VOLUME /test +CMD Hello world `, nil, }, @@ -105,7 +113,6 @@ func TestBuild(t *testing.T) { srv := &Server{ runtime: runtime, - lock: &sync.Mutex{}, pullingPool: make(map[string]struct{}), pushingPool: make(map[string]struct{}), } @@ -116,3 +123,39 @@ func TestBuild(t *testing.T) { } } } + +func TestVolume(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + buildfile := NewBuildFile(srv, ioutil.Discard) + imgId, err := buildfile.Build(mkTestContext(` +from %s +VOLUME /test +CMD Hello world +`, nil, t)) + if err != nil { + t.Fatal(err) + } + img, err := srv.ImageInspect(imgId) + if err != nil { + t.Fatal(err) + } + if len(img.Config.Volumes) == 0 { + t.Fail() + } + for key, _ := range img.Config.Volumes { + if key != "/test" { + t.Fail() + } + } +} diff --git a/components/engine/commands.go b/components/engine/commands.go index e275bdd671..4f894893fe 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -19,7 +19,6 @@ import ( "os/signal" "path/filepath" "reflect" - "regexp" "strconv" "strings" "syscall" @@ -73,7 +72,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error { return nil } } - help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=[tcp://%s:%d]: tcp://host:port to bind/connect to or unix://path/to/socker to use\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTHTTPHOST, DEFAULTHTTPPORT) + help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=[tcp://%s:%d]: tcp://host:port to bind/connect to or unix://path/to/socket to use\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTHTTPHOST, DEFAULTHTTPPORT) for _, command := range [][]string{ {"attach", "Attach to a running container"}, {"build", "Build a container from a Dockerfile"}, @@ -730,7 +729,6 @@ func (cli *DockerCli) CmdImport(args ...string) error { func (cli *DockerCli) CmdPush(args ...string) error { cmd := Subcmd("push", "[OPTION] NAME", "Push an image or a repository to the registry") - registry := cmd.String("registry", "", "Registry host to push the image to") if err := cmd.Parse(args); err != nil { return nil } @@ -741,28 +739,16 @@ func (cli *DockerCli) CmdPush(args ...string) error { return nil } - if err := cli.checkIfLogged(*registry == "", "push"); err != nil { + if err := cli.checkIfLogged("push"); err != nil { return err } - if *registry == "" { - // If we're not using a custom registry, we know the restrictions - // applied to repository names and can warn the user in advance. - // Custom repositories can have different rules, and we must also - // allow pushing by image ID. - if len(strings.SplitN(name, "/", 2)) == 1 { - return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.authConfig.Username, name) - } - - nameParts := strings.SplitN(name, "/", 2) - validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) - if !validNamespace.MatchString(nameParts[0]) { - return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0]) - } - validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) - if !validRepo.MatchString(nameParts[1]) { - return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1]) - } + // If we're not using a custom registry, we know the restrictions + // applied to repository names and can warn the user in advance. + // Custom repositories can have different rules, and we must also + // allow pushing by image ID. + if len(strings.SplitN(name, "/", 2)) == 1 { + return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.authConfig.Username, name) } buf, err := json.Marshal(cli.authConfig) @@ -771,7 +757,6 @@ func (cli *DockerCli) CmdPush(args ...string) error { } v := url.Values{} - v.Set("registry", *registry) if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out); err != nil { return err } @@ -781,7 +766,6 @@ func (cli *DockerCli) CmdPush(args ...string) error { func (cli *DockerCli) CmdPull(args ...string) error { cmd := Subcmd("pull", "NAME", "Pull an image or a repository from the registry") tag := cmd.String("t", "", "Download tagged image in repository") - registry := cmd.String("registry", "", "Registry to download from. Necessary if image is pulled by ID") if err := cmd.Parse(args); err != nil { return nil } @@ -791,17 +775,12 @@ func (cli *DockerCli) CmdPull(args ...string) error { return nil } - remote := cmd.Arg(0) - if strings.Contains(remote, ":") { - remoteParts := strings.Split(remote, ":") - tag = &remoteParts[1] - remote = remoteParts[0] - } + remote, parsedTag := utils.ParseRepositoryTag(cmd.Arg(0)) + *tag = parsedTag v := url.Values{} v.Set("fromImage", remote) v.Set("tag", *tag) - v.Set("registry", *registry) if err := cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.out); err != nil { return err @@ -1127,6 +1106,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error { func (cli *DockerCli) CmdSearch(args ...string) error { cmd := Subcmd("search", "NAME", "Search the docker index for images") + noTrunc := cmd.Bool("notrunc", false, "Don't truncate output") if err := cmd.Parse(args); err != nil { return nil } @@ -1148,13 +1128,19 @@ func (cli *DockerCli) CmdSearch(args ...string) error { return err } fmt.Fprintf(cli.out, "Found %d results matching your query (\"%s\")\n", len(outs), cmd.Arg(0)) - w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) + w := tabwriter.NewWriter(cli.out, 33, 1, 3, ' ', 0) fmt.Fprintf(w, "NAME\tDESCRIPTION\n") + _, width := cli.getTtySize() + if width == 0 { + width = 45 + } else { + width = width - 33 //remote the first column + } for _, out := range outs { desc := strings.Replace(out.Description, "\n", " ", -1) desc = strings.Replace(desc, "\r", " ", -1) - if len(desc) > 45 { - desc = utils.Trunc(desc, 42) + "..." + if !*noTrunc && len(desc) > width { + desc = utils.Trunc(desc, width-3) + "..." } fmt.Fprintf(w, "%s\t%s\n", out.Name, desc) } @@ -1265,7 +1251,9 @@ func (cli *DockerCli) CmdRun(args ...string) error { //if image not found try to pull it if statusCode == 404 { v := url.Values{} - v.Set("fromImage", config.Image) + repos, tag := utils.ParseRepositoryTag(config.Image) + v.Set("fromImage", repos) + v.Set("tag", tag) err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err) if err != nil { return err @@ -1286,7 +1274,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { } for _, warning := range runResult.Warnings { - fmt.Fprintln(cli.err, "WARNING: ", warning) + fmt.Fprintf(cli.err, "WARNING: %s\n", warning) } //start the container @@ -1338,9 +1326,9 @@ func (cli *DockerCli) CmdRun(args ...string) error { return nil } -func (cli *DockerCli) checkIfLogged(condition bool, action string) error { +func (cli *DockerCli) checkIfLogged(action string) error { // If condition AND the login failed - if condition && cli.authConfig.Username == "" { + if cli.authConfig.Username == "" { if err := cli.CmdLogin(""); err != nil { return err } @@ -1528,17 +1516,28 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea } -func (cli *DockerCli) resizeTty(id string) { +func (cli *DockerCli) getTtySize() (int, int) { if !cli.isTerminal { - return + return 0, 0 } ws, err := term.GetWinsize(cli.terminalFd) if err != nil { utils.Debugf("Error getting size: %s", err) + if ws == nil { + return 0, 0 + } + } + return int(ws.Height), int(ws.Width) +} + +func (cli *DockerCli) resizeTty(id string) { + height, width := cli.getTtySize() + if height == 0 && width == 0 { + return } v := url.Values{} - v.Set("h", strconv.Itoa(int(ws.Height))) - v.Set("w", strconv.Itoa(int(ws.Width))) + v.Set("h", strconv.Itoa(height)) + v.Set("w", strconv.Itoa(width)) if _, _, err := cli.call("POST", "/containers/"+id+"/resize?"+v.Encode(), nil); err != nil { utils.Debugf("Error resize: %s", err) } @@ -1574,7 +1573,7 @@ func Subcmd(name, signature, description string) *flag.FlagSet { func NewDockerCli(in io.ReadCloser, out, err io.Writer, proto, addr string) *DockerCli { var ( - isTerminal bool = false + isTerminal = false terminalFd uintptr ) diff --git a/components/engine/commands_test.go b/components/engine/commands_test.go index 87c4c02a52..fe1c51a2a2 100644 --- a/components/engine/commands_test.go +++ b/components/engine/commands_test.go @@ -132,17 +132,18 @@ func TestImages(t *testing.T) { } */ + // TestRunHostname checks that 'docker run -h' correctly sets a custom hostname func TestRunHostname(t *testing.T) { stdout, stdoutPipe := io.Pipe() - cli := NewDockerCli(nil, stdoutPipe, nil, testDaemonProto, testDaemonAddr) + cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalRuntime) c := make(chan struct{}) go func() { defer close(c) - if err := cli.CmdRun("-h", "foobar", unitTestImageId, "hostname"); err != nil { + if err := cli.CmdRun("-h", "foobar", unitTestImageID, "hostname"); err != nil { t.Fatal(err) } }() @@ -329,13 +330,13 @@ func TestRunAttachStdin(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := NewDockerCli(stdin, stdoutPipe, nil, testDaemonProto, testDaemonAddr) + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalRuntime) ch := make(chan struct{}) go func() { defer close(ch) - cli.CmdRun("-i", "-a", "stdin", unitTestImageId, "sh", "-c", "echo hello && cat") + cli.CmdRun("-i", "-a", "stdin", unitTestImageID, "sh", "-c", "echo hello && cat") }() // Send input to the command, close stdin diff --git a/components/engine/container.go b/components/engine/container.go index 508f39c0ef..48661a3098 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -202,20 +202,25 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, return config, hostConfig, cmd, nil } +type PortMapping map[string]string + type NetworkSettings struct { IPAddress string IPPrefixLen int Gateway string Bridge string - PortMapping map[string]string + PortMapping map[string]PortMapping } // String returns a human-readable description of the port mapping defined in the settings func (settings *NetworkSettings) PortMappingHuman() string { var mapping []string - for private, public := range settings.PortMapping { + for private, public := range settings.PortMapping["Tcp"] { mapping = append(mapping, fmt.Sprintf("%s->%s", public, private)) } + for private, public := range settings.PortMapping["Udp"] { + mapping = append(mapping, fmt.Sprintf("%s->%s/udp", public, private)) + } sort.Strings(mapping) return strings.Join(mapping, ", ") } @@ -466,8 +471,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s } func (container *Container) Start(hostConfig *HostConfig) error { - container.State.lock() - defer container.State.unlock() + container.State.Lock() + defer container.State.Unlock() if container.State.Running { return fmt.Errorf("The container %s is already running.", container.ID) @@ -494,7 +499,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { // Create the requested bind mounts binds := make(map[string]BindMap) // Define illegal container destinations - illegal_dsts := []string{"/", "."} + illegalDsts := []string{"/", "."} for _, bind := range hostConfig.Binds { // FIXME: factorize bind parsing in parseBind @@ -513,7 +518,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { } // Bail if trying to mount to an illegal destination - for _, illegal := range illegal_dsts { + for _, illegal := range illegalDsts { if dst == illegal { return fmt.Errorf("Illegal bind destination: %s", dst) } @@ -688,14 +693,18 @@ func (container *Container) allocateNetwork() error { if err != nil { return err } - container.NetworkSettings.PortMapping = make(map[string]string) + container.NetworkSettings.PortMapping = make(map[string]PortMapping) + container.NetworkSettings.PortMapping["Tcp"] = make(PortMapping) + container.NetworkSettings.PortMapping["Udp"] = make(PortMapping) for _, spec := range container.Config.PortSpecs { nat, err := iface.AllocatePort(spec) if err != nil { iface.Release() return err } - container.NetworkSettings.PortMapping[strconv.Itoa(nat.Backend)] = strconv.Itoa(nat.Frontend) + proto := strings.Title(nat.Proto) + backend, frontend := strconv.Itoa(nat.Backend), strconv.Itoa(nat.Frontend) + container.NetworkSettings.PortMapping[proto][backend] = frontend } container.network = iface container.NetworkSettings.Bridge = container.runtime.networkManager.bridgeIface @@ -821,8 +830,8 @@ func (container *Container) kill() error { } func (container *Container) Kill() error { - container.State.lock() - defer container.State.unlock() + container.State.Lock() + defer container.State.Unlock() if !container.State.Running { return nil } @@ -830,8 +839,8 @@ func (container *Container) Kill() error { } func (container *Container) Stop(seconds int) error { - container.State.lock() - defer container.State.unlock() + container.State.Lock() + defer container.State.Unlock() if !container.State.Running { return nil } diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index c508d8905e..fb7c465369 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -37,7 +37,7 @@ func main() { flag.Var(&flHosts, "H", "tcp://host:port to bind/connect to or unix://path/to/socket to use") flag.Parse() if len(flHosts) > 1 { - flHosts = flHosts[1:len(flHosts)] //trick to display a nice defaul value in the usage + flHosts = flHosts[1:] //trick to display a nice defaul value in the usage } for i, flHost := range flHosts { flHosts[i] = utils.ParseHost(docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT, flHost) diff --git a/components/engine/docs/sources/contributing/devenvironment.rst b/components/engine/docs/sources/contributing/devenvironment.rst index 1f39364cb1..6f5d6c1dc1 100644 --- a/components/engine/docs/sources/contributing/devenvironment.rst +++ b/components/engine/docs/sources/contributing/devenvironment.rst @@ -5,53 +5,52 @@ Setting Up a Dev Environment ============================ -Instructions that have been verified to work on Ubuntu Precise 12.04 (LTS) (64-bit), +To make it easier to contribute to Docker, we provide a standard development environment. It is important that +the same environment be used for all tests, builds and releases. The standard development environment defines +all build dependencies: system libraries and binaries, go environment, go dependencies, etc. -Dependencies ------------- +Step 1: install docker +---------------------- -**Linux kernel 3.8** +Docker's build environment itself is a docker container, so the first step is to install docker on your system. -Due to a bug in LXC docker works best on the 3.8 kernel. Precise comes with a 3.2 kernel, so we need to upgrade it. The kernel we install comes with AUFS built in. +You can follow the `install instructions most relevant to your system `. +Make sure you have a working, up-to-date docker installation, then continue to the next step. -.. code-block:: bash +Step 2: check out the source +---------------------------- - # install the backported kernel - sudo apt-get update && sudo apt-get install linux-image-generic-lts-raring +:: - # reboot - sudo reboot - - -Installation ------------- - -.. code-block:: bash - - sudo apt-get install python-software-properties - sudo add-apt-repository ppa:gophers/go - sudo apt-get update - sudo apt-get -y install lxc xz-utils curl golang-stable git aufs-tools - - export GOPATH=~/go/ - export PATH=$GOPATH/bin:$PATH - - mkdir -p $GOPATH/src/github.com/dotcloud - cd $GOPATH/src/github.com/dotcloud - git clone git://github.com/dotcloud/docker.git + git clone http://git@github.com/dotcloud/docker cd docker - go get -v github.com/dotcloud/docker/... - go install -v github.com/dotcloud/docker/... + +Step 3: build +------------- + +When you are ready to build docker, run this command: + +:: + + docker build -t docker . + +This will build the revision currently checked out in the repository. Feel free to check out the version +of your choice. + +If the build is successful, congratulations! You have produced a clean build of docker, neatly encapsulated +in a standard build environment. + +You can run an interactive session in the newly built container: + +:: + docker run -i -t docker bash -Then run the docker daemon, +To extract the binaries from the container: -.. code-block:: bash +:: + docker run docker sh -c 'cat $(which docker)' > docker-build && chmod +x docker-build - sudo $GOPATH/bin/docker -d - - -Run the ``go install`` command (above) to recompile docker. diff --git a/components/engine/docs/sources/use/builder.rst b/components/engine/docs/sources/use/builder.rst index 6a12beadf8..ab416281b4 100644 --- a/components/engine/docs/sources/use/builder.rst +++ b/components/engine/docs/sources/use/builder.rst @@ -160,6 +160,13 @@ files and directories are created with mode 0700, uid and gid 0. The `ENTRYPOINT` instruction adds an entry command that will not be overwritten when arguments are passed to docker run, unlike the behavior of `CMD`. This allows arguments to be passed to the entrypoint. i.e. `docker run -d` will pass the "-d" argument to the entrypoint. +2.9 VOLUME +---------- + + ``VOLUME ["/data"]`` + +The `VOLUME` instruction will add one or more new volumes to any container created from the image. + 3. Dockerfile Examples ====================== diff --git a/components/engine/graph.go b/components/engine/graph.go index 0bf7eccdbe..42d1bdbd4c 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -162,7 +162,7 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { // The archive is stored on disk and will be automatically deleted as soon as has been read. // If output is not nil, a human-readable progress bar will be written to it. // FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives? -func (graph *Graph) TempLayerArchive(id string, compression Compression, output io.Writer) (*TempArchive, error) { +func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *utils.StreamFormatter, output io.Writer) (*TempArchive, error) { image, err := graph.Get(id) if err != nil { return nil, err @@ -175,7 +175,6 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, output if err != nil { return nil, err } - sf := utils.NewStreamFormatter(false) return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("Buffering to disk", "%v/%v (%v)"), sf), tmp.Root) } diff --git a/components/engine/hack/dockerbuilder/Dockerfile b/components/engine/hack/dockerbuilder/Dockerfile index f377f0be76..60cd93b17a 100644 --- a/components/engine/hack/dockerbuilder/Dockerfile +++ b/components/engine/hack/dockerbuilder/Dockerfile @@ -5,8 +5,11 @@ # AUTHOR Solomon Hykes # Daniel Mizyrycki # BUILD_CMD docker build -t dockerbuilder . -# RUN_CMD docker run -e AWS_ID="$AWS_ID" -e AWS_KEY="$AWS_KEY" -e GPG_KEY="$GPG_KEY" dockerbuilder +# RUN_CMD docker run -e AWS_ID="$AWS_ID" -e AWS_KEY="$AWS_KEY" -e GPG_KEY="$GPG_KEY" -e PUBLISH_PPA="$PUBLISH_PPA" dockerbuilder # +# ENV_VARIABLES AWS_ID, AWS_KEY: S3 credentials for uploading Docker binary and tarball +# GPG_KEY: Signing key for docker package +# PUBLISH_PPA: 0 for staging release, 1 for production release # from ubuntu:12.04 maintainer Solomon Hykes diff --git a/components/engine/image.go b/components/engine/image.go index bb6598b262..e1b1ac0418 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -94,9 +94,12 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { } // If layerData is not nil, unpack it into the new layer if layerData != nil { + start := time.Now() + utils.Debugf("Start untar layer") if err := Untar(layerData, layer); err != nil { return err } + utils.Debugf("Untar time: %vs\n", time.Now().Sub(start).Seconds()) } return StoreSize(img, root) diff --git a/components/engine/network.go b/components/engine/network.go index 37037dd14a..0f98c899f1 100644 --- a/components/engine/network.go +++ b/components/engine/network.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "github.com/dotcloud/docker/utils" - "io" "log" "net" "os/exec" @@ -183,8 +182,10 @@ func getIfaceAddr(name string) (net.Addr, error) { // up iptables rules. // It keeps track of all mappings and is able to unmap at will type PortMapper struct { - mapping map[int]net.TCPAddr - proxies map[int]net.Listener + tcpMapping map[int]*net.TCPAddr + tcpProxies map[int]Proxy + udpMapping map[int]*net.UDPAddr + udpProxies map[int]Proxy } func (mapper *PortMapper) cleanup() error { @@ -197,8 +198,10 @@ func (mapper *PortMapper) cleanup() error { iptables("-t", "nat", "-D", "OUTPUT", "-j", "DOCKER") iptables("-t", "nat", "-F", "DOCKER") iptables("-t", "nat", "-X", "DOCKER") - mapper.mapping = make(map[int]net.TCPAddr) - mapper.proxies = make(map[int]net.Listener) + mapper.tcpMapping = make(map[int]*net.TCPAddr) + mapper.tcpProxies = make(map[int]Proxy) + mapper.udpMapping = make(map[int]*net.UDPAddr) + mapper.udpProxies = make(map[int]Proxy) return nil } @@ -215,76 +218,72 @@ func (mapper *PortMapper) setup() error { return nil } -func (mapper *PortMapper) iptablesForward(rule string, port int, dest net.TCPAddr) error { - return iptables("-t", "nat", rule, "DOCKER", "-p", "tcp", "--dport", strconv.Itoa(port), - "-j", "DNAT", "--to-destination", net.JoinHostPort(dest.IP.String(), strconv.Itoa(dest.Port))) +func (mapper *PortMapper) iptablesForward(rule string, port int, proto string, dest_addr string, dest_port int) error { + return iptables("-t", "nat", rule, "DOCKER", "-p", proto, "--dport", strconv.Itoa(port), + "-j", "DNAT", "--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port))) } -func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error { - if err := mapper.iptablesForward("-A", port, dest); err != nil { - return err +func (mapper *PortMapper) Map(port int, backendAddr net.Addr) error { + if _, isTCP := backendAddr.(*net.TCPAddr); isTCP { + backendPort := backendAddr.(*net.TCPAddr).Port + backendIP := backendAddr.(*net.TCPAddr).IP + if err := mapper.iptablesForward("-A", port, "tcp", backendIP.String(), backendPort); err != nil { + return err + } + mapper.tcpMapping[port] = backendAddr.(*net.TCPAddr) + proxy, err := NewProxy(&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}, backendAddr) + if err != nil { + mapper.Unmap(port, "tcp") + return err + } + mapper.tcpProxies[port] = proxy + go proxy.Run() + } else { + backendPort := backendAddr.(*net.UDPAddr).Port + backendIP := backendAddr.(*net.UDPAddr).IP + if err := mapper.iptablesForward("-A", port, "udp", backendIP.String(), backendPort); err != nil { + return err + } + mapper.udpMapping[port] = backendAddr.(*net.UDPAddr) + proxy, err := NewProxy(&net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}, backendAddr) + if err != nil { + mapper.Unmap(port, "udp") + return err + } + mapper.udpProxies[port] = proxy + go proxy.Run() } - - mapper.mapping[port] = dest - listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) - if err != nil { - mapper.Unmap(port) - return err - } - mapper.proxies[port] = listener - go proxy(listener, "tcp", dest.String()) return nil } -// proxy listens for socket connections on `listener`, and forwards them unmodified -// to `proto:address` -func proxy(listener net.Listener, proto, address string) error { - utils.Debugf("proxying to %s:%s", proto, address) - defer utils.Debugf("Done proxying to %s:%s", proto, address) - for { - utils.Debugf("Listening on %s", listener) - src, err := listener.Accept() - if err != nil { +func (mapper *PortMapper) Unmap(port int, proto string) error { + if proto == "tcp" { + backendAddr, ok := mapper.tcpMapping[port] + if !ok { + return fmt.Errorf("Port tcp/%v is not mapped", port) + } + if proxy, exists := mapper.tcpProxies[port]; exists { + proxy.Close() + delete(mapper.tcpProxies, port) + } + if err := mapper.iptablesForward("-D", port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil { return err } - utils.Debugf("Connecting to %s:%s", proto, address) - dst, err := net.Dial(proto, address) - if err != nil { - log.Printf("Error connecting to %s:%s: %s", proto, address, err) - src.Close() - continue + delete(mapper.tcpMapping, port) + } else { + backendAddr, ok := mapper.udpMapping[port] + if !ok { + return fmt.Errorf("Port udp/%v is not mapped", port) } - utils.Debugf("Connected to backend, splicing") - splice(src, dst) + if proxy, exists := mapper.udpProxies[port]; exists { + proxy.Close() + delete(mapper.udpProxies, port) + } + if err := mapper.iptablesForward("-D", port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil { + return err + } + delete(mapper.udpMapping, port) } -} - -func halfSplice(dst, src net.Conn) error { - _, err := io.Copy(dst, src) - // FIXME: on EOF from a tcp connection, pass WriteClose() - dst.Close() - src.Close() - return err -} - -func splice(a, b net.Conn) { - go halfSplice(a, b) - go halfSplice(b, a) -} - -func (mapper *PortMapper) Unmap(port int) error { - dest, ok := mapper.mapping[port] - if !ok { - return errors.New("Port is not mapped") - } - if proxy, exists := mapper.proxies[port]; exists { - proxy.Close() - delete(mapper.proxies, port) - } - if err := mapper.iptablesForward("-D", port, dest); err != nil { - return err - } - delete(mapper.mapping, port) return nil } @@ -301,9 +300,9 @@ func newPortMapper() (*PortMapper, error) { // Port allocator: Atomatically allocate and release networking ports type PortAllocator struct { + sync.Mutex inUse map[int]struct{} fountain chan (int) - lock sync.Mutex } func (alloc *PortAllocator) runFountain() { @@ -317,9 +316,9 @@ func (alloc *PortAllocator) runFountain() { // FIXME: Release can no longer fail, change its prototype to reflect that. func (alloc *PortAllocator) Release(port int) error { utils.Debugf("Releasing %d", port) - alloc.lock.Lock() + alloc.Lock() delete(alloc.inUse, port) - alloc.lock.Unlock() + alloc.Unlock() return nil } @@ -334,8 +333,8 @@ func (alloc *PortAllocator) Acquire(port int) (int, error) { } return -1, fmt.Errorf("Port generator ended unexpectedly") } - alloc.lock.Lock() - defer alloc.lock.Unlock() + alloc.Lock() + defer alloc.Unlock() if _, inUse := alloc.inUse[port]; inUse { return -1, fmt.Errorf("Port already in use: %d", port) } @@ -453,7 +452,7 @@ type NetworkInterface struct { Gateway net.IP manager *NetworkManager - extPorts []int + extPorts []*Nat } // Allocate an external TCP port and map it to the interface @@ -462,17 +461,32 @@ func (iface *NetworkInterface) AllocatePort(spec string) (*Nat, error) { if err != nil { return nil, err } - // Allocate a random port if Frontend==0 - extPort, err := iface.manager.portAllocator.Acquire(nat.Frontend) - if err != nil { - return nil, err + + if nat.Proto == "tcp" { + extPort, err := iface.manager.tcpPortAllocator.Acquire(nat.Frontend) + if err != nil { + return nil, err + } + backend := &net.TCPAddr{IP: iface.IPNet.IP, Port: nat.Backend} + if err := iface.manager.portMapper.Map(extPort, backend); err != nil { + iface.manager.tcpPortAllocator.Release(extPort) + return nil, err + } + nat.Frontend = extPort + } else { + extPort, err := iface.manager.udpPortAllocator.Acquire(nat.Frontend) + if err != nil { + return nil, err + } + backend := &net.UDPAddr{IP: iface.IPNet.IP, Port: nat.Backend} + if err := iface.manager.portMapper.Map(extPort, backend); err != nil { + iface.manager.udpPortAllocator.Release(extPort) + return nil, err + } + nat.Frontend = extPort } - nat.Frontend = extPort - if err := iface.manager.portMapper.Map(nat.Frontend, net.TCPAddr{IP: iface.IPNet.IP, Port: nat.Backend}); err != nil { - iface.manager.portAllocator.Release(nat.Frontend) - return nil, err - } - iface.extPorts = append(iface.extPorts, nat.Frontend) + iface.extPorts = append(iface.extPorts, nat) + return nat, nil } @@ -485,6 +499,21 @@ type Nat struct { func parseNat(spec string) (*Nat, error) { var nat Nat + if strings.Contains(spec, "/") { + specParts := strings.Split(spec, "/") + if len(specParts) != 2 { + return nil, fmt.Errorf("Invalid port format.") + } + proto := specParts[1] + spec = specParts[0] + if proto != "tcp" && proto != "udp" { + return nil, fmt.Errorf("Invalid port format: unknown protocol %v.", proto) + } + nat.Proto = proto + } else { + nat.Proto = "tcp" + } + if strings.Contains(spec, ":") { specParts := strings.Split(spec, ":") if len(specParts) != 2 { @@ -517,20 +546,24 @@ func parseNat(spec string) (*Nat, error) { } nat.Backend = int(port) } - nat.Proto = "tcp" + return &nat, nil } // Release: Network cleanup - release all resources func (iface *NetworkInterface) Release() { - for _, port := range iface.extPorts { - if err := iface.manager.portMapper.Unmap(port); err != nil { - log.Printf("Unable to unmap port %v: %v", port, err) + for _, nat := range iface.extPorts { + utils.Debugf("Unmaping %v/%v", nat.Proto, nat.Frontend) + if err := iface.manager.portMapper.Unmap(nat.Frontend, nat.Proto); err != nil { + log.Printf("Unable to unmap port %v/%v: %v", nat.Proto, nat.Frontend, err) } - if err := iface.manager.portAllocator.Release(port); err != nil { - log.Printf("Unable to release port %v: %v", port, err) + if nat.Proto == "tcp" { + if err := iface.manager.tcpPortAllocator.Release(nat.Frontend); err != nil { + log.Printf("Unable to release port tcp/%v: %v", nat.Frontend, err) + } + } else if err := iface.manager.udpPortAllocator.Release(nat.Frontend); err != nil { + log.Printf("Unable to release port udp/%v: %v", nat.Frontend, err) } - } iface.manager.ipAllocator.Release(iface.IPNet.IP) @@ -542,9 +575,10 @@ type NetworkManager struct { bridgeIface string bridgeNetwork *net.IPNet - ipAllocator *IPAllocator - portAllocator *PortAllocator - portMapper *PortMapper + ipAllocator *IPAllocator + tcpPortAllocator *PortAllocator + udpPortAllocator *PortAllocator + portMapper *PortMapper } // Allocate a network interface @@ -577,7 +611,11 @@ func newNetworkManager(bridgeIface string) (*NetworkManager, error) { ipAllocator := newIPAllocator(network) - portAllocator, err := newPortAllocator() + tcpPortAllocator, err := newPortAllocator() + if err != nil { + return nil, err + } + udpPortAllocator, err := newPortAllocator() if err != nil { return nil, err } @@ -588,11 +626,12 @@ func newNetworkManager(bridgeIface string) (*NetworkManager, error) { } manager := &NetworkManager{ - bridgeIface: bridgeIface, - bridgeNetwork: network, - ipAllocator: ipAllocator, - portAllocator: portAllocator, - portMapper: portMapper, + bridgeIface: bridgeIface, + bridgeNetwork: network, + ipAllocator: ipAllocator, + tcpPortAllocator: tcpPortAllocator, + udpPortAllocator: udpPortAllocator, + portMapper: portMapper, } return manager, nil } diff --git a/components/engine/network_proxy.go b/components/engine/network_proxy.go new file mode 100644 index 0000000000..905773e533 --- /dev/null +++ b/components/engine/network_proxy.go @@ -0,0 +1,257 @@ +package docker + +import ( + "encoding/binary" + "fmt" + "github.com/dotcloud/docker/utils" + "io" + "log" + "net" + "sync" + "syscall" + "time" +) + +const ( + UDPConnTrackTimeout = 90 * time.Second + UDPBufSize = 2048 +) + +type Proxy interface { + // Start forwarding traffic back and forth the front and back-end + // addresses. + Run() + // Stop forwarding traffic and close both ends of the Proxy. + Close() + // Return the address on which the proxy is listening. + FrontendAddr() net.Addr + // Return the proxied address. + BackendAddr() net.Addr +} + +type TCPProxy struct { + listener *net.TCPListener + frontendAddr *net.TCPAddr + backendAddr *net.TCPAddr +} + +func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) { + listener, err := net.ListenTCP("tcp", frontendAddr) + if err != nil { + return nil, err + } + // If the port in frontendAddr was 0 then ListenTCP will have a picked + // a port to listen on, hence the call to Addr to get that actual port: + return &TCPProxy{ + listener: listener, + frontendAddr: listener.Addr().(*net.TCPAddr), + backendAddr: backendAddr, + }, nil +} + +func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { + backend, err := net.DialTCP("tcp", nil, proxy.backendAddr) + if err != nil { + log.Printf("Can't forward traffic to backend tcp/%v: %v\n", proxy.backendAddr, err.Error()) + client.Close() + return + } + + event := make(chan int64) + var broker = func(to, from *net.TCPConn) { + written, err := io.Copy(to, from) + if err != nil { + err, ok := err.(*net.OpError) + // If the socket we are writing to is shutdown with + // SHUT_WR, forward it to the other end of the pipe: + if ok && err.Err == syscall.EPIPE { + from.CloseWrite() + } + } + event <- written + } + utils.Debugf("Forwarding traffic between tcp/%v and tcp/%v", client.RemoteAddr(), backend.RemoteAddr()) + go broker(client, backend) + go broker(backend, client) + + var transferred int64 = 0 + for i := 0; i < 2; i++ { + select { + case written := <-event: + transferred += written + case <-quit: + // Interrupt the two brokers and "join" them. + client.Close() + backend.Close() + for ; i < 2; i++ { + transferred += <-event + } + goto done + } + } + client.Close() + backend.Close() +done: + utils.Debugf("%v bytes transferred between tcp/%v and tcp/%v", transferred, client.RemoteAddr(), backend.RemoteAddr()) +} + +func (proxy *TCPProxy) Run() { + quit := make(chan bool) + defer close(quit) + utils.Debugf("Starting proxy on tcp/%v for tcp/%v", proxy.frontendAddr, proxy.backendAddr) + for { + client, err := proxy.listener.Accept() + if err != nil { + utils.Debugf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + return + } + go proxy.clientLoop(client.(*net.TCPConn), quit) + } +} + +func (proxy *TCPProxy) Close() { proxy.listener.Close() } +func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr } +func (proxy *TCPProxy) BackendAddr() net.Addr { return proxy.backendAddr } + +// A net.Addr where the IP is split into two fields so you can use it as a key +// in a map: +type connTrackKey struct { + IPHigh uint64 + IPLow uint64 + Port int +} + +func newConnTrackKey(addr *net.UDPAddr) *connTrackKey { + if len(addr.IP) == net.IPv4len { + return &connTrackKey{ + IPHigh: 0, + IPLow: uint64(binary.BigEndian.Uint32(addr.IP)), + Port: addr.Port, + } + } + return &connTrackKey{ + IPHigh: binary.BigEndian.Uint64(addr.IP[:8]), + IPLow: binary.BigEndian.Uint64(addr.IP[8:]), + Port: addr.Port, + } +} + +type connTrackMap map[connTrackKey]*net.UDPConn + +type UDPProxy struct { + listener *net.UDPConn + frontendAddr *net.UDPAddr + backendAddr *net.UDPAddr + connTrackTable connTrackMap + connTrackLock sync.Mutex +} + +func NewUDPProxy(frontendAddr, backendAddr *net.UDPAddr) (*UDPProxy, error) { + listener, err := net.ListenUDP("udp", frontendAddr) + if err != nil { + return nil, err + } + return &UDPProxy{ + listener: listener, + frontendAddr: listener.LocalAddr().(*net.UDPAddr), + backendAddr: backendAddr, + connTrackTable: make(connTrackMap), + }, nil +} + +func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) { + defer func() { + proxy.connTrackLock.Lock() + delete(proxy.connTrackTable, *clientKey) + proxy.connTrackLock.Unlock() + utils.Debugf("Done proxying between udp/%v and udp/%v", clientAddr.String(), proxy.backendAddr.String()) + proxyConn.Close() + }() + + readBuf := make([]byte, UDPBufSize) + for { + proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout)) + again: + read, err := proxyConn.Read(readBuf) + if err != nil { + if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED { + // This will happen if the last write failed + // (e.g: nothing is actually listening on the + // proxied port on the container), ignore it + // and continue until UDPConnTrackTimeout + // expires: + goto again + } + return + } + for i := 0; i != read; { + written, err := proxy.listener.WriteToUDP(readBuf[i:read], clientAddr) + if err != nil { + return + } + i += written + utils.Debugf("Forwarded %v/%v bytes to udp/%v", i, read, clientAddr.String()) + } + } +} + +func (proxy *UDPProxy) Run() { + readBuf := make([]byte, UDPBufSize) + utils.Debugf("Starting proxy on udp/%v for udp/%v", proxy.frontendAddr, proxy.backendAddr) + for { + read, from, err := proxy.listener.ReadFromUDP(readBuf) + if err != nil { + // NOTE: Apparently ReadFrom doesn't return + // ECONNREFUSED like Read do (see comment in + // UDPProxy.replyLoop) + utils.Debugf("Stopping proxy on udp/%v for udp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + break + } + + fromKey := newConnTrackKey(from) + proxy.connTrackLock.Lock() + proxyConn, hit := proxy.connTrackTable[*fromKey] + if !hit { + proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr) + if err != nil { + log.Printf("Can't proxy a datagram to udp/%s: %v\n", proxy.backendAddr.String(), err) + continue + } + proxy.connTrackTable[*fromKey] = proxyConn + go proxy.replyLoop(proxyConn, from, fromKey) + } + proxy.connTrackLock.Unlock() + for i := 0; i != read; { + written, err := proxyConn.Write(readBuf[i:read]) + if err != nil { + log.Printf("Can't proxy a datagram to udp/%s: %v\n", proxy.backendAddr.String(), err) + break + } + i += written + utils.Debugf("Forwarded %v/%v bytes to udp/%v", i, read, proxy.backendAddr.String()) + } + } +} + +func (proxy *UDPProxy) Close() { + proxy.listener.Close() + proxy.connTrackLock.Lock() + defer proxy.connTrackLock.Unlock() + for _, conn := range proxy.connTrackTable { + conn.Close() + } +} + +func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr } +func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr } + +func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) { + switch frontendAddr.(type) { + case *net.UDPAddr: + return NewUDPProxy(frontendAddr.(*net.UDPAddr), backendAddr.(*net.UDPAddr)) + case *net.TCPAddr: + return NewTCPProxy(frontendAddr.(*net.TCPAddr), backendAddr.(*net.TCPAddr)) + default: + panic(fmt.Errorf("Unsupported protocol")) + } +} diff --git a/components/engine/network_proxy_test.go b/components/engine/network_proxy_test.go new file mode 100644 index 0000000000..c27393eb54 --- /dev/null +++ b/components/engine/network_proxy_test.go @@ -0,0 +1,221 @@ +package docker + +import ( + "bytes" + "fmt" + "io" + "net" + "strings" + "testing" + "time" +) + +var testBuf = []byte("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo") +var testBufSize = len(testBuf) + +type EchoServer interface { + Run() + Close() + LocalAddr() net.Addr +} + +type TCPEchoServer struct { + listener net.Listener + testCtx *testing.T +} + +type UDPEchoServer struct { + conn net.PacketConn + testCtx *testing.T +} + +func NewEchoServer(t *testing.T, proto, address string) EchoServer { + var server EchoServer + if strings.HasPrefix(proto, "tcp") { + listener, err := net.Listen(proto, address) + if err != nil { + t.Fatal(err) + } + server = &TCPEchoServer{listener: listener, testCtx: t} + } else { + socket, err := net.ListenPacket(proto, address) + if err != nil { + t.Fatal(err) + } + server = &UDPEchoServer{conn: socket, testCtx: t} + } + t.Logf("EchoServer listening on %v/%v\n", proto, server.LocalAddr().String()) + return server +} + +func (server *TCPEchoServer) Run() { + go func() { + for { + client, err := server.listener.Accept() + if err != nil { + return + } + go func(client net.Conn) { + server.testCtx.Logf("TCP client accepted on the EchoServer\n") + written, err := io.Copy(client, client) + server.testCtx.Logf("%v bytes echoed back to the client\n", written) + if err != nil { + server.testCtx.Logf("can't echo to the client: %v\n", err.Error()) + } + client.Close() + }(client) + } + }() +} + +func (server *TCPEchoServer) LocalAddr() net.Addr { return server.listener.Addr() } +func (server *TCPEchoServer) Close() { server.listener.Addr() } + +func (server *UDPEchoServer) Run() { + go func() { + readBuf := make([]byte, 1024) + for { + read, from, err := server.conn.ReadFrom(readBuf) + if err != nil { + return + } + server.testCtx.Logf("Writing UDP datagram back") + for i := 0; i != read; { + written, err := server.conn.WriteTo(readBuf[i:read], from) + if err != nil { + break + } + i += written + } + } + }() +} + +func (server *UDPEchoServer) LocalAddr() net.Addr { return server.conn.LocalAddr() } +func (server *UDPEchoServer) Close() { server.conn.Close() } + +func testProxyAt(t *testing.T, proto string, proxy Proxy, addr string) { + defer proxy.Close() + go proxy.Run() + client, err := net.Dial(proto, addr) + if err != nil { + t.Fatalf("Can't connect to the proxy: %v", err) + } + defer client.Close() + client.SetDeadline(time.Now().Add(10 * time.Second)) + if _, err = client.Write(testBuf); err != nil { + t.Fatal(err) + } + recvBuf := make([]byte, testBufSize) + if _, err = client.Read(recvBuf); err != nil { + t.Fatal(err) + } + if !bytes.Equal(testBuf, recvBuf) { + t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf)) + } +} + +func testProxy(t *testing.T, proto string, proxy Proxy) { + testProxyAt(t, proto, proxy, proxy.FrontendAddr().String()) +} + +func TestTCP4Proxy(t *testing.T) { + backend := NewEchoServer(t, "tcp", "127.0.0.1:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + testProxy(t, "tcp", proxy) +} + +func TestTCP6Proxy(t *testing.T) { + backend := NewEchoServer(t, "tcp", "[::1]:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + testProxy(t, "tcp", proxy) +} + +func TestTCPDualStackProxy(t *testing.T) { + // If I understand `godoc -src net favoriteAddrFamily` (used by the + // net.Listen* functions) correctly this should work, but it doesn't. + t.Skip("No support for dual stack yet") + backend := NewEchoServer(t, "tcp", "[::1]:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + ipv4ProxyAddr := &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: proxy.FrontendAddr().(*net.TCPAddr).Port, + } + testProxyAt(t, "tcp", proxy, ipv4ProxyAddr.String()) +} + +func TestUDP4Proxy(t *testing.T) { + backend := NewEchoServer(t, "udp", "127.0.0.1:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + testProxy(t, "udp", proxy) +} + +func TestUDP6Proxy(t *testing.T) { + backend := NewEchoServer(t, "udp", "[::1]:0") + defer backend.Close() + backend.Run() + frontendAddr := &net.UDPAddr{IP: net.IPv6loopback, Port: 0} + proxy, err := NewProxy(frontendAddr, backend.LocalAddr()) + if err != nil { + t.Fatal(err) + } + testProxy(t, "udp", proxy) +} + +func TestUDPWriteError(t *testing.T) { + frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} + // Hopefully, this port will be free: */ + backendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 25587} + proxy, err := NewProxy(frontendAddr, backendAddr) + if err != nil { + t.Fatal(err) + } + defer proxy.Close() + go proxy.Run() + client, err := net.Dial("udp", "127.0.0.1:25587") + if err != nil { + t.Fatalf("Can't connect to the proxy: %v", err) + } + defer client.Close() + // Make sure the proxy doesn't stop when there is no actual backend: + client.Write(testBuf) + client.Write(testBuf) + backend := NewEchoServer(t, "udp", "127.0.0.1:25587") + defer backend.Close() + backend.Run() + client.SetDeadline(time.Now().Add(10 * time.Second)) + if _, err = client.Write(testBuf); err != nil { + t.Fatal(err) + } + recvBuf := make([]byte, testBufSize) + if _, err = client.Read(recvBuf); err != nil { + t.Fatal(err) + } + if !bytes.Equal(testBuf, recvBuf) { + t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf)) + } +} diff --git a/components/engine/network_test.go b/components/engine/network_test.go index 8e56c04ac2..8e6eaad773 100644 --- a/components/engine/network_test.go +++ b/components/engine/network_test.go @@ -20,28 +20,97 @@ func TestIptables(t *testing.T) { func TestParseNat(t *testing.T) { if nat, err := parseNat("4500"); err == nil { - if nat.Frontend != 0 || nat.Backend != 4500 { - t.Errorf("-p 4500 should produce 0->4500, got %d->%d", nat.Frontend, nat.Backend) + if nat.Frontend != 0 || nat.Backend != 4500 || nat.Proto != "tcp" { + t.Errorf("-p 4500 should produce 0->4500/tcp, got %d->%d/%s", + nat.Frontend, nat.Backend, nat.Proto) } } else { t.Fatal(err) } if nat, err := parseNat(":4501"); err == nil { - if nat.Frontend != 4501 || nat.Backend != 4501 { - t.Errorf("-p :4501 should produce 4501->4501, got %d->%d", nat.Frontend, nat.Backend) + if nat.Frontend != 4501 || nat.Backend != 4501 || nat.Proto != "tcp" { + t.Errorf("-p :4501 should produce 4501->4501/tcp, got %d->%d/%s", + nat.Frontend, nat.Backend, nat.Proto) } } else { t.Fatal(err) } if nat, err := parseNat("4502:4503"); err == nil { - if nat.Frontend != 4502 || nat.Backend != 4503 { - t.Errorf("-p 4502:4503 should produce 4502->4503, got %d->%d", nat.Frontend, nat.Backend) + if nat.Frontend != 4502 || nat.Backend != 4503 || nat.Proto != "tcp" { + t.Errorf("-p 4502:4503 should produce 4502->4503/tcp, got %d->%d/%s", + nat.Frontend, nat.Backend, nat.Proto) } } else { t.Fatal(err) } + + if nat, err := parseNat("4502:4503/tcp"); err == nil { + if nat.Frontend != 4502 || nat.Backend != 4503 || nat.Proto != "tcp" { + t.Errorf("-p 4502:4503/tcp should produce 4502->4503/tcp, got %d->%d/%s", + nat.Frontend, nat.Backend, nat.Proto) + } + } else { + t.Fatal(err) + } + + if nat, err := parseNat("4502:4503/udp"); err == nil { + if nat.Frontend != 4502 || nat.Backend != 4503 || nat.Proto != "udp" { + t.Errorf("-p 4502:4503/udp should produce 4502->4503/udp, got %d->%d/%s", + nat.Frontend, nat.Backend, nat.Proto) + } + } else { + t.Fatal(err) + } + + if nat, err := parseNat(":4503/udp"); err == nil { + if nat.Frontend != 4503 || nat.Backend != 4503 || nat.Proto != "udp" { + t.Errorf("-p :4503/udp should produce 4503->4503/udp, got %d->%d/%s", + nat.Frontend, nat.Backend, nat.Proto) + } + } else { + t.Fatal(err) + } + + if nat, err := parseNat(":4503/tcp"); err == nil { + if nat.Frontend != 4503 || nat.Backend != 4503 || nat.Proto != "tcp" { + t.Errorf("-p :4503/tcp should produce 4503->4503/tcp, got %d->%d/%s", + nat.Frontend, nat.Backend, nat.Proto) + } + } else { + t.Fatal(err) + } + + if nat, err := parseNat("4503/tcp"); err == nil { + if nat.Frontend != 0 || nat.Backend != 4503 || nat.Proto != "tcp" { + t.Errorf("-p 4503/tcp should produce 0->4503/tcp, got %d->%d/%s", + nat.Frontend, nat.Backend, nat.Proto) + } + } else { + t.Fatal(err) + } + + if nat, err := parseNat("4503/udp"); err == nil { + if nat.Frontend != 0 || nat.Backend != 4503 || nat.Proto != "udp" { + t.Errorf("-p 4503/udp should produce 0->4503/udp, got %d->%d/%s", + nat.Frontend, nat.Backend, nat.Proto) + } + } else { + t.Fatal(err) + } + + if _, err := parseNat("4503/tcpgarbage"); err == nil { + t.Fatal(err) + } + + if _, err := parseNat("4503/tcp/udp"); err == nil { + t.Fatal(err) + } + + if _, err := parseNat("4503/"); err == nil { + t.Fatal(err) + } } func TestPortAllocation(t *testing.T) { diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 0853a68e9f..fc84f19ec4 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -12,18 +12,90 @@ import ( "net/http" "net/http/cookiejar" "net/url" + "regexp" "strconv" "strings" ) var ErrAlreadyExists = errors.New("Image already exists") +var ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") -func UrlScheme() string { - u, err := url.Parse(auth.IndexServerAddress()) - if err != nil { - return "https" +func pingRegistryEndpoint(endpoint string) error { + if endpoint == auth.IndexServerAddress() { + // Skip the check, we now this one is valid + // (and we never want to fallback to http in case of error) + return nil } - return u.Scheme + resp, err := http.Get(endpoint + "_ping") + if err != nil { + return err + } + if resp.Header.Get("X-Docker-Registry-Version") == "" { + return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") + } + return nil +} + +func validateRepositoryName(repositoryName string) error { + var ( + namespace string + name string + ) + nameParts := strings.SplitN(repositoryName, "/", 2) + if len(nameParts) < 2 { + namespace = "library" + name = nameParts[0] + } else { + namespace = nameParts[0] + name = nameParts[1] + } + validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) + if !validNamespace.MatchString(namespace) { + return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace) + } + validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) + if !validRepo.MatchString(name) { + return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", name) + } + return nil +} + +// Resolves a repository name to a endpoint + name +func ResolveRepositoryName(reposName string) (string, string, error) { + if strings.Contains(reposName, "://") { + // It cannot contain a scheme! + return "", "", ErrInvalidRepositoryName + } + nameParts := strings.SplitN(reposName, "/", 2) + if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") { + // This is a Docker Index repos (ex: samalba/hipache or ubuntu) + err := validateRepositoryName(reposName) + return auth.IndexServerAddress(), reposName, err + } + if len(nameParts) < 2 { + // There is a dot in repos name (and no registry address) + // Is it a Registry address without repos name? + return "", "", ErrInvalidRepositoryName + } + hostname := nameParts[0] + reposName = nameParts[1] + if strings.Contains(hostname, "index.docker.io") { + return "", "", fmt.Errorf("Invalid repository name, try \"%s\" instead", reposName) + } + if err := validateRepositoryName(reposName); err != nil { + return "", "", err + } + endpoint := fmt.Sprintf("https://%s/v1/", hostname) + if err := pingRegistryEndpoint(endpoint); err != nil { + utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) + endpoint = fmt.Sprintf("http://%s/v1/", hostname) + if err = pingRegistryEndpoint(endpoint); err != nil { + //TODO: triggering highland build can be done there without "failing" + return "", "", errors.New("Invalid Registry endpoint: " + err.Error()) + } + } + err := validateRepositoryName(reposName) + return endpoint, reposName, err } func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { @@ -35,8 +107,8 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) +func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } @@ -44,7 +116,7 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s res, err := r.client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { - return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId) + return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) } return nil, err } @@ -64,10 +136,10 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s } // Check if an image exists in the Registry -func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool { +func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"/v1/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } @@ -79,44 +151,10 @@ func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) boo return res.StatusCode == 200 } -func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { - u := auth.IndexServerAddress() + "/repositories/" + repository + "/images" - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - if authConfig != nil && len(authConfig.Username) > 0 { - req.SetBasicAuth(authConfig.Username, authConfig.Password) - } - res, err := r.client.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - // Repository doesn't exist yet - if res.StatusCode == 404 { - return nil, nil - } - - jsonData, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - - imageList := []map[string]string{} - if err := json.Unmarshal(jsonData, &imageList); err != nil { - utils.Debugf("Body: %s (%s)\n", res.Body, u) - return nil, err - } - - return imageList, nil -} - // Retrieve an image from the Registry. -func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, int, error) { +func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -142,8 +180,8 @@ func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([ return jsonString, imageSize, nil } -func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) +func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } @@ -162,10 +200,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ repository = "library/" + repository } for _, host := range registries { - endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) - if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) { - endpoint = fmt.Sprintf("%s://%s", UrlScheme(), endpoint) - } + endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository) req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -198,8 +233,8 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, fmt.Errorf("Could not reach any registry endpoint") } -func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { - repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images" +func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) req, err := r.opaqueRequest("GET", repositoryTarget, nil) if err != nil { @@ -230,8 +265,12 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } var endpoints []string + var urlScheme = indexEp[:strings.Index(indexEp, ":")] if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints = res.Header["X-Docker-Endpoints"] + // The Registry's URL scheme has to match the Index' + for _, ep := range res.Header["X-Docker-Endpoints"] { + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep)) + } } else { return nil, fmt.Errorf("Index response didn't contain any endpoints") } @@ -260,9 +299,8 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - registry = registry + "/v1" // FIXME: try json with UTF8 - req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) if err != nil { return err } @@ -295,9 +333,8 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - registry = registry + "/v1" - req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error { + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer) if err != nil { return err } @@ -334,9 +371,8 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" - registry = registry + "/v1" - req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + req, err := r.opaqueRequest("PUT", registry+"repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { return err } @@ -354,7 +390,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return nil } -func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { +func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { imgListJSON, err := json.Marshal(imgList) if err != nil { return nil, err @@ -364,9 +400,10 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat suffix = "images" } + u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) + utils.Debugf("PUT %s", u) utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) - - req, err := r.opaqueRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJSON)) + req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { return nil, err } @@ -404,6 +441,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } var tokens, endpoints []string + var urlScheme = indexEp[:strings.Index(indexEp, ":")] if !validate { if res.StatusCode != 200 && res.StatusCode != 201 { errBody, err := ioutil.ReadAll(res.Body) @@ -420,7 +458,10 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints = res.Header["X-Docker-Endpoints"] + // The Registry's URL scheme has to match the Index' + for _, ep := range res.Header["X-Docker-Endpoints"] { + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep)) + } } else { return nil, fmt.Errorf("Index response didn't contain any endpoints") } @@ -442,7 +483,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { - u := auth.IndexServerAddress() + "/search?q=" + url.QueryEscape(term) + u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 06b1f8e1b9..5b0f7b2b2a 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -108,9 +108,6 @@ func (runtime *Runtime) Register(container *Container) error { // init the wait lock container.waitLock = make(chan struct{}) - // Even if not running, we init the lock (prevents races in start/stop/kill) - container.State.initLock() - container.runtime = runtime // Attach to stdout and stderr diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index c367ecd4c5..6c7ce5ed87 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -1,6 +1,7 @@ package docker import ( + "bytes" "fmt" "github.com/dotcloud/docker/utils" "io" @@ -17,11 +18,12 @@ import ( ) const ( - unitTestImageName = "docker-unit-tests" - unitTestImageId = "e9aa60c60128cad1" - unitTestStoreBase = "/var/lib/docker/unit-tests" - testDaemonAddr = "127.0.0.1:4270" - testDaemonProto = "tcp" + unitTestImageName = "docker-test-image" + unitTestImageID = "83599e29c455eb719f77d799bc7c51521b9551972f5a850d7ad265bc1b5292f6" // 1.0 + unitTestNetworkBridge = "testdockbr0" + unitTestStoreBase = "/var/lib/docker/unit-tests" + testDaemonAddr = "127.0.0.1:4270" + testDaemonProto = "tcp" ) var globalRuntime *Runtime @@ -49,7 +51,7 @@ func cleanup(runtime *Runtime) error { return err } for _, image := range images { - if image.ID != unitTestImageId { + if image.ID != unitTestImageID { runtime.graph.Delete(image.ID) } } @@ -73,10 +75,10 @@ func init() { } if uid := syscall.Geteuid(); uid != 0 { - log.Fatal("docker tests needs to be run as root") + log.Fatal("docker tests need to be run as root") } - NetworkBridgeIface = "testdockbr0" + NetworkBridgeIface = unitTestNetworkBridge // Make it our Store root runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false) @@ -89,13 +91,15 @@ func init() { srv := &Server{ runtime: runtime, enableCors: false, - lock: &sync.Mutex{}, pullingPool: make(map[string]struct{}), pushingPool: make(map[string]struct{}), } - // Retrieve the Image - if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil { - panic(err) + // If the unit test is not found, try to download it. + if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID { + // Retrieve the Image + if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil { + panic(err) + } } // Spawn a Daemon go func() { @@ -136,11 +140,11 @@ func GetTestImage(runtime *Runtime) *Image { panic(err) } for i := range imgs { - if imgs[i].ID == unitTestImageId { + if imgs[i].ID == unitTestImageID { return imgs[i] } } - panic(fmt.Errorf("Test image %v not found", unitTestImageId)) + panic(fmt.Errorf("Test image %v not found", unitTestImageID)) } func TestRuntimeCreate(t *testing.T) { @@ -318,52 +322,47 @@ func TestGet(t *testing.T) { } -func findAvailalblePort(runtime *Runtime, port int) (*Container, error) { - strPort := strconv.Itoa(port) - container, err := NewBuilder(runtime).Create(&Config{ - Image: GetTestImage(runtime).ID, - Cmd: []string{"sh", "-c", "echo well hello there | nc -l -p " + strPort}, - PortSpecs: []string{strPort}, - }, - ) - if err != nil { - return nil, err - } - hostConfig := &HostConfig{} - if err := container.Start(hostConfig); err != nil { - if strings.Contains(err.Error(), "address already in use") { - return nil, nil - } - return nil, err - } - return container, nil -} - -// Run a container with a TCP port allocated, and test that it can receive connections on localhost -func TestAllocatePortLocalhost(t *testing.T) { +func startEchoServerContainer(t *testing.T, proto string) (*Runtime, *Container, string) { runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - port := 5554 + port := 5554 var container *Container + var strPort string for { port += 1 - log.Println("Trying port", port) - t.Log("Trying port", port) - container, err = findAvailalblePort(runtime, port) + strPort = strconv.Itoa(port) + var cmd string + if proto == "tcp" { + cmd = "socat TCP-LISTEN:" + strPort + ",reuseaddr,fork EXEC:/bin/cat" + } else if proto == "udp" { + cmd = "socat UDP-RECVFROM:" + strPort + ",fork EXEC:/bin/cat" + } else { + t.Fatal(fmt.Errorf("Unknown protocol %v", proto)) + } + t.Log("Trying port", strPort) + container, err = NewBuilder(runtime).Create(&Config{ + Image: GetTestImage(runtime).ID, + Cmd: []string{"sh", "-c", cmd}, + PortSpecs: []string{fmt.Sprintf("%s/%s", strPort, proto)}, + }) if container != nil { break } if err != nil { + nuke(runtime) t.Fatal(err) } - log.Println("Port", port, "already in use") - t.Log("Port", port, "already in use") + t.Logf("Port %v already in use", strPort) } - defer container.Kill() + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { + nuke(runtime) + t.Fatal(err) + } setTimeout(t, "Waiting for the container to be started timed out", 2*time.Second, func() { for !container.State.Running { @@ -374,26 +373,93 @@ func TestAllocatePortLocalhost(t *testing.T) { // Even if the state is running, lets give some time to lxc to spawn the process container.WaitTimeout(500 * time.Millisecond) - conn, err := net.Dial("tcp", - fmt.Sprintf( - "localhost:%s", container.NetworkSettings.PortMapping[strconv.Itoa(port)], - ), - ) + strPort = container.NetworkSettings.PortMapping[strings.Title(proto)][strPort] + return runtime, container, strPort +} + +// Run a container with a TCP port allocated, and test that it can receive connections on localhost +func TestAllocateTCPPortLocalhost(t *testing.T) { + runtime, container, port := startEchoServerContainer(t, "tcp") + defer nuke(runtime) + defer container.Kill() + + for i := 0; i != 10; i++ { + conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%v", port)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + input := bytes.NewBufferString("well hello there\n") + _, err = conn.Write(input.Bytes()) + if err != nil { + t.Fatal(err) + } + buf := make([]byte, 16) + read := 0 + conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + read, err = conn.Read(buf) + if err != nil { + if err, ok := err.(*net.OpError); ok { + if err.Err == syscall.ECONNRESET { + t.Logf("Connection reset by the proxy, socat is probably not listening yet, trying again in a sec") + conn.Close() + time.Sleep(time.Second) + continue + } + if err.Timeout() { + t.Log("Timeout, trying again") + conn.Close() + continue + } + } + t.Fatal(err) + } + output := string(buf[:read]) + if !strings.Contains(output, "well hello there") { + t.Fatal(fmt.Errorf("[%v] doesn't contain [well hello there]", output)) + } else { + return + } + } + + t.Fatal("No reply from the container") +} + +// Run a container with an UDP port allocated, and test that it can receive connections on localhost +func TestAllocateUDPPortLocalhost(t *testing.T) { + runtime, container, port := startEchoServerContainer(t, "udp") + defer nuke(runtime) + defer container.Kill() + + conn, err := net.Dial("udp", fmt.Sprintf("localhost:%v", port)) if err != nil { t.Fatal(err) } defer conn.Close() - output, err := ioutil.ReadAll(conn) - if err != nil { - t.Fatal(err) + + input := bytes.NewBufferString("well hello there\n") + buf := make([]byte, 16) + // Try for a minute, for some reason the select in socat may take ages + // to return even though everything on the path seems fine (i.e: the + // UDPProxy forwards the traffic correctly and you can see the packets + // on the interface from within the container). + for i := 0; i != 120; i++ { + _, err := conn.Write(input.Bytes()) + if err != nil { + t.Fatal(err) + } + conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) + read, err := conn.Read(buf) + if err == nil { + output := string(buf[:read]) + if strings.Contains(output, "well hello there") { + return + } + } } - if string(output) != "well hello there\n" { - t.Fatalf("Received wrong output from network connection: should be '%s', not '%s'", - "well hello there\n", - string(output), - ) - } - container.Wait() + + t.Fatal("No reply from the container") } func TestRestore(t *testing.T) { diff --git a/components/engine/server.go b/components/engine/server.go index 944ab043eb..f1c0909516 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -29,7 +29,7 @@ func (srv *Server) DockerVersion() APIVersion { func (srv *Server) ContainerKill(name string) error { if container := srv.runtime.Get(name); container != nil { if err := container.Kill(); err != nil { - return fmt.Errorf("Error restarting container %s: %s", name, err.Error()) + return fmt.Errorf("Error restarting container %s: %s", name, err) } } else { return fmt.Errorf("No such container: %s", name) @@ -315,8 +315,8 @@ func (srv *Server) ContainerTag(name, repo, tag string, force bool) error { return nil } -func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoint string, token []string, sf *utils.StreamFormatter) error { - history, err := r.GetRemoteHistory(imgId, endpoint, token) +func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) error { + history, err := r.GetRemoteHistory(imgID, endpoint, token) if err != nil { return err } @@ -351,44 +351,32 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin return nil } -func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error { - out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress())) +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error { + out.Write(sf.FormatStatus("Pulling repository %s", localName)) - var repoData *registry.RepositoryData - var err error - if registryEp == "" { - repoData, err = r.GetRepositoryData(remote) - if err != nil { - return err - } + repoData, err := r.GetRepositoryData(indexEp, remoteName) + if err != nil { + return err + } - utils.Debugf("Updating checksums") - // Reload the json file to make sure not to overwrite faster sums - if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { - return err - } - } else { - repoData = ®istry.RepositoryData{ - Tokens: []string{}, - ImgList: make(map[string]*registry.ImgData), - Endpoints: []string{registryEp}, - } + utils.Debugf("Updating checksums") + // Reload the json file to make sure not to overwrite faster sums + if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { + return err } utils.Debugf("Retrieving the tag list") - tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens) + tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens) if err != nil { utils.Debugf("%v", err) return err } - if registryEp != "" { - for tag, id := range tagsList { - repoData.ImgList[id] = ®istry.ImgData{ - ID: id, - Tag: tag, - Checksum: "", - } + for tag, id := range tagsList { + repoData.ImgList[id] = ®istry.ImgData{ + ID: id, + Tag: tag, + Checksum: "", } } @@ -402,7 +390,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re // Otherwise, check that the tag exists and use only that one id, exists := tagsList[askedTag] if !exists { - return fmt.Errorf("Tag %s not found in repository %s", askedTag, local) + return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName) } repoData.ImgList[id].Tag = askedTag } @@ -417,13 +405,10 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) continue } - out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote)) + out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName)) success := false for _, ep := range repoData.Endpoints { - if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) { - ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep) - } - if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil { + if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) continue } @@ -438,7 +423,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re if askedTag != "" && tag != askedTag { continue } - if err := srv.runtime.repositories.Set(local, tag, id, true); err != nil { + if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil { return err } } @@ -450,8 +435,8 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re } func (srv *Server) poolAdd(kind, key string) error { - srv.lock.Lock() - defer srv.lock.Unlock() + srv.Lock() + defer srv.Unlock() if _, exists := srv.pullingPool[key]; exists { return fmt.Errorf("%s %s is already in progress", key, kind) @@ -483,25 +468,32 @@ func (srv *Server) poolRemove(kind, key string) error { } return nil } -func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { + +func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { r, err := registry.NewRegistry(srv.runtime.root, authConfig) if err != nil { return err } - if err := srv.poolAdd("pull", name+":"+tag); err != nil { + if err := srv.poolAdd("pull", localName+":"+tag); err != nil { return err } - defer srv.poolRemove("pull", name+":"+tag) + defer srv.poolRemove("pull", localName+":"+tag) - remote := name - parts := strings.Split(name, "/") - if len(parts) > 2 { - remote = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) + // Resolve the Repository name from fqn to endpoint + name + endpoint, remoteName, err := registry.ResolveRepositoryName(localName) + if err != nil { + return err } + + if endpoint == auth.IndexServerAddress() { + // If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar" + localName = remoteName + } + out = utils.NewWriteFlusher(out) - err = srv.pullRepository(r, out, name, remote, tag, endpoint, sf) - if err != nil && endpoint != "" { - if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { + err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf) + if err != nil { + if err := srv.pullImage(r, out, remoteName, endpoint, nil, sf); err != nil { return err } return nil @@ -516,20 +508,20 @@ func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *util // - Check if the archive exists, if it does not, ask the registry // - If the archive does exists, process the checksum from it // - If the archive does not exists and not found on registry, process checksum from layer -func (srv *Server) getChecksum(imageId string) (string, error) { +func (srv *Server) getChecksum(imageID string) (string, error) { // FIXME: Use in-memory map instead of reading the file each time if sums, err := srv.runtime.graph.getStoredChecksums(); err != nil { return "", err - } else if checksum, exists := sums[imageId]; exists { + } else if checksum, exists := sums[imageID]; exists { return checksum, nil } - img, err := srv.runtime.graph.Get(imageId) + img, err := srv.runtime.graph.Get(imageID) if err != nil { return "", err } - if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageId))); err != nil { + if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageID))); err != nil { if os.IsNotExist(err) { // TODO: Ask the registry for the checksum // As the archive is not there, it is supposed to come from a pull. @@ -576,7 +568,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return imgList, nil } -func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, registryEp string, localRepo map[string]string, sf *utils.StreamFormatter) error { +func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) out.Write(sf.FormatStatus("Processing checksums")) imgList, err := srv.getImageList(localRepo) @@ -584,94 +576,64 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, reg return err } out.Write(sf.FormatStatus("Sending image list")) - srvName := name - parts := strings.Split(name, "/") - if len(parts) > 2 { - srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) - } var repoData *registry.RepositoryData - if registryEp == "" { - repoData, err = r.PushImageJSONIndex(name, imgList, false, nil) - if err != nil { - return err - } - } else { - repoData = ®istry.RepositoryData{ - ImgList: make(map[string]*registry.ImgData), - Tokens: []string{}, - Endpoints: []string{registryEp}, - } - tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens) - if err != nil && err.Error() != "Repository not found" { - return err - } else if err == nil { - for tag, id := range tagsList { - repoData.ImgList[id] = ®istry.ImgData{ - ID: id, - Tag: tag, - Checksum: "", - } - } - } + repoData, err = r.PushImageJSONIndex(indexEp, remoteName, imgList, false, nil) + if err != nil { + return err } for _, ep := range repoData.Endpoints { - if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) { - ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep) - } - out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo))) + out.Write(sf.FormatStatus("Pushing repository %s (%d tags)", localName, len(localRepo))) // For each image within the repo, push them for _, elem := range imgList { if _, exists := repoData.ImgList[elem.ID]; exists { - out.Write(sf.FormatStatus("Image %s already on registry, skipping", name)) + out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) continue } else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) { - fmt.Fprintf(out, "Image %s already on registry, skipping\n", name) + out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) continue } - if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil { + if err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? return err } - out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"/repositories/"+srvName+"/tags/"+elem.Tag)) - if err := r.PushRegistryTag(srvName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { + out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) + if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { return err } } } - if registryEp == "" { - if _, err := r.PushImageJSONIndex(name, imgList, true, repoData.Endpoints); err != nil { - return err - } + if _, err := r.PushImageJSONIndex(indexEp, remoteName, imgList, true, repoData.Endpoints); err != nil { + return err } return nil } -func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId, ep string, token []string, sf *utils.StreamFormatter) error { +func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) - jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgId, "json")) + jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgID, "json")) if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgId, err) + return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) } - out.Write(sf.FormatStatus("Pushing %s", imgId)) + out.Write(sf.FormatStatus("Pushing %s", imgID)) // Make sure we have the image's checksum - checksum, err := srv.getChecksum(imgId) + checksum, err := srv.getChecksum(imgID) if err != nil { return err } imgData := ®istry.ImgData{ - ID: imgId, + ID: imgID, Checksum: checksum, } // Send the json if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { - out.Write(sf.FormatStatus("Image %s already uploaded ; skipping", imgData.ID)) + out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID)) return nil } return err @@ -680,11 +642,11 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId, // Retrieve the tarball to be sent var layerData *TempArchive // If the archive exists, use it - file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgId))) + file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgID))) if err != nil { if os.IsNotExist(err) { // If the archive does not exist, create one from the layer - layerData, err = srv.runtime.graph.TempLayerArchive(imgId, Xz, out) + layerData, err = srv.runtime.graph.TempLayerArchive(imgID, Xz, sf, out) if err != nil { return fmt.Errorf("Failed to generate layer archive: %s", err) } @@ -711,33 +673,41 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId, } // FIXME: Allow to interupt current push when new push of same image is done. -func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { - if err := srv.poolAdd("push", name); err != nil { +func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { + if err := srv.poolAdd("push", localName); err != nil { + return err + } + defer srv.poolRemove("push", localName) + + // Resolve the Repository name from fqn to endpoint + name + endpoint, remoteName, err := registry.ResolveRepositoryName(localName) + if err != nil { return err } - defer srv.poolRemove("push", name) out = utils.NewWriteFlusher(out) - img, err := srv.runtime.graph.Get(name) + img, err := srv.runtime.graph.Get(localName) r, err2 := registry.NewRegistry(srv.runtime.root, authConfig) if err2 != nil { return err2 } if err != nil { - out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name]))) + reposLen := len(srv.runtime.repositories.Repositories[localName]) + out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", localName, reposLen)) // If it fails, try to get the repository - if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists { - if err := srv.pushRepository(r, out, name, endpoint, localRepo, sf); err != nil { + if localRepo, exists := srv.runtime.repositories.Repositories[localName]; exists { + if err := srv.pushRepository(r, out, localName, remoteName, localRepo, endpoint, sf); err != nil { return err } return nil } - return err } - out.Write(sf.FormatStatus("The push refers to an image: [%s]", name)) - if err := srv.pushImage(r, out, name, img.ID, endpoint, nil, sf); err != nil { + + var token []string + out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName)) + if err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { return err } return nil @@ -809,7 +779,7 @@ func (srv *Server) ContainerCreate(config *Config) (string, error) { func (srv *Server) ContainerRestart(name string, t int) error { if container := srv.runtime.Get(name); container != nil { if err := container.Restart(t); err != nil { - return fmt.Errorf("Error restarting container %s: %s", name, err.Error()) + return fmt.Errorf("Error restarting container %s: %s", name, err) } } else { return fmt.Errorf("No such container: %s", name) @@ -828,7 +798,7 @@ func (srv *Server) ContainerDestroy(name string, removeVolume bool) error { volumes[volumeId] = struct{}{} } if err := srv.runtime.Destroy(container); err != nil { - return fmt.Errorf("Error destroying container %s: %s", name, err.Error()) + return fmt.Errorf("Error destroying container %s: %s", name, err) } if removeVolume { @@ -919,7 +889,7 @@ func (srv *Server) deleteImageParents(img *Image, imgs *[]APIRmi) error { func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, error) { //Untag the current image - var imgs []APIRmi + imgs := []APIRmi{} tagDeleted, err := srv.runtime.repositories.Delete(repoName, tag) if err != nil { return nil, err @@ -948,7 +918,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { } if !autoPrune { if err := srv.runtime.graph.Delete(img.ID); err != nil { - return nil, fmt.Errorf("Error deleting image %s: %s", name, err.Error()) + return nil, fmt.Errorf("Error deleting image %s: %s", name, err) } return nil, nil } @@ -963,7 +933,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { return srv.deleteImage(img, name, tag) } -func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error) { +func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) { // Retrieve all images images, err := srv.runtime.graph.All() @@ -981,7 +951,7 @@ func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error) } // Loop on the children of the given image and check the config - for elem := range imageMap[imgId] { + for elem := range imageMap[imgID] { img, err := srv.runtime.graph.Get(elem) if err != nil { return nil, err @@ -996,7 +966,7 @@ func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error) func (srv *Server) ContainerStart(name string, hostConfig *HostConfig) error { if container := srv.runtime.Get(name); container != nil { if err := container.Start(hostConfig); err != nil { - return fmt.Errorf("Error starting container %s: %s", name, err.Error()) + return fmt.Errorf("Error starting container %s: %s", name, err) } } else { return fmt.Errorf("No such container: %s", name) @@ -1007,7 +977,7 @@ func (srv *Server) ContainerStart(name string, hostConfig *HostConfig) error { func (srv *Server) ContainerStop(name string, t int) error { if container := srv.runtime.Get(name); container != nil { if err := container.Stop(t); err != nil { - return fmt.Errorf("Error stopping container %s: %s", name, err.Error()) + return fmt.Errorf("Error stopping container %s: %s", name, err) } } else { return fmt.Errorf("No such container: %s", name) @@ -1119,7 +1089,6 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( srv := &Server{ runtime: runtime, enableCors: enableCors, - lock: &sync.Mutex{}, pullingPool: make(map[string]struct{}), pushingPool: make(map[string]struct{}), } @@ -1128,9 +1097,9 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( } type Server struct { + sync.Mutex runtime *Runtime enableCors bool - lock *sync.Mutex pullingPool map[string]struct{} pushingPool map[string]struct{} } diff --git a/components/engine/server_test.go b/components/engine/server_test.go index 254a4a0c90..cf3e3f0bc8 100644 --- a/components/engine/server_test.go +++ b/components/engine/server_test.go @@ -31,7 +31,7 @@ func TestContainerTagImageDelete(t *testing.T) { } if len(images) != len(initialImages)+2 { - t.Errorf("Excepted %d images, %d found", len(initialImages)+2, len(images)) + t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) } if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil { @@ -44,7 +44,7 @@ func TestContainerTagImageDelete(t *testing.T) { } if len(images) != len(initialImages)+1 { - t.Errorf("Excepted %d images, %d found", len(initialImages)+1, len(images)) + t.Errorf("Expected %d images, %d found", len(initialImages)+1, len(images)) } if _, err := srv.ImageDelete("utest:tag1", true); err != nil { @@ -57,7 +57,7 @@ func TestContainerTagImageDelete(t *testing.T) { } if len(images) != len(initialImages) { - t.Errorf("Excepted %d image, %d found", len(initialImages), len(images)) + t.Errorf("Expected %d image, %d found", len(initialImages), len(images)) } } diff --git a/components/engine/state.go b/components/engine/state.go index a972e376a2..6480b9b428 100644 --- a/components/engine/state.go +++ b/components/engine/state.go @@ -8,11 +8,11 @@ import ( ) type State struct { + sync.Mutex Running bool Pid int ExitCode int StartedAt time.Time - l *sync.Mutex Ghost bool } @@ -29,6 +29,7 @@ func (s *State) String() string { func (s *State) setRunning(pid int) { s.Running = true + s.Ghost = false s.ExitCode = 0 s.Pid = pid s.StartedAt = time.Now() @@ -39,15 +40,3 @@ func (s *State) setStopped(exitCode int) { s.Pid = 0 s.ExitCode = exitCode } - -func (s *State) initLock() { - s.l = &sync.Mutex{} -} - -func (s *State) lock() { - s.l.Lock() -} - -func (s *State) unlock() { - s.l.Unlock() -} diff --git a/components/engine/tags.go b/components/engine/tags.go index 33ec4e149d..9ad9d10d05 100644 --- a/components/engine/tags.go +++ b/components/engine/tags.go @@ -70,11 +70,11 @@ func (store *TagStore) LookupImage(name string) (*Image, error) { if err != nil { // FIXME: standardize on returning nil when the image doesn't exist, and err for everything else // (so we can pass all errors here) - repoAndTag := strings.SplitN(name, ":", 2) - if len(repoAndTag) == 1 { - repoAndTag = append(repoAndTag, DEFAULTTAG) + repos, tag := utils.ParseRepositoryTag(name) + if tag == "" { + tag = DEFAULTTAG } - if i, err := store.GetImage(repoAndTag[0], repoAndTag[1]); err != nil { + if i, err := store.GetImage(repos, tag); err != nil { return nil, err } else if i == nil { return nil, fmt.Errorf("Image does not exist: %s", name) @@ -197,7 +197,7 @@ func (store *TagStore) Get(repoName string) (Repository, error) { return nil, nil } -func (store *TagStore) GetImage(repoName, tagOrId string) (*Image, error) { +func (store *TagStore) GetImage(repoName, tagOrID string) (*Image, error) { repo, err := store.Get(repoName) if err != nil { return nil, err @@ -206,11 +206,11 @@ func (store *TagStore) GetImage(repoName, tagOrId string) (*Image, error) { } //go through all the tags, to see if tag is in fact an ID for _, revision := range repo { - if strings.HasPrefix(revision, tagOrId) { + if strings.HasPrefix(revision, tagOrID) { return store.graph.Get(revision) } } - if revision, exists := repo[tagOrId]; exists { + if revision, exists := repo[tagOrID]; exists { return store.graph.Get(revision) } return nil, nil @@ -221,9 +221,6 @@ func validateRepoName(name string) error { if name == "" { return fmt.Errorf("Repository name can't be empty") } - if strings.Contains(name, ":") { - return fmt.Errorf("Illegal repository name: %s", name) - } return nil } diff --git a/components/engine/tags_test.go b/components/engine/tags_test.go index 90bc056406..1974e751be 100644 --- a/components/engine/tags_test.go +++ b/components/engine/tags_test.go @@ -35,13 +35,13 @@ func TestLookupImage(t *testing.T) { t.Errorf("Expected 0 image, 1 found") } - if img, err := runtime.repositories.LookupImage(unitTestImageId); err != nil { + if img, err := runtime.repositories.LookupImage(unitTestImageID); err != nil { t.Fatal(err) } else if img == nil { t.Errorf("Expected 1 image, none found") } - if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + unitTestImageId); err != nil { + if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + unitTestImageID); err != nil { t.Fatal(err) } else if img == nil { t.Errorf("Expected 1 image, none found") diff --git a/components/engine/term/term.go b/components/engine/term/term.go index 3f743d227f..f4d66a71d6 100644 --- a/components/engine/term/term.go +++ b/components/engine/term/term.go @@ -12,8 +12,8 @@ type State struct { } type Winsize struct { - Width uint16 Height uint16 + Width uint16 x uint16 y uint16 } diff --git a/components/engine/testing/README.rst b/components/engine/testing/README.rst index 405adaed94..3b11092f9f 100644 --- a/components/engine/testing/README.rst +++ b/components/engine/testing/README.rst @@ -30,6 +30,16 @@ Deployment export AWS_KEYPAIR_NAME=xxxxxxxxxxxx export AWS_SSH_PRIVKEY=xxxxxxxxxxxx + # Define email recipient and IRC channel + export EMAIL_RCP=xxxxxx@domain.com + export IRC_CHANNEL=docker + + # Define buildbot credentials + export BUILDBOT_PWD=xxxxxxxxxxxx + export IRC_PWD=xxxxxxxxxxxx + export SMTP_USER=xxxxxxxxxxxx + export SMTP_PWD=xxxxxxxxxxxx + # Checkout docker git clone git://github.com/dotcloud/docker.git diff --git a/components/engine/testing/Vagrantfile b/components/engine/testing/Vagrantfile index e3b25a6f9d..47257201dc 100644 --- a/components/engine/testing/Vagrantfile +++ b/components/engine/testing/Vagrantfile @@ -27,7 +27,9 @@ Vagrant::Config.run do |config| pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \ "pip install -r #{CFG_PATH}/requirements.txt; " \ "chown #{USER}.#{USER} /data; cd /data; " \ - "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH}; " + "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \ + "#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \ + "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ diff --git a/components/engine/testing/buildbot/master.cfg b/components/engine/testing/buildbot/master.cfg index 48df5835f7..65399bb1a1 100644 --- a/components/engine/testing/buildbot/master.cfg +++ b/components/engine/testing/buildbot/master.cfg @@ -5,9 +5,11 @@ from buildbot.schedulers.basic import SingleBranchScheduler from buildbot.changes import filter from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory +from buildbot.process.properties import Interpolate from buildbot.steps.shell import ShellCommand -from buildbot.status import html +from buildbot.status import html, words from buildbot.status.web import authz, auth +from buildbot.status.mail import MailNotifier PORT_WEB = 80 # Buildbot webserver port PORT_GITHUB = 8011 # Buildbot github hook port @@ -15,20 +17,27 @@ PORT_MASTER = 9989 # Port where buildbot master listen buildworkers TEST_USER = 'buildbot' # Credential to authenticate build triggers TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' -BUILDPASSWORD = 'pass-docker' # Credential to authenticate buildworkers -GITHUB_DOCKER = "github.com/dotcloud/docker" -DOCKER_PATH = "/data/docker" -BUILDER_PATH = "/data/buildbot/slave/{0}/build".format(BUILDER_NAME) +GITHUB_DOCKER = 'github.com/dotcloud/docker' +DOCKER_PATH = '/data/docker' +BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME) DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker' +# Credentials set by setup.sh and Vagrantfile +BUILDBOT_PWD = '' +IRC_PWD = '' +IRC_CHANNEL = '' +SMTP_USER = '' +SMTP_PWD = '' +EMAIL_RCP = '' + c = BuildmasterConfig = {} c['title'] = "Docker" c['titleURL'] = "waterfall" -c['buildbotURL'] = "http://0.0.0.0:{0}/".format(PORT_WEB) +c['buildbotURL'] = "http://docker-ci.dotcloud.com/" c['db'] = {'db_url':"sqlite:///state.sqlite"} -c['slaves'] = [BuildSlave('buildworker', BUILDPASSWORD)] +c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)] c['slavePortnum'] = PORT_MASTER c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])] @@ -36,20 +45,25 @@ c['schedulers'].append(SingleBranchScheduler(name="all", change_filter=filter.ChangeFilter(branch='master'),treeStableTimer=None, builderNames=[BUILDER_NAME])) -# Docker test command -test_cmd = ("cd /tmp; rm -rf {0}; export GOPATH={0}; go get -d {1}; cd {2}; " - "go test").format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH) - # Builder factory = BuildFactory() -factory.addStep(ShellCommand(description='Docker',logEnviron=False, - usePTY=True,command=test_cmd)) +factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, + command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; " + "go get -d {1}; cd {2}; git reset --hard %(src::revision:-unknown)s; " + "go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))])) c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], factory=factory)] # Status -authz_cfg=authz.Authz(auth=auth.BasicAuth([(TEST_USER,TEST_PWD)]), +authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]), forceBuild='auth') c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)] -c['status'].append(html.WebStatus(http_port=PORT_GITHUB,allowForce=True, - change_hook_dialects={ 'github' : True })) +c['status'].append(html.WebStatus(http_port=PORT_GITHUB, allowForce=True, + change_hook_dialects={ 'github': True })) +c['status'].append(MailNotifier(fromaddr='buildbot@docker.io', + sendToInterestedUsers=False, extraRecipients=[EMAIL_RCP], + mode='failing', relayhost='smtp.mailgun.org', smtpPort=587, useTls=True, + smtpUser=SMTP_USER, smtpPassword=SMTP_PWD)) +c['status'].append(words.IRC("irc.freenode.net", "dockerqabot", + channels=[IRC_CHANNEL], password=IRC_PWD, allowForce=True, + notify_events={'exception':1, 'successToFailure':1, 'failureToSuccess':1})) diff --git a/components/engine/testing/buildbot/setup.sh b/components/engine/testing/buildbot/setup.sh index 828ac3ebe5..937533ba1f 100755 --- a/components/engine/testing/buildbot/setup.sh +++ b/components/engine/testing/buildbot/setup.sh @@ -6,11 +6,16 @@ USER=$1 CFG_PATH=$2 +BUILDBOT_PWD=$3 +IRC_PWD=$4 +IRC_CHANNEL=$5 +SMTP_USER=$6 +SMTP_PWD=$7 +EMAIL_RCP=$8 BUILDBOT_PATH="/data/buildbot" DOCKER_PATH="/data/docker" SLAVE_NAME="buildworker" SLAVE_SOCKET="localhost:9989" -BUILDBOT_PWD="pass-docker" export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" function run { su $USER -c "$1"; } @@ -23,7 +28,12 @@ run "mkdir -p $BUILDBOT_PATH" cd $BUILDBOT_PATH run "buildbot create-master master" run "cp $CFG_PATH/master.cfg master" -run "sed -i -E 's#(DOCKER_PATH = ).+#\1\"$DOCKER_PATH\"#' master/master.cfg" +run "sed -i -E 's#(BUILDBOT_PWD = ).+#\1\"$BUILDBOT_PWD\"#' master/master.cfg" +run "sed -i -E 's#(IRC_PWD = ).+#\1\"$IRC_PWD\"#' master/master.cfg" +run "sed -i -E 's#(IRC_CHANNEL = ).+#\1\"$IRC_CHANNEL\"#' master/master.cfg" +run "sed -i -E 's#(SMTP_USER = ).+#\1\"$SMTP_USER\"#' master/master.cfg" +run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg" +run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg" run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" # Allow buildbot subprocesses (docker tests) to properly run in containers, diff --git a/components/engine/utils.go b/components/engine/utils.go index 22613e9f49..d55b1ff0bc 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -20,7 +20,8 @@ func CompareConfig(a, b *Config) bool { if len(a.Cmd) != len(b.Cmd) || len(a.Dns) != len(b.Dns) || len(a.Env) != len(b.Env) || - len(a.PortSpecs) != len(b.PortSpecs) { + len(a.PortSpecs) != len(b.PortSpecs) || + len(a.Entrypoint) != len(b.Entrypoint) { return false } @@ -53,9 +54,6 @@ func CompareConfig(a, b *Config) bool { } func MergeConfig(userConf, imageConf *Config) { - if userConf.Hostname == "" { - userConf.Hostname = imageConf.Hostname - } if userConf.User == "" { userConf.User = imageConf.User } @@ -92,4 +90,7 @@ func MergeConfig(userConf, imageConf *Config) { if userConf.Entrypoint == nil || len(userConf.Entrypoint) == 0 { userConf.Entrypoint = imageConf.Entrypoint } + if userConf.Volumes == nil || len(userConf.Volumes) == 0 { + userConf.Volumes = imageConf.Volumes + } } diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index b4a41ea420..a4e9ee2bfc 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -170,10 +170,9 @@ func SelfPath() string { return path } -type NopWriter struct { -} +type NopWriter struct{} -func (w *NopWriter) Write(buf []byte) (int, error) { +func (*NopWriter) Write(buf []byte) (int, error) { return len(buf), nil } @@ -188,10 +187,10 @@ func NopWriteCloser(w io.Writer) io.WriteCloser { } type bufReader struct { + sync.Mutex buf *bytes.Buffer reader io.Reader err error - l sync.Mutex wait sync.Cond } @@ -200,7 +199,7 @@ func NewBufReader(r io.Reader) *bufReader { buf: &bytes.Buffer{}, reader: r, } - reader.wait.L = &reader.l + reader.wait.L = &reader.Mutex go reader.drain() return reader } @@ -209,14 +208,14 @@ func (r *bufReader) drain() { buf := make([]byte, 1024) for { n, err := r.reader.Read(buf) - r.l.Lock() + r.Lock() if err != nil { r.err = err } else { r.buf.Write(buf[0:n]) } r.wait.Signal() - r.l.Unlock() + r.Unlock() if err != nil { break } @@ -224,8 +223,8 @@ func (r *bufReader) drain() { } func (r *bufReader) Read(p []byte) (n int, err error) { - r.l.Lock() - defer r.l.Unlock() + r.Lock() + defer r.Unlock() for { n, err = r.buf.Read(p) if n > 0 { @@ -247,27 +246,27 @@ func (r *bufReader) Close() error { } type WriteBroadcaster struct { - mu sync.Mutex + sync.Mutex writers map[io.WriteCloser]struct{} } func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser) { - w.mu.Lock() + w.Lock() w.writers[writer] = struct{}{} - w.mu.Unlock() + w.Unlock() } // FIXME: Is that function used? // FIXME: This relies on the concrete writer type used having equality operator func (w *WriteBroadcaster) RemoveWriter(writer io.WriteCloser) { - w.mu.Lock() + w.Lock() delete(w.writers, writer) - w.mu.Unlock() + w.Unlock() } func (w *WriteBroadcaster) Write(p []byte) (n int, err error) { - w.mu.Lock() - defer w.mu.Unlock() + w.Lock() + defer w.Unlock() for writer := range w.writers { if n, err := writer.Write(p); err != nil || n != len(p) { // On error, evict the writer @@ -278,8 +277,8 @@ func (w *WriteBroadcaster) Write(p []byte) (n int, err error) { } func (w *WriteBroadcaster) CloseWriters() error { - w.mu.Lock() - defer w.mu.Unlock() + w.Lock() + defer w.Unlock() for writer := range w.writers { writer.Close() } @@ -689,11 +688,7 @@ func ParseHost(host string, port int, addr string) string { } func GetReleaseVersion() string { - type githubTag struct { - Name string `json:"name"` - } - - resp, err := http.Get("https://api.github.com/repos/dotcloud/docker/tags") + resp, err := http.Get("http://get.docker.io/latest") if err != nil { return "" } @@ -702,10 +697,19 @@ func GetReleaseVersion() string { if err != nil { return "" } - var tags []githubTag - err = json.Unmarshal(body, &tags) - if err != nil || len(tags) == 0 { - return "" + return strings.TrimSpace(string(body)) +} + +// Get a repos name and returns the right reposName + tag +// The tag can be confusing because of a port in a repository name. +// Ex: localhost.localdomain:5000/samalba/hipache:latest +func ParseRepositoryTag(repos string) (string, string) { + n := strings.LastIndex(repos, ":") + if n < 0 { + return repos, "" } - return strings.TrimPrefix(tags[0].Name, "v") + if tag := repos[n+1:]; !strings.Contains(tag, "/") { + return repos[:n], tag + } + return repos, "" }