toolshed/docker-cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use http://get.docker.io/latest

Browse Source 
Upstream-commit: e9011122fbc4c65afe235a75ca8197ce4fd23753
Component: engine
This commit is contained in:
Victor Vieux
2013-07-12 11:45:40 +00:00
parent 2394c9c8dc 725d76d0d9
commit 18c42253ef
38 changed files with 1439 additions and 679 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/engine/Dockerfile
+30
View File
@@ -0,0 +1,30 @@
# This file describes the standard way to build Docker, using docker
docker-version 0.4.2
from ubuntu:12.04
maintainer Solomon Hykes <solomon@dotcloud.com>
# Build dependencies
run apt-get install -y -q curl
run apt-get install -y -q git
# Install Go
run curl -s https://go.googlecode.com/files/go1.1.1.linux-amd64.tar.gz | tar -v -C /usr/local -xz
env PATH /usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin
env GOPATH /go
env CGO_ENABLED 0
run cd /tmp && echo 'package main' > t.go && go test -a -i -v
# Download dependencies
run PKG=github.com/kr/pty REV=27435c699; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV
run PKG=github.com/gorilla/context/ REV=708054d61e5; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV
run PKG=github.com/gorilla/mux/ REV=9b36453141c; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV
# Run dependencies
run apt-get install -y iptables
# lxc requires updating ubuntu sources
run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
run apt-get update
run apt-get install -y lxc
run apt-get install -y aufs-tools
# Upload docker source
add . /go/src/github.com/dotcloud/docker
# Build the binary
run cd /go/src/github.com/dotcloud/docker/docker && go install -ldflags "-X main.GITCOMMIT '??' -d -w"
env PATH /usr/local/go/bin:/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin
cmd ["docker"]
components/engine/README.md
+9 -9
View File
@@ -23,15 +23,15 @@ happens, for a few reasons:
* *Size*: VMs are very large which makes them impractical to store and transfer.
* *Performance*: running VMs consumes significant CPU and memory, which makes them impractical in many scenarios, for example local development of multi-tier applications, and
large-scale deployment of cpu and memory-intensive applications on large numbers of machines.
large-scale deployment of cpu and memory-intensive applications on large numbers of machines.
* *Portability*: competing VM environments don't play well with each other. Although conversion tools do exist, they are limited and add even more overhead.
* *Hardware-centric*: VMs were designed with machine operators in mind, not software developers. As a result, they offer very limited tooling for what developers need most:
building, testing and running their software. For example, VMs offer no facilities for application versioning, monitoring, configuration, logging or service discovery.
building, testing and running their software. For example, VMs offer no facilities for application versioning, monitoring, configuration, logging or service discovery.
By contrast, Docker relies on a different sandboxing method known as *containerization*. Unlike traditional virtualization,
containerization takes place at the kernel level. Most modern operating system kernels now support the primitives necessary
for containerization, including Linux with [openvz](http://openvz.org), [vserver](http://linux-vserver.org) and more recently [lxc](http://lxc.sourceforge.net),
Solaris with [zones](http://docs.oracle.com/cd/E26502_01/html/E29024/preface-1.html#scrolltoc) and FreeBSD with [Jails](http://www.freebsd.org/doc/handbook/jails.html).
Solaris with [zones](http://docs.oracle.com/cd/E26502_01/html/E29024/preface-1.html#scrolltoc) and FreeBSD with [Jails](http://www.freebsd.org/doc/handbook/jails.html).
Docker builds on top of these low-level primitives to offer developers a portable format and runtime environment that solves
all 4 problems. Docker containers are small (and their transfer can be optimized with layers), they have basically zero memory and cpu overhead,
@@ -56,17 +56,17 @@ A common problem for developers is the difficulty of managing all their applicat
This is usually difficult for several reasons:
* *Cross-platform dependencies*. Modern applications often depend on a combination of system libraries and binaries, language-specific packages, framework-specific modules,
internal components developed for another project, etc. These dependencies live in different "worlds" and require different tools - these tools typically don't work
well with each other, requiring awkward custom integrations.
internal components developed for another project, etc. These dependencies live in different "worlds" and require different tools - these tools typically don't work
well with each other, requiring awkward custom integrations.
* Conflicting dependencies. Different applications may depend on different versions of the same dependency. Packaging tools handle these situations with various degrees of ease -
but they all handle them in different and incompatible ways, which again forces the developer to do extra work.
but they all handle them in different and incompatible ways, which again forces the developer to do extra work.
* Custom dependencies. A developer may need to prepare a custom version of his application's dependency. Some packaging systems can handle custom versions of a dependency,
others can't - and all of them handle it differently.
* Custom dependencies. A developer may need to prepare a custom version of their application's dependency. Some packaging systems can handle custom versions of a dependency,
others can't - and all of them handle it differently.
Docker solves dependency hell by giving the developer a simple way to express *all* his application's dependencies in one place,
Docker solves dependency hell by giving the developer a simple way to express *all* their application's dependencies in one place,
and streamline the process of assembling them. If this makes you think of [XKCD 927](http://xkcd.com/927/), don't worry. Docker doesn't
*replace* your favorite packaging systems. It simply orchestrates their use in a simple and repeatable way. How does it do that? With layers.
components/engine/api.go
+5 -7
View File
@@ -170,7 +170,7 @@ func getContainersExport(srv *Server, version float64, w http.ResponseWriter, r
name := vars["name"]
if err := srv.ContainerExport(name, w); err != nil {
utils.Debugf("%s", err.Error())
utils.Debugf("%s", err)
return err
}
return nil
@@ -306,7 +306,7 @@ func postCommit(srv *Server, version float64, w http.ResponseWriter, r *http.Req
}
config := &Config{}
if err := json.NewDecoder(r.Body).Decode(config); err != nil {
utils.Debugf("%s", err.Error())
utils.Debugf("%s", err)
}
repo := r.Form.Get("repo")
tag := r.Form.Get("tag")
@@ -342,8 +342,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht
}
sf := utils.NewStreamFormatter(version > 1.0)
if image != "" { //pull
registry := r.Form.Get("registry")
if err := srv.ImagePull(image, tag, registry, w, sf, &auth.AuthConfig{}); err != nil {
if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil {
if sf.Used() {
w.Write(sf.FormatError(err))
return nil
@@ -426,7 +425,6 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http
if err := parseForm(r); err != nil {
return err
}
registry := r.Form.Get("registry")
if vars == nil {
return fmt.Errorf("Missing parameter")
@@ -436,7 +434,7 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http
w.Header().Set("Content-Type", "application/json")
}
sf := utils.NewStreamFormatter(version > 1.0)
if err := srv.ImagePush(name, registry, w, sf, authConfig); err != nil {
if err := srv.ImagePush(name, w, sf, authConfig); err != nil {
if sf.Used() {
w.Write(sf.FormatError(err))
return nil
@@ -880,7 +878,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) {
localMethod := method
localFct := fct
f := func(w http.ResponseWriter, r *http.Request) {
utils.Debugf("Calling %s %s", localMethod, localRoute)
utils.Debugf("Calling %s %s from %s", localMethod, localRoute, r.RemoteAddr)
if logging {
log.Println(r.Method, r.RequestURI)
components/engine/api_test.go
+26 -86
View File
@@ -5,7 +5,6 @@ import (
"bufio"
"bytes"
"encoding/json"
"github.com/dotcloud/docker/auth"
"github.com/dotcloud/docker/utils"
"io"
"net"
@@ -41,44 +40,6 @@ func TestGetBoolParam(t *testing.T) {
}
}
func TestPostAuth(t *testing.T) {
runtime, err := newTestRuntime()
if err != nil {
t.Fatal(err)
}
defer nuke(runtime)
srv := &Server{
runtime: runtime,
}
r := httptest.NewRecorder()
authConfig := &auth.AuthConfig{
Username: "utest",
Password: "utest",
Email: "utest@yopmail.com",
}
authConfigJSON, err := json.Marshal(authConfig)
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest("POST", "/auth", bytes.NewReader(authConfigJSON))
if err != nil {
t.Fatal(err)
}
if err := postAuth(srv, APIVERSION, r, req, nil); err != nil {
t.Fatal(err)
}
if r.Code != http.StatusOK && r.Code != 0 {
t.Fatalf("%d OK or 0 expected, received %d\n", http.StatusOK, r.Code)
}
}
func TestGetVersion(t *testing.T) {
runtime, err := newTestRuntime()
if err != nil {
@@ -99,7 +60,7 @@ func TestGetVersion(t *testing.T) {
t.Fatal(err)
}
if v.Version != VERSION {
t.Errorf("Excepted version %s, %s found", VERSION, v.Version)
t.Errorf("Expected version %s, %s found", VERSION, v.Version)
}
}
@@ -129,7 +90,7 @@ func TestGetInfo(t *testing.T) {
t.Fatal(err)
}
if infos.Images != len(initialImages) {
t.Errorf("Excepted images: %d, %d found", len(initialImages), infos.Images)
t.Errorf("Expected images: %d, %d found", len(initialImages), infos.Images)
}
}
@@ -166,7 +127,7 @@ func TestGetImagesJSON(t *testing.T) {
}
if len(images) != len(initialImages) {
t.Errorf("Excepted %d image, %d found", len(initialImages), len(images))
t.Errorf("Expected %d image, %d found", len(initialImages), len(images))
}
found := false
@@ -177,7 +138,7 @@ func TestGetImagesJSON(t *testing.T) {
}
}
if !found {
t.Errorf("Excepted image %s, %+v found", unitTestImageName, images)
t.Errorf("Expected image %s, %+v found", unitTestImageName, images)
}
r2 := httptest.NewRecorder()
@@ -204,7 +165,7 @@ func TestGetImagesJSON(t *testing.T) {
}
if len(images2) != len(initialImages) {
t.Errorf("Excepted %d image, %d found", len(initialImages), len(images2))
t.Errorf("Expected %d image, %d found", len(initialImages), len(images2))
}
found = false
@@ -236,7 +197,7 @@ func TestGetImagesJSON(t *testing.T) {
}
if len(images3) != 0 {
t.Errorf("Excepted 0 image, %d found", len(images3))
t.Errorf("Expected 0 image, %d found", len(images3))
}
r4 := httptest.NewRecorder()
@@ -282,38 +243,7 @@ func TestGetImagesViz(t *testing.T) {
t.Fatal(err)
}
if line != "digraph docker {\n" {
t.Errorf("Excepted digraph docker {\n, %s found", line)
}
}
func TestGetImagesSearch(t *testing.T) {
runtime, err := newTestRuntime()
if err != nil {
t.Fatal(err)
}
defer nuke(runtime)
srv := &Server{
runtime: runtime,
}
r := httptest.NewRecorder()
req, err := http.NewRequest("GET", "/images/search?term=redis", nil)
if err != nil {
t.Fatal(err)
}
if err := getImagesSearch(srv, APIVERSION, r, req, nil); err != nil {
t.Fatal(err)
}
results := []APISearch{}
if err := json.Unmarshal(r.Body.Bytes(), &results); err != nil {
t.Fatal(err)
}
if len(results) < 2 {
t.Errorf("Excepted at least 2 lines, %d found", len(results))
t.Errorf("Expected digraph docker {\n, %s found", line)
}
}
@@ -337,7 +267,7 @@ func TestGetImagesHistory(t *testing.T) {
t.Fatal(err)
}
if len(history) != 1 {
t.Errorf("Excepted 1 line, %d found", len(history))
t.Errorf("Expected 1 line, %d found", len(history))
}
}
@@ -359,7 +289,7 @@ func TestGetImagesByName(t *testing.T) {
if err := json.Unmarshal(r.Body.Bytes(), img); err != nil {
t.Fatal(err)
}
if img.ID != unitTestImageId {
if img.ID != unitTestImageID {
t.Errorf("Error inspecting image")
}
}
@@ -396,7 +326,7 @@ func TestGetContainersJSON(t *testing.T) {
t.Fatal(err)
}
if len(containers) != 1 {
t.Fatalf("Excepted %d container, %d found", 1, len(containers))
t.Fatalf("Expected %d container, %d found", 1, len(containers))
}
if containers[0].ID != container.ID {
t.Fatalf("Container ID mismatch. Expected: %s, received: %s\n", container.ID, containers[0].ID)
@@ -1356,24 +1286,34 @@ func TestDeleteImages(t *testing.T) {
}
if len(images) != len(initialImages)+1 {
t.Errorf("Excepted %d images, %d found", len(initialImages)+1, len(images))
t.Errorf("Expected %d images, %d found", len(initialImages)+1, len(images))
}
req, err := http.NewRequest("DELETE", "/images/test:test", nil)
req, err := http.NewRequest("DELETE", "/images/"+unitTestImageID, nil)
if err != nil {
t.Fatal(err)
}
r := httptest.NewRecorder()
if err := deleteImages(srv, APIVERSION, r, req, map[string]string{"name": "test:test"}); err != nil {
if err := deleteImages(srv, APIVERSION, r, req, map[string]string{"name": unitTestImageID}); err == nil {
t.Fatalf("Expected conflict error, got none")
}
req2, err := http.NewRequest("DELETE", "/images/test:test", nil)
if err != nil {
t.Fatal(err)
}
if r.Code != http.StatusOK {
r2 := httptest.NewRecorder()
if err := deleteImages(srv, APIVERSION, r2, req2, map[string]string{"name": "test:test"}); err != nil {
t.Fatal(err)
}
if r2.Code != http.StatusOK {
t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code)
}
var outs []APIRmi
if err := json.Unmarshal(r.Body.Bytes(), &outs); err != nil {
if err := json.Unmarshal(r2.Body.Bytes(), &outs); err != nil {
t.Fatal(err)
}
if len(outs) != 1 {
@@ -1385,7 +1325,7 @@ func TestDeleteImages(t *testing.T) {
}
if len(images) != len(initialImages) {
t.Errorf("Excepted %d image, %d found", len(initialImages), len(images))
t.Errorf("Expected %d image, %d found", len(initialImages), len(images))
}
/* if c := runtime.Get(container.Id); c != nil {
components/engine/archive.go
+15 -12
View File
@@ -2,9 +2,7 @@ package docker
import (
"archive/tar"
"bufio"
"bytes"
"errors"
"fmt"
"github.com/dotcloud/docker/utils"
"io"
@@ -27,10 +25,6 @@ const (
)
func DetectCompression(source []byte) Compression {
for _, c := range source[:10] {
utils.Debugf("%x", c)
}
sourceLen := len(source)
for compression, m := range map[Compression][]byte{
Bzip2: {0x42, 0x5A, 0x68},
@@ -111,17 +105,26 @@ func Untar(archive io.Reader, path string) error {
if archive == nil {
return fmt.Errorf("Empty archive")
}
bufferedArchive := bufio.NewReaderSize(archive, 10)
buf, err := bufferedArchive.Peek(10)
if err != nil {
return err
buf := make([]byte, 10)
totalN := 0
for totalN < 10 {
if n, err := archive.Read(buf[totalN:]); err != nil {
if err == io.EOF {
return fmt.Errorf("Tarball too short")
}
return err
} else {
totalN += n
utils.Debugf("[tar autodetect] n: %d", n)
}
}
compression := DetectCompression(buf)
utils.Debugf("Archive compression detected: %s", compression.Extension())
cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag())
cmd.Stdin = bufferedArchive
cmd.Stdin = io.MultiReader(bytes.NewReader(buf), archive)
// Hardcode locale environment for predictable outcome regardless of host configuration.
// (see https://github.com/dotcloud/docker/issues/355)
cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}
@@ -251,7 +254,7 @@ func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
}
errText := <-errChan
if err := cmd.Wait(); err != nil {
pipeW.CloseWithError(errors.New(err.Error() + ": " + string(errText)))
pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errText))
} else {
pipeW.Close()
}
components/engine/archive_test.go
+4 -4
View File
@@ -16,7 +16,7 @@ func TestCmdStreamLargeStderr(t *testing.T) {
cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
out, err := CmdStream(cmd)
if err != nil {
t.Fatalf("Failed to start command: " + err.Error())
t.Fatalf("Failed to start command: %s", err)
}
errCh := make(chan error)
go func() {
@@ -26,7 +26,7 @@ func TestCmdStreamLargeStderr(t *testing.T) {
select {
case err := <-errCh:
if err != nil {
t.Fatalf("Command should not have failed (err=%s...)", err.Error()[:100])
t.Fatalf("Command should not have failed (err=%.100s...)", err)
}
case <-time.After(5 * time.Second):
t.Fatalf("Command did not complete in 5 seconds; probable deadlock")
@@ -37,12 +37,12 @@ func TestCmdStreamBad(t *testing.T) {
badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
out, err := CmdStream(badCmd)
if err != nil {
t.Fatalf("Failed to start command: " + err.Error())
t.Fatalf("Failed to start command: %s", err)
}
if output, err := ioutil.ReadAll(out); err == nil {
t.Fatalf("Command should have failed")
} else if err.Error() != "exit status 1: error couldn't reverse the phase pulser\n" {
t.Fatalf("Wrong error value (%s)", err.Error())
t.Fatalf("Wrong error value (%s)", err)
} else if s := string(output); s != "hello\n" {
t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
}
components/engine/auth/auth.go
+4 -7
View File
@@ -15,8 +15,8 @@ import (
// Where we store the config file
const CONFIGFILE = ".dockercfg"
// the registry server we want to login against
const INDEXSERVER = "https://index.docker.io/v1"
// Only used for user auth + account creation
const INDEXSERVER = "https://index.docker.io/v1/"
//const INDEXSERVER = "http://indexstaging-docker.dotcloud.com/"
@@ -41,9 +41,6 @@ func NewAuthConfig(username, password, email, rootPath string) *AuthConfig {
}
func IndexServerAddress() string {
if os.Getenv("DOCKER_INDEX_URL") != "" {
return os.Getenv("DOCKER_INDEX_URL") + "/v1"
}
return INDEXSERVER
}
@@ -132,7 +129,7 @@ func Login(authConfig *AuthConfig, store bool) (string, error) {
// using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status.
b := strings.NewReader(string(jsonBody))
req1, err := http.Post(IndexServerAddress()+"/users/", "application/json; charset=utf-8", b)
req1, err := http.Post(IndexServerAddress()+"users/", "application/json; charset=utf-8", b)
if err != nil {
return "", fmt.Errorf("Server Error: %s", err)
}
@@ -152,7 +149,7 @@ func Login(authConfig *AuthConfig, store bool) (string, error) {
"Please check your e-mail for a confirmation link.")
} else if reqStatusCode == 400 {
if string(reqBody) == "\"Username or email already exists\"" {
req, err := http.NewRequest("GET", IndexServerAddress()+"/users/", nil)
req, err := http.NewRequest("GET", IndexServerAddress()+"users/", nil)
req.SetBasicAuth(authConfig.Username, authConfig.Password)
resp, err := client.Do(req)
if err != nil {
components/engine/auth/auth_test.go
+1 -1
View File
@@ -68,6 +68,6 @@ func TestCreateAccount(t *testing.T) {
expectedError := "Login: Account is not Active"
if !strings.Contains(err.Error(), expectedError) {
t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err.Error())
t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err)
}
}
components/engine/buildfile.go
+55 -12
View File
@@ -29,6 +29,7 @@ type buildFile struct {
config *Config
context string
lastContainer *Container
tmpContainers map[string]struct{}
tmpImages map[string]struct{}
@@ -51,20 +52,10 @@ func (b *buildFile) CmdFrom(name string) error {
image, err := b.runtime.repositories.LookupImage(name)
if err != nil {
if b.runtime.graph.IsNotExist(err) {
var tag, remote string
if strings.Contains(name, ":") {
remoteParts := strings.Split(name, ":")
tag = remoteParts[1]
remote = remoteParts[0]
} else {
remote = name
}
if err := b.srv.ImagePull(remote, tag, "", b.out, utils.NewStreamFormatter(false), nil); err != nil {
remote, tag := utils.ParseRepositoryTag(name)
if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil); err != nil {
return err
}
image, err = b.runtime.repositories.LookupImage(name)
if err != nil {
return err
@@ -182,6 +173,27 @@ func (b *buildFile) CmdEntrypoint(args string) error {
return nil
}
func (b *buildFile) CmdVolume(args string) error {
if args == "" {
return fmt.Errorf("Volume cannot be empty")
}
var volume []string
if err := json.Unmarshal([]byte(args), &volume); err != nil {
volume = []string{args}
}
if b.config.Volumes == nil {
b.config.Volumes = NewPathOpts()
}
for _, v := range volume {
b.config.Volumes[v] = struct{}{}
}
if err := b.commit("", b.config.Cmd, fmt.Sprintf("VOLUME %s", args)); err != nil {
return err
}
return nil
}
func (b *buildFile) addRemote(container *Container, orig, dest string) error {
file, err := utils.Download(orig, ioutil.Discard)
if err != nil {
@@ -242,6 +254,7 @@ func (b *buildFile) CmdAdd(args string) error {
return err
}
b.tmpContainers[container.ID] = struct{}{}
b.lastContainer = container
if err := container.EnsureMounted(); err != nil {
return err
@@ -277,8 +290,13 @@ func (b *buildFile) run() (string, error) {
return "", err
}
b.tmpContainers[c.ID] = struct{}{}
b.lastContainer = c
fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(c.ID))
// override the entry point that may have been picked up from the base image
c.Path = b.config.Cmd[0]
c.Args = b.config.Cmd[1:]
//start the container
hostConfig := &HostConfig{}
if err := c.Start(hostConfig); err != nil {
@@ -319,6 +337,7 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error {
return err
}
b.tmpContainers[container.ID] = struct{}{}
b.lastContainer = container
fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID))
id = container.ID
if err := container.EnsureMounted(); err != nil {
@@ -346,6 +365,29 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error {
}
func (b *buildFile) Build(context io.Reader) (string, error) {
defer func() {
// If we have an error and a container, the display the logs
if b.lastContainer != nil {
fmt.Fprintf(b.out, "******** Logs from last container (%s) *******\n", b.lastContainer.ShortID())
cLog, err := b.lastContainer.ReadLog("stdout")
if err != nil {
utils.Debugf("Error reading logs (stdout): %s", err)
}
if _, err := io.Copy(b.out, cLog); err != nil {
utils.Debugf("Error streaming logs (stdout): %s", err)
}
cLog, err = b.lastContainer.ReadLog("stderr")
if err != nil {
utils.Debugf("Error reading logs (stderr): %s", err)
}
if _, err := io.Copy(b.out, cLog); err != nil {
utils.Debugf("Error streaming logs (stderr): %s", err)
}
fmt.Fprintf(b.out, "************* End of logs for %s *************\n", b.lastContainer.ShortID())
}
}()
// FIXME: @creack any reason for using /tmp instead of ""?
// FIXME: @creack "name" is a terrible variable name
name, err := ioutil.TempDir("/tmp", "docker-build")
@@ -398,6 +440,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) {
return "", ret.(error)
}
b.lastContainer = nil
fmt.Fprintf(b.out, " ---> %v\n", utils.TruncateID(b.image))
}
if b.image != "" {
components/engine/buildfile_test.go
+48 -5
View File
@@ -3,14 +3,13 @@ package docker
import (
"fmt"
"io/ioutil"
"sync"
"testing"
)
// mkTestContext generates a build context from the contents of the provided dockerfile.
// This context is suitable for use as an argument to BuildFile.Build()
func mkTestContext(dockerfile string, files [][2]string, t *testing.T) Archive {
context, err := mkBuildContext(fmt.Sprintf(dockerfile, unitTestImageId), files)
context, err := mkBuildContext(fmt.Sprintf(dockerfile, unitTestImageID), files)
if err != nil {
t.Fatal(err)
}
@@ -27,7 +26,7 @@ type testContextTemplate struct {
// A table of all the contexts to build and test.
// A new docker runtime will be created and torn down for each context.
var testContexts []testContextTemplate = []testContextTemplate{
var testContexts = []testContextTemplate{
{
`
from %s
@@ -85,9 +84,18 @@ run [ "$FOO" = "BAR" ]
{
`
from docker-ut
from %s
ENTRYPOINT /bin/echo
CMD Hello world
`,
nil,
},
{
`
from %s
VOLUME /test
CMD Hello world
`,
nil,
},
@@ -105,7 +113,6 @@ func TestBuild(t *testing.T) {
srv := &Server{
runtime: runtime,
lock: &sync.Mutex{},
pullingPool: make(map[string]struct{}),
pushingPool: make(map[string]struct{}),
}
@@ -116,3 +123,39 @@ func TestBuild(t *testing.T) {
}
}
}
func TestVolume(t *testing.T) {
runtime, err := newTestRuntime()
if err != nil {
t.Fatal(err)
}
defer nuke(runtime)
srv := &Server{
runtime: runtime,
pullingPool: make(map[string]struct{}),
pushingPool: make(map[string]struct{}),
}
buildfile := NewBuildFile(srv, ioutil.Discard)
imgId, err := buildfile.Build(mkTestContext(`
from %s
VOLUME /test
CMD Hello world
`, nil, t))
if err != nil {
t.Fatal(err)
}
img, err := srv.ImageInspect(imgId)
if err != nil {
t.Fatal(err)
}
if len(img.Config.Volumes) == 0 {
t.Fail()
}
for key, _ := range img.Config.Volumes {
if key != "/test" {
t.Fail()
}
}
}
components/engine/commands.go
+42 -43
View File
@@ -19,7 +19,6 @@ import (
"os/signal"
"path/filepath"
"reflect"
"regexp"
"strconv"
"strings"
"syscall"
@@ -73,7 +72,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error {
return nil
}
}
help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=[tcp://%s:%d]: tcp://host:port to bind/connect to or unix://path/to/socker to use\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTHTTPHOST, DEFAULTHTTPPORT)
help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=[tcp://%s:%d]: tcp://host:port to bind/connect to or unix://path/to/socket to use\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTHTTPHOST, DEFAULTHTTPPORT)
for _, command := range [][]string{
{"attach", "Attach to a running container"},
{"build", "Build a container from a Dockerfile"},
@@ -730,7 +729,6 @@ func (cli *DockerCli) CmdImport(args ...string) error {
func (cli *DockerCli) CmdPush(args ...string) error {
cmd := Subcmd("push", "[OPTION] NAME", "Push an image or a repository to the registry")
registry := cmd.String("registry", "", "Registry host to push the image to")
if err := cmd.Parse(args); err != nil {
return nil
}
@@ -741,28 +739,16 @@ func (cli *DockerCli) CmdPush(args ...string) error {
return nil
}
if err := cli.checkIfLogged(*registry == "", "push"); err != nil {
if err := cli.checkIfLogged("push"); err != nil {
return err
}
if *registry == "" {
// If we're not using a custom registry, we know the restrictions
// applied to repository names and can warn the user in advance.
// Custom repositories can have different rules, and we must also
// allow pushing by image ID.
if len(strings.SplitN(name, "/", 2)) == 1 {
return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in <user>/<repo> (ex: %s/%s)", cli.authConfig.Username, name)
}
nameParts := strings.SplitN(name, "/", 2)
validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`)
if !validNamespace.MatchString(nameParts[0]) {
return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0])
}
validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`)
if !validRepo.MatchString(nameParts[1]) {
return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1])
}
// If we're not using a custom registry, we know the restrictions
// applied to repository names and can warn the user in advance.
// Custom repositories can have different rules, and we must also
// allow pushing by image ID.
if len(strings.SplitN(name, "/", 2)) == 1 {
return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in <user>/<repo> (ex: %s/%s)", cli.authConfig.Username, name)
}
buf, err := json.Marshal(cli.authConfig)
@@ -771,7 +757,6 @@ func (cli *DockerCli) CmdPush(args ...string) error {
}
v := url.Values{}
v.Set("registry", *registry)
if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out); err != nil {
return err
}
@@ -781,7 +766,6 @@ func (cli *DockerCli) CmdPush(args ...string) error {
func (cli *DockerCli) CmdPull(args ...string) error {
cmd := Subcmd("pull", "NAME", "Pull an image or a repository from the registry")
tag := cmd.String("t", "", "Download tagged image in repository")
registry := cmd.String("registry", "", "Registry to download from. Necessary if image is pulled by ID")
if err := cmd.Parse(args); err != nil {
return nil
}
@@ -791,17 +775,12 @@ func (cli *DockerCli) CmdPull(args ...string) error {
return nil
}
remote := cmd.Arg(0)
if strings.Contains(remote, ":") {
remoteParts := strings.Split(remote, ":")
tag = &remoteParts[1]
remote = remoteParts[0]
}
remote, parsedTag := utils.ParseRepositoryTag(cmd.Arg(0))
*tag = parsedTag
v := url.Values{}
v.Set("fromImage", remote)
v.Set("tag", *tag)
v.Set("registry", *registry)
if err := cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.out); err != nil {
return err
@@ -1127,6 +1106,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error {
func (cli *DockerCli) CmdSearch(args ...string) error {
cmd := Subcmd("search", "NAME", "Search the docker index for images")
noTrunc := cmd.Bool("notrunc", false, "Don't truncate output")
if err := cmd.Parse(args); err != nil {
return nil
}
@@ -1148,13 +1128,19 @@ func (cli *DockerCli) CmdSearch(args ...string) error {
return err
}
fmt.Fprintf(cli.out, "Found %d results matching your query (\"%s\")\n", len(outs), cmd.Arg(0))
w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
w := tabwriter.NewWriter(cli.out, 33, 1, 3, ' ', 0)
fmt.Fprintf(w, "NAME\tDESCRIPTION\n")
_, width := cli.getTtySize()
if width == 0 {
width = 45
} else {
width = width - 33 //remote the first column
}
for _, out := range outs {
desc := strings.Replace(out.Description, "\n", " ", -1)
desc = strings.Replace(desc, "\r", " ", -1)
if len(desc) > 45 {
desc = utils.Trunc(desc, 42) + "..."
if !*noTrunc && len(desc) > width {
desc = utils.Trunc(desc, width-3) + "..."
}
fmt.Fprintf(w, "%s\t%s\n", out.Name, desc)
}
@@ -1265,7 +1251,9 @@ func (cli *DockerCli) CmdRun(args ...string) error {
//if image not found try to pull it
if statusCode == 404 {
v := url.Values{}
v.Set("fromImage", config.Image)
repos, tag := utils.ParseRepositoryTag(config.Image)
v.Set("fromImage", repos)
v.Set("tag", tag)
err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err)
if err != nil {
return err
@@ -1286,7 +1274,7 @@ func (cli *DockerCli) CmdRun(args ...string) error {
}
for _, warning := range runResult.Warnings {
fmt.Fprintln(cli.err, "WARNING: ", warning)
fmt.Fprintf(cli.err, "WARNING: %s\n", warning)
}
//start the container
@@ -1338,9 +1326,9 @@ func (cli *DockerCli) CmdRun(args ...string) error {
return nil
}
func (cli *DockerCli) checkIfLogged(condition bool, action string) error {
func (cli *DockerCli) checkIfLogged(action string) error {
// If condition AND the login failed
if condition && cli.authConfig.Username == "" {
if cli.authConfig.Username == "" {
if err := cli.CmdLogin(""); err != nil {
return err
}
@@ -1528,17 +1516,28 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea
}
func (cli *DockerCli) resizeTty(id string) {
func (cli *DockerCli) getTtySize() (int, int) {
if !cli.isTerminal {
return
return 0, 0
}
ws, err := term.GetWinsize(cli.terminalFd)
if err != nil {
utils.Debugf("Error getting size: %s", err)
if ws == nil {
return 0, 0
}
}
return int(ws.Height), int(ws.Width)
}
func (cli *DockerCli) resizeTty(id string) {
height, width := cli.getTtySize()
if height == 0 && width == 0 {
return
}
v := url.Values{}
v.Set("h", strconv.Itoa(int(ws.Height)))
v.Set("w", strconv.Itoa(int(ws.Width)))
v.Set("h", strconv.Itoa(height))
v.Set("w", strconv.Itoa(width))
if _, _, err := cli.call("POST", "/containers/"+id+"/resize?"+v.Encode(), nil); err != nil {
utils.Debugf("Error resize: %s", err)
}
@@ -1574,7 +1573,7 @@ func Subcmd(name, signature, description string) *flag.FlagSet {
func NewDockerCli(in io.ReadCloser, out, err io.Writer, proto, addr string) *DockerCli {
var (
isTerminal bool = false
isTerminal = false
terminalFd uintptr
)
components/engine/commands_test.go
+5 -4
View File
@@ -132,17 +132,18 @@ func TestImages(t *testing.T) {
}
*/
// TestRunHostname checks that 'docker run -h' correctly sets a custom hostname
func TestRunHostname(t *testing.T) {
stdout, stdoutPipe := io.Pipe()
cli := NewDockerCli(nil, stdoutPipe, nil, testDaemonProto, testDaemonAddr)
cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
defer cleanup(globalRuntime)
c := make(chan struct{})
go func() {
defer close(c)
if err := cli.CmdRun("-h", "foobar", unitTestImageId, "hostname"); err != nil {
if err := cli.CmdRun("-h", "foobar", unitTestImageID, "hostname"); err != nil {
t.Fatal(err)
}
}()
@@ -329,13 +330,13 @@ func TestRunAttachStdin(t *testing.T) {
stdin, stdinPipe := io.Pipe()
stdout, stdoutPipe := io.Pipe()
cli := NewDockerCli(stdin, stdoutPipe, nil, testDaemonProto, testDaemonAddr)
cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
defer cleanup(globalRuntime)
ch := make(chan struct{})
go func() {
defer close(ch)
cli.CmdRun("-i", "-a", "stdin", unitTestImageId, "sh", "-c", "echo hello && cat")
cli.CmdRun("-i", "-a", "stdin", unitTestImageID, "sh", "-c", "echo hello && cat")
}()
// Send input to the command, close stdin
components/engine/container.go
+21 -12
View File
@@ -202,20 +202,25 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
return config, hostConfig, cmd, nil
}
type PortMapping map[string]string
type NetworkSettings struct {
IPAddress string
IPPrefixLen int
Gateway string
Bridge string
PortMapping map[string]string
PortMapping map[string]PortMapping
}
// String returns a human-readable description of the port mapping defined in the settings
func (settings *NetworkSettings) PortMappingHuman() string {
var mapping []string
for private, public := range settings.PortMapping {
for private, public := range settings.PortMapping["Tcp"] {
mapping = append(mapping, fmt.Sprintf("%s->%s", public, private))
}
for private, public := range settings.PortMapping["Udp"] {
mapping = append(mapping, fmt.Sprintf("%s->%s/udp", public, private))
}
sort.Strings(mapping)
return strings.Join(mapping, ", ")
}
@@ -466,8 +471,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
}
func (container *Container) Start(hostConfig *HostConfig) error {
container.State.lock()
defer container.State.unlock()
container.State.Lock()
defer container.State.Unlock()
if container.State.Running {
return fmt.Errorf("The container %s is already running.", container.ID)
@@ -494,7 +499,7 @@ func (container *Container) Start(hostConfig *HostConfig) error {
// Create the requested bind mounts
binds := make(map[string]BindMap)
// Define illegal container destinations
illegal_dsts := []string{"/", "."}
illegalDsts := []string{"/", "."}
for _, bind := range hostConfig.Binds {
// FIXME: factorize bind parsing in parseBind
@@ -513,7 +518,7 @@ func (container *Container) Start(hostConfig *HostConfig) error {
}
// Bail if trying to mount to an illegal destination
for _, illegal := range illegal_dsts {
for _, illegal := range illegalDsts {
if dst == illegal {
return fmt.Errorf("Illegal bind destination: %s", dst)
}
@@ -688,14 +693,18 @@ func (container *Container) allocateNetwork() error {
if err != nil {
return err
}
container.NetworkSettings.PortMapping = make(map[string]string)
container.NetworkSettings.PortMapping = make(map[string]PortMapping)
container.NetworkSettings.PortMapping["Tcp"] = make(PortMapping)
container.NetworkSettings.PortMapping["Udp"] = make(PortMapping)
for _, spec := range container.Config.PortSpecs {
nat, err := iface.AllocatePort(spec)
if err != nil {
iface.Release()
return err
}
container.NetworkSettings.PortMapping[strconv.Itoa(nat.Backend)] = strconv.Itoa(nat.Frontend)
proto := strings.Title(nat.Proto)
backend, frontend := strconv.Itoa(nat.Backend), strconv.Itoa(nat.Frontend)
container.NetworkSettings.PortMapping[proto][backend] = frontend
}
container.network = iface
container.NetworkSettings.Bridge = container.runtime.networkManager.bridgeIface
@@ -821,8 +830,8 @@ func (container *Container) kill() error {
}
func (container *Container) Kill() error {
container.State.lock()
defer container.State.unlock()
container.State.Lock()
defer container.State.Unlock()
if !container.State.Running {
return nil
}
@@ -830,8 +839,8 @@ func (container *Container) Kill() error {
}
func (container *Container) Stop(seconds int) error {
container.State.lock()
defer container.State.unlock()
container.State.Lock()
defer container.State.Unlock()
if !container.State.Running {
return nil
}
components/engine/docker/docker.go
+1 -1
View File
@@ -37,7 +37,7 @@ func main() {
flag.Var(&flHosts, "H", "tcp://host:port to bind/connect to or unix://path/to/socket to use")
flag.Parse()
if len(flHosts) > 1 {
flHosts = flHosts[1:len(flHosts)] //trick to display a nice defaul value in the usage
flHosts = flHosts[1:] //trick to display a nice defaul value in the usage
}
for i, flHost := range flHosts {
flHosts[i] = utils.ParseHost(docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT, flHost)
components/engine/docs/sources/contributing/devenvironment.rst
+35 -36
View File
@@ -5,53 +5,52 @@
Setting Up a Dev Environment
============================
Instructions that have been verified to work on Ubuntu Precise 12.04 (LTS) (64-bit),
To make it easier to contribute to Docker, we provide a standard development environment. It is important that
the same environment be used for all tests, builds and releases. The standard development environment defines
all build dependencies: system libraries and binaries, go environment, go dependencies, etc.
Dependencies
------------
Step 1: install docker
----------------------
**Linux kernel 3.8**
Docker's build environment itself is a docker container, so the first step is to install docker on your system.
Due to a bug in LXC docker works best on the 3.8 kernel. Precise comes with a 3.2 kernel, so we need to upgrade it. The kernel we install comes with AUFS built in.
You can follow the `install instructions most relevant to your system <https://docs.docker.io/en/latest/installation/>`.
Make sure you have a working, up-to-date docker installation, then continue to the next step.
.. code-block:: bash
Step 2: check out the source
----------------------------
# install the backported kernel
sudo apt-get update && sudo apt-get install linux-image-generic-lts-raring
::
# reboot
sudo reboot
Installation
------------
.. code-block:: bash
sudo apt-get install python-software-properties
sudo add-apt-repository ppa:gophers/go
sudo apt-get update
sudo apt-get -y install lxc xz-utils curl golang-stable git aufs-tools
export GOPATH=~/go/
export PATH=$GOPATH/bin:$PATH
mkdir -p $GOPATH/src/github.com/dotcloud
cd $GOPATH/src/github.com/dotcloud
git clone git://github.com/dotcloud/docker.git
git clone http://git@github.com/dotcloud/docker
cd docker
go get -v github.com/dotcloud/docker/...
go install -v github.com/dotcloud/docker/...
Step 3: build
-------------
When you are ready to build docker, run this command:
::
docker build -t docker .
This will build the revision currently checked out in the repository. Feel free to check out the version
of your choice.
If the build is successful, congratulations! You have produced a clean build of docker, neatly encapsulated
in a standard build environment.
You can run an interactive session in the newly built container:
::
docker run -i -t docker bash
Then run the docker daemon,
To extract the binaries from the container:
.. code-block:: bash
::
docker run docker sh -c 'cat $(which docker)' > docker-build && chmod +x docker-build
sudo $GOPATH/bin/docker -d
Run the ``go install`` command (above) to recompile docker.
components/engine/docs/sources/use/builder.rst
+7
View File
@@ -160,6 +160,13 @@ files and directories are created with mode 0700, uid and gid 0.
The `ENTRYPOINT` instruction adds an entry command that will not be overwritten when arguments are passed to docker run, unlike the behavior of `CMD`. This allows arguments to be passed to the entrypoint. i.e. `docker run <image> -d` will pass the "-d" argument to the entrypoint.
2.9 VOLUME
----------
``VOLUME ["/data"]``
The `VOLUME` instruction will add one or more new volumes to any container created from the image.
3. Dockerfile Examples
======================
components/engine/graph.go
+1 -2
View File
@@ -162,7 +162,7 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error {
// The archive is stored on disk and will be automatically deleted as soon as has been read.
// If output is not nil, a human-readable progress bar will be written to it.
// FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives?
func (graph *Graph) TempLayerArchive(id string, compression Compression, output io.Writer) (*TempArchive, error) {
func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *utils.StreamFormatter, output io.Writer) (*TempArchive, error) {
image, err := graph.Get(id)
if err != nil {
return nil, err
@@ -175,7 +175,6 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, output
if err != nil {
return nil, err
}
sf := utils.NewStreamFormatter(false)
return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("Buffering to disk", "%v/%v (%v)"), sf), tmp.Root)
}
components/engine/hack/dockerbuilder/Dockerfile
+4 -1
View File
@@ -5,8 +5,11 @@
# AUTHOR Solomon Hykes <solomon@dotcloud.com>
# Daniel Mizyrycki <daniel@dotcloud.net>
# BUILD_CMD docker build -t dockerbuilder .
# RUN_CMD docker run -e AWS_ID="$AWS_ID" -e AWS_KEY="$AWS_KEY" -e GPG_KEY="$GPG_KEY" dockerbuilder
# RUN_CMD docker run -e AWS_ID="$AWS_ID" -e AWS_KEY="$AWS_KEY" -e GPG_KEY="$GPG_KEY" -e PUBLISH_PPA="$PUBLISH_PPA" dockerbuilder
#
# ENV_VARIABLES AWS_ID, AWS_KEY: S3 credentials for uploading Docker binary and tarball
# GPG_KEY: Signing key for docker package
# PUBLISH_PPA: 0 for staging release, 1 for production release
#
from ubuntu:12.04
maintainer Solomon Hykes <solomon@dotcloud.com>
components/engine/image.go
+3
View File
@@ -94,9 +94,12 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error {
}
// If layerData is not nil, unpack it into the new layer
if layerData != nil {
start := time.Now()
utils.Debugf("Start untar layer")
if err := Untar(layerData, layer); err != nil {
return err
}
utils.Debugf("Untar time: %vs\n", time.Now().Sub(start).Seconds())
}
return StoreSize(img, root)
components/engine/network.go
+136 -97
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"github.com/dotcloud/docker/utils"
"io"
"log"
"net"
"os/exec"
@@ -183,8 +182,10 @@ func getIfaceAddr(name string) (net.Addr, error) {
// up iptables rules.
// It keeps track of all mappings and is able to unmap at will
type PortMapper struct {
mapping map[int]net.TCPAddr
proxies map[int]net.Listener
tcpMapping map[int]*net.TCPAddr
tcpProxies map[int]Proxy
udpMapping map[int]*net.UDPAddr
udpProxies map[int]Proxy
}
func (mapper *PortMapper) cleanup() error {
@@ -197,8 +198,10 @@ func (mapper *PortMapper) cleanup() error {
iptables("-t", "nat", "-D", "OUTPUT", "-j", "DOCKER")
iptables("-t", "nat", "-F", "DOCKER")
iptables("-t", "nat", "-X", "DOCKER")
mapper.mapping = make(map[int]net.TCPAddr)
mapper.proxies = make(map[int]net.Listener)
mapper.tcpMapping = make(map[int]*net.TCPAddr)
mapper.tcpProxies = make(map[int]Proxy)
mapper.udpMapping = make(map[int]*net.UDPAddr)
mapper.udpProxies = make(map[int]Proxy)
return nil
}
@@ -215,76 +218,72 @@ func (mapper *PortMapper) setup() error {
return nil
}
func (mapper *PortMapper) iptablesForward(rule string, port int, dest net.TCPAddr) error {
return iptables("-t", "nat", rule, "DOCKER", "-p", "tcp", "--dport", strconv.Itoa(port),
"-j", "DNAT", "--to-destination", net.JoinHostPort(dest.IP.String(), strconv.Itoa(dest.Port)))
func (mapper *PortMapper) iptablesForward(rule string, port int, proto string, dest_addr string, dest_port int) error {
return iptables("-t", "nat", rule, "DOCKER", "-p", proto, "--dport", strconv.Itoa(port),
"-j", "DNAT", "--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port)))
}
func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error {
if err := mapper.iptablesForward("-A", port, dest); err != nil {
return err
func (mapper *PortMapper) Map(port int, backendAddr net.Addr) error {
if _, isTCP := backendAddr.(*net.TCPAddr); isTCP {
backendPort := backendAddr.(*net.TCPAddr).Port
backendIP := backendAddr.(*net.TCPAddr).IP
if err := mapper.iptablesForward("-A", port, "tcp", backendIP.String(), backendPort); err != nil {
return err
}
mapper.tcpMapping[port] = backendAddr.(*net.TCPAddr)
proxy, err := NewProxy(&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}, backendAddr)
if err != nil {
mapper.Unmap(port, "tcp")
return err
}
mapper.tcpProxies[port] = proxy
go proxy.Run()
} else {
backendPort := backendAddr.(*net.UDPAddr).Port
backendIP := backendAddr.(*net.UDPAddr).IP
if err := mapper.iptablesForward("-A", port, "udp", backendIP.String(), backendPort); err != nil {
return err
}
mapper.udpMapping[port] = backendAddr.(*net.UDPAddr)
proxy, err := NewProxy(&net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}, backendAddr)
if err != nil {
mapper.Unmap(port, "udp")
return err
}
mapper.udpProxies[port] = proxy
go proxy.Run()
}
mapper.mapping[port] = dest
listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
if err != nil {
mapper.Unmap(port)
return err
}
mapper.proxies[port] = listener
go proxy(listener, "tcp", dest.String())
return nil
}
// proxy listens for socket connections on `listener`, and forwards them unmodified
// to `proto:address`
func proxy(listener net.Listener, proto, address string) error {
utils.Debugf("proxying to %s:%s", proto, address)
defer utils.Debugf("Done proxying to %s:%s", proto, address)
for {
utils.Debugf("Listening on %s", listener)
src, err := listener.Accept()
if err != nil {
func (mapper *PortMapper) Unmap(port int, proto string) error {
if proto == "tcp" {
backendAddr, ok := mapper.tcpMapping[port]
if !ok {
return fmt.Errorf("Port tcp/%v is not mapped", port)
}
if proxy, exists := mapper.tcpProxies[port]; exists {
proxy.Close()
delete(mapper.tcpProxies, port)
}
if err := mapper.iptablesForward("-D", port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil {
return err
}
utils.Debugf("Connecting to %s:%s", proto, address)
dst, err := net.Dial(proto, address)
if err != nil {
log.Printf("Error connecting to %s:%s: %s", proto, address, err)
src.Close()
continue
delete(mapper.tcpMapping, port)
} else {
backendAddr, ok := mapper.udpMapping[port]
if !ok {
return fmt.Errorf("Port udp/%v is not mapped", port)
}
utils.Debugf("Connected to backend, splicing")
splice(src, dst)
if proxy, exists := mapper.udpProxies[port]; exists {
proxy.Close()
delete(mapper.udpProxies, port)
}
if err := mapper.iptablesForward("-D", port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil {
return err
}
delete(mapper.udpMapping, port)
}
}
func halfSplice(dst, src net.Conn) error {
_, err := io.Copy(dst, src)
// FIXME: on EOF from a tcp connection, pass WriteClose()
dst.Close()
src.Close()
return err
}
func splice(a, b net.Conn) {
go halfSplice(a, b)
go halfSplice(b, a)
}
func (mapper *PortMapper) Unmap(port int) error {
dest, ok := mapper.mapping[port]
if !ok {
return errors.New("Port is not mapped")
}
if proxy, exists := mapper.proxies[port]; exists {
proxy.Close()
delete(mapper.proxies, port)
}
if err := mapper.iptablesForward("-D", port, dest); err != nil {
return err
}
delete(mapper.mapping, port)
return nil
}
@@ -301,9 +300,9 @@ func newPortMapper() (*PortMapper, error) {
// Port allocator: Atomatically allocate and release networking ports
type PortAllocator struct {
sync.Mutex
inUse map[int]struct{}
fountain chan (int)
lock sync.Mutex
}
func (alloc *PortAllocator) runFountain() {
@@ -317,9 +316,9 @@ func (alloc *PortAllocator) runFountain() {
// FIXME: Release can no longer fail, change its prototype to reflect that.
func (alloc *PortAllocator) Release(port int) error {
utils.Debugf("Releasing %d", port)
alloc.lock.Lock()
alloc.Lock()
delete(alloc.inUse, port)
alloc.lock.Unlock()
alloc.Unlock()
return nil
}
@@ -334,8 +333,8 @@ func (alloc *PortAllocator) Acquire(port int) (int, error) {
}
return -1, fmt.Errorf("Port generator ended unexpectedly")
}
alloc.lock.Lock()
defer alloc.lock.Unlock()
alloc.Lock()
defer alloc.Unlock()
if _, inUse := alloc.inUse[port]; inUse {
return -1, fmt.Errorf("Port already in use: %d", port)
}
@@ -453,7 +452,7 @@ type NetworkInterface struct {
Gateway net.IP
manager *NetworkManager
extPorts []int
extPorts []*Nat
}
// Allocate an external TCP port and map it to the interface
@@ -462,17 +461,32 @@ func (iface *NetworkInterface) AllocatePort(spec string) (*Nat, error) {
if err != nil {
return nil, err
}
// Allocate a random port if Frontend==0
extPort, err := iface.manager.portAllocator.Acquire(nat.Frontend)
if err != nil {
return nil, err
if nat.Proto == "tcp" {
extPort, err := iface.manager.tcpPortAllocator.Acquire(nat.Frontend)
if err != nil {
return nil, err
}
backend := &net.TCPAddr{IP: iface.IPNet.IP, Port: nat.Backend}
if err := iface.manager.portMapper.Map(extPort, backend); err != nil {
iface.manager.tcpPortAllocator.Release(extPort)
return nil, err
}
nat.Frontend = extPort
} else {
extPort, err := iface.manager.udpPortAllocator.Acquire(nat.Frontend)
if err != nil {
return nil, err
}
backend := &net.UDPAddr{IP: iface.IPNet.IP, Port: nat.Backend}
if err := iface.manager.portMapper.Map(extPort, backend); err != nil {
iface.manager.udpPortAllocator.Release(extPort)
return nil, err
}
nat.Frontend = extPort
}
nat.Frontend = extPort
if err := iface.manager.portMapper.Map(nat.Frontend, net.TCPAddr{IP: iface.IPNet.IP, Port: nat.Backend}); err != nil {
iface.manager.portAllocator.Release(nat.Frontend)
return nil, err
}
iface.extPorts = append(iface.extPorts, nat.Frontend)
iface.extPorts = append(iface.extPorts, nat)
return nat, nil
}
@@ -485,6 +499,21 @@ type Nat struct {
func parseNat(spec string) (*Nat, error) {
var nat Nat
if strings.Contains(spec, "/") {
specParts := strings.Split(spec, "/")
if len(specParts) != 2 {
return nil, fmt.Errorf("Invalid port format.")
}
proto := specParts[1]
spec = specParts[0]
if proto != "tcp" && proto != "udp" {
return nil, fmt.Errorf("Invalid port format: unknown protocol %v.", proto)
}
nat.Proto = proto
} else {
nat.Proto = "tcp"
}
if strings.Contains(spec, ":") {
specParts := strings.Split(spec, ":")
if len(specParts) != 2 {
@@ -517,20 +546,24 @@ func parseNat(spec string) (*Nat, error) {
}
nat.Backend = int(port)
}
nat.Proto = "tcp"
return &nat, nil
}
// Release: Network cleanup - release all resources
func (iface *NetworkInterface) Release() {
for _, port := range iface.extPorts {
if err := iface.manager.portMapper.Unmap(port); err != nil {
log.Printf("Unable to unmap port %v: %v", port, err)
for _, nat := range iface.extPorts {
utils.Debugf("Unmaping %v/%v", nat.Proto, nat.Frontend)
if err := iface.manager.portMapper.Unmap(nat.Frontend, nat.Proto); err != nil {
log.Printf("Unable to unmap port %v/%v: %v", nat.Proto, nat.Frontend, err)
}
if err := iface.manager.portAllocator.Release(port); err != nil {
log.Printf("Unable to release port %v: %v", port, err)
if nat.Proto == "tcp" {
if err := iface.manager.tcpPortAllocator.Release(nat.Frontend); err != nil {
log.Printf("Unable to release port tcp/%v: %v", nat.Frontend, err)
}
} else if err := iface.manager.udpPortAllocator.Release(nat.Frontend); err != nil {
log.Printf("Unable to release port udp/%v: %v", nat.Frontend, err)
}
}
iface.manager.ipAllocator.Release(iface.IPNet.IP)
@@ -542,9 +575,10 @@ type NetworkManager struct {
bridgeIface string
bridgeNetwork *net.IPNet
ipAllocator *IPAllocator
portAllocator *PortAllocator
portMapper *PortMapper
ipAllocator *IPAllocator
tcpPortAllocator *PortAllocator
udpPortAllocator *PortAllocator
portMapper *PortMapper
}
// Allocate a network interface
@@ -577,7 +611,11 @@ func newNetworkManager(bridgeIface string) (*NetworkManager, error) {
ipAllocator := newIPAllocator(network)
portAllocator, err := newPortAllocator()
tcpPortAllocator, err := newPortAllocator()
if err != nil {
return nil, err
}
udpPortAllocator, err := newPortAllocator()
if err != nil {
return nil, err
}
@@ -588,11 +626,12 @@ func newNetworkManager(bridgeIface string) (*NetworkManager, error) {
}
manager := &NetworkManager{
bridgeIface: bridgeIface,
bridgeNetwork: network,
ipAllocator: ipAllocator,
portAllocator: portAllocator,
portMapper: portMapper,
bridgeIface: bridgeIface,
bridgeNetwork: network,
ipAllocator: ipAllocator,
tcpPortAllocator: tcpPortAllocator,
udpPortAllocator: udpPortAllocator,
portMapper: portMapper,
}
return manager, nil
}
components/engine/network_proxy.go
+257
View File
@@ -0,0 +1,257 @@
package docker
import (
"encoding/binary"
"fmt"
"github.com/dotcloud/docker/utils"
"io"
"log"
"net"
"sync"
"syscall"
"time"
)
const (
UDPConnTrackTimeout = 90 * time.Second
UDPBufSize = 2048
)
type Proxy interface {
// Start forwarding traffic back and forth the front and back-end
// addresses.
Run()
// Stop forwarding traffic and close both ends of the Proxy.
Close()
// Return the address on which the proxy is listening.
FrontendAddr() net.Addr
// Return the proxied address.
BackendAddr() net.Addr
}
type TCPProxy struct {
listener *net.TCPListener
frontendAddr *net.TCPAddr
backendAddr *net.TCPAddr
}
func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) {
listener, err := net.ListenTCP("tcp", frontendAddr)
if err != nil {
return nil, err
}
// If the port in frontendAddr was 0 then ListenTCP will have a picked
// a port to listen on, hence the call to Addr to get that actual port:
return &TCPProxy{
listener: listener,
frontendAddr: listener.Addr().(*net.TCPAddr),
backendAddr: backendAddr,
}, nil
}
func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) {
backend, err := net.DialTCP("tcp", nil, proxy.backendAddr)
if err != nil {
log.Printf("Can't forward traffic to backend tcp/%v: %v\n", proxy.backendAddr, err.Error())
client.Close()
return
}
event := make(chan int64)
var broker = func(to, from *net.TCPConn) {
written, err := io.Copy(to, from)
if err != nil {
err, ok := err.(*net.OpError)
// If the socket we are writing to is shutdown with
// SHUT_WR, forward it to the other end of the pipe:
if ok && err.Err == syscall.EPIPE {
from.CloseWrite()
}
}
event <- written
}
utils.Debugf("Forwarding traffic between tcp/%v and tcp/%v", client.RemoteAddr(), backend.RemoteAddr())
go broker(client, backend)
go broker(backend, client)
var transferred int64 = 0
for i := 0; i < 2; i++ {
select {
case written := <-event:
transferred += written
case <-quit:
// Interrupt the two brokers and "join" them.
client.Close()
backend.Close()
for ; i < 2; i++ {
transferred += <-event
}
goto done
}
}
client.Close()
backend.Close()
done:
utils.Debugf("%v bytes transferred between tcp/%v and tcp/%v", transferred, client.RemoteAddr(), backend.RemoteAddr())
}
func (proxy *TCPProxy) Run() {
quit := make(chan bool)
defer close(quit)
utils.Debugf("Starting proxy on tcp/%v for tcp/%v", proxy.frontendAddr, proxy.backendAddr)
for {
client, err := proxy.listener.Accept()
if err != nil {
utils.Debugf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error())
return
}
go proxy.clientLoop(client.(*net.TCPConn), quit)
}
}
func (proxy *TCPProxy) Close() { proxy.listener.Close() }
func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
func (proxy *TCPProxy) BackendAddr() net.Addr { return proxy.backendAddr }
// A net.Addr where the IP is split into two fields so you can use it as a key
// in a map:
type connTrackKey struct {
IPHigh uint64
IPLow uint64
Port int
}
func newConnTrackKey(addr *net.UDPAddr) *connTrackKey {
if len(addr.IP) == net.IPv4len {
return &connTrackKey{
IPHigh: 0,
IPLow: uint64(binary.BigEndian.Uint32(addr.IP)),
Port: addr.Port,
}
}
return &connTrackKey{
IPHigh: binary.BigEndian.Uint64(addr.IP[:8]),
IPLow: binary.BigEndian.Uint64(addr.IP[8:]),
Port: addr.Port,
}
}
type connTrackMap map[connTrackKey]*net.UDPConn
type UDPProxy struct {
listener *net.UDPConn
frontendAddr *net.UDPAddr
backendAddr *net.UDPAddr
connTrackTable connTrackMap
connTrackLock sync.Mutex
}
func NewUDPProxy(frontendAddr, backendAddr *net.UDPAddr) (*UDPProxy, error) {
listener, err := net.ListenUDP("udp", frontendAddr)
if err != nil {
return nil, err
}
return &UDPProxy{
listener: listener,
frontendAddr: listener.LocalAddr().(*net.UDPAddr),
backendAddr: backendAddr,
connTrackTable: make(connTrackMap),
}, nil
}
func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) {
defer func() {
proxy.connTrackLock.Lock()
delete(proxy.connTrackTable, *clientKey)
proxy.connTrackLock.Unlock()
utils.Debugf("Done proxying between udp/%v and udp/%v", clientAddr.String(), proxy.backendAddr.String())
proxyConn.Close()
}()
readBuf := make([]byte, UDPBufSize)
for {
proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout))
again:
read, err := proxyConn.Read(readBuf)
if err != nil {
if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED {
// This will happen if the last write failed
// (e.g: nothing is actually listening on the
// proxied port on the container), ignore it
// and continue until UDPConnTrackTimeout
// expires:
goto again
}
return
}
for i := 0; i != read; {
written, err := proxy.listener.WriteToUDP(readBuf[i:read], clientAddr)
if err != nil {
return
}
i += written
utils.Debugf("Forwarded %v/%v bytes to udp/%v", i, read, clientAddr.String())
}
}
}
func (proxy *UDPProxy) Run() {
readBuf := make([]byte, UDPBufSize)
utils.Debugf("Starting proxy on udp/%v for udp/%v", proxy.frontendAddr, proxy.backendAddr)
for {
read, from, err := proxy.listener.ReadFromUDP(readBuf)
if err != nil {
// NOTE: Apparently ReadFrom doesn't return
// ECONNREFUSED like Read do (see comment in
// UDPProxy.replyLoop)
utils.Debugf("Stopping proxy on udp/%v for udp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error())
break
}
fromKey := newConnTrackKey(from)
proxy.connTrackLock.Lock()
proxyConn, hit := proxy.connTrackTable[*fromKey]
if !hit {
proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr)
if err != nil {
log.Printf("Can't proxy a datagram to udp/%s: %v\n", proxy.backendAddr.String(), err)
continue
}
proxy.connTrackTable[*fromKey] = proxyConn
go proxy.replyLoop(proxyConn, from, fromKey)
}
proxy.connTrackLock.Unlock()
for i := 0; i != read; {
written, err := proxyConn.Write(readBuf[i:read])
if err != nil {
log.Printf("Can't proxy a datagram to udp/%s: %v\n", proxy.backendAddr.String(), err)
break
}
i += written
utils.Debugf("Forwarded %v/%v bytes to udp/%v", i, read, proxy.backendAddr.String())
}
}
}
func (proxy *UDPProxy) Close() {
proxy.listener.Close()
proxy.connTrackLock.Lock()
defer proxy.connTrackLock.Unlock()
for _, conn := range proxy.connTrackTable {
conn.Close()
}
}
func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr }
func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) {
switch frontendAddr.(type) {
case *net.UDPAddr:
return NewUDPProxy(frontendAddr.(*net.UDPAddr), backendAddr.(*net.UDPAddr))
case *net.TCPAddr:
return NewTCPProxy(frontendAddr.(*net.TCPAddr), backendAddr.(*net.TCPAddr))
default:
panic(fmt.Errorf("Unsupported protocol"))
}
}
components/engine/network_proxy_test.go
+221
View File
@@ -0,0 +1,221 @@
package docker
import (
"bytes"
"fmt"
"io"
"net"
"strings"
"testing"
"time"
)
var testBuf = []byte("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo")
var testBufSize = len(testBuf)
type EchoServer interface {
Run()
Close()
LocalAddr() net.Addr
}
type TCPEchoServer struct {
listener net.Listener
testCtx *testing.T
}
type UDPEchoServer struct {
conn net.PacketConn
testCtx *testing.T
}
func NewEchoServer(t *testing.T, proto, address string) EchoServer {
var server EchoServer
if strings.HasPrefix(proto, "tcp") {
listener, err := net.Listen(proto, address)
if err != nil {
t.Fatal(err)
}
server = &TCPEchoServer{listener: listener, testCtx: t}
} else {
socket, err := net.ListenPacket(proto, address)
if err != nil {
t.Fatal(err)
}
server = &UDPEchoServer{conn: socket, testCtx: t}
}
t.Logf("EchoServer listening on %v/%v\n", proto, server.LocalAddr().String())
return server
}
func (server *TCPEchoServer) Run() {
go func() {
for {
client, err := server.listener.Accept()
if err != nil {
return
}
go func(client net.Conn) {
server.testCtx.Logf("TCP client accepted on the EchoServer\n")
written, err := io.Copy(client, client)
server.testCtx.Logf("%v bytes echoed back to the client\n", written)
if err != nil {
server.testCtx.Logf("can't echo to the client: %v\n", err.Error())
}
client.Close()
}(client)
}
}()
}
func (server *TCPEchoServer) LocalAddr() net.Addr { return server.listener.Addr() }
func (server *TCPEchoServer) Close() { server.listener.Addr() }
func (server *UDPEchoServer) Run() {
go func() {
readBuf := make([]byte, 1024)
for {
read, from, err := server.conn.ReadFrom(readBuf)
if err != nil {
return
}
server.testCtx.Logf("Writing UDP datagram back")
for i := 0; i != read; {
written, err := server.conn.WriteTo(readBuf[i:read], from)
if err != nil {
break
}
i += written
}
}
}()
}
func (server *UDPEchoServer) LocalAddr() net.Addr { return server.conn.LocalAddr() }
func (server *UDPEchoServer) Close() { server.conn.Close() }
func testProxyAt(t *testing.T, proto string, proxy Proxy, addr string) {
defer proxy.Close()
go proxy.Run()
client, err := net.Dial(proto, addr)
if err != nil {
t.Fatalf("Can't connect to the proxy: %v", err)
}
defer client.Close()
client.SetDeadline(time.Now().Add(10 * time.Second))
if _, err = client.Write(testBuf); err != nil {
t.Fatal(err)
}
recvBuf := make([]byte, testBufSize)
if _, err = client.Read(recvBuf); err != nil {
t.Fatal(err)
}
if !bytes.Equal(testBuf, recvBuf) {
t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf))
}
}
func testProxy(t *testing.T, proto string, proxy Proxy) {
testProxyAt(t, proto, proxy, proxy.FrontendAddr().String())
}
func TestTCP4Proxy(t *testing.T) {
backend := NewEchoServer(t, "tcp", "127.0.0.1:0")
defer backend.Close()
backend.Run()
frontendAddr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}
proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
if err != nil {
t.Fatal(err)
}
testProxy(t, "tcp", proxy)
}
func TestTCP6Proxy(t *testing.T) {
backend := NewEchoServer(t, "tcp", "[::1]:0")
defer backend.Close()
backend.Run()
frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0}
proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
if err != nil {
t.Fatal(err)
}
testProxy(t, "tcp", proxy)
}
func TestTCPDualStackProxy(t *testing.T) {
// If I understand `godoc -src net favoriteAddrFamily` (used by the
// net.Listen* functions) correctly this should work, but it doesn't.
t.Skip("No support for dual stack yet")
backend := NewEchoServer(t, "tcp", "[::1]:0")
defer backend.Close()
backend.Run()
frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0}
proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
if err != nil {
t.Fatal(err)
}
ipv4ProxyAddr := &net.TCPAddr{
IP: net.IPv4(127, 0, 0, 1),
Port: proxy.FrontendAddr().(*net.TCPAddr).Port,
}
testProxyAt(t, "tcp", proxy, ipv4ProxyAddr.String())
}
func TestUDP4Proxy(t *testing.T) {
backend := NewEchoServer(t, "udp", "127.0.0.1:0")
defer backend.Close()
backend.Run()
frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}
proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
if err != nil {
t.Fatal(err)
}
testProxy(t, "udp", proxy)
}
func TestUDP6Proxy(t *testing.T) {
backend := NewEchoServer(t, "udp", "[::1]:0")
defer backend.Close()
backend.Run()
frontendAddr := &net.UDPAddr{IP: net.IPv6loopback, Port: 0}
proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
if err != nil {
t.Fatal(err)
}
testProxy(t, "udp", proxy)
}
func TestUDPWriteError(t *testing.T) {
frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}
// Hopefully, this port will be free: */
backendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 25587}
proxy, err := NewProxy(frontendAddr, backendAddr)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
go proxy.Run()
client, err := net.Dial("udp", "127.0.0.1:25587")
if err != nil {
t.Fatalf("Can't connect to the proxy: %v", err)
}
defer client.Close()
// Make sure the proxy doesn't stop when there is no actual backend:
client.Write(testBuf)
client.Write(testBuf)
backend := NewEchoServer(t, "udp", "127.0.0.1:25587")
defer backend.Close()
backend.Run()
client.SetDeadline(time.Now().Add(10 * time.Second))
if _, err = client.Write(testBuf); err != nil {
t.Fatal(err)
}
recvBuf := make([]byte, testBufSize)
if _, err = client.Read(recvBuf); err != nil {
t.Fatal(err)
}
if !bytes.Equal(testBuf, recvBuf) {
t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf))
}
}
components/engine/network_test.go
+75 -6
View File
@@ -20,28 +20,97 @@ func TestIptables(t *testing.T) {
func TestParseNat(t *testing.T) {
if nat, err := parseNat("4500"); err == nil {
if nat.Frontend != 0 || nat.Backend != 4500 {
t.Errorf("-p 4500 should produce 0->4500, got %d->%d", nat.Frontend, nat.Backend)
if nat.Frontend != 0 || nat.Backend != 4500 || nat.Proto != "tcp" {
t.Errorf("-p 4500 should produce 0->4500/tcp, got %d->%d/%s",
nat.Frontend, nat.Backend, nat.Proto)
}
} else {
t.Fatal(err)
}
if nat, err := parseNat(":4501"); err == nil {
if nat.Frontend != 4501 || nat.Backend != 4501 {
t.Errorf("-p :4501 should produce 4501->4501, got %d->%d", nat.Frontend, nat.Backend)
if nat.Frontend != 4501 || nat.Backend != 4501 || nat.Proto != "tcp" {
t.Errorf("-p :4501 should produce 4501->4501/tcp, got %d->%d/%s",
nat.Frontend, nat.Backend, nat.Proto)
}
} else {
t.Fatal(err)
}
if nat, err := parseNat("4502:4503"); err == nil {
if nat.Frontend != 4502 || nat.Backend != 4503 {
t.Errorf("-p 4502:4503 should produce 4502->4503, got %d->%d", nat.Frontend, nat.Backend)
if nat.Frontend != 4502 || nat.Backend != 4503 || nat.Proto != "tcp" {
t.Errorf("-p 4502:4503 should produce 4502->4503/tcp, got %d->%d/%s",
nat.Frontend, nat.Backend, nat.Proto)
}
} else {
t.Fatal(err)
}
if nat, err := parseNat("4502:4503/tcp"); err == nil {
if nat.Frontend != 4502 || nat.Backend != 4503 || nat.Proto != "tcp" {
t.Errorf("-p 4502:4503/tcp should produce 4502->4503/tcp, got %d->%d/%s",
nat.Frontend, nat.Backend, nat.Proto)
}
} else {
t.Fatal(err)
}
if nat, err := parseNat("4502:4503/udp"); err == nil {
if nat.Frontend != 4502 || nat.Backend != 4503 || nat.Proto != "udp" {
t.Errorf("-p 4502:4503/udp should produce 4502->4503/udp, got %d->%d/%s",
nat.Frontend, nat.Backend, nat.Proto)
}
} else {
t.Fatal(err)
}
if nat, err := parseNat(":4503/udp"); err == nil {
if nat.Frontend != 4503 || nat.Backend != 4503 || nat.Proto != "udp" {
t.Errorf("-p :4503/udp should produce 4503->4503/udp, got %d->%d/%s",
nat.Frontend, nat.Backend, nat.Proto)
}
} else {
t.Fatal(err)
}
if nat, err := parseNat(":4503/tcp"); err == nil {
if nat.Frontend != 4503 || nat.Backend != 4503 || nat.Proto != "tcp" {
t.Errorf("-p :4503/tcp should produce 4503->4503/tcp, got %d->%d/%s",
nat.Frontend, nat.Backend, nat.Proto)
}
} else {
t.Fatal(err)
}
if nat, err := parseNat("4503/tcp"); err == nil {
if nat.Frontend != 0 || nat.Backend != 4503 || nat.Proto != "tcp" {
t.Errorf("-p 4503/tcp should produce 0->4503/tcp, got %d->%d/%s",
nat.Frontend, nat.Backend, nat.Proto)
}
} else {
t.Fatal(err)
}
if nat, err := parseNat("4503/udp"); err == nil {
if nat.Frontend != 0 || nat.Backend != 4503 || nat.Proto != "udp" {
t.Errorf("-p 4503/udp should produce 0->4503/udp, got %d->%d/%s",
nat.Frontend, nat.Backend, nat.Proto)
}
} else {
t.Fatal(err)
}
if _, err := parseNat("4503/tcpgarbage"); err == nil {
t.Fatal(err)
}
if _, err := parseNat("4503/tcp/udp"); err == nil {
t.Fatal(err)
}
if _, err := parseNat("4503/"); err == nil {
t.Fatal(err)
}
}
func TestPortAllocation(t *testing.T) {
components/engine/registry/registry.go
+108 -67
View File
@@ -12,18 +12,90 @@ import (
"net/http"
"net/http/cookiejar"
"net/url"
"regexp"
"strconv"
"strings"
)
var ErrAlreadyExists = errors.New("Image already exists")
var ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")")
func UrlScheme() string {
u, err := url.Parse(auth.IndexServerAddress())
if err != nil {
return "https"
func pingRegistryEndpoint(endpoint string) error {
if endpoint == auth.IndexServerAddress() {
// Skip the check, we now this one is valid
// (and we never want to fallback to http in case of error)
return nil
}
return u.Scheme
resp, err := http.Get(endpoint + "_ping")
if err != nil {
return err
}
if resp.Header.Get("X-Docker-Registry-Version") == "" {
return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)")
}
return nil
}
func validateRepositoryName(repositoryName string) error {
var (
namespace string
name string
)
nameParts := strings.SplitN(repositoryName, "/", 2)
if len(nameParts) < 2 {
namespace = "library"
name = nameParts[0]
} else {
namespace = nameParts[0]
name = nameParts[1]
}
validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`)
if !validNamespace.MatchString(namespace) {
return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace)
}
validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`)
if !validRepo.MatchString(name) {
return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", name)
}
return nil
}
// Resolves a repository name to a endpoint + name
func ResolveRepositoryName(reposName string) (string, string, error) {
if strings.Contains(reposName, "://") {
// It cannot contain a scheme!
return "", "", ErrInvalidRepositoryName
}
nameParts := strings.SplitN(reposName, "/", 2)
if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") {
// This is a Docker Index repos (ex: samalba/hipache or ubuntu)
err := validateRepositoryName(reposName)
return auth.IndexServerAddress(), reposName, err
}
if len(nameParts) < 2 {
// There is a dot in repos name (and no registry address)
// Is it a Registry address without repos name?
return "", "", ErrInvalidRepositoryName
}
hostname := nameParts[0]
reposName = nameParts[1]
if strings.Contains(hostname, "index.docker.io") {
return "", "", fmt.Errorf("Invalid repository name, try \"%s\" instead", reposName)
}
if err := validateRepositoryName(reposName); err != nil {
return "", "", err
}
endpoint := fmt.Sprintf("https://%s/v1/", hostname)
if err := pingRegistryEndpoint(endpoint); err != nil {
utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err)
endpoint = fmt.Sprintf("http://%s/v1/", hostname)
if err = pingRegistryEndpoint(endpoint); err != nil {
//TODO: triggering highland build can be done there without "failing"
return "", "", errors.New("Invalid Registry endpoint: " + err.Error())
}
}
err := validateRepositoryName(reposName)
return endpoint, reposName, err
}
func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) {
@@ -35,8 +107,8 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) {
// Retrieve the history of a given image from the Registry.
// Return a list of the parent's json (requested image included)
func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) {
req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil)
func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) {
req, err := http.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil)
if err != nil {
return nil, err
}
@@ -44,7 +116,7 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s
res, err := r.client.Do(req)
if err != nil || res.StatusCode != 200 {
if res != nil {
return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId)
return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID)
}
return nil, err
}
@@ -64,10 +136,10 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s
}
// Check if an image exists in the Registry
func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool {
func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool {
rt := &http.Transport{Proxy: http.ProxyFromEnvironment}
req, err := http.NewRequest("GET", registry+"/v1/images/"+imgId+"/json", nil)
req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil)
if err != nil {
return false
}
@@ -79,44 +151,10 @@ func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) boo
return res.StatusCode == 200
}
func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) {
u := auth.IndexServerAddress() + "/repositories/" + repository + "/images"
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
}
if authConfig != nil && len(authConfig.Username) > 0 {
req.SetBasicAuth(authConfig.Username, authConfig.Password)
}
res, err := r.client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
// Repository doesn't exist yet
if res.StatusCode == 404 {
return nil, nil
}
jsonData, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
imageList := []map[string]string{}
if err := json.Unmarshal(jsonData, &imageList); err != nil {
utils.Debugf("Body: %s (%s)\n", res.Body, u)
return nil, err
}
return imageList, nil
}
// Retrieve an image from the Registry.
func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, int, error) {
func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) {
// Get the JSON
req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil)
req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil)
if err != nil {
return nil, -1, fmt.Errorf("Failed to download json: %s", err)
}
@@ -142,8 +180,8 @@ func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([
return jsonString, imageSize, nil
}
func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) {
req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil)
func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) {
req, err := http.NewRequest("GET", registry+"images/"+imgID+"/layer", nil)
if err != nil {
return nil, fmt.Errorf("Error while getting from the server: %s\n", err)
}
@@ -162,10 +200,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [
repository = "library/" + repository
}
for _, host := range registries {
endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository)
if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) {
endpoint = fmt.Sprintf("%s://%s", UrlScheme(), endpoint)
}
endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository)
req, err := r.opaqueRequest("GET", endpoint, nil)
if err != nil {
return nil, err
@@ -198,8 +233,8 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [
return nil, fmt.Errorf("Could not reach any registry endpoint")
}
func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) {
repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images"
func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) {
repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote)
req, err := r.opaqueRequest("GET", repositoryTarget, nil)
if err != nil {
@@ -230,8 +265,12 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) {
}
var endpoints []string
var urlScheme = indexEp[:strings.Index(indexEp, ":")]
if res.Header.Get("X-Docker-Endpoints") != "" {
endpoints = res.Header["X-Docker-Endpoints"]
// The Registry's URL scheme has to match the Index'
for _, ep := range res.Header["X-Docker-Endpoints"] {
endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep))
}
} else {
return nil, fmt.Errorf("Index response didn't contain any endpoints")
}
@@ -260,9 +299,8 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) {
// Push a local image to the registry
func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error {
registry = registry + "/v1"
// FIXME: try json with UTF8
req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw)))
req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw)))
if err != nil {
return err
}
@@ -295,9 +333,8 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis
return nil
}
func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error {
registry = registry + "/v1"
req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer)
func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error {
req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer)
if err != nil {
return err
}
@@ -334,9 +371,8 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R
func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error {
// "jsonify" the string
revision = "\"" + revision + "\""
registry = registry + "/v1"
req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision))
req, err := r.opaqueRequest("PUT", registry+"repositories/"+remote+"/tags/"+tag, strings.NewReader(revision))
if err != nil {
return err
}
@@ -354,7 +390,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token
return nil
}
func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) {
func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) {
imgListJSON, err := json.Marshal(imgList)
if err != nil {
return nil, err
@@ -364,9 +400,10 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat
suffix = "images"
}
u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix)
utils.Debugf("PUT %s", u)
utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON)
req, err := r.opaqueRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJSON))
req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON))
if err != nil {
return nil, err
}
@@ -404,6 +441,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat
}
var tokens, endpoints []string
var urlScheme = indexEp[:strings.Index(indexEp, ":")]
if !validate {
if res.StatusCode != 200 && res.StatusCode != 201 {
errBody, err := ioutil.ReadAll(res.Body)
@@ -420,7 +458,10 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat
}
if res.Header.Get("X-Docker-Endpoints") != "" {
endpoints = res.Header["X-Docker-Endpoints"]
// The Registry's URL scheme has to match the Index'
for _, ep := range res.Header["X-Docker-Endpoints"] {
endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep))
}
} else {
return nil, fmt.Errorf("Index response didn't contain any endpoints")
}
@@ -442,7 +483,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat
}
func (r *Registry) SearchRepositories(term string) (*SearchResults, error) {
u := auth.IndexServerAddress() + "/search?q=" + url.QueryEscape(term)
u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term)
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
components/engine/runtime.go
-3
View File
@@ -108,9 +108,6 @@ func (runtime *Runtime) Register(container *Container) error {
// init the wait lock
container.waitLock = make(chan struct{})
// Even if not running, we init the lock (prevents races in start/stop/kill)
container.State.initLock()
container.runtime = runtime
// Attach to stdout and stderr
components/engine/runtime_test.go
+125 -59
View File
@@ -1,6 +1,7 @@
package docker
import (
"bytes"
"fmt"
"github.com/dotcloud/docker/utils"
"io"
@@ -17,11 +18,12 @@ import (
)
const (
unitTestImageName = "docker-unit-tests"
unitTestImageId = "e9aa60c60128cad1"
unitTestStoreBase = "/var/lib/docker/unit-tests"
testDaemonAddr = "127.0.0.1:4270"
testDaemonProto = "tcp"
unitTestImageName = "docker-test-image"
unitTestImageID = "83599e29c455eb719f77d799bc7c51521b9551972f5a850d7ad265bc1b5292f6" // 1.0
unitTestNetworkBridge = "testdockbr0"
unitTestStoreBase = "/var/lib/docker/unit-tests"
testDaemonAddr = "127.0.0.1:4270"
testDaemonProto = "tcp"
)
var globalRuntime *Runtime
@@ -49,7 +51,7 @@ func cleanup(runtime *Runtime) error {
return err
}
for _, image := range images {
if image.ID != unitTestImageId {
if image.ID != unitTestImageID {
runtime.graph.Delete(image.ID)
}
}
@@ -73,10 +75,10 @@ func init() {
}
if uid := syscall.Geteuid(); uid != 0 {
log.Fatal("docker tests needs to be run as root")
log.Fatal("docker tests need to be run as root")
}
NetworkBridgeIface = "testdockbr0"
NetworkBridgeIface = unitTestNetworkBridge
// Make it our Store root
runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false)
@@ -89,13 +91,15 @@ func init() {
srv := &Server{
runtime: runtime,
enableCors: false,
lock: &sync.Mutex{},
pullingPool: make(map[string]struct{}),
pushingPool: make(map[string]struct{}),
}
// Retrieve the Image
if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil {
panic(err)
// If the unit test is not found, try to download it.
if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID {
// Retrieve the Image
if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil {
panic(err)
}
}
// Spawn a Daemon
go func() {
@@ -136,11 +140,11 @@ func GetTestImage(runtime *Runtime) *Image {
panic(err)
}
for i := range imgs {
if imgs[i].ID == unitTestImageId {
if imgs[i].ID == unitTestImageID {
return imgs[i]
}
}
panic(fmt.Errorf("Test image %v not found", unitTestImageId))
panic(fmt.Errorf("Test image %v not found", unitTestImageID))
}
func TestRuntimeCreate(t *testing.T) {
@@ -318,52 +322,47 @@ func TestGet(t *testing.T) {
}
func findAvailalblePort(runtime *Runtime, port int) (*Container, error) {
strPort := strconv.Itoa(port)
container, err := NewBuilder(runtime).Create(&Config{
Image: GetTestImage(runtime).ID,
Cmd: []string{"sh", "-c", "echo well hello there | nc -l -p " + strPort},
PortSpecs: []string{strPort},
},
)
if err != nil {
return nil, err
}
hostConfig := &HostConfig{}
if err := container.Start(hostConfig); err != nil {
if strings.Contains(err.Error(), "address already in use") {
return nil, nil
}
return nil, err
}
return container, nil
}
// Run a container with a TCP port allocated, and test that it can receive connections on localhost
func TestAllocatePortLocalhost(t *testing.T) {
func startEchoServerContainer(t *testing.T, proto string) (*Runtime, *Container, string) {
runtime, err := newTestRuntime()
if err != nil {
t.Fatal(err)
}
port := 5554
port := 5554
var container *Container
var strPort string
for {
port += 1
log.Println("Trying port", port)
t.Log("Trying port", port)
container, err = findAvailalblePort(runtime, port)
strPort = strconv.Itoa(port)
var cmd string
if proto == "tcp" {
cmd = "socat TCP-LISTEN:" + strPort + ",reuseaddr,fork EXEC:/bin/cat"
} else if proto == "udp" {
cmd = "socat UDP-RECVFROM:" + strPort + ",fork EXEC:/bin/cat"
} else {
t.Fatal(fmt.Errorf("Unknown protocol %v", proto))
}
t.Log("Trying port", strPort)
container, err = NewBuilder(runtime).Create(&Config{
Image: GetTestImage(runtime).ID,
Cmd: []string{"sh", "-c", cmd},
PortSpecs: []string{fmt.Sprintf("%s/%s", strPort, proto)},
})
if container != nil {
break
}
if err != nil {
nuke(runtime)
t.Fatal(err)
}
log.Println("Port", port, "already in use")
t.Log("Port", port, "already in use")
t.Logf("Port %v already in use", strPort)
}
defer container.Kill()
hostConfig := &HostConfig{}
if err := container.Start(hostConfig); err != nil {
nuke(runtime)
t.Fatal(err)
}
setTimeout(t, "Waiting for the container to be started timed out", 2*time.Second, func() {
for !container.State.Running {
@@ -374,26 +373,93 @@ func TestAllocatePortLocalhost(t *testing.T) {
// Even if the state is running, lets give some time to lxc to spawn the process
container.WaitTimeout(500 * time.Millisecond)
conn, err := net.Dial("tcp",
fmt.Sprintf(
"localhost:%s", container.NetworkSettings.PortMapping[strconv.Itoa(port)],
),
)
strPort = container.NetworkSettings.PortMapping[strings.Title(proto)][strPort]
return runtime, container, strPort
}
// Run a container with a TCP port allocated, and test that it can receive connections on localhost
func TestAllocateTCPPortLocalhost(t *testing.T) {
runtime, container, port := startEchoServerContainer(t, "tcp")
defer nuke(runtime)
defer container.Kill()
for i := 0; i != 10; i++ {
conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%v", port))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
input := bytes.NewBufferString("well hello there\n")
_, err = conn.Write(input.Bytes())
if err != nil {
t.Fatal(err)
}
buf := make([]byte, 16)
read := 0
conn.SetReadDeadline(time.Now().Add(3 * time.Second))
read, err = conn.Read(buf)
if err != nil {
if err, ok := err.(*net.OpError); ok {
if err.Err == syscall.ECONNRESET {
t.Logf("Connection reset by the proxy, socat is probably not listening yet, trying again in a sec")
conn.Close()
time.Sleep(time.Second)
continue
}
if err.Timeout() {
t.Log("Timeout, trying again")
conn.Close()
continue
}
}
t.Fatal(err)
}
output := string(buf[:read])
if !strings.Contains(output, "well hello there") {
t.Fatal(fmt.Errorf("[%v] doesn't contain [well hello there]", output))
} else {
return
}
}
t.Fatal("No reply from the container")
}
// Run a container with an UDP port allocated, and test that it can receive connections on localhost
func TestAllocateUDPPortLocalhost(t *testing.T) {
runtime, container, port := startEchoServerContainer(t, "udp")
defer nuke(runtime)
defer container.Kill()
conn, err := net.Dial("udp", fmt.Sprintf("localhost:%v", port))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
output, err := ioutil.ReadAll(conn)
if err != nil {
t.Fatal(err)
input := bytes.NewBufferString("well hello there\n")
buf := make([]byte, 16)
// Try for a minute, for some reason the select in socat may take ages
// to return even though everything on the path seems fine (i.e: the
// UDPProxy forwards the traffic correctly and you can see the packets
// on the interface from within the container).
for i := 0; i != 120; i++ {
_, err := conn.Write(input.Bytes())
if err != nil {
t.Fatal(err)
}
conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond))
read, err := conn.Read(buf)
if err == nil {
output := string(buf[:read])
if strings.Contains(output, "well hello there") {
return
}
}
}
if string(output) != "well hello there\n" {
t.Fatalf("Received wrong output from network connection: should be '%s', not '%s'",
"well hello there\n",
string(output),
)
}
container.Wait()
t.Fatal("No reply from the container")
}
func TestRestore(t *testing.T) {
components/engine/server.go
+94 -125
View File
@@ -29,7 +29,7 @@ func (srv *Server) DockerVersion() APIVersion {
func (srv *Server) ContainerKill(name string) error {
if container := srv.runtime.Get(name); container != nil {
if err := container.Kill(); err != nil {
return fmt.Errorf("Error restarting container %s: %s", name, err.Error())
return fmt.Errorf("Error restarting container %s: %s", name, err)
}
} else {
return fmt.Errorf("No such container: %s", name)
@@ -315,8 +315,8 @@ func (srv *Server) ContainerTag(name, repo, tag string, force bool) error {
return nil
}
func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoint string, token []string, sf *utils.StreamFormatter) error {
history, err := r.GetRemoteHistory(imgId, endpoint, token)
func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) error {
history, err := r.GetRemoteHistory(imgID, endpoint, token)
if err != nil {
return err
}
@@ -351,44 +351,32 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin
return nil
}
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error {
out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress()))
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error {
out.Write(sf.FormatStatus("Pulling repository %s", localName))
var repoData *registry.RepositoryData
var err error
if registryEp == "" {
repoData, err = r.GetRepositoryData(remote)
if err != nil {
return err
}
repoData, err := r.GetRepositoryData(indexEp, remoteName)
if err != nil {
return err
}
utils.Debugf("Updating checksums")
// Reload the json file to make sure not to overwrite faster sums
if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
return err
}
} else {
repoData = &registry.RepositoryData{
Tokens: []string{},
ImgList: make(map[string]*registry.ImgData),
Endpoints: []string{registryEp},
}
utils.Debugf("Updating checksums")
// Reload the json file to make sure not to overwrite faster sums
if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
return err
}
utils.Debugf("Retrieving the tag list")
tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens)
if err != nil {
utils.Debugf("%v", err)
return err
}
if registryEp != "" {
for tag, id := range tagsList {
repoData.ImgList[id] = &registry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
for tag, id := range tagsList {
repoData.ImgList[id] = &registry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
@@ -402,7 +390,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re
// Otherwise, check that the tag exists and use only that one
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, local)
return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName)
}
repoData.ImgList[id].Tag = askedTag
}
@@ -417,13 +405,10 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re
utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
continue
}
out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote))
out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName))
success := false
for _, ep := range repoData.Endpoints {
if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep)
}
if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil {
if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
continue
}
@@ -438,7 +423,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re
if askedTag != "" && tag != askedTag {
continue
}
if err := srv.runtime.repositories.Set(local, tag, id, true); err != nil {
if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil {
return err
}
}
@@ -450,8 +435,8 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re
}
func (srv *Server) poolAdd(kind, key string) error {
srv.lock.Lock()
defer srv.lock.Unlock()
srv.Lock()
defer srv.Unlock()
if _, exists := srv.pullingPool[key]; exists {
return fmt.Errorf("%s %s is already in progress", key, kind)
@@ -483,25 +468,32 @@ func (srv *Server) poolRemove(kind, key string) error {
}
return nil
}
func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error {
func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error {
r, err := registry.NewRegistry(srv.runtime.root, authConfig)
if err != nil {
return err
}
if err := srv.poolAdd("pull", name+":"+tag); err != nil {
if err := srv.poolAdd("pull", localName+":"+tag); err != nil {
return err
}
defer srv.poolRemove("pull", name+":"+tag)
defer srv.poolRemove("pull", localName+":"+tag)
remote := name
parts := strings.Split(name, "/")
if len(parts) > 2 {
remote = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
// Resolve the Repository name from fqn to endpoint + name
endpoint, remoteName, err := registry.ResolveRepositoryName(localName)
if err != nil {
return err
}
if endpoint == auth.IndexServerAddress() {
// If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar"
localName = remoteName
}
out = utils.NewWriteFlusher(out)
err = srv.pullRepository(r, out, name, remote, tag, endpoint, sf)
if err != nil && endpoint != "" {
if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil {
err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf)
if err != nil {
if err := srv.pullImage(r, out, remoteName, endpoint, nil, sf); err != nil {
return err
}
return nil
@@ -516,20 +508,20 @@ func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *util
// - Check if the archive exists, if it does not, ask the registry
// - If the archive does exists, process the checksum from it
// - If the archive does not exists and not found on registry, process checksum from layer
func (srv *Server) getChecksum(imageId string) (string, error) {
func (srv *Server) getChecksum(imageID string) (string, error) {
// FIXME: Use in-memory map instead of reading the file each time
if sums, err := srv.runtime.graph.getStoredChecksums(); err != nil {
return "", err
} else if checksum, exists := sums[imageId]; exists {
} else if checksum, exists := sums[imageID]; exists {
return checksum, nil
}
img, err := srv.runtime.graph.Get(imageId)
img, err := srv.runtime.graph.Get(imageID)
if err != nil {
return "", err
}
if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageId))); err != nil {
if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageID))); err != nil {
if os.IsNotExist(err) {
// TODO: Ask the registry for the checksum
// As the archive is not there, it is supposed to come from a pull.
@@ -576,7 +568,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat
return imgList, nil
}
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, registryEp string, localRepo map[string]string, sf *utils.StreamFormatter) error {
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
out.Write(sf.FormatStatus("Processing checksums"))
imgList, err := srv.getImageList(localRepo)
@@ -584,94 +576,64 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, reg
return err
}
out.Write(sf.FormatStatus("Sending image list"))
srvName := name
parts := strings.Split(name, "/")
if len(parts) > 2 {
srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
}
var repoData *registry.RepositoryData
if registryEp == "" {
repoData, err = r.PushImageJSONIndex(name, imgList, false, nil)
if err != nil {
return err
}
} else {
repoData = &registry.RepositoryData{
ImgList: make(map[string]*registry.ImgData),
Tokens: []string{},
Endpoints: []string{registryEp},
}
tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens)
if err != nil && err.Error() != "Repository not found" {
return err
} else if err == nil {
for tag, id := range tagsList {
repoData.ImgList[id] = &registry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
}
repoData, err = r.PushImageJSONIndex(indexEp, remoteName, imgList, false, nil)
if err != nil {
return err
}
for _, ep := range repoData.Endpoints {
if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep)
}
out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo)))
out.Write(sf.FormatStatus("Pushing repository %s (%d tags)", localName, len(localRepo)))
// For each image within the repo, push them
for _, elem := range imgList {
if _, exists := repoData.ImgList[elem.ID]; exists {
out.Write(sf.FormatStatus("Image %s already on registry, skipping", name))
out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID))
continue
} else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) {
fmt.Fprintf(out, "Image %s already on registry, skipping\n", name)
out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID))
continue
}
if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil {
if err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil {
// FIXME: Continue on error?
return err
}
out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"/repositories/"+srvName+"/tags/"+elem.Tag))
if err := r.PushRegistryTag(srvName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag))
if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
return err
}
}
}
if registryEp == "" {
if _, err := r.PushImageJSONIndex(name, imgList, true, repoData.Endpoints); err != nil {
return err
}
if _, err := r.PushImageJSONIndex(indexEp, remoteName, imgList, true, repoData.Endpoints); err != nil {
return err
}
return nil
}
func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId, ep string, token []string, sf *utils.StreamFormatter) error {
func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgId, "json"))
jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgID, "json"))
if err != nil {
return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgId, err)
return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err)
}
out.Write(sf.FormatStatus("Pushing %s", imgId))
out.Write(sf.FormatStatus("Pushing %s", imgID))
// Make sure we have the image's checksum
checksum, err := srv.getChecksum(imgId)
checksum, err := srv.getChecksum(imgID)
if err != nil {
return err
}
imgData := &registry.ImgData{
ID: imgId,
ID: imgID,
Checksum: checksum,
}
// Send the json
if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil {
if err == registry.ErrAlreadyExists {
out.Write(sf.FormatStatus("Image %s already uploaded ; skipping", imgData.ID))
out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID))
return nil
}
return err
@@ -680,11 +642,11 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId,
// Retrieve the tarball to be sent
var layerData *TempArchive
// If the archive exists, use it
file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgId)))
file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgID)))
if err != nil {
if os.IsNotExist(err) {
// If the archive does not exist, create one from the layer
layerData, err = srv.runtime.graph.TempLayerArchive(imgId, Xz, out)
layerData, err = srv.runtime.graph.TempLayerArchive(imgID, Xz, sf, out)
if err != nil {
return fmt.Errorf("Failed to generate layer archive: %s", err)
}
@@ -711,33 +673,41 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId,
}
// FIXME: Allow to interupt current push when new push of same image is done.
func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error {
if err := srv.poolAdd("push", name); err != nil {
func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error {
if err := srv.poolAdd("push", localName); err != nil {
return err
}
defer srv.poolRemove("push", localName)
// Resolve the Repository name from fqn to endpoint + name
endpoint, remoteName, err := registry.ResolveRepositoryName(localName)
if err != nil {
return err
}
defer srv.poolRemove("push", name)
out = utils.NewWriteFlusher(out)
img, err := srv.runtime.graph.Get(name)
img, err := srv.runtime.graph.Get(localName)
r, err2 := registry.NewRegistry(srv.runtime.root, authConfig)
if err2 != nil {
return err2
}
if err != nil {
out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name])))
reposLen := len(srv.runtime.repositories.Repositories[localName])
out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", localName, reposLen))
// If it fails, try to get the repository
if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists {
if err := srv.pushRepository(r, out, name, endpoint, localRepo, sf); err != nil {
if localRepo, exists := srv.runtime.repositories.Repositories[localName]; exists {
if err := srv.pushRepository(r, out, localName, remoteName, localRepo, endpoint, sf); err != nil {
return err
}
return nil
}
return err
}
out.Write(sf.FormatStatus("The push refers to an image: [%s]", name))
if err := srv.pushImage(r, out, name, img.ID, endpoint, nil, sf); err != nil {
var token []string
out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName))
if err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil {
return err
}
return nil
@@ -809,7 +779,7 @@ func (srv *Server) ContainerCreate(config *Config) (string, error) {
func (srv *Server) ContainerRestart(name string, t int) error {
if container := srv.runtime.Get(name); container != nil {
if err := container.Restart(t); err != nil {
return fmt.Errorf("Error restarting container %s: %s", name, err.Error())
return fmt.Errorf("Error restarting container %s: %s", name, err)
}
} else {
return fmt.Errorf("No such container: %s", name)
@@ -828,7 +798,7 @@ func (srv *Server) ContainerDestroy(name string, removeVolume bool) error {
volumes[volumeId] = struct{}{}
}
if err := srv.runtime.Destroy(container); err != nil {
return fmt.Errorf("Error destroying container %s: %s", name, err.Error())
return fmt.Errorf("Error destroying container %s: %s", name, err)
}
if removeVolume {
@@ -919,7 +889,7 @@ func (srv *Server) deleteImageParents(img *Image, imgs *[]APIRmi) error {
func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, error) {
//Untag the current image
var imgs []APIRmi
imgs := []APIRmi{}
tagDeleted, err := srv.runtime.repositories.Delete(repoName, tag)
if err != nil {
return nil, err
@@ -948,7 +918,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) {
}
if !autoPrune {
if err := srv.runtime.graph.Delete(img.ID); err != nil {
return nil, fmt.Errorf("Error deleting image %s: %s", name, err.Error())
return nil, fmt.Errorf("Error deleting image %s: %s", name, err)
}
return nil, nil
}
@@ -963,7 +933,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) {
return srv.deleteImage(img, name, tag)
}
func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error) {
func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) {
// Retrieve all images
images, err := srv.runtime.graph.All()
@@ -981,7 +951,7 @@ func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error)
}
// Loop on the children of the given image and check the config
for elem := range imageMap[imgId] {
for elem := range imageMap[imgID] {
img, err := srv.runtime.graph.Get(elem)
if err != nil {
return nil, err
@@ -996,7 +966,7 @@ func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error)
func (srv *Server) ContainerStart(name string, hostConfig *HostConfig) error {
if container := srv.runtime.Get(name); container != nil {
if err := container.Start(hostConfig); err != nil {
return fmt.Errorf("Error starting container %s: %s", name, err.Error())
return fmt.Errorf("Error starting container %s: %s", name, err)
}
} else {
return fmt.Errorf("No such container: %s", name)
@@ -1007,7 +977,7 @@ func (srv *Server) ContainerStart(name string, hostConfig *HostConfig) error {
func (srv *Server) ContainerStop(name string, t int) error {
if container := srv.runtime.Get(name); container != nil {
if err := container.Stop(t); err != nil {
return fmt.Errorf("Error stopping container %s: %s", name, err.Error())
return fmt.Errorf("Error stopping container %s: %s", name, err)
}
} else {
return fmt.Errorf("No such container: %s", name)
@@ -1119,7 +1089,6 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) (
srv := &Server{
runtime: runtime,
enableCors: enableCors,
lock: &sync.Mutex{},
pullingPool: make(map[string]struct{}),
pushingPool: make(map[string]struct{}),
}
@@ -1128,9 +1097,9 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) (
}
type Server struct {
sync.Mutex
runtime *Runtime
enableCors bool
lock *sync.Mutex
pullingPool map[string]struct{}
pushingPool map[string]struct{}
}
components/engine/server_test.go
+3 -3
View File
@@ -31,7 +31,7 @@ func TestContainerTagImageDelete(t *testing.T) {
}
if len(images) != len(initialImages)+2 {
t.Errorf("Excepted %d images, %d found", len(initialImages)+2, len(images))
t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images))
}
if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil {
@@ -44,7 +44,7 @@ func TestContainerTagImageDelete(t *testing.T) {
}
if len(images) != len(initialImages)+1 {
t.Errorf("Excepted %d images, %d found", len(initialImages)+1, len(images))
t.Errorf("Expected %d images, %d found", len(initialImages)+1, len(images))
}
if _, err := srv.ImageDelete("utest:tag1", true); err != nil {
@@ -57,7 +57,7 @@ func TestContainerTagImageDelete(t *testing.T) {
}
if len(images) != len(initialImages) {
t.Errorf("Excepted %d image, %d found", len(initialImages), len(images))
t.Errorf("Expected %d image, %d found", len(initialImages), len(images))
}
}
components/engine/state.go
+2 -13
View File
@@ -8,11 +8,11 @@ import (
)
type State struct {
sync.Mutex
Running bool
Pid int
ExitCode int
StartedAt time.Time
l *sync.Mutex
Ghost bool
}
@@ -29,6 +29,7 @@ func (s *State) String() string {
func (s *State) setRunning(pid int) {
s.Running = true
s.Ghost = false
s.ExitCode = 0
s.Pid = pid
s.StartedAt = time.Now()
@@ -39,15 +40,3 @@ func (s *State) setStopped(exitCode int) {
s.Pid = 0
s.ExitCode = exitCode
}
func (s *State) initLock() {
s.l = &sync.Mutex{}
}
func (s *State) lock() {
s.l.Lock()
}
func (s *State) unlock() {
s.l.Unlock()
}
components/engine/tags.go
+7 -10
View File
@@ -70,11 +70,11 @@ func (store *TagStore) LookupImage(name string) (*Image, error) {
if err != nil {
// FIXME: standardize on returning nil when the image doesn't exist, and err for everything else
// (so we can pass all errors here)
repoAndTag := strings.SplitN(name, ":", 2)
if len(repoAndTag) == 1 {
repoAndTag = append(repoAndTag, DEFAULTTAG)
repos, tag := utils.ParseRepositoryTag(name)
if tag == "" {
tag = DEFAULTTAG
}
if i, err := store.GetImage(repoAndTag[0], repoAndTag[1]); err != nil {
if i, err := store.GetImage(repos, tag); err != nil {
return nil, err
} else if i == nil {
return nil, fmt.Errorf("Image does not exist: %s", name)
@@ -197,7 +197,7 @@ func (store *TagStore) Get(repoName string) (Repository, error) {
return nil, nil
}
func (store *TagStore) GetImage(repoName, tagOrId string) (*Image, error) {
func (store *TagStore) GetImage(repoName, tagOrID string) (*Image, error) {
repo, err := store.Get(repoName)
if err != nil {
return nil, err
@@ -206,11 +206,11 @@ func (store *TagStore) GetImage(repoName, tagOrId string) (*Image, error) {
}
//go through all the tags, to see if tag is in fact an ID
for _, revision := range repo {
if strings.HasPrefix(revision, tagOrId) {
if strings.HasPrefix(revision, tagOrID) {
return store.graph.Get(revision)
}
}
if revision, exists := repo[tagOrId]; exists {
if revision, exists := repo[tagOrID]; exists {
return store.graph.Get(revision)
}
return nil, nil
@@ -221,9 +221,6 @@ func validateRepoName(name string) error {
if name == "" {
return fmt.Errorf("Repository name can't be empty")
}
if strings.Contains(name, ":") {
return fmt.Errorf("Illegal repository name: %s", name)
}
return nil
}
components/engine/tags_test.go
+2 -2
View File
@@ -35,13 +35,13 @@ func TestLookupImage(t *testing.T) {
t.Errorf("Expected 0 image, 1 found")
}
if img, err := runtime.repositories.LookupImage(unitTestImageId); err != nil {
if img, err := runtime.repositories.LookupImage(unitTestImageID); err != nil {
t.Fatal(err)
} else if img == nil {
t.Errorf("Expected 1 image, none found")
}
if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + unitTestImageId); err != nil {
if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + unitTestImageID); err != nil {
t.Fatal(err)
} else if img == nil {
t.Errorf("Expected 1 image, none found")
components/engine/term/term.go
+1 -1
View File
@@ -12,8 +12,8 @@ type State struct {
}
type Winsize struct {
Width uint16
Height uint16
Width uint16
x uint16
y uint16
}
components/engine/testing/README.rst
+10
View File
@@ -30,6 +30,16 @@ Deployment
export AWS_KEYPAIR_NAME=xxxxxxxxxxxx
export AWS_SSH_PRIVKEY=xxxxxxxxxxxx
# Define email recipient and IRC channel
export EMAIL_RCP=xxxxxx@domain.com
export IRC_CHANNEL=docker
# Define buildbot credentials
export BUILDBOT_PWD=xxxxxxxxxxxx
export IRC_PWD=xxxxxxxxxxxx
export SMTP_USER=xxxxxxxxxxxx
export SMTP_PWD=xxxxxxxxxxxx
# Checkout docker
git clone git://github.com/dotcloud/docker.git
components/engine/testing/Vagrantfile
Vendored
+3 -1
View File
@@ -27,7 +27,9 @@ Vagrant::Config.run do |config|
pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \
"pip install -r #{CFG_PATH}/requirements.txt; " \
"chown #{USER}.#{USER} /data; cd /data; " \
"#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH}; "
"#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \
"#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \
"#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; "
# Install docker dependencies
pkg_cmd << "apt-get install -q -y python-software-properties; " \
"add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \
components/engine/testing/buildbot/master.cfg
+30 -16
View File
@@ -5,9 +5,11 @@ from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.changes import filter
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Interpolate
from buildbot.steps.shell import ShellCommand
from buildbot.status import html
from buildbot.status import html, words
from buildbot.status.web import authz, auth
from buildbot.status.mail import MailNotifier
PORT_WEB = 80 # Buildbot webserver port
PORT_GITHUB = 8011 # Buildbot github hook port
@@ -15,20 +17,27 @@ PORT_MASTER = 9989 # Port where buildbot master listen buildworkers
TEST_USER = 'buildbot' # Credential to authenticate build triggers
TEST_PWD = 'docker' # Credential to authenticate build triggers
BUILDER_NAME = 'docker'
BUILDPASSWORD = 'pass-docker' # Credential to authenticate buildworkers
GITHUB_DOCKER = "github.com/dotcloud/docker"
DOCKER_PATH = "/data/docker"
BUILDER_PATH = "/data/buildbot/slave/{0}/build".format(BUILDER_NAME)
GITHUB_DOCKER = 'github.com/dotcloud/docker'
DOCKER_PATH = '/data/docker'
BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME)
DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker'
# Credentials set by setup.sh and Vagrantfile
BUILDBOT_PWD = ''
IRC_PWD = ''
IRC_CHANNEL = ''
SMTP_USER = ''
SMTP_PWD = ''
EMAIL_RCP = ''
c = BuildmasterConfig = {}
c['title'] = "Docker"
c['titleURL'] = "waterfall"
c['buildbotURL'] = "http://0.0.0.0:{0}/".format(PORT_WEB)
c['buildbotURL'] = "http://docker-ci.dotcloud.com/"
c['db'] = {'db_url':"sqlite:///state.sqlite"}
c['slaves'] = [BuildSlave('buildworker', BUILDPASSWORD)]
c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)]
c['slavePortnum'] = PORT_MASTER
c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])]
@@ -36,20 +45,25 @@ c['schedulers'].append(SingleBranchScheduler(name="all",
change_filter=filter.ChangeFilter(branch='master'),treeStableTimer=None,
builderNames=[BUILDER_NAME]))
# Docker test command
test_cmd = ("cd /tmp; rm -rf {0}; export GOPATH={0}; go get -d {1}; cd {2}; "
"go test").format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH)
# Builder
factory = BuildFactory()
factory.addStep(ShellCommand(description='Docker',logEnviron=False,
usePTY=True,command=test_cmd))
factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True,
command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; "
"go get -d {1}; cd {2}; git reset --hard %(src::revision:-unknown)s; "
"go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))]))
c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'],
factory=factory)]
# Status
authz_cfg=authz.Authz(auth=auth.BasicAuth([(TEST_USER,TEST_PWD)]),
authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]),
forceBuild='auth')
c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)]
c['status'].append(html.WebStatus(http_port=PORT_GITHUB,allowForce=True,
change_hook_dialects={ 'github' : True }))
c['status'].append(html.WebStatus(http_port=PORT_GITHUB, allowForce=True,
change_hook_dialects={ 'github': True }))
c['status'].append(MailNotifier(fromaddr='buildbot@docker.io',
sendToInterestedUsers=False, extraRecipients=[EMAIL_RCP],
mode='failing', relayhost='smtp.mailgun.org', smtpPort=587, useTls=True,
smtpUser=SMTP_USER, smtpPassword=SMTP_PWD))
c['status'].append(words.IRC("irc.freenode.net", "dockerqabot",
channels=[IRC_CHANNEL], password=IRC_PWD, allowForce=True,
notify_events={'exception':1, 'successToFailure':1, 'failureToSuccess':1}))
components/engine/testing/buildbot/setup.sh
+12 -2
View File
@@ -6,11 +6,16 @@
USER=$1
CFG_PATH=$2
BUILDBOT_PWD=$3
IRC_PWD=$4
IRC_CHANNEL=$5
SMTP_USER=$6
SMTP_PWD=$7
EMAIL_RCP=$8
BUILDBOT_PATH="/data/buildbot"
DOCKER_PATH="/data/docker"
SLAVE_NAME="buildworker"
SLAVE_SOCKET="localhost:9989"
BUILDBOT_PWD="pass-docker"
export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin"
function run { su $USER -c "$1"; }
@@ -23,7 +28,12 @@ run "mkdir -p $BUILDBOT_PATH"
cd $BUILDBOT_PATH
run "buildbot create-master master"
run "cp $CFG_PATH/master.cfg master"
run "sed -i -E 's#(DOCKER_PATH = ).+#\1\"$DOCKER_PATH\"#' master/master.cfg"
run "sed -i -E 's#(BUILDBOT_PWD = ).+#\1\"$BUILDBOT_PWD\"#' master/master.cfg"
run "sed -i -E 's#(IRC_PWD = ).+#\1\"$IRC_PWD\"#' master/master.cfg"
run "sed -i -E 's#(IRC_CHANNEL = ).+#\1\"$IRC_CHANNEL\"#' master/master.cfg"
run "sed -i -E 's#(SMTP_USER = ).+#\1\"$SMTP_USER\"#' master/master.cfg"
run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg"
run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg"
run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
# Allow buildbot subprocesses (docker tests) to properly run in containers,
components/engine/utils.go
+5 -4
View File
@@ -20,7 +20,8 @@ func CompareConfig(a, b *Config) bool {
if len(a.Cmd) != len(b.Cmd) ||
len(a.Dns) != len(b.Dns) ||
len(a.Env) != len(b.Env) ||
len(a.PortSpecs) != len(b.PortSpecs) {
len(a.PortSpecs) != len(b.PortSpecs) ||
len(a.Entrypoint) != len(b.Entrypoint) {
return false
}
@@ -53,9 +54,6 @@ func CompareConfig(a, b *Config) bool {
}
func MergeConfig(userConf, imageConf *Config) {
if userConf.Hostname == "" {
userConf.Hostname = imageConf.Hostname
}
if userConf.User == "" {
userConf.User = imageConf.User
}
@@ -92,4 +90,7 @@ func MergeConfig(userConf, imageConf *Config) {
if userConf.Entrypoint == nil || len(userConf.Entrypoint) == 0 {
userConf.Entrypoint = imageConf.Entrypoint
}
if userConf.Volumes == nil || len(userConf.Volumes) == 0 {
userConf.Volumes = imageConf.Volumes
}
}
components/engine/utils/utils.go
+32 -28
View File
@@ -170,10 +170,9 @@ func SelfPath() string {
return path
}
type NopWriter struct {
}
type NopWriter struct{}
func (w *NopWriter) Write(buf []byte) (int, error) {
func (*NopWriter) Write(buf []byte) (int, error) {
return len(buf), nil
}
@@ -188,10 +187,10 @@ func NopWriteCloser(w io.Writer) io.WriteCloser {
}
type bufReader struct {
sync.Mutex
buf *bytes.Buffer
reader io.Reader
err error
l sync.Mutex
wait sync.Cond
}
@@ -200,7 +199,7 @@ func NewBufReader(r io.Reader) *bufReader {
buf: &bytes.Buffer{},
reader: r,
}
reader.wait.L = &reader.l
reader.wait.L = &reader.Mutex
go reader.drain()
return reader
}
@@ -209,14 +208,14 @@ func (r *bufReader) drain() {
buf := make([]byte, 1024)
for {
n, err := r.reader.Read(buf)
r.l.Lock()
r.Lock()
if err != nil {
r.err = err
} else {
r.buf.Write(buf[0:n])
}
r.wait.Signal()
r.l.Unlock()
r.Unlock()
if err != nil {
break
}
@@ -224,8 +223,8 @@ func (r *bufReader) drain() {
}
func (r *bufReader) Read(p []byte) (n int, err error) {
r.l.Lock()
defer r.l.Unlock()
r.Lock()
defer r.Unlock()
for {
n, err = r.buf.Read(p)
if n > 0 {
@@ -247,27 +246,27 @@ func (r *bufReader) Close() error {
}
type WriteBroadcaster struct {
mu sync.Mutex
sync.Mutex
writers map[io.WriteCloser]struct{}
}
func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser) {
w.mu.Lock()
w.Lock()
w.writers[writer] = struct{}{}
w.mu.Unlock()
w.Unlock()
}
// FIXME: Is that function used?
// FIXME: This relies on the concrete writer type used having equality operator
func (w *WriteBroadcaster) RemoveWriter(writer io.WriteCloser) {
w.mu.Lock()
w.Lock()
delete(w.writers, writer)
w.mu.Unlock()
w.Unlock()
}
func (w *WriteBroadcaster) Write(p []byte) (n int, err error) {
w.mu.Lock()
defer w.mu.Unlock()
w.Lock()
defer w.Unlock()
for writer := range w.writers {
if n, err := writer.Write(p); err != nil || n != len(p) {
// On error, evict the writer
@@ -278,8 +277,8 @@ func (w *WriteBroadcaster) Write(p []byte) (n int, err error) {
}
func (w *WriteBroadcaster) CloseWriters() error {
w.mu.Lock()
defer w.mu.Unlock()
w.Lock()
defer w.Unlock()
for writer := range w.writers {
writer.Close()
}
@@ -689,11 +688,7 @@ func ParseHost(host string, port int, addr string) string {
}
func GetReleaseVersion() string {
type githubTag struct {
Name string `json:"name"`
}
resp, err := http.Get("https://api.github.com/repos/dotcloud/docker/tags")
resp, err := http.Get("http://get.docker.io/latest")
if err != nil {
return ""
}
@@ -702,10 +697,19 @@ func GetReleaseVersion() string {
if err != nil {
return ""
}
var tags []githubTag
err = json.Unmarshal(body, &tags)
if err != nil || len(tags) == 0 {
return ""
return strings.TrimSpace(string(body))
}
// Get a repos name and returns the right reposName + tag
// The tag can be confusing because of a port in a repository name.
// Ex: localhost.localdomain:5000/samalba/hipache:latest
func ParseRepositoryTag(repos string) (string, string) {
n := strings.LastIndex(repos, ":")
if n < 0 {
return repos, ""
}
return strings.TrimPrefix(tags[0].Name, "v")
if tag := repos[n+1:]; !strings.Contains(tag, "/") {
return repos[:n], tag
}
return repos, ""
}