From 02d950665461707b288f8021d2c0670c5988cdb8 Mon Sep 17 00:00:00 2001 From: Caleb Spare Date: Sun, 7 Jul 2013 23:24:52 -0700 Subject: [PATCH 1/4] Test pulling remote files using ADD in a buildfile. Upstream-commit: f236e62d9d288ac695129157d1753512f5cd2b0a Component: engine --- components/engine/buildfile_test.go | 117 +++++++++++++++++++++------- 1 file changed, 87 insertions(+), 30 deletions(-) diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 14edbc088f..602af5061b 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -3,13 +3,17 @@ package docker import ( "fmt" "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "strings" "testing" ) // mkTestContext generates a build context from the contents of the provided dockerfile. // This context is suitable for use as an argument to BuildFile.Build() func mkTestContext(dockerfile string, files [][2]string, t *testing.T) Archive { - context, err := mkBuildContext(fmt.Sprintf(dockerfile, unitTestImageID), files) + context, err := mkBuildContext(dockerfile, files) if err != nil { t.Fatal(err) } @@ -22,6 +26,8 @@ type testContextTemplate struct { dockerfile string // Additional files in the context, eg [][2]string{"./passwd", "gordon"} files [][2]string + // Additional remote files to host on a local HTTP server. + remoteFiles [][2]string } // A table of all the contexts to build and test. @@ -29,27 +35,31 @@ type testContextTemplate struct { var testContexts = []testContextTemplate{ { ` -from %s +from {IMAGE} run sh -c 'echo root:testpass > /tmp/passwd' run mkdir -p /var/run/sshd run [ "$(cat /tmp/passwd)" = "root:testpass" ] run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] `, nil, + nil, }, { ` -from %s +from {IMAGE} add foo /usr/lib/bla/bar -run [ "$(cat /usr/lib/bla/bar)" = 'hello world!' ] +run [ "$(cat /usr/lib/bla/bar)" = 'hello' ] +add http://{SERVERADDR}/baz /usr/lib/baz/quux +run [ "$(cat /usr/lib/baz/quux)" = 'world!' ] `, - [][2]string{{"foo", "hello world!"}}, + [][2]string{{"foo", "hello"}}, + [][2]string{{"/baz", "world!"}}, }, { ` -from %s +from {IMAGE} add f / run [ "$(cat /f)" = "hello" ] add f /abc @@ -71,38 +81,70 @@ run [ "$(cat /somewheeeere/over/the/rainbooow/ga)" = "bu" ] {"f", "hello"}, {"d/ga", "bu"}, }, + nil, }, { ` -from %s +from {IMAGE} env FOO BAR run [ "$FOO" = "BAR" ] `, nil, - }, - - { - ` -from %s -ENTRYPOINT /bin/echo -CMD Hello world -`, nil, }, { ` -from %s +from {IMAGE} +ENTRYPOINT /bin/echo +CMD Hello world +`, + nil, + nil, + }, + + { + ` +from {IMAGE} VOLUME /test CMD Hello world `, nil, + nil, }, } // FIXME: test building with 2 successive overlapping ADD commands +func constructDockerfile(template string, ip net.IP, port string) string { + serverAddr := fmt.Sprintf("%s:%s", ip, port) + replacer := strings.NewReplacer("{IMAGE}", unitTestImageID, "{SERVERADDR}", serverAddr) + return replacer.Replace(template) +} + +func mkTestingFileServer(files [][2]string) (*httptest.Server, error) { + mux := http.NewServeMux() + for _, file := range files { + name, contents := file[0], file[1] + mux.HandleFunc(name, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(contents)) + }) + } + + // This is how httptest.NewServer sets up a net.Listener, except that our listener must accept remote + // connections (from the container). + listener, err := net.Listen("tcp", ":0") + if err != nil { + return nil, err + } + + s := httptest.NewUnstartedServer(mux) + s.Listener = listener + s.Start() + return s, nil +} + func TestBuild(t *testing.T) { for _, ctx := range testContexts { buildImage(ctx, t) @@ -121,9 +163,24 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { pullingPool: make(map[string]struct{}), pushingPool: make(map[string]struct{}), } - buildfile := NewBuildFile(srv, ioutil.Discard, false) - id, err := buildfile.Build(mkTestContext(context.dockerfile, context.files, t)) + httpServer, err := mkTestingFileServer(context.remoteFiles) + if err != nil { + t.Fatal(err) + } + defer httpServer.Close() + + idx := strings.LastIndex(httpServer.URL, ":") + if idx < 0 { + t.Fatalf("could not get port from test http server address %s", httpServer.URL) + } + port := httpServer.URL[idx+1:] + + ip := runtime.networkManager.bridgeNetwork.IP + dockerfile := constructDockerfile(context.dockerfile, ip, port) + + buildfile := NewBuildFile(srv, ioutil.Discard, false) + id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { t.Fatal(err) } @@ -137,10 +194,10 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { func TestVolume(t *testing.T) { img := buildImage(testContextTemplate{` - from %s + from {IMAGE} volume /test cmd Hello world - `, nil}, t) + `, nil, nil}, t) if len(img.Config.Volumes) == 0 { t.Fail() @@ -154,9 +211,9 @@ func TestVolume(t *testing.T) { func TestBuildMaintainer(t *testing.T) { img := buildImage(testContextTemplate{` - from %s + from {IMAGE} maintainer dockerio - `, nil}, t) + `, nil, nil}, t) if img.Author != "dockerio" { t.Fail() @@ -165,10 +222,10 @@ func TestBuildMaintainer(t *testing.T) { func TestBuildEnv(t *testing.T) { img := buildImage(testContextTemplate{` - from %s + from {IMAGE} env port 4243 `, - nil}, t) + nil, nil}, t) if img.Config.Env[0] != "port=4243" { t.Fail() @@ -177,10 +234,10 @@ func TestBuildEnv(t *testing.T) { func TestBuildCmd(t *testing.T) { img := buildImage(testContextTemplate{` - from %s + from {IMAGE} cmd ["/bin/echo", "Hello World"] `, - nil}, t) + nil, nil}, t) if img.Config.Cmd[0] != "/bin/echo" { t.Log(img.Config.Cmd[0]) @@ -194,10 +251,10 @@ func TestBuildCmd(t *testing.T) { func TestBuildExpose(t *testing.T) { img := buildImage(testContextTemplate{` - from %s + from {IMAGE} expose 4243 `, - nil}, t) + nil, nil}, t) if img.Config.PortSpecs[0] != "4243" { t.Fail() @@ -206,10 +263,10 @@ func TestBuildExpose(t *testing.T) { func TestBuildEntrypoint(t *testing.T) { img := buildImage(testContextTemplate{` - from %s + from {IMAGE} entrypoint ["/bin/echo"] `, - nil}, t) + nil, nil}, t) if img.Config.Entrypoint[0] != "/bin/echo" { } From adbaef1554c2ef026c3bb1ecd59df1ce2fd6aa26 Mon Sep 17 00:00:00 2001 From: Caleb Spare Date: Mon, 8 Jul 2013 00:43:22 -0700 Subject: [PATCH 2/4] Buildfile: for ADD command, determine filename from URL. This is used if the destination is a directory. This makes the URL download behavior more closely match file copying. Fixes #1142. Upstream-commit: 2b0ebf5d32c65276ce50fce168f32483ffb9c311 Component: engine --- components/engine/buildfile.go | 21 ++++++++++++++++++++- components/engine/buildfile_test.go | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 7ade058c69..75ebdd7a7c 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -7,6 +7,7 @@ import ( "github.com/dotcloud/docker/utils" "io" "io/ioutil" + "net/url" "os" "path" "reflect" @@ -201,6 +202,24 @@ func (b *buildFile) addRemote(container *Container, orig, dest string) error { } defer file.Body.Close() + // If the destination is a directory, figure out the filename. + if strings.HasSuffix(dest, "/") { + u, err := url.Parse(orig) + if err != nil { + return err + } + path := u.Path + if strings.HasSuffix(path, "/") { + path = path[:len(path)-1] + } + parts := strings.Split(path, "/") + filename := parts[len(parts)-1] + if filename == "" { + return fmt.Errorf("cannot determine filename from url: %s", u) + } + dest = dest + filename + } + return container.Inject(file.Body, dest) } @@ -208,7 +227,7 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { origPath := path.Join(b.context, orig) destPath := path.Join(container.RootfsPath(), dest) // Preserve the trailing '/' - if dest[len(dest)-1] == '/' { + if strings.HasSuffix(dest, "/") { destPath = destPath + "/" } fi, err := os.Stat(origPath) diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 602af5061b..b7eca52336 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -84,6 +84,22 @@ run [ "$(cat /somewheeeere/over/the/rainbooow/ga)" = "bu" ] nil, }, + { + ` +from {IMAGE} +add http://{SERVERADDR}/x /a/b/c +run [ "$(cat /a/b/c)" = "hello" ] +add http://{SERVERADDR}/x?foo=bar / +run [ "$(cat /x)" = "hello" ] +add http://{SERVERADDR}/x /d/ +run [ "$(cat /d/x)" = "hello" ] +add http://{SERVERADDR} /e +run [ "$(cat /e)" = "blah" ] +`, + nil, + [][2]string{{"/x", "hello"}, {"/", "blah"}}, + }, + { ` from {IMAGE} From ed8e1bd7e6b319c614ffd1a2b50bae4ee2dd71c6 Mon Sep 17 00:00:00 2001 From: Caleb Spare Date: Mon, 8 Jul 2013 01:14:01 -0700 Subject: [PATCH 3/4] Remove some trailing whitespace. Upstream-commit: 416fdaa3d5d7b84f7e36fdfcff7153e3a38262c9 Component: engine --- .../engine/docs/sources/use/builder.rst | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/components/engine/docs/sources/use/builder.rst b/components/engine/docs/sources/use/builder.rst index 7f370609c8..ea98541dbf 100644 --- a/components/engine/docs/sources/use/builder.rst +++ b/components/engine/docs/sources/use/builder.rst @@ -30,7 +30,7 @@ build succeeds: ``docker build -t shykes/myapp .`` -Docker will run your steps one-by-one, committing the result if necessary, +Docker will run your steps one-by-one, committing the result if necessary, before finally outputting the ID of your new image. 2. Format @@ -43,7 +43,7 @@ The Dockerfile format is quite simple: # Comment INSTRUCTION arguments -The Instruction is not case-sensitive, however convention is for them to be +The Instruction is not case-sensitive, however convention is for them to be UPPERCASE in order to distinguish them from arguments more easily. Docker evaluates the instructions in a Dockerfile in order. **The first @@ -106,7 +106,7 @@ The ``CMD`` instruction sets the command to be executed when running the image. This is functionally equivalent to running ``docker commit -run '{"Cmd": }'`` outside the builder. -.. note:: +.. note:: Don't confuse `RUN` with `CMD`. `RUN` actually runs a command and commits the result; `CMD` does not execute anything at build time, but specifies the intended command for the image. @@ -131,7 +131,7 @@ value ````. This value will be passed to all future ``RUN`` instructions. This is functionally equivalent to prefixing the command with ``=`` -.. note:: +.. note:: The environment variables will persist when a container is run from the resulting image. @@ -158,10 +158,10 @@ The copy obeys the following rules: (identity, gzip, bzip2 or xz), it is unpacked as a directory. When a directory is copied or unpacked, it has the same behavior as - ``tar -x``: the result is the union of + ``tar -x``: the result is the union of 1. whatever existed at the destination path and - 2. the contents of the source tree, + 2. the contents of the source tree, with conflicts resolved in favor of 2) on a file-by-file basis. @@ -203,14 +203,14 @@ container created from the image. # Nginx # # VERSION 0.0.1 - + FROM ubuntu MAINTAINER Guillaume J. Charmes "guillaume@dotcloud.com" - + # make sure the package repository is up to date RUN echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list RUN apt-get update - + RUN apt-get install -y inotify-tools nginx apache2 openssh-server .. code-block:: bash @@ -218,12 +218,12 @@ container created from the image. # Firefox over VNC # # VERSION 0.3 - + FROM ubuntu # make sure the package repository is up to date RUN echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list RUN apt-get update - + # Install vnc, xvfb in order to create a 'fake' display and firefox RUN apt-get install -y x11vnc xvfb firefox RUN mkdir /.vnc @@ -231,7 +231,7 @@ container created from the image. RUN x11vnc -storepasswd 1234 ~/.vnc/passwd # Autostart firefox (might not be the best way, but it does the trick) RUN bash -c 'echo "firefox" >> /.bashrc' - + EXPOSE 5900 CMD ["x11vnc", "-forever", "-usepw", "-create"] From e3ecb715a38209f25483251b6f544fac97eb8052 Mon Sep 17 00:00:00 2001 From: Caleb Spare Date: Mon, 8 Jul 2013 01:29:13 -0700 Subject: [PATCH 4/4] Update ADD documentation to specify new behavior. Upstream-commit: c383d598809daec6bccd60bd211c10f6a4ef60b0 Component: engine --- components/engine/docs/sources/use/builder.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/engine/docs/sources/use/builder.rst b/components/engine/docs/sources/use/builder.rst index ea98541dbf..302527a740 100644 --- a/components/engine/docs/sources/use/builder.rst +++ b/components/engine/docs/sources/use/builder.rst @@ -152,6 +152,14 @@ destination container. The copy obeys the following rules: +* If ```` is a URL and ```` does not end with a trailing slash, + then a file is downloaded from the URL and copied to ````. +* If ```` is a URL and ```` does end with a trailing slash, + then the filename is inferred from the URL and the file is downloaded to + ``/``. For instance, ``ADD http://example.com/foobar /`` + would create the file ``/foobar``. The URL must have a nontrivial path + so that an appropriate filename can be discovered in this case + (``http://example.com`` will not work). * If ```` is a directory, the entire directory is copied, including filesystem metadata. * If ````` is a tar archive in a recognized compression format