From c7b15689018402d7940e26f91569d64cac7bb830 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 6 Sep 2013 19:02:59 -0700 Subject: [PATCH 01/21] Copy dind wrapper script from github.com/jpetazzo/dind Upstream-commit: c983023661deac0713eaa218c6bdfd20ec5eee45 Component: engine --- components/engine/hack/dind | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 components/engine/hack/dind diff --git a/components/engine/hack/dind b/components/engine/hack/dind new file mode 100644 index 0000000000..5d606ed42f --- /dev/null +++ b/components/engine/hack/dind @@ -0,0 +1,56 @@ +#!/bin/bash + +# First, make sure that cgroups are mounted correctly. +CGROUP=/sys/fs/cgroup + +[ -d $CGROUP ] || + mkdir $CGROUP + +mountpoint -q $CGROUP || + mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { + echo "Could not make a tmpfs mount. Did you use -privileged?" + exit 1 + } + +# Mount the cgroup hierarchies exactly as they are in the parent system. +for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) +do + [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS + mountpoint -q $CGROUP/$SUBSYS || + mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS +done + +# Note: as I write those lines, the LXC userland tools cannot setup +# a "sub-container" properly if the "devices" cgroup is not in its +# own hierarchy. Let's detect this and issue a warning. +grep -q :devices: /proc/1/cgroup || + echo "WARNING: the 'devices' cgroup should be in its own hierarchy." +grep -qw devices /proc/1/cgroup || + echo "WARNING: it looks like the 'devices' cgroup is not mounted." + +# Now, close extraneous file descriptors. +pushd /proc/self/fd +for FD in * +do + case "$FD" in + # Keep stdin/stdout/stderr + [012]) + ;; + # Nuke everything else + *) + eval exec "$FD>&-" + ;; + esac +done +popd + +# If we were given a PORT environment variable, start as a simple daemon; +# otherwise, spawn a shell as well +if [ "$PORT" ] +then + exec docker -d -H 0.0.0.0:$PORT +else + + docker -d & + exec bash +fi From ae4d5c759ff91a03bd17febf4222d400725ee61c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 6 Sep 2013 19:03:29 -0700 Subject: [PATCH 02/21] Adapt the original dind script and add a description Upstream-commit: 3c80bd76cf6aa8f6918439f8082b0a8c34fd2c0e Component: engine --- components/engine/hack/dind | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/engine/hack/dind b/components/engine/hack/dind index 5d606ed42f..f9a654670b 100644 --- a/components/engine/hack/dind +++ b/components/engine/hack/dind @@ -1,5 +1,14 @@ #!/bin/bash +# DinD: a wrapper script which allows docker to be run inside a docker container. +# Original version by Jerome Petazzoni +# See the blog post: http://blog.docker.io/2013/09/docker-can-now-run-within-docker/ +# +# This script should be executed inside a docker container in privilieged mode +# ('docker run -privileged', introduced in docker 0.6). + +# Usage: dind CMD [ARG...] + # First, make sure that cgroups are mounted correctly. CGROUP=/sys/fs/cgroup @@ -44,13 +53,4 @@ do done popd -# If we were given a PORT environment variable, start as a simple daemon; -# otherwise, spawn a shell as well -if [ "$PORT" ] -then - exec docker -d -H 0.0.0.0:$PORT -else - - docker -d & - exec bash -fi +exec $* From c090718b1011bb034ce1420601b9d796d3ec11f4 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 6 Sep 2013 19:19:03 -0700 Subject: [PATCH 03/21] Adapt Dockerfile to run docker tests inside docker Upstream-commit: 34eab428330ce4a56c22b03f3067dd8eeee7ed41 Component: engine --- components/engine/Dockerfile | 9 ++++++++- components/engine/hack/dind | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) mode change 100644 => 100755 components/engine/hack/dind diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 43f493d3ff..a0ef713c81 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -1,5 +1,5 @@ # This file describes the standard way to build Docker, using docker -docker-version 0.4.2 +docker-version 0.6.1 from ubuntu:12.04 maintainer Solomon Hykes # Build dependencies @@ -23,6 +23,9 @@ run apt-get install -y -q python-pip run pip install s3cmd run pip install python-magic run /bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY\n' > /.s3cfg +# Runtime dependencies +run apt-get install -y -q iptables +run apt-get install -y -q lxc # Download dependencies run PKG=github.com/kr/pty REV=27435c699; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV run PKG=github.com/gorilla/context/ REV=708054d61e5; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV @@ -32,6 +35,10 @@ run PKG=code.google.com/p/go.net/ REV=84a4013f96e0; hg clone http://$PKG /go/s # Upload docker source add . /go/src/github.com/dotcloud/docker run ln -s /go/src/github.com/dotcloud/docker /src +volume /var/lib/docker # Build the binary run cd /go/src/github.com/dotcloud/docker && hack/release/make.sh +workdir /go/src/github.com/dotcloud/docker +# Wrap all commands in the "docker-in-docker" script to allow nested containers +entrypoint ["hack/dind"] cmd cd /go/src/github.com/dotcloud/docker && hack/release/release.sh diff --git a/components/engine/hack/dind b/components/engine/hack/dind old mode 100644 new mode 100755 index f9a654670b..7810715675 --- a/components/engine/hack/dind +++ b/components/engine/hack/dind @@ -53,4 +53,7 @@ do done popd +# Mount /tmp +mount -t tmpfs none /tmp + exec $* From 4d40f8afe3d3f7cdeee3f6ce2f4f8e725107508a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 6 Sep 2013 19:58:05 -0700 Subject: [PATCH 04/21] Add usage instructions to the Dockerfile. Build, test and release docker using docker. Upstream-commit: fa806f26aff927eb0d98b41e79786ae98987da5b Component: engine --- components/engine/Dockerfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index a0ef713c81..1f3ae7e712 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -1,4 +1,24 @@ # This file describes the standard way to build Docker, using docker +# +# Usage: +# +# # Assemble the full dev environment. This is slow the first time. +# docker build -t docker . +# # Apparmor messes with privileged mode: disable it +# /etc/init.d/apparmor stop ; /etc/init.d/apparmor teardown +# +# # Run the test suite: +# docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker go test -v +# +# # Publish a release: +# docker run -privileged -lxc-conf=lxc.aa_profile=unconfined \ +# -e AWS_S3_BUCKET=baz \ +# -e AWS_ACCESS_KEY=foo \ +# -e AWS_SECRET_KEY=bar \ +# -e GPG_PASSPHRASE=gloubiboulga \ +# -lxc-conf=lxc.aa_profile=unconfined -privileged docker hack/release/release.sh +# + docker-version 0.6.1 from ubuntu:12.04 maintainer Solomon Hykes From 742fb66041377fe472ebc29508c4ea7f2140cb11 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 6 Sep 2013 20:14:03 -0700 Subject: [PATCH 05/21] Hack: improve the Dockerfile for an easier development workflow. Build dev container once, run a shell with source mount-binded, run tests as you edit. LIKE A BOSS. Upstream-commit: 47838051be695a4e74c80690ceef811dcd3ef62c Component: engine --- components/engine/Dockerfile | 10 +++------- .../hack/{release/README.md => RELEASE-CHECKLIST.md} | 0 components/engine/hack/{release => }/make.sh | 0 components/engine/hack/{release => }/release.sh | 0 4 files changed, 3 insertions(+), 7 deletions(-) rename components/engine/hack/{release/README.md => RELEASE-CHECKLIST.md} (100%) rename components/engine/hack/{release => }/make.sh (100%) rename components/engine/hack/{release => }/release.sh (100%) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 1f3ae7e712..0f04035ec4 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -16,7 +16,7 @@ # -e AWS_ACCESS_KEY=foo \ # -e AWS_SECRET_KEY=bar \ # -e GPG_PASSPHRASE=gloubiboulga \ -# -lxc-conf=lxc.aa_profile=unconfined -privileged docker hack/release/release.sh +# -lxc-conf=lxc.aa_profile=unconfined -privileged docker hack/release.sh # docker-version 0.6.1 @@ -52,13 +52,9 @@ run PKG=github.com/gorilla/context/ REV=708054d61e5; git clone http://$PKG /go/s run PKG=github.com/gorilla/mux/ REV=9b36453141c; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV run PKG=github.com/dotcloud/tar/ REV=e5ea6bb21a3294; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV run PKG=code.google.com/p/go.net/ REV=84a4013f96e0; hg clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && hg checkout $REV -# Upload docker source -add . /go/src/github.com/dotcloud/docker -run ln -s /go/src/github.com/dotcloud/docker /src volume /var/lib/docker -# Build the binary -run cd /go/src/github.com/dotcloud/docker && hack/release/make.sh workdir /go/src/github.com/dotcloud/docker # Wrap all commands in the "docker-in-docker" script to allow nested containers entrypoint ["hack/dind"] -cmd cd /go/src/github.com/dotcloud/docker && hack/release/release.sh +# Upload docker source +add . /go/src/github.com/dotcloud/docker diff --git a/components/engine/hack/release/README.md b/components/engine/hack/RELEASE-CHECKLIST.md similarity index 100% rename from components/engine/hack/release/README.md rename to components/engine/hack/RELEASE-CHECKLIST.md diff --git a/components/engine/hack/release/make.sh b/components/engine/hack/make.sh similarity index 100% rename from components/engine/hack/release/make.sh rename to components/engine/hack/make.sh diff --git a/components/engine/hack/release/release.sh b/components/engine/hack/release.sh similarity index 100% rename from components/engine/hack/release/release.sh rename to components/engine/hack/release.sh From 970480b8f8cf2b4730078f136de5e7341571be25 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 6 Sep 2013 20:16:13 -0700 Subject: [PATCH 06/21] Document using the Dockerfile for interactive dev/test cycles Upstream-commit: d757bd0904b71346b6c83622ae5685bc0c0d4349 Component: engine --- components/engine/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 0f04035ec4..122346c0dd 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -7,6 +7,10 @@ # # Apparmor messes with privileged mode: disable it # /etc/init.d/apparmor stop ; /etc/init.d/apparmor teardown # +# # Mount your source in an interactive container for quick testing: +# docker run -v `pwd`:/go/src/github.com/dotcloud/docker -privileged -lxc-conf=lxc.aa_profile=unconfined -i -t docker bash +# +# # # Run the test suite: # docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker go test -v # From 9327ca0955ad75cc1beb759784326304123c6b05 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Mon, 26 Aug 2013 15:51:22 -0700 Subject: [PATCH 07/21] move deps installation to vendor.sh script Upstream-commit: 20d24a450cbf22b6d5268bc1fe76022fa554d6be Component: engine --- components/engine/.gitignore | 1 + components/engine/Dockerfile | 9 ++------- components/engine/vendor.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) create mode 100755 components/engine/vendor.sh diff --git a/components/engine/.gitignore b/components/engine/.gitignore index ea62e34d19..034b2e6490 100644 --- a/components/engine/.gitignore +++ b/components/engine/.gitignore @@ -14,3 +14,4 @@ docs/_templates .gopath/ .dotcloud *.test +vendor/ diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 122346c0dd..6c8cb4cd6d 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -35,7 +35,7 @@ run apt-get install -y -q mercurial # Install Go run curl -s https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | tar -v -C /usr/local -xz env PATH /usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin -env GOPATH /go +env GOPATH /go:/vendor env CGO_ENABLED 0 run cd /tmp && echo 'package main' > t.go && go test -a -i -v # Ubuntu stuff @@ -50,15 +50,10 @@ run /bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_ # Runtime dependencies run apt-get install -y -q iptables run apt-get install -y -q lxc -# Download dependencies -run PKG=github.com/kr/pty REV=27435c699; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV -run PKG=github.com/gorilla/context/ REV=708054d61e5; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV -run PKG=github.com/gorilla/mux/ REV=9b36453141c; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV -run PKG=github.com/dotcloud/tar/ REV=e5ea6bb21a3294; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV -run PKG=code.google.com/p/go.net/ REV=84a4013f96e0; hg clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && hg checkout $REV volume /var/lib/docker workdir /go/src/github.com/dotcloud/docker # Wrap all commands in the "docker-in-docker" script to allow nested containers entrypoint ["hack/dind"] # Upload docker source +add vendor /vendor add . /go/src/github.com/dotcloud/docker diff --git a/components/engine/vendor.sh b/components/engine/vendor.sh new file mode 100755 index 0000000000..5dd9453cad --- /dev/null +++ b/components/engine/vendor.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Downloads dependencies into vendor/ directory +if [[ ! -d vendor ]]; then + mkdir vendor +fi +vendor_dir=${PWD}/vendor + +git_clone () { + PKG=$1 + REV=$2 + if [[ ! -d src/$PKG ]]; then + cd $vendor_dir && git clone http://$PKG src/$PKG && cd src/$PKG && git checkout -f $REV + fi +} + +git_clone github.com/kr/pty 27435c699 + +git_clone github.com/gorilla/context/ 708054d61e5 + +git_clone github.com/gorilla/mux/ 9b36453141c + +git_clone github.com/dotcloud/tar/ d06045a6d9 + +# Docker requires code.google.com/p/go.net/websocket +PKG=code.google.com/p/go.net REV=84a4013f96e0 +if [[ ! -d src/$PKG ]]; then + cd $vendor_dir && hg clone https://$PKG src/$PKG && cd src/$PKG && hg checkout -r $REV +fi From 911d6019d5706a1af52ae41eae9448aa462d7608 Mon Sep 17 00:00:00 2001 From: Brandon Philips Date: Mon, 26 Aug 2013 17:59:58 -0700 Subject: [PATCH 08/21] gitignore: ignore bundles directory Upstream-commit: 19dc3b0792272c2f0fbcfb654349fe5e86bc8046 Component: engine --- components/engine/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/.gitignore b/components/engine/.gitignore index 034b2e6490..ca58f735cb 100644 --- a/components/engine/.gitignore +++ b/components/engine/.gitignore @@ -15,3 +15,4 @@ docs/_templates .dotcloud *.test vendor/ +bundles/ From 3184ad242c95de0b98f874b43f7b51859ec7b6d7 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 7 Sep 2013 17:48:52 -0700 Subject: [PATCH 09/21] vendor.sh can cleanly update vendored dependencies Upstream-commit: 055bbb79c10a829e054d6741d1937f0304dbb7fd Component: engine --- components/engine/{ => hack}/vendor.sh | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) rename components/engine/{ => hack}/vendor.sh (63%) diff --git a/components/engine/vendor.sh b/components/engine/hack/vendor.sh similarity index 63% rename from components/engine/vendor.sh rename to components/engine/hack/vendor.sh index 5dd9453cad..d327911047 100755 --- a/components/engine/vendor.sh +++ b/components/engine/hack/vendor.sh @@ -9,9 +9,14 @@ vendor_dir=${PWD}/vendor git_clone () { PKG=$1 REV=$2 - if [[ ! -d src/$PKG ]]; then - cd $vendor_dir && git clone http://$PKG src/$PKG && cd src/$PKG && git checkout -f $REV - fi + ( + set -e + cd $vendor_dir + if [[ ! -d src/$PKG ]]; then + cd $vendor_dir && git clone http://$PKG src/$PKG + fi + cd src/$PKG && git checkout -f $REV + ) } git_clone github.com/kr/pty 27435c699 @@ -24,6 +29,11 @@ git_clone github.com/dotcloud/tar/ d06045a6d9 # Docker requires code.google.com/p/go.net/websocket PKG=code.google.com/p/go.net REV=84a4013f96e0 -if [[ ! -d src/$PKG ]]; then - cd $vendor_dir && hg clone https://$PKG src/$PKG && cd src/$PKG && hg checkout -r $REV -fi +( + set -e + cd $vendor_dir + if [[ ! -d src/$PKG ]]; then + hg clone https://$PKG src/$PKG + fi + cd src/$PKG && hg checkout -r $REV +) From 774dff2935b6b83c2de806f9e11d9feb1b9868b2 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 8 Sep 2013 18:45:23 -0700 Subject: [PATCH 10/21] Hack: we no longer need to generate test binaries. Upstream-commit: 4cd59b96ed93d8a4c35099a0a1171e17170071ab Component: engine --- components/engine/hack/make.sh | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index d68e4c0487..c618384b8f 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -73,21 +73,6 @@ bundle_binary() { ./docker } - -# Build Docker's test suite as a collection of binary files (one per -# sub-package to test) -bundle_test() { - mkdir -p bundles/$VERSION/test - for test_dir in $(find_test_dirs); do - test_binary=$( - cd $test_dir - go test -c -v -ldflags "-X main.GITCOMMIT $GITCOMMIT -X main.VERSION $VERSION -d -w" >&2 - find . -maxdepth 1 -type f -name '*.test' -executable - ) - cp $test_dir/$test_binary bundles/$VERSION/test/ - done -} - # Build docker as an ubuntu package using FPM and REPREPRO (sue me). # bundle_binary must be called first. bundle_ubuntu() { @@ -148,20 +133,9 @@ EOF } -# This helper function walks the current directory looking for directories -# holding Go test files, and prints their paths on standard output, one per -# line. -find_test_dirs() { - find . -name '*_test.go' | - { while read f; do dirname $f; done; } | - sort -u -} - - main() { bundle_binary bundle_ubuntu - #bundle_test cat < Date: Mon, 9 Sep 2013 15:05:25 -0700 Subject: [PATCH 11/21] Hack: use vendored dependencies in-place, for less moving parts when developing Upstream-commit: e37dcd726fe415193eaed18729251d293a018909 Component: engine --- components/engine/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 6c8cb4cd6d..36054d29b3 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -35,7 +35,7 @@ run apt-get install -y -q mercurial # Install Go run curl -s https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | tar -v -C /usr/local -xz env PATH /usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin -env GOPATH /go:/vendor +env GOPATH /go:/go/src/github.com/dotcloud/docker/vendor env CGO_ENABLED 0 run cd /tmp && echo 'package main' > t.go && go test -a -i -v # Ubuntu stuff @@ -55,5 +55,4 @@ workdir /go/src/github.com/dotcloud/docker # Wrap all commands in the "docker-in-docker" script to allow nested containers entrypoint ["hack/dind"] # Upload docker source -add vendor /vendor add . /go/src/github.com/dotcloud/docker From a4daaece9a11c8b002d70a57c9c9a367b98505c4 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 9 Sep 2013 16:20:30 -0700 Subject: [PATCH 12/21] Integrate unit tests into hack/make.sh Upstream-commit: b187cc40cda36fb1b4890d80fc7367b2df4fbd6e Component: engine --- components/engine/hack/make.sh | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index c618384b8f..1d79ed7396 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -18,6 +18,10 @@ # your checkout of the Docker repository. # +# FIXME: break down bundles into sub-scripts +# FIXME: create all bundles in a single run for consistency. +# If the bundles directory already exists, fail or erase it. + set -e # We're a nice, sexy, little shell script, and people might try to run us; @@ -39,6 +43,9 @@ then PKGVERSION="$PKGVERSION-$(date +%Y%m%d%H%M%S)-$GITCOMMIT" fi +# Use these flags when compiling the tests and final binary +LDFLAGS="-X main.GITCOMMIT $GITCOMMIT -X main.VERSION $VERSION -d -w" + PACKAGE_ARCHITECTURE="$(dpkg-architecture -qDEB_HOST_ARCH)" PACKAGE_URL="http://www.docker.io/" PACKAGE_MAINTAINER="docker@dotcloud.com" @@ -65,11 +72,34 @@ end script # Each "bundle" is a different type of build artefact: static binary, Ubuntu # package, etc. +# Run Docker's test suite, including sub-packages, and store their output as a bundle +bundle_test() { + mkdir -p bundles/$VERSION/test + { + date + for test_dir in $(find_test_dirs); do ( + set -x + cd $test_dir + go test -v -ldflags "$LDFLAGS" + ) done + } 2>&1 | tee bundles/$VERSION/test/test.log +} + + +# This helper function walks the current directory looking for directories +# holding Go test files, and prints their paths on standard output, one per +# line. +find_test_dirs() { + find . -name '*_test.go' | grep -v '^./vendor' | + { while read f; do dirname $f; done; } | + sort -u +} + # Build Docker as a static binary file bundle_binary() { mkdir -p bundles/$VERSION/binary go build -o bundles/$VERSION/binary/docker-$VERSION \ - -ldflags "-X main.GITCOMMIT $GITCOMMIT -X main.VERSION $VERSION -d -w" \ + -ldflags "$LDFLAGS" \ ./docker } @@ -134,6 +164,7 @@ EOF main() { + bundle_test bundle_binary bundle_ubuntu cat < Date: Mon, 9 Sep 2013 16:30:24 -0700 Subject: [PATCH 13/21] Add the output of the tests to each release Upstream-commit: 59856a20bfbad6f59308071fda220a04c3b0df09 Component: engine --- components/engine/hack/release.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/engine/hack/release.sh b/components/engine/hack/release.sh index 4d5bd00b22..237b5fb903 100755 --- a/components/engine/hack/release.sh +++ b/components/engine/hack/release.sh @@ -165,11 +165,18 @@ release_index() { ) | write_to_s3 s3://$BUCKET/index } +release_test() { + if [ -e "bundles/$VERSION/test" ]; then + s3cmd --acl-public sync bundles/$VERSION/test/ s3://$BUCKET/test/ + fi +} + main() { setup_s3 release_binary release_ubuntu release_index + release_test } main From 173e47873811ade685ea13d4bf4dc9f28d0c7b84 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 9 Sep 2013 18:45:40 -0700 Subject: [PATCH 14/21] Break down hack/make.sh into small scripts, one per 'bundle': test, binary, ubuntu etc. Upstream-commit: 3d39336a46a0d7f411467d29eb6328dc1ab3e900 Component: engine --- components/engine/hack/make.sh | 153 ++++++-------------------- components/engine/hack/make/README.md | 17 +++ components/engine/hack/make/binary | 4 + components/engine/hack/make/test | 27 +++++ components/engine/hack/make/ubuntu | 94 ++++++++++++++++ 5 files changed, 174 insertions(+), 121 deletions(-) create mode 100644 components/engine/hack/make/README.md create mode 100644 components/engine/hack/make/binary create mode 100644 components/engine/hack/make/test create mode 100644 components/engine/hack/make/ubuntu diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index 1d79ed7396..9910882e20 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # This script builds various binary artifacts from a checkout of the docker # source code. @@ -34,139 +34,50 @@ grep -q "$RESOLVCONF" /proc/mounts || { exit 1 } +# List of bundles to create when no argument is passed +DEFAULT_BUNDLES=( + test + binary + ubuntu +) + VERSION=$(cat ./VERSION) -PKGVERSION="$VERSION" GITCOMMIT=$(git rev-parse --short HEAD) if test -n "$(git status --porcelain)" then GITCOMMIT="$GITCOMMIT-dirty" - PKGVERSION="$PKGVERSION-$(date +%Y%m%d%H%M%S)-$GITCOMMIT" fi # Use these flags when compiling the tests and final binary LDFLAGS="-X main.GITCOMMIT $GITCOMMIT -X main.VERSION $VERSION -d -w" -PACKAGE_ARCHITECTURE="$(dpkg-architecture -qDEB_HOST_ARCH)" -PACKAGE_URL="http://www.docker.io/" -PACKAGE_MAINTAINER="docker@dotcloud.com" -PACKAGE_DESCRIPTION="lxc-docker is a Linux container runtime -Docker complements LXC with a high-level API which operates at the process -level. It runs unix processes with strong guarantees of isolation and -repeatability across servers. -Docker is a great building block for automating distributed systems: -large-scale web deployments, database clusters, continuous deployment systems, -private PaaS, service-oriented architectures, etc." -UPSTART_SCRIPT='description "Docker daemon" - -start on filesystem and started lxc-net -stop on runlevel [!2345] - -respawn - -script - /usr/bin/docker -d -end script -' - -# Each "bundle" is a different type of build artefact: static binary, Ubuntu -# package, etc. - -# Run Docker's test suite, including sub-packages, and store their output as a bundle -bundle_test() { - mkdir -p bundles/$VERSION/test - { - date - for test_dir in $(find_test_dirs); do ( - set -x - cd $test_dir - go test -v -ldflags "$LDFLAGS" - ) done - } 2>&1 | tee bundles/$VERSION/test/test.log +bundle() { + bundlescript=$1 + bundle=$(basename $bundlescript) + echo "---> Making bundle: $bundle" + mkdir -p bundles/$VERSION/$bundle + source $bundlescript $(pwd)/bundles/$VERSION/$bundle } - -# This helper function walks the current directory looking for directories -# holding Go test files, and prints their paths on standard output, one per -# line. -find_test_dirs() { - find . -name '*_test.go' | grep -v '^./vendor' | - { while read f; do dirname $f; done; } | - sort -u -} - -# Build Docker as a static binary file -bundle_binary() { - mkdir -p bundles/$VERSION/binary - go build -o bundles/$VERSION/binary/docker-$VERSION \ - -ldflags "$LDFLAGS" \ - ./docker -} - -# Build docker as an ubuntu package using FPM and REPREPRO (sue me). -# bundle_binary must be called first. -bundle_ubuntu() { - mkdir -p bundles/$VERSION/ubuntu - - DIR=$(pwd)/bundles/$VERSION/ubuntu/build - - # Generate an upstart config file (ubuntu-specific) - mkdir -p $DIR/etc/init - echo "$UPSTART_SCRIPT" > $DIR/etc/init/docker.conf - - # Copy the binary - mkdir -p $DIR/usr/bin - cp bundles/$VERSION/binary/docker-$VERSION $DIR/usr/bin/docker - - # Generate postinstall/prerm scripts - cat >/tmp/postinstall </tmp/prerm <&1 | tee $DEST/test.log +} + + +# This helper function walks the current directory looking for directories +# holding Go test files, and prints their paths on standard output, one per +# line. +find_test_dirs() { + find . -name '*_test.go' | grep -v '^./vendor' | + { while read f; do dirname $f; done; } | + sort -u +} + +bundle_test diff --git a/components/engine/hack/make/ubuntu b/components/engine/hack/make/ubuntu new file mode 100644 index 0000000000..dde11ae81e --- /dev/null +++ b/components/engine/hack/make/ubuntu @@ -0,0 +1,94 @@ +#!/bin/sh + +DEST=$1 + +PKGVERSION="$VERSION" +if test -n "$(git status --porcelain)" +then + PKGVERSION="$PKGVERSION-$(date +%Y%m%d%H%M%S)-$GITCOMMIT" +fi + +PACKAGE_ARCHITECTURE="$(dpkg-architecture -qDEB_HOST_ARCH)" +PACKAGE_URL="http://www.docker.io/" +PACKAGE_MAINTAINER="docker@dotcloud.com" +PACKAGE_DESCRIPTION="lxc-docker is a Linux container runtime +Docker complements LXC with a high-level API which operates at the process +level. It runs unix processes with strong guarantees of isolation and +repeatability across servers. +Docker is a great building block for automating distributed systems: +large-scale web deployments, database clusters, continuous deployment systems, +private PaaS, service-oriented architectures, etc." + +UPSTART_SCRIPT='description "Docker daemon" + +start on filesystem and started lxc-net +stop on runlevel [!2345] + +respawn + +script + /usr/bin/docker -d +end script +' + +# Build docker as an ubuntu package using FPM and REPREPRO (sue me). +# bundle_binary must be called first. +bundle_ubuntu() { + DIR=$DEST/build + + # Generate an upstart config file (ubuntu-specific) + mkdir -p $DIR/etc/init + echo "$UPSTART_SCRIPT" > $DIR/etc/init/docker.conf + + # Copy the binary + # This will fail if the binary bundle hasn't been built + mkdir -p $DIR/usr/bin + # Copy the binary + # This will fail if the binary bundle hasn't been built + cp $DEST/../binary/docker-$VERSION $DIR/usr/bin/docker + + # Generate postinstall/prerm scripts + cat >/tmp/postinstall </tmp/prerm < Date: Mon, 9 Sep 2013 23:39:55 -0700 Subject: [PATCH 15/21] PACKAGERS.md: a guide to packaging Docker for your favorite distro Upstream-commit: 14bbbcd57181f9d6e908b2e84474e731314d0d77 Component: engine --- components/engine/hack/PACKAGERS.md | 125 ++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 components/engine/hack/PACKAGERS.md diff --git a/components/engine/hack/PACKAGERS.md b/components/engine/hack/PACKAGERS.md new file mode 100644 index 0000000000..347f777186 --- /dev/null +++ b/components/engine/hack/PACKAGERS.md @@ -0,0 +1,125 @@ +Dear packager. + +If you are looking to make docker available on your favorite software distribution, +this document is for you. It summarizes the requirements for building and running +docker. + +## Getting started + +We really want to help you package Docker successfully. Before anything, a good first step +is to introduce yourself on the [docker-dev mailing list](https://groups.google.com/forum/?fromgroups#!forum/docker-dev) +, explain what you''re trying to achieve, and tell us how we can help. Don''t worry, we don''t bite! +There might even be someone already working on packaging for the same distro! + +You can also join the IRC channel - #docker and #docker-dev on Freenode are both active and friendly. + +## Package name + +If possible, your package should be called "docker". If that name is already taken, a second +choice is "lxc-docker". + +## System build dependencies + +To build docker, you will need the following system dependencies + +* An amd64 machine +* A recent version of git and mercurial +* Go version 1.1.2 +* A clean checkout of the source must be added to a valid Go [workspace](http://golang.org/doc/code.html#Workspaces) +under the path *src/github.com/dotcloud/docker*. See + + +## Go dependencies + +All Go dependencies are vendored under ,/vendor. They are used by the official build, +so the source of truth for the current version is whatever is in ./vendor. + +To use the vendored dependencies, simply make sure the path to ./vendor is included in $GOPATH. + +If you would rather package these dependencies yourself, take a look at ./hack/vendor.sh for an +easy-to-parse list of the exact version for each. + +NOTE: if you''re not able to package the exact version (to the exact commit) of a given dependency, +please get in touch so we can remediate! Who knows what discrepancies can be caused by even the +slightest deviation. We promise to do our best to make everybody happy. + + +## Disabling CGO + +Make sure to disable CGO on your system, and then recompile the standard library on the build +machine: + +```bash +export CGO_ENABLED=0 +cd /tmp && echo 'package main' > t.go && go test -a -i -v +``` + +## Building Docker + +To build the docker binary, run the following command with the source checkout as the +working directory: + +``` +./hack/make.sh binary +``` + +This will create a static binary under *./bundles/$VERSION/binary/docker-$VERSION*, where +*$VERSION* is the contents of the file *./VERSION*. + +You are encouraged to use ./hack/make.sh without modification. If you must absolutely write +your own script (are you really, really sure you need to? make.sh is really not that complicated), +then please take care the respect the following: + +* In *./hack/make.sh*: $LDFLAGS, $VERSION and $GITCOMMIT +* In *./hack/make/binary*: the exact build command to run + +You may be tempted to tweak these settings. In particular, being a rigorous maintainer, you may want +to disable static linking. Please don''t! Docker *needs* to be statically linked to function properly. +You would do the users of your distro a disservice and "void the docker warranty" by changing the flags. + +A good comparison is Busybox: all distros package it as a statically linked binary, because it just +makes sense. Docker is the same way. + +## Testing Docker + +Before releasing your binary, make sure to run the tests! Run the following command with the source +checkout as the working directory: + +```bash +./hack/make.sh test +``` + +The test suite includes both live integration tests and unit tests, so you will need all runtime +dependencies to be installed (see below). + +The test suite will also download a small test container, so you will need internet connectivity. + + +## Runtime dependencies + +To run properly, docker needs the following software to be installed at runtime: + +* GNU Tar version 1.26 or later +* A recent build of iproute2 (2012-05-21 or later), and specifically the "ip" utility. +* iptables version 1.4 or later +* The lxc utility scripts (http://lxc.sourceforge.net) version 0.8 or later. +* Git version 1.7 or later + +## Kernel dependencies + +Docker in daemon mode has specific kernel requirements. For details, see +http://docs.docker.io/en/latest/installation/kernel/ + +Note that Docker also has a client mode, which can run on virtually any linux kernel (it even builds +on OSX!). + +## Init script + +Docker expects to run as a daemon at machine startup. Your package will need to include a script +for your distro''s process supervisor of choice. + +Docker should be run as root, with the following arguments: + +``` +docker -d +``` From 47c1ee3661b384614cf2a20ea329ffaec9a052b4 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 10 Sep 2013 11:30:14 -0700 Subject: [PATCH 16/21] Packager's manual: official build vs distro build Upstream-commit: 5b361f31f7d4eb547cbf60fda2968537600772ca Component: engine --- components/engine/hack/PACKAGERS.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/engine/hack/PACKAGERS.md b/components/engine/hack/PACKAGERS.md index 347f777186..43280a0009 100644 --- a/components/engine/hack/PACKAGERS.md +++ b/components/engine/hack/PACKAGERS.md @@ -18,6 +18,18 @@ You can also join the IRC channel - #docker and #docker-dev on Freenode are both If possible, your package should be called "docker". If that name is already taken, a second choice is "lxc-docker". +## Official build vs distro build + +The Docker project maintains its own build and release toolchain. It is pretty neat and entirely +based on Docker (surprise!). This toolchain is the canonical way to build Docker, and the only +method supported by the development team. We encourage you to give it a try, and if the circumstances +allow you to use it, we recommend that you do. + +You might not be able to use the official build toolchain - usually because your distribution has a +toolchain and packaging policy of its own. We get it! Your house, your rules. The rest of this document +should give you the information you need to package Docker your way, without denaturing it in +the process. + ## System build dependencies To build docker, you will need the following system dependencies From 5e2f733ccee31efad2f625bb2714f39920e20ac9 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 10 Sep 2013 11:33:26 -0700 Subject: [PATCH 17/21] Update usage comments in hack/make.sh Upstream-commit: d14058bc29512918ffd4e7413c6d5ae8524088ed Component: engine --- components/engine/hack/make.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index 9910882e20..3df411f107 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -11,11 +11,12 @@ # will be used as Docker binary version and package version. # - The hash of the git commit will also be included in the Docker binary, # with the suffix -dirty if the repository isn't clean. -# - The script is intented to be run as part of a docker build, as defined +# - The script is intented to be run inside the docker container specified # in the Dockerfile at the root of the source. In other words: # DO NOT CALL THIS SCRIPT DIRECTLY. # - The right way to call this script is to invoke "docker build ." from -# your checkout of the Docker repository. +# your checkout of the Docker repository, and then +# "docker run hack/make.sh" in the resulting container image. # # FIXME: break down bundles into sub-scripts From c63c08e39f298ea47eb9c0ad009820bd87036b98 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 10 Sep 2013 11:33:37 -0700 Subject: [PATCH 18/21] Remove fixed FIXMEs Upstream-commit: 228b7af516e023984a3d0ceba98c639cec9695d0 Component: engine --- components/engine/hack/make.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index 3df411f107..f7412a94d1 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -19,10 +19,6 @@ # "docker run hack/make.sh" in the resulting container image. # -# FIXME: break down bundles into sub-scripts -# FIXME: create all bundles in a single run for consistency. -# If the bundles directory already exists, fail or erase it. - set -e # We're a nice, sexy, little shell script, and people might try to run us; From ed8387023efbe10fa36768e622ed86f6561697f5 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 10 Sep 2013 18:02:33 -0700 Subject: [PATCH 19/21] Fix typo and add dependency details in hack/PACKAGERS.md Upstream-commit: 03e36caeb12eb5b936de8cec29998c61fc02f037 Component: engine --- components/engine/hack/PACKAGERS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/hack/PACKAGERS.md b/components/engine/hack/PACKAGERS.md index 43280a0009..776ed47472 100644 --- a/components/engine/hack/PACKAGERS.md +++ b/components/engine/hack/PACKAGERS.md @@ -43,7 +43,7 @@ under the path *src/github.com/dotcloud/docker*. See ## Go dependencies -All Go dependencies are vendored under ,/vendor. They are used by the official build, +All Go dependencies are vendored under ./vendor. They are used by the official build, so the source of truth for the current version is whatever is in ./vendor. To use the vendored dependencies, simply make sure the path to ./vendor is included in $GOPATH. @@ -112,7 +112,7 @@ The test suite will also download a small test container, so you will need inter To run properly, docker needs the following software to be installed at runtime: * GNU Tar version 1.26 or later -* A recent build of iproute2 (2012-05-21 or later), and specifically the "ip" utility. +* iproute2 version 3.5 or later (build after 2012-05-21), and specifically the "ip" utility. * iptables version 1.4 or later * The lxc utility scripts (http://lxc.sourceforge.net) version 0.8 or later. * Git version 1.7 or later From 7ba861bda75af0cd98eeb69efdf2904dd794dabc Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 10 Sep 2013 18:08:02 -0700 Subject: [PATCH 20/21] hack/make.sh print a warning but don't exit if called outside a correct build environment. Upstream-commit: ebee8f28ac45e7907a35331383f8b9b4ab338353 Component: engine --- components/engine/hack/make.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index f7412a94d1..9414a5c1b6 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -25,10 +25,9 @@ set -e # but really, they shouldn't. We want to be in a container! RESOLVCONF=$(readlink --canonicalize /etc/resolv.conf) grep -q "$RESOLVCONF" /proc/mounts || { - echo "# I will only run within a container." - echo "# Try this instead:" - echo "docker build ." - exit 1 + echo "# WARNING! I don't seem to be running in a docker container. + echo "# The result of this command might be an incorrect build, and will not be officially supported." + echo "# Try this: 'docker build -t docker . && docker run docker ./hack/make.sh' } # List of bundles to create when no argument is passed From 04ac9582d357577cfcaefd63ce04619e181e8f50 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 11 Sep 2013 18:38:09 -0700 Subject: [PATCH 21/21] hack/vendor.sh: overwrite existing dependencies and remove .git so they can be checked in Upstream-commit: 45cedefadb444e8d7305defd246c63537fca04df Component: engine --- components/engine/.gitignore | 3 ++- components/engine/hack/vendor.sh | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/components/engine/.gitignore b/components/engine/.gitignore index ca58f735cb..5843eaf9cc 100644 --- a/components/engine/.gitignore +++ b/components/engine/.gitignore @@ -14,5 +14,6 @@ docs/_templates .gopath/ .dotcloud *.test -vendor/ bundles/ +.hg/ +.git/ diff --git a/components/engine/hack/vendor.sh b/components/engine/hack/vendor.sh index d327911047..5287449b4f 100755 --- a/components/engine/hack/vendor.sh +++ b/components/engine/hack/vendor.sh @@ -12,10 +12,12 @@ git_clone () { ( set -e cd $vendor_dir - if [[ ! -d src/$PKG ]]; then - cd $vendor_dir && git clone http://$PKG src/$PKG + if [[ -d src/$PKG ]]; then + echo "src/$PKG already exists. Removing." + rm -fr src/$PKG fi - cd src/$PKG && git checkout -f $REV + cd $vendor_dir && git clone http://$PKG src/$PKG + cd src/$PKG && git checkout -f $REV && rm -fr .git ) }