diff --git a/components/engine/.gitignore b/components/engine/.gitignore index ea62e34d19..5843eaf9cc 100644 --- a/components/engine/.gitignore +++ b/components/engine/.gitignore @@ -14,3 +14,6 @@ docs/_templates .gopath/ .dotcloud *.test +bundles/ +.hg/ +.git/ diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 43f493d3ff..36054d29b3 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -1,5 +1,29 @@ # This file describes the standard way to build Docker, using docker -docker-version 0.4.2 +# +# Usage: +# +# # Assemble the full dev environment. This is slow the first time. +# docker build -t docker . +# # Apparmor messes with privileged mode: disable it +# /etc/init.d/apparmor stop ; /etc/init.d/apparmor teardown +# +# # Mount your source in an interactive container for quick testing: +# docker run -v `pwd`:/go/src/github.com/dotcloud/docker -privileged -lxc-conf=lxc.aa_profile=unconfined -i -t docker bash +# +# +# # Run the test suite: +# docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker go test -v +# +# # Publish a release: +# docker run -privileged -lxc-conf=lxc.aa_profile=unconfined \ +# -e AWS_S3_BUCKET=baz \ +# -e AWS_ACCESS_KEY=foo \ +# -e AWS_SECRET_KEY=bar \ +# -e GPG_PASSPHRASE=gloubiboulga \ +# -lxc-conf=lxc.aa_profile=unconfined -privileged docker hack/release.sh +# + +docker-version 0.6.1 from ubuntu:12.04 maintainer Solomon Hykes # Build dependencies @@ -11,7 +35,7 @@ run apt-get install -y -q mercurial # Install Go run curl -s https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | tar -v -C /usr/local -xz env PATH /usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin -env GOPATH /go +env GOPATH /go:/go/src/github.com/dotcloud/docker/vendor env CGO_ENABLED 0 run cd /tmp && echo 'package main' > t.go && go test -a -i -v # Ubuntu stuff @@ -23,15 +47,12 @@ run apt-get install -y -q python-pip run pip install s3cmd run pip install python-magic run /bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY\n' > /.s3cfg -# Download dependencies -run PKG=github.com/kr/pty REV=27435c699; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV -run PKG=github.com/gorilla/context/ REV=708054d61e5; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV -run PKG=github.com/gorilla/mux/ REV=9b36453141c; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV -run PKG=github.com/dotcloud/tar/ REV=e5ea6bb21a3294; git clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && git checkout -f $REV -run PKG=code.google.com/p/go.net/ REV=84a4013f96e0; hg clone http://$PKG /go/src/$PKG && cd /go/src/$PKG && hg checkout $REV +# Runtime dependencies +run apt-get install -y -q iptables +run apt-get install -y -q lxc +volume /var/lib/docker +workdir /go/src/github.com/dotcloud/docker +# Wrap all commands in the "docker-in-docker" script to allow nested containers +entrypoint ["hack/dind"] # Upload docker source add . /go/src/github.com/dotcloud/docker -run ln -s /go/src/github.com/dotcloud/docker /src -# Build the binary -run cd /go/src/github.com/dotcloud/docker && hack/release/make.sh -cmd cd /go/src/github.com/dotcloud/docker && hack/release/release.sh diff --git a/components/engine/hack/PACKAGERS.md b/components/engine/hack/PACKAGERS.md new file mode 100644 index 0000000000..776ed47472 --- /dev/null +++ b/components/engine/hack/PACKAGERS.md @@ -0,0 +1,137 @@ +Dear packager. + +If you are looking to make docker available on your favorite software distribution, +this document is for you. It summarizes the requirements for building and running +docker. + +## Getting started + +We really want to help you package Docker successfully. Before anything, a good first step +is to introduce yourself on the [docker-dev mailing list](https://groups.google.com/forum/?fromgroups#!forum/docker-dev) +, explain what you''re trying to achieve, and tell us how we can help. Don''t worry, we don''t bite! +There might even be someone already working on packaging for the same distro! + +You can also join the IRC channel - #docker and #docker-dev on Freenode are both active and friendly. + +## Package name + +If possible, your package should be called "docker". If that name is already taken, a second +choice is "lxc-docker". + +## Official build vs distro build + +The Docker project maintains its own build and release toolchain. It is pretty neat and entirely +based on Docker (surprise!). This toolchain is the canonical way to build Docker, and the only +method supported by the development team. We encourage you to give it a try, and if the circumstances +allow you to use it, we recommend that you do. + +You might not be able to use the official build toolchain - usually because your distribution has a +toolchain and packaging policy of its own. We get it! Your house, your rules. The rest of this document +should give you the information you need to package Docker your way, without denaturing it in +the process. + +## System build dependencies + +To build docker, you will need the following system dependencies + +* An amd64 machine +* A recent version of git and mercurial +* Go version 1.1.2 +* A clean checkout of the source must be added to a valid Go [workspace](http://golang.org/doc/code.html#Workspaces) +under the path *src/github.com/dotcloud/docker*. See + + +## Go dependencies + +All Go dependencies are vendored under ./vendor. They are used by the official build, +so the source of truth for the current version is whatever is in ./vendor. + +To use the vendored dependencies, simply make sure the path to ./vendor is included in $GOPATH. + +If you would rather package these dependencies yourself, take a look at ./hack/vendor.sh for an +easy-to-parse list of the exact version for each. + +NOTE: if you''re not able to package the exact version (to the exact commit) of a given dependency, +please get in touch so we can remediate! Who knows what discrepancies can be caused by even the +slightest deviation. We promise to do our best to make everybody happy. + + +## Disabling CGO + +Make sure to disable CGO on your system, and then recompile the standard library on the build +machine: + +```bash +export CGO_ENABLED=0 +cd /tmp && echo 'package main' > t.go && go test -a -i -v +``` + +## Building Docker + +To build the docker binary, run the following command with the source checkout as the +working directory: + +``` +./hack/make.sh binary +``` + +This will create a static binary under *./bundles/$VERSION/binary/docker-$VERSION*, where +*$VERSION* is the contents of the file *./VERSION*. + +You are encouraged to use ./hack/make.sh without modification. If you must absolutely write +your own script (are you really, really sure you need to? make.sh is really not that complicated), +then please take care the respect the following: + +* In *./hack/make.sh*: $LDFLAGS, $VERSION and $GITCOMMIT +* In *./hack/make/binary*: the exact build command to run + +You may be tempted to tweak these settings. In particular, being a rigorous maintainer, you may want +to disable static linking. Please don''t! Docker *needs* to be statically linked to function properly. +You would do the users of your distro a disservice and "void the docker warranty" by changing the flags. + +A good comparison is Busybox: all distros package it as a statically linked binary, because it just +makes sense. Docker is the same way. + +## Testing Docker + +Before releasing your binary, make sure to run the tests! Run the following command with the source +checkout as the working directory: + +```bash +./hack/make.sh test +``` + +The test suite includes both live integration tests and unit tests, so you will need all runtime +dependencies to be installed (see below). + +The test suite will also download a small test container, so you will need internet connectivity. + + +## Runtime dependencies + +To run properly, docker needs the following software to be installed at runtime: + +* GNU Tar version 1.26 or later +* iproute2 version 3.5 or later (build after 2012-05-21), and specifically the "ip" utility. +* iptables version 1.4 or later +* The lxc utility scripts (http://lxc.sourceforge.net) version 0.8 or later. +* Git version 1.7 or later + +## Kernel dependencies + +Docker in daemon mode has specific kernel requirements. For details, see +http://docs.docker.io/en/latest/installation/kernel/ + +Note that Docker also has a client mode, which can run on virtually any linux kernel (it even builds +on OSX!). + +## Init script + +Docker expects to run as a daemon at machine startup. Your package will need to include a script +for your distro''s process supervisor of choice. + +Docker should be run as root, with the following arguments: + +``` +docker -d +``` diff --git a/components/engine/hack/release/README.md b/components/engine/hack/RELEASE-CHECKLIST.md similarity index 100% rename from components/engine/hack/release/README.md rename to components/engine/hack/RELEASE-CHECKLIST.md diff --git a/components/engine/hack/dind b/components/engine/hack/dind new file mode 100755 index 0000000000..7810715675 --- /dev/null +++ b/components/engine/hack/dind @@ -0,0 +1,59 @@ +#!/bin/bash + +# DinD: a wrapper script which allows docker to be run inside a docker container. +# Original version by Jerome Petazzoni +# See the blog post: http://blog.docker.io/2013/09/docker-can-now-run-within-docker/ +# +# This script should be executed inside a docker container in privilieged mode +# ('docker run -privileged', introduced in docker 0.6). + +# Usage: dind CMD [ARG...] + +# First, make sure that cgroups are mounted correctly. +CGROUP=/sys/fs/cgroup + +[ -d $CGROUP ] || + mkdir $CGROUP + +mountpoint -q $CGROUP || + mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { + echo "Could not make a tmpfs mount. Did you use -privileged?" + exit 1 + } + +# Mount the cgroup hierarchies exactly as they are in the parent system. +for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) +do + [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS + mountpoint -q $CGROUP/$SUBSYS || + mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS +done + +# Note: as I write those lines, the LXC userland tools cannot setup +# a "sub-container" properly if the "devices" cgroup is not in its +# own hierarchy. Let's detect this and issue a warning. +grep -q :devices: /proc/1/cgroup || + echo "WARNING: the 'devices' cgroup should be in its own hierarchy." +grep -qw devices /proc/1/cgroup || + echo "WARNING: it looks like the 'devices' cgroup is not mounted." + +# Now, close extraneous file descriptors. +pushd /proc/self/fd +for FD in * +do + case "$FD" in + # Keep stdin/stdout/stderr + [012]) + ;; + # Nuke everything else + *) + eval exec "$FD>&-" + ;; + esac +done +popd + +# Mount /tmp +mount -t tmpfs none /tmp + +exec $* diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh new file mode 100755 index 0000000000..9414a5c1b6 --- /dev/null +++ b/components/engine/hack/make.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# This script builds various binary artifacts from a checkout of the docker +# source code. +# +# Requirements: +# - The current directory should be a checkout of the docker source code +# (http://github.com/dotcloud/docker). Whatever version is checked out +# will be built. +# - The VERSION file, at the root of the repository, should exist, and +# will be used as Docker binary version and package version. +# - The hash of the git commit will also be included in the Docker binary, +# with the suffix -dirty if the repository isn't clean. +# - The script is intented to be run inside the docker container specified +# in the Dockerfile at the root of the source. In other words: +# DO NOT CALL THIS SCRIPT DIRECTLY. +# - The right way to call this script is to invoke "docker build ." from +# your checkout of the Docker repository, and then +# "docker run hack/make.sh" in the resulting container image. +# + +set -e + +# We're a nice, sexy, little shell script, and people might try to run us; +# but really, they shouldn't. We want to be in a container! +RESOLVCONF=$(readlink --canonicalize /etc/resolv.conf) +grep -q "$RESOLVCONF" /proc/mounts || { + echo "# WARNING! I don't seem to be running in a docker container. + echo "# The result of this command might be an incorrect build, and will not be officially supported." + echo "# Try this: 'docker build -t docker . && docker run docker ./hack/make.sh' +} + +# List of bundles to create when no argument is passed +DEFAULT_BUNDLES=( + test + binary + ubuntu +) + +VERSION=$(cat ./VERSION) +GITCOMMIT=$(git rev-parse --short HEAD) +if test -n "$(git status --porcelain)" +then + GITCOMMIT="$GITCOMMIT-dirty" +fi + +# Use these flags when compiling the tests and final binary +LDFLAGS="-X main.GITCOMMIT $GITCOMMIT -X main.VERSION $VERSION -d -w" + + +bundle() { + bundlescript=$1 + bundle=$(basename $bundlescript) + echo "---> Making bundle: $bundle" + mkdir -p bundles/$VERSION/$bundle + source $bundlescript $(pwd)/bundles/$VERSION/$bundle +} + +main() { + + # We want this to fail if the bundles already exist. + # This is to avoid mixing bundles from different versions of the code. + mkdir -p bundles + if [ -e "bundles/$VERSION" ]; then + echo "bundles/$VERSION already exists. Removing." + rm -fr bundles/$VERSION && mkdir bundles/$VERSION || exit 1 + fi + SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + if [ $# -lt 1 ]; then + bundles=($DEFAULT_BUNDLES) + else + bundles=($@) + fi + for bundle in ${bundles[@]}; do + bundle $SCRIPTDIR/make/$bundle + done + cat <&1 | tee $DEST/test.log +} + + +# This helper function walks the current directory looking for directories +# holding Go test files, and prints their paths on standard output, one per +# line. +find_test_dirs() { + find . -name '*_test.go' | grep -v '^./vendor' | + { while read f; do dirname $f; done; } | + sort -u +} + +bundle_test diff --git a/components/engine/hack/make/ubuntu b/components/engine/hack/make/ubuntu new file mode 100644 index 0000000000..dde11ae81e --- /dev/null +++ b/components/engine/hack/make/ubuntu @@ -0,0 +1,94 @@ +#!/bin/sh + +DEST=$1 + +PKGVERSION="$VERSION" +if test -n "$(git status --porcelain)" +then + PKGVERSION="$PKGVERSION-$(date +%Y%m%d%H%M%S)-$GITCOMMIT" +fi + +PACKAGE_ARCHITECTURE="$(dpkg-architecture -qDEB_HOST_ARCH)" +PACKAGE_URL="http://www.docker.io/" +PACKAGE_MAINTAINER="docker@dotcloud.com" +PACKAGE_DESCRIPTION="lxc-docker is a Linux container runtime +Docker complements LXC with a high-level API which operates at the process +level. It runs unix processes with strong guarantees of isolation and +repeatability across servers. +Docker is a great building block for automating distributed systems: +large-scale web deployments, database clusters, continuous deployment systems, +private PaaS, service-oriented architectures, etc." + +UPSTART_SCRIPT='description "Docker daemon" + +start on filesystem and started lxc-net +stop on runlevel [!2345] + +respawn + +script + /usr/bin/docker -d +end script +' + +# Build docker as an ubuntu package using FPM and REPREPRO (sue me). +# bundle_binary must be called first. +bundle_ubuntu() { + DIR=$DEST/build + + # Generate an upstart config file (ubuntu-specific) + mkdir -p $DIR/etc/init + echo "$UPSTART_SCRIPT" > $DIR/etc/init/docker.conf + + # Copy the binary + # This will fail if the binary bundle hasn't been built + mkdir -p $DIR/usr/bin + # Copy the binary + # This will fail if the binary bundle hasn't been built + cp $DEST/../binary/docker-$VERSION $DIR/usr/bin/docker + + # Generate postinstall/prerm scripts + cat >/tmp/postinstall </tmp/prerm <&2 - find . -maxdepth 1 -type f -name '*.test' -executable - ) - cp $test_dir/$test_binary bundles/$VERSION/test/ - done -} - -# Build docker as an ubuntu package using FPM and REPREPRO (sue me). -# bundle_binary must be called first. -bundle_ubuntu() { - mkdir -p bundles/$VERSION/ubuntu - - DIR=$(pwd)/bundles/$VERSION/ubuntu/build - - # Generate an upstart config file (ubuntu-specific) - mkdir -p $DIR/etc/init - echo "$UPSTART_SCRIPT" > $DIR/etc/init/docker.conf - - # Copy the binary - mkdir -p $DIR/usr/bin - cp bundles/$VERSION/binary/docker-$VERSION $DIR/usr/bin/docker - - # Generate postinstall/prerm scripts - cat >/tmp/postinstall </tmp/prerm <