From 716ff00c8b586385a85bb9f73fee92d752a52a14 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Fri, 29 Mar 2013 14:56:48 -0700 Subject: [PATCH 01/98] packaging; issue #30: Original files to make the Ubuntu PPA on launch date Upstream-commit: e99541e63729343e908b7c2ab6c901504f1311a3 Component: engine --- components/engine/deb/Makefile | 1 - components/engine/deb/README.md | 1 - components/engine/deb/debian/changelog | 5 - components/engine/deb/etc/docker-dev.upstart | 10 - .../ubuntu/Makefile} | 2 +- components/engine/packaging/ubuntu/README.md | 339 ++++++++++++++++++ .../engine/packaging/ubuntu/debian/changelog | 5 + .../{deb => packaging/ubuntu}/debian/compat | 0 .../{deb => packaging/ubuntu}/debian/control | 15 +- .../ubuntu}/debian/copyright | 2 +- .../{deb => packaging/ubuntu}/debian/docs | 0 .../{deb => packaging/ubuntu}/debian/rules | 0 .../ubuntu}/debian/source/format | 0 .../ubuntu}/etc/docker.upstart | 0 14 files changed, 353 insertions(+), 27 deletions(-) delete mode 120000 components/engine/deb/Makefile delete mode 120000 components/engine/deb/README.md delete mode 100644 components/engine/deb/debian/changelog delete mode 100644 components/engine/deb/etc/docker-dev.upstart rename components/engine/{deb/Makefile.deb => packaging/ubuntu/Makefile} (98%) create mode 100644 components/engine/packaging/ubuntu/README.md create mode 100644 components/engine/packaging/ubuntu/debian/changelog rename components/engine/{deb => packaging/ubuntu}/debian/compat (100%) rename components/engine/{deb => packaging/ubuntu}/debian/control (64%) rename components/engine/{deb => packaging/ubuntu}/debian/copyright (99%) rename components/engine/{deb => packaging/ubuntu}/debian/docs (100%) rename components/engine/{deb => packaging/ubuntu}/debian/rules (100%) rename components/engine/{deb => packaging/ubuntu}/debian/source/format (100%) rename components/engine/{deb => packaging/ubuntu}/etc/docker.upstart (100%) diff --git a/components/engine/deb/Makefile b/components/engine/deb/Makefile deleted file mode 120000 index d0b0e8e008..0000000000 --- a/components/engine/deb/Makefile +++ /dev/null @@ -1 +0,0 @@ -../Makefile \ No newline at end of file diff --git a/components/engine/deb/README.md b/components/engine/deb/README.md deleted file mode 120000 index 32d46ee883..0000000000 --- a/components/engine/deb/README.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/components/engine/deb/debian/changelog b/components/engine/deb/debian/changelog deleted file mode 100644 index 76cc04bee2..0000000000 --- a/components/engine/deb/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -dotcloud-docker (1) precise; urgency=low - - * Initial release - - -- dotCloud Mon, 14 Mar 2013 04:43:21 -0700 diff --git a/components/engine/deb/etc/docker-dev.upstart b/components/engine/deb/etc/docker-dev.upstart deleted file mode 100644 index 6cfe9d2616..0000000000 --- a/components/engine/deb/etc/docker-dev.upstart +++ /dev/null @@ -1,10 +0,0 @@ -description "Run docker" - -start on runlevel [2345] -stop on starting rc RUNLEVEL=[016] -respawn - -script - test -f /etc/default/locale && . /etc/default/locale || true - LANG=$LANG LC_ALL=$LANG /usr/bin/docker -d -end script diff --git a/components/engine/deb/Makefile.deb b/components/engine/packaging/ubuntu/Makefile similarity index 98% rename from components/engine/deb/Makefile.deb rename to components/engine/packaging/ubuntu/Makefile index c954b0f5b5..beec903fc9 100644 --- a/components/engine/deb/Makefile.deb +++ b/components/engine/packaging/ubuntu/Makefile @@ -1,4 +1,4 @@ -PKG_NAME=dotcloud-docker +PKG_NAME=lxc-docker PKG_ARCH=amd64 PKG_VERSION=1 ROOT_PATH:=$(PWD) diff --git a/components/engine/packaging/ubuntu/README.md b/components/engine/packaging/ubuntu/README.md new file mode 100644 index 0000000000..c955a1dcf2 --- /dev/null +++ b/components/engine/packaging/ubuntu/README.md @@ -0,0 +1,339 @@ +Docker: the Linux container runtime +=================================== + +Docker complements LXC with a high-level API which operates at the process level. It runs unix processes with strong guarantees of isolation and repeatability across servers. + +Docker is a great building block for automating distributed systems: large-scale web deployments, database clusters, continuous deployment systems, private PaaS, service-oriented architectures, etc. + + + +* *Heterogeneous payloads*: any combination of binaries, libraries, configuration files, scripts, virtualenvs, jars, gems, tarballs, you name it. No more juggling between domain-specific tools. Docker can deploy and run them all. + +* *Any server*: docker can run on any x64 machine with a modern linux kernel - whether it's a laptop, a bare metal server or a VM. This makes it perfect for multi-cloud deployments. + +* *Isolation*: docker isolates processes from each other and from the underlying host, using lightweight containers. + +* *Repeatability*: because containers are isolated in their own filesystem, they behave the same regardless of where, when, and alongside what they run. + + +Notable features +----------------- + +* Filesystem isolation: each process container runs in a completely separate root filesystem. + +* Resource isolation: system resources like cpu and memory can be allocated differently to each process container, using cgroups. + +* Network isolation: each process container runs in its own network namespace, with a virtual interface and IP address of its own. + +* Copy-on-write: root filesystems are created using copy-on-write, which makes deployment extremeley fast, memory-cheap and disk-cheap. + +* Logging: the standard streams (stdout/stderr/stdin) of each process container are collected and logged for real-time or batch retrieval. + +* Change management: changes to a container's filesystem can be committed into a new image and re-used to create more containers. No templating or manual configuration required. + +* Interactive shell: docker can allocate a pseudo-tty and attach to the standard input of any container, for example to run a throwaway interactive shell. + + + +Under the hood +-------------- + +Under the hood, Docker is built on the following components: + + +* The [cgroup](http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-24-c) and [namespacing](http://blog.dotcloud.com/under-the-hood-linux-kernels-on-dotcloud-part) capabilities of the Linux kernel; + +* [AUFS](http://aufs.sourceforge.net/aufs.html), a powerful union filesystem with copy-on-write capabilities; + +* The [Go](http://golang.org) programming language; + +* [lxc](http://lxc.sourceforge.net/), a set of convenience scripts to simplify the creation of linux containers. + + +Install instructions +================== + +Installing on Ubuntu 12.04 and 12.10 +------------------------------------ + +1. Install dependencies: + + ```bash + sudo apt-get install lxc wget bsdtar curl + sudo apt-get install linux-image-extra-`uname -r` + ``` + + The `linux-image-extra` package is needed on standard Ubuntu EC2 AMIs in order to install the aufs kernel module. + +2. Install the latest docker binary: + + ```bash + wget http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz + tar -xf docker-master.tgz + ``` + +3. Run your first container! + + ```bash + cd docker-master + sudo ./docker run -i -t base /bin/bash + ``` + + Consider adding docker to your `PATH` for simplicity. + +Installing on other Linux distributions +--------------------------------------- + +Right now, the officially supported distributions are: + +* Ubuntu 12.04 (precise LTS) +* Ubuntu 12.10 (quantal) + +Docker probably works on other distributions featuring a recent kernel, the AUFS patch, and up-to-date lxc. However this has not been tested. + +Installing with Vagrant +----------------------- + +Currently, Docker can be installed with Vagrant both on your localhost +with VirtualBox as well as on Amazon EC2. Vagrant 1.1 is required for +EC2, but deploying is as simple as: + +```bash +$ export AWS_ACCESS_KEY_ID=xxx \ + AWS_SECRET_ACCESS_KEY=xxx \ + AWS_KEYPAIR_NAME=xxx \ + AWS_SSH_PRIVKEY=xxx +$ vagrant plugin install vagrant-aws +$ vagrant up --provider=aws +``` + +The environment variables are: + +* `AWS_ACCESS_KEY_ID` - The API key used to make requests to AWS +* `AWS_SECRET_ACCESS_KEY` - The secret key to make AWS API requests +* `AWS_KEYPAIR_NAME` - The name of the keypair used for this EC2 instance +* `AWS_SSH_PRIVKEY` - The path to the private key for the named keypair + +For VirtualBox, you can simply ignore setting any of the environment +variables and omit the `provider` flag. VirtualBox is still supported with +Vagrant <= 1.1: + +```bash +$ vagrant up +``` + + + +Usage examples +============== + +Running an interactive shell +---------------------------- + +```bash +# Download a base image +docker pull base + +# Run an interactive shell in the base image, +# allocate a tty, attach stdin and stdout +docker run -i -t base /bin/bash +``` + + +Starting a long-running worker process +-------------------------------------- + +```bash +# Run docker in daemon mode +(docker -d || echo "Docker daemon already running") & + +# Start a very useful long-running process +JOB=$(docker run -d base /bin/sh -c "while true; do echo Hello world; sleep 1; done") + +# Collect the output of the job so far +docker logs $JOB + +# Kill the job +docker kill $JOB +``` + + +Listing all running containers +------------------------------ + +```bash +docker ps +``` + + +Expose a service on a TCP port +------------------------------ + +```bash +# Expose port 4444 of this container, and tell netcat to listen on it +JOB=$(docker run -d -p 4444 base /bin/nc -l -p 4444) + +# Which public port is NATed to my container? +PORT=$(docker port $JOB 4444) + +# Connect to the public port via the host's public address +echo hello world | nc $(hostname) $PORT + +# Verify that the network connection worked +echo "Daemon received: $(docker logs $JOB)" +``` + +Contributing to Docker +====================== + +Want to hack on Docker? Awesome! Here are instructions to get you started. They are probably not perfect, please let us know if anything feels wrong or incomplete. + +Contribution guidelines +----------------------- + +### Pull requests are always welcome + +We are always thrilled to receive pull requests, and do our best to process them as fast as possible. Not sure if that typo is worth a pull request? Do it! We will appreciate it. + +If your pull request is not accepted on the first try, don't be discouraged! If there's a problem with the implementation, hopefully you received feedback on what to improve. + +We're trying very hard to keep Docker lean and focused. We don't want it to do everything for everybody. This means that we might decide against incorporating a new feature. +However, there might be a way to implement that feature *on top of* docker. + +### Discuss your design on the mailing list + +We recommend discussing your plans [on the mailing list](https://groups.google.com/forum/?fromgroups#!forum/docker-club) before starting to code - especially for more ambitious contributions. This gives other contributors a chance to point +you in the right direction, give feedback on your design, and maybe point out if someone else is working on the same thing. + +### Create issues... + +Any significant improvement should be documented as [a github issue](https://github.com/dotcloud/docker/issues) before anybody starts working on it. + +### ...but check for existing issues first! + +Please take a moment to check that an issue doesn't already exist documenting your bug report or improvement proposal. +If it does, it never hurts to add a quick "+1" or "I have this problem too". This will help prioritize the most common problems and requests. + + +### Write tests + +Golang has a great testing suite built in: use it! Take a look at existing tests for inspiration. + + + +Setting up a dev environment +---------------------------- + +Instructions that have been verified to work on Ubuntu 12.10, + +```bash +sudo apt-get -y install lxc wget bsdtar curl golang git + +export GOPATH=~/go/ +export PATH=$GOPATH/bin:$PATH + +mkdir -p $GOPATH/src/github.com/dotcloud +cd $GOPATH/src/github.com/dotcloud +git clone git@github.com:dotcloud/docker.git +cd docker + +go get -v github.com/dotcloud/docker/... +go install -v github.com/dotcloud/docker/... +``` + +Then run the docker daemon, + +```bash +sudo $GOPATH/bin/docker -d +``` + +Run the `go install` command (above) to recompile docker. + + +What is a Standard Container? +============================= + +Docker defines a unit of software delivery called a Standard Container. The goal of a Standard Container is to encapsulate a software component and all its dependencies in +a format that is self-describing and portable, so that any compliant runtime can run it without extra dependencies, regardless of the underlying machine and the contents of the container. + +The spec for Standard Containers is currently a work in progress, but it is very straightforward. It mostly defines 1) an image format, 2) a set of standard operations, and 3) an execution environment. + +A great analogy for this is the shipping container. Just like Standard Containers are a fundamental unit of software delivery, shipping containers (http://bricks.argz.com/ins/7823-1/12) are a fundamental unit of physical delivery. + +### 1. STANDARD OPERATIONS + +Just like shipping containers, Standard Containers define a set of STANDARD OPERATIONS. Shipping containers can be lifted, stacked, locked, loaded, unloaded and labelled. Similarly, standard containers can be started, stopped, copied, snapshotted, downloaded, uploaded and tagged. + + +### 2. CONTENT-AGNOSTIC + +Just like shipping containers, Standard Containers are CONTENT-AGNOSTIC: all standard operations have the same effect regardless of the contents. A shipping container will be stacked in exactly the same way whether it contains Vietnamese powder coffee or spare Maserati parts. Similarly, Standard Containers are started or uploaded in the same way whether they contain a postgres database, a php application with its dependencies and application server, or Java build artifacts. + + +### 3. INFRASTRUCTURE-AGNOSTIC + +Both types of containers are INFRASTRUCTURE-AGNOSTIC: they can be transported to thousands of facilities around the world, and manipulated by a wide variety of equipment. A shipping container can be packed in a factory in Ukraine, transported by truck to the nearest routing center, stacked onto a train, loaded into a German boat by an Australian-built crane, stored in a warehouse at a US facility, etc. Similarly, a standard container can be bundled on my laptop, uploaded to S3, downloaded, run and snapshotted by a build server at Equinix in Virginia, uploaded to 10 staging servers in a home-made Openstack cluster, then sent to 30 production instances across 3 EC2 regions. + + +### 4. DESIGNED FOR AUTOMATION + +Because they offer the same standard operations regardless of content and infrastructure, Standard Containers, just like their physical counterpart, are extremely well-suited for automation. In fact, you could say automation is their secret weapon. + +Many things that once required time-consuming and error-prone human effort can now be programmed. Before shipping containers, a bag of powder coffee was hauled, dragged, dropped, rolled and stacked by 10 different people in 10 different locations by the time it reached its destination. 1 out of 50 disappeared. 1 out of 20 was damaged. The process was slow, inefficient and cost a fortune - and was entirely different depending on the facility and the type of goods. + +Similarly, before Standard Containers, by the time a software component ran in production, it had been individually built, configured, bundled, documented, patched, vendored, templated, tweaked and instrumented by 10 different people on 10 different computers. Builds failed, libraries conflicted, mirrors crashed, post-it notes were lost, logs were misplaced, cluster updates were half-broken. The process was slow, inefficient and cost a fortune - and was entirely different depending on the language and infrastructure provider. + + +### 5. INDUSTRIAL-GRADE DELIVERY + +There are 17 million shipping containers in existence, packed with every physical good imaginable. Every single one of them can be loaded on the same boats, by the same cranes, in the same facilities, and sent anywhere in the World with incredible efficiency. It is embarrassing to think that a 30 ton shipment of coffee can safely travel half-way across the World in *less time* than it takes a software team to deliver its code from one datacenter to another sitting 10 miles away. + +With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. + + + + +Standard Container Specification +-------------------------------- + +(TODO) + +### Image format + + +### Standard operations + +* Copy +* Run +* Stop +* Wait +* Commit +* Attach standard streams +* List filesystem changes +* ... + +### Execution environment + +#### Root filesystem + +#### Environment variables + +#### Process arguments + +#### Networking + +#### Process namespacing + +#### Resource limits + +#### Process monitoring + +#### Logging + +#### Signals + +#### Pseudo-terminal allocation + +#### Security + + diff --git a/components/engine/packaging/ubuntu/debian/changelog b/components/engine/packaging/ubuntu/debian/changelog new file mode 100644 index 0000000000..d8932885e6 --- /dev/null +++ b/components/engine/packaging/ubuntu/debian/changelog @@ -0,0 +1,5 @@ +lxc-docker (1) precise; urgency=low + + * Initial release + + -- dotCloud Mon, 25 Mar 2013 05:51:12 -0700 diff --git a/components/engine/deb/debian/compat b/components/engine/packaging/ubuntu/debian/compat similarity index 100% rename from components/engine/deb/debian/compat rename to components/engine/packaging/ubuntu/debian/compat diff --git a/components/engine/deb/debian/control b/components/engine/packaging/ubuntu/debian/control similarity index 64% rename from components/engine/deb/debian/control rename to components/engine/packaging/ubuntu/debian/control index 5245d7e238..1ad913854d 100644 --- a/components/engine/deb/debian/control +++ b/components/engine/packaging/ubuntu/debian/control @@ -1,16 +1,15 @@ -Source: dotcloud-docker +Source: lxc-docker Section: misc Priority: extra -Homepage: https://github.com/dotcloud/docker +Homepage: http://docker.io Maintainer: Daniel Mizyrycki -Build-Depends: debhelper (>= 8.0.0), git, golang -Vcs-Git: https://github.com/dotcloud/docker.git -Standards-Version: 3.9.2 +Build-Depends: debhelper (>= 8.0.0), pkg-config, git, golang, libsqlite3-dev +Vcs-Git: http://github.com/dotcloud/docker.git +Standards-Version: 3.9.3 -Package: dotcloud-docker +Package: lxc-docker Architecture: amd64 -Provides: dotcloud-docker -Depends: lxc, wget, bsdtar, curl +Depends: ${shlibs:Depends}, ${misc:Depends}, lxc, wget, bsdtar, curl, sqlite3 Conflicts: docker Description: A process manager with superpowers It encapsulates heterogeneous payloads in Standard Containers, and runs diff --git a/components/engine/deb/debian/copyright b/components/engine/packaging/ubuntu/debian/copyright similarity index 99% rename from components/engine/deb/debian/copyright rename to components/engine/packaging/ubuntu/debian/copyright index 6f3a66bbce..c6c97190a9 100644 --- a/components/engine/deb/debian/copyright +++ b/components/engine/packaging/ubuntu/debian/copyright @@ -1,5 +1,5 @@ Format: http://dep.debian.net/deps/dep5 -Upstream-Name: dotcloud-docker +Upstream-Name: docker Source: https://github.com/dotcloud/docker Files: * diff --git a/components/engine/deb/debian/docs b/components/engine/packaging/ubuntu/debian/docs similarity index 100% rename from components/engine/deb/debian/docs rename to components/engine/packaging/ubuntu/debian/docs diff --git a/components/engine/deb/debian/rules b/components/engine/packaging/ubuntu/debian/rules similarity index 100% rename from components/engine/deb/debian/rules rename to components/engine/packaging/ubuntu/debian/rules diff --git a/components/engine/deb/debian/source/format b/components/engine/packaging/ubuntu/debian/source/format similarity index 100% rename from components/engine/deb/debian/source/format rename to components/engine/packaging/ubuntu/debian/source/format diff --git a/components/engine/deb/etc/docker.upstart b/components/engine/packaging/ubuntu/etc/docker.upstart similarity index 100% rename from components/engine/deb/etc/docker.upstart rename to components/engine/packaging/ubuntu/etc/docker.upstart From 263f9f859184f88f604d36aa05739023b000cf0a Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Tue, 9 Apr 2013 14:30:10 -0700 Subject: [PATCH 02/98] testing: Add buildbot VM Upstream-commit: bbaa975ec8ea44f7df4f96a910360abc43b4d5f7 Component: engine --- components/engine/Makefile | 3 ++ components/engine/buildbot/README.rst | 20 ++++++++ components/engine/buildbot/Vagrantfile | 28 +++++++++++ .../buildbot/buildbot-cfg/buildbot-cfg.sh | 43 +++++++++++++++++ .../buildbot/buildbot-cfg/buildbot.conf | 18 ++++++++ .../engine/buildbot/buildbot-cfg/master.cfg | 46 +++++++++++++++++++ .../engine/buildbot/buildbot-cfg/post-commit | 21 +++++++++ components/engine/buildbot/buildbot.pp | 32 +++++++++++++ components/engine/buildbot/requirements.txt | 6 +++ 9 files changed, 217 insertions(+) create mode 100644 components/engine/buildbot/README.rst create mode 100644 components/engine/buildbot/Vagrantfile create mode 100755 components/engine/buildbot/buildbot-cfg/buildbot-cfg.sh create mode 100644 components/engine/buildbot/buildbot-cfg/buildbot.conf create mode 100644 components/engine/buildbot/buildbot-cfg/master.cfg create mode 100755 components/engine/buildbot/buildbot-cfg/post-commit create mode 100644 components/engine/buildbot/buildbot.pp create mode 100644 components/engine/buildbot/requirements.txt diff --git a/components/engine/Makefile b/components/engine/Makefile index a6eb613830..94d8d44436 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -49,3 +49,6 @@ test: all fmt: @gofmt -s -l -w . + +hack: + @(cd $(CURDIR)/buildbot; vagrant up) diff --git a/components/engine/buildbot/README.rst b/components/engine/buildbot/README.rst new file mode 100644 index 0000000000..a52b9769ef --- /dev/null +++ b/components/engine/buildbot/README.rst @@ -0,0 +1,20 @@ +Buildbot +======== + +Buildbot is a continuous integration system designed to automate the +build/test cycle. By automatically rebuilding and testing the tree each time +something has changed, build problems are pinpointed quickly, before other +developers are inconvenienced by the failure. + +When running 'make hack' at the docker root directory, it spawns a virtual +machine in the background running a buildbot instance and adds a git +post-commit hook that automatically run docker tests for you. + +You can check your buildbot instance at http://192.168.33.21:8010/waterfall + + +Buildbot dependencies +--------------------- + +vagrant, virtualbox packages and python package requests + diff --git a/components/engine/buildbot/Vagrantfile b/components/engine/buildbot/Vagrantfile new file mode 100644 index 0000000000..ea027f0666 --- /dev/null +++ b/components/engine/buildbot/Vagrantfile @@ -0,0 +1,28 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +$BUILDBOT_IP = '192.168.33.21' + +def v10(config) + config.vm.box = "quantal64_3.5.0-25" + config.vm.box_url = "http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box" + config.vm.share_folder 'v-data', '/data/docker', File.dirname(__FILE__) + '/..' + config.vm.network :hostonly, $BUILDBOT_IP + + # Ensure puppet is installed on the instance + config.vm.provision :shell, :inline => 'apt-get -qq update; apt-get install -y puppet' + + config.vm.provision :puppet do |puppet| + puppet.manifests_path = '.' + puppet.manifest_file = 'buildbot.pp' + puppet.options = ['--templatedir','.'] + end +end + +Vagrant::VERSION < '1.1.0' and Vagrant::Config.run do |config| + v10(config) +end + +Vagrant::VERSION >= '1.1.0' and Vagrant.configure('1') do |config| + v10(config) +end diff --git a/components/engine/buildbot/buildbot-cfg/buildbot-cfg.sh b/components/engine/buildbot/buildbot-cfg/buildbot-cfg.sh new file mode 100755 index 0000000000..5e4e7432fd --- /dev/null +++ b/components/engine/buildbot/buildbot-cfg/buildbot-cfg.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Auto setup of buildbot configuration. Package installation is being done +# on buildbot.pp +# Dependencies: buildbot, buildbot-slave, supervisor + +SLAVE_NAME='buildworker' +SLAVE_SOCKET='localhost:9989' +BUILDBOT_PWD='pass-docker' +USER='vagrant' +ROOT_PATH='/data/buildbot' +DOCKER_PATH='/data/docker' +BUILDBOT_CFG="$DOCKER_PATH/buildbot/buildbot-cfg" +IP=$(grep BUILDBOT_IP /data/docker/buildbot/Vagrantfile | awk -F "'" '{ print $2; }') + +function run { su $USER -c "$1"; } + +export PATH=/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin + +# Exit if buildbot has already been installed +[ -d "$ROOT_PATH" ] && exit 0 + +# Setup buildbot +run "mkdir -p ${ROOT_PATH}" +cd ${ROOT_PATH} +run "buildbot create-master master" +run "cp $BUILDBOT_CFG/master.cfg master" +run "sed -i 's/localhost/$IP/' master/master.cfg" +run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" + +# Allow buildbot subprocesses (docker tests) to properly run in containers, +# in particular with docker -u +run "sed -i 's/^umask = None/umask = 000/' ${ROOT_PATH}/slave/buildbot.tac" + +# Setup supervisor +cp $BUILDBOT_CFG/buildbot.conf /etc/supervisor/conf.d/buildbot.conf +sed -i "s/^chmod=0700.*0700./chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf +kill -HUP `pgrep -f "/usr/bin/python /usr/bin/supervisord"` + +# Add git hook +cp $BUILDBOT_CFG/post-commit $DOCKER_PATH/.git/hooks +sed -i "s/localhost/$IP/" $DOCKER_PATH/.git/hooks/post-commit + diff --git a/components/engine/buildbot/buildbot-cfg/buildbot.conf b/components/engine/buildbot/buildbot-cfg/buildbot.conf new file mode 100644 index 0000000000..b162f4e7c7 --- /dev/null +++ b/components/engine/buildbot/buildbot-cfg/buildbot.conf @@ -0,0 +1,18 @@ +[program:buildmaster] +command=su vagrant -c "buildbot start master" +directory=/data/buildbot +chown= root:root +redirect_stderr=true +stdout_logfile=/var/log/supervisor/buildbot-master.log +stderr_logfile=/var/log/supervisor/buildbot-master.log + +[program:buildworker] +command=buildslave start slave +directory=/data/buildbot +chown= root:root +redirect_stderr=true +stdout_logfile=/var/log/supervisor/buildbot-slave.log +stderr_logfile=/var/log/supervisor/buildbot-slave.log + +[group:buildbot] +programs=buildmaster,buildworker diff --git a/components/engine/buildbot/buildbot-cfg/master.cfg b/components/engine/buildbot/buildbot-cfg/master.cfg new file mode 100644 index 0000000000..c786e418ed --- /dev/null +++ b/components/engine/buildbot/buildbot-cfg/master.cfg @@ -0,0 +1,46 @@ +import os +from buildbot.buildslave import BuildSlave +from buildbot.schedulers.forcesched import ForceScheduler +from buildbot.config import BuilderConfig +from buildbot.process.factory import BuildFactory +from buildbot.steps.shell import ShellCommand +from buildbot.status import html +from buildbot.status.web import authz, auth + +PORT_WEB = 8010 # Buildbot webserver port +PORT_MASTER = 9989 # Port where buildbot master listen buildworkers +TEST_USER = 'buildbot' # Credential to authenticate build triggers +TEST_PWD = 'docker' # Credential to authenticate build triggers +BUILDER_NAME = 'docker' +BUILDPASSWORD = 'pass-docker' # Credential to authenticate buildworkers +DOCKER_PATH = '/data/docker' + + +c = BuildmasterConfig = {} + +c['title'] = "Docker" +c['titleURL'] = "waterfall" +c['buildbotURL'] = "http://localhost:{0}/".format(PORT_WEB) +c['db'] = {'db_url':"sqlite:///state.sqlite"} +c['slaves'] = [BuildSlave('buildworker', BUILDPASSWORD)] +c['slavePortnum'] = PORT_MASTER + +c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])] + +# Docker test command +test_cmd = """( + cd {0}/..; rm -rf docker-tmp; git clone docker docker-tmp; + cd docker-tmp; make test; exit_status=$?; + cd ..; rm -rf docker-tmp; exit $exit_status)""".format(DOCKER_PATH) + +# Builder +factory = BuildFactory() +factory.addStep(ShellCommand(description='Docker',logEnviron=False, + usePTY=True,command=test_cmd)) +c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], + factory=factory)] + +# Status +authz_cfg=authz.Authz(auth=auth.BasicAuth([(TEST_USER,TEST_PWD)]), + forceBuild='auth') +c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)] diff --git a/components/engine/buildbot/buildbot-cfg/post-commit b/components/engine/buildbot/buildbot-cfg/post-commit new file mode 100755 index 0000000000..8c5a06bf38 --- /dev/null +++ b/components/engine/buildbot/buildbot-cfg/post-commit @@ -0,0 +1,21 @@ +#!/usr/bin/python + +'''Trigger buildbot docker test build + + post-commit git hook designed to automatically trigger buildbot on + the provided vagrant docker VM.''' + +import requests + +USERNAME = 'buildbot' +PASSWORD = 'docker' +BASE_URL = 'http://localhost:8010' +path = lambda s: BASE_URL + '/' + s + +try: + session = requests.session() + session.post(path('login'),data={'username':USERNAME,'passwd':PASSWORD}) + session.post(path('builders/docker/force'), + data={'forcescheduler':'trigger','reason':'Test commit'}) +except: + pass diff --git a/components/engine/buildbot/buildbot.pp b/components/engine/buildbot/buildbot.pp new file mode 100644 index 0000000000..8109cdc2a0 --- /dev/null +++ b/components/engine/buildbot/buildbot.pp @@ -0,0 +1,32 @@ +node default { + $USER = 'vagrant' + $ROOT_PATH = '/data/buildbot' + $DOCKER_PATH = '/data/docker' + + exec {'apt_update': command => '/usr/bin/apt-get update' } + Package { require => Exec['apt_update'] } + group {'puppet': ensure => 'present'} + + # Install dependencies + Package { ensure => 'installed' } + package { ['python-dev','python-pip','supervisor','lxc','bsdtar','git','golang']: } + + file{[ '/data' ]: + owner => $USER, group => $USER, ensure => 'directory' } + + file {'/var/tmp/requirements.txt': + content => template('requirements.txt') } + + exec {'requirements': + require => [ Package['python-dev'], Package['python-pip'], + File['/var/tmp/requirements.txt'] ], + cwd => '/var/tmp', + command => "/bin/sh -c '(/usr/bin/pip install -r requirements.txt; + rm /var/tmp/requirements.txt)'" } + + exec {'buildbot-cfg-sh': + require => [ Package['supervisor'], Exec['requirements']], + path => '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin', + cwd => '/data', + command => "$DOCKER_PATH/buildbot/buildbot-cfg/buildbot-cfg.sh" } +} diff --git a/components/engine/buildbot/requirements.txt b/components/engine/buildbot/requirements.txt new file mode 100644 index 0000000000..0e451b017d --- /dev/null +++ b/components/engine/buildbot/requirements.txt @@ -0,0 +1,6 @@ +sqlalchemy<=0.7.9 +sqlalchemy-migrate>=0.7.2 +buildbot==0.8.7p1 +buildbot_slave==0.8.7p1 +nose==1.2.1 +requests==1.1.0 From 2f21f794cfa6aa6ef8c872afd829b575b5ced42f Mon Sep 17 00:00:00 2001 From: Sridhar Ratnakumar Date: Tue, 9 Apr 2013 15:00:05 -0700 Subject: [PATCH 03/98] serve goroutine must refer to the local conn Upstream-commit: f731835e4577c414d37fa050aaec1f541511ff6b Component: engine --- components/engine/rcli/tcp.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/rcli/tcp.go b/components/engine/rcli/tcp.go index e9dba7f319..cf111cdf71 100644 --- a/components/engine/rcli/tcp.go +++ b/components/engine/rcli/tcp.go @@ -138,7 +138,8 @@ func ListenAndServe(proto, addr string, service Service) error { if err != nil { return err } - go func() { + go func(conn DockerConn) { + defer conn.Close() if DEBUG_FLAG { CLIENT_SOCKET = conn } @@ -146,8 +147,7 @@ func ListenAndServe(proto, addr string, service Service) error { log.Println("Error:", err.Error()) fmt.Fprintln(conn, "Error:", err.Error()) } - conn.Close() - }() + }(conn) } } return nil From e53c739014a514f880293fb520310d11e48d157e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 9 Apr 2013 17:40:02 -0700 Subject: [PATCH 04/98] Make sure all containers (even previously stopped) get a correct wait lock Upstream-commit: 9f83b9df2256c7b34453d5da85c17fae516ee3a4 Component: engine --- components/engine/runtime.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 7971fe4f48..24194f74ad 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -134,6 +134,9 @@ func (runtime *Runtime) Register(container *Container) error { return err } + // init the wait lock + container.waitLock = make(chan struct{}) + // FIXME: if the container is supposed to be running but is not, auto restart it? // if so, then we need to restart monitor and init a new lock // If the container is supposed to be running, make sure of it @@ -150,6 +153,14 @@ func (runtime *Runtime) Register(container *Container) error { } } } + + // If the container is not running or just has been flagged not running + // then close the wait lock chan (will be reset upon start) + if !container.State.Running { + close(container.waitLock) + } + + // Even if not running, we init the lock (prevents races in start/stop/kill) container.State.initLock() container.runtime = runtime From 05a22d65274112ea66632016f4d3ad862f1f5dc2 Mon Sep 17 00:00:00 2001 From: Julien Barbier Date: Tue, 9 Apr 2013 19:07:50 -0700 Subject: [PATCH 05/98] Fix the Makefile, rule=hack to make it work on Windows Upstream-commit: b7cda3288ed857e4e2155846090b17a2d8fccdbf Component: engine --- components/engine/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index 94d8d44436..dea53206a6 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -51,4 +51,4 @@ fmt: @gofmt -s -l -w . hack: - @(cd $(CURDIR)/buildbot; vagrant up) + cd $(CURDIR)/buildbot && vagrant up From 551b762e136345d646c1b78b063ba3f6e0156530 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 10 Apr 2013 19:30:57 +0200 Subject: [PATCH 06/98] add -l to docker ps Upstream-commit: 8c3331dc978e3cbbe3255a436c8f18f01158eb27 Component: engine --- components/engine/commands.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 20d6b45c97..a231632f27 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -647,17 +647,25 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) quiet := cmd.Bool("q", false, "Only display numeric IDs") flAll := cmd.Bool("a", false, "Show all containers. Only running containers are shown by default.") flFull := cmd.Bool("notrunc", false, "Don't truncate output") + latest := cmd.Bool("l", false, "Show only the latest created container, include non-running ones.") + n_last := cmd.Int("last", -1, "Show last created containers, include non-running ones.") if err := cmd.Parse(args); err != nil { return nil } + if *n_last == -1 && *latest { + *n_last = 1 + } w := tabwriter.NewWriter(stdout, 12, 1, 3, ' ', 0) if !*quiet { fmt.Fprintln(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tCOMMENT") } - for _, container := range srv.runtime.List() { - if !container.State.Running && !*flAll { + for i, container := range srv.runtime.List() { + if !container.State.Running && !*flAll && *n_last == -1{ continue } + if i == *n_last { + break + } if !*quiet { command := fmt.Sprintf("%s %s", container.Path, strings.Join(container.Args, " ")) if !*flFull { From 226b7e3c89ae5d9d2e1bf07a0883ce61eb2ee5cb Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Wed, 10 Apr 2013 11:23:56 -0700 Subject: [PATCH 07/98] testing: Make postcommit more generic Upstream-commit: d2c1850fb5a210d59b87dcb8504c8c24b60cca3c Component: engine --- components/engine/buildbot/buildbot-cfg/post-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/buildbot/buildbot-cfg/post-commit b/components/engine/buildbot/buildbot-cfg/post-commit index 8c5a06bf38..0173fe504f 100755 --- a/components/engine/buildbot/buildbot-cfg/post-commit +++ b/components/engine/buildbot/buildbot-cfg/post-commit @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python '''Trigger buildbot docker test build From 053e80b29d547281a19ee081de0eb2a779cbef0f Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 10 Apr 2013 21:09:21 +0200 Subject: [PATCH 08/98] changed last to n Upstream-commit: 8bd192fb16a5ac9836d4b1d3df037af65b61ffce Component: engine --- components/engine/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index a231632f27..cc123b44a2 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -648,7 +648,7 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) flAll := cmd.Bool("a", false, "Show all containers. Only running containers are shown by default.") flFull := cmd.Bool("notrunc", false, "Don't truncate output") latest := cmd.Bool("l", false, "Show only the latest created container, include non-running ones.") - n_last := cmd.Int("last", -1, "Show last created containers, include non-running ones.") + n_last := cmd.Int("n", -1, "Show n last created containers, include non-running ones.") if err := cmd.Parse(args); err != nil { return nil } From 4fbbe500d2f7e399e5e9c8dd6a974b20246dcd83 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 10 Apr 2013 13:57:26 -0700 Subject: [PATCH 09/98] Added temporary spec for data volumes as requested on #111 Upstream-commit: e41fd24542d96c503cd923a403c43aceaf9af965 Component: engine --- components/engine/SPECS/data-volumes.md | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 components/engine/SPECS/data-volumes.md diff --git a/components/engine/SPECS/data-volumes.md b/components/engine/SPECS/data-volumes.md new file mode 100644 index 0000000000..d800656afc --- /dev/null +++ b/components/engine/SPECS/data-volumes.md @@ -0,0 +1,71 @@ + +## Spec for data volumes + +Spec owner: Solomon Hykes + +Data volumes (issue #111) are a much-requested feature which trigger much discussion and debate. Below is the current authoritative spec for implementing data volumes. +This spec will be deprecated once the feature is fully implemented. + +Discussion, requests, trolls, demands, offerings, threats and other forms of supplications concerning this spec should be addressed to Solomon here: https://github.com/dotcloud/docker/issues/111 + + +### 1. Creating data volumes + +At container creation, parts of a container's filesystem can be mounted as separate data volumes. Volumes are defined with the -v flag. + +For example: + +```bash +$ docker run -v /var/lib/postgres -v /var/log postgres /usr/bin/postgres +``` + +In this example, a new container is created from the 'postgres' image. At the same time, docker creates 2 new data volumes: one will be mapped to the container at /var/lib/postgres, the other at /var/log. + +2 important notes: + +1) Volumes don't have top-level names. At no point does the user provide a name, or is a name given to him. Volumes are identified by the path at which they are mounted inside their container. + +2) The user doesn't choose the source of the volume. Docker only mounts volumes it created itself, in the same way that it only runs containers that it created itself. That is by design. + + +### 2. Sharing data volumes + +Instead of creating its own volumes, a container can share another container's volumes. For example: + +```bash +$ docker run --volumes-from $OTHER_CONTAINER_ID postgres /usr/local/bin/postgres-backup +``` + +In this example, a new container is created from the 'postgres' example. At the same time, docker will *re-use* the 2 data volumes created in the previous example. One volume will be mounted on the /var/lib/postgres of *both* containers, and the other will be mounted on the /var/log of both containers. + +### 3. Under the hood + +Docker stores volumes in /var/lib/docker/volumes. Each volume receives a globally unique ID at creation, and is stored at /var/lib/docker/volumes/ID. + +At creation, volumes are attached to a single container - the source of truth for this mapping will be the container's configuration. + +Mounting a volume consists of calling "mount --bind" from the volume's directory to the appropriate sub-directory of the container mountpoint. This may be done by Docker itself, or farmed out to lxc (which supports mount-binding) if possible. + + +### 4. Backups, transfers and other volume operations + +Volumes sometimes need to be backed up, transfered between hosts, synchronized, etc. These operations typically are application-specific or site-specific, eg. rsync vs. S3 upload vs. replication vs... + +Rather than attempting to implement all these scenarios directly, Docker will allow for custom implementations using an extension mechanism. + +### 5. Custom volume handlers + +Docker allows for arbitrary code to be executed against a container's volumes, to implement any custom action: backup, transfer, synchronization across hosts, etc. + +Here's an example: + +```bash +$ DB=$(docker run -d -v /var/lib/postgres -v /var/log postgres /usr/bin/postgres) + +$ BACKUP_JOB=$(docker run -d --volumes-from $DB shykes/backuper /usr/local/bin/backup-postgres --s3creds=$S3CREDS) + +$ docker wait $BACKUP_JOB +``` + +Congratulations, you just implemented a custom volume handler, using Docker's built-in ability to 1) execute arbitrary code and 2) share volumes between containers. + From bcc22cca7dca74aaf2349033578b34d4eee4df1b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 10 Apr 2013 17:23:42 -0700 Subject: [PATCH 10/98] Allow rmi to remove image from its name Upstream-commit: 97f48e59fcfa8ca0f0a20ed80cab7330af93ae6d Component: engine --- components/engine/commands.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 20d6b45c97..0fc1c9a41d 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -351,7 +351,11 @@ func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) return nil } for _, name := range cmd.Args() { - if err := srv.runtime.graph.Delete(name); err != nil { + img, err := srv.runtime.repositories.LookupImage(name) + if err != nil { + return err + } + if err := srv.runtime.graph.Delete(img.Id); err != nil { return err } } From 7d008a24f89c155e23462c6ce26d2fef738f559e Mon Sep 17 00:00:00 2001 From: Louis Opter Date: Wed, 10 Apr 2013 17:40:28 -0700 Subject: [PATCH 11/98] Use ip to setup the gateway in sysinit.go ip from iproute2 replaces the legacy route tool which is often not installed by default on recent Linux distributions. The same patch has been done in network.go and is re-used here. Upstream-commit: 5e1a975b48c1d4e6743f68ca770f8fe06fb32ce4 Component: engine --- components/engine/sysinit.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/engine/sysinit.go b/components/engine/sysinit.go index a2c06239e9..2c1106db10 100644 --- a/components/engine/sysinit.go +++ b/components/engine/sysinit.go @@ -17,8 +17,7 @@ func setupNetworking(gw string) { if gw == "" { return } - cmd := exec.Command("/sbin/route", "add", "default", "gw", gw) - if err := cmd.Run(); err != nil { + if _, err := ip("route", "add", "default", "via", gw); err != nil { log.Fatalf("Unable to set up networking: %v", err) } } From 4af9c309f0aa3424f1e4fcd796a3fb2488071c63 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 10 Apr 2013 18:23:34 -0700 Subject: [PATCH 12/98] Put the resolv.conf path in a variable instead of being hardcoded within lxc Upstream-commit: 1f9f5eed5d004395886b8e93cec9389332d8fded Component: engine --- components/engine/container.go | 13 +++++++------ components/engine/lxc_template.go | 2 +- components/engine/runtime.go | 6 +++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index f180c7559b..a930cbd3e3 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -33,13 +33,14 @@ type Container struct { network *NetworkInterface NetworkSettings *NetworkSettings - SysInitPath string - cmd *exec.Cmd - stdout *writeBroadcaster - stderr *writeBroadcaster - stdin io.ReadCloser - stdinPipe io.WriteCloser + SysInitPath string + ResolvConfPath string + cmd *exec.Cmd + stdout *writeBroadcaster + stderr *writeBroadcaster + stdin io.ReadCloser + stdinPipe io.WriteCloser ptyMaster io.Closer runtime *Runtime diff --git a/components/engine/lxc_template.go b/components/engine/lxc_template.go index c6849cb0df..5ac62f52af 100644 --- a/components/engine/lxc_template.go +++ b/components/engine/lxc_template.go @@ -78,7 +78,7 @@ lxc.mount.entry = devpts {{$ROOTFS}}/dev/pts devpts newinstance,ptmxmode=0666,no lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/sbin/init none bind,ro 0 0 # In order to get a working DNS environment, mount bind (ro) the host's /etc/resolv.conf into the container -lxc.mount.entry = /etc/resolv.conf {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 +lxc.mount.entry = {{.ResolvConfPath}} {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 # drop linux capabilities (apply mainly to the user root in the container) diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 24194f74ad..2dd419868f 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -82,6 +82,9 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { if config.Hostname == "" { config.Hostname = id[:12] } + + resolvConfPath := "/etc/resolv.conf" + container := &Container{ // FIXME: we should generate the ID here instead of receiving it as an argument Id: id, @@ -92,7 +95,8 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { Image: img.Id, // Always use the resolved image id NetworkSettings: &NetworkSettings{}, // FIXME: do we need to store this in the container? - SysInitPath: sysInitPath, + SysInitPath: sysInitPath, + ResolvConfPath: resolvConfPath, } container.root = runtime.containerRoot(container.Id) // Step 1: create the container directory. From 1dcc582981cb7a0968adf03d70bca4336a48bf8e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 10 Apr 2013 19:08:46 -0700 Subject: [PATCH 13/98] use crlf in login in order to avoir issue due to real raw mode Upstream-commit: 9d1fd2317d5e3edee845410f10969b37ba37c9d5 Component: engine --- components/engine/commands.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 20d6b45c97..298890c51f 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -79,7 +79,7 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout rcli.DockerConn, args .. n, err := stdin.Read(char) if n > 0 { if char[0] == '\r' || char[0] == '\n' { - stdout.Write([]byte{'\n'}) + stdout.Write([]byte{'\r', '\n'}) break } else if char[0] == 127 || char[0] == '\b' { if i > 0 { @@ -99,7 +99,7 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout rcli.DockerConn, args .. } if err != nil { if err != io.EOF { - fmt.Fprintf(stdout, "Read error: %v\n", err) + fmt.Fprintf(stdout, "Read error: %v\r\n", err) } break } @@ -149,7 +149,7 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout rcli.DockerConn, args .. newAuthConfig := auth.NewAuthConfig(username, password, email, srv.runtime.root) status, err := auth.Login(newAuthConfig) if err != nil { - fmt.Fprintln(stdout, "Error:", err) + fmt.Fprintf(stdout, "Error: %s\r\n", err) } else { srv.runtime.authConfig = newAuthConfig } From d2aba9b3560285e354b972d7e56ca47c1416cffe Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 10 Apr 2013 19:13:15 -0700 Subject: [PATCH 14/98] Use crlf in registry function to avoid issue with autlogin in push and autopull in run Upstream-commit: 6dc4c74b5ab63e6de374fe839e05acf5998a6f72 Component: engine --- components/engine/registry.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/engine/registry.go b/components/engine/registry.go index 761fc335d3..428db1b968 100644 --- a/components/engine/registry.go +++ b/components/engine/registry.go @@ -97,7 +97,7 @@ func (graph *Graph) LookupRemoteImage(imgId string, authConfig *auth.AuthConfig) func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *auth.AuthConfig) (*Image, Archive, error) { client := &http.Client{} - fmt.Fprintf(stdout, "Pulling %s metadata\n", imgId) + fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) // Get the Json req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) if err != nil { @@ -125,7 +125,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *a img.Id = imgId // Get the layer - fmt.Fprintf(stdout, "Pulling %s fs layer\n", imgId) + fmt.Fprintf(stdout, "Pulling %s fs layer\r\n", imgId) req, err = http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/layer", nil) if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) @@ -164,7 +164,7 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.A func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} - fmt.Fprintf(stdout, "Pulling repository %s\n", remote) + fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) var repositoryTarget string // If we are asking for 'root' repository, lookup on the Library's registry @@ -196,7 +196,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return err } for tag, rev := range t { - fmt.Fprintf(stdout, "Pulling tag %s:%s\n", remote, tag) + fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) if err = graph.PullImage(stdout, rev, authConfig); err != nil { return err } @@ -223,7 +223,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) } - fmt.Fprintf(stdout, "Pushing %s metadata\n", img.Id) + fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) // FIXME: try json with UTF8 jsonData := strings.NewReader(string(jsonRaw)) @@ -253,7 +253,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth } } - fmt.Fprintf(stdout, "Pushing %s fs layer\n", img.Id) + fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil) req2.SetBasicAuth(authConfig.Username, authConfig.Password) res2, err := client.Do(req2) @@ -375,15 +375,15 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, a // Check if the local impage exists img, err := graph.Get(imgId) if err != nil { - fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\n", remote, tag, imgId) + fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId) return nil } - fmt.Fprintf(stdout, "Pushing tag %s:%s\n", remote, tag) + fmt.Fprintf(stdout, "Pushing tag %s:%s\r\n", remote, tag) // Push the image if err = graph.PushImage(stdout, img, authConfig); err != nil { return err } - fmt.Fprintf(stdout, "Registering tag %s:%s\n", remote, tag) + fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) // And then the tag if err = graph.pushTag(remote, imgId, tag, authConfig); err != nil { return err @@ -399,7 +399,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return fmt.Errorf("Permission denied on repository %s\n", remote) } - fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\n", remote, len(localRepo)) + fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) // For each image within the repo, push them for tag, imgId := range localRepo { if err := graph.pushPrimitive(stdout, remote, tag, imgId, authConfig); err != nil { From 5d8f85d0ac0d3fcdf39cb2301bc214c807e40af9 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Wed, 10 Apr 2013 21:00:39 -0700 Subject: [PATCH 15/98] packaging; issue #251: Add debian packaging Upstream-commit: 8ea1e9126f3d64216e2d8a91ce6f8ad1f3d8587c Component: engine --- components/engine/packaging/debian/Makefile | 35 +++ .../engine/packaging/debian/Vagrantfile | 22 ++ components/engine/packaging/debian/changelog | 14 ++ components/engine/packaging/debian/compat | 1 + components/engine/packaging/debian/control | 19 ++ components/engine/packaging/debian/copyright | 237 ++++++++++++++++++ .../engine/packaging/debian/docker.initd | 49 ++++ components/engine/packaging/debian/docs | 1 + .../packaging/debian/lxc-docker.postinst | 13 + components/engine/packaging/debian/rules | 13 + .../engine/packaging/debian/source/format | 1 + 11 files changed, 405 insertions(+) create mode 100644 components/engine/packaging/debian/Makefile create mode 100644 components/engine/packaging/debian/Vagrantfile create mode 100644 components/engine/packaging/debian/changelog create mode 100644 components/engine/packaging/debian/compat create mode 100644 components/engine/packaging/debian/control create mode 100644 components/engine/packaging/debian/copyright create mode 100644 components/engine/packaging/debian/docker.initd create mode 100644 components/engine/packaging/debian/docs create mode 100644 components/engine/packaging/debian/lxc-docker.postinst create mode 100755 components/engine/packaging/debian/rules create mode 100644 components/engine/packaging/debian/source/format diff --git a/components/engine/packaging/debian/Makefile b/components/engine/packaging/debian/Makefile new file mode 100644 index 0000000000..75ff8f34f5 --- /dev/null +++ b/components/engine/packaging/debian/Makefile @@ -0,0 +1,35 @@ +PKG_NAME=lxc-docker +DOCKER_VERSION=$(shell head -1 changelog | awk 'match($$0, /\(.+\)/) {print substr($$0, RSTART+1, RLENGTH-4)}') +GITHUB_PATH=github.com/dotcloud/docker +SOURCE_PKG=$(PKG_NAME)_$(DOCKER_VERSION).orig.tar.gz +BUILD_SRC=${CURDIR}/../../build_src + +all: + # Compile docker. Used by debian dpkg-buildpackage. + cd src/${GITHUB_PATH}/docker; GOPATH=${CURDIR} go build + +install: + # Used by debian dpkg-buildpackage + mkdir -p $(DESTDIR)/usr/bin + mkdir -p $(DESTDIR)/etc/init.d + install -m 0755 src/${GITHUB_PATH}/docker/docker $(DESTDIR)/usr/bin + install -o root -m 0755 debian/docker.initd $(DESTDIR)/etc/init.d/docker + +debian: + # This Makefile will compile the github master branch of dotcloud/docker + # Retrieve docker project and its go structure from internet + rm -rf ${BUILD_SRC} + GOPATH=${BUILD_SRC} go get ${GITHUB_PATH} + # Add debianization + mkdir ${BUILD_SRC}/debian + cp Makefile ${BUILD_SRC} + cp -r * ${BUILD_SRC}/debian + cp ../../README.md ${BUILD_SRC} + # Cleanup + for d in `find ${BUILD_SRC} -name '.git*'`; do rm -rf $$d; done + rm -rf ${BUILD_SRC}/../${SOURCE_PKG} + rm -rf ${BUILD_SRC}/pkg + # Create docker debian files + cd ${BUILD_SRC}; tar czf ../${SOURCE_PKG} . + cd ${BUILD_SRC}; dpkg-buildpackage + rm -rf ${BUILD_SRC} diff --git a/components/engine/packaging/debian/Vagrantfile b/components/engine/packaging/debian/Vagrantfile new file mode 100644 index 0000000000..2da2900605 --- /dev/null +++ b/components/engine/packaging/debian/Vagrantfile @@ -0,0 +1,22 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +$BUILDBOT_IP = '192.168.33.31' + +def v10(config) + config.vm.box = 'debian' + config.vm.share_folder 'v-data', '/data/docker', File.dirname(__FILE__) + '/../..' + config.vm.network :hostonly, $BUILDBOT_IP + + # Install debian packaging dependencies and create debian packages + config.vm.provision :shell, :inline => 'apt-get -qq update; apt-get install -y debhelper autotools-dev golang' + config.vm.provision :shell, :inline => 'cd /data/docker/packaging/debian; make debian' +end + +Vagrant::VERSION < '1.1.0' and Vagrant::Config.run do |config| + v10(config) +end + +Vagrant::VERSION >= '1.1.0' and Vagrant.configure('1') do |config| + v10(config) +end diff --git a/components/engine/packaging/debian/changelog b/components/engine/packaging/debian/changelog new file mode 100644 index 0000000000..761a879e8a --- /dev/null +++ b/components/engine/packaging/debian/changelog @@ -0,0 +1,14 @@ +lxc-docker (0.1.4-1) unstable; urgency=low + + Improvements [+], Updates [*], Bug fixes [-]: + * Changed default bridge interface do 'docker0' + - Fix a race condition when running the port allocator + + -- Daniel Mizyrycki Wed, 10 Apr 2013 18:06:21 -0700 + + +lxc-docker (0.1.0-1) unstable; urgency=low + + * Initial release + + -- Daniel Mizyrycki Mon, 29 Mar 2013 18:09:55 -0700 diff --git a/components/engine/packaging/debian/compat b/components/engine/packaging/debian/compat new file mode 100644 index 0000000000..ec635144f6 --- /dev/null +++ b/components/engine/packaging/debian/compat @@ -0,0 +1 @@ +9 diff --git a/components/engine/packaging/debian/control b/components/engine/packaging/debian/control new file mode 100644 index 0000000000..a09e9aee56 --- /dev/null +++ b/components/engine/packaging/debian/control @@ -0,0 +1,19 @@ +Source: lxc-docker +Section: admin +Priority: optional +Maintainer: Daniel Mizyrycki +Build-Depends: debhelper (>= 9),autotools-dev,golang +Standards-Version: 3.9.3 +Homepage: http://github.com/dotcloud/docker + +Package: lxc-docker +Architecture: linux-any +Depends: ${misc:Depends},${shlibs:Depends},lxc,bsdtar +Conflicts: docker +Description: lxc-docker is a Linux container runtime + Docker complements LXC with a high-level API which operates at the process + level. It runs unix processes with strong guarantees of isolation and + repeatability across servers. + Docker is a great building block for automating distributed systems: + large-scale web deployments, database clusters, continuous deployment systems, + private PaaS, service-oriented architectures, etc. diff --git a/components/engine/packaging/debian/copyright b/components/engine/packaging/debian/copyright new file mode 100644 index 0000000000..668c8635e4 --- /dev/null +++ b/components/engine/packaging/debian/copyright @@ -0,0 +1,237 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: docker +Upstream-Contact: DotCloud Inc +Source: http://github.com/dotcloud/docker + +Files: * +Copyright: 2012, DotCloud Inc +License: Apache-2.0 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2012 DotCloud Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +Files: src/github.com/kr/pty/* +Copyright: Copyright (c) 2011 Keith Rarick +License: Expat + Copyright (c) 2011 Keith Rarick + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall + be included in all copies or substantial portions of the + Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS + OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/components/engine/packaging/debian/docker.initd b/components/engine/packaging/debian/docker.initd new file mode 100644 index 0000000000..2b6a3c0979 --- /dev/null +++ b/components/engine/packaging/debian/docker.initd @@ -0,0 +1,49 @@ +#!/bin/sh + +### BEGIN INIT INFO +# Provides: docker +# Required-Start: $local_fs +# Required-Stop: $local_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: docker +# Description: docker daemon +### END INIT INFO + +DOCKER=/usr/bin/docker +PIDFILE=/var/run/docker.pid + +# Check docker is present +[ -x $DOCKER ] || log_success_msg "Docker not present" + +# Get lsb functions +. /lib/lsb/init-functions + + +case "$1" in + start) + log_begin_msg "Starting docker..." + start-stop-daemon --start --background --exec "$DOCKER" -- -d + log_end_msg $? + ;; + stop) + log_begin_msg "Stopping docker..." + docker_pid=`pgrep -f "$DOCKER -d"` + [ -n "$docker_pid" ] && kill $docker_pid + log_end_msg $? + ;; + status) + docker_pid=`pgrep -f "$DOCKER -d"` + if [ -z "$docker_pid" ] ; then + echo "docker not running" + else + echo "docker running (pid $docker_pid)" + fi + ;; + *) + echo "Usage: /etc/init.d/docker {start|stop|status}" + exit 1 + ;; +esac + +exit 0 diff --git a/components/engine/packaging/debian/docs b/components/engine/packaging/debian/docs new file mode 100644 index 0000000000..b43bf86b50 --- /dev/null +++ b/components/engine/packaging/debian/docs @@ -0,0 +1 @@ +README.md diff --git a/components/engine/packaging/debian/lxc-docker.postinst b/components/engine/packaging/debian/lxc-docker.postinst new file mode 100644 index 0000000000..91e251dc8d --- /dev/null +++ b/components/engine/packaging/debian/lxc-docker.postinst @@ -0,0 +1,13 @@ +#!/bin/sh + +# Ensure cgroup is mounted +if [ -z "`/bin/egrep -e '^cgroup' /etc/fstab`" ]; then + /bin/echo 'cgroup /sys/fs/cgroup cgroup defaults 0 0' >>/etc/fstab +fi +if [ -z "`/bin/mount | /bin/egrep -e '^cgroup'`" ]; then + /bin/mount /sys/fs/cgroup +fi + +# Start docker +/usr/sbin/update-rc.d docker defaults +/etc/init.d/docker start diff --git a/components/engine/packaging/debian/rules b/components/engine/packaging/debian/rules new file mode 100755 index 0000000000..25f16f9c61 --- /dev/null +++ b/components/engine/packaging/debian/rules @@ -0,0 +1,13 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + +%: + dh ${@} --with autotools_dev diff --git a/components/engine/packaging/debian/source/format b/components/engine/packaging/debian/source/format new file mode 100644 index 0000000000..163aaf8d82 --- /dev/null +++ b/components/engine/packaging/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) From 556b0802fe916a5140343dc50413879609de94be Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 11 Apr 2013 13:09:40 +0200 Subject: [PATCH 16/98] snake_case to camelCase Upstream-commit: 17136d58f29fdd921d41ebd7c67c73bd4078b023 Component: engine --- components/engine/commands.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index cc123b44a2..67689dda82 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -648,22 +648,22 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) flAll := cmd.Bool("a", false, "Show all containers. Only running containers are shown by default.") flFull := cmd.Bool("notrunc", false, "Don't truncate output") latest := cmd.Bool("l", false, "Show only the latest created container, include non-running ones.") - n_last := cmd.Int("n", -1, "Show n last created containers, include non-running ones.") + nLast := cmd.Int("n", -1, "Show n last created containers, include non-running ones.") if err := cmd.Parse(args); err != nil { return nil } - if *n_last == -1 && *latest { - *n_last = 1 + if *nLast == -1 && *latest { + *nLast = 1 } w := tabwriter.NewWriter(stdout, 12, 1, 3, ' ', 0) if !*quiet { fmt.Fprintln(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tCOMMENT") } for i, container := range srv.runtime.List() { - if !container.State.Running && !*flAll && *n_last == -1{ + if !container.State.Running && !*flAll && *nLast == -1 { continue } - if i == *n_last { + if i == *nLast { break } if !*quiet { From 0823d282a2bc061b13247bcf32f81307b0f4b3c8 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 11 Apr 2013 16:27:01 +0200 Subject: [PATCH 17/98] display usage when no parameter Upstream-commit: c105049f7ea5927ec9d7258624391f6a2b43a6a3 Component: engine --- components/engine/commands.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/engine/commands.go b/components/engine/commands.go index 20d6b45c97..9049596b39 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -389,6 +389,10 @@ func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) if err := cmd.Parse(args); err != nil { return nil } + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } for _, name := range cmd.Args() { container := srv.runtime.Get(name) if container == nil { From 46c83a02af18b7c56b35983c698015a2efa82eff Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 11 Apr 2013 09:02:34 -0700 Subject: [PATCH 18/98] force kill now use lxc-kill. Fixes #383 Upstream-commit: e68c04b722b4c001c999abc2cd5b60b1f1b34457 Component: engine --- components/engine/container.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index f180c7559b..06f39083d8 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -550,9 +550,21 @@ func (container *Container) kill() error { if !container.State.Running || container.cmd == nil { return nil } - if err := container.cmd.Process.Kill(); err != nil { - return err + + // Sending SIGINT to the process via lxc + output, err := exec.Command("lxc-kill", "-n", container.Id, "9").CombinedOutput() + if err != nil { + Debugf("error killing container %s (%s, %s)", container.Id, output, err) } + + // 2. Wait for the process to die, in last resort, try to kill the process directly + if err := container.WaitTimeout(10 * time.Second); err != nil { + log.Printf("Container %s failed to exit within 10 seconds of SIGINT - trying direct SIGKILL", container.Id) + if err := container.cmd.Process.Kill(); err != nil { + return err + } + } + // Wait for the container to be actually stopped container.Wait() return nil From c36cca18b40acba589566caffbbf4851c7e6569f Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 11 Apr 2013 09:26:17 -0700 Subject: [PATCH 19/98] Detect and mark ghost container. Upstream-commit: 313d13ea01f58c3e611fb85e435d36dd4907a6ac Component: engine --- components/engine/container.go | 9 +++++++++ components/engine/runtime.go | 3 +++ components/engine/state.go | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/components/engine/container.go b/components/engine/container.go index f180c7559b..eefbbe5c74 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -561,6 +561,12 @@ func (container *Container) kill() error { func (container *Container) Kill() error { container.State.lock() defer container.State.unlock() + if !container.State.Running { + return nil + } + if container.State.Ghost { + return fmt.Errorf("Impossible to kill ghost containers") + } return container.kill() } @@ -570,6 +576,9 @@ func (container *Container) Stop() error { if !container.State.Running { return nil } + if container.State.Ghost { + return fmt.Errorf("Impossible to stop ghost containers") + } // 1. Send a SIGTERM if output, err := exec.Command("lxc-kill", "-n", container.Id, "15").CombinedOutput(); err != nil { diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 24194f74ad..ec02939702 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -119,6 +119,9 @@ func (runtime *Runtime) Load(id string) (*Container, error) { if container.Id != id { return container, fmt.Errorf("Container %s is stored at %s", container.Id, id) } + if container.State.Running { + container.State.Ghost = true + } if err := runtime.Register(container); err != nil { return nil, err } diff --git a/components/engine/state.go b/components/engine/state.go index 2ca7130921..f09b289a6a 100644 --- a/components/engine/state.go +++ b/components/engine/state.go @@ -12,11 +12,15 @@ type State struct { ExitCode int StartedAt time.Time l *sync.Mutex + Ghost bool } // String returns a human-readable description of the state func (s *State) String() string { if s.Running { + if s.Ghost { + return fmt.Sprintf("Running ghost") + } return fmt.Sprintf("Up %s", HumanDuration(time.Now().Sub(s.StartedAt))) } return fmt.Sprintf("Exit %d", s.ExitCode) From 2470a0415a95d80490c7f5c203d847f1fe925005 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 11 Apr 2013 18:46:47 +0200 Subject: [PATCH 20/98] fixes some usages Upstream-commit: 3ba44d2d5fac067f7fcd3d5abed83b08c8ddb24d Component: engine --- components/engine/commands.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 9049596b39..1ca8f499c1 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -161,7 +161,7 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout rcli.DockerConn, args .. // 'docker wait': block until a container stops func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "wait", "[OPTIONS] NAME", "Block until a container stops, then print its exit code.") + cmd := rcli.Subcmd(stdout, "wait", "[OPTIONS] CONTAINER [CONTAINER...]", "Block until a container stops, then print its exit code.") if err := cmd.Parse(args); err != nil { return nil } @@ -217,7 +217,7 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "stop", "[OPTIONS] NAME", "Stop a running container") + cmd := rcli.Subcmd(stdout, "stop", "[OPTIONS] CONTAINER [CONTAINER...]", "Stop a running container") if err := cmd.Parse(args); err != nil { return nil } @@ -239,7 +239,7 @@ func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "restart", "[OPTIONS] NAME", "Restart a running container") + cmd := rcli.Subcmd(stdout, "restart", "[OPTIONS] CONTAINER [CONTAINER...]", "Restart a running container") if err := cmd.Parse(args); err != nil { return nil } @@ -261,7 +261,7 @@ func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...str } func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "start", "[OPTIONS] NAME", "Start a stopped container") + cmd := rcli.Subcmd(stdout, "start", "[OPTIONS] CONTAINER [CONTAINER...]", "Start a stopped container") if err := cmd.Parse(args); err != nil { return nil } @@ -340,9 +340,9 @@ func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } -// 'docker rmi NAME' removes all images with the name NAME +// 'docker rmi IMAGE' removes all images with the name IMAGE func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) (err error) { - cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") + cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE [IMAGE...]", "Remove an image") if err := cmd.Parse(args); err != nil { return nil } @@ -385,7 +385,7 @@ func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...str } func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "rm", "[OPTIONS] CONTAINER", "Remove a container") + cmd := rcli.Subcmd(stdout, "rm", "[OPTIONS] CONTAINER [CONTAINER...]", "Remove a container") if err := cmd.Parse(args); err != nil { return nil } From d6545d6e21be1e7ce09bc33c0afd9adbc1fbebfe Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 11 Apr 2013 11:30:35 -0700 Subject: [PATCH 21/98] Contrib post-commit hook for checking gofmt Upstream-commit: 2de953490d6a4ccba9668484fce1f028578c6f77 Component: engine --- components/engine/hack/README.md | 1 + components/engine/hack/fmt-check.hook | 46 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 components/engine/hack/README.md create mode 100644 components/engine/hack/fmt-check.hook diff --git a/components/engine/hack/README.md b/components/engine/hack/README.md new file mode 100644 index 0000000000..06cdd50854 --- /dev/null +++ b/components/engine/hack/README.md @@ -0,0 +1 @@ +This directory contains material helpful for hacking on docker. diff --git a/components/engine/hack/fmt-check.hook b/components/engine/hack/fmt-check.hook new file mode 100644 index 0000000000..cd18a18bcb --- /dev/null +++ b/components/engine/hack/fmt-check.hook @@ -0,0 +1,46 @@ +#!/bin/sh + +# This pre-commit hook will abort if a committed file doesn't pass gofmt. +# By Even Shaw +# http://github.com/edsrzf/gofmt-git-hook + +test_fmt() { + hash gofmt 2>&- || { echo >&2 "gofmt not in PATH."; exit 1; } + IFS=' +' + for file in `git diff --cached --name-only --diff-filter=ACM | grep '\.go$'` + do + output=`git cat-file -p :$file | gofmt -l 2>&1` + if test $? -ne 0 + then + output=`echo "$output" | sed "s,,$file,"` + syntaxerrors="${list}${output}\n" + elif test -n "$output" + then + list="${list}${file}\n" + fi + done + exitcode=0 + if test -n "$syntaxerrors" + then + echo >&2 "gofmt found syntax errors:" + printf "$syntaxerrors" + exitcode=1 + fi + if test -n "$list" + then + echo >&2 "gofmt needs to format these files (run gofmt -w and git add):" + printf "$list" + exitcode=1 + fi + exit $exitcode +} + +case "$1" in + --about ) + echo "Check Go code formatting" + ;; + * ) + test_fmt + ;; +esac From 2f118e2b3a2f1f073b253fa9781c59b248aeb487 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 11 Apr 2013 12:11:41 -0700 Subject: [PATCH 22/98] Simplify the output of 'docker images' by removing the PARENT column Upstream-commit: 79d934bfb0d2e5d30a89a42c1a106d9d3698775b Component: engine --- components/engine/commands.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index d120a8d6a4..d65a19bf3d 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -577,7 +577,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri } w := tabwriter.NewWriter(stdout, 20, 1, 3, ' ', 0) if !*quiet { - fmt.Fprintln(w, "REPOSITORY\tTAG\tID\tCREATED\tPARENT") + fmt.Fprintln(w, "REPOSITORY\tTAG\tID\tCREATED") } var allImages map[string]*Image var err error @@ -606,7 +606,6 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri /* TAG */ tag, /* ID */ TruncateId(id), /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", - /* PARENT */ srv.runtime.repositories.ImageName(image.Parent), } { if idx == 0 { w.Write([]byte(field)) @@ -629,7 +628,6 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri /* TAG */ "", /* ID */ TruncateId(id), /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", - /* PARENT */ srv.runtime.repositories.ImageName(image.Parent), } { if idx == 0 { w.Write([]byte(field)) From 7f83ef43deba3d90a81322ccdef2a94dfac93b92 Mon Sep 17 00:00:00 2001 From: Flavio Castelli Date: Wed, 10 Apr 2013 12:24:15 +0200 Subject: [PATCH 23/98] docker daemon: create file containing PID Ensure the docker daemon creates a file containing its PID under /var/run/docker.pid. The daemon takes care of removing the pid file when it receives either SIGTERM, SIGINT or SIGKILL. The daemon also refuses to start when the pidfile is found. An explanation message is shown to the user when this happens. This change is required to make docker easier to manage by tools like checkproc which rely on this information. Upstream-commit: fb0b375be70e79eaa8349143dceef048db6e0e19 Component: engine --- components/engine/docker/docker.go | 44 ++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 1b1c21990d..a8f4a9e9a6 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -2,12 +2,15 @@ package main import ( "flag" + "fmt" "github.com/dotcloud/docker" "github.com/dotcloud/docker/rcli" "github.com/dotcloud/docker/term" "io" "log" "os" + "os/signal" + "syscall" ) var GIT_COMMIT string @@ -22,6 +25,7 @@ func main() { flDaemon := flag.Bool("d", false, "Daemon mode") flDebug := flag.Bool("D", false, "Debug mode") bridgeName := flag.String("b", "", "Attach containers to a pre-existing network bridge") + pidfile := flag.String("p", "/var/run/docker.pid", "File containing process PID") flag.Parse() if *bridgeName != "" { docker.NetworkBridgeIface = *bridgeName @@ -37,7 +41,7 @@ func main() { flag.Usage() return } - if err := daemon(); err != nil { + if err := daemon(*pidfile); err != nil { log.Fatal(err) } } else { @@ -47,7 +51,43 @@ func main() { } } -func daemon() error { +func createPidFile(pidfile string) error { + if _, err := os.Stat(pidfile); err == nil { + return fmt.Errorf("pid file found, ensure docker is not running or delete %s", pidfile) + } + + file, err := os.Create(pidfile) + if err != nil { + return err + } + + defer file.Close() + + _, err = fmt.Fprintf(file, "%d", os.Getpid()) + return err +} + +func removePidFile(pidfile string) { + if err := os.Remove(pidfile); err != nil { + log.Printf("Error removing %s: %s", pidfile, err) + } +} + +func daemon(pidfile string) error { + if err := createPidFile(pidfile); err != nil { + log.Fatal(err) + } + defer removePidFile(pidfile) + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, os.Kill, os.Signal(syscall.SIGTERM)) + go func() { + sig := <-c + log.Printf("Received signal '%v', exiting\n", sig) + removePidFile(pidfile) + os.Exit(0) + }() + service, err := docker.NewServer() if err != nil { return err From c1ad5969266f3784122e53c3634a2cdf3f7d9e72 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 11 Apr 2013 14:44:39 -0700 Subject: [PATCH 24/98] Added docker-build (formerly github.com/shykes/changer) as a contrib script Upstream-commit: 048f9f4974ac74fac3b03af132c392daaa9f134e Component: engine --- components/engine/contrib/docker-build/README | 68 ++++++++++++ .../engine/contrib/docker-build/docker-build | 104 ++++++++++++++++++ .../contrib/docker-build/example.changefile | 11 ++ 3 files changed, 183 insertions(+) create mode 100644 components/engine/contrib/docker-build/README create mode 100755 components/engine/contrib/docker-build/docker-build create mode 100644 components/engine/contrib/docker-build/example.changefile diff --git a/components/engine/contrib/docker-build/README b/components/engine/contrib/docker-build/README new file mode 100644 index 0000000000..f648753b90 --- /dev/null +++ b/components/engine/contrib/docker-build/README @@ -0,0 +1,68 @@ +# docker-build: build your software with docker + +## Description + +docker-build is a script to build docker images from source. It will be deprecated once the 'build' feature is incorporated into docker itself (See https://github.com/dotcloud/docker/issues/278) + +Author: Solomon Hykes + + +## Install + +docker-builder requires: + +1) A reasonably recent Python setup (tested on 2.7.2). + +2) A running docker daemon at version 0.1.4 or more recent (http://www.docker.io/gettingstarted) + + +## Usage + +First create a valid Changefile, which defines a sequence of changes to apply to a base image. + + $ cat Changefile + # Start build from a know base image + from base:ubuntu-12.10 + # Update ubuntu sources + run echo 'deb http://archive.ubuntu.com/ubuntu quantal main universe multiverse' > /etc/apt/sources.list + run apt-get update + # Install system packages + run DEBIAN_FRONTEND=noninteractive apt-get install -y -q git + run DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl + run DEBIAN_FRONTEND=noninteractive apt-get install -y -q golang + # Insert files from the host (./myscript must be present in the current directory) + copy myscript /usr/local/bin/myscript + + +Run docker-build, and pass the contents of your Changefile as standard input. + + $ IMG=$(./docker-build < Changefile) + +This will take a while: for each line of the changefile, docker-build will: + +1. Create a new container to execute the given command or insert the given file +2. Wait for the container to complete execution +3. Commit the resulting changes as a new image +4. Use the resulting image as the input of the next step + + +If all the steps succeed, the result will be an image containing the combined results of each build step. +You can trace back those build steps by inspecting the image's history: + + $ docker history $IMG + ID CREATED CREATED BY + 1e9e2045de86 A few seconds ago /bin/sh -c cat > /usr/local/bin/myscript; chmod +x /usr/local/bin/git + 77db140aa62a A few seconds ago /bin/sh -c DEBIAN_FRONTEND=noninteractive apt-get install -y -q golang + 77db140aa62a A few seconds ago /bin/sh -c DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl + 77db140aa62a A few seconds ago /bin/sh -c DEBIAN_FRONTEND=noninteractive apt-get install -y -q git + 83e85d155451 A few seconds ago /bin/sh -c apt-get update + bfd53b36d9d3 A few seconds ago /bin/sh -c echo 'deb http://archive.ubuntu.com/ubuntu quantal main universe multiverse' > /etc/apt/sources.list + base 2 weeks ago /bin/bash + 27cf78414709 2 weeks ago + + +Note that your build started from 'base', as instructed by your Changefile. But that base image itself seems to have been built in 2 steps - hence the extra step in the history. + + +You can use this build technique to create any image you want: a database, a web application, or anything else that can be build by a sequence of unix commands - in other words, anything else. + diff --git a/components/engine/contrib/docker-build/docker-build b/components/engine/contrib/docker-build/docker-build new file mode 100755 index 0000000000..934e47ec8f --- /dev/null +++ b/components/engine/contrib/docker-build/docker-build @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +# docker-build is a script to build docker images from source. +# It will be deprecated once the 'build' feature is incorporated into docker itself. +# (See https://github.com/dotcloud/docker/issues/278) +# +# Author: Solomon Hykes + + + +# First create a valid Changefile, which defines a sequence of changes to apply to a base image. +# +# $ cat Changefile +# # Start build from a know base image +# from base:ubuntu-12.10 +# # Update ubuntu sources +# run echo 'deb http://archive.ubuntu.com/ubuntu quantal main universe multiverse' > /etc/apt/sources.list +# run apt-get update +# # Install system packages +# run DEBIAN_FRONTEND=noninteractive apt-get install -y -q git +# run DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl +# run DEBIAN_FRONTEND=noninteractive apt-get install -y -q golang +# # Insert files from the host (./myscript must be present in the current directory) +# copy myscript /usr/local/bin/myscript +# +# +# Run docker-build, and pass the contents of your Changefile as standard input. +# +# $ IMG=$(./docker-build < Changefile) +# +# This will take a while: for each line of the changefile, docker-build will: +# +# 1. Create a new container to execute the given command or insert the given file +# 2. Wait for the container to complete execution +# 3. Commit the resulting changes as a new image +# 4. Use the resulting image as the input of the next step + + +import sys +import subprocess +import json +import hashlib + +def docker(args, stdin=None): + print "# docker " + " ".join(args) + p = subprocess.Popen(["docker"] + list(args), stdin=stdin, stdout=subprocess.PIPE) + return p.stdout + +def image_exists(img): + return docker(["inspect", img]).read().strip() != "" + +def run_and_commit(img_in, cmd, stdin=None): + run_id = docker(["run"] + (["-i", "-a", "stdin"] if stdin else ["-d"]) + [img_in, "/bin/sh", "-c", cmd], stdin=stdin).read().rstrip() + print "---> Waiting for " + run_id + result=int(docker(["wait", run_id]).read().rstrip()) + if result != 0: + print "!!! '{}' return non-zero exit code '{}'. Aborting.".format(cmd, result) + sys.exit(1) + return docker(["commit", run_id]).read().rstrip() + +def insert(base, src, dst): + print "COPY {} to {} in {}".format(src, dst, base) + if dst == "": + raise Exception("Missing destination path") + stdin = file(src) + stdin.seek(0) + return run_and_commit(base, "cat > {0}; chmod +x {0}".format(dst), stdin=stdin) + + +def main(): + base="" + steps = [] + try: + for line in sys.stdin.readlines(): + line = line.strip() + # Skip comments and empty lines + if line == "" or line[0] == "#": + continue + op, param = line.split(" ", 1) + if op == "from": + print "FROM " + param + base = param + steps.append(base) + elif op == "run": + print "RUN " + param + result = run_and_commit(base, param) + steps.append(result) + base = result + print "===> " + base + elif op == "copy": + src, dst = param.split(" ", 1) + result = insert(base, src, dst) + steps.append(result) + base = result + print "===> " + base + else: + print "Skipping uknown op " + op + except: + docker(["rmi"] + steps) + raise + print base + +if __name__ == "__main__": + main() diff --git a/components/engine/contrib/docker-build/example.changefile b/components/engine/contrib/docker-build/example.changefile new file mode 100644 index 0000000000..19261de82b --- /dev/null +++ b/components/engine/contrib/docker-build/example.changefile @@ -0,0 +1,11 @@ +# Start build from a know base image +from base:ubuntu-12.10 +# Update ubuntu sources +run echo 'deb http://archive.ubuntu.com/ubuntu quantal main universe multiverse' > /etc/apt/sources.list +run apt-get update +# Install system packages +run DEBIAN_FRONTEND=noninteractive apt-get install -y -q git +run DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl +run DEBIAN_FRONTEND=noninteractive apt-get install -y -q golang +# Insert files from the host (./myscript must be present in the current directory) +copy myscript /usr/local/bin/myscript From 34381c942fdd4ea3432bd6c04268ea27cdded7c3 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 11 Apr 2013 16:21:19 -0700 Subject: [PATCH 25/98] Add unit test for hanging kill + fix other tests behaviour Upstream-commit: bb22cd492efdc67855bbcf8ab4a669e45ea9ee8e Component: engine --- components/engine/commands_test.go | 17 +++++++++ components/engine/container.go | 6 ++-- components/engine/container_test.go | 56 +++++++++++++++++++++++++++++ components/engine/runtime_test.go | 1 - 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/components/engine/commands_test.go b/components/engine/commands_test.go index 30e2579d20..1be4a2abe0 100644 --- a/components/engine/commands_test.go +++ b/components/engine/commands_test.go @@ -59,6 +59,20 @@ func assertPipe(input, output string, r io.Reader, w io.Writer, count int) error return nil } +func cmdWait(srv *Server, container *Container) error { + stdout, stdoutPipe := io.Pipe() + + go func() { + srv.CmdWait(nil, stdoutPipe, container.Id) + }() + + if _, err := bufio.NewReader(stdout).ReadString('\n'); err != nil { + return err + } + // Cleanup pipes + return closeWrap(stdout, stdoutPipe) +} + // TestRunHostname checks that 'docker run -h' correctly sets a custom hostname func TestRunHostname(t *testing.T) { runtime, err := newTestRuntime() @@ -89,7 +103,9 @@ func TestRunHostname(t *testing.T) { setTimeout(t, "CmdRun timed out", 2*time.Second, func() { <-c + cmdWait(srv, srv.runtime.List()[0]) }) + } func TestRunExit(t *testing.T) { @@ -129,6 +145,7 @@ func TestRunExit(t *testing.T) { // as the process exited, CmdRun must finish and unblock. Wait for it setTimeout(t, "Waiting for CmdRun timed out", 2*time.Second, func() { <-c1 + cmdWait(srv, container) }) // Make sure that the client has been disconnected diff --git a/components/engine/container.go b/components/engine/container.go index 06f39083d8..d644ac7115 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -551,15 +551,15 @@ func (container *Container) kill() error { return nil } - // Sending SIGINT to the process via lxc + // Sending SIGKILL to the process via lxc output, err := exec.Command("lxc-kill", "-n", container.Id, "9").CombinedOutput() if err != nil { - Debugf("error killing container %s (%s, %s)", container.Id, output, err) + log.Printf("error killing container %s (%s, %s)", container.Id, output, err) } // 2. Wait for the process to die, in last resort, try to kill the process directly if err := container.WaitTimeout(10 * time.Second); err != nil { - log.Printf("Container %s failed to exit within 10 seconds of SIGINT - trying direct SIGKILL", container.Id) + log.Printf("Container %s failed to exit within 10 seconds of lxc SIGKILL - trying direct SIGKILL", container.Id) if err := container.cmd.Process.Kill(); err != nil { return err } diff --git a/components/engine/container_test.go b/components/engine/container_test.go index ac47f84bf0..d5f3694c59 100644 --- a/components/engine/container_test.go +++ b/components/engine/container_test.go @@ -324,6 +324,54 @@ func TestOutput(t *testing.T) { } } +func TestKillDifferentUser(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"tail", "-f", "/etc/resolv.conf"}, + User: "daemon", + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + + if container.State.Running { + t.Errorf("Container shouldn't be running") + } + if err := container.Start(); err != nil { + t.Fatal(err) + } + + // Give some time to lxc to spawn the process (setuid might take some time) + container.WaitTimeout(500 * time.Millisecond) + + if !container.State.Running { + t.Errorf("Container should be running") + } + + if err := container.Kill(); err != nil { + t.Fatal(err) + } + + if container.State.Running { + t.Errorf("Container shouldn't be running") + } + container.Wait() + if container.State.Running { + t.Errorf("Container shouldn't be running") + } + // Try stopping twice + if err := container.Kill(); err != nil { + t.Fatal(err) + } +} + func TestKill(t *testing.T) { runtime, err := newTestRuntime() if err != nil { @@ -346,6 +394,10 @@ func TestKill(t *testing.T) { if err := container.Start(); err != nil { t.Fatal(err) } + + // Give some time to lxc to spawn the process + container.WaitTimeout(500 * time.Millisecond) + if !container.State.Running { t.Errorf("Container should be running") } @@ -657,6 +709,10 @@ func TestMultipleContainers(t *testing.T) { t.Fatal(err) } + // Make sure they are running before trying to kill them + container1.WaitTimeout(250 * time.Millisecond) + container2.WaitTimeout(250 * time.Millisecond) + // If we are here, both containers should be running if !container1.State.Running { t.Fatal("Container not running") diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index 9ab8b9b1e7..55d2b54e7e 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -17,7 +17,6 @@ const testLayerPath string = "/var/lib/docker/docker-ut.tar" const unitTestImageName string = "docker-ut" var unitTestStoreBase string -var srv *Server func nuke(runtime *Runtime) error { var wg sync.WaitGroup From 5b2bb885c11502f391f511e78dd7a28238757817 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Thu, 11 Apr 2013 17:39:47 -0700 Subject: [PATCH 26/98] Packaging: Add README documentation Upstream-commit: b14164879b4461b77b630719e67d619acf1a7648 Component: engine --- components/engine/packaging/README.rst | 8 + .../engine/packaging/debian/README.debian | 31 ++ components/engine/packaging/ubuntu/README.md | 339 ------------------ 3 files changed, 39 insertions(+), 339 deletions(-) create mode 100644 components/engine/packaging/README.rst create mode 100644 components/engine/packaging/debian/README.debian delete mode 100644 components/engine/packaging/ubuntu/README.md diff --git a/components/engine/packaging/README.rst b/components/engine/packaging/README.rst new file mode 100644 index 0000000000..7e927ccffe --- /dev/null +++ b/components/engine/packaging/README.rst @@ -0,0 +1,8 @@ +Docker packaging +================ + +This directory has one subdirectory per packaging distribution. +At minimum, each of these subdirectories should contain a +README.$DISTRIBUTION explaining how to create the native +docker package and how to install it. + diff --git a/components/engine/packaging/debian/README.debian b/components/engine/packaging/debian/README.debian new file mode 100644 index 0000000000..83dc42268b --- /dev/null +++ b/components/engine/packaging/debian/README.debian @@ -0,0 +1,31 @@ +Docker on Debian +================ + +Docker has been built and tested on Wheezy. All docker functionality works +out of the box, except for memory limitation as the stock debian kernel +does not support it yet. + + +Building docker package +~~~~~~~~~~~~~~~~~~~~~~~ + +Building Dependencies: debhelper, autotools-dev and golang + + +Assuming you have a wheezy system up and running + +# Download a fresh copy of the docker project +git clone https://github.com/dotcloud/docker.git +cd docker + +# Get building dependencies +sudo apt-get update ; sudo apt-get install -y debhelper autotools-dev golang + +# Make the debian package, with no memory limitation support +(cd packaging/debian; make debian NO_MEMORY_LIMIT=1) + + +Install docker package +~~~~~~~~~~~~~~~~~~~~~~ + +sudo dpkg -i lxc-docker_0.1.4-1_amd64.deb; sudo apt-get install -f -y diff --git a/components/engine/packaging/ubuntu/README.md b/components/engine/packaging/ubuntu/README.md deleted file mode 100644 index c955a1dcf2..0000000000 --- a/components/engine/packaging/ubuntu/README.md +++ /dev/null @@ -1,339 +0,0 @@ -Docker: the Linux container runtime -=================================== - -Docker complements LXC with a high-level API which operates at the process level. It runs unix processes with strong guarantees of isolation and repeatability across servers. - -Docker is a great building block for automating distributed systems: large-scale web deployments, database clusters, continuous deployment systems, private PaaS, service-oriented architectures, etc. - - - -* *Heterogeneous payloads*: any combination of binaries, libraries, configuration files, scripts, virtualenvs, jars, gems, tarballs, you name it. No more juggling between domain-specific tools. Docker can deploy and run them all. - -* *Any server*: docker can run on any x64 machine with a modern linux kernel - whether it's a laptop, a bare metal server or a VM. This makes it perfect for multi-cloud deployments. - -* *Isolation*: docker isolates processes from each other and from the underlying host, using lightweight containers. - -* *Repeatability*: because containers are isolated in their own filesystem, they behave the same regardless of where, when, and alongside what they run. - - -Notable features ------------------ - -* Filesystem isolation: each process container runs in a completely separate root filesystem. - -* Resource isolation: system resources like cpu and memory can be allocated differently to each process container, using cgroups. - -* Network isolation: each process container runs in its own network namespace, with a virtual interface and IP address of its own. - -* Copy-on-write: root filesystems are created using copy-on-write, which makes deployment extremeley fast, memory-cheap and disk-cheap. - -* Logging: the standard streams (stdout/stderr/stdin) of each process container are collected and logged for real-time or batch retrieval. - -* Change management: changes to a container's filesystem can be committed into a new image and re-used to create more containers. No templating or manual configuration required. - -* Interactive shell: docker can allocate a pseudo-tty and attach to the standard input of any container, for example to run a throwaway interactive shell. - - - -Under the hood --------------- - -Under the hood, Docker is built on the following components: - - -* The [cgroup](http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-24-c) and [namespacing](http://blog.dotcloud.com/under-the-hood-linux-kernels-on-dotcloud-part) capabilities of the Linux kernel; - -* [AUFS](http://aufs.sourceforge.net/aufs.html), a powerful union filesystem with copy-on-write capabilities; - -* The [Go](http://golang.org) programming language; - -* [lxc](http://lxc.sourceforge.net/), a set of convenience scripts to simplify the creation of linux containers. - - -Install instructions -================== - -Installing on Ubuntu 12.04 and 12.10 ------------------------------------- - -1. Install dependencies: - - ```bash - sudo apt-get install lxc wget bsdtar curl - sudo apt-get install linux-image-extra-`uname -r` - ``` - - The `linux-image-extra` package is needed on standard Ubuntu EC2 AMIs in order to install the aufs kernel module. - -2. Install the latest docker binary: - - ```bash - wget http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz - tar -xf docker-master.tgz - ``` - -3. Run your first container! - - ```bash - cd docker-master - sudo ./docker run -i -t base /bin/bash - ``` - - Consider adding docker to your `PATH` for simplicity. - -Installing on other Linux distributions ---------------------------------------- - -Right now, the officially supported distributions are: - -* Ubuntu 12.04 (precise LTS) -* Ubuntu 12.10 (quantal) - -Docker probably works on other distributions featuring a recent kernel, the AUFS patch, and up-to-date lxc. However this has not been tested. - -Installing with Vagrant ------------------------ - -Currently, Docker can be installed with Vagrant both on your localhost -with VirtualBox as well as on Amazon EC2. Vagrant 1.1 is required for -EC2, but deploying is as simple as: - -```bash -$ export AWS_ACCESS_KEY_ID=xxx \ - AWS_SECRET_ACCESS_KEY=xxx \ - AWS_KEYPAIR_NAME=xxx \ - AWS_SSH_PRIVKEY=xxx -$ vagrant plugin install vagrant-aws -$ vagrant up --provider=aws -``` - -The environment variables are: - -* `AWS_ACCESS_KEY_ID` - The API key used to make requests to AWS -* `AWS_SECRET_ACCESS_KEY` - The secret key to make AWS API requests -* `AWS_KEYPAIR_NAME` - The name of the keypair used for this EC2 instance -* `AWS_SSH_PRIVKEY` - The path to the private key for the named keypair - -For VirtualBox, you can simply ignore setting any of the environment -variables and omit the `provider` flag. VirtualBox is still supported with -Vagrant <= 1.1: - -```bash -$ vagrant up -``` - - - -Usage examples -============== - -Running an interactive shell ----------------------------- - -```bash -# Download a base image -docker pull base - -# Run an interactive shell in the base image, -# allocate a tty, attach stdin and stdout -docker run -i -t base /bin/bash -``` - - -Starting a long-running worker process --------------------------------------- - -```bash -# Run docker in daemon mode -(docker -d || echo "Docker daemon already running") & - -# Start a very useful long-running process -JOB=$(docker run -d base /bin/sh -c "while true; do echo Hello world; sleep 1; done") - -# Collect the output of the job so far -docker logs $JOB - -# Kill the job -docker kill $JOB -``` - - -Listing all running containers ------------------------------- - -```bash -docker ps -``` - - -Expose a service on a TCP port ------------------------------- - -```bash -# Expose port 4444 of this container, and tell netcat to listen on it -JOB=$(docker run -d -p 4444 base /bin/nc -l -p 4444) - -# Which public port is NATed to my container? -PORT=$(docker port $JOB 4444) - -# Connect to the public port via the host's public address -echo hello world | nc $(hostname) $PORT - -# Verify that the network connection worked -echo "Daemon received: $(docker logs $JOB)" -``` - -Contributing to Docker -====================== - -Want to hack on Docker? Awesome! Here are instructions to get you started. They are probably not perfect, please let us know if anything feels wrong or incomplete. - -Contribution guidelines ------------------------ - -### Pull requests are always welcome - -We are always thrilled to receive pull requests, and do our best to process them as fast as possible. Not sure if that typo is worth a pull request? Do it! We will appreciate it. - -If your pull request is not accepted on the first try, don't be discouraged! If there's a problem with the implementation, hopefully you received feedback on what to improve. - -We're trying very hard to keep Docker lean and focused. We don't want it to do everything for everybody. This means that we might decide against incorporating a new feature. -However, there might be a way to implement that feature *on top of* docker. - -### Discuss your design on the mailing list - -We recommend discussing your plans [on the mailing list](https://groups.google.com/forum/?fromgroups#!forum/docker-club) before starting to code - especially for more ambitious contributions. This gives other contributors a chance to point -you in the right direction, give feedback on your design, and maybe point out if someone else is working on the same thing. - -### Create issues... - -Any significant improvement should be documented as [a github issue](https://github.com/dotcloud/docker/issues) before anybody starts working on it. - -### ...but check for existing issues first! - -Please take a moment to check that an issue doesn't already exist documenting your bug report or improvement proposal. -If it does, it never hurts to add a quick "+1" or "I have this problem too". This will help prioritize the most common problems and requests. - - -### Write tests - -Golang has a great testing suite built in: use it! Take a look at existing tests for inspiration. - - - -Setting up a dev environment ----------------------------- - -Instructions that have been verified to work on Ubuntu 12.10, - -```bash -sudo apt-get -y install lxc wget bsdtar curl golang git - -export GOPATH=~/go/ -export PATH=$GOPATH/bin:$PATH - -mkdir -p $GOPATH/src/github.com/dotcloud -cd $GOPATH/src/github.com/dotcloud -git clone git@github.com:dotcloud/docker.git -cd docker - -go get -v github.com/dotcloud/docker/... -go install -v github.com/dotcloud/docker/... -``` - -Then run the docker daemon, - -```bash -sudo $GOPATH/bin/docker -d -``` - -Run the `go install` command (above) to recompile docker. - - -What is a Standard Container? -============================= - -Docker defines a unit of software delivery called a Standard Container. The goal of a Standard Container is to encapsulate a software component and all its dependencies in -a format that is self-describing and portable, so that any compliant runtime can run it without extra dependencies, regardless of the underlying machine and the contents of the container. - -The spec for Standard Containers is currently a work in progress, but it is very straightforward. It mostly defines 1) an image format, 2) a set of standard operations, and 3) an execution environment. - -A great analogy for this is the shipping container. Just like Standard Containers are a fundamental unit of software delivery, shipping containers (http://bricks.argz.com/ins/7823-1/12) are a fundamental unit of physical delivery. - -### 1. STANDARD OPERATIONS - -Just like shipping containers, Standard Containers define a set of STANDARD OPERATIONS. Shipping containers can be lifted, stacked, locked, loaded, unloaded and labelled. Similarly, standard containers can be started, stopped, copied, snapshotted, downloaded, uploaded and tagged. - - -### 2. CONTENT-AGNOSTIC - -Just like shipping containers, Standard Containers are CONTENT-AGNOSTIC: all standard operations have the same effect regardless of the contents. A shipping container will be stacked in exactly the same way whether it contains Vietnamese powder coffee or spare Maserati parts. Similarly, Standard Containers are started or uploaded in the same way whether they contain a postgres database, a php application with its dependencies and application server, or Java build artifacts. - - -### 3. INFRASTRUCTURE-AGNOSTIC - -Both types of containers are INFRASTRUCTURE-AGNOSTIC: they can be transported to thousands of facilities around the world, and manipulated by a wide variety of equipment. A shipping container can be packed in a factory in Ukraine, transported by truck to the nearest routing center, stacked onto a train, loaded into a German boat by an Australian-built crane, stored in a warehouse at a US facility, etc. Similarly, a standard container can be bundled on my laptop, uploaded to S3, downloaded, run and snapshotted by a build server at Equinix in Virginia, uploaded to 10 staging servers in a home-made Openstack cluster, then sent to 30 production instances across 3 EC2 regions. - - -### 4. DESIGNED FOR AUTOMATION - -Because they offer the same standard operations regardless of content and infrastructure, Standard Containers, just like their physical counterpart, are extremely well-suited for automation. In fact, you could say automation is their secret weapon. - -Many things that once required time-consuming and error-prone human effort can now be programmed. Before shipping containers, a bag of powder coffee was hauled, dragged, dropped, rolled and stacked by 10 different people in 10 different locations by the time it reached its destination. 1 out of 50 disappeared. 1 out of 20 was damaged. The process was slow, inefficient and cost a fortune - and was entirely different depending on the facility and the type of goods. - -Similarly, before Standard Containers, by the time a software component ran in production, it had been individually built, configured, bundled, documented, patched, vendored, templated, tweaked and instrumented by 10 different people on 10 different computers. Builds failed, libraries conflicted, mirrors crashed, post-it notes were lost, logs were misplaced, cluster updates were half-broken. The process was slow, inefficient and cost a fortune - and was entirely different depending on the language and infrastructure provider. - - -### 5. INDUSTRIAL-GRADE DELIVERY - -There are 17 million shipping containers in existence, packed with every physical good imaginable. Every single one of them can be loaded on the same boats, by the same cranes, in the same facilities, and sent anywhere in the World with incredible efficiency. It is embarrassing to think that a 30 ton shipment of coffee can safely travel half-way across the World in *less time* than it takes a software team to deliver its code from one datacenter to another sitting 10 miles away. - -With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. - - - - -Standard Container Specification --------------------------------- - -(TODO) - -### Image format - - -### Standard operations - -* Copy -* Run -* Stop -* Wait -* Commit -* Attach standard streams -* List filesystem changes -* ... - -### Execution environment - -#### Root filesystem - -#### Environment variables - -#### Process arguments - -#### Networking - -#### Process namespacing - -#### Resource limits - -#### Process monitoring - -#### Logging - -#### Signals - -#### Pseudo-terminal allocation - -#### Security - - From f8333f7048494ea6ddc415271677b579f3cb5a94 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Thu, 11 Apr 2013 20:17:41 -0700 Subject: [PATCH 27/98] Packaging, Debian: Add maintainer documentation Upstream-commit: f226842aa1f9148449558c90de7cdfb61105df28 Component: engine --- .../engine/packaging/debian/maintainer.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 components/engine/packaging/debian/maintainer.rst diff --git a/components/engine/packaging/debian/maintainer.rst b/components/engine/packaging/debian/maintainer.rst new file mode 100644 index 0000000000..111d4fcc3e --- /dev/null +++ b/components/engine/packaging/debian/maintainer.rst @@ -0,0 +1,16 @@ +Maintainer duty +=============== + +The Debian project specifies the role of a 'maintainer' which is the person +making the Debian package of the program. This role requires an 'sponsor' to +upload the package. As a maintainer you should follow the guide +http://www.debian.org/doc/manuals/maint-guide . Your sponsor will be there +helping you succeed. + +The most relevant information to update is the changelog file: +Each new release should create a new first paragraph with new release version, +changes, and the maintainer information. + +After this is done, follow README.debian to generate the actual source +packages and talk with your sponsor to upload them into the official Debian +package archive. From b5ccb3f71cfcfd7296817c7493a74b4140129c73 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 12 Apr 2013 12:26:31 +0200 Subject: [PATCH 28/98] removed not needed [OPTIONS] and remove poor messages like 'Not enough arguments' Upstream-commit: 8987bd5832d8162805d34b89c567fcf2e933f746 Component: engine --- components/engine/commands.go | 41 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 1ca8f499c1..9eb5b865c3 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -161,7 +161,7 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout rcli.DockerConn, args .. // 'docker wait': block until a container stops func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "wait", "[OPTIONS] CONTAINER [CONTAINER...]", "Block until a container stops, then print its exit code.") + cmd := rcli.Subcmd(stdout, "wait", "CONTAINER [CONTAINER...]", "Block until a container stops, then print its exit code.") if err := cmd.Parse(args); err != nil { return nil } @@ -217,7 +217,7 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "stop", "[OPTIONS] CONTAINER [CONTAINER...]", "Stop a running container") + cmd := rcli.Subcmd(stdout, "stop", "CONTAINER [CONTAINER...]", "Stop a running container") if err := cmd.Parse(args); err != nil { return nil } @@ -239,7 +239,7 @@ func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "restart", "[OPTIONS] CONTAINER [CONTAINER...]", "Restart a running container") + cmd := rcli.Subcmd(stdout, "restart", "CONTAINER [CONTAINER...]", "Restart a running container") if err := cmd.Parse(args); err != nil { return nil } @@ -261,7 +261,7 @@ func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...str } func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "start", "[OPTIONS] CONTAINER [CONTAINER...]", "Start a stopped container") + cmd := rcli.Subcmd(stdout, "start", "CONTAINER [CONTAINER...]", "Start a stopped container") if err := cmd.Parse(args); err != nil { return nil } @@ -283,7 +283,7 @@ func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...strin } func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") + cmd := rcli.Subcmd(stdout, "inspect", "CONTAINER", "Return low-level information on a container") if err := cmd.Parse(args); err != nil { return nil } @@ -318,7 +318,7 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str } func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "port", "[OPTIONS] CONTAINER PRIVATE_PORT", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT") + cmd := rcli.Subcmd(stdout, "port", "CONTAINER PRIVATE_PORT", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT") if err := cmd.Parse(args); err != nil { return nil } @@ -342,7 +342,7 @@ func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string // 'docker rmi IMAGE' removes all images with the name IMAGE func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) (err error) { - cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE [IMAGE...]", "Remove an image") + cmd := rcli.Subcmd(stdout, "rmimage", "IMAGE [IMAGE...]", "Remove an image") if err := cmd.Parse(args); err != nil { return nil } @@ -359,7 +359,7 @@ func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) } func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "history", "[OPTIONS] IMAGE", "Show the history of an image") + cmd := rcli.Subcmd(stdout, "history", "IMAGE", "Show the history of an image") if err := cmd.Parse(args); err != nil { return nil } @@ -385,7 +385,7 @@ func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...str } func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "rm", "[OPTIONS] CONTAINER [CONTAINER...]", "Remove a container") + cmd := rcli.Subcmd(stdout, "rm", "CONTAINER [CONTAINER...]", "Remove a container") if err := cmd.Parse(args); err != nil { return nil } @@ -407,10 +407,14 @@ func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) // 'docker kill NAME' kills a running container func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "kill", "[OPTIONS] CONTAINER [CONTAINER...]", "Kill a running container") + cmd := rcli.Subcmd(stdout, "kill", "CONTAINER [CONTAINER...]", "Kill a running container") if err := cmd.Parse(args); err != nil { return nil } + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } for _, name := range cmd.Args() { container := srv.runtime.Get(name) if container == nil { @@ -425,17 +429,19 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdImport(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { stdout.Flush() - cmd := rcli.Subcmd(stdout, "import", "[OPTIONS] URL|- [REPOSITORY [TAG]]", "Create a new filesystem image from the contents of a tarball") + cmd := rcli.Subcmd(stdout, "import", "URL|- [REPOSITORY [TAG]]", "Create a new filesystem image from the contents of a tarball") var archive io.Reader var resp *http.Response if err := cmd.Parse(args); err != nil { return nil } + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } src := cmd.Arg(0) - if src == "" { - return fmt.Errorf("Not enough arguments") - } else if src == "-" { + if src == "-" { archive = stdin } else { u, err := url.Parse(src) @@ -737,13 +743,14 @@ func (srv *Server) CmdExport(stdin io.ReadCloser, stdout io.Writer, args ...stri func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, - "diff", "CONTAINER [OPTIONS]", + "diff", "CONTAINER", "Inspect changes on a container's filesystem") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() < 1 { - return fmt.Errorf("Not enough arguments") + cmd.Usage() + return nil } if container := srv.runtime.Get(cmd.Arg(0)); container == nil { return fmt.Errorf("No such container") @@ -760,7 +767,7 @@ func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "logs", "[OPTIONS] CONTAINER", "Fetch the logs of a container") + cmd := rcli.Subcmd(stdout, "logs", "CONTAINER", "Fetch the logs of a container") if err := cmd.Parse(args); err != nil { return nil } From ebb6835d2050fb2e454b6dfa70e58ff564bc3104 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 12 Apr 2013 07:37:24 -0700 Subject: [PATCH 29/98] Go fmt Upstream-commit: 22893429ed29dbb7a01eed7a90a4d1fbecfd5d90 Component: engine --- components/engine/docker/docker.go | 24 ++++++++++++------------ components/engine/network.go | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index a8f4a9e9a6..76a0467be8 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -74,19 +74,19 @@ func removePidFile(pidfile string) { } func daemon(pidfile string) error { - if err := createPidFile(pidfile); err != nil { - log.Fatal(err) - } - defer removePidFile(pidfile) + if err := createPidFile(pidfile); err != nil { + log.Fatal(err) + } + defer removePidFile(pidfile) - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, os.Kill, os.Signal(syscall.SIGTERM)) - go func() { - sig := <-c - log.Printf("Received signal '%v', exiting\n", sig) - removePidFile(pidfile) - os.Exit(0) - }() + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, os.Kill, os.Signal(syscall.SIGTERM)) + go func() { + sig := <-c + log.Printf("Received signal '%v', exiting\n", sig) + removePidFile(pidfile) + os.Exit(0) + }() service, err := docker.NewServer() if err != nil { diff --git a/components/engine/network.go b/components/engine/network.go index 9164c1d72e..706c31fa4e 100644 --- a/components/engine/network.go +++ b/components/engine/network.go @@ -293,7 +293,7 @@ func (alloc *PortAllocator) Acquire(port int) (int, error) { func newPortAllocator() (*PortAllocator, error) { allocator := &PortAllocator{ - inUse: make(map[int]struct{}), + inUse: make(map[int]struct{}), fountain: make(chan int), } go allocator.runFountain() From 9b95a8b869adfe9c52ac897c6fbbeb3120c67742 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 12 Apr 2013 10:44:50 -0400 Subject: [PATCH 30/98] Update AUTHORS Upstream-commit: 343ed6b53f225e35bb5464efb556f555a31056a1 Component: engine --- components/engine/.mailmap | 3 +++ components/engine/AUTHORS | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/components/engine/.mailmap b/components/engine/.mailmap index 2570683f85..83c18fa29c 100644 --- a/components/engine/.mailmap +++ b/components/engine/.mailmap @@ -14,3 +14,6 @@ Joffrey F Tim Terhorst Andy Smith + + + diff --git a/components/engine/AUTHORS b/components/engine/AUTHORS index fefd748422..e8979aac6b 100644 --- a/components/engine/AUTHORS +++ b/components/engine/AUTHORS @@ -10,6 +10,8 @@ Daniel Robinson Dominik Honnef Don Spaulding ezbercih +Flavio Castelli +Francisco Souza Frederick F. Kautz IV Guillaume J. Charmes Hunter Blanks @@ -23,10 +25,13 @@ Jérôme Petazzoni Ken Cochrane Kevin J. Lynagh Louis Opter +Maxim Treskin Mikhail Sobolev Nelson Chen Niall O'Higgins +Paul Hammond Piotr Bogdan +Robert Obryk Sam Alba Shawn Siefkas Silas Sewell @@ -35,4 +40,6 @@ Sridhar Ratnakumar Thatcher Peskens Tim Terhorst Troy Howard +unclejack +Victor Vieux Vivek Agarwal From c74dd47b04a8f60f29d3133aae5146b4d2a2b305 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 12 Apr 2013 10:39:08 -0700 Subject: [PATCH 31/98] Fixed wording of ghost-related messages Upstream-commit: b1fbebb4a3611d340176ad637a22d2250085a077 Component: engine --- components/engine/container.go | 4 ++-- components/engine/state.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index fb95be0ca3..e0b09935b5 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -577,7 +577,7 @@ func (container *Container) Kill() error { return nil } if container.State.Ghost { - return fmt.Errorf("Impossible to kill ghost containers") + return fmt.Errorf("Can't kill ghost container") } return container.kill() } @@ -589,7 +589,7 @@ func (container *Container) Stop() error { return nil } if container.State.Ghost { - return fmt.Errorf("Impossible to stop ghost containers") + return fmt.Errorf("Can't stop ghot container") } // 1. Send a SIGTERM diff --git a/components/engine/state.go b/components/engine/state.go index f09b289a6a..f51a06b01a 100644 --- a/components/engine/state.go +++ b/components/engine/state.go @@ -19,7 +19,7 @@ type State struct { func (s *State) String() string { if s.Running { if s.Ghost { - return fmt.Sprintf("Running ghost") + return fmt.Sprintf("Ghost") } return fmt.Sprintf("Up %s", HumanDuration(time.Now().Sub(s.StartedAt))) } From 1875a00c380cbec448f897b000919c361f5b0daa Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 10 Apr 2013 19:02:23 -0700 Subject: [PATCH 32/98] Allow use to set his own dns via -dns Upstream-commit: 7673afc8438a9e46a1d11aff0f8c0f7eca169809 Component: engine --- components/engine/container.go | 5 +++++ components/engine/runtime.go | 23 +++++++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index a930cbd3e3..848c2ae215 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -62,6 +62,7 @@ type Config struct { StdinOnce bool // If true, close stdin after the 1 attached client disconnects. Env []string Cmd []string + Dns []string Image string // Name of the image as it was passed by the operator (eg. could be symbolic) } @@ -86,6 +87,9 @@ func ParseRun(args []string, stdout io.Writer) (*Config, error) { var flEnv ListOpts cmd.Var(&flEnv, "e", "Set environment variables") + var flDns ListOpts + cmd.Var(&flDns, "dns", "Set custom dns servers") + if err := cmd.Parse(args); err != nil { return nil, err } @@ -123,6 +127,7 @@ func ParseRun(args []string, stdout io.Writer) (*Config, error) { AttachStderr: flAttach.Get("stderr"), Env: flEnv, Cmd: runCmd, + Dns: flDns, Image: image, } // When allocating stdin in attached mode, close stdin at client disconnect diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 2dd419868f..e66b32f31a 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -83,8 +83,6 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { config.Hostname = id[:12] } - resolvConfPath := "/etc/resolv.conf" - container := &Container{ // FIXME: we should generate the ID here instead of receiving it as an argument Id: id, @@ -95,8 +93,7 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { Image: img.Id, // Always use the resolved image id NetworkSettings: &NetworkSettings{}, // FIXME: do we need to store this in the container? - SysInitPath: sysInitPath, - ResolvConfPath: resolvConfPath, + SysInitPath: sysInitPath, } container.root = runtime.containerRoot(container.Id) // Step 1: create the container directory. @@ -104,6 +101,24 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { if err := os.Mkdir(container.root, 0700); err != nil { return nil, err } + + // If custom dns exists, then create a resolv.conf for the container + if len(config.Dns) > 0 { + container.ResolvConfPath = path.Join(container.root, "resolv.conf") + f, err := os.Create(container.ResolvConfPath) + if err != nil { + return nil, err + } + defer f.Close() + for _, dns := range config.Dns { + if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil { + return nil, err + } + } + } else { + container.ResolvConfPath = "/etc/resolv.conf" + } + // Step 2: save the container json if err := container.ToDisk(); err != nil { return nil, err From 3fcc60d0f78695e54648f0efcb100ab5082fb874 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 11 Apr 2013 12:58:23 -0700 Subject: [PATCH 33/98] Allow to disable memory limit at compilation time Upstream-commit: 1967c8342aef1a81eb33a41d673bb8e702daa5df Component: engine --- components/engine/Makefile | 5 ++++- components/engine/commands.go | 8 +++++++- components/engine/container.go | 11 +++++++++++ components/engine/docker/docker.go | 9 ++++++++- components/engine/runtime_test.go | 2 ++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index a6eb613830..c89f0f33b0 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -13,7 +13,10 @@ endif GIT_COMMIT = $(shell git rev-parse --short HEAD) GIT_STATUS = $(shell test -n "`git status --porcelain`" && echo "+CHANGES") -BUILD_OPTIONS = -ldflags "-X main.GIT_COMMIT $(GIT_COMMIT)$(GIT_STATUS)" +NO_MEMORY_LIMIT ?= 0 +export NO_MEMORY_LIMIT + +BUILD_OPTIONS = -ldflags "-X main.GIT_COMMIT $(GIT_COMMIT)$(GIT_STATUS) -X main.NO_MEMORY_LIMIT $(NO_MEMORY_LIMIT)" SRC_DIR := $(GOPATH)/src diff --git a/components/engine/commands.go b/components/engine/commands.go index dc2bb20ac6..0229b1dee3 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -20,7 +20,10 @@ import ( const VERSION = "0.1.4" -var GIT_COMMIT string +var ( + GIT_COMMIT string + NO_MEMORY_LIMIT bool +) func (srv *Server) Name() string { return "docker" @@ -183,6 +186,9 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdVersion(stdin io.ReadCloser, stdout io.Writer, args ...string) error { fmt.Fprintf(stdout, "Version:%s\n", VERSION) fmt.Fprintf(stdout, "Git Commit:%s\n", GIT_COMMIT) + if NO_MEMORY_LIMIT { + fmt.Fprintf(stdout, "Memory limit disabled\n") + } return nil } diff --git a/components/engine/container.go b/components/engine/container.go index e0b09935b5..53f29917dd 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -79,6 +79,11 @@ func ParseRun(args []string, stdout io.Writer) (*Config, error) { flTty := cmd.Bool("t", false, "Allocate a pseudo-tty") flMemory := cmd.Int64("m", 0, "Memory limit (in bytes)") + if *flMemory > 0 && NO_MEMORY_LIMIT { + fmt.Fprintf(stdout, "WARNING: This version of docker has been compiled without memory limit support. Discarding -m.") + *flMemory = 0 + } + var flPorts ListOpts cmd.Var(&flPorts, "p", "Expose a container's port to the host (use 'docker port' to see the actual mapping)") @@ -355,6 +360,12 @@ func (container *Container) Start() error { if err := container.allocateNetwork(); err != nil { return err } + + if container.Config.Memory > 0 && NO_MEMORY_LIMIT { + log.Printf("WARNING: This version of docker has been compiled without memory limit support. Discarding the limit.") + container.Config.Memory = 0 + } + if err := container.generateLXCConfig(); err != nil { return err } diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 76a0467be8..bbe0a0e7a8 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -13,7 +13,10 @@ import ( "syscall" ) -var GIT_COMMIT string +var ( + GIT_COMMIT string + NO_MEMORY_LIMIT string +) func main() { if docker.SelfPath() == "/sbin/init" { @@ -36,11 +39,15 @@ func main() { os.Setenv("DEBUG", "1") } docker.GIT_COMMIT = GIT_COMMIT + docker.NO_MEMORY_LIMIT = NO_MEMORY_LIMIT == "1" if *flDaemon { if flag.NArg() != 0 { flag.Usage() return } + if NO_MEMORY_LIMIT == "1" { + log.Printf("WARNING: This version of docker has been compiled without memory limit support.") + } if err := daemon(*pidfile); err != nil { log.Fatal(err) } diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index 55d2b54e7e..b990deaf38 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -48,6 +48,8 @@ func layerArchive(tarfile string) (io.Reader, error) { } func init() { + NO_MEMORY_LIMIT = os.Getenv("NO_MEMORY_LIMIT") == "1" + // Hack to run sys init during unit testing if SelfPath() == "/sbin/init" { SysInit() From 0a3b5be4e06eb08ddf2f214d8e87993993b2e42f Mon Sep 17 00:00:00 2001 From: Shawn Siefkas Date: Fri, 12 Apr 2013 13:20:03 -0500 Subject: [PATCH 34/98] Issue #405: Documentation for Arch Linux Install Upstream-commit: 45809e9a055defd073bde794026aa611333b7023 Component: engine --- .../docs/sources/installation/archlinux.rst | 43 +++++++++++++++++++ .../docs/sources/installation/index.rst | 1 + 2 files changed, 44 insertions(+) create mode 100644 components/engine/docs/sources/installation/archlinux.rst diff --git a/components/engine/docs/sources/installation/archlinux.rst b/components/engine/docs/sources/installation/archlinux.rst new file mode 100644 index 0000000000..c6de247d6c --- /dev/null +++ b/components/engine/docs/sources/installation/archlinux.rst @@ -0,0 +1,43 @@ +.. _arch_linux: + +Arch Linux +========== + +Installing on Arch Linux is not officially supported but can be handled via +either of the following AUR packages: + +* `dotcloud-docker `_ +* `dotcloud-docker-git `_ + +The dotcloud-docker package will install the latest tagged version of docker. +The dotcloud-docker-git package will build from the current master branch. + +Dependencies +------------ + +Docker depends on several packages which will be installed automatically with +either AUR package. + +* aufs3 +* bridge-utils +* go +* iproute2 +* linux-aufs_friendly + +Installation +------------ + +The instructions here assume **yaourt** is installed. See +`Arch User Repository `_ +for information on building and installing packages from the AUR if you have not +done so before. + +Keep in mind that if **linux-aufs_friendly** is not already installed that a +new kernel will be compiled and this can take quite a while. + +:: + + yaourt -S dotcloud-docker-git + +Prior to starting docker modify your bootloader to use the +**linux-aufs_friendly** kernel and reboot your system. diff --git a/components/engine/docs/sources/installation/index.rst b/components/engine/docs/sources/installation/index.rst index b02e9c83ac..b2718c72b1 100644 --- a/components/engine/docs/sources/installation/index.rst +++ b/components/engine/docs/sources/installation/index.rst @@ -13,6 +13,7 @@ Contents: :maxdepth: 1 ubuntulinux + archlinux macos windows amazon From 1f46dfdd512eb03ff536e3aedf94b8d8c9bc90bd Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sat, 13 Apr 2013 15:03:24 -0700 Subject: [PATCH 35/98] Add \r to error message in run raw mode Upstream-commit: c2c72bcfd796a43b9be33705ec695455c9823489 Component: engine --- components/engine/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index dc2bb20ac6..8cd149b3cb 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -925,7 +925,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...s if err != nil { // If container not found, try to pull it if srv.runtime.graph.IsNotExist(err) { - fmt.Fprintf(stdout, "Image %s not found, trying to pull it from registry.\n", config.Image) + fmt.Fprintf(stdout, "Image %s not found, trying to pull it from registry.\r\n", config.Image) if err = srv.CmdPull(stdin, stdout, config.Image); err != nil { return err } From a2aceab579730640b41da015326a13667ec713de Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sun, 14 Apr 2013 15:13:32 -0700 Subject: [PATCH 36/98] Add a script to help reproduce #407 Upstream-commit: 1ec6c223c9e3a22e9cf182d1d4d69f16a9d083f4 Component: engine --- components/engine/contrib/crashTest.go | 94 ++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 components/engine/contrib/crashTest.go diff --git a/components/engine/contrib/crashTest.go b/components/engine/contrib/crashTest.go new file mode 100644 index 0000000000..34749b52d3 --- /dev/null +++ b/components/engine/contrib/crashTest.go @@ -0,0 +1,94 @@ +package main + +import ( + "io" + "log" + "os" + "os/exec" + "time" +) + +const DOCKER_PATH = "/home/creack/dotcloud/docker/docker/docker" + +func runDaemon() (*exec.Cmd, error) { + cmd := exec.Command(DOCKER_PATH, "-d") + outPipe, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + errPipe, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + go func() { + io.Copy(os.Stdout, outPipe) + }() + go func() { + io.Copy(os.Stderr, errPipe) + }() + return cmd, nil +} + +func crashTest() error { + if err := exec.Command("/bin/bash", "-c", "while true; do true; done").Start(); err != nil { + return err + } + + for { + daemon, err := runDaemon() + if err != nil { + return err + } + time.Sleep(5000 * time.Millisecond) + go func() error { + for i := 0; i < 100; i++ { + go func() error { + cmd := exec.Command(DOCKER_PATH, "run", "base", "echo", "hello", "world") + log.Printf("%d", i) + outPipe, err := cmd.StdoutPipe() + if err != nil { + return err + } + inPipe, err := cmd.StdinPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + go func() { + io.Copy(os.Stdout, outPipe) + }() + // Expecting error, do not check + inPipe.Write([]byte("hello world!!!!!\n")) + go inPipe.Write([]byte("hello world!!!!!\n")) + go inPipe.Write([]byte("hello world!!!!!\n")) + inPipe.Close() + + if err := cmd.Wait(); err != nil { + return err + } + outPipe.Close() + return nil + }() + time.Sleep(250 * time.Millisecond) + } + return nil + }() + + time.Sleep(20 * time.Second) + if err := daemon.Process.Kill(); err != nil { + return err + } + } + return nil +} + +func main() { + if err := crashTest(); err != nil { + log.Println(err) + } +} From fd2c37bc5eb26fae12ac339411b0fcb2d9b65bf6 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 15 Apr 2013 12:05:46 +0200 Subject: [PATCH 37/98] try to load aufs module, disply error on failure Upstream-commit: ac49a797b469bda89384dc1cd31d349761340693 Component: engine --- components/engine/image.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/components/engine/image.go b/components/engine/image.go index 83bf9481ae..f9595bd976 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "io/ioutil" + "log" "os" + "os/exec" "path" "strings" "time" @@ -92,7 +94,19 @@ func MountAUFS(ro []string, rw string, target string) error { roBranches += fmt.Sprintf("%v=ro:", layer) } branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) - return mount("none", target, "aufs", 0, branches) + + //if error, try to load aufs kernel module + if err := mount("none", target, "aufs", 0, branches); err != nil { + log.Printf("Kernel does not support AUFS, trying to load the AUFS module with modprobe...") + if err := exec.Command("modprobe", "aufs").Run(); err != nil { + return fmt.Errorf("Unable to load the AUFS module") + } + log.Printf("...module loaded.") + if err := mount("none", target, "aufs", 0, branches); err != nil { + return fmt.Errorf("Unable to mount using aufs") + } + } + return nil } func (image *Image) Mount(root, rw string) error { From 34287914c5e41cea4aeeb9ada1494583a9015213 Mon Sep 17 00:00:00 2001 From: "Kevin J. Lynagh" Date: Mon, 15 Apr 2013 08:49:48 -0700 Subject: [PATCH 38/98] install.sh script's dockerd.conf should set docker daemon environment's LANG to en_US.UTF-8. See #355. Upstream-commit: 468fb901172f2534972cc44cd4459f14b3745a98 Component: engine --- components/engine/contrib/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/contrib/install.sh b/components/engine/contrib/install.sh index b0a998332e..d7c6e66466 100755 --- a/components/engine/contrib/install.sh +++ b/components/engine/contrib/install.sh @@ -45,7 +45,7 @@ then echo "Upstart script already exists." else echo "Creating /etc/init/dockerd.conf..." - echo "exec /usr/local/bin/docker -d" > /etc/init/dockerd.conf + echo "exec env LANG=\"en_US.UTF-8\" /usr/local/bin/docker -d" > /etc/init/dockerd.conf fi echo "Starting dockerd..." From b71d62255034be946637aad9d91937107a1fe769 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 15 Apr 2013 11:44:11 -0700 Subject: [PATCH 39/98] Disabled standalone mode (fixes #364) Upstream-commit: 2a47df020246cc08a0b72cf355c9d803404245b3 Component: engine --- components/engine/docker/docker.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index bbe0a0e7a8..83c47c6f1e 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -138,15 +138,7 @@ func runCommand(args []string) error { } } } else { - service, err := docker.NewServer() - if err != nil { - return err - } - dockerConn := rcli.NewDockerLocalConn(os.Stdout) - defer dockerConn.Close() - if err := rcli.LocalCall(service, os.Stdin, dockerConn, args...); err != nil { - return err - } + return fmt.Errorf("Can't connect to docker daemon. Is 'docker -d' running on this host?") } return nil } From 5c53eceb79cf3966b4a79b282e2d960a93b9893e Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 16 Apr 2013 12:10:16 -0700 Subject: [PATCH 40/98] Remove unneeded dependencies from README Upstream-commit: fc72a809c1c60569fb1d70a95f5296feaa5220f8 Component: engine --- components/engine/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/README.md b/components/engine/README.md index c186d9a063..4ba9222f85 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -83,7 +83,7 @@ Installing on Ubuntu 12.04 and 12.10 1. Install dependencies: ```bash - sudo apt-get install lxc wget bsdtar curl + sudo apt-get install lxc bsdtar sudo apt-get install linux-image-extra-`uname -r` ``` From a7d3c2e8bf87fe630619d6b7ec81208b08a7fdbc Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 16 Apr 2013 00:25:55 -0700 Subject: [PATCH 41/98] Manually pass the env to docker-init instead of relying on lxc to pass it Upstream-commit: 7b0e96f1f4639f869703ed32f885cec9b666b127 Component: engine --- components/engine/container.go | 27 ++++++++++++++------------- components/engine/sysinit.go | 8 +++++--- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index 74706a4079..9f175e42aa 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -390,21 +390,26 @@ func (container *Container) Start() error { params = append(params, "-u", container.Config.User) } + if container.Config.Tty { + params = append(params, "-e", "TERM=xterm") + } + + // Setup environment + params = append(params, + "-e", "HOME=/", + "-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + ) + + for _, elem := range container.Config.Env { + params = append(params, "-e", elem) + } + // Program params = append(params, "--", container.Path) params = append(params, container.Args...) container.cmd = exec.Command("lxc-start", params...) - // Setup environment - container.cmd.Env = append( - []string{ - "HOME=/", - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - }, - container.Config.Env..., - ) - // Setup logging of stdout and stderr to disk if err := container.runtime.LogToDisk(container.stdout, container.logPath("stdout")); err != nil { return err @@ -415,10 +420,6 @@ func (container *Container) Start() error { var err error if container.Config.Tty { - container.cmd.Env = append( - []string{"TERM=xterm"}, - container.cmd.Env..., - ) err = container.startPty() } else { err = container.start() diff --git a/components/engine/sysinit.go b/components/engine/sysinit.go index 2c1106db10..4b2d6c3032 100644 --- a/components/engine/sysinit.go +++ b/components/engine/sysinit.go @@ -53,8 +53,7 @@ func changeUser(u string) { } // Clear environment pollution introduced by lxc-start -func cleanupEnv() { - env := os.Environ() +func cleanupEnv(env ListOpts) { os.Clearenv() for _, kv := range env { parts := strings.SplitN(kv, "=", 2) @@ -91,10 +90,13 @@ func SysInit() { var u = flag.String("u", "", "username or uid") var gw = flag.String("g", "", "gateway address") + var flEnv ListOpts + flag.Var(&flEnv, "e", "Set environment variables") + flag.Parse() + cleanupEnv(flEnv) setupNetworking(*gw) - cleanupEnv() changeUser(*u) executeProgram(flag.Arg(0), flag.Args()) } From fd84a2d5c055a1ba7be7ac43c29883c2d8f188e1 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 16 Apr 2013 15:20:04 -0700 Subject: [PATCH 42/98] improve the crashTest script Upstream-commit: c4cd224d901ece4d9a2f15d10a80998f2b970c07 Component: engine --- components/engine/contrib/crashTest.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/components/engine/contrib/crashTest.go b/components/engine/contrib/crashTest.go index 34749b52d3..fa9cda6056 100644 --- a/components/engine/contrib/crashTest.go +++ b/components/engine/contrib/crashTest.go @@ -11,6 +11,7 @@ import ( const DOCKER_PATH = "/home/creack/dotcloud/docker/docker/docker" func runDaemon() (*exec.Cmd, error) { + os.Remove("/var/run/docker.pid") cmd := exec.Command(DOCKER_PATH, "-d") outPipe, err := cmd.StdoutPipe() if err != nil { @@ -42,10 +43,12 @@ func crashTest() error { if err != nil { return err } - time.Sleep(5000 * time.Millisecond) + // time.Sleep(5000 * time.Millisecond) + var stop bool go func() error { - for i := 0; i < 100; i++ { - go func() error { + stop = false + for i := 0; i < 100 && !stop; i++ { + func() error { cmd := exec.Command(DOCKER_PATH, "run", "base", "echo", "hello", "world") log.Printf("%d", i) outPipe, err := cmd.StdoutPipe() @@ -74,12 +77,11 @@ func crashTest() error { outPipe.Close() return nil }() - time.Sleep(250 * time.Millisecond) } return nil }() - time.Sleep(20 * time.Second) + stop = true if err := daemon.Process.Kill(); err != nil { return err } From 2638d9a6cc20626a6bda7a0c0317647dc45be9eb Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 16 Apr 2013 18:43:44 +0200 Subject: [PATCH 43/98] added -t in docker stop and restart to choose grace period Upstream-commit: 1615bb08c7c3fc6c4b22db0a633edda516f97cf0 Component: engine --- components/engine/commands.go | 8 +++++--- components/engine/container.go | 10 +++++----- components/engine/runtime.go | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 82a0ce103d..2b0c91a586 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -223,7 +223,8 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "stop", "CONTAINER [CONTAINER...]", "Stop a running container") + cmd := rcli.Subcmd(stdout, "stop", "[OPTIONS] CONTAINER [CONTAINER...]", "Stop a running container") + nSeconds := cmd.Int("t", 10, "wait t seconds before killing the container") if err := cmd.Parse(args); err != nil { return nil } @@ -233,7 +234,7 @@ func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string } for _, name := range cmd.Args() { if container := srv.runtime.Get(name); container != nil { - if err := container.Stop(); err != nil { + if err := container.Stop(*nSeconds); err != nil { return err } fmt.Fprintln(stdout, container.ShortId()) @@ -246,6 +247,7 @@ func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "restart", "CONTAINER [CONTAINER...]", "Restart a running container") + nSeconds := cmd.Int("t", 10, "wait t seconds before killing the container") if err := cmd.Parse(args); err != nil { return nil } @@ -255,7 +257,7 @@ func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...str } for _, name := range cmd.Args() { if container := srv.runtime.Get(name); container != nil { - if err := container.Restart(); err != nil { + if err := container.Restart(*nSeconds); err != nil { return err } fmt.Fprintln(stdout, container.ShortId()) diff --git a/components/engine/container.go b/components/engine/container.go index 74706a4079..2bd56180a7 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -599,7 +599,7 @@ func (container *Container) Kill() error { return container.kill() } -func (container *Container) Stop() error { +func (container *Container) Stop(seconds int) error { container.State.lock() defer container.State.unlock() if !container.State.Running { @@ -619,8 +619,8 @@ func (container *Container) Stop() error { } // 2. Wait for the process to exit on its own - if err := container.WaitTimeout(10 * time.Second); err != nil { - log.Printf("Container %v failed to exit within 10 seconds of SIGTERM - using the force", container.Id) + if err := container.WaitTimeout(time.Duration(seconds) * time.Second); err != nil { + log.Printf("Container %v failed to exit within %d seconds of SIGTERM - using the force", container.Id, seconds) if err := container.kill(); err != nil { return err } @@ -628,8 +628,8 @@ func (container *Container) Stop() error { return nil } -func (container *Container) Restart() error { - if err := container.Stop(); err != nil { +func (container *Container) Restart(seconds int) error { + if err := container.Stop(seconds); err != nil { return err } if err := container.Start(); err != nil { diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 3fe07c7ea6..0955eb8703 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -217,7 +217,7 @@ func (runtime *Runtime) Destroy(container *Container) error { return fmt.Errorf("Container %v not found - maybe it was already destroyed?", container.Id) } - if err := container.Stop(); err != nil { + if err := container.Stop(10); err != nil { return err } if mounted, err := container.Mounted(); err != nil { From e35cce33e7b4cfac229bfa7f1242ca45b8bda7c2 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 17 Apr 2013 11:32:13 -0700 Subject: [PATCH 44/98] Keep a cache of the unit-tests image. So I can code in conferences with crappy wifi. Upstream-commit: ca6cd5b557279facb4831e8380be4eb40ac38362 Component: engine --- components/engine/runtime_test.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index b990deaf38..20e7ee140d 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -12,11 +12,9 @@ import ( "time" ) -// FIXME: this is no longer needed -const testLayerPath string = "/var/lib/docker/docker-ut.tar" const unitTestImageName string = "docker-ut" -var unitTestStoreBase string +const unitTestStoreBase string = "/var/lib/docker/unit-tests" func nuke(runtime *Runtime) error { var wg sync.WaitGroup @@ -62,15 +60,8 @@ func init() { panic("docker tests needs to be run as root") } - // Create a temp directory - root, err := ioutil.TempDir("", "docker-test") - if err != nil { - panic(err) - } - unitTestStoreBase = root - // Make it our Store root - runtime, err := NewRuntimeFromDirectory(root) + runtime, err := NewRuntimeFromDirectory(unitTestStoreBase) if err != nil { panic(err) } From 2c9483cdd26bb6e045a860fe6f943c6604204f17 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 17 Apr 2013 16:35:22 -0700 Subject: [PATCH 45/98] Fix the behavior of Graph.Register so that it can be interrupted without side effect Upstream-commit: 13d9e26edd62228597cf8f060ed397f7ae00298a Component: engine --- components/engine/graph.go | 2 +- components/engine/graph_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/components/engine/graph.go b/components/engine/graph.go index e7044c25a0..afb89cd02b 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -111,7 +111,7 @@ func (graph *Graph) Register(layerData Archive, img *Image) error { if graph.Exists(img.Id) { return fmt.Errorf("Image %s already exists", img.Id) } - tmp, err := graph.Mktemp(img.Id) + tmp, err := graph.Mktemp("") defer os.RemoveAll(tmp) if err != nil { return fmt.Errorf("Mktemp failed: %s", err) diff --git a/components/engine/graph_test.go b/components/engine/graph_test.go index 7c40330aa4..8f28983494 100644 --- a/components/engine/graph_test.go +++ b/components/engine/graph_test.go @@ -3,6 +3,7 @@ package docker import ( "archive/tar" "bytes" + "errors" "io" "io/ioutil" "os" @@ -26,6 +27,32 @@ func TestInit(t *testing.T) { } } +// Test that Register can be interrupted cleanly without side effects +func TestInterruptedRegister(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + badArchive, w := io.Pipe() // Use a pipe reader as a fake archive which never yields data + image := &Image{ + Id: GenerateId(), + Comment: "testing", + Created: time.Now(), + } + go graph.Register(badArchive, image) + time.Sleep(200 * time.Millisecond) + w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling) + if _, err := graph.Get(image.Id); err == nil { + t.Fatal("Image should not exist after Register is interrupted") + } + // Registering the same image again should succeed if the first register was interrupted + goodArchive, err := fakeTar() + if err != nil { + t.Fatal(err) + } + if err := graph.Register(goodArchive, image); err != nil { + t.Fatal(err) + } +} + // FIXME: Do more extensive tests (ex: create multiple, delete, recreate; // create multiple, check the amount of images and paths, etc..) func TestGraphCreate(t *testing.T) { From 973c2340e9969de395f8f73fbaf4265a8b77e5cd Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 17 Apr 2013 17:12:08 -0700 Subject: [PATCH 46/98] Bumped version to 0.1.5 Upstream-commit: e34e44e8e6097e0a54bc0a96581f79909f7d9a42 Component: engine --- components/engine/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 82a0ce103d..cfb1d03008 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -18,7 +18,7 @@ import ( "unicode" ) -const VERSION = "0.1.4" +const VERSION = "0.1.5" var ( GIT_COMMIT string From 51cf529c261685dc4b8068c9918203fba48090ad Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 17 Apr 2013 19:58:17 -0700 Subject: [PATCH 47/98] Added 'author' field to the image format Upstream-commit: 4ef2d5c1e6e65b1e214071388618fc9fa4345be9 Component: engine --- components/engine/commands.go | 4 ++-- components/engine/container_test.go | 2 +- components/engine/graph.go | 3 ++- components/engine/graph_test.go | 12 ++++++------ components/engine/image.go | 1 + components/engine/runtime.go | 4 ++-- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index cfb1d03008..cfc4b714cc 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -472,7 +472,7 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout rcli.DockerConn, args . } archive = ProgressReader(resp.Body, int(resp.ContentLength), stdout) } - img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src) + img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "") if err != nil { return err } @@ -727,7 +727,7 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd.Usage() return nil } - img, err := srv.runtime.Commit(containerName, repository, tag, *flComment) + img, err := srv.runtime.Commit(containerName, repository, tag, *flComment, "") if err != nil { return err } diff --git a/components/engine/container_test.go b/components/engine/container_test.go index d5f3694c59..ff2dc4c4c7 100644 --- a/components/engine/container_test.go +++ b/components/engine/container_test.go @@ -182,7 +182,7 @@ func TestCommitRun(t *testing.T) { if err != nil { t.Error(err) } - img, err := runtime.graph.Create(rwTar, container1, "unit test commited image") + img, err := runtime.graph.Create(rwTar, container1, "unit test commited image", "") if err != nil { t.Error(err) } diff --git a/components/engine/graph.go b/components/engine/graph.go index afb89cd02b..b7dbf2e113 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -83,12 +83,13 @@ func (graph *Graph) Get(name string) (*Image, error) { } // Create creates a new image and registers it in the graph. -func (graph *Graph) Create(layerData Archive, container *Container, comment string) (*Image, error) { +func (graph *Graph) Create(layerData Archive, container *Container, comment, author string) (*Image, error) { img := &Image{ Id: GenerateId(), Comment: comment, Created: time.Now(), DockerVersion: VERSION, + Author: author, } if container != nil { img.Parent = container.Image diff --git a/components/engine/graph_test.go b/components/engine/graph_test.go index 8f28983494..1bd05aaa97 100644 --- a/components/engine/graph_test.go +++ b/components/engine/graph_test.go @@ -62,7 +62,7 @@ func TestGraphCreate(t *testing.T) { if err != nil { t.Fatal(err) } - image, err := graph.Create(archive, nil, "Testing") + image, err := graph.Create(archive, nil, "Testing", "") if err != nil { t.Fatal(err) } @@ -122,7 +122,7 @@ func TestMount(t *testing.T) { if err != nil { t.Fatal(err) } - image, err := graph.Create(archive, nil, "Testing") + image, err := graph.Create(archive, nil, "Testing", "") if err != nil { t.Fatal(err) } @@ -166,7 +166,7 @@ func createTestImage(graph *Graph, t *testing.T) *Image { if err != nil { t.Fatal(err) } - img, err := graph.Create(archive, nil, "Test image") + img, err := graph.Create(archive, nil, "Test image", "") if err != nil { t.Fatal(err) } @@ -181,7 +181,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } assertNImages(graph, t, 0) - img, err := graph.Create(archive, nil, "Bla bla") + img, err := graph.Create(archive, nil, "Bla bla", "") if err != nil { t.Fatal(err) } @@ -192,11 +192,11 @@ func TestDelete(t *testing.T) { assertNImages(graph, t, 0) // Test 2 create (same name) / 1 delete - img1, err := graph.Create(archive, nil, "Testing") + img1, err := graph.Create(archive, nil, "Testing", "") if err != nil { t.Fatal(err) } - if _, err = graph.Create(archive, nil, "Testing"); err != nil { + if _, err = graph.Create(archive, nil, "Testing", ""); err != nil { t.Fatal(err) } assertNImages(graph, t, 2) diff --git a/components/engine/image.go b/components/engine/image.go index f9595bd976..9369fc3f48 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -23,6 +23,7 @@ type Image struct { Container string `json:"container,omitempty"` ContainerConfig Config `json:"container_config,omitempty"` DockerVersion string `json:"docker_version,omitempty"` + Author string `json:"author,omitempty"` graph *Graph } diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 3fe07c7ea6..72de9f8476 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -238,7 +238,7 @@ func (runtime *Runtime) Destroy(container *Container) error { // Commit creates a new filesystem image from the current state of a container. // The image can optionally be tagged into a repository -func (runtime *Runtime) Commit(id, repository, tag, comment string) (*Image, error) { +func (runtime *Runtime) Commit(id, repository, tag, comment, author string) (*Image, error) { container := runtime.Get(id) if container == nil { return nil, fmt.Errorf("No such container: %s", id) @@ -250,7 +250,7 @@ func (runtime *Runtime) Commit(id, repository, tag, comment string) (*Image, err return nil, err } // Create a new image from the container's base layers + a new layer from container changes - img, err := runtime.graph.Create(rwTar, container, comment) + img, err := runtime.graph.Create(rwTar, container, comment, author) if err != nil { return nil, err } From ea3911cd8cf46adbc78804f42dc6e7c3daea3a2e Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 17 Apr 2013 20:13:11 -0700 Subject: [PATCH 48/98] Record the author of an image with 'docker commit -author' Upstream-commit: 227a8142a3c2ea2fd3b085214ef39989ebd57fe1 Component: engine --- components/engine/commands.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index cfc4b714cc..d66eaa8398 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -719,6 +719,7 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri "commit", "[OPTIONS] CONTAINER [REPOSITORY [TAG]]", "Create a new image from a container's changes") flComment := cmd.String("m", "", "Commit message") + flAuthor := cmd.String("author", "", "Author (eg. \"John Hannibal Smith \"") if err := cmd.Parse(args); err != nil { return nil } @@ -727,7 +728,7 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd.Usage() return nil } - img, err := srv.runtime.Commit(containerName, repository, tag, *flComment, "") + img, err := srv.runtime.Commit(containerName, repository, tag, *flComment, *flAuthor) if err != nil { return err } From 3adf253fc37f267b65b5254913c470ce345c5480 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 17 Apr 2013 20:18:35 -0700 Subject: [PATCH 49/98] Bumped version to 0.1.6 to mark image format change (author field) Upstream-commit: ee82870ff78e6d9e0f0ce7674d6a065f9d02f67a Component: engine --- components/engine/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index d66eaa8398..ba501dd5e4 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -18,7 +18,7 @@ import ( "unicode" ) -const VERSION = "0.1.5" +const VERSION = "0.1.6" var ( GIT_COMMIT string From 7548a05b9764a4cd83fc0c8273e78d0033e4ae76 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Fri, 12 Apr 2013 11:42:35 -0700 Subject: [PATCH 50/98] packaging-ubuntu: move original files in place for update Upstream-commit: fd39af7f859b5bd1c202b4c0ddee53de9c82437d Component: engine --- components/engine/packaging/ubuntu/{debian => }/changelog | 0 components/engine/packaging/ubuntu/{debian => }/compat | 0 components/engine/packaging/ubuntu/{debian => }/control | 0 components/engine/packaging/ubuntu/{debian => }/copyright | 0 components/engine/packaging/ubuntu/{etc => }/docker.upstart | 0 components/engine/packaging/ubuntu/{debian => }/docs | 0 components/engine/packaging/ubuntu/{debian => }/rules | 0 components/engine/packaging/ubuntu/{debian => }/source/format | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename components/engine/packaging/ubuntu/{debian => }/changelog (100%) rename components/engine/packaging/ubuntu/{debian => }/compat (100%) rename components/engine/packaging/ubuntu/{debian => }/control (100%) rename components/engine/packaging/ubuntu/{debian => }/copyright (100%) rename components/engine/packaging/ubuntu/{etc => }/docker.upstart (100%) rename components/engine/packaging/ubuntu/{debian => }/docs (100%) rename components/engine/packaging/ubuntu/{debian => }/rules (100%) rename components/engine/packaging/ubuntu/{debian => }/source/format (100%) diff --git a/components/engine/packaging/ubuntu/debian/changelog b/components/engine/packaging/ubuntu/changelog similarity index 100% rename from components/engine/packaging/ubuntu/debian/changelog rename to components/engine/packaging/ubuntu/changelog diff --git a/components/engine/packaging/ubuntu/debian/compat b/components/engine/packaging/ubuntu/compat similarity index 100% rename from components/engine/packaging/ubuntu/debian/compat rename to components/engine/packaging/ubuntu/compat diff --git a/components/engine/packaging/ubuntu/debian/control b/components/engine/packaging/ubuntu/control similarity index 100% rename from components/engine/packaging/ubuntu/debian/control rename to components/engine/packaging/ubuntu/control diff --git a/components/engine/packaging/ubuntu/debian/copyright b/components/engine/packaging/ubuntu/copyright similarity index 100% rename from components/engine/packaging/ubuntu/debian/copyright rename to components/engine/packaging/ubuntu/copyright diff --git a/components/engine/packaging/ubuntu/etc/docker.upstart b/components/engine/packaging/ubuntu/docker.upstart similarity index 100% rename from components/engine/packaging/ubuntu/etc/docker.upstart rename to components/engine/packaging/ubuntu/docker.upstart diff --git a/components/engine/packaging/ubuntu/debian/docs b/components/engine/packaging/ubuntu/docs similarity index 100% rename from components/engine/packaging/ubuntu/debian/docs rename to components/engine/packaging/ubuntu/docs diff --git a/components/engine/packaging/ubuntu/debian/rules b/components/engine/packaging/ubuntu/rules similarity index 100% rename from components/engine/packaging/ubuntu/debian/rules rename to components/engine/packaging/ubuntu/rules diff --git a/components/engine/packaging/ubuntu/debian/source/format b/components/engine/packaging/ubuntu/source/format similarity index 100% rename from components/engine/packaging/ubuntu/debian/source/format rename to components/engine/packaging/ubuntu/source/format From a4e3c63817eee6d5a241487655073dd93b51cd14 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Mon, 15 Apr 2013 18:01:54 -0700 Subject: [PATCH 51/98] packaging-ubuntu, issue #30: streamline building and uploading to PPA Upstream-commit: 523cd8e29cb871271a8e2a55ab009aa980ae5572 Component: engine --- components/engine/packaging/ubuntu/Makefile | 120 +++-- .../engine/packaging/ubuntu/README.ubuntu | 37 ++ .../engine/packaging/ubuntu/Vagrantfile | 12 + components/engine/packaging/ubuntu/changelog | 19 +- components/engine/packaging/ubuntu/control | 22 +- components/engine/packaging/ubuntu/copyright | 440 ++++++++++-------- .../engine/packaging/ubuntu/docker.upstart | 2 +- .../packaging/ubuntu/lxc-docker.postinst | 4 + .../engine/packaging/ubuntu/lxc-docker.prerm | 4 + .../engine/packaging/ubuntu/maintainer.ubuntu | 34 ++ 10 files changed, 407 insertions(+), 287 deletions(-) create mode 100644 components/engine/packaging/ubuntu/README.ubuntu create mode 100644 components/engine/packaging/ubuntu/Vagrantfile create mode 100644 components/engine/packaging/ubuntu/lxc-docker.postinst create mode 100644 components/engine/packaging/ubuntu/lxc-docker.prerm create mode 100644 components/engine/packaging/ubuntu/maintainer.ubuntu diff --git a/components/engine/packaging/ubuntu/Makefile b/components/engine/packaging/ubuntu/Makefile index beec903fc9..0443d8b3e6 100644 --- a/components/engine/packaging/ubuntu/Makefile +++ b/components/engine/packaging/ubuntu/Makefile @@ -1,73 +1,57 @@ +# Ubuntu package Makefile +# +# Dependencies: debhelper autotools-dev devscripts golang +# Notes: +# Use 'make ubuntu' to create the ubuntu package +# GPG_KEY environment variable needs to contain a GPG private key for package to be signed +# and uploaded to docker PPA. +# If GPG_KEY is not defined, make ubuntu will create docker package and exit with +# status code 2 + PKG_NAME=lxc-docker -PKG_ARCH=amd64 -PKG_VERSION=1 -ROOT_PATH:=$(PWD) -BUILD_PATH=build # Do not change, decided by dpkg-buildpackage -BUILD_SRC=build_src -GITHUB_PATH=src/github.com/dotcloud/docker -INSDIR=usr/bin -SOURCE_PACKAGE=$(PKG_NAME)_$(PKG_VERSION).orig.tar.gz -DEB_PACKAGE=$(PKG_NAME)_$(PKG_VERSION)_$(PKG_ARCH).deb -EXTRA_GO_PKG=./auth +VERSION=$(shell head -1 changelog | sed 's/^.\+(\(.\+\)..).\+$$/\1/') +GITHUB_PATH=github.com/dotcloud/docker +DOCKER_VERSION=${PKG_NAME}_${VERSION} +DOCKER_FVERSION=${PKG_NAME}_$(shell head -1 changelog | sed 's/^.\+(\(.\+\)).\+$$/\1/') +BUILD_SRC=${CURDIR}/../../build_src -TMPDIR=$(shell mktemp -d -t XXXXXX) +all: + # Compile docker. Used by dpkg-buildpackage. + cd src/${GITHUB_PATH}/docker; GOPATH=${CURDIR} go build - -# Build a debian source package -all: clean build_in_deb - -build_in_deb: - echo "GOPATH = " $(ROOT_PATH) - mkdir bin - cd $(GITHUB_PATH)/docker; GOPATH=$(ROOT_PATH) go build -o $(ROOT_PATH)/bin/docker - -# DESTDIR provided by Debian packaging install: - # Call this from a go environment (as packaged for deb source package) - mkdir -p $(DESTDIR)/$(INSDIR) - mkdir -p $(DESTDIR)/etc/init - install -m 0755 bin/docker $(DESTDIR)/$(INSDIR) - install -o root -m 0755 etc/docker.upstart $(DESTDIR)/etc/init/docker.conf + # Used by dpkg-buildpackage + mkdir -p ${DESTDIR}/usr/bin + mkdir -p ${DESTDIR}/etc/init + install -m 0755 src/${GITHUB_PATH}/docker/docker ${DESTDIR}/usr/bin + install -o root -m 0755 debian/docker.upstart ${DESTDIR}/etc/init/docker.conf -$(BUILD_SRC): clean - # Copy ourselves into $BUILD_SRC to comply with unusual golang constraints - tar --exclude=*.tar.gz --exclude=checkout.tgz -f checkout.tgz -cz * - mkdir -p $(BUILD_SRC)/$(GITHUB_PATH) - tar -f checkout.tgz -C $(BUILD_SRC)/$(GITHUB_PATH) -xz - cd $(BUILD_SRC)/$(GITHUB_PATH)/docker; GOPATH=$(ROOT_PATH)/$(BUILD_SRC) go get -d - for d in `find $(BUILD_SRC) -name '.git*'`; do rm -rf $$d; done - # Populate source build with debian stuff - cp -R -L ./deb/* $(BUILD_SRC) - -$(SOURCE_PACKAGE): $(BUILD_SRC) - rm -f $(SOURCE_PACKAGE) - # Create the debian source package - tar -f $(SOURCE_PACKAGE) -C ${ROOT_PATH}/${BUILD_SRC} -cz . - -# Build deb package fetching go dependencies and cleaning up git repositories -deb: $(DEB_PACKAGE) - -$(DEB_PACKAGE): $(SOURCE_PACKAGE) - # dpkg-buildpackage looks for source package tarball in ../ - cd $(BUILD_SRC); dpkg-buildpackage - rm -rf $(BUILD_PATH) debian/$(PKG_NAME)* debian/files - -debsrc: $(SOURCE_PACKAGE) - -# Build local sources -#$(PKG_NAME): build_local - -build_local: - -@mkdir -p bin - cd docker && go build -o ../bin/docker - -gotest: - @echo "\033[36m[Testing]\033[00m docker..." - @sudo -E GOPATH=$(ROOT_PATH)/$(BUILD_SRC) go test -v . $(EXTRA_GO_PKG) && \ - echo -n "\033[32m[OK]\033[00m" || \ - echo -n "\033[31m[FAIL]\033[00m"; \ - echo " docker" - @sudo rm -rf /tmp/docker-* - -clean: - rm -rf $(BUILD_PATH) debian/$(PKG_NAME)* debian/files $(BUILD_SRC) checkout.tgz bin +ubuntu: + # This Makefile will compile the github master branch of dotcloud/docker + # Retrieve docker project and its go structure from internet + rm -rf ${BUILD_SRC} + GOPATH=${BUILD_SRC} go get ${GITHUB_PATH} + # Add debianization + mkdir ${BUILD_SRC}/debian + cp Makefile ${BUILD_SRC} + cp -r * ${BUILD_SRC}/debian + cp ../../README.md ${BUILD_SRC} + # Cleanup + for d in `find ${BUILD_SRC} -name '.git*'`; do rm -rf $$d; done + rm -rf ${BUILD_SRC}/../${DOCKER_VERSION}.orig.tar.gz + rm -rf ${BUILD_SRC}/pkg + # Create docker debian files + cd ${BUILD_SRC}; tar czf ../${DOCKER_VERSION}.orig.tar.gz . + cd ${BUILD_SRC}; dpkg-buildpackage -us -uc + rm -rf ${BUILD_SRC} + # Sign package and upload it to PPA if GPG_KEY environment variable + # holds a private GPG KEY + if /usr/bin/test "$${GPG_KEY}" == ""; then exit 2; fi + mkdir ${BUILD_SRC} + # Import gpg signing key + echo "$${GPG_KEY}" | gpg --allow-secret-key-import --import + # Sign the package + cd ${BUILD_SRC}; dpkg-source -x ${BUILD_SRC}/../${DOCKER_FVERSION}.dsc + cd ${BUILD_SRC}/${PKG_NAME}-${VERSION}; debuild -S -sa + cd ${BUILD_SRC};dput ppa:dotcloud/lxc-docker ${DOCKER_FVERSION}_source.changes + rm -rf ${BUILD_SRC} diff --git a/components/engine/packaging/ubuntu/README.ubuntu b/components/engine/packaging/ubuntu/README.ubuntu new file mode 100644 index 0000000000..286a6f8d52 --- /dev/null +++ b/components/engine/packaging/ubuntu/README.ubuntu @@ -0,0 +1,37 @@ +Docker on Ubuntu +================ + +The easiest way to get docker up and running natively on Ubuntu is installing +it from its official PPA:: + + sudo sh -c "echo 'deb http://ppa.launchpad.net/dotcloud/lxc-docker/ubuntu precise main' >>/etc/apt/sources.list" + sudo apt-get update + sudo apt-get install lxc-docker + + +Building docker package +~~~~~~~~~~~~~~~~~~~~~~~ + +The building process is shared by both, developers and maintainers. If you are +a developer, the Makefile will stop with exit status 2 right before signing +the built packages. + +Assuming you are working on an Ubuntu 12.04 TLS system :: + + # Download a fresh copy of the docker project + git clone https://github.com/dotcloud/docker.git + cd docker + + # Get building dependencies + sudo apt-get update; sudo apt-get install -y debhelper autotools-dev devscripts golang + + # Make the ubuntu package + (cd packaging/ubuntu; make ubuntu) + + +Install docker built package +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + sudo dpkg -i lxc-docker_*_amd64.deb; sudo apt-get install -f -y diff --git a/components/engine/packaging/ubuntu/Vagrantfile b/components/engine/packaging/ubuntu/Vagrantfile new file mode 100644 index 0000000000..2c9018ab98 --- /dev/null +++ b/components/engine/packaging/ubuntu/Vagrantfile @@ -0,0 +1,12 @@ +BUILDBOT_IP = '192.168.33.32' + +Vagrant::Config.run do |config| + config.vm.box = 'quantal64_3.5.0-25' + config.vm.box_url = 'http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box' + config.vm.share_folder 'v-data', '/data/docker', "#{File.dirname(__FILE__)}/../.." + config.vm.network :hostonly,BUILDBOT_IP + + # Install ubuntu packaging dependencies and create ubuntu packages + config.vm.provision :shell, :inline => 'export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; apt-get install -qq -y debhelper autotools-dev devscripts golang' + config.vm.provision :shell, :inline => "export GPG_KEY='#{ENV['GPG_KEY']}'; cd /data/docker/packaging/ubuntu; make ubuntu" +end diff --git a/components/engine/packaging/ubuntu/changelog b/components/engine/packaging/ubuntu/changelog index d8932885e6..515a927f5e 100644 --- a/components/engine/packaging/ubuntu/changelog +++ b/components/engine/packaging/ubuntu/changelog @@ -1,4 +1,21 @@ -lxc-docker (1) precise; urgency=low +lxc-docker (0.1.4.1-1) precise; urgency=low + + Improvements [+], Updates [*], Bug fixes [-]: + * Test PPA + + -- dotCloud Fri, 15 Apr 2013 12:14:50 -0700 + + +lxc-docker (0.1.4-1) precise; urgency=low + + Improvements [+], Updates [*], Bug fixes [-]: + * Changed default bridge interface do 'docker0' + - Fix a race condition when running the port allocator + + -- dotCloud Fri, 12 Apr 2013 12:20:06 -0700 + + +lxc-docker (0.1.0-1) unstable; urgency=low * Initial release diff --git a/components/engine/packaging/ubuntu/control b/components/engine/packaging/ubuntu/control index 1ad913854d..c52303a88b 100644 --- a/components/engine/packaging/ubuntu/control +++ b/components/engine/packaging/ubuntu/control @@ -1,19 +1,19 @@ Source: lxc-docker Section: misc Priority: extra -Homepage: http://docker.io Maintainer: Daniel Mizyrycki -Build-Depends: debhelper (>= 8.0.0), pkg-config, git, golang, libsqlite3-dev -Vcs-Git: http://github.com/dotcloud/docker.git +Build-Depends: debhelper,autotools-dev,devscripts,golang Standards-Version: 3.9.3 +Homepage: http://github.com/dotcloud/docker Package: lxc-docker -Architecture: amd64 -Depends: ${shlibs:Depends}, ${misc:Depends}, lxc, wget, bsdtar, curl, sqlite3 +Architecture: linux-any +Depends: ${misc:Depends},${shlibs:Depends},lxc,bsdtar Conflicts: docker -Description: A process manager with superpowers - It encapsulates heterogeneous payloads in Standard Containers, and runs - them on any server with strong guarantees of isolation and repeatability. - Is is a great building block for automating distributed systems: - large-scale web deployments, database clusters, continuous deployment - systems, private PaaS, service-oriented architectures, etc. +Description: lxc-docker is a Linux container runtime + Docker complements LXC with a high-level API which operates at the process + level. It runs unix processes with strong guarantees of isolation and + repeatability across servers. + Docker is a great building block for automating distributed systems: + large-scale web deployments, database clusters, continuous deployment systems, + private PaaS, service-oriented architectures, etc. diff --git a/components/engine/packaging/ubuntu/copyright b/components/engine/packaging/ubuntu/copyright index c6c97190a9..668c8635e4 100644 --- a/components/engine/packaging/ubuntu/copyright +++ b/components/engine/packaging/ubuntu/copyright @@ -1,209 +1,237 @@ -Format: http://dep.debian.net/deps/dep5 +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: docker -Source: https://github.com/dotcloud/docker +Upstream-Contact: DotCloud Inc +Source: http://github.com/dotcloud/docker Files: * -Copyright: 2012 DotCloud Inc (opensource@dotcloud.com) -License: Apache License Version 2.0 - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2012 DotCloud Inc (opensource@dotcloud.com) - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Copyright: 2012, DotCloud Inc +License: Apache-2.0 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2012 DotCloud Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +Files: src/github.com/kr/pty/* +Copyright: Copyright (c) 2011 Keith Rarick +License: Expat + Copyright (c) 2011 Keith Rarick + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall + be included in all copies or substantial portions of the + Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS + OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/components/engine/packaging/ubuntu/docker.upstart b/components/engine/packaging/ubuntu/docker.upstart index 6cfe9d2616..4e49a3fa9a 100644 --- a/components/engine/packaging/ubuntu/docker.upstart +++ b/components/engine/packaging/ubuntu/docker.upstart @@ -5,6 +5,6 @@ stop on starting rc RUNLEVEL=[016] respawn script - test -f /etc/default/locale && . /etc/default/locale || true + /usr/bin/test -f /etc/default/locale && . /etc/default/locale || true LANG=$LANG LC_ALL=$LANG /usr/bin/docker -d end script diff --git a/components/engine/packaging/ubuntu/lxc-docker.postinst b/components/engine/packaging/ubuntu/lxc-docker.postinst new file mode 100644 index 0000000000..5d04c5b55d --- /dev/null +++ b/components/engine/packaging/ubuntu/lxc-docker.postinst @@ -0,0 +1,4 @@ +#!/bin/sh + +# Start docker +/sbin/start docker diff --git a/components/engine/packaging/ubuntu/lxc-docker.prerm b/components/engine/packaging/ubuntu/lxc-docker.prerm new file mode 100644 index 0000000000..824f15cff0 --- /dev/null +++ b/components/engine/packaging/ubuntu/lxc-docker.prerm @@ -0,0 +1,4 @@ +#!/bin/sh + +# Stop docker +/sbin/stop docker diff --git a/components/engine/packaging/ubuntu/maintainer.ubuntu b/components/engine/packaging/ubuntu/maintainer.ubuntu new file mode 100644 index 0000000000..d69ba462a4 --- /dev/null +++ b/components/engine/packaging/ubuntu/maintainer.ubuntu @@ -0,0 +1,34 @@ +Maintainer duty +=============== + +Ubuntu allows developers to use their PPA (Personal Package Archive) +repository. This is very convenient for the users as they just need to add +the PPA address, update their package database and use the apt-get tool. + +The official lxc-docker package is located on launchpad and can be accessed +adding the following line to /etc/apt/sources.list :: + + deb http://ppa.launchpad.net/dotcloud/lxc-docker/ubuntu precise main + + +Realeasing a new package +~~~~~~~~~~~~~~~~~~~~~~~~ + +The most relevant information to update is the changelog file: +Each new release should create a new first paragraph with new release version, +changes, and the maintainer information. + +Assuming your PPA GPG signing key is on /media/usbdrive/docker.key, load it +into the GPG_KEY environment variable with:: + + export GPG_KEY=`cat /media/usbdrive/docker.key` + + +After this is done and you are ready to upload the package to the PPA, you have +a couple of choices: + +* Follow README.debian to generate the actual source packages and upload them + to the PPA +* Let vagrant do all the work for you:: + + ( cd docker/packaging/ubuntu; vagrant up ) From e0e0c2c96ea08c66ec6d6b346b7ba916545d3c42 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Wed, 17 Apr 2013 21:10:53 -0700 Subject: [PATCH 52/98] packaging-ubuntu, issue #30: Ensure docker package installs and passes tests on official vagrant Ubuntu 12.04 box Upstream-commit: 8e6ba343bfb93ed7782d50a3ac1b581ae0e1fb00 Component: engine --- components/engine/packaging/ubuntu/Makefile | 3 +++ components/engine/packaging/ubuntu/Vagrantfile | 6 +++--- components/engine/packaging/ubuntu/changelog | 10 +++++++++- components/engine/packaging/ubuntu/docker.upstart | 4 ++-- components/engine/packaging/ubuntu/maintainer.ubuntu | 9 +++++---- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/components/engine/packaging/ubuntu/Makefile b/components/engine/packaging/ubuntu/Makefile index 0443d8b3e6..8e0634870e 100644 --- a/components/engine/packaging/ubuntu/Makefile +++ b/components/engine/packaging/ubuntu/Makefile @@ -23,8 +23,11 @@ install: # Used by dpkg-buildpackage mkdir -p ${DESTDIR}/usr/bin mkdir -p ${DESTDIR}/etc/init + mkdir -p ${DESTDIR}/DEBIAN install -m 0755 src/${GITHUB_PATH}/docker/docker ${DESTDIR}/usr/bin install -o root -m 0755 debian/docker.upstart ${DESTDIR}/etc/init/docker.conf + install debian/lxc-docker.prerm ${DESTDIR}/DEBIAN/prerm + install debian/lxc-docker.postinst ${DESTDIR}/DEBIAN/postinst ubuntu: # This Makefile will compile the github master branch of dotcloud/docker diff --git a/components/engine/packaging/ubuntu/Vagrantfile b/components/engine/packaging/ubuntu/Vagrantfile index 2c9018ab98..0689eea1c2 100644 --- a/components/engine/packaging/ubuntu/Vagrantfile +++ b/components/engine/packaging/ubuntu/Vagrantfile @@ -1,12 +1,12 @@ BUILDBOT_IP = '192.168.33.32' Vagrant::Config.run do |config| - config.vm.box = 'quantal64_3.5.0-25' - config.vm.box_url = 'http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box' + config.vm.box = 'precise64' + config.vm.box_url = 'http://files.vagrantup.com/precise64.box' config.vm.share_folder 'v-data', '/data/docker', "#{File.dirname(__FILE__)}/../.." config.vm.network :hostonly,BUILDBOT_IP # Install ubuntu packaging dependencies and create ubuntu packages - config.vm.provision :shell, :inline => 'export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; apt-get install -qq -y debhelper autotools-dev devscripts golang' + config.vm.provision :shell, :inline => 'export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; apt-get install -qq -y git debhelper autotools-dev devscripts golang' config.vm.provision :shell, :inline => "export GPG_KEY='#{ENV['GPG_KEY']}'; cd /data/docker/packaging/ubuntu; make ubuntu" end diff --git a/components/engine/packaging/ubuntu/changelog b/components/engine/packaging/ubuntu/changelog index 515a927f5e..aa5ea6cc87 100644 --- a/components/engine/packaging/ubuntu/changelog +++ b/components/engine/packaging/ubuntu/changelog @@ -1,9 +1,17 @@ +lxc-docker (0.1.6-1) precise; urgency=low + + Improvements [+], Updates [*], Bug fixes [-]: + + Multiple improvements, updates and bug fixes + + -- dotCloud Wed, 17 Apr 2013 20:43:43 -0700 + + lxc-docker (0.1.4.1-1) precise; urgency=low Improvements [+], Updates [*], Bug fixes [-]: * Test PPA - -- dotCloud Fri, 15 Apr 2013 12:14:50 -0700 + -- dotCloud Mon, 15 Apr 2013 12:14:50 -0700 lxc-docker (0.1.4-1) precise; urgency=low diff --git a/components/engine/packaging/ubuntu/docker.upstart b/components/engine/packaging/ubuntu/docker.upstart index 4e49a3fa9a..07e7e8a890 100644 --- a/components/engine/packaging/ubuntu/docker.upstart +++ b/components/engine/packaging/ubuntu/docker.upstart @@ -5,6 +5,6 @@ stop on starting rc RUNLEVEL=[016] respawn script - /usr/bin/test -f /etc/default/locale && . /etc/default/locale || true - LANG=$LANG LC_ALL=$LANG /usr/bin/docker -d + # FIXME: docker should not depend on the system having en_US.UTF-8 + LC_ALL='en_US.UTF-8' /usr/bin/docker -d end script diff --git a/components/engine/packaging/ubuntu/maintainer.ubuntu b/components/engine/packaging/ubuntu/maintainer.ubuntu index d69ba462a4..406498ebad 100644 --- a/components/engine/packaging/ubuntu/maintainer.ubuntu +++ b/components/engine/packaging/ubuntu/maintainer.ubuntu @@ -5,14 +5,15 @@ Ubuntu allows developers to use their PPA (Personal Package Archive) repository. This is very convenient for the users as they just need to add the PPA address, update their package database and use the apt-get tool. -The official lxc-docker package is located on launchpad and can be accessed -adding the following line to /etc/apt/sources.list :: +For now, the official lxc-docker package is located on launchpad and can be +accessed adding the following line to /etc/apt/sources.list :: + deb http://ppa.launchpad.net/dotcloud/lxc-docker/ubuntu precise main -Realeasing a new package -~~~~~~~~~~~~~~~~~~~~~~~~ +Releasing a new package +~~~~~~~~~~~~~~~~~~~~~~~ The most relevant information to update is the changelog file: Each new release should create a new first paragraph with new release version, From 847e9fb0f99ab57715bd064cbaf349392e322825 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 18 Apr 2013 16:03:50 +0200 Subject: [PATCH 53/98] fixed test Upstream-commit: 90602ab62a4dddadff6671d80815465556a267ee Component: engine --- components/engine/container_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/container_test.go b/components/engine/container_test.go index d5f3694c59..9e17390116 100644 --- a/components/engine/container_test.go +++ b/components/engine/container_test.go @@ -97,7 +97,7 @@ func TestMultipleAttachRestart(t *testing.T) { t.Fatalf("Unexpected output. Expected [%s], received [%s]", "hello", l3) } - if err := container.Stop(); err != nil { + if err := container.Stop(10); err != nil { t.Fatal(err) } From 921b5daf0607f0dfa325c75ac875bc8b70c6bcb0 Mon Sep 17 00:00:00 2001 From: Shawn Siefkas Date: Thu, 18 Apr 2013 09:17:31 -0500 Subject: [PATCH 54/98] Adding archlinux packaging documentation Upstream-commit: 84c13a3dcf8a0f884d3b95b8f3515e0040816691 Component: engine --- .../packaging/archlinux/README.archlinux | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 components/engine/packaging/archlinux/README.archlinux diff --git a/components/engine/packaging/archlinux/README.archlinux b/components/engine/packaging/archlinux/README.archlinux new file mode 100644 index 0000000000..f20d2d25bc --- /dev/null +++ b/components/engine/packaging/archlinux/README.archlinux @@ -0,0 +1,25 @@ +Docker on Arch +============== + +The AUR lxc-docker and lxc-docker-git packages handle building docker on Arch +linux. The PKGBUILD specifies all dependencies, build, and packaging steps. + +Dependencies +============ + +The only buildtime dependencies are git and go which are available via pacman. +The -s flag can be used on makepkg commands below to automatically install +these dependencies. + +Building Package +================ + +Download the tarball for either AUR packaged to a local directory. In that +directory makepkg can be run to build the package. + +# Build the binary package +makepkg + +# Build an updated source tarball +makepkg --source + From 15ab010e20a1846b4416ef629ab53e17e8f7dd11 Mon Sep 17 00:00:00 2001 From: Shawn Siefkas Date: Thu, 18 Apr 2013 09:17:57 -0500 Subject: [PATCH 55/98] Updating the arch linux installation docs New AUR package name Adding systemd service unit info Upstream-commit: 7eda9c64b80ff43af5e90b37a3830615034fefee Component: engine --- .../docs/sources/installation/archlinux.rst | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/components/engine/docs/sources/installation/archlinux.rst b/components/engine/docs/sources/installation/archlinux.rst index c6de247d6c..e809974053 100644 --- a/components/engine/docs/sources/installation/archlinux.rst +++ b/components/engine/docs/sources/installation/archlinux.rst @@ -6,16 +6,16 @@ Arch Linux Installing on Arch Linux is not officially supported but can be handled via either of the following AUR packages: -* `dotcloud-docker `_ -* `dotcloud-docker-git `_ +* `lxc-docker `_ +* `lxc-docker-git `_ -The dotcloud-docker package will install the latest tagged version of docker. -The dotcloud-docker-git package will build from the current master branch. +The lxc-docker package will install the latest tagged version of docker. +The lxc-docker-git package will build from the current master branch. Dependencies ------------ -Docker depends on several packages which will be installed automatically with +Docker depends on several packages which are specified as dependencies in either AUR package. * aufs3 @@ -23,6 +23,7 @@ either AUR package. * go * iproute2 * linux-aufs_friendly +* lxc Installation ------------ @@ -37,7 +38,22 @@ new kernel will be compiled and this can take quite a while. :: - yaourt -S dotcloud-docker-git + yaourt -S lxc-docker-git + +Starting Docker +--------------- Prior to starting docker modify your bootloader to use the **linux-aufs_friendly** kernel and reboot your system. + +There is a systemd service unit created for docker. To start the docker service: + +:: + + sudo systemctl start docker + +To start on system boot: + +:: + + sudo systemctl enable docker From faf8875469ec25b17de7b2794b1d93230266894b Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 18 Apr 2013 14:46:17 -0700 Subject: [PATCH 56/98] contrib/docker-build: don't remove the base image if the first build step fails Upstream-commit: deb55e416e068a9b51c5973dfeeb2bb2ce6fd1be Component: engine --- components/engine/contrib/docker-build/docker-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/contrib/docker-build/docker-build b/components/engine/contrib/docker-build/docker-build index 934e47ec8f..f2fc340680 100755 --- a/components/engine/contrib/docker-build/docker-build +++ b/components/engine/contrib/docker-build/docker-build @@ -96,7 +96,7 @@ def main(): else: print "Skipping uknown op " + op except: - docker(["rmi"] + steps) + docker(["rmi"] + steps[1:]) raise print base From 4408440fb33745627196b4f6c37a6d2ba120261c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 18 Apr 2013 20:47:24 -0700 Subject: [PATCH 57/98] Check kernel version and display warning if too low Upstream-commit: 003622c8b6587814744a9903f3286dc1b07554c2 Component: engine --- components/engine/runtime.go | 19 +++++++- components/engine/utils.go | 94 ++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 72de9f8476..d8c6d4259c 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -6,6 +6,7 @@ import ( "github.com/dotcloud/docker/auth" "io" "io/ioutil" + "log" "os" "os/exec" "path" @@ -23,6 +24,7 @@ type Runtime struct { repositories *TagStore authConfig *auth.AuthConfig idIndex *TruncIndex + kernelVersion *KernelVersionInfo } var sysInitPath string @@ -282,7 +284,22 @@ func (runtime *Runtime) restore() error { // FIXME: harmonize with NewGraph() func NewRuntime() (*Runtime, error) { - return NewRuntimeFromDirectory("/var/lib/docker") + runtime, err := NewRuntimeFromDirectory("/var/lib/docker") + if err != nil { + return nil, err + } + + k, err := GetKernelVersion() + if err != nil { + return nil, err + } + runtime.kernelVersion = k + + if CompareKernelVersion(k, &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}) < 0 { + log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String()) + } + + return runtime, nil } func NewRuntimeFromDirectory(root string) (*Runtime, error) { diff --git a/components/engine/utils.go b/components/engine/utils.go index 68e12b20bd..8daf404481 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -13,8 +13,10 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "strings" "sync" + "syscall" "time" ) @@ -384,3 +386,95 @@ func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) } return written, err } + +type KernelVersionInfo struct { + Kernel int + Major int + Minor int + Specific int +} + +func GetKernelVersion() (*KernelVersionInfo, error) { + var uts syscall.Utsname + + if err := syscall.Uname(&uts); err != nil { + return nil, err + } + + release := make([]byte, len(uts.Release)) + + i := 0 + for _, c := range uts.Release { + release[i] = byte(c) + i++ + } + + tmp := strings.SplitN(string(release), "-", 2) + if len(tmp) != 2 { + return nil, fmt.Errorf("Unrecognized kernel version") + } + tmp2 := strings.SplitN(tmp[0], ".", 3) + if len(tmp2) != 3 { + return nil, fmt.Errorf("Unrecognized kernel version") + } + + kernel, err := strconv.Atoi(tmp2[0]) + if err != nil { + return nil, err + } + + major, err := strconv.Atoi(tmp2[1]) + if err != nil { + return nil, err + } + + minor, err := strconv.Atoi(tmp2[2]) + if err != nil { + return nil, err + } + + specific, err := strconv.Atoi(strings.Split(tmp[1], "-")[0]) + if err != nil { + return nil, err + } + + return &KernelVersionInfo{ + Kernel: kernel, + Major: major, + Minor: minor, + Specific: specific, + }, nil +} + +func (k *KernelVersionInfo) String() string { + return fmt.Sprintf("%d.%d.%d-%d", k.Kernel, k.Major, k.Minor, k.Specific) +} + +// Compare two KernelVersionInfo struct. +// Returns -1 if a < b, = if a == b, 1 it a > b +func CompareKernelVersion(a, b *KernelVersionInfo) int { + if a.Kernel < b.Kernel { + return -1 + } else if a.Kernel > b.Kernel { + return 1 + } + + if a.Major < b.Major { + return -1 + } else if a.Major > b.Major { + return 1 + } + + if a.Minor < b.Minor { + return -1 + } else if a.Minor > b.Minor { + return 1 + } + + if a.Specific < b.Specific { + return -1 + } else if a.Specific > b.Specific { + return 1 + } + return 0 +} From b6710144e6a04b22773dc48098345d6e321764db Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 18 Apr 2013 20:55:41 -0700 Subject: [PATCH 58/98] Add capabilities check to allow docker to run on kernel that does not have all options Upstream-commit: 640efc2ed2244dc16e161cc954af59d20a5e6ed2 Component: engine --- components/engine/commands.go | 2 +- components/engine/container.go | 12 +++++++++--- components/engine/runtime.go | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index ba501dd5e4..6ab164d647 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -907,7 +907,7 @@ func (srv *Server) CmdTag(stdin io.ReadCloser, stdout io.Writer, args ...string) } func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { - config, err := ParseRun(args, stdout) + config, err := ParseRun(args, stdout, srv.runtime.capabilities) if err != nil { return err } diff --git a/components/engine/container.go b/components/engine/container.go index 9f175e42aa..0d3427d9c0 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -66,7 +66,7 @@ type Config struct { Image string // Name of the image as it was passed by the operator (eg. could be symbolic) } -func ParseRun(args []string, stdout io.Writer) (*Config, error) { +func ParseRun(args []string, stdout io.Writer, capabilities *Capabilities) (*Config, error) { cmd := rcli.Subcmd(stdout, "run", "[OPTIONS] IMAGE COMMAND [ARG...]", "Run a command in a new container") if len(args) > 0 && args[0] != "--help" { cmd.SetOutput(ioutil.Discard) @@ -81,8 +81,8 @@ func ParseRun(args []string, stdout io.Writer) (*Config, error) { flTty := cmd.Bool("t", false, "Allocate a pseudo-tty") flMemory := cmd.Int64("m", 0, "Memory limit (in bytes)") - if *flMemory > 0 && NO_MEMORY_LIMIT { - fmt.Fprintf(stdout, "WARNING: This version of docker has been compiled without memory limit support. Discarding -m.") + if *flMemory > 0 && !capabilities.MemoryLimit { + fmt.Fprintf(stdout, "WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.\n") *flMemory = 0 } @@ -135,6 +135,12 @@ func ParseRun(args []string, stdout io.Writer) (*Config, error) { Dns: flDns, Image: image, } + + if *flMemory > 0 && !capabilities.SwapLimit { + fmt.Fprintf(stdout, "WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n") + config.MemorySwap = -1 + } + // When allocating stdin in attached mode, close stdin at client disconnect if config.OpenStdin && config.AttachStdin { config.StdinOnce = true diff --git a/components/engine/runtime.go b/components/engine/runtime.go index d8c6d4259c..43b1a7815e 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -15,6 +15,11 @@ import ( "time" ) +type Capabilities struct { + MemoryLimit bool + SwapLimit bool +} + type Runtime struct { root string repository string @@ -24,6 +29,7 @@ type Runtime struct { repositories *TagStore authConfig *auth.AuthConfig idIndex *TruncIndex + capabilities *Capabilities kernelVersion *KernelVersionInfo } @@ -299,6 +305,13 @@ func NewRuntime() (*Runtime, error) { log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String()) } + _, err1 := ioutil.ReadFile("/sys/fs/cgroup/memory/memory.limit_in_bytes") + _, err2 := ioutil.ReadFile("/sys/fs/cgroup/memory/memory.soft_limit_in_bytes") + runtime.capabilities.MemoryLimit = err1 == nil && err2 == nil + + _, err = ioutil.ReadFile("/sys/fs/cgroup/memory/memeory.memsw.limit_in_bytes") + runtime.capabilities.SwapLimit = err == nil + return runtime, nil } @@ -338,6 +351,7 @@ func NewRuntimeFromDirectory(root string) (*Runtime, error) { repositories: repositories, authConfig: authConfig, idIndex: NewTruncIndex(), + capabilities: &Capabilities{}, } if err := runtime.restore(); err != nil { From 630116b2c045ca9ab5b8fd878bfb00db3590e4ab Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 18 Apr 2013 21:08:20 -0700 Subject: [PATCH 59/98] Remove the NO_MEMORY_LIMIT constant Upstream-commit: f68d107a1368b3d4f3342456a2b0675659688354 Component: engine --- components/engine/Makefile | 5 +---- components/engine/commands.go | 3 +-- components/engine/container.go | 9 +++++++-- components/engine/docker/docker.go | 7 +------ components/engine/runtime_test.go | 2 -- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index c89f0f33b0..a6eb613830 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -13,10 +13,7 @@ endif GIT_COMMIT = $(shell git rev-parse --short HEAD) GIT_STATUS = $(shell test -n "`git status --porcelain`" && echo "+CHANGES") -NO_MEMORY_LIMIT ?= 0 -export NO_MEMORY_LIMIT - -BUILD_OPTIONS = -ldflags "-X main.GIT_COMMIT $(GIT_COMMIT)$(GIT_STATUS) -X main.NO_MEMORY_LIMIT $(NO_MEMORY_LIMIT)" +BUILD_OPTIONS = -ldflags "-X main.GIT_COMMIT $(GIT_COMMIT)$(GIT_STATUS)" SRC_DIR := $(GOPATH)/src diff --git a/components/engine/commands.go b/components/engine/commands.go index 6ab164d647..274acf4992 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -21,8 +21,7 @@ import ( const VERSION = "0.1.6" var ( - GIT_COMMIT string - NO_MEMORY_LIMIT bool + GIT_COMMIT string ) func (srv *Server) Name() string { diff --git a/components/engine/container.go b/components/engine/container.go index 0d3427d9c0..c23578875d 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -373,10 +373,15 @@ func (container *Container) Start() error { return err } - if container.Config.Memory > 0 && NO_MEMORY_LIMIT { - log.Printf("WARNING: This version of docker has been compiled without memory limit support. Discarding the limit.") + // Make sure the config is compatible with the current kernel + if container.Config.Memory > 0 && !container.runtime.capabilities.MemoryLimit { + log.Printf("WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.\n") container.Config.Memory = 0 } + if container.Config.Memory > 0 && !container.runtime.capabilities.SwapLimit { + log.Printf("WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n") + container.Config.MemorySwap = -1 + } if err := container.generateLXCConfig(); err != nil { return err diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 83c47c6f1e..411e4d0c96 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -14,8 +14,7 @@ import ( ) var ( - GIT_COMMIT string - NO_MEMORY_LIMIT string + GIT_COMMIT string ) func main() { @@ -39,15 +38,11 @@ func main() { os.Setenv("DEBUG", "1") } docker.GIT_COMMIT = GIT_COMMIT - docker.NO_MEMORY_LIMIT = NO_MEMORY_LIMIT == "1" if *flDaemon { if flag.NArg() != 0 { flag.Usage() return } - if NO_MEMORY_LIMIT == "1" { - log.Printf("WARNING: This version of docker has been compiled without memory limit support.") - } if err := daemon(*pidfile); err != nil { log.Fatal(err) } diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index 20e7ee140d..48786fd5be 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -46,8 +46,6 @@ func layerArchive(tarfile string) (io.Reader, error) { } func init() { - NO_MEMORY_LIMIT = os.Getenv("NO_MEMORY_LIMIT") == "1" - // Hack to run sys init during unit testing if SelfPath() == "/sbin/init" { SysInit() From 4bb07b273a64c12d4256db3c71ed37b3b5f1b23a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 18 Apr 2013 21:08:33 -0700 Subject: [PATCH 60/98] Improve the docker version output Upstream-commit: 2d32ac8cffe08b9c5d562e6ea30c796ae32b8fe1 Component: engine --- components/engine/commands.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 274acf4992..2a73da7f6c 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -183,10 +183,14 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string // 'docker version': show version information func (srv *Server) CmdVersion(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - fmt.Fprintf(stdout, "Version:%s\n", VERSION) - fmt.Fprintf(stdout, "Git Commit:%s\n", GIT_COMMIT) - if NO_MEMORY_LIMIT { - fmt.Fprintf(stdout, "Memory limit disabled\n") + fmt.Fprintf(stdout, "Version: %s\n", VERSION) + fmt.Fprintf(stdout, "Git Commit: %s\n", GIT_COMMIT) + fmt.Fprintf(stdout, "Kernel: %s\n", srv.runtime.kernelVersion) + if !srv.runtime.capabilities.MemoryLimit { + fmt.Fprintf(stdout, "WARNING: No memory limit support\n") + } + if !srv.runtime.capabilities.SwapLimit { + fmt.Fprintf(stdout, "WARNING: No swap limit support\n") } return nil } From be5c830b2c17cbb4724e803f2b0a97845cee0abf Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 18 Apr 2013 21:34:34 -0700 Subject: [PATCH 61/98] Add unit tests for CompareKernelVersion Upstream-commit: c42a4179fc6954a2363b181969978641553955c4 Component: engine --- components/engine/utils_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/components/engine/utils_test.go b/components/engine/utils_test.go index c15084f61e..1ee223ee3c 100644 --- a/components/engine/utils_test.go +++ b/components/engine/utils_test.go @@ -228,3 +228,36 @@ func assertIndexGet(t *testing.T, index *TruncIndex, input, expectedResult strin t.Fatalf("Getting '%s' returned '%s' instead of '%s'", input, result, expectedResult) } } + +func assertKernelVersion(t *testing.T, a, b *KernelVersionInfo, result int) { + if r := CompareKernelVersion(a, b); r != result { + t.Fatalf("Unepected kernel version comparaison result. Found %d, expected %d", r, result) + } +} + +func TestCompareKernelVersion(t *testing.T) { + assertKernelVersion(t, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + 0) + assertKernelVersion(t, + &KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0, Specific: 0}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + -1) + assertKernelVersion(t, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + &KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0, Specific: 0}, + 1) + assertKernelVersion(t, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 16}, + -1) + assertKernelVersion(t, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 5, Specific: 0}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + 1) + assertKernelVersion(t, + &KernelVersionInfo{Kernel: 3, Major: 0, Minor: 20, Specific: 25}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + -1) +} From b4f4a1379123ed7497f7854d22478cf169b3f77a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 18 Apr 2013 21:57:58 -0700 Subject: [PATCH 62/98] Use mount to determine the cgroup mountpoint Upstream-commit: f3e89fae287778cb8b7056e228170e0073c9a046 Component: engine --- components/engine/runtime.go | 11 ++++++++--- components/engine/utils.go | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 43b1a7815e..ca850d347c 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -305,11 +305,16 @@ func NewRuntime() (*Runtime, error) { log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String()) } - _, err1 := ioutil.ReadFile("/sys/fs/cgroup/memory/memory.limit_in_bytes") - _, err2 := ioutil.ReadFile("/sys/fs/cgroup/memory/memory.soft_limit_in_bytes") + cgroupMemoryMountpoint, err := FindCgroupMountpoint("memory") + if err != nil { + return nil, err + } + + _, err1 := ioutil.ReadFile(path.Join(cgroupMemoryMountpoint, "/memory.limit_in_bytes")) + _, err2 := ioutil.ReadFile(path.Join(cgroupMemoryMountpoint, "memory.soft_limit_in_bytes")) runtime.capabilities.MemoryLimit = err1 == nil && err2 == nil - _, err = ioutil.ReadFile("/sys/fs/cgroup/memory/memeory.memsw.limit_in_bytes") + _, err = ioutil.ReadFile(path.Join(cgroupMemoryMountpoint, "memeory.memsw.limit_in_bytes")) runtime.capabilities.SwapLimit = err == nil return runtime, nil diff --git a/components/engine/utils.go b/components/engine/utils.go index 8daf404481..8763a43933 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "runtime" "strconv" "strings" @@ -478,3 +479,20 @@ func CompareKernelVersion(a, b *KernelVersionInfo) int { } return 0 } + +func FindCgroupMountpoint(cgroupType string) (string, error) { + output, err := exec.Command("mount").CombinedOutput() + if err != nil { + return "", err + } + + reg := regexp.MustCompile(`^cgroup on (.*) type cgroup \(.*` + cgroupType + `[,\)]`) + for _, line := range strings.Split(string(output), "\n") { + r := reg.FindStringSubmatch(line) + if len(r) == 2 { + return r[1], nil + } + fmt.Printf("line: %s (%d)\n", line, len(r)) + } + return "", fmt.Errorf("cgroup mountpoint not found") +} From b79707c599974a8c8fa3e8bdd61e2ca4ef343322 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 18 Apr 2013 22:22:00 -0700 Subject: [PATCH 63/98] Fix examples in README to no longer rely on standalone mode Upstream-commit: 3ae5c45d9a75befdd144d689a70c32180b0a16c6 Component: engine --- components/engine/README.md | 25 +++++++++++++++---- .../sources/examples/running_examples.rst | 21 ++++------------ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 4ba9222f85..2c40658429 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -122,8 +122,26 @@ Some streamlined (but possibly outdated) installation paths' are available from Usage examples ============== -Running an interactive shell ----------------------------- +First run the docker daemon +--------------------------- + +All the examples assume your machine is running the docker daemon. To run the docker daemon in the background, simply type: + + .. code-block:: bash + + sudo docker -d & + +Now you can run docker in client mode: all commands will be forwarded to the docker daemon, so the client +can run from any account. + + .. code-block:: bash + + # now you can run docker commands from any account. + docker help + + +Throwaway shell in a base ubuntu image +-------------------------------------- ```bash # Download a base image @@ -145,9 +163,6 @@ Starting a long-running worker process -------------------------------------- ```bash -# Run docker in daemon mode -(docker -d || echo "Docker daemon already running") & - # Start a very useful long-running process JOB=$(docker run -d base /bin/sh -c "while true; do echo Hello world; sleep 1; done") diff --git a/components/engine/docs/sources/examples/running_examples.rst b/components/engine/docs/sources/examples/running_examples.rst index 4042add487..3d2593c710 100644 --- a/components/engine/docs/sources/examples/running_examples.rst +++ b/components/engine/docs/sources/examples/running_examples.rst @@ -7,27 +7,16 @@ Running The Examples -------------------- -There are two ways to run docker, daemon mode and standalone mode. - -When you run the docker command it will first check if there is a docker daemon running in the background it can connect to. - -* If it exists it will use that daemon to run all of the commands. -* If it does not exist docker will run in standalone mode (docker will exit after each command). - -Docker needs to be run from a privileged account (root). - -1. The most common (and recommended) way is to run a docker daemon as root in the background, and then connect to it from the docker client from any account. +All the examples assume your machine is running the docker daemon. To run the docker daemon in the background, simply type: .. code-block:: bash - # starting docker daemon in the background sudo docker -d & - # now you can run docker commands from any account. - docker - -2. Standalone: You need to run every command as root, or using sudo +Now you can run docker in client mode: all commands will be forwarded to the docker daemon, so the client +can run from any account. .. code-block:: bash - sudo docker + # now you can run docker commands from any account. + docker help From a59816d9a8240c4ec8444b5bad838ee8869771f6 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 18 Apr 2013 22:24:29 -0700 Subject: [PATCH 64/98] Add examples to the README Upstream-commit: 79a78d37e7102f0aadfa49d89b84b095e86824d1 Component: engine --- components/engine/README.md | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 2c40658429..a3fe1313ad 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -144,12 +144,11 @@ Throwaway shell in a base ubuntu image -------------------------------------- ```bash -# Download a base image -docker pull base +docker pull ubuntu:12.10 -# Run an interactive shell in the base image, +# Run an interactive shell # allocate a tty, attach stdin and stdout -docker run -i -t base /bin/bash +docker run -i -t ubuntu:12.10 /bin/bash ``` Detaching from the interactive shell @@ -164,7 +163,7 @@ Starting a long-running worker process ```bash # Start a very useful long-running process -JOB=$(docker run -d base /bin/sh -c "while true; do echo Hello world; sleep 1; done") +JOB=$(docker run -d ubuntu /bin/sh -c "while true; do echo Hello world; sleep 1; done") # Collect the output of the job so far docker logs $JOB @@ -173,21 +172,27 @@ docker logs $JOB docker kill $JOB ``` - -Listing all running containers ------------------------------- +Run an irc bouncer +------------------ ```bash -docker ps +BOUNCER_ID=$(docker run -d -p 6667 -u irc shykes/znc $USER $PASSWORD) +echo "Configure your irc client to connect to port $(port $BOUNCER_ID 6667) of this machine" ``` +Run Redis +--------- + +```bash +REDIS_ID=$(docker run -d -p 6379 shykes/redis redis-server) +echo "Configure your redis client to connect to port $(port $REDIS_ID 6379) of this machine" +``` Share your own image! --------------------- ```bash -docker pull base -CONTAINER=$(docker run -d base apt-get install -y curl) +CONTAINER=$(docker run -d ubuntu:12.10 apt-get install -y curl) docker commit -m "Installed curl" $CONTAINER $USER/betterbase docker push $USER/betterbase ``` From a5929adc352101372622c3a689971e9310cd6a39 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 18 Apr 2013 22:24:52 -0700 Subject: [PATCH 65/98] Small wording fix in README Upstream-commit: 3d2fd8a65024a503456fc8cd64d4d2565c73e304 Component: engine --- components/engine/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/README.md b/components/engine/README.md index a3fe1313ad..420c79500f 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -154,7 +154,7 @@ docker run -i -t ubuntu:12.10 /bin/bash Detaching from the interactive shell ------------------------------------ ``` -# In order to detach without killing the shell, you can use the escape sequence Ctrl-p + Ctrl-q +# To detach without killing the shell, you can use the escape sequence Ctrl-p + Ctrl-q # Note: this works only in tty mode (run with -t option). ``` From 338f50deb70312ae111dff2d66ff86a3ff9fa8bf Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 18 Apr 2013 22:37:45 -0700 Subject: [PATCH 66/98] Cleanup examples on README Upstream-commit: e8a67f632ea2e83c24f32c243b72fd7653e94d4b Component: engine --- components/engine/README.md | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 420c79500f..54fe0a22b2 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -127,17 +127,17 @@ First run the docker daemon All the examples assume your machine is running the docker daemon. To run the docker daemon in the background, simply type: - .. code-block:: bash +```bash +# On a production system you want this running in an init script +sudo docker -d & +``` - sudo docker -d & +Now you can run docker in client mode: all commands will be forwarded to the docker daemon, so the client can run from any account. -Now you can run docker in client mode: all commands will be forwarded to the docker daemon, so the client -can run from any account. - - .. code-block:: bash - - # now you can run docker commands from any account. - docker help +```bash +# Now you can run docker commands from any account. +docker help +``` Throwaway shell in a base ubuntu image @@ -146,18 +146,11 @@ Throwaway shell in a base ubuntu image ```bash docker pull ubuntu:12.10 -# Run an interactive shell -# allocate a tty, attach stdin and stdout +# Run an interactive shell, allocate a tty, attach stdin and stdout +# To detach the tty without exiting the shell, use the escape sequence Ctrl-p + Ctrl-q docker run -i -t ubuntu:12.10 /bin/bash ``` -Detaching from the interactive shell ------------------------------------- -``` -# To detach without killing the shell, you can use the escape sequence Ctrl-p + Ctrl-q -# Note: this works only in tty mode (run with -t option). -``` - Starting a long-running worker process -------------------------------------- @@ -172,16 +165,16 @@ docker logs $JOB docker kill $JOB ``` -Run an irc bouncer ------------------- +Running an irc bouncer +---------------------- ```bash BOUNCER_ID=$(docker run -d -p 6667 -u irc shykes/znc $USER $PASSWORD) echo "Configure your irc client to connect to port $(port $BOUNCER_ID 6667) of this machine" ``` -Run Redis ---------- +Running Redis +------------- ```bash REDIS_ID=$(docker run -d -p 6379 shykes/redis redis-server) From 11280be410e3e310dbfac7b0b85263fd03f0bd03 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 19 Apr 2013 12:39:40 -0700 Subject: [PATCH 67/98] Fix ubuntu packaging to build from a clean checkout of the correct git tag Upstream-commit: 152a9f77b4475bcec95ac58229c5db0e24b42557 Component: engine --- components/engine/packaging/ubuntu/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/engine/packaging/ubuntu/Makefile b/components/engine/packaging/ubuntu/Makefile index 8e0634870e..dbdf1af7a9 100644 --- a/components/engine/packaging/ubuntu/Makefile +++ b/components/engine/packaging/ubuntu/Makefile @@ -14,6 +14,7 @@ GITHUB_PATH=github.com/dotcloud/docker DOCKER_VERSION=${PKG_NAME}_${VERSION} DOCKER_FVERSION=${PKG_NAME}_$(shell head -1 changelog | sed 's/^.\+(\(.\+\)).\+$$/\1/') BUILD_SRC=${CURDIR}/../../build_src +VERSION_TAG=v$(shell head -1 changelog | sed 's/^.\+(\(.\+\)-[0-9]\+).\+$$/\1/') all: # Compile docker. Used by dpkg-buildpackage. @@ -33,7 +34,8 @@ ubuntu: # This Makefile will compile the github master branch of dotcloud/docker # Retrieve docker project and its go structure from internet rm -rf ${BUILD_SRC} - GOPATH=${BUILD_SRC} go get ${GITHUB_PATH} + git clone $(shell git rev-parse --show-toplevel) ${BUILD_SRC}/${GITHUB_PATH} + cd ${BUILD_SRC}/${GITHUB_PATH}; git checkout ${VERSION_TAG} && GOPATH=${BUILD_SRC} go get -d # Add debianization mkdir ${BUILD_SRC}/debian cp Makefile ${BUILD_SRC} From f7536bf8027fad330dc9769acb25297dde3a17dd Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 19 Apr 2013 12:55:17 -0700 Subject: [PATCH 68/98] Fixed 'hack' rule in Makefile Upstream-commit: e81ddb2dc76ab2e502ad35363a436dd823289598 Component: engine --- components/engine/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index a4096b1bbb..ab5682a09f 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -26,7 +26,7 @@ DOCKER_MAIN := $(DOCKER_DIR)/docker DOCKER_BIN_RELATIVE := bin/docker DOCKER_BIN := $(CURDIR)/$(DOCKER_BIN_RELATIVE) -.PHONY: all clean test +.PHONY: all clean test hack all: $(DOCKER_BIN) From 6dd897a9210d53fc1e0d1c19b4e3d57565a05028 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 19 Apr 2013 15:55:34 -0700 Subject: [PATCH 69/98] contrib/vagrant-docker: a placeholder to centralize collaboration on an official docker provider for vagrant Upstream-commit: d8416539b3b068a6a1a17f6dba441913b648a774 Component: engine --- components/engine/contrib/vagrant-docker/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 components/engine/contrib/vagrant-docker/README.md diff --git a/components/engine/contrib/vagrant-docker/README.md b/components/engine/contrib/vagrant-docker/README.md new file mode 100644 index 0000000000..5852ea1927 --- /dev/null +++ b/components/engine/contrib/vagrant-docker/README.md @@ -0,0 +1,3 @@ +# Vagrant-docker + +This is a placeholder for the official vagrant-docker, a plugin for Vagrant (http://vagrantup.com) which exposes Docker as a provider. From 2c563ad6e869486890271409b523542ca2c702d1 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 19 Apr 2013 16:33:25 -0700 Subject: [PATCH 70/98] Use aufs to handle parents whitouts instead of doing it manually Upstream-commit: e49af5b6def4e69ae234549fbcd2dd2987949cab Component: engine --- components/engine/image.go | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/components/engine/image.go b/components/engine/image.go index 9369fc3f48..3b5b4be6ea 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -92,7 +92,7 @@ func MountAUFS(ro []string, rw string, target string) error { rwBranch := fmt.Sprintf("%v=rw", rw) roBranches := "" for _, layer := range ro { - roBranches += fmt.Sprintf("%v=ro:", layer) + roBranches += fmt.Sprintf("%v=ro+wh:", layer) } branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) @@ -127,34 +127,9 @@ func (image *Image) Mount(root, rw string) error { if err := os.Mkdir(rw, 0755); err != nil && !os.IsExist(err) { return err } - // FIXME: @creack shouldn't we do this after going over changes? if err := MountAUFS(layers, rw, root); err != nil { return err } - // FIXME: Create tests for deletion - // FIXME: move this part to change.go - // Retrieve the changeset from the parent and apply it to the container - // - Retrieve the changes - changes, err := Changes(layers, layers[0]) - if err != nil { - return err - } - // Iterate on changes - for _, c := range changes { - // If there is a delete - if c.Kind == ChangeDelete { - // Make sure the directory exists - file_path, file_name := path.Dir(c.Path), path.Base(c.Path) - if err := os.MkdirAll(path.Join(rw, file_path), 0755); err != nil { - return err - } - // And create the whiteout (we just need to create empty file, discard the return) - if _, err := os.Create(path.Join(path.Join(rw, file_path), - ".wh."+path.Base(file_name))); err != nil { - return err - } - } - } return nil } From d01b620bcbcdafffbea54a37be29e92f57a7fcc8 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 19 Apr 2013 18:06:13 -0700 Subject: [PATCH 71/98] Wait for the container terminate at the end of CmdRun Fixes the race condition between docker run and docker logs from #428. Upstream-commit: 3bfb70db243d35b969eba9781dec619b7c52be98 Component: engine --- components/engine/commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/commands.go b/components/engine/commands.go index 2ca026eb81..4fe91bb8eb 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -975,6 +975,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...s } Debugf("Waiting for attach to return\n") <-attachErr + container.Wait() // Expecting I/O pipe error, discarding return nil } From 00b1b1ebe575910d1ff798e84721c116aed54db4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 19 Apr 2013 17:51:41 -0700 Subject: [PATCH 72/98] update TestRunDisconnectTty to reflect the correct behavior of CmdRun Upstream-commit: cc5a044a8c140e32868bb0f312f3780161515f0b Component: engine --- components/engine/commands_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/components/engine/commands_test.go b/components/engine/commands_test.go index 1be4a2abe0..9615e877e6 100644 --- a/components/engine/commands_test.go +++ b/components/engine/commands_test.go @@ -239,11 +239,8 @@ func TestRunDisconnectTty(t *testing.T) { t.Fatal(err) } - // as the pipes are close, we expect the process to die, - // therefore CmdRun to unblock. Wait for CmdRun - setTimeout(t, "Waiting for CmdRun timed out", 2*time.Second, func() { - <-c1 - }) + // In tty mode, we expect the process to stay alive even after client's stdin closes. + // Do not wait for run to finish // Client disconnect after run -i should keep stdin out in TTY mode container := runtime.List()[0] From 82eedaa832d5cd827d47f7745a8c810e75aed0ce Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 19 Apr 2013 19:29:13 -0700 Subject: [PATCH 73/98] 'docker ps' shows port mappings Upstream-commit: 931ca464a7db038fa78ee51a3f5c4cbe85db1e21 Component: engine --- components/engine/commands.go | 3 ++- components/engine/container.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 4fe91bb8eb..7f82775690 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -677,7 +677,7 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) } w := tabwriter.NewWriter(stdout, 12, 1, 3, ' ', 0) if !*quiet { - fmt.Fprintln(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tCOMMENT") + fmt.Fprintln(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tCOMMENT\tPORTS") } for i, container := range srv.runtime.List() { if !container.State.Running && !*flAll && *nLast == -1 { @@ -698,6 +698,7 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) /* CREATED */ HumanDuration(time.Now().Sub(container.Created)) + " ago", /* STATUS */ container.State.String(), /* COMMENT */ "", + /* PORTS */ container.NetworkSettings.PortMappingHuman(), } { if idx == 0 { w.Write([]byte(field)) diff --git a/components/engine/container.go b/components/engine/container.go index dc04a91609..4719e9f1cf 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -11,7 +11,9 @@ import ( "os" "os/exec" "path" + "sort" "strconv" + "strings" "syscall" "time" ) @@ -150,6 +152,16 @@ type NetworkSettings struct { PortMapping map[string]string } +// String returns a human-readable description of the port mapping defined in the settings +func (settings *NetworkSettings) PortMappingHuman() string { + var mapping []string + for private, public := range settings.PortMapping { + mapping = append(mapping, fmt.Sprintf("%s->%s", public, private)) + } + sort.Strings(mapping) + return strings.Join(mapping, ", ") +} + func (container *Container) Cmd() *exec.Cmd { return container.cmd } From 66366b4f1d1289e63c5584a14ef23da3e93c95b0 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 19 Apr 2013 19:32:32 -0700 Subject: [PATCH 74/98] Exclude loopback-to-loopback connections from DNAT rules, to allow userland proxying Upstream-commit: 61259ab4b4bfe3404e75dd811a2da7c88e7c7133 Component: engine --- components/engine/network.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/network.go b/components/engine/network.go index 706c31fa4e..85c6083316 100644 --- a/components/engine/network.go +++ b/components/engine/network.go @@ -188,7 +188,8 @@ type PortMapper struct { func (mapper *PortMapper) cleanup() error { // Ignore errors - This could mean the chains were never set up iptables("-t", "nat", "-D", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER") - iptables("-t", "nat", "-D", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER") + iptables("-t", "nat", "-D", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8", "-j", "DOCKER") + iptables("-t", "nat", "-D", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER") // Created in versions <= 0.1.6 // Also cleanup rules created by older versions, or -X might fail. iptables("-t", "nat", "-D", "PREROUTING", "-j", "DOCKER") iptables("-t", "nat", "-D", "OUTPUT", "-j", "DOCKER") @@ -205,7 +206,7 @@ func (mapper *PortMapper) setup() error { if err := iptables("-t", "nat", "-A", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER"); err != nil { return fmt.Errorf("Failed to inject docker in PREROUTING chain: %s", err) } - if err := iptables("-t", "nat", "-A", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER"); err != nil { + if err := iptables("-t", "nat", "-A", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8", "-j", "DOCKER"); err != nil { return fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err) } return nil From 1cb0701d9b0197028fe05625f53dda0881bdb893 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 19 Apr 2013 19:35:44 -0700 Subject: [PATCH 75/98] Emulate DNAT in userland for loopback-to-loopback connections. This makes container ports available from localhost. Upstream-commit: 930e9a7e430a3d78e09a95bb32d9fb6052e7dae1 Component: engine --- components/engine/network.go | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/components/engine/network.go b/components/engine/network.go index 85c6083316..54b8dbfe31 100644 --- a/components/engine/network.go +++ b/components/engine/network.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "errors" "fmt" + "io" "log" "net" "os/exec" @@ -221,10 +222,55 @@ func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error { if err := mapper.iptablesForward("-A", port, dest); err != nil { return err } + mapper.mapping[port] = dest + listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + mapper.Unmap(port) + return err + } + // FIXME: store the listener so we can close it at Unmap + go proxy(listener, "tcp", dest.String()) return nil } +// proxy listens for socket connections on `listener`, and forwards them unmodified +// to `proto:address` +func proxy(listener net.Listener, proto, address string) error { + Debugf("proxying to %s:%s", proto, address) + defer Debugf("Done proxying to %s:%s", proto, address) + for { + Debugf("Listening on %s", listener) + src, err := listener.Accept() + if err != nil { + return err + } + Debugf("Connecting to %s:%s", proto, address) + dst, err := net.Dial(proto, address) + if err != nil { + log.Printf("Error connecting to %s:%s: %s", proto, address, err) + src.Close() + continue + } + Debugf("Connected to backend, splicing") + splice(src, dst) + } + return nil +} + +func halfSplice(dst, src net.Conn) error { + _, err := io.Copy(dst, src) + // FIXME: on EOF from a tcp connection, pass WriteClose() + dst.Close() + src.Close() + return err +} + +func splice(a, b net.Conn) { + go halfSplice(a, b) + go halfSplice(b, a) +} + func (mapper *PortMapper) Unmap(port int) error { dest, ok := mapper.mapping[port] if !ok { From 2d0d73c6f31a36ae08e4271c62f95cf968f496af Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 19 Apr 2013 20:44:25 -0700 Subject: [PATCH 76/98] Shutdown loopback-to-loopback proxy when unmapping a port Upstream-commit: 7f1a32b9ff31bd931e9495acc1d5ccdef4bd51b6 Component: engine --- components/engine/network.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/components/engine/network.go b/components/engine/network.go index 54b8dbfe31..373625d59c 100644 --- a/components/engine/network.go +++ b/components/engine/network.go @@ -184,6 +184,7 @@ func getIfaceAddr(name string) (net.Addr, error) { // It keeps track of all mappings and is able to unmap at will type PortMapper struct { mapping map[int]net.TCPAddr + proxies map[int]net.Listener } func (mapper *PortMapper) cleanup() error { @@ -197,6 +198,7 @@ func (mapper *PortMapper) cleanup() error { iptables("-t", "nat", "-F", "DOCKER") iptables("-t", "nat", "-X", "DOCKER") mapper.mapping = make(map[int]net.TCPAddr) + mapper.proxies = make(map[int]net.Listener) return nil } @@ -229,7 +231,7 @@ func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error { mapper.Unmap(port) return err } - // FIXME: store the listener so we can close it at Unmap + mapper.proxies[port] = listener go proxy(listener, "tcp", dest.String()) return nil } @@ -276,6 +278,10 @@ func (mapper *PortMapper) Unmap(port int) error { if !ok { return errors.New("Port is not mapped") } + if proxy, exists := mapper.proxies[port]; exists { + proxy.Close() + delete(mapper.proxies, port) + } if err := mapper.iptablesForward("-D", port, dest); err != nil { return err } From 5fe93d40b13618536b9151a7442e72c86a3560a4 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 19 Apr 2013 20:46:07 -0700 Subject: [PATCH 77/98] Add a test for allocating tcp ports and reaching them on localhost Upstream-commit: 911925b54a42fffa0fed8bac2a3eeba14bf3ec4b Component: engine --- components/engine/runtime_test.go | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index 20e7ee140d..eef8db3a8f 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -1,9 +1,11 @@ package docker import ( + "fmt" "github.com/dotcloud/docker/rcli" "io" "io/ioutil" + "net" "os" "os/exec" "os/user" @@ -254,6 +256,47 @@ func TestGet(t *testing.T) { } +// Run a container with a TCP port allocated, and test that it can receive connections on localhost +func TestAllocatePortLocalhost(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"sh", "-c", "echo well hello there | nc -l -p 5555"}, + PortSpecs: []string{"5555"}, + }, + ) + if err != nil { + t.Fatal(err) + } + if err := container.Start(); err != nil { + t.Fatal(err) + } + defer container.Kill() + time.Sleep(300 * time.Millisecond) // Wait for the container to run + conn, err := net.Dial("tcp", + fmt.Sprintf( + "localhost:%s", container.NetworkSettings.PortMapping["5555"], + ), + ) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + output, err := ioutil.ReadAll(conn) + if err != nil { + t.Fatal(err) + } + if string(output) != "well hello there\n" { + t.Fatalf("Received wrong output from network connection: should be '%s', not '%s'", + "well hello there\n", + string(output), + ) + } +} + func TestRestore(t *testing.T) { root, err := ioutil.TempDir("", "docker-test") From a5e52704ac029ff1bb4045e65de6464866659a80 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 20 Apr 2013 17:26:50 -0700 Subject: [PATCH 78/98] Cleaned up install instructions in the README * Addded quick install on ubuntu as the 1st install option * Grouped other binary installs under "binary installs" * Removed duplicate binary ubuntu installs (linked to the docs) * Improved "build from source" instructions Upstream-commit: c40f01319f741ef895090334ffa637f099c529b3 Component: engine --- components/engine/README.md | 47 ++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 54fe0a22b2..774e107a5b 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -53,29 +53,48 @@ Under the hood, Docker is built on the following components: Install instructions ================== -Building from source --------------------- +Quick install on Ubuntu 12.04 and 12.10 +--------------------------------------- -1. Make sure you have a [Go language](http://golang.org) compiler. +```bash +curl get.docker.io | sh -x +``` - On a Debian/wheezy or Ubuntu 12.10 install the package: +Binary installs +---------------- - ```bash +Docker supports the following binary installation methods. +Note that some methods are community contributions and not yet officially supported. - $ sudo apt-get install golang-go - ``` +* [Ubuntu 12.04 and 12.10 (officially supported)](http://docs.docker.io/en/latest/installation/ubuntulinux/) +* [Arch Linux](http://docs.docker.io/en/latest/installation/archlinux/) +* [MacOS X (with Vagrant)](http://docs.docker.io/en/latest/installation/macos/) +* [Windows (with Vagrant)](http://docs.docker.io/en/latest/installation/windows/) +* [Amazon EC2 (with Vagrant)](http://docs.docker.io/en/latest/installation/amazon/) -2. Execute ``make`` +Installing from source +---------------------- - This command will install all necessary dependencies and build the - executable that you can find in ``bin/docker`` +1. Make sure you have a [Go language](http://golang.org/doc/install) compiler and [git](http://git-scm.com) installed. -3. Should you like to see what's happening, run ``make`` with ``VERBOSE=1`` parameter: +2. Checkout the source code - ```bash + ```bash + git clone http://github.com/dotcloud/docker + ``` - $ make VERBOSE=1 - ``` +3. Build the docker binary + + ```bash + cd docker + make VERBOSE=1 + ``` + +4. Execute the docker daemon + + ```bash + sudo ./bin/docker -d + ``` Installing on Ubuntu 12.04 and 12.10 ------------------------------------ From eec939c8e4e64f318ef7b8fe9c009eded31d173b Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 20 Apr 2013 17:29:12 -0700 Subject: [PATCH 79/98] Removed duplicate ubuntu binary install instructions from README Upstream-commit: 70cf467fdf8c4809429089ceae0f3ac1b3a6f729 Component: engine --- components/engine/README.md | 49 +------------------------------------ 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 774e107a5b..d112575632 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -88,56 +88,9 @@ Installing from source ```bash cd docker make VERBOSE=1 + sudo cp ./bin/docker /usr/local/bin/docker ``` -4. Execute the docker daemon - - ```bash - sudo ./bin/docker -d - ``` - -Installing on Ubuntu 12.04 and 12.10 ------------------------------------- - -1. Install dependencies: - - ```bash - sudo apt-get install lxc bsdtar - sudo apt-get install linux-image-extra-`uname -r` - ``` - - The `linux-image-extra` package is needed on standard Ubuntu EC2 AMIs in order to install the aufs kernel module. - -2. Install the latest docker binary: - - ```bash - wget http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz - tar -xf docker-master.tgz - ``` - -3. Run your first container! - - ```bash - cd docker-master - sudo ./docker pull base - sudo ./docker run -i -t base /bin/bash - ``` - - Consider adding docker to your `PATH` for simplicity. - -Installing on other Linux distributions ---------------------------------------- - -Right now, the officially supported distributions are: - -* Ubuntu 12.04 (precise LTS) -* Ubuntu 12.10 (quantal) - -Docker probably works on other distributions featuring a recent kernel, the AUFS patch, and up-to-date lxc. However this has not been tested. - -Some streamlined (but possibly outdated) installation paths' are available from the website: http://docker.io/documentation/ - - Usage examples ============== From 1f72ca9771ae9f7fc55abf02a71db06b2bc72b15 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 20 Apr 2013 17:29:41 -0700 Subject: [PATCH 80/98] Link to public images list in the README Upstream-commit: 28831a412fa51094124f17d242908f308a2a973a Component: engine --- components/engine/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/README.md b/components/engine/README.md index d112575632..22359c9269 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -162,6 +162,7 @@ docker commit -m "Installed curl" $CONTAINER $USER/betterbase docker push $USER/betterbase ``` +A list of publicly available images is [available here](https://github.com/dotcloud/docker/wiki/Public-docker-images). Expose a service on a TCP port ------------------------------ From 3aa701de71db2dac3800d2c67e5cb8a35bd26aa8 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 20 Apr 2013 17:30:33 -0700 Subject: [PATCH 81/98] Formatting fix in ubuntu install doc Upstream-commit: 4a9c3a92e1552fab5702eeddc0307f7f87f96496 Component: engine --- components/engine/docs/sources/installation/ubuntulinux.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index bf83a5bc88..4d777d52ab 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -11,9 +11,8 @@ Installing on Ubuntu 12.04 and 12.10 Right now, the officially supported distributions are: -Ubuntu 12.04 (precise LTS) -Ubuntu 12.10 (quantal) -Docker probably works on other distributions featuring a recent kernel, the AUFS patch, and up-to-date lxc. However this has not been tested. +* Ubuntu 12.04 (precise LTS) +* Ubuntu 12.10 (quantal) Install dependencies: --------------------- From 4b4475fccb6b66959e06802dec811f58d1ff9ea0 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 20 Apr 2013 17:31:49 -0700 Subject: [PATCH 82/98] Fix Ubuntu install doc Upstream-commit: 1201c418cd6f02ca8c206cfdd4659b8ec2d2f856 Component: engine --- .../docs/sources/installation/ubuntulinux.rst | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index 4d777d52ab..306e43813d 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -19,37 +19,38 @@ Install dependencies: :: - sudo apt-get install lxc wget bsdtar curl + sudo apt-get install lxc bsdtar sudo apt-get install linux-image-extra-`uname -r` The linux-image-extra package is needed on standard Ubuntu EC2 AMIs in order to install the aufs kernel module. -Install the latest docker binary: +Install the docker binary +------------------------- :: - wget http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz + wget http://get.docker.io/builds/Linux/x86_64/docker-master.tgz tar -xf docker-master.tgz + sudo cp ./docker-master /usr/local/bin + +Note: docker currently only supports 64-bit Linux hosts. + + +Run the docker daemon +--------------------- + +:: + + sudo docker -d & Run your first container! +------------------------- :: - - cd docker-master - -:: - - sudo ./docker run -i -t base /bin/bash + docker run -i -t ubuntu /bin/bash -To run docker as a daemon, in the background, and allow non-root users to run ``docker`` start -docker -d - -:: - - sudo ./docker -d & - - -Consider adding docker to your PATH for simplicity. +Check out more examples +----------------------- Continue with the :ref:`hello_world` example. \ No newline at end of file From 3440da267601c247247a8b203c2072ea4c9b3fe1 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 20 Apr 2013 17:33:02 -0700 Subject: [PATCH 83/98] Moved 'under the hood' below install instructions and examples in README Upstream-commit: d49a273071eaa579f59799b679a9f7b316e28fb8 Component: engine --- components/engine/README.md | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 22359c9269..058004c078 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -33,23 +33,6 @@ Notable features * Interactive shell: docker can allocate a pseudo-tty and attach to the standard input of any container, for example to run a throwaway interactive shell. - - -Under the hood --------------- - -Under the hood, Docker is built on the following components: - - -* The [cgroup](http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-24-c) and [namespacing](http://blog.dotcloud.com/under-the-hood-linux-kernels-on-dotcloud-part) capabilities of the Linux kernel; - -* [AUFS](http://aufs.sourceforge.net/aufs.html), a powerful union filesystem with copy-on-write capabilities; - -* The [Go](http://golang.org) programming language; - -* [lxc](http://lxc.sourceforge.net/), a set of convenience scripts to simplify the creation of linux containers. - - Install instructions ================== @@ -183,6 +166,22 @@ echo hello world | nc $IP $PORT echo "Daemon received: $(docker logs $JOB)" ``` +Under the hood +-------------- + +Under the hood, Docker is built on the following components: + + +* The [cgroup](http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-24-c) and [namespacing](http://blog.dotcloud.com/under-the-hood-linux-kernels-on-dotcloud-part) capabilities of the Linux kernel; + +* [AUFS](http://aufs.sourceforge.net/aufs.html), a powerful union filesystem with copy-on-write capabilities; + +* The [Go](http://golang.org) programming language; + +* [lxc](http://lxc.sourceforge.net/), a set of convenience scripts to simplify the creation of linux containers. + + + Contributing to Docker ====================== From 4295918675a6bca8c1833cf78a3fa4734fe191ec Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 20 Apr 2013 17:35:00 -0700 Subject: [PATCH 84/98] Bumped version to 0.1.7 Upstream-commit: 3b6c540fe8313035de0f0b00de52d7197f8bc665 Component: engine --- components/engine/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 7f82775690..fea1c109d8 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -18,7 +18,7 @@ import ( "unicode" ) -const VERSION = "0.1.6" +const VERSION = "0.1.7" var ( GIT_COMMIT string From 53559b1043b4264a1b71cad3bf6e0987c5ce33d0 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 20 Apr 2013 18:26:15 -0700 Subject: [PATCH 85/98] Better title in ubuntu install doc Upstream-commit: 52cedb8a05182c7d6f5e5918453cf356b1efafc8 Component: engine --- components/engine/docs/sources/installation/ubuntulinux.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index 306e43813d..6149a26f28 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -1,7 +1,7 @@ .. _ubuntu_linux: -Ubuntu Linux -============ +Installing on Ubuntu Linux +========================== **Please note this project is currently under heavy development. It should not be used in production.** @@ -53,4 +53,4 @@ Run your first container! Check out more examples ----------------------- -Continue with the :ref:`hello_world` example. \ No newline at end of file +Continue with the :ref:`hello_world` example. From 41f179ebb12a75090fda99cd2c262540e1a822fd Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 21 Apr 2013 14:23:55 -0700 Subject: [PATCH 86/98] 'docker push' buffers filesystem archives on disk instead of memory. Upstream-commit: baacae8345febd688579ac29832c200c41602ed2 Component: engine --- components/engine/archive.go | 36 +++++++++++++++++++++++++++++++++++ components/engine/graph.go | 24 ++++++++++++++++++++++- components/engine/image.go | 9 +++++++++ components/engine/registry.go | 21 +++++++++----------- 4 files changed, 77 insertions(+), 13 deletions(-) diff --git a/components/engine/archive.go b/components/engine/archive.go index d09d3d6b97..8a011eb6e1 100644 --- a/components/engine/archive.go +++ b/components/engine/archive.go @@ -4,6 +4,7 @@ import ( "errors" "io" "io/ioutil" + "os" "os/exec" ) @@ -86,3 +87,38 @@ func CmdStream(cmd *exec.Cmd) (io.Reader, error) { } return pipeR, nil } + +// NewTempArchive reads the content of src into a temporary file, and returns the contents +// of that file as an archive. The archive can only be read once - as soon as reading completes, +// the file will be deleted. +func NewTempArchive(src Archive, dir string) (*TempArchive, error) { + f, err := ioutil.TempFile(dir, "") + if err != nil { + return nil, err + } + if _, err := io.Copy(f, src); err != nil { + return nil, err + } + if _, err := f.Seek(0, 0); err != nil { + return nil, err + } + st, err := f.Stat() + if err != nil { + return nil, err + } + size := st.Size() + return &TempArchive{f, size}, nil +} + +type TempArchive struct { + *os.File + Size int64 // Pre-computed from Stat().Size() as a convenience +} + +func (archive *TempArchive) Read(data []byte) (int, error) { + n, err := archive.File.Read(data) + if err != nil { + os.Remove(archive.File.Name()) + } + return n, err +} diff --git a/components/engine/graph.go b/components/engine/graph.go index b7dbf2e113..d2692fc822 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -129,12 +129,30 @@ func (graph *Graph) Register(layerData Archive, img *Image) error { return nil } +// TempLayerArchive creates a temporary archive of the given image's filesystem layer. +// The archive is stored on disk and will be automatically deleted as soon as has been read. +func (graph *Graph) TempLayerArchive(id string, compression Compression) (*TempArchive, error) { + image, err := graph.Get(id) + if err != nil { + return nil, err + } + tmp, err := graph.tmp() + if err != nil { + return nil, err + } + archive, err := image.TarLayer(compression) + if err != nil { + return nil, err + } + return NewTempArchive(archive, tmp.Root) +} + // Mktemp creates a temporary sub-directory inside the graph's filesystem. func (graph *Graph) Mktemp(id string) (string, error) { if id == "" { id = GenerateId() } - tmp, err := NewGraph(path.Join(graph.Root, ":tmp:")) + tmp, err := graph.tmp() if err != nil { return "", fmt.Errorf("Couldn't create temp: %s", err) } @@ -144,6 +162,10 @@ func (graph *Graph) Mktemp(id string) (string, error) { return tmp.imageRoot(id), nil } +func (graph *Graph) tmp() (*Graph, error) { + return NewGraph(path.Join(graph.Root, ":tmp:")) +} + // Check if given error is "not empty". // Note: this is the way golang does it internally with os.IsNotExists. func isNotEmpty(err error) bool { diff --git a/components/engine/image.go b/components/engine/image.go index 9369fc3f48..403731d6e9 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -110,6 +110,15 @@ func MountAUFS(ro []string, rw string, target string) error { return nil } +// TarLayer returns a tar archive of the image's filesystem layer. +func (image *Image) TarLayer(compression Compression) (Archive, error) { + layerPath, err := image.layer() + if err != nil { + return nil, err + } + return Tar(layerPath, compression) +} + func (image *Image) Mount(root, rw string) error { if mounted, err := Mounted(root); err != nil { return err diff --git a/components/engine/registry.go b/components/engine/registry.go index 428db1b968..2f461cc8ec 100644 --- a/components/engine/registry.go +++ b/components/engine/registry.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "net/http" + "os" "path" "strings" ) @@ -269,24 +270,20 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth return fmt.Errorf("Failed to retrieve layer upload location: %s", err) } - // FIXME: Don't do this :D. Check the S3 requierement and implement chunks of 5MB - // FIXME2: I won't stress it enough, DON'T DO THIS! very high priority - layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) - tmp, err := ioutil.ReadAll(layerData2) + // FIXME: stream the archive directly to the registry instead of buffering it on disk. This requires either: + // a) Implementing S3's proprietary streaming logic, or + // b) Stream directly to the registry instead of S3. + // I prefer option b. because it doesn't lock us into a proprietary cloud service. + tmpLayer, err := graph.TempLayerArchive(img.Id, Xz) if err != nil { return err } - layerLength := len(tmp) - - layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) - if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) - } - req3, err := http.NewRequest("PUT", url.String(), ProgressReader(layerData.(io.ReadCloser), layerLength, stdout)) + defer os.Remove(tmpLayer.Name()) + req3, err := http.NewRequest("PUT", url.String(), ProgressReader(tmpLayer, int(tmpLayer.Size), stdout)) if err != nil { return err } - req3.ContentLength = int64(layerLength) + req3.ContentLength = int64(tmpLayer.Size) req3.TransferEncoding = []string{"none"} res3, err := client.Do(req3) From 262fafb31ff820c9f5820d8fa36d6a299c10e380 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 21 Apr 2013 15:29:26 -0700 Subject: [PATCH 87/98] 'docker push' shows an additional progress bar while it buffers the archive to disk. Fixes #451. Upstream-commit: 965e8a02d24a35aa7a7f64f2baf2b652a04f5983 Component: engine --- components/engine/commands.go | 2 +- components/engine/graph.go | 7 +++++-- components/engine/registry.go | 6 +++--- components/engine/utils.go | 31 +++++++++++++++++++++---------- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 2feb648c3b..1f951ff331 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -475,7 +475,7 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout rcli.DockerConn, args . if err != nil { return err } - archive = ProgressReader(resp.Body, int(resp.ContentLength), stdout) + archive = ProgressReader(resp.Body, int(resp.ContentLength), stdout, "Importing %v/%v (%v)") } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "") if err != nil { diff --git a/components/engine/graph.go b/components/engine/graph.go index d2692fc822..c0e5000913 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -2,6 +2,7 @@ package docker import ( "fmt" + "io" "io/ioutil" "os" "path" @@ -131,7 +132,9 @@ func (graph *Graph) Register(layerData Archive, img *Image) error { // TempLayerArchive creates a temporary archive of the given image's filesystem layer. // The archive is stored on disk and will be automatically deleted as soon as has been read. -func (graph *Graph) TempLayerArchive(id string, compression Compression) (*TempArchive, error) { +// If output is not nil, a human-readable progress bar will be written to it. +// FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives? +func (graph *Graph) TempLayerArchive(id string, compression Compression, output io.Writer) (*TempArchive, error) { image, err := graph.Get(id) if err != nil { return nil, err @@ -144,7 +147,7 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression) (*TempA if err != nil { return nil, err } - return NewTempArchive(archive, tmp.Root) + return NewTempArchive(ProgressReader(ioutil.NopCloser(archive), 0, output, "Buffering to disk %v/%v (%v)"), tmp.Root) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. diff --git a/components/engine/registry.go b/components/engine/registry.go index 2f461cc8ec..74b166906f 100644 --- a/components/engine/registry.go +++ b/components/engine/registry.go @@ -136,7 +136,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *a if err != nil { return nil, nil, err } - return img, ProgressReader(res.Body, int(res.ContentLength), stdout), nil + return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil } func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.AuthConfig) error { @@ -274,12 +274,12 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth // a) Implementing S3's proprietary streaming logic, or // b) Stream directly to the registry instead of S3. // I prefer option b. because it doesn't lock us into a proprietary cloud service. - tmpLayer, err := graph.TempLayerArchive(img.Id, Xz) + tmpLayer, err := graph.TempLayerArchive(img.Id, Xz, stdout) if err != nil { return err } defer os.Remove(tmpLayer.Name()) - req3, err := http.NewRequest("PUT", url.String(), ProgressReader(tmpLayer, int(tmpLayer.Size), stdout)) + req3, err := http.NewRequest("PUT", url.String(), ProgressReader(tmpLayer, int(tmpLayer.Size), stdout, "Uploading %v/%v (%v)")) if err != nil { return err } diff --git a/components/engine/utils.go b/components/engine/utils.go index 8763a43933..bb891b18a9 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -72,23 +72,30 @@ type progressReader struct { readTotal int // Expected stream length (bytes) readProgress int // How much has been read so far (bytes) lastUpdate int // How many bytes read at least update + template string // Template to print. Default "%v/%v (%v)" } func (r *progressReader) Read(p []byte) (n int, err error) { read, err := io.ReadCloser(r.reader).Read(p) r.readProgress += read - // Only update progress for every 1% read - updateEvery := int(0.01 * float64(r.readTotal)) - if r.readProgress-r.lastUpdate > updateEvery || r.readProgress == r.readTotal { - fmt.Fprintf(r.output, "%d/%d (%.0f%%)\r", - r.readProgress, - r.readTotal, - float64(r.readProgress)/float64(r.readTotal)*100) + updateEvery := 4096 + if r.readTotal > 0 { + // Only update progress for every 1% read + if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery { + updateEvery = increment + } + } + if r.readProgress-r.lastUpdate > updateEvery || err != nil { + if r.readTotal > 0 { + fmt.Fprintf(r.output, r.template+"\r", r.readProgress, r.readTotal, fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) + } else { + fmt.Fprintf(r.output, r.template+"\r", r.readProgress, "?", "n/a") + } r.lastUpdate = r.readProgress } // Send newline when complete - if err == io.EOF { + if err != nil { fmt.Fprintf(r.output, "\n") } @@ -97,8 +104,11 @@ func (r *progressReader) Read(p []byte) (n int, err error) { func (r *progressReader) Close() error { return io.ReadCloser(r.reader).Close() } -func ProgressReader(r io.ReadCloser, size int, output io.Writer) *progressReader { - return &progressReader{r, output, size, 0, 0} +func ProgressReader(r io.ReadCloser, size int, output io.Writer, template string) *progressReader { + if template == "" { + template = "%v/%v (%v)" + } + return &progressReader{r, output, size, 0, 0, template} } // HumanDuration returns a human-readable approximation of a duration @@ -395,6 +405,7 @@ type KernelVersionInfo struct { Specific int } +// FIXME: this doens't build on Darwin func GetKernelVersion() (*KernelVersionInfo, error) { var uts syscall.Utsname From 547af82c3d6d6664365a54dd05b10fb19d76f795 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 21 Apr 2013 19:19:38 -0600 Subject: [PATCH 88/98] Update utils.go to not enforce extra constraints on the kernel "flavor" (such as being integral or even comparable one to another) This is especially to fix the current docker on kernels such as gentoo-sources, where the "flavor" is the string "gentoo", and that obviously fails to be converted to an integer. Upstream-commit: 1f65c6bf4c9dcfc513eeb9da34a7cac60c6bc4ff Component: engine --- components/engine/utils.go | 28 ++++++++++------------------ components/engine/utils_test.go | 26 +++++++++++++------------- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/components/engine/utils.go b/components/engine/utils.go index bb891b18a9..c6a8c9465f 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -399,10 +399,10 @@ func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) } type KernelVersionInfo struct { - Kernel int - Major int - Minor int - Specific int + Kernel int + Major int + Minor int + Flavor string } // FIXME: this doens't build on Darwin @@ -445,21 +445,18 @@ func GetKernelVersion() (*KernelVersionInfo, error) { return nil, err } - specific, err := strconv.Atoi(strings.Split(tmp[1], "-")[0]) - if err != nil { - return nil, err - } + flavor := tmp[1] return &KernelVersionInfo{ - Kernel: kernel, - Major: major, - Minor: minor, - Specific: specific, + Kernel: kernel, + Major: major, + Minor: minor, + Flavor: flavor, }, nil } func (k *KernelVersionInfo) String() string { - return fmt.Sprintf("%d.%d.%d-%d", k.Kernel, k.Major, k.Minor, k.Specific) + return fmt.Sprintf("%d.%d.%d-%s", k.Kernel, k.Major, k.Minor, k.Flavor) } // Compare two KernelVersionInfo struct. @@ -483,11 +480,6 @@ func CompareKernelVersion(a, b *KernelVersionInfo) int { return 1 } - if a.Specific < b.Specific { - return -1 - } else if a.Specific > b.Specific { - return 1 - } return 0 } diff --git a/components/engine/utils_test.go b/components/engine/utils_test.go index 1ee223ee3c..aa2a1b9682 100644 --- a/components/engine/utils_test.go +++ b/components/engine/utils_test.go @@ -237,27 +237,27 @@ func assertKernelVersion(t *testing.T, a, b *KernelVersionInfo, result int) { func TestCompareKernelVersion(t *testing.T) { assertKernelVersion(t, - &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, - &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}, 0) assertKernelVersion(t, - &KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0, Specific: 0}, - &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + &KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}, -1) assertKernelVersion(t, - &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, - &KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0, Specific: 0}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}, + &KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0}, 1) assertKernelVersion(t, - &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, - &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 16}, - -1) + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "0"}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "16"}, + 0) assertKernelVersion(t, - &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 5, Specific: 0}, - &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 5}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}, 1) assertKernelVersion(t, - &KernelVersionInfo{Kernel: 3, Major: 0, Minor: 20, Specific: 25}, - &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Specific: 0}, + &KernelVersionInfo{Kernel: 3, Major: 0, Minor: 20, Flavor: "25"}, + &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "0"}, -1) } From 1e9641c3d76ca04d837cf52a859c4d59f8fbca8c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Apr 2013 00:44:57 -0400 Subject: [PATCH 89/98] Do not stop execution if cgroup mountpoint is not found Upstream-commit: 71b580661451c35f01ee3824506f572c52d86ac8 Component: engine --- components/engine/runtime.go | 26 +++++++++++++++----------- components/engine/utils.go | 3 +-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/components/engine/runtime.go b/components/engine/runtime.go index a12d0c92d0..b894a2cdaf 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -305,18 +305,22 @@ func NewRuntime() (*Runtime, error) { log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String()) } - cgroupMemoryMountpoint, err := FindCgroupMountpoint("memory") - if err != nil { - return nil, err + if cgroupMemoryMountpoint, err := FindCgroupMountpoint("memory"); err != nil { + log.Printf("WARNING: %s\n", err) + } else { + _, err1 := ioutil.ReadFile(path.Join(cgroupMemoryMountpoint, "memory.limit_in_bytes")) + _, err2 := ioutil.ReadFile(path.Join(cgroupMemoryMountpoint, "memory.soft_limit_in_bytes")) + runtime.capabilities.MemoryLimit = err1 == nil && err2 == nil + if !runtime.capabilities.MemoryLimit { + log.Printf("WARNING: Your kernel does not support cgroup memory limit.") + } + + _, err = ioutil.ReadFile(path.Join(cgroupMemoryMountpoint, "memory.memsw.limit_in_bytes")) + runtime.capabilities.SwapLimit = err == nil + if !runtime.capabilities.SwapLimit { + log.Printf("WARNING: Your kernel does not support cgroup swap limit.") + } } - - _, err1 := ioutil.ReadFile(path.Join(cgroupMemoryMountpoint, "/memory.limit_in_bytes")) - _, err2 := ioutil.ReadFile(path.Join(cgroupMemoryMountpoint, "memory.soft_limit_in_bytes")) - runtime.capabilities.MemoryLimit = err1 == nil && err2 == nil - - _, err = ioutil.ReadFile(path.Join(cgroupMemoryMountpoint, "memeory.memsw.limit_in_bytes")) - runtime.capabilities.SwapLimit = err == nil - return runtime, nil } diff --git a/components/engine/utils.go b/components/engine/utils.go index bb891b18a9..ff918d1eef 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -503,7 +503,6 @@ func FindCgroupMountpoint(cgroupType string) (string, error) { if len(r) == 2 { return r[1], nil } - fmt.Printf("line: %s (%d)\n", line, len(r)) } - return "", fmt.Errorf("cgroup mountpoint not found") + return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) } From e2e7fde6545af6cc7d7080d967823fa7f6471279 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Apr 2013 11:16:32 -0700 Subject: [PATCH 90/98] Fix race within TestRunDisconnectTty Upstream-commit: acb546cd1bf45a248f4bb51637c795ef63feb6cb Component: engine --- components/engine/commands.go | 7 ++++++- components/engine/commands_test.go | 17 +++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 1f951ff331..b0440a9766 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -979,8 +979,13 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...s } Debugf("Waiting for attach to return\n") <-attachErr - container.Wait() // Expecting I/O pipe error, discarding + + // If we are in stdinonce mode, wait for the process to end + // otherwise, simply return + if config.StdinOnce && !config.Tty { + container.Wait() + } return nil } diff --git a/components/engine/commands_test.go b/components/engine/commands_test.go index 9615e877e6..a64b4f4dc7 100644 --- a/components/engine/commands_test.go +++ b/components/engine/commands_test.go @@ -228,6 +228,21 @@ func TestRunDisconnectTty(t *testing.T) { close(c1) }() + setTimeout(t, "Waiting for the container to be started timed out", 2*time.Second, func() { + for { + // Client disconnect after run -i should keep stdin out in TTY mode + l := runtime.List() + if len(l) == 1 && l[0].State.Running { + break + } + + time.Sleep(10 * time.Millisecond) + } + }) + + // Client disconnect after run -i should keep stdin out in TTY mode + container := runtime.List()[0] + setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() { if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { t.Fatal(err) @@ -242,8 +257,6 @@ func TestRunDisconnectTty(t *testing.T) { // In tty mode, we expect the process to stay alive even after client's stdin closes. // Do not wait for run to finish - // Client disconnect after run -i should keep stdin out in TTY mode - container := runtime.List()[0] // Give some time to monitor to do his thing container.WaitTimeout(500 * time.Millisecond) if !container.State.Running { From 14226bf4645831d6ff37b4001ced893db3975885 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Apr 2013 11:26:34 -0700 Subject: [PATCH 91/98] Do not prevent docker from running when kernel detection fails Upstream-commit: 3514e47edc2d3cdaae2d92a78cc5c618d9549f13 Component: engine --- components/engine/runtime.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/components/engine/runtime.go b/components/engine/runtime.go index b894a2cdaf..1dfc22d632 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -295,14 +295,13 @@ func NewRuntime() (*Runtime, error) { return nil, err } - k, err := GetKernelVersion() - if err != nil { - return nil, err - } - runtime.kernelVersion = k - - if CompareKernelVersion(k, &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}) < 0 { - log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String()) + if k, err := GetKernelVersion(); err != nil { + log.Printf("WARNING: %s\n", err) + } else { + runtime.kernelVersion = k + if CompareKernelVersion(k, &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}) < 0 { + log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String()) + } } if cgroupMemoryMountpoint, err := FindCgroupMountpoint("memory"); err != nil { From 59824ca6708d009561ccaae65d4c327224ef18ce Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Apr 2013 11:39:56 -0700 Subject: [PATCH 92/98] Make the kernel version detection more generic Upstream-commit: 4ac3b803b9b7d0db1840a12f3332034938eb9550 Component: engine --- components/engine/utils.go | 43 +++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/components/engine/utils.go b/components/engine/utils.go index a039ca6eb6..8f9e84f2b8 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -407,7 +407,12 @@ type KernelVersionInfo struct { // FIXME: this doens't build on Darwin func GetKernelVersion() (*KernelVersionInfo, error) { - var uts syscall.Utsname + var ( + uts syscall.Utsname + flavor string + kernel, major, minor int + err error + ) if err := syscall.Uname(&uts); err != nil { return nil, err @@ -422,31 +427,35 @@ func GetKernelVersion() (*KernelVersionInfo, error) { } tmp := strings.SplitN(string(release), "-", 2) - if len(tmp) != 2 { - return nil, fmt.Errorf("Unrecognized kernel version") - } tmp2 := strings.SplitN(tmp[0], ".", 3) - if len(tmp2) != 3 { - return nil, fmt.Errorf("Unrecognized kernel version") + + if len(tmp2) > 0 { + kernel, err = strconv.Atoi(tmp2[0]) + if err != nil { + return nil, err + } } - kernel, err := strconv.Atoi(tmp2[0]) - if err != nil { - return nil, err + if len(tmp2) > 1 { + major, err = strconv.Atoi(tmp2[1]) + if err != nil { + return nil, err + } } - major, err := strconv.Atoi(tmp2[1]) - if err != nil { - return nil, err + if len(tmp2) > 2 { + minor, err = strconv.Atoi(tmp2[2]) + if err != nil { + return nil, err + } } - minor, err := strconv.Atoi(tmp2[2]) - if err != nil { - return nil, err + if len(tmp) == 2 { + flavor = tmp[1] + } else { + flavor = "" } - flavor := tmp[1] - return &KernelVersionInfo{ Kernel: kernel, Major: major, From 6b46021e640a545aa086216f8e959981244cb95b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Apr 2013 12:08:59 -0700 Subject: [PATCH 93/98] Move the kernel detection to arch specific files Upstream-commit: 16aeb77d5155cf33d1637073b9ca56728d472b49 Component: engine --- components/engine/getKernelVersion_darwin.go | 5 ++ components/engine/getKernelVersion_linux.go | 65 ++++++++++++++++++++ components/engine/utils.go | 58 +---------------- 3 files changed, 71 insertions(+), 57 deletions(-) create mode 100644 components/engine/getKernelVersion_darwin.go create mode 100644 components/engine/getKernelVersion_linux.go diff --git a/components/engine/getKernelVersion_darwin.go b/components/engine/getKernelVersion_darwin.go new file mode 100644 index 0000000000..36a959b8d6 --- /dev/null +++ b/components/engine/getKernelVersion_darwin.go @@ -0,0 +1,5 @@ +package docker + +func getKernelVersion() (*KernelVersionInfo, error) { + return nil, fmt.Errorf("Kernel version detection is not available on darwin") +} diff --git a/components/engine/getKernelVersion_linux.go b/components/engine/getKernelVersion_linux.go new file mode 100644 index 0000000000..bbf5a4fad1 --- /dev/null +++ b/components/engine/getKernelVersion_linux.go @@ -0,0 +1,65 @@ +package docker + +import ( + "strconv" + "strings" + "syscall" +) + +func getKernelVersion() (*KernelVersionInfo, error) { + var ( + uts syscall.Utsname + flavor string + kernel, major, minor int + err error + ) + + if err := syscall.Uname(&uts); err != nil { + return nil, err + } + + release := make([]byte, len(uts.Release)) + + i := 0 + for _, c := range uts.Release { + release[i] = byte(c) + i++ + } + + tmp := strings.SplitN(string(release), "-", 2) + tmp2 := strings.SplitN(tmp[0], ".", 3) + + if len(tmp2) > 0 { + kernel, err = strconv.Atoi(tmp2[0]) + if err != nil { + return nil, err + } + } + + if len(tmp2) > 1 { + major, err = strconv.Atoi(tmp2[1]) + if err != nil { + return nil, err + } + } + + if len(tmp2) > 2 { + minor, err = strconv.Atoi(tmp2[2]) + if err != nil { + return nil, err + } + } + + if len(tmp) == 2 { + flavor = tmp[1] + } else { + flavor = "" + } + + return &KernelVersionInfo{ + Kernel: kernel, + Major: major, + Minor: minor, + Flavor: flavor, + }, nil +} diff --git a/components/engine/utils.go b/components/engine/utils.go index 8f9e84f2b8..5974b7df3b 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -14,10 +14,8 @@ import ( "path/filepath" "regexp" "runtime" - "strconv" "strings" "sync" - "syscall" "time" ) @@ -407,61 +405,7 @@ type KernelVersionInfo struct { // FIXME: this doens't build on Darwin func GetKernelVersion() (*KernelVersionInfo, error) { - var ( - uts syscall.Utsname - flavor string - kernel, major, minor int - err error - ) - - if err := syscall.Uname(&uts); err != nil { - return nil, err - } - - release := make([]byte, len(uts.Release)) - - i := 0 - for _, c := range uts.Release { - release[i] = byte(c) - i++ - } - - tmp := strings.SplitN(string(release), "-", 2) - tmp2 := strings.SplitN(tmp[0], ".", 3) - - if len(tmp2) > 0 { - kernel, err = strconv.Atoi(tmp2[0]) - if err != nil { - return nil, err - } - } - - if len(tmp2) > 1 { - major, err = strconv.Atoi(tmp2[1]) - if err != nil { - return nil, err - } - } - - if len(tmp2) > 2 { - minor, err = strconv.Atoi(tmp2[2]) - if err != nil { - return nil, err - } - } - - if len(tmp) == 2 { - flavor = tmp[1] - } else { - flavor = "" - } - - return &KernelVersionInfo{ - Kernel: kernel, - Major: major, - Minor: minor, - Flavor: flavor, - }, nil + return getKernelVersion() } func (k *KernelVersionInfo) String() string { From 7793ea0a7bb8deb17d85d659d1dcde7eb5f6de07 Mon Sep 17 00:00:00 2001 From: Alexey Shamrin Date: Tue, 23 Apr 2013 00:27:23 +0400 Subject: [PATCH 94/98] README.md: `docker port` instead of just `port` Upstream-commit: 038e1d174bed0d73bcf4ba197f60651907aa1116 Component: engine --- components/engine/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 058004c078..13ec817e2b 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -125,7 +125,7 @@ Running an irc bouncer ```bash BOUNCER_ID=$(docker run -d -p 6667 -u irc shykes/znc $USER $PASSWORD) -echo "Configure your irc client to connect to port $(port $BOUNCER_ID 6667) of this machine" +echo "Configure your irc client to connect to port $(docker port $BOUNCER_ID 6667) of this machine" ``` Running Redis @@ -133,7 +133,7 @@ Running Redis ```bash REDIS_ID=$(docker run -d -p 6379 shykes/redis redis-server) -echo "Configure your redis client to connect to port $(port $REDIS_ID 6379) of this machine" +echo "Configure your redis client to connect to port $(docker port $REDIS_ID 6379) of this machine" ``` Share your own image! From 3c0fcda55980b55c1c5d8d66966b3a27f9d9d68b Mon Sep 17 00:00:00 2001 From: Evan Wies Date: Mon, 22 Apr 2013 18:37:06 -0400 Subject: [PATCH 95/98] Fix typo (ghot -> ghost) Upstream-commit: ffe16e32243da4f02d325dae1bf08f71181ed439 Component: engine --- components/engine/container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/container.go b/components/engine/container.go index 7c6f7614f1..c2c6fddd40 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -630,7 +630,7 @@ func (container *Container) Stop(seconds int) error { return nil } if container.State.Ghost { - return fmt.Errorf("Can't stop ghot container") + return fmt.Errorf("Can't stop ghost container") } // 1. Send a SIGTERM From 2a6be2d20230359489a24e2a8d08782c604ee9cb Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Apr 2013 15:57:31 -0700 Subject: [PATCH 96/98] Check that the pid in pidfile exists before preventing docker to start Upstream-commit: f079fbe3fa90d58d414341bed5246539c89b61b2 Component: engine --- components/engine/docker/docker.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 411e4d0c96..f2194c06cc 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -7,9 +7,11 @@ import ( "github.com/dotcloud/docker/rcli" "github.com/dotcloud/docker/term" "io" + "io/ioutil" "log" "os" "os/signal" + "strconv" "syscall" ) @@ -54,8 +56,13 @@ func main() { } func createPidFile(pidfile string) error { - if _, err := os.Stat(pidfile); err == nil { - return fmt.Errorf("pid file found, ensure docker is not running or delete %s", pidfile) + if pidString, err := ioutil.ReadFile(pidfile); err == nil { + pid, err := strconv.Atoi(string(pidString)) + if err == nil { + if _, err := os.Stat(fmt.Sprintf("/proc/%d/", pid)); err == nil { + return fmt.Errorf("pid file found, ensure docker is not running or delete %s", pidfile) + } + } } file, err := os.Create(pidfile) From 9868eec5a412046fdb0941861103c4d8d885a90f Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Apr 2013 17:51:09 -0700 Subject: [PATCH 97/98] Add unit test for file deletion Upstream-commit: 97535e5a6466dcad6ed9767960e869eadf68204f Component: engine --- components/engine/container_test.go | 76 +++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/components/engine/container_test.go b/components/engine/container_test.go index e6525f0a79..fef5331e3a 100644 --- a/components/engine/container_test.go +++ b/components/engine/container_test.go @@ -150,6 +150,82 @@ func TestMultipleAttachRestart(t *testing.T) { } } +func TestDiff(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + // Create a container and remove a file + container1, err := runtime.Create( + &Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"/bin/rm", "/etc/passwd"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container1) + + if err := container1.Run(); err != nil { + t.Fatal(err) + } + + // Check the changelog + c, err := container1.Changes() + if err != nil { + t.Fatal(err) + } + success := false + for _, elem := range c { + if elem.Path == "/etc/passwd" && elem.Kind == 2 { + success = true + } + } + if !success { + t.Fatalf("/etc/passwd as been removed but is not present in the diff") + } + + // Commit the container + rwTar, err := container1.ExportRw() + if err != nil { + t.Error(err) + } + img, err := runtime.graph.Create(rwTar, container1, "unit test commited image - diff", "") + if err != nil { + t.Error(err) + } + + // Create a new container from the commited image + container2, err := runtime.Create( + &Config{ + Image: img.Id, + Cmd: []string{"cat", "/etc/passwd"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container2) + + if err := container2.Run(); err != nil { + t.Fatal(err) + } + + // Check the changelog + c, err = container2.Changes() + if err != nil { + t.Fatal(err) + } + for _, elem := range c { + if elem.Path == "/etc/passwd" { + t.Fatalf("/etc/passwd should not be present in the diff after commit.") + } + } +} + func TestCommitRun(t *testing.T) { runtime, err := newTestRuntime() if err != nil { From 56cb20082e7f0a51e7feb91364a1392f21e93c60 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Apr 2013 13:19:50 -0700 Subject: [PATCH 98/98] Make kernel detection work without suffix Upstream-commit: c05c91ca3bb69d2c9faa0a2b15bc23b7c2a7bd8a Component: engine --- components/engine/getKernelVersion_darwin.go | 4 ++++ components/engine/getKernelVersion_linux.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/components/engine/getKernelVersion_darwin.go b/components/engine/getKernelVersion_darwin.go index 36a959b8d6..be3b733b68 100644 --- a/components/engine/getKernelVersion_darwin.go +++ b/components/engine/getKernelVersion_darwin.go @@ -1,5 +1,9 @@ package docker +import ( + "fmt" +) + func getKernelVersion() (*KernelVersionInfo, error) { return nil, fmt.Errorf("Kernel version detection is not available on darwin") } diff --git a/components/engine/getKernelVersion_linux.go b/components/engine/getKernelVersion_linux.go index bbf5a4fad1..04bb1edcb8 100644 --- a/components/engine/getKernelVersion_linux.go +++ b/components/engine/getKernelVersion_linux.go @@ -1,6 +1,7 @@ package docker import ( + "bytes" "strconv" "strings" "syscall" @@ -26,6 +27,9 @@ func getKernelVersion() (*KernelVersionInfo, error) { i++ } + // Remove the \x00 from the release for Atoi to parse correctly + release = release[:bytes.IndexByte(release, 0)] + tmp := strings.SplitN(string(release), "-", 2) tmp2 := strings.SplitN(tmp[0], ".", 3)