Merge pull request #2584 from dotcloud/docker-ci

docker-ci production sync
Upstream-commit: 5cf6ad4565374f924cdf64427d4e7b4adce50b41
Component: engine

components/engine/.gitignore

@@ -18,3 +18,4 @@ bundles/
 .hg/
 .git/
 vendor/pkg/
+pyenv

components/engine/hack/infrastructure/docker-ci/Dockerfile

@@ -1,14 +1,16 @@
-# VERSION:        0.22
-# DOCKER-VERSION  0.6.3
-# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
-# DESCRIPTION:    Deploy docker-ci on Amazon EC2
+# VERSION:        0.25
+# DOCKER-VERSION  0.6.6
+# AUTHOR:         Daniel Mizyrycki <daniel@docker.com>
+# DESCRIPTION:    Deploy docker-ci on Digital Ocean
 # COMMENTS:
 #     CONFIG_JSON is an environment variable json string loaded as:
 #
 # export CONFIG_JSON='
-#     { "AWS_TAG":             "EC2_instance_name",
-#       "AWS_ACCESS_KEY":      "EC2_access_key",
-#       "AWS_SECRET_KEY":      "EC2_secret_key",
+#     { "DROPLET_NAME":        "docker-ci",
+#       "DO_CLIENT_ID":        "Digital_Ocean_client_id",
+#       "DO_API_KEY":          "Digital_Ocean_api_key",
+#       "DOCKER_KEY_ID":       "Digital_Ocean_ssh_key_id",
+#       "DOCKER_CI_KEY_PATH":  "docker-ci_private_key_path",
 #       "DOCKER_CI_PUB":       "$(cat docker-ci_ssh_public_key.pub)",
 #       "DOCKER_CI_KEY":       "$(cat docker-ci_ssh_private_key.key)",
 #       "BUILDBOT_PWD":        "Buildbot_server_password",
@@ -33,9 +35,11 @@
 
 from ubuntu:12.04
 
-run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
-run apt-get update; apt-get install -y python2.7 python-dev python-pip ssh rsync less vim
-run pip install boto fabric
+run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' \
+    > /etc/apt/sources.list
+run apt-get update; apt-get install -y git python2.7 python-dev libevent-dev \
+    python-pip ssh rsync less vim
+run pip install requests fabric
 
 # Add deployment code and set default container command
 add . /docker-ci

components/engine/hack/infrastructure/docker-ci/VERSION

@@ -0,0 +1 @@
+0.4.5

components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg

@@ -43,7 +43,7 @@ c['slavePortnum'] = PORT_MASTER
 
 # Schedulers
 c['schedulers'] = [ForceScheduler(name='trigger', builderNames=['docker',
-    'index','registry','coverage','nightlyrelease'])]
+    'index','registry','docker-coverage','registry-coverage','nightlyrelease'])]
 c['schedulers'] += [SingleBranchScheduler(name="all", treeStableTimer=None,
     change_filter=filter.ChangeFilter(branch='master',
     repository='https://github.com/dotcloud/docker'), builderNames=['docker'])]
@@ -51,7 +51,7 @@ c['schedulers'] += [SingleBranchScheduler(name='pullrequest',
     change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None,
     builderNames=['pullrequest'])]
 c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease',
-    'coverage'], hour=7, minute=00)]
+    'docker-coverage','registry-coverage'], hour=7, minute=00)]
 c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'],
     hour=range(0,24,4), minute=15)]
 
@@ -76,17 +76,25 @@ c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'],
 
 # Docker coverage test
 factory = BuildFactory()
-factory.addStep(ShellCommand(description='Coverage', logEnviron=False,
+factory.addStep(ShellCommand(description='docker-coverage', logEnviron=False,
     usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format(
     DOCKER_CI_PATH)))
-c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'],
+c['builders'] += [BuilderConfig(name='docker-coverage',slavenames=['buildworker'],
+    factory=factory)]
+
+# Docker registry coverage test
+factory = BuildFactory()
+factory.addStep(ShellCommand(description='registry-coverage', logEnviron=False,
+    usePTY=True, command='docker run registry_coverage'.format(
+    DOCKER_CI_PATH)))
+c['builders'] += [BuilderConfig(name='registry-coverage',slavenames=['buildworker'],
     factory=factory)]
 
 # Registry functional test
 factory = BuildFactory()
 factory.addStep(ShellCommand(description='registry', logEnviron=False,
     command='. {0}/master/credentials.cfg; '
-    '/docker-ci/functionaltests/test_registry.sh'.format(BUILDBOT_PATH),
+    '{1}/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, DOCKER_CI_PATH),
     usePTY=True))
 c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'],
     factory=factory)]
@@ -95,16 +103,17 @@ c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'],
 factory = BuildFactory()
 factory.addStep(ShellCommand(description='index', logEnviron=False,
     command='. {0}/master/credentials.cfg; '
-    '/docker-ci/functionaltests/test_index.py'.format(BUILDBOT_PATH),
+    '{1}/functionaltests/test_index.py'.format(BUILDBOT_PATH, DOCKER_CI_PATH),
     usePTY=True))
 c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'],
     factory=factory)]
 
 # Docker nightly release
+nightlyrelease_cmd = ('docker version; docker run -i -t -privileged -e AWS_S3_BUCKET='
+    'test.docker.io dockerbuilder hack/dind dockerbuild.sh')
 factory = BuildFactory()
-factory.addStep(ShellCommand(description='NightlyRelease', logEnviron=False,
-    usePTY=True, command='docker run -privileged'
-    ' -e AWS_S3_BUCKET=test.docker.io dockerbuilder'))
+factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False,
+    usePTY=True, command=nightlyrelease_cmd))
 c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'],
     factory=factory)]
 

components/engine/hack/infrastructure/docker-ci/deployment.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 
-import os, sys, re, json, base64
-from boto.ec2.connection import EC2Connection
+import os, sys, re, json, requests, base64
 from subprocess import call
 from fabric import api
 from fabric.api import cd, run, put, sudo
 from os import environ as env
+from datetime import datetime
 from time import sleep
 
 # Remove SSH private key as it needs more processing
@@ -20,42 +20,41 @@ for key in CONFIG:
 env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1',
     env['CONFIG_JSON'],flags=re.DOTALL)
 
-
-AWS_TAG = env.get('AWS_TAG','docker-ci')
-AWS_KEY_NAME = 'dotcloud-dev'       # Same as CONFIG_JSON['DOCKER_CI_PUB']
-AWS_AMI = 'ami-d582d6bc'            # Ubuntu 13.04
-AWS_REGION = 'us-east-1'
-AWS_TYPE = 'm1.small'
-AWS_SEC_GROUPS = 'gateway'
-AWS_IMAGE_USER = 'ubuntu'
+DROPLET_NAME = env.get('DROPLET_NAME','docker-ci')
+TIMEOUT = 120            # Seconds before timeout droplet creation
+IMAGE_ID = 1004145       # Docker on Ubuntu 13.04
+REGION_ID = 4            # New York 2
+SIZE_ID = 62             # memory 2GB
+DO_IMAGE_USER = 'root'   # Image user on Digital Ocean
+API_URL = 'https://api.digitalocean.com/'
 DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
 DOCKER_CI_PATH = '/docker-ci'
 CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH)
 
 
-class AWS_EC2:
-    '''Amazon EC2'''
-    def __init__(self, access_key, secret_key):
+class DigitalOcean():
+
+    def __init__(self, key, client):
         '''Set default API parameters'''
-        self.handler = EC2Connection(access_key, secret_key)
-    def create_instance(self, tag, instance_type):
-        reservation = self.handler.run_instances(**instance_type)
-        instance = reservation.instances[0]
-        sleep(10)
-        while instance.state != 'running':
-            sleep(5)
-            instance.update()
-            print "Instance state: %s" % (instance.state)
-        instance.add_tag("Name",tag)
-        print "instance %s done!" % (instance.id)
-        return instance.ip_address
-    def get_instances(self):
-        return self.handler.get_all_instances()
-    def get_tags(self):
-        return dict([(i.instances[0].id, i.instances[0].tags['Name'])
-            for i in self.handler.get_all_instances() if i.instances[0].tags])
-    def del_instance(self, instance_id):
-        self.handler.terminate_instances(instance_ids=[instance_id])
+        self.key = key
+        self.client = client
+        self.api_url = API_URL
+
+    def api(self, cmd_path, api_arg={}):
+        '''Make api call'''
+        api_arg.update({'api_key':self.key, 'client_id':self.client})
+        resp = requests.get(self.api_url + cmd_path, params=api_arg).text
+        resp = json.loads(resp)
+        if resp['status'] != 'OK':
+            raise Exception(resp['error_message'])
+        return resp
+
+    def droplet_data(self, name):
+        '''Get droplet data'''
+        data = self.api('droplets')
+        data = [droplet for droplet in data['droplets']
+            if droplet['name'] == name]
+        return data[0] if data else {}
 
 
 def json_fmt(data):
@@ -63,20 +62,36 @@ def json_fmt(data):
     return json.dumps(data, sort_keys = True, indent = 2)
 
 
-# Create EC2 API handler
-ec2 = AWS_EC2(env['AWS_ACCESS_KEY'], env['AWS_SECRET_KEY'])
+do = DigitalOcean(env['DO_API_KEY'], env['DO_CLIENT_ID'])
 
-# Stop processing if AWS_TAG exists on EC2
-if AWS_TAG in ec2.get_tags().values():
-    print ('Instance: {} already deployed. Not further processing.'
-        .format(AWS_TAG))
+# Get DROPLET_NAME data
+data = do.droplet_data(DROPLET_NAME)
+
+# Stop processing if DROPLET_NAME exists on Digital Ocean
+if data:
+    print ('Droplet: {} already deployed. Not further processing.'
+        .format(DROPLET_NAME))
     exit(1)
 
-ip = ec2.create_instance(AWS_TAG, {'image_id':AWS_AMI, 'instance_type':AWS_TYPE,
-    'security_groups':[AWS_SEC_GROUPS], 'key_name':AWS_KEY_NAME})
+# Create droplet
+do.api('droplets/new', {'name':DROPLET_NAME, 'region_id':REGION_ID,
+    'image_id':IMAGE_ID, 'size_id':SIZE_ID,
+    'ssh_key_ids':[env['DOCKER_KEY_ID']]})
 
-# Wait 30 seconds for the machine to boot
-sleep(30)
+# Wait for droplet to be created.
+start_time = datetime.now()
+while (data.get('status','') != 'active' and (
+ datetime.now()-start_time).seconds < TIMEOUT):
+    data = do.droplet_data(DROPLET_NAME)
+    print data['status']
+    sleep(3)
+
+# Wait for the machine to boot
+sleep(15)
+
+# Get droplet IP
+ip = str(data['ip_address'])
+print 'droplet: {}    ip: {}'.format(DROPLET_NAME, ip)
 
 # Create docker-ci ssh private key so docker-ci docker container can communicate
 # with its EC2 instance
@@ -86,7 +101,7 @@ os.chmod('/root/.ssh/id_rsa',0600)
 open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n')
 
 api.env.host_string = ip
-api.env.user = AWS_IMAGE_USER
+api.env.user = DO_IMAGE_USER
 api.env.key_filename = '/root/.ssh/id_rsa'
 
 # Correct timezone
@@ -100,20 +115,17 @@ sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB']))
 credentials = {
     'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'],
     'AWS_SECRET_KEY': env['PKG_SECRET_KEY'],
-    'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE'],
-    'INDEX_AUTH': env['INDEX_AUTH']}
+    'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE']}
 open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write(
     base64.b64encode(json.dumps(credentials)))
 
 # Transfer docker
 sudo('mkdir -p ' + DOCKER_CI_PATH)
-sudo('chown {}.{} {}'.format(AWS_IMAGE_USER, AWS_IMAGE_USER, DOCKER_CI_PATH))
-call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, AWS_IMAGE_USER, ip,
+sudo('chown {}.{} {}'.format(DO_IMAGE_USER, DO_IMAGE_USER, DOCKER_CI_PATH))
+call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, DO_IMAGE_USER, ip,
     os.path.dirname(DOCKER_CI_PATH)), shell=True)
 
 # Install Docker and Buildbot dependencies
-sudo('addgroup docker')
-sudo('usermod -a -G docker ubuntu')
 sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker')
 sudo('wget -q -O - https://get.docker.io/gpg | apt-key add -')
 sudo('echo deb https://get.docker.io/ubuntu docker main >'
@@ -123,7 +135,7 @@ sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n'
     ' > /etc/apt/sources.list; apt-get update')
 sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev'
     ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)'
-    ' aufs-tools make libfontconfig libevent-dev')
+    ' aufs-tools make libfontconfig libevent-dev libsqlite3-dev libssl-dev')
 sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | '
     'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go')
 sudo('GOPATH=/go go get -d github.com/dotcloud/docker')
@@ -135,13 +147,13 @@ sudo('curl -s https://phantomjs.googlecode.com/files/'
     'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin'
     ' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs')
 
-# Preventively reboot docker-ci daily
-sudo('ln -s /sbin/reboot /etc/cron.daily')
-
 # Build docker-ci containers
 sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH))
+sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH))
 sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format(
     DOCKER_CI_PATH))
+sudo('cd {}/registry-coverage; docker build -t registry_coverage .'.format(
+    DOCKER_CI_PATH))
 
 # Download docker-ci testing container
 sudo('docker pull mzdaniel/test_docker')
@@ -154,3 +166,6 @@ sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'
     env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'],
     env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'],
     env['REGISTRY_SECRET_KEY']))
+
+# Preventively reboot docker-ci daily
+sudo('ln -s /sbin/reboot /etc/cron.daily')

components/engine/hack/infrastructure/docker-ci/docker-test/Dockerfile

@@ -1,6 +1,6 @@
-# VERSION:        0.3
-# DOCKER-VERSION  0.6.3
-# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
+# VERSION:        0.4
+# DOCKER-VERSION  0.6.6
+# AUTHOR:         Daniel Mizyrycki <daniel@docker.com>
 # DESCRIPTION:    Testing docker PRs and commits on top of master using
 # REFERENCES:     This code reuses the excellent implementation of
 #                 Docker in Docker made by Jerome Petazzoni.
@@ -15,15 +15,10 @@
 # TO_RUN:         docker run -privileged test_docker hack/dind test_docker.sh [commit] [repo] [branch]
 
 from docker
-maintainer Daniel Mizyrycki <daniel@dotcloud.com>
+maintainer Daniel Mizyrycki <daniel@docker.com>
 
-# Setup go environment. Extracted from /Dockerfile
-env     CGO_ENABLED 0
-env     GOROOT  /goroot
-env     PATH    $PATH:/goroot/bin
-env     GOPATH  /go:/go/src/github.com/dotcloud/docker/vendor
-volume  /var/lib/docker
-workdir /go/src/github.com/dotcloud/docker
+# Setup go in PATH. Extracted from /Dockerfile
+env PATH /usr/local/go/bin:$PATH
 
 # Add test_docker.sh
 add test_docker.sh /usr/bin/test_docker.sh

components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh

@@ -8,31 +8,26 @@ BRANCH=${3-master}
 # Compute test paths
 DOCKER_PATH=/go/src/github.com/dotcloud/docker
 
+# Timestamp
+echo
+date; echo
+
 # Fetch latest master
+cd /
 rm -rf /go
-mkdir -p $DOCKER_PATH
+git clone -q -b master http://github.com/dotcloud/docker $DOCKER_PATH
 cd $DOCKER_PATH
-git init .
-git fetch -q http://github.com/dotcloud/docker master
-git reset --hard FETCH_HEAD
 
 # Merge commit
-#echo FIXME. Temporarily skip TestPrivilegedCanMount until DinD works reliable on AWS
-git pull -q https://github.com/mzdaniel/docker.git dind-aws || exit 1
-
-# Merge commit in top of master
 git fetch -q "$REPO" "$BRANCH"
-git merge --no-edit $COMMIT || exit 1
+git merge --no-edit $COMMIT || exit 255
 
 # Test commit
-go test -v; exit_status=$?
+./hack/make.sh test; exit_status=$?
 
 # Display load if test fails
-if [ $exit_status -eq 1 ] ; then
+if [ $exit_status -ne 0 ] ; then
     uptime; echo; free
 fi
 
-# Cleanup testing directory
-rm -rf $BASE_PATH
-
 exit $exit_status

components/engine/hack/infrastructure/docker-ci/functionaltests/test_registry.sh

@@ -8,10 +8,12 @@ rm -rf docker-registry
 # Setup the environment
 export SETTINGS_FLAVOR=test
 export DOCKER_REGISTRY_CONFIG=config_test.yml
+export PYTHONPATH=$(pwd)/docker-registry/test
 
 # Get latest docker registry
 git clone -q https://github.com/dotcloud/docker-registry.git
 cd docker-registry
+sed -Ei "s#(boto_bucket: ).+#\1_env:S3_BUCKET#" config_test.yml
 
 # Get dependencies
 pip install -q -r requirements.txt
@@ -20,7 +22,6 @@ pip install -q tox
 
 # Run registry tests
 tox || exit 1
-export PYTHONPATH=$(pwd)/docker-registry
 python -m unittest discover -p s3.py -s test || exit 1
 python -m unittest discover -p workflow.py -s test
 

components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile

@@ -1,20 +1,19 @@
-# VERSION:        1.2
-# DOCKER-VERSION  0.6.3
-# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
+# VERSION:        1.6
+# DOCKER-VERSION  0.6.6
+# AUTHOR:         Daniel Mizyrycki <daniel@docker.com>
 # DESCRIPTION:    Build docker nightly release using Docker in Docker.
 # REFERENCES:     This code reuses the excellent implementation of docker in docker
 #                 made by Jerome Petazzoni.  https://github.com/jpetazzo/dind
 # COMMENTS:
 #   release_credentials.json is a base64 json encoded file containing:
 #       { "AWS_ACCESS_KEY": "Test_docker_AWS_S3_bucket_id",
-#         "AWS_SECRET_KEY='Test_docker_AWS_S3_bucket_key'
-#         "GPG_PASSPHRASE='Test_docker_GPG_passphrase_signature'
-#         "INDEX_AUTH='Encripted_index_authentication' }
+#         "AWS_SECRET_KEY": "Test_docker_AWS_S3_bucket_key",
+#         "GPG_PASSPHRASE": "Test_docker_GPG_passphrase_signature" }
 # TO_BUILD:       docker build -t dockerbuilder .
-# TO_RELEASE:     docker run -i -t -privileged  -e AWS_S3_BUCKET="test.docker.io" dockerbuilder
+# TO_RELEASE:     docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder hack/dind dockerbuild.sh
 
 from docker
-maintainer Daniel Mizyrycki <daniel@dotcloud.com>
+maintainer Daniel Mizyrycki <daniel@docker.com>
 
 # Add docker dependencies and downloading packages
 run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
@@ -24,11 +23,8 @@ run apt-get update; apt-get install -y -q wget python2.7
 run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker
 
 # Add proto docker builder
-add ./dockerbuild /usr/bin/dockerbuild
-run chmod +x /usr/bin/dockerbuild
+add ./dockerbuild.sh /usr/bin/dockerbuild.sh
+run chmod +x /usr/bin/dockerbuild.sh
 
 # Add release credentials
 add ./release_credentials.json /root/release_credentials.json
-
-# Launch build process in a container
-cmd dockerbuild

components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild

@@ -1,50 +0,0 @@
-#!/bin/bash
-
-# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY, PG_PASSPHRASE and INDEX_AUTH
-# are decoded from /root/release_credentials.json
-# Variable AWS_S3_BUCKET is passed to the environment from docker run -e
-
-# Enable debugging
-set -x
-
-# Fetch docker master branch
-rm -rf  /go/src/github.com/dotcloud/docker
-cd /
-git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker
-cd /go/src/github.com/dotcloud/docker
-
-# Launch docker daemon using dind inside the container
-./hack/dind /usr/bin/docker -d &
-sleep 5
-
-# Add an uncommitted change to generate a timestamped release
-date > timestamp
-
-# Build the docker package using /Dockerfile
-docker build -t docker .
-
-# Run Docker unittests binary and Ubuntu package
-docker run -privileged docker hack/make.sh
-exit_status=$?
-
-# Display load if test fails
-if [ $exit_status -eq 1 ] ; then
-    uptime; echo; free
-    exit 1
-fi
-
-# Commit binary and ubuntu bundles for release
-docker commit -run '{"Env": ["PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin"], "WorkingDir": "/go/src/github.com/dotcloud/docker"}' $(docker ps -l -q) release
-
-# Turn debug off to load credentials from the environment
-set +x
-eval $(cat /root/release_credentials.json  | python -c '
-import sys,json,base64;
-d=json.loads(base64.b64decode(sys.stdin.read()));
-exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")')
-set -x
-
-# Push docker nightly
-echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX release  hack/release.sh
-set +x
-docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE  release  hack/release.sh

components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY and PG_PASSPHRASE are decoded
+# from /root/release_credentials.json
+# Variable AWS_S3_BUCKET is passed to the environment from docker run -e
+
+# Turn debug off to load credentials from the environment
+set +x
+eval $(cat /root/release_credentials.json  | python -c '
+import sys,json,base64;
+d=json.loads(base64.b64decode(sys.stdin.read()));
+exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")')
+
+# Fetch docker master branch
+set -x
+cd /
+rm -rf /go
+git clone -q -b master http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker
+cd /go/src/github.com/dotcloud/docker
+
+# Launch docker daemon using dind inside the container
+/usr/bin/docker version
+/usr/bin/docker -d &
+sleep 5
+
+# Build Docker release container
+docker build -t docker .
+
+# Test docker and if everything works well, release
+echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX docker hack/release.sh
+set +x
+docker run -privileged -i -t -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE docker hack/release.sh
+exit_status=$?
+
+# Display load if test fails
+set -x
+if [ $exit_status -ne 0 ] ; then
+    uptime; echo; free
+    exit 1
+fi

components/engine/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json

@@ -1 +0,0 @@
-eyAiQVdTX0FDQ0VTU19LRVkiOiAiIiwKICAiQVdTX1NFQ1JFVF9LRVkiOiAiIiwKICAiR1BHX1BBU1NQSFJBU0UiOiAiIiwKICAiSU5ERVhfQVVUSCI6ICIiIH0=

components/engine/hack/infrastructure/docker-ci/registry-coverage/Dockerfile

@@ -0,0 +1,18 @@
+# VERSION:        0.1
+# DOCKER-VERSION  0.6.4
+# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
+# DESCRIPTION:    Docker registry coverage
+# COMMENTS:       Add registry coverage into the docker-ci image
+# TO_BUILD:       docker build -t registry_coverage .
+# TO_RUN:         docker run registry_coverage
+
+from docker-ci
+maintainer Daniel Mizyrycki <daniel@dotcloud.com>
+
+# Add registry_coverager.sh and dependencies
+run pip install coverage flask pyyaml requests simplejson python-glanceclient \
+    blinker redis boto gevent rsa mock
+add registry_coverage.sh /usr/bin/registry_coverage.sh
+run chmod +x /usr/bin/registry_coverage.sh
+
+cmd "/usr/bin/registry_coverage.sh"

components/engine/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -x
+
+# Setup the environment
+REGISTRY_PATH=/data/docker-registry
+export SETTINGS_FLAVOR=test
+export DOCKER_REGISTRY_CONFIG=config_test.yml
+export PYTHONPATH=$REGISTRY_PATH/test
+
+# Fetch latest docker-registry master
+rm -rf $REGISTRY_PATH
+git clone https://github.com/dotcloud/docker-registry -b master $REGISTRY_PATH
+cd $REGISTRY_PATH
+
+# Generate coverage
+coverage run -m unittest discover test || exit 1
+coverage report --include='./*' --omit='./test/*'

components/engine/hack/infrastructure/docker-ci/report/deployment.py

@@ -34,7 +34,7 @@ env['DOCKER_CI_KEY'] = open(env['DOCKER_CI_KEY_PATH']).read()
 
 DROPLET_NAME = env.get('DROPLET_NAME','report')
 TIMEOUT = 120            # Seconds before timeout droplet creation
-IMAGE_ID = 894856        # Docker on Ubuntu 13.04
+IMAGE_ID = 1004145       # Docker on Ubuntu 13.04
 REGION_ID = 4            # New York 2
 SIZE_ID = 66             # memory 512MB
 DO_IMAGE_USER = 'root'   # Image user on Digital Ocean