diff --git a/components/engine/.gitignore b/components/engine/.gitignore index 8cf66168eb..00d66de3ed 100644 --- a/components/engine/.gitignore +++ b/components/engine/.gitignore @@ -18,3 +18,4 @@ bundles/ .hg/ .git/ vendor/pkg/ +pyenv diff --git a/components/engine/hack/infrastructure/docker-ci/Dockerfile b/components/engine/hack/infrastructure/docker-ci/Dockerfile index 3ac8d90d24..d894330ffa 100644 --- a/components/engine/hack/infrastructure/docker-ci/Dockerfile +++ b/components/engine/hack/infrastructure/docker-ci/Dockerfile @@ -1,14 +1,16 @@ -# VERSION: 0.22 -# DOCKER-VERSION 0.6.3 -# AUTHOR: Daniel Mizyrycki -# DESCRIPTION: Deploy docker-ci on Amazon EC2 +# VERSION: 0.25 +# DOCKER-VERSION 0.6.6 +# AUTHOR: Daniel Mizyrycki +# DESCRIPTION: Deploy docker-ci on Digital Ocean # COMMENTS: # CONFIG_JSON is an environment variable json string loaded as: # # export CONFIG_JSON=' -# { "AWS_TAG": "EC2_instance_name", -# "AWS_ACCESS_KEY": "EC2_access_key", -# "AWS_SECRET_KEY": "EC2_secret_key", +# { "DROPLET_NAME": "docker-ci", +# "DO_CLIENT_ID": "Digital_Ocean_client_id", +# "DO_API_KEY": "Digital_Ocean_api_key", +# "DOCKER_KEY_ID": "Digital_Ocean_ssh_key_id", +# "DOCKER_CI_KEY_PATH": "docker-ci_private_key_path", # "DOCKER_CI_PUB": "$(cat docker-ci_ssh_public_key.pub)", # "DOCKER_CI_KEY": "$(cat docker-ci_ssh_private_key.key)", # "BUILDBOT_PWD": "Buildbot_server_password", @@ -33,9 +35,11 @@ from ubuntu:12.04 -run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list -run apt-get update; apt-get install -y python2.7 python-dev python-pip ssh rsync less vim -run pip install boto fabric +run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' \ + > /etc/apt/sources.list +run apt-get update; apt-get install -y git python2.7 python-dev libevent-dev \ + python-pip ssh rsync less vim +run pip install requests fabric # Add deployment code and set default container command add . /docker-ci diff --git a/components/engine/hack/infrastructure/docker-ci/VERSION b/components/engine/hack/infrastructure/docker-ci/VERSION new file mode 100644 index 0000000000..0bfccb0804 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/VERSION @@ -0,0 +1 @@ +0.4.5 diff --git a/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg b/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg index 52bf495df3..9ca5fc035a 100644 --- a/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg +++ b/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg @@ -43,7 +43,7 @@ c['slavePortnum'] = PORT_MASTER # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=['docker', - 'index','registry','coverage','nightlyrelease'])] + 'index','registry','docker-coverage','registry-coverage','nightlyrelease'])] c['schedulers'] += [SingleBranchScheduler(name="all", treeStableTimer=None, change_filter=filter.ChangeFilter(branch='master', repository='https://github.com/dotcloud/docker'), builderNames=['docker'])] @@ -51,7 +51,7 @@ c['schedulers'] += [SingleBranchScheduler(name='pullrequest', change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None, builderNames=['pullrequest'])] c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease', - 'coverage'], hour=7, minute=00)] + 'docker-coverage','registry-coverage'], hour=7, minute=00)] c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'], hour=range(0,24,4), minute=15)] @@ -76,17 +76,25 @@ c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'], # Docker coverage test factory = BuildFactory() -factory.addStep(ShellCommand(description='Coverage', logEnviron=False, +factory.addStep(ShellCommand(description='docker-coverage', logEnviron=False, usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format( DOCKER_CI_PATH))) -c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], +c['builders'] += [BuilderConfig(name='docker-coverage',slavenames=['buildworker'], + factory=factory)] + +# Docker registry coverage test +factory = BuildFactory() +factory.addStep(ShellCommand(description='registry-coverage', logEnviron=False, + usePTY=True, command='docker run registry_coverage'.format( + DOCKER_CI_PATH))) +c['builders'] += [BuilderConfig(name='registry-coverage',slavenames=['buildworker'], factory=factory)] # Registry functional test factory = BuildFactory() factory.addStep(ShellCommand(description='registry', logEnviron=False, command='. {0}/master/credentials.cfg; ' - '/docker-ci/functionaltests/test_registry.sh'.format(BUILDBOT_PATH), + '{1}/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, DOCKER_CI_PATH), usePTY=True)) c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], factory=factory)] @@ -95,16 +103,17 @@ c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], factory = BuildFactory() factory.addStep(ShellCommand(description='index', logEnviron=False, command='. {0}/master/credentials.cfg; ' - '/docker-ci/functionaltests/test_index.py'.format(BUILDBOT_PATH), + '{1}/functionaltests/test_index.py'.format(BUILDBOT_PATH, DOCKER_CI_PATH), usePTY=True)) c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'], factory=factory)] # Docker nightly release +nightlyrelease_cmd = ('docker version; docker run -i -t -privileged -e AWS_S3_BUCKET=' + 'test.docker.io dockerbuilder hack/dind dockerbuild.sh') factory = BuildFactory() -factory.addStep(ShellCommand(description='NightlyRelease', logEnviron=False, - usePTY=True, command='docker run -privileged' - ' -e AWS_S3_BUCKET=test.docker.io dockerbuilder')) +factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False, + usePTY=True, command=nightlyrelease_cmd)) c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'], factory=factory)] diff --git a/components/engine/hack/infrastructure/docker-ci/deployment.py b/components/engine/hack/infrastructure/docker-ci/deployment.py index 655bdd43bc..c04219d523 100755 --- a/components/engine/hack/infrastructure/docker-ci/deployment.py +++ b/components/engine/hack/infrastructure/docker-ci/deployment.py @@ -1,11 +1,11 @@ #!/usr/bin/env python -import os, sys, re, json, base64 -from boto.ec2.connection import EC2Connection +import os, sys, re, json, requests, base64 from subprocess import call from fabric import api from fabric.api import cd, run, put, sudo from os import environ as env +from datetime import datetime from time import sleep # Remove SSH private key as it needs more processing @@ -20,42 +20,41 @@ for key in CONFIG: env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1', env['CONFIG_JSON'],flags=re.DOTALL) - -AWS_TAG = env.get('AWS_TAG','docker-ci') -AWS_KEY_NAME = 'dotcloud-dev' # Same as CONFIG_JSON['DOCKER_CI_PUB'] -AWS_AMI = 'ami-d582d6bc' # Ubuntu 13.04 -AWS_REGION = 'us-east-1' -AWS_TYPE = 'm1.small' -AWS_SEC_GROUPS = 'gateway' -AWS_IMAGE_USER = 'ubuntu' +DROPLET_NAME = env.get('DROPLET_NAME','docker-ci') +TIMEOUT = 120 # Seconds before timeout droplet creation +IMAGE_ID = 1004145 # Docker on Ubuntu 13.04 +REGION_ID = 4 # New York 2 +SIZE_ID = 62 # memory 2GB +DO_IMAGE_USER = 'root' # Image user on Digital Ocean +API_URL = 'https://api.digitalocean.com/' DOCKER_PATH = '/go/src/github.com/dotcloud/docker' DOCKER_CI_PATH = '/docker-ci' CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH) -class AWS_EC2: - '''Amazon EC2''' - def __init__(self, access_key, secret_key): +class DigitalOcean(): + + def __init__(self, key, client): '''Set default API parameters''' - self.handler = EC2Connection(access_key, secret_key) - def create_instance(self, tag, instance_type): - reservation = self.handler.run_instances(**instance_type) - instance = reservation.instances[0] - sleep(10) - while instance.state != 'running': - sleep(5) - instance.update() - print "Instance state: %s" % (instance.state) - instance.add_tag("Name",tag) - print "instance %s done!" % (instance.id) - return instance.ip_address - def get_instances(self): - return self.handler.get_all_instances() - def get_tags(self): - return dict([(i.instances[0].id, i.instances[0].tags['Name']) - for i in self.handler.get_all_instances() if i.instances[0].tags]) - def del_instance(self, instance_id): - self.handler.terminate_instances(instance_ids=[instance_id]) + self.key = key + self.client = client + self.api_url = API_URL + + def api(self, cmd_path, api_arg={}): + '''Make api call''' + api_arg.update({'api_key':self.key, 'client_id':self.client}) + resp = requests.get(self.api_url + cmd_path, params=api_arg).text + resp = json.loads(resp) + if resp['status'] != 'OK': + raise Exception(resp['error_message']) + return resp + + def droplet_data(self, name): + '''Get droplet data''' + data = self.api('droplets') + data = [droplet for droplet in data['droplets'] + if droplet['name'] == name] + return data[0] if data else {} def json_fmt(data): @@ -63,20 +62,36 @@ def json_fmt(data): return json.dumps(data, sort_keys = True, indent = 2) -# Create EC2 API handler -ec2 = AWS_EC2(env['AWS_ACCESS_KEY'], env['AWS_SECRET_KEY']) +do = DigitalOcean(env['DO_API_KEY'], env['DO_CLIENT_ID']) -# Stop processing if AWS_TAG exists on EC2 -if AWS_TAG in ec2.get_tags().values(): - print ('Instance: {} already deployed. Not further processing.' - .format(AWS_TAG)) +# Get DROPLET_NAME data +data = do.droplet_data(DROPLET_NAME) + +# Stop processing if DROPLET_NAME exists on Digital Ocean +if data: + print ('Droplet: {} already deployed. Not further processing.' + .format(DROPLET_NAME)) exit(1) -ip = ec2.create_instance(AWS_TAG, {'image_id':AWS_AMI, 'instance_type':AWS_TYPE, - 'security_groups':[AWS_SEC_GROUPS], 'key_name':AWS_KEY_NAME}) +# Create droplet +do.api('droplets/new', {'name':DROPLET_NAME, 'region_id':REGION_ID, + 'image_id':IMAGE_ID, 'size_id':SIZE_ID, + 'ssh_key_ids':[env['DOCKER_KEY_ID']]}) -# Wait 30 seconds for the machine to boot -sleep(30) +# Wait for droplet to be created. +start_time = datetime.now() +while (data.get('status','') != 'active' and ( + datetime.now()-start_time).seconds < TIMEOUT): + data = do.droplet_data(DROPLET_NAME) + print data['status'] + sleep(3) + +# Wait for the machine to boot +sleep(15) + +# Get droplet IP +ip = str(data['ip_address']) +print 'droplet: {} ip: {}'.format(DROPLET_NAME, ip) # Create docker-ci ssh private key so docker-ci docker container can communicate # with its EC2 instance @@ -86,7 +101,7 @@ os.chmod('/root/.ssh/id_rsa',0600) open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n') api.env.host_string = ip -api.env.user = AWS_IMAGE_USER +api.env.user = DO_IMAGE_USER api.env.key_filename = '/root/.ssh/id_rsa' # Correct timezone @@ -100,20 +115,17 @@ sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB'])) credentials = { 'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'], 'AWS_SECRET_KEY': env['PKG_SECRET_KEY'], - 'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE'], - 'INDEX_AUTH': env['INDEX_AUTH']} + 'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE']} open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write( base64.b64encode(json.dumps(credentials))) # Transfer docker sudo('mkdir -p ' + DOCKER_CI_PATH) -sudo('chown {}.{} {}'.format(AWS_IMAGE_USER, AWS_IMAGE_USER, DOCKER_CI_PATH)) -call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, AWS_IMAGE_USER, ip, +sudo('chown {}.{} {}'.format(DO_IMAGE_USER, DO_IMAGE_USER, DOCKER_CI_PATH)) +call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, DO_IMAGE_USER, ip, os.path.dirname(DOCKER_CI_PATH)), shell=True) # Install Docker and Buildbot dependencies -sudo('addgroup docker') -sudo('usermod -a -G docker ubuntu') sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker') sudo('wget -q -O - https://get.docker.io/gpg | apt-key add -') sudo('echo deb https://get.docker.io/ubuntu docker main >' @@ -123,7 +135,7 @@ sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n' ' > /etc/apt/sources.list; apt-get update') sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev' ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)' - ' aufs-tools make libfontconfig libevent-dev') + ' aufs-tools make libfontconfig libevent-dev libsqlite3-dev libssl-dev') sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | ' 'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go') sudo('GOPATH=/go go get -d github.com/dotcloud/docker') @@ -135,13 +147,13 @@ sudo('curl -s https://phantomjs.googlecode.com/files/' 'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin' ' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs') -# Preventively reboot docker-ci daily -sudo('ln -s /sbin/reboot /etc/cron.daily') - # Build docker-ci containers sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH)) +sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH)) sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format( DOCKER_CI_PATH)) +sudo('cd {}/registry-coverage; docker build -t registry_coverage .'.format( + DOCKER_CI_PATH)) # Download docker-ci testing container sudo('docker pull mzdaniel/test_docker') @@ -154,3 +166,6 @@ sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}' env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'], env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'], env['REGISTRY_SECRET_KEY'])) + +# Preventively reboot docker-ci daily +sudo('ln -s /sbin/reboot /etc/cron.daily') diff --git a/components/engine/hack/infrastructure/docker-ci/docker-test/Dockerfile b/components/engine/hack/infrastructure/docker-ci/docker-test/Dockerfile index 66cb9762eb..0f3a63f5f1 100644 --- a/components/engine/hack/infrastructure/docker-ci/docker-test/Dockerfile +++ b/components/engine/hack/infrastructure/docker-ci/docker-test/Dockerfile @@ -1,6 +1,6 @@ -# VERSION: 0.3 -# DOCKER-VERSION 0.6.3 -# AUTHOR: Daniel Mizyrycki +# VERSION: 0.4 +# DOCKER-VERSION 0.6.6 +# AUTHOR: Daniel Mizyrycki # DESCRIPTION: Testing docker PRs and commits on top of master using # REFERENCES: This code reuses the excellent implementation of # Docker in Docker made by Jerome Petazzoni. @@ -15,15 +15,10 @@ # TO_RUN: docker run -privileged test_docker hack/dind test_docker.sh [commit] [repo] [branch] from docker -maintainer Daniel Mizyrycki +maintainer Daniel Mizyrycki -# Setup go environment. Extracted from /Dockerfile -env CGO_ENABLED 0 -env GOROOT /goroot -env PATH $PATH:/goroot/bin -env GOPATH /go:/go/src/github.com/dotcloud/docker/vendor -volume /var/lib/docker -workdir /go/src/github.com/dotcloud/docker +# Setup go in PATH. Extracted from /Dockerfile +env PATH /usr/local/go/bin:$PATH # Add test_docker.sh add test_docker.sh /usr/bin/test_docker.sh diff --git a/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh b/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh index 895e4d9642..cf8fdb90bb 100755 --- a/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh +++ b/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh @@ -8,31 +8,26 @@ BRANCH=${3-master} # Compute test paths DOCKER_PATH=/go/src/github.com/dotcloud/docker +# Timestamp +echo +date; echo + # Fetch latest master +cd / rm -rf /go -mkdir -p $DOCKER_PATH +git clone -q -b master http://github.com/dotcloud/docker $DOCKER_PATH cd $DOCKER_PATH -git init . -git fetch -q http://github.com/dotcloud/docker master -git reset --hard FETCH_HEAD # Merge commit -#echo FIXME. Temporarily skip TestPrivilegedCanMount until DinD works reliable on AWS -git pull -q https://github.com/mzdaniel/docker.git dind-aws || exit 1 - -# Merge commit in top of master git fetch -q "$REPO" "$BRANCH" -git merge --no-edit $COMMIT || exit 1 +git merge --no-edit $COMMIT || exit 255 # Test commit -go test -v; exit_status=$? +./hack/make.sh test; exit_status=$? # Display load if test fails -if [ $exit_status -eq 1 ] ; then +if [ $exit_status -ne 0 ] ; then uptime; echo; free fi -# Cleanup testing directory -rm -rf $BASE_PATH - exit $exit_status diff --git a/components/engine/hack/infrastructure/docker-ci/functionaltests/test_registry.sh b/components/engine/hack/infrastructure/docker-ci/functionaltests/test_registry.sh index 8bcd355c7c..58642529cc 100755 --- a/components/engine/hack/infrastructure/docker-ci/functionaltests/test_registry.sh +++ b/components/engine/hack/infrastructure/docker-ci/functionaltests/test_registry.sh @@ -8,10 +8,12 @@ rm -rf docker-registry # Setup the environment export SETTINGS_FLAVOR=test export DOCKER_REGISTRY_CONFIG=config_test.yml +export PYTHONPATH=$(pwd)/docker-registry/test # Get latest docker registry git clone -q https://github.com/dotcloud/docker-registry.git cd docker-registry +sed -Ei "s#(boto_bucket: ).+#\1_env:S3_BUCKET#" config_test.yml # Get dependencies pip install -q -r requirements.txt @@ -20,7 +22,6 @@ pip install -q tox # Run registry tests tox || exit 1 -export PYTHONPATH=$(pwd)/docker-registry python -m unittest discover -p s3.py -s test || exit 1 python -m unittest discover -p workflow.py -s test diff --git a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile index 541f3a9584..2100a9e8e9 100644 --- a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile +++ b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile @@ -1,20 +1,19 @@ -# VERSION: 1.2 -# DOCKER-VERSION 0.6.3 -# AUTHOR: Daniel Mizyrycki +# VERSION: 1.6 +# DOCKER-VERSION 0.6.6 +# AUTHOR: Daniel Mizyrycki # DESCRIPTION: Build docker nightly release using Docker in Docker. # REFERENCES: This code reuses the excellent implementation of docker in docker # made by Jerome Petazzoni. https://github.com/jpetazzo/dind # COMMENTS: # release_credentials.json is a base64 json encoded file containing: # { "AWS_ACCESS_KEY": "Test_docker_AWS_S3_bucket_id", -# "AWS_SECRET_KEY='Test_docker_AWS_S3_bucket_key' -# "GPG_PASSPHRASE='Test_docker_GPG_passphrase_signature' -# "INDEX_AUTH='Encripted_index_authentication' } +# "AWS_SECRET_KEY": "Test_docker_AWS_S3_bucket_key", +# "GPG_PASSPHRASE": "Test_docker_GPG_passphrase_signature" } # TO_BUILD: docker build -t dockerbuilder . -# TO_RELEASE: docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder +# TO_RELEASE: docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder hack/dind dockerbuild.sh from docker -maintainer Daniel Mizyrycki +maintainer Daniel Mizyrycki # Add docker dependencies and downloading packages run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list @@ -24,11 +23,8 @@ run apt-get update; apt-get install -y -q wget python2.7 run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker # Add proto docker builder -add ./dockerbuild /usr/bin/dockerbuild -run chmod +x /usr/bin/dockerbuild +add ./dockerbuild.sh /usr/bin/dockerbuild.sh +run chmod +x /usr/bin/dockerbuild.sh # Add release credentials add ./release_credentials.json /root/release_credentials.json - -# Launch build process in a container -cmd dockerbuild diff --git a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild deleted file mode 100644 index 83a7157a3e..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY, PG_PASSPHRASE and INDEX_AUTH -# are decoded from /root/release_credentials.json -# Variable AWS_S3_BUCKET is passed to the environment from docker run -e - -# Enable debugging -set -x - -# Fetch docker master branch -rm -rf /go/src/github.com/dotcloud/docker -cd / -git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker -cd /go/src/github.com/dotcloud/docker - -# Launch docker daemon using dind inside the container -./hack/dind /usr/bin/docker -d & -sleep 5 - -# Add an uncommitted change to generate a timestamped release -date > timestamp - -# Build the docker package using /Dockerfile -docker build -t docker . - -# Run Docker unittests binary and Ubuntu package -docker run -privileged docker hack/make.sh -exit_status=$? - -# Display load if test fails -if [ $exit_status -eq 1 ] ; then - uptime; echo; free - exit 1 -fi - -# Commit binary and ubuntu bundles for release -docker commit -run '{"Env": ["PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin"], "WorkingDir": "/go/src/github.com/dotcloud/docker"}' $(docker ps -l -q) release - -# Turn debug off to load credentials from the environment -set +x -eval $(cat /root/release_credentials.json | python -c ' -import sys,json,base64; -d=json.loads(base64.b64decode(sys.stdin.read())); -exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")') -set -x - -# Push docker nightly -echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX release hack/release.sh -set +x -docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE release hack/release.sh diff --git a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh new file mode 100644 index 0000000000..80caaec25e --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY and PG_PASSPHRASE are decoded +# from /root/release_credentials.json +# Variable AWS_S3_BUCKET is passed to the environment from docker run -e + +# Turn debug off to load credentials from the environment +set +x +eval $(cat /root/release_credentials.json | python -c ' +import sys,json,base64; +d=json.loads(base64.b64decode(sys.stdin.read())); +exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")') + +# Fetch docker master branch +set -x +cd / +rm -rf /go +git clone -q -b master http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker +cd /go/src/github.com/dotcloud/docker + +# Launch docker daemon using dind inside the container +/usr/bin/docker version +/usr/bin/docker -d & +sleep 5 + +# Build Docker release container +docker build -t docker . + +# Test docker and if everything works well, release +echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX docker hack/release.sh +set +x +docker run -privileged -i -t -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE docker hack/release.sh +exit_status=$? + +# Display load if test fails +set -x +if [ $exit_status -ne 0 ] ; then + uptime; echo; free + exit 1 +fi diff --git a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json deleted file mode 100644 index ed6d53ecd1..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json +++ /dev/null @@ -1 +0,0 @@ -eyAiQVdTX0FDQ0VTU19LRVkiOiAiIiwKICAiQVdTX1NFQ1JFVF9LRVkiOiAiIiwKICAiR1BHX1BBU1NQSFJBU0UiOiAiIiwKICAiSU5ERVhfQVVUSCI6ICIiIH0= diff --git a/components/engine/hack/infrastructure/docker-ci/registry-coverage/Dockerfile b/components/engine/hack/infrastructure/docker-ci/registry-coverage/Dockerfile new file mode 100644 index 0000000000..e544645b67 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/registry-coverage/Dockerfile @@ -0,0 +1,18 @@ +# VERSION: 0.1 +# DOCKER-VERSION 0.6.4 +# AUTHOR: Daniel Mizyrycki +# DESCRIPTION: Docker registry coverage +# COMMENTS: Add registry coverage into the docker-ci image +# TO_BUILD: docker build -t registry_coverage . +# TO_RUN: docker run registry_coverage + +from docker-ci +maintainer Daniel Mizyrycki + +# Add registry_coverager.sh and dependencies +run pip install coverage flask pyyaml requests simplejson python-glanceclient \ + blinker redis boto gevent rsa mock +add registry_coverage.sh /usr/bin/registry_coverage.sh +run chmod +x /usr/bin/registry_coverage.sh + +cmd "/usr/bin/registry_coverage.sh" diff --git a/components/engine/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh b/components/engine/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh new file mode 100755 index 0000000000..e16cea8e3c --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -x + +# Setup the environment +REGISTRY_PATH=/data/docker-registry +export SETTINGS_FLAVOR=test +export DOCKER_REGISTRY_CONFIG=config_test.yml +export PYTHONPATH=$REGISTRY_PATH/test + +# Fetch latest docker-registry master +rm -rf $REGISTRY_PATH +git clone https://github.com/dotcloud/docker-registry -b master $REGISTRY_PATH +cd $REGISTRY_PATH + +# Generate coverage +coverage run -m unittest discover test || exit 1 +coverage report --include='./*' --omit='./test/*' diff --git a/components/engine/hack/infrastructure/docker-ci/report/deployment.py b/components/engine/hack/infrastructure/docker-ci/report/deployment.py index d5efb4a960..5b2eaf3cab 100755 --- a/components/engine/hack/infrastructure/docker-ci/report/deployment.py +++ b/components/engine/hack/infrastructure/docker-ci/report/deployment.py @@ -34,7 +34,7 @@ env['DOCKER_CI_KEY'] = open(env['DOCKER_CI_KEY_PATH']).read() DROPLET_NAME = env.get('DROPLET_NAME','report') TIMEOUT = 120 # Seconds before timeout droplet creation -IMAGE_ID = 894856 # Docker on Ubuntu 13.04 +IMAGE_ID = 1004145 # Docker on Ubuntu 13.04 REGION_ID = 4 # New York 2 SIZE_ID = 66 # memory 512MB DO_IMAGE_USER = 'root' # Image user on Digital Ocean