docker-cli/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg

import os, re
from buildbot.buildslave import BuildSlave
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.timed import Nightly
from buildbot.changes import filter
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Property
from buildbot.steps.shell import ShellCommand
from buildbot.status import html, words
from buildbot.status.web import authz, auth
from buildbot.status.mail import MailNotifier


def ENV(x):
    '''Promote an environment variable for global use returning its value'''
    retval = os.environ.get(x, '')
    globals()[x] = retval
    return retval


class TestCommand(ShellCommand):
    '''Extend ShellCommand with optional summary logs'''
    def __init__(self, *args, **kwargs):
        super(TestCommand, self).__init__(*args, **kwargs)

    def createSummary(self, log):
        exit_status = re.sub(r'.+\n\+ exit (\d+).+',
            r'\1', log.getText()[-100:], flags=re.DOTALL)
        if exit_status != '0':
            return
        # Infer coverage path from log
        if '+ COVERAGE_PATH' in log.getText():
            path = re.sub(r'.+\+ COVERAGE_PATH=((.+?)-\d+).+',
              r'\2/\1', log.getText(), flags=re.DOTALL)
            url = '{}coverage/{}/index.html'.format(c['buildbotURL'], path)
            self.addURL('coverage', url)
        elif 'COVERAGE_FILE' in log.getText():
            path = re.sub(r'.+\+ COVERAGE_FILE=((.+?)-\d+).+',
              r'\2/\1', log.getText(), flags=re.DOTALL)
            url = '{}coverage/{}/index.html'.format(c['buildbotURL'], path)
            self.addURL('coverage', url)


PORT_WEB = 8000      # Buildbot webserver port
PORT_GITHUB = 8011   # Buildbot github hook port
PORT_MASTER = 9989   # Port where buildbot master listen buildworkers

BUILDBOT_URL = '//localhost:{}/'.format(PORT_WEB)
DOCKER_REPO = 'https://github.com/docker-test/docker'
DOCKER_TEST_ARGV = 'HEAD {}'.format(DOCKER_REPO)
REGISTRY_REPO = 'https://github.com/docker-test/docker-registry'
REGISTRY_TEST_ARGV = 'HEAD {}'.format(REGISTRY_REPO)
if ENV('DEPLOYMENT') == 'staging':
    BUILDBOT_URL = "//docker-ci-stage.docker.io/"
if ENV('DEPLOYMENT') == 'production':
    BUILDBOT_URL = '//docker-ci.docker.io/'
    DOCKER_REPO = 'https://github.com/dotcloud/docker'
    DOCKER_TEST_ARGV = ''
    REGISTRY_REPO = 'https://github.com/dotcloud/docker-registry'
    REGISTRY_TEST_ARGV = ''

# Credentials set by setup.sh from deployment.py
ENV('WEB_USER')
ENV('WEB_IRC_PWD')
ENV('BUILDBOT_PWD')
ENV('SMTP_USER')
ENV('SMTP_PWD')
ENV('EMAIL_RCP')
ENV('IRC_CHANNEL')


c = BuildmasterConfig = {}

c['title'] = "docker-ci"
c['titleURL'] = "waterfall"
c['buildbotURL'] = BUILDBOT_URL
c['db'] = {'db_url':"sqlite:///state.sqlite"}
c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)]
c['slavePortnum'] = PORT_MASTER


# Schedulers
c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[
    'docker', 'docker-registry', 'nightlyrelease', 'backup'])]
c['schedulers'] += [SingleBranchScheduler(name="docker", treeStableTimer=None,
    change_filter=filter.ChangeFilter(branch='master',
    repository=DOCKER_REPO), builderNames=['docker'])]
c['schedulers'] += [SingleBranchScheduler(name="registry", treeStableTimer=None,
    change_filter=filter.ChangeFilter(branch='master',
    repository=REGISTRY_REPO), builderNames=['docker-registry'])]
c['schedulers'] += [SingleBranchScheduler(name='docker-pr', treeStableTimer=None,
    change_filter=filter.ChangeFilter(category='github_pullrequest',
    project='docker'), builderNames=['docker-pr'])]
c['schedulers'] += [SingleBranchScheduler(name='docker-registry-pr', treeStableTimer=None,
    change_filter=filter.ChangeFilter(category='github_pullrequest',
    project='docker-registry'), builderNames=['docker-registry-pr'])]
c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=[
    'nightlyrelease', 'backup'], hour=7, minute=00)]


# Builders

# Backup
factory = BuildFactory()
factory.addStep(TestCommand(description='backup', logEnviron=False,
    usePTY=True, command='/docker-ci/tool/backup.py'))
c['builders'] = [BuilderConfig(name='backup',slavenames=['buildworker'],
    factory=factory)]

# Docker test
factory = BuildFactory()
factory.addStep(TestCommand(description='docker', logEnviron=False,
    usePTY=True, command='/docker-ci/dockertest/docker {}'.format(DOCKER_TEST_ARGV)))
c['builders'] += [BuilderConfig(name='docker',slavenames=['buildworker'],
    factory=factory)]

# Docker pull request test
factory = BuildFactory()
factory.addStep(TestCommand(description='docker-pr', logEnviron=False,
    usePTY=True, command=['/docker-ci/dockertest/docker',
    Property('revision'), Property('repository'), Property('branch')]))
c['builders'] += [BuilderConfig(name='docker-pr',slavenames=['buildworker'],
    factory=factory)]

# docker-registry test
factory = BuildFactory()
factory.addStep(TestCommand(description='docker-registry', logEnviron=False,
    usePTY=True, command='/docker-ci/dockertest/docker-registry {}'.format(REGISTRY_TEST_ARGV)))
c['builders'] += [BuilderConfig(name='docker-registry',slavenames=['buildworker'],
    factory=factory)]

# Docker registry pull request test
factory = BuildFactory()
factory.addStep(TestCommand(description='docker-registry-pr', logEnviron=False,
    usePTY=True, command=['/docker-ci/dockertest/docker-registry',
    Property('revision'), Property('repository'), Property('branch')]))
c['builders'] += [BuilderConfig(name='docker-registry-pr',slavenames=['buildworker'],
    factory=factory)]

# Docker nightly release
factory = BuildFactory()
factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False,
    usePTY=True, command=['/docker-ci/dockertest/nightlyrelease']))
c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'],
    factory=factory)]

# Status
authz_cfg = authz.Authz(auth=auth.BasicAuth([(WEB_USER, WEB_IRC_PWD)]),
    forceBuild='auth')
c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)]
c['status'].append(html.WebStatus(http_port=PORT_GITHUB, allowForce=True,
    change_hook_dialects={ 'github': True }))
c['status'].append(MailNotifier(fromaddr='docker-test@docker.io',
    sendToInterestedUsers=False, extraRecipients=[EMAIL_RCP],
    mode='failing', relayhost='smtp.mailgun.org', smtpPort=587, useTls=True,
    smtpUser=SMTP_USER, smtpPassword=SMTP_PWD))
c['status'].append(words.IRC("irc.freenode.net", "dockerqabot",
    channels=[IRC_CHANNEL], password=WEB_IRC_PWD, allowForce=True,
    notify_events={'exception':1, 'successToFailure':1, 'failureToSuccess':1}))