diff options
Diffstat (limited to 'WebKitTools/BuildSlaveSupport/build.webkit.org-config')
12 files changed, 583 insertions, 0 deletions
diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/Makefile b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/Makefile new file mode 100644 index 0000000..c3a935f --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/Makefile @@ -0,0 +1,19 @@ +# -*- makefile -*- + +# This is a simple makefile which lives in a buildmaster/buildslave +# directory (next to the buildbot.tac file). It allows you to start/stop the +# master or slave by doing 'make start' or 'make stop'. + +# The 'reconfig' target will tell a buildmaster to reload its config file. + +start: + twistd --no_save -y buildbot.tac + +stop: + kill `cat twistd.pid` + +reconfig: + kill -HUP `cat twistd.pid` + +log: + tail -f twistd.log diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.css b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.css new file mode 100644 index 0000000..81de950 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.css @@ -0,0 +1,108 @@ +* { + font-family: Verdana, Cursor; + font-size: 10px; + font-weight: bold; +} + +a:link,a:visited,a:active { + color: #444; +} +a:hover { + color: #FFFFFF; +} + +table { + border-spacing: 1px 1px; +} + +table td { + padding: 3px 0px 3px 0px; + text-align: center; +} + +.Project { + width: 100px; +} + +.LastBuild, .Activity { + width: 230px; + padding: 0 0 0 4px; +} + +td.Time { + color: #000; + border-bottom: 1px solid #aaa; + background-color: #eee; +} + +td.Activity, td.Change, td.Builder { + color: #333333; + background-color: #CCCCCC; +} + +td.Change { + border-radius: 5px; + -webkit-border-radius: 5px; +} +td.Event { + color: #777; + background-color: #ddd; + border-radius: 5px; + -webkit-border-radius: 5px; +} + +td.Activity { + border-top-left-radius: 10px; + -webkit-border-top-left-radius: 10px; + min-height: 20px; + padding: 8px 0 8px 0; +} + +td.idle, td.waiting, td.offline, td.building { + border-top-left-radius: 0px; + -webkit-border-top-left-radius: 0px; +} + +.LastBuild { + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + border-top-right-radius: 5px; + -webkit-border-top-right-radius: 5px; +} + +/* LastBuild, BuildStep states */ +.success { + color: #FFFFFF; + background-color: #8fdf5f; +} + +.failure { + color: #FFFFFF; + background-color: #e98080; +} + +.warnings { + color: #FFFFFF; + background-color: #ffc343; +} + +.exception, td.offline { + color: #FFFFFF; + background-color: #e0b0ff; +} + +.start,.running, td.building { + color: #666666; + background-color: #fffc6c; +} + +.start { + border-bottom-left-radius: 10px; + -webkit-border-bottom-left-radius: 10px; + border-bottom-right-radius: 10px; + -webkit-border-bottom-right-radius: 10px; +} + +td.Project a:hover, td.start a:hover { + color: #000; +} diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.tac b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.tac new file mode 100644 index 0000000..f66e068 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.tac @@ -0,0 +1,10 @@ + +from twisted.application import service +from buildbot.master import BuildMaster + +basedir = r'/home/mrowe/sites/build.webkit.org/buildbot' +configfile = r'master.cfg' + +application = service.Application('buildmaster') +BuildMaster(basedir, configfile).setServiceParent(application) + diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg new file mode 100644 index 0000000..ea7966a --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg @@ -0,0 +1,22 @@ +# -*- python -*- + +import os +from twisted.web import static +static.File.contentTypes = static.loadMimeTypes(['/etc/mime.types', os.path.join(basedir, 'mime.types')]) + +from buildbot.changes.pb import PBChangeSource +import webkit +reload(webkit) + +c = BuildmasterConfig = {} + +c['slaves'] = webkit.auth.getSlaveAuthenticationDetails() +c['change_source'] = [PBChangeSource(prefix="trunk")] +c['builders'] = webkit.builders.getBuilders() +c['schedulers'] = webkit.schedulers.getSchedulers(c['builders']) +c['status'] = webkit.status.getStatusListeners() + +c['slavePortnum'] = 9989 +c['projectName'] = "WebKit" +c['projectURL'] = "http://webkit.org/" +c['buildbotURL'] = "http://build.webkit.org/" diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/__init__.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/__init__.py new file mode 100644 index 0000000..f81fcae --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/__init__.py @@ -0,0 +1,5 @@ +from webkit import auth, status, builders, schedulers +reload(auth) +reload(status) +reload(builders) +reload(schedulers) diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/auth.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/auth.py new file mode 100644 index 0000000..b182e16 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/auth.py @@ -0,0 +1,9 @@ +from buildbot.buildslave import BuildSlave + +def getSlaveAuthenticationDetails(): + def createBuildSlave((name, password)): + return BuildSlave(name, password, max_builds=1) + return map(createBuildSlave, _getSlaveAuthenticationDetails()) + +def _getSlaveAuthenticationDetails(): + return [("slave-name", "password")] diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/basesteps.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/basesteps.py new file mode 100644 index 0000000..8bba881 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/basesteps.py @@ -0,0 +1,51 @@ +from buildbot.steps import shell, source +import os + + +def buildStepWithDefaultTimeout(klass, default_timeout=75*60): + class Step(klass): + timeout = default_timeout + def __init__(self, *args, **kwargs): + kwargs['timeout'] = self.timeout + klass.__init__(self, *args, **kwargs) + + return Step + + +Test = buildStepWithDefaultTimeout(shell.Test) +Compile = buildStepWithDefaultTimeout(shell.Compile) +ShellCommand = buildStepWithDefaultTimeout(shell.ShellCommand) +SVN = buildStepWithDefaultTimeout(source.SVN) + + +class UploadCommand: + def initializeForUpload(self): + try: + try: + umask = os.umask(0) + os.makedirs(self.getDestinationPath(), 042770) + except OSError, e: + if e.errno != 17: + raise + finally: + os.umask(umask) + + def getDestinationPath(self): + return "/home/buildresults%s" % (self.getURLPath(), ) + + def getRemotePath(self): + return "buildresults@build.webkit.org:%s" % (self.getDestinationPath(), ) + + def getURLPath(self): + return '/results/%s/%s/' % (self.getBuild().builder.name, self.getBuild().getProperty("buildnumber"), ) + + def getBuild(self): + return self.build + + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + return ['<a href="%s">%s</a>' % (self.getURLPath(), self.name)] + diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/builders.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/builders.py new file mode 100644 index 0000000..b4e087e --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/builders.py @@ -0,0 +1,35 @@ +from webkit.factories import * +from buildbot import locks + +# There are four build slaves that take care of the majority of builds, with two other specialist slaves at Apple +# Slave 1 isn older G4 PowerMac dedicated to the PLT builds, as it needs extra configuration +# Slave 2 is a Windows PC dedicated to the Windows builds +# Slaves 3 and 4 are older G4 PowerMacs with relatively low amounts of RAM which leads to insanely slow leaks tests +# Slaves 4 and 5 are newer G5 PowerMacs with ATI graphics cards that lead to kernel panics during pixel tests + +nonATIPowerPCBuilders = ['apple-slave-%d' % i for i in (3, 4)] +ATIPowerPCBuilders = ['apple-slave-%d' % i for i in (1, 5, 6)] +allPowerPCBuilders = nonATIPowerPCBuilders + ATIPowerPCBuilders +allIntelBuilders = ['bdash-slave-1', 'bdash-slave-2'] + +_builders = [('post-commit-powerpc-mac-os-x', StandardBuildFactory, allPowerPCBuilders), + ('post-commit-intel-mac-os-x', StandardBuildFactory, allIntelBuilders), + ('post-commit-leaks-powerpc-mac-os-x', LeakBuildFactory, allPowerPCBuilders), + ('post-commit-leaks-intel-mac-os-x', LeakBuildFactory, allIntelBuilders), +# ('page-layout-test-mac-os-x', PageLoadTestBuildFactory, ['apple-slave-1']), +# ('post-commit-pixel-powerpc-mac-os-x', PixelTestBuildFactory, nonATIPowerPCBuilders), + ('post-commit-win32', Win32BuildFactory, ['apple-slave-2']), + ('post-commit-linux-qt', StandardBuildFactory, ['webtroll-slave-1']), + ('post-commit-linux-gtk', GtkBuildFactory, ['zecke-slave-1']), + ('periodic-powerpc-mac-os-x-no-svg', NoSVGBuildFactory, allPowerPCBuilders), + ('periodic-intel-mac-os-x-coverage', CoverageDataBuildFactory, allIntelBuilders), + ] + +def getBuilders(): + result = [] + for name, factory, slaves in _builders: + result.append({'name': name, + 'slavenames': slaves, + 'builddir': name, + 'factory': factory()}) + return result diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/factories.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/factories.py new file mode 100644 index 0000000..781375f --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/factories.py @@ -0,0 +1,70 @@ +from webkit.steps import * +from buildbot.process import factory + +s = factory.s + +class BuildFactory(factory.BuildFactory): + useProgress = False + def __init__(self): + factory.BuildFactory.__init__(self, [s(CheckOutSource)]) + +class StandardBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(SetConfiguration, configuration="release")) + self.steps.append(s(self.getCompileStep(), configuration="release")) + self.steps.append(s(self.getJavaScriptCoreTestStep())) + self.steps.append(s(LayoutTest)) + self.steps.append(s(UploadLayoutResults)) +# self.steps.append(s(UploadDiskImage)) + + def getCompileStep(self): + return CompileWebKit + + def getJavaScriptCoreTestStep(self): + return JavaScriptCoreTest + + +class NoSVGBuildFactory(StandardBuildFactory): + def getCompileStep(self): + return CompileWebKitNoSVG + + +class PixelTestBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(SetConfiguration, configuration="release")) + self.steps.append(s(CompileWebKit, configuration="release")) + self.steps.append(s(PixelLayoutTest)) + self.steps.append(s(UploadLayoutResults)) + +class LeakBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(SetConfiguration, configuration="debug")) + self.steps.append(s(CompileWebKit, configuration="debug")) + self.steps.append(s(JavaScriptCoreTest)) + self.steps.append(s(LeakTest)) + self.steps.append(s(UploadLayoutResults)) +# self.steps.append(s(UploadDiskImage)) + +class PageLoadTestBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(CompileWebKit, configuration="release")) + self.steps.append(s(PageLoadTest)) + +Win32BuildFactory = StandardBuildFactory + +class GtkBuildFactory(StandardBuildFactory): + def getCompileStep(self): + return CompileWebKitGtk + + def getJavaScriptCoreTestStep(self): + return JavaScriptCoreTestGtk + +class CoverageDataBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(GenerateCoverageData)) + self.steps.append(s(UploadCoverageData)) diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/schedulers.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/schedulers.py new file mode 100644 index 0000000..11f9d41 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/schedulers.py @@ -0,0 +1,15 @@ +from buildbot.scheduler import Scheduler, Periodic + +def getSchedulers(builders): + builder_names = map(lambda builder: builder['name'], builders) + post_commit_builders = [name for name in builder_names if name.startswith('post-commit-')] + ['page-layout-test-mac-os-x'] + post_commit_builders = [name for name in post_commit_builders if name in builder_names] + post_commit_builders.sort() + + periodic_builders = [b['name'] for b in builders if b['name'].startswith('periodic-')] + periodic_builders.sort() + + post_commit = Scheduler(name="post-commit", branch=None, treeStableTimer=90, builderNames=post_commit_builders) + periodic = Periodic("periodic", periodic_builders, 6 * 60 * 60) + + return [post_commit, periodic] diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/status.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/status.py new file mode 100644 index 0000000..ad5519e --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/status.py @@ -0,0 +1,19 @@ +from buildbot.status import html, mail, words + +web = html.WebStatus(http_port=8010) + +allBuildsEmail = mail.MailNotifier(fromaddr="buildbot@webkit.org", + extraRecipients=["mark+webkit-builds@bdash.net.nz"], + sendToInterestedUsers=False) +breakageEmail = mail.MailNotifier(fromaddr="buildbot@webkit.org", + lookup=mail.Domain("webkit.org"), + mode="failing") + +IRC = words.IRC(host="irc.freenode.net", + nick="webkit-build", + channels=["#webkit-build"], +# announceAllBuilds=True + ) + +def getStatusListeners(): + return [web, allBuildsEmail, breakageEmail, IRC] diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/steps.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/steps.py new file mode 100644 index 0000000..c1933a3 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/steps.py @@ -0,0 +1,220 @@ +from webkit.basesteps import ShellCommand, SVN, Test, Compile, UploadCommand +from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS + +class CheckOutSource(SVN): + svnurl = "http://svn.webkit.org/repository/webkit/trunk" + mode = "update" + def __init__(self, *args, **kwargs): + SVN.__init__(self, svnurl=self.svnurl, mode=self.mode, *args, **kwargs) + +class SetConfiguration(ShellCommand): + command = ["./WebKitTools/Scripts/set-webkit-configuration"] + + def __init__(self, *args, **kwargs): + configuration = kwargs.pop('configuration') + self.command = self.command + ['--' + configuration] + self.name = "set-configuration-%s" % (configuration, ) + self.description = ["set configuration %s" % (configuration, )] + self.descriptionDone = ["set configuration %s" % (configuration, )] + ShellCommand.__init__(self, *args, **kwargs) + + +class LayoutTest(Test): + name = "layout-test" + description = ["layout-tests running"] + descriptionDone = ["layout-tests"] + command = ["./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results", "--results-directory", "layout-test-results"] + + def commandComplete(self, cmd): + Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + incorrectLayoutLines = [line for line in logText.splitlines() if line.find('had incorrect layout') >= 0 or (line.find('test case') >= 0 and line.find(' crashed') >= 0)] + if incorrectLayoutLines: + self.incorrectLayoutLines = incorrectLayoutLines + else: + self.incorrectLayoutLines = None + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.incorrectLayoutLines: + return self.incorrectLayoutLines + + return [self.name] + + +class JavaScriptCoreTest(Test): + name = "jscore-test" + description = ["jscore-tests running"] + descriptionDone = ["jscore-tests"] + command = ["./WebKitTools/Scripts/run-javascriptcore-tests"] + logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'} + + def commandComplete(self, cmd): + Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0] + if statusLines and statusLines[0].split()[0] != '0': + self.regressionLine = statusLines[0] + else: + self.regressionLine = None + + def evaluateCommand(self, cmd): + if cmd.rc != 0: + return FAILURE + + if self.regressionLine: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.regressionLine: + return [self.name, self.regressionLine] + + return [self.name] + +class PixelLayoutTest(LayoutTest): + name = "pixel-layout-test" + description = ["pixel-layout-tests running"] + descriptionDone = ["pixel-layout-tests"] + command = LayoutTest.command + ["--pixel"] + + +class LeakTest(Test): + name = "leak-test" + description = ["leak-tests running"] + descriptionDone = ["leak-tests"] + command = ["./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--leaks", "--results-directory", "layout-test-results"] + + def commandComplete(self, cmd): + Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + lines = logText.splitlines() + self.totalLeakLines = [line for line in lines if line.find('total leaks found!') >= 0] + self.totalLeakLines += [line for line in lines if line.find('LEAK: ') >= 0] + self.totalLeakLines = [' '.join(x.split()[1:]) for x in self.totalLeakLines] + self.totalLeakLines += [line for line in lines if line.find('test case') >= 0 and line.find('crashed') >= 0] + + + def evaluateCommand(self, cmd): + if cmd.rc != 0: + return FAILURE + + if self.totalLeakLines: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.totalLeakLines: + return self.totalLeakLines + return [self.name] + + +class UploadLayoutResults(UploadCommand, ShellCommand): + name = "upload-results" + description = ["uploading results"] + descriptionDone = ["uploaded-results"] + command = "echo Disabled for now" + + def __init__(self, *args, **kwargs): + ShellCommand.__init__(self, *args, **kwargs) + + def setBuild(self, build): + ShellCommand.setBuild(self, build) + self.initializeForUpload() + + self.command = '''\ + if [[ -d layout-test-results ]]; then \ + find layout-test-results -type d -print0 | xargs -0 chmod ug+rx; \ + find layout-test-results -type f -print0 | xargs -0 chmod ug+r; \ + rsync -rlvzP --rsync-path="/home/buildresults/bin/rsync" layout-test-results/ %s && rm -rf layout-test-results; \ + fi; \ + CRASH_LOG=~/Library/Logs/CrashReporter/DumpRenderTree*.crash*; \ + if [[ -f $(ls -1 $CRASH_LOG | head -n 1 ) ]]; then \ + chmod ug+r $CRASH_LOG; \ + rsync -rlvzP --rsync-path="/home/buildresults/bin/rsync" $CRASH_LOG %s && rm -rf $CRASH_LOG; \ + fi;''' % (self.getRemotePath(), self.getRemotePath()) + + self.addFactoryArguments(command=self.command) + + +class CompileWebKit(Compile): + command = ["./WebKitTools/Scripts/build-webkit", "--no-color"] + env = {'WEBKITSUPPORTLIBRARIESZIPDIR': 'C:\\cygwin\\home\\buildbot'} + def __init__(self, *args, **kwargs): + configuration = kwargs.pop('configuration') + + self.name = "compile-" + configuration + self.description = ["compiling " + configuration] + self.descriptionDone = ["compiled " + configuration] + + Compile.__init__(self, *args, **kwargs) + + +class CompileWebKitNoSVG(CompileWebKit): + command = 'rm -rf WebKitBuild && ./WebKitTools/Scripts/build-webkit --no-svg --no-color' + +class CompileWebKitGtk(CompileWebKit): + command = CompileWebKit.command + ['--gtk'] + +class JavaScriptCoreTestGtk(JavaScriptCoreTest): + command = JavaScriptCoreTest.command + ['--gtk'] + +class InstallWin32Dependencies(ShellCommand): + description = ["installing Windows dependencies"] + descriptionDone = ["installed Windows dependencies"] + command = ["WebKitTools/Scripts/update-webkit-auxiliary-libs"] + + +# class UploadDiskImage(UploadCommand, ShellCommand): +# description = ["uploading disk image"] +# descriptionDone = ["uploaded disk image"] +# name = "upload-disk-image" + +# def __init__(self, *args, **kwargs): +# UploadCommand.__init__(self, *args, **kwargs) +# self.command = 'umask 002 && ./WebKitTools/BuildSlaveSupport/build-launcher-app && ./WebKitTools/BuildSlaveSupport/build-launcher-dmg --upload-to-host %s' % (self.getRemotePath(), ) +# ShellCommand.__init__(self, *args, **kwargs) + +class GenerateCoverageData(Compile): + command = ["./WebKitTools/Scripts/generate-coverage-data"] + description = ["generating coverage data"] + descriptionDone = ["generated coverage data"] + + +class UploadCoverageData(UploadCommand, ShellCommand): + name = "upload-coverage-data" + description = ["uploading coverage data"] + descriptionDone = ["uploaded-coverage-data"] + command = "echo Disabled for now" + + def __init__(self, *args, **kwargs): + ShellCommand.__init__(self, *args, **kwargs) + + def setBuild(self, build): + ShellCommand.setBuild(self, build) + self.initializeForUpload() + + self.command = '''\ + if [[ -d WebKitBuild/Coverage/html ]]; then \ + find WebKitBuild/Coverage/html -type d -print0 | xargs -0 chmod ug+rx; \ + find WebKitBuild/Coverage/html -type f -print0 | xargs -0 chmod ug+r; \ + rsync -rlvzP --rsync-path="/home/buildresults/bin/rsync" WebKitBuild/Coverage/html/ %s && rm -rf WebKitBuild/Coverage/html; \ + fi;''' % (self.getRemotePath(), ) + + self.addFactoryArguments(command=self.command) + + def getURLPath(self): + return "/results/code-coverage/" |