diff options
Diffstat (limited to 'Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg')
| -rw-r--r-- | Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg | 567 |
1 files changed, 567 insertions, 0 deletions
diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg b/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg new file mode 100644 index 0000000..df9dcb0 --- /dev/null +++ b/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg @@ -0,0 +1,567 @@ +# -*- python -*- +# ex: set syntax=python: + +c = BuildmasterConfig = {} + +from buildbot.buildslave import BuildSlave +from buildbot.changes.pb import PBChangeSource +from buildbot.scheduler import AnyBranchScheduler, Triggerable +from buildbot.schedulers.filter import ChangeFilter +from buildbot.status import html +from buildbot.status.web.authz import Authz +from buildbot.process import buildstep, factory, properties +from buildbot.steps import master, shell, source, transfer, trigger +from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED + +from twisted.internet import defer + +import re +import simplejson + +from webkitpy.common.config import build as wkbuild +from webkitpy.common.net.buildbot import BuildBot as wkbuildbot + +WithProperties = properties.WithProperties + +class ConfigureBuild(buildstep.BuildStep): + name = "configure build" + description = ["configuring build"] + descriptionDone = ["configured build"] + def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs): + buildstep.BuildStep.__init__(self, *args, **kwargs) + self.platform = platform.split('-', 1)[0] + self.fullPlatform = platform + self.configuration = configuration + self.architecture = architecture + self.buildOnly = buildOnly + self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly) + + def start(self): + self.setProperty("platform", self.platform) + self.setProperty("fullPlatform", self.fullPlatform) + self.setProperty("configuration", self.configuration) + self.setProperty("architecture", self.architecture) + self.setProperty("buildOnly", self.buildOnly) + self.finished(SUCCESS) + return defer.succeed(None) + + +class CheckOutSource(source.SVN): + baseURL = "http://svn.webkit.org/repository/webkit/" + mode = "update" + def __init__(self, *args, **kwargs): + source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs) + + +class InstallWin32Dependencies(shell.Compile): + description = ["installing dependencies"] + descriptionDone = ["installed dependencies"] + command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"] + +class KillOldProcesses(shell.Compile): + name = "kill old processes" + description = ["killing old processes"] + descriptionDone = ["killed old processes"] + command = ["python", "./Tools/BuildSlaveSupport/win/kill-old-processes"] + +class InstallChromiumDependencies(shell.ShellCommand): + name = "gclient" + description = ["updating chromium dependencies"] + descriptionDone = ["updated chromium dependencies"] + command = ["perl", "./Tools/Scripts/update-webkit-chromium", "--force"] + haltOnFailure = True + +class CleanupChromiumLinuxCrashLogs(shell.ShellCommand): + name = "cleanup crash logs" + description = ["removing crash logs"] + descriptionDone = ["removed crash logs"] + command = ["sh", "-c", "rm -rf /tmp/.org.chromium.*"] + haltOnFailure = False + + +def appendCustomBuildFlags(step, platform): + if platform in ('gtk', 'wx', 'qt', 'chromium', 'wincairo', 'efl'): + step.setCommand(step.command + ['--' + platform]) + + +class CompileWebKit(shell.Compile): + command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")] + env = {'MFLAGS':''} + name = "compile-webkit" + description = ["compiling"] + descriptionDone = ["compiled"] + warningPattern = ".*arning: .*" + + def start(self): + platform = self.getProperty('platform') + buildOnly = self.getProperty('buildOnly') + if platform == 'mac' and buildOnly: + self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym']) + + appendCustomBuildFlags(self, platform) + return shell.Compile.start(self) + + +class ArchiveBuiltProduct(shell.ShellCommand): + command = ["python", "./Tools/BuildSlaveSupport/built-product-archive", + WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] + name = "archive-built-product" + description = ["archiving built product"] + descriptionDone = ["archived built product"] + haltOnFailure = True + + +class ExtractBuiltProduct(shell.ShellCommand): + command = ["python", "./Tools/BuildSlaveSupport/built-product-archive", + WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"] + name = "extract-built-product" + description = ["extracting built product"] + descriptionDone = ["extracted built product"] + haltOnFailure = True + + +class UploadBuiltProduct(transfer.FileUpload): + slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip") + masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") + haltOnFailure = True + + def __init__(self): + transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) + + +class DownloadBuiltProduct(transfer.FileDownload): + slavedest = WithProperties("WebKitBuild/%(configuration)s.zip") + mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") + haltOnFailure = True + flunkOnFailure = True + + def __init__(self): + transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest) + + +class RunJavaScriptCoreTests(shell.Test): + name = "jscore-test" + description = ["jscore-tests running"] + descriptionDone = ["jscore-tests"] + command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")] + logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'} + + def __init__(self, skipBuild=False, *args, **kwargs): + self.skipBuild = skipBuild + shell.Test.__init__(self, *args, **kwargs) + self.addFactoryArguments(skipBuild=skipBuild) + + def start(self): + appendCustomBuildFlags(self, self.getProperty('platform')) + if self.skipBuild: + self.setCommand(self.command + ['--skip-build']) + return shell.Test.start(self) + + def commandComplete(self, cmd): + shell.Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0] + if statusLines and statusLines[0].split()[0] != '0': + self.regressionLine = statusLines[0] + else: + self.regressionLine = None + + def evaluateCommand(self, cmd): + if self.regressionLine: + return FAILURE + + if cmd.rc != 0: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.regressionLine: + return [self.name, self.regressionLine] + + return [self.name] + + +class RunWebKitTests(shell.Test): + name = "layout-test" + description = ["layout-tests running"] + descriptionDone = ["layout-tests"] + command = ["perl", "./Tools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results", + "--no-sample-on-timeout", "--results-directory", "layout-test-results", "--use-remote-links-to-tests", + WithProperties("--%(configuration)s"), "--exit-after-n-crashes-or-timeouts", "20"] + + def __init__(self, skipBuild=False, *args, **kwargs): + self.skipBuild = skipBuild + shell.Test.__init__(self, *args, **kwargs) + self.addFactoryArguments(skipBuild=skipBuild) + + def start(self): + appendCustomBuildFlags(self, self.getProperty('platform')) + if self.skipBuild: + self.setCommand(self.command + ['--root=WebKitBuild/bin']) + return shell.Test.start(self) + + def commandComplete(self, cmd): + shell.Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + incorrectLayoutLines = [] + for line in logText.splitlines(): + if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0: + incorrectLayoutLines.append(line) + elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0): + incorrectLayoutLines.append(line) + elif line.startswith("WARNING:") and line.find(' leak') >= 0: + incorrectLayoutLines.append(line.replace('WARNING: ', '')) + elif line.find('Exiting early') >= 0: + incorrectLayoutLines.append(line) + + # FIXME: Detect and summarize leaks of RefCounted objects + + self.incorrectLayoutLines = incorrectLayoutLines + + def evaluateCommand(self, cmd): + if self.incorrectLayoutLines: + if len(self.incorrectLayoutLines) == 1: + line = self.incorrectLayoutLines[0] + if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0: + return WARNINGS + + return FAILURE + + if cmd.rc != 0: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.incorrectLayoutLines: + return self.incorrectLayoutLines + + return [self.name] + + +class NewRunWebKitTests(RunWebKitTests): + command = ["python", "./Tools/Scripts/new-run-webkit-tests", "--noshow-results", + "--verbose", "--results-directory", "layout-test-results", + "--builder-name", WithProperties("%(buildername)s"), + "--build-number", WithProperties("%(buildnumber)s"), + "--master-name", "webkit.org", + "--test-results-server", "test-results.appspot.com", + WithProperties("--%(configuration)s")] + + +class RunPythonTests(shell.Test): + name = "webkitpy-test" + description = ["python-tests running"] + descriptionDone = ["python-tests"] + command = ["python", "./Tools/Scripts/test-webkitpy"] + + +class RunPerlTests(shell.Test): + name = "webkitperl-test" + description = ["perl-tests running"] + descriptionDone = ["perl-tests"] + command = ["perl", "./Tools/Scripts/test-webkitperl"] + + +class RunGtkAPITests(shell.Test): + name = "API tests" + description = ["API tests running"] + descriptionDone = ["API tests"] + command = ["perl", "./Tools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")] + + def commandComplete(self, cmd): + shell.Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + incorrectLines = [] + for line in logText.splitlines(): + if line.startswith('ERROR'): + incorrectLines.append(line) + + self.incorrectLines = incorrectLines + + def evaluateCommand(self, cmd): + if self.incorrectLines: + return FAILURE + + if cmd.rc != 0: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.incorrectLines: + return ["%d API tests failed" % len(self.incorrectLines)] + + return [self.name] + +class RunQtAPITests(shell.Test): + name = "API tests" + description = ["API tests running"] + descriptionDone = ["API tests"] + command = ["python", "./Tools/Scripts/run-qtwebkit-tests", + "--output-file=qt-unit-tests.html", "--do-not-open-results", + WithProperties("WebKitBuild/%(configuration_pretty)s/WebKit/qt/tests/")] + + def start(self): + self.setProperty("configuration_pretty", self.getProperty("configuration").title()) + return shell.Test.start(self) + + def commandComplete(self, cmd): + shell.Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped", logText) + + self.incorrectTests = 0 + self.statusLine = [] + + if foundItems: + self.incorrectTests = int(foundItems[0][1]) + if self.incorrectTests > 0: + self.statusLine = [ + "%s passed, %s failed, %s skipped" % (foundItems[0][0], foundItems[0][1], foundItems[0][2]) + ] + + def evaluateCommand(self, cmd): + if self.incorrectTests: + return WARNINGS + + if cmd.rc != 0: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.incorrectTests: + return self.statusLine + + return [self.name] + +class RunWebKitLeakTests(RunWebKitTests): + def start(self): + self.setCommand(self.command + ["--leaks"]) + return RunWebKitTests.start(self) + + +class RunWebKit2Tests(RunWebKitTests): + def start(self): + self.setCommand(self.command + ["--webkit-test-runner"]) + return RunWebKitTests.start(self) + + +class RunChromiumWebKitUnitTests(shell.Test): + name = "webkit-unit-tests" + description = ["webkit-unit-tests running"] + descriptionDone = ["webkit-unit-tests"] + command = ["perl", "./Tools/Scripts/run-chromium-webkit-unit-tests", + WithProperties("--%(configuration)s")] + + +class ArchiveTestResults(shell.ShellCommand): + command = ["python", "./Tools/BuildSlaveSupport/test-result-archive", + WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] + name = "archive-test-results" + description = ["archiving test results"] + descriptionDone = ["archived test results"] + haltOnFailure = True + + +class UploadTestResults(transfer.FileUpload): + slavesrc = "layout-test-results.zip" + masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") + + def __init__(self): + transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) + + +class ExtractTestResults(master.MasterShellCommand): + zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") + resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)") + + def __init__(self): + master.MasterShellCommand.__init__(self, "") + + def start(self): + self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)] + return master.MasterShellCommand.start(self) + + def finished(self, result): + url = self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + self.addURL("view results", url) + result = master.MasterShellCommand.finished(self, result) + self.step_status.setText(["uploaded results"]) + return result + + +class Factory(factory.BuildFactory): + def __init__(self, platform, configuration, architectures, buildOnly): + factory.BuildFactory.__init__(self) + self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly) + self.addStep(CheckOutSource) + if platform == "win": + self.addStep(KillOldProcesses) + self.addStep(InstallWin32Dependencies) + if platform.startswith("chromium"): + self.addStep(InstallChromiumDependencies) + +class BuildFactory(Factory): + def __init__(self, platform, configuration, architectures, triggers=None): + Factory.__init__(self, platform, configuration, architectures, True) + self.addStep(CompileWebKit) + if triggers: + self.addStep(ArchiveBuiltProduct) + self.addStep(UploadBuiltProduct) + self.addStep(trigger.Trigger, schedulerNames=triggers) + +class TestFactory(Factory): + TestClass = RunWebKitTests + def __init__(self, platform, configuration, architectures): + Factory.__init__(self, platform, configuration, architectures, False) + self.addStep(DownloadBuiltProduct) + self.addStep(ExtractBuiltProduct) + self.addStep(RunJavaScriptCoreTests, skipBuild=True) + self.addStep(self.TestClass, skipBuild=(platform == 'win')) + # Tiger's Python 2.3 is too old. WebKit Python requires 2.5+. + # Sadly we have no way to detect the version on the slave from here. + if platform != "mac-tiger": + self.addStep(RunPythonTests) + self.addStep(RunPerlTests) + self.addStep(ArchiveTestResults) + self.addStep(UploadTestResults) + self.addStep(ExtractTestResults) + +class BuildAndTestFactory(Factory): + TestClass = RunWebKitTests + def __init__(self, platform, configuration, architectures): + Factory.__init__(self, platform, configuration, architectures, False) + if platform == "chromium-linux": + self.addStep(CleanupChromiumLinuxCrashLogs) + self.addStep(CompileWebKit) + if not platform.startswith("chromium"): + self.addStep(RunJavaScriptCoreTests) + if platform.startswith("chromium"): + self.addStep(RunChromiumWebKitUnitTests) + self.addStep(self.TestClass) + # Tiger's Python 2.3 is too old. WebKit Python requires 2.5+. + # Sadly we have no way to detect the version on the slave from here. + if platform != "mac-tiger": + self.addStep(RunPythonTests) + self.addStep(RunPerlTests) + self.addStep(ArchiveTestResults) + self.addStep(UploadTestResults) + self.addStep(ExtractTestResults) + if platform == "gtk": + self.addStep(RunGtkAPITests) + if platform == "qt": + self.addStep(RunQtAPITests) + +class BuildAndTestLeaksFactory(BuildAndTestFactory): + TestClass = RunWebKitLeakTests + +class NewBuildAndTestFactory(BuildAndTestFactory): + TestClass = NewRunWebKitTests + +class TestWebKit2Factory(TestFactory): + TestClass = RunWebKit2Tests + +class PlatformSpecificScheduler(AnyBranchScheduler): + def __init__(self, platform, branch, **kwargs): + self.platform = platform + filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter) + AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs) + + def filter(self, change): + return wkbuild.should_build(self.platform, change.files) + +trunk_filter = ChangeFilter(branch=["trunk", None]) + +def loadBuilderConfig(c): + # FIXME: These file handles are leaked. + passwords = simplejson.load(open('passwords.json')) + config = simplejson.load(open('config.json')) + + # use webkitpy's buildbot module to test for core builders + wkbb = wkbuildbot() + + c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']] + + c['schedulers'] = [] + for scheduler in config['schedulers']: + if "change_filter" in scheduler: + scheduler["change_filter"] = globals()[scheduler["change_filter"]] + kls = globals()[scheduler.pop('type')] + c['schedulers'].append(kls(**scheduler)) + + c['builders'] = [] + for builder in config['builders']: + for slaveName in builder['slavenames']: + for slave in config['slaves']: + if slave['name'] != slaveName or slave['platform'] == '*': + continue + + if slave['platform'] != builder['platform']: + raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform']) + + break + + factory = globals()["%sFactory" % builder.pop('type')] + factoryArgs = [] + for key in "platform", "configuration", "architectures", "triggers": + value = builder.pop(key, None) + if value: + factoryArgs.append(value) + + builder["factory"] = factory(*factoryArgs) + + builder["category"] = "noncore" + if wkbb._is_core_builder(builder['name']): + builder["category"] = "core" + + c['builders'].append(builder) + +loadBuilderConfig(c) + +c['change_source'] = PBChangeSource() + +# permissions for WebStatus +authz = Authz( + forceBuild=True, + forceAllBuilds=True, + pingBuilder=True, + gracefulShutdown=False, + stopBuild=True, + stopAllBuilds=True, + cancelPendingBuild=True, + stopChange=True, + cleanShutdown=False) + +c['status'] = [] +c['status'].append(html.WebStatus(http_port=8710, + revlink="http://trac.webkit.org/changeset/%s", + authz=authz)) + +c['slavePortnum'] = 17000 +c['projectName'] = "WebKit" +c['projectURL'] = "http://webkit.org" +c['buildbotURL'] = "http://build.webkit.org/" + +c['buildHorizon'] = 1000 +c['logHorizon'] = 500 +c['eventHorizon'] = 200 +c['buildCacheSize'] = 60 |