diff options
Diffstat (limited to 'WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg')
| -rw-r--r-- | WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg | 552 |
1 files changed, 0 insertions, 552 deletions
diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg deleted file mode 100644 index c28abb6..0000000 --- a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg +++ /dev/null @@ -1,552 +0,0 @@ -# -*- python -*- -# ex: set syntax=python: - -c = BuildmasterConfig = {} - -from buildbot.buildslave import BuildSlave -from buildbot.changes.pb import PBChangeSource -from buildbot.scheduler import AnyBranchScheduler, Triggerable -from buildbot.schedulers.filter import ChangeFilter -from buildbot.status import html -from buildbot.process import buildstep, factory, properties -from buildbot.steps import master, shell, source, transfer, trigger -from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED - -from twisted.internet import defer - -import re -import simplejson - -from webkitpy.common.config import build as wkbuild -from webkitpy.common.net.buildbot import BuildBot as wkbuildbot - -WithProperties = properties.WithProperties - -class ConfigureBuild(buildstep.BuildStep): - name = "configure build" - description = ["configuring build"] - descriptionDone = ["configured build"] - def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs): - buildstep.BuildStep.__init__(self, *args, **kwargs) - self.platform = platform.split('-', 1)[0] - self.fullPlatform = platform - self.configuration = configuration - self.architecture = architecture - self.buildOnly = buildOnly - self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly) - - def start(self): - self.setProperty("platform", self.platform) - self.setProperty("fullPlatform", self.fullPlatform) - self.setProperty("configuration", self.configuration) - self.setProperty("architecture", self.architecture) - self.setProperty("buildOnly", self.buildOnly) - self.finished(SUCCESS) - return defer.succeed(None) - - -class CheckOutSource(source.SVN): - baseURL = "http://svn.webkit.org/repository/webkit/" - mode = "update" - def __init__(self, *args, **kwargs): - source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs) - - -class InstallWin32Dependencies(shell.Compile): - description = ["installing dependencies"] - descriptionDone = ["installed dependencies"] - command = ["perl", "./WebKitTools/Scripts/update-webkit-auxiliary-libs"] - -class KillOldProcesses(shell.Compile): - name = "kill old processes" - description = ["killing old processes"] - descriptionDone = ["killed old processes"] - command = ["python", "./WebKitTools/BuildSlaveSupport/win/kill-old-processes"] - -class InstallChromiumDependencies(shell.ShellCommand): - name = "gclient" - description = ["updating chromium dependencies"] - descriptionDone = ["updated chromium dependencies"] - command = ["perl", "./WebKitTools/Scripts/update-webkit-chromium", "--force"] - haltOnFailure = True - -class CleanupChromiumLinuxCrashLogs(shell.ShellCommand): - name = "cleanup crash logs" - description = ["removing crash logs"] - descriptionDone = ["removed crash logs"] - command = ["sh", "-c", "rm -rf /tmp/.org.chromium.*"] - haltOnFailure = False - - -def appendCustomBuildFlags(step, platform): - if platform in ('gtk', 'wx', 'qt', 'chromium', 'wincairo', 'efl'): - step.setCommand(step.command + ['--' + platform]) - - -class CompileWebKit(shell.Compile): - command = ["perl", "./WebKitTools/Scripts/build-webkit", WithProperties("--%(configuration)s")] - env = {'MFLAGS':''} - name = "compile-webkit" - description = ["compiling"] - descriptionDone = ["compiled"] - warningPattern = ".*arning: .*" - - def start(self): - platform = self.getProperty('platform') - buildOnly = self.getProperty('buildOnly') - if platform == 'mac' and buildOnly: - self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym']) - - appendCustomBuildFlags(self, platform) - return shell.Compile.start(self) - - -class ArchiveBuiltProduct(shell.ShellCommand): - command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive", - WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] - name = "archive-built-product" - description = ["archiving built product"] - descriptionDone = ["archived built product"] - haltOnFailure = True - - -class ExtractBuiltProduct(shell.ShellCommand): - command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive", - WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"] - name = "extract-built-product" - description = ["extracting built product"] - descriptionDone = ["extracted built product"] - haltOnFailure = True - - -class UploadBuiltProduct(transfer.FileUpload): - slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip") - masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") - haltOnFailure = True - - def __init__(self): - transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) - - -class DownloadBuiltProduct(transfer.FileDownload): - slavedest = WithProperties("WebKitBuild/%(configuration)s.zip") - mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") - haltOnFailure = True - flunkOnFailure = True - - def __init__(self): - transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest) - - -class RunJavaScriptCoreTests(shell.Test): - name = "jscore-test" - description = ["jscore-tests running"] - descriptionDone = ["jscore-tests"] - command = ["perl", "./WebKitTools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")] - logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'} - - def __init__(self, skipBuild=False, *args, **kwargs): - self.skipBuild = skipBuild - shell.Test.__init__(self, *args, **kwargs) - self.addFactoryArguments(skipBuild=skipBuild) - - def start(self): - appendCustomBuildFlags(self, self.getProperty('platform')) - if self.skipBuild: - self.setCommand(self.command + ['--skip-build']) - return shell.Test.start(self) - - def commandComplete(self, cmd): - shell.Test.commandComplete(self, cmd) - - logText = cmd.logs['stdio'].getText() - statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0] - if statusLines and statusLines[0].split()[0] != '0': - self.regressionLine = statusLines[0] - else: - self.regressionLine = None - - def evaluateCommand(self, cmd): - if self.regressionLine: - return FAILURE - - if cmd.rc != 0: - return FAILURE - - return SUCCESS - - def getText(self, cmd, results): - return self.getText2(cmd, results) - - def getText2(self, cmd, results): - if results != SUCCESS and self.regressionLine: - return [self.name, self.regressionLine] - - return [self.name] - - -class RunWebKitTests(shell.Test): - name = "layout-test" - description = ["layout-tests running"] - descriptionDone = ["layout-tests"] - command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results", - "--no-sample-on-timeout", "--results-directory", "layout-test-results", "--use-remote-links-to-tests", - WithProperties("--%(configuration)s"), "--exit-after-n-crashes-or-timeouts", "20"] - - def __init__(self, skipBuild=False, *args, **kwargs): - self.skipBuild = skipBuild - shell.Test.__init__(self, *args, **kwargs) - self.addFactoryArguments(skipBuild=skipBuild) - - def start(self): - appendCustomBuildFlags(self, self.getProperty('platform')) - if self.skipBuild: - self.setCommand(self.command + ['--root=WebKitBuild/bin']) - return shell.Test.start(self) - - def commandComplete(self, cmd): - shell.Test.commandComplete(self, cmd) - - logText = cmd.logs['stdio'].getText() - incorrectLayoutLines = [] - for line in logText.splitlines(): - if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0: - incorrectLayoutLines.append(line) - elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0): - incorrectLayoutLines.append(line) - elif line.startswith("WARNING:") and line.find(' leak') >= 0: - incorrectLayoutLines.append(line.replace('WARNING: ', '')) - elif line.find('Exiting early') >= 0: - incorrectLayoutLines.append(line) - - # FIXME: Detect and summarize leaks of RefCounted objects - - self.incorrectLayoutLines = incorrectLayoutLines - - def evaluateCommand(self, cmd): - if self.incorrectLayoutLines: - if len(self.incorrectLayoutLines) == 1: - line = self.incorrectLayoutLines[0] - if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0: - return WARNINGS - - return FAILURE - - if cmd.rc != 0: - return FAILURE - - return SUCCESS - - def getText(self, cmd, results): - return self.getText2(cmd, results) - - def getText2(self, cmd, results): - if results != SUCCESS and self.incorrectLayoutLines: - return self.incorrectLayoutLines - - return [self.name] - - -class NewRunWebKitTests(RunWebKitTests): - command = ["python", "./WebKitTools/Scripts/new-run-webkit-tests", "--noshow-results", - "--verbose", "--results-directory", "layout-test-results", - "--builder-name", WithProperties("%(buildername)s"), - "--build-number", WithProperties("%(buildnumber)s"), - "--master-name", "webkit.org", - "--test-results-server", "test-results.appspot.com", - WithProperties("--%(configuration)s"), "--use-drt"] - - -class RunPythonTests(shell.Test): - name = "webkitpy-test" - description = ["python-tests running"] - descriptionDone = ["python-tests"] - command = ["python", "./WebKitTools/Scripts/test-webkitpy"] - - -class RunPerlTests(shell.Test): - name = "webkitperl-test" - description = ["perl-tests running"] - descriptionDone = ["perl-tests"] - command = ["perl", "./WebKitTools/Scripts/test-webkitperl"] - - -class RunGtkAPITests(shell.Test): - name = "API tests" - description = ["API tests running"] - descriptionDone = ["API tests"] - command = ["perl", "./WebKitTools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")] - - def commandComplete(self, cmd): - shell.Test.commandComplete(self, cmd) - - logText = cmd.logs['stdio'].getText() - incorrectLines = [] - for line in logText.splitlines(): - if line.startswith('ERROR'): - incorrectLines.append(line) - - self.incorrectLines = incorrectLines - - def evaluateCommand(self, cmd): - if self.incorrectLines: - return FAILURE - - if cmd.rc != 0: - return FAILURE - - return SUCCESS - - def getText(self, cmd, results): - return self.getText2(cmd, results) - - def getText2(self, cmd, results): - if results != SUCCESS and self.incorrectLines: - return ["%d API tests failed" % len(self.incorrectLines)] - - return [self.name] - -class RunQtAPITests(shell.Test): - name = "API tests" - description = ["API tests running"] - descriptionDone = ["API tests"] - command = ["python", "./WebKitTools/Scripts/run-qtwebkit-tests", - "--output-file=qt-unit-tests.html", "--do-not-open-results", - WithProperties("WebKitBuild/%(configuration_pretty)s/WebKit/qt/tests/")] - - def start(self): - self.setProperty("configuration_pretty", self.getProperty("configuration").title()) - return shell.Test.start(self) - - def commandComplete(self, cmd): - shell.Test.commandComplete(self, cmd) - - logText = cmd.logs['stdio'].getText() - foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped", logText) - - self.incorrectTests = 0 - self.statusLine = [] - - if foundItems: - self.incorrectTests = int(foundItems[0][1]) - if self.incorrectTests > 0: - self.statusLine = [ - "%s passed, %s failed, %s skipped" % (foundItems[0][0], foundItems[0][1], foundItems[0][2]) - ] - - def evaluateCommand(self, cmd): - if self.incorrectTests: - return WARNINGS - - if cmd.rc != 0: - return FAILURE - - return SUCCESS - - def getText(self, cmd, results): - return self.getText2(cmd, results) - - def getText2(self, cmd, results): - if results != SUCCESS and self.incorrectTests: - return self.statusLine - - return [self.name] - -class RunWebKitLeakTests(RunWebKitTests): - def start(self): - self.setCommand(self.command + ["--leaks"]) - return RunWebKitTests.start(self) - - -class RunWebKit2Tests(RunWebKitTests): - def start(self): - self.setCommand(self.command + ["--webkit-test-runner"]) - return RunWebKitTests.start(self) - - -class RunChromiumWebKitUnitTests(shell.Test): - name = "webkit-unit-tests" - description = ["webkit-unit-tests running"] - descriptionDone = ["webkit-unit-tests"] - command = ["perl", "./WebKitTools/Scripts/run-chromium-webkit-unit-tests", - WithProperties("--%(configuration)s")] - - -class ArchiveTestResults(shell.ShellCommand): - command = ["python", "./WebKitTools/BuildSlaveSupport/test-result-archive", - WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] - name = "archive-test-results" - description = ["archiving test results"] - descriptionDone = ["archived test results"] - haltOnFailure = True - - -class UploadTestResults(transfer.FileUpload): - slavesrc = "layout-test-results.zip" - masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") - - def __init__(self): - transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) - - -class ExtractTestResults(master.MasterShellCommand): - zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") - resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)") - - def __init__(self): - master.MasterShellCommand.__init__(self, "") - - def start(self): - self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)] - return master.MasterShellCommand.start(self) - - def finished(self, result): - url = self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") - self.addURL("view results", url) - result = master.MasterShellCommand.finished(self, result) - self.step_status.setText(["uploaded results"]) - return result - - -class Factory(factory.BuildFactory): - def __init__(self, platform, configuration, architectures, buildOnly): - factory.BuildFactory.__init__(self) - self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly) - self.addStep(CheckOutSource) - if platform == "win": - self.addStep(KillOldProcesses) - self.addStep(InstallWin32Dependencies) - if platform.startswith("chromium"): - self.addStep(InstallChromiumDependencies) - -class BuildFactory(Factory): - def __init__(self, platform, configuration, architectures, triggers=None): - Factory.__init__(self, platform, configuration, architectures, True) - self.addStep(CompileWebKit) - if triggers: - self.addStep(ArchiveBuiltProduct) - self.addStep(UploadBuiltProduct) - self.addStep(trigger.Trigger, schedulerNames=triggers) - -class TestFactory(Factory): - TestClass = RunWebKitTests - def __init__(self, platform, configuration, architectures): - Factory.__init__(self, platform, configuration, architectures, False) - self.addStep(DownloadBuiltProduct) - self.addStep(ExtractBuiltProduct) - self.addStep(RunJavaScriptCoreTests, skipBuild=True) - self.addStep(self.TestClass, skipBuild=(platform == 'win')) - # Tiger's Python 2.3 is too old. WebKit Python requires 2.5+. - # Sadly we have no way to detect the version on the slave from here. - if platform != "mac-tiger": - self.addStep(RunPythonTests) - self.addStep(RunPerlTests) - self.addStep(ArchiveTestResults) - self.addStep(UploadTestResults) - self.addStep(ExtractTestResults) - -class BuildAndTestFactory(Factory): - TestClass = RunWebKitTests - def __init__(self, platform, configuration, architectures): - Factory.__init__(self, platform, configuration, architectures, False) - if platform == "chromium-linux": - self.addStep(CleanupChromiumLinuxCrashLogs) - self.addStep(CompileWebKit) - if not platform.startswith("chromium"): - self.addStep(RunJavaScriptCoreTests) - if platform.startswith("chromium"): - self.addStep(RunChromiumWebKitUnitTests) - self.addStep(self.TestClass) - # Tiger's Python 2.3 is too old. WebKit Python requires 2.5+. - # Sadly we have no way to detect the version on the slave from here. - if platform != "mac-tiger": - self.addStep(RunPythonTests) - self.addStep(RunPerlTests) - self.addStep(ArchiveTestResults) - self.addStep(UploadTestResults) - self.addStep(ExtractTestResults) - if platform == "gtk": - self.addStep(RunGtkAPITests) - if platform == "qt": - self.addStep(RunQtAPITests) - -class BuildAndTestLeaksFactory(BuildAndTestFactory): - TestClass = RunWebKitLeakTests - -class NewBuildAndTestFactory(BuildAndTestFactory): - TestClass = NewRunWebKitTests - -class TestWebKit2Factory(TestFactory): - TestClass = RunWebKit2Tests - -class PlatformSpecificScheduler(AnyBranchScheduler): - def __init__(self, platform, branch, **kwargs): - self.platform = platform - filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter) - AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs) - - def filter(self, change): - return wkbuild.should_build(self.platform, change.files) - -trunk_filter = ChangeFilter(branch=["trunk", None]) - -def loadBuilderConfig(c): - # FIXME: These file handles are leaked. - passwords = simplejson.load(open('passwords.json')) - config = simplejson.load(open('config.json')) - - # use webkitpy's buildbot module to test for core builders - wkbb = wkbuildbot() - - c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']] - - c['schedulers'] = [] - for scheduler in config['schedulers']: - if "change_filter" in scheduler: - scheduler["change_filter"] = globals()[scheduler["change_filter"]] - kls = globals()[scheduler.pop('type')] - c['schedulers'].append(kls(**scheduler)) - - c['builders'] = [] - for builder in config['builders']: - for slaveName in builder['slavenames']: - for slave in config['slaves']: - if slave['name'] != slaveName or slave['platform'] == '*': - continue - - if slave['platform'] != builder['platform']: - raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform']) - - break - - factory = globals()["%sFactory" % builder.pop('type')] - factoryArgs = [] - for key in "platform", "configuration", "architectures", "triggers": - value = builder.pop(key, None) - if value: - factoryArgs.append(value) - - builder["factory"] = factory(*factoryArgs) - - builder["category"] = "noncore" - if wkbb._is_core_builder(builder['name']): - builder["category"] = "core" - - c['builders'].append(builder) - -loadBuilderConfig(c) - -c['change_source'] = PBChangeSource() - -c['status'] = [] -c['status'].append(html.WebStatus(http_port=8710, allowForce=True)) - -c['slavePortnum'] = 17000 -c['projectName'] = "WebKit" -c['projectURL'] = "http://webkit.org" -c['buildbotURL'] = "http://build.webkit.org/" - -c['buildHorizon'] = 1000 -c['logHorizon'] = 500 -c['eventHorizon'] = 200 -c['buildCacheSize'] = 60 |