# -*- python -*- # ex: set syntax=python: c = BuildmasterConfig = {} from buildbot.buildslave import BuildSlave from buildbot.changes.pb import PBChangeSource from buildbot.scheduler import AnyBranchScheduler, Triggerable from buildbot.status import html from buildbot.process import buildstep, factory, properties from buildbot.steps import master, shell, source, transfer, trigger from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED from twisted.internet import defer import simplejson WithProperties = properties.WithProperties class ConfigureBuild(buildstep.BuildStep): name = "configure build" description = ["configuring build"] descriptionDone = ["configured build"] def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs): buildstep.BuildStep.__init__(self, *args, **kwargs) self.platform = platform.split('-', 1)[0] self.fullPlatform = platform self.configuration = configuration self.architecture = architecture self.buildOnly = buildOnly self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly) def start(self): self.setProperty("platform", self.platform) self.setProperty("fullPlatform", self.fullPlatform) self.setProperty("configuration", self.configuration) self.setProperty("architecture", self.architecture) self.setProperty("buildOnly", self.buildOnly) self.finished(SUCCESS) return defer.succeed(None) class CheckOutSource(source.SVN): baseURL = "http://svn.webkit.org/repository/webkit/" mode = "update" def __init__(self, *args, **kwargs): source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs) class InstallWin32Dependencies(shell.Compile): description = ["installing dependencies"] descriptionDone = ["installed dependencies"] command = ["perl", "./WebKitTools/Scripts/update-webkit-auxiliary-libs"] def appendCustomBuildFlags(step, platform): if platform in ('gtk', 'wx', 'qt'): step.setCommand(step.command + ['--' + platform]) class CompileWebKit(shell.Compile): command = ["perl", "./WebKitTools/Scripts/build-webkit", WithProperties("--%(configuration)s")] env = {'MFLAGS':''} name = "compile-webkit" description = ["compiling"] descriptionDone = ["compiled"] warningPattern = ".*arning: .*" def start(self): platform = self.getProperty('platform') buildOnly = self.getProperty('buildOnly') if platform == 'mac' and buildOnly: self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym']) appendCustomBuildFlags(self, platform) return shell.Compile.start(self) class ArchiveBuiltProduct(shell.ShellCommand): command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive", WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] name = "archive-built-product" description = ["archiving built product"] descriptionDone = ["archived built product"] haltOnFailure = True class ExtractBuiltProduct(shell.ShellCommand): command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive", WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"] name = "extract-built-product" description = ["extracting built product"] descriptionDone = ["extracted built product"] haltOnFailure = True class UploadBuiltProduct(transfer.FileUpload): slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip") masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") haltOnFailure = True def __init__(self): transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) class DownloadBuiltProduct(transfer.FileDownload): slavedest = WithProperties("WebKitBuild/%(configuration)s.zip") mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") haltOnFailure = True def __init__(self): transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest) class RunJavaScriptCoreTests(shell.Test): name = "jscore-test" description = ["jscore-tests running"] descriptionDone = ["jscore-tests"] command = ["perl", "./WebKitTools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")] logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'} def __init__(self, skipBuild=False, *args, **kwargs): self.skipBuild = skipBuild shell.Test.__init__(self, *args, **kwargs) self.addFactoryArguments(skipBuild=skipBuild) def start(self): appendCustomBuildFlags(self, self.getProperty('platform')) if self.skipBuild: self.setCommand(self.command + ['--skip-build']) return shell.Test.start(self) def commandComplete(self, cmd): shell.Test.commandComplete(self, cmd) logText = cmd.logs['stdio'].getText() statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0] if statusLines and statusLines[0].split()[0] != '0': self.regressionLine = statusLines[0] else: self.regressionLine = None def evaluateCommand(self, cmd): if self.regressionLine: return FAILURE if cmd.rc != 0: return FAILURE return SUCCESS def getText(self, cmd, results): return self.getText2(cmd, results) def getText2(self, cmd, results): if results != SUCCESS and self.regressionLine: return [self.name, self.regressionLine] return [self.name] class RunWebKitTests(shell.Test): name = "layout-test" description = ["layout-tests running"] descriptionDone = ["layout-tests"] command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results", "--no-sample-on-timeout", "--results-directory", "layout-test-results", "--use-remote-links-to-tests", WithProperties("--%(configuration)s")] def __init__(self, skipBuild=False, *args, **kwargs): self.skipBuild = skipBuild shell.Test.__init__(self, *args, **kwargs) self.addFactoryArguments(skipBuild=skipBuild) def start(self): appendCustomBuildFlags(self, self.getProperty('platform')) if self.skipBuild: self.setCommand(self.command + ['--root=WebKitBuild/bin']) return shell.Test.start(self) def commandComplete(self, cmd): shell.Test.commandComplete(self, cmd) logText = cmd.logs['stdio'].getText() incorrectLayoutLines = [] for line in logText.splitlines(): if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0: incorrectLayoutLines.append(line) elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0): incorrectLayoutLines.append(line) self.incorrectLayoutLines = incorrectLayoutLines def evaluateCommand(self, cmd): if self.incorrectLayoutLines: if len(self.incorrectLayoutLines) == 1 and (self.incorrectLayoutLines[0].find('were new') >= 0 or self.incorrectLayoutLines[0].find('was new') >= 0): return WARNINGS return FAILURE if cmd.rc != 0: return FAILURE return SUCCESS def getText(self, cmd, results): return self.getText2(cmd, results) def getText2(self, cmd, results): if results != SUCCESS and self.incorrectLayoutLines: return self.incorrectLayoutLines return [self.name] class ArchiveTestResults(shell.ShellCommand): command = ["python", "./WebKitTools/BuildSlaveSupport/test-result-archive", WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] name = "archive-test-results" description = ["archiving test results"] descriptionDone = ["archived test results"] haltOnFailure = True class UploadTestResults(transfer.FileUpload): slavesrc = "layout-test-results.zip" masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") def __init__(self): transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) class ExtractTestResults(master.MasterShellCommand): zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)") def __init__(self): master.MasterShellCommand.__init__(self, "") def start(self): self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)] return master.MasterShellCommand.start(self) def finished(self, result): url = self.build.getProperties().render(self.resultDirectory).replace("public_html/", "") self.addURL("view results", url) result = master.MasterShellCommand.finished(self, result) self.step_status.setText(["uploaded results"]) return result class Factory(factory.BuildFactory): def __init__(self, platform, configuration, architectures, buildOnly): factory.BuildFactory.__init__(self) self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly) self.addStep(CheckOutSource) if platform == "win": self.addStep(InstallWin32Dependencies) class BuildFactory(Factory): def __init__(self, platform, configuration, architectures, triggers): Factory.__init__(self, platform, configuration, architectures, True) self.addStep(CompileWebKit) self.addStep(ArchiveBuiltProduct) self.addStep(UploadBuiltProduct) self.addStep(trigger.Trigger, schedulerNames=triggers) class TestFactory(Factory): def __init__(self, platform, configuration, architectures): Factory.__init__(self, platform, configuration, architectures, False) self.addStep(DownloadBuiltProduct) self.addStep(ExtractBuiltProduct) self.addStep(RunJavaScriptCoreTests, skipBuild=True) self.addStep(RunWebKitTests, skipBuild=(platform == 'win')) self.addStep(ArchiveTestResults) self.addStep(UploadTestResults) self.addStep(ExtractTestResults) class BuildAndTestFactory(Factory): def __init__(self, platform, configuration, architectures): Factory.__init__(self, platform, configuration, architectures, False) self.addStep(CompileWebKit) self.addStep(RunJavaScriptCoreTests) self.addStep(RunWebKitTests) self.addStep(ArchiveTestResults) self.addStep(UploadTestResults) self.addStep(ExtractTestResults) def loadBuilderConfig(c): passwords = simplejson.load(open('passwords.json')) config = simplejson.load(open('config.json')) c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']] c['schedulers'] = [] for scheduler in config['schedulers']: kls = globals()[scheduler.pop('type')] c['schedulers'].append(kls(**scheduler)) c['builders'] = [] for builder in config['builders']: for slaveName in builder['slavenames']: for slave in config['slaves']: if slave['name'] != slaveName or slave['platform'] == '*': continue if slave['platform'] != builder['platform']: raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform']) break factory = globals()["%sFactory" % builder.pop('type')] factoryArgs = [] for key in "platform", "configuration", "architectures", "triggers": value = builder.pop(key, None) if value: factoryArgs.append(value) builder["factory"] = factory(*factoryArgs) c['builders'].append(builder) loadBuilderConfig(c) c['change_source'] = PBChangeSource() c['status'] = [] c['status'].append(html.WebStatus(http_port=8710, allowForce=True)) c['slavePortnum'] = 17000 c['projectName'] = "WebKit" c['projectURL'] = "http://webkit.org" c['buildbotURL'] = "http://build.webkit.org/"