diff options
author | Upstream <upstream-import@none> | 1970-01-12 13:46:40 +0000 |
---|---|---|
committer | Upstream <upstream-import@none> | 1970-01-12 13:46:40 +0000 |
commit | d8543bb6618c17b12da906afa77d216f58cf4058 (patch) | |
tree | c58dc05ed86825bd0ef8d305d58c8205106b540f /WebKitTools/BuildSlaveSupport | |
download | external_webkit-d8543bb6618c17b12da906afa77d216f58cf4058.zip external_webkit-d8543bb6618c17b12da906afa77d216f58cf4058.tar.gz external_webkit-d8543bb6618c17b12da906afa77d216f58cf4058.tar.bz2 |
external/webkit r30707
Diffstat (limited to 'WebKitTools/BuildSlaveSupport')
15 files changed, 964 insertions, 0 deletions
diff --git a/WebKitTools/BuildSlaveSupport/build-launcher-app b/WebKitTools/BuildSlaveSupport/build-launcher-app new file mode 100755 index 0000000..79f47bf --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build-launcher-app @@ -0,0 +1,165 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2006 Apple Computer, Inc. All rights reserved. +# Copyright (C) 2006 Mark Rowe <opendarwin.org@bdash.net.nz>. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Creates the launcher WebKit.app with bundled frameworks. + +use strict; + +use FindBin; +use lib "$FindBin::Bin/../Scripts"; +use Cwd 'realpath'; +use webkitdirs; +use VCSUtils; + +my $tigerProductDir = realpath("$FindBin::Bin/../../WebKitBuildTiger"); +my $leopardProductDir = realpath("$FindBin::Bin/../../WebKitBuildLeopard"); +setBaseProductDir($tigerProductDir); + +my @xcodeBuildArguments = XcodeOptions(); +unshift @xcodeBuildArguments, "SYMROOT=$tigerProductDir"; +unshift @xcodeBuildArguments, "OBJROOT=$tigerProductDir"; + +my $nightlyLauncherTemplatePath = "$FindBin::Bin/../WebKitLauncher"; +my $nightlyLauncherStagingPath = productDir() . "/WebKit.app"; +my $droseraProjectPath = "$FindBin::Bin/../Drosera/mac"; +my $droseraStagingPath = productDir() . "/DroseraLauncher.app"; + + +sub buildNightlyLauncher +{ + chdir($nightlyLauncherTemplatePath); + system("xcodebuild", "clean", "-alltargets", @xcodeBuildArguments, @ARGV) == 0 or die "Failed cleaning WebKitLauncher project"; + system("xcodebuild", @xcodeBuildArguments, @ARGV) == 0 or die "Failed building WebKitLauncher project"; + chdirWebKit(); +} + +sub currentRevision +{ + my $sourceDir = sourceDir(); + if (isSVNDirectory($sourceDir)) { + return currentSVNRevision(); + } elsif (isGitDirectory($sourceDir)) { + my $gitLog = `cd $sourceDir && LC_ALL=C git log --grep='git-svn-id: ' -n 1 | grep git-svn-id:`; + (my $revision) = ($gitLog =~ m/ +git-svn-id: .+@(\d+) /g); + return $revision; + } +} + +sub currentBranch +{ + my $sourceDir = sourceDir(); + my ($url, $branch); + if (isSVNDirectory($sourceDir)) { + my $svnInfo = `LC_ALL=C svn info $sourceDir | grep URL:`; + ($url) = ($svnInfo =~ m/URL: (.+)/g); + } elsif (isGitDirectory($sourceDir)) { + my $gitLog = `cd $sourceDir && LC_ALL=C git log --grep='git-svn-id: ' -n 1 | grep git-svn-id:`; + ($url) = ($gitLog =~ m/ +git-svn-id: (.+)@\d+ /g); + } + ($branch) = ($url =~ m/\/webkit\/(trunk|branches\/[^\/]+)/); + die "Unable to determine current SVN branch in $sourceDir" unless (defined $branch); + $branch =~ s/^branches\///; + return $branch; +} + +sub copyNightlyLauncher +{ + my $revision = currentRevision(); + my $branch = currentBranch(); + my $configuration = configuration(); + + my $infoPlist = "$nightlyLauncherStagingPath/Contents/Info.plist"; + my $versionFile = "$nightlyLauncherStagingPath/Contents/Resources/VERSION"; + my $branchFile = "$nightlyLauncherStagingPath/Contents/Resources/BRANCH"; + my $data; + open(IN, $infoPlist) or die "Couldn't open Info.plist in built application for reading"; + { + undef $/; + $data = <IN>; + } + close(IN); + open(OUT, ">$infoPlist") or die "Couldn't open Info.plist in built application for writing"; + $data =~ s/VERSION/$revision/g; + print OUT $data; + close(OUT); + + open(OUT, ">$versionFile") or die "Couldn't open VERSION in built application for writing"; + print OUT "$revision\n"; + close(OUT); + + open(OUT, ">$branchFile") or die "Couldn't open BRANCH in built application for writing"; + print OUT "$branch\n"; + close(OUT); + + my @frameworks = ("JavaScriptCore", "JavaScriptGlue", "WebCore", "WebKit"); + for my $framework (@frameworks) { + my $tigerFramework = "$tigerProductDir/$configuration/$framework.framework"; + my $leopardFramework = "$leopardProductDir/$configuration/$framework.framework"; + system("ditto", $tigerFramework, "$nightlyLauncherStagingPath/Contents/Frameworks/10.4/$framework.framework") == 0 or die "Failed copying $tigerFramework into $nightlyLauncherStagingPath"; + system("ditto", $leopardFramework, "$nightlyLauncherStagingPath/Contents/Frameworks/10.5/$framework.framework") == 0 or die "Failed copying $leopardFramework into $nightlyLauncherStagingPath"; + } +} + +sub buildDroseraLauncher +{ + chdir($droseraProjectPath); + system("xcodebuild", "clean", "-alltargets", @xcodeBuildArguments, @ARGV) == 0 or die "Failed cleaning Drosera project"; + # Build native platform only right now, as building universal with the 10.4u SDK cause Xcode to look for WebKit, + # WebCore & JavaScriptCore in the SDK under /Developer/SDKs/MacOSX10.4u.sdk/$(BUILT_PRODUCTS_DIR) where they do not exist + system("xcodebuild", "-alltargets", @xcodeBuildArguments, @ARGV) == 0 or die "Failed building Drosera project"; + chdirWebKit(); +} + +sub setDroseraLauncherVersion +{ + my $revision = currentRevision(); + my $infoPlist = "$droseraStagingPath/Contents/Info.plist"; + my $data; + open(IN, $infoPlist) or die "Couldn't open Info.plist in built application for reading"; + { + undef $/; + $data = <IN>; + } + close(IN); + open(OUT, ">$infoPlist") or die "Couldn't open Info.plist in built application for writing"; + $data =~ s/VERSION/$revision/g; + print OUT $data; + close(OUT); +} + +my $b = currentBranch(); +my $r = currentRevision(); +print "Branch: ", $b, "\n"; +print "Revision: ", $r, "\n"; + +chdirWebKit(); +buildNightlyLauncher(); +copyNightlyLauncher(); +buildDroseraLauncher(); +setDroseraLauncherVersion(); diff --git a/WebKitTools/BuildSlaveSupport/build-launcher-dmg b/WebKitTools/BuildSlaveSupport/build-launcher-dmg new file mode 100755 index 0000000..ff1a22d --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build-launcher-dmg @@ -0,0 +1,136 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2006 Apple Computer, Inc. All rights reserved. +# Copyright (C) 2006 Mark Rowe <opendarwin.org@bdash.net.nz>. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Script used by build slaves to create a disk-image containing WebKit.app. + +use strict; + +use File::Basename; +use Getopt::Long; +use FindBin; +use lib "$FindBin::Bin/../Scripts"; +use Cwd 'realpath'; +use webkitdirs; +use VCSUtils; + +my $tigerProductDir = realpath("$FindBin::Bin/../../WebKitBuildTiger"); +my $leopardProductDir = realpath("$FindBin::Bin/../../WebKitBuildLeopard"); +setBaseProductDir($tigerProductDir); + +my $nightlyLauncherStagingPath = productDir() . "/WebKit.app"; +my $droseraStagingPath = productDir() . "/DroseraLauncher.app"; +my $nightlyLauncherDiskImagePath; + +my $nightlyRemoteHost = 'webkit-nightlies@live.nightly.webkit.org'; +my $nightlyRemotePath = "/home/webkit-nightlies"; +my $nightlyRemoteLatestPath = "$nightlyRemotePath/update-latest.sh"; + +sub currentRevision +{ + my $sourceDir = sourceDir(); + if (isSVNDirectory($sourceDir)) { + return currentSVNRevision(); + } elsif (isGitDirectory($sourceDir)) { + my $gitLog = `cd $sourceDir && LC_ALL=C git log --grep='git-svn-id: ' -n 1 | grep git-svn-id:`; + (my $revision) = ($gitLog =~ m/ +git-svn-id: .+@(\d+) /g); + return $revision; + } +} + +sub buildDiskImage +{ + my $revision = currentRevision(); + my $productDir = productDir(); + $nightlyLauncherDiskImagePath = productDir() . "/WebKit-SVN-r$revision.dmg"; + + print "Removing previous temp source directory (if any)...\n"; + `rm -rf /tmp/WebKitNightly`; + die "Removing previous temp source directory failed" if $?; + + print "Making a new temp source directory...\n"; + `mkdir /tmp/WebKitNightly`; + die "Making a new temp source directory failed" if $?; + + print "Copying WebKit.app to temp source directory...\n"; + `cp -R \"$nightlyLauncherStagingPath\" /tmp/WebKitNightly/WebKit.app`; + die "Copying WebKit.app to temp source directory failed" if $?; + + print "Copying Drosera.app to temp source directory...\n"; + `cp -R \"$droseraStagingPath\" /tmp/WebKitNightly/Drosera.app`; + die "Copying Drosera.app to temp source directory failed" if $?; + + print "Creating disk image...\n"; + `hdiutil create \"$nightlyLauncherDiskImagePath\" -ov -srcfolder /tmp/WebKitNightly -fs HFS+ -volname \"WebKit\"`; + die "Creating disk image failed" if $?; + + print "Removing temp source directory...\n"; + `rm -rf /tmp/WebKitNightly`; + die "Removing temp source directory failed" if $?; + + print "Compressing disk image...\n"; + system("mv", "-f", $nightlyLauncherDiskImagePath, "$nightlyLauncherDiskImagePath.uncompressed.dmg") == 0 or die "Renaming disk image failed"; + system("hdiutil", "convert", "-quiet", "$nightlyLauncherDiskImagePath.uncompressed.dmg", "-format", "UDBZ", "-imagekey", "zlib-level=9", "-o", "$nightlyLauncherDiskImagePath"); + die "Compressing disk image failed" if $?; + + unlink "$nightlyLauncherDiskImagePath.uncompressed.dmg"; +} + +sub uploadNightlyDiskImage +{ + my $buildTag = shift(@_); + my $nightlyRemoteDiskImagePath = "$nightlyRemotePath/builds/$buildTag/mac/" . basename($nightlyLauncherDiskImagePath); + my $revision = currentRevision(); + system("rsync", "-vP", $nightlyLauncherDiskImagePath, "$nightlyRemoteHost:$nightlyRemoteDiskImagePath") == 0 or die "Failed uploading disk image"; + system("ssh", $nightlyRemoteHost, $nightlyRemoteLatestPath, $buildTag, "mac", $nightlyRemoteDiskImagePath, $revision) == 0 or die "Failed linking disk image to latest"; +} + +sub uploadBuildSlaveDiskImage +{ + my $remoteDiskImagePath = shift(@_) . basename($nightlyLauncherDiskImagePath); + system("rsync", "-vP", $nightlyLauncherDiskImagePath, $remoteDiskImagePath) == 0 or die "Failed uploading disk image"; +} + + +my $uploadTo; +my $nightlyBuild = 0; +my $buildTag = 'trunk'; +GetOptions('upload-to-host=s' => \$uploadTo, + 'upload-as-nightly!' => \$nightlyBuild, + 'tag=s' => \$buildTag); + +chdirWebKit(); +buildDiskImage($buildTag); + +if ($nightlyBuild) { + uploadNightlyDiskImage($buildTag); +} elsif ($uploadTo) { + uploadBuildSlaveDiskImage($uploadTo); +} else { + print "Disk image left at $nightlyLauncherDiskImagePath\n"; +} diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/Makefile b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/Makefile new file mode 100644 index 0000000..c3a935f --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/Makefile @@ -0,0 +1,19 @@ +# -*- makefile -*- + +# This is a simple makefile which lives in a buildmaster/buildslave +# directory (next to the buildbot.tac file). It allows you to start/stop the +# master or slave by doing 'make start' or 'make stop'. + +# The 'reconfig' target will tell a buildmaster to reload its config file. + +start: + twistd --no_save -y buildbot.tac + +stop: + kill `cat twistd.pid` + +reconfig: + kill -HUP `cat twistd.pid` + +log: + tail -f twistd.log diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.css b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.css new file mode 100644 index 0000000..81de950 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.css @@ -0,0 +1,108 @@ +* { + font-family: Verdana, Cursor; + font-size: 10px; + font-weight: bold; +} + +a:link,a:visited,a:active { + color: #444; +} +a:hover { + color: #FFFFFF; +} + +table { + border-spacing: 1px 1px; +} + +table td { + padding: 3px 0px 3px 0px; + text-align: center; +} + +.Project { + width: 100px; +} + +.LastBuild, .Activity { + width: 230px; + padding: 0 0 0 4px; +} + +td.Time { + color: #000; + border-bottom: 1px solid #aaa; + background-color: #eee; +} + +td.Activity, td.Change, td.Builder { + color: #333333; + background-color: #CCCCCC; +} + +td.Change { + border-radius: 5px; + -webkit-border-radius: 5px; +} +td.Event { + color: #777; + background-color: #ddd; + border-radius: 5px; + -webkit-border-radius: 5px; +} + +td.Activity { + border-top-left-radius: 10px; + -webkit-border-top-left-radius: 10px; + min-height: 20px; + padding: 8px 0 8px 0; +} + +td.idle, td.waiting, td.offline, td.building { + border-top-left-radius: 0px; + -webkit-border-top-left-radius: 0px; +} + +.LastBuild { + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + border-top-right-radius: 5px; + -webkit-border-top-right-radius: 5px; +} + +/* LastBuild, BuildStep states */ +.success { + color: #FFFFFF; + background-color: #8fdf5f; +} + +.failure { + color: #FFFFFF; + background-color: #e98080; +} + +.warnings { + color: #FFFFFF; + background-color: #ffc343; +} + +.exception, td.offline { + color: #FFFFFF; + background-color: #e0b0ff; +} + +.start,.running, td.building { + color: #666666; + background-color: #fffc6c; +} + +.start { + border-bottom-left-radius: 10px; + -webkit-border-bottom-left-radius: 10px; + border-bottom-right-radius: 10px; + -webkit-border-bottom-right-radius: 10px; +} + +td.Project a:hover, td.start a:hover { + color: #000; +} diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.tac b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.tac new file mode 100644 index 0000000..f66e068 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/buildbot.tac @@ -0,0 +1,10 @@ + +from twisted.application import service +from buildbot.master import BuildMaster + +basedir = r'/home/mrowe/sites/build.webkit.org/buildbot' +configfile = r'master.cfg' + +application = service.Application('buildmaster') +BuildMaster(basedir, configfile).setServiceParent(application) + diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg new file mode 100644 index 0000000..ea7966a --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/master.cfg @@ -0,0 +1,22 @@ +# -*- python -*- + +import os +from twisted.web import static +static.File.contentTypes = static.loadMimeTypes(['/etc/mime.types', os.path.join(basedir, 'mime.types')]) + +from buildbot.changes.pb import PBChangeSource +import webkit +reload(webkit) + +c = BuildmasterConfig = {} + +c['slaves'] = webkit.auth.getSlaveAuthenticationDetails() +c['change_source'] = [PBChangeSource(prefix="trunk")] +c['builders'] = webkit.builders.getBuilders() +c['schedulers'] = webkit.schedulers.getSchedulers(c['builders']) +c['status'] = webkit.status.getStatusListeners() + +c['slavePortnum'] = 9989 +c['projectName'] = "WebKit" +c['projectURL'] = "http://webkit.org/" +c['buildbotURL'] = "http://build.webkit.org/" diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/__init__.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/__init__.py new file mode 100644 index 0000000..f81fcae --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/__init__.py @@ -0,0 +1,5 @@ +from webkit import auth, status, builders, schedulers +reload(auth) +reload(status) +reload(builders) +reload(schedulers) diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/auth.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/auth.py new file mode 100644 index 0000000..b182e16 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/auth.py @@ -0,0 +1,9 @@ +from buildbot.buildslave import BuildSlave + +def getSlaveAuthenticationDetails(): + def createBuildSlave((name, password)): + return BuildSlave(name, password, max_builds=1) + return map(createBuildSlave, _getSlaveAuthenticationDetails()) + +def _getSlaveAuthenticationDetails(): + return [("slave-name", "password")] diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/basesteps.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/basesteps.py new file mode 100644 index 0000000..8bba881 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/basesteps.py @@ -0,0 +1,51 @@ +from buildbot.steps import shell, source +import os + + +def buildStepWithDefaultTimeout(klass, default_timeout=75*60): + class Step(klass): + timeout = default_timeout + def __init__(self, *args, **kwargs): + kwargs['timeout'] = self.timeout + klass.__init__(self, *args, **kwargs) + + return Step + + +Test = buildStepWithDefaultTimeout(shell.Test) +Compile = buildStepWithDefaultTimeout(shell.Compile) +ShellCommand = buildStepWithDefaultTimeout(shell.ShellCommand) +SVN = buildStepWithDefaultTimeout(source.SVN) + + +class UploadCommand: + def initializeForUpload(self): + try: + try: + umask = os.umask(0) + os.makedirs(self.getDestinationPath(), 042770) + except OSError, e: + if e.errno != 17: + raise + finally: + os.umask(umask) + + def getDestinationPath(self): + return "/home/buildresults%s" % (self.getURLPath(), ) + + def getRemotePath(self): + return "buildresults@build.webkit.org:%s" % (self.getDestinationPath(), ) + + def getURLPath(self): + return '/results/%s/%s/' % (self.getBuild().builder.name, self.getBuild().getProperty("buildnumber"), ) + + def getBuild(self): + return self.build + + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + return ['<a href="%s">%s</a>' % (self.getURLPath(), self.name)] + diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/builders.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/builders.py new file mode 100644 index 0000000..b4e087e --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/builders.py @@ -0,0 +1,35 @@ +from webkit.factories import * +from buildbot import locks + +# There are four build slaves that take care of the majority of builds, with two other specialist slaves at Apple +# Slave 1 isn older G4 PowerMac dedicated to the PLT builds, as it needs extra configuration +# Slave 2 is a Windows PC dedicated to the Windows builds +# Slaves 3 and 4 are older G4 PowerMacs with relatively low amounts of RAM which leads to insanely slow leaks tests +# Slaves 4 and 5 are newer G5 PowerMacs with ATI graphics cards that lead to kernel panics during pixel tests + +nonATIPowerPCBuilders = ['apple-slave-%d' % i for i in (3, 4)] +ATIPowerPCBuilders = ['apple-slave-%d' % i for i in (1, 5, 6)] +allPowerPCBuilders = nonATIPowerPCBuilders + ATIPowerPCBuilders +allIntelBuilders = ['bdash-slave-1', 'bdash-slave-2'] + +_builders = [('post-commit-powerpc-mac-os-x', StandardBuildFactory, allPowerPCBuilders), + ('post-commit-intel-mac-os-x', StandardBuildFactory, allIntelBuilders), + ('post-commit-leaks-powerpc-mac-os-x', LeakBuildFactory, allPowerPCBuilders), + ('post-commit-leaks-intel-mac-os-x', LeakBuildFactory, allIntelBuilders), +# ('page-layout-test-mac-os-x', PageLoadTestBuildFactory, ['apple-slave-1']), +# ('post-commit-pixel-powerpc-mac-os-x', PixelTestBuildFactory, nonATIPowerPCBuilders), + ('post-commit-win32', Win32BuildFactory, ['apple-slave-2']), + ('post-commit-linux-qt', StandardBuildFactory, ['webtroll-slave-1']), + ('post-commit-linux-gtk', GtkBuildFactory, ['zecke-slave-1']), + ('periodic-powerpc-mac-os-x-no-svg', NoSVGBuildFactory, allPowerPCBuilders), + ('periodic-intel-mac-os-x-coverage', CoverageDataBuildFactory, allIntelBuilders), + ] + +def getBuilders(): + result = [] + for name, factory, slaves in _builders: + result.append({'name': name, + 'slavenames': slaves, + 'builddir': name, + 'factory': factory()}) + return result diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/factories.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/factories.py new file mode 100644 index 0000000..781375f --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/factories.py @@ -0,0 +1,70 @@ +from webkit.steps import * +from buildbot.process import factory + +s = factory.s + +class BuildFactory(factory.BuildFactory): + useProgress = False + def __init__(self): + factory.BuildFactory.__init__(self, [s(CheckOutSource)]) + +class StandardBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(SetConfiguration, configuration="release")) + self.steps.append(s(self.getCompileStep(), configuration="release")) + self.steps.append(s(self.getJavaScriptCoreTestStep())) + self.steps.append(s(LayoutTest)) + self.steps.append(s(UploadLayoutResults)) +# self.steps.append(s(UploadDiskImage)) + + def getCompileStep(self): + return CompileWebKit + + def getJavaScriptCoreTestStep(self): + return JavaScriptCoreTest + + +class NoSVGBuildFactory(StandardBuildFactory): + def getCompileStep(self): + return CompileWebKitNoSVG + + +class PixelTestBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(SetConfiguration, configuration="release")) + self.steps.append(s(CompileWebKit, configuration="release")) + self.steps.append(s(PixelLayoutTest)) + self.steps.append(s(UploadLayoutResults)) + +class LeakBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(SetConfiguration, configuration="debug")) + self.steps.append(s(CompileWebKit, configuration="debug")) + self.steps.append(s(JavaScriptCoreTest)) + self.steps.append(s(LeakTest)) + self.steps.append(s(UploadLayoutResults)) +# self.steps.append(s(UploadDiskImage)) + +class PageLoadTestBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(CompileWebKit, configuration="release")) + self.steps.append(s(PageLoadTest)) + +Win32BuildFactory = StandardBuildFactory + +class GtkBuildFactory(StandardBuildFactory): + def getCompileStep(self): + return CompileWebKitGtk + + def getJavaScriptCoreTestStep(self): + return JavaScriptCoreTestGtk + +class CoverageDataBuildFactory(BuildFactory): + def __init__(self): + BuildFactory.__init__(self) + self.steps.append(s(GenerateCoverageData)) + self.steps.append(s(UploadCoverageData)) diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/schedulers.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/schedulers.py new file mode 100644 index 0000000..11f9d41 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/schedulers.py @@ -0,0 +1,15 @@ +from buildbot.scheduler import Scheduler, Periodic + +def getSchedulers(builders): + builder_names = map(lambda builder: builder['name'], builders) + post_commit_builders = [name for name in builder_names if name.startswith('post-commit-')] + ['page-layout-test-mac-os-x'] + post_commit_builders = [name for name in post_commit_builders if name in builder_names] + post_commit_builders.sort() + + periodic_builders = [b['name'] for b in builders if b['name'].startswith('periodic-')] + periodic_builders.sort() + + post_commit = Scheduler(name="post-commit", branch=None, treeStableTimer=90, builderNames=post_commit_builders) + periodic = Periodic("periodic", periodic_builders, 6 * 60 * 60) + + return [post_commit, periodic] diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/status.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/status.py new file mode 100644 index 0000000..ad5519e --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/status.py @@ -0,0 +1,19 @@ +from buildbot.status import html, mail, words + +web = html.WebStatus(http_port=8010) + +allBuildsEmail = mail.MailNotifier(fromaddr="buildbot@webkit.org", + extraRecipients=["mark+webkit-builds@bdash.net.nz"], + sendToInterestedUsers=False) +breakageEmail = mail.MailNotifier(fromaddr="buildbot@webkit.org", + lookup=mail.Domain("webkit.org"), + mode="failing") + +IRC = words.IRC(host="irc.freenode.net", + nick="webkit-build", + channels=["#webkit-build"], +# announceAllBuilds=True + ) + +def getStatusListeners(): + return [web, allBuildsEmail, breakageEmail, IRC] diff --git a/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/steps.py b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/steps.py new file mode 100644 index 0000000..c1933a3 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/steps.py @@ -0,0 +1,220 @@ +from webkit.basesteps import ShellCommand, SVN, Test, Compile, UploadCommand +from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS + +class CheckOutSource(SVN): + svnurl = "http://svn.webkit.org/repository/webkit/trunk" + mode = "update" + def __init__(self, *args, **kwargs): + SVN.__init__(self, svnurl=self.svnurl, mode=self.mode, *args, **kwargs) + +class SetConfiguration(ShellCommand): + command = ["./WebKitTools/Scripts/set-webkit-configuration"] + + def __init__(self, *args, **kwargs): + configuration = kwargs.pop('configuration') + self.command = self.command + ['--' + configuration] + self.name = "set-configuration-%s" % (configuration, ) + self.description = ["set configuration %s" % (configuration, )] + self.descriptionDone = ["set configuration %s" % (configuration, )] + ShellCommand.__init__(self, *args, **kwargs) + + +class LayoutTest(Test): + name = "layout-test" + description = ["layout-tests running"] + descriptionDone = ["layout-tests"] + command = ["./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results", "--results-directory", "layout-test-results"] + + def commandComplete(self, cmd): + Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + incorrectLayoutLines = [line for line in logText.splitlines() if line.find('had incorrect layout') >= 0 or (line.find('test case') >= 0 and line.find(' crashed') >= 0)] + if incorrectLayoutLines: + self.incorrectLayoutLines = incorrectLayoutLines + else: + self.incorrectLayoutLines = None + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.incorrectLayoutLines: + return self.incorrectLayoutLines + + return [self.name] + + +class JavaScriptCoreTest(Test): + name = "jscore-test" + description = ["jscore-tests running"] + descriptionDone = ["jscore-tests"] + command = ["./WebKitTools/Scripts/run-javascriptcore-tests"] + logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'} + + def commandComplete(self, cmd): + Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0] + if statusLines and statusLines[0].split()[0] != '0': + self.regressionLine = statusLines[0] + else: + self.regressionLine = None + + def evaluateCommand(self, cmd): + if cmd.rc != 0: + return FAILURE + + if self.regressionLine: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.regressionLine: + return [self.name, self.regressionLine] + + return [self.name] + +class PixelLayoutTest(LayoutTest): + name = "pixel-layout-test" + description = ["pixel-layout-tests running"] + descriptionDone = ["pixel-layout-tests"] + command = LayoutTest.command + ["--pixel"] + + +class LeakTest(Test): + name = "leak-test" + description = ["leak-tests running"] + descriptionDone = ["leak-tests"] + command = ["./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--leaks", "--results-directory", "layout-test-results"] + + def commandComplete(self, cmd): + Test.commandComplete(self, cmd) + + logText = cmd.logs['stdio'].getText() + lines = logText.splitlines() + self.totalLeakLines = [line for line in lines if line.find('total leaks found!') >= 0] + self.totalLeakLines += [line for line in lines if line.find('LEAK: ') >= 0] + self.totalLeakLines = [' '.join(x.split()[1:]) for x in self.totalLeakLines] + self.totalLeakLines += [line for line in lines if line.find('test case') >= 0 and line.find('crashed') >= 0] + + + def evaluateCommand(self, cmd): + if cmd.rc != 0: + return FAILURE + + if self.totalLeakLines: + return FAILURE + + return SUCCESS + + def getText(self, cmd, results): + return self.getText2(cmd, results) + + def getText2(self, cmd, results): + if results != SUCCESS and self.totalLeakLines: + return self.totalLeakLines + return [self.name] + + +class UploadLayoutResults(UploadCommand, ShellCommand): + name = "upload-results" + description = ["uploading results"] + descriptionDone = ["uploaded-results"] + command = "echo Disabled for now" + + def __init__(self, *args, **kwargs): + ShellCommand.__init__(self, *args, **kwargs) + + def setBuild(self, build): + ShellCommand.setBuild(self, build) + self.initializeForUpload() + + self.command = '''\ + if [[ -d layout-test-results ]]; then \ + find layout-test-results -type d -print0 | xargs -0 chmod ug+rx; \ + find layout-test-results -type f -print0 | xargs -0 chmod ug+r; \ + rsync -rlvzP --rsync-path="/home/buildresults/bin/rsync" layout-test-results/ %s && rm -rf layout-test-results; \ + fi; \ + CRASH_LOG=~/Library/Logs/CrashReporter/DumpRenderTree*.crash*; \ + if [[ -f $(ls -1 $CRASH_LOG | head -n 1 ) ]]; then \ + chmod ug+r $CRASH_LOG; \ + rsync -rlvzP --rsync-path="/home/buildresults/bin/rsync" $CRASH_LOG %s && rm -rf $CRASH_LOG; \ + fi;''' % (self.getRemotePath(), self.getRemotePath()) + + self.addFactoryArguments(command=self.command) + + +class CompileWebKit(Compile): + command = ["./WebKitTools/Scripts/build-webkit", "--no-color"] + env = {'WEBKITSUPPORTLIBRARIESZIPDIR': 'C:\\cygwin\\home\\buildbot'} + def __init__(self, *args, **kwargs): + configuration = kwargs.pop('configuration') + + self.name = "compile-" + configuration + self.description = ["compiling " + configuration] + self.descriptionDone = ["compiled " + configuration] + + Compile.__init__(self, *args, **kwargs) + + +class CompileWebKitNoSVG(CompileWebKit): + command = 'rm -rf WebKitBuild && ./WebKitTools/Scripts/build-webkit --no-svg --no-color' + +class CompileWebKitGtk(CompileWebKit): + command = CompileWebKit.command + ['--gtk'] + +class JavaScriptCoreTestGtk(JavaScriptCoreTest): + command = JavaScriptCoreTest.command + ['--gtk'] + +class InstallWin32Dependencies(ShellCommand): + description = ["installing Windows dependencies"] + descriptionDone = ["installed Windows dependencies"] + command = ["WebKitTools/Scripts/update-webkit-auxiliary-libs"] + + +# class UploadDiskImage(UploadCommand, ShellCommand): +# description = ["uploading disk image"] +# descriptionDone = ["uploaded disk image"] +# name = "upload-disk-image" + +# def __init__(self, *args, **kwargs): +# UploadCommand.__init__(self, *args, **kwargs) +# self.command = 'umask 002 && ./WebKitTools/BuildSlaveSupport/build-launcher-app && ./WebKitTools/BuildSlaveSupport/build-launcher-dmg --upload-to-host %s' % (self.getRemotePath(), ) +# ShellCommand.__init__(self, *args, **kwargs) + +class GenerateCoverageData(Compile): + command = ["./WebKitTools/Scripts/generate-coverage-data"] + description = ["generating coverage data"] + descriptionDone = ["generated coverage data"] + + +class UploadCoverageData(UploadCommand, ShellCommand): + name = "upload-coverage-data" + description = ["uploading coverage data"] + descriptionDone = ["uploaded-coverage-data"] + command = "echo Disabled for now" + + def __init__(self, *args, **kwargs): + ShellCommand.__init__(self, *args, **kwargs) + + def setBuild(self, build): + ShellCommand.setBuild(self, build) + self.initializeForUpload() + + self.command = '''\ + if [[ -d WebKitBuild/Coverage/html ]]; then \ + find WebKitBuild/Coverage/html -type d -print0 | xargs -0 chmod ug+rx; \ + find WebKitBuild/Coverage/html -type f -print0 | xargs -0 chmod ug+r; \ + rsync -rlvzP --rsync-path="/home/buildresults/bin/rsync" WebKitBuild/Coverage/html/ %s && rm -rf WebKitBuild/Coverage/html; \ + fi;''' % (self.getRemotePath(), ) + + self.addFactoryArguments(command=self.command) + + def getURLPath(self): + return "/results/code-coverage/" diff --git a/WebKitTools/BuildSlaveSupport/run-performance-tests b/WebKitTools/BuildSlaveSupport/run-performance-tests new file mode 100755 index 0000000..5d6ea44 --- /dev/null +++ b/WebKitTools/BuildSlaveSupport/run-performance-tests @@ -0,0 +1,80 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2006 Apple Computer, Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Script used by WebKit build slave to kick off performance tests. + +use strict; + +use FindBin; +use lib "$FindBin::Bin/../Scripts"; +use webkitdirs; + +use Getopt::Long; +use IO::Socket; +my $shouldPrebuild = 1; + +my $perfMaster = "webkit-build-1.local"; +my $perfSlave = "webkit-build-2.local"; +my $slaveUser = "buildbot"; +my $reportPort = 54481; # Something that looks sorta like SAFARI +my $slaveDirectory = "/ToTest"; + +my $buildDirectory = determineConfigurationProductDir(); + +my $userAndHost = $slaveUser . "@" . $perfSlave; +my $resultsUploadDestination; + +GetOptions('upload-results=s' => \$resultsUploadDestination); + +print "Copying built frameworks to PLT slave...\n"; +my @frameworks = ("$buildDirectory/JavaScriptCore.framework", "$buildDirectory/WebCore.framework", "$buildDirectory/WebKit.framework"); +die "Failed to copy to slave\n" unless (system("rsync", "-avz", @frameworks, "$userAndHost:$slaveDirectory") == 0); + +print "Opening reponse port for PLT slave...\n"; +my $sock = new IO::Socket::INET(LocalHost => $perfMaster, + LocalPort => $reportPort, + Proto => 'tcp', + Listen => 1, + Reuse => 1); +die "Could not create socket for port $reportPort: $!\n" unless $sock; + +print "Starting performance tests on PLT slave...\n"; +die "Failed to start slave!\n" unless (system("ssh", $userAndHost, "autovicki", $slaveDirectory, "--safari", "$slaveDirectory/Safari.app", "--count", 5, "--clean-exit", "--webkit-revision", currentSVNRevision(), "--show-results", "send-completed-results.command") == 0); + +print "Waiting for PLT slave to respond...\n"; +my $new_sock = $sock->accept(); +while(<$new_sock>) { + print $_; +} +close($sock); + +if ($resultsUploadDestination) { + print "Uploading results to $resultsUploadDestination\n"; + die "Failed to upload xml results file." unless (system("scp", "$userAndHost:/Results/PerformanceReportSummary.xml", $resultsUploadDestination) == 0); + die "Failed to upload svg results file." unless (system("scp", "$userAndHost:/Results/PerformanceGraph.svg", $resultsUploadDestination) == 0); +} |