diff options
Diffstat (limited to 'WebKitTools/Scripts')
50 files changed, 1641 insertions, 737 deletions
diff --git a/WebKitTools/Scripts/build-webkit b/WebKitTools/Scripts/build-webkit index d9534f8..800e610 100755 --- a/WebKitTools/Scripts/build-webkit +++ b/WebKitTools/Scripts/build-webkit @@ -100,6 +100,7 @@ my ( $videoSupport, $wcssSupport, $webAudioSupport, + $webInspectorSupport, $webSocketsSupport, $webTimingSupport, $wmlSupport, @@ -176,6 +177,9 @@ my @features = ( { option => "input-speech", desc => "Speech Input API support", define => "ENABLE_INPUT_SPEECH", default => 0, value => \$inputSpeechSupport }, + { option => "inspector", desc => "Toggle Web Inspector support", + define => "ENABLE_INSPECTOR", default => 1, value => \$webInspectorSupport }, + { option => "javascript-debugger", desc => "Toggle JavaScript Debugger/Profiler support", define => "ENABLE_JAVASCRIPT_DEBUGGER", default => 1, value => \$javaScriptDebuggerSupport }, diff --git a/WebKitTools/Scripts/generate-forwarding-headers.pl b/WebKitTools/Scripts/generate-forwarding-headers.pl index ed58702..d5abb5b 100755 --- a/WebKitTools/Scripts/generate-forwarding-headers.pl +++ b/WebKitTools/Scripts/generate-forwarding-headers.pl @@ -30,6 +30,7 @@ use strict; use Cwd qw(abs_path realpath); use File::Find; use File::Basename; +use File::Path qw(mkpath); use File::Spec::Functions; my $srcRoot = realpath(File::Spec->catfile(dirname(abs_path($0)), "../..")); @@ -81,8 +82,10 @@ sub collectFameworkHeaderPaths { } sub createForwardingHeadersForFramework { + my $targetDirectory = File::Spec->catfile($outputDirectory, $framework); + mkpath($targetDirectory); foreach my $header (@frameworkHeaders) { - my $forwardingHeaderPath = File::Spec->catfile($outputDirectory, $framework, basename($header)); + my $forwardingHeaderPath = File::Spec->catfile($targetDirectory, basename($header)); my $expectedIncludeStatement = "#include \"$header\""; my $foundIncludeStatement = 0; $foundIncludeStatement = <EXISTING_HEADER> if open(EXISTING_HEADER, "<$forwardingHeaderPath"); diff --git a/WebKitTools/Scripts/old-run-webkit-tests b/WebKitTools/Scripts/old-run-webkit-tests index eeaaab3..5780c5a 100755 --- a/WebKitTools/Scripts/old-run-webkit-tests +++ b/WebKitTools/Scripts/old-run-webkit-tests @@ -418,14 +418,14 @@ if (!defined($root)) { my $dumpToolName = $useWebKitTestRunner ? "WebKitTestRunner" : "DumpRenderTree"; if (isAppleWinWebKit()) { - $dumpToolName .= "_debug" if configurationForVisualStudio() !~ /^Release|Debug_Internal$/; + $dumpToolName .= "_debug" if configurationForVisualStudio() eq "Debug_All"; $dumpToolName .= $Config{_exe}; } my $dumpTool = File::Spec->catfile($productDir, $dumpToolName); die "can't find executable $dumpToolName (looked in $productDir)\n" unless -x $dumpTool; my $imageDiffTool = "$productDir/ImageDiff"; -$imageDiffTool .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/; +$imageDiffTool .= "_debug" if isCygwin() && configurationForVisualStudio() eq "Debug_All"; die "can't find executable $imageDiffTool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool; checkFrameworks() unless isCygwin(); diff --git a/WebKitTools/Scripts/run-api-tests b/WebKitTools/Scripts/run-api-tests index 3d08013..9db08fc 100755 --- a/WebKitTools/Scripts/run-api-tests +++ b/WebKitTools/Scripts/run-api-tests @@ -139,7 +139,7 @@ sub runTest($$) } } elsif (isAppleWinWebKit()) { my $apiTesterNameSuffix; - if (configurationForVisualStudio() =~ /^Release|Debug_Internal$/) { + if (configurationForVisualStudio() ne "Debug_All") { $apiTesterNameSuffix = ""; } else { $apiTesterNameSuffix = "_debug"; @@ -187,7 +187,7 @@ sub populateTests() } } elsif (isAppleWinWebKit()) { my $apiTesterNameSuffix; - if (configurationForVisualStudio() =~ /^Release|Debug_Internal$/) { + if (configurationForVisualStudio() ne "Debug_All") { $apiTesterNameSuffix = ""; } else { $apiTesterNameSuffix = "_debug"; diff --git a/WebKitTools/Scripts/run-javascriptcore-tests b/WebKitTools/Scripts/run-javascriptcore-tests index fb4c388..cbf8cbc 100755 --- a/WebKitTools/Scripts/run-javascriptcore-tests +++ b/WebKitTools/Scripts/run-javascriptcore-tests @@ -108,7 +108,7 @@ sub testapiPath($) { my ($productDir) = @_; my $jscName = "testapi"; - $jscName .= "_debug" if (isCygwin() && ($configuration eq "Debug")); + $jscName .= "_debug" if configurationForVisualStudio() eq "Debug_All"; return "$productDir/$jscName"; } @@ -116,7 +116,10 @@ sub testapiPath($) if (isAppleMacWebKit() || isAppleWinWebKit()) { chdirWebKit(); chdir($productDir) or die; - my $testapiResult = system testapiPath($productDir); + my $path = testapiPath($productDir); + # Use an "indirect object" so that system() won't get confused if the path + # contains spaces (see perldoc -f exec). + my $testapiResult = system { $path } $path; exit exitStatus($testapiResult) if $testapiResult; } diff --git a/WebKitTools/Scripts/sunspider-compare-results b/WebKitTools/Scripts/sunspider-compare-results index 193ee8f..97e0b67 100755 --- a/WebKitTools/Scripts/sunspider-compare-results +++ b/WebKitTools/Scripts/sunspider-compare-results @@ -93,7 +93,7 @@ sub pathToBuiltJSC($) { my ($productDir) = @_; my $jscName = "jsc"; - $jscName .= "_debug" if (isCygwin() && ($configuration eq "Debug")); + $jscName .= "_debug" if configurationForVisualStudio() eq "Debug_All"; return "$productDir/$jscName"; } diff --git a/WebKitTools/Scripts/update-webkit b/WebKitTools/Scripts/update-webkit index 3fc2efd..fd40dcd 100755 --- a/WebKitTools/Scripts/update-webkit +++ b/WebKitTools/Scripts/update-webkit @@ -126,5 +126,8 @@ sub runSvnUpdate() sub runGitUpdate() { + # Doing a git fetch first allows setups with svn-remote.svn.fetch = trunk:refs/remotes/origin/master + # to perform the rebase much much faster. + system("git", "fetch") == 0 or die; system("git", "svn", "rebase") == 0 or die; } diff --git a/WebKitTools/Scripts/update-webkit-support-libs b/WebKitTools/Scripts/update-webkit-support-libs index fa2afd0..f0c897e 100755 --- a/WebKitTools/Scripts/update-webkit-support-libs +++ b/WebKitTools/Scripts/update-webkit-support-libs @@ -1,6 +1,7 @@ #!/usr/bin/perl -w # Copyright (C) 2005, 2006, 2007 Apple Computer, Inc. All rights reserved. +# Copyright (C) Research In Motion Limited 2010. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -38,6 +39,8 @@ use FindBin; use lib $FindBin::Bin; use webkitdirs; +use constant NOTAVERSION => "-1"; + my $sourceDir = sourceDir(); my $file = "WebKitSupportLibrary"; my $zipFile = "$file.zip"; @@ -47,23 +50,25 @@ my $webkitLibrariesDir = toUnixPath($ENV{'WEBKITLIBRARIESDIR'}) || "$sourceDir/W my $versionFile = $file . "Version"; my $pathToVersionFile = File::Spec->catfile($webkitLibrariesDir, $versionFile); my $tmpDir = File::Spec->rel2abs(File::Temp::tempdir("webkitlibsXXXXXXX", TMPDIR => 1, CLEANUP => 1)); +my $versionFileURL = "http://developer.apple.com/opensource/internet/$versionFile"; -chomp(my $expectedVersion = `curl -s http://developer.apple.com/opensource/internet/$versionFile`); +my $extractedVersion = extractedVersion(); # Check whether the extracted library is up-to-date. If it is, we don't have anything to do. -if (open VERSION, "<", $pathToVersionFile) { - chomp(my $extractedVersion = <VERSION>); - close VERSION; - if ($extractedVersion eq $expectedVersion) { - print "$file is up-to-date.\n"; - exit; - } +my $expectedVersion = downloadExpectedVersionNumber(); +if ($extractedVersion ne NOTAVERSION && $extractedVersion eq $expectedVersion) { + print "$file is up-to-date.\n"; + exit; } # Check whether the downloaded library is up-to-date. If it isn't, the user needs to download it. --f $pathToZip or dieAndInstructToDownload("$zipFile could not be found in $zipDirectory."); -chomp(my $zipFileVersion = `unzip -p "$pathToZip" $file/win/$versionFile`); -dieAndInstructToDownload("$zipFile is out-of-date.") if $zipFileVersion ne $expectedVersion; +my $zipFileVersion = zipFileVersion(); +dieAndInstructToDownload("$zipFile could not be found in $zipDirectory.") if $zipFileVersion eq NOTAVERSION; +dieAndInstructToDownload("$zipFile is out-of-date.") if $expectedVersion ne NOTAVERSION && $zipFileVersion ne $expectedVersion; +if ($zipFileVersion eq $extractedVersion) { + print "Falling back to existing version of $file.\n"; + exit; +} my $result = system "unzip", "-q", "-d", $tmpDir, $pathToZip; die "Couldn't unzip $zipFile." if $result; @@ -96,6 +101,29 @@ sub toUnixPath return $path; } +sub extractedVersion +{ + if (open VERSION, "<", $pathToVersionFile) { + chomp(my $extractedVersion = <VERSION>); + close VERSION; + return $extractedVersion; + } + return NOTAVERSION; +} + +sub downloadExpectedVersionNumber +{ + chomp(my $expectedVersion = `curl -s $versionFileURL`); + return WEXITSTATUS($?) ? NOTAVERSION : $expectedVersion; +} + +sub zipFileVersion +{ + return NOTAVERSION unless -f $pathToZip; + chomp(my $zipFileVersion = `unzip -p "$pathToZip" $file/win/$versionFile`); + return $zipFileVersion; +} + sub dieAndInstructToDownload { my $message = shift; diff --git a/WebKitTools/Scripts/webkitdirs.pm b/WebKitTools/Scripts/webkitdirs.pm index 2c1d8da..73288e0 100644 --- a/WebKitTools/Scripts/webkitdirs.pm +++ b/WebKitTools/Scripts/webkitdirs.pm @@ -253,7 +253,7 @@ sub jscPath($) { my ($productDir) = @_; my $jscName = "jsc"; - $jscName .= "_debug" if (isCygwin() && ($configuration eq "Debug")); + $jscName .= "_debug" if configurationForVisualStudio() eq "Debug_All"; $jscName .= ".exe" if (isWindows() || isCygwin()); return "$productDir/$jscName" if -e "$productDir/$jscName"; return "$productDir/JavaScriptCore.framework/Resources/$jscName"; @@ -282,12 +282,8 @@ sub determineConfigurationForVisualStudio { return if defined $configurationForVisualStudio; determineConfiguration(); + # FIXME: We should detect when Debug_All or Release_LTCG has been chosen. $configurationForVisualStudio = $configuration; - return unless $configuration eq "Debug"; - setupCygwinEnv(); - my $dir = $ENV{WEBKITLIBRARIESDIR}; - chomp($dir = `cygpath -ua '$dir'`) if isCygwin(); - $configurationForVisualStudio = "Debug_Internal" if -f File::Spec->catfile($dir, "bin", "CoreFoundation_debug.dll"); } sub determineConfigurationProductDir @@ -419,19 +415,19 @@ sub determinePassedConfiguration if ($opt =~ /^--debug$/i || $opt =~ /^--devel/i) { splice(@ARGV, $i, 1); $passedConfiguration = "Debug"; - $passedConfiguration .= "_Cairo" if ($isWinCairo && isCygwin()); + $passedConfiguration .= "_Cairo_CFLite" if ($isWinCairo && isCygwin()); return; } if ($opt =~ /^--release$/i || $opt =~ /^--deploy/i) { splice(@ARGV, $i, 1); $passedConfiguration = "Release"; - $passedConfiguration .= "_Cairo" if ($isWinCairo && isCygwin()); + $passedConfiguration .= "_Cairo_CFLite" if ($isWinCairo && isCygwin()); return; } if ($opt =~ /^--profil(e|ing)$/i) { splice(@ARGV, $i, 1); $passedConfiguration = "Profiling"; - $passedConfiguration .= "_Cairo" if ($isWinCairo && isCygwin()); + $passedConfiguration .= "_Cairo_CFLite" if ($isWinCairo && isCygwin()); return; } } @@ -548,7 +544,7 @@ sub safariPath my $path = "$configurationProductDir/Safari.exe"; my $debugPath = "$configurationProductDir/Safari_debug.exe"; - if (configurationForVisualStudio() =~ /Debug/ && -x $debugPath) { + if (configurationForVisualStudio() eq "Debug_All" && -x $debugPath) { $safariBundle = $debugPath; } elsif (-x $path) { $safariBundle = $path; @@ -684,15 +680,26 @@ sub determineQtFeatureDefaults() sub checkForArgumentAndRemoveFromARGV { my $argToCheck = shift; - foreach my $opt (@ARGV) { + return checkForArgumentAndRemoveFromArrayRef($argToCheck, \@ARGV); +} + +sub checkForArgumentAndRemoveFromArrayRef +{ + my ($argToCheck, $arrayRef) = @_; + my @indicesToRemove; + foreach my $index (0 .. $#$arrayRef) { + my $opt = $$arrayRef[$index]; if ($opt =~ /^$argToCheck$/i ) { - @ARGV = grep(!/^$argToCheck$/i, @ARGV); - return 1; + push(@indicesToRemove, $index); } } - return 0; + foreach my $index (@indicesToRemove) { + splice(@$arrayRef, $index, 1); + } + return $#indicesToRemove > -1; } + sub determineIsQt() { return if defined($isQt); @@ -1188,7 +1195,7 @@ sub buildXCodeProject($$@) sub usingVisualStudioExpress() { - determineConfigurationForVisualStudio(); + setupCygwinEnv(); return $willUseVCExpressWhenBuilding; } @@ -1681,7 +1688,9 @@ sub buildChromium($@) my ($clean, @options) = @_; # We might need to update DEPS or re-run GYP if things have changed. - system("perl", "WebKitTools/Scripts/update-webkit-chromium") == 0 or die $!; + if (checkForArgumentAndRemoveFromArrayRef("--update-chromium", \@options)) { + system("perl", "WebKitTools/Scripts/update-webkit-chromium") == 0 or die $!; + } my $result = 1; if (isDarwin()) { diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/scm.py b/WebKitTools/Scripts/webkitpy/common/checkout/scm.py index 11e82ac..d39b8b4 100644 --- a/WebKitTools/Scripts/webkitpy/common/checkout/scm.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/scm.py @@ -597,7 +597,8 @@ class Git(SCM): @classmethod def read_git_config(cls, key): # FIXME: This should probably use cwd=self.checkout_root. - return run_command(["git", "config", key], + # Pass --get-all for cases where the config has multiple values + return run_command(["git", "config", "--get-all", key], error_handler=Executive.ignore_error).rstrip('\n') @staticmethod @@ -854,19 +855,17 @@ class Git(SCM): def remote_branch_ref(self): # Use references so that we can avoid collisions, e.g. we don't want to operate on refs/heads/trunk if it exists. - - # FIXME: This should so something like: Git.read_git_config('svn-remote.svn.fetch').split(':')[1] - # but that doesn't work if the git repo is tracking multiple svn branches. - remote_branch_refs = [ - 'refs/remotes/trunk', # A git-svn checkout as per http://trac.webkit.org/wiki/UsingGitWithWebKit. - 'refs/remotes/origin/master', # A git clone of git://git.webkit.org/WebKit.git that is not tracking svn. - ] - - for ref in remote_branch_refs: - if self._branch_ref_exists(ref): - return ref - - raise ScriptError(message="Can't find a branch to diff against. %s branches do not exist." % " and ".join(remote_branch_refs)) + remote_branch_refs = Git.read_git_config('svn-remote.svn.fetch') + if not remote_branch_refs: + remote_master_ref = 'refs/remotes/origin/master' + if not self._branch_ref_exists(remote_master_ref): + raise ScriptError(message="Can't find a branch to diff against. svn-remote.svn.fetch is not in the git config and %s does not exist" % remote_master_ref) + return remote_master_ref + + # FIXME: What's the right behavior when there are multiple svn-remotes listed? + # For now, just use the first one. + first_remote_branch_ref = remote_branch_refs.split('\n')[0] + return first_remote_branch_ref.split(':')[1] def commit_locally_with_message(self, message): self.run(['git', 'commit', '--all', '-F', '-'], input=message) diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/scm_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/scm_unittest.py index 8af9ad5..46a2acf 100644 --- a/WebKitTools/Scripts/webkitpy/common/checkout/scm_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/scm_unittest.py @@ -803,6 +803,10 @@ class GitTest(SCMTest): os.chdir(self.untracking_checkout_path) self.assertRaises(ScriptError, self.untracking_scm.remote_branch_ref) + def test_multiple_remotes(self): + run_command(['git', 'config', '--add', 'svn-remote.svn.fetch', 'trunk:remote1']) + run_command(['git', 'config', '--add', 'svn-remote.svn.fetch', 'trunk:remote2']) + self.assertEqual(self.tracking_scm.remote_branch_ref(), 'remote1') class GitSVNTest(SCMTest): diff --git a/WebKitTools/Scripts/webkitpy/common/config/committers.py b/WebKitTools/Scripts/webkitpy/common/config/committers.py index 0967340..bb2d551 100644 --- a/WebKitTools/Scripts/webkitpy/common/config/committers.py +++ b/WebKitTools/Scripts/webkitpy/common/config/committers.py @@ -74,7 +74,7 @@ committers_unable_to_review = [ Committer("Andrei Popescu", "andreip@google.com", "andreip"), Committer("Andrew Wellington", ["andrew@webkit.org", "proton@wiretapped.net"], "proton"), Committer("Andrey Kosyakov", "caseq@chromium.org", "caseq"), - Committer("Andras Becsi", "abecsi@webkit.org", "bbandix"), + Committer("Andras Becsi", ["abecsi@webkit.org", "abecsi@inf.u-szeged.hu"], "bbandix"), Committer("Andy Estes", "aestes@apple.com", "estes"), Committer("Anthony Ricaud", "rik@webkit.org", "rik"), Committer("Anton Muhin", "antonm@chromium.org", "antonm"), @@ -130,6 +130,7 @@ committers_unable_to_review = [ Committer("Jochen Eisinger", "jochen@chromium.org", "jochen__"), Committer("John Abd-El-Malek", "jam@chromium.org", "jam"), Committer("John Gregg", ["johnnyg@google.com", "johnnyg@chromium.org"], "johnnyg"), + Committer("Johnny Ding", ["jnd@chromium.org", "johnnyding.webkit@gmail.com"], "johnnyding"), Committer("Joost de Valk", ["joost@webkit.org", "webkit-dev@joostdevalk.nl"], "Altha"), Committer("Julie Parent", ["jparent@google.com", "jparent@chromium.org"], "jparent"), Committer("Julien Chaffraix", ["jchaffraix@webkit.org", "julien.chaffraix@gmail.com"]), diff --git a/WebKitTools/Scripts/webkitpy/common/config/ports.py b/WebKitTools/Scripts/webkitpy/common/config/ports.py index d268865..5f15e88 100644 --- a/WebKitTools/Scripts/webkitpy/common/config/ports.py +++ b/WebKitTools/Scripts/webkitpy/common/config/ports.py @@ -221,6 +221,7 @@ class ChromiumPort(WebKitPort): def build_webkit_command(cls, build_style=None): command = WebKitPort.build_webkit_command(build_style=build_style) command.append("--chromium") + command.append("--update-chromium") return command @classmethod diff --git a/WebKitTools/Scripts/webkitpy/common/config/ports_unittest.py b/WebKitTools/Scripts/webkitpy/common/config/ports_unittest.py index 3bdf0e6..125981a 100644 --- a/WebKitTools/Scripts/webkitpy/common/config/ports_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/config/ports_unittest.py @@ -65,8 +65,8 @@ class WebKitPortTest(unittest.TestCase): self.assertEquals(ChromiumPort.name(), "Chromium") self.assertEquals(ChromiumPort.flag(), "--port=chromium") self.assertEquals(ChromiumPort.run_webkit_tests_command(), [WebKitPort.script_path("new-run-webkit-tests"), "--chromium", "--use-drt", "--no-pixel-tests"]) - self.assertEquals(ChromiumPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--chromium"]) - self.assertEquals(ChromiumPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--chromium"]) + self.assertEquals(ChromiumPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--chromium", "--update-chromium"]) + self.assertEquals(ChromiumPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--chromium", "--update-chromium"]) self.assertEquals(ChromiumPort.update_webkit_command(), [WebKitPort.script_path("update-webkit"), "--chromium"]) def test_chromium_xvfb_port(self): diff --git a/WebKitTools/Scripts/webkitpy/common/system/executive_mock.py b/WebKitTools/Scripts/webkitpy/common/system/executive_mock.py index 7347ff9..c1cf999 100644 --- a/WebKitTools/Scripts/webkitpy/common/system/executive_mock.py +++ b/WebKitTools/Scripts/webkitpy/common/system/executive_mock.py @@ -32,10 +32,12 @@ class MockExecutive2(object): - def __init__(self, output='', exit_code=0, exception=None): + def __init__(self, output='', exit_code=0, exception=None, + run_command_fn=None): self._output = output self._exit_code = exit_code self._exception = exception + self._run_command_fn = run_command_fn def cpu_count(self): return 2 @@ -52,4 +54,6 @@ class MockExecutive2(object): raise self._exception if return_exit_code: return self._exit_code + if self._run_command_fn: + return self._run_command_fn(arg_list) return self._output diff --git a/WebKitTools/Scripts/webkitpy/common/system/filesystem_mock.py b/WebKitTools/Scripts/webkitpy/common/system/filesystem_mock.py index d2cde4f..2dbc1e8 100644 --- a/WebKitTools/Scripts/webkitpy/common/system/filesystem_mock.py +++ b/WebKitTools/Scripts/webkitpy/common/system/filesystem_mock.py @@ -39,11 +39,7 @@ class MockFileSystem(object): Args: files: a dict of filenames -> file contents. A file contents value of None is used to indicate that the file should - not exist (even if standalone is False). - standalone: If True, only the files listed in _files_ exist. - If False, the object will pass through read calls to the - underlying filesystem. Writes are never passed through. - + not exist. """ self.files = files diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py b/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py deleted file mode 100644 index 633dfe8..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# -# FIXME: this is a poor attempt at a unit tests driver. We should replace -# this with something that actually uses a unit testing framework or -# at least produces output that could be useful. - -"""Simple test client for the port/Driver interface.""" - -import os -import optparse -import port - - -def run_tests(port, options, tests): - # |image_path| is a path to the image capture from the driver. - image_path = 'image_result.png' - driver = port.create_driver(image_path, None) - driver.start() - for t in tests: - uri = port.filename_to_uri(os.path.join(port.layout_tests_dir(), t)) - print "uri: " + uri - crash, timeout, checksum, output, err = \ - driver.run_test(uri, int(options.timeout), None) - print "crash: " + str(crash) - print "timeout: " + str(timeout) - print "checksum: " + str(checksum) - print 'stdout: """' - print ''.join(output) - print '"""' - print 'stderr: """' - print ''.join(err) - print '"""' - print - driver.stop() - - -if __name__ == '__main__': - # FIXME: configuration_options belong in a shared location. - configuration_options = [ - optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), - optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), - ] - misc_options = [ - optparse.make_option('-p', '--platform', action='store', default='mac', help='Platform to test (e.g., "mac", "chromium-mac", etc.'), - optparse.make_option('--timeout', action='store', default='2000', help='test timeout in milliseconds (2000 by default)'), - optparse.make_option('--wrapper', action='store'), - optparse.make_option('--no-pixel-tests', action='store_true', default=False, help='disable pixel-to-pixel PNG comparisons'), - ] - option_list = configuration_options + misc_options - optparser = optparse.OptionParser(option_list=option_list) - options, args = optparser.parse_args() - p = port.get(options.platform, options) - run_tests(p, options, args) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py index 88f493d..fdb8da6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py @@ -48,7 +48,11 @@ import sys import thread import threading import time -import traceback + + +from webkitpy.layout_tests.test_types import image_diff +from webkitpy.layout_tests.test_types import test_type_base +from webkitpy.layout_tests.test_types import text_diff import test_failures import test_output @@ -58,23 +62,6 @@ _log = logging.getLogger("webkitpy.layout_tests.layout_package." "dump_render_tree_thread") -def find_thread_stack(id): - """Returns a stack object that can be used to dump a stack trace for - the given thread id (or None if the id is not found).""" - for thread_id, stack in sys._current_frames().items(): - if thread_id == id: - return stack - return None - - -def log_stack(stack): - """Log a stack trace to log.error().""" - for filename, lineno, name, line in traceback.extract_stack(stack): - _log.error('File: "%s", line %d, in %s' % (filename, lineno, name)) - if line: - _log.error(' %s' % line.strip()) - - def _expected_test_output(port, filename): """Returns an expected TestOutput object.""" return test_output.TestOutput(port.expected_text(filename), @@ -82,7 +69,7 @@ def _expected_test_output(port, filename): port.expected_checksum(filename)) def _process_output(port, options, test_input, test_types, test_args, - test_output): + test_output, worker_name): """Receives the output from a DumpRenderTree process, subjects it to a number of tests, and returns a list of failure types the test produced. @@ -94,6 +81,7 @@ def _process_output(port, options, test_input, test_types, test_args, test_types: list of test types to subject the output to test_args: arguments to be passed to each test test_output: a TestOutput object containing the output of the test + worker_name: worker name for logging Returns: a TestResult object """ @@ -104,20 +92,18 @@ def _process_output(port, options, test_input, test_types, test_args, if test_output.timeout: failures.append(test_failures.FailureTimeout()) + test_name = port.relative_test_filename(test_input.filename) if test_output.crash: - _log.debug("Stacktrace for %s:\n%s" % (test_input.filename, - test_output.error)) - # Strip off "file://" since RelativeTestFilename expects - # filesystem paths. - filename = os.path.join(options.results_directory, - port.relative_test_filename( - test_input.filename)) + _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name, + test_output.error)) + filename = os.path.join(options.results_directory, test_name) filename = os.path.splitext(filename)[0] + "-stack.txt" port.maybe_make_directory(os.path.split(filename)[0]) with codecs.open(filename, "wb", "utf-8") as file: file.write(test_output.error) elif test_output.error: - _log.debug("Previous test output stderr lines:\n%s" % test_output.error) + _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name, + test_output.error)) expected_test_output = _expected_test_output(port, test_input.filename) @@ -161,7 +147,7 @@ def _should_fetch_expected_checksum(options): return options.pixel_tests and not (options.new_baseline or options.reset_results) -def _run_single_test(port, options, test_input, test_types, test_args, driver): +def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name): # FIXME: Pull this into TestShellThread._run(). # The image hash is used to avoid doing an image dump if the @@ -169,23 +155,23 @@ def _run_single_test(port, options, test_input, test_types, test_args, driver): # are generating a new baseline. (Otherwise, an image from a # previous run will be copied into the baseline.""" if _should_fetch_expected_checksum(options): - image_hash_to_driver = port.expected_checksum(test_input.filename) - else: - image_hash_to_driver = None - uri = port.filename_to_uri(test_input.filename) - test_output = driver.run_test(uri, test_input.timeout, image_hash_to_driver) + test_input.image_hash = port.expected_checksum(test_input.filename) + test_output = driver.run_test(test_input) return _process_output(port, options, test_input, test_types, test_args, - test_output) + test_output, worker_name) class SingleTestThread(threading.Thread): """Thread wrapper for running a single test file.""" - def __init__(self, port, options, test_input, test_types, test_args): + def __init__(self, port, options, worker_number, worker_name, + test_input, test_types, test_args): """ Args: port: object implementing port-specific hooks options: command line argument object from optparse + worker_number: worker number for tests + worker_name: for logging test_input: Object containing the test filename and timeout test_types: A list of TestType objects to run the test output against. @@ -199,6 +185,8 @@ class SingleTestThread(threading.Thread): self._test_types = test_types self._test_args = test_args self._driver = None + self._worker_number = worker_number + self._name = worker_name def run(self): self._covered_run() @@ -206,12 +194,12 @@ class SingleTestThread(threading.Thread): def _covered_run(self): # FIXME: this is a separate routine to work around a bug # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85. - self._driver = self._port.create_driver(self._test_args.png_path, - self._options) + self._driver = self._port.create_driver(self._worker_number) self._driver.start() self._test_result = _run_single_test(self._port, self._options, self._test_input, self._test_types, - self._test_args, self._driver) + self._test_args, self._driver, + self._name) self._driver.stop() def get_test_result(self): @@ -254,29 +242,28 @@ class WatchableThread(threading.Thread): class TestShellThread(WatchableThread): - def __init__(self, port, options, filename_list_queue, result_queue, - test_types, test_args): + def __init__(self, port, options, worker_number, worker_name, + filename_list_queue, result_queue): """Initialize all the local state for this DumpRenderTree thread. Args: port: interface to port-specific hooks options: command line options argument from optparse + worker_number: identifier for a particular worker thread. + worker_name: for logging. filename_list_queue: A thread safe Queue class that contains lists of tuples of (filename, uri) pairs. result_queue: A thread safe Queue class that will contain serialized TestResult objects. - test_types: A list of TestType objects to run the test output - against. - test_args: A TestArguments object to pass to each TestType. """ WatchableThread.__init__(self) self._port = port self._options = options + self._worker_number = worker_number + self._name = worker_name self._filename_list_queue = filename_list_queue self._result_queue = result_queue self._filename_list = [] - self._test_types = test_types - self._test_args = test_args self._driver = None self._test_group_timing_stats = {} self._test_results = [] @@ -287,6 +274,12 @@ class TestShellThread(WatchableThread): self._http_lock_wait_begin = 0 self._http_lock_wait_end = 0 + self._test_types = [] + for cls in self._get_test_type_classes(): + self._test_types.append(cls(self._port, + self._options.results_directory)) + self._test_args = self._get_test_args(worker_number) + # Current group of tests we're running. self._current_group = None # Number of tests in self._current_group. @@ -294,6 +287,20 @@ class TestShellThread(WatchableThread): # Time at which we started running tests from self._current_group. self._current_group_start_time = None + def _get_test_args(self, worker_number): + """Returns the tuple of arguments for tests and for DumpRenderTree.""" + test_args = test_type_base.TestArguments() + test_args.new_baseline = self._options.new_baseline + test_args.reset_results = self._options.reset_results + + return test_args + + def _get_test_type_classes(self): + classes = [text_diff.TestTextDiff] + if self._options.pixel_tests: + classes.append(image_diff.ImageDiff) + return classes + def get_test_group_timing_stats(self): """Returns a dictionary mapping test group to a tuple of (number of tests in that group, time to run the tests)""" @@ -417,9 +424,9 @@ class TestShellThread(WatchableThread): batch_count += 1 self._num_tests += 1 if self._options.run_singly: - result = self._run_test_singly(test_input) + result = self._run_test_in_another_thread(test_input) else: - result = self._run_test(test_input) + result = self._run_test_in_this_thread(test_input) filename = test_input.filename tests_run_file.write(filename + "\n") @@ -449,7 +456,7 @@ class TestShellThread(WatchableThread): if test_runner: test_runner.update_summary(result_summary) - def _run_test_singly(self, test_input): + def _run_test_in_another_thread(self, test_input): """Run a test in a separate thread, enforcing a hard time limit. Since we can only detect the termination of a thread, not any internal @@ -461,10 +468,11 @@ class TestShellThread(WatchableThread): Returns: A TestResult - """ worker = SingleTestThread(self._port, self._options, + self._worker_number, + self._name, test_input, self._test_types, self._test_args) @@ -496,11 +504,11 @@ class TestShellThread(WatchableThread): _log.error('Cannot get results of test: %s' % test_input.filename) result = test_results.TestResult(test_input.filename, failures=[], - test_run_time=0, total_time_for_all_diffs=0, time_for_diffs=0) + test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={}) return result - def _run_test(self, test_input): + def _run_test_in_this_thread(self, test_input): """Run a single test file using a shared DumpRenderTree process. Args: @@ -514,7 +522,7 @@ class TestShellThread(WatchableThread): self._next_timeout = time.time() + thread_timeout test_result = _run_single_test(self._port, self._options, test_input, self._test_types, self._test_args, - self._driver) + self._driver, self._name) self._test_results.append(test_result) return test_result @@ -527,9 +535,8 @@ class TestShellThread(WatchableThread): """ # poll() is not threadsafe and can throw OSError due to: # http://bugs.python.org/issue1731717 - if (not self._driver or self._driver.poll() is not None): - self._driver = self._port.create_driver(self._test_args.png_path, - self._options) + if not self._driver or self._driver.poll() is not None: + self._driver = self._port.create_driver(self._worker_number) self._driver.start() def _start_servers_with_lock(self): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py deleted file mode 100644 index 63f86d9..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -""""Tests code paths not covered by the regular unit tests.""" - -import sys -import unittest - -import dump_render_tree_thread - - -class Test(unittest.TestCase): - def test_find_thread_stack_found(self): - id, stack = sys._current_frames().items()[0] - found_stack = dump_render_tree_thread.find_thread_stack(id) - self.assertNotEqual(found_stack, None) - - def test_find_thread_stack_not_found(self): - found_stack = dump_render_tree_thread.find_thread_stack(0) - self.assertEqual(found_stack, None) - - -if __name__ == '__main__': - unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py index 101d30b..b054c5b 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py @@ -129,6 +129,10 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase return self.PASS_RESULT # override + def _get_result_char(self, test_name): + return self._get_modifier_char(test_name) + + # override def _convert_json_to_current_version(self, results_json): archive_version = None if self.VERSION_KEY in results_json: diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py index 3267718..331e330 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py @@ -80,7 +80,7 @@ class TestResult(object): class JSONResultsGeneratorBase(object): """A JSON results generator for generic tests.""" - MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 1500 + MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 # Min time (seconds) that will be added to the JSON. MIN_TIME = 1 JSON_PREFIX = "ADD_RESULTS(" @@ -303,6 +303,23 @@ class JSONResultsGeneratorBase(object): return JSONResultsGenerator.PASS_RESULT + def _get_result_char(self, test_name): + """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, + PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result + for the given test_name. + """ + if test_name not in self._test_results_map: + return JSONResultsGenerator.NO_DATA_RESULT + + test_result = self._test_results_map[test_name] + if test_result.modifier == TestResult.DISABLED: + return JSONResultsGenerator.SKIP_RESULT + + if test_result.failed: + return JSONResultsGenerator.FAIL_RESULT + + return JSONResultsGenerator.PASS_RESULT + # FIXME: Callers should use scm.py instead. # FIXME: Identify and fix the run-time errors that were observed on Windows # chromium buildbot when we had updated this code to use scm.py once before. @@ -484,7 +501,7 @@ class JSONResultsGeneratorBase(object): tests: Dictionary containing test result entries. """ - result = self._get_modifier_char(test_name) + result = self._get_result_char(test_name) time = self._get_test_timing(test_name) if test_name not in tests: diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py index 606a613..d6275ee 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py @@ -56,15 +56,6 @@ class JSONGeneratorTest(unittest.TestCase): self._FLAKY_tests = set([]) self._FAILS_tests = set([]) - def _get_test_modifier(self, test_name): - if test_name.startswith('DISABLED_'): - return json_results_generator.JSONResultsGenerator.SKIP_RESULT - elif test_name.startswith('FLAKY_'): - return json_results_generator.JSONResultsGenerator.FLAKY_RESULT - elif test_name.startswith('FAILS_'): - return json_results_generator.JSONResultsGenerator.FAIL_RESULT - return json_results_generator.JSONResultsGenerator.PASS_RESULT - def _test_json_generation(self, passed_tests_list, failed_tests_list): tests_set = set(passed_tests_list) | set(failed_tests_list) @@ -74,9 +65,9 @@ class JSONGeneratorTest(unittest.TestCase): if t.startswith('FLAKY_')]) FAILS_tests = set([t for t in tests_set if t.startswith('FAILS_')]) - PASS_tests = tests_set ^ (DISABLED_tests | FLAKY_tests | FAILS_tests) + PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests) - passed_tests = set(passed_tests_list) ^ DISABLED_tests + passed_tests = set(passed_tests_list) - DISABLED_tests failed_tests = set(failed_tests_list) test_timings = {} @@ -180,10 +171,10 @@ class JSONGeneratorTest(unittest.TestCase): test = tests[test_name] failed = 0 - modifier = self._get_test_modifier(test_name) for result in test[JRG.RESULTS]: - if result[1] == modifier: + if result[1] == JRG.FAIL_RESULT: failed = result[0] + self.assertEqual(1, failed) timing_count = 0 diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py new file mode 100644 index 0000000..e520a9c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py @@ -0,0 +1,197 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Module for handling messages, threads, processes, and concurrency for run-webkit-tests. + +Testing is accomplished by having a manager (TestRunner) gather all of the +tests to be run, and sending messages to a pool of workers (TestShellThreads) +to run each test. Each worker communicates with one driver (usually +DumpRenderTree) to run one test at a time and then compare the output against +what we expected to get. + +This modules provides a message broker that connects the manager to the +workers: it provides a messaging abstraction and message loops, and +handles launching threads and/or processes depending on the +requested configuration. +""" + +import logging +import sys +import time +import traceback + +import dump_render_tree_thread + +_log = logging.getLogger(__name__) + + +def get(port, options): + """Return an instance of a WorkerMessageBroker.""" + worker_model = options.worker_model + if worker_model == 'inline': + return InlineBroker(port, options) + if worker_model == 'threads': + return MultiThreadedBroker(port, options) + raise ValueError('unsupported value for --worker-model: %s' % worker_model) + + +class _WorkerState(object): + def __init__(self, name): + self.name = name + self.thread = None + + +class WorkerMessageBroker(object): + def __init__(self, port, options): + self._port = port + self._options = options + self._num_workers = int(self._options.child_processes) + + # This maps worker names to their _WorkerState values. + self._workers = {} + + def _threads(self): + return tuple([w.thread for w in self._workers.values()]) + + def start_workers(self, test_runner): + """Starts up the pool of workers for running the tests. + + Args: + test_runner: a handle to the manager/TestRunner object + """ + self._test_runner = test_runner + for worker_number in xrange(self._num_workers): + worker = _WorkerState('worker-%d' % worker_number) + worker.thread = self._start_worker(worker_number, worker.name) + self._workers[worker.name] = worker + return self._threads() + + def _start_worker(self, worker_number, worker_name): + raise NotImplementedError + + def run_message_loop(self): + """Loop processing messages until done.""" + raise NotImplementedError + + def cancel_workers(self): + """Cancel/interrupt any workers that are still alive.""" + pass + + def cleanup(self): + """Perform any necessary cleanup on shutdown.""" + pass + + +class InlineBroker(WorkerMessageBroker): + def _start_worker(self, worker_number, worker_name): + # FIXME: Replace with something that isn't a thread. + thread = dump_render_tree_thread.TestShellThread(self._port, + self._options, worker_number, worker_name, + self._test_runner._current_filename_queue, + self._test_runner._result_queue) + # Note: Don't start() the thread! If we did, it would actually + # create another thread and start executing it, and we'd no longer + # be single-threaded. + return thread + + def run_message_loop(self): + thread = self._threads()[0] + thread.run_in_main_thread(self._test_runner, + self._test_runner._current_result_summary) + self._test_runner.update() + + +class MultiThreadedBroker(WorkerMessageBroker): + def _start_worker(self, worker_number, worker_name): + thread = dump_render_tree_thread.TestShellThread(self._port, + self._options, worker_number, worker_name, + self._test_runner._current_filename_queue, + self._test_runner._result_queue) + thread.start() + return thread + + def run_message_loop(self): + threads = self._threads() + + # Loop through all the threads waiting for them to finish. + some_thread_is_alive = True + while some_thread_is_alive: + some_thread_is_alive = False + t = time.time() + for thread in threads: + exception_info = thread.exception_info() + if exception_info is not None: + # Re-raise the thread's exception here to make it + # clear that testing was aborted. Otherwise, + # the tests that did not run would be assumed + # to have passed. + raise exception_info[0], exception_info[1], exception_info[2] + + if thread.isAlive(): + some_thread_is_alive = True + next_timeout = thread.next_timeout() + if next_timeout and t > next_timeout: + log_wedged_worker(thread.getName(), thread.id()) + thread.clear_next_timeout() + + self._test_runner.update() + + if some_thread_is_alive: + time.sleep(0.01) + + def cancel_workers(self): + threads = self._threads() + for thread in threads: + thread.cancel() + + +def log_wedged_worker(name, id): + """Log information about the given worker state.""" + stack = _find_thread_stack(id) + assert(stack is not None) + _log.error("") + _log.error("%s (tid %d) is wedged" % (name, id)) + _log_stack(stack) + _log.error("") + + +def _find_thread_stack(id): + """Returns a stack object that can be used to dump a stack trace for + the given thread id (or None if the id is not found).""" + for thread_id, stack in sys._current_frames().items(): + if thread_id == id: + return stack + return None + + +def _log_stack(stack): + """Log a stack trace to log.error().""" + for filename, lineno, name, line in traceback.extract_stack(stack): + _log.error('File: "%s", line %d, in %s' % (filename, lineno, name)) + if line: + _log.error(' %s' % line.strip()) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py new file mode 100644 index 0000000..6f04fd3 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py @@ -0,0 +1,183 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +import Queue +import sys +import thread +import threading +import time +import unittest + +from webkitpy.common import array_stream +from webkitpy.common.system import outputcapture +from webkitpy.tool import mocktool + +from webkitpy.layout_tests import run_webkit_tests + +import message_broker + + +class TestThread(threading.Thread): + def __init__(self, started_queue, stopping_queue): + threading.Thread.__init__(self) + self._thread_id = None + self._started_queue = started_queue + self._stopping_queue = stopping_queue + self._timeout = False + self._timeout_queue = Queue.Queue() + self._exception_info = None + + def id(self): + return self._thread_id + + def getName(self): + return "worker-0" + + def run(self): + self._covered_run() + + def _covered_run(self): + # FIXME: this is a separate routine to work around a bug + # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85. + self._thread_id = thread.get_ident() + try: + self._started_queue.put('') + msg = self._stopping_queue.get() + if msg == 'KeyboardInterrupt': + raise KeyboardInterrupt + elif msg == 'Exception': + raise ValueError() + elif msg == 'Timeout': + self._timeout = True + self._timeout_queue.get() + except: + self._exception_info = sys.exc_info() + + def exception_info(self): + return self._exception_info + + def next_timeout(self): + if self._timeout: + self._timeout_queue.put('done') + return time.time() - 10 + return time.time() + + def clear_next_timeout(self): + self._next_timeout = None + +class TestHandler(logging.Handler): + def __init__(self, astream): + logging.Handler.__init__(self) + self._stream = astream + + def emit(self, record): + self._stream.write(self.format(record)) + + +class MultiThreadedBrokerTest(unittest.TestCase): + class MockTestRunner(object): + def __init__(self): + pass + + def __del__(self): + pass + + def update(self): + pass + + def run_one_thread(self, msg): + runner = self.MockTestRunner() + port = None + options = mocktool.MockOptions(child_processes='1') + starting_queue = Queue.Queue() + stopping_queue = Queue.Queue() + broker = message_broker.MultiThreadedBroker(port, options) + broker._test_runner = runner + child_thread = TestThread(starting_queue, stopping_queue) + broker._workers['worker-0'] = message_broker._WorkerState('worker-0') + broker._workers['worker-0'].thread = child_thread + child_thread.start() + started_msg = starting_queue.get() + stopping_queue.put(msg) + return broker.run_message_loop() + + def test_basic(self): + interrupted = self.run_one_thread('') + self.assertFalse(interrupted) + + def test_interrupt(self): + self.assertRaises(KeyboardInterrupt, self.run_one_thread, 'KeyboardInterrupt') + + def test_timeout(self): + oc = outputcapture.OutputCapture() + oc.capture_output() + interrupted = self.run_one_thread('Timeout') + self.assertFalse(interrupted) + oc.restore_output() + + def test_exception(self): + self.assertRaises(ValueError, self.run_one_thread, 'Exception') + + +class Test(unittest.TestCase): + def test_find_thread_stack_found(self): + id, stack = sys._current_frames().items()[0] + found_stack = message_broker._find_thread_stack(id) + self.assertNotEqual(found_stack, None) + + def test_find_thread_stack_not_found(self): + found_stack = message_broker._find_thread_stack(0) + self.assertEqual(found_stack, None) + + def test_log_wedged_worker(self): + oc = outputcapture.OutputCapture() + oc.capture_output() + logger = message_broker._log + astream = array_stream.ArrayStream() + handler = TestHandler(astream) + logger.addHandler(handler) + + starting_queue = Queue.Queue() + stopping_queue = Queue.Queue() + child_thread = TestThread(starting_queue, stopping_queue) + child_thread.start() + msg = starting_queue.get() + + message_broker.log_wedged_worker(child_thread.getName(), + child_thread.id()) + stopping_queue.put('') + child_thread.join(timeout=1.0) + + self.assertFalse(astream.empty()) + self.assertFalse(child_thread.isAlive()) + oc.restore_output() + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py index fb9fe6d..7a6aad1 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py @@ -126,7 +126,6 @@ def print_options(): ] - def parse_print_options(print_options, verbose, child_processes, is_fully_parallel): """Parse the options provided to --print and dedup and rank them. @@ -182,8 +181,8 @@ def _configure_logging(stream, verbose): log_datefmt = '%y%m%d %H:%M:%S' log_level = logging.INFO if verbose: - log_fmt = ('%(asctime)s %(process)d %(filename)s:%(lineno)-4d %(levelname)s' - '%(message)s') + log_fmt = ('%(asctime)s %(process)d %(filename)s:%(lineno)d ' + '%(levelname)s %(message)s') log_level = logging.DEBUG root = logging.getLogger() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py index 9a0f4ee..27a6a29 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py @@ -78,8 +78,9 @@ class TestUtilityFunctions(unittest.TestCase): self.assertTrue(options is not None) def test_parse_print_options(self): - def test_switches(args, verbose, child_processes, is_fully_parallel, - expected_switches_str): + def test_switches(args, expected_switches_str, + verbose=False, child_processes=1, + is_fully_parallel=False): options, args = get_options(args) if expected_switches_str: expected_switches = set(expected_switches_str.split(',')) @@ -92,28 +93,23 @@ class TestUtilityFunctions(unittest.TestCase): self.assertEqual(expected_switches, switches) # test that we default to the default set of switches - test_switches([], False, 1, False, - printing.PRINT_DEFAULT) + test_switches([], printing.PRINT_DEFAULT) # test that verbose defaults to everything - test_switches([], True, 1, False, - printing.PRINT_EVERYTHING) + test_switches([], printing.PRINT_EVERYTHING, verbose=True) # test that --print default does what it's supposed to - test_switches(['--print', 'default'], False, 1, False, - printing.PRINT_DEFAULT) + test_switches(['--print', 'default'], printing.PRINT_DEFAULT) # test that --print nothing does what it's supposed to - test_switches(['--print', 'nothing'], False, 1, False, - None) + test_switches(['--print', 'nothing'], None) # test that --print everything does what it's supposed to - test_switches(['--print', 'everything'], False, 1, False, - printing.PRINT_EVERYTHING) + test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING) # this tests that '--print X' overrides '--verbose' - test_switches(['--print', 'actual'], True, 1, False, - 'actual') + test_switches(['--print', 'actual'], 'actual', verbose=True) + class Testprinter(unittest.TestCase): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py index 680b848..033c8c6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py @@ -27,45 +27,81 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import with_statement + +import codecs import mimetypes import socket +import urllib2 from webkitpy.common.net.networktransaction import NetworkTransaction -from webkitpy.thirdparty.autoinstalled.mechanize import Browser - def get_mime_type(filename): - return mimetypes.guess_type(filename)[0] or "text/plain" + return mimetypes.guess_type(filename)[0] or 'application/octet-stream' + + +def _encode_multipart_form_data(fields, files): + """Encode form fields for multipart/form-data. + + Args: + fields: A sequence of (name, value) elements for regular form fields. + files: A sequence of (name, filename, value) elements for data to be + uploaded as files. + Returns: + (content_type, body) ready for httplib.HTTP instance. + + Source: + http://code.google.com/p/rietveld/source/browse/trunk/upload.py + """ + BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' + CRLF = '\r\n' + lines = [] + + for key, value in fields: + lines.append('--' + BOUNDARY) + lines.append('Content-Disposition: form-data; name="%s"' % key) + lines.append('') + if isinstance(value, unicode): + value = value.encode('utf-8') + lines.append(value) + + for key, filename, value in files: + lines.append('--' + BOUNDARY) + lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) + lines.append('Content-Type: %s' % get_mime_type(filename)) + lines.append('') + if isinstance(value, unicode): + value = value.encode('utf-8') + lines.append(value) + + lines.append('--' + BOUNDARY + '--') + lines.append('') + body = CRLF.join(lines) + content_type = 'multipart/form-data; boundary=%s' % BOUNDARY + return content_type, body class TestResultsUploader: def __init__(self, host): self._host = host - self._browser = Browser() def _upload_files(self, attrs, file_objs): - self._browser.open("http://%s/testfile/uploadform" % self._host) - self._browser.select_form("test_result_upload") - for (name, data) in attrs: - self._browser[name] = str(data) - - for (filename, handle) in file_objs: - self._browser.add_file(handle, get_mime_type(filename), filename, - "file") - - self._browser.submit() + url = "http://%s/testfile/upload" % self._host + content_type, data = _encode_multipart_form_data(attrs, file_objs) + headers = {"Content-Type": content_type} + request = urllib2.Request(url, data, headers) + urllib2.urlopen(request) def upload(self, params, files, timeout_seconds): - orig_timeout = socket.getdefaulttimeout() file_objs = [] - try: - file_objs = [(filename, open(path, "rb")) for (filename, path) - in files] + for filename, path in files: + with codecs.open(path, "rb") as file: + file_objs.append(('file', filename, file.read())) + orig_timeout = socket.getdefaulttimeout() + try: socket.setdefaulttimeout(timeout_seconds) NetworkTransaction(timeout_seconds=timeout_seconds).run( lambda: self._upload_files(params, file_objs)) finally: socket.setdefaulttimeout(orig_timeout) - for (filename, handle) in file_objs: - handle.close() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py index 632806f..bc5a9aa 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py @@ -384,6 +384,11 @@ class Port(object): # valid test and by printing.py to determine if baselines exist. return self._filesystem.exists(path) + def driver_cmd_line(self): + """Prints the DRT command line that will be used.""" + driver = self.create_driver(0) + return driver.cmd_line() + def update_baseline(self, path, data, encoding): """Updates the baseline for a test. @@ -487,7 +492,7 @@ class Port(object): """Relative unix-style path for a filename under the LayoutTests directory. Filenames outside the LayoutTests directory should raise an error.""" - #assert(filename.startswith(self.layout_tests_dir())) + assert filename.startswith(self.layout_tests_dir()), "%s did not start with %s" % (filename, self.layout_tests_dir()) return filename[len(self.layout_tests_dir()) + 1:] def results_directory(self): @@ -511,7 +516,7 @@ class Port(object): results_filename in a users' browser.""" return self._user.open_url(results_filename) - def create_driver(self, image_path, options): + def create_driver(self, worker_number): """Return a newly created base.Driver subclass for starting/stopping the test driver.""" raise NotImplementedError('Port.create_driver') @@ -741,7 +746,7 @@ class Port(object): def _path_to_driver(self, configuration=None): """Returns the full path to the test driver (DumpRenderTree).""" - raise NotImplementedError('Port.path_to_driver') + raise NotImplementedError('Port._path_to_driver') def _path_to_webcore_library(self): """Returns the full path to a built copy of WebCore.""" @@ -804,33 +809,26 @@ class Port(object): class Driver: """Abstract interface for the DumpRenderTree interface.""" - def __init__(self, port, png_path, options, executive): + def __init__(self, port, worker_number): """Initialize a Driver to subsequently run tests. Typically this routine will spawn DumpRenderTree in a config ready for subsequent input. port - reference back to the port object. - png_path - an absolute path for the driver to write any image - data for a test (as a PNG). If no path is provided, that - indicates that pixel test results will not be checked. - options - command line options argument from optparse - executive - reference to the process-wide Executive object - + worker_number - identifier for a particular worker/driver instance """ raise NotImplementedError('Driver.__init__') - def run_test(self, uri, timeout, checksum): + def run_test(self, test_input): """Run a single test and return the results. Note that it is okay if a test times out or crashes and leaves the driver in an indeterminate state. The upper layers of the program are responsible for cleaning up and ensuring things are okay. - uri - a full URI for the given test - timeout - number of milliseconds to wait before aborting this test. - checksum - if present, the expected checksum for the image for this - test + Args: + test_input: a TestInput object Returns a TestOutput object. """ diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py index 1e9c2b7..8d586e3 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py @@ -258,7 +258,7 @@ class VirtualTest(unittest.TestCase): self.assertVirtual(port.baseline_search_path) self.assertVirtual(port.check_build, None) self.assertVirtual(port.check_image_diff) - self.assertVirtual(port.create_driver, None, None) + self.assertVirtual(port.create_driver, 0) self.assertVirtual(port.diff_image, None, None) self.assertVirtual(port.path_to_test_expectations_file) self.assertVirtual(port.test_platform_name) @@ -282,7 +282,7 @@ class VirtualTest(unittest.TestCase): def test_virtual_driver_method(self): self.assertRaises(NotImplementedError, base.Driver, base.Port(), - "", None, None) + 0) def test_virtual_driver_methods(self): class VirtualDriver(base.Driver): @@ -290,7 +290,7 @@ class VirtualTest(unittest.TestCase): pass driver = VirtualDriver() - self.assertVirtual(driver.run_test, None, None, None) + self.assertVirtual(driver.run_test, None) self.assertVirtual(driver.poll) self.assertVirtual(driver.stop) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py index 3149290..8fe685a 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -32,6 +32,7 @@ from __future__ import with_statement import codecs +import errno import logging import os import re @@ -43,7 +44,6 @@ import tempfile import time import webbrowser -from webkitpy.common.system.executive import Executive from webkitpy.common.system.path import cygpath from webkitpy.layout_tests.layout_package import test_expectations from webkitpy.layout_tests.layout_package import test_output @@ -175,6 +175,8 @@ class ChromiumPort(base.Port): return result def driver_name(self): + if self._options.use_drt: + return "DumpRenderTree" return "test_shell" def path_from_chromium_base(self, *comps): @@ -212,13 +214,11 @@ class ChromiumPort(base.Port): if os.path.exists(cachedir): shutil.rmtree(cachedir) - def create_driver(self, image_path, options): + def create_driver(self, worker_number): """Starts a new Driver and returns a handle to it.""" - if options.use_drt and sys.platform == 'darwin': - return webkit.WebKitDriver(self, image_path, options, - executive=self._executive) - return ChromiumDriver(self, image_path, options, - executive=self._executive) + if self.get_option('use_drt') and sys.platform == 'darwin': + return webkit.WebKitDriver(self, worker_number) + return ChromiumDriver(self, worker_number) def start_helper(self): helper_path = self._path_to_helper() @@ -359,48 +359,50 @@ class ChromiumPort(base.Port): class ChromiumDriver(base.Driver): """Abstract interface for test_shell.""" - def __init__(self, port, image_path, options, executive=Executive()): + def __init__(self, port, worker_number): self._port = port - self._options = options - self._image_path = image_path - self._executive = executive - - def _driver_args(self): - driver_args = [] - if self._image_path: + self._worker_number = worker_number + self._image_path = None + if self._port.get_option('pixel_tests'): + self._image_path = os.path.join( + self._port.get_option('results_directory'), + 'png_result%s.png' % self._worker_number) + + def cmd_line(self): + cmd = self._command_wrapper(self._port.get_option('wrapper')) + cmd.append(self._port._path_to_driver()) + if self._port.get_option('pixel_tests'): # See note above in diff_image() for why we need _convert_path(). - driver_args.append("--pixel-tests=" + - self._port._convert_path(self._image_path)) + cmd.append("--pixel-tests=" + + self._port._convert_path(self._image_path)) if self._port.get_option('use_drt'): - driver_args.append('--test-shell') + cmd.append('--test-shell') else: - driver_args.append('--layout-tests') + cmd.append('--layout-tests') if self._port.get_option('startup_dialog'): - driver_args.append('--testshell-startup-dialog') + cmd.append('--testshell-startup-dialog') if self._port.get_option('gp_fault_error_box'): - driver_args.append('--gp-fault-error-box') + cmd.append('--gp-fault-error-box') - if self._options.js_flags is not None: - driver_args.append('--js-flags="' + self._options.js_flags + '"') + if self._port.get_option('js_flags') is not None: + cmd.append('--js-flags="' + self._port.get_option('js_flags') + '"') - if self._options.multiple_loads is not None and self._options.multiple_loads > 0: - driver_args.append('--multiple-loads=' + str(self._options.multiple_loads)) + if self._port.get_option('multiple_loads') > 0: + cmd.append('--multiple-loads=' + str(self._port.get_option('multiple_loads'))) if self._port.get_option('accelerated_compositing'): - driver_args.append('--enable-accelerated-compositing') + cmd.append('--enable-accelerated-compositing') if self._port.get_option('accelerated_2d_canvas'): - driver_args.append('--enable-accelerated-2d-canvas') - return driver_args + cmd.append('--enable-accelerated-2d-canvas') + return cmd def start(self): # FIXME: Should be an error to call this method twice. - cmd = self._command_wrapper(self._port.get_option('wrapper')) - cmd.append(self._port._path_to_driver()) - cmd += self._driver_args() + cmd = self.cmd_line() # We need to pass close_fds=True to work around Python bug #2320 # (otherwise we can hang when we kill DumpRenderTree when we are running @@ -454,7 +456,22 @@ class ChromiumDriver(base.Driver): else: return None - def run_test(self, uri, timeoutms, checksum): + def _output_image_with_retry(self): + # Retry a few more times because open() sometimes fails on Windows, + # raising "IOError: [Errno 13] Permission denied:" + retry_num = 50 + timeout_seconds = 5.0 + for i in range(retry_num): + try: + return self._output_image() + except IOError, e: + if e.errno == errno.EACCES: + time.sleep(timeout_seconds / retry_num) + else: + raise e + return self._output_image() + + def run_test(self, test_input): output = [] error = [] crash = False @@ -464,7 +481,9 @@ class ChromiumDriver(base.Driver): start_time = time.time() - cmd = self._test_shell_command(uri, timeoutms, checksum) + uri = self._port.filename_to_uri(test_input.filename) + cmd = self._test_shell_command(uri, test_input.timeout, + test_input.image_hash) (line, crash) = self._write_command_and_read_line(input=cmd) while not crash and line.rstrip() != "#EOF": @@ -505,9 +524,10 @@ class ChromiumDriver(base.Driver): (line, crash) = self._write_command_and_read_line(input=None) + run_time = time.time() - start_time return test_output.TestOutput( - ''.join(output), self._output_image(), actual_checksum, - crash, time.time() - start_time, timeout, ''.join(error)) + ''.join(output), self._output_image_with_retry(), actual_checksum, + crash, run_time, timeout, ''.join(error)) def stop(self): if self._proc: @@ -532,4 +552,4 @@ class ChromiumDriver(base.Driver): if self._proc.poll() is None: _log.warning('stopping test driver timed out, ' 'killing it') - self._executive.kill_process(self._proc.pid) + self._port._executive.kill_process(self._proc.pid) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py index 92a31fb..5396522 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py @@ -42,7 +42,8 @@ class ChromiumDriverTest(unittest.TestCase): def setUp(self): mock_port = Mock() - self.driver = chromium.ChromiumDriver(mock_port, image_path=None, options=None) + mock_port.get_option = lambda option_name: '' + self.driver = chromium.ChromiumDriver(mock_port, worker_number=0) def test_test_shell_command(self): expected_command = "test.html 2 checksum\n" diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py index cad5e37..88f1146 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py @@ -75,7 +75,6 @@ class Config(object): if configuration: flags = ["--configuration", self._FLAGS_FROM_CONFIGURATIONS[configuration]] - configuration = "" else: configuration = "" flags = ["--top-level"] @@ -133,7 +132,7 @@ class Config(object): # This code will also work if there is no SCM system at all. if not self._webkit_base_dir: abspath = os.path.abspath(__file__) - self._webkit_base_dir = abspath[0:abspath.find('WebKitTools')] + self._webkit_base_dir = abspath[0:abspath.find('WebKitTools') - 1] return self._webkit_base_dir def _script_path(self, script_name): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py index 9bea014..8ec28fc 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py @@ -38,13 +38,37 @@ from webkitpy.common.system import outputcapture import config + +def mock_run_command(arg_list): + # Set this to True to test actual output (where possible). + integration_test = False + if integration_test: + return executive.Executive().run_command(arg_list) + + if 'webkit-build-directory' in arg_list[1]: + return mock_webkit_build_directory(arg_list[2:]) + return 'Error' + + +def mock_webkit_build_directory(arg_list): + if arg_list == ['--top-level']: + return '/WebKitBuild' + elif arg_list == ['--configuration', '--debug']: + return '/WebKitBuild/Debug' + elif arg_list == ['--configuration', '--release']: + return '/WebKitBuild/Release' + return 'Error' + + class ConfigTest(unittest.TestCase): def tearDown(self): config.clear_cached_configuration() - def make_config(self, output='', files={}, exit_code=0, exception=None): + def make_config(self, output='', files={}, exit_code=0, exception=None, + run_command_fn=None): e = executive_mock.MockExecutive2(output=output, exit_code=exit_code, - exception=exception) + exception=exception, + run_command_fn=run_command_fn) fs = filesystem_mock.MockFileSystem(files) return config.Config(e, fs) @@ -54,23 +78,17 @@ class ConfigTest(unittest.TestCase): c = self.make_config('foo', {'foo/Configuration': contents}) self.assertEqual(c.default_configuration(), expected) - def test_build_directory_toplevel(self): - c = self.make_config('toplevel') - self.assertEqual(c.build_directory(None), 'toplevel') + def test_build_directory(self): + # --top-level + c = self.make_config(run_command_fn=mock_run_command) + self.assertTrue(c.build_directory(None).endswith('WebKitBuild')) # Test again to check caching - self.assertEqual(c.build_directory(None), 'toplevel') - - def test_build_directory__release(self): - c = self.make_config('release') - self.assertEqual(c.build_directory('Release'), 'release') - - def test_build_directory__debug(self): - c = self.make_config('debug') - self.assertEqual(c.build_directory('Debug'), 'debug') + self.assertTrue(c.build_directory(None).endswith('WebKitBuild')) - def test_build_directory__unknown(self): - c = self.make_config("unknown") + # Test other values + self.assertTrue(c.build_directory('Release').endswith('/Release')) + self.assertTrue(c.build_directory('Debug').endswith('/Debug')) self.assertRaises(KeyError, c.build_directory, 'Unknown') def test_build_dumprendertree__success(self): @@ -168,6 +186,7 @@ class ConfigTest(unittest.TestCase): c = config.Config(executive.Executive(), filesystem.FileSystem()) base_dir = c.webkit_base_dir() self.assertTrue(base_dir) + self.assertNotEqual(base_dir[-1], '/') orig_cwd = os.getcwd() os.chdir(os.environ['HOME']) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py index 96d0d55..4ed34e6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py @@ -95,31 +95,30 @@ class DryRunPort(object): def stop_websocket_server(self): pass - def create_driver(self, image_path, options): - return DryrunDriver(self, image_path, options, executive=None) + def create_driver(self, worker_number): + return DryrunDriver(self, worker_number) class DryrunDriver(base.Driver): """Dryrun implementation of the DumpRenderTree / Driver interface.""" - def __init__(self, port, image_path, options, executive): + def __init__(self, port, worker_number): self._port = port - self._image_path = image_path - self._executive = executive - self._layout_tests_dir = None + self._worker_number = worker_number + + def cmd_line(self): + return ['None'] def poll(self): return None - def run_test(self, uri, timeoutms, image_hash): + def run_test(self, test_input): start_time = time.time() - test_name = self._port.uri_to_test_name(uri) - path = os.path.join(self._port.layout_tests_dir(), test_name) - text_output = self._port.expected_text(path) + text_output = self._port.expected_text(test_input.filename) - if image_hash is not None: - image = self._port.expected_image(path) - hash = self._port.expected_checksum(path) + if test_input.image_hash is not None: + image = self._port.expected_image(test_input.filename) + hash = self._port.expected_checksum(test_input.filename) else: image = None hash = None diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py index 04ada50..c4b36ac 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py @@ -37,6 +37,8 @@ mock_options = mocktool.MockOptions(results_directory='layout-test-results', use_apache=True, configuration='Release') +# FIXME: This should be used for all ports, not just WebKit Mac. See +# https://bugs.webkit.org/show_bug.cgi?id=50043 . class PortTestCase(unittest.TestCase): """Tests the WebKit port implementation.""" @@ -44,6 +46,12 @@ class PortTestCase(unittest.TestCase): """Override in subclass.""" raise NotImplementedError() + def test_driver_cmd_line(self): + port = self.make_port() + if not port: + return + self.assertTrue(len(port.driver_cmd_line())) + def test_http_server(self): port = self.make_port() if not port: diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py index 0a27821..8e27f35 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py @@ -226,8 +226,8 @@ class TestPort(base.Port): def setup_test_run(self): pass - def create_driver(self, image_path, options): - return TestDriver(self, image_path, options, executive=None) + def create_driver(self, worker_number): + return TestDriver(self, worker_number) def start_http_server(self): pass @@ -281,25 +281,25 @@ WONTFIX SKIP : failures/expected/exception.html = CRASH class TestDriver(base.Driver): """Test/Dummy implementation of the DumpRenderTree interface.""" - def __init__(self, port, image_path, options, executive): + def __init__(self, port, worker_number): self._port = port - self._image_path = image_path - self._executive = executive - self._image_written = False + + def cmd_line(self): + return ['None'] def poll(self): return True - def run_test(self, uri, timeoutms, image_hash): + def run_test(self, test_input): start_time = time.time() - test_name = self._port.uri_to_test_name(uri) + test_name = self._port.relative_test_filename(test_input.filename) test = self._port._tests[test_name] if test.keyboard: raise KeyboardInterrupt if test.exception: raise ValueError('exception from ' + test_name) if test.hang: - time.sleep((float(timeoutms) * 4) / 1000.0) + time.sleep((float(test_input.timeout) * 4) / 1000.0) return test_output.TestOutput(test.actual_text, test.actual_image, test.actual_checksum, test.crash, time.time() - start_time, test.timeout, diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py index 3fa0fb3..2c0a7b6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py @@ -78,7 +78,7 @@ def find(port, paths): # Now walk all the paths passed in on the command line and get filenames test_files = set() for path in paths_to_walk: - if os.path.isfile(path) and _has_supported_extension(path): + if os.path.isfile(path) and _is_test_file(path): test_files.add(os.path.normpath(path)) continue @@ -95,7 +95,7 @@ def find(port, paths): dirs.remove(directory) for filename in files: - if _has_supported_extension(filename): + if _is_test_file(filename): filename = os.path.join(root, filename) filename = os.path.normpath(filename) test_files.add(filename) @@ -111,3 +111,18 @@ def _has_supported_extension(filename): test on.""" extension = os.path.splitext(filename)[1] return extension in _supported_file_extensions + + +def _is_reference_html_file(filename): + """Return true if the filename points to a reference HTML file.""" + if (filename.endswith('-expected.html') or + filename.endswith('-expected-mismatch.html')): + _log.warn("Reftests are not supported - ignoring %s" % filename) + return True + return False + + +def _is_test_file(filename): + """Return true if the filename points to a test file.""" + return (_has_supported_extension(filename) and + not _is_reference_html_file(filename)) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py index c37eb92..83525c8 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py @@ -63,6 +63,13 @@ class TestFilesTest(unittest.TestCase): tests = test_files.find(port, ['userscripts/resources']) self.assertEqual(tests, set([])) + def test_is_test_file(self): + self.assertTrue(test_files._is_test_file('foo.html')) + self.assertTrue(test_files._is_test_file('foo.shtml')) + self.assertFalse(test_files._is_test_file('foo.png')) + self.assertFalse(test_files._is_test_file('foo-expected.html')) + self.assertFalse(test_files._is_test_file('foo-expected-mismatch.html')) + if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py index 06797c6..09be833 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -46,8 +46,6 @@ import operator import tempfile import shutil -from webkitpy.common.system.executive import Executive - import webkitpy.common.system.ospath as ospath import webkitpy.layout_tests.layout_package.test_output as test_output import webkitpy.layout_tests.port.base as base @@ -185,9 +183,8 @@ class WebKitPort(base.Port): # This port doesn't require any specific configuration. pass - def create_driver(self, image_path, options): - return WebKitDriver(self, image_path, options, - executive=self._executive) + def create_driver(self, worker_number): + return WebKitDriver(self, worker_number) def test_base_platform_names(self): # At the moment we don't use test platform names, but we have @@ -389,40 +386,36 @@ class WebKitPort(base.Port): class WebKitDriver(base.Driver): """WebKit implementation of the DumpRenderTree interface.""" - def __init__(self, port, image_path, options, executive=Executive()): + def __init__(self, port, worker_number): + self._worker_number = worker_number self._port = port - self._image_path = image_path - self._executive = executive self._driver_tempdir = tempfile.mkdtemp(prefix='DumpRenderTree-') def __del__(self): shutil.rmtree(self._driver_tempdir) - def _driver_args(self): - driver_args = [] + def cmd_line(self): + cmd = self._command_wrapper(self._port.get_option('wrapper')) + cmd += [self._port._path_to_driver(), '-'] - if self._image_path: - driver_args.append('--pixel-tests') + if self._port.get_option('pixel_tests'): + cmd.append('--pixel-tests') if self._port.get_option('use_drt'): if self._port.get_option('accelerated_compositing'): - driver_args.append('--enable-accelerated-compositing') + cmd.append('--enable-accelerated-compositing') if self._port.get_option('accelerated_2d_canvas'): - driver_args.append('--enable-accelerated-2d-canvas') + cmd.append('--enable-accelerated-2d-canvas') - return driver_args + return cmd def start(self): - command = self._command_wrapper(self._port.get_option('wrapper')) - command += [self._port._path_to_driver(), '-'] - command += self._driver_args() - environment = self._port.setup_environ_for_server() environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path() environment['DUMPRENDERTREE_TEMP'] = self._driver_tempdir self._server_process = server_process.ServerProcess(self._port, - "DumpRenderTree", command, environment) + "DumpRenderTree", self.cmd_line(), environment) def poll(self): return self._server_process.poll() @@ -433,14 +426,15 @@ class WebKitDriver(base.Driver): return # FIXME: This function is huge. - def run_test(self, uri, timeoutms, image_hash): + def run_test(self, test_input): + uri = self._port.filename_to_uri(test_input.filename) if uri.startswith("file:///"): command = uri[7:] else: command = uri - if image_hash: - command += "'" + image_hash + if test_input.image_hash: + command += "'" + test_input.image_hash command += "\n" start_time = time.time() @@ -451,7 +445,7 @@ class WebKitDriver(base.Driver): output = str() # Use a byte array for output, even though it should be UTF-8. image = str() - timeout = int(timeoutms) / 1000.0 + timeout = int(test_input.timeout) / 1000.0 deadline = time.time() + timeout line = self._server_process.read_line(timeout) while (not self._server_process.timed_out diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index 119de8c..f4e92a6 100755 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -66,17 +66,16 @@ import traceback from layout_package import dump_render_tree_thread from layout_package import json_layout_results_generator +from layout_package import message_broker from layout_package import printing from layout_package import test_expectations from layout_package import test_failures from layout_package import test_results from layout_package import test_results_uploader -from test_types import image_diff -from test_types import text_diff -from test_types import test_type_base from webkitpy.common.system import user from webkitpy.thirdparty import simplejson +from webkitpy.tool import grammar import port @@ -102,6 +101,10 @@ class TestInput: # FIXME: filename should really be test_name as a relative path. self.filename = filename self.timeout = timeout + # The image_hash is used to avoid doing an image dump if the + # checksums match. The image_hash is set later, and only if it is needed + # for the test. + self.image_hash = None class ResultSummary(object): @@ -237,27 +240,24 @@ class TestRunner: # in DumpRenderTree. DEFAULT_TEST_TIMEOUT_MS = 6 * 1000 - def __init__(self, port, options, printer): + def __init__(self, port, options, printer, message_broker): """Initialize test runner data structures. Args: port: an object implementing port-specific options: a dictionary of command line options printer: a Printer object to record updates to. + message_broker: object used to communicate with workers. """ self._port = port self._options = options self._printer = printer + self._message_broker = message_broker # disable wss server. need to install pyOpenSSL on buildbots. # self._websocket_secure_server = websocket_server.PyWebSocket( # options.results_directory, use_tls=True, port=9323) - # a list of TestType objects - self._test_types = [text_diff.TestTextDiff] - if options.pixel_tests: - self._test_types.append(image_diff.ImageDiff) - # a set of test files, and the same tests as a list self._test_files = set() self._test_files_list = None @@ -488,7 +488,7 @@ class TestRunner: """Returns the appropriate TestInput object for the file. Mostly this is used for looking up the timeout value (in ms) to use for the given test.""" - if self._expectations.has_modifier(test_file, test_expectations.SLOW): + if self._test_is_slow(test_file): return TestInput(test_file, self._options.slow_time_out_ms) return TestInput(test_file, self._options.time_out_ms) @@ -498,23 +498,30 @@ class TestRunner: split_path = test_file.split(os.sep) return 'http' in split_path or 'websocket' in split_path - def _get_test_file_queue(self, test_files): - """Create the thread safe queue of lists of (test filenames, test URIs) - tuples. Each TestShellThread pulls a list from this queue and runs - those tests in order before grabbing the next available list. + def _test_is_slow(self, test_file): + return self._expectations.has_modifier(test_file, + test_expectations.SLOW) - Shard the lists by directory. This helps ensure that tests that depend - on each other (aka bad tests!) continue to run together as most - cross-tests dependencies tend to occur within the same directory. + def _shard_tests(self, test_files, use_real_shards): + """Groups tests into batches. + This helps ensure that tests that depend on each other (aka bad tests!) + continue to run together as most cross-tests dependencies tend to + occur within the same directory. If use_real_shards is false, we + put each (non-HTTP/websocket) test into its own shard for maximum + concurrency instead of trying to do any sort of real sharding. Return: - The Queue of lists of TestInput objects. + A list of lists of TestInput objects. """ + # FIXME: when we added http locking, we changed how this works such + # that we always lump all of the HTTP threads into a single shard. + # That will slow down experimental-fully-parallel, but it's unclear + # what the best alternative is completely revamping how we track + # when to grab the lock. test_lists = [] tests_to_http_lock = [] - if (self._options.experimental_fully_parallel or - self._is_single_threaded()): + if not use_real_shards: for test_file in test_files: test_input = self._get_test_input_for_file(test_file) if self._test_requires_lock(test_file): @@ -553,23 +560,7 @@ class TestRunner: tests_to_http_lock.reverse() test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock)) - filename_queue = Queue.Queue() - for item in test_lists: - filename_queue.put(item) - return filename_queue - - def _get_test_args(self, index): - """Returns the tuple of arguments for tests and for DumpRenderTree.""" - test_args = test_type_base.TestArguments() - test_args.png_path = None - if self._options.pixel_tests: - png_path = os.path.join(self._options.results_directory, - "png_result%s.png" % index) - test_args.png_path = png_path - test_args.new_baseline = self._options.new_baseline - test_args.reset_results = self._options.reset_results - - return test_args + return test_lists def _contains_tests(self, subdir): for test_file in self._test_files: @@ -577,39 +568,8 @@ class TestRunner: return True return False - def _instantiate_dump_render_tree_threads(self, test_files, - result_summary): - """Instantitates and starts the TestShellThread(s). - - Return: - The list of threads. - """ - filename_queue = self._get_test_file_queue(test_files) - - # Instantiate TestShellThreads and start them. - threads = [] - for i in xrange(int(self._options.child_processes)): - # Create separate TestTypes instances for each thread. - test_types = [] - for test_type in self._test_types: - test_types.append(test_type(self._port, - self._options.results_directory)) - - test_args = self._get_test_args(i) - thread = dump_render_tree_thread.TestShellThread(self._port, - self._options, filename_queue, self._result_queue, - test_types, test_args) - if self._is_single_threaded(): - thread.run_in_main_thread(self, result_summary) - else: - thread.start() - threads.append(thread) - - return threads - - def _is_single_threaded(self): - """Returns whether we should run all the tests in the main thread.""" - return int(self._options.child_processes) == 1 + def _num_workers(self): + return int(self._options.child_processes) def _run_tests(self, file_list, result_summary): """Runs the tests in the file_list. @@ -625,59 +585,48 @@ class TestRunner: in the form {filename:filename, test_run_time:test_run_time} result_summary: summary object to populate with the results """ - # FIXME: We should use webkitpy.tool.grammar.pluralize here. - plural = "" - if not self._is_single_threaded(): - plural = "s" - self._printer.print_update('Starting %s%s ...' % - (self._port.driver_name(), plural)) - threads = self._instantiate_dump_render_tree_threads(file_list, - result_summary) + + self._printer.print_update('Sharding tests ...') + num_workers = self._num_workers() + test_lists = self._shard_tests(file_list, + num_workers > 1 and not self._options.experimental_fully_parallel) + filename_queue = Queue.Queue() + for item in test_lists: + filename_queue.put(item) + + self._printer.print_update('Starting %s ...' % + grammar.pluralize('worker', num_workers)) + message_broker = self._message_broker + self._current_filename_queue = filename_queue + self._current_result_summary = result_summary + + if not self._options.dry_run: + threads = message_broker.start_workers(self) + else: + threads = {} + self._printer.print_update("Starting testing ...") + keyboard_interrupted = False + if not self._options.dry_run: + try: + message_broker.run_message_loop() + except KeyboardInterrupt: + _log.info("Interrupted, exiting") + message_broker.cancel_workers() + keyboard_interrupted = True + except: + # Unexpected exception; don't try to clean up workers. + _log.info("Exception raised, exiting") + raise - keyboard_interrupted = self._wait_for_threads_to_finish(threads, - result_summary) - (thread_timings, test_timings, individual_test_timings) = \ + thread_timings, test_timings, individual_test_timings = \ self._collect_timing_info(threads) return (keyboard_interrupted, thread_timings, test_timings, individual_test_timings) - def _wait_for_threads_to_finish(self, threads, result_summary): - keyboard_interrupted = False - try: - # Loop through all the threads waiting for them to finish. - some_thread_is_alive = True - while some_thread_is_alive: - some_thread_is_alive = False - t = time.time() - for thread in threads: - exception_info = thread.exception_info() - if exception_info is not None: - # Re-raise the thread's exception here to make it - # clear that testing was aborted. Otherwise, - # the tests that did not run would be assumed - # to have passed. - raise exception_info[0], exception_info[1], exception_info[2] - - if thread.isAlive(): - some_thread_is_alive = True - next_timeout = thread.next_timeout() - if (next_timeout and t > next_timeout): - _log_wedged_thread(thread) - thread.clear_next_timeout() - - self.update_summary(result_summary) - - if some_thread_is_alive: - time.sleep(0.01) - - except KeyboardInterrupt: - keyboard_interrupted = True - for thread in threads: - thread.cancel() - - return keyboard_interrupted + def update(self): + self.update_summary(self._current_result_summary) def _collect_timing_info(self, threads): test_timings = {} @@ -793,16 +742,18 @@ class TestRunner: self._expectations, result_summary, retry_summary) self._printer.print_unexpected_results(unexpected_results) - if self._options.record_results: + if (self._options.record_results and not self._options.dry_run and + not keyboard_interrupted): # Write the same data to log files and upload generated JSON files # to appengine server. self._upload_json_files(unexpected_results, result_summary, individual_test_timings) # Write the summary to disk (results.html) and display it if requested. - wrote_results = self._write_results_html_file(result_summary) - if self._options.show_results and wrote_results: - self._show_results_html_file() + if not self._options.dry_run: + wrote_results = self._write_results_html_file(result_summary) + if self._options.show_results and wrote_results: + self._show_results_html_file() # Now that we've completed all the processing we can, we re-raise # a KeyboardInterrupt if necessary so the caller can handle it. @@ -947,12 +898,15 @@ class TestRunner: (self._options.time_out_ms, self._options.slow_time_out_ms)) - if self._is_single_threaded(): + if self._num_workers() == 1: p.print_config("Running one %s" % self._port.driver_name()) else: p.print_config("Running %s %ss in parallel" % (self._options.child_processes, self._port.driver_name())) + p.print_config('Command line: ' + + ' '.join(self._port.driver_cmd_line())) + p.print_config("Worker model: %s" % self._options.worker_model) p.print_config("") def _print_expected_results_of_type(self, result_summary, @@ -1067,8 +1021,7 @@ class TestRunner: for test_tuple in individual_test_timings: filename = test_tuple.filename is_timeout_crash_or_slow = False - if self._expectations.has_modifier(filename, - test_expectations.SLOW): + if self._test_is_slow(filename): is_timeout_crash_or_slow = True slow_tests.append(test_tuple) @@ -1342,11 +1295,13 @@ def run(port, options, args, regular_output=sys.stderr, printer.cleanup() return 0 + broker = message_broker.get(port, options) + # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. num_unexpected_results = -1 try: - test_runner = TestRunner(port, options, printer) + test_runner = TestRunner(port, options, printer, broker) test_runner._print_config() printer.print_update("Collecting tests ...") @@ -1375,6 +1330,7 @@ def run(port, options, args, regular_output=sys.stderr, _log.debug("Testing completed, Exit status: %d" % num_unexpected_results) finally: + broker.cleanup() printer.cleanup() return num_unexpected_results @@ -1383,8 +1339,11 @@ def run(port, options, args, regular_output=sys.stderr, def _set_up_derived_options(port_obj, options): """Sets the options values that depend on other options values.""" + if options.worker_model == 'inline': + if options.child_processes and int(options.child_processes) > 1: + _log.warning("--worker-model=inline overrides --child-processes") + options.child_processes = "1" if not options.child_processes: - # FIXME: Investigate perf/flakiness impact of using cpu_count + 1. options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES", str(port_obj.default_child_processes())) @@ -1568,6 +1527,9 @@ def parse_args(args=None): optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), + optparse.make_option("-n", "--dry-run", action="store_true", + default=False, + help="Do everything but actually run the tests or upload results."), # old-run-webkit-tests has --valgrind instead of wrapper. optparse.make_option("--wrapper", help="wrapper command to insert before invocations of " @@ -1607,6 +1569,9 @@ def parse_args(args=None): optparse.make_option("--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. + optparse.make_option("--worker-model", action="store", + default="threads", help=("controls worker model. Valid values are " + "'inline' and 'threads' (default).")), optparse.make_option("--experimental-fully-parallel", action="store_true", default=False, help="run all tests in parallel"), @@ -1618,7 +1583,7 @@ def parse_args(args=None): # Number of times to run the set of tests (e.g. ABCABCABC) optparse.make_option("--print-last-failures", action="store_true", default=False, help="Print the tests in the last run that " - "had unexpected failures (or passes)."), + "had unexpected failures (or passes) and then exit."), optparse.make_option("--retest-last-failures", action="store_true", default=False, help="re-test the tests in the last run that " "had unexpected failures (or passes)."), @@ -1662,20 +1627,7 @@ def parse_args(args=None): old_run_webkit_tests_compat) option_parser = optparse.OptionParser(option_list=option_list) - options, args = option_parser.parse_args(args) - - return options, args - - -def _log_wedged_thread(thread): - """Log information about the given thread state.""" - id = thread.id() - stack = dump_render_tree_thread.find_thread_stack(id) - assert(stack is not None) - _log.error("") - _log.error("thread %s (%d) is wedged" % (thread.getName(), id)) - dump_render_tree_thread.log_stack(stack) - _log.error("") + return option_parser.parse_args(args) def main(): @@ -1683,6 +1635,7 @@ def main(): port_obj = port.get(options.platform, options) return run(port_obj, options, args) + if '__main__' == __name__: try: sys.exit(main()) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py index 54e1dc0..6bb741a 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py @@ -72,12 +72,14 @@ def passing_run(extra_args=None, port_obj=None, record_results=False, args.extend(['--platform', 'test']) if not record_results: args.append('--no-record-results') + if not '--child-processes' in extra_args: + args.extend(['--worker-model', 'inline']) args.extend(extra_args) if not tests_included: # We use the glob to test that globbing works. args.extend(['passes', 'http/tests', - 'http/tests/websocket/tests', + 'websocket/tests', 'failures/expected/*']) options, parsed_args = run_webkit_tests.parse_args(args) if not port_obj: @@ -92,21 +94,30 @@ def logging_run(extra_args=None, port_obj=None, tests_included=False): args = ['--no-record-results'] if not '--platform' in extra_args: args.extend(['--platform', 'test']) + if not '--child-processes' in extra_args: + args.extend(['--worker-model', 'inline']) args.extend(extra_args) if not tests_included: args.extend(['passes', 'http/tests', - 'http/tests/websocket/tests', + 'websocket/tests', 'failures/expected/*']) - options, parsed_args = run_webkit_tests.parse_args(args) - user = MockUser() - if not port_obj: - port_obj = port.get(port_name=options.platform, options=options, user=user) - buildbot_output = array_stream.ArrayStream() - regular_output = array_stream.ArrayStream() - res = run_webkit_tests.run(port_obj, options, parsed_args, - buildbot_output=buildbot_output, - regular_output=regular_output) + + oc = outputcapture.OutputCapture() + try: + oc.capture_output() + options, parsed_args = run_webkit_tests.parse_args(args) + user = MockUser() + if not port_obj: + port_obj = port.get(port_name=options.platform, options=options, + user=user) + buildbot_output = array_stream.ArrayStream() + regular_output = array_stream.ArrayStream() + res = run_webkit_tests.run(port_obj, options, parsed_args, + buildbot_output=buildbot_output, + regular_output=regular_output) + finally: + oc.restore_output() return (res, buildbot_output, regular_output, user) @@ -116,7 +127,7 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False): '--print', 'nothing', '--platform', 'test', '--no-record-results', - '--child-processes', '1'] + '--worker-model', 'inline'] args.extend(extra_args) if not tests_included: # Not including http tests since they get run out of order (that @@ -128,8 +139,8 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False): test_batches = [] class RecordingTestDriver(TestDriver): - def __init__(self, port, image_path, options): - TestDriver.__init__(self, port, image_path, options, executive=None) + def __init__(self, port, worker_number): + TestDriver.__init__(self, port, worker_number) self._current_test_batch = None def poll(self): @@ -139,16 +150,17 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False): def stop(self): self._current_test_batch = None - def run_test(self, uri, timeoutms, image_hash): + def run_test(self, test_input): if self._current_test_batch is None: self._current_test_batch = [] test_batches.append(self._current_test_batch) - self._current_test_batch.append(self._port.uri_to_test_name(uri)) - return TestDriver.run_test(self, uri, timeoutms, image_hash) + test_name = self._port.relative_test_filename(test_input.filename) + self._current_test_batch.append(test_name) + return TestDriver.run_test(self, test_input) class RecordingTestPort(TestPort): - def create_driver(self, image_path, options): - return RecordingTestDriver(self, image_path, options) + def create_driver(self, worker_number): + return RecordingTestDriver(self, worker_number) recording_port = RecordingTestPort(options=options, user=user) logging_run(extra_args=args, port_obj=recording_port, tests_included=True) @@ -189,6 +201,13 @@ class MainTest(unittest.TestCase): self.assertTrue('Running 2 DumpRenderTrees in parallel\n' in regular_output.get()) + def test_dryrun(self): + batch_tests_run = get_tests_run(['--dry-run']) + self.assertEqual(batch_tests_run, []) + + batch_tests_run = get_tests_run(['-n']) + self.assertEqual(batch_tests_run, []) + def test_exception_raised(self): self.assertRaises(ValueError, logging_run, ['failures/expected/exception.html'], tests_included=True) @@ -214,7 +233,7 @@ class MainTest(unittest.TestCase): def test_keyboard_interrupt(self): # Note that this also tests running a test marked as SKIP if # you specify it explicitly. - self.assertRaises(KeyboardInterrupt, passing_run, + self.assertRaises(KeyboardInterrupt, logging_run, ['failures/expected/keyboard.html'], tests_included=True) def test_last_results(self): @@ -359,9 +378,24 @@ class MainTest(unittest.TestCase): test_port = get_port_for_run(base_args) self.assertEqual(None, test_port.tolerance_used_for_diff_image) + def test_worker_model__inline(self): + self.assertTrue(passing_run(['--worker-model', 'inline'])) + + def test_worker_model__threads(self): + self.assertTrue(passing_run(['--worker-model', 'threads'])) + + def test_worker_model__processes(self): + self.assertRaises(ValueError, logging_run, + ['--worker-model', 'processes']) + + def test_worker_model__unknown(self): + self.assertRaises(ValueError, logging_run, + ['--worker-model', 'unknown']) + MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2') + def _mocked_open(original_open, file_list): def _wrapper(name, mode, encoding): if name.find("-expected.") != -1 and mode.find("w") != -1: @@ -439,7 +473,8 @@ class TestRunnerTest(unittest.TestCase): mock_port.relative_test_filename = lambda name: name mock_port.filename_to_uri = lambda name: name - runner = run_webkit_tests.TestRunner(port=mock_port, options=Mock(), printer=Mock()) + runner = run_webkit_tests.TestRunner(port=mock_port, options=Mock(), + printer=Mock(), message_broker=Mock()) expected_html = u"""<html> <head> <title>Layout Test Results (time)</title> @@ -453,20 +488,11 @@ class TestRunnerTest(unittest.TestCase): html = runner._results_html(["test_path"], {}, "Title", override_time="time") self.assertEqual(html, expected_html) - def queue_to_list(self, queue): - queue_list = [] - while(True): - try: - queue_list.append(queue.get_nowait()) - except Queue.Empty: - break - return queue_list - - def test_get_test_file_queue(self): - # Test that _get_test_file_queue in run_webkit_tests.TestRunner really + def test_shard_tests(self): + # Test that _shard_tests in run_webkit_tests.TestRunner really # put the http tests first in the queue. - runner = TestRunnerWrapper(port=Mock(), options=Mock(), printer=Mock()) - runner._options.experimental_fully_parallel = False + runner = TestRunnerWrapper(port=Mock(), options=Mock(), + printer=Mock(), message_broker=Mock()) test_list = [ "LayoutTests/websocket/tests/unicode.htm", @@ -487,19 +513,16 @@ class TestRunnerTest(unittest.TestCase): 'LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html', ]) - runner._options.child_processes = 1 - test_queue_for_single_thread = runner._get_test_file_queue(test_list) - runner._options.child_processes = 2 - test_queue_for_multi_thread = runner._get_test_file_queue(test_list) - - single_thread_results = self.queue_to_list(test_queue_for_single_thread) - multi_thread_results = self.queue_to_list(test_queue_for_multi_thread) + # FIXME: Ideally the HTTP tests don't have to all be in one shard. + single_thread_results = runner._shard_tests(test_list, False) + multi_thread_results = runner._shard_tests(test_list, True) self.assertEqual("tests_to_http_lock", single_thread_results[0][0]) self.assertEqual(expected_tests_to_http_lock, set(single_thread_results[0][1])) self.assertEqual("tests_to_http_lock", multi_thread_results[0][0]) self.assertEqual(expected_tests_to_http_lock, set(multi_thread_results[0][1])) + class DryrunTest(unittest.TestCase): # FIXME: it's hard to know which platforms are safe to test; the # chromium platforms require a chromium checkout, and the mac platform @@ -520,114 +543,5 @@ class DryrunTest(unittest.TestCase): '--pixel-tests'])) -class TestThread(dump_render_tree_thread.WatchableThread): - def __init__(self, started_queue, stopping_queue): - dump_render_tree_thread.WatchableThread.__init__(self) - self._started_queue = started_queue - self._stopping_queue = stopping_queue - self._timeout = False - self._timeout_queue = Queue.Queue() - - def run(self): - self._covered_run() - - def _covered_run(self): - # FIXME: this is a separate routine to work around a bug - # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85. - self._thread_id = thread.get_ident() - try: - self._started_queue.put('') - msg = self._stopping_queue.get() - if msg == 'KeyboardInterrupt': - raise KeyboardInterrupt - elif msg == 'Exception': - raise ValueError() - elif msg == 'Timeout': - self._timeout = True - self._timeout_queue.get() - except: - self._exception_info = sys.exc_info() - - def next_timeout(self): - if self._timeout: - self._timeout_queue.put('done') - return time.time() - 10 - return time.time() - - -class TestHandler(logging.Handler): - def __init__(self, astream): - logging.Handler.__init__(self) - self._stream = astream - - def emit(self, record): - self._stream.write(self.format(record)) - - -class WaitForThreadsToFinishTest(unittest.TestCase): - class MockTestRunner(run_webkit_tests.TestRunner): - def __init__(self): - pass - - def __del__(self): - pass - - def update_summary(self, result_summary): - pass - - def run_one_thread(self, msg): - runner = self.MockTestRunner() - starting_queue = Queue.Queue() - stopping_queue = Queue.Queue() - child_thread = TestThread(starting_queue, stopping_queue) - child_thread.start() - started_msg = starting_queue.get() - stopping_queue.put(msg) - threads = [child_thread] - return runner._wait_for_threads_to_finish(threads, None) - - def test_basic(self): - interrupted = self.run_one_thread('') - self.assertFalse(interrupted) - - def test_interrupt(self): - interrupted = self.run_one_thread('KeyboardInterrupt') - self.assertTrue(interrupted) - - def test_timeout(self): - oc = outputcapture.OutputCapture() - oc.capture_output() - interrupted = self.run_one_thread('Timeout') - self.assertFalse(interrupted) - oc.restore_output() - - def test_exception(self): - self.assertRaises(ValueError, self.run_one_thread, 'Exception') - - -class StandaloneFunctionsTest(unittest.TestCase): - def test_log_wedged_thread(self): - oc = outputcapture.OutputCapture() - oc.capture_output() - logger = run_webkit_tests._log - astream = array_stream.ArrayStream() - handler = TestHandler(astream) - logger.addHandler(handler) - - starting_queue = Queue.Queue() - stopping_queue = Queue.Queue() - child_thread = TestThread(starting_queue, stopping_queue) - child_thread.start() - msg = starting_queue.get() - - run_webkit_tests._log_wedged_thread(child_thread) - stopping_queue.put('') - child_thread.join(timeout=1.0) - - self.assertFalse(astream.empty()) - self.assertFalse(child_thread.isAlive()) - oc.restore_output() - - if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py index 41fe9bd..da466c8 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py @@ -103,8 +103,8 @@ class ImageDiff(test_type_base.TestTypeBase): # If we're generating a new baseline, we pass. if test_args.new_baseline or test_args.reset_results: - self._save_baseline_files(filename, actual_test_output.image_hash, - actual_test_output.image, + self._save_baseline_files(filename, actual_test_output.image, + actual_test_output.image_hash, test_args.new_baseline) return failures diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py index 66e42ba..ca4b17d 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py @@ -66,7 +66,8 @@ class TestTextDiff(test_type_base.TestTypeBase): # Although all test_shell/DumpRenderTree output should be utf-8, # we do not ever decode it inside run-webkit-tests. For some tests # DumpRenderTree may not output utf-8 text (e.g. webarchives). - self._save_baseline_data(filename, output, ".txt", encoding=None, + self._save_baseline_data(filename, actual_test_output.text, + ".txt", encoding=None, generate_new_baseline=test_args.new_baseline) return failures diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/index.html b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/index.html index 8cc48c1..5b58301 100644 --- a/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/index.html +++ b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/index.html @@ -35,13 +35,24 @@ <script src="/util.js"></script> <script src="/loupe.js"></script> <script src="/main.js"></script> + <script src="/queue.js"></script> </head> <body class="loading"> +<pre id="log" style="display: none"></pre> +<div id="queue" style="display: none"> + Queue: + <select id="queue-select" size="10"></select> + <button id="remove-queue-selection">Remove selection</button> + <button id="rebaseline-queue">Rebaseline queue</button> +</div> + <div id="header"> <div id="controls"> <!-- Add a dummy <select> node so that this lines up with the text on the left --> <select style="visibility: hidden"></select> + <span id="toggle-log" class="link">Log</span> + <span class="divider">|</span> <a href="/quitquitquit">Exit</a> </div> @@ -62,7 +73,7 @@ </label> </span> - <a id="test-link">View test</a> + <a id="test-link" target="_blank">View test</a> <span id="nav-buttons"> <button id="previous-test">«</button> @@ -86,7 +97,14 @@ <tr> <td><img id="expected-image"></td> <td><img id="actual-image"></td> - <td><canvas id="diff-canvas" width="800" height="600"></canvas></td> + <td> + <canvas id="diff-canvas" width="800" height="600"></canvas> + <div id="diff-checksum" style="display: none"> + <h3>Checksum mismatch</h3> + Expected: <span id="expected-checksum"></span><br> + Actual: <span id="actual-checksum"></span> + </div> + </td> </tr> </tbody> <tbody id="text-outputs" style="display: none"> @@ -101,6 +119,29 @@ </tbody> </table> +<div id="footer"> + <label>State: <span id="state"></span></label> + <label>Existing baselines: <span id="current-baselines"></span></label> + <label> + Baseline target: + <select id="baseline-target"></select> + </label> + <label> + Move current baselines to: + <select id="baseline-move-to"> + <option value="none">Nowhere (replace)</option> + </select> + </label> + + <!-- Add a dummy <button> node so that this lines up with the text on the right --> + <button style="visibility: hidden; padding-left: 0; padding-right: 0;"></button> + + <div id="action-buttons"> + <span id="toggle-queue" class="link">Queue</span> + <button id="add-to-rebaseline-queue">Add to rebaseline queue</button> + </div> +</div> + <table id="loupe" style="display: none"> <tr> <td colspan="3" id="loupe-info"> diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/main.css b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/main.css index 6e90fe4..aff2bf6 100644 --- a/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/main.css +++ b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/main.css @@ -55,15 +55,52 @@ a, .link { text-decoration: none; } -#header { +#log, +#queue { + padding: .25em 0 0 .25em; + position: absolute; + right: 0; + height: 200px; + overflow: auto; + background: #fff; + -webkit-box-shadow: 1px 1px 5px rgba(0, 0, 0, .5); +} + +#log { + top: 2em; + width: 500px; +} + +#queue { + bottom: 3em; + width: 400px; +} + +#queue-select { + display: block; + width: 390px; +} + +#header, +#footer { padding: .5em 1em; background: #333; color: #fff; -webkit-box-shadow: 0 1px 5px rgba(0, 0, 0, 0.5); +} + +#header { margin-bottom: 1em; } -#header label { +#header .divider, +#footer .divider { + opacity: .3; + padding: 0 .5em; +} + +#header label, +#footer label { padding-right: 1em; color: #ccc; } @@ -72,7 +109,8 @@ a, .link { margin-right: 1em; } -#header label span { +#header label span, +#footer label span { color: #fff; font-weight: bold; } @@ -114,13 +152,18 @@ a, .link { } #image-outputs img, -#image-outputs canvas { +#image-outputs canvas, +#image-outputs #diff-checksum { width: 800px; height: 600px; border: solid 1px #ddd; -webkit-user-select: none; -webkit-user-drag: none; - cursor: crosshair; +} + +#image-outputs img, +#image-outputs canvas { + cursor: crosshair; } #image-outputs img.loading, @@ -150,6 +193,59 @@ a, .link { background: #eee; } +#footer { + position: absolute; + bottom: 0; + left: 0; + right: 0; + margin-top: 1em; +} + +#state.needs_rebaseline { + color: yellow; +} + +#state.rebaseline_failed { + color: red; +} + +#state.rebaseline_succeeded { + color: green; +} + +#state.in_queue { + color: gray; +} + +#current-baselines { + font-weight: normal !important; +} + +#current-baselines .platform { + font-weight: bold; +} + +#current-baselines a { + color: #ddf; +} + +#current-baselines .was-used-for-test { + color: #aaf; + font-weight: bold; +} + +#action-buttons { + float: right; +} + +#action-buttons .link { + margin-right: 1em; +} + +#footer button { + padding: 1em; +} + #loupe { -webkit-box-shadow: 2px 2px 5px rgba(0, 0, 0, .5); position: absolute; @@ -165,7 +261,7 @@ a, .link { #loupe td { padding: 0; - border: solid 1px #ccc; + border: solid 1px #ccc; } #loupe label { diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/main.js b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/main.js index fa037b3..66990d0 100644 --- a/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/main.js +++ b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/main.js @@ -30,11 +30,22 @@ var ALL_DIRECTORY_PATH = '[all]'; +var STATE_NEEDS_REBASELINE = 'needs_rebaseline'; +var STATE_REBASELINE_FAILED = 'rebaseline_failed'; +var STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded'; +var STATE_IN_QUEUE = 'in_queue'; +var STATE_TO_DISPLAY_STATE = {}; +STATE_TO_DISPLAY_STATE[STATE_NEEDS_REBASELINE] = 'Needs rebaseline'; +STATE_TO_DISPLAY_STATE[STATE_REBASELINE_FAILED] = 'Rebaseline failed'; +STATE_TO_DISPLAY_STATE[STATE_REBASELINE_SUCCEEDED] = 'Rebaseline succeeded'; +STATE_TO_DISPLAY_STATE[STATE_IN_QUEUE] = 'In queue'; + var results; var testsByFailureType = {}; var testsByDirectory = {}; var selectedTests = []; var loupe; +var queue; function main() { @@ -44,7 +55,10 @@ function main() $('next-test').addEventListener('click', nextTest); $('previous-test').addEventListener('click', previousTest); + $('toggle-log').addEventListener('click', function() { toggle('log'); }); + loupe = new Loupe(); + queue = new RebaselineQueue(); document.addEventListener('keydown', function(event) { if (event.altKey || event.ctrlKey || event.metaKey || event.shiftKey) { @@ -60,9 +74,32 @@ function main() event.preventDefault(); nextTest(); break; + case 'U+0051': // q + queue.addCurrentTest(); + break; + case 'U+0058': // x + queue.removeCurrentTest(); + break; + case 'U+0052': // r + queue.rebaseline(); + break; } }); + loadText('/platforms.json', function(text) { + var platforms = JSON.parse(text); + platforms.platforms.forEach(function(platform) { + var platformOption = document.createElement('option'); + platformOption.value = platform; + platformOption.textContent = platform; + + var targetOption = platformOption.cloneNode(true); + targetOption.selected = platform == platforms.defaultPlatform; + $('baseline-target').appendChild(targetOption); + $('baseline-move-to').appendChild(platformOption.cloneNode(true)); + }); + }); + loadText('/results.json', function(text) { results = JSON.parse(text); displayResults(); @@ -104,7 +141,7 @@ function displayResults() selectFailureType(); - document.body.classList.remove('loading'); + document.body.className = ''; } /** @@ -212,8 +249,61 @@ function selectTest() $('text-outputs').style.display = 'none'; } + var currentBaselines = $('current-baselines'); + currentBaselines.textContent = ''; + var baselines = results.tests[selectedTest].baselines; + var testName = selectedTest.split('.').slice(0, -1).join('.'); + getSortedKeys(baselines).forEach(function(platform, i) { + if (i != 0) { + currentBaselines.appendChild(document.createTextNode('; ')); + } + var platformName = document.createElement('span'); + platformName.className = 'platform'; + platformName.textContent = platform; + currentBaselines.appendChild(platformName); + currentBaselines.appendChild(document.createTextNode(' (')); + getSortedKeys(baselines[platform]).forEach(function(extension, j) { + if (j != 0) { + currentBaselines.appendChild(document.createTextNode(', ')); + } + var link = document.createElement('a'); + var baselinePath = ''; + if (platform != 'base') { + baselinePath += 'platform/' + platform + '/'; + } + baselinePath += testName + '-expected' + extension; + link.href = getTracUrl(baselinePath); + if (extension == '.checksum') { + link.textContent = 'chk'; + } else { + link.textContent = extension.substring(1); + } + link.target = '_blank'; + if (baselines[platform][extension]) { + link.className = 'was-used-for-test'; + } + currentBaselines.appendChild(link); + }); + currentBaselines.appendChild(document.createTextNode(')')); + }); + updateState(); loupe.hide(); + + prefetchNextImageTest(); +} + +function prefetchNextImageTest() +{ + var testSelector = $('test-selector'); + if (testSelector.selectedIndex == testSelector.options.length - 1) { + return; + } + var nextTest = testSelector.options[testSelector.selectedIndex + 1].value; + if (results.tests[nextTest].actual.indexOf('IMAGE') != -1) { + new Image().src = getTestResultUrl(nextTest, 'expected-image'); + new Image().src = getTestResultUrl(nextTest, 'actual-image'); + } } function updateState() @@ -227,8 +317,13 @@ function updateState() $('next-test').disabled = testIndex == testCount - 1; $('previous-test').disabled = testIndex == 0; - $('test-link').href = - 'http://trac.webkit.org/browser/trunk/LayoutTests/' + testName; + $('test-link').href = getTracUrl(testName); + + var state = results.tests[testName].state; + $('state').className = state; + $('state').innerHTML = STATE_TO_DISPLAY_STATE[state]; + + queue.updateState(); } function getTestResultUrl(testName, mode) @@ -266,6 +361,7 @@ function displayImageResults(testName) $('diff-canvas').className = 'loading'; $('diff-canvas').style.display = ''; + $('diff-checksum').style.display = 'none'; } /** @@ -317,6 +413,7 @@ function updateImageDiff() { var diffWidth = diffImageData.width; var diff = diffImageData.data; + var hadDiff = false; for (var x = 0; x < expectedWidth; x++) { for (var y = 0; y < expectedHeight; y++) { var expectedOffset = (y * expectedWidth + x) * 4; @@ -326,6 +423,7 @@ function updateImageDiff() { expected[expectedOffset + 1] != actual[actualOffset + 1] || expected[expectedOffset + 2] != actual[actualOffset + 2] || expected[expectedOffset + 3] != actual[actualOffset + 3]) { + hadDiff = true; diff[diffOffset] = 255; diff[diffOffset + 1] = 0; diff[diffOffset + 2] = 0; @@ -345,19 +443,27 @@ function updateImageDiff() { 0, 0, diffImageData.width, diffImageData.height); diffCanvas.className = ''; + + if (!hadDiff) { + diffCanvas.style.display = 'none'; + $('diff-checksum').style.display = ''; + loadTextResult(currentExpectedImageTest, 'expected-checksum'); + loadTextResult(currentExpectedImageTest, 'actual-checksum'); + } } -function displayTextResults(testName) +function loadTextResult(testName, mode) { - function loadTextResult(mode) { - loadText(getTestResultUrl(testName, mode), function(text) { - $(mode).textContent = text; - }); - } + loadText(getTestResultUrl(testName, mode), function(text) { + $(mode).textContent = text; + }); +} - loadTextResult('expected-text'); - loadTextResult('actual-text'); - loadTextResult('diff-text'); +function displayTextResults(testName) +{ + loadTextResult(testName, 'expected-text'); + loadTextResult(testName, 'actual-text'); + loadTextResult(testName, 'diff-text'); } function nextTest() diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/queue.js b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/queue.js new file mode 100644 index 0000000..f57c919 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/queue.js @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2010 Google Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +function RebaselineQueue() +{ + this._selectNode = $('queue-select'); + this._rebaselineButtonNode = $('rebaseline-queue'); + this._toggleNode = $('toggle-queue'); + this._removeSelectionButtonNode = $('remove-queue-selection'); + + this._inProgressRebaselineCount = 0; + + var self = this; + $('add-to-rebaseline-queue').addEventListener( + 'click', function() { self.addCurentTest(); }); + this._selectNode.addEventListener('change', updateState); + this._removeSelectionButtonNode.addEventListener( + 'click', function() { self._removeSelection(); }); + this._rebaselineButtonNode.addEventListener( + 'click', function() { self.rebaseline(); }); + this._toggleNode.addEventListener( + 'click', function() { toggle('queue'); }); +} + +RebaselineQueue.prototype.updateState = function() +{ + var testName = getSelectedTest(); + + var state = results.tests[testName].state; + $('add-to-rebaseline-queue').disabled = state != STATE_NEEDS_REBASELINE; + + var queueLength = this._selectNode.options.length; + if (this._inProgressRebaselineCount > 0) { + this._rebaselineButtonNode.disabled = true; + this._rebaselineButtonNode.textContent = + 'Rebaseline in progress (' + this._inProgressRebaselineCount + + ' tests left)'; + } else if (queueLength == 0) { + this._rebaselineButtonNode.disabled = true; + this._rebaselineButtonNode.textContent = 'Rebaseline queue'; + this._toggleNode.textContent = 'Queue'; + } else { + this._rebaselineButtonNode.disabled = false; + this._rebaselineButtonNode.textContent = + 'Rebaseline queue (' + queueLength + ' tests)'; + this._toggleNode.textContent = 'Queue (' + queueLength + ' tests)'; + } + this._removeSelectionButtonNode.disabled = + this._selectNode.selectedIndex == -1; +}; + +RebaselineQueue.prototype.addCurrentTest = function() +{ + var testName = getSelectedTest(); + var test = results.tests[testName]; + + if (test.state != STATE_NEEDS_REBASELINE) { + log('Cannot add test with state "' + test.state + '" to queue.', + log.WARNING); + return; + } + + var queueOption = document.createElement('option'); + queueOption.value = testName; + queueOption.textContent = testName; + this._selectNode.appendChild(queueOption); + test.state = STATE_IN_QUEUE; + updateState(); +}; + +RebaselineQueue.prototype.removeCurrentTest = function() +{ + this._removeTest(getSelectedTest()); +}; + +RebaselineQueue.prototype._removeSelection = function() +{ + if (this._selectNode.selectedIndex == -1) + return; + + this._removeTest( + this._selectNode.options[this._selectNode.selectedIndex].value); +}; + +RebaselineQueue.prototype._removeTest = function(testName) +{ + var queueOption = this._selectNode.firstChild; + + while (queueOption && queueOption.value != testName) { + queueOption = queueOption.nextSibling; + } + + if (!queueOption) + return; + + this._selectNode.removeChild(queueOption); + var test = results.tests[testName]; + test.state = STATE_NEEDS_REBASELINE; + updateState(); +}; + +RebaselineQueue.prototype.rebaseline = function() +{ + var testNames = []; + for (var queueOption = this._selectNode.firstChild; + queueOption; + queueOption = queueOption.nextSibling) { + testNames.push(queueOption.value); + } + + this._inProgressRebaselineCount = testNames.length; + updateState(); + + testNames.forEach(this._rebaselineTest, this); +}; + +RebaselineQueue.prototype._rebaselineTest = function(testName) +{ + var baselineTarget = getSelectValue('baseline-target'); + var baselineMoveTo = getSelectValue('baseline-move-to'); + + // FIXME: actually rebaseline + log('Rebaselining ' + testName + ' for platform ' + baselineTarget + '...'); + var test = results.tests[testName]; + this._removeTest(testName); + this._inProgressRebaselineCount--; + test.state = STATE_REBASELINE_SUCCEEDED; + updateState(); + log('Rebaselined add test with state ' + test.state + ' to queue', + log.SUCCESS); +}; diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/util.js b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/util.js index 1c8782b..5ad7612 100644 --- a/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/util.js +++ b/WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/util.js @@ -55,3 +55,50 @@ function loadText(url, callback) xhr.addEventListener('load', function() { callback(xhr.responseText); }); xhr.send(); } + +function log(text, type) +{ + var node = $('log'); + + if (type) { + var typeNode = document.createElement('span'); + typeNode.textContent = type.text; + typeNode.style.color = type.color; + node.appendChild(typeNode); + } + + node.appendChild(document.createTextNode(text + '\n')); + node.scrollTop = node.scrollHeight; +} + +log.WARNING = {text: 'Warning: ', color: '#aa3'}; +log.SUCCESS = {text: 'Success: ', color: 'green'}; +log.ERROR = {text: 'Error: ', color: 'red'}; + +function toggle(id) +{ + var element = $(id); + var toggler = $('toggle-' + id); + if (element.style.display == 'none') { + element.style.display = ''; + toggler.className = 'link selected'; + } else { + element.style.display = 'none'; + toggler.className = 'link'; + } +} + +function getTracUrl(layoutTestPath) +{ + return 'http://trac.webkit.org/browser/trunk/LayoutTests/' + layoutTestPath; +} + +function getSortedKeys(obj) +{ + var keys = []; + for (var key in obj) { + keys.push(key); + } + keys.sort(); + return keys; +}
\ No newline at end of file diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/rebaselineserver.py b/WebKitTools/Scripts/webkitpy/tool/commands/rebaselineserver.py index abb2af4..2dcc566 100644 --- a/WebKitTools/Scripts/webkitpy/tool/commands/rebaselineserver.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/rebaselineserver.py @@ -45,14 +45,22 @@ import BaseHTTPServer from optparse import make_option from wsgiref.handlers import format_date_time +from webkitpy.common import system +from webkitpy.layout_tests.port import factory +from webkitpy.layout_tests.port.webkit import WebKitPort from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand -import webkitpy.thirdparty.simplejson as simplejson +from webkitpy.thirdparty import simplejson + +STATE_NEEDS_REBASELINE = 'needs_rebaseline' +STATE_REBASELINE_FAILED = 'rebaseline_failed' +STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded' class RebaselineHTTPServer(BaseHTTPServer.HTTPServer): - def __init__(self, httpd_port, results_directory, results_json): + def __init__(self, httpd_port, results_directory, results_json, platforms_json): BaseHTTPServer.HTTPServer.__init__(self, ("", httpd_port), RebaselineHTTPRequestHandler) self.results_directory = results_directory self.results_json = results_json + self.platforms_json = platforms_json class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): @@ -61,6 +69,7 @@ class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): "loupe.js", "main.js", "main.css", + "queue.js", "util.js", ]) @@ -141,10 +150,16 @@ class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): self._serve_file(file_path, cacheable_seconds=60) def results_json(self): + self._serve_json(self.server.results_json) + + def platforms_json(self): + self._serve_json(self.server.platforms_json) + + def _serve_json(self, json): self.send_response(200) self.send_header('Content-type', 'application/json') self.end_headers() - simplejson.dump(self.server.results_json, self.wfile) + simplejson.dump(json, self.wfile) def _serve_file(self, file_path, cacheable_seconds=0): if not os.path.exists(file_path): @@ -168,6 +183,44 @@ class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): shutil.copyfileobj(static_file, self.wfile) +def _get_test_baselines(test_file, test_port, layout_tests_directory, platforms, filesystem): + class AllPlatformsPort(WebKitPort): + def __init__(self): + WebKitPort.__init__(self, filesystem=filesystem) + self._platforms_by_directory = dict( + [(self._webkit_baseline_path(p), p) for p in platforms]) + + def baseline_search_path(self): + return self._platforms_by_directory.keys() + + def platform_from_directory(self, directory): + return self._platforms_by_directory[directory] + + test_path = filesystem.join(layout_tests_directory, test_file) + + all_platforms_port = AllPlatformsPort() + + all_test_baselines = {} + for baseline_extension in ('.txt', '.checksum', '.png'): + test_baselines = test_port.expected_baselines( + test_path, baseline_extension) + baselines = all_platforms_port.expected_baselines( + test_path, baseline_extension, all_baselines=True) + for platform_directory, expected_filename in baselines: + if not platform_directory: + continue + if platform_directory == layout_tests_directory: + platform = 'base' + else: + platform = all_platforms_port.platform_from_directory( + platform_directory) + platform_baselines = all_test_baselines.setdefault(platform, {}) + was_used_for_test = ( + platform_directory, expected_filename) in test_baselines + platform_baselines[baseline_extension] = was_used_for_test + + return all_test_baselines + class RebaselineServer(AbstractDeclarativeCommand): name = "rebaseline-server" help_text = __doc__ @@ -181,20 +234,41 @@ class RebaselineServer(AbstractDeclarativeCommand): def execute(self, options, args, tool): results_directory = args[0] + filesystem = system.filesystem.FileSystem() print 'Parsing unexpected_results.json...' - results_json_path = os.path.join( + results_json_path = filesystem.join( results_directory, 'unexpected_results.json') with codecs.open(results_json_path, "r") as results_json_file: results_json_file = file(results_json_path) results_json = simplejson.load(results_json_file) - print "Starting server at http://localhost:%d/" % options.httpd_port - print ("Use the 'Exit' link in the UI, http://localhost:%d/" - "quitquitquit or Ctrl-C to stop") % options.httpd_port + port = factory.get() + layout_tests_directory = port.layout_tests_dir() + platforms = filesystem.listdir( + filesystem.join(layout_tests_directory, 'platform')) + + print 'Gathering current baselines...' + for test_file, test_json in results_json['tests'].items(): + test_json['state'] = STATE_NEEDS_REBASELINE + test_path = filesystem.join(layout_tests_directory, test_file) + test_json['baselines'] = _get_test_baselines( + test_file, port, layout_tests_directory, platforms, filesystem) + + server_url = "http://localhost:%d/" % options.httpd_port + print "Starting server at %s" % server_url + print ("Use the 'Exit' link in the UI, %squitquitquit " + "or Ctrl-C to stop") % server_url + + threading.Timer( + .1, lambda: self._tool.user.open_url(server_url)).start() httpd = RebaselineHTTPServer( httpd_port=options.httpd_port, results_directory=results_directory, - results_json=results_json) + results_json=results_json, + platforms_json={ + 'platforms': platforms, + 'defaultPlatform': port.name(), + }) httpd.serve_forever() diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/rebaselineserver_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/rebaselineserver_unittest.py new file mode 100644 index 0000000..b37da3d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/rebaselineserver_unittest.py @@ -0,0 +1,99 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.system import filesystem_mock +from webkitpy.layout_tests.port import base +from webkitpy.layout_tests.port.webkit import WebKitPort +from webkitpy.tool.commands import rebaselineserver + + +class GetBaselinesTest(unittest.TestCase): + def test_no_baselines(self): + self._assertBaselines( + test_files=(), + test_name='fast/missing.html', + expected_baselines={}) + + def test_text_baselines(self): + self._assertBaselines( + test_files=( + 'fast/text-expected.txt', + 'platform/mac/fast/text-expected.txt', + ), + test_name='fast/text.html', + expected_baselines={ + 'mac': {'.txt': True}, + 'base': {'.txt': False}, + }) + + def test_image_and_text_baselines(self): + self._assertBaselines( + test_files=( + 'fast/image-expected.txt', + 'platform/mac/fast/image-expected.png', + 'platform/mac/fast/image-expected.checksum', + 'platform/win/fast/image-expected.png', + 'platform/win/fast/image-expected.checksum', + ), + test_name='fast/image.html', + expected_baselines={ + 'base': {'.txt': True}, + 'mac': {'.checksum': True, '.png': True}, + 'win': {'.checksum': False, '.png': False}, + }) + + def test_extra_baselines(self): + self._assertBaselines( + test_files=( + 'fast/text-expected.txt', + 'platform/nosuchplatform/fast/text-expected.txt', + ), + test_name='fast/text.html', + expected_baselines={'base': {'.txt': True}}) + + def _assertBaselines(self, test_files, test_name, expected_baselines): + layout_tests_directory = base.Port().layout_tests_dir() + mock_filesystem = filesystem_mock.MockFileSystem() + for file in test_files + (test_name,): + file_path = mock_filesystem.join(layout_tests_directory, file) + mock_filesystem.files[file_path] = '' + + class TestMacPort(WebKitPort): + def __init__(self): + WebKitPort.__init__(self, filesystem=mock_filesystem) + self._name = 'mac' + + actual_baselines = rebaselineserver._get_test_baselines( + test_name, + TestMacPort(), + layout_tests_directory, + ('mac', 'win', 'linux'), + mock_filesystem) + self.assertEqual(expected_baselines, actual_baselines) |