diff options
author | Steve Block <steveblock@google.com> | 2010-04-27 16:31:00 +0100 |
---|---|---|
committer | Steve Block <steveblock@google.com> | 2010-05-11 14:42:12 +0100 |
commit | dcc8cf2e65d1aa555cce12431a16547e66b469ee (patch) | |
tree | 92a8d65cd5383bca9749f5327fb5e440563926e6 /WebKitTools/Scripts | |
parent | ccac38a6b48843126402088a309597e682f40fe6 (diff) | |
download | external_webkit-dcc8cf2e65d1aa555cce12431a16547e66b469ee.zip external_webkit-dcc8cf2e65d1aa555cce12431a16547e66b469ee.tar.gz external_webkit-dcc8cf2e65d1aa555cce12431a16547e66b469ee.tar.bz2 |
Merge webkit.org at r58033 : Initial merge by git
Change-Id: If006c38561af287c50cd578d251629b51e4d8cd1
Diffstat (limited to 'WebKitTools/Scripts')
272 files changed, 21580 insertions, 6587 deletions
diff --git a/WebKitTools/Scripts/VCSUtils.pm b/WebKitTools/Scripts/VCSUtils.pm index 022c72a..25a319b 100644 --- a/WebKitTools/Scripts/VCSUtils.pm +++ b/WebKitTools/Scripts/VCSUtils.pm @@ -61,6 +61,7 @@ BEGIN { &isSVNDirectory &isSVNVersion16OrNewer &makeFilePathRelative + &mergeChangeLogs &normalizePath &parsePatch &pathRelativeToSVNRepositoryRootForPath @@ -815,6 +816,76 @@ sub runPatchCommand($$$;$) return $exitStatus; } +# Merge ChangeLog patches using a three-file approach. +# +# This is used by resolve-ChangeLogs when it's operated as a merge driver +# and when it's used to merge conflicts after a patch is applied or after +# an svn update. +# +# It's also used for traditional rejected patches. +# +# Args: +# $fileMine: The merged version of the file. Also known in git as the +# other branch's version (%B) or "ours". +# For traditional patch rejects, this is the *.rej file. +# $fileOlder: The base version of the file. Also known in git as the +# ancestor version (%O) or "base". +# For traditional patch rejects, this is the *.orig file. +# $fileNewer: The current version of the file. Also known in git as the +# current version (%A) or "theirs". +# For traditional patch rejects, this is the original-named +# file. +# +# Returns 1 if merge was successful, else 0. +sub mergeChangeLogs($$$) +{ + my ($fileMine, $fileOlder, $fileNewer) = @_; + + my $traditionalReject = $fileMine =~ /\.rej$/ ? 1 : 0; + + local $/ = undef; + + my $patch; + if ($traditionalReject) { + open(DIFF, "<", $fileMine) or die $!; + $patch = <DIFF>; + close(DIFF); + rename($fileMine, "$fileMine.save"); + rename($fileOlder, "$fileOlder.save"); + } else { + open(DIFF, "-|", qw(diff -u -a --binary), $fileOlder, $fileMine) or die $!; + $patch = <DIFF>; + close(DIFF); + } + + unlink("${fileNewer}.orig"); + unlink("${fileNewer}.rej"); + + open(PATCH, "| patch --force --fuzz=3 --binary $fileNewer > " . File::Spec->devnull()) or die $!; + print PATCH ($traditionalReject ? $patch : fixChangeLogPatch($patch)); + close(PATCH); + + my $result = !exitStatus($?); + + # Refuse to merge the patch if it did not apply cleanly + if (-e "${fileNewer}.rej") { + unlink("${fileNewer}.rej"); + if (-f "${fileNewer}.orig") { + unlink($fileNewer); + rename("${fileNewer}.orig", $fileNewer); + } + } else { + unlink("${fileNewer}.orig"); + } + + if ($traditionalReject) { + rename("$fileMine.save", $fileMine); + rename("$fileOlder.save", $fileOlder); + } + + return $result; +} + sub gitConfig($) { return unless $isGit; diff --git a/WebKitTools/Scripts/build-dumprendertree b/WebKitTools/Scripts/build-dumprendertree index 72e81b0..14690a8 100755 --- a/WebKitTools/Scripts/build-dumprendertree +++ b/WebKitTools/Scripts/build-dumprendertree @@ -45,6 +45,7 @@ Usage: $programName [options] [options to pass to build system] --gtk Build the GTK+ port --qt Build the Qt port --wx Build the wxWindows port + --chromium Build the Chromium port EOF GetOptions( @@ -72,6 +73,17 @@ if (isAppleMacWebKit()) { } elsif (isQt() || isGtk() || isWx()) { # Qt, Gtk and wxWindows build everything in one shot. No need to build anything here. $result = 0; +} elsif (isChromium()) { + if (isDarwin()) { + $result = buildXCodeProject("DumpRenderTree.gyp/DumpRenderTree", $clean, @ARGV); + } elsif (isCygwin() || isWindows()) { + # Windows build - builds the root visual studio solution. + $result = buildChromiumVisualStudioProject("DumpRenderTree.gyp/DumpRenderTree.sln", $clean); + } elsif (isLinux()) { + $result = buildChromiumMakefile("../../WebKit/chromium/", "DumpRenderTree", $clean); + } else { + die "This platform is not supported by Chromium.\n"; + } } else { die "Building not defined for this platform!\n"; } diff --git a/WebKitTools/Scripts/build-webkit b/WebKitTools/Scripts/build-webkit index 5ae1aae..2d172c5 100755 --- a/WebKitTools/Scripts/build-webkit +++ b/WebKitTools/Scripts/build-webkit @@ -48,15 +48,17 @@ chdirWebKit(); my $showHelp = 0; my $clean = 0; my $minimal = 0; +my $webkit2 = 0; my $makeArgs; my $startTime = time(); my ($threeDCanvasSupport, $threeDRenderingSupport, $channelMessagingSupport, $clientBasedGeolocationSupport, $databaseSupport, $datagridSupport, $datalistSupport, $domStorageSupport, $eventsourceSupport, $filtersSupport, $geolocationSupport, $iconDatabaseSupport, $indexedDatabaseSupport, - $javaScriptDebuggerSupport, $mathmlSupport, $offlineWebApplicationSupport, $rubySupport, $sharedWorkersSupport, + $javaScriptDebuggerSupport, $mathmlSupport, $offlineWebApplicationSupport, $rubySupport, $systemMallocSupport, $sandboxSupport, $sharedWorkersSupport, $svgSupport, $svgAnimationSupport, $svgAsImageSupport, $svgDOMObjCBindingsSupport, $svgFontsSupport, $svgForeignObjectSupport, $svgUseSupport, $videoSupport, $webSocketsSupport, $wmlSupport, $wcssSupport, $xhtmlmpSupport, $workersSupport, - $xpathSupport, $xsltSupport, $coverageSupport, $notificationsSupport); + $xpathSupport, $xsltSupport, $coverageSupport, $notificationsSupport, $blobSliceSupport, $tiledBackingStoreSupport, + $fileReaderSupport, $fileWriterSupport); my @features = ( { option => "3d-canvas", desc => "Toggle 3D canvas support", @@ -65,6 +67,9 @@ my @features = ( { option => "3d-rendering", desc => "Toggle 3D rendering support", define => "ENABLE_3D_RENDERING", default => (isAppleMacWebKit() && !isTiger()), value => \$threeDRenderingSupport }, + { option => "blob-slice", desc => "Toggle Blob.slice support", + define => "ENABLE_BLOB_SLICE", default => (isAppleMacWebKit()), value => \$blobSliceSupport }, + { option => "channel-messaging", desc => "Toggle MessageChannel and MessagePort support", define => "ENABLE_CHANNEL_MESSAGING", default => 1, value => \$channelMessagingSupport }, @@ -116,6 +121,12 @@ my @features = ( { option => "ruby", desc => "Toggle HTML5 Ruby support", define => "ENABLE_RUBY", default => 1, value => \$rubySupport }, + { option => "system-malloc", desc => "Toggle system allocator instead of TCmalloc", + define => "USE_SYSTEM_MALLOC", default => 0, value => \$systemMallocSupport }, + + { option => "sandbox", desc => "Toggle HTML5 Sandboxed iframe support", + define => "ENABLE_SANDBOX", default => 1, value => \$sandboxSupport }, + { option => "shared-workers", desc => "Toggle SharedWorkers support", define => "ENABLE_SHARED_WORKERS", default => (isAppleWebKit() || isGtk()), value => \$sharedWorkersSupport }, @@ -140,6 +151,9 @@ my @features = ( { option => "svg-use", desc => "Toggle SVG use element support (implies SVG support)", define => "ENABLE_SVG_USE", default => 1, value => \$svgUseSupport }, + { option => "tiled-backing-store", desc => "Toggle Tiled Backing Store support", + define => "ENABLE_TILED_BACKING_STORE", default => isQt(), value => \$tiledBackingStoreSupport }, + { option => "video", desc => "Toggle Video support", define => "ENABLE_VIDEO", default => (isAppleWebKit() || isGtk()), value => \$videoSupport }, @@ -163,6 +177,12 @@ my @features = ( { option => "xslt", desc => "Toggle XSLT support", define => "ENABLE_XSLT", default => 1, value => \$xsltSupport }, + + { option => "file-reader", desc => "Toggle FileReader support", + define => "ENABLE_FILE_READER", default => 0, value => \$fileReaderSupport }, + + { option => "file-writer", desc => "Toggle FileWriter support", + define => "ENABLE_FILE_WRITER", default => 0, value => \$fileWriterSupport }, ); # Update defaults from Qt's project file @@ -204,6 +224,7 @@ Usage: $programName [options] [options to pass to build system] --chromium Build the Chromium port on Mac/Win/Linux --gtk Build the GTK+ port --qt Build the Qt port + --webkit2 Build the WebKit2 framework --inspector-frontend Copy changes to the inspector front-end files to the build directory --makeargs=<arguments> Optional Makefile flags @@ -217,6 +238,7 @@ my %options = ( 'clean' => \$clean, 'makeargs=s' => \$makeArgs, 'minimal' => \$minimal, + 'webkit2' => \$webkit2, ); # Build usage text and options list from features @@ -239,7 +261,14 @@ setConfiguration(); my $productDir = productDir(); # Check that all the project directories are there. -my @projects = ("JavaScriptCore", "WebCore", "WebKit"); +my @projects = ("JavaScriptCore", "WebCore"); + +if (!$webkit2) { + push @projects, "WebKit"; +} else { + push @projects, ("WebKit2", "WebKitTools/MiniBrowser"); +} + # Only Apple builds JavaScriptGlue, and only on the Mac splice @projects, 1, 0, "JavaScriptGlue" if isAppleMacWebKit(); @@ -254,6 +283,7 @@ my @options = (); # enable autotool options accordingly if (isGtk()) { + @options = @ARGV; foreach (@features) { push @options, autotoolsFlag(${$_->{value}}, $_->{option}); } @@ -359,7 +389,18 @@ for my $dir (@projects) { } elsif (isQt()) { $result = buildQMakeQtProject($dir, $clean, @options); } elsif (isAppleMacWebKit()) { - $result = buildXCodeProject($dir, $clean, @options, @ARGV); + my @completeOptions = @options; + if ($webkit2 && $dir eq "WebCore") { + my @webKit2SpecificOverrides = ( + 'UMBRELLA_LDFLAGS=', + 'GCC_PREPROCESSOR_DEFINITIONS=$(GCC_PREPROCESSOR_DEFINITIONS) ' . + 'ENABLE_EXPERIMENTAL_SINGLE_VIEW_MODE=1 ' . + 'WTF_USE_WEB_THREAD=1 ' + ); + push @completeOptions, @webKit2SpecificOverrides; + } + + $result = buildXCodeProject($dir, $clean, @completeOptions, @ARGV); } elsif (isAppleWinWebKit()) { if ($dir eq "WebKit") { $result = buildVisualStudioProject("win/WebKit.vcproj/WebKit.sln", $clean); @@ -420,10 +461,16 @@ sub writeCongrats() print "\n"; print "===========================================================\n"; - print " WebKit is now built ($buildTime). \n"; - if (!isChromium()) { - print " To run $launcherName with this newly-built code, use the\n"; - print " \"$launcherPath\" script.\n"; + if ($webkit2) { + print " WebKit2 is now built ($buildTime). \n"; + print " To run MiniBrowser with this newly-built code, use the\n"; + print " \"run-minibrowser\" script.\n"; + } else { + print " WebKit is now built ($buildTime). \n"; + if (!isChromium()) { + print " To run $launcherName with this newly-built code, use the\n"; + print " \"$launcherPath\" script.\n"; + } } print "===========================================================\n"; } diff --git a/WebKitTools/Scripts/check-for-global-initializers b/WebKitTools/Scripts/check-for-global-initializers index cf83048..0472901 100755 --- a/WebKitTools/Scripts/check-for-global-initializers +++ b/WebKitTools/Scripts/check-for-global-initializers @@ -37,6 +37,7 @@ use strict; use File::Basename; sub touch($); +sub demangle($); my $arch = $ENV{'CURRENT_ARCH'}; my $configuration = $ENV{'CONFIGURATION'}; @@ -78,9 +79,14 @@ for my $file (sort @files) { next; } my $sawGlobal = 0; + my @globals; while (<NM>) { if (/^STDOUT:/) { - $sawGlobal = 1 if /__GLOBAL__I/; + my $line = $_; + if ($line =~ /__GLOBAL__I(.+)$/) { + $sawGlobal = 1; + push(@globals, demangle($1)); + } } else { print STDERR if $_ ne "nm: no name list\n"; } @@ -119,7 +125,7 @@ for my $file (sort @files) { } } - print "ERROR: $shortName has a global initializer in it! ($file)\n"; + print "ERROR: $shortName has one or more global initializers in it! ($file), near @globals\n"; $sawError = 1; } } @@ -138,3 +144,17 @@ sub touch($) open(TOUCH, ">", $path) or die "$!"; close(TOUCH); } + +sub demangle($) +{ + my ($symbol) = @_; + if (!open FILT, "c++filt $symbol |") { + print "ERROR: Could not open c++filt\n"; + return; + } + my $result = <FILT>; + close FILT; + chomp $result; + return $result; +} + diff --git a/WebKitTools/Scripts/check-for-inappropriate-files-in-framework b/WebKitTools/Scripts/check-for-inappropriate-files-in-framework new file mode 100755 index 0000000..2bc65d4 --- /dev/null +++ b/WebKitTools/Scripts/check-for-inappropriate-files-in-framework @@ -0,0 +1,66 @@ +#!/usr/bin/env ruby + +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +base_directory = ENV['TARGET_BUILD_DIR'] or throw "Unable to find TARGET_BUILD_DIR in the environment!" +project_name = ENV['PROJECT_NAME'] or throw "Unable to find PROJECT_NAME in the environment!" + +$INAPPROPRIATE_FILES = { "WebCore" => { "Resources" => ["*.css", "*.in", "*.idl"] } } + +Dir.chdir base_directory + +$error_printed = false + +def print_error msg + $error_printed = true + STDERR.puts "ERROR: #{msg}" +end + +def print_inappropriate_file_error framework, relative_path + print_error "#{framework}.framework/#{relative_path} should not be present in the framework." +end + +def check_framework framework + $INAPPROPRIATE_FILES[framework].each do |directory, patterns| + Dir.chdir "#{framework}.framework/Versions/A/#{directory}" do + patterns.each do |pattern| + Dir.glob(pattern).each do |inappropriate_file| + print_inappropriate_file_error framework, "Resources/#{inappropriate_file}" + File.unlink inappropriate_file + end + end + end + end +end + +check_framework project_name + +if $error_printed + STDERR.puts + STDERR.puts " Inappropriate files were detected and have been removed from the framework." + STDERR.puts " If this error continues to appear after building again then the build system needs" + STDERR.puts " to be modified so that the inappropriate files are no longer copied in to the framework." + STDERR.puts + exit 1 +end diff --git a/WebKitTools/Scripts/check-for-webkit-framework-include-consistency b/WebKitTools/Scripts/check-for-webkit-framework-include-consistency new file mode 100755 index 0000000..693dd6a --- /dev/null +++ b/WebKitTools/Scripts/check-for-webkit-framework-include-consistency @@ -0,0 +1,109 @@ +#!/usr/bin/env ruby + +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + + +base_directory = ENV['TARGET_BUILD_DIR'] + +unless base_directory + throw "Unable to find TARGET_BUILD_DIR in the environment!" +end + +Dir.chdir base_directory + +$PERMITTED_INCLUDE_TYPES = { :public => [ :public ], :private => [ :public, :private ] } + +$HEADER_NAMES_TO_TYPE = { } +$HEADERS_BY_TYPE = { :public => [], :private => [] } + +$error_printed = false + +def print_error msg + $error_printed = true + STDERR.puts "ERROR: #{msg}" +end + +def build_header_maps + all_headers = `find WebKit.framework/Versions/A/{,Private}Headers -type f -name '*.h'`.split + + all_headers.each do |header| + if /\/Headers\/(.*)/.match(header) + $HEADER_NAMES_TO_TYPE[$1] = :public + $HEADERS_BY_TYPE[:public] << header + elsif /\/PrivateHeaders\/(.*)/.match(header) + $HEADER_NAMES_TO_TYPE[$1] = :private + $HEADERS_BY_TYPE[:private] << header + else + print_error "Unknown header type: #{header}" + end + end +end + +def resolve_include(header, included_header, permitted_types) + # Ignore includes that aren't in the typical framework style. + return unless /<([^\/]+)\/(.*)>/.match(included_header) + + framework, included_header_name = [$1, $2] + + # Ignore includes that aren't related to other WebKit headers. + return unless framework =~ /^Web/ + + # A header of any type including a WebCore header is a recipe for disaster. + if framework == "WebCore" + # <rdar://problem/7718826> WebKeyGenerator.h should not include a WebCore header + return if header =~ /\/WebKeyGenerator.h$/ and included_header_name == "WebCoreKeyGenerator.h" + + print_error "#{header} included #{included_header}!" + return + end + + header_type = $HEADER_NAMES_TO_TYPE[included_header_name] + + if not header_type + print_error "#{header} included #{included_header} but I could not find a header of that name!" + elsif not permitted_types.member?(header_type) + print_error "#{header} included #{included_header} which is #{header_type}!" + end +end + +def verify_includes(header, permitted_types) + File.open(header) do |file| + file.each_line do |line| + if /#(include|import) (.*)/.match(line) + resolve_include(header, $2, permitted_types) + end + end + end +end + +build_header_maps + +$HEADERS_BY_TYPE.each do |header_type, headers| + permitted_types = $PERMITTED_INCLUDE_TYPES[header_type] + headers.each do |header| + verify_includes header, permitted_types + end +end + +exit 1 if $error_printed diff --git a/WebKitTools/Scripts/check-webkit-style b/WebKitTools/Scripts/check-webkit-style index ea2e943..9897fbd 100755 --- a/WebKitTools/Scripts/check-webkit-style +++ b/WebKitTools/Scripts/check-webkit-style @@ -43,51 +43,84 @@ same line, but it is far from perfect (in either direction). """ import codecs +import logging import os import os.path import sys +from webkitpy.style_references import detect_checkout import webkitpy.style.checker as checker -from webkitpy.style_references import SimpleScm +from webkitpy.style.checker import PatchChecker +from webkitpy.style.main import change_directory +_log = logging.getLogger("check-webkit-style") + + +# FIXME: Move this code to style.main. def main(): # Change stderr to write with replacement characters so we don't die # if we try to print something containing non-ASCII characters. - sys.stderr = codecs.StreamReaderWriter(sys.stderr, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace') + stderr = codecs.StreamReaderWriter(sys.stderr, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace') + # Setting an "encoding" attribute on the stream is necessary to + # prevent the logging module from raising an error. See + # the checker.configure_logging() function for more information. + stderr.encoding = "UTF-8" + + # FIXME: Change webkitpy.style so that we do not need to overwrite + # the global sys.stderr. This involves updating the code to + # accept a stream parameter where necessary, and not calling + # sys.stderr explicitly anywhere. + sys.stderr = stderr + + args = sys.argv[1:] + + # Checking for the verbose flag before calling check_webkit_style_parser() + # lets us enable verbose logging earlier. + is_verbose = "-v" in args or "--verbose" in args + + checker.configure_logging(stream=stderr, is_verbose=is_verbose) + _log.debug("Verbose logging enabled.") + + checkout = detect_checkout() + + if checkout is None: + checkout_root = None + _log.debug("WebKit checkout not found for current directory.") + else: + checkout_root = checkout.root_path() + _log.debug("WebKit checkout found with root: %s" % checkout_root) + parser = checker.check_webkit_style_parser() - (files, options) = parser.parse(sys.argv[1:]) + (paths, options) = parser.parse(args) + + if checkout is None and not paths: + _log.error("WebKit checkout not found: You must run this script " + "from within a WebKit checkout if you are not passing " + "specific paths to check.") + sys.exit(1) configuration = checker.check_webkit_style_configuration(options) style_checker = checker.StyleChecker(configuration) - if files: - for filename in files: - style_checker.check_file(filename) + paths = change_directory(checkout_root=checkout_root, paths=paths) + if paths: + style_checker.check_paths(paths) else: - scm = SimpleScm() - - os.chdir(scm.checkout_root()) - if options.git_commit: - commit = options.git_commit - if '..' in commit: - # FIXME: If the range is a "...", the code should find the common ancestor and - # start there (see git diff --help for information about how ... usually works). - commit = commit[:commit.find('..')] - print >> sys.stderr, "Warning: Ranges are not supported for --git-commit. Checking all changes since %s.\n" % commit - patch = scm.create_patch_since_local_commit(commit) + patch = checkout.create_patch_since_local_commit(options.git_commit) else: - patch = scm.create_patch() - style_checker.check_patch(patch) + patch = checkout.create_patch() + patch_checker = PatchChecker(style_checker) + patch_checker.check(patch) error_count = style_checker.error_count file_count = style_checker.file_count - sys.stderr.write('Total errors found: %d in %d files\n' - % (error_count, file_count)) + _log.info("Total errors found: %d in %d files" + % (error_count, file_count)) # We fail when style errors are found or there are no checked files. sys.exit(error_count > 0 or file_count == 0) diff --git a/WebKitTools/Scripts/commit-log-editor b/WebKitTools/Scripts/commit-log-editor index 75017e3..a642731 100755 --- a/WebKitTools/Scripts/commit-log-editor +++ b/WebKitTools/Scripts/commit-log-editor @@ -1,6 +1,6 @@ #!/usr/bin/perl -w -# Copyright (C) 2006, 2007, 2008, 2009 Apple Inc. All rights reserved. +# Copyright (C) 2006, 2007, 2008, 2009, 2010 Apple Inc. All rights reserved. # Copyright (C) 2009 Torch Mobile Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -40,6 +40,7 @@ use webkitdirs; sub normalizeLineEndings($$); sub removeLongestCommonPrefixEndingInDoubleNewline(\%); +sub isCommitLogEditor($); sub usage { @@ -61,30 +62,33 @@ if (!$log) { my $baseDir = baseProductDir(); my $editor = $ENV{SVN_LOG_EDITOR}; -if (!$editor) { +if (!$editor || isCommitLogEditor($editor)) { $editor = $ENV{CVS_LOG_EDITOR}; } -if (!$editor) { +if (!$editor || isCommitLogEditor($editor)) { my $builtEditorApplication = "$baseDir/Release/Commit Log Editor.app/Contents/MacOS/Commit Log Editor"; $editor = $builtEditorApplication if -x $builtEditorApplication; } -if (!$editor) { +if (!$editor || isCommitLogEditor($editor)) { my $builtEditorApplication = "$baseDir/Debug/Commit Log Editor.app/Contents/MacOS/Commit Log Editor"; $editor = $builtEditorApplication if -x $builtEditorApplication; } -if (!$editor) { +if (!$editor || isCommitLogEditor($editor)) { my $installedEditorApplication = "$ENV{HOME}/Applications/Commit Log Editor.app/Contents/MacOS/Commit Log Editor"; $editor = $installedEditorApplication if -x $installedEditorApplication; } -if (!$editor) { - $editor = $ENV{EDITOR} || "/usr/bin/vi"; +if (!$editor || isCommitLogEditor($editor)) { + $editor = $ENV{EDITOR}; +} +if (!$editor || isCommitLogEditor($editor)) { + $editor = "/usr/bin/vi"; } my $inChangesToBeCommitted = !isGit(); my @changeLogs = (); my $logContents = ""; my $existingLog = 0; -open LOG, $log or die; +open LOG, $log or die "Could not open the log file."; while (<LOG>) { if (isGit()) { if (/^# Changes to be committed:$/) { @@ -102,7 +106,7 @@ while (<LOG>) { } $existingLog = isGit() && !(/^#/ || /^\s*$/) unless $existingLog; - push @changeLogs, makeFilePathRelative($1) if $inChangesToBeCommitted && (/^M....(.*ChangeLog)\r?\n?$/ || /^#\tmodified: (.*ChangeLog)/) && !/-ChangeLog/; + push @changeLogs, makeFilePathRelative($1) if $inChangesToBeCommitted && (/^(?:M|A)....(.*ChangeLog)\r?\n?$/ || /^#\t(?:modified|new file): (.*ChangeLog)$/) && !/-ChangeLog$/; } close LOG; @@ -151,8 +155,8 @@ for my $changeLog (@changeLogs) { # Remove indentation spaces $line =~ s/^ {8}//; - # Save the reviewed by line - if ($line =~ m/^Reviewed by .*/) { + # Save the reviewed / rubber stamped by line. + if ($line =~ m/^Reviewed by .*/ || $line =~ m/^Rubber[ \-]?stamped by .*/) { $reviewedByLine = $line; next; } @@ -184,7 +188,6 @@ for my $changeLog (@changeLogs) { $reviewedByLine = ""; } - $lineCount++; $contents .= $line; } else { @@ -204,12 +207,6 @@ for my $changeLog (@changeLogs) { my $sortKey = lc $label; if ($label eq "top level") { $sortKey = ""; - } elsif ($label eq "Tools") { - $sortKey = "-, just after top level"; - } elsif ($label eq "WebBrowser") { - $sortKey = lc "WebKit, WebBrowser after"; - } elsif ($label eq "WebCore") { - $sortKey = lc "WebFoundation, WebCore after"; } elsif ($label eq "LayoutTests") { $sortKey = lc "~, LayoutTests last"; } @@ -307,3 +304,9 @@ sub removeLongestCommonPrefixEndingInDoubleNewline(\%) } return substr($prefix, 0, $lastDoubleNewline + 2); } + +sub isCommitLogEditor($) +{ + my $editor = shift; + return $editor =~ m/commit-log-editor/; +} diff --git a/WebKitTools/Scripts/debug-minibrowser b/WebKitTools/Scripts/debug-minibrowser new file mode 100755 index 0000000..06685b4 --- /dev/null +++ b/WebKitTools/Scripts/debug-minibrowser @@ -0,0 +1,38 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2005, 2007 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Simplified "debug" script for debugging the WebKit2 MiniBrowser. + +use strict; +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; + +setConfiguration(); + +exit exitStatus(debugMiniBrowser()); diff --git a/WebKitTools/Scripts/new-run-webkit-httpd b/WebKitTools/Scripts/new-run-webkit-httpd new file mode 100755 index 0000000..88ae84e --- /dev/null +++ b/WebKitTools/Scripts/new-run-webkit-httpd @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""A utility script for starting and stopping the HTTP server with the + same configuration as used in the layout tests.""" + +# +# FIXME: currently this code only works with the Chromium ports and LigHTTPd. +# It should be made to work on all ports. +# +# This script is also used by Chromium's ui_tests to run http layout tests +# in a browser. +# +import optparse +import os +import sys +import tempfile + +scripts_directory = os.path.dirname(os.path.abspath(sys.argv[0])) +webkitpy_directory = os.path.join(scripts_directory, "webkitpy") +sys.path.append(os.path.join(webkitpy_directory, "layout_tests")) + +import port +from port import http_server + +def run(options): + if not options.server: + print ('Usage: %s --server {start|stop} [--root=root_dir]' + ' [--port=port_number]' % sys.argv[0]) + else: + if (options.root is None) and (options.port is not None): + # specifying root but not port means we want httpd on default + # set of ports that LayoutTest use, but pointing to a different + # source of tests. Specifying port but no root does not seem + # meaningful. + raise 'Specifying port requires also a root.' + port_obj = port.get(None, options) + httpd = http_server.Lighttpd(port_obj, + tempfile.gettempdir(), + port=options.port, + root=options.root, + register_cygwin=options.register_cygwin, + run_background=options.run_background) + if options.server == 'start': + httpd.start() + else: + httpd.stop(force=True) + + +def main(): + option_parser = optparse.OptionParser() + option_parser.add_option('-k', '--server', + help='Server action (start|stop)') + option_parser.add_option('-p', '--port', + help='Port to listen on (overrides layout test ports)') + option_parser.add_option('-r', '--root', + help='Absolute path to DocumentRoot (overrides layout test roots)') + option_parser.add_option('--register_cygwin', action="store_true", + dest="register_cygwin", help='Register Cygwin paths (on Win try bots)') + option_parser.add_option('--run_background', action="store_true", + dest="run_background", + help='Run on background (for running as UI test)') + options, args = option_parser.parse_args() + + # FIXME: Make this work with other ports as well. + options.chromium = True + + run(options) + + +if '__main__' == __name__: + main() diff --git a/WebKitTools/Scripts/webkitpy/steps/commit.py b/WebKitTools/Scripts/new-run-webkit-tests index dd1fed7..2ebe1da 100644..100755 --- a/WebKitTools/Scripts/webkitpy/steps/commit.py +++ b/WebKitTools/Scripts/new-run-webkit-tests @@ -1,9 +1,10 @@ +#!/usr/bin/env python # Copyright (C) 2010 Google Inc. All rights reserved. -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above @@ -13,7 +14,7 @@ # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -26,10 +27,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep +"""Wrapper around webkitpy/layout_tests/run_webkit_tests.py""" +import sys +import webkitpy.layout_tests.run_webkit_tests as run_webkit_tests -class Commit(AbstractStep): - def run(self, state): - commit_message = self._tool.scm().commit_message_for_this_commit() - state["commit_text"] = self._tool.scm().commit_with_message(commit_message.message()) +if __name__ == '__main__': + options, args = run_webkit_tests.parse_args() + sys.exit(run_webkit_tests.main(options, args)) diff --git a/WebKitTools/Scripts/new-run-webkit-websocketserver b/WebKitTools/Scripts/new-run-webkit-websocketserver new file mode 100644 index 0000000..8e4aeaa --- /dev/null +++ b/WebKitTools/Scripts/new-run-webkit-websocketserver @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""A utility script for starting and stopping the web socket server with the + same configuration as used in the layout tests.""" + +import logging +import optparse +import tempfile + +import webkitpy.layout_tests.port.factory as factory +import webkitpy.layout_tests.port.websocket_server as websocket_server + + +def main(): + option_parser = optparse.OptionParser() + option_parser.add_option('--server', type='choice', + choices=['start', 'stop'], default='start', + help='Server action (start|stop)') + option_parser.add_option('-p', '--port', dest='port', + default=None, help='Port to listen on') + option_parser.add_option('-r', '--root', + help='Absolute path to DocumentRoot ' + '(overrides layout test roots)') + option_parser.add_option('-t', '--tls', dest='use_tls', + action='store_true', + default=False, help='use TLS (wss://)') + option_parser.add_option('-k', '--private_key', dest='private_key', + default='', help='TLS private key file.') + option_parser.add_option('-c', '--certificate', dest='certificate', + default='', help='TLS certificate file.') + option_parser.add_option('--register_cygwin', action="store_true", + dest="register_cygwin", + help='Register Cygwin paths (on Win try bots)') + option_parser.add_option('--pidfile', help='path to pid file.') + option_parser.add_option('-v', '--verbose', action='store_true', + default=False, help='include debug-level logging') + options, args = option_parser.parse_args() + + if not options.port: + if options.use_tls: + # FIXME: We shouldn't grab at this private variable. + options.port = websocket_server._DEFAULT_WSS_PORT + else: + # FIXME: We shouldn't grab at this private variable. + options.port = websocket_server._DEFAULT_WS_PORT + + kwds = {'port': options.port, 'use_tls': options.use_tls} + if options.root: + kwds['root'] = options.root + if options.private_key: + kwds['private_key'] = options.private_key + if options.certificate: + kwds['certificate'] = options.certificate + kwds['register_cygwin'] = options.register_cygwin + if options.pidfile: + kwds['pidfile'] = options.pidfile + + port_obj = factory.get() + pywebsocket = websocket_server.PyWebSocket(port_obj, tempfile.gettempdir(), **kwds) + + log_level = logging.WARN + if options.verbose: + log_level = logging.DEBUG + logging.basicConfig(level=log_level) + + if 'start' == options.server: + pywebsocket.start() + else: + pywebsocket.stop(force=True) + +if '__main__' == __name__: + main() diff --git a/WebKitTools/Scripts/old-run-webkit-tests b/WebKitTools/Scripts/old-run-webkit-tests new file mode 100755 index 0000000..d5d7349 --- /dev/null +++ b/WebKitTools/Scripts/old-run-webkit-tests @@ -0,0 +1,2281 @@ +#!/usr/bin/perl + +# Copyright (C) 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved. +# Copyright (C) 2006 Alexey Proskuryakov (ap@nypop.com) +# Copyright (C) 2007 Matt Lilek (pewtermoose@gmail.com) +# Copyright (C) 2007 Eric Seidel <eric@webkit.org> +# Copyright (C) 2009 Google Inc. All rights reserved. +# Copyright (C) 2009 Andras Becsi (becsi.andras@stud.u-szeged.hu), University of Szeged +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Script to run the WebKit Open Source Project layout tests. + +# Run all the tests passed in on the command line. +# If no tests are passed, find all the .html, .shtml, .xml, .xhtml, .xhtmlmp, .pl, .php (and svg) files in the test directory. + +# Run each text. +# Compare against the existing file xxx-expected.txt. +# If there is a mismatch, generate xxx-actual.txt and xxx-diffs.txt. + +# At the end, report: +# the number of tests that got the expected results +# the number of tests that ran, but did not get the expected results +# the number of tests that failed to run +# the number of tests that were run but had no expected results to compare against + +use strict; +use warnings; + +use Cwd; +use Data::Dumper; +use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK); +use File::Basename; +use File::Copy; +use File::Find; +use File::Path; +use File::Spec; +use File::Spec::Functions; +use File::Temp; +use FindBin; +use Getopt::Long; +use IPC::Open2; +use IPC::Open3; +use Time::HiRes qw(time usleep); + +use List::Util 'shuffle'; + +use lib $FindBin::Bin; +use webkitperl::features; +use webkitperl::httpd; +use webkitdirs; +use VCSUtils; +use POSIX; + +sub buildPlatformResultHierarchy(); +sub buildPlatformTestHierarchy(@); +sub closeCygpaths(); +sub closeDumpTool(); +sub closeWebSocketServer(); +sub configureAndOpenHTTPDIfNeeded(); +sub countAndPrintLeaks($$$); +sub countFinishedTest($$$$); +sub deleteExpectedAndActualResults($); +sub dumpToolDidCrash(); +sub epiloguesAndPrologues($$); +sub expectedDirectoryForTest($;$;$); +sub fileNameWithNumber($$); +sub htmlForResultsSection(\@$&); +sub isTextOnlyTest($); +sub launchWithEnv(\@\%); +sub resolveAndMakeTestResultsDirectory(); +sub numericcmp($$); +sub openDiffTool(); +sub openDumpTool(); +sub parseLeaksandPrintUniqueLeaks(); +sub openWebSocketServerIfNeeded(); +sub pathcmp($$); +sub printFailureMessageForTest($$); +sub processIgnoreTests($$); +sub readFromDumpToolWithTimer(**); +sub readSkippedFiles($); +sub recordActualResultsAndDiff($$); +sub sampleDumpTool(); +sub setFileHandleNonBlocking(*$); +sub slowestcmp($$); +sub splitpath($); +sub stopRunningTestsEarlyIfNeeded(); +sub stripExtension($); +sub stripMetrics($$); +sub testCrashedOrTimedOut($$$$$); +sub toURL($); +sub toWindowsPath($); +sub validateSkippedArg($$;$); +sub writeToFile($$); + +# Argument handling +my $addPlatformExceptions = 0; +my $complexText = 0; +my $exitAfterNFailures = 0; +my $generateNewResults = isAppleMacWebKit() ? 1 : 0; +my $guardMalloc = ''; +# FIXME: Dynamic HTTP-port configuration in this file is wrong. The various +# apache config files in LayoutTests/http/config govern the port numbers. +# Dynamic configuration as-written will also cause random failures in +# an IPv6 environment. See https://bugs.webkit.org/show_bug.cgi?id=37104. +my $httpdPort = 8000; +my $httpdSSLPort = 8443; +my $ignoreMetrics = 0; +my $webSocketPort = 8880; +# wss is disabled until all platforms support pyOpenSSL. +# my $webSocketSecurePort = 9323; +my $ignoreTests = ''; +my $iterations = 1; +my $launchSafari = 1; +my $mergeDepth; +my $pixelTests = ''; +my $platform; +my $quiet = ''; +my $randomizeTests = 0; +my $repeatEach = 1; +my $report10Slowest = 0; +my $resetResults = 0; +my $reverseTests = 0; +my $root; +my $runSample = 1; +my $shouldCheckLeaks = 0; +my $showHelp = 0; +my $stripEditingCallbacks = isCygwin(); +my $testHTTP = 1; +my $testMedia = 1; +my $tmpDir = "/tmp"; +my $testResultsDirectory = File::Spec->catfile($tmpDir, "layout-test-results"); +my $testsPerDumpTool = 1000; +my $threaded = 0; +# DumpRenderTree has an internal timeout of 30 seconds, so this must be > 30. +my $timeoutSeconds = 35; +my $tolerance = 0; +my $treatSkipped = "default"; +my $useRemoteLinksToTests = 0; +my $useValgrind = 0; +my $verbose = 0; +my $shouldWaitForHTTPD = 0; + +my @leaksFilenames; + +if (isWindows() || isMsys()) { + print "This script has to be run under Cygwin to function correctly.\n"; + exit 1; +} + +# Default to --no-http for wx for now. +$testHTTP = 0 if (isWx()); + +my $expectedTag = "expected"; +my $actualTag = "actual"; +my $prettyDiffTag = "pretty-diff"; +my $diffsTag = "diffs"; +my $errorTag = "stderr"; + +my @macPlatforms = ("mac-tiger", "mac-leopard", "mac-snowleopard", "mac"); + +if (isAppleMacWebKit()) { + if (isTiger()) { + $platform = "mac-tiger"; + $tolerance = 1.0; + } elsif (isLeopard()) { + $platform = "mac-leopard"; + $tolerance = 0.1; + } elsif (isSnowLeopard()) { + $platform = "mac-snowleopard"; + $tolerance = 0.1; + } else { + $platform = "mac"; + } +} elsif (isQt()) { + if (isDarwin()) { + $platform = "qt-mac"; + } elsif (isLinux()) { + $platform = "qt-linux"; + } elsif (isWindows() || isCygwin()) { + $platform = "qt-win"; + } else { + $platform = "qt"; + } +} elsif (isGtk()) { + $platform = "gtk"; + if (!$ENV{"WEBKIT_TESTFONTS"}) { + print "The WEBKIT_TESTFONTS environment variable is not defined.\n"; + print "You must set it before running the tests.\n"; + print "Use git to grab the actual fonts from http://gitorious.org/qtwebkit/testfonts\n"; + exit 1; + } +} elsif (isWx()) { + $platform = "wx"; +} elsif (isCygwin()) { + $platform = "win"; +} + +if (!defined($platform)) { + print "WARNING: Your platform is not recognized. Any platform-specific results will be generated in platform/undefined.\n"; + $platform = "undefined"; +} + +my $programName = basename($0); +my $launchSafariDefault = $launchSafari ? "launch" : "do not launch"; +my $httpDefault = $testHTTP ? "run" : "do not run"; +my $sampleDefault = $runSample ? "run" : "do not run"; + +my $usage = <<EOF; +Usage: $programName [options] [testdir|testpath ...] + --add-platform-exceptions Put new results for non-platform-specific failing tests into the platform-specific results directory + --complex-text Use the complex text code path for all text (Mac OS X and Windows only) + -c|--configuration config Set DumpRenderTree build configuration + -g|--guard-malloc Enable malloc guard + --exit-after-n-failures N Exit after the first N failures instead of running all tests + -h|--help Show this help message + --[no-]http Run (or do not run) http tests (default: $httpDefault) + --[no-]wait-for-httpd Wait for httpd if some other test session is using it already (same as WEBKIT_WAIT_FOR_HTTPD=1). (default: $shouldWaitForHTTPD) + -i|--ignore-tests Comma-separated list of directories or tests to ignore + --iterations n Number of times to run the set of tests (e.g. ABCABCABC) + --[no-]launch-safari Launch (or do not launch) Safari to display test results (default: $launchSafariDefault) + -l|--leaks Enable leaks checking + --[no-]new-test-results Generate results for new tests + --nthly n Restart DumpRenderTree every n tests (default: $testsPerDumpTool) + -p|--pixel-tests Enable pixel tests + --tolerance t Ignore image differences less than this percentage (default: $tolerance) + --platform Override the detected platform to use for tests and results (default: $platform) + --port Web server port to use with http tests + -q|--quiet Less verbose output + --reset-results Reset ALL results (including pixel tests if --pixel-tests is set) + -o|--results-directory Output results directory (default: $testResultsDirectory) + --random Run the tests in a random order + --repeat-each n Number of times to run each test (e.g. AAABBBCCC) + --reverse Run the tests in reverse alphabetical order + --root Path to root tools build + --[no-]sample-on-timeout Run sample on timeout (default: $sampleDefault) (Mac OS X only) + -1|--singly Isolate each test case run (implies --nthly 1 --verbose) + --skipped=[default|ignore|only] Specifies how to treat the Skipped file + default: Tests/directories listed in the Skipped file are not tested + ignore: The Skipped file is ignored + only: Only those tests/directories listed in the Skipped file will be run + --slowest Report the 10 slowest tests + --ignore-metrics Ignore metrics in tests + --[no-]strip-editing-callbacks Remove editing callbacks from expected results + -t|--threaded Run a concurrent JavaScript thead with each test + --timeout t Sets the number of seconds before a test times out (default: $timeoutSeconds) + --valgrind Run DumpRenderTree inside valgrind (Qt/Linux only) + -v|--verbose More verbose output (overrides --quiet) + -m|--merge-leak-depth arg Merges leak callStacks and prints the number of unique leaks beneath a callstack depth of arg. Defaults to 5. + --use-remote-links-to-tests Link to test files within the SVN repository in the results. +EOF + +setConfiguration(); + +my $getOptionsResult = GetOptions( + 'add-platform-exceptions' => \$addPlatformExceptions, + 'complex-text' => \$complexText, + 'exit-after-n-failures=i' => \$exitAfterNFailures, + 'guard-malloc|g' => \$guardMalloc, + 'help|h' => \$showHelp, + 'http!' => \$testHTTP, + 'wait-for-httpd!' => \$shouldWaitForHTTPD, + 'ignore-metrics!' => \$ignoreMetrics, + 'ignore-tests|i=s' => \$ignoreTests, + 'iterations=i' => \$iterations, + 'launch-safari!' => \$launchSafari, + 'leaks|l' => \$shouldCheckLeaks, + 'merge-leak-depth|m:5' => \$mergeDepth, + 'new-test-results!' => \$generateNewResults, + 'nthly=i' => \$testsPerDumpTool, + 'pixel-tests|p' => \$pixelTests, + 'platform=s' => \$platform, + 'port=i' => \$httpdPort, + 'quiet|q' => \$quiet, + 'random' => \$randomizeTests, + 'repeat-each=i' => \$repeatEach, + 'reset-results' => \$resetResults, + 'results-directory|o=s' => \$testResultsDirectory, + 'reverse' => \$reverseTests, + 'root=s' => \$root, + 'sample-on-timeout!' => \$runSample, + 'singly|1' => sub { $testsPerDumpTool = 1; }, + 'skipped=s' => \&validateSkippedArg, + 'slowest' => \$report10Slowest, + 'strip-editing-callbacks!' => \$stripEditingCallbacks, + 'threaded|t' => \$threaded, + 'timeout=i' => \$timeoutSeconds, + 'tolerance=f' => \$tolerance, + 'use-remote-links-to-tests' => \$useRemoteLinksToTests, + 'valgrind' => \$useValgrind, + 'verbose|v' => \$verbose, +); + +if (!$getOptionsResult || $showHelp) { + print STDERR $usage; + exit 1; +} + +my $ignoreSkipped = $treatSkipped eq "ignore"; +my $skippedOnly = $treatSkipped eq "only"; + +my $configuration = configuration(); + +# We need an environment variable to be able to enable the feature per-slave +$shouldWaitForHTTPD = $ENV{"WEBKIT_WAIT_FOR_HTTPD"} unless ($shouldWaitForHTTPD); +$verbose = 1 if $testsPerDumpTool == 1; + +if ($shouldCheckLeaks && $testsPerDumpTool > 1000) { + print STDERR "\nWARNING: Running more than 1000 tests at a time with MallocStackLogging enabled may cause a crash.\n\n"; +} + +# Stack logging does not play well with QuickTime on Tiger (rdar://problem/5537157) +$testMedia = 0 if $shouldCheckLeaks && isTiger(); + +# Generating remote links causes a lot of unnecessary spew on GTK build bot +$useRemoteLinksToTests = 0 if isGtk(); + +setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root)); +my $productDir = productDir(); +$productDir .= "/bin" if isQt(); +$productDir .= "/Programs" if isGtk(); + +chdirWebKit(); + +if (!defined($root)) { + print STDERR "Running build-dumprendertree\n"; + + local *DEVNULL; + my ($childIn, $childOut, $childErr); + if ($quiet) { + open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null"; + $childOut = ">&DEVNULL"; + $childErr = ">&DEVNULL"; + } else { + # When not quiet, let the child use our stdout/stderr. + $childOut = ">&STDOUT"; + $childErr = ">&STDERR"; + } + + my @args = argumentsForConfiguration(); + my $buildProcess = open3($childIn, $childOut, $childErr, "WebKitTools/Scripts/build-dumprendertree", @args) or die "Failed to run build-dumprendertree"; + close($childIn); + waitpid $buildProcess, 0; + my $buildResult = $?; + close($childOut); + close($childErr); + + close DEVNULL if ($quiet); + + if ($buildResult) { + print STDERR "Compiling DumpRenderTree failed!\n"; + exit exitStatus($buildResult); + } +} + +my $dumpToolName = "DumpRenderTree"; +$dumpToolName .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/; +my $dumpTool = "$productDir/$dumpToolName"; +die "can't find executable $dumpToolName (looked in $productDir)\n" unless -x $dumpTool; + +my $imageDiffTool = "$productDir/ImageDiff"; +$imageDiffTool .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/; +die "can't find executable $imageDiffTool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool; + +checkFrameworks() unless isCygwin(); + +if (isAppleMacWebKit()) { + push @INC, $productDir; + require DumpRenderTreeSupport; +} + +my $layoutTestsName = "LayoutTests"; +my $testDirectory = File::Spec->rel2abs($layoutTestsName); +my $expectedDirectory = $testDirectory; +my $platformBaseDirectory = catdir($testDirectory, "platform"); +my $platformTestDirectory = catdir($platformBaseDirectory, $platform); +my @platformResultHierarchy = buildPlatformResultHierarchy(); +my @platformTestHierarchy = buildPlatformTestHierarchy(@platformResultHierarchy); + +$expectedDirectory = $ENV{"WebKitExpectedTestResultsDirectory"} if $ENV{"WebKitExpectedTestResultsDirectory"}; + +$testResultsDirectory = File::Spec->rel2abs($testResultsDirectory); +my $testResults = File::Spec->catfile($testResultsDirectory, "results.html"); + +if (isAppleMacWebKit()) { + print STDERR "Compiling Java tests\n"; + my $javaTestsDirectory = catdir($testDirectory, "java"); + + if (system("/usr/bin/make", "-C", "$javaTestsDirectory")) { + exit 1; + } +} + + +print "Running tests from $testDirectory\n"; +if ($pixelTests) { + print "Enabling pixel tests with a tolerance of $tolerance%\n"; + if (isDarwin()) { + print "WARNING: Temporarily changing the main display color profile:\n"; + print "\tThe colors on your screen will change for the duration of the testing.\n"; + print "\tThis allows the pixel tests to have consistent color values across all machines.\n"; + + if (isPerianInstalled()) { + print "WARNING: Perian's QuickTime component is installed and this may affect pixel test results!\n"; + print "\tYou should avoid generating new pixel results in this environment.\n"; + print "\tSee https://bugs.webkit.org/show_bug.cgi?id=22615 for details.\n"; + } + } +} + +system "ln", "-s", $testDirectory, "/tmp/LayoutTests" unless -x "/tmp/LayoutTests"; + +my %ignoredFiles = ( "results.html" => 1 ); +my %ignoredDirectories = map { $_ => 1 } qw(platform); +my %ignoredLocalDirectories = map { $_ => 1 } qw(.svn _svn resources script-tests); +my %supportedFileExtensions = map { $_ => 1 } qw(html shtml xml xhtml xhtmlmp pl php); + +if (!checkWebCoreFeatureSupport("MathML", 0)) { + $ignoredDirectories{'mathml'} = 1; +} + +# FIXME: We should fix webkitperl/features.pm:hasFeature() to do the correct feature detection for Cygwin. +if (checkWebCoreFeatureSupport("SVG", 0)) { + $supportedFileExtensions{'svg'} = 1; +} elsif (isCygwin()) { + $supportedFileExtensions{'svg'} = 1; +} else { + $ignoredLocalDirectories{'svg'} = 1; +} + +if (!$testHTTP) { + $ignoredDirectories{'http'} = 1; + $ignoredDirectories{'websocket'} = 1; +} + +if (!$testMedia) { + $ignoredDirectories{'media'} = 1; + $ignoredDirectories{'http/tests/media'} = 1; +} + +my $supportedFeaturesResult = ""; + +if (isCygwin()) { + # Collect supported features list + setPathForRunningWebKitApp(\%ENV); + my $supportedFeaturesCommand = $dumpTool . " --print-supported-features 2>&1"; + $supportedFeaturesResult = `$supportedFeaturesCommand 2>&1`; +} + +my $hasAcceleratedCompositing = 0; +my $has3DRendering = 0; + +if (isCygwin()) { + $hasAcceleratedCompositing = $supportedFeaturesResult =~ /AcceleratedCompositing/; + $has3DRendering = $supportedFeaturesResult =~ /3DRendering/; +} else { + $hasAcceleratedCompositing = checkWebCoreFeatureSupport("Accelerated Compositing", 0); + $has3DRendering = checkWebCoreFeatureSupport("3D Rendering", 0); +} + +if (!$hasAcceleratedCompositing) { + $ignoredDirectories{'compositing'} = 1; +} + +if (!$has3DRendering) { + $ignoredDirectories{'animations/3d'} = 1; + $ignoredDirectories{'transforms/3d'} = 1; +} + +if (!checkWebCoreFeatureSupport("3D Canvas", 0)) { + $ignoredDirectories{'fast/canvas/webgl'} = 1; +} + +if (checkWebCoreFeatureSupport("WML", 0)) { + $supportedFileExtensions{'wml'} = 1; +} else { + $ignoredDirectories{'http/tests/wml'} = 1; + $ignoredDirectories{'fast/wml'} = 1; + $ignoredDirectories{'wml'} = 1; +} + +if (!checkWebCoreFeatureSupport("XHTMLMP", 0)) { + $ignoredDirectories{'fast/xhtmlmp'} = 1; +} + +processIgnoreTests($ignoreTests, "ignore-tests") if $ignoreTests; +if (!$ignoreSkipped) { + if (!$skippedOnly || @ARGV == 0) { + readSkippedFiles(""); + } else { + # Since readSkippedFiles() appends to @ARGV, we must use a foreach + # loop so that we only iterate over the original argument list. + foreach my $argnum (0 .. $#ARGV) { + readSkippedFiles(shift @ARGV); + } + } +} + +my @tests = findTestsToRun(); + +die "no tests to run\n" if !@tests; + +my %counts; +my %tests; +my %imagesPresent; +my %imageDifferences; +my %durations; +my $count = 0; +my $leaksOutputFileNumber = 1; +my $totalLeaks = 0; + +my @toolArgs = (); +push @toolArgs, "--pixel-tests" if $pixelTests; +push @toolArgs, "--threaded" if $threaded; +push @toolArgs, "--complex-text" if $complexText; +push @toolArgs, "-"; + +my @diffToolArgs = (); +push @diffToolArgs, "--tolerance", $tolerance; + +$| = 1; + +my $dumpToolPID; +my $isDumpToolOpen = 0; +my $dumpToolCrashed = 0; +my $imageDiffToolPID; +my $isDiffToolOpen = 0; + +my $atLineStart = 1; +my $lastDirectory = ""; + +my $isHttpdOpen = 0; +my $isWebSocketServerOpen = 0; +my $webSocketServerPID = 0; +my $failedToStartWebSocketServer = 0; +# wss is disabled until all platforms support pyOpenSSL. +# my $webSocketSecureServerPID = 0; + +sub catch_pipe { $dumpToolCrashed = 1; } +$SIG{"PIPE"} = "catch_pipe"; + +print "Testing ", scalar @tests, " test cases"; +print " $iterations times" if ($iterations > 1); +print ", repeating each test $repeatEach times" if ($repeatEach > 1); +print ".\n"; + +my $overallStartTime = time; + +my %expectedResultPaths; + +my @originalTests = @tests; +# Add individual test repetitions +if ($repeatEach > 1) { + @tests = (); + foreach my $test (@originalTests) { + for (my $i = 0; $i < $repeatEach; $i++) { + push(@tests, $test); + } + } +} +# Add test set repetitions +for (my $i = 1; $i < $iterations; $i++) { + push(@tests, @originalTests); +} + +for my $test (@tests) { + my $newDumpTool = not $isDumpToolOpen; + openDumpTool(); + + my $base = stripExtension($test); + my $expectedExtension = ".txt"; + + my $dir = $base; + $dir =~ s|/[^/]+$||; + + if ($newDumpTool || $dir ne $lastDirectory) { + foreach my $logue (epiloguesAndPrologues($newDumpTool ? "" : $lastDirectory, $dir)) { + if (isCygwin()) { + $logue = toWindowsPath($logue); + } else { + $logue = canonpath($logue); + } + if ($verbose) { + print "running epilogue or prologue $logue\n"; + } + print OUT "$logue\n"; + # Throw away output from DumpRenderTree. + # Once for the test output and once for pixel results (empty) + while (<IN>) { + last if /#EOF/; + } + while (<IN>) { + last if /#EOF/; + } + } + } + + if ($verbose) { + print "running $test -> "; + $atLineStart = 0; + } elsif (!$quiet) { + if ($dir ne $lastDirectory) { + print "\n" unless $atLineStart; + print "$dir "; + } + print "."; + $atLineStart = 0; + } + + $lastDirectory = $dir; + + my $result; + + my $startTime = time if $report10Slowest; + + # Try to read expected hash file for pixel tests + my $suffixExpectedHash = ""; + if ($pixelTests && !$resetResults) { + my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); + if (open EXPECTEDHASH, "$expectedPixelDir/$base-$expectedTag.checksum") { + my $expectedHash = <EXPECTEDHASH>; + chomp($expectedHash); + close EXPECTEDHASH; + + # Format expected hash into a suffix string that is appended to the path / URL passed to DRT + $suffixExpectedHash = "'$expectedHash"; + } + } + + if ($test =~ /^http\//) { + configureAndOpenHTTPDIfNeeded(); + if ($test !~ /^http\/tests\/local\// && $test !~ /^http\/tests\/ssl\// && $test !~ /^http\/tests\/wml\// && $test !~ /^http\/tests\/media\//) { + my $path = canonpath($test); + $path =~ s/^http\/tests\///; + print OUT "http://127.0.0.1:$httpdPort/$path$suffixExpectedHash\n"; + } elsif ($test =~ /^http\/tests\/ssl\//) { + my $path = canonpath($test); + $path =~ s/^http\/tests\///; + print OUT "https://127.0.0.1:$httpdSSLPort/$path$suffixExpectedHash\n"; + } else { + my $testPath = "$testDirectory/$test"; + if (isCygwin()) { + $testPath = toWindowsPath($testPath); + } else { + $testPath = canonpath($testPath); + } + print OUT "$testPath$suffixExpectedHash\n"; + } + } elsif ($test =~ /^websocket\//) { + if ($test =~ /^websocket\/tests\/local\//) { + my $testPath = "$testDirectory/$test"; + if (isCygwin()) { + $testPath = toWindowsPath($testPath); + } else { + $testPath = canonpath($testPath); + } + print OUT "$testPath\n"; + } else { + if (openWebSocketServerIfNeeded()) { + my $path = canonpath($test); + if ($test =~ /^websocket\/tests\/ssl\//) { + # wss is disabled until all platforms support pyOpenSSL. + print STDERR "Error: wss is disabled until all platforms support pyOpenSSL."; + # print OUT "https://127.0.0.1:$webSocketSecurePort/$path\n"; + } else { + print OUT "http://127.0.0.1:$webSocketPort/$path\n"; + } + } else { + # We failed to launch the WebSocket server. Display a useful error message rather than attempting + # to run tests that expect the server to be available. + my $errorMessagePath = "$testDirectory/websocket/resources/server-failed-to-start.html"; + $errorMessagePath = isCygwin() ? toWindowsPath($errorMessagePath) : canonpath($errorMessagePath); + print OUT "$errorMessagePath\n"; + } + } + } else { + my $testPath = "$testDirectory/$test"; + if (isCygwin()) { + $testPath = toWindowsPath($testPath); + } else { + $testPath = canonpath($testPath); + } + print OUT "$testPath$suffixExpectedHash\n" if defined $testPath; + } + + # DumpRenderTree is expected to dump two "blocks" to stdout for each test. + # Each block is terminated by a #EOF on a line by itself. + # The first block is the output of the test (in text, RenderTree or other formats). + # The second block is for optional pixel data in PNG format, and may be empty if + # pixel tests are not being run, or the test does not dump pixels (e.g. text tests). + my $readResults = readFromDumpToolWithTimer(IN, ERROR); + + my $actual = $readResults->{output}; + my $error = $readResults->{error}; + + $expectedExtension = $readResults->{extension}; + my $expectedFileName = "$base-$expectedTag.$expectedExtension"; + + my $isText = isTextOnlyTest($actual); + + my $expectedDir = expectedDirectoryForTest($base, $isText, $expectedExtension); + $expectedResultPaths{$base} = "$expectedDir/$expectedFileName"; + + unless ($readResults->{status} eq "success") { + my $crashed = $readResults->{status} eq "crashed"; + testCrashedOrTimedOut($test, $base, $crashed, $actual, $error); + countFinishedTest($test, $base, $crashed ? "crash" : "timedout", 0); + last if stopRunningTestsEarlyIfNeeded(); + next; + } + + $durations{$test} = time - $startTime if $report10Slowest; + + my $expected; + + if (!$resetResults && open EXPECTED, "<", "$expectedDir/$expectedFileName") { + $expected = ""; + while (<EXPECTED>) { + next if $stripEditingCallbacks && $_ =~ /^EDITING DELEGATE:/; + $expected .= $_; + } + close EXPECTED; + } + + if ($ignoreMetrics && !$isText && defined $expected) { + ($actual, $expected) = stripMetrics($actual, $expected); + } + + if ($shouldCheckLeaks && $testsPerDumpTool == 1) { + print " $test -> "; + } + + my $actualPNG = ""; + my $diffPNG = ""; + my $diffPercentage = 0; + my $diffResult = "passed"; + + my $actualHash = ""; + my $expectedHash = ""; + my $actualPNGSize = 0; + + while (<IN>) { + last if /#EOF/; + if (/ActualHash: ([a-f0-9]{32})/) { + $actualHash = $1; + } elsif (/ExpectedHash: ([a-f0-9]{32})/) { + $expectedHash = $1; + } elsif (/Content-Length: (\d+)\s*/) { + $actualPNGSize = $1; + read(IN, $actualPNG, $actualPNGSize); + } + } + + if ($verbose && $pixelTests && !$resetResults && $actualPNGSize) { + if ($actualHash eq "" && $expectedHash eq "") { + printFailureMessageForTest($test, "WARNING: actual & expected pixel hashes are missing!"); + } elsif ($actualHash eq "") { + printFailureMessageForTest($test, "WARNING: actual pixel hash is missing!"); + } elsif ($expectedHash eq "") { + printFailureMessageForTest($test, "WARNING: expected pixel hash is missing!"); + } + } + + if ($actualPNGSize > 0) { + my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); + + if (!$resetResults && ($expectedHash ne $actualHash || ($actualHash eq "" && $expectedHash eq ""))) { + if (-f "$expectedPixelDir/$base-$expectedTag.png") { + my $expectedPNGSize = -s "$expectedPixelDir/$base-$expectedTag.png"; + my $expectedPNG = ""; + open EXPECTEDPNG, "$expectedPixelDir/$base-$expectedTag.png"; + read(EXPECTEDPNG, $expectedPNG, $expectedPNGSize); + + openDiffTool(); + print DIFFOUT "Content-Length: $actualPNGSize\n"; + print DIFFOUT $actualPNG; + + print DIFFOUT "Content-Length: $expectedPNGSize\n"; + print DIFFOUT $expectedPNG; + + while (<DIFFIN>) { + last if /^error/ || /^diff:/; + if (/Content-Length: (\d+)\s*/) { + read(DIFFIN, $diffPNG, $1); + } + } + + if (/^diff: (.+)% (passed|failed)/) { + $diffPercentage = $1 + 0; + $imageDifferences{$base} = $diffPercentage; + $diffResult = $2; + } + + if (!$diffPercentage) { + printFailureMessageForTest($test, "pixel hash failed (but pixel test still passes)"); + } + } elsif ($verbose) { + printFailureMessageForTest($test, "WARNING: expected image is missing!"); + } + } + + if ($resetResults || !-f "$expectedPixelDir/$base-$expectedTag.png") { + mkpath catfile($expectedPixelDir, dirname($base)) if $testDirectory ne $expectedPixelDir; + writeToFile("$expectedPixelDir/$base-$expectedTag.png", $actualPNG); + } + + if ($actualHash ne "" && ($resetResults || !-f "$expectedPixelDir/$base-$expectedTag.checksum")) { + writeToFile("$expectedPixelDir/$base-$expectedTag.checksum", $actualHash); + } + } + + if (dumpToolDidCrash()) { + $result = "crash"; + testCrashedOrTimedOut($test, $base, 1, $actual, $error); + } elsif (!defined $expected) { + if ($verbose) { + print "new " . ($resetResults ? "result" : "test") ."\n"; + $atLineStart = 1; + } + $result = "new"; + + if ($generateNewResults || $resetResults) { + mkpath catfile($expectedDir, dirname($base)) if $testDirectory ne $expectedDir; + writeToFile("$expectedDir/$expectedFileName", $actual); + } + deleteExpectedAndActualResults($base); + recordActualResultsAndDiff($base, $actual); + if (!$resetResults) { + # Always print the file name for new tests, as they will probably need some manual inspection. + # in verbose mode we already printed the test case, so no need to do it again. + unless ($verbose) { + print "\n" unless $atLineStart; + print "$test -> "; + } + my $resultsDir = catdir($expectedDir, dirname($base)); + if ($generateNewResults) { + print "new (results generated in $resultsDir)\n"; + } else { + print "new\n"; + } + $atLineStart = 1; + } + } elsif ($actual eq $expected && $diffResult eq "passed") { + if ($verbose) { + print "succeeded\n"; + $atLineStart = 1; + } + $result = "match"; + deleteExpectedAndActualResults($base); + } else { + $result = "mismatch"; + + my $pixelTestFailed = $pixelTests && $diffPNG && $diffPNG ne ""; + my $testFailed = $actual ne $expected; + + my $message = !$testFailed ? "pixel test failed" : "failed"; + + if (($testFailed || $pixelTestFailed) && $addPlatformExceptions) { + my $testBase = catfile($testDirectory, $base); + my $expectedBase = catfile($expectedDir, $base); + my $testIsMaximallyPlatformSpecific = $testBase =~ m|^\Q$platformTestDirectory\E/|; + my $expectedResultIsMaximallyPlatformSpecific = $expectedBase =~ m|^\Q$platformTestDirectory\E/|; + if (!$testIsMaximallyPlatformSpecific && !$expectedResultIsMaximallyPlatformSpecific) { + mkpath catfile($platformTestDirectory, dirname($base)); + if ($testFailed) { + my $expectedFile = catfile($platformTestDirectory, "$expectedFileName"); + writeToFile("$expectedFile", $actual); + } + if ($pixelTestFailed) { + my $expectedFile = catfile($platformTestDirectory, "$base-$expectedTag.checksum"); + writeToFile("$expectedFile", $actualHash); + + $expectedFile = catfile($platformTestDirectory, "$base-$expectedTag.png"); + writeToFile("$expectedFile", $actualPNG); + } + $message .= " (results generated in $platformTestDirectory)"; + } + } + + printFailureMessageForTest($test, $message); + + my $dir = "$testResultsDirectory/$base"; + $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; + my $testName = $1; + mkpath $dir; + + deleteExpectedAndActualResults($base); + recordActualResultsAndDiff($base, $actual); + + if ($pixelTestFailed) { + $imagesPresent{$base} = 1; + + writeToFile("$testResultsDirectory/$base-$actualTag.png", $actualPNG); + writeToFile("$testResultsDirectory/$base-$diffsTag.png", $diffPNG); + + my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); + copy("$expectedPixelDir/$base-$expectedTag.png", "$testResultsDirectory/$base-$expectedTag.png"); + + open DIFFHTML, ">$testResultsDirectory/$base-$diffsTag.html" or die; + print DIFFHTML "<html>\n"; + print DIFFHTML "<head>\n"; + print DIFFHTML "<title>$base Image Compare</title>\n"; + print DIFFHTML "<script language=\"Javascript\" type=\"text/javascript\">\n"; + print DIFFHTML "var currentImage = 0;\n"; + print DIFFHTML "var imageNames = new Array(\"Actual\", \"Expected\");\n"; + print DIFFHTML "var imagePaths = new Array(\"$testName-$actualTag.png\", \"$testName-$expectedTag.png\");\n"; + if (-f "$testDirectory/$base-w3c.png") { + copy("$testDirectory/$base-w3c.png", "$testResultsDirectory/$base-w3c.png"); + print DIFFHTML "imageNames.push(\"W3C\");\n"; + print DIFFHTML "imagePaths.push(\"$testName-w3c.png\");\n"; + } + print DIFFHTML "function animateImage() {\n"; + print DIFFHTML " var image = document.getElementById(\"animatedImage\");\n"; + print DIFFHTML " var imageText = document.getElementById(\"imageText\");\n"; + print DIFFHTML " image.src = imagePaths[currentImage];\n"; + print DIFFHTML " imageText.innerHTML = imageNames[currentImage] + \" Image\";\n"; + print DIFFHTML " currentImage = (currentImage + 1) % imageNames.length;\n"; + print DIFFHTML " setTimeout('animateImage()',2000);\n"; + print DIFFHTML "}\n"; + print DIFFHTML "</script>\n"; + print DIFFHTML "</head>\n"; + print DIFFHTML "<body onLoad=\"animateImage();\">\n"; + print DIFFHTML "<table>\n"; + if ($diffPercentage) { + print DIFFHTML "<tr>\n"; + print DIFFHTML "<td>Difference between images: <a href=\"$testName-$diffsTag.png\">$diffPercentage%</a></td>\n"; + print DIFFHTML "</tr>\n"; + } + print DIFFHTML "<tr>\n"; + print DIFFHTML "<td><a href=\"" . toURL("$testDirectory/$test") . "\">test file</a></td>\n"; + print DIFFHTML "</tr>\n"; + print DIFFHTML "<tr>\n"; + print DIFFHTML "<td id=\"imageText\" style=\"text-weight: bold;\">Actual Image</td>\n"; + print DIFFHTML "</tr>\n"; + print DIFFHTML "<tr>\n"; + print DIFFHTML "<td><img src=\"$testName-$actualTag.png\" id=\"animatedImage\"></td>\n"; + print DIFFHTML "</tr>\n"; + print DIFFHTML "</table>\n"; + print DIFFHTML "</body>\n"; + print DIFFHTML "</html>\n"; + } + } + + if ($error) { + my $dir = "$testResultsDirectory/$base"; + $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; + mkpath $dir; + + writeToFile("$testResultsDirectory/$base-$errorTag.txt", $error); + + $counts{error}++; + push @{$tests{error}}, $test; + } + + countFinishedTest($test, $base, $result, $isText); + last if stopRunningTestsEarlyIfNeeded(); +} + +my $totalTestingTime = time - $overallStartTime; +my $waitTime = getWaitTime(); +if ($waitTime > 0.1) { + my $normalizedTestingTime = $totalTestingTime - $waitTime; + printf "\n%0.2fs HTTPD waiting time\n", $waitTime . ""; + printf "%0.2fs normalized testing time", $normalizedTestingTime . ""; +} +printf "\n%0.2fs total testing time\n", $totalTestingTime . ""; + +!$isDumpToolOpen || die "Failed to close $dumpToolName.\n"; + +$isHttpdOpen = !closeHTTPD(); +closeWebSocketServer(); + +# Because multiple instances of this script are running concurrently we cannot +# safely delete this symlink. +# system "rm /tmp/LayoutTests"; + +# FIXME: Do we really want to check the image-comparison tool for leaks every time? +if ($isDiffToolOpen && $shouldCheckLeaks) { + $totalLeaks += countAndPrintLeaks("ImageDiff", $imageDiffToolPID, "$testResultsDirectory/ImageDiff-leaks.txt"); +} + +if ($totalLeaks) { + if ($mergeDepth) { + parseLeaksandPrintUniqueLeaks(); + } else { + print "\nWARNING: $totalLeaks total leaks found!\n"; + print "See above for individual leaks results.\n" if ($leaksOutputFileNumber > 2); + } +} + +close IN; +close OUT; +close ERROR; + +if ($report10Slowest) { + print "\n\nThe 10 slowest tests:\n\n"; + my $count = 0; + for my $test (sort slowestcmp keys %durations) { + printf "%0.2f secs: %s\n", $durations{$test}, $test; + last if ++$count == 10; + } +} + +print "\n"; + +if ($skippedOnly && $counts{"match"}) { + print "The following tests are in the Skipped file (" . File::Spec->abs2rel("$platformTestDirectory/Skipped", $testDirectory) . "), but succeeded:\n"; + foreach my $test (@{$tests{"match"}}) { + print " $test\n"; + } +} + +if ($resetResults || ($counts{match} && $counts{match} == $count)) { + print "all $count test cases succeeded\n"; + unlink $testResults; + exit; +} + +printResults(); + +mkpath $testResultsDirectory; + +open HTML, ">", $testResults or die "Failed to open $testResults. $!"; +print HTML "<html>\n"; +print HTML "<head>\n"; +print HTML "<title>Layout Test Results</title>\n"; +print HTML "</head>\n"; +print HTML "<body>\n"; + +if ($ignoreMetrics) { + print HTML "<h4>Tested with metrics ignored.</h4>"; +} + +print HTML htmlForResultsSection(@{$tests{mismatch}}, "Tests where results did not match expected results", \&linksForMismatchTest); +print HTML htmlForResultsSection(@{$tests{timedout}}, "Tests that timed out", \&linksForErrorTest); +print HTML htmlForResultsSection(@{$tests{crash}}, "Tests that caused the DumpRenderTree tool to crash", \&linksForErrorTest); +print HTML htmlForResultsSection(@{$tests{error}}, "Tests that had stderr output", \&linksForErrorTest); +print HTML htmlForResultsSection(@{$tests{new}}, "Tests that had no expected results (probably new)", \&linksForNewTest); + +print HTML "</body>\n"; +print HTML "</html>\n"; +close HTML; + +my @configurationArgs = argumentsForConfiguration(); + +if (isGtk()) { + system "WebKitTools/Scripts/run-launcher", @configurationArgs, "file://".$testResults if $launchSafari; +} elsif (isQt()) { + unshift @configurationArgs, qw(-graphicssystem raster -style windows); + if (isCygwin()) { + $testResults = "/" . toWindowsPath($testResults); + $testResults =~ s/\\/\//g; + } + system "WebKitTools/Scripts/run-launcher", @configurationArgs, "file://".$testResults if $launchSafari; +} elsif (isCygwin()) { + system "cygstart", $testResults if $launchSafari; +} else { + system "WebKitTools/Scripts/run-safari", @configurationArgs, "-NSOpen", $testResults if $launchSafari; +} + +closeCygpaths() if isCygwin(); + +exit 1; + +sub countAndPrintLeaks($$$) +{ + my ($dumpToolName, $dumpToolPID, $leaksFilePath) = @_; + + print "\n" unless $atLineStart; + $atLineStart = 1; + + # We are excluding the following reported leaks so they don't get in our way when looking for WebKit leaks: + # This allows us ignore known leaks and only be alerted when new leaks occur. Some leaks are in the old + # versions of the system frameworks that are being used by the leaks bots. Even though a leak has been + # fixed, it will be listed here until the bot has been updated with the newer frameworks. + + my @typesToExclude = ( + ); + + my @callStacksToExclude = ( + "Flash_EnforceLocalSecurity" # leaks in Flash plug-in code, rdar://problem/4449747 + ); + + if (isTiger()) { + # Leak list for the version of Tiger used on the build bot. + push @callStacksToExclude, ( + "CFRunLoopRunSpecific \\| malloc_zone_malloc", "CFRunLoopRunSpecific \\| CFAllocatorAllocate ", # leak in CFRunLoopRunSpecific, rdar://problem/4670839 + "CGImageSourceGetPropertiesAtIndex", # leak in ImageIO, rdar://problem/4628809 + "FOGetCoveredUnicodeChars", # leak in ATS, rdar://problem/3943604 + "GetLineDirectionPreference", "InitUnicodeUtilities", # leaks tool falsely reporting leak in CFNotificationCenterAddObserver, rdar://problem/4964790 + "ICCFPrefWrapper::GetPrefDictionary", # leaks in Internet Config. code, rdar://problem/4449794 + "NSHTTPURLProtocol setResponseHeader:", # leak in multipart/mixed-replace handling in Foundation, no Radar, but fixed in Leopard + "NSURLCache cachedResponseForRequest", # leak in CFURL cache, rdar://problem/4768430 + "PCFragPrepareClosureFromFile", # leak in Code Fragment Manager, rdar://problem/3426998 + "WebCore::Selection::toRange", # bug in 'leaks', rdar://problem/4967949 + "WebCore::SubresourceLoader::create", # bug in 'leaks', rdar://problem/4985806 + "_CFPreferencesDomainDeepCopyDictionary", # leak in CFPreferences, rdar://problem/4220786 + "_objc_msgForward", # leak in NSSpellChecker, rdar://problem/4965278 + "gldGetString", # leak in OpenGL, rdar://problem/5013699 + "_setDefaultUserInfoFromURL", # leak in NSHTTPAuthenticator, rdar://problem/5546453 + "SSLHandshake", # leak in SSL, rdar://problem/5546440 + "SecCertificateCreateFromData", # leak in SSL code, rdar://problem/4464397 + ); + push @typesToExclude, ( + "THRD", # bug in 'leaks', rdar://problem/3387783 + "DRHT", # ditto (endian little hate i) + ); + } + + if (isLeopard()) { + # Leak list for the version of Leopard used on the build bot. + push @callStacksToExclude, ( + "CFHTTPMessageAppendBytes", # leak in CFNetwork, rdar://problem/5435912 + "sendDidReceiveDataCallback", # leak in CFNetwork, rdar://problem/5441619 + "_CFHTTPReadStreamReadMark", # leak in CFNetwork, rdar://problem/5441468 + "httpProtocolStart", # leak in CFNetwork, rdar://problem/5468837 + "_CFURLConnectionSendCallbacks", # leak in CFNetwork, rdar://problem/5441600 + "DispatchQTMsg", # leak in QuickTime, PPC only, rdar://problem/5667132 + "QTMovieContentView createVisualContext", # leak in QuickTime, PPC only, rdar://problem/5667132 + "_CopyArchitecturesForJVMVersion", # leak in Java, rdar://problem/5910823 + ); + } + + if (isSnowLeopard()) { + push @callStacksToExclude, ( + "readMakerNoteProps", # <rdar://problem/7156432> leak in ImageIO + "QTKitMovieControllerView completeUISetup", # <rdar://problem/7155156> leak in QTKit + "getVMInitArgs", # <rdar://problem/7714444> leak in Java + "Java_java_lang_System_initProperties", # <rdar://problem/7714465> leak in Java + ); + } + + if (isDarwin() && !isTiger() && !isLeopard() && !isSnowLeopard()) { + push @callStacksToExclude, ( + "CGGradientCreateWithColorComponents", # leak in CoreGraphics, <rdar://problem/7888492> + ); + } + + my $leaksTool = sourceDir() . "/WebKitTools/Scripts/run-leaks"; + my $excludeString = "--exclude-callstack '" . (join "' --exclude-callstack '", @callStacksToExclude) . "'"; + $excludeString .= " --exclude-type '" . (join "' --exclude-type '", @typesToExclude) . "'" if @typesToExclude; + + print " ? checking for leaks in $dumpToolName\n"; + my $leaksOutput = `$leaksTool $excludeString $dumpToolPID`; + my ($count, $bytes) = $leaksOutput =~ /Process $dumpToolPID: (\d+) leaks? for (\d+) total/; + my ($excluded) = $leaksOutput =~ /(\d+) leaks? excluded/; + + my $adjustedCount = $count; + $adjustedCount -= $excluded if $excluded; + + if (!$adjustedCount) { + print " - no leaks found\n"; + unlink $leaksFilePath; + return 0; + } else { + my $dir = $leaksFilePath; + $dir =~ s|/[^/]+$|| or die; + mkpath $dir; + + if ($excluded) { + print " + $adjustedCount leaks ($bytes bytes including $excluded excluded leaks) were found, details in $leaksFilePath\n"; + } else { + print " + $count leaks ($bytes bytes) were found, details in $leaksFilePath\n"; + } + + writeToFile($leaksFilePath, $leaksOutput); + + push @leaksFilenames, $leaksFilePath; + } + + return $adjustedCount; +} + +sub writeToFile($$) +{ + my ($filePath, $contents) = @_; + open NEWFILE, ">", "$filePath" or die "Could not create $filePath. $!\n"; + print NEWFILE $contents; + close NEWFILE; +} + +# Break up a path into the directory (with slash) and base name. +sub splitpath($) +{ + my ($path) = @_; + + my $pathSeparator = "/"; + my $dirname = dirname($path) . $pathSeparator; + $dirname = "" if $dirname eq "." . $pathSeparator; + + return ($dirname, basename($path)); +} + +# Sort first by directory, then by file, so all paths in one directory are grouped +# rather than being interspersed with items from subdirectories. +# Use numericcmp to sort directory and filenames to make order logical. +sub pathcmp($$) +{ + my ($patha, $pathb) = @_; + + my ($dira, $namea) = splitpath($patha); + my ($dirb, $nameb) = splitpath($pathb); + + return numericcmp($dira, $dirb) if $dira ne $dirb; + return numericcmp($namea, $nameb); +} + +# Sort numeric parts of strings as numbers, other parts as strings. +# Makes 1.33 come after 1.3, which is cool. +sub numericcmp($$) +{ + my ($aa, $bb) = @_; + + my @a = split /(\d+)/, $aa; + my @b = split /(\d+)/, $bb; + + # Compare one chunk at a time. + # Each chunk is either all numeric digits, or all not numeric digits. + while (@a && @b) { + my $a = shift @a; + my $b = shift @b; + + # Use numeric comparison if chunks are non-equal numbers. + return $a <=> $b if $a =~ /^\d/ && $b =~ /^\d/ && $a != $b; + + # Use string comparison if chunks are any other kind of non-equal string. + return $a cmp $b if $a ne $b; + } + + # One of the two is now empty; compare lengths for result in this case. + return @a <=> @b; +} + +# Sort slowest tests first. +sub slowestcmp($$) +{ + my ($testa, $testb) = @_; + + my $dura = $durations{$testa}; + my $durb = $durations{$testb}; + return $durb <=> $dura if $dura != $durb; + return pathcmp($testa, $testb); +} + +sub launchWithEnv(\@\%) +{ + my ($args, $env) = @_; + + # Dump the current environment as perl code and then put it in quotes so it is one parameter. + my $environmentDumper = Data::Dumper->new([\%{$env}], [qw(*ENV)]); + $environmentDumper->Indent(0); + $environmentDumper->Purity(1); + my $allEnvVars = $environmentDumper->Dump(); + unshift @{$args}, "\"$allEnvVars\""; + + my $execScript = File::Spec->catfile(sourceDir(), qw(WebKitTools Scripts execAppWithEnv)); + unshift @{$args}, $execScript; + return @{$args}; +} + +sub resolveAndMakeTestResultsDirectory() +{ + my $absTestResultsDirectory = File::Spec->rel2abs(glob $testResultsDirectory); + mkpath $absTestResultsDirectory; + return $absTestResultsDirectory; +} + +sub openDiffTool() +{ + return if $isDiffToolOpen; + return if !$pixelTests; + + my %CLEAN_ENV; + $CLEAN_ENV{MallocStackLogging} = 1 if $shouldCheckLeaks; + $imageDiffToolPID = open2(\*DIFFIN, \*DIFFOUT, $imageDiffTool, launchWithEnv(@diffToolArgs, %CLEAN_ENV)) or die "unable to open $imageDiffTool\n"; + $isDiffToolOpen = 1; +} + +sub openDumpTool() +{ + return if $isDumpToolOpen; + + my %CLEAN_ENV; + + # Generic environment variables + if (defined $ENV{'WEBKIT_TESTFONTS'}) { + $CLEAN_ENV{WEBKIT_TESTFONTS} = $ENV{'WEBKIT_TESTFONTS'}; + } + + # unique temporary directory for each DumpRendertree - needed for running more DumpRenderTree in parallel + $CLEAN_ENV{DUMPRENDERTREE_TEMP} = File::Temp::tempdir('DumpRenderTree-XXXXXX', TMPDIR => 1, CLEANUP => 1); + $CLEAN_ENV{XML_CATALOG_FILES} = ""; # work around missing /etc/catalog <rdar://problem/4292995> + + # Platform spesifics + if (isLinux()) { + if (defined $ENV{'DISPLAY'}) { + $CLEAN_ENV{DISPLAY} = $ENV{'DISPLAY'}; + } else { + $CLEAN_ENV{DISPLAY} = ":1"; + } + if (defined $ENV{'XAUTHORITY'}) { + $CLEAN_ENV{XAUTHORITY} = $ENV{'XAUTHORITY'}; + } + + $CLEAN_ENV{HOME} = $ENV{'HOME'}; + + if (defined $ENV{'LD_LIBRARY_PATH'}) { + $CLEAN_ENV{LD_LIBRARY_PATH} = $ENV{'LD_LIBRARY_PATH'}; + } + if (defined $ENV{'DBUS_SESSION_BUS_ADDRESS'}) { + $CLEAN_ENV{DBUS_SESSION_BUS_ADDRESS} = $ENV{'DBUS_SESSION_BUS_ADDRESS'}; + } + } elsif (isDarwin()) { + if (defined $ENV{'DYLD_LIBRARY_PATH'}) { + $CLEAN_ENV{DYLD_LIBRARY_PATH} = $ENV{'DYLD_LIBRARY_PATH'}; + } + + $CLEAN_ENV{DYLD_FRAMEWORK_PATH} = $productDir; + $CLEAN_ENV{DYLD_INSERT_LIBRARIES} = "/usr/lib/libgmalloc.dylib" if $guardMalloc; + } elsif (isCygwin()) { + $CLEAN_ENV{HOMEDRIVE} = $ENV{'HOMEDRIVE'}; + $CLEAN_ENV{HOMEPATH} = $ENV{'HOMEPATH'}; + + setPathForRunningWebKitApp(\%CLEAN_ENV); + } + + # Port spesifics + if (isQt()) { + $CLEAN_ENV{QTWEBKIT_PLUGIN_PATH} = productDir() . "/lib/plugins"; + } + + my @args = ($dumpTool, @toolArgs); + if (isAppleMacWebKit() and !isTiger()) { + unshift @args, "arch", "-" . architecture(); + } + + if ($useValgrind) { + unshift @args, "valgrind", "--suppressions=$platformBaseDirectory/qt/SuppressedValgrindErrors"; + } + + $CLEAN_ENV{MallocStackLogging} = 1 if $shouldCheckLeaks; + + $dumpToolPID = open3(\*OUT, \*IN, \*ERROR, launchWithEnv(@args, %CLEAN_ENV)) or die "Failed to start tool: $dumpTool\n"; + $isDumpToolOpen = 1; + $dumpToolCrashed = 0; +} + +sub closeDumpTool() +{ + return if !$isDumpToolOpen; + + close IN; + close OUT; + waitpid $dumpToolPID, 0; + + # check for WebCore counter leaks. + if ($shouldCheckLeaks) { + while (<ERROR>) { + print; + } + } + close ERROR; + $isDumpToolOpen = 0; +} + +sub dumpToolDidCrash() +{ + return 1 if $dumpToolCrashed; + return 0 unless $isDumpToolOpen; + my $pid = waitpid(-1, WNOHANG); + return 1 if ($pid == $dumpToolPID); + + # On Mac OS X, crashing may be significantly delayed by crash reporter. + return 0 unless isAppleMacWebKit(); + + return DumpRenderTreeSupport::processIsCrashing($dumpToolPID); +} + +sub configureAndOpenHTTPDIfNeeded() +{ + return if $isHttpdOpen; + my $absTestResultsDirectory = resolveAndMakeTestResultsDirectory(); + my $listen = "127.0.0.1:$httpdPort"; + my @args = ( + "-c", "CustomLog \"$absTestResultsDirectory/access_log.txt\" common", + "-c", "ErrorLog \"$absTestResultsDirectory/error_log.txt\"", + "-C", "Listen $listen" + ); + + my @defaultArgs = getDefaultConfigForTestDirectory($testDirectory); + @args = (@defaultArgs, @args); + + waitForHTTPDLock() if $shouldWaitForHTTPD; + $isHttpdOpen = openHTTPD(@args); +} + +sub openWebSocketServerIfNeeded() +{ + return 1 if $isWebSocketServerOpen; + return 0 if $failedToStartWebSocketServer; + + my $webSocketServerPath = "/usr/bin/python"; + my $webSocketPythonPath = "WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket"; + my $webSocketHandlerDir = "$testDirectory"; + my $webSocketHandlerScanDir = "$testDirectory/websocket/tests"; + my $webSocketHandlerMapFile = "$webSocketHandlerScanDir/handler_map.txt"; + my $sslCertificate = "$testDirectory/http/conf/webkit-httpd.pem"; + my $absTestResultsDirectory = resolveAndMakeTestResultsDirectory(); + my $logFile = "$absTestResultsDirectory/pywebsocket_log.txt"; + + my @args = ( + "WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/standalone.py", + "--server-host", "127.0.0.1", + "--port", "$webSocketPort", + "--document-root", "$webSocketHandlerDir", + "--scan-dir", "$webSocketHandlerScanDir", + "--websock-handlers-map-file", "$webSocketHandlerMapFile", + "--cgi-paths", "/websocket/tests", + "--log-file", "$logFile", + "--strict", + ); + # wss is disabled until all platforms support pyOpenSSL. + # my @argsSecure = ( + # "WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/standalone.py", + # "-p", "$webSocketSecurePort", + # "-d", "$webSocketHandlerDir", + # "-t", + # "-k", "$sslCertificate", + # "-c", "$sslCertificate", + # ); + + $ENV{"PYTHONPATH"} = $webSocketPythonPath; + $webSocketServerPID = open3(\*WEBSOCKETSERVER_IN, \*WEBSOCKETSERVER_OUT, \*WEBSOCKETSERVER_ERR, $webSocketServerPath, @args); + # wss is disabled until all platforms support pyOpenSSL. + # $webSocketSecureServerPID = open3(\*WEBSOCKETSECURESERVER_IN, \*WEBSOCKETSECURESERVER_OUT, \*WEBSOCKETSECURESERVER_ERR, $webSocketServerPath, @argsSecure); + # my @listen = ("http://127.0.0.1:$webSocketPort", "https://127.0.0.1:$webSocketSecurePort"); + my @listen = ("http://127.0.0.1:$webSocketPort"); + for (my $i = 0; $i < @listen; $i++) { + my $retryCount = 10; + while (system("/usr/bin/curl -k -q --silent --stderr - --output /dev/null $listen[$i]") && $retryCount) { + sleep 1; + --$retryCount; + } + unless ($retryCount) { + print STDERR "Timed out waiting for WebSocketServer to start.\n"; + $failedToStartWebSocketServer = 1; + return 0; + } + } + + $isWebSocketServerOpen = 1; + return 1; +} + +sub closeWebSocketServer() +{ + return if !$isWebSocketServerOpen; + + close WEBSOCKETSERVER_IN; + close WEBSOCKETSERVER_OUT; + close WEBSOCKETSERVER_ERR; + kill 15, $webSocketServerPID; + + # wss is disabled until all platforms support pyOpenSSL. + # close WEBSOCKETSECURESERVER_IN; + # close WEBSOCKETSECURESERVER_OUT; + # close WEBSOCKETSECURESERVER_ERR; + # kill 15, $webSocketSecureServerPID; + + $isWebSocketServerOpen = 0; +} + +sub fileNameWithNumber($$) +{ + my ($base, $number) = @_; + return "$base$number" if ($number > 1); + return $base; +} + +sub processIgnoreTests($$) +{ + my @ignoreList = split(/\s*,\s*/, shift); + my $listName = shift; + + my $disabledSuffix = "-disabled"; + + my $addIgnoredDirectories = sub { + return () if exists $ignoredLocalDirectories{basename($File::Find::dir)}; + $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)} = 1; + return @_; + }; + foreach my $item (@ignoreList) { + my $path = catfile($testDirectory, $item); + if (-d $path) { + $ignoredDirectories{$item} = 1; + find({ preprocess => $addIgnoredDirectories, wanted => sub {} }, $path); + } + elsif (-f $path) { + $ignoredFiles{$item} = 1; + } elsif (-f $path . $disabledSuffix) { + # The test is disabled, so do nothing. + } else { + print "$listName list contained '$item', but no file of that name could be found\n"; + } + } +} + +sub stripExtension($) +{ + my ($test) = @_; + + $test =~ s/\.[a-zA-Z]+$//; + return $test; +} + +sub isTextOnlyTest($) +{ + my ($actual) = @_; + my $isText; + if ($actual =~ /^layer at/ms) { + $isText = 0; + } else { + $isText = 1; + } + return $isText; +} + +sub expectedDirectoryForTest($;$;$) +{ + my ($base, $isText, $expectedExtension) = @_; + + my @directories = @platformResultHierarchy; + push @directories, map { catdir($platformBaseDirectory, $_) } qw(mac-snowleopard mac) if isCygwin(); + push @directories, $expectedDirectory; + + # If we already have expected results, just return their location. + foreach my $directory (@directories) { + return $directory if (-f "$directory/$base-$expectedTag.$expectedExtension"); + } + + # For cross-platform tests, text-only results should go in the cross-platform directory, + # while render tree dumps should go in the least-specific platform directory. + return $isText ? $expectedDirectory : $platformResultHierarchy[$#platformResultHierarchy]; +} + +sub countFinishedTest($$$$) +{ + my ($test, $base, $result, $isText) = @_; + + if (($count + 1) % $testsPerDumpTool == 0 || $count == $#tests) { + if ($shouldCheckLeaks) { + my $fileName; + if ($testsPerDumpTool == 1) { + $fileName = "$testResultsDirectory/$base-leaks.txt"; + } else { + $fileName = "$testResultsDirectory/" . fileNameWithNumber($dumpToolName, $leaksOutputFileNumber) . "-leaks.txt"; + } + my $leakCount = countAndPrintLeaks($dumpToolName, $dumpToolPID, $fileName); + $totalLeaks += $leakCount; + $leaksOutputFileNumber++ if ($leakCount); + } + + closeDumpTool(); + } + + $count++; + $counts{$result}++; + push @{$tests{$result}}, $test; +} + +sub testCrashedOrTimedOut($$$$$) +{ + my ($test, $base, $didCrash, $actual, $error) = @_; + + printFailureMessageForTest($test, $didCrash ? "crashed" : "timed out"); + + sampleDumpTool() unless $didCrash; + + my $dir = "$testResultsDirectory/$base"; + $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; + mkpath $dir; + + deleteExpectedAndActualResults($base); + + if (defined($error) && length($error)) { + writeToFile("$testResultsDirectory/$base-$errorTag.txt", $error); + } + + recordActualResultsAndDiff($base, $actual); + + kill 9, $dumpToolPID unless $didCrash; + + closeDumpTool(); +} + +sub printFailureMessageForTest($$) +{ + my ($test, $description) = @_; + + unless ($verbose) { + print "\n" unless $atLineStart; + print "$test -> "; + } + print "$description\n"; + $atLineStart = 1; +} + +my %cygpaths = (); + +sub openCygpathIfNeeded($) +{ + my ($options) = @_; + + return unless isCygwin(); + return $cygpaths{$options} if $cygpaths{$options} && $cygpaths{$options}->{"open"}; + + local (*CYGPATHIN, *CYGPATHOUT); + my $pid = open2(\*CYGPATHIN, \*CYGPATHOUT, "cygpath -f - $options"); + my $cygpath = { + "pid" => $pid, + "in" => *CYGPATHIN, + "out" => *CYGPATHOUT, + "open" => 1 + }; + + $cygpaths{$options} = $cygpath; + + return $cygpath; +} + +sub closeCygpaths() +{ + return unless isCygwin(); + + foreach my $cygpath (values(%cygpaths)) { + close $cygpath->{"in"}; + close $cygpath->{"out"}; + waitpid($cygpath->{"pid"}, 0); + $cygpath->{"open"} = 0; + + } +} + +sub convertPathUsingCygpath($$) +{ + my ($path, $options) = @_; + + # cygpath -f (at least in Cygwin 1.7) converts spaces into newlines. We remove spaces here and + # add them back in after conversion to work around this. + my $spaceSubstitute = "__NOTASPACE__"; + $path =~ s/ /\Q$spaceSubstitute\E/g; + + my $cygpath = openCygpathIfNeeded($options); + local *inFH = $cygpath->{"in"}; + local *outFH = $cygpath->{"out"}; + print outFH $path . "\n"; + my $convertedPath = <inFH>; + chomp($convertedPath) if defined $convertedPath; + + $convertedPath =~ s/\Q$spaceSubstitute\E/ /g; + return $convertedPath; +} + +sub toWindowsPath($) +{ + my ($path) = @_; + return unless isCygwin(); + + return convertPathUsingCygpath($path, "-w"); +} + +sub toURL($) +{ + my ($path) = @_; + + if ($useRemoteLinksToTests) { + my $relativePath = File::Spec->abs2rel($path, $testDirectory); + + # If the file is below the test directory then convert it into a link to the file in SVN + if ($relativePath !~ /^\.\.\//) { + my $revision = svnRevisionForDirectory($testDirectory); + my $svnPath = pathRelativeToSVNRepositoryRootForPath($path); + return "http://trac.webkit.org/export/$revision/$svnPath"; + } + } + + return $path unless isCygwin(); + + return "file:///" . convertPathUsingCygpath($path, "-m"); +} + +sub validateSkippedArg($$;$) +{ + my ($option, $value, $value2) = @_; + my %validSkippedValues = map { $_ => 1 } qw(default ignore only); + $value = lc($value); + die "Invalid argument '" . $value . "' for option $option" unless $validSkippedValues{$value}; + $treatSkipped = $value; +} + +sub htmlForResultsSection(\@$&) +{ + my ($tests, $description, $linkGetter) = @_; + + my @html = (); + return join("\n", @html) unless @{$tests}; + + push @html, "<p>$description:</p>"; + push @html, "<table>"; + foreach my $test (@{$tests}) { + push @html, "<tr>"; + push @html, "<td><a href=\"" . toURL("$testDirectory/$test") . "\">$test</a></td>"; + foreach my $link (@{&{$linkGetter}($test)}) { + push @html, "<td><a href=\"$link->{href}\">$link->{text}</a></td>"; + } + push @html, "</tr>"; + } + push @html, "</table>"; + + return join("\n", @html); +} + +sub linksForExpectedAndActualResults($) +{ + my ($base) = @_; + + my @links = (); + + return \@links unless -s "$testResultsDirectory/$base-$diffsTag.txt"; + + my $expectedResultPath = $expectedResultPaths{$base}; + my ($expectedResultFileName, $expectedResultsDirectory, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); + + push @links, { href => "$base-$expectedTag$expectedResultExtension", text => "expected" }; + push @links, { href => "$base-$actualTag$expectedResultExtension", text => "actual" }; + push @links, { href => "$base-$diffsTag.txt", text => "diff" }; + push @links, { href => "$base-$prettyDiffTag.html", text => "pretty diff" }; + + return \@links; +} + +sub linksForMismatchTest +{ + my ($test) = @_; + + my @links = (); + + my $base = stripExtension($test); + + push @links, @{linksForExpectedAndActualResults($base)}; + return \@links unless $pixelTests && $imagesPresent{$base}; + + push @links, { href => "$base-$expectedTag.png", text => "expected image" }; + push @links, { href => "$base-$diffsTag.html", text => "image diffs" }; + push @links, { href => "$base-$diffsTag.png", text => "$imageDifferences{$base}%" }; + + return \@links; +} + +sub linksForErrorTest +{ + my ($test) = @_; + + my @links = (); + + my $base = stripExtension($test); + + push @links, @{linksForExpectedAndActualResults($base)}; + push @links, { href => "$base-$errorTag.txt", text => "stderr" }; + + return \@links; +} + +sub linksForNewTest +{ + my ($test) = @_; + + my @links = (); + + my $base = stripExtension($test); + + my $expectedResultPath = $expectedResultPaths{$base}; + my ($expectedResultFileName, $expectedResultsDirectory, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); + + push @links, { href => "$base-$actualTag$expectedResultExtension", text => "result" }; + if ($pixelTests && $imagesPresent{$base}) { + push @links, { href => "$base-$expectedTag.png", text => "image" }; + } + + return \@links; +} + +sub deleteExpectedAndActualResults($) +{ + my ($base) = @_; + + unlink "$testResultsDirectory/$base-$actualTag.txt"; + unlink "$testResultsDirectory/$base-$diffsTag.txt"; + unlink "$testResultsDirectory/$base-$errorTag.txt"; +} + +sub recordActualResultsAndDiff($$) +{ + my ($base, $actualResults) = @_; + + return unless defined($actualResults) && length($actualResults); + + my $expectedResultPath = $expectedResultPaths{$base}; + my ($expectedResultFileNameMinusExtension, $expectedResultDirectoryPath, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); + my $actualResultsPath = "$testResultsDirectory/$base-$actualTag$expectedResultExtension"; + my $copiedExpectedResultsPath = "$testResultsDirectory/$base-$expectedTag$expectedResultExtension"; + + mkpath(dirname($actualResultsPath)); + writeToFile("$actualResultsPath", $actualResults); + + if (-f $expectedResultPath) { + copy("$expectedResultPath", "$copiedExpectedResultsPath"); + } else { + open EMPTY, ">$copiedExpectedResultsPath"; + close EMPTY; + } + + my $diffOuputBasePath = "$testResultsDirectory/$base"; + my $diffOutputPath = "$diffOuputBasePath-$diffsTag.txt"; + system "diff -u \"$copiedExpectedResultsPath\" \"$actualResultsPath\" > \"$diffOutputPath\""; + + my $prettyDiffOutputPath = "$diffOuputBasePath-$prettyDiffTag.html"; + my $prettyPatchPath = "BugsSite/PrettyPatch/"; + my $prettifyPath = "$prettyPatchPath/prettify.rb"; + system "ruby -I \"$prettyPatchPath\" \"$prettifyPath\" \"$diffOutputPath\" > \"$prettyDiffOutputPath\""; +} + +sub buildPlatformResultHierarchy() +{ + mkpath($platformTestDirectory) if ($platform eq "undefined" && !-d "$platformTestDirectory"); + + my @platforms; + if ($platform =~ /^mac-/) { + my $i; + for ($i = 0; $i < @macPlatforms; $i++) { + last if $macPlatforms[$i] eq $platform; + } + for (; $i < @macPlatforms; $i++) { + push @platforms, $macPlatforms[$i]; + } + } elsif ($platform =~ /^qt-/) { + push @platforms, $platform; + push @platforms, "qt"; + } else { + @platforms = $platform; + } + + my @hierarchy; + for (my $i = 0; $i < @platforms; $i++) { + my $scoped = catdir($platformBaseDirectory, $platforms[$i]); + push(@hierarchy, $scoped) if (-d $scoped); + } + + return @hierarchy; +} + +sub buildPlatformTestHierarchy(@) +{ + my (@platformHierarchy) = @_; + return @platformHierarchy if (@platformHierarchy < 2); + + return ($platformHierarchy[0], $platformHierarchy[$#platformHierarchy]); +} + +sub epiloguesAndPrologues($$) +{ + my ($lastDirectory, $directory) = @_; + my @lastComponents = split('/', $lastDirectory); + my @components = split('/', $directory); + + while (@lastComponents) { + if (!defined($components[0]) || $lastComponents[0] ne $components[0]) { + last; + } + shift @components; + shift @lastComponents; + } + + my @result; + my $leaving = $lastDirectory; + foreach (@lastComponents) { + my $epilogue = $leaving . "/resources/run-webkit-tests-epilogue.html"; + foreach (@platformResultHierarchy) { + push @result, catdir($_, $epilogue) if (stat(catdir($_, $epilogue))); + } + push @result, catdir($testDirectory, $epilogue) if (stat(catdir($testDirectory, $epilogue))); + $leaving =~ s|(^\|/)[^/]+$||; + } + + my $entering = $leaving; + foreach (@components) { + $entering .= '/' . $_; + my $prologue = $entering . "/resources/run-webkit-tests-prologue.html"; + push @result, catdir($testDirectory, $prologue) if (stat(catdir($testDirectory, $prologue))); + foreach (reverse @platformResultHierarchy) { + push @result, catdir($_, $prologue) if (stat(catdir($_, $prologue))); + } + } + return @result; +} + +sub parseLeaksandPrintUniqueLeaks() +{ + return unless @leaksFilenames; + + my $mergedFilenames = join " ", @leaksFilenames; + my $parseMallocHistoryTool = sourceDir() . "/WebKitTools/Scripts/parse-malloc-history"; + + open MERGED_LEAKS, "cat $mergedFilenames | $parseMallocHistoryTool --merge-depth $mergeDepth - |" ; + my @leakLines = <MERGED_LEAKS>; + close MERGED_LEAKS; + + my $uniqueLeakCount = 0; + my $totalBytes; + foreach my $line (@leakLines) { + ++$uniqueLeakCount if ($line =~ /^(\d*)\scalls/); + $totalBytes = $1 if $line =~ /^total\:\s(.*)\s\(/; + } + + print "\nWARNING: $totalLeaks total leaks found for a total of $totalBytes!\n"; + print "WARNING: $uniqueLeakCount unique leaks found!\n"; + print "See above for individual leaks results.\n" if ($leaksOutputFileNumber > 2); + +} + +sub extensionForMimeType($) +{ + my ($mimeType) = @_; + + if ($mimeType eq "application/x-webarchive") { + return "webarchive"; + } elsif ($mimeType eq "application/pdf") { + return "pdf"; + } + return "txt"; +} + +# Read up to the first #EOF (the content block of the test), or until detecting crashes or timeouts. +sub readFromDumpToolWithTimer(**) +{ + my ($fhIn, $fhError) = @_; + + setFileHandleNonBlocking($fhIn, 1); + setFileHandleNonBlocking($fhError, 1); + + my $maximumSecondsWithoutOutput = $timeoutSeconds; + $maximumSecondsWithoutOutput *= 10 if $guardMalloc; + my $microsecondsToWaitBeforeReadingAgain = 1000; + + my $timeOfLastSuccessfulRead = time; + + my @output = (); + my @error = (); + my $status = "success"; + my $mimeType = "text/plain"; + # We don't have a very good way to know when the "headers" stop + # and the content starts, so we use this as a hack: + my $haveSeenContentType = 0; + my $haveSeenEofIn = 0; + my $haveSeenEofError = 0; + + while (1) { + if (time - $timeOfLastSuccessfulRead > $maximumSecondsWithoutOutput) { + $status = dumpToolDidCrash() ? "crashed" : "timedOut"; + last; + } + + # Once we've seen the EOF, we must not read anymore. + my $lineIn = readline($fhIn) unless $haveSeenEofIn; + my $lineError = readline($fhError) unless $haveSeenEofError; + if (!defined($lineIn) && !defined($lineError)) { + last if ($haveSeenEofIn && $haveSeenEofError); + + if ($! != EAGAIN) { + $status = "crashed"; + last; + } + + # No data ready + usleep($microsecondsToWaitBeforeReadingAgain); + next; + } + + $timeOfLastSuccessfulRead = time; + + if (defined($lineIn)) { + if (!$haveSeenContentType && $lineIn =~ /^Content-Type: (\S+)$/) { + $mimeType = $1; + $haveSeenContentType = 1; + } elsif ($lineIn =~ /#EOF/) { + $haveSeenEofIn = 1; + } else { + push @output, $lineIn; + } + } + if (defined($lineError)) { + if ($lineError =~ /#EOF/) { + $haveSeenEofError = 1; + } else { + push @error, $lineError; + } + } + } + + setFileHandleNonBlocking($fhIn, 0); + setFileHandleNonBlocking($fhError, 0); + return { + output => join("", @output), + error => join("", @error), + status => $status, + mimeType => $mimeType, + extension => extensionForMimeType($mimeType) + }; +} + +sub setFileHandleNonBlocking(*$) +{ + my ($fh, $nonBlocking) = @_; + + my $flags = fcntl($fh, F_GETFL, 0) or die "Couldn't get filehandle flags"; + + if ($nonBlocking) { + $flags |= O_NONBLOCK; + } else { + $flags &= ~O_NONBLOCK; + } + + fcntl($fh, F_SETFL, $flags) or die "Couldn't set filehandle flags"; + + return 1; +} + +sub sampleDumpTool() +{ + return unless isAppleMacWebKit(); + return unless $runSample; + + my $outputDirectory = "$ENV{HOME}/Library/Logs/DumpRenderTree"; + -d $outputDirectory or mkdir $outputDirectory; + + my $outputFile = "$outputDirectory/HangReport.txt"; + system "/usr/bin/sample", $dumpToolPID, qw(10 10 -file), $outputFile; +} + +sub stripMetrics($$) +{ + my ($actual, $expected) = @_; + + foreach my $result ($actual, $expected) { + $result =~ s/at \(-?[0-9]+,-?[0-9]+\) *//g; + $result =~ s/size -?[0-9]+x-?[0-9]+ *//g; + $result =~ s/text run width -?[0-9]+: //g; + $result =~ s/text run width -?[0-9]+ [a-zA-Z ]+: //g; + $result =~ s/RenderButton {BUTTON} .*/RenderButton {BUTTON}/g; + $result =~ s/RenderImage {INPUT} .*/RenderImage {INPUT}/g; + $result =~ s/RenderBlock {INPUT} .*/RenderBlock {INPUT}/g; + $result =~ s/RenderTextControl {INPUT} .*/RenderTextControl {INPUT}/g; + $result =~ s/\([0-9]+px/px/g; + $result =~ s/ *" *\n +" */ /g; + $result =~ s/" +$/"/g; + + $result =~ s/- /-/g; + $result =~ s/\n( *)"\s+/\n$1"/g; + $result =~ s/\s+"\n/"\n/g; + $result =~ s/scrollWidth [0-9]+/scrollWidth/g; + $result =~ s/scrollHeight [0-9]+/scrollHeight/g; + } + + return ($actual, $expected); +} + +sub fileShouldBeIgnored +{ + my ($filePath) = @_; + foreach my $ignoredDir (keys %ignoredDirectories) { + if ($filePath =~ m/^$ignoredDir/) { + return 1; + } + } + return 0; +} + +sub readSkippedFiles($) +{ + my ($constraintPath) = @_; + + foreach my $level (@platformTestHierarchy) { + if (open SKIPPED, "<", "$level/Skipped") { + if ($verbose) { + my ($dir, $name) = splitpath($level); + print "Skipped tests in $name:\n"; + } + + while (<SKIPPED>) { + my $skipped = $_; + chomp $skipped; + $skipped =~ s/^[ \n\r]+//; + $skipped =~ s/[ \n\r]+$//; + if ($skipped && $skipped !~ /^#/) { + if ($skippedOnly) { + if (!fileShouldBeIgnored($skipped)) { + if (!$constraintPath) { + # Always add $skipped since no constraint path was specified on the command line. + push(@ARGV, $skipped); + } elsif ($skipped =~ /^($constraintPath)/) { + # Add $skipped only if it matches the current path constraint, e.g., + # "--skipped=only dir1" with "dir1/file1.html" on the skipped list. + push(@ARGV, $skipped); + } elsif ($constraintPath =~ /^($skipped)/) { + # Add current path constraint if it is more specific than the skip list entry, + # e.g., "--skipped=only dir1/dir2/dir3" with "dir1" on the skipped list. + push(@ARGV, $constraintPath); + } + } elsif ($verbose) { + print " $skipped\n"; + } + } else { + if ($verbose) { + print " $skipped\n"; + } + processIgnoreTests($skipped, "Skipped"); + } + } + } + close SKIPPED; + } + } +} + +my @testsFound; + +sub directoryFilter +{ + return () if exists $ignoredLocalDirectories{basename($File::Find::dir)}; + return () if exists $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)}; + return @_; +} + +sub fileFilter +{ + my $filename = $_; + if ($filename =~ /\.([^.]+)$/) { + if (exists $supportedFileExtensions{$1}) { + my $path = File::Spec->abs2rel(catfile($File::Find::dir, $filename), $testDirectory); + push @testsFound, $path if !exists $ignoredFiles{$path}; + } + } +} + +sub findTestsToRun +{ + my @testsToRun = (); + + for my $test (@ARGV) { + $test =~ s/^($layoutTestsName|$testDirectory)\///; + my $fullPath = catfile($testDirectory, $test); + if (file_name_is_absolute($test)) { + print "can't run test $test outside $testDirectory\n"; + } elsif (-f $fullPath) { + my ($filename, $pathname, $fileExtension) = fileparse($test, qr{\.[^.]+$}); + if (!exists $supportedFileExtensions{substr($fileExtension, 1)}) { + print "test $test does not have a supported extension\n"; + } elsif ($testHTTP || $pathname !~ /^http\//) { + push @testsToRun, $test; + } + } elsif (-d $fullPath) { + @testsFound = (); + find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $fullPath); + for my $level (@platformTestHierarchy) { + my $platformPath = catfile($level, $test); + find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $platformPath) if (-d $platformPath); + } + push @testsToRun, sort pathcmp @testsFound; + @testsFound = (); + } else { + print "test $test not found\n"; + } + } + + if (!scalar @ARGV) { + @testsFound = (); + find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $testDirectory); + for my $level (@platformTestHierarchy) { + find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $level); + } + push @testsToRun, sort pathcmp @testsFound; + @testsFound = (); + + # We need to minimize the time when Apache and WebSocketServer is locked by tests + # so run them last if no explicit order was specified in the argument list. + my @httpTests; + my @websocketTests; + my @otherTests; + foreach my $test (@testsToRun) { + if ($test =~ /^http\//) { + push(@httpTests, $test); + } elsif ($test =~ /^websocket\//) { + push(@websocketTests, $test); + } else { + push(@otherTests, $test); + } + } + @testsToRun = (@otherTests, @httpTests, @websocketTests); + } + + # Reverse the tests + @testsToRun = reverse @testsToRun if $reverseTests; + + # Shuffle the array + @testsToRun = shuffle(@testsToRun) if $randomizeTests; + + return @testsToRun; +} + +sub printResults +{ + my %text = ( + match => "succeeded", + mismatch => "had incorrect layout", + new => "were new", + timedout => "timed out", + crash => "crashed", + error => "had stderr output" + ); + + for my $type ("match", "mismatch", "new", "timedout", "crash", "error") { + my $typeCount = $counts{$type}; + next unless $typeCount; + my $typeText = $text{$type}; + my $message; + if ($typeCount == 1) { + $typeText =~ s/were/was/; + $message = sprintf "1 test case (%d%%) %s\n", 1 * 100 / $count, $typeText; + } else { + $message = sprintf "%d test cases (%d%%) %s\n", $typeCount, $typeCount * 100 / $count, $typeText; + } + $message =~ s-\(0%\)-(<1%)-; + print $message; + } +} + +sub stopRunningTestsEarlyIfNeeded() +{ + # --reset-results does not check pass vs. fail, so exitAfterNFailures makes no sense with --reset-results. + return 0 if !$exitAfterNFailures || $resetResults; + + my $passCount = $counts{match} || 0; # $counts{match} will be undefined if we've not yet passed a test (e.g. the first test fails). + my $failureCount = $count - $passCount; # "Failure" here includes new tests, timeouts, crashes, etc. + return 0 if $failureCount < $exitAfterNFailures; + + print "\nExiting early after $failureCount failures. $count tests run."; + closeDumpTool(); + return 1; +} diff --git a/WebKitTools/Scripts/prepare-ChangeLog b/WebKitTools/Scripts/prepare-ChangeLog index 3350aa3..b087f67 100755 --- a/WebKitTools/Scripts/prepare-ChangeLog +++ b/WebKitTools/Scripts/prepare-ChangeLog @@ -89,6 +89,7 @@ sub get_function_line_ranges($$); sub get_function_line_ranges_for_c($$); sub get_function_line_ranges_for_java($$); sub get_function_line_ranges_for_javascript($$); +sub get_selector_line_ranges_for_css($$); sub method_decl_to_selector($); sub processPaths(\@); sub reviewerAndDescriptionForGitCommit($); @@ -101,6 +102,7 @@ my $changeLogTimeZone = "PST8PDT"; my $bugNumber; my $name; my $emailAddress; +my $mergeBase = 0; my $gitCommit = 0; my $gitIndex = ""; my $gitReviewer = ""; @@ -114,6 +116,7 @@ my $parseOptionsResult = "bug:i" => \$bugNumber, "name:s" => \$name, "email:s" => \$emailAddress, + "merge-base:s" => \$mergeBase, "git-commit:s" => \$gitCommit, "git-index" => \$gitIndex, "git-reviewer:s" => \$gitReviewer, @@ -125,6 +128,7 @@ if (!$parseOptionsResult || $showHelp) { print STDERR basename($0) . " [--bug] [-d|--diff] [-h|--help] [-o|--open] [--git-commit=<committish>] [--git-reviewer=<name>] [svndir1 [svndir2 ...]]\n"; print STDERR " --bug Fill in the ChangeLog bug information from the given bug.\n"; print STDERR " -d|--diff Spew diff to stdout when running\n"; + print STDERR " --merge-base Populate the ChangeLogs with the diff to this branch\n"; print STDERR " --git-commit Populate the ChangeLogs from the specified git commit\n"; print STDERR " --git-index Populate the ChangeLogs from the git index only\n"; print STDERR " --git-reviewer When populating the ChangeLogs from a git commit claim that the spcified name reviewed the change.\n"; @@ -474,6 +478,8 @@ sub get_function_line_ranges($$) return get_function_line_ranges_for_java ($file_handle, $file_name); } elsif ($file_name =~ /\.js$/) { return get_function_line_ranges_for_javascript ($file_handle, $file_name); + } elsif ($file_name =~ /\.css$/) { + return get_selector_line_ranges_for_css ($file_handle, $file_name); } return (); } @@ -1173,6 +1179,41 @@ sub get_function_line_ranges_for_javascript($$) return @ranges; } +# Read a file and get all the line ranges of the things that look like CSS selectors. A selector is +# anything before an opening brace on a line. A selector starts at the line containing the opening +# brace and ends at the closing brace. +# FIXME: Comments are parsed just like uncommented text. +# +# Result is a list of triples: [ start_line, end_line, selector ]. + +sub get_selector_line_ranges_for_css($$) +{ + my ($fileHandle, $fileName) = @_; + + my @ranges; + + my $currentSelector = ""; + my $start = 0; + + while (<$fileHandle>) { + if (/^[ \t]*(.*[^ \t])[ \t]*{/) { + $currentSelector = $1; + $start = $.; + } + if (index($_, "}") >= 0) { + unless ($start) { + warn "mismatched braces in $fileName\n"; + next; + } + push(@ranges, [$start, $., $currentSelector]); + $currentSelector = ""; + $start = 0; + next; + } + } + + return @ranges; +} sub processPaths(\@) { @@ -1216,6 +1257,7 @@ sub diffFromToString() return $gitCommit if $gitCommit =~ m/.+\.\..+/; return "\"$gitCommit^\" \"$gitCommit\"" if $gitCommit; return "--cached" if $gitIndex; + return $mergeBase if $mergeBase; return "HEAD" if $isGit; } @@ -1230,7 +1272,7 @@ sub diffCommand(@) $command = "$SVN diff --diff-cmd diff -x -N $pathsString"; } elsif ($isGit) { $command = "$GIT diff --no-ext-diff -U0 " . diffFromToString(); - $command .= " -- $pathsString" unless $gitCommit; + $command .= " -- $pathsString" unless $gitCommit or $mergeBase; } return $command; diff --git a/WebKitTools/Scripts/rebaseline-chromium-webkit-tests b/WebKitTools/Scripts/rebaseline-chromium-webkit-tests index 302995c..8d14b86 100755 --- a/WebKitTools/Scripts/rebaseline-chromium-webkit-tests +++ b/WebKitTools/Scripts/rebaseline-chromium-webkit-tests @@ -31,9 +31,12 @@ import os import sys -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), - "webkitpy", "layout_tests")) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) +scripts_directory = os.path.dirname(os.path.abspath(sys.argv[0])) +webkitpy_directory = os.path.join(scripts_directory, "webkitpy") +sys.path.append(os.path.join(webkitpy_directory, "layout_tests")) + +# For simplejson +sys.path.append(os.path.join(webkitpy_directory, "thirdparty")) import rebaseline_chromium_webkit_tests diff --git a/WebKitTools/Scripts/resolve-ChangeLogs b/WebKitTools/Scripts/resolve-ChangeLogs index 3238350..6635711 100755 --- a/WebKitTools/Scripts/resolve-ChangeLogs +++ b/WebKitTools/Scripts/resolve-ChangeLogs @@ -49,7 +49,6 @@ sub fixMergedChangeLogs($;@); sub fixOneMergedChangeLog($); sub hasGitUnmergedFiles(); sub isInGitFilterBranch(); -sub mergeChanges($$$); sub parseFixMerged($$;$); sub removeChangeLogArguments($); sub resolveChangeLog($); @@ -130,11 +129,11 @@ if (defined $fixMerged && length($fixMerged) > 0) { fixMergedChangeLogs($commitRange, @changeLogFiles); } elsif ($mergeDriver) { my ($base, $theirs, $ours) = @ARGV; - if (mergeChanges($ours, $base, $theirs)) { + if (mergeChangeLogs($ours, $base, $theirs)) { unlink($ours); copy($theirs, $ours) or die $!; } else { - exit 1; + exec qw(git merge-file -L THEIRS -L BASE -L OURS), $theirs, $base, $ours; } } elsif (@changeLogFiles) { for my $file (@changeLogFiles) { @@ -401,55 +400,6 @@ sub isInGitFilterBranch() return exists $ENV{MAPPED_PREVIOUS_COMMIT} && $ENV{MAPPED_PREVIOUS_COMMIT}; } -sub mergeChanges($$$) -{ - my ($fileMine, $fileOlder, $fileNewer) = @_; - - my $traditionalReject = $fileMine =~ /\.rej$/ ? 1 : 0; - - local $/ = undef; - - my $patch; - if ($traditionalReject) { - open(DIFF, "<", $fileMine) or die $!; - $patch = <DIFF>; - close(DIFF); - rename($fileMine, "$fileMine.save"); - rename($fileOlder, "$fileOlder.save"); - } else { - open(DIFF, "-|", qw(diff -u -a --binary), $fileOlder, $fileMine) or die $!; - $patch = <DIFF>; - close(DIFF); - } - - unlink("${fileNewer}.orig"); - unlink("${fileNewer}.rej"); - - open(PATCH, "| patch --fuzz=3 --binary $fileNewer > " . File::Spec->devnull()) or die $!; - print PATCH fixChangeLogPatch($patch); - close(PATCH); - - my $result; - - # Refuse to merge the patch if it did not apply cleanly - if (-e "${fileNewer}.rej") { - unlink("${fileNewer}.rej"); - unlink($fileNewer); - rename("${fileNewer}.orig", $fileNewer); - $result = 0; - } else { - unlink("${fileNewer}.orig"); - $result = 1; - } - - if ($traditionalReject) { - rename("$fileMine.save", $fileMine); - rename("$fileOlder.save", $fileOlder); - } - - return $result; -} - sub parseFixMerged($$;$) { my ($switchName, $key, $value) = @_; @@ -491,7 +441,7 @@ sub resolveChangeLog($) return unless $fileMine && $fileOlder && $fileNewer; - if (mergeChanges($fileMine, $fileOlder, $fileNewer)) { + if (mergeChangeLogs($fileMine, $fileOlder, $fileNewer)) { if ($file ne $fileNewer) { unlink($file); rename($fileNewer, $file) or die $!; diff --git a/WebKitTools/Scripts/run-launcher b/WebKitTools/Scripts/run-launcher index e12a64a..bc00aac 100755 --- a/WebKitTools/Scripts/run-launcher +++ b/WebKitTools/Scripts/run-launcher @@ -64,6 +64,10 @@ if (isQt()) { $launcherPath = catdir($launcherPath, "Programs", "GtkLauncher"); } + if (isEfl()) { + $launcherPath = catdir($launcherPath, "Programs", "EWebLauncher"); + } + if (isWx()) { if (isDarwin()) { $launcherPath = catdir($launcherPath, 'wxBrowser.app', 'Contents', 'MacOS', 'wxBrowser'); diff --git a/WebKitTools/Scripts/run-minibrowser b/WebKitTools/Scripts/run-minibrowser new file mode 100755 index 0000000..c2fd412 --- /dev/null +++ b/WebKitTools/Scripts/run-minibrowser @@ -0,0 +1,38 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2005, 2007 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Simplified "run" script for launching the WebKit2 MiniBrowser. + +use strict; +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; + +setConfiguration(); + +exit exitStatus(runMiniBrowser()); diff --git a/WebKitTools/Scripts/run-webkit-httpd b/WebKitTools/Scripts/run-webkit-httpd index 018f64c..9ea2551 100755 --- a/WebKitTools/Scripts/run-webkit-httpd +++ b/WebKitTools/Scripts/run-webkit-httpd @@ -42,6 +42,10 @@ use lib $FindBin::Bin; use webkitperl::httpd; use webkitdirs; +# FIXME: Dynamic HTTP-port configuration in this file is wrong. The various +# apache config files in LayoutTests/http/config govern the port numbers. +# Dynamic configuration as-written will also cause random failures in +# an IPv6 environment. See https://bugs.webkit.org/show_bug.cgi?id=37104. # Argument handling my $httpdPort = 8000; my $allInterfaces = 0; diff --git a/WebKitTools/Scripts/run-webkit-tests b/WebKitTools/Scripts/run-webkit-tests index 809e078..f28f20a 100755 --- a/WebKitTools/Scripts/run-webkit-tests +++ b/WebKitTools/Scripts/run-webkit-tests @@ -1,2222 +1,81 @@ #!/usr/bin/perl - -# Copyright (C) 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved. -# Copyright (C) 2006 Alexey Proskuryakov (ap@nypop.com) -# Copyright (C) 2007 Matt Lilek (pewtermoose@gmail.com) -# Copyright (C) 2007 Eric Seidel <eric@webkit.org> -# Copyright (C) 2009 Google Inc. All rights reserved. -# Copyright (C) 2009 Andras Becsi (becsi.andras@stud.u-szeged.hu), University of Szeged +# Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: +# modification, are permitted provided that the following conditions are +# met: # -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of -# its contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. # -# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Script to run the WebKit Open Source Project layout tests. - -# Run all the tests passed in on the command line. -# If no tests are passed, find all the .html, .shtml, .xml, .xhtml, .pl, .php (and svg) files in the test directory. - -# Run each text. -# Compare against the existing file xxx-expected.txt. -# If there is a mismatch, generate xxx-actual.txt and xxx-diffs.txt. - -# At the end, report: -# the number of tests that got the expected results -# the number of tests that ran, but did not get the expected results -# the number of tests that failed to run -# the number of tests that were run but had no expected results to compare against +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This file is a temporary hack. +# It will be removed as soon as all platforms are are ready to move to +# new-run-webkit-tests and we can then update the buildbots to explicitly +# call old-run-webkit-tests for any platforms which will never support +# a Python run-webkit-tests. + +# This is intentionally written in Perl to guarantee support on +# the same set of platforms as old-run-webkit-tests currently supports. +# The buildbot master.cfg also currently passes run-webkit-tests to +# perl directly instead of executing it in a shell. use strict; use warnings; -use Cwd; -use Data::Dumper; -use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK); -use File::Basename; -use File::Copy; -use File::Find; -use File::Path; -use File::Spec; -use File::Spec::Functions; use FindBin; -use Getopt::Long; -use IPC::Open2; -use IPC::Open3; -use Time::HiRes qw(time usleep); - -use List::Util 'shuffle'; - use lib $FindBin::Bin; -use webkitperl::features; -use webkitperl::httpd; use webkitdirs; -use VCSUtils; -use POSIX; - -sub buildPlatformResultHierarchy(); -sub buildPlatformTestHierarchy(@); -sub closeCygpaths(); -sub closeDumpTool(); -sub closeWebSocketServer(); -sub configureAndOpenHTTPDIfNeeded(); -sub countAndPrintLeaks($$$); -sub countFinishedTest($$$$); -sub deleteExpectedAndActualResults($); -sub dumpToolDidCrash(); -sub epiloguesAndPrologues($$); -sub expectedDirectoryForTest($;$;$); -sub fileNameWithNumber($$); -sub htmlForResultsSection(\@$&); -sub isTextOnlyTest($); -sub launchWithEnv(\@\%); -sub resolveAndMakeTestResultsDirectory(); -sub numericcmp($$); -sub openDiffTool(); -sub openDumpTool(); -sub parseLeaksandPrintUniqueLeaks(); -sub openWebSocketServerIfNeeded(); -sub pathcmp($$); -sub printFailureMessageForTest($$); -sub processIgnoreTests($$); -sub readFromDumpToolWithTimer(**); -sub readSkippedFiles($); -sub recordActualResultsAndDiff($$); -sub sampleDumpTool(); -sub setFileHandleNonBlocking(*$); -sub slowestcmp($$); -sub splitpath($); -sub stripExtension($); -sub stripMetrics($$); -sub testCrashedOrTimedOut($$$$$); -sub toURL($); -sub toWindowsPath($); -sub validateSkippedArg($$;$); -sub writeToFile($$); - -# Argument handling -my $addPlatformExceptions = 0; -my $complexText = 0; -my $exitAfterNFailures = 0; -my $generateNewResults = isAppleMacWebKit() ? 1 : 0; -my $guardMalloc = ''; -my $httpdPort = 8000; -my $httpdSSLPort = 8443; -my $ignoreMetrics = 0; -my $webSocketPort = 8880; -# wss is disabled until all platforms support pyOpenSSL. -# my $webSocketSecurePort = 9323; -my $ignoreTests = ''; -my $iterations = 1; -my $launchSafari = 1; -my $mergeDepth; -my $pixelTests = ''; -my $platform; -my $quiet = ''; -my $randomizeTests = 0; -my $repeatEach = 1; -my $report10Slowest = 0; -my $resetResults = 0; -my $reverseTests = 0; -my $root; -my $runSample = 1; -my $shouldCheckLeaks = 0; -my $showHelp = 0; -my $stripEditingCallbacks = isCygwin(); -my $testHTTP = 1; -my $testMedia = 1; -my $tmpDir = "/tmp"; -my $testResultsDirectory = File::Spec->catfile($tmpDir, "layout-test-results"); -my $testsPerDumpTool = 1000; -my $threaded = 0; -# DumpRenderTree has an internal timeout of 15 seconds, so this must be > 15. -my $timeoutSeconds = 20; -my $tolerance = 0; -my $treatSkipped = "default"; -my $useRemoteLinksToTests = 0; -my $useValgrind = 0; -my $verbose = 0; -my $shouldWaitForHTTPD = 0; - -my @leaksFilenames; - -if (isWindows() || isMsys()) { - print "This script has to be run under Cygwin to function correctly.\n"; - exit 1; -} - -# Default to --no-http for wx for now. -$testHTTP = 0 if (isWx()); - -my $expectedTag = "expected"; -my $actualTag = "actual"; -my $prettyDiffTag = "pretty-diff"; -my $diffsTag = "diffs"; -my $errorTag = "stderr"; - -my @macPlatforms = ("mac-tiger", "mac-leopard", "mac-snowleopard", "mac"); - -if (isAppleMacWebKit()) { - if (isTiger()) { - $platform = "mac-tiger"; - $tolerance = 1.0; - } elsif (isLeopard()) { - $platform = "mac-leopard"; - $tolerance = 0.1; - } elsif (isSnowLeopard()) { - $platform = "mac-snowleopard"; - $tolerance = 0.1; - } else { - $platform = "mac"; - } -} elsif (isQt()) { - if (isDarwin()) { - $platform = "qt-mac"; - } elsif (isLinux()) { - $platform = "qt-linux"; - } elsif (isWindows() || isCygwin()) { - $platform = "qt-win"; - } else { - $platform = "qt"; - } -} elsif (isGtk()) { - $platform = "gtk"; - if (!$ENV{"WEBKIT_TESTFONTS"}) { - print "The WEBKIT_TESTFONTS environment variable is not defined.\n"; - print "You must set it before running the tests.\n"; - print "Use git to grab the actual fonts from http://gitorious.org/qtwebkit/testfonts\n"; - exit 1; - } -} elsif (isWx()) { - $platform = "wx"; -} elsif (isCygwin()) { - $platform = "win"; -} - -if (!defined($platform)) { - print "WARNING: Your platform is not recognized. Any platform-specific results will be generated in platform/undefined.\n"; - $platform = "undefined"; -} - -my $programName = basename($0); -my $launchSafariDefault = $launchSafari ? "launch" : "do not launch"; -my $httpDefault = $testHTTP ? "run" : "do not run"; -my $sampleDefault = $runSample ? "run" : "do not run"; - -my $usage = <<EOF; -Usage: $programName [options] [testdir|testpath ...] - --add-platform-exceptions Put new results for non-platform-specific failing tests into the platform-specific results directory - --complex-text Use the complex text code path for all text (Mac OS X and Windows only) - -c|--configuration config Set DumpRenderTree build configuration - -g|--guard-malloc Enable malloc guard - --exit-after-n-failures N Exit after the first N failures instead of running all tests - -h|--help Show this help message - --[no-]http Run (or do not run) http tests (default: $httpDefault) - --[no-]wait-for-httpd Wait for httpd if some other test session is using it already (same as WEBKIT_WAIT_FOR_HTTPD=1). (default: $shouldWaitForHTTPD) - -i|--ignore-tests Comma-separated list of directories or tests to ignore - --iterations n Number of times to run the set of tests (e.g. ABCABCABC) - --[no-]launch-safari Launch (or do not launch) Safari to display test results (default: $launchSafariDefault) - -l|--leaks Enable leaks checking - --[no-]new-test-results Generate results for new tests - --nthly n Restart DumpRenderTree every n tests (default: $testsPerDumpTool) - -p|--pixel-tests Enable pixel tests - --tolerance t Ignore image differences less than this percentage (default: $tolerance) - --platform Override the detected platform to use for tests and results (default: $platform) - --port Web server port to use with http tests - -q|--quiet Less verbose output - --reset-results Reset ALL results (including pixel tests if --pixel-tests is set) - -o|--results-directory Output results directory (default: $testResultsDirectory) - --random Run the tests in a random order - --repeat-each n Number of times to run each test (e.g. AAABBBCCC) - --reverse Run the tests in reverse alphabetical order - --root Path to root tools build - --[no-]sample-on-timeout Run sample on timeout (default: $sampleDefault) (Mac OS X only) - -1|--singly Isolate each test case run (implies --nthly 1 --verbose) - --skipped=[default|ignore|only] Specifies how to treat the Skipped file - default: Tests/directories listed in the Skipped file are not tested - ignore: The Skipped file is ignored - only: Only those tests/directories listed in the Skipped file will be run - --slowest Report the 10 slowest tests - --ignore-metrics Ignore metrics in tests - --[no-]strip-editing-callbacks Remove editing callbacks from expected results - -t|--threaded Run a concurrent JavaScript thead with each test - --timeout t Sets the number of seconds before a test times out (default: $timeoutSeconds) - --valgrind Run DumpRenderTree inside valgrind (Qt/Linux only) - -v|--verbose More verbose output (overrides --quiet) - -m|--merge-leak-depth arg Merges leak callStacks and prints the number of unique leaks beneath a callstack depth of arg. Defaults to 5. - --use-remote-links-to-tests Link to test files within the SVN repository in the results. -EOF - -setConfiguration(); - -my $getOptionsResult = GetOptions( - 'add-platform-exceptions' => \$addPlatformExceptions, - 'complex-text' => \$complexText, - 'exit-after-n-failures=i' => \$exitAfterNFailures, - 'guard-malloc|g' => \$guardMalloc, - 'help|h' => \$showHelp, - 'http!' => \$testHTTP, - 'wait-for-httpd!' => \$shouldWaitForHTTPD, - 'ignore-metrics!' => \$ignoreMetrics, - 'ignore-tests|i=s' => \$ignoreTests, - 'iterations=i' => \$iterations, - 'launch-safari!' => \$launchSafari, - 'leaks|l' => \$shouldCheckLeaks, - 'merge-leak-depth|m:5' => \$mergeDepth, - 'new-test-results!' => \$generateNewResults, - 'nthly=i' => \$testsPerDumpTool, - 'pixel-tests|p' => \$pixelTests, - 'platform=s' => \$platform, - 'port=i' => \$httpdPort, - 'quiet|q' => \$quiet, - 'random' => \$randomizeTests, - 'repeat-each=i' => \$repeatEach, - 'reset-results' => \$resetResults, - 'results-directory|o=s' => \$testResultsDirectory, - 'reverse' => \$reverseTests, - 'root=s' => \$root, - 'sample-on-timeout!' => \$runSample, - 'singly|1' => sub { $testsPerDumpTool = 1; }, - 'skipped=s' => \&validateSkippedArg, - 'slowest' => \$report10Slowest, - 'strip-editing-callbacks!' => \$stripEditingCallbacks, - 'threaded|t' => \$threaded, - 'timeout=i' => \$timeoutSeconds, - 'tolerance=f' => \$tolerance, - 'use-remote-links-to-tests' => \$useRemoteLinksToTests, - 'valgrind' => \$useValgrind, - 'verbose|v' => \$verbose, -); - -if (!$getOptionsResult || $showHelp) { - print STDERR $usage; - exit 1; -} - -my $ignoreSkipped = $treatSkipped eq "ignore"; -my $skippedOnly = $treatSkipped eq "only"; - -my $configuration = configuration(); - -# We need an environment variable to be able to enable the feature per-slave -$shouldWaitForHTTPD = $ENV{"WEBKIT_WAIT_FOR_HTTPD"} unless ($shouldWaitForHTTPD); -$verbose = 1 if $testsPerDumpTool == 1; - -if ($shouldCheckLeaks && $testsPerDumpTool > 1000) { - print STDERR "\nWARNING: Running more than 1000 tests at a time with MallocStackLogging enabled may cause a crash.\n\n"; -} - -# Stack logging does not play well with QuickTime on Tiger (rdar://problem/5537157) -$testMedia = 0 if $shouldCheckLeaks && isTiger(); - -# Generating remote links causes a lot of unnecessary spew on GTK build bot -$useRemoteLinksToTests = 0 if isGtk(); - -setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root)); -my $productDir = productDir(); -$productDir .= "/bin" if isQt(); -$productDir .= "/Programs" if isGtk(); - -chdirWebKit(); - -if (!defined($root)) { - print STDERR "Running build-dumprendertree\n"; - - local *DEVNULL; - my ($childIn, $childOut, $childErr); - if ($quiet) { - open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null"; - $childOut = ">&DEVNULL"; - $childErr = ">&DEVNULL"; - } else { - # When not quiet, let the child use our stdout/stderr. - $childOut = ">&STDOUT"; - $childErr = ">&STDERR"; - } - - my @args = argumentsForConfiguration(); - my $buildProcess = open3($childIn, $childOut, $childErr, "WebKitTools/Scripts/build-dumprendertree", @args) or die "Failed to run build-dumprendertree"; - close($childIn); - waitpid $buildProcess, 0; - my $buildResult = $?; - close($childOut); - close($childErr); - - close DEVNULL if ($quiet); - - if ($buildResult) { - print STDERR "Compiling DumpRenderTree failed!\n"; - exit exitStatus($buildResult); - } -} - -my $dumpToolName = "DumpRenderTree"; -$dumpToolName .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/; -my $dumpTool = "$productDir/$dumpToolName"; -die "can't find executable $dumpToolName (looked in $productDir)\n" unless -x $dumpTool; - -my $imageDiffTool = "$productDir/ImageDiff"; -$imageDiffTool .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/; -die "can't find executable $imageDiffTool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool; - -checkFrameworks() unless isCygwin(); - -if (isAppleMacWebKit()) { - push @INC, $productDir; - require DumpRenderTreeSupport; -} - -my $layoutTestsName = "LayoutTests"; -my $testDirectory = File::Spec->rel2abs($layoutTestsName); -my $expectedDirectory = $testDirectory; -my $platformBaseDirectory = catdir($testDirectory, "platform"); -my $platformTestDirectory = catdir($platformBaseDirectory, $platform); -my @platformResultHierarchy = buildPlatformResultHierarchy(); -my @platformTestHierarchy = buildPlatformTestHierarchy(@platformResultHierarchy); - -$expectedDirectory = $ENV{"WebKitExpectedTestResultsDirectory"} if $ENV{"WebKitExpectedTestResultsDirectory"}; - -$testResultsDirectory = File::Spec->rel2abs($testResultsDirectory); -my $testResults = File::Spec->catfile($testResultsDirectory, "results.html"); - -print "Running tests from $testDirectory\n"; -if ($pixelTests) { - print "Enabling pixel tests with a tolerance of $tolerance%\n"; - if (isDarwin()) { - print "WARNING: Temporarily changing the main display color profile:\n"; - print "\tThe colors on your screen will change for the duration of the testing.\n"; - print "\tThis allows the pixel tests to have consistent color values across all machines.\n"; - - if (isPerianInstalled()) { - print "WARNING: Perian's QuickTime component is installed and this may affect pixel test results!\n"; - print "\tYou should avoid generating new pixel results in this environment.\n"; - print "\tSee https://bugs.webkit.org/show_bug.cgi?id=22615 for details.\n"; - } - } -} - -system "ln", "-s", $testDirectory, "/tmp/LayoutTests" unless -x "/tmp/LayoutTests"; - -my %ignoredFiles = ( "results.html" => 1 ); -my %ignoredDirectories = map { $_ => 1 } qw(platform); -my %ignoredLocalDirectories = map { $_ => 1 } qw(.svn _svn resources script-tests); -my %supportedFileExtensions = map { $_ => 1 } qw(html shtml xml xhtml pl php); - -if (!checkWebCoreFeatureSupport("MathML", 0)) { - $ignoredDirectories{'mathml'} = 1; -} - -# FIXME: We should fix webkitperl/features.pm:hasFeature() to do the correct feature detection for Cygwin. -if (checkWebCoreFeatureSupport("SVG", 0)) { - $supportedFileExtensions{'svg'} = 1; -} elsif (isCygwin()) { - $supportedFileExtensions{'svg'} = 1; -} else { - $ignoredLocalDirectories{'svg'} = 1; -} - -if (!$testHTTP) { - $ignoredDirectories{'http'} = 1; - $ignoredDirectories{'websocket'} = 1; -} - -if (!$testMedia) { - $ignoredDirectories{'media'} = 1; - $ignoredDirectories{'http/tests/media'} = 1; -} - -if (!checkWebCoreFeatureSupport("Accelerated Compositing", 0)) { - $ignoredDirectories{'compositing'} = 1; -} - -if (!checkWebCoreFeatureSupport("3D Rendering", 0)) { - $ignoredDirectories{'animations/3d'} = 1; - $ignoredDirectories{'transforms/3d'} = 1; -} - -if (!checkWebCoreFeatureSupport("3D Canvas", 0)) { - $ignoredDirectories{'fast/canvas/webgl'} = 1; -} - -if (checkWebCoreFeatureSupport("WML", 0)) { - $supportedFileExtensions{'wml'} = 1; -} else { - $ignoredDirectories{'http/tests/wml'} = 1; - $ignoredDirectories{'fast/wml'} = 1; - $ignoredDirectories{'wml'} = 1; -} - -if (!checkWebCoreFeatureSupport("XHTMLMP", 0)) { - $ignoredDirectories{'fast/xhtmlmp'} = 1; -} - -processIgnoreTests($ignoreTests, "ignore-tests") if $ignoreTests; -if (!$ignoreSkipped) { - if (!$skippedOnly || @ARGV == 0) { - readSkippedFiles(""); - } else { - # Since readSkippedFiles() appends to @ARGV, we must use a foreach - # loop so that we only iterate over the original argument list. - foreach my $argnum (0 .. $#ARGV) { - readSkippedFiles(shift @ARGV); - } - } -} - -my @tests = findTestsToRun(); - -die "no tests to run\n" if !@tests; - -my %counts; -my %tests; -my %imagesPresent; -my %imageDifferences; -my %durations; -my $count = 0; -my $leaksOutputFileNumber = 1; -my $totalLeaks = 0; - -my @toolArgs = (); -push @toolArgs, "--pixel-tests" if $pixelTests; -push @toolArgs, "--threaded" if $threaded; -push @toolArgs, "--complex-text" if $complexText; -push @toolArgs, "-"; - -my @diffToolArgs = (); -push @diffToolArgs, "--tolerance", $tolerance; - -$| = 1; - -my $dumpToolPID; -my $isDumpToolOpen = 0; -my $dumpToolCrashed = 0; -my $imageDiffToolPID; -my $isDiffToolOpen = 0; - -my $atLineStart = 1; -my $lastDirectory = ""; - -my $isHttpdOpen = 0; -my $isWebSocketServerOpen = 0; -my $webSocketServerPID = 0; -my $failedToStartWebSocketServer = 0; -# wss is disabled until all platforms support pyOpenSSL. -# my $webSocketSecureServerPID = 0; - -sub catch_pipe { $dumpToolCrashed = 1; } -$SIG{"PIPE"} = "catch_pipe"; - -print "Testing ", scalar @tests, " test cases"; -print " $iterations times" if ($iterations > 1); -print ", repeating each test $repeatEach times" if ($repeatEach > 1); -print ".\n"; - -my $overallStartTime = time; - -my %expectedResultPaths; - -my @originalTests = @tests; -# Add individual test repetitions -if ($repeatEach > 1) { - @tests = (); - foreach my $test (@originalTests) { - for (my $i = 0; $i < $repeatEach; $i++) { - push(@tests, $test); - } - } -} -# Add test set repetitions -for (my $i = 1; $i < $iterations; $i++) { - push(@tests, @originalTests); -} - -for my $test (@tests) { - my $newDumpTool = not $isDumpToolOpen; - openDumpTool(); - - my $base = stripExtension($test); - my $expectedExtension = ".txt"; - - my $dir = $base; - $dir =~ s|/[^/]+$||; - - if ($newDumpTool || $dir ne $lastDirectory) { - foreach my $logue (epiloguesAndPrologues($newDumpTool ? "" : $lastDirectory, $dir)) { - if (isCygwin()) { - $logue = toWindowsPath($logue); - } else { - $logue = canonpath($logue); - } - if ($verbose) { - print "running epilogue or prologue $logue\n"; - } - print OUT "$logue\n"; - # Throw away output from DumpRenderTree. - # Once for the test output and once for pixel results (empty) - while (<IN>) { - last if /#EOF/; - } - while (<IN>) { - last if /#EOF/; - } - } - } - - if ($verbose) { - print "running $test -> "; - $atLineStart = 0; - } elsif (!$quiet) { - if ($dir ne $lastDirectory) { - print "\n" unless $atLineStart; - print "$dir "; - } - print "."; - $atLineStart = 0; - } - - $lastDirectory = $dir; - - my $result; - - my $startTime = time if $report10Slowest; - - # Try to read expected hash file for pixel tests - my $suffixExpectedHash = ""; - if ($pixelTests && !$resetResults) { - my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); - if (open EXPECTEDHASH, "$expectedPixelDir/$base-$expectedTag.checksum") { - my $expectedHash = <EXPECTEDHASH>; - chomp($expectedHash); - close EXPECTEDHASH; - - # Format expected hash into a suffix string that is appended to the path / URL passed to DRT - $suffixExpectedHash = "'$expectedHash"; - } - } - - if ($test =~ /^http\//) { - configureAndOpenHTTPDIfNeeded(); - if ($test !~ /^http\/tests\/local\// && $test !~ /^http\/tests\/ssl\// && $test !~ /^http\/tests\/wml\// && $test !~ /^http\/tests\/media\//) { - my $path = canonpath($test); - $path =~ s/^http\/tests\///; - print OUT "http://127.0.0.1:$httpdPort/$path$suffixExpectedHash\n"; - } elsif ($test =~ /^http\/tests\/ssl\//) { - my $path = canonpath($test); - $path =~ s/^http\/tests\///; - print OUT "https://127.0.0.1:$httpdSSLPort/$path$suffixExpectedHash\n"; - } else { - my $testPath = "$testDirectory/$test"; - if (isCygwin()) { - $testPath = toWindowsPath($testPath); - } else { - $testPath = canonpath($testPath); - } - print OUT "$testPath$suffixExpectedHash\n"; - } - } elsif ($test =~ /^websocket\//) { - if ($test =~ /^websocket\/tests\/local\//) { - my $testPath = "$testDirectory/$test"; - if (isCygwin()) { - $testPath = toWindowsPath($testPath); - } else { - $testPath = canonpath($testPath); - } - print OUT "$testPath\n"; - } else { - if (openWebSocketServerIfNeeded()) { - my $path = canonpath($test); - if ($test =~ /^websocket\/tests\/ssl\//) { - # wss is disabled until all platforms support pyOpenSSL. - print STDERR "Error: wss is disabled until all platforms support pyOpenSSL."; - # print OUT "https://127.0.0.1:$webSocketSecurePort/$path\n"; - } else { - print OUT "http://127.0.0.1:$webSocketPort/$path\n"; - } - } else { - # We failed to launch the WebSocket server. Display a useful error message rather than attempting - # to run tests that expect the server to be available. - my $errorMessagePath = "$testDirectory/websocket/resources/server-failed-to-start.html"; - $errorMessagePath = isCygwin() ? toWindowsPath($errorMessagePath) : canonpath($errorMessagePath); - print OUT "$errorMessagePath\n"; - } - } - } else { - my $testPath = "$testDirectory/$test"; - if (isCygwin()) { - $testPath = toWindowsPath($testPath); - } else { - $testPath = canonpath($testPath); - } - print OUT "$testPath$suffixExpectedHash\n" if defined $testPath; - } - - # DumpRenderTree is expected to dump two "blocks" to stdout for each test. - # Each block is terminated by a #EOF on a line by itself. - # The first block is the output of the test (in text, RenderTree or other formats). - # The second block is for optional pixel data in PNG format, and may be empty if - # pixel tests are not being run, or the test does not dump pixels (e.g. text tests). - my $readResults = readFromDumpToolWithTimer(IN, ERROR); - - my $actual = $readResults->{output}; - my $error = $readResults->{error}; - - $expectedExtension = $readResults->{extension}; - my $expectedFileName = "$base-$expectedTag.$expectedExtension"; - - my $isText = isTextOnlyTest($actual); - - my $expectedDir = expectedDirectoryForTest($base, $isText, $expectedExtension); - $expectedResultPaths{$base} = "$expectedDir/$expectedFileName"; - - unless ($readResults->{status} eq "success") { - my $crashed = $readResults->{status} eq "crashed"; - testCrashedOrTimedOut($test, $base, $crashed, $actual, $error); - countFinishedTest($test, $base, $crashed ? "crash" : "timedout", 0); - next; - } - - $durations{$test} = time - $startTime if $report10Slowest; - - my $expected; - - if (!$resetResults && open EXPECTED, "<", "$expectedDir/$expectedFileName") { - $expected = ""; - while (<EXPECTED>) { - next if $stripEditingCallbacks && $_ =~ /^EDITING DELEGATE:/; - $expected .= $_; - } - close EXPECTED; - } - - if ($ignoreMetrics && !$isText && defined $expected) { - ($actual, $expected) = stripMetrics($actual, $expected); - } - - if ($shouldCheckLeaks && $testsPerDumpTool == 1) { - print " $test -> "; - } - - my $actualPNG = ""; - my $diffPNG = ""; - my $diffPercentage = 0; - my $diffResult = "passed"; - - my $actualHash = ""; - my $expectedHash = ""; - my $actualPNGSize = 0; - - while (<IN>) { - last if /#EOF/; - if (/ActualHash: ([a-f0-9]{32})/) { - $actualHash = $1; - } elsif (/ExpectedHash: ([a-f0-9]{32})/) { - $expectedHash = $1; - } elsif (/Content-Length: (\d+)\s*/) { - $actualPNGSize = $1; - read(IN, $actualPNG, $actualPNGSize); - } - } - - if ($verbose && $pixelTests && !$resetResults && $actualPNGSize) { - if ($actualHash eq "" && $expectedHash eq "") { - printFailureMessageForTest($test, "WARNING: actual & expected pixel hashes are missing!"); - } elsif ($actualHash eq "") { - printFailureMessageForTest($test, "WARNING: actual pixel hash is missing!"); - } elsif ($expectedHash eq "") { - printFailureMessageForTest($test, "WARNING: expected pixel hash is missing!"); - } - } - if ($actualPNGSize > 0) { - my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); - - if (!$resetResults && ($expectedHash ne $actualHash || ($actualHash eq "" && $expectedHash eq ""))) { - if (-f "$expectedPixelDir/$base-$expectedTag.png") { - my $expectedPNGSize = -s "$expectedPixelDir/$base-$expectedTag.png"; - my $expectedPNG = ""; - open EXPECTEDPNG, "$expectedPixelDir/$base-$expectedTag.png"; - read(EXPECTEDPNG, $expectedPNG, $expectedPNGSize); - - openDiffTool(); - print DIFFOUT "Content-Length: $actualPNGSize\n"; - print DIFFOUT $actualPNG; - - print DIFFOUT "Content-Length: $expectedPNGSize\n"; - print DIFFOUT $expectedPNG; - - while (<DIFFIN>) { - last if /^error/ || /^diff:/; - if (/Content-Length: (\d+)\s*/) { - read(DIFFIN, $diffPNG, $1); - } - } - - if (/^diff: (.+)% (passed|failed)/) { - $diffPercentage = $1 + 0; - $imageDifferences{$base} = $diffPercentage; - $diffResult = $2; - } - - if (!$diffPercentage) { - printFailureMessageForTest($test, "pixel hash failed (but pixel test still passes)"); - } - } elsif ($verbose) { - printFailureMessageForTest($test, "WARNING: expected image is missing!"); - } - } - - if ($resetResults || !-f "$expectedPixelDir/$base-$expectedTag.png") { - mkpath catfile($expectedPixelDir, dirname($base)) if $testDirectory ne $expectedPixelDir; - writeToFile("$expectedPixelDir/$base-$expectedTag.png", $actualPNG); - } - - if ($actualHash ne "" && ($resetResults || !-f "$expectedPixelDir/$base-$expectedTag.checksum")) { - writeToFile("$expectedPixelDir/$base-$expectedTag.checksum", $actualHash); - } - } - - if (dumpToolDidCrash()) { - $result = "crash"; - testCrashedOrTimedOut($test, $base, 1, $actual, $error); - } elsif (!defined $expected) { - if ($verbose) { - print "new " . ($resetResults ? "result" : "test") ."\n"; - $atLineStart = 1; - } - $result = "new"; - - if ($generateNewResults || $resetResults) { - mkpath catfile($expectedDir, dirname($base)) if $testDirectory ne $expectedDir; - writeToFile("$expectedDir/$expectedFileName", $actual); - } - deleteExpectedAndActualResults($base); - recordActualResultsAndDiff($base, $actual); - if (!$resetResults) { - # Always print the file name for new tests, as they will probably need some manual inspection. - # in verbose mode we already printed the test case, so no need to do it again. - unless ($verbose) { - print "\n" unless $atLineStart; - print "$test -> "; - } - my $resultsDir = catdir($expectedDir, dirname($base)); - if ($generateNewResults) { - print "new (results generated in $resultsDir)\n"; - } else { - print "new\n"; - } - $atLineStart = 1; - } - } elsif ($actual eq $expected && $diffResult eq "passed") { - if ($verbose) { - print "succeeded\n"; - $atLineStart = 1; - } - $result = "match"; - deleteExpectedAndActualResults($base); - } else { - $result = "mismatch"; - - my $pixelTestFailed = $pixelTests && $diffPNG && $diffPNG ne ""; - my $testFailed = $actual ne $expected; - - my $message = !$testFailed ? "pixel test failed" : "failed"; - - if (($testFailed || $pixelTestFailed) && $addPlatformExceptions) { - my $testBase = catfile($testDirectory, $base); - my $expectedBase = catfile($expectedDir, $base); - my $testIsMaximallyPlatformSpecific = $testBase =~ m|^\Q$platformTestDirectory\E/|; - my $expectedResultIsMaximallyPlatformSpecific = $expectedBase =~ m|^\Q$platformTestDirectory\E/|; - if (!$testIsMaximallyPlatformSpecific && !$expectedResultIsMaximallyPlatformSpecific) { - mkpath catfile($platformTestDirectory, dirname($base)); - if ($testFailed) { - my $expectedFile = catfile($platformTestDirectory, "$expectedFileName"); - writeToFile("$expectedFile", $actual); - } - if ($pixelTestFailed) { - my $expectedFile = catfile($platformTestDirectory, "$base-$expectedTag.checksum"); - writeToFile("$expectedFile", $actualHash); - - $expectedFile = catfile($platformTestDirectory, "$base-$expectedTag.png"); - writeToFile("$expectedFile", $actualPNG); - } - $message .= " (results generated in $platformTestDirectory)"; - } - } - - printFailureMessageForTest($test, $message); - - my $dir = "$testResultsDirectory/$base"; - $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; - my $testName = $1; - mkpath $dir; - - deleteExpectedAndActualResults($base); - recordActualResultsAndDiff($base, $actual); - - if ($pixelTestFailed) { - $imagesPresent{$base} = 1; - - writeToFile("$testResultsDirectory/$base-$actualTag.png", $actualPNG); - writeToFile("$testResultsDirectory/$base-$diffsTag.png", $diffPNG); - - my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); - copy("$expectedPixelDir/$base-$expectedTag.png", "$testResultsDirectory/$base-$expectedTag.png"); - - open DIFFHTML, ">$testResultsDirectory/$base-$diffsTag.html" or die; - print DIFFHTML "<html>\n"; - print DIFFHTML "<head>\n"; - print DIFFHTML "<title>$base Image Compare</title>\n"; - print DIFFHTML "<script language=\"Javascript\" type=\"text/javascript\">\n"; - print DIFFHTML "var currentImage = 0;\n"; - print DIFFHTML "var imageNames = new Array(\"Actual\", \"Expected\");\n"; - print DIFFHTML "var imagePaths = new Array(\"$testName-$actualTag.png\", \"$testName-$expectedTag.png\");\n"; - if (-f "$testDirectory/$base-w3c.png") { - copy("$testDirectory/$base-w3c.png", "$testResultsDirectory/$base-w3c.png"); - print DIFFHTML "imageNames.push(\"W3C\");\n"; - print DIFFHTML "imagePaths.push(\"$testName-w3c.png\");\n"; - } - print DIFFHTML "function animateImage() {\n"; - print DIFFHTML " var image = document.getElementById(\"animatedImage\");\n"; - print DIFFHTML " var imageText = document.getElementById(\"imageText\");\n"; - print DIFFHTML " image.src = imagePaths[currentImage];\n"; - print DIFFHTML " imageText.innerHTML = imageNames[currentImage] + \" Image\";\n"; - print DIFFHTML " currentImage = (currentImage + 1) % imageNames.length;\n"; - print DIFFHTML " setTimeout('animateImage()',2000);\n"; - print DIFFHTML "}\n"; - print DIFFHTML "</script>\n"; - print DIFFHTML "</head>\n"; - print DIFFHTML "<body onLoad=\"animateImage();\">\n"; - print DIFFHTML "<table>\n"; - if ($diffPercentage) { - print DIFFHTML "<tr>\n"; - print DIFFHTML "<td>Difference between images: <a href=\"$testName-$diffsTag.png\">$diffPercentage%</a></td>\n"; - print DIFFHTML "</tr>\n"; - } - print DIFFHTML "<tr>\n"; - print DIFFHTML "<td><a href=\"" . toURL("$testDirectory/$test") . "\">test file</a></td>\n"; - print DIFFHTML "</tr>\n"; - print DIFFHTML "<tr>\n"; - print DIFFHTML "<td id=\"imageText\" style=\"text-weight: bold;\">Actual Image</td>\n"; - print DIFFHTML "</tr>\n"; - print DIFFHTML "<tr>\n"; - print DIFFHTML "<td><img src=\"$testName-$actualTag.png\" id=\"animatedImage\"></td>\n"; - print DIFFHTML "</tr>\n"; - print DIFFHTML "</table>\n"; - print DIFFHTML "</body>\n"; - print DIFFHTML "</html>\n"; - } - } - - if ($error) { - my $dir = "$testResultsDirectory/$base"; - $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; - mkpath $dir; - - writeToFile("$testResultsDirectory/$base-$errorTag.txt", $error); - - $counts{error}++; - push @{$tests{error}}, $test; - } - - countFinishedTest($test, $base, $result, $isText); - - # --reset-results does not check pass vs. fail, so exitAfterNFailures makes no sense with --reset-results. - if ($exitAfterNFailures && !$resetResults) { - my $passCount = $counts{match} || 0; # $counts{match} will be undefined if we've not yet passed a test (e.g. the first test fails). - my $failureCount = $count - $passCount; # "Failure" here includes new tests, timeouts, crashes, etc. - if ($failureCount >= $exitAfterNFailures) { - print "\nExiting early after $failureCount failures. $count tests run."; - closeDumpTool(); - last; - } - } -} -my $totalTestingTime = time - $overallStartTime; -my $waitTime = getWaitTime(); -if ($waitTime > 0.1) { - my $normalizedTestingTime = $totalTestingTime - $waitTime; - printf "\n%0.2fs HTTPD waiting time\n", $waitTime . ""; - printf "%0.2fs normalized testing time", $normalizedTestingTime . ""; -} -printf "\n%0.2fs total testing time\n", $totalTestingTime . ""; - -!$isDumpToolOpen || die "Failed to close $dumpToolName.\n"; - -$isHttpdOpen = !closeHTTPD(); -closeWebSocketServer(); - -# Because multiple instances of this script are running concurrently we cannot -# safely delete this symlink. -# system "rm /tmp/LayoutTests"; - -# FIXME: Do we really want to check the image-comparison tool for leaks every time? -if ($isDiffToolOpen && $shouldCheckLeaks) { - $totalLeaks += countAndPrintLeaks("ImageDiff", $imageDiffToolPID, "$testResultsDirectory/ImageDiff-leaks.txt"); -} - -if ($totalLeaks) { - if ($mergeDepth) { - parseLeaksandPrintUniqueLeaks(); - } else { - print "\nWARNING: $totalLeaks total leaks found!\n"; - print "See above for individual leaks results.\n" if ($leaksOutputFileNumber > 2); - } -} - -close IN; -close OUT; -close ERROR; - -if ($report10Slowest) { - print "\n\nThe 10 slowest tests:\n\n"; - my $count = 0; - for my $test (sort slowestcmp keys %durations) { - printf "%0.2f secs: %s\n", $durations{$test}, $test; - last if ++$count == 10; - } -} - -print "\n"; - -if ($skippedOnly && $counts{"match"}) { - print "The following tests are in the Skipped file (" . File::Spec->abs2rel("$platformTestDirectory/Skipped", $testDirectory) . "), but succeeded:\n"; - foreach my $test (@{$tests{"match"}}) { - print " $test\n"; - } -} - -if ($resetResults || ($counts{match} && $counts{match} == $count)) { - print "all $count test cases succeeded\n"; - unlink $testResults; - exit; -} - -printResults(); - -mkpath $testResultsDirectory; - -open HTML, ">", $testResults or die "Failed to open $testResults. $!"; -print HTML "<html>\n"; -print HTML "<head>\n"; -print HTML "<title>Layout Test Results</title>\n"; -print HTML "</head>\n"; -print HTML "<body>\n"; - -if ($ignoreMetrics) { - print HTML "<h4>Tested with metrics ignored.</h4>"; -} - -print HTML htmlForResultsSection(@{$tests{mismatch}}, "Tests where results did not match expected results", \&linksForMismatchTest); -print HTML htmlForResultsSection(@{$tests{timedout}}, "Tests that timed out", \&linksForErrorTest); -print HTML htmlForResultsSection(@{$tests{crash}}, "Tests that caused the DumpRenderTree tool to crash", \&linksForErrorTest); -print HTML htmlForResultsSection(@{$tests{error}}, "Tests that had stderr output", \&linksForErrorTest); -print HTML htmlForResultsSection(@{$tests{new}}, "Tests that had no expected results (probably new)", \&linksForNewTest); - -print HTML "</body>\n"; -print HTML "</html>\n"; -close HTML; - -my @configurationArgs = argumentsForConfiguration(); - -if (isGtk()) { - system "WebKitTools/Scripts/run-launcher", @configurationArgs, "file://".$testResults if $launchSafari; -} elsif (isQt()) { - unshift @configurationArgs, qw(-graphicssystem raster -style windows); - if (isCygwin()) { - $testResults = "/" . toWindowsPath($testResults); - $testResults =~ s/\\/\//g; - } - system "WebKitTools/Scripts/run-launcher", @configurationArgs, "file://".$testResults if $launchSafari; -} elsif (isCygwin()) { - system "cygstart", $testResults if $launchSafari; -} else { - system "WebKitTools/Scripts/run-safari", @configurationArgs, "-NSOpen", $testResults if $launchSafari; -} - -closeCygpaths() if isCygwin(); - -exit 1; - -sub countAndPrintLeaks($$$) +sub runningOnBuildBot() { - my ($dumpToolName, $dumpToolPID, $leaksFilePath) = @_; - - print "\n" unless $atLineStart; - $atLineStart = 1; - - # We are excluding the following reported leaks so they don't get in our way when looking for WebKit leaks: - # This allows us ignore known leaks and only be alerted when new leaks occur. Some leaks are in the old - # versions of the system frameworks that are being used by the leaks bots. Even though a leak has been - # fixed, it will be listed here until the bot has been updated with the newer frameworks. - - my @typesToExclude = ( - ); - - my @callStacksToExclude = ( - "Flash_EnforceLocalSecurity" # leaks in Flash plug-in code, rdar://problem/4449747 - ); - - if (isTiger()) { - # Leak list for the version of Tiger used on the build bot. - push @callStacksToExclude, ( - "CFRunLoopRunSpecific \\| malloc_zone_malloc", "CFRunLoopRunSpecific \\| CFAllocatorAllocate ", # leak in CFRunLoopRunSpecific, rdar://problem/4670839 - "CGImageSourceGetPropertiesAtIndex", # leak in ImageIO, rdar://problem/4628809 - "FOGetCoveredUnicodeChars", # leak in ATS, rdar://problem/3943604 - "GetLineDirectionPreference", "InitUnicodeUtilities", # leaks tool falsely reporting leak in CFNotificationCenterAddObserver, rdar://problem/4964790 - "ICCFPrefWrapper::GetPrefDictionary", # leaks in Internet Config. code, rdar://problem/4449794 - "NSHTTPURLProtocol setResponseHeader:", # leak in multipart/mixed-replace handling in Foundation, no Radar, but fixed in Leopard - "NSURLCache cachedResponseForRequest", # leak in CFURL cache, rdar://problem/4768430 - "PCFragPrepareClosureFromFile", # leak in Code Fragment Manager, rdar://problem/3426998 - "WebCore::Selection::toRange", # bug in 'leaks', rdar://problem/4967949 - "WebCore::SubresourceLoader::create", # bug in 'leaks', rdar://problem/4985806 - "_CFPreferencesDomainDeepCopyDictionary", # leak in CFPreferences, rdar://problem/4220786 - "_objc_msgForward", # leak in NSSpellChecker, rdar://problem/4965278 - "gldGetString", # leak in OpenGL, rdar://problem/5013699 - "_setDefaultUserInfoFromURL", # leak in NSHTTPAuthenticator, rdar://problem/5546453 - "SSLHandshake", # leak in SSL, rdar://problem/5546440 - "SecCertificateCreateFromData", # leak in SSL code, rdar://problem/4464397 - ); - push @typesToExclude, ( - "THRD", # bug in 'leaks', rdar://problem/3387783 - "DRHT", # ditto (endian little hate i) - ); - } - - if (isLeopard()) { - # Leak list for the version of Leopard used on the build bot. - push @callStacksToExclude, ( - "CFHTTPMessageAppendBytes", # leak in CFNetwork, rdar://problem/5435912 - "sendDidReceiveDataCallback", # leak in CFNetwork, rdar://problem/5441619 - "_CFHTTPReadStreamReadMark", # leak in CFNetwork, rdar://problem/5441468 - "httpProtocolStart", # leak in CFNetwork, rdar://problem/5468837 - "_CFURLConnectionSendCallbacks", # leak in CFNetwork, rdar://problem/5441600 - "DispatchQTMsg", # leak in QuickTime, PPC only, rdar://problem/5667132 - "QTMovieContentView createVisualContext", # leak in QuickTime, PPC only, rdar://problem/5667132 - "_CopyArchitecturesForJVMVersion", # leak in Java, rdar://problem/5910823 - ); - } - - if (isSnowLeopard()) { - push @callStacksToExclude, ( - "readMakerNoteProps", # <rdar://problem/7156432> leak in ImageIO - "QTKitMovieControllerView completeUISetup", # <rdar://problem/7155156> leak in QTKit - ); - } - - my $leaksTool = sourceDir() . "/WebKitTools/Scripts/run-leaks"; - my $excludeString = "--exclude-callstack '" . (join "' --exclude-callstack '", @callStacksToExclude) . "'"; - $excludeString .= " --exclude-type '" . (join "' --exclude-type '", @typesToExclude) . "'" if @typesToExclude; - - print " ? checking for leaks in $dumpToolName\n"; - my $leaksOutput = `$leaksTool $excludeString $dumpToolPID`; - my ($count, $bytes) = $leaksOutput =~ /Process $dumpToolPID: (\d+) leaks? for (\d+) total/; - my ($excluded) = $leaksOutput =~ /(\d+) leaks? excluded/; - - my $adjustedCount = $count; - $adjustedCount -= $excluded if $excluded; - - if (!$adjustedCount) { - print " - no leaks found\n"; - unlink $leaksFilePath; - return 0; - } else { - my $dir = $leaksFilePath; - $dir =~ s|/[^/]+$|| or die; - mkpath $dir; - - if ($excluded) { - print " + $adjustedCount leaks ($bytes bytes including $excluded excluded leaks) were found, details in $leaksFilePath\n"; - } else { - print " + $count leaks ($bytes bytes) were found, details in $leaksFilePath\n"; - } - - writeToFile($leaksFilePath, $leaksOutput); - - push @leaksFilenames, $leaksFilePath; - } - - return $adjustedCount; -} - -sub writeToFile($$) -{ - my ($filePath, $contents) = @_; - open NEWFILE, ">", "$filePath" or die "Could not create $filePath. $!\n"; - print NEWFILE $contents; - close NEWFILE; -} - -# Break up a path into the directory (with slash) and base name. -sub splitpath($) -{ - my ($path) = @_; - - my $pathSeparator = "/"; - my $dirname = dirname($path) . $pathSeparator; - $dirname = "" if $dirname eq "." . $pathSeparator; - - return ($dirname, basename($path)); -} - -# Sort first by directory, then by file, so all paths in one directory are grouped -# rather than being interspersed with items from subdirectories. -# Use numericcmp to sort directory and filenames to make order logical. -sub pathcmp($$) -{ - my ($patha, $pathb) = @_; - - my ($dira, $namea) = splitpath($patha); - my ($dirb, $nameb) = splitpath($pathb); - - return numericcmp($dira, $dirb) if $dira ne $dirb; - return numericcmp($namea, $nameb); -} - -# Sort numeric parts of strings as numbers, other parts as strings. -# Makes 1.33 come after 1.3, which is cool. -sub numericcmp($$) -{ - my ($aa, $bb) = @_; - - my @a = split /(\d+)/, $aa; - my @b = split /(\d+)/, $bb; - - # Compare one chunk at a time. - # Each chunk is either all numeric digits, or all not numeric digits. - while (@a && @b) { - my $a = shift @a; - my $b = shift @b; - - # Use numeric comparison if chunks are non-equal numbers. - return $a <=> $b if $a =~ /^\d/ && $b =~ /^\d/ && $a != $b; - - # Use string comparison if chunks are any other kind of non-equal string. - return $a cmp $b if $a ne $b; - } - - # One of the two is now empty; compare lengths for result in this case. - return @a <=> @b; -} - -# Sort slowest tests first. -sub slowestcmp($$) -{ - my ($testa, $testb) = @_; - - my $dura = $durations{$testa}; - my $durb = $durations{$testb}; - return $durb <=> $dura if $dura != $durb; - return pathcmp($testa, $testb); -} - -sub launchWithEnv(\@\%) -{ - my ($args, $env) = @_; - - # Dump the current environment as perl code and then put it in quotes so it is one parameter. - my $environmentDumper = Data::Dumper->new([\%{$env}], [qw(*ENV)]); - $environmentDumper->Indent(0); - $environmentDumper->Purity(1); - my $allEnvVars = $environmentDumper->Dump(); - unshift @{$args}, "\"$allEnvVars\""; - - my $execScript = File::Spec->catfile(sourceDir(), qw(WebKitTools Scripts execAppWithEnv)); - unshift @{$args}, $execScript; - return @{$args}; -} - -sub resolveAndMakeTestResultsDirectory() -{ - my $absTestResultsDirectory = File::Spec->rel2abs(glob $testResultsDirectory); - mkpath $absTestResultsDirectory; - return $absTestResultsDirectory; -} - -sub openDiffTool() -{ - return if $isDiffToolOpen; - return if !$pixelTests; - - my %CLEAN_ENV; - $CLEAN_ENV{MallocStackLogging} = 1 if $shouldCheckLeaks; - $imageDiffToolPID = open2(\*DIFFIN, \*DIFFOUT, $imageDiffTool, launchWithEnv(@diffToolArgs, %CLEAN_ENV)) or die "unable to open $imageDiffTool\n"; - $isDiffToolOpen = 1; + # This is a hack to detect if we're running on the buildbot so we can + # pass --verbose to new-run-webkit-tests. This will be removed when we + # update the buildbot config to call new-run-webkit-tests explicitly. + my %isBuildBotUser = ("apple" => 1, "buildbot" => 1); + return $isBuildBotUser{$ENV{"USER"}}; } -sub openDumpTool() +sub useNewRunWebKitTests() { - return if $isDumpToolOpen; - - my %CLEAN_ENV; - - # Generic environment variables - if (defined $ENV{'WEBKIT_TESTFONTS'}) { - $CLEAN_ENV{WEBKIT_TESTFONTS} = $ENV{'WEBKIT_TESTFONTS'}; - } - - $CLEAN_ENV{XML_CATALOG_FILES} = ""; # work around missing /etc/catalog <rdar://problem/4292995> - - # Platform spesifics - if (isLinux()) { - if (defined $ENV{'DISPLAY'}) { - $CLEAN_ENV{DISPLAY} = $ENV{'DISPLAY'}; - } else { - $CLEAN_ENV{DISPLAY} = ":1"; - } - if (defined $ENV{'XAUTHORITY'}) { - $CLEAN_ENV{XAUTHORITY} = $ENV{'XAUTHORITY'}; - } - - $CLEAN_ENV{HOME} = $ENV{'HOME'}; - - if (defined $ENV{'LD_LIBRARY_PATH'}) { - $CLEAN_ENV{LD_LIBRARY_PATH} = $ENV{'LD_LIBRARY_PATH'}; - } - if (defined $ENV{'DBUS_SESSION_BUS_ADDRESS'}) { - $CLEAN_ENV{DBUS_SESSION_BUS_ADDRESS} = $ENV{'DBUS_SESSION_BUS_ADDRESS'}; - } - } elsif (isDarwin()) { - if (defined $ENV{'DYLD_LIBRARY_PATH'}) { - $CLEAN_ENV{DYLD_LIBRARY_PATH} = $ENV{'DYLD_LIBRARY_PATH'}; - } - - $CLEAN_ENV{DYLD_FRAMEWORK_PATH} = $productDir; - $CLEAN_ENV{DYLD_INSERT_LIBRARIES} = "/usr/lib/libgmalloc.dylib" if $guardMalloc; - } elsif (isCygwin()) { - $CLEAN_ENV{HOMEDRIVE} = $ENV{'HOMEDRIVE'}; - $CLEAN_ENV{HOMEPATH} = $ENV{'HOMEPATH'}; - - setPathForRunningWebKitApp(\%CLEAN_ENV); - } - - # Port spesifics - if (isQt()) { - $CLEAN_ENV{QTWEBKIT_PLUGIN_PATH} = productDir() . "/lib/plugins"; - } - - my @args = ($dumpTool, @toolArgs); - if (isAppleMacWebKit() and !isTiger()) { - unshift @args, "arch", "-" . architecture(); - } - - if ($useValgrind) { - unshift @args, "valgrind", "--suppressions=$platformBaseDirectory/qt/SuppressedValgrindErrors"; - } - - $CLEAN_ENV{MallocStackLogging} = 1 if $shouldCheckLeaks; - - $dumpToolPID = open3(\*OUT, \*IN, \*ERROR, launchWithEnv(@args, %CLEAN_ENV)) or die "Failed to start tool: $dumpTool\n"; - $isDumpToolOpen = 1; - $dumpToolCrashed = 0; -} - -sub closeDumpTool() -{ - return if !$isDumpToolOpen; - - close IN; - close OUT; - waitpid $dumpToolPID, 0; - - # check for WebCore counter leaks. - if ($shouldCheckLeaks) { - while (<ERROR>) { - print; - } - } - close ERROR; - $isDumpToolOpen = 0; -} - -sub dumpToolDidCrash() -{ - return 1 if $dumpToolCrashed; - return 0 unless $isDumpToolOpen; - my $pid = waitpid(-1, WNOHANG); - return 1 if ($pid == $dumpToolPID); - - # On Mac OS X, crashing may be significantly delayed by crash reporter. - return 0 unless isAppleMacWebKit(); - - return DumpRenderTreeSupport::processIsCrashing($dumpToolPID); -} - -sub configureAndOpenHTTPDIfNeeded() -{ - return if $isHttpdOpen; - my $absTestResultsDirectory = resolveAndMakeTestResultsDirectory(); - my $listen = "127.0.0.1:$httpdPort"; - my @args = ( - "-c", "CustomLog \"$absTestResultsDirectory/access_log.txt\" common", - "-c", "ErrorLog \"$absTestResultsDirectory/error_log.txt\"", - "-C", "Listen $listen" - ); - - my @defaultArgs = getDefaultConfigForTestDirectory($testDirectory); - @args = (@defaultArgs, @args); - - waitForHTTPDLock() if $shouldWaitForHTTPD; - $isHttpdOpen = openHTTPD(@args); -} - -sub openWebSocketServerIfNeeded() -{ - return 1 if $isWebSocketServerOpen; - return 0 if $failedToStartWebSocketServer; - - my $webSocketServerPath = "/usr/bin/python"; - my $webSocketPythonPath = "WebKitTools/pywebsocket"; - my $webSocketHandlerDir = "$testDirectory"; - my $webSocketHandlerScanDir = "$testDirectory/websocket/tests"; - my $webSocketHandlerMapFile = "$webSocketHandlerScanDir/handler_map.txt"; - my $sslCertificate = "$testDirectory/http/conf/webkit-httpd.pem"; - my $absTestResultsDirectory = resolveAndMakeTestResultsDirectory(); - my $logFile = "$absTestResultsDirectory/pywebsocket_log.txt"; - - my @args = ( - "WebKitTools/pywebsocket/mod_pywebsocket/standalone.py", - "-p", "$webSocketPort", - "-d", "$webSocketHandlerDir", - "-s", "$webSocketHandlerScanDir", - "-m", "$webSocketHandlerMapFile", - "-x", "/websocket/tests/cookies", - "-l", "$logFile", - "--strict", - ); - # wss is disabled until all platforms support pyOpenSSL. - # my @argsSecure = ( - # "WebKitTools/pywebsocket/mod_pywebsocket/standalone.py", - # "-p", "$webSocketSecurePort", - # "-d", "$webSocketHandlerDir", - # "-t", - # "-k", "$sslCertificate", - # "-c", "$sslCertificate", - # ); - - $ENV{"PYTHONPATH"} = $webSocketPythonPath; - $webSocketServerPID = open3(\*WEBSOCKETSERVER_IN, \*WEBSOCKETSERVER_OUT, \*WEBSOCKETSERVER_ERR, $webSocketServerPath, @args); - # wss is disabled until all platforms support pyOpenSSL. - # $webSocketSecureServerPID = open3(\*WEBSOCKETSECURESERVER_IN, \*WEBSOCKETSECURESERVER_OUT, \*WEBSOCKETSECURESERVER_ERR, $webSocketServerPath, @argsSecure); - # my @listen = ("http://127.0.0.1:$webSocketPort", "https://127.0.0.1:$webSocketSecurePort"); - my @listen = ("http://127.0.0.1:$webSocketPort"); - for (my $i = 0; $i < @listen; $i++) { - my $retryCount = 10; - while (system("/usr/bin/curl -k -q --silent --stderr - --output /dev/null $listen[$i]") && $retryCount) { - sleep 1; - --$retryCount; - } - unless ($retryCount) { - print STDERR "Timed out waiting for WebSocketServer to start.\n"; - $failedToStartWebSocketServer = 1; - return 0; - } - } - - $isWebSocketServerOpen = 1; - return 1; -} - -sub closeWebSocketServer() -{ - return if !$isWebSocketServerOpen; - - close WEBSOCKETSERVER_IN; - close WEBSOCKETSERVER_OUT; - close WEBSOCKETSERVER_ERR; - kill 15, $webSocketServerPID; - - # wss is disabled until all platforms support pyOpenSSL. - # close WEBSOCKETSECURESERVER_IN; - # close WEBSOCKETSECURESERVER_OUT; - # close WEBSOCKETSECURESERVER_ERR; - # kill 15, $webSocketSecureServerPID; - - $isWebSocketServerOpen = 0; -} - -sub fileNameWithNumber($$) -{ - my ($base, $number) = @_; - return "$base$number" if ($number > 1); - return $base; -} - -sub processIgnoreTests($$) -{ - my @ignoreList = split(/\s*,\s*/, shift); - my $listName = shift; - - my $disabledSuffix = "-disabled"; - - my $addIgnoredDirectories = sub { - return () if exists $ignoredLocalDirectories{basename($File::Find::dir)}; - $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)} = 1; - return @_; - }; - foreach my $item (@ignoreList) { - my $path = catfile($testDirectory, $item); - if (-d $path) { - $ignoredDirectories{$item} = 1; - find({ preprocess => $addIgnoredDirectories, wanted => sub {} }, $path); - } - elsif (-f $path) { - $ignoredFiles{$item} = 1; - } elsif (-f $path . $disabledSuffix) { - # The test is disabled, so do nothing. - } else { - print "$listName list contained '$item', but no file of that name could be found\n"; - } - } -} - -sub stripExtension($) -{ - my ($test) = @_; - - $test =~ s/\.[a-zA-Z]+$//; - return $test; -} - -sub isTextOnlyTest($) -{ - my ($actual) = @_; - my $isText; - if ($actual =~ /^layer at/ms) { - $isText = 0; - } else { - $isText = 1; - } - return $isText; -} - -sub expectedDirectoryForTest($;$;$) -{ - my ($base, $isText, $expectedExtension) = @_; - - my @directories = @platformResultHierarchy; - push @directories, map { catdir($platformBaseDirectory, $_) } qw(mac-snowleopard mac) if isCygwin(); - push @directories, $expectedDirectory; - - # If we already have expected results, just return their location. - foreach my $directory (@directories) { - return $directory if (-f "$directory/$base-$expectedTag.$expectedExtension"); - } - - # For cross-platform tests, text-only results should go in the cross-platform directory, - # while render tree dumps should go in the least-specific platform directory. - return $isText ? $expectedDirectory : $platformResultHierarchy[$#platformResultHierarchy]; -} - -sub countFinishedTest($$$$) -{ - my ($test, $base, $result, $isText) = @_; - - if (($count + 1) % $testsPerDumpTool == 0 || $count == $#tests) { - if ($shouldCheckLeaks) { - my $fileName; - if ($testsPerDumpTool == 1) { - $fileName = "$testResultsDirectory/$base-leaks.txt"; - } else { - $fileName = "$testResultsDirectory/" . fileNameWithNumber($dumpToolName, $leaksOutputFileNumber) . "-leaks.txt"; - } - my $leakCount = countAndPrintLeaks($dumpToolName, $dumpToolPID, $fileName); - $totalLeaks += $leakCount; - $leaksOutputFileNumber++ if ($leakCount); - } - - closeDumpTool(); - } - - $count++; - $counts{$result}++; - push @{$tests{$result}}, $test; -} - -sub testCrashedOrTimedOut($$$$$) -{ - my ($test, $base, $didCrash, $actual, $error) = @_; - - printFailureMessageForTest($test, $didCrash ? "crashed" : "timed out"); - - sampleDumpTool() unless $didCrash; - - my $dir = "$testResultsDirectory/$base"; - $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; - mkpath $dir; - - deleteExpectedAndActualResults($base); - - if (defined($error) && length($error)) { - writeToFile("$testResultsDirectory/$base-$errorTag.txt", $error); - } - - recordActualResultsAndDiff($base, $actual); - - kill 9, $dumpToolPID unless $didCrash; - - closeDumpTool(); -} - -sub printFailureMessageForTest($$) -{ - my ($test, $description) = @_; - - unless ($verbose) { - print "\n" unless $atLineStart; - print "$test -> "; - } - print "$description\n"; - $atLineStart = 1; -} - -my %cygpaths = (); - -sub openCygpathIfNeeded($) -{ - my ($options) = @_; - - return unless isCygwin(); - return $cygpaths{$options} if $cygpaths{$options} && $cygpaths{$options}->{"open"}; - - local (*CYGPATHIN, *CYGPATHOUT); - my $pid = open2(\*CYGPATHIN, \*CYGPATHOUT, "cygpath -f - $options"); - my $cygpath = { - "pid" => $pid, - "in" => *CYGPATHIN, - "out" => *CYGPATHOUT, - "open" => 1 - }; - - $cygpaths{$options} = $cygpath; - - return $cygpath; -} - -sub closeCygpaths() -{ - return unless isCygwin(); - - foreach my $cygpath (values(%cygpaths)) { - close $cygpath->{"in"}; - close $cygpath->{"out"}; - waitpid($cygpath->{"pid"}, 0); - $cygpath->{"open"} = 0; - - } -} - -sub convertPathUsingCygpath($$) -{ - my ($path, $options) = @_; - - my $cygpath = openCygpathIfNeeded($options); - local *inFH = $cygpath->{"in"}; - local *outFH = $cygpath->{"out"}; - print outFH $path . "\n"; - my $convertedPath = <inFH>; - chomp($convertedPath) if defined $convertedPath; - return $convertedPath; -} - -sub toWindowsPath($) -{ - my ($path) = @_; - return unless isCygwin(); - - return convertPathUsingCygpath($path, "-w"); -} - -sub toURL($) -{ - my ($path) = @_; - - if ($useRemoteLinksToTests) { - my $relativePath = File::Spec->abs2rel($path, $testDirectory); - - # If the file is below the test directory then convert it into a link to the file in SVN - if ($relativePath !~ /^\.\.\//) { - my $revision = svnRevisionForDirectory($testDirectory); - my $svnPath = pathRelativeToSVNRepositoryRootForPath($path); - return "http://trac.webkit.org/export/$revision/$svnPath"; - } - } - - return $path unless isCygwin(); - - return "file:///" . convertPathUsingCygpath($path, "-m"); -} - -sub validateSkippedArg($$;$) -{ - my ($option, $value, $value2) = @_; - my %validSkippedValues = map { $_ => 1 } qw(default ignore only); - $value = lc($value); - die "Invalid argument '" . $value . "' for option $option" unless $validSkippedValues{$value}; - $treatSkipped = $value; -} - -sub htmlForResultsSection(\@$&) -{ - my ($tests, $description, $linkGetter) = @_; - - my @html = (); - return join("\n", @html) unless @{$tests}; - - push @html, "<p>$description:</p>"; - push @html, "<table>"; - foreach my $test (@{$tests}) { - push @html, "<tr>"; - push @html, "<td><a href=\"" . toURL("$testDirectory/$test") . "\">$test</a></td>"; - foreach my $link (@{&{$linkGetter}($test)}) { - push @html, "<td><a href=\"$link->{href}\">$link->{text}</a></td>"; - } - push @html, "</tr>"; - } - push @html, "</table>"; - - return join("\n", @html); -} - -sub linksForExpectedAndActualResults($) -{ - my ($base) = @_; - - my @links = (); - - return \@links unless -s "$testResultsDirectory/$base-$diffsTag.txt"; - - my $expectedResultPath = $expectedResultPaths{$base}; - my ($expectedResultFileName, $expectedResultsDirectory, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); - - push @links, { href => "$base-$expectedTag$expectedResultExtension", text => "expected" }; - push @links, { href => "$base-$actualTag$expectedResultExtension", text => "actual" }; - push @links, { href => "$base-$diffsTag.txt", text => "diff" }; - push @links, { href => "$base-$prettyDiffTag.html", text => "pretty diff" }; - - return \@links; -} - -sub linksForMismatchTest -{ - my ($test) = @_; - - my @links = (); - - my $base = stripExtension($test); - - push @links, @{linksForExpectedAndActualResults($base)}; - return \@links unless $pixelTests && $imagesPresent{$base}; - - push @links, { href => "$base-$expectedTag.png", text => "expected image" }; - push @links, { href => "$base-$diffsTag.html", text => "image diffs" }; - push @links, { href => "$base-$diffsTag.png", text => "$imageDifferences{$base}%" }; - - return \@links; -} - -sub linksForErrorTest -{ - my ($test) = @_; - - my @links = (); - - my $base = stripExtension($test); - - push @links, @{linksForExpectedAndActualResults($base)}; - push @links, { href => "$base-$errorTag.txt", text => "stderr" }; - - return \@links; -} - -sub linksForNewTest -{ - my ($test) = @_; - - my @links = (); - - my $base = stripExtension($test); - - my $expectedResultPath = $expectedResultPaths{$base}; - my ($expectedResultFileName, $expectedResultsDirectory, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); - - push @links, { href => "$base-$actualTag$expectedResultExtension", text => "result" }; - if ($pixelTests && $imagesPresent{$base}) { - push @links, { href => "$base-$expectedTag.png", text => "image" }; - } - - return \@links; -} - -sub deleteExpectedAndActualResults($) -{ - my ($base) = @_; - - unlink "$testResultsDirectory/$base-$actualTag.txt"; - unlink "$testResultsDirectory/$base-$diffsTag.txt"; - unlink "$testResultsDirectory/$base-$errorTag.txt"; -} - -sub recordActualResultsAndDiff($$) -{ - my ($base, $actualResults) = @_; - - return unless defined($actualResults) && length($actualResults); - - my $expectedResultPath = $expectedResultPaths{$base}; - my ($expectedResultFileNameMinusExtension, $expectedResultDirectoryPath, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); - my $actualResultsPath = "$testResultsDirectory/$base-$actualTag$expectedResultExtension"; - my $copiedExpectedResultsPath = "$testResultsDirectory/$base-$expectedTag$expectedResultExtension"; - - mkpath(dirname($actualResultsPath)); - writeToFile("$actualResultsPath", $actualResults); - - if (-f $expectedResultPath) { - copy("$expectedResultPath", "$copiedExpectedResultsPath"); - } else { - open EMPTY, ">$copiedExpectedResultsPath"; - close EMPTY; - } - - my $diffOuputBasePath = "$testResultsDirectory/$base"; - my $diffOutputPath = "$diffOuputBasePath-$diffsTag.txt"; - system "diff -u \"$copiedExpectedResultsPath\" \"$actualResultsPath\" > \"$diffOutputPath\""; - - my $prettyDiffOutputPath = "$diffOuputBasePath-$prettyDiffTag.html"; - my $prettyPatchPath = "BugsSite/PrettyPatch/"; - my $prettifyPath = "$prettyPatchPath/prettify.rb"; - system "ruby -I \"$prettyPatchPath\" \"$prettifyPath\" \"$diffOutputPath\" > \"$prettyDiffOutputPath\""; -} - -sub buildPlatformResultHierarchy() -{ - mkpath($platformTestDirectory) if ($platform eq "undefined" && !-d "$platformTestDirectory"); - - my @platforms; - if ($platform =~ /^mac-/) { - my $i; - for ($i = 0; $i < @macPlatforms; $i++) { - last if $macPlatforms[$i] eq $platform; - } - for (; $i < @macPlatforms; $i++) { - push @platforms, $macPlatforms[$i]; - } - } elsif ($platform =~ /^qt-/) { - push @platforms, $platform; - push @platforms, "qt"; - } else { - @platforms = $platform; - } - - my @hierarchy; - for (my $i = 0; $i < @platforms; $i++) { - my $scoped = catdir($platformBaseDirectory, $platforms[$i]); - push(@hierarchy, $scoped) if (-d $scoped); - } - - return @hierarchy; -} - -sub buildPlatformTestHierarchy(@) -{ - my (@platformHierarchy) = @_; - return @platformHierarchy if (@platformHierarchy < 2); - - return ($platformHierarchy[0], $platformHierarchy[$#platformHierarchy]); -} - -sub epiloguesAndPrologues($$) -{ - my ($lastDirectory, $directory) = @_; - my @lastComponents = split('/', $lastDirectory); - my @components = split('/', $directory); - - while (@lastComponents) { - if (!defined($components[0]) || $lastComponents[0] ne $components[0]) { - last; - } - shift @components; - shift @lastComponents; - } - - my @result; - my $leaving = $lastDirectory; - foreach (@lastComponents) { - my $epilogue = $leaving . "/resources/run-webkit-tests-epilogue.html"; - foreach (@platformResultHierarchy) { - push @result, catdir($_, $epilogue) if (stat(catdir($_, $epilogue))); - } - push @result, catdir($testDirectory, $epilogue) if (stat(catdir($testDirectory, $epilogue))); - $leaving =~ s|(^\|/)[^/]+$||; - } - - my $entering = $leaving; - foreach (@components) { - $entering .= '/' . $_; - my $prologue = $entering . "/resources/run-webkit-tests-prologue.html"; - push @result, catdir($testDirectory, $prologue) if (stat(catdir($testDirectory, $prologue))); - foreach (reverse @platformResultHierarchy) { - push @result, catdir($_, $prologue) if (stat(catdir($_, $prologue))); - } - } - return @result; -} - -sub parseLeaksandPrintUniqueLeaks() -{ - return unless @leaksFilenames; - - my $mergedFilenames = join " ", @leaksFilenames; - my $parseMallocHistoryTool = sourceDir() . "/WebKitTools/Scripts/parse-malloc-history"; - - open MERGED_LEAKS, "cat $mergedFilenames | $parseMallocHistoryTool --merge-depth $mergeDepth - |" ; - my @leakLines = <MERGED_LEAKS>; - close MERGED_LEAKS; - - my $uniqueLeakCount = 0; - my $totalBytes; - foreach my $line (@leakLines) { - ++$uniqueLeakCount if ($line =~ /^(\d*)\scalls/); - $totalBytes = $1 if $line =~ /^total\:\s(.*)\s\(/; - } - - print "\nWARNING: $totalLeaks total leaks found for a total of $totalBytes!\n"; - print "WARNING: $uniqueLeakCount unique leaks found!\n"; - print "See above for individual leaks results.\n" if ($leaksOutputFileNumber > 2); - -} - -sub extensionForMimeType($) -{ - my ($mimeType) = @_; - - if ($mimeType eq "application/x-webarchive") { - return "webarchive"; - } elsif ($mimeType eq "application/pdf") { - return "pdf"; - } - return "txt"; -} - -# Read up to the first #EOF (the content block of the test), or until detecting crashes or timeouts. -sub readFromDumpToolWithTimer(**) -{ - my ($fhIn, $fhError) = @_; - - setFileHandleNonBlocking($fhIn, 1); - setFileHandleNonBlocking($fhError, 1); - - my $maximumSecondsWithoutOutput = $timeoutSeconds; - $maximumSecondsWithoutOutput *= 10 if $guardMalloc; - my $microsecondsToWaitBeforeReadingAgain = 1000; - - my $timeOfLastSuccessfulRead = time; - - my @output = (); - my @error = (); - my $status = "success"; - my $mimeType = "text/plain"; - # We don't have a very good way to know when the "headers" stop - # and the content starts, so we use this as a hack: - my $haveSeenContentType = 0; - my $haveSeenEofIn = 0; - my $haveSeenEofError = 0; - - while (1) { - if (time - $timeOfLastSuccessfulRead > $maximumSecondsWithoutOutput) { - $status = dumpToolDidCrash() ? "crashed" : "timedOut"; - last; - } - - # Once we've seen the EOF, we must not read anymore. - my $lineIn = readline($fhIn) unless $haveSeenEofIn; - my $lineError = readline($fhError) unless $haveSeenEofError; - if (!defined($lineIn) && !defined($lineError)) { - last if ($haveSeenEofIn && $haveSeenEofError); - - if ($! != EAGAIN) { - $status = "crashed"; - last; - } - - # No data ready - usleep($microsecondsToWaitBeforeReadingAgain); - next; - } - - $timeOfLastSuccessfulRead = time; - - if (defined($lineIn)) { - if (!$haveSeenContentType && $lineIn =~ /^Content-Type: (\S+)$/) { - $mimeType = $1; - $haveSeenContentType = 1; - } elsif ($lineIn =~ /#EOF/) { - $haveSeenEofIn = 1; - } else { - push @output, $lineIn; - } - } - if (defined($lineError)) { - if ($lineError =~ /#EOF/) { - $haveSeenEofError = 1; - } else { - push @error, $lineError; - } - } - } - - setFileHandleNonBlocking($fhIn, 0); - setFileHandleNonBlocking($fhError, 0); - return { - output => join("", @output), - error => join("", @error), - status => $status, - mimeType => $mimeType, - extension => extensionForMimeType($mimeType) - }; -} - -sub setFileHandleNonBlocking(*$) -{ - my ($fh, $nonBlocking) = @_; - - my $flags = fcntl($fh, F_GETFL, 0) or die "Couldn't get filehandle flags"; - - if ($nonBlocking) { - $flags |= O_NONBLOCK; - } else { - $flags &= ~O_NONBLOCK; - } - - fcntl($fh, F_SETFL, $flags) or die "Couldn't set filehandle flags"; - - return 1; -} - -sub sampleDumpTool() -{ - return unless isAppleMacWebKit(); - return unless $runSample; - - my $outputDirectory = "$ENV{HOME}/Library/Logs/DumpRenderTree"; - -d $outputDirectory or mkdir $outputDirectory; - - my $outputFile = "$outputDirectory/HangReport.txt"; - system "/usr/bin/sample", $dumpToolPID, qw(10 10 -file), $outputFile; -} - -sub stripMetrics($$) -{ - my ($actual, $expected) = @_; - - foreach my $result ($actual, $expected) { - $result =~ s/at \(-?[0-9]+,-?[0-9]+\) *//g; - $result =~ s/size -?[0-9]+x-?[0-9]+ *//g; - $result =~ s/text run width -?[0-9]+: //g; - $result =~ s/text run width -?[0-9]+ [a-zA-Z ]+: //g; - $result =~ s/RenderButton {BUTTON} .*/RenderButton {BUTTON}/g; - $result =~ s/RenderImage {INPUT} .*/RenderImage {INPUT}/g; - $result =~ s/RenderBlock {INPUT} .*/RenderBlock {INPUT}/g; - $result =~ s/RenderTextControl {INPUT} .*/RenderTextControl {INPUT}/g; - $result =~ s/\([0-9]+px/px/g; - $result =~ s/ *" *\n +" */ /g; - $result =~ s/" +$/"/g; - - $result =~ s/- /-/g; - $result =~ s/\n( *)"\s+/\n$1"/g; - $result =~ s/\s+"\n/"\n/g; - $result =~ s/scrollWidth [0-9]+/scrollWidth/g; - $result =~ s/scrollHeight [0-9]+/scrollHeight/g; - } - - return ($actual, $expected); -} - -sub fileShouldBeIgnored -{ - my ($filePath) = @_; - foreach my $ignoredDir (keys %ignoredDirectories) { - if ($filePath =~ m/^$ignoredDir/) { - return 1; - } - } + # Change this check to control which platforms use + # new-run-webkit-tests by default. return 0; } -sub readSkippedFiles($) -{ - my ($constraintPath) = @_; - - foreach my $level (@platformTestHierarchy) { - if (open SKIPPED, "<", "$level/Skipped") { - if ($verbose) { - my ($dir, $name) = splitpath($level); - print "Skipped tests in $name:\n"; - } - - while (<SKIPPED>) { - my $skipped = $_; - chomp $skipped; - $skipped =~ s/^[ \n\r]+//; - $skipped =~ s/[ \n\r]+$//; - if ($skipped && $skipped !~ /^#/) { - if ($skippedOnly) { - if (!fileShouldBeIgnored($skipped)) { - if (!$constraintPath) { - # Always add $skipped since no constraint path was specified on the command line. - push(@ARGV, $skipped); - } elsif ($skipped =~ /^($constraintPath)/) { - # Add $skipped only if it matches the current path constraint, e.g., - # "--skipped=only dir1" with "dir1/file1.html" on the skipped list. - push(@ARGV, $skipped); - } elsif ($constraintPath =~ /^($skipped)/) { - # Add current path constraint if it is more specific than the skip list entry, - # e.g., "--skipped=only dir1/dir2/dir3" with "dir1" on the skipped list. - push(@ARGV, $constraintPath); - } - } elsif ($verbose) { - print " $skipped\n"; - } - } else { - if ($verbose) { - print " $skipped\n"; - } - processIgnoreTests($skipped, "Skipped"); - } - } - } - close SKIPPED; - } - } -} - -my @testsToRun; - -sub directoryFilter -{ - return () if exists $ignoredLocalDirectories{basename($File::Find::dir)}; - return () if exists $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)}; - return @_; -} - -sub fileFilter -{ - my $filename = $_; - if ($filename =~ /\.([^.]+)$/) { - if (exists $supportedFileExtensions{$1}) { - my $path = File::Spec->abs2rel(catfile($File::Find::dir, $filename), $testDirectory); - push @testsToRun, $path if !exists $ignoredFiles{$path}; - } - } -} - -sub findTestsToRun -{ - @testsToRun = (); - - for my $test (@ARGV) { - $test =~ s/^($layoutTestsName|$testDirectory)\///; - my $fullPath = catfile($testDirectory, $test); - if (file_name_is_absolute($test)) { - print "can't run test $test outside $testDirectory\n"; - } elsif (-f $fullPath) { - my ($filename, $pathname, $fileExtension) = fileparse($test, qr{\.[^.]+$}); - if (!exists $supportedFileExtensions{substr($fileExtension, 1)}) { - print "test $test does not have a supported extension\n"; - } elsif ($testHTTP || $pathname !~ /^http\//) { - push @testsToRun, $test; - } - } elsif (-d $fullPath) { - find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $fullPath); - for my $level (@platformTestHierarchy) { - my $platformPath = catfile($level, $test); - find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $platformPath) if (-d $platformPath); - } - } else { - print "test $test not found\n"; - } - } - - if (!scalar @ARGV) { - find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $testDirectory); - for my $level (@platformTestHierarchy) { - find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $level); - } - } - - # Remove duplicate tests - @testsToRun = keys %{{ map { $_ => 1 } @testsToRun }}; +my $harnessName = "old-run-webkit-tests"; - @testsToRun = sort pathcmp @testsToRun; - - # We need to minimize the time when Apache and WebSocketServer is locked by tests - # so run them last if no explicit order was specified in the argument list. - if (!scalar @ARGV) { - my @httpTests; - my @websocketTests; - my @otherTests; - foreach my $test (@testsToRun) { - if ($test =~ /^http\//) { - push(@httpTests, $test); - } elsif ($test =~ /^websocket\//) { - push(@websocketTests, $test); - } else { - push(@otherTests, $test); - } - } - @testsToRun = (@otherTests, @httpTests, @websocketTests); +if (useNewRunWebKitTests()) { + $harnessName = "new-run-webkit-tests"; + if (runningOnBuildBot()) { + push(@ARGV, "--verbose"); + # old-run-webkit-tests treats --results-directory as $CWD relative. + # new-run-webkit-tests treats --results-directory as build directory relative. + # Override the passed in --results-directory by appending a new one + # (later arguments override earlier ones in Python's optparse). + push(@ARGV, "--results-directory"); + # The buildbot always uses $SRCDIR/layout-test-results, hardcode it: + push(@ARGV, sourceDir() . "/layout-test-results"); } - - # Reverse the tests - @testsToRun = reverse @testsToRun if $reverseTests; - - # Shuffle the array - @testsToRun = shuffle(@testsToRun) if $randomizeTests; - - return @testsToRun; } -sub printResults -{ - my %text = ( - match => "succeeded", - mismatch => "had incorrect layout", - new => "were new", - timedout => "timed out", - crash => "crashed", - error => "had stderr output" - ); - - for my $type ("match", "mismatch", "new", "timedout", "crash", "error") { - my $typeCount = $counts{$type}; - next unless $typeCount; - my $typeText = $text{$type}; - my $message; - if ($typeCount == 1) { - $typeText =~ s/were/was/; - $message = sprintf "1 test case (%d%%) %s\n", 1 * 100 / $count, $typeText; - } else { - $message = sprintf "%d test cases (%d%%) %s\n", $typeCount, $typeCount * 100 / $count, $typeText; - } - $message =~ s-\(0%\)-(<1%)-; - print $message; - } -} +my $harnessPath = sprintf("%s/%s", relativeScriptsDir(), $harnessName); +exec $harnessPath ($harnessPath, @ARGV) or die "Failed to execute $harnessPath"; diff --git a/WebKitTools/Scripts/run-webkit-websocketserver b/WebKitTools/Scripts/run-webkit-websocketserver index 64a724d..06f9079 100755 --- a/WebKitTools/Scripts/run-webkit-websocketserver +++ b/WebKitTools/Scripts/run-webkit-websocketserver @@ -64,18 +64,19 @@ exit 0; sub openWebSocketServer() { my $webSocketServerPath = "/usr/bin/python"; - my $webSocketPythonPath = "$srcDir/WebKitTools/pywebsocket"; + my $webSocketPythonPath = "$srcDir/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket"; my $webSocketHandlerDir = "$testDirectory"; my $webSocketHandlerScanDir = "$testDirectory/websocket/tests"; my $webSocketHandlerMapFile = "$webSocketHandlerScanDir/handler_map.txt"; my @args = ( - "$srcDir/WebKitTools/pywebsocket/mod_pywebsocket/standalone.py", - "-p", "$webSocketPort", - "-d", "$webSocketHandlerDir", - "-s", "$webSocketHandlerScanDir", - "-m", "$webSocketHandlerMapFile", - "-x", "/websocket/tests/cookies", + "$srcDir/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/standalone.py", + "--server-host", "127.0.0.1", + "--port", "$webSocketPort", + "--document-root", "$webSocketHandlerDir", + "--scan-dir", "$webSocketHandlerScanDir", + "--websock-handlers-map-file", "$webSocketHandlerMapFile", + "--cgi-paths", "/websocket/tests", ); $ENV{"PYTHONPATH"} = $webSocketPythonPath; diff --git a/WebKitTools/Scripts/svn-apply b/WebKitTools/Scripts/svn-apply index f586211..61d193d 100755 --- a/WebKitTools/Scripts/svn-apply +++ b/WebKitTools/Scripts/svn-apply @@ -310,6 +310,7 @@ sub handleGitBinaryChange($$) sub isDirectoryEmptyForRemoval($) { my ($dir) = @_; + return 1 unless -d $dir; my $directoryIsEmpty = 1; opendir DIR, $dir or die "Could not open '$dir' to list files: $?"; for (my $item = readdir DIR; $item && $directoryIsEmpty; $item = readdir DIR) { @@ -481,6 +482,12 @@ sub scmRemove($) close SVN; print $svnOutput if $svnOutput; } elsif (isGit()) { - system("git", "rm", "--force", $path) == 0 or die "Failed to git rm --force $path."; + # Git removes a directory if it becomes empty when the last file it contains is + # removed by `git rm`. In svn-apply this can happen when a directory is being + # removed in a patch, and all of the files inside of the directory are removed + # before attemping to remove the directory itself. In this case, Git will have + # already deleted the directory and `git rm` would exit with an error claiming + # there was no file. The --ignore-unmatch switch gracefully handles this case. + system("git", "rm", "--force", "--ignore-unmatch", $path) == 0 or die "Failed to git rm --force --ignore-unmatch $path."; } } diff --git a/WebKitTools/Scripts/svn-create-patch b/WebKitTools/Scripts/svn-create-patch index 768a8ed..5aead2e 100755 --- a/WebKitTools/Scripts/svn-create-patch +++ b/WebKitTools/Scripts/svn-create-patch @@ -56,12 +56,14 @@ use Time::gmtime; use VCSUtils; sub binarycmp($$); +sub diffOptionsForFile($); sub findBaseUrl($); sub findMimeType($;$); sub findModificationType($); sub findSourceFileAndRevision($); sub generateDiff($$); sub generateFileList($\%); +sub hunkHeaderLineRegExForFile($); sub isBinaryMimeType($); sub manufacturePatchForAdditionWithHistory($); sub numericcmp($$); @@ -99,9 +101,16 @@ for my $path (keys %paths) { my $svnRoot = determineSVNRoot(); my $prefix = chdirReturningRelativePath($svnRoot); +my $patchSize = 0; + # Generate the diffs, in a order chosen for easy reviewing. for my $path (sort patchpathcmp values %diffFiles) { - generateDiff($path, $prefix); + $patchSize += generateDiff($path, $prefix); +} + +if ($patchSize > 20480) { + print STDERR "WARNING: Patch's size is " . int($patchSize/1024) . " kbytes.\n"; + print STDERR "Patches 20k or smaller are more likely to be reviewed. Larger patches may sit unreviewed for a long time.\n"; } exit 0; @@ -130,6 +139,19 @@ sub binarycmp($$) return $fileDataA->{isBinary} <=> $fileDataB->{isBinary}; } +sub diffOptionsForFile($) +{ + my ($file) = @_; + + my $options = "uaNp"; + + if (my $hunkHeaderLineRegEx = hunkHeaderLineRegExForFile($file)) { + $options .= "F'$hunkHeaderLineRegEx'"; + } + + return $options; +} + sub findBaseUrl($) { my ($infoPath) = @_; @@ -196,24 +218,27 @@ sub generateDiff($$) my $file = File::Spec->catdir($prefix, $fileData->{path}); if ($ignoreChangelogs && basename($file) eq "ChangeLog") { - return; + return 0; } - my $patch; + my $patch = ""; if ($fileData->{modificationType} eq "additionWithHistory") { manufacturePatchForAdditionWithHistory($fileData); } - open DIFF, "svn diff --diff-cmd diff -x -uaNp '$file' |" or die; + + my $diffOptions = diffOptionsForFile($file); + open DIFF, "svn diff --diff-cmd diff -x -$diffOptions '$file' |" or die; while (<DIFF>) { $patch .= $_; } close DIFF; $patch = fixChangeLogPatch($patch) if basename($file) eq "ChangeLog"; - print $patch if $patch; + print $patch; if ($fileData->{isBinary}) { print "\n" if ($patch && $patch =~ m/\n\S+$/m); outputBinaryContent($file); } + return length($patch); } sub generateFileList($\%) @@ -252,6 +277,15 @@ sub generateFileList($\%) close STAT; } +sub hunkHeaderLineRegExForFile($) +{ + my ($file) = @_; + + my $startOfObjCInterfaceRegEx = "@(implementation\\|interface\\|protocol)"; + return "^[-+]\\|$startOfObjCInterfaceRegEx" if $file =~ /\.mm?$/; + return "^$startOfObjCInterfaceRegEx" if $file =~ /^(.*\/)?(mac|objc)\// && $file =~ /\.h$/; +} + sub isBinaryMimeType($) { my ($file) = @_; diff --git a/WebKitTools/Scripts/test-webkitpy b/WebKitTools/Scripts/test-webkitpy index 8617330..e35c6e6 100755 --- a/WebKitTools/Scripts/test-webkitpy +++ b/WebKitTools/Scripts/test-webkitpy @@ -1,5 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2009 Google Inc. All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -27,41 +28,213 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +import os import sys -import unittest - -from webkitpy.bugzilla_unittest import * -from webkitpy.buildbot_unittest import * -from webkitpy.changelogs_unittest import * -from webkitpy.commands.download_unittest import * -from webkitpy.commands.early_warning_system_unittest import * -from webkitpy.commands.openbugs_unittest import OpenBugsTest -from webkitpy.commands.upload_unittest import * -from webkitpy.commands.queries_unittest import * -from webkitpy.commands.queues_unittest import * -from webkitpy.committers_unittest import * -from webkitpy.credentials_unittest import * -from webkitpy.diff_parser_unittest import * -from webkitpy.executive_unittest import * -from webkitpy.grammar_unittest import * -from webkitpy.layout_tests.port.mac_unittest import * -from webkitpy.multicommandtool_unittest import * -from webkitpy.networktransaction_unittest import * -from webkitpy.patchcollection_unittest import * -from webkitpy.queueengine_unittest import * -from webkitpy.steps.steps_unittest import * -from webkitpy.steps.closebugforlanddiff_unittest import * -from webkitpy.steps.updatechangelogswithreview_unittests import * -from webkitpy.style.unittests import * # for check-webkit-style -from webkitpy.user_unittest import * -from webkitpy.webkit_logging_unittest import * -from webkitpy.webkitport_unittest import * + +# Do not import anything from webkitpy prior to cleaning webkitpy of +# orphaned *.pyc files. This ensures that no orphaned *.pyc files are +# accidentally imported during the course of this script. +# +# Also, do not import or execute any Python code incompatible with +# Python 2.4 until after execution of the init() method below. + + +_log = logging.getLogger("test-webkitpy") + + +# Verbose logging is useful for debugging test-webkitpy code that runs +# before the actual unit tests -- things like autoinstall downloading and +# unit-test auto-detection logic. This is different from verbose logging +# of the unit tests themselves (i.e. the unittest module's --verbose flag). +def configure_logging(is_verbose_logging): + """Configure the root logger. + + Configure the root logger not to log any messages from webkitpy -- + except for messages from the autoinstall module. Also set the + logging level as described below. + + Args: + is_verbose_logging: A boolean value of whether logging should be + verbose. If this parameter is true, the logging + level for the handler on the root logger is set to + logging.DEBUG. Otherwise, it is set to logging.INFO. + + """ + # Don't use the Python ternary operator here so that this method will + # work with Python 2.4. + if is_verbose_logging: + logging_level = logging.DEBUG + else: + logging_level = logging.INFO + + handler = logging.StreamHandler(sys.stderr) + # We constrain the level on the handler rather than on the root + # logger itself. This is probably better because the handler is + # configured and known only to this module, whereas the root logger + # is an object shared (and potentially modified) by many modules. + # Modifying the handler, then, is less intrusive and less likely to + # interfere with modifications made by other modules (e.g. in unit + # tests). + handler.setLevel(logging_level) + formatter = logging.Formatter("%(name)s: %(levelname)-8s %(message)s") + handler.setFormatter(formatter) + + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.NOTSET) + + # Filter out most webkitpy messages. + # + # Messages can be selectively re-enabled for this script by updating + # this method accordingly. + def filter(record): + """Filter out autoinstall and non-third-party webkitpy messages.""" + # FIXME: Figure out a way not to use strings here, for example by + # using syntax like webkitpy.test.__name__. We want to be + # sure not to import any non-Python 2.4 code, though, until + # after the version-checking code has executed. + if (record.name.startswith("webkitpy.common.system.autoinstall") or + record.name.startswith("webkitpy.test")): + return True + if record.name.startswith("webkitpy"): + return False + return True + + testing_filter = logging.Filter() + testing_filter.filter = filter + + # Display a message so developers are not mystified as to why + # logging does not work in the unit tests. + _log.info("Suppressing most webkitpy logging while running unit tests.") + handler.addFilter(testing_filter) + + +def _clean_pyc_files(dir_to_clean, paths_not_to_log): + """Delete from a directory all .pyc files that have no .py file. + + Args: + dir_to_clean: The path to the directory to clean. + paths_not_to_log: A list of paths to .pyc files whose deletions should + not be logged. This list should normally include + only test .pyc files. + + """ + _log.debug("Cleaning orphaned *.pyc files from: %s" % dir_to_clean) + + # Normalize paths not to log. + paths_not_to_log = [os.path.abspath(path) for path in paths_not_to_log] + + for dir_path, dir_names, file_names in os.walk(dir_to_clean): + for file_name in file_names: + if file_name.endswith(".pyc") and file_name[:-1] not in file_names: + file_path = os.path.join(dir_path, file_name) + if os.path.abspath(file_path) not in paths_not_to_log: + _log.info("Deleting orphan *.pyc file: %s" % file_path) + os.remove(file_path) + + +# As a substitute for a unit test, this method tests _clean_pyc_files() +# in addition to calling it. We chose not to use the unittest module +# because _clean_pyc_files() is called only once and is not used elsewhere. +def _clean_webkitpy_with_test(): + webkitpy_dir = os.path.join(os.path.dirname(__file__), "webkitpy") + + # The test .pyc file is-- + # webkitpy/python24/TEMP_test-webkitpy_test_pyc_file.pyc. + test_path = os.path.join(webkitpy_dir, "python24", + "TEMP_test-webkitpy_test_pyc_file.pyc") + + test_file = open(test_path, "w") + try: + test_file.write("Test .pyc file generated by test-webkitpy.") + finally: + test_file.close() + + # Confirm that the test file exists so that when we check that it does + # not exist, the result is meaningful. + if not os.path.exists(test_path): + raise Exception("Test .pyc file not created: %s" % test_path) + + _clean_pyc_files(webkitpy_dir, [test_path]) + + if os.path.exists(test_path): + raise Exception("Test .pyc file not deleted: %s" % test_path) + + +def init(command_args): + """Execute code prior to importing from webkitpy.unittests. + + Args: + command_args: The list of command-line arguments -- usually + sys.argv[1:]. + + """ + verbose_logging_flag = "--verbose-logging" + is_verbose_logging = verbose_logging_flag in command_args + if is_verbose_logging: + # Remove the flag so it doesn't cause unittest.main() to error out. + # + # FIXME: Get documentation for the --verbose-logging flag to show + # up in the usage instructions, which are currently generated + # by unittest.main(). It's possible that this will require + # re-implementing the option parser for unittest.main() + # since there may not be an easy way to modify its existing + # option parser. + sys.argv.remove(verbose_logging_flag) + + configure_logging(is_verbose_logging) + _log.debug("Verbose WebKit logging enabled.") + + # We clean orphaned *.pyc files from webkitpy prior to importing from + # webkitpy to make sure that no import statements falsely succeed. + # This helps to check that import statements have been updated correctly + # after any file moves. Otherwise, incorrect import statements can + # be masked. + # + # For example, if webkitpy/python24/versioning.py were moved to a + # different location without changing any import statements, and if + # the corresponding .pyc file were left behind without deleting it, + # then "import webkitpy.python24.versioning" would continue to succeed + # even though it would fail for someone checking out a fresh copy + # of the source tree. This is because of a Python feature: + # + # "It is possible to have a file called spam.pyc (or spam.pyo when -O + # is used) without a file spam.py for the same module. This can be used + # to distribute a library of Python code in a form that is moderately + # hard to reverse engineer." + # + # ( http://docs.python.org/tutorial/modules.html#compiled-python-files ) + # + # Deleting the orphaned .pyc file prior to importing, however, would + # cause an ImportError to occur on import as desired. + _clean_webkitpy_with_test() + + import webkitpy.python24.versioning as versioning + + versioning.check_version(log=_log) + + (comparison, current_version, minimum_version) = \ + versioning.compare_version() + + if comparison > 0: + # Then the current version is later than the minimum version. + message = ("You are testing webkitpy with a Python version (%s) " + "higher than the minimum version (%s) it was meant " + "to support." % (current_version, minimum_version)) + _log.warn(message) + if __name__ == "__main__": - # FIXME: This is a hack, but I'm tired of commenting out the test. - # See https://bugs.webkit.org/show_bug.cgi?id=31818 - if len(sys.argv) > 1 and sys.argv[1] == "--all": - sys.argv.remove("--all") - from webkitpy.scm_unittest import * - unittest.main() + init(sys.argv[1:]) + + # We import the unit test code after init() to ensure that any + # Python version warnings are displayed in case an error occurs + # while interpreting webkitpy.unittests. This also allows + # logging to be configured prior to importing -- for example to + # enable the display of autoinstall logging.log messages while + # running the unit tests. + from webkitpy.test.main import Tester + + Tester().run_tests(sys.argv) diff --git a/WebKitTools/Scripts/update-iexploder-cssproperties b/WebKitTools/Scripts/update-iexploder-cssproperties index b7ae6cb..3fbcf83 100755 --- a/WebKitTools/Scripts/update-iexploder-cssproperties +++ b/WebKitTools/Scripts/update-iexploder-cssproperties @@ -1,6 +1,7 @@ #!/usr/bin/perl # Copyright (C) 2007 Apple Inc. All rights reserved. +# Copyright (C) 2010 Holger Hans Peter Freyther # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,87 +27,103 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# This script updates WebKitTools/iExploder/htdocs/cssproperties.in based on -# WebCore/css/CSSPropertyNames.in. +# This script updates WebKitTools/iExploder/htdocs/*.in based on +# WebCore/css/CSSPropertyNames.in, WebCore/html/HTMLTagNames.in +# and WebCore/html/HTMLAttributeNames.in use warnings; use strict; use FindBin; use lib $FindBin::Bin; +use VCSUtils; use webkitdirs; use File::Spec; -sub generateSectionFromCSSPropertyNamesFile(); -sub readiExploderFile(); -sub svnRevision($); -sub writeiExploderFile(); +sub generateEntityListFromFile($); +sub readiExploderFile($); +sub update($$); +sub writeiExploderFile($@); -my $iExploderFile = File::Spec->catfile(sourceDir(), split("/", "WebKitTools/iExploder/htdocs/cssproperties.in")); -my $cssPropertyNamesFile = File::Spec->catfile(sourceDir(), split("/", "WebCore/css/CSSPropertyNames.in")); - -my @sections = readiExploderFile(); -$sections[0] = generateSectionFromCSSPropertyNamesFile(); -writeiExploderFile(); - -print `svn stat $iExploderFile`; +update("cssproperties.in", "css/CSSPropertyNames.in"); +update("htmlattrs.in", "html/HTMLAttributeNames.in"); +update("htmltags.in", "html/HTMLTagNames.in"); print "Successfully updated!\n"; exit 0; -sub generateSectionFromCSSPropertyNamesFile() +sub generateEntityListFromFile($) { - my $revision = svnRevision($cssPropertyNamesFile); - my $path = File::Spec->abs2rel($cssPropertyNamesFile, sourceDir()); + my ($filename) = @_; + + my $revision = svnRevisionForDirectory(dirname($filename)); + my $path = File::Spec->abs2rel($filename, sourceDir()); my $result = "# From WebKit svn r" . $revision . " (" . $path . ")\n"; - my @properties = (); + my @entities = (); + my $in_namespace = 0; - open(IN, $cssPropertyNamesFile) || die "$!"; + open(IN, $filename) || die "$!"; while (my $l = <IN>) { chomp $l; + if ($l =~ m/^namespace=\"/) { + $in_namespace = 1; + } elsif ($in_namespace && $l =~ m/^$/) { + $in_namespace = 0; + } + + next if $in_namespace; next if $l =~ m/^\s*#/ || $l =~ m/^\s*$/; - push(@properties, $l); + + # For HTML Tags that can have additional information + if ($l =~ m/ /) { + my @split = split / /, $l; + $l = $split[0] + } + + push(@entities, $l); } close(IN); - $result .= join("\n", sort { $a cmp $b } @properties) . "\n\n"; + $result .= join("\n", sort { $a cmp $b } @entities) . "\n\n"; return $result; } -sub readiExploderFile() +sub readiExploderFile($) { + my ($filename) = @_; + my @sections = (); local $/ = "\n\n"; - open(IN, $iExploderFile) || die "$!"; + open(IN, $filename) || die "$!"; @sections = <IN>; close(IN); return @sections; } -sub svnRevision($) +sub update($$) { - my ($file) = @_; - my $revision = ""; + my ($iexploderPath, $webcorePath) = @_; - open INFO, "svn info '$file' |" or die; - while (<INFO>) { - if (/^Revision: (.+)/) { - $revision = $1; - } - } - close INFO; + $iexploderPath = File::Spec->catfile(sourceDir(), "WebKitTools", "iExploder", "htdocs", split("/", $iexploderPath)); + $webcorePath = File::Spec->catfile(sourceDir(), "WebCore", split("/", $webcorePath)); - return $revision ? $revision : "UNKNOWN"; + my @sections = readiExploderFile($iexploderPath); + $sections[0] = generateEntityListFromFile($webcorePath); + writeiExploderFile($iexploderPath, @sections); } -sub writeiExploderFile() + +sub writeiExploderFile($@) { - open(OUT, "> $iExploderFile") || die "$!"; + my ($filename, @sections) = @_; + + open(OUT, "> $filename") || die "$!"; print OUT join("", @sections); close(OUT); } + diff --git a/WebKitTools/Scripts/validate-committer-lists b/WebKitTools/Scripts/validate-committer-lists index 2f2dd32..ad3d358 100755 --- a/WebKitTools/Scripts/validate-committer-lists +++ b/WebKitTools/Scripts/validate-committer-lists @@ -36,13 +36,13 @@ import subprocess import re import urllib2 from datetime import date, datetime, timedelta -from webkitpy.committers import CommitterList -from webkitpy.webkit_logging import log, error +from webkitpy.common.config.committers import CommitterList +from webkitpy.common.system.deprecated_logging import log, error from webkitpy.scm import Git # WebKit includes a built copy of BeautifulSoup in Scripts/webkitpy # so this import should always succeed. -from webkitpy.BeautifulSoup import BeautifulSoup +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup def print_list_if_non_empty(title, list_to_print): if not list_to_print: diff --git a/WebKitTools/Scripts/webkit-patch b/WebKitTools/Scripts/webkit-patch index b4bcc4c..e0170ed 100755 --- a/WebKitTools/Scripts/webkit-patch +++ b/WebKitTools/Scripts/webkit-patch @@ -1,6 +1,7 @@ #!/usr/bin/env python # Copyright (c) 2009, Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -31,79 +32,25 @@ # A tool for automating dealing with bugzilla, posting patches, committing patches, etc. import os +import sys -from webkitpy.bugzilla import Bugzilla -from webkitpy.buildbot import BuildBot -from webkitpy.commands.download import * -from webkitpy.commands.early_warning_system import * -from webkitpy.commands.openbugs import OpenBugs -from webkitpy.commands.queries import * -from webkitpy.commands.queues import * -from webkitpy.commands.upload import * -from webkitpy.executive import Executive -from webkitpy.webkit_logging import log -from webkitpy.multicommandtool import MultiCommandTool -from webkitpy.scm import detect_scm_system -from webkitpy.user import User +from webkitpy.common.system.logutils import configure_logging +import webkitpy.python24.versioning as versioning -class WebKitPatch(MultiCommandTool): - global_options = [ - make_option("--dry-run", action="store_true", dest="dry_run", default=False, help="do not touch remote servers"), - make_option("--status-host", action="store", dest="status_host", type="string", nargs=1, help="Hostname (e.g. localhost or commit.webkit.org) where status updates should be posted."), - ] +def main(): + configure_logging() - def __init__(self): - MultiCommandTool.__init__(self) + versioning.check_version() - self.bugs = Bugzilla() - self.buildbot = BuildBot() - self.executive = Executive() - self.user = User() - self._scm = None - self.status_server = StatusServer() + # Import webkit-patch code only after version-checking so that + # script doesn't error out before having a chance to report the + # version warning. + from webkitpy.tool.main import WebKitPatch - def scm(self): - # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands). - original_cwd = os.path.abspath(".") - if not self._scm: - self._scm = detect_scm_system(original_cwd) - - if not self._scm: - script_directory = os.path.abspath(sys.path[0]) - webkit_directory = os.path.abspath(os.path.join(script_directory, "../..")) - self._scm = detect_scm_system(webkit_directory) - if self._scm: - log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory)) - else: - error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory)) - - return self._scm - - def path(self): - return __file__ - - def should_show_in_main_help(self, command): - if not command.show_in_main_help: - return False - if command.requires_local_commits: - return self.scm().supports_local_commits() - return True - - # FIXME: This may be unnecessary since we pass global options to all commands during execute() as well. - def handle_global_options(self, options): - if options.dry_run: - self.scm().dryrun = True - self.bugs.dryrun = True - if options.status_host: - self.status_server.set_host(options.status_host) - - def should_execute_command(self, command): - if command.requires_local_commits and not self.scm().supports_local_commits(): - failure_reason = "%s requires local commits using %s in %s." % (command.name, self.scm().display_name(), self.scm().checkout_root) - return (False, failure_reason) - return (True, None) + WebKitPatch(__file__).main() if __name__ == "__main__": - WebKitPatch().main() + + main() diff --git a/WebKitTools/Scripts/webkitdirs.pm b/WebKitTools/Scripts/webkitdirs.pm index 7985790..0b18373 100644 --- a/WebKitTools/Scripts/webkitdirs.pm +++ b/WebKitTools/Scripts/webkitdirs.pm @@ -63,6 +63,7 @@ my $isSymbian; my %qtFeatureDefaults; my $isGtk; my $isWx; +my $isEfl; my @wxArgs; my $isChromium; my $isInspectorFrontend; @@ -71,6 +72,7 @@ my $isInspectorFrontend; my $vcBuildPath; my $windowsTmpPath; my $windowsSourceDir; +my $willUseVCExpressWhenBuilding = 0; # Defined in VCSUtils. sub exitStatus($); @@ -245,6 +247,7 @@ sub argumentsForConfiguration() push(@args, '--qt') if isQt(); push(@args, '--symbian') if isSymbian(); push(@args, '--gtk') if isGtk(); + push(@args, '--efl') if isEfl(); push(@args, '--wx') if isWx(); push(@args, '--chromium') if isChromium(); push(@args, '--inspector-frontend') if isInspectorFrontend(); @@ -270,11 +273,11 @@ sub determineConfigurationProductDir if (isAppleWinWebKit() && !isWx()) { $configurationProductDir = "$baseProductDir/bin"; } else { - # [Gtk] We don't have Release/Debug configurations in straight + # [Gtk][Efl] We don't have Release/Debug configurations in straight # autotool builds (non build-webkit). In this case and if # WEBKITOUTPUTDIR exist, use that as our configuration dir. This will # allows us to run run-webkit-tests without using build-webkit. - if ($ENV{"WEBKITOUTPUTDIR"} && isGtk()) { + if ($ENV{"WEBKITOUTPUTDIR"} && (isGtk() || isEfl())) { $configurationProductDir = "$baseProductDir"; } else { $configurationProductDir = "$baseProductDir/$configuration"; @@ -325,7 +328,7 @@ sub jscProductDir my $productDir = productDir(); $productDir .= "/JavaScriptCore" if isQt(); $productDir .= "/$configuration" if (isQt() && isWindows()); - $productDir .= "/Programs" if isGtk(); + $productDir .= "/Programs" if (isGtk() || isEfl()); return $productDir; } @@ -541,6 +544,11 @@ sub builtDylibPathForName if (isDarwin() and -d "$configurationProductDir/lib/$libraryName.framework") { return "$configurationProductDir/lib/$libraryName.framework/$libraryName"; } elsif (isWindows()) { + if (configuration() eq "Debug") { + # On Windows, there is a "d" suffix to the library name. See <http://trac.webkit.org/changeset/53924/>. + $libraryName .= "d"; + } + my $mkspec = `qmake -query QMAKE_MKSPECS`; $mkspec =~ s/[\n|\r]$//g; my $qtMajorVersion = retrieveQMakespecVar("$mkspec/qconfig.pri", "QT_MAJOR_VERSION"); @@ -558,6 +566,9 @@ sub builtDylibPathForName if (isGtk()) { return "$configurationProductDir/$libraryName/../.libs/libwebkit-1.0.so"; } + if (isEfl()) { + return "$configurationProductDir/$libraryName/../.libs/libewebkit.so"; + } if (isAppleMacWebKit()) { return "$configurationProductDir/$libraryName.framework/Versions/A/$libraryName"; } @@ -569,7 +580,7 @@ sub builtDylibPathForName } } - die "Unsupported platform, can't determine built library locations."; + die "Unsupported platform, can't determine built library locations.\nTry `build-webkit --help` for more information.\n"; } # Check to see that all the frameworks are built. @@ -657,8 +668,8 @@ sub determineIsQt() return; } - # The presence of QTDIR only means Qt if --gtk is not on the command-line - if (isGtk() || isWx()) { + # The presence of QTDIR only means Qt if --gtk or --wx or --efl are not on the command-line + if (isGtk() || isWx() || isEfl()) { $isQt = 0; return; } @@ -678,6 +689,18 @@ sub determineIsSymbian() $isSymbian = defined($ENV{'EPOCROOT'}); } +sub determineIsEfl() +{ + return if defined($isEfl); + $isEfl = checkForArgumentAndRemoveFromARGV("--efl"); +} + +sub isEfl() +{ + determineIsEfl(); + return $isEfl; +} + sub isGtk() { determineIsGtk(); @@ -769,7 +792,7 @@ sub isLinux() sub isAppleWebKit() { - return !(isQt() or isGtk() or isWx() or isChromium()); + return !(isQt() or isGtk() or isWx() or isChromium() or isEfl()); } sub isAppleMacWebKit() @@ -856,7 +879,7 @@ sub relativeScriptsDir() sub launcherPath() { my $relativeScriptsPath = relativeScriptsDir(); - if (isGtk() || isQt() || isWx()) { + if (isGtk() || isQt() || isWx() || isEfl()) { return "$relativeScriptsPath/run-launcher"; } elsif (isAppleWebKit()) { return "$relativeScriptsPath/run-safari"; @@ -873,6 +896,8 @@ sub launcherName() return "wxBrowser"; } elsif (isAppleWebKit()) { return "Safari"; + } elsif (isEfl()) { + return "EWebLauncher"; } } @@ -895,7 +920,7 @@ sub checkRequiredSystemConfig print "http://developer.apple.com/tools/xcode\n"; print "*************************************************************\n"; } - } elsif (isGtk() or isQt() or isWx()) { + } elsif (isGtk() or isQt() or isWx() or isEfl()) { my @cmds = qw(flex bison gperf); my @missing = (); foreach my $cmd (@cmds) { @@ -1002,6 +1027,7 @@ sub setupCygwinEnv() print "*************************************************************\n"; die; } + $willUseVCExpressWhenBuilding = 1; } my $qtSDKPath = "$programFilesPath/QuickTime SDK"; @@ -1023,6 +1049,23 @@ sub setupCygwinEnv() print "WEBKITLIBRARIESDIR is set to: ", $ENV{"WEBKITLIBRARIESDIR"}, "\n"; } +sub dieIfWindowsPlatformSDKNotInstalled +{ + my $windowsPlatformSDKRegistryEntry = "/proc/registry/HKEY_LOCAL_MACHINE/SOFTWARE/Microsoft/MicrosoftSDK/InstalledSDKs/D2FF9F89-8AA2-4373-8A31-C838BF4DBBE1"; + + return if -e $windowsPlatformSDKRegistryEntry; + + print "*************************************************************\n"; + print "Cannot find '$windowsPlatformSDKRegistryEntry'.\n"; + print "Please download and install the Microsoft Windows Server 2003 R2\n"; + print "Platform SDK from <http://www.microsoft.com/downloads/details.aspx?\n"; + print "familyid=0baf2b35-c656-4969-ace8-e4c0c0716adb&displaylang=en>.\n\n"; + print "Then follow step 2 in the Windows section of the \"Installing Developer\n"; + print "Tools\" instructions at <http://www.webkit.org/building/tools.html>.\n"; + print "*************************************************************\n"; + die; +} + sub copyInspectorFrontendFiles { my $productDir = productDir(); @@ -1040,6 +1083,9 @@ sub copyInspectorFrontendFiles } elsif (isQt() || isGtk()) { my $prefix = $ENV{"WebKitInstallationPrefix"}; $inspectorResourcesDirPath = (defined($prefix) ? $prefix : "/usr/share") . "/webkit-1.0/webinspector"; + } elsif (isEfl()) { + my $prefix = $ENV{"WebKitInstallationPrefix"}; + $inspectorResourcesDirPath = (defined($prefix) ? $prefix : "/usr/share") . "/ewebkit/webinspector"; } if (! -d $inspectorResourcesDirPath) { @@ -1074,6 +1120,8 @@ sub buildVisualStudioProject my $config = configurationForVisualStudio(); + dieIfWindowsPlatformSDKNotInstalled() if $willUseVCExpressWhenBuilding; + chomp(my $winProjectPath = `cygpath -w "$project"`); my $action = "/build"; @@ -1403,6 +1451,19 @@ sub buildChromiumVisualStudioProject($$) $vsInstallDir = `cygpath "$vsInstallDir"` if isCygwin(); chomp $vsInstallDir; $vcBuildPath = "$vsInstallDir/Common7/IDE/devenv.com"; + if (! -e $vcBuildPath) { + # Visual Studio not found, try VC++ Express + $vcBuildPath = "$vsInstallDir/Common7/IDE/VCExpress.exe"; + if (! -e $vcBuildPath) { + print "*************************************************************\n"; + print "Cannot find '$vcBuildPath'\n"; + print "Please execute the file 'vcvars32.bat' from\n"; + print "'$programFilesPath\\Microsoft Visual Studio 8\\VC\\bin\\'\n"; + print "to setup the necessary environment variables.\n"; + print "*************************************************************\n"; + die; + } + } # Create command line and execute it. my @command = ($vcBuildPath, $projectPath, $action, $config); @@ -1491,4 +1552,44 @@ sub runSafari return 1; } +sub runMiniBrowser +{ + if (isAppleMacWebKit()) { + my $productDir = productDir(); + print "Starting MiniBrowser with DYLD_FRAMEWORK_PATH set to point to $productDir.\n"; + $ENV{DYLD_FRAMEWORK_PATH} = $productDir; + $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES"; + my $miniBrowserPath = "$productDir/MiniBrowser.app/Contents/MacOS/MiniBrowser"; + if (!isTiger() && architecture()) { + return system "arch", "-" . architecture(), $miniBrowserPath, @ARGV; + } else { + return system $miniBrowserPath, @ARGV; + } + } + + return 1; +} + +sub debugMiniBrowser +{ + if (isAppleMacWebKit()) { + my $gdbPath = "/usr/bin/gdb"; + die "Can't find gdb executable. Is gdb installed?\n" unless -x $gdbPath; + + my $productDir = productDir(); + + $ENV{DYLD_FRAMEWORK_PATH} = $productDir; + $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = 'YES'; + + my $miniBrowserPath = "$productDir/MiniBrowser.app/Contents/MacOS/MiniBrowser"; + + print "Starting MiniBrowser under gdb with DYLD_FRAMEWORK_PATH set to point to built WebKit2 in $productDir.\n"; + my @architectureFlags = ("-arch", architecture()) if !isTiger(); + exec $gdbPath, @architectureFlags, $miniBrowserPath or die; + return; + } + + return 1; +} + 1; diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/mergeChangeLogs.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/mergeChangeLogs.pl new file mode 100644 index 0000000..a226e43 --- /dev/null +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/mergeChangeLogs.pl @@ -0,0 +1,336 @@ +#!/usr/bin/perl +# +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Unit tests of VCSUtils::mergeChangeLogs(). + +use strict; + +use Test::Simple tests => 16; +use File::Temp qw(tempfile); +use VCSUtils; + +# Read contents of a file and return it. +sub readFile($) +{ + my ($fileName) = @_; + + local $/; + open(FH, "<", $fileName); + my $content = <FH>; + close(FH); + + return $content; +} + +# Write a temporary file and return the filename. +sub writeTempFile($$$) +{ + my ($name, $extension, $content) = @_; + + my ($FH, $fileName) = tempfile( + $name . "-XXXXXXXX", + DIR => ($ENV{'TMPDIR'} || "/tmp"), + UNLINK => 0, + ); + print $FH $content; + close $FH; + + if ($extension) { + my $newFileName = $fileName . $extension; + rename($fileName, $newFileName); + $fileName = $newFileName; + } + + return $fileName; +} + +# -------------------------------------------------------------------------------- + +{ + # New test + my $title = "mergeChangeLogs: traditional rejected patch success"; + + my $fileNewerContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileNewer = writeTempFile("file", "", $fileNewerContent); + + my $fileMineContent = <<'EOF'; +*************** +*** 1,3 **** + 2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. +--- 1,9 ---- ++ 2010-01-29 Oliver Hunt <oliver@apple.com> ++ ++ Reviewed by Darin Adler. ++ ++ JSC is failing to propagate anonymous slot count on some transitions ++ + 2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. +EOF + my $fileMine = writeTempFile("file", ".rej", $fileMineContent); + rename($fileMine, $fileNewer . ".rej"); + $fileMine = $fileNewer . ".rej"; + + my $fileOlderContent = $fileNewerContent; + my $fileOlder = writeTempFile("file", ".orig", $fileOlderContent); + rename($fileOlder, $fileNewer . ".orig"); + $fileOlder = $fileNewer . ".orig"; + + my $exitStatus = mergeChangeLogs($fileMine, $fileOlder, $fileNewer); + + # mergeChangeLogs() should return 1 since the patch succeeded. + ok($exitStatus == 1, "$title: should return 1 for success"); + + ok(readFile($fileMine) eq $fileMineContent, "$title: \$fileMine should be unchanged"); + ok(readFile($fileOlder) eq $fileOlderContent, "$title: \$fileOlder should be unchanged"); + + my $expectedContent = <<'EOF'; +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +EOF + $expectedContent .= $fileNewerContent; + ok(readFile($fileNewer) eq $expectedContent, "$title: \$fileNewer should be updated to include patch"); + + unlink($fileMine, $fileOlder, $fileNewer); +} + +# -------------------------------------------------------------------------------- + +{ + # New test + my $title = "mergeChangeLogs: traditional rejected patch failure"; + + my $fileNewerContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileNewer = writeTempFile("file", "", $fileNewerContent); + + my $fileMineContent = <<'EOF'; +*************** +*** 1,9 **** +- 2010-01-29 Oliver Hunt <oliver@apple.com> +- +- Reviewed by Darin Adler. +- +- JSC is failing to propagate anonymous slot count on some transitions +- + 2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. +--- 1,3 ---- + 2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. +EOF + my $fileMine = writeTempFile("file", ".rej", $fileMineContent); + rename($fileMine, $fileNewer . ".rej"); + $fileMine = $fileNewer . ".rej"; + + my $fileOlderContent = $fileNewerContent; + my $fileOlder = writeTempFile("file", ".orig", $fileOlderContent); + rename($fileOlder, $fileNewer . ".orig"); + $fileOlder = $fileNewer . ".orig"; + + my $exitStatus = mergeChangeLogs($fileMine, $fileOlder, $fileNewer); + + # mergeChangeLogs() should return 0 since the patch failed. + ok($exitStatus == 0, "$title: should return 0 for failure"); + + ok(readFile($fileMine) eq $fileMineContent, "$title: \$fileMine should be unchanged"); + ok(readFile($fileOlder) eq $fileOlderContent, "$title: \$fileOlder should be unchanged"); + ok(readFile($fileNewer) eq $fileNewerContent, "$title: \$fileNewer should be unchanged"); + + unlink($fileMine, $fileOlder, $fileNewer); +} + +# -------------------------------------------------------------------------------- + +{ + # New test + my $title = "mergeChangeLogs: patch succeeds"; + + my $fileMineContent = <<'EOF'; +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileMine = writeTempFile("fileMine", "", $fileMineContent); + + my $fileOlderContent = <<'EOF'; +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileOlder = writeTempFile("fileOlder", "", $fileOlderContent); + + my $fileNewerContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileNewer = writeTempFile("fileNewer", "", $fileNewerContent); + + my $exitStatus = mergeChangeLogs($fileMine, $fileOlder, $fileNewer); + + # mergeChangeLogs() should return 1 since the patch succeeded. + ok($exitStatus == 1, "$title: should return 1 for success"); + + ok(readFile($fileMine) eq $fileMineContent, "$title: \$fileMine should be unchanged"); + ok(readFile($fileOlder) eq $fileOlderContent, "$title: \$fileOlder should be unchanged"); + + my $expectedContent = <<'EOF'; +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +EOF + $expectedContent .= $fileNewerContent; + + ok(readFile($fileNewer) eq $expectedContent, "$title: \$fileNewer should be patched"); + + unlink($fileMine, $fileOlder, $fileNewer); +} + +# -------------------------------------------------------------------------------- + +{ + # New test + my $title = "mergeChangeLogs: patch fails"; + + my $fileMineContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileMine = writeTempFile("fileMine", "", $fileMineContent); + + my $fileOlderContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileOlder = writeTempFile("fileOlder", "", $fileOlderContent); + + my $fileNewerContent = <<'EOF'; +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileNewer = writeTempFile("fileNewer", "", $fileNewerContent); + + my $exitStatus = mergeChangeLogs($fileMine, $fileOlder, $fileNewer); + + # mergeChangeLogs() should return a non-zero exit status since the patch failed. + ok($exitStatus == 0, "$title: return non-zero exit status for failure"); + + ok(readFile($fileMine) eq $fileMineContent, "$title: \$fileMine should be unchanged"); + ok(readFile($fileOlder) eq $fileOlderContent, "$title: \$fileOlder should be unchanged"); + + # $fileNewer should still exist unchanged because the patch failed + ok(readFile($fileNewer) eq $fileNewerContent, "$title: \$fileNewer should be unchanged"); + + unlink($fileMine, $fileOlder, $fileNewer); +} + +# -------------------------------------------------------------------------------- + diff --git a/WebKitTools/Scripts/webkitperl/httpd.pm b/WebKitTools/Scripts/webkitperl/httpd.pm index 05eb21c..240f368 100644 --- a/WebKitTools/Scripts/webkitperl/httpd.pm +++ b/WebKitTools/Scripts/webkitperl/httpd.pm @@ -31,6 +31,7 @@ use strict; use warnings; +use File::Copy; use File::Path; use File::Spec; use File::Spec::Functions; diff --git a/WebKitTools/Scripts/webkitpy/__init__.py b/WebKitTools/Scripts/webkitpy/__init__.py index 94ecc70..b376bf2 100644 --- a/WebKitTools/Scripts/webkitpy/__init__.py +++ b/WebKitTools/Scripts/webkitpy/__init__.py @@ -1,8 +1,13 @@ # Required for Python to search this directory for module files -import autoinstall - -# List our third-party library dependencies here and where they can be -# downloaded. -autoinstall.bind("ClientForm", "http://pypi.python.org/packages/source/C/ClientForm/ClientForm-0.2.10.zip", "ClientForm-0.2.10") -autoinstall.bind("mechanize", "http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", "mechanize-0.1.11") +# Keep this file free of any code or import statements that could +# cause either an error to occur or a log message to be logged. +# This ensures that calling code can import initialization code from +# webkitpy before any errors or log messages due to code in this file. +# Initialization code can include things like version-checking code and +# logging configuration code. +# +# We do not execute any version-checking code or logging configuration +# code in this file so that callers can opt-in as they want. This also +# allows different callers to choose different initialization code, +# as necessary. diff --git a/WebKitTools/Scripts/webkitpy/autoinstall.py b/WebKitTools/Scripts/webkitpy/autoinstall.py deleted file mode 100644 index 467e6b4..0000000 --- a/WebKitTools/Scripts/webkitpy/autoinstall.py +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright (c) 2009, Daniel Krech All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the Daniel Krech nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""\ -package loader for auto installing Python packages. - -A package loader in the spirit of Zero Install that can be used to -inject dependencies into the import process. - - -To install:: - - easy_install -U autoinstall - - or - - download, unpack, python setup.py install - - or - - try the bootstrap loader. See below. - - -To use:: - - # You can bind any package name to a URL pointing to something - # that can be imported using the zipimporter. - - autoinstall.bind("pymarc", "http://pypi.python.org/packages/2.5/p/pymarc/pymarc-2.1-py2.5.egg") - - import pymarc - - print pymarc.__version__, pymarc.__file__ - - -Changelog:: - -- added support for non top level packages. -- cache files now use filename part from URL. -- applied patch from Eric Seidel <eseidel@google.com> to add support -for loading modules where the module is not at the root of the .zip -file. - - -TODO:: - -- a description of the intended use case -- address other issues pointed out in: - - http://mail.python.org/pipermail/python-dev/2008-March/077926.html - -Scribbles:: - -pull vs. push -user vs. system -web vs. filesystem -auto vs. manual - -manage development sandboxes - -optional interfaces... - - def get_data(pathname) -> string with file data. - - Return the data associated with 'pathname'. Raise IOError if - the file wasn't found."); - - def is_package, - "is_package(fullname) -> bool. - - Return True if the module specified by fullname is a package. - Raise ZipImportError is the module couldn't be found."); - - def get_code, - "get_code(fullname) -> code object. - - Return the code object for the specified module. Raise ZipImportError - is the module couldn't be found."); - - def get_source, - "get_source(fullname) -> source string. - - Return the source code for the specified module. Raise ZipImportError - is the module couldn't be found, return None if the archive does - contain the module, but has no source for it."); - - -Autoinstall can also be bootstraped with the nascent package loader -bootstrap module. For example:: - - # or via the bootstrap - # loader. - - try: - _version = "0.2" - import autoinstall - if autoinstall.__version__ != _version: - raise ImportError("A different version than expected found.") - except ImportError, e: - # http://svn.python.org/projects/sandbox/trunk/bootstrap/bootstrap.py - import bootstrap - pypi = "http://pypi.python.org" - dir = "packages/source/a/autoinstall" - url = "%s/%s/autoinstall-%s.tar.gz" % (pypi, dir, _version) - bootstrap.main((url,)) - import autoinstall - -References:: - - http://0install.net/ - http://www.python.org/dev/peps/pep-0302/ - http://svn.python.org/projects/sandbox/trunk/import_in_py - http://0install.net/injector-find.html - http://roscidus.com/desktop/node/903 - -""" - -# To allow use of the "with" keyword for Python 2.5 users. -from __future__ import with_statement - -__version__ = "0.2" -__docformat__ = "restructuredtext en" - -import os -import new -import sys -import urllib -import logging -import tempfile -import zipimport - -_logger = logging.getLogger(__name__) - - -_importer = None - -def _getImporter(): - global _importer - if _importer is None: - _importer = Importer() - sys.meta_path.append(_importer) - return _importer - -def bind(package_name, url, zip_subpath=None): - """bind a top level package name to a URL. - - The package name should be a package name and the url should be a - url to something that can be imported using the zipimporter. - - Optional zip_subpath parameter allows searching for modules - below the root level of the zip file. - """ - _getImporter().bind(package_name, url, zip_subpath) - - -class Cache(object): - - def __init__(self, directory=None): - if directory is None: - # Default to putting the cache directory in the same directory - # as this file. - containing_directory = os.path.dirname(__file__) - directory = os.path.join(containing_directory, "autoinstall.cache.d"); - - self.directory = directory - try: - if not os.path.exists(self.directory): - self._create_cache_directory() - except Exception, err: - _logger.exception(err) - self.cache_directry = tempfile.mkdtemp() - _logger.info("Using cache directory '%s'." % self.directory) - - def _create_cache_directory(self): - _logger.debug("Creating cache directory '%s'." % self.directory) - os.mkdir(self.directory) - readme_path = os.path.join(self.directory, "README") - with open(readme_path, "w") as f: - f.write("This directory was auto-generated by '%s'.\n" - "It is safe to delete.\n" % __file__) - - def get(self, url): - _logger.info("Getting '%s' from cache." % url) - filename = url.rsplit("/")[-1] - - # so that source url is significant in determining cache hits - d = os.path.join(self.directory, "%s" % hash(url)) - if not os.path.exists(d): - os.mkdir(d) - - filename = os.path.join(d, filename) - - if os.path.exists(filename): - _logger.debug("... already cached in file '%s'." % filename) - else: - _logger.debug("... not in cache. Caching in '%s'." % filename) - stream = file(filename, "wb") - self.download(url, stream) - stream.close() - return filename - - def download(self, url, stream): - _logger.info("Downloading: %s" % url) - try: - netstream = urllib.urlopen(url) - code = 200 - if hasattr(netstream, "getcode"): - code = netstream.getcode() - if not 200 <= code < 300: - raise ValueError("HTTP Error code %s" % code) - except Exception, err: - _logger.exception(err) - - BUFSIZE = 2**13 # 8KB - size = 0 - while True: - data = netstream.read(BUFSIZE) - if not data: - break - stream.write(data) - size += len(data) - netstream.close() - _logger.info("Downloaded %d bytes." % size) - - -class Importer(object): - - def __init__(self): - self.packages = {} - self.__cache = None - - def __get_store(self): - return self.__store - store = property(__get_store) - - def _get_cache(self): - if self.__cache is None: - self.__cache = Cache() - return self.__cache - def _set_cache(self, cache): - self.__cache = cache - cache = property(_get_cache, _set_cache) - - def find_module(self, fullname, path=None): - """-> self or None. - - Search for a module specified by 'fullname'. 'fullname' must be - the fully qualified (dotted) module name. It returns the - zipimporter instance itself if the module was found, or None if - it wasn't. The optional 'path' argument is ignored -- it's - there for compatibility with the importer protocol."); - """ - _logger.debug("find_module(%s, path=%s)" % (fullname, path)) - - if fullname in self.packages: - (url, zip_subpath) = self.packages[fullname] - filename = self.cache.get(url) - zip_path = "%s/%s" % (filename, zip_subpath) if zip_subpath else filename - _logger.debug("fullname: %s url: %s path: %s zip_path: %s" % (fullname, url, path, zip_path)) - try: - loader = zipimport.zipimporter(zip_path) - _logger.debug("returning: %s" % loader) - except Exception, e: - _logger.exception(e) - return None - return loader - return None - - def bind(self, package_name, url, zip_subpath): - _logger.info("binding: %s -> %s subpath: %s" % (package_name, url, zip_subpath)) - self.packages[package_name] = (url, zip_subpath) - - -if __name__=="__main__": - import logging - #logging.basicConfig() - logger = logging.getLogger() - - console = logging.StreamHandler() - console.setLevel(logging.DEBUG) - # set a format which is simpler for console use - formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') - # tell the handler to use this format - console.setFormatter(formatter) - # add the handler to the root logger - logger.addHandler(console) - logger.setLevel(logging.INFO) - - bind("pymarc", "http://pypi.python.org/packages/2.5/p/pymarc/pymarc-2.1-py2.5.egg") - - import pymarc - - print pymarc.__version__, pymarc.__file__ - - assert pymarc.__version__=="2.1" - - d = _getImporter().cache.directory - assert d in pymarc.__file__, "'%s' not found in pymarc.__file__ (%s)" % (d, pymarc.__file__) - - # Can now also bind to non top level packages. The packages - # leading up to the package being bound will need to be defined - # however. - # - # bind("rdf.plugins.stores.memory", - # "http://pypi.python.org/packages/2.5/r/rdf.plugins.stores.memeory/rdf.plugins.stores.memory-0.9a-py2.5.egg") - # - # from rdf.plugins.stores.memory import Memory - - diff --git a/WebKitTools/Scripts/webkitpy/buildbot.py b/WebKitTools/Scripts/webkitpy/buildbot.py deleted file mode 100644 index 38828fd..0000000 --- a/WebKitTools/Scripts/webkitpy/buildbot.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# WebKit's Python module for interacting with WebKit's buildbot - -import re -import urllib2 - -# Import WebKit-specific modules. -from webkitpy.webkit_logging import log - -# WebKit includes a built copy of BeautifulSoup in Scripts/webkitpy -# so this import should always succeed. -from .BeautifulSoup import BeautifulSoup - - -class BuildBot: - - default_host = "build.webkit.org" - - def __init__(self, host=default_host): - self.buildbot_host = host - self.buildbot_server_url = "http://%s/" % self.buildbot_host - - # If any Leopard builder/tester, Windows builder or Chromium builder is - # red we should not be landing patches. Other builders should be added - # to this list once they are known to be reliable. - # See https://bugs.webkit.org/show_bug.cgi?id=33296 and related bugs. - self.core_builder_names_regexps = [ - "Leopard", - "Windows.*Build", - "Chromium", - ] - - def _parse_builder_status_from_row(self, status_row): - # If WebKit's buildbot has an XMLRPC interface we could use, we could - # do something more sophisticated here. For now we just parse out the - # basics, enough to support basic questions like "is the tree green?" - status_cells = status_row.findAll('td') - builder = {} - - name_link = status_cells[0].find('a') - builder['name'] = name_link.string - # We could generate the builder_url from the name in a future version - # of this code. - builder['builder_url'] = self.buildbot_server_url + name_link['href'] - - status_link = status_cells[1].find('a') - if not status_link: - # We failed to find a link in the first cell, just give up. This - # can happen if a builder is just-added, the first cell will just - # be "no build" - # Other parts of the code depend on is_green being present. - builder['is_green'] = False - return builder - # Will be either a revision number or a build number - revision_string = status_link.string - # If revision_string has non-digits assume it's not a revision number. - builder['built_revision'] = int(revision_string) \ - if not re.match('\D', revision_string) \ - else None - builder['is_green'] = not re.search('fail', - status_cells[1].renderContents()) - # We could parse out the build number instead, but for now just store - # the URL. - builder['build_url'] = self.buildbot_server_url + status_link['href'] - - # We could parse out the current activity too. - - return builder - - def _builder_statuses_with_names_matching_regexps(self, - builder_statuses, - name_regexps): - builders = [] - for builder in builder_statuses: - for name_regexp in name_regexps: - if re.match(name_regexp, builder['name']): - builders.append(builder) - return builders - - def red_core_builders(self): - red_builders = [] - for builder in self._builder_statuses_with_names_matching_regexps( - self.builder_statuses(), - self.core_builder_names_regexps): - if not builder['is_green']: - red_builders.append(builder) - return red_builders - - def red_core_builders_names(self): - red_builders = self.red_core_builders() - return map(lambda builder: builder['name'], red_builders) - - def core_builders_are_green(self): - return not self.red_core_builders() - - def builder_statuses(self): - build_status_url = self.buildbot_server_url + 'one_box_per_builder' - page = urllib2.urlopen(build_status_url) - soup = BeautifulSoup(page) - - builders = [] - status_table = soup.find('table') - for status_row in status_table.findAll('tr'): - builder = self._parse_builder_status_from_row(status_row) - builders.append(builder) - return builders diff --git a/WebKitTools/Scripts/webkitpy/buildbot_unittest.py b/WebKitTools/Scripts/webkitpy/buildbot_unittest.py deleted file mode 100644 index bde3e04..0000000 --- a/WebKitTools/Scripts/webkitpy/buildbot_unittest.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (C) 2009 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import unittest - -from webkitpy.buildbot import BuildBot - -from webkitpy.BeautifulSoup import BeautifulSoup - -class BuildBotTest(unittest.TestCase): - - _example_one_box_status = ''' - <table> - <tr> - <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td> - <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td> - <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td> - <tr> - <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td> - <td class="LastBuild box" >no build</td> - <td align="center" class="Activity building">building<br />< 1 min</td> - <tr> - <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td> - <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td> - <td align="center" class="Activity idle">idle</td> - </table> -''' - _expected_example_one_box_parsings = [ - { - 'builder_url': u'http://build.webkit.org/builders/Windows%20Debug%20%28Tests%29', - 'build_url': u'http://build.webkit.org/builders/Windows%20Debug%20%28Tests%29/builds/3693', - 'is_green': True, - 'name': u'Windows Debug (Tests)', - 'built_revision': 47380 - }, - { - 'builder_url': u'http://build.webkit.org/builders/SnowLeopard%20Intel%20Release', - 'is_green': False, - 'name': u'SnowLeopard Intel Release', - }, - { - 'builder_url': u'http://build.webkit.org/builders/Qt%20Linux%20Release', - 'build_url': u'http://build.webkit.org/builders/Qt%20Linux%20Release/builds/654', - 'is_green': False, - 'name': u'Qt Linux Release', - 'built_revision': 47383 - }, - ] - - def test_status_parsing(self): - buildbot = BuildBot() - - soup = BeautifulSoup(self._example_one_box_status) - status_table = soup.find("table") - input_rows = status_table.findAll('tr') - - for x in range(len(input_rows)): - status_row = input_rows[x] - expected_parsing = self._expected_example_one_box_parsings[x] - - builder = buildbot._parse_builder_status_from_row(status_row) - - # Make sure we aren't parsing more or less than we expect - self.assertEquals(builder.keys(), expected_parsing.keys()) - - for key, expected_value in expected_parsing.items(): - self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value))) - - def test_core_builder_methods(self): - buildbot = BuildBot() - - # Override builder_statuses function to not touch the network. - def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to. - return BuildBotTest._expected_example_one_box_parsings - buildbot.builder_statuses = example_builder_statuses - - buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ] - self.assertEquals(buildbot.red_core_builders_names(), []) - self.assertTrue(buildbot.core_builders_are_green()) - - buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ] - self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ]) - self.assertFalse(buildbot.core_builders_are_green()) - - def test_builder_name_regexps(self): - buildbot = BuildBot() - - # For complete testing, this list should match the list of builders at build.webkit.org: - example_builders = [ - { 'name': u'Tiger Intel Release', }, - { 'name': u'Leopard Intel Release (Build)', }, - { 'name': u'Leopard Intel Release (Tests)', }, - { 'name': u'Leopard Intel Debug (Build)', }, - { 'name': u'Leopard Intel Debug (Tests)', }, - { 'name': u'SnowLeopard Intel Release (Build)', }, - { 'name': u'SnowLeopard Intel Release (Tests)', }, - { 'name': u'SnowLeopard Intel Leaks', }, - { 'name': u'Windows Release (Build)', }, - { 'name': u'Windows Release (Tests)', }, - { 'name': u'Windows Debug (Build)', }, - { 'name': u'Windows Debug (Tests)', }, - { 'name': u'Qt Linux Release', }, - { 'name': u'Gtk Linux Release', }, - { 'name': u'Gtk Linux 32-bit Debug', }, - { 'name': u'Gtk Linux 64-bit Debug', }, - { 'name': u'Chromium Linux Release', }, - { 'name': u'Chromium Mac Release', }, - { 'name': u'Chromium Win Release', }, - ] - name_regexps = [ "Leopard", "Windows.*Build", "Chromium" ] - expected_builders = [ - { 'name': u'Leopard Intel Release (Build)', }, - { 'name': u'Leopard Intel Release (Tests)', }, - { 'name': u'Leopard Intel Debug (Build)', }, - { 'name': u'Leopard Intel Debug (Tests)', }, - { 'name': u'Windows Release (Build)', }, - { 'name': u'Windows Debug (Build)', }, - { 'name': u'Chromium Linux Release', }, - { 'name': u'Chromium Mac Release', }, - { 'name': u'Chromium Win Release', }, - ] - - # This test should probably be updated if the default regexp list changes - self.assertEquals(buildbot.core_builder_names_regexps, name_regexps) - - builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps) - self.assertEquals(builders, expected_builders) - -if __name__ == '__main__': - unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/commands/queries.py b/WebKitTools/Scripts/webkitpy/commands/queries.py deleted file mode 100644 index 3ca4f42..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/queries.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. -# Copyright (c) 2009 Apple Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -from optparse import make_option - -from webkitpy.buildbot import BuildBot -from webkitpy.committers import CommitterList -from webkitpy.webkit_logging import log -from webkitpy.multicommandtool import AbstractDeclarativeCommand - - -class BugsToCommit(AbstractDeclarativeCommand): - name = "bugs-to-commit" - help_text = "List bugs in the commit-queue" - - def execute(self, options, args, tool): - # FIXME: This command is poorly named. It's fetching the commit-queue list here. The name implies it's fetching pending-commit (all r+'d patches). - bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue() - for bug_id in bug_ids: - print "%s" % bug_id - - -class PatchesInCommitQueue(AbstractDeclarativeCommand): - name = "patches-in-commit-queue" - help_text = "List patches in the commit-queue" - - def execute(self, options, args, tool): - patches = tool.bugs.queries.fetch_patches_from_commit_queue() - log("Patches in commit queue:") - for patch in patches: - print patch.url() - - -class PatchesToCommitQueue(AbstractDeclarativeCommand): - name = "patches-to-commit-queue" - help_text = "Patches which should be added to the commit queue" - def __init__(self): - options = [ - make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"), - ] - AbstractDeclarativeCommand.__init__(self, options=options) - - @staticmethod - def _needs_commit_queue(patch): - if patch.commit_queue() == "+": # If it's already cq+, ignore the patch. - log("%s already has cq=%s" % (patch.id(), patch.commit_queue())) - return False - - # We only need to worry about patches from contributers who are not yet committers. - committer_record = CommitterList().committer_by_email(patch.attacher_email()) - if committer_record: - log("%s committer = %s" % (patch.id(), committer_record)) - return not committer_record - - def execute(self, options, args, tool): - patches = tool.bugs.queries.fetch_patches_from_pending_commit_list() - patches_needing_cq = filter(self._needs_commit_queue, patches) - if options.bugs: - bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq) - bugs_needing_cq = sorted(set(bugs_needing_cq)) - for bug_id in bugs_needing_cq: - print "%s" % tool.bugs.bug_url_for_bug_id(bug_id) - else: - for patch in patches_needing_cq: - print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit") - - -class PatchesToReview(AbstractDeclarativeCommand): - name = "patches-to-review" - help_text = "List patches that are pending review" - - def execute(self, options, args, tool): - patch_ids = tool.bugs.queries.fetch_attachment_ids_from_review_queue() - log("Patches pending review:") - for patch_id in patch_ids: - print patch_id - - -class TreeStatus(AbstractDeclarativeCommand): - name = "tree-status" - help_text = "Print the status of the %s buildbots" % BuildBot.default_host - long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder -and displayes the status of each builder.""" - - def execute(self, options, args, tool): - for builder in tool.buildbot.builder_statuses(): - status_string = "ok" if builder["is_green"] else "FAIL" - print "%s : %s" % (status_string.ljust(4), builder["name"]) diff --git a/WebKitTools/Scripts/webkitpy/commands/__init__.py b/WebKitTools/Scripts/webkitpy/common/__init__.py index ef65bee..ef65bee 100644 --- a/WebKitTools/Scripts/webkitpy/commands/__init__.py +++ b/WebKitTools/Scripts/webkitpy/common/__init__.py diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/__init__.py b/WebKitTools/Scripts/webkitpy/common/checkout/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/api.py b/WebKitTools/Scripts/webkitpy/common/checkout/api.py new file mode 100644 index 0000000..c4e2b69 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/api.py @@ -0,0 +1,140 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import subprocess +import StringIO + +from webkitpy.common.checkout.changelog import ChangeLog +from webkitpy.common.checkout.commitinfo import CommitInfo +from webkitpy.common.checkout.scm import CommitMessage +from webkitpy.common.net.bugzilla import parse_bug_id +from webkitpy.common.system.executive import Executive, run_command, ScriptError +from webkitpy.common.system.deprecated_logging import log + + +# This class represents the WebKit-specific parts of the checkout (like +# ChangeLogs). +# FIXME: Move a bunch of ChangeLog-specific processing from SCM to this object. +class Checkout(object): + def __init__(self, scm): + self._scm = scm + + def _is_path_to_changelog(self, path): + return os.path.basename(path) == "ChangeLog" + + def _latest_entry_for_changelog_at_revision(self, changelog_path, revision): + changelog_contents = self._scm.contents_at_revision(changelog_path, revision) + return ChangeLog.parse_latest_entry_from_file(StringIO.StringIO(changelog_contents)) + + def changelog_entries_for_revision(self, revision): + changed_files = self._scm.changed_files_for_revision(revision) + return [self._latest_entry_for_changelog_at_revision(path, revision) for path in changed_files if self._is_path_to_changelog(path)] + + def commit_info_for_revision(self, revision): + committer_email = self._scm.committer_email_for_revision(revision) + changelog_entries = self.changelog_entries_for_revision(revision) + # Assume for now that the first entry has everything we need: + # FIXME: This will throw an exception if there were no ChangeLogs. + if not len(changelog_entries): + return None + changelog_entry = changelog_entries[0] + changelog_data = { + "bug_id": parse_bug_id(changelog_entry.contents()), + "author_name": changelog_entry.author_name(), + "author_email": changelog_entry.author_email(), + "author": changelog_entry.author(), + "reviewer_text": changelog_entry.reviewer_text(), + "reviewer": changelog_entry.reviewer(), + } + # We could pass the changelog_entry instead of a dictionary here, but that makes + # mocking slightly more involved, and would make aggregating data from multiple + # entries more difficult to wire in if we need to do that in the future. + return CommitInfo(revision, committer_email, changelog_data) + + def bug_id_for_revision(self, revision): + return self.commit_info_for_revision(revision).bug_id() + + def modified_changelogs(self): + # SCM returns paths relative to scm.checkout_root + # Callers (especially those using the ChangeLog class) may + # expect absolute paths, so this method returns absolute paths. + changed_files = self._scm.changed_files() + absolute_paths = [os.path.join(self._scm.checkout_root, path) for path in changed_files] + return [path for path in absolute_paths if self._is_path_to_changelog(path)] + + def commit_message_for_this_commit(self): + changelog_paths = self.modified_changelogs() + if not len(changelog_paths): + raise ScriptError(message="Found no modified ChangeLogs, cannot create a commit message.\n" + "All changes require a ChangeLog. See:\n" + "http://webkit.org/coding/contributing.html") + + changelog_messages = [] + for changelog_path in changelog_paths: + log("Parsing ChangeLog: %s" % changelog_path) + changelog_entry = ChangeLog(changelog_path).latest_entry() + if not changelog_entry: + raise ScriptError(message="Failed to parse ChangeLog: %s" % os.path.abspath(changelog_path)) + changelog_messages.append(changelog_entry.contents()) + + # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does. + return CommitMessage("".join(changelog_messages).splitlines()) + + def bug_id_for_this_commit(self): + try: + return parse_bug_id(self.commit_message_for_this_commit().message()) + except ScriptError, e: + pass # We might not have ChangeLogs. + + def apply_patch(self, patch, force=False): + # It's possible that the patch was not made from the root directory. + # We should detect and handle that case. + # FIXME: Use Executive instead of subprocess here. + curl_process = subprocess.Popen(['curl', '--location', '--silent', '--show-error', patch.url()], stdout=subprocess.PIPE) + # FIXME: Move _scm.script_path here once we get rid of all the dependencies. + args = [self._scm.script_path('svn-apply')] + if patch.reviewer(): + args += ['--reviewer', patch.reviewer().full_name] + if force: + args.append('--force') + + run_command(args, input=curl_process.stdout) + + def apply_reverse_diff(self, revision): + self._scm.apply_reverse_diff(revision) + + # We revert the ChangeLogs because removing lines from a ChangeLog + # doesn't make sense. ChangeLogs are append only. + changelog_paths = self.modified_changelogs() + if len(changelog_paths): + self._scm.revert_files(changelog_paths) + + conflicts = self._scm.conflicted_files() + if len(conflicts): + raise ScriptError(message="Failed to apply reverse diff for revision %s because of the following conflicts:\n%s" % (revision, "\n".join(conflicts))) diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/api_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/api_unittest.py new file mode 100644 index 0000000..e99caee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/api_unittest.py @@ -0,0 +1,169 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import shutil +import tempfile +import unittest + +from webkitpy.common.checkout.api import Checkout +from webkitpy.common.checkout.changelog import ChangeLogEntry +from webkitpy.common.checkout.scm import detect_scm_system, CommitMessage +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock + +# FIXME: Copied from scm_unittest.py +def write_into_file_at_path(file_path, contents): + new_file = open(file_path, 'w') + new_file.write(contents) + new_file.close() + + +_changelog1entry1 = """2010-03-25 Eric Seidel <eric@webkit.org> + + Unreviewed build fix to un-break webkit-patch land. + + Move commit_message_for_this_commit from scm to checkout + https://bugs.webkit.org/show_bug.cgi?id=36629 + + * Scripts/webkitpy/common/checkout/api.py: import scm.CommitMessage +""" +_changelog1entry2 = """2010-03-25 Adam Barth <abarth@webkit.org> + + Reviewed by Eric Seidel. + + Move commit_message_for_this_commit from scm to checkout + https://bugs.webkit.org/show_bug.cgi?id=36629 + + * Scripts/webkitpy/common/checkout/api.py: +""" +_changelog1 = "\n".join([_changelog1entry1, _changelog1entry2]) +_changelog2 = """2010-03-25 Eric Seidel <eric@webkit.org> + + Unreviewed build fix to un-break webkit-patch land. + + Second part of this complicated change. + + * Path/To/Complicated/File: Added. + +2010-03-25 Adam Barth <abarth@webkit.org> + + Reviewed by Eric Seidel. + + Filler change. +""" + +class CommitMessageForThisCommitTest(unittest.TestCase): + expected_commit_message = """2010-03-25 Eric Seidel <eric@webkit.org> + + Unreviewed build fix to un-break webkit-patch land. + + Move commit_message_for_this_commit from scm to checkout + https://bugs.webkit.org/show_bug.cgi?id=36629 + + * Scripts/webkitpy/common/checkout/api.py: import scm.CommitMessage +2010-03-25 Eric Seidel <eric@webkit.org> + + Unreviewed build fix to un-break webkit-patch land. + + Second part of this complicated change. + + * Path/To/Complicated/File: Added. +""" + + def setUp(self): + self.temp_dir = tempfile.mkdtemp(suffix="changelogs") + self.old_cwd = os.getcwd() + os.chdir(self.temp_dir) + write_into_file_at_path("ChangeLog1", _changelog1) + write_into_file_at_path("ChangeLog2", _changelog2) + + def tearDown(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + os.chdir(self.old_cwd) + + # FIXME: This should not need to touch the file system, however + # ChangeLog is difficult to mock at current. + def test_commit_message_for_this_commit(self): + checkout = Checkout(None) + checkout.modified_changelogs = lambda: ["ChangeLog1", "ChangeLog2"] + output = OutputCapture() + expected_stderr = "Parsing ChangeLog: ChangeLog1\nParsing ChangeLog: ChangeLog2\n" + commit_message = output.assert_outputs(self, checkout.commit_message_for_this_commit, expected_stderr=expected_stderr) + self.assertEqual(commit_message.message(), self.expected_commit_message) + + +class CheckoutTest(unittest.TestCase): + def test_latest_entry_for_changelog_at_revision(self): + scm = Mock() + def mock_contents_at_revision(changelog_path, revision): + self.assertEqual(changelog_path, "foo") + self.assertEqual(revision, "bar") + return _changelog1 + scm.contents_at_revision = mock_contents_at_revision + checkout = Checkout(scm) + entry = checkout._latest_entry_for_changelog_at_revision("foo", "bar") + self.assertEqual(entry.contents(), _changelog1entry1) + + def test_commit_info_for_revision(self): + scm = Mock() + scm.committer_email_for_revision = lambda revision: "committer@example.com" + checkout = Checkout(scm) + checkout.changelog_entries_for_revision = lambda revision: [ChangeLogEntry(_changelog1entry1)] + commitinfo = checkout.commit_info_for_revision(4) + self.assertEqual(commitinfo.bug_id(), 36629) + self.assertEqual(commitinfo.author_name(), "Eric Seidel") + self.assertEqual(commitinfo.author_email(), "eric@webkit.org") + self.assertEqual(commitinfo.reviewer_text(), None) + self.assertEqual(commitinfo.reviewer(), None) + self.assertEqual(commitinfo.committer_email(), "committer@example.com") + self.assertEqual(commitinfo.committer(), None) + + checkout.changelog_entries_for_revision = lambda revision: [] + self.assertEqual(checkout.commit_info_for_revision(1), None) + + def test_bug_id_for_revision(self): + scm = Mock() + scm.committer_email_for_revision = lambda revision: "committer@example.com" + checkout = Checkout(scm) + checkout.changelog_entries_for_revision = lambda revision: [ChangeLogEntry(_changelog1entry1)] + self.assertEqual(checkout.bug_id_for_revision(4), 36629) + + def test_bug_id_for_this_commit(self): + scm = Mock() + checkout = Checkout(scm) + checkout.commit_message_for_this_commit = lambda: CommitMessage(ChangeLogEntry(_changelog1entry1).contents().splitlines()) + self.assertEqual(checkout.bug_id_for_this_commit(), 36629) + + def test_modified_changelogs(self): + scm = Mock() + scm.checkout_root = "/foo/bar" + scm.changed_files = lambda:["file1", "ChangeLog", "relative/path/ChangeLog"] + checkout = Checkout(scm) + expected_changlogs = ["/foo/bar/ChangeLog", "/foo/bar/relative/path/ChangeLog"] + self.assertEqual(checkout.modified_changelogs(), expected_changlogs) diff --git a/WebKitTools/Scripts/webkitpy/changelogs.py b/WebKitTools/Scripts/webkitpy/common/checkout/changelog.py index ebc89c4..e93896f 100644 --- a/WebKitTools/Scripts/webkitpy/changelogs.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/changelog.py @@ -28,10 +28,14 @@ # # WebKit's Python module for parsing and modifying ChangeLog files +import codecs import fileinput # inplace file editing for set_reviewer_in_changelog +import os.path import re import textwrap +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.config.committers import CommitterList def view_source_url(revision_number): # FIMXE: This doesn't really belong in this file, but we don't have a @@ -40,39 +44,82 @@ def view_source_url(revision_number): return "http://trac.webkit.org/changeset/%s" % revision_number -class ChangeLog: +class ChangeLogEntry(object): + # e.g. 2009-06-03 Eric Seidel <eric@webkit.org> + date_line_regexp = r'^(?P<date>\d{4}-\d{2}-\d{2})\s+(?P<name>.+?)\s+<(?P<email>[^<>]+)>$' + + def __init__(self, contents, committer_list=CommitterList()): + self._contents = contents + self._committer_list = committer_list + self._parse_entry() + + def _parse_entry(self): + match = re.match(self.date_line_regexp, self._contents, re.MULTILINE) + if not match: + log("WARNING: Creating invalid ChangeLogEntry:\n%s" % self._contents) + + # FIXME: group("name") does not seem to be Unicode? Probably due to self._contents not being unicode. + self._author_name = match.group("name") if match else None + self._author_email = match.group("email") if match else None + + match = re.search("^\s+Reviewed by (?P<reviewer>.*?)[\.,]?\s*$", self._contents, re.MULTILINE) # Discard everything after the first period + self._reviewer_text = match.group("reviewer") if match else None + + self._reviewer = self._committer_list.committer_by_name(self._reviewer_text) + self._author = self._committer_list.committer_by_email(self._author_email) or self._committer_list.committer_by_name(self._author_name) + + def author_name(self): + return self._author_name + + def author_email(self): + return self._author_email + + def author(self): + return self._author # Might be None + + # FIXME: Eventually we would like to map reviwer names to reviewer objects. + # See https://bugs.webkit.org/show_bug.cgi?id=26533 + def reviewer_text(self): + return self._reviewer_text + + def reviewer(self): + return self._reviewer # Might be None + + def contents(self): + return self._contents + + +# FIXME: Various methods on ChangeLog should move into ChangeLogEntry instead. +class ChangeLog(object): def __init__(self, path): self.path = path _changelog_indent = " " * 8 - # e.g. 2009-06-03 Eric Seidel <eric@webkit.org> - date_line_regexp = re.compile('^(\d{4}-\d{2}-\d{2})' # Consume the date. - + '\s+(.+)\s+' # Consume the name. - + '<([^<>]+)>$') # And the email address. - @staticmethod - def _parse_latest_entry_from_file(changelog_file): + def parse_latest_entry_from_file(changelog_file): + date_line_regexp = re.compile(ChangeLogEntry.date_line_regexp) entry_lines = [] # The first line should be a date line. first_line = changelog_file.readline() - if not ChangeLog.date_line_regexp.match(first_line): + if not date_line_regexp.match(first_line): return None entry_lines.append(first_line) for line in changelog_file: # If we've hit the next entry, return. - if ChangeLog.date_line_regexp.match(line): + if date_line_regexp.match(line): # Remove the extra newline at the end - return ''.join(entry_lines[:-1]) + return ChangeLogEntry(''.join(entry_lines[:-1])) entry_lines.append(line) return None # We never found a date line! def latest_entry(self): - changelog_file = open(self.path) + # ChangeLog files are always UTF-8, we read them in as such to support Reviewers with unicode in their names. + changelog_file = codecs.open(self.path, "r", "utf-8") try: - return self._parse_latest_entry_from_file(changelog_file) + return self.parse_latest_entry_from_file(changelog_file) finally: changelog_file.close() @@ -96,7 +143,7 @@ class ChangeLog: # This probably does not belong in changelogs.py def _message_for_revert(self, revision, reason, bug_url): - message = "No review, rolling out r%s.\n" % revision + message = "Unreviewed, rolling out r%s.\n" % revision message += "%s\n" % view_source_url(revision) if bug_url: message += "%s\n" % bug_url diff --git a/WebKitTools/Scripts/webkitpy/changelogs_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/changelog_unittest.py index de3e60c..9210c9c 100644 --- a/WebKitTools/Scripts/webkitpy/changelogs_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/changelog_unittest.py @@ -27,17 +27,19 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from changelogs import * - import os import tempfile + from StringIO import StringIO +from webkitpy.common.checkout.changelog import * + + class ChangeLogsTest(unittest.TestCase): - _example_entry = '''2009-08-17 Peter Kasting <pkasting@google.com> + _example_entry = u'''2009-08-17 Peter Kasting <pkasting@google.com> - Reviewed by Steve Falkenburg. + Reviewed by Tor Arne Vestb\xf8. https://bugs.webkit.org/show_bug.cgi?id=27323 Only add Cygwin to the path when it isn't already there. This avoids @@ -87,8 +89,12 @@ class ChangeLogsTest(unittest.TestCase): def test_latest_entry_parse(self): changelog_contents = "%s\n%s" % (self._example_entry, self._example_changelog) changelog_file = StringIO(changelog_contents) - latest_entry = ChangeLog._parse_latest_entry_from_file(changelog_file) - self.assertEquals(self._example_entry, latest_entry) + latest_entry = ChangeLog.parse_latest_entry_from_file(changelog_file) + self.assertEquals(latest_entry.contents(), self._example_entry) + self.assertEquals(latest_entry.author_name(), "Peter Kasting") + self.assertEquals(latest_entry.author_email(), "pkasting@google.com") + self.assertEquals(latest_entry.reviewer_text(), u"Tor Arne Vestb\xf8") + self.assertTrue(latest_entry.reviewer()) # Make sure that our UTF8-based lookup of Tor works. @staticmethod def _write_tmp_file_with_contents(contents): @@ -124,7 +130,7 @@ class ChangeLogsTest(unittest.TestCase): os.remove(changelog_path) self.assertEquals(actual_contents, expected_contents) - _revert_message = """ No review, rolling out r12345. + _revert_message = """ Unreviewed, rolling out r12345. http://trac.webkit.org/changeset/12345 http://example.com/123 @@ -143,7 +149,7 @@ class ChangeLogsTest(unittest.TestCase): _revert_entry_with_bug_url = '''2009-08-19 Eric Seidel <eric@webkit.org> - No review, rolling out r12345. + Unreviewed, rolling out r12345. http://trac.webkit.org/changeset/12345 http://example.com/123 @@ -154,7 +160,7 @@ class ChangeLogsTest(unittest.TestCase): _revert_entry_without_bug_url = '''2009-08-19 Eric Seidel <eric@webkit.org> - No review, rolling out r12345. + Unreviewed, rolling out r12345. http://trac.webkit.org/changeset/12345 Reason @@ -169,11 +175,16 @@ class ChangeLogsTest(unittest.TestCase): changelog.update_for_revert(*args) actual_entry = changelog.latest_entry() os.remove(changelog_path) - self.assertEquals(actual_entry, expected_entry) + self.assertEquals(actual_entry.contents(), expected_entry) + self.assertEquals(actual_entry.reviewer_text(), None) + # These checks could be removed to allow this to work on other entries: + self.assertEquals(actual_entry.author_name(), "Eric Seidel") + self.assertEquals(actual_entry.author_email(), "eric@webkit.org") def test_update_for_revert(self): self._assert_update_for_revert_output([12345, "Reason"], self._revert_entry_without_bug_url) self._assert_update_for_revert_output([12345, "Reason", "http://example.com/123"], self._revert_entry_with_bug_url) + if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo.py b/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo.py new file mode 100644 index 0000000..7c3315f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo.py @@ -0,0 +1,95 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# WebKit's python module for holding information on a commit + +import StringIO + +from webkitpy.common.checkout.changelog import view_source_url +from webkitpy.common.config.committers import CommitterList + + +class CommitInfo(object): + def __init__(self, revision, committer_email, changelog_data, committer_list=CommitterList()): + self._revision = revision + self._committer_email = committer_email + self._bug_id = changelog_data["bug_id"] + self._author_name = changelog_data["author_name"] + self._author_email = changelog_data["author_email"] + self._author = changelog_data["author"] + self._reviewer_text = changelog_data["reviewer_text"] + self._reviewer = changelog_data["reviewer"] + + # Derived values: + self._committer = committer_list.committer_by_email(committer_email) + + def revision(self): + return self._revision + + def committer(self): + return self._committer # None if committer isn't in committers.py + + def committer_email(self): + return self._committer_email + + def bug_id(self): + return self._bug_id # May be None + + def author(self): + return self._author # May be None + + def author_name(self): + return self._author_name + + def author_email(self): + return self._author_email + + def reviewer(self): + return self._reviewer # May be None + + def reviewer_text(self): + return self._reviewer_text # May be None + + def responsible_parties(self): + responsible_parties = [ + self.committer(), + self.author(), + self.reviewer(), + ] + return set([party for party in responsible_parties if party]) # Filter out None + + # FIXME: It is slightly lame that this "view" method is on this "model" class (in MVC terms) + def blame_string(self, bugs): + string = "r%s:\n" % self.revision() + string += " %s\n" % view_source_url(self.revision()) + string += " Bug: %s (%s)\n" % (self.bug_id(), bugs.bug_url_for_bug_id(self.bug_id())) + author_line = "\"%s\" <%s>" % (self.author_name(), self.author_email()) + string += " Author: %s\n" % (self.author() or author_line) + string += " Reviewer: %s\n" % (self.reviewer() or self.reviewer_text()) + string += " Committer: %s" % self.committer() + return string diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo_unittest.py new file mode 100644 index 0000000..f58e6f1 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo_unittest.py @@ -0,0 +1,61 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.checkout.commitinfo import CommitInfo +from webkitpy.common.config.committers import CommitterList, Committer, Reviewer + +class CommitInfoTest(unittest.TestCase): + + def test_commit_info_creation(self): + author = Committer("Author", "author@example.com") + committer = Committer("Committer", "committer@example.com") + reviewer = Reviewer("Reviewer", "reviewer@example.com") + committer_list = CommitterList(committers=[author, committer], reviewers=[reviewer]) + + changelog_data = { + "bug_id": 1234, + "author_name": "Committer", + "author_email": "author@example.com", + "author": author, + "reviewer_text": "Reviewer", + "reviewer": reviewer, + } + commit = CommitInfo(123, "committer@example.com", changelog_data, committer_list) + + self.assertEqual(commit.revision(), 123) + self.assertEqual(commit.bug_id(), 1234) + self.assertEqual(commit.author_name(), "Committer") + self.assertEqual(commit.author_email(), "author@example.com") + self.assertEqual(commit.author(), author) + self.assertEqual(commit.reviewer_text(), "Reviewer") + self.assertEqual(commit.reviewer(), reviewer) + self.assertEqual(commit.committer(), committer) + self.assertEqual(commit.committer_email(), "committer@example.com") + self.assertEqual(commit.responsible_parties(), set([author, committer, reviewer])) diff --git a/WebKitTools/Scripts/webkitpy/diff_parser.py b/WebKitTools/Scripts/webkitpy/common/checkout/diff_parser.py index 7dce7e8..d8ebae6 100644 --- a/WebKitTools/Scripts/webkitpy/diff_parser.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/diff_parser.py @@ -31,6 +31,7 @@ import logging import re +_log = logging.getLogger("webkitpy.common.checkout.diff_parser") _regexp_compile_cache = {} @@ -138,7 +139,8 @@ class DiffParser: lines_changed = match(r"^@@ -(?P<OldStartLine>\d+)(,\d+)? \+(?P<NewStartLine>\d+)(,\d+)? @@", line) if lines_changed: if state != _DECLARED_FILE_PATH and state != _PROCESSING_CHUNK: - logging.error('Unexpected line change without file path declaration: %r' % line) + _log.error('Unexpected line change without file path ' + 'declaration: %r' % line) old_diff_line = int(lines_changed.group('OldStartLine')) new_diff_line = int(lines_changed.group('NewStartLine')) state = _PROCESSING_CHUNK @@ -159,4 +161,5 @@ class DiffParser: # Nothing to do. We may still have some added lines. pass else: - logging.error('Unexpected diff format when parsing a chunk: %r' % line) + _log.error('Unexpected diff format when parsing a ' + 'chunk: %r' % line) diff --git a/WebKitTools/Scripts/webkitpy/diff_parser_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/diff_parser_unittest.py index 7eb0eab..7eb0eab 100644 --- a/WebKitTools/Scripts/webkitpy/diff_parser_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/diff_parser_unittest.py diff --git a/WebKitTools/Scripts/webkitpy/scm.py b/WebKitTools/Scripts/webkitpy/common/checkout/scm.py index 743f3fe..2704f07 100644 --- a/WebKitTools/Scripts/webkitpy/scm.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/scm.py @@ -31,12 +31,14 @@ import os import re -import subprocess -# Import WebKit-specific modules. -from webkitpy.changelogs import ChangeLog -from webkitpy.executive import Executive, run_command, ScriptError -from webkitpy.webkit_logging import error, log +# FIXME: Instead of using run_command directly, most places in this +# class would rather use an SCM.run method which automatically set +# cwd=self.checkout_root. +from webkitpy.common.system.executive import Executive, run_command, ScriptError +from webkitpy.common.system.user import User +from webkitpy.common.system.deprecated_logging import error, log + def detect_scm_system(path): if SVN.in_working_directory(path): @@ -47,6 +49,7 @@ def detect_scm_system(path): return None + def first_non_empty_line_after_index(lines, index=0): first_non_empty_line = index for line in lines[index:]: @@ -90,20 +93,29 @@ def commit_error_handler(error): Executive.default_error_handler(error) +# SCM methods are expected to return paths relative to self.checkout_root. class SCM: - def __init__(self, cwd, dryrun=False): + def __init__(self, cwd): self.cwd = cwd self.checkout_root = self.find_checkout_root(self.cwd) - self.dryrun = dryrun + self.dryrun = False + # SCM always returns repository relative path, but sometimes we need + # absolute paths to pass to rm, etc. + def absolute_path(self, repository_relative_path): + return os.path.join(self.checkout_root, repository_relative_path) + + # FIXME: This belongs in Checkout, not SCM. def scripts_directory(self): return os.path.join(self.checkout_root, "WebKitTools", "Scripts") + # FIXME: This belongs in Checkout, not SCM. def script_path(self, script_name): return os.path.join(self.scripts_directory(), script_name) def ensure_clean_working_directory(self, force_clean): if not force_clean and not self.working_directory_is_clean(): + # FIXME: Shouldn't this use cwd=self.checkout_root? print run_command(self.status_command(), error_handler=Executive.ignore_error) raise ScriptError(message="Working directory has modifications, pass --force-clean or --no-clean to continue.") @@ -120,22 +132,10 @@ class SCM: error("Working directory has local commits, pass --force-clean to continue.") self.discard_local_commits() - def apply_patch(self, patch, force=False): - # It's possible that the patch was not made from the root directory. - # We should detect and handle that case. - # FIXME: scm.py should not deal with fetching Attachment data. Attachment should just have a .data() accessor. - curl_process = subprocess.Popen(['curl', '--location', '--silent', '--show-error', patch.url()], stdout=subprocess.PIPE) - args = [self.script_path('svn-apply')] - if patch.reviewer(): - args += ['--reviewer', patch.reviewer().full_name] - if force: - args.append('--force') - - run_command(args, input=curl_process.stdout) - def run_status_and_extract_filenames(self, status_command, status_regexp): filenames = [] - for line in run_command(status_command).splitlines(): + # We run with cwd=self.checkout_root so that returned-paths are root-relative. + for line in run_command(status_command, cwd=self.checkout_root).splitlines(): match = re.search(status_regexp, line) if not match: continue @@ -154,37 +154,6 @@ class SCM: match = re.search(self.commit_success_regexp(), commit_text, re.MULTILINE) return match.group('svn_revision') - # ChangeLog-specific code doesn't really belong in scm.py, but this function is very useful. - def modified_changelogs(self): - changelog_paths = [] - paths = self.changed_files() - for path in paths: - if os.path.basename(path) == "ChangeLog": - changelog_paths.append(path) - return changelog_paths - - # FIXME: Requires unit test - # FIXME: commit_message_for_this_commit and modified_changelogs don't - # really belong here. We should have a separate module for - # handling ChangeLogs. - def commit_message_for_this_commit(self): - changelog_paths = self.modified_changelogs() - if not len(changelog_paths): - raise ScriptError(message="Found no modified ChangeLogs, cannot create a commit message.\n" - "All changes require a ChangeLog. See:\n" - "http://webkit.org/coding/contributing.html") - - changelog_messages = [] - for changelog_path in changelog_paths: - log("Parsing ChangeLog: %s" % changelog_path) - changelog_entry = ChangeLog(changelog_path).latest_entry() - if not changelog_entry: - raise ScriptError(message="Failed to parse ChangeLog: " + os.path.abspath(changelog_path)) - changelog_messages.append(changelog_entry) - - # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does. - return CommitMessage("".join(changelog_messages).splitlines()) - @staticmethod def in_working_directory(path): raise NotImplementedError, "subclasses must implement" @@ -206,15 +175,33 @@ class SCM: def status_command(self): raise NotImplementedError, "subclasses must implement" + def add(self, path): + raise NotImplementedError, "subclasses must implement" + def changed_files(self): raise NotImplementedError, "subclasses must implement" + def changed_files_for_revision(self): + raise NotImplementedError, "subclasses must implement" + + def added_files(self): + raise NotImplementedError, "subclasses must implement" + + def conflicted_files(self): + raise NotImplementedError, "subclasses must implement" + def display_name(self): raise NotImplementedError, "subclasses must implement" def create_patch(self): raise NotImplementedError, "subclasses must implement" + def committer_email_for_revision(self, revision): + raise NotImplementedError, "subclasses must implement" + + def contents_at_revision(self, path, revision): + raise NotImplementedError, "subclasses must implement" + def diff_for_revision(self, revision): raise NotImplementedError, "subclasses must implement" @@ -224,7 +211,7 @@ class SCM: def revert_files(self, file_paths): raise NotImplementedError, "subclasses must implement" - def commit_with_message(self, message): + def commit_with_message(self, message, username=None): raise NotImplementedError, "subclasses must implement" def svn_commit_log(self, svn_revision): @@ -239,6 +226,9 @@ class SCM: def supports_local_commits(): raise NotImplementedError, "subclasses must implement" + def svn_merge_base(): + raise NotImplementedError, "subclasses must implement" + def create_patch_from_local_commit(self, commit_id): error("Your source control manager does not support creating a patch from a local commit.") @@ -256,8 +246,12 @@ class SCM: class SVN(SCM): - def __init__(self, cwd, dryrun=False): - SCM.__init__(self, cwd, dryrun) + # FIXME: We should move these values to a WebKit-specific config. file. + svn_server_host = "svn.webkit.org" + svn_server_realm = "<http://svn.webkit.org:80> Mac OS Forge" + + def __init__(self, cwd): + SCM.__init__(self, cwd) self.cached_version = None @staticmethod @@ -299,6 +293,14 @@ class SVN(SCM): def commit_success_regexp(): return "^Committed revision (?P<svn_revision>\d+)\.$" + def has_authorization_for_realm(self, realm=svn_server_realm, home_directory=os.getenv("HOME")): + # Assumes find and grep are installed. + if not os.path.isdir(os.path.join(home_directory, ".subversion")): + return False + find_args = ["find", ".subversion", "-type", "f", "-exec", "grep", "-q", realm, "{}", ";", "-print"]; + find_output = run_command(find_args, cwd=home_directory, error_handler=Executive.ignore_error).rstrip() + return find_output and os.path.isfile(os.path.join(home_directory, find_output)) + def svn_version(self): if not self.cached_version: self.cached_version = run_command(['svn', '--version', '--quiet']) @@ -306,20 +308,50 @@ class SVN(SCM): return self.cached_version def working_directory_is_clean(self): - return run_command(['svn', 'diff']) == "" + return run_command(["svn", "diff"], cwd=self.checkout_root) == "" def clean_working_directory(self): - run_command(['svn', 'revert', '-R', '.']) + # svn revert -R is not as awesome as git reset --hard. + # It will leave added files around, causing later svn update + # calls to fail on the bots. We make this mirror git reset --hard + # by deleting any added files as well. + added_files = reversed(sorted(self.added_files())) + # added_files() returns directories for SVN, we walk the files in reverse path + # length order so that we remove files before we try to remove the directories. + run_command(["svn", "revert", "-R", "."], cwd=self.checkout_root) + for path in added_files: + # This is robust against cwd != self.checkout_root + absolute_path = self.absolute_path(path) + # Completely lame that there is no easy way to remove both types with one call. + if os.path.isdir(path): + os.rmdir(absolute_path) + else: + os.remove(absolute_path) def status_command(self): return ['svn', 'status'] + def _status_regexp(self, expected_types): + field_count = 6 if self.svn_version() > "1.6" else 5 + return "^(?P<status>[%s]).{%s} (?P<filename>.+)$" % (expected_types, field_count) + + def add(self, path): + # path is assumed to be cwd relative? + run_command(["svn", "add", path]) + def changed_files(self): - if self.svn_version() > "1.6": - status_regexp = "^(?P<status>[ACDMR]).{6} (?P<filename>.+)$" - else: - status_regexp = "^(?P<status>[ACDMR]).{5} (?P<filename>.+)$" - return self.run_status_and_extract_filenames(self.status_command(), status_regexp) + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("ACDMR")) + + def changed_files_for_revision(self, revision): + # As far as I can tell svn diff --summarize output looks just like svn status output. + status_command = ["svn", "diff", "--summarize", "-c", str(revision)] + return self.run_status_and_extract_filenames(status_command, self._status_regexp("ACDMR")) + + def conflicted_files(self): + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("C")) + + def added_files(self): + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("A")) @staticmethod def supports_local_commits(): @@ -331,7 +363,15 @@ class SVN(SCM): def create_patch(self): return run_command(self.script_path("svn-create-patch"), cwd=self.checkout_root, return_stderr=False) + def committer_email_for_revision(self, revision): + return run_command(["svn", "propget", "svn:author", "--revprop", "-r", str(revision)]).rstrip() + + def contents_at_revision(self, path, revision): + remote_path = "%s/%s" % (self._repository_url(), path) + return run_command(["svn", "cat", "-r", str(revision), remote_path]) + def diff_for_revision(self, revision): + # FIXME: This should probably use cwd=self.checkout_root return run_command(['svn', 'diff', '-c', str(revision)]) def _repository_url(self): @@ -342,16 +382,27 @@ class SVN(SCM): svn_merge_args = ['svn', 'merge', '--non-interactive', '-c', '-%s' % revision, self._repository_url()] log("WARNING: svn merge has been known to take more than 10 minutes to complete. It is recommended you use git for rollouts.") log("Running '%s'" % " ".join(svn_merge_args)) + # FIXME: Should this use cwd=self.checkout_root? run_command(svn_merge_args) def revert_files(self, file_paths): + # FIXME: This should probably use cwd=self.checkout_root. run_command(['svn', 'revert'] + file_paths) - def commit_with_message(self, message): + def commit_with_message(self, message, username=None): if self.dryrun: # Return a string which looks like a commit so that things which parse this output will succeed. return "Dry run, no commit.\nCommitted revision 0." - return run_command(['svn', 'commit', '-m', message], error_handler=commit_error_handler) + svn_commit_args = ["svn", "commit"] + if not username and not self.has_authorization_for_realm(): + username = User.prompt("%s login: " % self.svn_server_host, repeat=5) + if not username: + raise Exception("You need to specify the username on %s to perform the commit as." % self.svn_server_host) + if username: + svn_commit_args.extend(["--username", username]) + svn_commit_args.extend(["-m", message]) + # FIXME: Should this use cwd=self.checkout_root? + return run_command(svn_commit_args, error_handler=commit_error_handler) def svn_commit_log(self, svn_revision): svn_revision = self.strip_r_from_svn_revision(str(svn_revision)) @@ -364,8 +415,8 @@ class SVN(SCM): # All git-specific logic should go here. class Git(SCM): - def __init__(self, cwd, dryrun=False): - SCM.__init__(self, cwd, dryrun) + def __init__(self, cwd): + SCM.__init__(self, cwd) @classmethod def in_working_directory(cls, path): @@ -379,25 +430,34 @@ class Git(SCM): if not os.path.isabs(checkout_root): # Sometimes git returns relative paths checkout_root = os.path.join(path, checkout_root) return checkout_root - + + @classmethod + def read_git_config(cls, key): + # FIXME: This should probably use cwd=self.checkout_root. + return run_command(["git", "config", key], + error_handler=Executive.ignore_error).rstrip('\n') + @staticmethod def commit_success_regexp(): return "^Committed r(?P<svn_revision>\d+)$" - def discard_local_commits(self): - run_command(['git', 'reset', '--hard', 'trunk']) + # FIXME: This should probably use cwd=self.checkout_root + run_command(['git', 'reset', '--hard', self.svn_branch_name()]) def local_commits(self): - return run_command(['git', 'log', '--pretty=oneline', 'HEAD...trunk']).splitlines() + # FIXME: This should probably use cwd=self.checkout_root + return run_command(['git', 'log', '--pretty=oneline', 'HEAD...' + self.svn_branch_name()]).splitlines() def rebase_in_progress(self): return os.path.exists(os.path.join(self.checkout_root, '.git/rebase-apply')) def working_directory_is_clean(self): - return run_command(['git', 'diff-index', 'HEAD']) == "" + # FIXME: This should probably use cwd=self.checkout_root + return run_command(['git', 'diff', 'HEAD', '--name-only']) == "" def clean_working_directory(self): + # FIXME: These should probably use cwd=self.checkout_root. # Could run git clean here too, but that wouldn't match working_directory_is_clean run_command(['git', 'reset', '--hard', 'HEAD']) # Aborting rebase even though this does not match working_directory_is_clean @@ -405,13 +465,37 @@ class Git(SCM): run_command(['git', 'rebase', '--abort']) def status_command(self): - return ['git', 'status'] + # git status returns non-zero when there are changes, so we use git diff name --name-status HEAD instead. + return ["git", "diff", "--name-status", "HEAD"] + + def _status_regexp(self, expected_types): + return '^(?P<status>[%s])\t(?P<filename>.+)$' % expected_types + + def add(self, path): + # path is assumed to be cwd relative? + run_command(["git", "add", path]) def changed_files(self): status_command = ['git', 'diff', '-r', '--name-status', '-C', '-M', 'HEAD'] - status_regexp = '^(?P<status>[ADM])\t(?P<filename>.+)$' - return self.run_status_and_extract_filenames(status_command, status_regexp) - + return self.run_status_and_extract_filenames(status_command, self._status_regexp("ADM")) + + def _changes_files_for_commit(self, git_commit): + # --pretty="format:" makes git show not print the commit log header, + changed_files = run_command(["git", "show", "--pretty=format:", "--name-only", git_commit]).splitlines() + # instead it just prints a blank line at the top, so we skip the blank line: + return changed_files[1:] + + def changed_files_for_revision(self, revision): + commit_id = self.git_commit_from_svn_revision(revision) + return self._changes_files_for_commit(commit_id) + + def conflicted_files(self): + status_command = ['git', 'diff', '--name-status', '-C', '-M', '--diff-filter=U'] + return self.run_status_and_extract_filenames(status_command, self._status_regexp("U")) + + def added_files(self): + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("A")) + @staticmethod def supports_local_commits(): return True @@ -420,36 +504,42 @@ class Git(SCM): return "git" def create_patch(self): + # FIXME: This should probably use cwd=self.checkout_root return run_command(['git', 'diff', '--binary', 'HEAD']) @classmethod def git_commit_from_svn_revision(cls, revision): + # FIXME: This should probably use cwd=self.checkout_root + git_commit = run_command(['git', 'svn', 'find-rev', 'r%s' % revision]).rstrip() # git svn find-rev always exits 0, even when the revision is not found. - return run_command(['git', 'svn', 'find-rev', 'r%s' % revision]).rstrip() + if not git_commit: + raise ScriptError(message='Failed to find git commit for revision %s, your checkout likely needs an update.' % revision) + return git_commit + + def contents_at_revision(self, path, revision): + return run_command(["git", "show", "%s:%s" % (self.git_commit_from_svn_revision(revision), path)]) def diff_for_revision(self, revision): git_commit = self.git_commit_from_svn_revision(revision) return self.create_patch_from_local_commit(git_commit) + def committer_email_for_revision(self, revision): + git_commit = self.git_commit_from_svn_revision(revision) + committer_email = run_command(["git", "log", "-1", "--pretty=format:%ce", git_commit]) + # Git adds an extra @repository_hash to the end of every committer email, remove it: + return committer_email.rsplit("@", 1)[0] + def apply_reverse_diff(self, revision): # Assume the revision is an svn revision. git_commit = self.git_commit_from_svn_revision(revision) - if not git_commit: - raise ScriptError(message='Failed to find git commit for revision %s, git svn log output: "%s"' % (revision, git_commit)) - # I think this will always fail due to ChangeLogs. - # FIXME: We need to detec specific failure conditions and handle them. run_command(['git', 'revert', '--no-commit', git_commit], error_handler=Executive.ignore_error) - # Fix any ChangeLogs if necessary. - changelog_paths = self.modified_changelogs() - if len(changelog_paths): - run_command([self.script_path('resolve-ChangeLogs')] + changelog_paths) - def revert_files(self, file_paths): run_command(['git', 'checkout', 'HEAD'] + file_paths) - def commit_with_message(self, message): + def commit_with_message(self, message, username=None): + # Username is ignored during Git commits. self.commit_locally_with_message(message) return self.push_local_commits_to_server() @@ -462,6 +552,16 @@ class Git(SCM): # Git-specific methods: + def delete_branch(self, branch): + if run_command(['git', 'show-ref', '--quiet', '--verify', 'refs/heads/' + branch], return_exit_code=True) == 0: + run_command(['git', 'branch', '-D', branch]) + + def svn_merge_base(self): + return run_command(['git', 'merge-base', self.svn_branch_name(), 'HEAD']).strip() + + def svn_branch_name(self): + return Git.read_git_config('svn-remote.svn.fetch').split(':')[1] + def create_patch_from_local_commit(self, commit_id): return run_command(['git', 'diff', '--binary', commit_id + "^.." + commit_id]) @@ -470,12 +570,16 @@ class Git(SCM): def commit_locally_with_message(self, message): run_command(['git', 'commit', '--all', '-F', '-'], input=message) - + def push_local_commits_to_server(self): + dcommit_command = ['git', 'svn', 'dcommit'] if self.dryrun: - # Return a string which looks like a commit so that things which parse this output will succeed. - return "Dry run, no remote commit.\nCommitted r0" - return run_command(['git', 'svn', 'dcommit'], error_handler=commit_error_handler) + dcommit_command.append('--dry-run') + output = run_command(dcommit_command, error_handler=commit_error_handler) + # Return a string which looks like a commit so that things which parse this output will succeed. + if self.dryrun: + output += "\nCommitted r0" + return output # This function supports the following argument formats: # no args : rev-list trunk..HEAD @@ -484,8 +588,7 @@ class Git(SCM): # A B : [A, B] (different from git diff, which would use "rev-list A..B") def commit_ids_from_commitish_arguments(self, args): if not len(args): - # FIXME: trunk is not always the remote branch name, need a way to detect the name. - args.append('trunk..HEAD') + args.append('%s..HEAD' % self.svn_branch_name()) commit_ids = [] for commitish in args: diff --git a/WebKitTools/Scripts/webkitpy/scm_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/scm_unittest.py index 73faf40..c0a64d4 100644 --- a/WebKitTools/Scripts/webkitpy/scm_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/scm_unittest.py @@ -28,6 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import base64 +import getpass import os import os.path import re @@ -38,9 +39,11 @@ import unittest import urllib from datetime import date -from webkitpy.executive import Executive, run_command, ScriptError -from webkitpy.scm import detect_scm_system, SCM, CheckoutNeedsUpdate, commit_error_handler -from webkitpy.bugzilla import Attachment # FIXME: This should not be needed +from webkitpy.common.checkout.api import Checkout +from webkitpy.common.checkout.scm import detect_scm_system, SCM, SVN, CheckoutNeedsUpdate, commit_error_handler +from webkitpy.common.config.committers import Committer # FIXME: This should not be needed +from webkitpy.common.net.bugzilla import Attachment # FIXME: This should not be needed +from webkitpy.common.system.executive import Executive, run_command, ScriptError # Eventually we will want to write tests which work for both scms. (like update_webkit, changed_files, etc.) # Perhaps through some SCMTest base-class which both SVNTest and GitTest inherit from. @@ -66,31 +69,42 @@ def read_from_path(file_path): # Exists to share svn repository creation code between the git and svn tests class SVNTestRepository: - @staticmethod - def _setup_test_commits(test_object): + @classmethod + def _svn_add(cls, path): + run_command(["svn", "add", path]) + + @classmethod + def _svn_commit(cls, message): + run_command(["svn", "commit", "--quiet", "--message", message]) + + @classmethod + def _setup_test_commits(cls, test_object): # Add some test commits os.chdir(test_object.svn_checkout_path) - test_file = open('test_file', 'w') - test_file.write("test1") - test_file.flush() - - run_command(['svn', 'add', 'test_file']) - run_command(['svn', 'commit', '--quiet', '--message', 'initial commit']) - - test_file.write("test2") - test_file.flush() - - run_command(['svn', 'commit', '--quiet', '--message', 'second commit']) - - test_file.write("test3\n") - test_file.flush() - - run_command(['svn', 'commit', '--quiet', '--message', 'third commit']) - - test_file.write("test4\n") - test_file.close() - - run_command(['svn', 'commit', '--quiet', '--message', 'fourth commit']) + + write_into_file_at_path("test_file", "test1") + cls._svn_add("test_file") + cls._svn_commit("initial commit") + + write_into_file_at_path("test_file", "test1test2") + # This used to be the last commit, but doing so broke + # GitTest.test_apply_git_patch which use the inverse diff of the last commit. + # svn-apply fails to remove directories in Git, see: + # https://bugs.webkit.org/show_bug.cgi?id=34871 + os.mkdir("test_dir") + # Slash should always be the right path separator since we use cygwin on Windows. + test_file3_path = "test_dir/test_file3" + write_into_file_at_path(test_file3_path, "third file") + cls._svn_add("test_dir") + cls._svn_commit("second commit") + + write_into_file_at_path("test_file", "test1test2test3\n") + write_into_file_at_path("test_file2", "second file") + cls._svn_add("test_file2") + cls._svn_commit("third commit") + + write_into_file_at_path("test_file", "test1test2test3\ntest4\n") + cls._svn_commit("fourth commit") # svn does not seem to update after commit as I would expect. run_command(['svn', 'update']) @@ -115,6 +129,10 @@ class SVNTestRepository: run_command(['rm', '-rf', test_object.svn_repo_path]) run_command(['rm', '-rf', test_object.svn_checkout_path]) + # Now that we've deleted the checkout paths, cwddir may be invalid + # Change back to a valid directory so that later calls to os.getcwd() do not fail. + os.chdir(detect_scm_system(os.path.dirname(__file__)).checkout_root) + # For testing the SCM baseclass directly. class SCMClassTests(unittest.TestCase): def setUp(self): @@ -166,10 +184,14 @@ class SCMTest(unittest.TestCase): patch_path = os.path.join(self.svn_checkout_path, 'patch.diff') write_into_file_at_path(patch_path, patch_contents) patch = {} - patch['reviewer'] = 'Joe Cool' patch['bug_id'] = '12345' patch['url'] = 'file://%s' % urllib.pathname2url(patch_path) - return Attachment(patch, None) # FIXME: This is a hack, scm.py shouldn't be fetching attachment data. + + attachment = Attachment(patch, None) # FIXME: This is a hack, scm.py shouldn't be fetching attachment data. + joe_cool = Committer(name="Joe Cool", email_or_emails=None) + attachment._reviewer = joe_cool + + return attachment def _setup_webkittools_scripts_symlink(self, local_scm): webkit_scm = detect_scm_system(os.path.dirname(os.path.abspath(__file__))) @@ -180,16 +202,74 @@ class SCMTest(unittest.TestCase): # Tests which both GitTest and SVNTest should run. # FIXME: There must be a simpler way to add these w/o adding a wrapper method to both subclasses - def _shared_test_commit_with_message(self): + def _shared_test_commit_with_message(self, username="dbates@webkit.org"): write_into_file_at_path('test_file', 'more test content') - commit_text = self.scm.commit_with_message('another test commit') + commit_text = self.scm.commit_with_message("another test commit", username) self.assertEqual(self.scm.svn_revision_from_commit_text(commit_text), '5') self.scm.dryrun = True write_into_file_at_path('test_file', 'still more test content') - commit_text = self.scm.commit_with_message('yet another test commit') + commit_text = self.scm.commit_with_message("yet another test commit", username) self.assertEqual(self.scm.svn_revision_from_commit_text(commit_text), '0') + def _shared_test_changed_files(self): + write_into_file_at_path("test_file", "changed content") + self.assertEqual(self.scm.changed_files(), ["test_file"]) + write_into_file_at_path("test_dir/test_file3", "new stuff") + self.assertEqual(self.scm.changed_files(), ["test_dir/test_file3", "test_file"]) + old_cwd = os.getcwd() + os.chdir("test_dir") + # Validate that changed_files does not change with our cwd, see bug 37015. + self.assertEqual(self.scm.changed_files(), ["test_dir/test_file3", "test_file"]) + os.chdir(old_cwd) + + def _shared_test_added_files(self): + write_into_file_at_path("test_file", "changed content") + self.assertEqual(self.scm.added_files(), []) + + write_into_file_at_path("added_file", "new stuff") + self.scm.add("added_file") + + os.mkdir("added_dir") + write_into_file_at_path("added_dir/added_file2", "new stuff") + self.scm.add("added_dir") + + # SVN reports directory changes, Git does not. + added_files = self.scm.added_files() + if "added_dir" in added_files: + added_files.remove("added_dir") + self.assertEqual(added_files, ["added_dir/added_file2", "added_file"]) + + # Test also to make sure clean_working_directory removes added files + self.scm.clean_working_directory() + self.assertEqual(self.scm.added_files(), []) + self.assertFalse(os.path.exists("added_file")) + self.assertFalse(os.path.exists("added_dir")) + + def _shared_test_changed_files_for_revision(self): + # SVN reports directory changes, Git does not. + changed_files = self.scm.changed_files_for_revision(2) + if "test_dir" in changed_files: + changed_files.remove("test_dir") + self.assertEqual(changed_files, ["test_dir/test_file3", "test_file"]) + self.assertEqual(sorted(self.scm.changed_files_for_revision(3)), sorted(["test_file", "test_file2"])) # Git and SVN return different orders. + self.assertEqual(self.scm.changed_files_for_revision(4), ["test_file"]) + + def _shared_test_contents_at_revision(self): + self.assertEqual(self.scm.contents_at_revision("test_file", 2), "test1test2") + self.assertEqual(self.scm.contents_at_revision("test_file", 3), "test1test2test3\n") + self.assertEqual(self.scm.contents_at_revision("test_file", 4), "test1test2test3\ntest4\n") + + self.assertEqual(self.scm.contents_at_revision("test_file2", 3), "second file") + # Files which don't exist: + # Currently we raise instead of returning None because detecting the difference between + # "file not found" and any other error seems impossible with svn (git seems to expose such through the return code). + self.assertRaises(ScriptError, self.scm.contents_at_revision, "test_file2", 2) + self.assertRaises(ScriptError, self.scm.contents_at_revision, "does_not_exist", 2) + + def _shared_test_committer_email_for_revision(self): + self.assertEqual(self.scm.committer_email_for_revision(2), getpass.getuser()) # Committer "email" will be the current user + def _shared_test_reverse_diff(self): self._setup_webkittools_scripts_symlink(self.scm) # Git's apply_reverse_diff uses resolve-ChangeLogs # Only test the simple case, as any other will end up with conflict markers. @@ -227,14 +307,14 @@ literal 0 HcmV?d00001 """ - self.scm.apply_patch(self._create_patch(git_binary_addition)) + self.checkout.apply_patch(self._create_patch(git_binary_addition)) added = read_from_path('fizzbuzz7.gif') self.assertEqual(512, len(added)) self.assertTrue(added.startswith('GIF89a')) self.assertTrue('fizzbuzz7.gif' in self.scm.changed_files()) # The file already exists. - self.assertRaises(ScriptError, self.scm.apply_patch, self._create_patch(git_binary_addition)) + self.assertRaises(ScriptError, self.checkout.apply_patch, self._create_patch(git_binary_addition)) git_binary_modification = """diff --git a/fizzbuzz7.gif b/fizzbuzz7.gif index 64a9532e7794fcd791f6f12157406d9060151690..323fae03f4606ea9991df8befbb2fca7 @@ -255,13 +335,13 @@ z{89I|WPyD!*M?gv?q`;L=2YFeXrJQNti4?}s!zFo=5CzeBxC69xA<zrjP<wUcCRh4 ptUl-ZG<%a~#LwkIWv&q!KSCH7tQ8cJDiw+|GV?MN)RjY50RTb-xvT&H """ - self.scm.apply_patch(self._create_patch(git_binary_modification)) + self.checkout.apply_patch(self._create_patch(git_binary_modification)) modified = read_from_path('fizzbuzz7.gif') self.assertEqual('foobar\n', modified) self.assertTrue('fizzbuzz7.gif' in self.scm.changed_files()) # Applying the same modification should fail. - self.assertRaises(ScriptError, self.scm.apply_patch, self._create_patch(git_binary_modification)) + self.assertRaises(ScriptError, self.checkout.apply_patch, self._create_patch(git_binary_modification)) git_binary_deletion = """diff --git a/fizzbuzz7.gif b/fizzbuzz7.gif deleted file mode 100644 @@ -274,12 +354,12 @@ literal 7 OcmYex&reD$;sO8*F9L)B """ - self.scm.apply_patch(self._create_patch(git_binary_deletion)) + self.checkout.apply_patch(self._create_patch(git_binary_deletion)) self.assertFalse(os.path.exists('fizzbuzz7.gif')) self.assertFalse('fizzbuzz7.gif' in self.scm.changed_files()) # Cannot delete again. - self.assertRaises(ScriptError, self.scm.apply_patch, self._create_patch(git_binary_deletion)) + self.assertRaises(ScriptError, self.checkout.apply_patch, self._create_patch(git_binary_deletion)) class SVNTest(SCMTest): @@ -314,24 +394,24 @@ class SVNTest(SCMTest): +++ ChangeLog (working copy) @@ -1,5 +1,13 @@ 2009-10-26 Eric Seidel <eric@webkit.org> - + + Reviewed by NOBODY (OOPS!). + -+ Second most awsome change ever. ++ Second most awesome change ever. + + * scm_unittest.py: + +2009-10-26 Eric Seidel <eric@webkit.org> + Reviewed by Foo Bar. - + Most awesome change ever. """ one_line_overlap_entry = """DATE_HERE Eric Seidel <eric@webkit.org> Reviewed by REVIEWER_HERE. - Second most awsome change ever. + Second most awesome change ever. * scm_unittest.py: """ @@ -340,10 +420,10 @@ class SVNTest(SCMTest): --- ChangeLog (revision 5) +++ ChangeLog (working copy) @@ -2,6 +2,14 @@ - + Reviewed by Foo Bar. - -+ Second most awsome change ever. + ++ Second most awesome change ever. + + * scm_unittest.py: + @@ -352,14 +432,14 @@ class SVNTest(SCMTest): + Reviewed by Foo Bar. + Most awesome change ever. - + * scm_unittest.py: """ two_line_overlap_entry = """DATE_HERE Eric Seidel <eric@webkit.org> Reviewed by Foo Bar. - Second most awsome change ever. + Second most awesome change ever. * scm_unittest.py: """ @@ -374,12 +454,12 @@ class SVNTest(SCMTest): run_command(['svn', 'commit', '--quiet', '--message', 'Intermediate commit']) self._setup_webkittools_scripts_symlink(self.scm) - self.scm.apply_patch(self._create_patch(one_line_overlap_patch)) + self.checkout.apply_patch(self._create_patch(one_line_overlap_patch)) expected_changelog_contents = "%s\n%s" % (self._set_date_and_reviewer(one_line_overlap_entry), changelog_contents) self.assertEquals(read_from_path('ChangeLog'), expected_changelog_contents) self.scm.revert_files(['ChangeLog']) - self.scm.apply_patch(self._create_patch(two_line_overlap_patch)) + self.checkout.apply_patch(self._create_patch(two_line_overlap_patch)) expected_changelog_contents = "%s\n%s" % (self._set_date_and_reviewer(two_line_overlap_entry), changelog_contents) self.assertEquals(read_from_path('ChangeLog'), expected_changelog_contents) @@ -387,16 +467,18 @@ class SVNTest(SCMTest): SVNTestRepository.setup(self) os.chdir(self.svn_checkout_path) self.scm = detect_scm_system(self.svn_checkout_path) + # For historical reasons, we test some checkout code here too. + self.checkout = Checkout(self.scm) def tearDown(self): SVNTestRepository.tear_down(self) def test_create_patch_is_full_patch(self): - test_dir_path = os.path.join(self.svn_checkout_path, 'test_dir') + test_dir_path = os.path.join(self.svn_checkout_path, "test_dir2") os.mkdir(test_dir_path) test_file_path = os.path.join(test_dir_path, 'test_file2') write_into_file_at_path(test_file_path, 'test content') - run_command(['svn', 'add', 'test_dir']) + run_command(['svn', 'add', 'test_dir2']) # create_patch depends on 'svn-create-patch', so make a dummy version. scripts_path = os.path.join(self.svn_checkout_path, 'WebKitTools', 'Scripts') @@ -435,7 +517,7 @@ Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== expected_contents = base64.b64decode("Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA==") self._setup_webkittools_scripts_symlink(self.scm) patch_file = self._create_patch(patch_contents) - self.scm.apply_patch(patch_file) + self.checkout.apply_patch(patch_file) actual_contents = read_from_path("test_file.swf") self.assertEqual(actual_contents, expected_contents) @@ -443,13 +525,13 @@ Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== scm = detect_scm_system(self.svn_checkout_path) patch = self._create_patch(run_command(['svn', 'diff', '-r4:3'])) self._setup_webkittools_scripts_symlink(scm) - scm.apply_patch(patch) + Checkout(scm).apply_patch(patch) def test_apply_svn_patch_force(self): scm = detect_scm_system(self.svn_checkout_path) patch = self._create_patch(run_command(['svn', 'diff', '-r2:4'])) self._setup_webkittools_scripts_symlink(scm) - self.assertRaises(ScriptError, scm.apply_patch, patch, force=True) + self.assertRaises(ScriptError, Checkout(scm).apply_patch, patch, force=True) def test_commit_logs(self): # Commits have dates and usernames in them, so we can't just direct compare. @@ -459,6 +541,30 @@ Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== def test_commit_text_parsing(self): self._shared_test_commit_with_message() + def test_commit_with_username(self): + self._shared_test_commit_with_message("dbates@webkit.org") + + def test_has_authorization_for_realm(self): + scm = detect_scm_system(self.svn_checkout_path) + fake_home_dir = tempfile.mkdtemp(suffix="fake_home_dir") + svn_config_dir_path = os.path.join(fake_home_dir, ".subversion") + os.mkdir(svn_config_dir_path) + fake_webkit_auth_file = os.path.join(svn_config_dir_path, "fake_webkit_auth_file") + write_into_file_at_path(fake_webkit_auth_file, SVN.svn_server_realm) + self.assertTrue(scm.has_authorization_for_realm(home_directory=fake_home_dir)) + os.remove(fake_webkit_auth_file) + os.rmdir(svn_config_dir_path) + os.rmdir(fake_home_dir) + + def test_not_have_authorization_for_realm(self): + scm = detect_scm_system(self.svn_checkout_path) + fake_home_dir = tempfile.mkdtemp(suffix="fake_home_dir") + svn_config_dir_path = os.path.join(fake_home_dir, ".subversion") + os.mkdir(svn_config_dir_path) + self.assertFalse(scm.has_authorization_for_realm(home_directory=fake_home_dir)) + os.rmdir(svn_config_dir_path) + os.rmdir(fake_home_dir) + def test_reverse_diff(self): self._shared_test_reverse_diff() @@ -468,6 +574,22 @@ Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== def test_svn_apply_git_patch(self): self._shared_test_svn_apply_git_patch() + def test_changed_files(self): + self._shared_test_changed_files() + + def test_changed_files_for_revision(self): + self._shared_test_changed_files_for_revision() + + def test_added_files(self): + self._shared_test_added_files() + + def test_contents_at_revision(self): + self._shared_test_contents_at_revision() + + def test_committer_email_for_revision(self): + self._shared_test_committer_email_for_revision() + + class GitTest(SCMTest): def _setup_git_clone_of_svn_repository(self): @@ -483,6 +605,8 @@ class GitTest(SCMTest): self._setup_git_clone_of_svn_repository() os.chdir(self.git_checkout_path) self.scm = detect_scm_system(self.git_checkout_path) + # For historical reasons, we test some checkout code here too. + self.checkout = Checkout(self.scm) def tearDown(self): SVNTestRepository.tear_down(self) @@ -493,6 +617,52 @@ class GitTest(SCMTest): self.assertEqual(scm.display_name(), "git") self.assertEqual(scm.supports_local_commits(), True) + def test_read_git_config(self): + key = 'test.git-config' + value = 'git-config value' + run_command(['git', 'config', key, value]) + self.assertEqual(self.scm.read_git_config(key), value) + + def test_local_commits(self): + test_file = os.path.join(self.git_checkout_path, 'test_file') + write_into_file_at_path(test_file, 'foo') + run_command(['git', 'commit', '-a', '-m', 'local commit']) + + self.assertEqual(len(self.scm.local_commits()), 1) + + def test_discard_local_commits(self): + test_file = os.path.join(self.git_checkout_path, 'test_file') + write_into_file_at_path(test_file, 'foo') + run_command(['git', 'commit', '-a', '-m', 'local commit']) + + self.assertEqual(len(self.scm.local_commits()), 1) + self.scm.discard_local_commits() + self.assertEqual(len(self.scm.local_commits()), 0) + + def test_delete_branch(self): + old_branch = run_command(['git', 'symbolic-ref', 'HEAD']).strip() + new_branch = 'foo' + + run_command(['git', 'checkout', '-b', new_branch]) + self.assertEqual(run_command(['git', 'symbolic-ref', 'HEAD']).strip(), 'refs/heads/' + new_branch) + + run_command(['git', 'checkout', old_branch]) + self.scm.delete_branch(new_branch) + + self.assertFalse(re.search(r'foo', run_command(['git', 'branch']))) + + def test_svn_merge_base(self): + # Diff to merge-base should include working-copy changes, + # which the diff to svn_branch.. doesn't. + test_file = os.path.join(self.git_checkout_path, 'test_file') + write_into_file_at_path(test_file, 'foo') + + diff_to_common_base = run_command(['git', 'diff', self.scm.svn_branch_name() + '..']) + diff_to_merge_base = run_command(['git', 'diff', self.scm.svn_merge_base()]) + + self.assertFalse(re.search(r'foo', diff_to_common_base)) + self.assertTrue(re.search(r'foo', diff_to_merge_base)) + def test_rebase_in_progress(self): svn_test_file = os.path.join(self.svn_checkout_path, 'test_file') write_into_file_at_path(svn_test_file, "svn_checkout") @@ -538,15 +708,18 @@ class GitTest(SCMTest): def test_apply_git_patch(self): scm = detect_scm_system(self.git_checkout_path) + # We carefullly pick a diff which does not have a directory addition + # as currently svn-apply will error out when trying to remove directories + # in Git: https://bugs.webkit.org/show_bug.cgi?id=34871 patch = self._create_patch(run_command(['git', 'diff', 'HEAD..HEAD^'])) self._setup_webkittools_scripts_symlink(scm) - scm.apply_patch(patch) + Checkout(scm).apply_patch(patch) def test_apply_git_patch_force(self): scm = detect_scm_system(self.git_checkout_path) patch = self._create_patch(run_command(['git', 'diff', 'HEAD~2..HEAD'])) self._setup_webkittools_scripts_symlink(scm) - self.assertRaises(ScriptError, scm.apply_patch, patch, force=True) + self.assertRaises(ScriptError, Checkout(scm).apply_patch, patch, force=True) def test_commit_text_parsing(self): self._shared_test_commit_with_message() @@ -575,7 +748,7 @@ class GitTest(SCMTest): # Check if we can apply the created patch. run_command(['git', 'rm', '-f', test_file_name]) self._setup_webkittools_scripts_symlink(scm) - self.scm.apply_patch(self._create_patch(patch)) + self.checkout.apply_patch(self._create_patch(patch)) self.assertEqual(file_contents, read_from_path(test_file_path)) # Check if we can create a patch from a local commit. @@ -590,6 +763,21 @@ class GitTest(SCMTest): self.assertTrue(re.search(r'\nliteral 256\n', patch_since_local_commit)) self.assertEqual(patch_from_local_commit, patch_since_local_commit) + def test_changed_files(self): + self._shared_test_changed_files() + + def test_changed_files_for_revision(self): + self._shared_test_changed_files_for_revision() + + def test_contents_at_revision(self): + self._shared_test_contents_at_revision() + + def test_added_files(self): + self._shared_test_added_files() + + def test_committer_email_for_revision(self): + self._shared_test_committer_email_for_revision() + if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/common/config/__init__.py b/WebKitTools/Scripts/webkitpy/common/config/__init__.py new file mode 100644 index 0000000..03f1bc7 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/config/__init__.py @@ -0,0 +1,7 @@ +# Required for Python to search this directory for module files + +import re + +codereview_server_host = "wkrietveld.appspot.com" +codereview_server_regex = "https?://%s/" % re.sub('\.', '\\.', codereview_server_host) +codereview_server_url = "https://%s/" % codereview_server_host diff --git a/WebKitTools/Scripts/webkitpy/committers.py b/WebKitTools/Scripts/webkitpy/common/config/committers.py index 6413243..a92dbd3 100644 --- a/WebKitTools/Scripts/webkitpy/committers.py +++ b/WebKitTools/Scripts/webkitpy/common/config/committers.py @@ -31,12 +31,13 @@ class Committer: - def __init__(self, name, email_or_emails): + def __init__(self, name, email_or_emails, irc_nickname=None): self.full_name = name if isinstance(email_or_emails, str): self.emails = [email_or_emails] else: self.emails = email_or_emails + self.irc_nickname = irc_nickname self.can_review = False def bugzilla_email(self): @@ -50,8 +51,8 @@ class Committer: class Reviewer(Committer): - def __init__(self, name, email_or_emails): - Committer.__init__(self, name, email_or_emails) + def __init__(self, name, email_or_emails, irc_nickname=None): + Committer.__init__(self, name, email_or_emails, irc_nickname) self.can_review = True @@ -62,103 +63,110 @@ class Reviewer(Committer): committers_unable_to_review = [ - Committer("Aaron Boodman", "aa@chromium.org"), - Committer("Adam Langley", "agl@chromium.org"), + Committer("Aaron Boodman", "aa@chromium.org", "aboodman"), + Committer("Adam Langley", "agl@chromium.org", "agl"), Committer("Albert J. Wong", "ajwong@chromium.org"), Committer("Alejandro G. Castro", ["alex@igalia.com", "alex@webkit.org"]), - Committer("Alexander Kellett", ["lypanov@mac.com", "a-lists001@lypanov.net", "lypanov@kde.org"]), + Committer("Alexander Kellett", ["lypanov@mac.com", "a-lists001@lypanov.net", "lypanov@kde.org"], "lypanov"), Committer("Alexander Pavlov", "apavlov@chromium.org"), Committer("Andre Boule", "aboule@apple.com"), - Committer("Andrew Wellington", ["andrew@webkit.org", "proton@wiretapped.net"]), - Committer("Andras Becsi", "abecsi@webkit.org"), - Committer("Anthony Ricaud", "rik@webkit.org"), - Committer("Anton Muhin", "antonm@chromium.org"), - Committer("Antonio Gomes", "tonikitoo@webkit.org"), - Committer("Ben Murdoch", "benm@google.com"), - Committer("Benjamin C Meyer", ["ben@meyerhome.net", "ben@webkit.org"]), - Committer("Benjamin Otte", ["otte@gnome.org", "otte@webkit.org"]), - Committer("Brent Fulgham", "bfulgham@webkit.org"), - Committer("Brett Wilson", "brettw@chromium.org"), - Committer("Brian Weinstein", "bweinstein@apple.com"), - Committer("Cameron McCormack", "cam@webkit.org"), + Committer("Andrew Wellington", ["andrew@webkit.org", "proton@wiretapped.net"], "proton"), + Committer("Andras Becsi", "abecsi@webkit.org", "bbandix"), + Committer("Andy Estes", "aestes@apple.com", "estes"), + Committer("Anthony Ricaud", "rik@webkit.org", "rik"), + Committer("Anton Muhin", "antonm@chromium.org", "antonm"), + Committer("Antonio Gomes", "tonikitoo@webkit.org", "tonikitoo"), + Committer("Ben Murdoch", "benm@google.com", "benm"), + Committer("Benjamin C Meyer", ["ben@meyerhome.net", "ben@webkit.org"], "icefox"), + Committer("Benjamin Otte", ["otte@gnome.org", "otte@webkit.org"], "otte"), + Committer("Brent Fulgham", "bfulgham@webkit.org", "bfulgham"), + Committer("Brett Wilson", "brettw@chromium.org", "brettx"), + Committer("Brian Weinstein", "bweinstein@apple.com", "bweinstein"), + Committer("Cameron McCormack", "cam@webkit.org", "heycam"), Committer("Carol Szabo", "carol.szabo@nokia.com"), Committer("Chang Shu", "Chang.Shu@nokia.com"), Committer("Chris Fleizach", "cfleizach@apple.com"), - Committer("Chris Jerdonek", "cjerdonek@webkit.org"), - Committer("Chris Marrin", "cmarrin@apple.com"), - Committer("Chris Petersen", "cpetersen@apple.com"), + Committer("Chris Jerdonek", "cjerdonek@webkit.org", "cjerdonek"), + Committer("Chris Marrin", "cmarrin@apple.com", "cmarrin"), + Committer("Chris Petersen", "cpetersen@apple.com", "cpetersen"), Committer("Christian Dywan", ["christian@twotoasts.de", "christian@webkit.org"]), Committer("Collin Jackson", "collinj@webkit.org"), - Committer("Csaba Osztrogonac", "ossy@webkit.org"), - Committer("Daniel Bates", "dbates@webkit.org"), - Committer("David Smith", ["catfish.man@gmail.com", "dsmith@webkit.org"]), - Committer("Dean Jackson", "dino@apple.com"), + Committer("Csaba Osztrogonac", "ossy@webkit.org", "ossy"), + Committer("David Smith", ["catfish.man@gmail.com", "dsmith@webkit.org"], "catfishman"), + Committer("Dean Jackson", "dino@apple.com", "dino"), Committer("Dirk Pranke", "dpranke@chromium.org"), - Committer("Drew Wilson", "atwilson@chromium.org"), - Committer("Dumitru Daniliuc", "dumi@chromium.org"), - Committer("Eli Fidler", "eli@staikos.net"), + Committer("Drew Wilson", "atwilson@chromium.org", "atwilson"), + Committer("Dumitru Daniliuc", "dumi@chromium.org", "dumi"), + Committer("Eli Fidler", "eli@staikos.net", "QBin"), Committer("Enrica Casucci", "enrica@apple.com"), - Committer("Erik Arvidsson", "arv@chromium.org"), - Committer("Eric Roman", "eroman@chromium.org"), + Committer("Erik Arvidsson", "arv@chromium.org", "arv"), + Committer("Eric Roman", "eroman@chromium.org", "eroman"), Committer("Feng Qian", "feng@chromium.org"), - Committer("Fumitoshi Ukai", "ukai@chromium.org"), - Committer("Gabor Loki", "loki@webkit.org"), + Committer("Fumitoshi Ukai", "ukai@chromium.org", "ukai"), + Committer("Gabor Loki", "loki@webkit.org", "loki04"), Committer("Girish Ramakrishnan", ["girish@forwardbias.in", "ramakrishnan.girish@gmail.com"]), Committer("Graham Dennis", ["Graham.Dennis@gmail.com", "gdennis@webkit.org"]), Committer("Greg Bolsinga", "bolsinga@apple.com"), Committer("Hin-Chung Lam", ["hclam@google.com", "hclam@chromium.org"]), - Committer("Jakob Petsovits", ["jpetsovits@rim.com", "jpetso@gmx.at"]), + Committer("Ilya Tikhonovsky", "loislo@chromium.org", "loislo"), + Committer("Jakob Petsovits", ["jpetsovits@rim.com", "jpetso@gmx.at"], "jpetso"), + Committer("Jakub Wieczorek", "jwieczorek@webkit.org", "fawek"), + Committer("James Hawkins", ["jhawkins@chromium.org", "jhawkins@google.com"], "jhawkins"), + Committer("James Robinson", ["jamesr@chromium.org", "jamesr@google.com"]), Committer("Jens Alfke", ["snej@chromium.org", "jens@apple.com"]), - Committer("Jeremy Moskovich", ["playmobil@google.com", "jeremy@chromium.org"]), + Committer("Jeremy Moskovich", ["playmobil@google.com", "jeremy@chromium.org"], "jeremymos"), Committer("Jessie Berlin", ["jberlin@webkit.org", "jberlin@apple.com"]), - Committer("Jian Li", "jianli@chromium.org"), - Committer("John Abd-El-Malek", "jam@chromium.org"), - Committer("Joost de Valk", ["joost@webkit.org", "webkit-dev@joostdevalk.nl"]), - Committer("Joseph Pecoraro", "joepeck@webkit.org"), - Committer("Julie Parent", ["jparent@google.com", "jparent@chromium.org"]), + Committer("Jesus Sanchez-Palencia", ["jesus@webkit.org", "jesus.palencia@openbossa.org"], "jeez_"), + Committer("John Abd-El-Malek", "jam@chromium.org", "jam"), + Committer("John Gregg", ["johnnyg@google.com", "johnnyg@chromium.org"], "johnnyg"), + Committer("Joost de Valk", ["joost@webkit.org", "webkit-dev@joostdevalk.nl"], "Altha"), + Committer("Julie Parent", ["jparent@google.com", "jparent@chromium.org"], "jparent"), Committer("Julien Chaffraix", ["jchaffraix@webkit.org", "julien.chaffraix@gmail.com"]), Committer("Jungshik Shin", "jshin@chromium.org"), - Committer("Keishi Hattori", "keishi@webkit.org"), + Committer("Keishi Hattori", "keishi@webkit.org", "keishi"), Committer("Kelly Norton", "knorton@google.com"), Committer("Kenneth Russell", "kbr@google.com"), - Committer("Kent Tamura", "tkent@chromium.org"), + Committer("Kent Tamura", "tkent@chromium.org", "tkent"), + Committer("Kinuko Yasuda", "kinuko@chromium.org", "kinuko"), Committer("Krzysztof Kowalczyk", "kkowalczyk@gmail.com"), Committer("Levi Weintraub", "lweintraub@apple.com"), Committer("Mads Ager", "ager@chromium.org"), Committer("Matt Lilek", ["webkit@mattlilek.com", "pewtermoose@webkit.org"]), Committer("Matt Perry", "mpcomplete@chromium.org"), Committer("Maxime Britto", ["maxime.britto@gmail.com", "britto@apple.com"]), - Committer("Maxime Simon", ["simon.maxime@gmail.com", "maxime.simon@webkit.org"]), + Committer("Maxime Simon", ["simon.maxime@gmail.com", "maxime.simon@webkit.org"], "maxime.simon"), Committer("Martin Robinson", ["mrobinson@webkit.org", "martin.james.robinson@gmail.com"]), - Committer("Michelangelo De Simone", "michelangelo@webkit.org"), + Committer("Michelangelo De Simone", "michelangelo@webkit.org", "michelangelo"), Committer("Mike Belshe", ["mbelshe@chromium.org", "mike@belshe.com"]), - Committer("Mike Fenton", ["mike.fenton@torchmobile.com", "mifenton@rim.com"]), + Committer("Mike Fenton", ["mike.fenton@torchmobile.com", "mifenton@rim.com"], "mfenton"), Committer("Mike Thole", ["mthole@mikethole.com", "mthole@apple.com"]), Committer("Mikhail Naganov", "mnaganov@chromium.org"), - Committer("Ojan Vafai", "ojan@chromium.org"), - Committer("Pam Greene", "pam@chromium.org"), - Committer("Peter Kasting", ["pkasting@google.com", "pkasting@chromium.org"]), - Committer("Philippe Normand", ["pnormand@igalia.com", "philn@webkit.org"]), - Committer("Pierre d'Herbemont", ["pdherbemont@free.fr", "pdherbemont@apple.com"]), - Committer("Pierre-Olivier Latour", "pol@apple.com"), + Committer("MORITA Hajime", "morrita@google.com", "morrita"), + Committer("Ojan Vafai", "ojan@chromium.org", "ojan"), + Committer("Pam Greene", "pam@chromium.org", "pamg"), + Committer("Peter Kasting", ["pkasting@google.com", "pkasting@chromium.org"], "pkasting"), + Committer("Philippe Normand", ["pnormand@igalia.com", "philn@webkit.org"], "pnormand"), + Committer("Pierre d'Herbemont", ["pdherbemont@free.fr", "pdherbemont@apple.com"], "pdherbemont"), + Committer("Pierre-Olivier Latour", "pol@apple.com", "pol"), + Committer("Robert Hogan", ["robert@webkit.org", "robert@roberthogan.net"], "mwenge"), Committer("Roland Steiner", "rolandsteiner@chromium.org"), - Committer("Ryosuke Niwa", "rniwa@webkit.org"), - Committer("Scott Violet", "sky@chromium.org"), - Committer("Stephen White", "senorblanco@chromium.org"), + Committer("Ryosuke Niwa", "rniwa@webkit.org", "rniwa"), + Committer("Scott Violet", "sky@chromium.org", "sky"), + Committer("Stephen White", "senorblanco@chromium.org", "senorblanco"), Committer("Steve Block", "steveblock@google.com"), - Committer("Tony Chang", "tony@chromium.org"), - Committer("Trey Matteson", "trey@usa.net"), + Committer("Tony Chang", "tony@chromium.org", "tony^work"), + Committer("Trey Matteson", "trey@usa.net", "trey"), Committer("Tristan O'Tierney", ["tristan@otierney.net", "tristan@apple.com"]), Committer("Victor Wang", "victorw@chromium.org"), - Committer("William Siegrist", "wsiegrist@apple.com"), + Committer("Vitaly Repeshko", "vitalyr@chromium.org"), + Committer("William Siegrist", "wsiegrist@apple.com", "wms"), Committer("Yael Aharon", "yael.aharon@nokia.com"), Committer("Yaar Schnitman", ["yaar@chromium.org", "yaar@google.com"]), - Committer("Yong Li", ["yong.li@torchmobile.com", "yong.li.webkit@gmail.com"]), + Committer("Yong Li", ["yong.li@torchmobile.com", "yong.li.webkit@gmail.com"], "yong"), Committer("Yongjun Zhang", "yongjun.zhang@nokia.com"), - Committer("Yury Semikhatsky", "yurys@chromium.org"), - Committer("Yuzo Fujishima", "yuzo@google.com"), - Committer("Zoltan Herczeg", "zherczeg@webkit.org"), - Committer("Zoltan Horvath", "zoltan@webkit.org"), + Committer("Yuzo Fujishima", "yuzo@google.com", "yuzo"), + Committer("Zoltan Herczeg", "zherczeg@webkit.org", "zherczeg"), + Committer("Zoltan Horvath", "zoltan@webkit.org", "zoltan"), ] @@ -168,71 +176,75 @@ committers_unable_to_review = [ reviewers_list = [ - Reviewer("Ada Chan", "adachan@apple.com"), - Reviewer("Adam Barth", "abarth@webkit.org"), - Reviewer("Adam Roben", "aroben@apple.com"), - Reviewer("Adam Treat", ["treat@kde.org", "treat@webkit.org"]), - Reviewer("Adele Peterson", "adele@apple.com"), - Reviewer("Alexey Proskuryakov", ["ap@webkit.org", "ap@apple.com"]), - Reviewer("Alice Liu", "alice.liu@apple.com"), - Reviewer("Alp Toker", ["alp@nuanti.com", "alp@atoker.com", "alp@webkit.org"]), - Reviewer("Anders Carlsson", ["andersca@apple.com", "acarlsson@apple.com"]), - Reviewer("Antti Koivisto", ["koivisto@iki.fi", "antti@apple.com"]), - Reviewer("Ariya Hidayat", ["ariya.hidayat@gmail.com", "ariya@webkit.org"]), - Reviewer("Beth Dakin", "bdakin@apple.com"), - Reviewer("Brady Eidson", "beidson@apple.com"), + Reviewer("Ada Chan", "adachan@apple.com", "chanada"), + Reviewer("Adam Barth", "abarth@webkit.org", "abarth"), + Reviewer("Adam Roben", "aroben@apple.com", "aroben"), + Reviewer("Adam Treat", ["treat@kde.org", "treat@webkit.org"], "manyoso"), + Reviewer("Adele Peterson", "adele@apple.com", "adele"), + Reviewer("Alexey Proskuryakov", ["ap@webkit.org", "ap@apple.com"], "ap"), + Reviewer("Alice Liu", "alice.liu@apple.com", "aliu"), + Reviewer("Alp Toker", ["alp@nuanti.com", "alp@atoker.com", "alp@webkit.org"], "alp"), + Reviewer("Anders Carlsson", ["andersca@apple.com", "acarlsson@apple.com"], "andersca"), + Reviewer("Antti Koivisto", ["koivisto@iki.fi", "antti@apple.com"], "anttik"), + Reviewer("Ariya Hidayat", ["ariya.hidayat@gmail.com", "ariya@webkit.org"], "ariya"), + Reviewer("Beth Dakin", "bdakin@apple.com", "dethbakin"), + Reviewer("Brady Eidson", "beidson@apple.com", "bradee-oh"), Reviewer("Cameron Zwarich", ["zwarich@apple.com", "cwzwarich@apple.com", "cwzwarich@webkit.org"]), - Reviewer("Chris Blumenberg", "cblu@apple.com"), - Reviewer("Dan Bernstein", ["mitz@webkit.org", "mitz@apple.com"]), - Reviewer("Darin Adler", "darin@apple.com"), - Reviewer("Darin Fisher", ["fishd@chromium.org", "darin@chromium.org"]), - Reviewer("David Harrison", "harrison@apple.com"), - Reviewer("David Hyatt", "hyatt@apple.com"), - Reviewer("David Kilzer", ["ddkilzer@webkit.org", "ddkilzer@apple.com"]), - Reviewer("David Levin", "levin@chromium.org"), - Reviewer("Dimitri Glazkov", "dglazkov@chromium.org"), - Reviewer("Dirk Schulze", "krit@webkit.org"), - Reviewer("Dmitry Titov", "dimich@chromium.org"), - Reviewer("Don Melton", "gramps@apple.com"), + Reviewer("Chris Blumenberg", "cblu@apple.com", "cblu"), + Reviewer("Dan Bernstein", ["mitz@webkit.org", "mitz@apple.com"], "mitzpettel"), + Reviewer("Daniel Bates", "dbates@webkit.org", "dydz"), + Reviewer("Darin Adler", "darin@apple.com", "darin"), + Reviewer("Darin Fisher", ["fishd@chromium.org", "darin@chromium.org"], "fishd"), + Reviewer("David Harrison", "harrison@apple.com", "harrison"), + Reviewer("David Hyatt", "hyatt@apple.com", "hyatt"), + Reviewer("David Kilzer", ["ddkilzer@webkit.org", "ddkilzer@apple.com"], "ddkilzer"), + Reviewer("David Levin", "levin@chromium.org", "dave_levin"), + Reviewer("Dimitri Glazkov", "dglazkov@chromium.org", "dglazkov"), + Reviewer("Dirk Schulze", "krit@webkit.org", "krit"), + Reviewer("Dmitry Titov", "dimich@chromium.org", "dimich"), + Reviewer("Don Melton", "gramps@apple.com", "gramps"), Reviewer("Eric Carlson", "eric.carlson@apple.com"), - Reviewer("Eric Seidel", "eric@webkit.org"), - Reviewer("Gavin Barraclough", "barraclough@apple.com"), - Reviewer("Geoffrey Garen", "ggaren@apple.com"), + Reviewer("Eric Seidel", "eric@webkit.org", "eseidel"), + Reviewer("Gavin Barraclough", "barraclough@apple.com", "gbarra"), + Reviewer("Geoffrey Garen", "ggaren@apple.com", "ggaren"), Reviewer("George Staikos", ["staikos@kde.org", "staikos@webkit.org"]), - Reviewer("Gustavo Noronha Silva", ["gns@gnome.org", "kov@webkit.org"]), - Reviewer("Holger Freyther", ["zecke@selfish.org", "zecke@webkit.org"]), - Reviewer("Jan Alonzo", ["jmalonzo@gmail.com", "jmalonzo@webkit.org"]), - Reviewer("Jeremy Orlow", "jorlow@chromium.org"), - Reviewer("John Sullivan", "sullivan@apple.com"), - Reviewer("Jon Honeycutt", "jhoneycutt@apple.com"), - Reviewer("Justin Garcia", "justin.garcia@apple.com"), + Reviewer("Gustavo Noronha Silva", ["gns@gnome.org", "kov@webkit.org"], "kov"), + Reviewer("Holger Freyther", ["zecke@selfish.org", "zecke@webkit.org"], "zecke"), + Reviewer("Jan Alonzo", ["jmalonzo@gmail.com", "jmalonzo@webkit.org"], "janm"), + Reviewer("Jeremy Orlow", "jorlow@chromium.org", "jorlow"), + Reviewer("Jian Li", "jianli@chromium.org", "jianli"), + Reviewer("John Sullivan", "sullivan@apple.com", "sullivan"), + Reviewer("Jon Honeycutt", "jhoneycutt@apple.com", "jhoneycutt"), + Reviewer("Joseph Pecoraro", "joepeck@webkit.org", "JoePeck"), + Reviewer("Justin Garcia", "justin.garcia@apple.com", "justing"), Reviewer("Ken Kocienda", "kocienda@apple.com"), - Reviewer("Kenneth Rohde Christiansen", ["kenneth@webkit.org", "kenneth.christiansen@openbossa.org"]), - Reviewer("Kevin Decker", "kdecker@apple.com"), - Reviewer("Kevin McCullough", "kmccullough@apple.com"), - Reviewer("Kevin Ollivier", ["kevino@theolliviers.com", "kevino@webkit.org"]), - Reviewer("Lars Knoll", ["lars@trolltech.com", "lars@kde.org"]), - Reviewer("Laszlo Gombos", "laszlo.1.gombos@nokia.com"), - Reviewer("Maciej Stachowiak", "mjs@apple.com"), - Reviewer("Mark Rowe", "mrowe@apple.com"), - Reviewer("Nate Chapin", "japhet@chromium.org"), - Reviewer("Nikolas Zimmermann", ["zimmermann@kde.org", "zimmermann@physik.rwth-aachen.de", "zimmermann@webkit.org"]), - Reviewer("Oliver Hunt", "oliver@apple.com"), - Reviewer("Pavel Feldman", "pfeldman@chromium.org"), - Reviewer("Richard Williamson", "rjw@apple.com"), - Reviewer("Rob Buis", ["rwlbuis@gmail.com", "rwlbuis@webkit.org"]), - Reviewer("Sam Weinig", ["sam@webkit.org", "weinig@apple.com"]), - Reviewer("Shinichiro Hamaji", "hamaji@chromium.org"), - Reviewer("Simon Fraser", "simon.fraser@apple.com"), - Reviewer("Simon Hausmann", ["hausmann@webkit.org", "hausmann@kde.org", "simon.hausmann@nokia.com"]), - Reviewer("Stephanie Lewis", "slewis@apple.com"), - Reviewer("Steve Falkenburg", "sfalken@apple.com"), + Reviewer("Kenneth Rohde Christiansen", ["kenneth@webkit.org", "kenneth.christiansen@openbossa.org"], "kenne"), + Reviewer("Kevin Decker", "kdecker@apple.com", "superkevin"), + Reviewer("Kevin McCullough", "kmccullough@apple.com", "maculloch"), + Reviewer("Kevin Ollivier", ["kevino@theolliviers.com", "kevino@webkit.org"], "kollivier"), + Reviewer("Lars Knoll", ["lars@trolltech.com", "lars@kde.org"], "lars"), + Reviewer("Laszlo Gombos", "laszlo.1.gombos@nokia.com", "lgombos"), + Reviewer("Maciej Stachowiak", "mjs@apple.com", "othermaciej"), + Reviewer("Mark Rowe", "mrowe@apple.com", "bdash"), + Reviewer("Nate Chapin", "japhet@chromium.org", "japhet"), + Reviewer("Nikolas Zimmermann", ["zimmermann@kde.org", "zimmermann@physik.rwth-aachen.de", "zimmermann@webkit.org"], "wildfox"), + Reviewer("Oliver Hunt", "oliver@apple.com", "olliej"), + Reviewer("Pavel Feldman", "pfeldman@chromium.org", "pfeldman"), + Reviewer("Richard Williamson", "rjw@apple.com", "rjw"), + Reviewer("Rob Buis", ["rwlbuis@gmail.com", "rwlbuis@webkit.org"], "rwlbuis"), + Reviewer("Sam Weinig", ["sam@webkit.org", "weinig@apple.com"], "weinig"), + Reviewer("Shinichiro Hamaji", "hamaji@chromium.org", "hamaji"), + Reviewer("Simon Fraser", "simon.fraser@apple.com", "smfr"), + Reviewer("Simon Hausmann", ["hausmann@webkit.org", "hausmann@kde.org", "simon.hausmann@nokia.com"], "tronical"), + Reviewer("Stephanie Lewis", "slewis@apple.com", "sundiamonde"), + Reviewer("Steve Falkenburg", "sfalken@apple.com", "sfalken"), Reviewer("Tim Omernick", "timo@apple.com"), - Reviewer("Timothy Hatcher", ["timothy@hatcher.name", "timothy@apple.com"]), - Reviewer(u'Tor Arne Vestb\xf8', "vestbo@webkit.org"), + Reviewer("Timothy Hatcher", ["timothy@hatcher.name", "timothy@apple.com"], "xenon"), + Reviewer(u'Tor Arne Vestb\xf8', "vestbo@webkit.org", "torarne"), Reviewer("Vicki Murley", "vicki@apple.com"), - Reviewer("Xan Lopez", ["xan.lopez@gmail.com", "xan@gnome.org", "xan@webkit.org"]), - Reviewer("Zack Rusin", "zack@kde.org"), + Reviewer("Xan Lopez", ["xan.lopez@gmail.com", "xan@gnome.org", "xan@webkit.org"], "xan"), + Reviewer("Yury Semikhatsky", "yurys@chromium.org", "yurys"), + Reviewer("Zack Rusin", "zack@kde.org", "zackr"), ] @@ -260,6 +272,12 @@ class CommitterList: self._committers_by_email[email] = committer return self._committers_by_email + def committer_by_name(self, name): + # This could be made into a hash lookup if callers need it to be fast. + for committer in self.committers(): + if committer.full_name == name: + return committer + def committer_by_email(self, email): return self._email_to_committer_map().get(email) diff --git a/WebKitTools/Scripts/webkitpy/committers_unittest.py b/WebKitTools/Scripts/webkitpy/common/config/committers_unittest.py index f5dc539..068c0ee 100644 --- a/WebKitTools/Scripts/webkitpy/committers_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/config/committers_unittest.py @@ -27,12 +27,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from committers import CommitterList, Committer, Reviewer +from webkitpy.common.config.committers import CommitterList, Committer, Reviewer class CommittersTest(unittest.TestCase): def test_committer_lookup(self): - committer = Committer('Test One', 'one@test.com') + committer = Committer('Test One', 'one@test.com', 'one') reviewer = Reviewer('Test Two', ['two@test.com', 'two@rad.com', 'so_two@gmail.com']) committer_list = CommitterList(committers=[committer], reviewers=[reviewer]) @@ -43,6 +43,11 @@ class CommittersTest(unittest.TestCase): self.assertEqual(committer_list.committer_by_email('two@rad.com'), reviewer) self.assertEqual(committer_list.reviewer_by_email('so_two@gmail.com'), reviewer) + # Test valid committer and reviewer lookup + self.assertEqual(committer_list.committer_by_name("Test One"), committer) + self.assertEqual(committer_list.committer_by_name("Test Two"), reviewer) + self.assertEqual(committer_list.committer_by_name("Test Three"), None) + # Test that the first email is assumed to be the Bugzilla email address (for now) self.assertEqual(committer_list.committer_by_email('two@rad.com').bugzilla_email(), 'two@test.com') @@ -56,6 +61,8 @@ class CommittersTest(unittest.TestCase): # Test that emails returns a list. self.assertEqual(committer.emails, ['one@test.com']) + self.assertEqual(committer.irc_nickname, 'one') + # Test that committers returns committers and reviewers and reviewers() just reviewers. self.assertEqual(committer_list.committers(), [committer, reviewer]) self.assertEqual(committer_list.reviewers(), [reviewer]) diff --git a/WebKitTools/Scripts/webkitpy/common/config/irc.py b/WebKitTools/Scripts/webkitpy/common/config/irc.py new file mode 100644 index 0000000..950c573 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/config/irc.py @@ -0,0 +1,31 @@ +# Copyright (c) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +server="irc.freenode.net" +port=6667 +channel="#webkit" diff --git a/WebKitTools/Scripts/webkitpy/webkitport.py b/WebKitTools/Scripts/webkitpy/common/config/ports.py index cd60a54..a881a67 100644 --- a/WebKitTools/Scripts/webkitpy/webkitport.py +++ b/WebKitTools/Scripts/webkitpy/common/config/ports.py @@ -29,9 +29,9 @@ # WebKit's Python module for understanding the various ports import os +import platform -from optparse import make_option -from webkitpy.executive import Executive +from webkitpy.common.system.executive import Executive class WebKitPort(object): @@ -47,10 +47,22 @@ class WebKitPort(object): "chromium": ChromiumPort, "gtk": GtkPort, "mac": MacPort, + "win": WinPort, "qt": QtPort, } - # FIXME: We should default to WinPort on Windows. - return ports.get(port_name, MacPort) + default_port = { + "Windows": WinPort, + "Darwin": MacPort, + } + # Do we really need MacPort as the ultimate default? + return ports.get(port_name, default_port.get(platform.system(), MacPort)) + + @staticmethod + def makeArgs(): + args = '--makeargs="-j%s"' % Executive().cpu_count() + if os.environ.has_key('MAKEFLAGS'): + args = '--makeargs="%s"' % os.environ['MAKEFLAGS'] + return args @classmethod def name(cls): @@ -101,6 +113,18 @@ class MacPort(WebKitPort): return "--port=mac" +class WinPort(WebKitPort): + + @classmethod + def name(cls): + return "Win" + + @classmethod + def flag(cls): + # FIXME: This is lame. We should autogenerate this from a codename or something. + return "--port=win" + + class GtkPort(WebKitPort): @classmethod @@ -115,7 +139,7 @@ class GtkPort(WebKitPort): def build_webkit_command(cls, build_style=None): command = WebKitPort.build_webkit_command(build_style=build_style) command.append("--gtk") - command.append('--makeargs="-j%s"' % Executive.cpu_count()) + command.append(WebKitPort.makeArgs()) return command @classmethod @@ -139,7 +163,7 @@ class QtPort(WebKitPort): def build_webkit_command(cls, build_style=None): command = WebKitPort.build_webkit_command(build_style=build_style) command.append("--qt") - command.append('--makeargs="-j%s"' % Executive.cpu_count()) + command.append(WebKitPort.makeArgs()) return command diff --git a/WebKitTools/Scripts/webkitpy/webkitport_unittest.py b/WebKitTools/Scripts/webkitpy/common/config/ports_unittest.py index 202234f..c42d2d0 100644 --- a/WebKitTools/Scripts/webkitpy/webkitport_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/config/ports_unittest.py @@ -29,8 +29,7 @@ import unittest -from webkitpy.executive import Executive -from webkitpy.webkitport import WebKitPort, MacPort, GtkPort, QtPort, ChromiumPort +from webkitpy.common.config.ports import WebKitPort, MacPort, GtkPort, QtPort, ChromiumPort class WebKitPortTest(unittest.TestCase): @@ -46,15 +45,15 @@ class WebKitPortTest(unittest.TestCase): self.assertEquals(GtkPort.name(), "Gtk") self.assertEquals(GtkPort.flag(), "--port=gtk") self.assertEquals(GtkPort.run_webkit_tests_command(), [WebKitPort.script_path("run-webkit-tests"), "--gtk"]) - self.assertEquals(GtkPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--gtk", '--makeargs="-j%s"' % Executive.cpu_count()]) - self.assertEquals(GtkPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--gtk", '--makeargs="-j%s"' % Executive.cpu_count()]) + self.assertEquals(GtkPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--gtk", WebKitPort.makeArgs()]) + self.assertEquals(GtkPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--gtk", WebKitPort.makeArgs()]) def test_qt_port(self): self.assertEquals(QtPort.name(), "Qt") self.assertEquals(QtPort.flag(), "--port=qt") self.assertEquals(QtPort.run_webkit_tests_command(), [WebKitPort.script_path("run-webkit-tests")]) - self.assertEquals(QtPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--qt", '--makeargs="-j%s"' % Executive.cpu_count()]) - self.assertEquals(QtPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--qt", '--makeargs="-j%s"' % Executive.cpu_count()]) + self.assertEquals(QtPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--qt", WebKitPort.makeArgs()]) + self.assertEquals(QtPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--qt", WebKitPort.makeArgs()]) def test_chromium_port(self): self.assertEquals(ChromiumPort.name(), "Chromium") diff --git a/WebKitTools/Scripts/webkitpy/common/net/__init__.py b/WebKitTools/Scripts/webkitpy/common/net/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/bugzilla.py b/WebKitTools/Scripts/webkitpy/common/net/bugzilla.py index 4506af2..6920d67 100644 --- a/WebKitTools/Scripts/webkitpy/bugzilla.py +++ b/WebKitTools/Scripts/webkitpy/common/net/bugzilla.py @@ -30,25 +30,24 @@ # # WebKit's Python module for interacting with Bugzilla +import os.path import re import subprocess from datetime import datetime # used in timestamp() -# Import WebKit-specific modules. -from webkitpy.webkit_logging import error, log -from webkitpy.committers import CommitterList -from webkitpy.credentials import Credentials -from webkitpy.user import User - -# WebKit includes a built copy of BeautifulSoup in Scripts/webkitpy -# so this import should always succeed. -from .BeautifulSoup import BeautifulSoup, SoupStrainer - -from mechanize import Browser +from webkitpy.common.system.deprecated_logging import error, log +from webkitpy.common.config import committers +from webkitpy.common.net.credentials import Credentials +from webkitpy.common.system.ospath import relpath +from webkitpy.common.system.user import User +from webkitpy.thirdparty.autoinstalled.mechanize import Browser +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, SoupStrainer def parse_bug_id(message): + if not message: + return None match = re.search("http\://webkit\.org/b/(?P<bug_id>\d+)", message) if match: return int(match.group('bug_id')) @@ -66,6 +65,8 @@ def timestamp(): class Attachment(object): + rollout_preamble = "ROLLOUT of r" + def __init__(self, attachment_dictionary, bug): self._attachment_dictionary = attachment_dictionary self._bug = bug @@ -97,6 +98,9 @@ class Attachment(object): def is_obsolete(self): return not not self._attachment_dictionary.get("is_obsolete") + def is_rollout(self): + return self.name().startswith(self.rollout_preamble) + def name(self): return self._attachment_dictionary.get("name") @@ -150,6 +154,14 @@ class Bug(object): def assigned_to_email(self): return self.bug_dictionary["assigned_to_email"] + # FIXME: This information should be stored in some sort of webkit_config.py instead of here. + unassigned_emails = frozenset([ + "webkit-unassigned@lists.webkit.org", + "webkit-qt-unassigned@trolltech.com", + ]) + def is_unassigned(self): + return self.assigned_to_email() in self.unassigned_emails + # Rarely do we actually want obsolete attachments def attachments(self, include_obsolete=False): attachments = self.bug_dictionary["attachments"] @@ -218,6 +230,14 @@ class BugzillaQueries(object): def _fetch_attachment_ids_request_query(self, query): return self._parse_attachment_ids_request_query(self._load_query(query)) + def _parse_quips(self, page): + soup = BeautifulSoup(page, convertEntities=BeautifulSoup.HTML_ENTITIES) + quips = soup.find(text=re.compile(r"Existing quips:")).findNext("ul").findAll("li") + return [unicode(quip_entry.string) for quip_entry in quips] + + def fetch_quips(self): + return self._parse_quips(self._load_query("/quips.cgi?action=show")) + # List of all r+'d bugs. def fetch_bug_ids_from_pending_commit_list(self): needs_commit_query_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B" @@ -264,23 +284,37 @@ class CommitterValidator(object): def _view_source_url(self, local_path): return "http://trac.webkit.org/browser/trunk/%s" % local_path + def _checkout_root(self): + # FIXME: This is a hack, we would have this from scm.checkout_root + # if we had any way to get to an scm object here. + components = __file__.split(os.sep) + tools_index = components.index("WebKitTools") + return os.sep.join(components[:tools_index]) + + def _committers_py_path(self): + # extension can sometimes be .pyc, we always want .py + (path, extension) = os.path.splitext(committers.__file__) + # FIXME: When we're allowed to use python 2.6 we can use the real + # os.path.relpath + path = relpath(path, self._checkout_root()) + return ".".join([path, "py"]) + def _flag_permission_rejection_message(self, setter_email, flag_name): - # This could be computed from CommitterList.__file__ - committer_list = "WebKitTools/Scripts/webkitpy/committers.py" # Should come from some webkit_config.py contribution_guidlines = "http://webkit.org/coding/contributing.html" # This could be queried from the status_server. queue_administrator = "eseidel@chromium.org" # This could be queried from the tool. queue_name = "commit-queue" + committers_list = self._committers_py_path() message = "%s does not have %s permissions according to %s." % ( setter_email, flag_name, - self._view_source_url(committer_list)) + self._view_source_url(committers_list)) message += "\n\n- If you do not have %s rights please read %s for instructions on how to use bugzilla flags." % ( flag_name, contribution_guidlines) message += "\n\n- If you have %s rights please correct the error in %s by adding yourself to the file (no review needed). " % ( - flag_name, committer_list) + flag_name, committers_list) message += "Due to bug 30084 the %s will require a restart after your change. " % queue_name message += "Please contact %s to request a %s restart. " % ( queue_administrator, queue_name) @@ -333,11 +367,12 @@ class CommitterValidator(object): class Bugzilla(object): - def __init__(self, dryrun=False, committers=CommitterList()): + def __init__(self, dryrun=False, committers=committers.CommitterList()): self.dryrun = dryrun self.authenticated = False self.queries = BugzillaQueries(self) self.committers = committers + self.cached_quips = [] # FIXME: We should use some sort of Browser mock object when in dryrun # mode (to prevent any mistakes). @@ -350,18 +385,30 @@ class Bugzilla(object): bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) bug_server_url = "https://%s/" % bug_server_host - unassigned_email = "webkit-unassigned@lists.webkit.org" + + def quips(self): + # We only fetch and parse the list of quips once per instantiation + # so that we do not burden bugs.webkit.org. + if not self.cached_quips and not self.dryrun: + self.cached_quips = self.queries.fetch_quips() + return self.cached_quips def bug_url_for_bug_id(self, bug_id, xml=False): + if not bug_id: + return None content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def short_bug_url_for_bug_id(self, bug_id): + if not bug_id: + return None return "http://webkit.org/b/%s" % bug_id def attachment_url_for_id(self, attachment_id, action="view"): + if not attachment_id: + return None action_param = "" if action and action != "view": action_param = "&action=%s" % action @@ -418,7 +465,11 @@ class Bugzilla(object): return self.browser.open(bug_url) def fetch_bug_dictionary(self, bug_id): - return self._parse_bug_page(self._fetch_bug_page(bug_id)) + try: + return self._parse_bug_page(self._fetch_bug_page(bug_id)) + except: + self.authenticate() + return self._parse_bug_page(self._fetch_bug_page(bug_id)) # FIXME: A BugzillaCache object should provide all these fetch_ methods. @@ -554,15 +605,6 @@ class Bugzilla(object): self.browser['comment'] = comment_text self.browser.submit() - def prompt_for_component(self, components): - log("Please pick a component:") - i = 0 - for name in components: - i += 1 - log("%2d. %s" % (i, name)) - result = int(User.prompt("Enter a number: ")) - 1 - return components[result] - def _check_create_bug_response(self, response_html): match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html) @@ -589,6 +631,7 @@ class Bugzilla(object): patch_file_object=None, patch_description=None, cc=None, + blocked=None, mark_for_review=False, mark_for_commit_queue=False): self.authenticate() @@ -605,12 +648,14 @@ class Bugzilla(object): if not component: component = "New Bugs" if component not in component_names: - component = self.prompt_for_component(component_names) - self.browser['component'] = [component] + component = User.prompt_with_list("Please pick a component:", component_names) + self.browser["component"] = [component] if cc: - self.browser['cc'] = cc - self.browser['short_desc'] = bug_title - self.browser['comment'] = bug_description + self.browser["cc"] = cc + if blocked: + self.browser["blocked"] = str(blocked) + self.browser["short_desc"] = bug_title + self.browser["comment"] = bug_description if patch_file_object: self._fill_attachment_form( diff --git a/WebKitTools/Scripts/webkitpy/bugzilla_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/bugzilla_unittest.py index d555f78..4c44cdf 100644 --- a/WebKitTools/Scripts/webkitpy/bugzilla_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/net/bugzilla_unittest.py @@ -28,12 +28,11 @@ import unittest -from webkitpy.committers import CommitterList, Reviewer, Committer -from webkitpy.bugzilla import Bugzilla, BugzillaQueries, parse_bug_id, CommitterValidator -from webkitpy.outputcapture import OutputCapture -from webkitpy.mock import Mock - -from webkitpy.BeautifulSoup import BeautifulSoup +from webkitpy.common.config.committers import CommitterList, Reviewer, Committer +from webkitpy.common.net.bugzilla import Bugzilla, BugzillaQueries, parse_bug_id, CommitterValidator, Bug +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup class MockBrowser(object): @@ -49,14 +48,25 @@ class MockBrowser(object): def submit(self): pass + +class BugTest(unittest.TestCase): + def test_is_unassigned(self): + for email in Bug.unassigned_emails: + bug = Bug({"assigned_to_email" : email}, bugzilla=None) + self.assertTrue(bug.is_unassigned()) + bug = Bug({"assigned_to_email" : "test@test.com"}, bugzilla=None) + self.assertFalse(bug.is_unassigned()) + + class CommitterValidatorTest(unittest.TestCase): def test_flag_permission_rejection_message(self): validator = CommitterValidator(bugzilla=None) - expected_messsage="""foo@foo.com does not have review permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/committers.py. + self.assertEqual(validator._committers_py_path(), "WebKitTools/Scripts/webkitpy/common/config/committers.py") + expected_messsage="""foo@foo.com does not have review permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/common/config/committers.py. - If you do not have review rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags. -- If you have review rights please correct the error in WebKitTools/Scripts/webkitpy/committers.py by adding yourself to the file (no review needed). Due to bug 30084 the commit-queue will require a restart after your change. Please contact eseidel@chromium.org to request a commit-queue restart. After restart the commit-queue will correctly respect your review rights.""" +- If you have review rights please correct the error in WebKitTools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed). Due to bug 30084 the commit-queue will require a restart after your change. Please contact eseidel@chromium.org to request a commit-queue restart. After restart the commit-queue will correctly respect your review rights.""" self.assertEqual(validator._flag_permission_rejection_message("foo@foo.com", "review"), expected_messsage) @@ -101,6 +111,13 @@ class BugzillaTest(unittest.TestCase): 'attacher_email' : 'christian.plesner.hansen@gmail.com', } + def test_url_creation(self): + # FIXME: These would be all better as doctests + bugs = Bugzilla() + self.assertEquals(None, bugs.bug_url_for_bug_id(None)) + self.assertEquals(None, bugs.short_bug_url_for_bug_id(None)) + self.assertEquals(None, bugs.attachment_url_for_id(None)) + def test_parse_bug_id(self): # FIXME: These would be all better as doctests bugs = Bugzilla() @@ -199,6 +216,7 @@ ZEZpbmlzaExvYWRXaXRoUmVhc29uOnJlYXNvbl07Cit9CisKIEBlbmQKIAogI2VuZGlmCg== }], } + # FIXME: This should move to a central location and be shared by more unit tests. def _assert_dictionaries_equal(self, actual, expected): # Make sure we aren't parsing more or less than we expect self.assertEquals(sorted(actual.keys()), sorted(expected.keys())) @@ -289,11 +307,37 @@ class BugzillaQueriesTest(unittest.TestCase): </body> </html> """ + _sample_quip_page = u""" +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" + "http://www.w3.org/TR/html4/loose.dtd"> +<html> + <head> + <title>Bugzilla Quip System</title> + </head> + <body> + <h2> + + Existing quips: + </h2> + <ul> + <li>Everything should be made as simple as possible, but not simpler. - Albert Einstein</li> + <li>Good artists copy. Great artists steal. - Pablo Picasso</li> + <li>\u00e7gua mole em pedra dura, tanto bate at\u008e que fura.</li> + + </ul> + </body> +</html> +""" def test_request_page_parsing(self): queries = BugzillaQueries(None) self.assertEquals([40511, 40722, 40723], queries._parse_attachment_ids_request_query(self._sample_request_page)) + def test_quip_page_parsing(self): + queries = BugzillaQueries(None) + expected_quips = ["Everything should be made as simple as possible, but not simpler. - Albert Einstein", "Good artists copy. Great artists steal. - Pablo Picasso", u"\u00e7gua mole em pedra dura, tanto bate at\u008e que fura."] + self.assertEquals(expected_quips, queries._parse_quips(self._sample_quip_page)) + def test_load_query(self): queries = BugzillaQueries(Mock()) queries._load_query("request.cgi?action=queue&type=review&group=type") diff --git a/WebKitTools/Scripts/webkitpy/common/net/buildbot.py b/WebKitTools/Scripts/webkitpy/common/net/buildbot.py new file mode 100644 index 0000000..753e909 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/buildbot.py @@ -0,0 +1,495 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# WebKit's Python module for interacting with WebKit's buildbot + +import operator +import re +import urllib +import urllib2 +import xmlrpclib + +from webkitpy.common.system.logutils import get_logger +from webkitpy.thirdparty.autoinstalled.mechanize import Browser +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup + + +_log = get_logger(__file__) + + +class Builder(object): + def __init__(self, name, buildbot): + self._name = unicode(name) + self._buildbot = buildbot + self._builds_cache = {} + self._revision_to_build_number = None + self._browser = Browser() + self._browser.set_handle_robots(False) # The builder pages are excluded by robots.txt + + def name(self): + return self._name + + def results_url(self): + return "http://%s/results/%s" % (self._buildbot.buildbot_host, self.url_encoded_name()) + + def url_encoded_name(self): + return urllib.quote(self._name) + + def url(self): + return "http://%s/builders/%s" % (self._buildbot.buildbot_host, self.url_encoded_name()) + + # This provides a single place to mock + def _fetch_build(self, build_number): + build_dictionary = self._buildbot._fetch_xmlrpc_build_dictionary(self, build_number) + if not build_dictionary: + return None + return Build(self, + build_number=int(build_dictionary['number']), + revision=int(build_dictionary['revision']), + is_green=(build_dictionary['results'] == 0) # Undocumented, buildbot XMLRPC, 0 seems to mean "pass" + ) + + def build(self, build_number): + if not build_number: + return None + cached_build = self._builds_cache.get(build_number) + if cached_build: + return cached_build + + build = self._fetch_build(build_number) + self._builds_cache[build_number] = build + return build + + def force_build(self, username="webkit-patch", comments=None): + def predicate(form): + try: + return form.find_control("username") + except Exception, e: + return False + self._browser.open(self.url()) + self._browser.select_form(predicate=predicate) + self._browser["username"] = username + if comments: + self._browser["comments"] = comments + return self._browser.submit() + + file_name_regexp = re.compile(r"r(?P<revision>\d+) \((?P<build_number>\d+)\)") + def _revision_and_build_for_filename(self, filename): + # Example: "r47483 (1)/" or "r47483 (1).zip" + match = self.file_name_regexp.match(filename) + return (int(match.group("revision")), int(match.group("build_number"))) + + def _fetch_revision_to_build_map(self): + # All _fetch requests go through _buildbot for easier mocking + try: + # FIXME: This method is horribly slow due to the huge network load. + # FIXME: This is a poor way to do revision -> build mapping. + # Better would be to ask buildbot through some sort of API. + print "Loading revision/build list from %s." % self.results_url() + print "This may take a while..." + result_files = self._buildbot._fetch_twisted_directory_listing(self.results_url()) + except urllib2.HTTPError, error: + if error.code != 404: + raise + result_files = [] + + # This assumes there was only one build per revision, which is false but we don't care for now. + return dict([self._revision_and_build_for_filename(file_info["filename"]) for file_info in result_files]) + + def _revision_to_build_map(self): + if not self._revision_to_build_number: + self._revision_to_build_number = self._fetch_revision_to_build_map() + return self._revision_to_build_number + + def revision_build_pairs_with_results(self): + return self._revision_to_build_map().items() + + # This assumes there can be only one build per revision, which is false, but we don't care for now. + def build_for_revision(self, revision, allow_failed_lookups=False): + # NOTE: This lookup will fail if that exact revision was never built. + build_number = self._revision_to_build_map().get(int(revision)) + if not build_number: + return None + build = self.build(build_number) + if not build and allow_failed_lookups: + # Builds for old revisions with fail to lookup via buildbot's xmlrpc api. + build = Build(self, + build_number=build_number, + revision=revision, + is_green=False, + ) + return build + + def find_failure_transition(self, red_build, look_back_limit=30): + if not red_build or red_build.is_green(): + return (None, None) + common_failures = None + current_build = red_build + build_after_current_build = None + look_back_count = 0 + while current_build: + if current_build.is_green(): + # current_build can't possibly have any failures in common + # with red_build because it's green. + break + results = current_build.layout_test_results() + # We treat a lack of results as if all the test failed. + # This occurs, for example, when we can't compile at all. + if results: + failures = set(results.failing_tests()) + if common_failures == None: + common_failures = failures + common_failures = common_failures.intersection(failures) + if not common_failures: + # current_build doesn't have any failures in common with + # the red build we're worried about. We assume that any + # failures in current_build were due to flakiness. + break + look_back_count += 1 + if look_back_count > look_back_limit: + return (None, current_build) + build_after_current_build = current_build + current_build = current_build.previous_build() + # We must iterate at least once because red_build is red. + assert(build_after_current_build) + # Current build must either be green or have no failures in common + # with red build, so we've found our failure transition. + return (current_build, build_after_current_build) + + # FIXME: This likely does not belong on Builder + def suspect_revisions_for_transition(self, last_good_build, first_bad_build): + suspect_revisions = range(first_bad_build.revision(), + last_good_build.revision(), + -1) + suspect_revisions.reverse() + return suspect_revisions + + def blameworthy_revisions(self, red_build_number, look_back_limit=30, avoid_flakey_tests=True): + red_build = self.build(red_build_number) + (last_good_build, first_bad_build) = \ + self.find_failure_transition(red_build, look_back_limit) + if not last_good_build: + return [] # We ran off the limit of our search + # If avoid_flakey_tests, require at least 2 bad builds before we + # suspect a real failure transition. + if avoid_flakey_tests and first_bad_build == red_build: + return [] + return self.suspect_revisions_for_transition(last_good_build, first_bad_build) + + +# FIXME: This should be unified with all the layout test results code in the layout_tests package +class LayoutTestResults(object): + stderr_key = u'Tests that had stderr output:' + fail_key = u'Tests where results did not match expected results:' + timeout_key = u'Tests that timed out:' + crash_key = u'Tests that caused the DumpRenderTree tool to crash:' + missing_key = u'Tests that had no expected results (probably new):' + + expected_keys = [ + stderr_key, + fail_key, + crash_key, + timeout_key, + missing_key, + ] + + @classmethod + def _parse_results_html(cls, page): + parsed_results = {} + tables = BeautifulSoup(page).findAll("table") + for table in tables: + table_title = table.findPreviousSibling("p").string + if table_title not in cls.expected_keys: + # This Exception should only ever be hit if run-webkit-tests changes its results.html format. + raise Exception("Unhandled title: %s" % str(table_title)) + # We might want to translate table titles into identifiers before storing. + parsed_results[table_title] = [row.find("a").string for row in table.findAll("tr")] + + return parsed_results + + @classmethod + def _fetch_results_html(cls, base_url): + results_html = "%s/results.html" % base_url + # FIXME: We need to move this sort of 404 logic into NetworkTransaction or similar. + try: + page = urllib2.urlopen(results_html) + return cls._parse_results_html(page) + except urllib2.HTTPError, error: + if error.code != 404: + raise + + @classmethod + def results_from_url(cls, base_url): + parsed_results = cls._fetch_results_html(base_url) + if not parsed_results: + return None + return cls(base_url, parsed_results) + + def __init__(self, base_url, parsed_results): + self._base_url = base_url + self._parsed_results = parsed_results + + def parsed_results(self): + return self._parsed_results + + def failing_tests(self): + failing_keys = [self.fail_key, self.crash_key, self.timeout_key] + return sorted(sum([tests for key, tests in self._parsed_results.items() if key in failing_keys], [])) + + +class Build(object): + def __init__(self, builder, build_number, revision, is_green): + self._builder = builder + self._number = build_number + self._revision = revision + self._is_green = is_green + self._layout_test_results = None + + @staticmethod + def build_url(builder, build_number): + return "%s/builds/%s" % (builder.url(), build_number) + + def url(self): + return self.build_url(self.builder(), self._number) + + def results_url(self): + results_directory = "r%s (%s)" % (self.revision(), self._number) + return "%s/%s" % (self._builder.results_url(), urllib.quote(results_directory)) + + def layout_test_results(self): + if not self._layout_test_results: + self._layout_test_results = LayoutTestResults.results_from_url(self.results_url()) + return self._layout_test_results + + def builder(self): + return self._builder + + def revision(self): + return self._revision + + def is_green(self): + return self._is_green + + def previous_build(self): + # previous_build() allows callers to avoid assuming build numbers are sequential. + # They may not be sequential across all master changes, or when non-trunk builds are made. + return self._builder.build(self._number - 1) + + +class BuildBot(object): + # FIXME: This should move into some sort of webkit_config.py + default_host = "build.webkit.org" + + def __init__(self, host=default_host): + self.buildbot_host = host + self._builder_by_name = {} + + # If any core builder is red we should not be landing patches. Other + # builders should be added to this list once they are known to be + # reliable. + # See https://bugs.webkit.org/show_bug.cgi?id=33296 and related bugs. + self.core_builder_names_regexps = [ + "SnowLeopard.*Build", + "SnowLeopard.*Test", + "Leopard", + "Tiger", + "Windows.*Build", + "Windows.*Debug.*Test", + "GTK", + "Qt", + "Chromium", + ] + + def _parse_last_build_cell(self, builder, cell): + status_link = cell.find('a') + if status_link: + # Will be either a revision number or a build number + revision_string = status_link.string + # If revision_string has non-digits assume it's not a revision number. + builder['built_revision'] = int(revision_string) \ + if not re.match('\D', revision_string) \ + else None + builder['is_green'] = not re.search('fail', cell.renderContents()) + + status_link_regexp = r"builders/(?P<builder_name>.*)/builds/(?P<build_number>\d+)" + link_match = re.match(status_link_regexp, status_link['href']) + builder['build_number'] = int(link_match.group("build_number")) + else: + # We failed to find a link in the first cell, just give up. This + # can happen if a builder is just-added, the first cell will just + # be "no build" + # Other parts of the code depend on is_green being present. + builder['is_green'] = False + builder['built_revision'] = None + builder['build_number'] = None + + def _parse_current_build_cell(self, builder, cell): + activity_lines = cell.renderContents().split("<br />") + builder["activity"] = activity_lines[0] # normally "building" or "idle" + # The middle lines document how long left for any current builds. + match = re.match("(?P<pending_builds>\d) pending", activity_lines[-1]) + builder["pending_builds"] = int(match.group("pending_builds")) if match else 0 + + def _parse_builder_status_from_row(self, status_row): + status_cells = status_row.findAll('td') + builder = {} + + # First cell is the name + name_link = status_cells[0].find('a') + builder["name"] = name_link.string + + self._parse_last_build_cell(builder, status_cells[1]) + self._parse_current_build_cell(builder, status_cells[2]) + return builder + + def _matches_regexps(self, builder_name, name_regexps): + for name_regexp in name_regexps: + if re.match(name_regexp, builder_name): + return True + return False + + # FIXME: Should move onto Builder + def _is_core_builder(self, builder_name): + return self._matches_regexps(builder_name, self.core_builder_names_regexps) + + # FIXME: This method needs to die, but is used by a unit test at the moment. + def _builder_statuses_with_names_matching_regexps(self, builder_statuses, name_regexps): + return [builder for builder in builder_statuses if self._matches_regexps(builder["name"], name_regexps)] + + def red_core_builders(self): + return [builder for builder in self.core_builder_statuses() if not builder["is_green"]] + + def red_core_builders_names(self): + return [builder["name"] for builder in self.red_core_builders()] + + def idle_red_core_builders(self): + return [builder for builder in self.red_core_builders() if builder["activity"] == "idle"] + + def core_builders_are_green(self): + return not self.red_core_builders() + + # FIXME: These _fetch methods should move to a networking class. + def _fetch_xmlrpc_build_dictionary(self, builder, build_number): + # The buildbot XMLRPC API is super-limited. + # For one, you cannot fetch info on builds which are incomplete. + proxy = xmlrpclib.ServerProxy("http://%s/xmlrpc" % self.buildbot_host, allow_none=True) + try: + return proxy.getBuild(builder.name(), int(build_number)) + except xmlrpclib.Fault, err: + build_url = Build.build_url(builder, build_number) + _log.error("Error fetching data for %s build %s (%s): %s" % (builder.name(), build_number, build_url, err)) + return None + + def _fetch_one_box_per_builder(self): + build_status_url = "http://%s/one_box_per_builder" % self.buildbot_host + return urllib2.urlopen(build_status_url) + + def _parse_twisted_file_row(self, file_row): + string_or_empty = lambda string: str(string) if string else "" + file_cells = file_row.findAll('td') + return { + "filename" : string_or_empty(file_cells[0].find("a").string), + "size" : string_or_empty(file_cells[1].string), + "type" : string_or_empty(file_cells[2].string), + "encoding" : string_or_empty(file_cells[3].string), + } + + def _parse_twisted_directory_listing(self, page): + soup = BeautifulSoup(page) + # HACK: Match only table rows with a class to ignore twisted header/footer rows. + file_rows = soup.find('table').findAll('tr', { "class" : True }) + return [self._parse_twisted_file_row(file_row) for file_row in file_rows] + + # FIXME: There should be a better way to get this information directly from twisted. + def _fetch_twisted_directory_listing(self, url): + return self._parse_twisted_directory_listing(urllib2.urlopen(url)) + + def builders(self): + return [self.builder_with_name(status["name"]) for status in self.builder_statuses()] + + # This method pulls from /one_box_per_builder as an efficient way to get information about + def builder_statuses(self): + soup = BeautifulSoup(self._fetch_one_box_per_builder()) + return [self._parse_builder_status_from_row(status_row) for status_row in soup.find('table').findAll('tr')] + + def core_builder_statuses(self): + return [builder for builder in self.builder_statuses() if self._is_core_builder(builder["name"])] + + def builder_with_name(self, name): + builder = self._builder_by_name.get(name) + if not builder: + builder = Builder(name, self) + self._builder_by_name[name] = builder + return builder + + def revisions_causing_failures(self, only_core_builders=True): + builder_statuses = self.core_builder_statuses() if only_core_builders else self.builder_statuses() + revision_to_failing_bots = {} + for builder_status in builder_statuses: + if builder_status["is_green"]: + continue + builder = self.builder_with_name(builder_status["name"]) + revisions = builder.blameworthy_revisions(builder_status["build_number"]) + for revision in revisions: + failing_bots = revision_to_failing_bots.get(revision, []) + failing_bots.append(builder) + revision_to_failing_bots[revision] = failing_bots + return revision_to_failing_bots + + # This makes fewer requests than calling Builder.latest_build would. It grabs all builder + # statuses in one request using self.builder_statuses (fetching /one_box_per_builder instead of builder pages). + def _latest_builds_from_builders(self, only_core_builders=True): + builder_statuses = self.core_builder_statuses() if only_core_builders else self.builder_statuses() + return [self.builder_with_name(status["name"]).build(status["build_number"]) for status in builder_statuses] + + def _build_at_or_before_revision(self, build, revision): + while build: + if build.revision() <= revision: + return build + build = build.previous_build() + + def last_green_revision(self, only_core_builders=True): + builds = self._latest_builds_from_builders(only_core_builders) + target_revision = builds[0].revision() + # An alternate way to do this would be to start at one revision and walk backwards + # checking builder.build_for_revision, however build_for_revision is very slow on first load. + while True: + # Make builds agree on revision + builds = [self._build_at_or_before_revision(build, target_revision) for build in builds] + if None in builds: # One of the builds failed to load from the server. + return None + min_revision = min(map(lambda build: build.revision(), builds)) + if min_revision != target_revision: + target_revision = min_revision + continue # Builds don't all agree on revision, keep searching + # Check to make sure they're all green + all_are_green = reduce(operator.and_, map(lambda build: build.is_green(), builds)) + if not all_are_green: + target_revision -= 1 + continue + return min_revision diff --git a/WebKitTools/Scripts/webkitpy/common/net/buildbot_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/buildbot_unittest.py new file mode 100644 index 0000000..f765f6e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/buildbot_unittest.py @@ -0,0 +1,433 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.net.buildbot import BuildBot, Builder, Build, LayoutTestResults + +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup + + +class BuilderTest(unittest.TestCase): + def _install_fetch_build(self, failure): + def _mock_fetch_build(build_number): + build = Build( + builder=self.builder, + build_number=build_number, + revision=build_number + 1000, + is_green=build_number < 4 + ) + build._layout_test_results = LayoutTestResults( + "http://buildbot.example.com/foo", { + LayoutTestResults.fail_key: failure(build_number), + }) + return build + self.builder._fetch_build = _mock_fetch_build + + def setUp(self): + self.buildbot = BuildBot() + self.builder = Builder("Test Builder", self.buildbot) + self._install_fetch_build(lambda build_number: ["test1", "test2"]) + + def test_find_failure_transition(self): + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1003) + self.assertEqual(red_build.revision(), 1004) + + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10), look_back_limit=2) + self.assertEqual(green_build, None) + self.assertEqual(red_build.revision(), 1008) + + def test_none_build(self): + self.builder._fetch_build = lambda build_number: None + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build, None) + self.assertEqual(red_build, None) + + def test_flaky_tests(self): + self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1009) + self.assertEqual(red_build.revision(), 1010) + + def test_failure_and_flaky(self): + self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1003) + self.assertEqual(red_build.revision(), 1004) + + def test_no_results(self): + self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1003) + self.assertEqual(red_build.revision(), 1004) + + def test_failure_after_flaky(self): + self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1006) + self.assertEqual(red_build.revision(), 1007) + + def test_blameworthy_revisions(self): + self.assertEqual(self.builder.blameworthy_revisions(10), [1004]) + self.assertEqual(self.builder.blameworthy_revisions(10, look_back_limit=2), []) + # Flakey test avoidance requires at least 2 red builds: + self.assertEqual(self.builder.blameworthy_revisions(4), []) + self.assertEqual(self.builder.blameworthy_revisions(4, avoid_flakey_tests=False), [1004]) + # Green builder: + self.assertEqual(self.builder.blameworthy_revisions(3), []) + + def test_build_caching(self): + self.assertEqual(self.builder.build(10), self.builder.build(10)) + + def test_build_and_revision_for_filename(self): + expectations = { + "r47483 (1)/" : (47483, 1), + "r47483 (1).zip" : (47483, 1), + } + for filename, revision_and_build in expectations.items(): + self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build) + + +class LayoutTestResultsTest(unittest.TestCase): + _example_results_html = """ +<html> +<head> +<title>Layout Test Results</title> +</head> +<body> +<p>Tests that had stderr output:</p> +<table> +<tr> +<td><a href="/var/lib/buildbot/build/gtk-linux-64-release/build/LayoutTests/accessibility/aria-activedescendant-crash.html">accessibility/aria-activedescendant-crash.html</a></td> +<td><a href="accessibility/aria-activedescendant-crash-stderr.txt">stderr</a></td> +</tr> +<td><a href="/var/lib/buildbot/build/gtk-linux-64-release/build/LayoutTests/http/tests/security/canvas-remote-read-svg-image.html">http/tests/security/canvas-remote-read-svg-image.html</a></td> +<td><a href="http/tests/security/canvas-remote-read-svg-image-stderr.txt">stderr</a></td> +</tr> +</table><p>Tests that had no expected results (probably new):</p> +<table> +<tr> +<td><a href="/var/lib/buildbot/build/gtk-linux-64-release/build/LayoutTests/fast/repaint/no-caret-repaint-in-non-content-editable-element.html">fast/repaint/no-caret-repaint-in-non-content-editable-element.html</a></td> +<td><a href="fast/repaint/no-caret-repaint-in-non-content-editable-element-actual.txt">result</a></td> +</tr> +</table></body> +</html> +""" + + _expected_layout_test_results = { + 'Tests that had stderr output:' : [ + 'accessibility/aria-activedescendant-crash.html' + ], + 'Tests that had no expected results (probably new):' : [ + 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html' + ] + } + def test_parse_layout_test_results(self): + results = LayoutTestResults._parse_results_html(self._example_results_html) + self.assertEqual(self._expected_layout_test_results, results) + + +class BuildBotTest(unittest.TestCase): + + _example_one_box_status = ''' + <table> + <tr> + <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td> + <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td> + <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td> + <tr> + <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td> + <td class="LastBuild box" >no build</td> + <td align="center" class="Activity building">building<br />< 1 min</td> + <tr> + <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td> + <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td> + <td align="center" class="Activity idle">idle<br />3 pending</td> + </table> +''' + _expected_example_one_box_parsings = [ + { + 'is_green': True, + 'build_number' : 3693, + 'name': u'Windows Debug (Tests)', + 'built_revision': 47380, + 'activity': 'building', + 'pending_builds': 0, + }, + { + 'is_green': False, + 'build_number' : None, + 'name': u'SnowLeopard Intel Release', + 'built_revision': None, + 'activity': 'building', + 'pending_builds': 0, + }, + { + 'is_green': False, + 'build_number' : 654, + 'name': u'Qt Linux Release', + 'built_revision': 47383, + 'activity': 'idle', + 'pending_builds': 3, + }, + ] + + def test_status_parsing(self): + buildbot = BuildBot() + + soup = BeautifulSoup(self._example_one_box_status) + status_table = soup.find("table") + input_rows = status_table.findAll('tr') + + for x in range(len(input_rows)): + status_row = input_rows[x] + expected_parsing = self._expected_example_one_box_parsings[x] + + builder = buildbot._parse_builder_status_from_row(status_row) + + # Make sure we aren't parsing more or less than we expect + self.assertEquals(builder.keys(), expected_parsing.keys()) + + for key, expected_value in expected_parsing.items(): + self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value))) + + def test_core_builder_methods(self): + buildbot = BuildBot() + + # Override builder_statuses function to not touch the network. + def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to. + return BuildBotTest._expected_example_one_box_parsings + buildbot.builder_statuses = example_builder_statuses + + buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ] + self.assertEquals(buildbot.red_core_builders_names(), []) + self.assertTrue(buildbot.core_builders_are_green()) + + buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ] + self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ]) + self.assertFalse(buildbot.core_builders_are_green()) + + def test_builder_name_regexps(self): + buildbot = BuildBot() + + # For complete testing, this list should match the list of builders at build.webkit.org: + example_builders = [ + {'name': u'Tiger Intel Release', }, + {'name': u'Leopard Intel Release (Build)', }, + {'name': u'Leopard Intel Release (Tests)', }, + {'name': u'Leopard Intel Debug (Build)', }, + {'name': u'Leopard Intel Debug (Tests)', }, + {'name': u'SnowLeopard Intel Release (Build)', }, + {'name': u'SnowLeopard Intel Release (Tests)', }, + {'name': u'SnowLeopard Intel Leaks', }, + {'name': u'Windows Release (Build)', }, + {'name': u'Windows Release (Tests)', }, + {'name': u'Windows Debug (Build)', }, + {'name': u'Windows Debug (Tests)', }, + {'name': u'GTK Linux 32-bit Release', }, + {'name': u'GTK Linux 32-bit Debug', }, + {'name': u'GTK Linux 64-bit Debug', }, + {'name': u'GTK Linux 64-bit Release', }, + {'name': u'Qt Linux Release', }, + {'name': u'Qt Linux Release minimal', }, + {'name': u'Qt Linux ARMv5 Release', }, + {'name': u'Qt Linux ARMv7 Release', }, + {'name': u'Qt Windows 32-bit Release', }, + {'name': u'Qt Windows 32-bit Debug', }, + {'name': u'Chromium Linux Release', }, + {'name': u'Chromium Mac Release', }, + {'name': u'Chromium Win Release', }, + {'name': u'New run-webkit-tests', }, + ] + name_regexps = [ + "SnowLeopard.*Build", + "SnowLeopard.*Test", + "Leopard", + "Tiger", + "Windows.*Build", + "Windows.*Debug.*Test", + "GTK", + "Qt", + "Chromium", + ] + expected_builders = [ + {'name': u'Tiger Intel Release', }, + {'name': u'Leopard Intel Release (Build)', }, + {'name': u'Leopard Intel Release (Tests)', }, + {'name': u'Leopard Intel Debug (Build)', }, + {'name': u'Leopard Intel Debug (Tests)', }, + {'name': u'SnowLeopard Intel Release (Build)', }, + {'name': u'SnowLeopard Intel Release (Tests)', }, + {'name': u'Windows Release (Build)', }, + {'name': u'Windows Debug (Build)', }, + {'name': u'Windows Debug (Tests)', }, + {'name': u'GTK Linux 32-bit Release', }, + {'name': u'GTK Linux 32-bit Debug', }, + {'name': u'GTK Linux 64-bit Debug', }, + {'name': u'GTK Linux 64-bit Release', }, + {'name': u'Qt Linux Release', }, + {'name': u'Qt Linux Release minimal', }, + {'name': u'Qt Linux ARMv5 Release', }, + {'name': u'Qt Linux ARMv7 Release', }, + {'name': u'Qt Windows 32-bit Release', }, + {'name': u'Qt Windows 32-bit Debug', }, + {'name': u'Chromium Linux Release', }, + {'name': u'Chromium Mac Release', }, + {'name': u'Chromium Win Release', }, + ] + + # This test should probably be updated if the default regexp list changes + self.assertEquals(buildbot.core_builder_names_regexps, name_regexps) + + builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps) + self.assertEquals(builders, expected_builders) + + def test_builder_with_name(self): + buildbot = BuildBot() + + builder = buildbot.builder_with_name("Test Builder") + self.assertEqual(builder.name(), "Test Builder") + self.assertEqual(builder.url(), "http://build.webkit.org/builders/Test%20Builder") + self.assertEqual(builder.url_encoded_name(), "Test%20Builder") + self.assertEqual(builder.results_url(), "http://build.webkit.org/results/Test%20Builder") + + # Override _fetch_xmlrpc_build_dictionary function to not touch the network. + def mock_fetch_xmlrpc_build_dictionary(self, build_number): + build_dictionary = { + "revision" : 2 * build_number, + "number" : int(build_number), + "results" : build_number % 2, # 0 means pass + } + return build_dictionary + buildbot._fetch_xmlrpc_build_dictionary = mock_fetch_xmlrpc_build_dictionary + + build = builder.build(10) + self.assertEqual(build.builder(), builder) + self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/10") + self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r20%20%2810%29") + self.assertEqual(build.revision(), 20) + self.assertEqual(build.is_green(), True) + + build = build.previous_build() + self.assertEqual(build.builder(), builder) + self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/9") + self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r18%20%289%29") + self.assertEqual(build.revision(), 18) + self.assertEqual(build.is_green(), False) + + self.assertEqual(builder.build(None), None) + + _example_directory_listing = ''' +<h1>Directory listing for /results/SnowLeopard Intel Leaks/</h1> + +<table> + <thead> + <tr> + <th>Filename</th> + <th>Size</th> + <th>Content type</th> + <th>Content encoding</th> + </tr> + </thead> + <tbody> +<tr class="odd"> + <td><a href="r47483%20%281%29/">r47483 (1)/</a></td> + <td></td> + <td>[Directory]</td> + <td></td> +</tr> +<tr class="odd"> + <td><a href="r47484%20%282%29.zip">r47484 (2).zip</a></td> + <td>89K</td> + <td>[application/zip]</td> + <td></td> +</tr> +''' + _expected_files = [ + { + "filename" : "r47483 (1)/", + "size" : "", + "type" : "[Directory]", + "encoding" : "", + }, + { + "filename" : "r47484 (2).zip", + "size" : "89K", + "type" : "[application/zip]", + "encoding" : "", + }, + ] + + def test_parse_build_to_revision_map(self): + buildbot = BuildBot() + files = buildbot._parse_twisted_directory_listing(self._example_directory_listing) + self.assertEqual(self._expected_files, files) + + # Revision, is_green + # Ordered from newest (highest number) to oldest. + fake_builder1 = [ + [2, False], + [1, True], + ] + fake_builder2 = [ + [2, False], + [1, True], + ] + fake_builders = [ + fake_builder1, + fake_builder2, + ] + def _build_from_fake(self, fake_builder, index): + if index >= len(fake_builder): + return None + fake_build = fake_builder[index] + build = Build( + builder=fake_builder, + build_number=index, + revision=fake_build[0], + is_green=fake_build[1], + ) + def mock_previous_build(): + return self._build_from_fake(fake_builder, index + 1) + build.previous_build = mock_previous_build + return build + + def _fake_builds_at_index(self, index): + return [self._build_from_fake(builder, index) for builder in self.fake_builders] + + def test_last_green_revision(self): + buildbot = BuildBot() + def mock_builds_from_builders(only_core_builders): + return self._fake_builds_at_index(0) + buildbot._latest_builds_from_builders = mock_builds_from_builders + self.assertEqual(buildbot.last_green_revision(), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/credentials.py b/WebKitTools/Scripts/webkitpy/common/net/credentials.py index 295c576..1d5f83d 100644 --- a/WebKitTools/Scripts/webkitpy/credentials.py +++ b/WebKitTools/Scripts/webkitpy/common/net/credentials.py @@ -34,30 +34,23 @@ import os import platform import re -from webkitpy.executive import Executive, ScriptError -from webkitpy.webkit_logging import log -from webkitpy.scm import Git -from webkitpy.user import User +from webkitpy.common.checkout.scm import Git +from webkitpy.common.system.executive import Executive, ScriptError +from webkitpy.common.system.user import User +from webkitpy.common.system.deprecated_logging import log class Credentials(object): def __init__(self, host, git_prefix=None, executive=None, cwd=os.getcwd()): self.host = host - self.git_prefix = git_prefix + self.git_prefix = "%s." % git_prefix if git_prefix else "" self.executive = executive or Executive() self.cwd = cwd def _credentials_from_git(self): - return [self._read_git_config("username"), - self._read_git_config("password")] - - def _read_git_config(self, key): - config_key = "%s.%s" % (self.git_prefix, key) if self.git_prefix \ - else key - return self.executive.run_command( - ["git", "config", "--get", config_key], - error_handler=Executive.ignore_error).rstrip('\n') + return [Git.read_git_config(self.git_prefix + "username"), + Git.read_git_config(self.git_prefix + "password")] def _keychain_value_with_label(self, label, source_text): match = re.search("%s\"(?P<value>.+)\"" % label, diff --git a/WebKitTools/Scripts/webkitpy/credentials_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/credentials_unittest.py index 0bd5340..9a42bdd 100644 --- a/WebKitTools/Scripts/webkitpy/credentials_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/net/credentials_unittest.py @@ -29,10 +29,10 @@ import os import tempfile import unittest -from webkitpy.credentials import Credentials -from webkitpy.executive import Executive -from webkitpy.outputcapture import OutputCapture -from webkitpy.mock import Mock +from webkitpy.common.net.credentials import Credentials +from webkitpy.common.system.executive import Executive +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock class CredentialsTest(unittest.TestCase): example_security_output = """keychain: "/Users/test/Library/Keychains/login.keychain" @@ -101,16 +101,6 @@ password: "SECRETSAUCE" self._assert_security_call() self._assert_security_call(username="foo") - def test_git_config_calls(self): - executive_mock = Mock() - credentials = Credentials("example.com", executive=executive_mock) - credentials._read_git_config("foo") - executive_mock.run_command.assert_called_with(["git", "config", "--get", "foo"], error_handler=Executive.ignore_error) - - credentials = Credentials("example.com", git_prefix="test_prefix", executive=executive_mock) - credentials._read_git_config("foo") - executive_mock.run_command.assert_called_with(["git", "config", "--get", "test_prefix.foo"], error_handler=Executive.ignore_error) - def test_read_credentials_without_git_repo(self): class FakeCredentials(Credentials): def _is_mac_os_x(self): diff --git a/WebKitTools/Scripts/webkitpy/common/net/irc/__init__.py b/WebKitTools/Scripts/webkitpy/common/net/irc/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/irc/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/common/net/irc/ircbot.py b/WebKitTools/Scripts/webkitpy/common/net/irc/ircbot.py new file mode 100644 index 0000000..f742867 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/irc/ircbot.py @@ -0,0 +1,91 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import webkitpy.common.config.irc as config_irc + +from webkitpy.common.thread.messagepump import MessagePump, MessagePumpDelegate +from webkitpy.thirdparty.autoinstalled.irc import ircbot +from webkitpy.thirdparty.autoinstalled.irc import irclib + + +class IRCBotDelegate(object): + def irc_message_received(self, nick, message): + raise NotImplementedError, "subclasses must implement" + + def irc_nickname(self): + raise NotImplementedError, "subclasses must implement" + + def irc_password(self): + raise NotImplementedError, "subclasses must implement" + + +class IRCBot(ircbot.SingleServerIRCBot, MessagePumpDelegate): + # FIXME: We should get this information from a config file. + def __init__(self, + message_queue, + delegate): + self._message_queue = message_queue + self._delegate = delegate + ircbot.SingleServerIRCBot.__init__( + self, + [( + config_irc.server, + config_irc.port, + self._delegate.irc_password() + )], + self._delegate.irc_nickname(), + self._delegate.irc_nickname()) + self._channel = config_irc.channel + + # ircbot.SingleServerIRCBot methods + + def on_nicknameinuse(self, connection, event): + connection.nick(connection.get_nickname() + "_") + + def on_welcome(self, connection, event): + connection.join(self._channel) + self._message_pump = MessagePump(self, self._message_queue) + + def on_pubmsg(self, connection, event): + nick = irclib.nm_to_n(event.source()) + request = event.arguments()[0].split(":", 1) + if len(request) > 1 and irclib.irc_lower(request[0]) == irclib.irc_lower(self.connection.get_nickname()): + response = self._delegate.irc_message_received(nick, request[1]) + if response: + connection.privmsg(self._channel, response) + + # MessagePumpDelegate methods + + def schedule(self, interval, callback): + self.connection.execute_delayed(interval, callback) + + def message_available(self, message): + self.connection.privmsg(self._channel, message) + + def final_message_delivered(self): + self.die() diff --git a/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy.py b/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy.py new file mode 100644 index 0000000..13348b4 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy.py @@ -0,0 +1,62 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import threading + +from webkitpy.common.net.irc.ircbot import IRCBot +from webkitpy.common.thread.threadedmessagequeue import ThreadedMessageQueue +from webkitpy.common.system.deprecated_logging import log + + +class _IRCThread(threading.Thread): + def __init__(self, message_queue, irc_delegate, irc_bot): + threading.Thread.__init__(self) + self.setDaemon(True) + self._message_queue = message_queue + self._irc_delegate = irc_delegate + self._irc_bot = irc_bot + + def run(self): + bot = self._irc_bot(self._message_queue, self._irc_delegate) + bot.start() + + +class IRCProxy(object): + def __init__(self, irc_delegate, irc_bot=IRCBot): + log("Connecting to IRC") + self._message_queue = ThreadedMessageQueue() + self._child_thread = _IRCThread(self._message_queue, irc_delegate, irc_bot) + self._child_thread.start() + + def post(self, message): + self._message_queue.post(message) + + def disconnect(self): + log("Disconnecting from IRC...") + self._message_queue.stop() + self._child_thread.join() diff --git a/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy_unittest.py new file mode 100644 index 0000000..b44ce40 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy_unittest.py @@ -0,0 +1,43 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.net.irc.ircproxy import IRCProxy +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock + +class IRCProxyTest(unittest.TestCase): + def test_trivial(self): + def fun(): + proxy = IRCProxy(Mock(), Mock()) + proxy.post("hello") + proxy.disconnect() + + expected_stderr = "Connecting to IRC\nDisconnecting from IRC...\n" + OutputCapture().assert_outputs(self, fun, expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/networktransaction.py b/WebKitTools/Scripts/webkitpy/common/net/networktransaction.py index 65ea27d..c82fc6f 100644 --- a/WebKitTools/Scripts/webkitpy/networktransaction.py +++ b/WebKitTools/Scripts/webkitpy/common/net/networktransaction.py @@ -26,10 +26,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging import time -from mechanize import HTTPError -from webkitpy.webkit_logging import log +from webkitpy.thirdparty.autoinstalled.mechanize import HTTPError +from webkitpy.common.system.deprecated_logging import log + + +_log = logging.getLogger(__name__) class NetworkTimeout(Exception): @@ -37,7 +41,7 @@ class NetworkTimeout(Exception): class NetworkTransaction(object): - def __init__(self, initial_backoff_seconds=10, grown_factor=1.1, timeout_seconds=5*60*60): + def __init__(self, initial_backoff_seconds=10, grown_factor=1.5, timeout_seconds=10*60): self._initial_backoff_seconds = initial_backoff_seconds self._grown_factor = grown_factor self._timeout_seconds = timeout_seconds @@ -50,7 +54,8 @@ class NetworkTransaction(object): return request() except HTTPError, e: self._check_for_timeout() - log("Received HTTP status %s from server. Retrying in %s seconds..." % (e.code, self._backoff_seconds)) + _log.warn("Received HTTP status %s from server. Retrying in " + "%s seconds..." % (e.code, self._backoff_seconds)) self._sleep() def _check_for_timeout(self): diff --git a/WebKitTools/Scripts/webkitpy/networktransaction_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/networktransaction_unittest.py index 3cffe02..cd0702b 100644 --- a/WebKitTools/Scripts/webkitpy/networktransaction_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/net/networktransaction_unittest.py @@ -28,10 +28,12 @@ import unittest -from mechanize import HTTPError -from webkitpy.networktransaction import NetworkTransaction, NetworkTimeout +from webkitpy.common.net.networktransaction import NetworkTransaction, NetworkTimeout +from webkitpy.common.system.logtesting import LoggingTestCase +from webkitpy.thirdparty.autoinstalled.mechanize import HTTPError -class NetworkTransactionTest(unittest.TestCase): + +class NetworkTransactionTest(LoggingTestCase): exception = Exception("Test exception") def test_success(self): @@ -65,6 +67,10 @@ class NetworkTransactionTest(unittest.TestCase): transaction = NetworkTransaction(initial_backoff_seconds=0) self.assertEqual(transaction.run(lambda: self._raise_http_error()), 42) self.assertEqual(self._run_count, 3) + self.assertLog(['WARNING: Received HTTP status 500 from server. ' + 'Retrying in 0 seconds...\n', + 'WARNING: Received HTTP status 500 from server. ' + 'Retrying in 0.0 seconds...\n']) def test_timeout(self): self._run_count = 0 diff --git a/WebKitTools/Scripts/webkitpy/common/net/rietveld.py b/WebKitTools/Scripts/webkitpy/common/net/rietveld.py new file mode 100644 index 0000000..a9e5b1a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/rietveld.py @@ -0,0 +1,89 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import re +import stat + +import webkitpy.common.config as config +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.system.executive import ScriptError +import webkitpy.thirdparty.autoinstalled.rietveld.upload as upload + + +def parse_codereview_issue(message): + if not message: + return None + match = re.search(config.codereview_server_regex + + "(?P<codereview_issue>\d+)", + message) + if match: + return int(match.group('codereview_issue')) + + +class Rietveld(object): + def __init__(self, executive, dryrun=False): + self.dryrun = dryrun + self._executive = executive + self._upload_py = upload.__file__ + # Chop off the last character so we modify permissions on the py file instead of the pyc. + if os.path.splitext(self._upload_py)[1] == ".pyc": + self._upload_py = self._upload_py[:-1] + os.chmod(self._upload_py, os.stat(self._upload_py).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + def url_for_issue(self, codereview_issue): + if not codereview_issue: + return None + return "%s%s" % (config.codereview_server_url, codereview_issue) + + def post(self, message=None, codereview_issue=None, cc=None): + if not message: + raise ScriptError("Rietveld requires a message.") + + args = [ + self._upload_py, + "--assume_yes", + "--server=%s" % config.codereview_server_host, + "--message=%s" % message, + ] + if codereview_issue: + args.append("--issue=%s" % codereview_issue) + if cc: + args.append("--cc=%s" % cc) + + if self.dryrun: + log("Would have run %s" % args) + return + + output = self._executive.run_and_throw_if_fail(args) + match = re.search("Issue created\. URL: " + + config.codereview_server_regex + + "(?P<codereview_issue>\d+)", + output) + if match: + return int(match.group('codereview_issue')) diff --git a/WebKitTools/Scripts/webkitpy/common/net/rietveld_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/rietveld_unittest.py new file mode 100644 index 0000000..9c5a29e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/rietveld_unittest.py @@ -0,0 +1,39 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.net.rietveld import Rietveld +from webkitpy.thirdparty.mock import Mock + + +class RietveldTest(unittest.TestCase): + def test_url_for_issue(self): + rietveld = Rietveld(Mock()) + self.assertEqual(rietveld.url_for_issue(34223), + "https://wkrietveld.appspot.com/34223") diff --git a/WebKitTools/Scripts/webkitpy/statusserver.py b/WebKitTools/Scripts/webkitpy/common/net/statusserver.py index ff0ddfa..e8987a9 100644 --- a/WebKitTools/Scripts/webkitpy/statusserver.py +++ b/WebKitTools/Scripts/webkitpy/common/net/statusserver.py @@ -26,13 +26,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.networktransaction import NetworkTransaction -from webkitpy.webkit_logging import log -from mechanize import Browser - -# WebKit includes a built copy of BeautifulSoup in Scripts/webkitpy -# so this import should always succeed. -from .BeautifulSoup import BeautifulSoup +from webkitpy.common.net.networktransaction import NetworkTransaction +from webkitpy.common.system.deprecated_logging import log +from webkitpy.thirdparty.autoinstalled.mechanize import Browser +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup import urllib2 @@ -64,7 +61,7 @@ class StatusServer: return self.browser.add_file(results_file, "text/plain", "results.txt", 'results_file') - def _post_to_server(self, queue_name, status, patch, results_file): + def _post_status_to_server(self, queue_name, status, patch, results_file): if results_file: # We might need to re-wind the file if we've already tried to post it. results_file.seek(0) @@ -72,25 +69,40 @@ class StatusServer: update_status_url = "%s/update-status" % self.url self.browser.open(update_status_url) self.browser.select_form(name="update_status") - self.browser['queue_name'] = queue_name + self.browser["queue_name"] = queue_name self._add_patch(patch) - self.browser['status'] = status + self.browser["status"] = status self._add_results_file(results_file) return self.browser.submit().read() # This is the id of the newly created status object. - def update_status(self, queue_name, status, patch=None, results_file=None): - # During unit testing, host is None - if not self.host: - return + def _post_svn_revision_to_server(self, svn_revision_number, broken_bot): + update_svn_revision_url = "%s/update-svn-revision" % self.url + self.browser.open(update_svn_revision_url) + self.browser.select_form(name="update_svn_revision") + self.browser["number"] = str(svn_revision_number) + self.browser["broken_bot"] = broken_bot + return self.browser.submit().read() + def update_status(self, queue_name, status, patch=None, results_file=None): log(status) - return NetworkTransaction().run(lambda: self._post_to_server(queue_name, status, patch, results_file)) + return NetworkTransaction().run(lambda: self._post_status_to_server(queue_name, status, patch, results_file)) - def patch_status(self, queue_name, patch_id): - update_status_url = "%s/patch-status/%s/%s" % (self.url, queue_name, patch_id) + def update_svn_revision(self, svn_revision_number, broken_bot): + log("SVN revision: %s broke %s" % (svn_revision_number, broken_bot)) + return NetworkTransaction().run(lambda: self._post_svn_revision_to_server(svn_revision_number, broken_bot)) + + def _fetch_url(self, url): try: - return urllib2.urlopen(update_status_url).read() + return urllib2.urlopen(url).read() except urllib2.HTTPError, e: if e.code == 404: return None raise e + + def patch_status(self, queue_name, patch_id): + patch_status_url = "%s/patch-status/%s/%s" % (self.url, queue_name, patch_id) + return self._fetch_url(patch_status_url) + + def svn_revision(self, svn_revision_number): + svn_revision_url = "%s/svn-revision/%s" % (self.url, svn_revision_number) + return self._fetch_url(svn_revision_url) diff --git a/WebKitTools/Scripts/webkitpy/common/prettypatch.py b/WebKitTools/Scripts/webkitpy/common/prettypatch.py new file mode 100644 index 0000000..8157f9c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/prettypatch.py @@ -0,0 +1,55 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import tempfile + + +class PrettyPatch(object): + def __init__(self, executive, checkout_root): + self._executive = executive + self._checkout_root = checkout_root + + def pretty_diff_file(self, diff): + pretty_diff = self.pretty_diff(diff) + diff_file = tempfile.NamedTemporaryFile(suffix=".html") + diff_file.write(pretty_diff) + diff_file.flush() + return diff_file + + def pretty_diff(self, diff): + pretty_patch_path = os.path.join(self._checkout_root, + "BugsSite", "PrettyPatch") + prettify_path = os.path.join(pretty_patch_path, "prettify.rb") + args = [ + "ruby", + "-I", + pretty_patch_path, + prettify_path, + ] + return self._executive.run_command(args, input=diff) diff --git a/WebKitTools/Scripts/webkitpy/common/system/__init__.py b/WebKitTools/Scripts/webkitpy/common/system/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/common/system/autoinstall.py b/WebKitTools/Scripts/webkitpy/common/system/autoinstall.py new file mode 100755 index 0000000..32fd2cf --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/autoinstall.py @@ -0,0 +1,518 @@ +# Copyright (c) 2009, Daniel Krech All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of the Daniel Krech nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Support for automatically downloading Python packages from an URL.""" + +import logging +import new +import os +import shutil +import sys +import tarfile +import tempfile +import urllib +import urlparse +import zipfile +import zipimport + +_log = logging.getLogger(__name__) + + +class AutoInstaller(object): + + """Supports automatically installing Python packages from an URL. + + Supports uncompressed files, .tar.gz, and .zip formats. + + Basic usage: + + installer = AutoInstaller() + + installer.install(url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", + url_subpath="pep8-0.5.0/pep8.py") + installer.install(url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", + url_subpath="mechanize") + + """ + + def __init__(self, append_to_search_path=False, make_package=True, + target_dir=None, temp_dir=None): + """Create an AutoInstaller instance, and set up the target directory. + + Args: + append_to_search_path: A boolean value of whether to append the + target directory to the sys.path search path. + make_package: A boolean value of whether to make the target + directory a package. This adds an __init__.py file + to the target directory -- allowing packages and + modules within the target directory to be imported + explicitly using dotted module names. + target_dir: The directory path to which packages should be installed. + Defaults to a subdirectory of the folder containing + this module called "autoinstalled". + temp_dir: The directory path to use for any temporary files + generated while downloading, unzipping, and extracting + packages to install. Defaults to a standard temporary + location generated by the tempfile module. This + parameter should normally be used only for development + testing. + + """ + if target_dir is None: + this_dir = os.path.dirname(__file__) + target_dir = os.path.join(this_dir, "autoinstalled") + + # Ensure that the target directory exists. + self._set_up_target_dir(target_dir, append_to_search_path, make_package) + + self._target_dir = target_dir + self._temp_dir = temp_dir + + def _log_transfer(self, message, source, target, log_method=None): + """Log a debug message that involves a source and target.""" + if log_method is None: + log_method = _log.debug + + log_method("%s" % message) + log_method(' From: "%s"' % source) + log_method(' To: "%s"' % target) + + def _create_directory(self, path, name=None): + """Create a directory.""" + log = _log.debug + + name = name + " " if name is not None else "" + log('Creating %sdirectory...' % name) + log(' "%s"' % path) + + os.makedirs(path) + + def _write_file(self, path, text): + """Create a file at the given path with given text. + + This method overwrites any existing file. + + """ + _log.debug("Creating file...") + _log.debug(' "%s"' % path) + file = open(path, "w") + try: + file.write(text) + finally: + file.close() + + def _set_up_target_dir(self, target_dir, append_to_search_path, + make_package): + """Set up a target directory. + + Args: + target_dir: The path to the target directory to set up. + append_to_search_path: A boolean value of whether to append the + target directory to the sys.path search path. + make_package: A boolean value of whether to make the target + directory a package. This adds an __init__.py file + to the target directory -- allowing packages and + modules within the target directory to be imported + explicitly using dotted module names. + + """ + if not os.path.exists(target_dir): + self._create_directory(target_dir, "autoinstall target") + + if append_to_search_path: + sys.path.append(target_dir) + + if make_package: + init_path = os.path.join(target_dir, "__init__.py") + if not os.path.exists(init_path): + text = ("# This file is required for Python to search this " + "directory for modules.\n") + self._write_file(init_path, text) + + def _create_scratch_directory_inner(self, prefix): + """Create a scratch directory without exception handling. + + Creates a scratch directory inside the AutoInstaller temp + directory self._temp_dir, or inside a platform-dependent temp + directory if self._temp_dir is None. Returns the path to the + created scratch directory. + + Raises: + OSError: [Errno 2] if the containing temp directory self._temp_dir + is not None and does not exist. + + """ + # The tempfile.mkdtemp() method function requires that the + # directory corresponding to the "dir" parameter already exist + # if it is not None. + scratch_dir = tempfile.mkdtemp(prefix=prefix, dir=self._temp_dir) + return scratch_dir + + def _create_scratch_directory(self, target_name): + """Create a temporary scratch directory, and return its path. + + The scratch directory is generated inside the temp directory + of this AutoInstaller instance. This method also creates the + temp directory if it does not already exist. + + """ + prefix = target_name + "_" + try: + scratch_dir = self._create_scratch_directory_inner(prefix) + except OSError: + # Handle case of containing temp directory not existing-- + # OSError: [Errno 2] No such file or directory:... + temp_dir = self._temp_dir + if temp_dir is None or os.path.exists(temp_dir): + raise + # Else try again after creating the temp directory. + self._create_directory(temp_dir, "autoinstall temp") + scratch_dir = self._create_scratch_directory_inner(prefix) + + return scratch_dir + + def _url_downloaded_path(self, target_name): + """Return the path to the file containing the URL downloaded.""" + filename = ".%s.url" % target_name + path = os.path.join(self._target_dir, filename) + return path + + def _is_downloaded(self, target_name, url): + """Return whether a package version has been downloaded.""" + version_path = self._url_downloaded_path(target_name) + + _log.debug('Checking %s URL downloaded...' % target_name) + _log.debug(' "%s"' % version_path) + + if not os.path.exists(version_path): + # Then no package version has been downloaded. + _log.debug("No URL file found.") + return False + + file = open(version_path, "r") + try: + version = file.read() + finally: + file.close() + + return version.strip() == url.strip() + + def _record_url_downloaded(self, target_name, url): + """Record the URL downloaded to a file.""" + version_path = self._url_downloaded_path(target_name) + _log.debug("Recording URL downloaded...") + _log.debug(' URL: "%s"' % url) + _log.debug(' To: "%s"' % version_path) + + self._write_file(version_path, url) + + def _extract_targz(self, path, scratch_dir): + # tarfile.extractall() extracts to a path without the + # trailing ".tar.gz". + target_basename = os.path.basename(path[:-len(".tar.gz")]) + target_path = os.path.join(scratch_dir, target_basename) + + self._log_transfer("Starting gunzip/extract...", path, target_path) + + try: + tar_file = tarfile.open(path) + except tarfile.ReadError, err: + # Append existing Error message to new Error. + message = ("Could not open tar file: %s\n" + " The file probably does not have the correct format.\n" + " --> Inner message: %s" + % (path, err)) + raise Exception(message) + + try: + # This is helpful for debugging purposes. + _log.debug("Listing tar file contents...") + for name in tar_file.getnames(): + _log.debug(' * "%s"' % name) + _log.debug("Extracting gzipped tar file...") + tar_file.extractall(target_path) + finally: + tar_file.close() + + return target_path + + # This is a replacement for ZipFile.extractall(), which is + # available in Python 2.6 but not in earlier versions. + def _extract_all(self, zip_file, target_dir): + self._log_transfer("Extracting zip file...", zip_file, target_dir) + + # This is helpful for debugging purposes. + _log.debug("Listing zip file contents...") + for name in zip_file.namelist(): + _log.debug(' * "%s"' % name) + + for name in zip_file.namelist(): + path = os.path.join(target_dir, name) + self._log_transfer("Extracting...", name, path) + + if not os.path.basename(path): + # Then the path ends in a slash, so it is a directory. + self._create_directory(path) + continue + # Otherwise, it is a file. + + try: + outfile = open(path, 'wb') + except IOError, err: + # Not all zip files seem to list the directories explicitly, + # so try again after creating the containing directory. + _log.debug("Got IOError: retrying after creating directory...") + dir = os.path.dirname(path) + self._create_directory(dir) + outfile = open(path, 'wb') + + try: + outfile.write(zip_file.read(name)) + finally: + outfile.close() + + def _unzip(self, path, scratch_dir): + # zipfile.extractall() extracts to a path without the + # trailing ".zip". + target_basename = os.path.basename(path[:-len(".zip")]) + target_path = os.path.join(scratch_dir, target_basename) + + self._log_transfer("Starting unzip...", path, target_path) + + try: + zip_file = zipfile.ZipFile(path, "r") + except zipfile.BadZipfile, err: + message = ("Could not open zip file: %s\n" + " --> Inner message: %s" + % (path, err)) + raise Exception(message) + + try: + self._extract_all(zip_file, scratch_dir) + finally: + zip_file.close() + + return target_path + + def _prepare_package(self, path, scratch_dir): + """Prepare a package for use, if necessary, and return the new path. + + For example, this method unzips zipped files and extracts + tar files. + + Args: + path: The path to the downloaded URL contents. + scratch_dir: The scratch directory. Note that the scratch + directory contains the file designated by the + path parameter. + + """ + # FIXME: Add other natural extensions. + if path.endswith(".zip"): + new_path = self._unzip(path, scratch_dir) + elif path.endswith(".tar.gz"): + new_path = self._extract_targz(path, scratch_dir) + else: + # No preparation is needed. + new_path = path + + return new_path + + def _download_to_stream(self, url, stream): + """Download an URL to a stream, and return the number of bytes.""" + try: + netstream = urllib.urlopen(url) + except IOError, err: + # Append existing Error message to new Error. + message = ('Could not download Python modules from URL "%s".\n' + " Make sure you are connected to the internet.\n" + " You must be connected to the internet when " + "downloading needed modules for the first time.\n" + " --> Inner message: %s" + % (url, err)) + raise IOError(message) + code = 200 + if hasattr(netstream, "getcode"): + code = netstream.getcode() + if not 200 <= code < 300: + raise ValueError("HTTP Error code %s" % code) + + BUFSIZE = 2**13 # 8KB + bytes = 0 + while True: + data = netstream.read(BUFSIZE) + if not data: + break + stream.write(data) + bytes += len(data) + netstream.close() + return bytes + + def _download(self, url, scratch_dir): + """Download URL contents, and return the download path.""" + url_path = urlparse.urlsplit(url)[2] + url_path = os.path.normpath(url_path) # Removes trailing slash. + target_filename = os.path.basename(url_path) + target_path = os.path.join(scratch_dir, target_filename) + + self._log_transfer("Starting download...", url, target_path) + + stream = file(target_path, "wb") + bytes = self._download_to_stream(url, stream) + stream.close() + + _log.debug("Downloaded %s bytes." % bytes) + + return target_path + + def _install(self, scratch_dir, package_name, target_path, url, + url_subpath): + """Install a python package from an URL. + + This internal method overwrites the target path if the target + path already exists. + + """ + path = self._download(url=url, scratch_dir=scratch_dir) + path = self._prepare_package(path, scratch_dir) + + if url_subpath is None: + source_path = path + else: + source_path = os.path.join(path, url_subpath) + + if os.path.exists(target_path): + _log.debug('Refreshing install: deleting "%s".' % target_path) + if os.path.isdir(target_path): + shutil.rmtree(target_path) + else: + os.remove(target_path) + + self._log_transfer("Moving files into place...", source_path, target_path) + + # The shutil.move() command creates intermediate directories if they + # do not exist, but we do not rely on this behavior since we + # need to create the __init__.py file anyway. + shutil.move(source_path, target_path) + + self._record_url_downloaded(package_name, url) + + def install(self, url, should_refresh=False, target_name=None, + url_subpath=None): + """Install a python package from an URL. + + Args: + url: The URL from which to download the package. + + Optional Args: + should_refresh: A boolean value of whether the package should be + downloaded again if the package is already present. + target_name: The name of the folder or file in the autoinstaller + target directory at which the package should be + installed. Defaults to the base name of the + URL sub-path. This parameter must be provided if + the URL sub-path is not specified. + url_subpath: The relative path of the URL directory that should + be installed. Defaults to the full directory, or + the entire URL contents. + + """ + if target_name is None: + if not url_subpath: + raise ValueError('The "target_name" parameter must be ' + 'provided if the "url_subpath" parameter ' + "is not provided.") + # Remove any trailing slashes. + url_subpath = os.path.normpath(url_subpath) + target_name = os.path.basename(url_subpath) + + target_path = os.path.join(self._target_dir, target_name) + if not should_refresh and self._is_downloaded(target_name, url): + _log.debug('URL for %s already downloaded. Skipping...' + % target_name) + _log.debug(' "%s"' % url) + return + + self._log_transfer("Auto-installing package: %s" % target_name, + url, target_path, log_method=_log.info) + + # The scratch directory is where we will download and prepare + # files specific to this install until they are ready to move + # into place. + scratch_dir = self._create_scratch_directory(target_name) + + try: + self._install(package_name=target_name, + target_path=target_path, + scratch_dir=scratch_dir, + url=url, + url_subpath=url_subpath) + except Exception, err: + # Append existing Error message to new Error. + message = ("Error auto-installing the %s package to:\n" + ' "%s"\n' + " --> Inner message: %s" + % (target_name, target_path, err)) + raise Exception(message) + finally: + _log.debug('Cleaning up: deleting "%s".' % scratch_dir) + shutil.rmtree(scratch_dir) + _log.debug('Auto-installed %s to:' % target_name) + _log.debug(' "%s"' % target_path) + + +if __name__=="__main__": + + # Configure the autoinstall logger to log DEBUG messages for + # development testing purposes. + console = logging.StreamHandler() + + formatter = logging.Formatter('%(name)s: %(levelname)-8s %(message)s') + console.setFormatter(formatter) + _log.addHandler(console) + _log.setLevel(logging.DEBUG) + + # Use a more visible temp directory for debug purposes. + this_dir = os.path.dirname(__file__) + target_dir = os.path.join(this_dir, "autoinstalled") + temp_dir = os.path.join(target_dir, "Temp") + + installer = AutoInstaller(target_dir=target_dir, + temp_dir=temp_dir) + + installer.install(should_refresh=False, + target_name="pep8.py", + url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", + url_subpath="pep8-0.5.0/pep8.py") + installer.install(should_refresh=False, + target_name="mechanize", + url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", + url_subpath="mechanize") + diff --git a/WebKitTools/Scripts/webkitpy/webkit_logging.py b/WebKitTools/Scripts/webkitpy/common/system/deprecated_logging.py index ba1c5eb..ba1c5eb 100644 --- a/WebKitTools/Scripts/webkitpy/webkit_logging.py +++ b/WebKitTools/Scripts/webkitpy/common/system/deprecated_logging.py diff --git a/WebKitTools/Scripts/webkitpy/webkit_logging_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/deprecated_logging_unittest.py index b940a4d..2b71803 100644 --- a/WebKitTools/Scripts/webkitpy/webkit_logging_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/system/deprecated_logging_unittest.py @@ -32,8 +32,8 @@ import StringIO import tempfile import unittest -from webkitpy.executive import ScriptError -from webkitpy.webkit_logging import * +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.system.deprecated_logging import * class LoggingTest(unittest.TestCase): @@ -46,7 +46,7 @@ class LoggingTest(unittest.TestCase): log(log_input) actual_output = test_stderr.getvalue() finally: - original_stderr = original_stderr + sys.stderr = original_stderr self.assertEquals(actual_output, expected_output, "log(\"%s\") expected: %s actual: %s" % (log_input, expected_output, actual_output)) diff --git a/WebKitTools/Scripts/webkitpy/executive.py b/WebKitTools/Scripts/webkitpy/common/system/executive.py index 50b119b..b6126e4 100644 --- a/WebKitTools/Scripts/webkitpy/executive.py +++ b/WebKitTools/Scripts/webkitpy/common/system/executive.py @@ -27,12 +27,20 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +try: + # This API exists only in Python 2.6 and higher. :( + import multiprocessing +except ImportError: + multiprocessing = None + import os +import platform import StringIO +import signal import subprocess import sys -from webkitpy.webkit_logging import tee +from webkitpy.common.system.deprecated_logging import tee class ScriptError(Exception): @@ -96,9 +104,11 @@ class Executive(object): def run_and_throw_if_fail(self, args, quiet=False): # Cache the child's output locally so it can be used for error reports. child_out_file = StringIO.StringIO() + tee_stdout = sys.stdout if quiet: dev_null = open(os.devnull, "w") - child_stdout = tee(child_out_file, dev_null if quiet else sys.stdout) + tee_stdout = dev_null + child_stdout = tee(child_out_file, tee_stdout) exit_code = self._run_command_with_teed_output(args, child_stdout) if quiet: dev_null.close() @@ -110,17 +120,37 @@ class Executive(object): raise ScriptError(script_args=args, exit_code=exit_code, output=child_output) + return child_output - @staticmethod - def cpu_count(): - # This API exists only in Python 2.6 and higher. :( - try: - import multiprocessing + def cpu_count(self): + if multiprocessing: return multiprocessing.cpu_count() - except (ImportError, NotImplementedError): - # This quantity is a lie but probably a reasonable guess for modern - # machines. - return 2 + # Darn. We don't have the multiprocessing package. + system_name = platform.system() + if system_name == "Darwin": + return int(self.run_command(["sysctl", "-n", "hw.ncpu"])) + elif system_name == "Windows": + return int(os.environ.get('NUMBER_OF_PROCESSORS', 1)) + elif system_name == "Linux": + num_cores = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(num_cores, int) and num_cores > 0: + return num_cores + # This quantity is a lie but probably a reasonable guess for modern + # machines. + return 2 + + def kill_process(self, pid): + if platform.system() == "Windows": + # According to http://docs.python.org/library/os.html + # os.kill isn't available on Windows. However, when I tried it + # using Cygwin, it worked fine. We should investigate whether + # we need this platform specific code here. + subprocess.call(('taskkill.exe', '/f', '/pid', str(pid)), + stdin=open(os.devnull, 'r'), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return + os.kill(pid, signal.SIGKILL) # Error handlers do not need to be static methods once all callers are # updated to use an Executive object. @@ -146,8 +176,14 @@ class Executive(object): stdin = input string_to_communicate = None else: - stdin = subprocess.PIPE if input else None - string_to_communicate = input + stdin = None + if input: + stdin = subprocess.PIPE + # string_to_communicate seems to need to be a str for proper + # communication with shell commands. + # See https://bugs.webkit.org/show_bug.cgi?id=37528 + # For an example of a regresion caused by passing a unicode string through. + string_to_communicate = str(input) if return_stderr: stderr = subprocess.STDOUT else: @@ -160,12 +196,14 @@ class Executive(object): cwd=cwd) output = process.communicate(string_to_communicate)[0] exit_code = process.wait() + + if return_exit_code: + return exit_code + if exit_code: script_error = ScriptError(script_args=args, exit_code=exit_code, output=output, cwd=cwd) (error_handler or self.default_error_handler)(script_error) - if return_exit_code: - return exit_code return output diff --git a/WebKitTools/Scripts/webkitpy/executive_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/executive_unittest.py index f78e301..ac380f8 100644 --- a/WebKitTools/Scripts/webkitpy/executive_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/system/executive_unittest.py @@ -28,7 +28,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from webkitpy.executive import Executive, run_command + +from webkitpy.common.system.executive import Executive, run_command class ExecutiveTest(unittest.TestCase): diff --git a/WebKitTools/Scripts/webkitpy/common/system/logtesting.py b/WebKitTools/Scripts/webkitpy/common/system/logtesting.py new file mode 100644 index 0000000..e361cb5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/logtesting.py @@ -0,0 +1,258 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports the unit-testing of logging code. + +Provides support for unit-testing messages logged using the built-in +logging module. + +Inherit from the LoggingTestCase class for basic testing needs. For +more advanced needs (e.g. unit-testing methods that configure logging), +see the TestLogStream class, and perhaps also the LogTesting class. + +""" + +import logging +import unittest + + +class TestLogStream(object): + + """Represents a file-like object for unit-testing logging. + + This is meant for passing to the logging.StreamHandler constructor. + Log messages captured by instances of this object can be tested + using self.assertMessages() below. + + """ + + def __init__(self, test_case): + """Create an instance. + + Args: + test_case: A unittest.TestCase instance. + + """ + self._test_case = test_case + self.messages = [] + """A list of log messages written to the stream.""" + + # Python documentation says that any object passed to the StreamHandler + # constructor should support write() and flush(): + # + # http://docs.python.org/library/logging.html#module-logging.handlers + def write(self, message): + self.messages.append(message) + + def flush(self): + pass + + def assertMessages(self, messages): + """Assert that the given messages match the logged messages. + + messages: A list of log message strings. + + """ + self._test_case.assertEquals(messages, self.messages) + + +class LogTesting(object): + + """Supports end-to-end unit-testing of log messages. + + Sample usage: + + class SampleTest(unittest.TestCase): + + def setUp(self): + self._log = LogTesting.setUp(self) # Turn logging on. + + def tearDown(self): + self._log.tearDown() # Turn off and reset logging. + + def test_logging_in_some_method(self): + call_some_method() # Contains calls to _log.info(), etc. + + # Check the resulting log messages. + self._log.assertMessages(["INFO: expected message #1", + "WARNING: expected message #2"]) + + """ + + def __init__(self, test_stream, handler): + """Create an instance. + + This method should never be called directly. Instances should + instead be created using the static setUp() method. + + Args: + test_stream: A TestLogStream instance. + handler: The handler added to the logger. + + """ + self._test_stream = test_stream + self._handler = handler + + @staticmethod + def _getLogger(): + """Return the logger being tested.""" + # It is possible we might want to return something other than + # the root logger in some special situation. For now, the + # root logger seems to suffice. + return logging.getLogger() + + @staticmethod + def setUp(test_case, logging_level=logging.INFO): + """Configure logging for unit testing. + + Configures the root logger to log to a testing log stream. + Only messages logged at or above the given level are logged + to the stream. Messages logged to the stream are formatted + in the following way, for example-- + + "INFO: This is a test log message." + + This method should normally be called in the setUp() method + of a unittest.TestCase. See the docstring of this class + for more details. + + Returns: + A LogTesting instance. + + Args: + test_case: A unittest.TestCase instance. + logging_level: An integer logging level that is the minimum level + of log messages you would like to test. + + """ + stream = TestLogStream(test_case) + handler = logging.StreamHandler(stream) + handler.setLevel(logging_level) + formatter = logging.Formatter("%(levelname)s: %(message)s") + handler.setFormatter(formatter) + + # Notice that we only change the root logger by adding a handler + # to it. In particular, we do not reset its level using + # logger.setLevel(). This ensures that we have not interfered + # with how the code being tested may have configured the root + # logger. + logger = LogTesting._getLogger() + logger.addHandler(handler) + + return LogTesting(stream, handler) + + def tearDown(self): + """Assert there are no remaining log messages, and reset logging. + + This method asserts that there are no more messages in the array of + log messages, and then restores logging to its original state. + This method should normally be called in the tearDown() method of a + unittest.TestCase. See the docstring of this class for more details. + + """ + self.assertMessages([]) + logger = LogTesting._getLogger() + logger.removeHandler(self._handler) + + def messages(self): + """Return the current list of log messages.""" + return self._test_stream.messages + + # FIXME: Add a clearMessages() method for cases where the caller + # deliberately doesn't want to assert every message. + + # We clear the log messages after asserting since they are no longer + # needed after asserting. This serves two purposes: (1) it simplifies + # the calling code when we want to check multiple logging calls in a + # single test method, and (2) it lets us check in the tearDown() method + # that there are no remaining log messages to be asserted. + # + # The latter ensures that no extra log messages are getting logged that + # the caller might not be aware of or may have forgotten to check for. + # This gets us a bit more mileage out of our tests without writing any + # additional code. + def assertMessages(self, messages): + """Assert the current array of log messages, and clear its contents. + + Args: + messages: A list of log message strings. + + """ + try: + self._test_stream.assertMessages(messages) + finally: + # We want to clear the array of messages even in the case of + # an Exception (e.g. an AssertionError). Otherwise, another + # AssertionError can occur in the tearDown() because the + # array might not have gotten emptied. + self._test_stream.messages = [] + + +# This class needs to inherit from unittest.TestCase. Otherwise, the +# setUp() and tearDown() methods will not get fired for test case classes +# that inherit from this class -- even if the class inherits from *both* +# unittest.TestCase and LoggingTestCase. +# +# FIXME: Rename this class to LoggingTestCaseBase to be sure that +# the unittest module does not interpret this class as a unittest +# test case itself. +class LoggingTestCase(unittest.TestCase): + + """Supports end-to-end unit-testing of log messages. + + Sample usage: + + class SampleTest(LoggingTestCase): + + def test_logging_in_some_method(self): + call_some_method() # Contains calls to _log.info(), etc. + + # Check the resulting log messages. + self.assertLog(["INFO: expected message #1", + "WARNING: expected message #2"]) + + """ + + def setUp(self): + self._log = LogTesting.setUp(self) + + def tearDown(self): + self._log.tearDown() + + def logMessages(self): + """Return the current list of log messages.""" + return self._log.messages() + + # FIXME: Add a clearMessages() method for cases where the caller + # deliberately doesn't want to assert every message. + + # See the code comments preceding LogTesting.assertMessages() for + # an explanation of why we clear the array of messages after + # asserting its contents. + def assertLog(self, messages): + """Assert the current array of log messages, and clear its contents. + + Args: + messages: A list of log message strings. + + """ + self._log.assertMessages(messages) diff --git a/WebKitTools/Scripts/webkitpy/common/system/logutils.py b/WebKitTools/Scripts/webkitpy/common/system/logutils.py new file mode 100644 index 0000000..cd4e60f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/logutils.py @@ -0,0 +1,207 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports webkitpy logging.""" + +# FIXME: Move this file to webkitpy/python24 since logging needs to +# be configured prior to running version-checking code. + +import logging +import os +import sys + +import webkitpy + + +_log = logging.getLogger(__name__) + +# We set these directory paths lazily in get_logger() below. +_scripts_dir = "" +"""The normalized, absolute path to the ...Scripts directory.""" + +_webkitpy_dir = "" +"""The normalized, absolute path to the ...Scripts/webkitpy directory.""" + + +def _normalize_path(path): + """Return the given path normalized. + + Converts a path to an absolute path, removes any trailing slashes, + removes any extension, and lower-cases it. + + """ + path = os.path.abspath(path) + path = os.path.normpath(path) + path = os.path.splitext(path)[0] # Remove the extension, if any. + path = path.lower() + + return path + + +# Observe that the implementation of this function does not require +# the use of any hard-coded strings like "webkitpy", etc. +# +# The main benefit this function has over using-- +# +# _log = logging.getLogger(__name__) +# +# is that get_logger() returns the same value even if __name__ is +# "__main__" -- i.e. even if the module is the script being executed +# from the command-line. +def get_logger(path): + """Return a logging.logger for the given path. + + Returns: + A logger whose name is the name of the module corresponding to + the given path. If the module is in webkitpy, the name is + the fully-qualified dotted module name beginning with webkitpy.... + Otherwise, the name is the base name of the module (i.e. without + any dotted module name prefix). + + Args: + path: The path of the module. Normally, this parameter should be + the __file__ variable of the module. + + Sample usage: + + import webkitpy.common.system.logutils as logutils + + _log = logutils.get_logger(__file__) + + """ + # Since we assign to _scripts_dir and _webkitpy_dir in this function, + # we need to declare them global. + global _scripts_dir + global _webkitpy_dir + + path = _normalize_path(path) + + # Lazily evaluate _webkitpy_dir and _scripts_dir. + if not _scripts_dir: + # The normalized, absolute path to ...Scripts/webkitpy/__init__. + webkitpy_path = _normalize_path(webkitpy.__file__) + + _webkitpy_dir = os.path.split(webkitpy_path)[0] + _scripts_dir = os.path.split(_webkitpy_dir)[0] + + if path.startswith(_webkitpy_dir): + # Remove the initial Scripts directory portion, so the path + # starts with /webkitpy, for example "/webkitpy/init/logutils". + path = path[len(_scripts_dir):] + + parts = [] + while True: + (path, tail) = os.path.split(path) + if not tail: + break + parts.insert(0, tail) + + logger_name = ".".join(parts) # For example, webkitpy.common.system.logutils. + else: + # The path is outside of webkitpy. Default to the basename + # without the extension. + basename = os.path.basename(path) + logger_name = os.path.splitext(basename)[0] + + return logging.getLogger(logger_name) + + +def _default_handlers(stream): + """Return a list of the default logging handlers to use. + + Args: + stream: See the configure_logging() docstring. + + """ + # Create the filter. + def should_log(record): + """Return whether a logging.LogRecord should be logged.""" + # FIXME: Enable the logging of autoinstall messages once + # autoinstall is adjusted. Currently, autoinstall logs + # INFO messages when importing already-downloaded packages, + # which is too verbose. + if record.name.startswith("webkitpy.thirdparty.autoinstall"): + return False + return True + + logging_filter = logging.Filter() + logging_filter.filter = should_log + + # Create the handler. + handler = logging.StreamHandler(stream) + formatter = logging.Formatter("%(name)s: [%(levelname)s] %(message)s") + handler.setFormatter(formatter) + handler.addFilter(logging_filter) + + return [handler] + + +def configure_logging(logging_level=None, logger=None, stream=None, + handlers=None): + """Configure logging for standard purposes. + + Returns: + A list of references to the logging handlers added to the root + logger. This allows the caller to later remove the handlers + using logger.removeHandler. This is useful primarily during unit + testing where the caller may want to configure logging temporarily + and then undo the configuring. + + Args: + logging_level: The minimum logging level to log. Defaults to + logging.INFO. + logger: A logging.logger instance to configure. This parameter + should be used only in unit tests. Defaults to the + root logger. + stream: A file-like object to which to log used in creating the default + handlers. The stream must define an "encoding" data attribute, + or else logging raises an error. Defaults to sys.stderr. + handlers: A list of logging.Handler instances to add to the logger + being configured. If this parameter is provided, then the + stream parameter is not used. + + """ + # If the stream does not define an "encoding" data attribute, the + # logging module can throw an error like the following: + # + # Traceback (most recent call last): + # File "/System/Library/Frameworks/Python.framework/Versions/2.6/... + # lib/python2.6/logging/__init__.py", line 761, in emit + # self.stream.write(fs % msg.encode(self.stream.encoding)) + # LookupError: unknown encoding: unknown + if logging_level is None: + logging_level = logging.INFO + if logger is None: + logger = logging.getLogger() + if stream is None: + stream = sys.stderr + if handlers is None: + handlers = _default_handlers(stream) + + logger.setLevel(logging_level) + + for handler in handlers: + logger.addHandler(handler) + + _log.debug("Debug logging enabled.") + + return handlers diff --git a/WebKitTools/Scripts/webkitpy/common/system/logutils_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/logutils_unittest.py new file mode 100644 index 0000000..a4a6496 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/logutils_unittest.py @@ -0,0 +1,142 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for logutils.py.""" + +import logging +import os +import unittest + +from webkitpy.common.system.logtesting import LogTesting +from webkitpy.common.system.logtesting import TestLogStream +import webkitpy.common.system.logutils as logutils + + +class GetLoggerTest(unittest.TestCase): + + """Tests get_logger().""" + + def test_get_logger_in_webkitpy(self): + logger = logutils.get_logger(__file__) + self.assertEquals(logger.name, "webkitpy.common.system.logutils_unittest") + + def test_get_logger_not_in_webkitpy(self): + # Temporarily change the working directory so that we + # can test get_logger() for a path outside of webkitpy. + working_directory = os.getcwd() + root_dir = "/" + os.chdir(root_dir) + + logger = logutils.get_logger("/WebKitTools/Scripts/test-webkitpy") + self.assertEquals(logger.name, "test-webkitpy") + + logger = logutils.get_logger("/WebKitTools/Scripts/test-webkitpy.py") + self.assertEquals(logger.name, "test-webkitpy") + + os.chdir(working_directory) + + +class ConfigureLoggingTestBase(unittest.TestCase): + + """Base class for configure_logging() unit tests.""" + + def _logging_level(self): + raise Exception("Not implemented.") + + def setUp(self): + log_stream = TestLogStream(self) + + # Use a logger other than the root logger or one prefixed with + # "webkitpy." so as not to conflict with test-webkitpy logging. + logger = logging.getLogger("unittest") + + # Configure the test logger not to pass messages along to the + # root logger. This prevents test messages from being + # propagated to loggers used by test-webkitpy logging (e.g. + # the root logger). + logger.propagate = False + + logging_level = self._logging_level() + self._handlers = logutils.configure_logging(logging_level=logging_level, + logger=logger, + stream=log_stream) + self._log = logger + self._log_stream = log_stream + + def tearDown(self): + """Reset logging to its original state. + + This method ensures that the logging configuration set up + for a unit test does not affect logging in other unit tests. + + """ + logger = self._log + for handler in self._handlers: + logger.removeHandler(handler) + + def _assert_log_messages(self, messages): + """Assert that the logged messages equal the given messages.""" + self._log_stream.assertMessages(messages) + + +class ConfigureLoggingTest(ConfigureLoggingTestBase): + + """Tests configure_logging() with the default logging level.""" + + def _logging_level(self): + return None + + def test_info_message(self): + self._log.info("test message") + self._assert_log_messages(["unittest: [INFO] test message\n"]) + + def test_below_threshold_message(self): + # We test the boundary case of a logging level equal to 19. + # In practice, we will probably only be calling log.debug(), + # which corresponds to a logging level of 10. + level = logging.INFO - 1 # Equals 19. + self._log.log(level, "test message") + self._assert_log_messages([]) + + def test_two_messages(self): + self._log.info("message1") + self._log.info("message2") + self._assert_log_messages(["unittest: [INFO] message1\n", + "unittest: [INFO] message2\n"]) + + +class ConfigureLoggingCustomLevelTest(ConfigureLoggingTestBase): + + """Tests configure_logging() with a custom logging level.""" + + _level = 36 + + def _logging_level(self): + return self._level + + def test_logged_message(self): + self._log.log(self._level, "test message") + self._assert_log_messages(["unittest: [Level 36] test message\n"]) + + def test_below_threshold_message(self): + self._log.log(self._level - 1, "test message") + self._assert_log_messages([]) diff --git a/WebKitTools/Scripts/webkitpy/common/system/ospath.py b/WebKitTools/Scripts/webkitpy/common/system/ospath.py new file mode 100644 index 0000000..aed7a3d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/ospath.py @@ -0,0 +1,83 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains a substitute for Python 2.6's os.path.relpath().""" + +import os + + +# This function is a replacement for os.path.relpath(), which is only +# available in Python 2.6: +# +# http://docs.python.org/library/os.path.html#os.path.relpath +# +# It should behave essentially the same as os.path.relpath(), except for +# returning None on paths not contained in abs_start_path. +def relpath(path, start_path, os_path_abspath=None): + """Return a path relative to the given start path, or None. + + Returns None if the path is not contained in the directory start_path. + + Args: + path: An absolute or relative path to convert to a relative path. + start_path: The path relative to which the given path should be + converted. + os_path_abspath: A replacement function for unit testing. This + function should strip trailing slashes just like + os.path.abspath(). Defaults to os.path.abspath. + + """ + if os_path_abspath is None: + os_path_abspath = os.path.abspath + + # Since os_path_abspath() calls os.path.normpath()-- + # + # (see http://docs.python.org/library/os.path.html#os.path.abspath ) + # + # it also removes trailing slashes and converts forward and backward + # slashes to the preferred slash os.sep. + start_path = os_path_abspath(start_path) + path = os_path_abspath(path) + + if not path.lower().startswith(start_path.lower()): + # Then path is outside the directory given by start_path. + return None + + rel_path = path[len(start_path):] + + if not rel_path: + # Then the paths are the same. + pass + elif rel_path[0] == os.sep: + # It is probably sufficient to remove just the first character + # since os.path.normpath() collapses separators, but we use + # lstrip() just to be sure. + rel_path = rel_path.lstrip(os.sep) + else: + # We are in the case typified by the following example: + # + # start_path = "/tmp/foo" + # path = "/tmp/foobar" + # rel_path = "bar" + return None + + return rel_path diff --git a/WebKitTools/Scripts/webkitpy/common/system/ospath_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/ospath_unittest.py new file mode 100644 index 0000000..0493c68 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/ospath_unittest.py @@ -0,0 +1,62 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for ospath.py.""" + +import os +import unittest + +from webkitpy.common.system.ospath import relpath + + +# Make sure the tests in this class are platform independent. +class RelPathTest(unittest.TestCase): + + """Tests relpath().""" + + os_path_abspath = lambda self, path: path + + def _rel_path(self, path, abs_start_path): + return relpath(path, abs_start_path, self.os_path_abspath) + + def test_same_path(self): + rel_path = self._rel_path("WebKit", "WebKit") + self.assertEquals(rel_path, "") + + def test_long_rel_path(self): + start_path = "WebKit" + expected_rel_path = os.path.join("test", "Foo.txt") + path = os.path.join(start_path, expected_rel_path) + + rel_path = self._rel_path(path, start_path) + self.assertEquals(expected_rel_path, rel_path) + + def test_none_rel_path(self): + """Test _rel_path() with None return value.""" + start_path = "WebKit" + path = os.path.join("other_dir", "foo.txt") + + rel_path = self._rel_path(path, start_path) + self.assertTrue(rel_path is None) + + rel_path = self._rel_path("WebKitTools", "WebKit") + self.assertTrue(rel_path is None) diff --git a/WebKitTools/Scripts/webkitpy/outputcapture.py b/WebKitTools/Scripts/webkitpy/common/system/outputcapture.py index 592a669..592a669 100644 --- a/WebKitTools/Scripts/webkitpy/outputcapture.py +++ b/WebKitTools/Scripts/webkitpy/common/system/outputcapture.py diff --git a/WebKitTools/Scripts/webkitpy/user.py b/WebKitTools/Scripts/webkitpy/common/system/user.py index b2ec19e..076f965 100644 --- a/WebKitTools/Scripts/webkitpy/user.py +++ b/WebKitTools/Scripts/webkitpy/common/system/user.py @@ -31,15 +31,34 @@ import shlex import subprocess import webbrowser +try: + import readline +except ImportError: + print "Unable to import readline. If you're using MacPorts, try running:" + print " sudo port install py25-readline" + exit(1) + + class User(object): - @staticmethod - def prompt(message, repeat=1, raw_input=raw_input): + # FIXME: These are @classmethods because scm.py and bugzilla.py don't have a Tool object (thus no User instance). + @classmethod + def prompt(cls, message, repeat=1, raw_input=raw_input): response = None while (repeat and not response): repeat -= 1 response = raw_input(message) return response + @classmethod + def prompt_with_list(cls, list_title, list_items): + print list_title + i = 0 + for item in list_items: + i += 1 + print "%2d. %s" % (i, item) + result = int(cls.prompt("Enter a number: ")) - 1 + return list_items[result] + def edit(self, files): editor = os.environ.get("EDITOR") or "vi" args = shlex.split(editor) diff --git a/WebKitTools/Scripts/webkitpy/user_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/user_unittest.py index 34d9983..dadead3 100644 --- a/WebKitTools/Scripts/webkitpy/user_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/system/user_unittest.py @@ -27,7 +27,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from webkitpy.user import User + +from webkitpy.common.system.user import User class UserTest(unittest.TestCase): diff --git a/WebKitTools/Scripts/webkitpy/common/thread/__init__.py b/WebKitTools/Scripts/webkitpy/common/thread/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/thread/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/common/thread/messagepump.py b/WebKitTools/Scripts/webkitpy/common/thread/messagepump.py new file mode 100644 index 0000000..0e39285 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/thread/messagepump.py @@ -0,0 +1,59 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class MessagePumpDelegate(object): + def schedule(self, interval, callback): + raise NotImplementedError, "subclasses must implement" + + def message_available(self, message): + raise NotImplementedError, "subclasses must implement" + + def final_message_delivered(self): + raise NotImplementedError, "subclasses must implement" + + +class MessagePump(object): + interval = 10 # seconds + + def __init__(self, delegate, message_queue): + self._delegate = delegate + self._message_queue = message_queue + self._schedule() + + def _schedule(self): + self._delegate.schedule(self.interval, self._callback) + + def _callback(self): + (messages, is_running) = self._message_queue.take_all() + for message in messages: + self._delegate.message_available(message) + if not is_running: + self._delegate.final_message_delivered() + return + self._schedule() diff --git a/WebKitTools/Scripts/webkitpy/common/thread/messagepump_unittest.py b/WebKitTools/Scripts/webkitpy/common/thread/messagepump_unittest.py new file mode 100644 index 0000000..f731db2 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/thread/messagepump_unittest.py @@ -0,0 +1,83 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.thread.messagepump import MessagePump, MessagePumpDelegate +from webkitpy.common.thread.threadedmessagequeue import ThreadedMessageQueue + + +class TestDelegate(MessagePumpDelegate): + def __init__(self): + self.log = [] + + def schedule(self, interval, callback): + self.callback = callback + self.log.append("schedule") + + def message_available(self, message): + self.log.append("message_available: %s" % message) + + def final_message_delivered(self): + self.log.append("final_message_delivered") + + +class MessagePumpTest(unittest.TestCase): + + def test_basic(self): + queue = ThreadedMessageQueue() + delegate = TestDelegate() + pump = MessagePump(delegate, queue) + self.assertEqual(delegate.log, [ + 'schedule' + ]) + delegate.callback() + queue.post("Hello") + queue.post("There") + delegate.callback() + self.assertEqual(delegate.log, [ + 'schedule', + 'schedule', + 'message_available: Hello', + 'message_available: There', + 'schedule' + ]) + queue.post("More") + queue.post("Messages") + queue.stop() + delegate.callback() + self.assertEqual(delegate.log, [ + 'schedule', + 'schedule', + 'message_available: Hello', + 'message_available: There', + 'schedule', + 'message_available: More', + 'message_available: Messages', + 'final_message_delivered' + ]) diff --git a/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue.py b/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue.py new file mode 100644 index 0000000..6cb6f8c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue.py @@ -0,0 +1,55 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import threading + + +class ThreadedMessageQueue(object): + def __init__(self): + self._messages = [] + self._is_running = True + self._lock = threading.Lock() + + def post(self, message): + self._lock.acquire() + self._messages.append(message) + self._lock.release() + + def stop(self): + self._lock.acquire() + self._is_running = False + self._lock.release() + + def take_all(self): + self._lock.acquire() + messages = self._messages + is_running = self._is_running + self._messages = [] + self._lock.release() + return (messages, is_running) + diff --git a/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue_unittest.py b/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue_unittest.py new file mode 100644 index 0000000..cb67c1e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue_unittest.py @@ -0,0 +1,53 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.thread.threadedmessagequeue import ThreadedMessageQueue + +class ThreadedMessageQueueTest(unittest.TestCase): + + def test_basic(self): + queue = ThreadedMessageQueue() + queue.post("Hello") + queue.post("There") + (messages, is_running) = queue.take_all() + self.assertEqual(messages, ["Hello", "There"]) + self.assertTrue(is_running) + (messages, is_running) = queue.take_all() + self.assertEqual(messages, []) + self.assertTrue(is_running) + queue.post("More") + queue.stop() + queue.post("Messages") + (messages, is_running) = queue.take_all() + self.assertEqual(messages, ["More", "Messages"]) + self.assertFalse(is_running) + (messages, is_running) = queue.take_all() + self.assertEqual(messages, []) + self.assertFalse(is_running) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py b/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py index 6e4ba99..231ed70 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py @@ -61,17 +61,19 @@ def run_tests(port, options, tests): if __name__ == '__main__': - optparser = optparse.OptionParser() - optparser.add_option('-p', '--platform', action='store', default='mac', - help='Platform to test (e.g., "mac", "chromium-mac", etc.') - optparser.add_option('-t', '--target', action='store', default='Release', - help='build type ("Debug" or "Release")') - optparser.add_option('', '--timeout', action='store', default='2000', - help='test timeout in milliseconds (2000 by default)') - optparser.add_option('', '--wrapper', action='store') - optparser.add_option('', '--no-pixel-tests', action='store_true', - default=False, - help='disable pixel-to-pixel PNG comparisons') + # FIXME: configuration_options belong in a shared location. + configuration_options = [ + optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), + optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), + ] + misc_options = [ + optparse.make_option('-p', '--platform', action='store', default='mac', help='Platform to test (e.g., "mac", "chromium-mac", etc.'), + optparse.make_option('--timeout', action='store', default='2000', help='test timeout in milliseconds (2000 by default)'), + optparse.make_option('--wrapper', action='store'), + optparse.make_option('--no-pixel-tests', action='store_true', default=False, help='disable pixel-to-pixel PNG comparisons'), + ] + option_list = configuration_options + misc_options + optparser = optparse.OptionParser(option_list=option_list) options, args = optparser.parse_args() p = port.get(options.platform, options) run_tests(p, options, args) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_shell_thread.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py index 3452035..e61d11f 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_shell_thread.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py @@ -27,10 +27,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""A Thread object for running the test shell and processing URLs from a +"""A Thread object for running DumpRenderTree and processing URLs from a shared queue. -Each thread runs a separate instance of the test_shell binary and validates +Each thread runs a separate instance of the DumpRenderTree binary and validates the output. When there are no more URLs to process in the shared queue, the thread exits. """ @@ -47,23 +47,26 @@ import time import test_failures +_log = logging.getLogger("webkitpy.layout_tests.layout_package." + "dump_render_tree_thread") -def process_output(port, test_info, test_types, test_args, target, output_dir, - crash, timeout, test_run_time, actual_checksum, + +def process_output(port, test_info, test_types, test_args, configuration, + output_dir, crash, timeout, test_run_time, actual_checksum, output, error): - """Receives the output from a test_shell process, subjects it to a number - of tests, and returns a list of failure types the test produced. + """Receives the output from a DumpRenderTree process, subjects it to a + number of tests, and returns a list of failure types the test produced. Args: port: port-specific hooks - proc: an active test_shell process + proc: an active DumpRenderTree process test_info: Object containing the test filename, uri and timeout test_types: list of test types to subject the output to test_args: arguments to be passed to each test - target: Debug or Release + configuration: Debug or Release output_dir: directory to put crash stack traces into - Returns: a list of failure objects and times for the test being processed + Returns: a TestResult object """ failures = [] @@ -79,16 +82,17 @@ def process_output(port, test_info, test_types, test_args, target, output_dir, failures.append(test_failures.FailureTimeout()) if crash: - logging.debug("Stacktrace for %s:\n%s" % (test_info.filename, error)) + _log.debug("Stacktrace for %s:\n%s" % (test_info.filename, error)) # Strip off "file://" since RelativeTestFilename expects # filesystem paths. - filename = os.path.join(output_dir, test_info.filename) + filename = os.path.join(output_dir, port.relative_test_filename( + test_info.filename)) filename = os.path.splitext(filename)[0] + "-stack.txt" port.maybe_make_directory(os.path.split(filename)[0]) - open(filename, "wb").write(error) + open(filename, "wb").write(error) # FIXME: This leaks a file handle. elif error: - logging.debug("Previous test output extra lines after dump:\n%s" % - error) + _log.debug("Previous test output extra lines after dump:\n%s" % + error) # Check the output and save the results. start_time = time.time() @@ -97,7 +101,7 @@ def process_output(port, test_info, test_types, test_args, target, output_dir, start_diff_time = time.time() new_failures = test_type.compare_output(port, test_info.filename, output, local_test_args, - target) + configuration) # Don't add any more failures if we already have a crash, so we don't # double-report those tests. We do double-report for timeouts since # we still want to see the text and image output. @@ -107,26 +111,27 @@ def process_output(port, test_info, test_types, test_args, target, output_dir, time.time() - start_diff_time) total_time_for_all_diffs = time.time() - start_diff_time - return TestStats(test_info.filename, failures, test_run_time, - total_time_for_all_diffs, time_for_diffs) + return TestResult(test_info.filename, failures, test_run_time, + total_time_for_all_diffs, time_for_diffs) -class TestStats: +class TestResult(object): def __init__(self, filename, failures, test_run_time, total_time_for_all_diffs, time_for_diffs): - self.filename = filename self.failures = failures + self.filename = filename self.test_run_time = test_run_time - self.total_time_for_all_diffs = total_time_for_all_diffs self.time_for_diffs = time_for_diffs + self.total_time_for_all_diffs = total_time_for_all_diffs + self.type = test_failures.determine_result_type(failures) class SingleTestThread(threading.Thread): """Thread wrapper for running a single test file.""" def __init__(self, port, image_path, shell_args, test_info, - test_types, test_args, target, output_dir): + test_types, test_args, configuration, output_dir): """ Args: port: object implementing port-specific hooks @@ -142,32 +147,32 @@ class SingleTestThread(threading.Thread): self._test_info = test_info self._test_types = test_types self._test_args = test_args - self._target = target + self._configuration = configuration self._output_dir = output_dir def run(self): - driver = self._port.start_test_driver(self._image_path, - self._shell_args) + test_info = self._test_info + driver = self._port.start_driver(self._image_path, self._shell_args) start = time.time() crash, timeout, actual_checksum, output, error = \ driver.run_test(test_info.uri.strip(), test_info.timeout, - test_info.image_hash) + test_info.image_hash()) end = time.time() - self._test_stats = process_output(self._port, - self._test_info, self._test_types, self._test_args, - self._target, self._output_dir, crash, timeout, end - start, + self._test_result = process_output(self._port, + test_info, self._test_types, self._test_args, + self._configuration, self._output_dir, crash, timeout, end - start, actual_checksum, output, error) driver.stop() - def get_test_stats(self): - return self._test_stats + def get_test_result(self): + return self._test_result class TestShellThread(threading.Thread): def __init__(self, port, filename_list_queue, result_queue, test_types, test_args, image_path, shell_args, options): - """Initialize all the local state for this test shell thread. + """Initialize all the local state for this DumpRenderTree thread. Args: port: interface to port-specific hooks @@ -178,7 +183,7 @@ class TestShellThread(threading.Thread): test_types: A list of TestType objects to run the test output against. test_args: A TestArguments object to pass to each TestType. - shell_args: Any extra arguments to be passed to test_shell.exe. + shell_args: Any extra arguments to be passed to DumpRenderTree. options: A property dictionary as produced by optparse. The command-line options should match those expected by run_webkit_tests; they are typically passed via the @@ -197,7 +202,7 @@ class TestShellThread(threading.Thread): self._canceled = False self._exception_info = None self._directory_timing_stats = {} - self._test_stats = [] + self._test_results = [] self._num_tests = 0 self._start_time = 0 self._stop_time = 0 @@ -214,10 +219,13 @@ class TestShellThread(threading.Thread): (number of tests in that directory, time to run the tests)""" return self._directory_timing_stats - def get_individual_test_stats(self): - """Returns a list of (test_filename, time_to_run_test, - total_time_for_all_diffs, time_for_diffs) tuples.""" - return self._test_stats + def get_test_results(self): + """Return the list of all tests run on this thread. + + This is used to calculate per-thread statistics. + + """ + return self._test_results def cancel(self): """Set a flag telling this thread to quit.""" @@ -242,17 +250,17 @@ class TestShellThread(threading.Thread): self._start_time = time.time() self._num_tests = 0 try: - logging.debug('%s starting' % (self.getName())) + _log.debug('%s starting' % (self.getName())) self._run(test_runner=None, result_summary=None) - logging.debug('%s done (%d tests)' % (self.getName(), - self.get_num_tests())) + _log.debug('%s done (%d tests)' % (self.getName(), + self.get_num_tests())) except: # Save the exception for our caller to see. self._exception_info = sys.exc_info() self._stop_time = time.time() # Re-raise it and die. - logging.error('%s dying: %s' % (self.getName(), - self._exception_info)) + _log.error('%s dying: %s' % (self.getName(), + self._exception_info)) raise self._stop_time = time.time() @@ -275,8 +283,8 @@ class TestShellThread(threading.Thread): try: batch_size = int(self._options.batch_size) except: - logging.info("Ignoring invalid batch size '%s'" % - self._options.batch_size) + _log.info("Ignoring invalid batch size '%s'" % + self._options.batch_size) # Append tests we're running to the existing tests_run.txt file. # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput. @@ -286,7 +294,7 @@ class TestShellThread(threading.Thread): while True: if self._canceled: - logging.info('Testing canceled') + _log.info('Testing canceled') tests_run_file.close() return @@ -300,7 +308,7 @@ class TestShellThread(threading.Thread): self._current_dir, self._filename_list = \ self._filename_list_queue.get_nowait() except Queue.Empty: - self._kill_test_shell() + self._kill_dump_render_tree() tests_run_file.close() return @@ -313,31 +321,33 @@ class TestShellThread(threading.Thread): batch_count += 1 self._num_tests += 1 if self._options.run_singly: - failures = self._run_test_singly(test_info) + result = self._run_test_singly(test_info) else: - failures = self._run_test(test_info) + result = self._run_test(test_info) filename = test_info.filename tests_run_file.write(filename + "\n") - if failures: - # Check and kill test shell if we need too. - if len([1 for f in failures if f.should_kill_test_shell()]): - self._kill_test_shell() + if result.failures: + # Check and kill DumpRenderTree if we need to. + if len([1 for f in result.failures + if f.should_kill_dump_render_tree()]): + self._kill_dump_render_tree() # Reset the batch count since the shell just bounced. batch_count = 0 # Print the error message(s). - error_str = '\n'.join([' ' + f.message() for f in failures]) - logging.debug("%s %s failed:\n%s" % (self.getName(), - self._port.relative_test_filename(filename), - error_str)) + error_str = '\n'.join([' ' + f.message() for + f in result.failures]) + _log.debug("%s %s failed:\n%s" % (self.getName(), + self._port.relative_test_filename(filename), + error_str)) else: - logging.debug("%s %s passed" % (self.getName(), - self._port.relative_test_filename(filename))) - self._result_queue.put((filename, failures)) + _log.debug("%s %s passed" % (self.getName(), + self._port.relative_test_filename(filename))) + self._result_queue.put(result) if batch_size > 0 and batch_count > batch_size: # Bounce the shell and reset count. - self._kill_test_shell() + self._kill_dump_render_tree() batch_count = 0 if test_runner: @@ -353,61 +363,64 @@ class TestShellThread(threading.Thread): Args: test_info: Object containing the test filename, uri and timeout - Return: - A list of TestFailure objects describing the error. + Returns: + A TestResult + """ worker = SingleTestThread(self._port, self._image_path, self._shell_args, test_info, self._test_types, self._test_args, - self._options.target, + self._options.configuration, self._options.results_directory) worker.start() - # When we're running one test per test_shell process, we can enforce - # a hard timeout. the test_shell watchdog uses 2.5x the timeout - # We want to be larger than that. + # When we're running one test per DumpRenderTree process, we can + # enforce a hard timeout. The DumpRenderTree watchdog uses 2.5x + # the timeout; we want to be larger than that. worker.join(int(test_info.timeout) * 3.0 / 1000.0) if worker.isAlive(): # If join() returned with the thread still running, the - # test_shell.exe is completely hung and there's nothing + # DumpRenderTree is completely hung and there's nothing # more we can do with it. We have to kill all the - # test_shells to free it up. If we're running more than - # one test_shell thread, we'll end up killing the other - # test_shells too, introducing spurious crashes. We accept that - # tradeoff in order to avoid losing the rest of this thread's - # results. - logging.error('Test thread hung: killing all test_shells') + # DumpRenderTrees to free it up. If we're running more than + # one DumpRenderTree thread, we'll end up killing the other + # DumpRenderTrees too, introducing spurious crashes. We accept + # that tradeoff in order to avoid losing the rest of this + # thread's results. + _log.error('Test thread hung: killing all DumpRenderTrees') worker._driver.stop() try: - stats = worker.get_test_stats() - self._test_stats.append(stats) - failures = stats.failures + result = worker.get_test_result() except AttributeError, e: failures = [] - logging.error('Cannot get results of test: %s' % - test_info.filename) + _log.error('Cannot get results of test: %s' % + test_info.filename) + result = TestResult(test_info.filename, failures=[], + test_run_time=0, total_time_for_all_diffs=0, + time_for_diffs=0) - return failures + return result def _run_test(self, test_info): - """Run a single test file using a shared test_shell process. + """Run a single test file using a shared DumpRenderTree process. Args: test_info: Object containing the test filename, uri and timeout - Return: + Returns: A list of TestFailure objects describing the error. + """ - self._ensure_test_shell_is_running() + self._ensure_dump_render_tree_is_running() # The pixel_hash is used to avoid doing an image dump if the # checksums match, so it should be set to a blank value if we # are generating a new baseline. (Otherwise, an image from a # previous run will be copied into the baseline.) - image_hash = test_info.image_hash + image_hash = test_info.image_hash() if image_hash and self._test_args.new_baseline: image_hash = "" start = time.time() @@ -415,26 +428,27 @@ class TestShellThread(threading.Thread): self._driver.run_test(test_info.uri, test_info.timeout, image_hash) end = time.time() - stats = process_output(self._port, test_info, self._test_types, - self._test_args, self._options.target, - self._options.results_directory, crash, - timeout, end - start, actual_checksum, - output, error) + result = process_output(self._port, test_info, self._test_types, + self._test_args, self._options.configuration, + self._options.results_directory, crash, + timeout, end - start, actual_checksum, + output, error) + self._test_results.append(result) + return result + + def _ensure_dump_render_tree_is_running(self): + """Start the shared DumpRenderTree, if it's not running. - self._test_stats.append(stats) - return stats.failures + This is not for use when running tests singly, since those each start + a separate DumpRenderTree in their own thread. - def _ensure_test_shell_is_running(self): - """Start the shared test shell, if it's not running. Not for use when - running tests singly, since those each start a separate test shell in - their own thread. """ if (not self._driver or self._driver.poll() is not None): self._driver = self._port.start_driver( self._image_path, self._shell_args) - def _kill_test_shell(self): - """Kill the test shell process if it's running.""" + def _kill_dump_render_tree(self): + """Kill the DumpRenderTree process if it's running.""" if self._driver: self._driver.stop() self._driver = None diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py index 520ab1f..cee44ad 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py @@ -29,12 +29,11 @@ import logging import os -import simplejson - -from layout_package import json_results_generator -from layout_package import test_expectations -from layout_package import test_failures +from webkitpy.layout_tests.layout_package import json_results_generator +from webkitpy.layout_tests.layout_package import test_expectations +from webkitpy.layout_tests.layout_package import test_failures +import webkitpy.thirdparty.simplejson as simplejson class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator): """A JSON results generator for layout tests.""" diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py index 84be0e1..6263540 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py @@ -29,14 +29,17 @@ import logging import os -import simplejson import subprocess import sys import time import urllib2 import xml.dom.minidom -from layout_package import test_expectations +from webkitpy.layout_tests.layout_package import test_expectations +import webkitpy.thirdparty.simplejson as simplejson + +_log = logging.getLogger("webkitpy.layout_tests.layout_package." + "json_results_generator") class JSONResultsGenerator(object): @@ -154,8 +157,8 @@ class JSONResultsGenerator(object): # Check if we have the archived JSON file on the buildbot server. results_file_url = (self._builder_base_url + self._build_name + "/" + self.RESULTS_FILENAME) - logging.error("Local results.json file does not exist. Grabbing " - "it off the archive at " + results_file_url) + _log.error("Local results.json file does not exist. Grabbing " + "it off the archive at " + results_file_url) try: results_file = urllib2.urlopen(results_file_url) @@ -177,11 +180,11 @@ class JSONResultsGenerator(object): try: results_json = simplejson.loads(old_results) except: - logging.debug("results.json was not valid JSON. Clobbering.") + _log.debug("results.json was not valid JSON. Clobbering.") # The JSON file is not valid JSON. Just clobber the results. results_json = {} else: - logging.debug('Old JSON results do not exist. Starting fresh.') + _log.debug('Old JSON results do not exist. Starting fresh.') results_json = {} return results_json, error @@ -192,14 +195,14 @@ class JSONResultsGenerator(object): if error: # If there was an error don't write a results.json # file at all as it would lose all the information on the bot. - logging.error("Archive directory is inaccessible. Not modifying " - "or clobbering the results.json file: " + str(error)) + _log.error("Archive directory is inaccessible. Not modifying " + "or clobbering the results.json file: " + str(error)) return None builder_name = self._builder_name if results_json and builder_name not in results_json: - logging.debug("Builder name (%s) is not in the results.json file." - % builder_name) + _log.debug("Builder name (%s) is not in the results.json file." + % builder_name) self._convert_json_to_current_version(results_json) @@ -307,16 +310,20 @@ class JSONResultsGenerator(object): # These next two branches test to see which source repos we can # pull revisions from. if hasattr(self._port, 'path_from_webkit_base'): - path_to_webkit = self._port.path_from_webkit_base() + path_to_webkit = self._port.path_from_webkit_base('WebCore') self._insert_item_into_raw_list(results_for_builder, self._get_svn_revision(path_to_webkit), self.WEBKIT_SVN) if hasattr(self._port, 'path_from_chromium_base'): - path_to_chrome = self._port.path_from_chromium_base() - self._insert_item_into_raw_list(results_for_builder, - self._get_svn_revision(path_to_chrome), - self.CHROME_SVN) + try: + path_to_chrome = self._port.path_from_chromium_base() + self._insert_item_into_raw_list(results_for_builder, + self._get_svn_revision(path_to_chrome), + self.CHROME_SVN) + except AssertionError: + # We're not in a Chromium checkout, that's ok. + pass self._insert_item_into_raw_list(results_for_builder, int(time.time()), diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py index 72b30a1..930b9e4 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py @@ -34,6 +34,10 @@ and rewritten repeatedly, without producing multiple lines of output. It can be used to produce effects like progress bars. """ +import logging + +_log = logging.getLogger("webkitpy.layout_tests.metered_stream") + class MeteredStream: """This class is a wrapper around a stream that allows you to implement @@ -57,8 +61,7 @@ class MeteredStream: self._last_update = "" def write(self, txt): - """Write text directly to the stream, overwriting and resetting the - meter.""" + """Write to the stream, overwriting and resetting the meter.""" if self._dirty: self.update("") self._dirty = False @@ -68,22 +71,43 @@ class MeteredStream: """Flush any buffered output.""" self._stream.flush() - def update(self, str): - """Write an update to the stream that will get overwritten by the next - update() or by a write(). + def progress(self, str): + """ + Write a message to the stream that will get overwritten. This is used for progress updates that don't need to be preserved in - the log. Note that verbose disables this routine; we have this in - case we are logging lots of output and the update()s will get lost - or won't work properly (typically because verbose streams are - redirected to files. - - TODO(dpranke): figure out if there is a way to detect if we're writing - to a stream that handles CRs correctly (e.g., terminals). That might - be a cleaner way of handling this. + the log. If the MeteredStream was initialized with verbose==True, + then this output is discarded. We have this in case we are logging + lots of output and the update()s will get lost or won't work + properly (typically because verbose streams are redirected to files). + """ if self._verbose: return + self._write(str) + + def update(self, str): + """ + Write a message that is also included when logging verbosely. + + This routine preserves the same console logging behavior as progress(), + but will also log the message if verbose() was true. + + """ + # Note this is a separate routine that calls either into the logger + # or the metering stream. We have to be careful to avoid a layering + # inversion (stream calling back into the logger). + if self._verbose: + _log.info(str) + else: + self._write(str) + + def _write(self, str): + """Actually write the message to the stream.""" + + # FIXME: Figure out if there is a way to detect if we're writing + # to a stream that handles CRs correctly (e.g., terminals). That might + # be a cleaner way of handling this. # Print the necessary number of backspaces to erase the previous # message. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py index 01add62..38223dd 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py @@ -36,7 +36,10 @@ import os import re import sys -import simplejson +import webkitpy.thirdparty.simplejson as simplejson + +_log = logging.getLogger("webkitpy.layout_tests.layout_package." + "test_expectations") # Test expectation and modifier constants. (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, TIMEOUT, CRASH, SKIP, WONTFIX, @@ -46,11 +49,46 @@ import simplejson (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4) +def result_was_expected(result, expected_results, test_needs_rebaselining, + test_is_skipped): + """Returns whether we got a result we were expecting. + Args: + result: actual result of a test execution + expected_results: set of results listed in test_expectations + test_needs_rebaselining: whether test was marked as REBASELINE + test_is_skipped: whether test was marked as SKIP""" + if result in expected_results: + return True + if result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and FAIL in expected_results: + return True + if result == MISSING and test_needs_rebaselining: + return True + if result == SKIP and test_is_skipped: + return True + return False + + +def remove_pixel_failures(expected_results): + """Returns a copy of the expected results for a test, except that we + drop any pixel failures and return the remaining expectations. For example, + if we're not running pixel tests, then tests expected to fail as IMAGE + will PASS.""" + expected_results = expected_results.copy() + if IMAGE in expected_results: + expected_results.remove(IMAGE) + expected_results.add(PASS) + if IMAGE_PLUS_TEXT in expected_results: + expected_results.remove(IMAGE_PLUS_TEXT) + expected_results.add(TEXT) + return expected_results + + class TestExpectations: TEST_LIST = "test_expectations.txt" def __init__(self, port, tests, expectations, test_platform_name, - is_debug_mode, is_lint_mode, tests_are_present=True): + is_debug_mode, is_lint_mode, tests_are_present=True, + overrides=None): """Loads and parses the test expectations given in the string. Args: port: handle to object containing platform-specific functionality @@ -67,10 +105,14 @@ class TestExpectations: system and can be probed for. This is useful for distinguishing test files from directories, and is needed by the LTTF dashboard, where the files aren't actually locally present. + overrides: test expectations that are allowed to override any + entries in |expectations|. This is used by callers + that need to manage two sets of expectations (e.g., upstream + and downstream expectations). """ self._expected_failures = TestExpectationsFile(port, expectations, tests, test_platform_name, is_debug_mode, is_lint_mode, - tests_are_present=tests_are_present) + tests_are_present=tests_are_present, overrides=overrides) # TODO(ojan): Allow for removing skipped tests when getting the list of # tests to run, but not when getting metrics. @@ -101,12 +143,16 @@ class TestExpectations: retval = [] for expectation in expectations: - for item in TestExpectationsFile.EXPECTATIONS.items(): - if item[1] == expectation: - retval.append(item[0]) - break + retval.append(self.expectation_to_string(expectation)) - return " ".join(retval).upper() + return " ".join(retval) + + def expectation_to_string(self, expectation): + """Return the uppercased string equivalent of a given expectation.""" + for item in TestExpectationsFile.EXPECTATIONS.items(): + if item[1] == expectation: + return item[0].upper() + return "" def get_timeline_for_test(self, test): return self._expected_failures.get_timeline_for_test(test) @@ -117,14 +163,13 @@ class TestExpectations: def get_tests_with_timeline(self, timeline): return self._expected_failures.get_tests_with_timeline(timeline) - def matches_an_expected_result(self, test, result): - """Returns whether we got one of the expected results for this test.""" - return (result in self._expected_failures.get_expectations(test) or - (result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and - FAIL in self._expected_failures.get_expectations(test)) or - result == MISSING and self.is_rebaselining(test) or - result == SKIP and self._expected_failures.has_modifier(test, - SKIP)) + def matches_an_expected_result(self, test, result, + pixel_tests_are_enabled): + expected_results = self._expected_failures.get_expectations(test) + if not pixel_tests_are_enabled: + expected_results = remove_pixel_failures(expected_results) + return result_was_expected(result, expected_results, + self.is_rebaselining(test), self.has_modifier(test, SKIP)) def is_rebaselining(self, test): return self._expected_failures.has_modifier(test, REBASELINE) @@ -232,8 +277,8 @@ class TestExpectationsFile: IMAGE: ('image mismatch', 'image mismatch'), IMAGE_PLUS_TEXT: ('image and text mismatch', 'image and text mismatch'), - CRASH: ('test shell crash', - 'test shell crashes'), + CRASH: ('DumpRenderTree crash', + 'DumpRenderTree crashes'), TIMEOUT: ('test timed out', 'tests timed out'), MISSING: ('no expected result found', 'no expected results found')} @@ -261,7 +306,7 @@ class TestExpectationsFile: def __init__(self, port, expectations, full_test_list, test_platform_name, is_debug_mode, is_lint_mode, suppress_errors=False, - tests_are_present=True): + tests_are_present=True, overrides=None): """ expectations: Contents of the expectations file full_test_list: The list of all tests to be run pending processing of @@ -275,6 +320,10 @@ class TestExpectationsFile: tests_are_present: Whether the test files are present in the local filesystem. The LTTF Dashboard uses False here to avoid having to keep a local copy of the tree. + overrides: test expectations that are allowed to override any + entries in |expectations|. This is used by callers + that need to manage two sets of expectations (e.g., upstream + and downstream expectations). """ self._port = port @@ -284,6 +333,7 @@ class TestExpectationsFile: self._is_debug_mode = is_debug_mode self._is_lint_mode = is_lint_mode self._tests_are_present = tests_are_present + self._overrides = overrides self._suppress_errors = suppress_errors self._errors = [] self._non_fatal_errors = [] @@ -311,7 +361,50 @@ class TestExpectationsFile: self._timeline_to_tests = self._dict_of_sets(self.TIMELINES) self._result_type_to_tests = self._dict_of_sets(self.RESULT_TYPES) - self._read(self._get_iterable_expectations()) + self._read(self._get_iterable_expectations(self._expectations), + overrides_allowed=False) + + # List of tests that are in the overrides file (used for checking for + # duplicates inside the overrides file itself). Note that just because + # a test is in this set doesn't mean it's necessarily overridding a + # expectation in the regular expectations; the test might not be + # mentioned in the regular expectations file at all. + self._overridding_tests = set() + + if overrides: + self._read(self._get_iterable_expectations(self._overrides), + overrides_allowed=True) + + self._handle_any_read_errors() + self._process_tests_without_expectations() + + def _handle_any_read_errors(self): + if not self._suppress_errors and ( + len(self._errors) or len(self._non_fatal_errors)): + if self._is_debug_mode: + build_type = 'DEBUG' + else: + build_type = 'RELEASE' + _log.error('') + _log.error("FAILURES FOR PLATFORM: %s, BUILD_TYPE: %s" % + (self._test_platform_name.upper(), build_type)) + + for error in self._non_fatal_errors: + _log.error(error) + _log.error('') + + if len(self._errors): + raise SyntaxError('\n'.join(map(str, self._errors))) + + def _process_tests_without_expectations(self): + expectations = set([PASS]) + options = [] + modifiers = [] + if self._full_test_list: + for test in self._full_test_list: + if not test in self._test_list_paths: + self._add_test(test, modifiers, expectations, options, + overrides_allowed=False) def _dict_of_sets(self, strings_to_constants): """Takes a dict of strings->constants and returns a dict mapping @@ -321,12 +414,11 @@ class TestExpectationsFile: d[c] = set() return d - def _get_iterable_expectations(self): + def _get_iterable_expectations(self, expectations_str): """Returns an object that can be iterated over. Allows for not caring about whether we're iterating over a file or a new-line separated string.""" - iterable = [x + "\n" for x in - self._expectations.split("\n")] + iterable = [x + "\n" for x in expectations_str.split("\n")] # Strip final entry if it's empty to avoid added in an extra # newline. if iterable[-1] == "\n": @@ -388,7 +480,7 @@ class TestExpectationsFile: the updated string. """ - f_orig = self._get_iterable_expectations() + f_orig = self._get_iterable_expectations(self._expectations) f_new = [] tests_removed = 0 @@ -400,20 +492,20 @@ class TestExpectationsFile: platform) if action == NO_CHANGE: # Save the original line back to the file - logging.debug('No change to test: %s', line) + _log.debug('No change to test: %s', line) f_new.append(line) elif action == REMOVE_TEST: tests_removed += 1 - logging.info('Test removed: %s', line) + _log.info('Test removed: %s', line) elif action == REMOVE_PLATFORM: parts = line.split(':') new_options = parts[0].replace(platform.upper() + ' ', '', 1) new_line = ('%s:%s' % (new_options, parts[1])) f_new.append(new_line) tests_updated += 1 - logging.info('Test updated: ') - logging.info(' old: %s', line) - logging.info(' new: %s', new_line) + _log.info('Test updated: ') + _log.info(' old: %s', line) + _log.info(' new: %s', new_line) elif action == ADD_PLATFORMS_EXCEPT_THIS: parts = line.split(':') new_options = parts[0] @@ -430,15 +522,15 @@ class TestExpectationsFile: new_line = ('%s:%s' % (new_options, parts[1])) f_new.append(new_line) tests_updated += 1 - logging.info('Test updated: ') - logging.info(' old: %s', line) - logging.info(' new: %s', new_line) + _log.info('Test updated: ') + _log.info(' old: %s', line) + _log.info(' new: %s', new_line) else: - logging.error('Unknown update action: %d; line: %s', - action, line) + _log.error('Unknown update action: %d; line: %s', + action, line) - logging.info('Total tests removed: %d', tests_removed) - logging.info('Total tests updated: %d', tests_updated) + _log.info('Total tests removed: %d', tests_removed) + _log.info('Total tests updated: %d', tests_updated) return "".join(f_new) @@ -574,7 +666,7 @@ class TestExpectationsFile: self._all_expectations[test].append( ModifiersAndExpectations(options, expectations)) - def _read(self, expectations): + def _read(self, expectations, overrides_allowed): """For each test in an expectations iterable, generate the expectations for it.""" lineno = 0 @@ -625,30 +717,7 @@ class TestExpectationsFile: tests = self._expand_tests(test_list_path) self._add_tests(tests, expectations, test_list_path, lineno, - modifiers, options) - - if not self._suppress_errors and ( - len(self._errors) or len(self._non_fatal_errors)): - if self._is_debug_mode: - build_type = 'DEBUG' - else: - build_type = 'RELEASE' - print "\nFAILURES FOR PLATFORM: %s, BUILD_TYPE: %s" \ - % (self._test_platform_name.upper(), build_type) - - for error in self._non_fatal_errors: - logging.error(error) - if len(self._errors): - raise SyntaxError('\n'.join(map(str, self._errors))) - - # Now add in the tests that weren't present in the expectations file - expectations = set([PASS]) - options = [] - modifiers = [] - if self._full_test_list: - for test in self._full_test_list: - if not test in self._test_list_paths: - self._add_test(test, modifiers, expectations, options) + modifiers, options, overrides_allowed) def _get_options_list(self, listString): return [part.strip().lower() for part in listString.strip().split(' ')] @@ -692,15 +761,18 @@ class TestExpectationsFile: return path def _add_tests(self, tests, expectations, test_list_path, lineno, - modifiers, options): + modifiers, options, overrides_allowed): for test in tests: - if self._already_seen_test(test, test_list_path, lineno): + if self._already_seen_test(test, test_list_path, lineno, + overrides_allowed): continue self._clear_expectations_for_test(test, test_list_path) - self._add_test(test, modifiers, expectations, options) + self._add_test(test, modifiers, expectations, options, + overrides_allowed) - def _add_test(self, test, modifiers, expectations, options): + def _add_test(self, test, modifiers, expectations, options, + overrides_allowed): """Sets the expected state for a given test. This routine assumes the test has not been added before. If it has, @@ -711,7 +783,9 @@ class TestExpectationsFile: test: test to add modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.) expectations: sequence of expectations (PASS, IMAGE, etc.) - options: sequence of keywords and bug identifiers.""" + options: sequence of keywords and bug identifiers. + overrides_allowed: whether we're parsing the regular expectations + or the overridding expectations""" self._test_to_expectations[test] = expectations for expectation in expectations: self._expectation_to_tests[expectation].add(test) @@ -739,6 +813,9 @@ class TestExpectationsFile: else: self._result_type_to_tests[FAIL].add(test) + if overrides_allowed: + self._overridding_tests.add(test) + def _clear_expectations_for_test(self, test, test_list_path): """Remove prexisting expectations for this test. This happens if we are seeing a more precise path @@ -763,7 +840,8 @@ class TestExpectationsFile: if test in set_of_tests: set_of_tests.remove(test) - def _already_seen_test(self, test, test_list_path, lineno): + def _already_seen_test(self, test, test_list_path, lineno, + allow_overrides): """Returns true if we've already seen a more precise path for this test than the test_list_path. """ @@ -772,8 +850,19 @@ class TestExpectationsFile: prev_base_path = self._test_list_paths[test] if (prev_base_path == os.path.normpath(test_list_path)): - self._add_error(lineno, 'Duplicate expectations.', test) - return True + if (not allow_overrides or test in self._overridding_tests): + if allow_overrides: + expectation_source = "override" + else: + expectation_source = "expectation" + self._add_error(lineno, 'Duplicate %s.' % expectation_source, + test) + return True + else: + # We have seen this path, but that's okay because its + # in the overrides and the earlier path was in the + # expectations. + return False # Check if we've already seen a more precise path. return prev_base_path.startswith(os.path.normpath(test_list_path)) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py new file mode 100644 index 0000000..d11f3e2 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py @@ -0,0 +1,169 @@ +#!/usr/bin/python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for test_expectations.py.""" + +import os +import sys +import unittest + +try: + d = os.path.dirname(__file__) +except NameError: + d = os.path.dirname(sys.argv[0]) + +sys.path.append(os.path.abspath(os.path.join(d, '..'))) +sys.path.append(os.path.abspath(os.path.join(d, '../../thirdparty'))) + +import port +from test_expectations import * + +class FunctionsTest(unittest.TestCase): + def test_result_was_expected(self): + # test basics + self.assertEquals(result_was_expected(PASS, set([PASS]), + False, False), True) + self.assertEquals(result_was_expected(TEXT, set([PASS]), + False, False), False) + + # test handling of FAIL expectations + self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]), + False, False), True) + self.assertEquals(result_was_expected(IMAGE, set([FAIL]), + False, False), True) + self.assertEquals(result_was_expected(TEXT, set([FAIL]), + False, False), True) + self.assertEquals(result_was_expected(CRASH, set([FAIL]), + False, False), False) + + # test handling of SKIPped tests and results + self.assertEquals(result_was_expected(SKIP, set([CRASH]), + False, True), True) + self.assertEquals(result_was_expected(SKIP, set([CRASH]), + False, False), False) + + # test handling of MISSING results and the REBASELINE modifier + self.assertEquals(result_was_expected(MISSING, set([PASS]), + True, False), True) + self.assertEquals(result_was_expected(MISSING, set([PASS]), + False, False), False) + + def test_remove_pixel_failures(self): + self.assertEquals(remove_pixel_failures(set([TEXT])), + set([TEXT])) + self.assertEquals(remove_pixel_failures(set([PASS])), + set([PASS])) + self.assertEquals(remove_pixel_failures(set([IMAGE])), + set([PASS])) + self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])), + set([TEXT])) + self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])), + set([PASS, CRASH])) + + +class TestExpectationsTest(unittest.TestCase): + + def __init__(self, testFunc, setUp=None, tearDown=None, description=None): + self._port = port.get('test', None) + self._exp = None + unittest.TestCase.__init__(self, testFunc) + + def get_test(self, test_name): + return os.path.join(self._port.layout_tests_dir(), test_name) + + def get_basic_tests(self): + return [self.get_test('fast/html/article-element.html'), + self.get_test('fast/html/header-element.html'), + self.get_test('fast/html/keygen.html'), + self.get_test('fast/html/tab-order.html'), + self.get_test('fast/events/space-scroll-event.html'), + self.get_test('fast/events/tab-imagemap.html')] + + def get_basic_expectations(self): + return """ +BUG_TEST : fast/html/article-element.html = TEXT +BUG_TEST SKIP : fast/html/keygen.html = CRASH +BUG_TEST REBASELINE : fast/htmltab-order.html = MISSING +BUG_TEST : fast/events = IMAGE +""" + + def parse_exp(self, expectations, overrides=None): + self._exp = TestExpectations(self._port, + tests=self.get_basic_tests(), + expectations=expectations, + test_platform_name=self._port.test_platform_name(), + is_debug_mode=False, + is_lint_mode=False, + tests_are_present=True, + overrides=overrides) + + def assert_exp(self, test, result): + self.assertEquals(self._exp.get_expectations(self.get_test(test)), + set([result])) + + def test_basic(self): + self.parse_exp(self.get_basic_expectations()) + self.assert_exp('fast/html/article-element.html', TEXT) + self.assert_exp('fast/events/tab-imagemap.html', IMAGE) + self.assert_exp('fast/html/header-element.html', PASS) + + def test_duplicates(self): + self.assertRaises(SyntaxError, self.parse_exp, """ +BUG_TEST : fast/html/article-element.html = TEXT +BUG_TEST : fast/html/article-element.html = IMAGE""") + self.assertRaises(SyntaxError, self.parse_exp, + self.get_basic_expectations(), """ +BUG_TEST : fast/html/article-element.html = TEXT +BUG_TEST : fast/html/article-element.html = IMAGE""") + + def test_overrides(self): + self.parse_exp(self.get_basic_expectations(), """ +BUG_OVERRIDE : fast/html/article-element.html = IMAGE""") + self.assert_exp('fast/html/article-element.html', IMAGE) + + def test_matches_an_expected_result(self): + + def match(test, result, pixel_tests_enabled): + return self._exp.matches_an_expected_result( + self.get_test(test), result, pixel_tests_enabled) + + self.parse_exp(self.get_basic_expectations()) + self.assertTrue(match('fast/html/article-element.html', TEXT, True)) + self.assertTrue(match('fast/html/article-element.html', TEXT, False)) + self.assertFalse(match('fast/html/article-element.html', CRASH, True)) + self.assertFalse(match('fast/html/article-element.html', CRASH, False)) + + self.assertTrue(match('fast/events/tab-imagemap.html', IMAGE, True)) + self.assertTrue(match('fast/events/tab-imagemap.html', PASS, False)) + + self.assertTrue(match('fast/html/keygen.html', SKIP, False)) + self.assertTrue(match('fast/html/tab-order.html', PASS, False)) + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py index 56d7b5a..60bdbca 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py @@ -79,8 +79,8 @@ class TestFailure(object): """Returns an HTML string to be included on the results.html page.""" raise NotImplemented - def should_kill_test_shell(self): - """Returns True if we should kill the test shell before the next + def should_kill_dump_render_tree(self): + """Returns True if we should kill DumpRenderTree before the next test.""" return False @@ -110,7 +110,7 @@ class FailureWithType(TestFailure): def __init__(self, test_type): TestFailure.__init__(self) - # TODO(ojan): This class no longer needs to know the test_type. + # FIXME: This class no longer needs to know the test_type. self._test_type = test_type # Filename suffixes used by ResultHtmlOutput. @@ -127,6 +127,9 @@ class FailureWithType(TestFailure): single item is the [actual] filename suffix. If out_names is empty, returns the empty string. """ + # FIXME: Seems like a bad idea to separate the display name data + # from the path data by hard-coding the display name here + # and passing in the path information via out_names. links = [''] uris = [self.relative_output_filename(filename, fn) for fn in out_names] @@ -138,6 +141,8 @@ class FailureWithType(TestFailure): links.append("<a href='%s'>diff</a>" % uris[2]) if len(uris) > 3: links.append("<a href='%s'>wdiff</a>" % uris[3]) + if len(uris) > 4: + links.append("<a href='%s'>pretty diff</a>" % uris[4]) return ' '.join(links) def result_html_output(self, filename): @@ -145,7 +150,7 @@ class FailureWithType(TestFailure): class FailureTimeout(TestFailure): - """Test timed out. We also want to restart the test shell if this + """Test timed out. We also want to restart DumpRenderTree if this happens.""" @staticmethod @@ -155,7 +160,7 @@ class FailureTimeout(TestFailure): def result_html_output(self, filename): return "<strong>%s</strong>" % self.message() - def should_kill_test_shell(self): + def should_kill_dump_render_tree(self): return True @@ -172,7 +177,7 @@ class FailureCrash(TestFailure): return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(), stack) - def should_kill_test_shell(self): + def should_kill_dump_render_tree(self): return True @@ -192,9 +197,10 @@ class FailureMissingResult(FailureWithType): class FailureTextMismatch(FailureWithType): """Text diff output failed.""" # Filename suffixes used by ResultHtmlOutput. + # FIXME: Why don't we use the constants from TestTypeBase here? OUT_FILENAMES = ["-actual.txt", "-expected.txt", "-diff.txt"] OUT_FILENAMES_WDIFF = ["-actual.txt", "-expected.txt", "-diff.txt", - "-wdiff.html"] + "-wdiff.html", "-pretty-diff.html"] def __init__(self, test_type, has_wdiff): FailureWithType.__init__(self, test_type) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_files.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_files.py index 3c087c0..6754fa6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_files.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_files.py @@ -36,9 +36,16 @@ under that directory.""" import glob import os +import time + +from webkitpy.common.system import logutils + + +_log = logutils.get_logger(__file__) + # When collecting test cases, we include any file with these extensions. -_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl', +_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.xhtmlmp', '.pl', '.php', '.svg']) # When collecting test cases, skip these directories _skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests']) @@ -51,6 +58,7 @@ def gather_test_files(port, paths): paths: a list of command line paths relative to the webkit/tests directory. glob patterns are ok. """ + gather_start_time = time.time() paths_to_walk = set() # if paths is empty, provide a pre-defined list. if paths: @@ -73,10 +81,16 @@ def gather_test_files(port, paths): continue for root, dirs, files in os.walk(path): - # don't walk skipped directories and sub directories + # Don't walk skipped directories or their sub-directories. if os.path.basename(root) in _skipped_directories: del dirs[:] continue + # This copy and for-in is slightly inefficient, but + # the extra walk avoidance consistently shaves .5 seconds + # off of total walk() time on my MacBook Pro. + for directory in dirs[:]: + if directory in _skipped_directories: + dirs.remove(directory) for filename in files: if _has_supported_extension(filename): @@ -84,6 +98,9 @@ def gather_test_files(port, paths): filename = os.path.normpath(filename) test_files.add(filename) + gather_time = time.time() - gather_start_time + _log.debug("Test gathering took %f seconds" % gather_time) + return test_files diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/__init__.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/__init__.py index 3509675..e3ad6f4 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/__init__.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/__init__.py @@ -29,37 +29,4 @@ """Port-specific entrypoints for the layout tests test infrastructure.""" - -import sys - - -def get(port_name=None, options=None): - """Returns an object implementing the Port interface. If - port_name is None, this routine attempts to guess at the most - appropriate port on this platform.""" - port_to_use = port_name - if port_to_use is None: - if sys.platform == 'win32': - port_to_use = 'chromium-win' - elif sys.platform == 'linux2': - port_to_use = 'chromium-linux' - elif sys.platform == 'darwin': - port_to_use = 'chromium-mac' - - if port_to_use == 'test': - import test - return test.TestPort(port_name, options) - elif port_to_use.startswith('mac'): - import mac - return mac.MacPort(port_name, options) - elif port_to_use.startswith('chromium-mac'): - import chromium_mac - return chromium_mac.ChromiumMacPort(port_name, options) - elif port_to_use.startswith('chromium-linux'): - import chromium_linux - return chromium_linux.ChromiumLinuxPort(port_name, options) - elif port_to_use.startswith('chromium-win'): - import chromium_win - return chromium_win.ChromiumWinPort(port_name, options) - - raise NotImplementedError('unsupported port: %s' % port_to_use) +from factory import get diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/apache_http_server.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/apache_http_server.py index 9ff3671..1dd5b93 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/apache_http_server.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/apache_http_server.py @@ -38,6 +38,8 @@ import sys import http_server_base +_log = logging.getLogger("webkitpy.layout_tests.port.apache_http_server") + class LayoutTestApacheHttpd(http_server_base.HttpServerBase): @@ -77,14 +79,15 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): error_log = self._cygwin_safe_join(output_dir, "error_log.txt") document_root = self._cygwin_safe_join(test_dir, "http", "tests") + # FIXME: We shouldn't be calling a protected method of _port_obj! executable = self._port_obj._path_to_apache() if self._is_cygwin(): executable = self._get_cygwin_path(executable) cmd = [executable, - '-f', self._get_apache_config_file_path(test_dir, output_dir), - '-C', "\'DocumentRoot %s\'" % document_root, - '-c', "\'Alias /js-test-resources %s\'" % js_test_resources_dir, + '-f', "\"%s\"" % self._get_apache_config_file_path(test_dir, output_dir), + '-C', "\'DocumentRoot \"%s\"\'" % document_root, + '-c', "\'Alias /js-test-resources \"%s\"'" % js_test_resources_dir, '-C', "\'Listen %s\'" % "127.0.0.1:8000", '-C', "\'Listen %s\'" % "127.0.0.1:8081", '-c', "\'TypesConfig \"%s\"\'" % mime_types_path, @@ -174,7 +177,7 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): It will listen to 127.0.0.1 on each of the given port. """ return '\n'.join(('<VirtualHost 127.0.0.1:%s>' % port, - 'DocumentRoot %s' % document_root, + 'DocumentRoot "%s"' % document_root, ssl and 'SSLEngine On' or '', '</VirtualHost>', '')) @@ -188,7 +191,7 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): shell=True) err = self._httpd_proc.stderr.read() if len(err): - logging.debug(err) + _log.debug(err) return False return True @@ -197,22 +200,23 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): # Stop any currently running servers. self.stop() - logging.debug("Starting apache http server") + _log.debug("Starting apache http server") server_started = self.wait_for_action(self._start_httpd_process) if server_started: - logging.debug("Apache started. Testing ports") + _log.debug("Apache started. Testing ports") server_started = self.wait_for_action( self.is_server_running_on_all_ports) if server_started: - logging.debug("Server successfully started") + _log.debug("Server successfully started") else: raise Exception('Failed to start http server') def stop(self): """Stops the apache http server.""" - logging.debug("Shutting down any running http servers") + _log.debug("Shutting down any running http servers") httpd_pid = None if os.path.exists(self._pid_file): httpd_pid = int(open(self._pid_file).readline()) + # FIXME: We shouldn't be calling a protected method of _port_obj! self._port_obj._shut_down_http_server(httpd_pid) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py index 2b25e29..fb6fddf 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py @@ -36,27 +36,49 @@ import errno import os import subprocess import sys +import time import apache_http_server import http_server import websocket_server +from webkitpy.common.system import logutils +from webkitpy.common.system.executive import Executive, ScriptError + + +_log = logutils.get_logger(__file__) + + # Python bug workaround. See Port.wdiff_text() for an explanation. _wdiff_available = True - +_pretty_patch_available = True # FIXME: This class should merge with webkitpy.webkit_port at some point. class Port(object): """Abstract class for Port-specific hooks for the layout_test package. """ - def __init__(self, port_name=None, options=None): + @staticmethod + def flag_from_configuration(configuration): + flags_by_configuration = { + "Debug": "--debug", + "Release": "--release", + } + return flags_by_configuration[configuration] + + def __init__(self, port_name=None, options=None, executive=Executive()): self._name = port_name self._options = options self._helper = None self._http_server = None self._webkit_base_dir = None self._websocket_server = None + self._executive = executive + + def default_child_processes(self): + """Return the number of DumpRenderTree instances to use for this + port.""" + return self._executive.cpu_count() def baseline_path(self): """Return the absolute path to the directory to store new baselines @@ -68,38 +90,53 @@ class Port(object): baselines. The directories are searched in order.""" raise NotImplementedError('Port.baseline_search_path') - def check_sys_deps(self): + def check_build(self, needs_http): + """This routine is used to ensure that the build is up to date + and all the needed binaries are present.""" + raise NotImplementedError('Port.check_build') + + def check_sys_deps(self, needs_http): """If the port needs to do some runtime checks to ensure that the - tests can be run successfully, they should be done here. + tests can be run successfully, it should override this routine. + This step can be skipped with --nocheck-sys-deps. Returns whether the system is properly configured.""" - raise NotImplementedError('Port.check_sys_deps') + return True + + def check_image_diff(self, override_step=None, logging=True): + """This routine is used to check whether image_diff binary exists.""" + raise NotImplemented('Port.check_image_diff') - def compare_text(self, actual_text, expected_text): + def compare_text(self, expected_text, actual_text): """Return whether or not the two strings are *not* equal. This routine is used to diff text output. While this is a generic routine, we include it in the Port interface so that it can be overriden for testing purposes.""" - return actual_text != expected_text + return expected_text != actual_text - def diff_image(self, actual_filename, expected_filename, + def diff_image(self, expected_filename, actual_filename, diff_filename=None): """Compare two image files and produce a delta image file. - Return 1 if the two files are different, 0 if they are the same. + Return True if the two files are different, False if they are the same. Also produce a delta image of the two images and write that into |diff_filename| if it is not None. While this is a generic routine, we include it in the Port interface so that it can be overriden for testing purposes.""" executable = self._path_to_image_diff() - cmd = [executable, '--diff', actual_filename, expected_filename] + if diff_filename: - cmd.append(diff_filename) - result = 1 + cmd = [executable, '--diff', expected_filename, actual_filename, + diff_filename] + else: + cmd = [executable, expected_filename, actual_filename] + + result = True try: - result = subprocess.call(cmd) + if subprocess.call(cmd) == 0: + return False except OSError, e: if e.errno == errno.ENOENT or e.errno == errno.EACCES: _compare_available = False @@ -111,8 +148,8 @@ class Port(object): pass return result - def diff_text(self, actual_text, expected_text, - actual_filename, expected_filename): + def diff_text(self, expected_text, actual_text, + expected_filename, actual_filename): """Returns a string containing the diff of the two text strings in 'unified diff' format. @@ -124,6 +161,13 @@ class Port(object): actual_filename) return ''.join(diff) + def driver_name(self): + """Returns the name of the actual binary that is performing the test, + so that it can be referred to in log messages. In most cases this + will be DumpRenderTree, but if a port uses a binary with a different + name, it can be overridden here.""" + return "DumpRenderTree" + def expected_baselines(self, filename, suffix, all_baselines=False): """Given a test name, finds where the baseline results are located. @@ -262,14 +306,7 @@ class Port(object): may be different (e.g., 'win-xp' instead of 'chromium-win-xp'.""" return self._name - def num_cores(self): - """Return the number of cores/cpus available on this machine. - - This routine is used to determine the default amount of parallelism - used by run-chromium-webkit-tests.""" - raise NotImplementedError('Port.num_cores') - - # FIXME: This could be replaced by functions in webkitpy.scm. + # FIXME: This could be replaced by functions in webkitpy.common.checkout.scm. def path_from_webkit_base(self, *comps): """Returns the full path to path made by joining the top of the WebKit source tree and the list of path components in |*comps|.""" @@ -288,7 +325,7 @@ class Port(object): This is used by the rebaselining tool. Raises NotImplementedError if the port does not use expectations files.""" raise NotImplementedError('Port.path_to_test_expectations_file') - + def remove_directory(self, *path): """Recursively removes a directory, even if it's marked read-only. @@ -321,7 +358,7 @@ class Port(object): win32 = False def remove_with_retry(rmfunc, path): - os.chmod(path, stat.S_IWRITE) + os.chmod(path, os.stat.S_IWRITE) if win32: win32api.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_NORMAL) @@ -381,10 +418,10 @@ class Port(object): raise NotImplementedError('Port.start_driver') def start_helper(self): - """Start a layout test helper if needed on this port. The test helper - is used to reconfigure graphics settings and do other things that - may be necessary to ensure a known test configuration.""" - raise NotImplementedError('Port.start_helper') + """If a port needs to reconfigure graphics settings or do other + things to ensure a known test configuration, it should override this + method.""" + pass def start_http_server(self): """Start a web server if it is available. Do nothing if @@ -408,8 +445,9 @@ class Port(object): def stop_helper(self): """Shut down the test helper if it is running. Do nothing if - it isn't, or it isn't available.""" - raise NotImplementedError('Port.stop_helper') + it isn't, or it isn't available. If a port overrides start_helper() + it must override this routine as well.""" + pass def stop_http_server(self): """Shut down the http server if it is running. Do nothing if @@ -430,6 +468,15 @@ class Port(object): test_expectations file. See test_expectations.py for more details.""" raise NotImplementedError('Port.test_expectations') + def test_expectations_overrides(self): + """Returns an optional set of overrides for the test_expectations. + + This is used by ports that have code in two repositories, and where + it is possible that you might need "downstream" expectations that + temporarily override the "upstream" expectations until the port can + sync up the two repos.""" + return None + def test_base_platform_names(self): """Return a list of the 'base' platforms on your port. The base platforms represent different architectures, operating systems, @@ -458,6 +505,12 @@ class Port(object): might return 'mac' as a test_platform name'.""" raise NotImplementedError('Port.platforms') + def test_platform_name_to_name(self, test_platform_name): + """Returns the Port platform name that corresponds to the name as + referenced in the expectations file. E.g., "mac" returns + "chromium-mac" on the Chromium ports.""" + raise NotImplementedError('Port.test_platform_name_to_name') + def version(self): """Returns a string indicating the version of a given platform, e.g. '-leopard' or '-xp'. @@ -476,8 +529,9 @@ class Port(object): '--end-delete=##WDIFF_END##', '--start-insert=##WDIFF_ADD##', '--end-insert=##WDIFF_END##', - expected_filename, - actual_filename] + actual_filename, + expected_filename] + # FIXME: Why not just check os.exists(executable) once? global _wdiff_available result = '' try: @@ -500,6 +554,7 @@ class Port(object): # http://bugs.python.org/issue1236 if _wdiff_available: try: + # FIXME: Use Executive() here. wdiff = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] except ValueError, e: @@ -521,6 +576,31 @@ class Port(object): raise e return result + _pretty_patch_error_html = "Failed to run PrettyPatch, see error console." + + def pretty_patch_text(self, diff_path): + global _pretty_patch_available + if not _pretty_patch_available: + return self._pretty_patch_error_html + pretty_patch_path = self.path_from_webkit_base("BugsSite", "PrettyPatch") + prettify_path = os.path.join(pretty_patch_path, "prettify.rb") + command = ["ruby", "-I", pretty_patch_path, prettify_path, diff_path] + try: + return self._executive.run_command(command) + except OSError, e: + # If the system is missing ruby log the error and stop trying. + _pretty_patch_available = False + _log.error("Failed to run PrettyPatch (%s): %s" % (command, e)) + return self._pretty_patch_error_html + except ScriptError, e: + # If ruby failed to run for some reason, log the command output and stop trying. + _pretty_patch_available = False + _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output())) + return self._pretty_patch_error_html + + def default_configuration(self): + return "Release" + # # PROTECTED ROUTINES # @@ -528,13 +608,6 @@ class Port(object): # or any of its subclasses. # - def _kill_process(self, pid): - """Forcefully kill a process. - - This routine should not be used or needed generically, but can be - used in helper files like http_server.py.""" - raise NotImplementedError('Port.kill_process') - def _path_to_apache(self): """Returns the full path to the apache binary. @@ -547,7 +620,7 @@ class Port(object): This is needed only by ports that use the apache_http_server module.""" raise NotImplementedError('Port.path_to_apache_config_file') - def _path_to_driver(self): + def _path_to_driver(self, configuration=None): """Returns the full path to the test driver (DumpRenderTree).""" raise NotImplementedError('Port.path_to_driver') diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py index 1123376..8bae2a9 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -36,11 +36,43 @@ import signal import subprocess import sys import time +import webbrowser import base import http_server + +# FIXME: To use the DRT-based version of this file, we need to be able to +# run the webkit code, which uses server_process, which requires UNIX-style +# non-blocking I/O with selects(), which requires fcntl() which doesn't exist +# on Windows. +if sys.platform not in ('win32', 'cygwin'): + import webkit + import websocket_server +_log = logging.getLogger("webkitpy.layout_tests.port.chromium") + + +# FIXME: This function doesn't belong in this package. +def check_file_exists(path_to_file, file_description, override_step=None, + logging=True): + """Verify the file is present where expected or log an error. + + Args: + file_name: The (human friendly) name or description of the file + you're looking for (e.g., "HTTP Server"). Used for error logging. + override_step: An optional string to be logged if the check fails. + logging: Whether or not log the error messages.""" + if not os.path.exists(path_to_file): + if logging: + _log.error('Unable to find %s' % file_description) + _log.error(' at %s' % path_to_file) + if override_step: + _log.error(' %s' % override_step) + _log.error('') + return False + return True + class ChromiumPort(base.Port): """Abstract base class for Chromium implementations of the Port class.""" @@ -50,81 +82,116 @@ class ChromiumPort(base.Port): self._chromium_base_dir = None def baseline_path(self): - return self._chromium_baseline_path(self._name) + return self._webkit_baseline_path(self._name) - def check_sys_deps(self): + def check_build(self, needs_http): result = True - test_shell_binary_path = self._path_to_driver() - if os.path.exists(test_shell_binary_path): - proc = subprocess.Popen([test_shell_binary_path, - '--check-layout-test-sys-deps']) - if proc.wait() != 0: - logging.error("Aborting because system dependencies check " - "failed.") - logging.error("To override, invoke with --nocheck-sys-deps") - result = False - else: - logging.error('test driver is not found at %s' % - test_shell_binary_path) - result = False - image_diff_path = self._path_to_image_diff() - if (not os.path.exists(image_diff_path) and not - self._options.no_pixel_tests): - logging.error('image diff not found at %s' % image_diff_path) - logging.error("To override, invoke with --no-pixel-tests") + # FIXME: see comment above re: import webkit + if (sys.platform in ('win32', 'cygwin') and self._options and + hasattr(self._options, 'use_drt') and self._options.use_drt): + _log.error('--use-drt is not supported on Windows yet') + _log.error('') result = False + dump_render_tree_binary_path = self._path_to_driver() + result = check_file_exists(dump_render_tree_binary_path, + 'test driver') and result + if result and self._options.build: + result = self._check_driver_build_up_to_date( + self._options.configuration) + else: + _log.error('') + + helper_path = self._path_to_helper() + if helper_path: + result = check_file_exists(helper_path, + 'layout test helper') and result + + if self._options.pixel_tests: + result = self.check_image_diff( + 'To override, invoke with --no-pixel-tests') and result + return result - def compare_text(self, actual_text, expected_text): - return actual_text != expected_text + def check_sys_deps(self, needs_http): + dump_render_tree_binary_path = self._path_to_driver() + proc = subprocess.Popen([dump_render_tree_binary_path, + '--check-layout-test-sys-deps']) + if proc.wait(): + _log.error('System dependencies check failed.') + _log.error('To override, invoke with --nocheck-sys-deps') + _log.error('') + return False + return True + + def check_image_diff(self, override_step=None, logging=True): + image_diff_path = self._path_to_image_diff() + return check_file_exists(image_diff_path, 'image diff exe', + override_step, logging) + + def driver_name(self): + return "test_shell" def path_from_chromium_base(self, *comps): """Returns the full path to path made by joining the top of the Chromium source tree and the list of path components in |*comps|.""" if not self._chromium_base_dir: abspath = os.path.abspath(__file__) - self._chromium_base_dir = abspath[0:abspath.find('third_party')] + offset = abspath.find('third_party') + if offset == -1: + raise AssertionError('could not find Chromium base dir from ' + + abspath) + self._chromium_base_dir = abspath[0:offset] return os.path.join(self._chromium_base_dir, *comps) def path_to_test_expectations_file(self): - return self.path_from_chromium_base('webkit', 'tools', 'layout_tests', - 'test_expectations.txt') + return self.path_from_webkit_base('LayoutTests', 'platform', + 'chromium', 'test_expectations.txt') def results_directory(self): - return self.path_from_chromium_base('webkit', self._options.target, - self._options.results_directory) + try: + return self.path_from_chromium_base('webkit', + self._options.configuration, self._options.results_directory) + except AssertionError: + return self.path_from_webkit_base('WebKit', 'chromium', + 'xcodebuild', self._options.configuration, + self._options.results_directory) def setup_test_run(self): # Delete the disk cache if any to ensure a clean test run. - test_shell_binary_path = self._path_to_driver() - cachedir = os.path.split(test_shell_binary_path)[0] + dump_render_tree_binary_path = self._path_to_driver() + cachedir = os.path.split(dump_render_tree_binary_path)[0] cachedir = os.path.join(cachedir, "cache") if os.path.exists(cachedir): shutil.rmtree(cachedir) def show_results_html_file(self, results_filename): - subprocess.Popen([self._path_to_driver(), - self.filename_to_uri(results_filename)]) + uri = self.filename_to_uri(results_filename) + if self._options.use_drt: + webbrowser.open(uri, new=1) + else: + subprocess.Popen([self._path_to_driver(), uri]) def start_driver(self, image_path, options): """Starts a new Driver and returns a handle to it.""" + if self._options.use_drt: + return webkit.WebKitDriver(self, image_path, options) return ChromiumDriver(self, image_path, options) def start_helper(self): helper_path = self._path_to_helper() if helper_path: - logging.debug("Starting layout helper %s" % helper_path) + _log.debug("Starting layout helper %s" % helper_path) self._helper = subprocess.Popen([helper_path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None) is_ready = self._helper.stdout.readline() if not is_ready.startswith('ready'): - logging.error("layout_test_helper failed to be ready") + _log.error("layout_test_helper failed to be ready") def stop_helper(self): if self._helper: - logging.debug("Stopping layout test helper") + _log.debug("Stopping layout test helper") self._helper.stdin.write("x\n") self._helper.stdin.close() self._helper.wait() @@ -140,10 +207,27 @@ class ChromiumPort(base.Port): expectations_file = self.path_to_test_expectations_file() return file(expectations_file, "r").read() + def test_expectations_overrides(self): + try: + overrides_file = self.path_from_chromium_base('webkit', 'tools', + 'layout_tests', 'test_expectations.txt') + except AssertionError: + return None + if os.path.exists(overrides_file): + return file(overrides_file, "r").read() + else: + return None + def test_platform_names(self): return self.test_base_platform_names() + ('win-xp', 'win-vista', 'win-7') + def test_platform_name_to_name(self, test_platform_name): + if test_platform_name in self.test_platform_names(): + return 'chromium-' + test_platform_name + raise ValueError('Unsupported test_platform_name: %s' % + test_platform_name) + # # PROTECTED METHODS # @@ -151,11 +235,34 @@ class ChromiumPort(base.Port): # or any subclasses. # + def _check_driver_build_up_to_date(self, configuration): + if configuration in ('Debug', 'Release'): + try: + debug_path = self._path_to_driver('Debug') + release_path = self._path_to_driver('Release') + + debug_mtime = os.stat(debug_path).st_mtime + release_mtime = os.stat(release_path).st_mtime + + if (debug_mtime > release_mtime and configuration == 'Release' or + release_mtime > debug_mtime and configuration == 'Debug'): + _log.warning('You are not running the most ' + 'recent DumpRenderTree binary. You need to ' + 'pass --debug or not to select between ' + 'Debug and Release.') + _log.warning('') + # This will fail if we don't have both a debug and release binary. + # That's fine because, in this case, we must already be running the + # most up-to-date one. + except OSError: + pass + return True + def _chromium_baseline_path(self, platform): if platform is None: platform = self.name() - return self.path_from_chromium_base('webkit', 'data', 'layout_tests', - 'platform', platform, 'LayoutTests') + return self.path_from_webkit_base('LayoutTests', 'platform', platform) + class ChromiumDriver(base.Driver): """Abstract interface for the DumpRenderTree interface.""" @@ -163,7 +270,7 @@ class ChromiumDriver(base.Driver): def __init__(self, port, image_path, options): self._port = port self._options = options - self._target = port._options.target + self._configuration = port._options.configuration self._image_path = image_path cmd = [] @@ -181,10 +288,17 @@ class ChromiumDriver(base.Driver): cmd += [port._path_to_driver(), '--layout-tests'] if options: cmd += options + + # We need to pass close_fds=True to work around Python bug #2320 + # (otherwise we can hang when we kill DumpRenderTree when we are running + # multiple threads). See http://bugs.python.org/issue2320 . + # Note that close_fds isn't supported on Windows, but this bug only + # shows up on Mac and Linux. + close_flag = sys.platform not in ('win32', 'cygwin') self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - + stderr=subprocess.STDOUT, + close_fds=close_flag) def poll(self): return self._proc.poll() @@ -207,14 +321,19 @@ class ChromiumDriver(base.Driver): cmd += ' ' + checksum cmd += "\n" - self._proc.stdin.write(cmd) - line = self._proc.stdout.readline() - while line.rstrip() != "#EOF": + try: + self._proc.stdin.write(cmd) + line = self._proc.stdout.readline() + except IOError, e: + _log.error("IOError communicating w/ test_shell: " + str(e)) + crash = True + + while not crash and line.rstrip() != "#EOF": # Make sure we haven't crashed. if line == '' and self.poll() is not None: # This is hex code 0xc000001d, which is used for abrupt # termination. This happens if we hit ctrl+c from the prompt - # and we happen to be waiting on the test_shell. + # and we happen to be waiting on the DumpRenderTree. # sdoyon: Not sure for which OS and in what circumstances the # above code is valid. What works for me under Linux to detect # ctrl+c is for the subprocess returncode to be negative @@ -229,8 +348,8 @@ class ChromiumDriver(base.Driver): if line.startswith("#URL:"): actual_uri = line.rstrip()[5:] if uri != actual_uri: - logging.fatal("Test got out of sync:\n|%s|\n|%s|" % - (uri, actual_uri)) + _log.fatal("Test got out of sync:\n|%s|\n|%s|" % + (uri, actual_uri)) raise AssertionError("test out of sync") elif line.startswith("#MD5:"): actual_checksum = line.rstrip()[5:] @@ -242,7 +361,11 @@ class ChromiumDriver(base.Driver): else: error.append(line) - line = self._proc.stdout.readline() + try: + line = self._proc.stdout.readline() + except IOError, e: + _log.error("IOError while reading: " + str(e)) + crash = True return (crash, timeout, actual_checksum, ''.join(output), ''.join(error)) @@ -253,10 +376,20 @@ class ChromiumDriver(base.Driver): self._proc.stdout.close() if self._proc.stderr: self._proc.stderr.close() - if (sys.platform not in ('win32', 'cygwin') and - not self._proc.poll()): - # Closing stdin/stdout/stderr hangs sometimes on OS X. - null = open(os.devnull, "w") - subprocess.Popen(["kill", "-9", - str(self._proc.pid)], stderr=null) - null.close() + if sys.platform not in ('win32', 'cygwin'): + # Closing stdin/stdout/stderr hangs sometimes on OS X, + # (see __init__(), above), and anyway we don't want to hang + # the harness if DumpRenderTree is buggy, so we wait a couple + # seconds to give DumpRenderTree a chance to clean up, but then + # force-kill the process if necessary. + KILL_TIMEOUT = 3.0 + timeout = time.time() + KILL_TIMEOUT + while self._proc.poll() is None and time.time() < timeout: + time.sleep(0.1) + if self._proc.poll() is None: + _log.warning('stopping test driver timed out, ' + 'killing it') + null = open(os.devnull, "w") + subprocess.Popen(["kill", "-9", + str(self._proc.pid)], stderr=null) + null.close() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py index b817251..9a595f2 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py @@ -27,8 +27,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Chromium Mac implementation of the Port interface.""" +"""Chromium Linux implementation of the Port interface.""" +import logging import os import platform import signal @@ -36,6 +37,8 @@ import subprocess import chromium +_log = logging.getLogger("webkitpy.layout_tests.port.chromium_linux") + class ChromiumLinuxPort(chromium.ChromiumPort): """Chromium Linux implementation of the Port class.""" @@ -43,25 +46,32 @@ class ChromiumLinuxPort(chromium.ChromiumPort): def __init__(self, port_name=None, options=None): if port_name is None: port_name = 'chromium-linux' - if options and not hasattr(options, 'target'): - options.target = 'Release' + if options and not hasattr(options, 'configuration'): + options.configuration = 'Release' chromium.ChromiumPort.__init__(self, port_name, options) def baseline_search_path(self): - return [self.baseline_path(), - self._chromium_baseline_path('chromium-win'), + return [self._webkit_baseline_path('chromium-linux'), + self._webkit_baseline_path('chromium-win'), + self._webkit_baseline_path('chromium'), self._webkit_baseline_path('win'), self._webkit_baseline_path('mac')] - def check_sys_deps(self): - # We have no platform-specific dependencies to check. - return True - - def num_cores(self): - num_cores = os.sysconf("SC_NPROCESSORS_ONLN") - if isinstance(num_cores, int) and num_cores > 0: - return num_cores - return 1 + def check_build(self, needs_http): + result = chromium.ChromiumPort.check_build(self, needs_http) + if needs_http: + if self._options.use_apache: + result = self._check_apache_install() and result + else: + result = self._check_lighttpd_install() and result + result = self._check_wdiff_install() and result + + if not result: + _log.error('For complete Linux build requirements, please see:') + _log.error('') + _log.error(' http://code.google.com/p/chromium/wiki/' + 'LinuxBuildInstructions') + return result def test_platform_name(self): # We use 'linux' instead of 'chromium-linux' in test_expectations.txt. @@ -78,19 +88,42 @@ class ChromiumLinuxPort(chromium.ChromiumPort): def _build_path(self, *comps): base = self.path_from_chromium_base() if os.path.exists(os.path.join(base, 'sconsbuild')): - return self.path_from_chromium_base('sconsbuild', - self._options.target, *comps) + return self.path_from_chromium_base('sconsbuild', *comps) else: - return self.path_from_chromium_base('out', - self._options.target, *comps) - - def _kill_process(self, pid): - """Forcefully kill the process. + return self.path_from_chromium_base('out', *comps) + + def _check_apache_install(self): + result = chromium.check_file_exists(self._path_to_apache(), + "apache2") + result = chromium.check_file_exists(self._path_to_apache_config_file(), + "apache2 config file") and result + if not result: + _log.error(' Please install using: "sudo apt-get install ' + 'apache2 libapache2-mod-php5"') + _log.error('') + return result + + def _check_lighttpd_install(self): + result = chromium.check_file_exists( + self._path_to_lighttpd(), "LigHTTPd executable") + result = chromium.check_file_exists(self._path_to_lighttpd_php(), + "PHP CGI executable") and result + result = chromium.check_file_exists(self._path_to_lighttpd_modules(), + "LigHTTPd modules") and result + if not result: + _log.error(' Please install using: "sudo apt-get install ' + 'lighttpd php5-cgi"') + _log.error('') + return result + + def _check_wdiff_install(self): + result = chromium.check_file_exists(self._path_to_wdiff(), 'wdiff') + if not result: + _log.error(' Please install using: "sudo apt-get install ' + 'wdiff"') + _log.error('') + return result - Args: - pid: The id of the process to be killed. - """ - os.kill(pid, signal.SIGKILL) def _kill_all_process(self, process_name): null = open(os.devnull) @@ -99,11 +132,19 @@ class ChromiumLinuxPort(chromium.ChromiumPort): null.close() def _path_to_apache(self): - return '/usr/sbin/apache2' + if self._is_redhat_based(): + return '/usr/sbin/httpd' + else: + return '/usr/sbin/apache2' def _path_to_apache_config_file(self): + if self._is_redhat_based(): + config_name = 'fedora-httpd.conf' + else: + config_name = 'apache2-debian-httpd.conf' + return os.path.join(self.layout_tests_dir(), 'http', 'conf', - 'apache2-debian-httpd.conf') + config_name) def _path_to_lighttpd(self): return "/usr/sbin/lighttpd" @@ -114,17 +155,25 @@ class ChromiumLinuxPort(chromium.ChromiumPort): def _path_to_lighttpd_php(self): return "/usr/bin/php-cgi" - def _path_to_driver(self): - return self._build_path('test_shell') + def _path_to_driver(self, configuration=None): + if not configuration: + configuration = self._options.configuration + return self._build_path(configuration, 'test_shell') def _path_to_helper(self): return None def _path_to_image_diff(self): - return self._build_path('image_diff') + return self._build_path(self._options.configuration, 'image_diff') def _path_to_wdiff(self): - return 'wdiff' + if self._is_redhat_based(): + return '/usr/bin/dwdiff' + else: + return '/usr/bin/wdiff' + + def _is_redhat_based(self): + return os.path.exists(os.path.join('/etc', 'redhat-release')) def _shut_down_http_server(self, server_pid): """Shut down the lighttpd web server. Blocks until it's fully diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py index bcffcf8..d5e1757 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py @@ -29,6 +29,7 @@ """Chromium Mac implementation of the Port interface.""" +import logging import os import platform import signal @@ -36,6 +37,8 @@ import subprocess import chromium +_log = logging.getLogger("webkitpy.layout_tests.port.chromium_mac") + class ChromiumMacPort(chromium.ChromiumPort): """Chromium Mac implementation of the Port class.""" @@ -43,22 +46,31 @@ class ChromiumMacPort(chromium.ChromiumPort): def __init__(self, port_name=None, options=None): if port_name is None: port_name = 'chromium-mac' - if options and not hasattr(options, 'target'): - options.target = 'Release' + if options and not hasattr(options, 'configuration'): + options.configuration = 'Release' chromium.ChromiumPort.__init__(self, port_name, options) def baseline_search_path(self): - return [self.baseline_path(), + return [self._webkit_baseline_path('chromium-mac'), + self._webkit_baseline_path('chromium'), self._webkit_baseline_path('mac' + self.version()), self._webkit_baseline_path('mac')] - def check_sys_deps(self): - # We have no specific platform dependencies. - return True - - def num_cores(self): - return int(subprocess.Popen(['sysctl','-n','hw.ncpu'], - stdout=subprocess.PIPE).stdout.read()) + def check_build(self, needs_http): + result = chromium.ChromiumPort.check_build(self, needs_http) + result = self._check_wdiff_install() and result + if not result: + _log.error('For complete Mac build requirements, please see:') + _log.error('') + _log.error(' http://code.google.com/p/chromium/wiki/' + 'MacBuildInstructions') + return result + + def driver_name(self): + """name for this port's equivalent of DumpRenderTree.""" + if self._options.use_drt: + return "DumpRenderTree" + return "TestShell" def test_platform_name(self): # We use 'mac' instead of 'chromium-mac' @@ -81,21 +93,27 @@ class ChromiumMacPort(chromium.ChromiumPort): # def _build_path(self, *comps): - return self.path_from_chromium_base('xcodebuild', self._options.target, - *comps) + if self._options.use_drt: + return self.path_from_webkit_base('WebKit', 'chromium', + 'xcodebuild', *comps) + return self.path_from_chromium_base('xcodebuild', *comps) + + def _check_wdiff_install(self): + f = open(os.devnull, 'w') + rcode = 0 + try: + rcode = subprocess.call(['wdiff'], stderr=f) + except OSError: + _log.warning('wdiff not found. Install using MacPorts or some ' + 'other means') + pass + f.close() + return True def _lighttpd_path(self, *comps): return self.path_from_chromium_base('third_party', 'lighttpd', 'mac', *comps) - def _kill_process(self, pid): - """Forcefully kill the process. - - Args: - pid: The id of the process to be killed. - """ - os.kill(pid, signal.SIGKILL) - def _kill_all_process(self, process_name): """Kill any processes running under this name.""" # On Mac OS X 10.6, killall has a new constraint: -SIGNALNAME or @@ -116,25 +134,33 @@ class ChromiumMacPort(chromium.ChromiumPort): 'apache2-httpd.conf') def _path_to_lighttpd(self): - return self._lighttp_path('bin', 'lighttp') + return self._lighttpd_path('bin', 'lighttpd') def _path_to_lighttpd_modules(self): - return self._lighttp_path('lib') + return self._lighttpd_path('lib') def _path_to_lighttpd_php(self): return self._lighttpd_path('bin', 'php-cgi') - def _path_to_driver(self): - # TODO(pinkerton): make |target| happy with case-sensitive file + def _path_to_driver(self, configuration=None): + # FIXME: make |configuration| happy with case-sensitive file # systems. - return self._build_path('TestShell.app', 'Contents', 'MacOS', - 'TestShell') + if not configuration: + configuration = self._options.configuration + return self._build_path(configuration, self.driver_name() + '.app', + 'Contents', 'MacOS', self.driver_name()) def _path_to_helper(self): - return self._build_path('layout_test_helper') + binary_name = 'layout_test_helper' + if self._options.use_drt: + binary_name = 'LayoutTestHelper' + return self._build_path(self._options.configuration, binary_name) def _path_to_image_diff(self): - return self._build_path('image_diff') + binary_name = 'image_diff' + if self._options.use_drt: + binary_name = 'ImageDiff' + return self._build_path(self._options.configuration, binary_name) def _path_to_wdiff(self): return 'wdiff' diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py index 5eb0ba1..2e3de85 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py @@ -29,6 +29,7 @@ """Chromium Win implementation of the Port interface.""" +import logging import os import platform import signal @@ -37,6 +38,8 @@ import sys import chromium +_log = logging.getLogger("webkitpy.layout_tests.port.chromium_win") + class ChromiumWinPort(chromium.ChromiumPort): """Chromium Win implementation of the Port class.""" @@ -44,33 +47,37 @@ class ChromiumWinPort(chromium.ChromiumPort): def __init__(self, port_name=None, options=None): if port_name is None: port_name = 'chromium-win' + self.version() - if options and not hasattr(options, 'target'): - options.target = 'Release' + if options and not hasattr(options, 'configuration'): + options.configuration = 'Release' chromium.ChromiumPort.__init__(self, port_name, options) def baseline_search_path(self): dirs = [] if self._name == 'chromium-win-xp': - dirs.append(self._chromium_baseline_path(self._name)) + dirs.append(self._webkit_baseline_path('chromium-win-xp')) if self._name in ('chromium-win-xp', 'chromium-win-vista'): - dirs.append(self._chromium_baseline_path('chromium-win-vista')) - dirs.append(self._chromium_baseline_path('chromium-win')) + dirs.append(self._webkit_baseline_path('chromium-win-vista')) + dirs.append(self._webkit_baseline_path('chromium-win')) + dirs.append(self._webkit_baseline_path('chromium')) dirs.append(self._webkit_baseline_path('win')) dirs.append(self._webkit_baseline_path('mac')) return dirs - def check_sys_deps(self): - # TODO(dpranke): implement this - return True + def check_build(self, needs_http): + result = chromium.ChromiumPort.check_build(self, needs_http) + if not result: + _log.error('For complete Windows build requirements, please ' + 'see:') + _log.error('') + _log.error(' http://dev.chromium.org/developers/how-tos/' + 'build-instructions-windows') + return result def get_absolute_path(self, filename): """Return the absolute path in unix format for the given filename.""" abspath = os.path.abspath(filename) return abspath.replace('\\', '/') - def num_cores(self): - return int(os.environ.get('NUMBER_OF_PROCESSORS', 1)) - def relative_test_filename(self, filename): path = filename[len(self.layout_tests_dir()) + 1:] return path.replace('\\', '/') @@ -80,6 +87,8 @@ class ChromiumWinPort(chromium.ChromiumPort): return 'win' + self.version() def version(self): + if not hasattr(sys, 'getwindowsversion'): + return '' winver = sys.getwindowsversion() if winver[0] == 6 and (winver[1] == 1): return '-7' @@ -94,24 +103,15 @@ class ChromiumWinPort(chromium.ChromiumPort): # def _build_path(self, *comps): - # FIXME(dpranke): allow for builds under 'chrome' as well. - return self.path_from_chromium_base('webkit', self._options.target, - *comps) + p = self.path_from_chromium_base('webkit', *comps) + if os.path.exists(p): + return p + return self.path_from_chromium_base('chrome', *comps) def _lighttpd_path(self, *comps): return self.path_from_chromium_base('third_party', 'lighttpd', 'win', *comps) - def _kill_process(self, pid): - """Forcefully kill the process. - - Args: - pid: The id of the process to be killed. - """ - subprocess.call(('taskkill.exe', '/f', '/pid', str(pid)), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - def _path_to_apache(self): return self.path_from_chromium_base('third_party', 'cygwin', 'usr', 'sbin', 'httpd') @@ -129,14 +129,16 @@ class ChromiumWinPort(chromium.ChromiumPort): def _path_to_lighttpd_php(self): return self._lighttpd_path('php5', 'php-cgi.exe') - def _path_to_driver(self): - return self._build_path('test_shell.exe') + def _path_to_driver(self, configuration=None): + if not configuration: + configuration = self._options.configuration + return self._build_path(configuration, 'test_shell.exe') def _path_to_helper(self): - return self._build_path('layout_test_helper.exe') + return self._build_path(self._options.configuration, 'layout_test_helper.exe') def _path_to_image_diff(self): - return self._build_path('image_diff.exe') + return self._build_path(self._options.configuration, 'image_diff.exe') def _path_to_wdiff(self): return self.path_from_chromium_base('third_party', 'cygwin', 'bin', @@ -150,8 +152,10 @@ class ChromiumWinPort(chromium.ChromiumPort): server_pid: The process ID of the running server. """ subprocess.Popen(('taskkill.exe', '/f', '/im', 'LightTPD.exe'), + stdin=open(os.devnull, 'r'), stdout=subprocess.PIPE, stderr=subprocess.PIPE).wait() subprocess.Popen(('taskkill.exe', '/f', '/im', 'httpd.exe'), + stdin=open(os.devnull, 'r'), stdout=subprocess.PIPE, stderr=subprocess.PIPE).wait() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py new file mode 100644 index 0000000..7a6717f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""This is a test implementation of the Port interface that generates the + correct output for every test. It can be used for perf testing, because + it is pretty much a lower limit on how fast a port can possibly run. + + This implementation acts as a wrapper around a real port (the real port + is held as a delegate object). To specify which port, use the port name + 'dryrun-XXX' (e.g., 'dryrun-chromium-mac-leopard'). If you use just + 'dryrun', it uses the default port. + + Note that because this is really acting as a wrapper around the underlying + port, you must be able to run the underlying port as well + (check_build() and check_sys_deps() must pass and auxiliary binaries + like layout_test_helper and httpd must work). + + This implementation also modifies the test expectations so that all + tests are either SKIPPED or expected to PASS.""" + +from __future__ import with_statement + +import base +import factory + + +def _read_file(path, mode='r'): + """Return the contents of a file as a string. + + Returns '' if anything goes wrong, instead of throwing an IOError. + + """ + contents = '' + try: + with open(path, mode) as f: + contents = f.read() + except IOError: + pass + return contents + + +def _write_file(path, contents, mode='w'): + """Write the string to the specified path. + + Returns nothing if the write fails, instead of raising an IOError. + + """ + try: + with open(path, mode) as f: + f.write(contents) + except IOError: + pass + + +class DryRunPort(object): + """DryRun implementation of the Port interface.""" + + def __init__(self, port_name=None, options=None): + pfx = 'dryrun-' + if port_name.startswith(pfx): + port_name = port_name[len(pfx):] + else: + port_name = None + self.__delegate = factory.get(port_name, options) + + def __getattr__(self, name): + return getattr(self.__delegate, name) + + def check_build(self, needs_http): + return True + + def check_sys_deps(self, needs_http): + return True + + def start_helper(self): + pass + + def start_http_server(self): + pass + + def start_websocket_server(self): + pass + + def stop_helper(self): + pass + + def stop_http_server(self): + pass + + def stop_websocket_server(self): + pass + + def start_driver(self, image_path, options): + return DryrunDriver(self, image_path, options) + + +class DryrunDriver(base.Driver): + """Dryrun implementation of the DumpRenderTree / Driver interface.""" + + def __init__(self, port, image_path, test_driver_options): + self._port = port + self._driver_options = test_driver_options + self._image_path = image_path + self._layout_tests_dir = None + + def poll(self): + return None + + def returncode(self): + return 0 + + def run_test(self, uri, timeoutms, image_hash): + test_name = self._uri_to_test(uri) + + text_filename = self._port.expected_filename(test_name, '.txt') + text_output = _read_file(text_filename) + + if image_hash: + image_filename = self._port.expected_filename(test_name, '.png') + image = _read_file(image_filename, 'rb') + if self._image_path: + _write_file(self._image_path, image) + hash_filename = self._port.expected_filename(test_name, + '.checksum') + hash = _read_file(hash_filename) + else: + hash = None + return (False, False, hash, text_output, None) + + def stop(self): + pass + + def _uri_to_test(self, uri): + """Return the base layout test name for a given URI. + + This returns the test name for a given URI, e.g., if you passed in + "file:///src/LayoutTests/fast/html/keygen.html" it would return + "fast/html/keygen.html". + + """ + if not self._layout_tests_dir: + self._layout_tests_dir = self._port.layout_tests_dir() + test = uri + + if uri.startswith("file:///"): + test = test.replace('file://', '') + return test + elif uri.startswith("http://127.0.0.1:8880/"): + # websocket tests + test = test.replace('http://127.0.0.1:8880/', + self._layout_tests_dir + '/') + return test + elif uri.startswith("http://"): + # regular HTTP test + test = test.replace('http://127.0.0.1:8000/', + self._layout_tests_dir + '/http/tests/') + return test + elif uri.startswith("https://"): + test = test.replace('https://127.0.0.1:8443/', + self._layout_tests_dir + '/http/tests/') + return test + else: + raise NotImplementedError('unknown url type: %s' % uri) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py new file mode 100644 index 0000000..95b90da --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Factory method to retrieve the appropriate port implementation.""" + + +import sys + + +def get(port_name=None, options=None): + """Returns an object implementing the Port interface. If + port_name is None, this routine attempts to guess at the most + appropriate port on this platform.""" + port_to_use = port_name + if port_to_use is None: + if sys.platform == 'win32' or sys.platform == 'cygwin': + if options and hasattr(options, 'chromium') and options.chromium: + port_to_use = 'chromium-win' + else: + port_to_use = 'win' + elif sys.platform == 'linux2': + port_to_use = 'chromium-linux' + elif sys.platform == 'darwin': + if options and hasattr(options, 'chromium') and options.chromium: + port_to_use = 'chromium-mac' + else: + port_to_use = 'mac' + + if port_to_use is None: + raise NotImplementedError('unknown port; sys.platform = "%s"' % + sys.platform) + + if port_to_use == 'test': + import test + return test.TestPort(port_name, options) + elif port_to_use.startswith('dryrun'): + import dryrun + return dryrun.DryRunPort(port_name, options) + elif port_to_use.startswith('mac'): + import mac + return mac.MacPort(port_name, options) + elif port_to_use.startswith('win'): + import win + return win.WinPort(port_name, options) + elif port_to_use.startswith('gtk'): + import gtk + return gtk.GtkPort(port_name, options) + elif port_to_use.startswith('qt'): + import qt + return qt.QtPort(port_name, options) + elif port_to_use.startswith('chromium-mac'): + import chromium_mac + return chromium_mac.ChromiumMacPort(port_name, options) + elif port_to_use.startswith('chromium-linux'): + import chromium_linux + return chromium_linux.ChromiumLinuxPort(port_name, options) + elif port_to_use.startswith('chromium-win'): + import chromium_win + return chromium_win.ChromiumWinPort(port_name, options) + + raise NotImplementedError('unsupported port: %s' % port_to_use) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py new file mode 100644 index 0000000..de5e28a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py @@ -0,0 +1,91 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""WebKit Gtk implementation of the Port interface.""" + +import logging +import os +import subprocess + +from webkitpy.layout_tests.port.webkit import WebKitPort + +_log = logging.getLogger("webkitpy.layout_tests.port.gtk") + + +class GtkPort(WebKitPort): + """WebKit Gtk implementation of the Port class.""" + + def __init__(self, port_name=None, options=None): + if port_name is None: + port_name = 'gtk' + WebKitPort.__init__(self, port_name, options) + + def _tests_for_other_platforms(self): + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + # This really need to be automated. + return [ + "platform/chromium", + "platform/win", + "platform/qt", + "platform/mac", + ] + + def _path_to_apache_config_file(self): + # FIXME: This needs to detect the distribution and change config files. + return os.path.join(self.layout_tests_dir(), 'http', 'conf', + 'apache2-debian-httpd.conf') + + def _kill_all_process(self, process_name): + null = open(os.devnull) + subprocess.call(['killall', '-TERM', '-u', os.getenv('USER'), + process_name], stderr=null) + null.close() + + def _shut_down_http_server(self, server_pid): + """Shut down the httpd web server. Blocks until it's fully + shut down. + + Args: + server_pid: The process ID of the running server. + """ + # server_pid is not set when "http_server.py stop" is run manually. + if server_pid is None: + # FIXME: This isn't ideal, since it could conflict with + # lighttpd processes not started by http_server.py, + # but good enough for now. + self._kill_all_process('apache2') + else: + try: + os.kill(server_pid, signal.SIGTERM) + # TODO(mmoss) Maybe throw in a SIGKILL just to be sure? + except OSError: + # Sometimes we get a bad PID (e.g. from a stale httpd.pid + # file), so if kill fails on the given PID, just try to + # 'killall' web servers. + self._shut_down_http_server(None) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server.py index 0315704..cc434bc 100755 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server.py @@ -40,8 +40,11 @@ import tempfile import time import urllib +import factory import http_server_base +_log = logging.getLogger("webkitpy.layout_tests.port.http_server") + class HttpdNotStarted(Exception): pass @@ -200,11 +203,11 @@ class Lighttpd(http_server_base.HttpServerBase): env['PATH']) if sys.platform == 'win32' and self._register_cygwin: - setup_mount = port.path_from_chromium_base('third_party', + setup_mount = self._port_obj.path_from_chromium_base('third_party', 'cygwin', 'setup_mount.bat') subprocess.Popen(setup_mount).wait() - logging.debug('Starting http server') + _log.debug('Starting http server') self._process = subprocess.Popen(start_cmd, env=env) # Wait for server to start. @@ -216,7 +219,7 @@ class Lighttpd(http_server_base.HttpServerBase): if not server_started or self._process.returncode != None: raise google.httpd_utils.HttpdNotStarted('Failed to start httpd.') - logging.debug("Server successfully started") + _log.debug("Server successfully started") # TODO(deanm): Find a nicer way to shutdown cleanly. Our log files are # probably not being flushed, etc... why doesn't our python have os.kill ? @@ -233,40 +236,3 @@ class Lighttpd(http_server_base.HttpServerBase): if self._process: self._process.wait() self._process = None - -if '__main__' == __name__: - # Provide some command line params for starting/stopping the http server - # manually. Also used in ui_tests to run http layout tests in a browser. - option_parser = optparse.OptionParser() - option_parser.add_option('-k', '--server', - help='Server action (start|stop)') - option_parser.add_option('-p', '--port', - help='Port to listen on (overrides layout test ports)') - option_parser.add_option('-r', '--root', - help='Absolute path to DocumentRoot (overrides layout test roots)') - option_parser.add_option('--register_cygwin', action="store_true", - dest="register_cygwin", help='Register Cygwin paths (on Win try bots)') - option_parser.add_option('--run_background', action="store_true", - dest="run_background", - help='Run on background (for running as UI test)') - options, args = option_parser.parse_args() - - if not options.server: - print ('Usage: %s --server {start|stop} [--root=root_dir]' - ' [--port=port_number]' % sys.argv[0]) - else: - if (options.root is None) and (options.port is not None): - # specifying root but not port means we want httpd on default - # set of ports that LayoutTest use, but pointing to a different - # source of tests. Specifying port but no root does not seem - # meaningful. - raise 'Specifying port requires also a root.' - httpd = Lighttpd(tempfile.gettempdir(), - port=options.port, - root=options.root, - register_cygwin=options.register_cygwin, - run_background=options.run_background) - if 'start' == options.server: - httpd.start() - else: - httpd.stop(force=True) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server_base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server_base.py index e82943e..c9805d6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server_base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server_base.py @@ -34,6 +34,8 @@ import os import time import urllib +_log = logging.getLogger("webkitpy.layout_tests.port.http_server_base") + class HttpServerBase(object): @@ -47,6 +49,7 @@ class HttpServerBase(object): while time.time() - start_time < 20: if action(): return True + _log.debug("Waiting for action: %s" % action) time.sleep(1) return False @@ -63,9 +66,9 @@ class HttpServerBase(object): try: response = urllib.urlopen(url) - logging.debug("Server running at %s" % url) + _log.debug("Server running at %s" % url) except IOError: - logging.debug("Server NOT running at %s" % url) + _log.debug("Server NOT running at %s" % url) return False return True diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf b/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf index d3150dd..2e9c82e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf @@ -21,6 +21,7 @@ mimetype.assign = ( ".html" => "text/html", ".htm" => "text/html", ".xhtml" => "application/xhtml+xml", + ".xhtmlmp" => "application/vnd.wap.xhtml+xml", ".js" => "text/javascript", ".log" => "text/plain", ".conf" => "text/plain", diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py index d355f62..cf4daa8 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -29,31 +28,41 @@ """WebKit Mac implementation of the Port interface.""" -import fcntl import logging import os import pdb import platform -import select +import re +import shutil import signal import subprocess import sys import time import webbrowser -import base +import webkitpy.common.system.ospath as ospath +import webkitpy.layout_tests.port.server_process as server_process +from webkitpy.layout_tests.port.webkit import WebKitPort, WebKitDriver -import webkitpy -from webkitpy import executive +_log = logging.getLogger("webkitpy.layout_tests.port.mac") -class MacPort(base.Port): + +class MacPort(WebKitPort): """WebKit Mac implementation of the Port class.""" def __init__(self, port_name=None, options=None): if port_name is None: port_name = 'mac' + self.version() - base.Port.__init__(self, port_name, options) - self._cached_build_root = None + WebKitPort.__init__(self, port_name, options) + + def default_child_processes(self): + # FIXME: new-run-webkit-tests is unstable on Mac running more than + # four threads in parallel. + # See https://bugs.webkit.org/show_bug.cgi?id=36622 + child_processes = WebKitPort.default_child_processes(self) + if child_processes > 4: + return 4 + return child_processes def baseline_search_path(self): dirs = [] @@ -66,53 +75,13 @@ class MacPort(base.Port): dirs.append(self._webkit_baseline_path('mac')) return dirs - def check_sys_deps(self): - if executive.run_command([self.script_path("build-dumprendertree")], return_exit_code=True) != 0: - return False - - driver_path = self._path_to_driver() - if not os.path.exists(driver_path): - logging.error("DumpRenderTree was not found at %s" % driver_path) - return False - - # This should also validate that the ImageDiff path is valid (once this script knows how to use ImageDiff). - # https://bugs.webkit.org/show_bug.cgi?id=34826 - return True - - def num_cores(self): - return int(os.popen2("sysctl -n hw.ncpu")[1].read()) - - def results_directory(self): - return ('/tmp/run-chromium-webkit-tests-' + - self._options.results_directory) - - def setup_test_run(self): - # This port doesn't require any specific configuration. - pass - - def show_results_html_file(self, results_filename): - uri = self.filename_to_uri(results_filename) - webbrowser.open(uri, new=1) - - def start_driver(self, image_path, options): - """Starts a new Driver and returns a handle to it.""" - return MacDriver(self, image_path, options) - - def start_helper(self): - # This port doesn't use a helper process. - pass - - def stop_helper(self): - # This port doesn't use a helper process. - pass - - def test_base_platform_names(self): - # At the moment we don't use test platform names, but we have - # to return something. - return ('mac',) + def path_to_test_expectations_file(self): + return self.path_from_webkit_base('LayoutTests', 'platform', + 'mac', 'test_expectations.txt') def _skipped_file_paths(self): - # FIXME: This method will need to be made work for non-mac platforms and moved into base.Port. + # FIXME: This method will need to be made work for non-mac + # platforms and moved into base.Port. skipped_files = [] if self._name in ('mac-tiger', 'mac-leopard', 'mac-snowleopard'): skipped_files.append(os.path.join( @@ -121,79 +90,8 @@ class MacPort(base.Port): 'Skipped')) return skipped_files - def _tests_for_other_platforms(self): - # The original run-webkit-tests builds up a "whitelist" of tests to run, and passes that to DumpRenderTree. - # run-chromium-webkit-tests assumes we run *all* tests and test_expectations.txt functions as a blacklist. - # FIXME: This list could be dynamic based on platform name and pushed into base.Port. - return [ - "platform/chromium", - "platform/gtk", - "platform/qt", - "platform/win", - ] - - def _tests_for_disabled_features(self): - # FIXME: This should use the feature detection from webkitperl/features.pm to match run-webkit-tests. - # For now we hard-code a list of features known to be disabled on the Mac platform. - disabled_feature_tests = [ - "fast/xhtmlmp", - "http/tests/wml", - "mathml", - "wml", - ] - # FIXME: webarchive tests expect to read-write from -expected.webarchive files instead of .txt files. - # This script doesn't know how to do that yet, so pretend they're just "disabled". - webarchive_tests = [ - "webarchive", - "svg/webarchive", - "http/tests/webarchive", - "svg/custom/image-with-prefix-in-webarchive.svg", - ] - return disabled_feature_tests + webarchive_tests - - def _tests_from_skipped_file(self, skipped_file): - tests_to_skip = [] - for line in skipped_file.readlines(): - line = line.strip() - if line.startswith('#') or not len(line): - continue - tests_to_skip.append(line) - return tests_to_skip - - def _expectations_from_skipped_files(self): - tests_to_skip = [] - for filename in self._skipped_file_paths(): - if not os.path.exists(filename): - logging.warn("Failed to open Skipped file: %s" % filename) - continue - skipped_file = file(filename) - tests_to_skip.extend(self._tests_from_skipped_file(skipped_file)) - skipped_file.close() - return tests_to_skip - - def test_expectations(self): - # The WebKit mac port uses 'Skipped' files at the moment. Each - # file contains a list of files or directories to be skipped during - # the test run. The total list of tests to skipped is given by the - # contents of the generic Skipped file found in platform/X plus - # a version-specific file found in platform/X-version. Duplicate - # entries are allowed. This routine reads those files and turns - # contents into the format expected by test_expectations. - tests_to_skip = set(self._expectations_from_skipped_files()) # Use a set to allow duplicates - tests_to_skip.update(self._tests_for_other_platforms()) - tests_to_skip.update(self._tests_for_disabled_features()) - expectations = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" % test_path, tests_to_skip) - return "\n".join(expectations) - def test_platform_name(self): - # At the moment we don't use test platform names, but we have - # to return something. - return 'mac' - - def test_platform_names(self): - # At the moment we don't use test platform names, but we have - # to return something. - return ('mac',) + return 'mac' + self.version() def version(self): os_version_string = platform.mac_ver()[0] # e.g. "10.5.6" @@ -208,23 +106,32 @@ class MacPort(base.Port): return '-snowleopard' return '' - # - # PROTECTED METHODS - # - - def _build_path(self, *comps): - if not self._cached_build_root: - self._cached_build_root = executive.run_command([self.script_path("webkit-build-directory"), "--top-level"]).rstrip() - return os.path.join(self._cached_build_root, self._options.target, *comps) + def _build_java_test_support(self): + java_tests_path = os.path.join(self.layout_tests_dir(), "java") + build_java = ["/usr/bin/make", "-C", java_tests_path] + if self._executive.run_command(build_java, return_exit_code=True): + _log.error("Failed to build Java support files: %s" % build_java) + return False + return True - def _kill_process(self, pid): - """Forcefully kill the process. + def _check_port_build(self): + return self._build_java_test_support() - Args: - pid: The id of the process to be killed. - """ - os.kill(pid, signal.SIGKILL) + def _tests_for_other_platforms(self): + # The original run-webkit-tests builds up a "whitelist" of tests to + # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes + # we run *all* tests and test_expectations.txt functions as a + # blacklist. + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + return [ + "platform/chromium", + "platform/gtk", + "platform/qt", + "platform/win", + ] + # FIXME: This doesn't have anything to do with WebKit. def _kill_all_process(self, process_name): # On Mac OS X 10.6, killall has a new constraint: -SIGNALNAME or # -SIGNALNUMBER must come first. Example problem: @@ -236,25 +143,11 @@ class MacPort(base.Port): process_name], stderr=null) null.close() - def _path_to_apache(self): - return '/usr/sbin/httpd' - def _path_to_apache_config_file(self): return os.path.join(self.layout_tests_dir(), 'http', 'conf', 'apache2-httpd.conf') - def _path_to_driver(self): - return self._build_path('DumpRenderTree') - - def _path_to_helper(self): - return None - - def _path_to_image_diff(self): - return self._build_path('image_diff') # FIXME: This is wrong and should be "ImageDiff", but having the correct path causes other parts of the script to hang. - - def _path_to_wdiff(self): - return 'wdiff' # FIXME: This does not exist on a default Mac OS X Leopard install. - + # FIXME: This doesn't have anything to do with WebKit. def _shut_down_http_server(self, server_pid): """Shut down the lighttpd web server. Blocks until it's fully shut down. @@ -264,209 +157,16 @@ class MacPort(base.Port): """ # server_pid is not set when "http_server.py stop" is run manually. if server_pid is None: - # TODO(mmoss) This isn't ideal, since it could conflict with + # FIXME: This isn't ideal, since it could conflict with # lighttpd processes not started by http_server.py, # but good enough for now. self._kill_all_process('httpd') else: try: os.kill(server_pid, signal.SIGTERM) - # TODO(mmoss) Maybe throw in a SIGKILL just to be sure? + # FIXME: Maybe throw in a SIGKILL just to be sure? except OSError: # Sometimes we get a bad PID (e.g. from a stale httpd.pid # file), so if kill fails on the given PID, just try to # 'killall' web servers. self._shut_down_http_server(None) - - -class MacDriver(base.Driver): - """implementation of the DumpRenderTree interface.""" - - def __init__(self, port, image_path, driver_options): - self._port = port - self._driver_options = driver_options - self._target = port._options.target - self._image_path = image_path - self._stdout_fd = None - self._cmd = None - self._env = None - self._proc = None - self._read_buffer = '' - - cmd = [] - # Hook for injecting valgrind or other runtime instrumentation, - # used by e.g. tools/valgrind/valgrind_tests.py. - wrapper = os.environ.get("BROWSER_WRAPPER", None) - if wrapper != None: - cmd += [wrapper] - if self._port._options.wrapper: - # This split() isn't really what we want -- it incorrectly will - # split quoted strings within the wrapper argument -- but in - # practice it shouldn't come up and the --help output warns - # about it anyway. - cmd += self._options.wrapper.split() - # FIXME: Using arch here masks any possible file-not-found errors from a non-existant driver executable. - cmd += ['arch', '-i386', port._path_to_driver(), '-'] - - # FIXME: This is a hack around our lack of ImageDiff support for now. - if not self._port._options.no_pixel_tests: - logging.warn("This port does not yet support pixel tests.") - self._port._options.no_pixel_tests = True - #cmd.append('--pixel-tests') - - #if driver_options: - # cmd += driver_options - env = os.environ - env['DYLD_FRAMEWORK_PATH'] = self._port._build_path() - self._cmd = cmd - self._env = env - self.restart() - - def poll(self): - return self._proc.poll() - - def restart(self): - self.stop() - self._proc = subprocess.Popen(self._cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self._env) - - def returncode(self): - return self._proc.returncode - - def run_test(self, uri, timeoutms, image_hash): - output = [] - error = [] - image = '' - crash = False - timeout = False - actual_uri = None - actual_image_hash = None - - if uri.startswith("file:///"): - cmd = uri[7:] - else: - cmd = uri - - if image_hash: - cmd += "'" + image_hash - cmd += "\n" - - self._proc.stdin.write(cmd) - self._stdout_fd = self._proc.stdout.fileno() - fl = fcntl.fcntl(self._stdout_fd, fcntl.F_GETFL) - fcntl.fcntl(self._stdout_fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) - - stop_time = time.time() + (int(timeoutms) / 1000.0) - resp = '' - (timeout, line) = self._read_line(timeout, stop_time) - resp += line - have_seen_content_type = False - while not timeout and line.rstrip() != "#EOF": - # Make sure we haven't crashed. - if line == '' and self.poll() is not None: - # This is hex code 0xc000001d, which is used for abrupt - # termination. This happens if we hit ctrl+c from the prompt - # and we happen to be waiting on the test_shell. - # sdoyon: Not sure for which OS and in what circumstances the - # above code is valid. What works for me under Linux to detect - # ctrl+c is for the subprocess returncode to be negative - # SIGINT. And that agrees with the subprocess documentation. - if (-1073741510 == self.returncode() or - - signal.SIGINT == self.returncode()): - raise KeyboardInterrupt - crash = True - break - - elif (line.startswith('Content-Type:') and not - have_seen_content_type): - have_seen_content_type = True - pass - else: - output.append(line) - - (timeout, line) = self._read_line(timeout, stop_time) - resp += line - - # Now read a second block of text for the optional image data - image_length = 0 - (timeout, line) = self._read_line(timeout, stop_time) - resp += line - HASH_HEADER = 'ActualHash: ' - LENGTH_HEADER = 'Content-Length: ' - while not timeout and not crash and line.rstrip() != "#EOF": - if line == '' and self.poll() is not None: - if (-1073741510 == self.returncode() or - - signal.SIGINT == self.returncode()): - raise KeyboardInterrupt - crash = True - break - elif line.startswith(HASH_HEADER): - actual_image_hash = line[len(HASH_HEADER):].strip() - elif line.startswith('Content-Type:'): - pass - elif line.startswith(LENGTH_HEADER): - image_length = int(line[len(LENGTH_HEADER):]) - elif image_length: - image += line - - (timeout, line) = self._read_line(timeout, stop_time, image_length) - resp += line - - if timeout: - self.restart() - - if self._image_path and len(self._image_path): - image_file = file(self._image_path, "wb") - image_file.write(image) - image_file.close() - - return (crash, timeout, actual_image_hash, - ''.join(output), ''.join(error)) - - def stop(self): - if self._proc: - self._proc.stdin.close() - self._proc.stdout.close() - if self._proc.stderr: - self._proc.stderr.close() - if (sys.platform not in ('win32', 'cygwin') and - not self._proc.poll()): - # Closing stdin/stdout/stderr hangs sometimes on OS X. - null = open(os.devnull, "w") - subprocess.Popen(["kill", "-9", - str(self._proc.pid)], stderr=null) - null.close() - - def _read_line(self, timeout, stop_time, image_length=0): - now = time.time() - read_fds = [] - - # first check to see if we have a line already read or if we've - # read the entire image - if image_length and len(self._read_buffer) >= image_length: - out = self._read_buffer[0:image_length] - self._read_buffer = self._read_buffer[image_length:] - return (timeout, out) - - idx = self._read_buffer.find('\n') - if not image_length and idx != -1: - out = self._read_buffer[0:idx + 1] - self._read_buffer = self._read_buffer[idx + 1:] - return (timeout, out) - - # If we've timed out, return just what we have, if anything - if timeout or now >= stop_time: - out = self._read_buffer - self._read_buffer = '' - return (True, out) - - (read_fds, write_fds, err_fds) = select.select( - [self._stdout_fd], [], [], stop_time - now) - try: - if timeout or len(read_fds) == 1: - self._read_buffer += self._proc.stdout.read() - except IOError, e: - read = [] - return self._read_line(timeout, stop_time) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py new file mode 100644 index 0000000..67cdefe --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py @@ -0,0 +1,99 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""QtWebKit implementation of the Port interface.""" + +import logging +import os +import subprocess +import signal + +from webkitpy.layout_tests.port.webkit import WebKitPort + +_log = logging.getLogger("webkitpy.layout_tests.port.qt") + + +class QtPort(WebKitPort): + """QtWebKit implementation of the Port class.""" + + def __init__(self, port_name=None, options=None): + if port_name is None: + port_name = 'qt' + WebKitPort.__init__(self, port_name, options) + + def _tests_for_other_platforms(self): + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + # This really need to be automated. + return [ + "platform/chromium", + "platform/win", + "platform/gtk", + "platform/mac", + ] + + def _path_to_apache_config_file(self): + # FIXME: This needs to detect the distribution and change config files. + return os.path.join(self.layout_tests_dir(), 'http', 'conf', + 'apache2-debian-httpd.conf') + + def _kill_all_process(self, process_name): + null = open(os.devnull) + subprocess.call(['killall', '-TERM', '-u', os.getenv('USER'), + process_name], stderr=null) + null.close() + + def _shut_down_http_server(self, server_pid): + """Shut down the httpd web server. Blocks until it's fully + shut down. + + Args: + server_pid: The process ID of the running server. + """ + # server_pid is not set when "http_server.py stop" is run manually. + if server_pid is None: + # FIXME: This isn't ideal, since it could conflict with + # lighttpd processes not started by http_server.py, + # but good enough for now. + self._kill_all_process('apache2') + else: + try: + os.kill(server_pid, signal.SIGTERM) + # TODO(mmoss) Maybe throw in a SIGKILL just to be sure? + except OSError: + # Sometimes we get a bad PID (e.g. from a stale httpd.pid + # file), so if kill fails on the given PID, just try to + # 'killall' web servers. + self._shut_down_http_server(None) + + def _build_driver(self): + # The Qt port builds DRT as part of the main build step + return True + + def _path_to_driver(self): + return self._build_path('bin/DumpRenderTree') diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py new file mode 100644 index 0000000..f1c6d73 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Package that implements the ServerProcess wrapper class""" + +import fcntl +import logging +import os +import select +import signal +import subprocess +import sys +import time + +_log = logging.getLogger("webkitpy.layout_tests.port.server_process") + + +class ServerProcess: + """This class provides a wrapper around a subprocess that + implements a simple request/response usage model. The primary benefit + is that reading responses takes a timeout, so that we don't ever block + indefinitely. The class also handles transparently restarting processes + as necessary to keep issuing commands.""" + + def __init__(self, port_obj, name, cmd, env=None): + self._port = port_obj + self._name = name + self._cmd = cmd + self._env = env + self._reset() + + def _reset(self): + self._proc = None + self._output = '' + self.crashed = False + self.timed_out = False + self.error = '' + + def _start(self): + if self._proc: + raise ValueError("%s already running" % self._name) + self._reset() + close_fds = sys.platform not in ('win32', 'cygwin') + self._proc = subprocess.Popen(self._cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=close_fds, + env=self._env) + fd = self._proc.stdout.fileno() + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) + fd = self._proc.stderr.fileno() + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) + + def handle_interrupt(self): + """This routine checks to see if the process crashed or exited + because of a keyboard interrupt and raises KeyboardInterrupt + accordingly.""" + if self.crashed: + # This is hex code 0xc000001d, which is used for abrupt + # termination. This happens if we hit ctrl+c from the prompt + # and we happen to be waiting on the DumpRenderTree. + # sdoyon: Not sure for which OS and in what circumstances the + # above code is valid. What works for me under Linux to detect + # ctrl+c is for the subprocess returncode to be negative + # SIGINT. And that agrees with the subprocess documentation. + if (-1073741510 == self._proc.returncode or + - signal.SIGINT == self._proc.returncode): + raise KeyboardInterrupt + return + + def poll(self): + """Check to see if the underlying process is running; returns None + if it still is (wrapper around subprocess.poll).""" + if self._proc: + return self._proc.poll() + return None + + def returncode(self): + """Returns the exit code from the subprcoess; returns None if the + process hasn't exited (this is a wrapper around subprocess.returncode). + """ + if self._proc: + return self._proc.returncode + return None + + def write(self, input): + """Write a request to the subprocess. The subprocess is (re-)start()'ed + if is not already running.""" + if not self._proc: + self._start() + self._proc.stdin.write(input) + + def read_line(self, timeout): + """Read a single line from the subprocess, waiting until the deadline. + If the deadline passes, the call times out. Note that even if the + subprocess has crashed or the deadline has passed, if there is output + pending, it will be returned. + + Args: + timeout: floating-point number of seconds the call is allowed + to block for. A zero or negative number will attempt to read + any existing data, but will not block. There is no way to + block indefinitely. + Returns: + output: data returned, if any. If no data is available and the + call times out or crashes, an empty string is returned. Note + that the returned string includes the newline ('\n').""" + return self._read(timeout, size=0) + + def read(self, timeout, size): + """Attempts to read size characters from the subprocess, waiting until + the deadline passes. If the deadline passes, any available data will be + returned. Note that even if the deadline has passed or if the + subprocess has crashed, any available data will still be returned. + + Args: + timeout: floating-point number of seconds the call is allowed + to block for. A zero or negative number will attempt to read + any existing data, but will not block. There is no way to + block indefinitely. + size: amount of data to read. Must be a postive integer. + Returns: + output: data returned, if any. If no data is available, an empty + string is returned. + """ + if size <= 0: + raise ValueError('ServerProcess.read() called with a ' + 'non-positive size: %d ' % size) + return self._read(timeout, size) + + def _read(self, timeout, size): + """Internal routine that actually does the read.""" + index = -1 + out_fd = self._proc.stdout.fileno() + err_fd = self._proc.stderr.fileno() + select_fds = (out_fd, err_fd) + deadline = time.time() + timeout + while not self.timed_out and not self.crashed: + if self._proc.poll() != None: + self.crashed = True + self.handle_interrupt() + + now = time.time() + if now > deadline: + self.timed_out = True + + # Check to see if we have any output we can return. + if size and len(self._output) >= size: + index = size + elif size == 0: + index = self._output.find('\n') + 1 + + if index or self.crashed or self.timed_out: + output = self._output[0:index] + self._output = self._output[index:] + return output + + # Nope - wait for more data. + (read_fds, write_fds, err_fds) = select.select(select_fds, [], + select_fds, + deadline - now) + try: + if out_fd in read_fds: + self._output += self._proc.stdout.read() + if err_fd in read_fds: + self.error += self._proc.stderr.read() + except IOError, e: + pass + + def stop(self): + """Stop (shut down) the subprocess), if it is running.""" + pid = self._proc.pid + self._proc.stdin.close() + self._proc.stdout.close() + if self._proc.stderr: + self._proc.stderr.close() + if sys.platform not in ('win32', 'cygwin'): + # Closing stdin/stdout/stderr hangs sometimes on OS X, + # (see restart(), above), and anyway we don't want to hang + # the harness if DumpRenderTree is buggy, so we wait a couple + # seconds to give DumpRenderTree a chance to clean up, but then + # force-kill the process if necessary. + KILL_TIMEOUT = 3.0 + timeout = time.time() + KILL_TIMEOUT + while self._proc.poll() is None and time.time() < timeout: + time.sleep(0.1) + if self._proc.poll() is None: + _log.warning('stopping %s timed out, killing it' % + self._name) + null = open(os.devnull, "w") + subprocess.Popen(["kill", "-9", + str(self._proc.pid)], stderr=null) + null.close() + _log.warning('killed') + self._reset() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py index c3e97be..edef485 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py @@ -52,31 +52,28 @@ class TestPort(base.Port): def baseline_search_path(self): return [self.baseline_path()] - def check_sys_deps(self): + def check_build(self, needs_http): return True - def diff_image(self, actual_filename, expected_filename, - diff_filename=None): + def compare_text(self, expected_text, actual_text): return False - def compare_text(self, actual_text, expected_text): + def diff_image(self, expected_filename, actual_filename, + diff_filename=None): return False - def diff_text(self, actual_text, expected_text, - actual_filename, expected_filename): + def diff_text(self, expected_text, actual_text, + expected_filename, actual_filename): return '' def name(self): return self._name - def num_cores(self): - return int(os.popen2("sysctl -n hw.ncpu")[1].read()) - def options(self): return self._options def results_directory(self): - return '/tmp' + self._options.results_directory + return '/tmp/' + self._options.results_directory def setup_test_run(self): pass @@ -93,18 +90,12 @@ class TestPort(base.Port): def start_websocket_server(self): pass - def start_helper(self): - pass - def stop_http_server(self): pass def stop_websocket_server(self): pass - def stop_helper(self): - pass - def test_expectations(self): return '' @@ -120,7 +111,7 @@ class TestPort(base.Port): def version(): return '' - def wdiff_text(self, actual_filename, expected_filename): + def wdiff_text(self, expected_filename, actual_filename): return '' diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py new file mode 100644 index 0000000..f2f5237 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""WebKit implementations of the Port interface.""" + +import logging +import os +import pdb +import platform +import re +import shutil +import signal +import subprocess +import sys +import time +import webbrowser + +import webkitpy.common.system.ospath as ospath +import webkitpy.layout_tests.port.base as base +import webkitpy.layout_tests.port.server_process as server_process + +_log = logging.getLogger("webkitpy.layout_tests.port.webkit") + + +class WebKitPort(base.Port): + """WebKit implementation of the Port class.""" + + def __init__(self, port_name=None, options=None): + base.Port.__init__(self, port_name, options) + self._cached_build_root = None + self._cached_apache_path = None + + # FIXME: disable pixel tests until they are run by default on the + # build machines. + if options and (not hasattr(options, "pixel_tests") or + options.pixel_tests is None): + options.pixel_tests = False + + def baseline_path(self): + return self._webkit_baseline_path(self._name) + + def baseline_search_path(self): + return [self._webkit_baseline_path(self._name)] + + def path_to_test_expectations_file(self): + return os.path.join(self._webkit_baseline_path(self._name), + 'test_expectations.txt') + + # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard) + def version(self): + return '' + + def _build_driver(self): + return not self._executive.run_command([ + self.script_path("build-dumprendertree"), + self.flag_from_configuration(self._options.configuration), + ], return_exit_code=True) + + def _check_driver(self): + driver_path = self._path_to_driver() + if not os.path.exists(driver_path): + _log.error("DumpRenderTree was not found at %s" % driver_path) + return False + return True + + def check_build(self, needs_http): + if self._options.build and not self._build_driver(): + return False + if not self._check_driver(): + return False + if self._options.pixel_tests: + if not self.check_image_diff(): + return False + if not self._check_port_build(): + return False + return True + + def _check_port_build(self): + # Ports can override this method to do additional checks. + return True + + def check_image_diff(self, override_step=None, logging=True): + image_diff_path = self._path_to_image_diff() + if not os.path.exists(image_diff_path): + _log.error("ImageDiff was not found at %s" % image_diff_path) + return False + return True + + def diff_image(self, expected_filename, actual_filename, + diff_filename=None): + """Return True if the two files are different. Also write a delta + image of the two images into |diff_filename| if it is not None.""" + + # Handle the case where the test didn't actually generate an image. + actual_length = os.stat(actual_filename).st_size + if actual_length == 0: + if diff_filename: + shutil.copyfile(actual_filename, expected_filename) + return True + + sp = self._diff_image_request(expected_filename, actual_filename) + return self._diff_image_reply(sp, expected_filename, diff_filename) + + def _diff_image_request(self, expected_filename, actual_filename): + # FIXME: either expose the tolerance argument as a command-line + # parameter, or make it go away and aways use exact matches. + command = [self._path_to_image_diff(), '--tolerance', '0.1'] + sp = server_process.ServerProcess(self, 'ImageDiff', command) + + actual_length = os.stat(actual_filename).st_size + actual_file = open(actual_filename).read() + expected_length = os.stat(expected_filename).st_size + expected_file = open(expected_filename).read() + sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' % + (actual_length, actual_file, expected_length, expected_file)) + + return sp + + def _diff_image_reply(self, sp, expected_filename, diff_filename): + timeout = 2.0 + deadline = time.time() + timeout + output = sp.read_line(timeout) + while not sp.timed_out and not sp.crashed and output: + if output.startswith('Content-Length'): + m = re.match('Content-Length: (\d+)', output) + content_length = int(m.group(1)) + timeout = deadline - time.time() + output = sp.read(timeout, content_length) + break + elif output.startswith('diff'): + break + else: + timeout = deadline - time.time() + output = sp.read_line(deadline) + + result = True + if output.startswith('diff'): + m = re.match('diff: (.+)% (passed|failed)', output) + if m.group(2) == 'passed': + result = False + elif output and diff_filename: + open(diff_filename, 'w').write(output) # FIXME: This leaks a file handle. + elif sp.timed_out: + _log.error("ImageDiff timed out on %s" % expected_filename) + elif sp.crashed: + _log.error("ImageDiff crashed") + sp.stop() + return result + + def results_directory(self): + # Results are store relative to the built products to make it easy + # to have multiple copies of webkit checked out and built. + return self._build_path(self._options.results_directory) + + def setup_test_run(self): + # This port doesn't require any specific configuration. + pass + + def show_results_html_file(self, results_filename): + uri = self.filename_to_uri(results_filename) + # FIXME: We should open results in the version of WebKit we built. + webbrowser.open(uri, new=1) + + def start_driver(self, image_path, options): + return WebKitDriver(self, image_path, options) + + def test_base_platform_names(self): + # At the moment we don't use test platform names, but we have + # to return something. + return ('mac', 'win') + + def _tests_for_other_platforms(self): + raise NotImplementedError('WebKitPort._tests_for_other_platforms') + # The original run-webkit-tests builds up a "whitelist" of tests to + # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes + # we run *all* tests and test_expectations.txt functions as a + # blacklist. + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + return [ + "platform/chromium", + "platform/gtk", + "platform/qt", + "platform/win", + ] + + def _tests_for_disabled_features(self): + # FIXME: This should use the feature detection from + # webkitperl/features.pm to match run-webkit-tests. + # For now we hard-code a list of features known to be disabled on + # the Mac platform. + disabled_feature_tests = [ + "fast/xhtmlmp", + "http/tests/wml", + "mathml", + "wml", + ] + # FIXME: webarchive tests expect to read-write from + # -expected.webarchive files instead of .txt files. + # This script doesn't know how to do that yet, so pretend they're + # just "disabled". + webarchive_tests = [ + "webarchive", + "svg/webarchive", + "http/tests/webarchive", + "svg/custom/image-with-prefix-in-webarchive.svg", + ] + return disabled_feature_tests + webarchive_tests + + def _tests_from_skipped_file(self, skipped_file): + tests_to_skip = [] + for line in skipped_file.readlines(): + line = line.strip() + if line.startswith('#') or not len(line): + continue + tests_to_skip.append(line) + return tests_to_skip + + def _skipped_file_paths(self): + return [os.path.join(self._webkit_baseline_path(self._name), + 'Skipped')] + + def _expectations_from_skipped_files(self): + tests_to_skip = [] + for filename in self._skipped_file_paths(): + if not os.path.exists(filename): + _log.warn("Failed to open Skipped file: %s" % filename) + continue + skipped_file = file(filename) + tests_to_skip.extend(self._tests_from_skipped_file(skipped_file)) + skipped_file.close() + return tests_to_skip + + def test_expectations(self): + # The WebKit mac port uses a combination of a test_expectations file + # and 'Skipped' files. + expectations_file = self.path_to_test_expectations_file() + expectations = file(expectations_file, "r").read() + return expectations + self._skips() + + def _skips(self): + # Each Skipped file contains a list of files + # or directories to be skipped during the test run. The total list + # of tests to skipped is given by the contents of the generic + # Skipped file found in platform/X plus a version-specific file + # found in platform/X-version. Duplicate entries are allowed. + # This routine reads those files and turns contents into the + # format expected by test_expectations. + + # Use a set to allow duplicates + tests_to_skip = set(self._expectations_from_skipped_files()) + + tests_to_skip.update(self._tests_for_other_platforms()) + tests_to_skip.update(self._tests_for_disabled_features()) + skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" % + test_path, tests_to_skip) + return "\n".join(skip_lines) + + def test_platform_name(self): + return self._name + self.version() + + def test_platform_names(self): + return self.test_base_platform_names() + ( + 'mac-tiger', 'mac-leopard', 'mac-snowleopard') + + def default_configuration(self): + # This is a bit of a hack. This state exists in a much nicer form in + # perl-land. + configuration = ospath.relpath( + self._webkit_build_directory(["--configuration"]), + self._webkit_build_directory(["--top-level"])) + assert(configuration == "Debug" or configuration == "Release") + return configuration + + def _webkit_build_directory(self, args): + args = [self.script_path("webkit-build-directory")] + args + return self._executive.run_command(args).rstrip() + + def _build_path(self, *comps): + if not self._cached_build_root: + self._cached_build_root = self._webkit_build_directory([ + "--configuration", + self.flag_from_configuration(self._options.configuration), + ]) + return os.path.join(self._cached_build_root, *comps) + + def _path_to_driver(self): + return self._build_path('DumpRenderTree') + + def _path_to_helper(self): + return None + + def _path_to_image_diff(self): + return self._build_path('ImageDiff') + + def _path_to_wdiff(self): + # FIXME: This does not exist on a default Mac OS X Leopard install. + return 'wdiff' + + def _path_to_apache(self): + if not self._cached_apache_path: + # The Apache binary path can vary depending on OS and distribution + # See http://wiki.apache.org/httpd/DistrosDefaultLayout + for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]: + if os.path.exists(path): + self._cached_apache_path = path + break + + if not self._cached_apache_path: + _log.error("Could not find apache. Not installed or unknown path.") + + return self._cached_apache_path + + +class WebKitDriver(base.Driver): + """WebKit implementation of the DumpRenderTree interface.""" + + def __init__(self, port, image_path, driver_options): + self._port = port + self._driver_options = driver_options + self._image_path = image_path + + command = [] + # Hook for injecting valgrind or other runtime instrumentation, + # used by e.g. tools/valgrind/valgrind_tests.py. + wrapper = os.environ.get("BROWSER_WRAPPER", None) + if wrapper != None: + command += [wrapper] + if self._port._options.wrapper: + # This split() isn't really what we want -- it incorrectly will + # split quoted strings within the wrapper argument -- but in + # practice it shouldn't come up and the --help output warns + # about it anyway. + # FIXME: Use a real shell parser. + command += self._options.wrapper.split() + + command += [port._path_to_driver(), '-'] + + if image_path: + command.append('--pixel-tests') + environment = os.environ + environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path() + self._server_process = server_process.ServerProcess(self._port, + "DumpRenderTree", command, environment) + + def poll(self): + return self._server_process.poll() + + def restart(self): + self._server_process.stop() + self._server_process.start() + return + + def returncode(self): + return self._server_process.returncode() + + # FIXME: This function is huge. + def run_test(self, uri, timeoutms, image_hash): + if uri.startswith("file:///"): + command = uri[7:] + else: + command = uri + + if image_hash: + command += "'" + image_hash + command += "\n" + + # pdb.set_trace() + self._server_process.write(command) + + have_seen_content_type = False + actual_image_hash = None + output = '' + image = '' + + timeout = int(timeoutms) / 1000.0 + deadline = time.time() + timeout + line = self._server_process.read_line(timeout) + while (not self._server_process.timed_out + and not self._server_process.crashed + and line.rstrip() != "#EOF"): + if (line.startswith('Content-Type:') and not + have_seen_content_type): + have_seen_content_type = True + else: + output += line + line = self._server_process.read_line(timeout) + timeout = deadline - time.time() + + # Now read a second block of text for the optional image data + remaining_length = -1 + HASH_HEADER = 'ActualHash: ' + LENGTH_HEADER = 'Content-Length: ' + line = self._server_process.read_line(timeout) + while (not self._server_process.timed_out + and not self._server_process.crashed + and line.rstrip() != "#EOF"): + if line.startswith(HASH_HEADER): + actual_image_hash = line[len(HASH_HEADER):].strip() + elif line.startswith('Content-Type:'): + pass + elif line.startswith(LENGTH_HEADER): + timeout = deadline - time.time() + content_length = int(line[len(LENGTH_HEADER):]) + image = self._server_process.read(timeout, content_length) + timeout = deadline - time.time() + line = self._server_process.read_line(timeout) + + if self._image_path and len(self._image_path): + image_file = file(self._image_path, "wb") + image_file.write(image) + image_file.close() + return (self._server_process.crashed, + self._server_process.timed_out, + actual_image_hash, + output, + self._server_process.error) + + def stop(self): + if self._server_process: + self._server_process.stop() + self._server_process = None diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py index 54c2f6f..a9ba160 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py @@ -39,8 +39,13 @@ import tempfile import time import urllib +import factory import http_server +from webkitpy.common.system.executive import Executive + +_log = logging.getLogger("webkitpy.layout_tests.port.websocket_server") + _WS_LOG_PREFIX = 'pywebsocket.ws.log-' _WSS_LOG_PREFIX = 'pywebsocket.wss.log-' @@ -59,6 +64,7 @@ def url_is_alive(url): Return: True if the url is alive. """ + sleep_time = 0.5 wait_time = 5 while wait_time > 0: try: @@ -67,9 +73,9 @@ def url_is_alive(url): return True except IOError: pass - wait_time -= 1 - # Wait a second and try again. - time.sleep(1) + # Wait for sleep_time before trying again. + wait_time -= sleep_time + time.sleep(sleep_time) return False @@ -86,7 +92,7 @@ class PyWebSocket(http_server.Lighttpd): def __init__(self, port_obj, output_dir, port=_DEFAULT_WS_PORT, root=None, use_tls=False, - register_cygwin=None, + register_cygwin=True, pidfile=None): """Args: output_dir: the absolute path to the layout test result directory @@ -126,7 +132,7 @@ class PyWebSocket(http_server.Lighttpd): def start(self): if not self._web_socket_tests: - logging.info('No need to start %s server.' % self._server_name) + _log.info('No need to start %s server.' % self._server_name) return if self.is_running(): raise PyWebSocketNotStarted('%s is already running.' % @@ -150,27 +156,27 @@ class PyWebSocket(http_server.Lighttpd): python_interp = sys.executable pywebsocket_base = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))))), 'pywebsocket') + os.path.abspath(__file__)))), 'thirdparty', 'pywebsocket') pywebsocket_script = os.path.join(pywebsocket_base, 'mod_pywebsocket', 'standalone.py') start_cmd = [ python_interp, pywebsocket_script, - '-p', str(self._port), - '-d', self._layout_tests, - '-s', self._web_socket_tests, - '-x', '/websocket/tests/cookies', - '-l', error_log, + '--server-host', '127.0.0.1', + '--port', str(self._port), + '--document-root', self._layout_tests, + '--scan-dir', self._web_socket_tests, + '--cgi-paths', '/websocket/tests', + '--log-file', error_log, ] handler_map_file = os.path.join(self._web_socket_tests, 'handler_map.txt') if os.path.exists(handler_map_file): - logging.debug('Using handler_map_file: %s' % handler_map_file) - start_cmd.append('-m') + _log.debug('Using handler_map_file: %s' % handler_map_file) + start_cmd.append('--websock-handlers-map-file') start_cmd.append(handler_map_file) else: - logging.warning('No handler_map_file found') + _log.warning('No handler_map_file found') if self._use_tls: start_cmd.extend(['-t', '-k', self._private_key, @@ -183,6 +189,8 @@ class PyWebSocket(http_server.Lighttpd): self._port_obj.path_from_chromium_base('third_party', 'cygwin', 'bin'), env['PATH']) + env['CYGWIN_PATH'] = self._port_obj.path_from_chromium_base( + 'third_party', 'cygwin', 'bin') if sys.platform == 'win32' and self._register_cygwin: setup_mount = self._port_obj.path_from_chromium_base( @@ -192,16 +200,16 @@ class PyWebSocket(http_server.Lighttpd): env['PYTHONPATH'] = (pywebsocket_base + os.path.pathsep + env.get('PYTHONPATH', '')) - logging.debug('Starting %s server on %d.' % ( - self._server_name, self._port)) - logging.debug('cmdline: %s' % ' '.join(start_cmd)) - self._process = subprocess.Popen(start_cmd, stdout=self._wsout, + _log.debug('Starting %s server on %d.' % ( + self._server_name, self._port)) + _log.debug('cmdline: %s' % ' '.join(start_cmd)) + # FIXME: We should direct this call through Executive for testing. + self._process = subprocess.Popen(start_cmd, + stdin=open(os.devnull, 'r'), + stdout=self._wsout, stderr=subprocess.STDOUT, env=env) - # Wait a bit before checking the liveness of the server. - time.sleep(0.5) - if self._use_tls: url = 'https' else: @@ -211,7 +219,7 @@ class PyWebSocket(http_server.Lighttpd): fp = open(output_log) try: for line in fp: - logging.error(line) + _log.error(line) finally: fp.close() raise PyWebSocketNotStarted( @@ -231,6 +239,7 @@ class PyWebSocket(http_server.Lighttpd): if not force and not self.is_running(): return + pid = None if self._process: pid = self._process.pid elif self._pidfile: @@ -242,8 +251,9 @@ class PyWebSocket(http_server.Lighttpd): raise PyWebSocketNotFound( 'Failed to find %s server pid.' % self._server_name) - logging.debug('Shutting down %s server %d.' % (self._server_name, pid)) - self._port_obj._kill_process(pid) + _log.debug('Shutting down %s server %d.' % (self._server_name, pid)) + # FIXME: We should use a non-static Executive for easier testing. + Executive().kill_process(pid) if self._process: self._process.wait() @@ -252,53 +262,3 @@ class PyWebSocket(http_server.Lighttpd): if self._wsout: self._wsout.close() self._wsout = None - - -if '__main__' == __name__: - # Provide some command line params for starting the PyWebSocket server - # manually. - option_parser = optparse.OptionParser() - option_parser.add_option('--server', type='choice', - choices=['start', 'stop'], default='start', - help='Server action (start|stop)') - option_parser.add_option('-p', '--port', dest='port', - default=None, help='Port to listen on') - option_parser.add_option('-r', '--root', - help='Absolute path to DocumentRoot ' - '(overrides layout test roots)') - option_parser.add_option('-t', '--tls', dest='use_tls', - action='store_true', - default=False, help='use TLS (wss://)') - option_parser.add_option('-k', '--private_key', dest='private_key', - default='', help='TLS private key file.') - option_parser.add_option('-c', '--certificate', dest='certificate', - default='', help='TLS certificate file.') - option_parser.add_option('--register_cygwin', action="store_true", - dest="register_cygwin", - help='Register Cygwin paths (on Win try bots)') - option_parser.add_option('--pidfile', help='path to pid file.') - options, args = option_parser.parse_args() - - if not options.port: - if options.use_tls: - options.port = _DEFAULT_WSS_PORT - else: - options.port = _DEFAULT_WS_PORT - - kwds = {'port': options.port, 'use_tls': options.use_tls} - if options.root: - kwds['root'] = options.root - if options.private_key: - kwds['private_key'] = options.private_key - if options.certificate: - kwds['certificate'] = options.certificate - kwds['register_cygwin'] = options.register_cygwin - if options.pidfile: - kwds['pidfile'] = options.pidfile - - pywebsocket = PyWebSocket(tempfile.gettempdir(), **kwds) - - if 'start' == options.server: - pywebsocket.start() - else: - pywebsocket.stop(force=True) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py new file mode 100644 index 0000000..2bf692b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py @@ -0,0 +1,75 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""WebKit Win implementation of the Port interface.""" + +import logging +import os +import subprocess + +from webkitpy.layout_tests.port.webkit import WebKitPort + +_log = logging.getLogger("webkitpy.layout_tests.port.win") + + +class WinPort(WebKitPort): + """WebKit Win implementation of the Port class.""" + + def __init__(self, port_name=None, options=None): + if port_name is None: + port_name = 'win' + WebKitPort.__init__(self, port_name, options) + + def _tests_for_other_platforms(self): + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + # This really need to be automated. + return [ + "platform/chromium", + "platform/gtk", + "platform/qt", + "platform/mac", + ] + + def _path_to_apache_config_file(self): + return os.path.join(self.layout_tests_dir(), 'http', 'conf', + 'cygwin-httpd.conf') + + def _shut_down_http_server(self, server_pid): + """Shut down the httpd web server. Blocks until it's fully + shut down. + + Args: + server_pid: The process ID of the running server. + """ + # Looks like we ignore server_pid. + # Copy/pasted from chromium-win. + subprocess.Popen(('taskkill.exe', '/f', '/im', 'httpd.exe'), + stdin=open(os.devnull, 'r'), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).wait() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py index 4604a1a..b972154 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py @@ -41,6 +41,7 @@ The script does the following for each platform specified: At the end, the script generates a html that compares old and new baselines. """ +import copy import logging import optparse import os @@ -59,6 +60,9 @@ from layout_package import test_expectations from test_types import image_diff from test_types import text_diff +_log = logging.getLogger("webkitpy.layout_tests." + "rebaseline_chromium_webkit_tests") + # Repository type constants. REPO_SVN, REPO_UNKNOWN = range(2) @@ -137,11 +141,11 @@ def log_dashed_string(text, platform, logging_level=logging.INFO): msg = '%s %s %s' % (dashes, msg, dashes) if logging_level == logging.ERROR: - logging.error(msg) + _log.error(msg) elif logging_level == logging.WARNING: - logging.warn(msg) + _log.warn(msg) else: - logging.info(msg) + _log.info(msg) def setup_html_directory(html_directory): @@ -163,11 +167,11 @@ def setup_html_directory(html_directory): os.mkdir(html_directory) html_directory = os.path.join(html_directory, 'rebaseline_html') - logging.info('Html directory: "%s"', html_directory) + _log.info('Html directory: "%s"', html_directory) if os.path.exists(html_directory): shutil.rmtree(html_directory, True) - logging.info('Deleted file at html directory: "%s"', html_directory) + _log.info('Deleted file at html directory: "%s"', html_directory) if not os.path.exists(html_directory): os.mkdir(html_directory) @@ -191,7 +195,7 @@ def get_result_file_fullpath(html_directory, baseline_filename, platform, base, ext = os.path.splitext(baseline_filename) result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext) fullpath = os.path.join(html_directory, result_filename) - logging.debug(' Result file full path: "%s".', fullpath) + _log.debug(' Result file full path: "%s".', fullpath) return fullpath @@ -200,12 +204,21 @@ class Rebaseliner(object): REVISION_REGEX = r'<a href=\"(\d+)/\">' - def __init__(self, port, platform, options): - self._file_dir = port.path_from_chromium_base('webkit', 'tools', - 'layout_tests') - self._port = port + def __init__(self, running_port, target_port, platform, options): + """ + Args: + running_port: the Port the script is running on. + target_port: the Port the script uses to find port-specific + configuration information like the test_expectations.txt + file location and the list of test platforms. + platform: the test platform to rebaseline + options: the command-line options object.""" self._platform = platform self._options = options + self._port = running_port + self._target_port = target_port + self._rebaseline_port = port.get( + self._target_port.test_platform_name_to_name(platform), options) self._rebaselining_tests = [] self._rebaselined_tests = [] @@ -213,9 +226,9 @@ class Rebaseliner(object): # -. compile list of tests that need rebaselining. # -. update the tests in test_expectations file after rebaseline # is done. - expectations_str = self._port.test_expectations() + expectations_str = self._rebaseline_port.test_expectations() self._test_expectations = \ - test_expectations.TestExpectations(self._port, + test_expectations.TestExpectations(self._rebaseline_port, None, expectations_str, self._platform, @@ -233,9 +246,9 @@ class Rebaseliner(object): log_dashed_string('Downloading archive', self._platform) archive_file = self._download_buildbot_archive() - logging.info('') + _log.info('') if not archive_file: - logging.error('No archive found.') + _log.error('No archive found.') return False log_dashed_string('Extracting and adding new baselines', @@ -246,19 +259,19 @@ class Rebaseliner(object): log_dashed_string('Updating rebaselined tests in file', self._platform) self._update_rebaselined_tests_in_file(backup) - logging.info('') + _log.info('') if len(self._rebaselining_tests) != len(self._rebaselined_tests): - logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN ' - 'REBASELINED.') - logging.warning(' Total tests needing rebaselining: %d', - len(self._rebaselining_tests)) - logging.warning(' Total tests rebaselined: %d', - len(self._rebaselined_tests)) + _log.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN ' + 'REBASELINED.') + _log.warning(' Total tests needing rebaselining: %d', + len(self._rebaselining_tests)) + _log.warning(' Total tests rebaselined: %d', + len(self._rebaselined_tests)) return False - logging.warning('All tests needing rebaselining were successfully ' - 'rebaselined.') + _log.warning('All tests needing rebaselining were successfully ' + 'rebaselined.') return True @@ -285,16 +298,16 @@ class Rebaseliner(object): self._rebaselining_tests = \ self._test_expectations.get_rebaselining_failures() if not self._rebaselining_tests: - logging.warn('No tests found that need rebaselining.') + _log.warn('No tests found that need rebaselining.') return None - logging.info('Total number of tests needing rebaselining ' - 'for "%s": "%d"', self._platform, - len(self._rebaselining_tests)) + _log.info('Total number of tests needing rebaselining ' + 'for "%s": "%d"', self._platform, + len(self._rebaselining_tests)) test_no = 1 for test in self._rebaselining_tests: - logging.info(' %d: %s', test_no, test) + _log.info(' %d: %s', test_no, test) test_no += 1 return self._rebaselining_tests @@ -310,7 +323,7 @@ class Rebaseliner(object): None on failure. """ - logging.debug('Url to retrieve revision: "%s"', url) + _log.debug('Url to retrieve revision: "%s"', url) f = urllib.urlopen(url) content = f.read() @@ -318,11 +331,11 @@ class Rebaseliner(object): revisions = re.findall(self.REVISION_REGEX, content) if not revisions: - logging.error('Failed to find revision, content: "%s"', content) + _log.error('Failed to find revision, content: "%s"', content) return None revisions.sort(key=int) - logging.info('Latest revision: "%s"', revisions[len(revisions) - 1]) + _log.info('Latest revision: "%s"', revisions[len(revisions) - 1]) return revisions[len(revisions) - 1] def _get_archive_dir_name(self, platform, webkit_canary): @@ -339,8 +352,8 @@ class Rebaseliner(object): if platform in ARCHIVE_DIR_NAME_DICT: return ARCHIVE_DIR_NAME_DICT[platform] else: - logging.error('Cannot find platform key %s in archive ' - 'directory name dictionary', platform) + _log.error('Cannot find platform key %s in archive ' + 'directory name dictionary', platform) return None def _get_archive_url(self): @@ -356,7 +369,7 @@ class Rebaseliner(object): if not dir_name: return None - logging.debug('Buildbot platform dir name: "%s"', dir_name) + _log.debug('Buildbot platform dir name: "%s"', dir_name) url_base = '%s/%s/' % (self._options.archive_url, dir_name) latest_revision = self._get_latest_revision(url_base) @@ -364,7 +377,7 @@ class Rebaseliner(object): return None archive_url = ('%s%s/layout-test-results.zip' % (url_base, latest_revision)) - logging.info('Archive url: "%s"', archive_url) + _log.info('Archive url: "%s"', archive_url) return archive_url def _download_buildbot_archive(self): @@ -380,7 +393,7 @@ class Rebaseliner(object): return None fn = urllib.urlretrieve(url)[0] - logging.info('Archive downloaded and saved to file: "%s"', fn) + _log.info('Archive downloaded and saved to file: "%s"', fn) return fn def _extract_and_add_new_baselines(self, archive_file): @@ -397,17 +410,18 @@ class Rebaseliner(object): zip_file = zipfile.ZipFile(archive_file, 'r') zip_namelist = zip_file.namelist() - logging.debug('zip file namelist:') + _log.debug('zip file namelist:') for name in zip_namelist: - logging.debug(' ' + name) + _log.debug(' ' + name) - platform = self._port.name() - logging.debug('Platform dir: "%s"', platform) + platform = self._rebaseline_port.test_platform_name_to_name( + self._platform) + _log.debug('Platform dir: "%s"', platform) test_no = 1 self._rebaselined_tests = [] for test in self._rebaselining_tests: - logging.info('Test %d: %s', test_no, test) + _log.info('Test %d: %s', test_no, test) found = False svn_error = False @@ -415,14 +429,14 @@ class Rebaseliner(object): for suffix in BASELINE_SUFFIXES: archive_test_name = ('layout-test-results/%s-actual%s' % (test_basename, suffix)) - logging.debug(' Archive test file name: "%s"', - archive_test_name) + _log.debug(' Archive test file name: "%s"', + archive_test_name) if not archive_test_name in zip_namelist: - logging.info(' %s file not in archive.', suffix) + _log.info(' %s file not in archive.', suffix) continue found = True - logging.info(' %s file found in archive.', suffix) + _log.info(' %s file found in archive.', suffix) # Extract new baseline from archive and save it to a temp file. data = zip_file.read(archive_test_name) @@ -433,11 +447,10 @@ class Rebaseliner(object): expected_filename = '%s-expected%s' % (test_basename, suffix) expected_fullpath = os.path.join( - self._port._chromium_baseline_path(platform), - expected_filename) + self._rebaseline_port.baseline_path(), expected_filename) expected_fullpath = os.path.normpath(expected_fullpath) - logging.debug(' Expected file full path: "%s"', - expected_fullpath) + _log.debug(' Expected file full path: "%s"', + expected_fullpath) # TODO(victorw): for now, the rebaselining tool checks whether # or not THIS baseline is duplicate and should be skipped. @@ -466,12 +479,12 @@ class Rebaseliner(object): self._create_html_baseline_files(expected_fullpath) if not found: - logging.warn(' No new baselines found in archive.') + _log.warn(' No new baselines found in archive.') else: if svn_error: - logging.warn(' Failed to add baselines to SVN.') + _log.warn(' Failed to add baselines to SVN.') else: - logging.info(' Rebaseline succeeded.') + _log.info(' Rebaseline succeeded.') self._rebaselined_tests.append(test) test_no += 1 @@ -499,9 +512,10 @@ class Rebaseliner(object): True if the baseline is unnecessary. False otherwise. """ - test_filepath = os.path.join(self._port.layout_tests_dir(), test) - all_baselines = self._port.expected_baselines(test_filepath, - suffix, True) + test_filepath = os.path.join(self._target_port.layout_tests_dir(), + test) + all_baselines = self._rebaseline_port.expected_baselines( + test_filepath, suffix, True) for (fallback_dir, fallback_file) in all_baselines: if fallback_dir and fallback_file: fallback_fullpath = os.path.normpath( @@ -509,8 +523,8 @@ class Rebaseliner(object): if fallback_fullpath.lower() != baseline_path.lower(): if not self._diff_baselines(new_baseline, fallback_fullpath): - logging.info(' Found same baseline at %s', - fallback_fullpath) + _log.info(' Found same baseline at %s', + fallback_fullpath) return True else: return False @@ -531,15 +545,15 @@ class Rebaseliner(object): ext1 = os.path.splitext(file1)[1].upper() ext2 = os.path.splitext(file2)[1].upper() if ext1 != ext2: - logging.warn('Files to compare have different ext. ' - 'File1: %s; File2: %s', file1, file2) + _log.warn('Files to compare have different ext. ' + 'File1: %s; File2: %s', file1, file2) return True if ext1 == '.PNG': - return image_diff.ImageDiff(self._port, self._platform, - '').diff_files(self._port, file1, file2) + return image_diff.ImageDiff(self._port, + '').diff_files(self._port, file1, file2) else: - return text_diff.TestTextDiff(self._port, self._platform, + return text_diff.TestTextDiff(self._port, '').diff_files(self._port, file1, file2) def _delete_baseline(self, filename): @@ -575,20 +589,20 @@ class Rebaseliner(object): new_expectations = ( self._test_expectations.remove_platform_from_expectations( self._rebaselined_tests, self._platform)) - path = self._port.path_to_test_expectations_file() + path = self._target_port.path_to_test_expectations_file() if backup: date_suffix = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) backup_file = ('%s.orig.%s' % (path, date_suffix)) if os.path.exists(backup_file): os.remove(backup_file) - logging.info('Saving original file to "%s"', backup_file) + _log.info('Saving original file to "%s"', backup_file) os.rename(path, backup_file) f = open(path, "w") f.write(new_expectations) f.close() else: - logging.info('No test was rebaselined so nothing to remove.') + _log.info('No test was rebaselined so nothing to remove.') def _svn_add(self, filename): """Add the file to SVN repository. @@ -607,7 +621,7 @@ class Rebaseliner(object): parent_dir, basename = os.path.split(filename) if self._repo_type != REPO_SVN or parent_dir == filename: - logging.info("No svn checkout found, skip svn add.") + _log.info("No svn checkout found, skip svn add.") return True original_dir = os.getcwd() @@ -616,12 +630,12 @@ class Rebaseliner(object): os.chdir(original_dir) output = status_output.upper() if output.startswith('A') or output.startswith('M'): - logging.info(' File already added to SVN: "%s"', filename) + _log.info(' File already added to SVN: "%s"', filename) return True if output.find('IS NOT A WORKING COPY') >= 0: - logging.info(' File is not a working copy, add its parent: "%s"', - parent_dir) + _log.info(' File is not a working copy, add its parent: "%s"', + parent_dir) return self._svn_add(parent_dir) os.chdir(parent_dir) @@ -629,19 +643,19 @@ class Rebaseliner(object): os.chdir(original_dir) output = add_output.upper().rstrip() if output.startswith('A') and output.find(basename.upper()) >= 0: - logging.info(' Added new file: "%s"', filename) + _log.info(' Added new file: "%s"', filename) self._svn_prop_set(filename) return True if (not status_output) and (add_output.upper().find( 'ALREADY UNDER VERSION CONTROL') >= 0): - logging.info(' File already under SVN and has no change: "%s"', - filename) + _log.info(' File already under SVN and has no change: "%s"', + filename) return True - logging.warn(' Failed to add file to SVN: "%s"', filename) - logging.warn(' Svn status output: "%s"', status_output) - logging.warn(' Svn add output: "%s"', add_output) + _log.warn(' Failed to add file to SVN: "%s"', filename) + _log.warn(' Svn status output: "%s"', status_output) + _log.warn(' Svn add output: "%s"', add_output) return False def _svn_prop_set(self, filename): @@ -667,7 +681,7 @@ class Rebaseliner(object): else: cmd = ['svn', 'pset', 'svn:eol-style', 'LF', basename] - logging.debug(' Set svn prop: %s', ' '.join(cmd)) + _log.debug(' Set svn prop: %s', ' '.join(cmd)) run_shell(cmd, False) os.chdir(original_dir) @@ -689,14 +703,14 @@ class Rebaseliner(object): baseline_filename, self._platform, 'new') shutil.copyfile(baseline_fullpath, new_file) - logging.info(' Html: copied new baseline file from "%s" to "%s".', - baseline_fullpath, new_file) + _log.info(' Html: copied new baseline file from "%s" to "%s".', + baseline_fullpath, new_file) # Get the old baseline from SVN and save to the html directory. output = run_shell(['svn', 'cat', '-r', 'BASE', baseline_fullpath]) if (not output) or (output.upper().rstrip().endswith( 'NO SUCH FILE OR DIRECTORY')): - logging.info(' No base file: "%s"', baseline_fullpath) + _log.info(' No base file: "%s"', baseline_fullpath) return base_file = get_result_file_fullpath(self._options.html_directory, baseline_filename, self._platform, @@ -704,8 +718,8 @@ class Rebaseliner(object): f = open(base_file, 'wb') f.write(output) f.close() - logging.info(' Html: created old baseline file: "%s".', - base_file) + _log.info(' Html: created old baseline file: "%s".', + base_file) # Get the diff between old and new baselines and save to the html dir. if baseline_filename.upper().endswith('.TXT'): @@ -721,7 +735,7 @@ class Rebaseliner(object): else: parent_dir = sys.path[0] # tempdir is not secure. bogus_dir = os.path.join(parent_dir, "temp_svn_config") - logging.debug(' Html: temp config dir: "%s".', bogus_dir) + _log.debug(' Html: temp config dir: "%s".', bogus_dir) if not os.path.exists(bogus_dir): os.mkdir(bogus_dir) delete_bogus_dir = True @@ -737,13 +751,13 @@ class Rebaseliner(object): f = open(diff_file, 'wb') f.write(output) f.close() - logging.info(' Html: created baseline diff file: "%s".', - diff_file) + _log.info(' Html: created baseline diff file: "%s".', + diff_file) if delete_bogus_dir: shutil.rmtree(bogus_dir, True) - logging.debug(' Html: removed temp config dir: "%s".', - bogus_dir) + _log.debug(' Html: removed temp config dir: "%s".', + bogus_dir) class HtmlGenerator(object): @@ -792,9 +806,9 @@ class HtmlGenerator(object): '<img style="width: 200" src="%(uri)s" /></a></td>') HTML_TR = '<tr>%s</tr>' - def __init__(self, port, options, platforms, rebaselining_tests): + def __init__(self, target_port, options, platforms, rebaselining_tests): self._html_directory = options.html_directory - self._port = port + self._target_port = target_port self._platforms = platforms self._rebaselining_tests = rebaselining_tests self._html_file = os.path.join(options.html_directory, @@ -803,7 +817,7 @@ class HtmlGenerator(object): def generate_html(self): """Generate html file for rebaselining result comparison.""" - logging.info('Generating html file') + _log.info('Generating html file') html_body = '' if not self._rebaselining_tests: @@ -814,29 +828,29 @@ class HtmlGenerator(object): test_no = 1 for test in tests: - logging.info('Test %d: %s', test_no, test) + _log.info('Test %d: %s', test_no, test) html_body += self._generate_html_for_one_test(test) html = self.HTML_REBASELINE % ({'time': time.asctime(), 'body': html_body}) - logging.debug(html) + _log.debug(html) f = open(self._html_file, 'w') f.write(html) f.close() - logging.info('Baseline comparison html generated at "%s"', - self._html_file) + _log.info('Baseline comparison html generated at "%s"', + self._html_file) def show_html(self): """Launch the rebaselining html in brwoser.""" - logging.info('Launching html: "%s"', self._html_file) + _log.info('Launching html: "%s"', self._html_file) - html_uri = self._port.filename_to_uri(self._html_file) + html_uri = self._target_port.filename_to_uri(self._html_file) webbrowser.open(html_uri, 1) - logging.info('Html launched.') + _log.info('Html launched.') def _generate_baseline_links(self, test_basename, suffix, platform): """Generate links for baseline results (old, new and diff). @@ -851,18 +865,18 @@ class HtmlGenerator(object): """ baseline_filename = '%s-expected%s' % (test_basename, suffix) - logging.debug(' baseline filename: "%s"', baseline_filename) + _log.debug(' baseline filename: "%s"', baseline_filename) new_file = get_result_file_fullpath(self._html_directory, baseline_filename, platform, 'new') - logging.info(' New baseline file: "%s"', new_file) + _log.info(' New baseline file: "%s"', new_file) if not os.path.exists(new_file): - logging.info(' No new baseline file: "%s"', new_file) + _log.info(' No new baseline file: "%s"', new_file) return '' old_file = get_result_file_fullpath(self._html_directory, baseline_filename, platform, 'old') - logging.info(' Old baseline file: "%s"', old_file) + _log.info(' Old baseline file: "%s"', old_file) if suffix == '.png': html_td_link = self.HTML_TD_LINK_IMG else: @@ -871,24 +885,25 @@ class HtmlGenerator(object): links = '' if os.path.exists(old_file): links += html_td_link % { - 'uri': self._port.filename_to_uri(old_file), + 'uri': self._target_port.filename_to_uri(old_file), 'name': baseline_filename} else: - logging.info(' No old baseline file: "%s"', old_file) + _log.info(' No old baseline file: "%s"', old_file) links += self.HTML_TD_NOLINK % '' - links += html_td_link % {'uri': self._port.filename_to_uri(new_file), + links += html_td_link % {'uri': self._target_port.filename_to_uri( + new_file), 'name': baseline_filename} diff_file = get_result_file_fullpath(self._html_directory, baseline_filename, platform, 'diff') - logging.info(' Baseline diff file: "%s"', diff_file) + _log.info(' Baseline diff file: "%s"', diff_file) if os.path.exists(diff_file): - links += html_td_link % {'uri': self._port.filename_to_uri( + links += html_td_link % {'uri': self._target_port.filename_to_uri( diff_file), 'name': 'Diff'} else: - logging.info(' No baseline diff file: "%s"', diff_file) + _log.info(' No baseline diff file: "%s"', diff_file) links += self.HTML_TD_NOLINK % '' return links @@ -904,13 +919,13 @@ class HtmlGenerator(object): """ test_basename = os.path.basename(os.path.splitext(test)[0]) - logging.info(' basename: "%s"', test_basename) + _log.info(' basename: "%s"', test_basename) rows = [] for suffix in BASELINE_SUFFIXES: if suffix == '.checksum': continue - logging.info(' Checking %s files', suffix) + _log.info(' Checking %s files', suffix) for platform in self._platforms: links = self._generate_baseline_links(test_basename, suffix, platform) @@ -919,17 +934,18 @@ class HtmlGenerator(object): suffix) row += self.HTML_TD_NOLINK % platform row += links - logging.debug(' html row: %s', row) + _log.debug(' html row: %s', row) rows.append(self.HTML_TR % row) if rows: - test_path = os.path.join(self._port.layout_tests_dir(), test) - html = self.HTML_TR_TEST % (self._port.filename_to_uri(test_path), - test) + test_path = os.path.join(self._target_port.layout_tests_dir(), + test) + html = self.HTML_TR_TEST % ( + self._target_port.filename_to_uri(test_path), test) html += self.HTML_TEST_DETAIL % ' '.join(rows) - logging.debug(' html for test: %s', html) + _log.debug(' html for test: %s', html) return self.HTML_TABLE_TEST % html return '' @@ -982,8 +998,23 @@ def main(): help=('The directory that stores the results for' ' rebaselining comparison.')) + option_parser.add_option('', '--target-platform', + default='chromium', + help=('The target platform to rebaseline ' + '("mac", "chromium", "qt", etc.). Defaults ' + 'to "chromium".')) options = option_parser.parse_args()[0] - port_obj = port.get(None, options) + + # We need to create three different Port objects over the life of this + # script. |target_port_obj| is used to determine configuration information: + # location of the expectations file, names of ports to rebaseline, etc. + # |port_obj| is used for runtime functionality like actually diffing + # Then we create a rebaselining port to actual find and manage the + # baselines. + target_options = copy.copy(options) + if options.target_platform == 'chromium': + target_options.chromium = True + target_port_obj = port.get(None, target_options) # Set up our logging format. log_level = logging.INFO @@ -994,15 +1025,27 @@ def main(): '%(levelname)s %(message)s'), datefmt='%y%m%d %H:%M:%S') + # options.configuration is used by port to locate image_diff binary. + # Check the imgage_diff release binary, if it does not exist, + # fallback to debug. + options.configuration = "Release" + port_obj = port.get(None, options) + if not port_obj.check_image_diff(override_step=None, logging=False): + _log.debug('No release version image diff binary found.') + options.configuration = "Debug" + port_obj = port.get(None, options) + else: + _log.debug('Found release version image diff binary.') + # Verify 'platforms' option is valid if not options.platforms: - logging.error('Invalid "platforms" option. --platforms must be ' - 'specified in order to rebaseline.') + _log.error('Invalid "platforms" option. --platforms must be ' + 'specified in order to rebaseline.') sys.exit(1) platforms = [p.strip().lower() for p in options.platforms.split(',')] for platform in platforms: if not platform in REBASELINE_PLATFORM_ORDER: - logging.error('Invalid platform: "%s"' % (platform)) + _log.error('Invalid platform: "%s"' % (platform)) sys.exit(1) # Adjust the platform order so rebaseline tool is running at the order of @@ -1019,9 +1062,9 @@ def main(): rebaselining_tests = set() backup = options.backup for platform in rebaseline_platforms: - rebaseliner = Rebaseliner(port_obj, platform, options) + rebaseliner = Rebaseliner(port_obj, target_port_obj, platform, options) - logging.info('') + _log.info('') log_dashed_string('Rebaseline started', platform) if rebaseliner.run(backup): # Only need to backup one original copy of test expectation file. @@ -1032,9 +1075,9 @@ def main(): rebaselining_tests |= set(rebaseliner.get_rebaselining_tests()) - logging.info('') + _log.info('') log_dashed_string('Rebaselining result comparison started', None) - html_generator = HtmlGenerator(port_obj, + html_generator = HtmlGenerator(target_port_obj, options, rebaseline_platforms, rebaselining_tests) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_chromium_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index f0b68ee..73195b3 100755 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_chromium_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Run layout tests using the test_shell. +"""Run layout tests. This is a port of the existing webkit test script run-webkit-tests. @@ -44,12 +44,16 @@ directory. Entire lines starting with '//' (comments) will be ignored. For details of the files' contents and purposes, see test_lists/README. """ +from __future__ import with_statement + +import codecs import errno import glob import logging import math import optparse import os +import platform import Queue import random import re @@ -58,27 +62,50 @@ import sys import time import traceback -import simplejson - from layout_package import test_expectations from layout_package import json_layout_results_generator from layout_package import metered_stream from layout_package import test_failures -from layout_package import test_shell_thread +from layout_package import dump_render_tree_thread from layout_package import test_files from test_types import fuzzy_image_diff from test_types import image_diff from test_types import test_type_base from test_types import text_diff +from webkitpy.common.system.executive import Executive +from webkitpy.thirdparty import simplejson + import port +_log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests") + +# dummy value used for command-line explicitness to disable defaults +LOG_NOTHING = 'nothing' + +# Display the one-line progress bar (% completed) while testing +LOG_PROGRESS = 'progress' + # Indicates that we want detailed progress updates in the output (prints # directory-by-directory feedback). LOG_DETAILED_PROGRESS = 'detailed-progress' +# Log the one-line summary at the end of the run +LOG_SUMMARY = 'summary' + +# "Trace" the test - log the expected result, the actual result, and the +# baselines used +LOG_TRACE = 'trace' + # Log any unexpected results while running (instead of just at the end). LOG_UNEXPECTED = 'unexpected' +LOG_UNEXPECTED_RESULTS = 'unexpected-results' + +LOG_VALUES = ",".join(("actual", "config", LOG_DETAILED_PROGRESS, "expected", + LOG_NOTHING, LOG_PROGRESS, LOG_SUMMARY, "timing", + LOG_UNEXPECTED, LOG_UNEXPECTED_RESULTS)) +LOG_DEFAULT_VALUE = ",".join((LOG_DETAILED_PROGRESS, LOG_SUMMARY, + LOG_UNEXPECTED, LOG_UNEXPECTED_RESULTS)) # Builder base URL where we have the archived test results. BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/" @@ -98,13 +125,27 @@ class TestInfo: self.filename = filename self.uri = port.filename_to_uri(filename) self.timeout = timeout - expected_hash_file = port.expected_filename(filename, '.checksum') + # FIXME: Confusing that the file is .checksum and we call it "hash" + self._expected_hash_path = port.expected_filename(filename, '.checksum') + self._have_read_expected_hash = False + self._image_hash = None + + def _read_image_hash(self): try: - self.image_hash = open(expected_hash_file, "r").read() + with codecs.open(self._expected_hash_path, "r", "ascii") as hash_file: + return hash_file.read() except IOError, e: if errno.ENOENT != e.errno: raise - self.image_hash = None + + def image_hash(self): + # Read the image_hash lazily to reduce startup time. + # This class is accessed across threads, but only one thread should + # ever be dealing with any given TestInfo so no locking is needed. + if not self._have_read_expected_hash: + self._have_read_expected_hash = True + self._image_hash = self._read_image_hash() + return self._image_hash class ResultSummary(object): @@ -131,25 +172,23 @@ class ResultSummary(object): self.tests_by_timeline[timeline] = ( expectations.get_tests_with_timeline(timeline)) - def add(self, test, failures, result, expected): - """Add a result into the appropriate bin. + def add(self, result, expected): + """Add a TestResult into the appropriate bin. Args: - test: test file name - failures: list of failure objects from test execution - result: result of test (PASS, IMAGE, etc.). + result: TestResult from dump_render_tree_thread. expected: whether the result was what we expected it to be. """ - self.tests_by_expectation[result].add(test) - self.results[test] = result + self.tests_by_expectation[result.type].add(result.filename) + self.results[result.filename] = result.type self.remaining -= 1 - if len(failures): - self.failures[test] = failures + if len(result.failures): + self.failures[result.filename] = result.failures if expected: self.expected += 1 else: - self.unexpected_results[test] = result + self.unexpected_results[result.filename] = result.type self.unexpected += 1 @@ -162,7 +201,7 @@ class TestRunner: # The per-test timeout in milliseconds, if no --time-out-ms option was # given to run_webkit_tests. This should correspond to the default timeout - # in test_shell.exe. + # in DumpRenderTree. DEFAULT_TEST_TIMEOUT_MS = 6 * 1000 NUM_RETRY_ON_UNEXPECTED_FAILURE = 1 @@ -197,14 +236,16 @@ class TestRunner: self._current_progress_str = "" self._current_test_number = 0 + self._retries = 0 + def __del__(self): - logging.debug("flushing stdout") + _log.debug("flushing stdout") sys.stdout.flush() - logging.debug("flushing stderr") + _log.debug("flushing stderr") sys.stderr.flush() - logging.debug("stopping http server") + _log.debug("stopping http server") self._port.stop_http_server() - logging.debug("stopping websocket server") + _log.debug("stopping websocket server") self._port.stop_websocket_server() def gather_file_paths(self, paths): @@ -225,11 +266,13 @@ class TestRunner: try: expectations_str = self._port.test_expectations() + overrides_str = self._port.test_expectations_overrides() self._expectations = test_expectations.TestExpectations( self._port, test_files, expectations_str, test_platform_name, - is_debug_mode, self._options.lint_test_files) + is_debug_mode, self._options.lint_test_files, + tests_are_present=True, overrides=overrides_str) return self._expectations - except Exception, err: + except SyntaxError, err: if self._options.lint_test_files: print str(err) else: @@ -274,7 +317,7 @@ class TestRunner: test_size = int(chunk_len) assert(test_size > 0) except: - logging.critical("invalid chunk '%s'" % chunk_value) + _log.critical("invalid chunk '%s'" % chunk_value) sys.exit(1) # Get the number of tests @@ -343,7 +386,7 @@ class TestRunner: self._expectations = self.parse_expectations( self._port.test_platform_name(), - self._options.target == 'Debug') + self._options.configuration == 'Debug') self._test_files = set(files) self._test_files_list = files @@ -361,7 +404,6 @@ class TestRunner: self._print_expected_results_of_type(write, result_summary, test_expectations.SKIP, "skipped") - if self._options.force: write('Running all tests, including skips (--force)') else: @@ -369,8 +411,11 @@ class TestRunner: # subtracted out of self._test_files, above), but we stub out the # results here so the statistics can remain accurate. for test in skip_chunk: - result_summary.add(test, [], test_expectations.SKIP, - expected=True) + result = dump_render_tree_thread.TestResult(test, + failures=[], test_run_time=0, total_time_for_all_diffs=0, + time_for_diffs=0) + result.type = test_expectations.SKIP + result_summary.add(result, expected=True) write("") return result_summary @@ -471,12 +516,12 @@ class TestRunner: filename_queue.put(item) return filename_queue - def _get_test_shell_args(self, index): - """Returns the tuple of arguments for tests and for test_shell.""" + def _get_dump_render_tree_args(self, index): + """Returns the tuple of arguments for tests and for DumpRenderTree.""" shell_args = [] test_args = test_type_base.TestArguments() png_path = None - if not self._options.no_pixel_tests: + if self._options.pixel_tests: png_path = os.path.join(self._options.results_directory, "png_result%s.png" % index) shell_args.append("--pixel-tests=" + png_path) @@ -495,12 +540,13 @@ class TestRunner: return test_args, png_path, shell_args def _contains_tests(self, subdir): - for test_file in self._test_files_list: + for test_file in self._test_files: if test_file.find(subdir) >= 0: return True return False - def _instantiate_test_shell_threads(self, test_files, result_summary): + def _instantiate_dump_render_tree_threads(self, test_files, + result_summary): """Instantitates and starts the TestShellThread(s). Return: @@ -510,22 +556,18 @@ class TestRunner: # Instantiate TestShellThreads and start them. threads = [] - for i in xrange(int(self._options.num_test_shells)): + for i in xrange(int(self._options.child_processes)): # Create separate TestTypes instances for each thread. test_types = [] - for t in self._test_types: - test_types.append(t(self._port, self._options.platform, + for test_type in self._test_types: + test_types.append(test_type(self._port, self._options.results_directory)) - test_args, png_path, shell_args = self._get_test_shell_args(i) - thread = test_shell_thread.TestShellThread(self._port, - filename_queue, - self._result_queue, - test_types, - test_args, - png_path, - shell_args, - self._options) + test_args, png_path, shell_args = \ + self._get_dump_render_tree_args(i) + thread = dump_render_tree_thread.TestShellThread(self._port, + filename_queue, self._result_queue, test_types, test_args, + png_path, shell_args, self._options) if self._is_single_threaded(): thread.run_in_main_thread(self, result_summary) else: @@ -536,7 +578,7 @@ class TestRunner: def _is_single_threaded(self): """Returns whether we should run all the tests in the main thread.""" - return int(self._options.num_test_shells) == 1 + return int(self._options.child_processes) == 1 def _run_tests(self, file_list, result_summary): """Runs the tests in the file_list. @@ -552,8 +594,14 @@ class TestRunner: in the form {filename:filename, test_run_time:test_run_time} result_summary: summary object to populate with the results """ - threads = self._instantiate_test_shell_threads(file_list, - result_summary) + plural = "" + if self._options.child_processes > 1: + plural = "s" + self._meter.update('Starting %s%s ...' % + (self._port.driver_name(), plural)) + threads = self._instantiate_dump_render_tree_threads(file_list, + result_summary) + self._meter.update("Starting testing ...") # Wait for the threads to finish and collect test failures. failures = {} @@ -575,11 +623,10 @@ class TestRunner: 'total_time': thread.get_total_time()}) test_timings.update(thread.get_directory_timing_stats()) individual_test_timings.extend( - thread.get_individual_test_stats()) + thread.get_test_results()) except KeyboardInterrupt: for thread in threads: thread.cancel() - self._port.stop_helper() raise for thread in threads: # Check whether a TestShellThread died before normal completion. @@ -594,7 +641,11 @@ class TestRunner: self.update_summary(result_summary) return (thread_timings, test_timings, individual_test_timings) - def run(self, result_summary): + def needs_http(self): + """Returns whether the test runner needs an HTTP server.""" + return self._contains_tests(self.HTTP_SUBDIR) + + def run(self, result_summary, print_results): """Run all our tests on all our test files. For each test file, we run each test type. If there are any failures, @@ -602,22 +653,21 @@ class TestRunner: Args: result_summary: a summary object tracking the test results. + print_results: whether or not to print the summary at the end Return: - We return nonzero if there are regressions compared to the last run. + The number of unexpected results (0 == success) """ if not self._test_files: return 0 start_time = time.time() - # Start up any helper needed - if not self._options.no_pixel_tests: - self._port.start_helper() - - if self._contains_tests(self.HTTP_SUBDIR): + if self.needs_http(): + self._meter.update('Starting HTTP server ...') self._port.start_http_server() if self._contains_tests(self.WEBSOCKET_SUBDIR): + self._meter.update('Starting WebSocket server ...') self._port.start_websocket_server() # self._websocket_secure_server.Start() @@ -627,17 +677,17 @@ class TestRunner: # We exclude the crashes from the list of results to retry, because # we want to treat even a potentially flaky crash as an error. failures = self._get_failures(result_summary, include_crashes=False) - retries = 0 retry_summary = result_summary - while (retries < self.NUM_RETRY_ON_UNEXPECTED_FAILURE and + while (self._retries < self.NUM_RETRY_ON_UNEXPECTED_FAILURE and len(failures)): - logging.debug("Retrying %d unexpected failure(s)" % len(failures)) - retries += 1 + _log.info('') + _log.info("Retrying %d unexpected failure(s)" % len(failures)) + _log.info('') + self._retries += 1 retry_summary = ResultSummary(self._expectations, failures.keys()) self._run_tests(failures.keys(), retry_summary) failures = self._get_failures(retry_summary, include_crashes=True) - self._port.stop_helper() end_time = time.time() write = create_logging_writer(self._options, 'timing') @@ -660,27 +710,29 @@ class TestRunner: sys.stdout.flush() sys.stderr.flush() - if (LOG_DETAILED_PROGRESS in self._options.log or - (LOG_UNEXPECTED in self._options.log and - result_summary.total != result_summary.expected)): - print - # This summary data gets written to stdout regardless of log level - self._print_one_line_summary(result_summary.total, - result_summary.expected) + # (unless of course we're printing nothing). + if print_results: + if (LOG_DETAILED_PROGRESS in self._options.log or + (LOG_UNEXPECTED in self._options.log and + result_summary.total != result_summary.expected)): + print + if LOG_SUMMARY in self._options.log: + self._print_one_line_summary(result_summary.total, + result_summary.expected) unexpected_results = self._summarize_unexpected_results(result_summary, retry_summary) - self._print_unexpected_results(unexpected_results) + if LOG_UNEXPECTED_RESULTS in self._options.log: + self._print_unexpected_results(unexpected_results) # Write the same data to log files. self._write_json_files(unexpected_results, result_summary, individual_test_timings) - # Write the summary to disk (results.html) and maybe open the - # test_shell to this file. + # Write the summary to disk (results.html) and display it if requested. wrote_results = self._write_results_html_file(result_summary) - if not self._options.noshow_results and wrote_results: + if self._options.show_results and wrote_results: self._show_results_html_file() # Ignore flaky failures and unexpected passes so we don't turn the @@ -688,32 +740,69 @@ class TestRunner: return unexpected_results['num_regressions'] def update_summary(self, result_summary): - """Update the summary while running tests.""" + """Update the summary and print results with any completed tests.""" while True: try: - (test, fail_list) = self._result_queue.get_nowait() - result = test_failures.determine_result_type(fail_list) - expected = self._expectations.matches_an_expected_result(test, - result) - result_summary.add(test, fail_list, result, expected) - if (LOG_DETAILED_PROGRESS in self._options.log and - (self._options.experimental_fully_parallel or - self._is_single_threaded())): - self._display_detailed_progress(result_summary) - else: - if not expected and LOG_UNEXPECTED in self._options.log: - self._print_unexpected_test_result(test, result) - self._display_one_line_progress(result_summary) + result = self._result_queue.get_nowait() except Queue.Empty: return - - def _display_one_line_progress(self, result_summary): + expected = self._expectations.matches_an_expected_result( + result.filename, result.type, self._options.pixel_tests) + result_summary.add(result, expected) + self._print_test_results(result, expected, result_summary) + + def _print_test_results(self, result, expected, result_summary): + "Print the result of the test as determined by the --log switches." + if LOG_TRACE in self._options.log: + self._print_test_trace(result) + elif (LOG_DETAILED_PROGRESS in self._options.log and + (self._options.experimental_fully_parallel or + self._is_single_threaded())): + self._print_detailed_progress(result_summary) + else: + if (not expected and LOG_UNEXPECTED in self._options.log): + self._print_unexpected_test_result(result) + self._print_one_line_progress(result_summary) + + def _print_test_trace(self, result): + """Print detailed results of a test (triggered by --log trace). + For each test, print: + - location of the expected baselines + - expected results + - actual result + - timing info + """ + filename = result.filename + test_name = self._port.relative_test_filename(filename) + _log.info('trace: %s' % test_name) + _log.info(' txt: %s' % + self._port.relative_test_filename( + self._port.expected_filename(filename, '.txt'))) + png_file = self._port.expected_filename(filename, '.png') + if os.path.exists(png_file): + _log.info(' png: %s' % + self._port.relative_test_filename(filename)) + else: + _log.info(' png: <none>') + _log.info(' exp: %s' % + self._expectations.get_expectations_string(filename)) + _log.info(' got: %s' % + self._expectations.expectation_to_string(result.type)) + _log.info(' took: %-.3f' % result.test_run_time) + _log.info('') + + def _print_one_line_progress(self, result_summary): """Displays the progress through the test run.""" - self._meter.update("Testing: %d ran as expected, %d didn't, %d left" % - (result_summary.expected, result_summary.unexpected, - result_summary.remaining)) - - def _display_detailed_progress(self, result_summary): + percent_complete = 100 * (result_summary.expected + + result_summary.unexpected) / result_summary.total + action = "Testing" + if self._retries > 0: + action = "Retrying" + self._meter.progress("%s (%d%%): %d ran as expected, %d didn't," + " %d left" % (action, percent_complete, result_summary.expected, + result_summary.unexpected, result_summary.remaining)) + + def _print_detailed_progress(self, result_summary): """Display detailed progress output where we print the directory name and one dot for each completed test. This is triggered by "--log detailed-progress".""" @@ -752,10 +841,17 @@ class TestRunner: if result_summary.remaining: remain_str = " (%d)" % (result_summary.remaining) - self._meter.update("%s%s" % - (self._current_progress_str, remain_str)) + self._meter.progress("%s%s" % + (self._current_progress_str, remain_str)) else: - self._meter.write("%s\n" % (self._current_progress_str)) + self._meter.progress("%s\n" % (self._current_progress_str)) + + def _print_unexpected_test_result(self, result): + """Prints one unexpected test result line.""" + desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result.type][0] + self._meter.write(" %s -> unexpected %s\n" % + (self._port.relative_test_filename(result.filename), + desc)) def _get_failures(self, result_summary, include_crashes): """Filters a dict of results and returns only the failures. @@ -870,8 +966,8 @@ class TestRunner: individual_test_timings: list of test times (used by the flakiness dashboard). """ - logging.debug("Writing JSON files in %s." % - self._options.results_directory) + _log.debug("Writing JSON files in %s." % + self._options.results_directory) unexpected_file = open(os.path.join(self._options.results_directory, "unexpected_results.json"), "w") unexpected_file.write(simplejson.dumps(unexpected_results, @@ -893,7 +989,7 @@ class TestRunner: BUILDER_BASE_URL, individual_test_timings, self._expectations, result_summary, self._test_files_list) - logging.debug("Finished writing JSON files.") + _log.debug("Finished writing JSON files.") def _print_expected_results_of_type(self, write, result_summary, result_type, result_type_str): @@ -951,7 +1047,7 @@ class TestRunner: (t['name'], t['num_tests'], t['total_time'])) cuml_time += t['total_time'] write(" %6.2f cumulative, %6.2f optimal" % - (cuml_time, cuml_time / int(self._options.num_test_shells))) + (cuml_time, cuml_time / int(self._options.child_processes))) write("") self._print_aggregate_test_statistics(write, individual_test_timings) @@ -964,18 +1060,20 @@ class TestRunner: Args: write: A callback to write info to (e.g., a LoggingWriter) or sys.stdout.write. - individual_test_timings: List of test_shell_thread.TestStats for all - tests. + individual_test_timings: List of dump_render_tree_thread.TestStats + for all tests. """ - test_types = individual_test_timings[0].time_for_diffs.keys() - times_for_test_shell = [] + test_types = [] # Unit tests don't actually produce any timings. + if individual_test_timings: + test_types = individual_test_timings[0].time_for_diffs.keys() + times_for_dump_render_tree = [] times_for_diff_processing = [] times_per_test_type = {} for test_type in test_types: times_per_test_type[test_type] = [] for test_stats in individual_test_timings: - times_for_test_shell.append(test_stats.test_run_time) + times_for_dump_render_tree.append(test_stats.test_run_time) times_for_diff_processing.append( test_stats.total_time_for_all_diffs) time_for_diffs = test_stats.time_for_diffs @@ -984,7 +1082,8 @@ class TestRunner: time_for_diffs[test_type]) self._print_statistics_for_test_timings(write, - "PER TEST TIME IN TESTSHELL (seconds):", times_for_test_shell) + "PER TEST TIME IN TESTSHELL (seconds):", + times_for_dump_render_tree) self._print_statistics_for_test_timings(write, "PER TEST DIFF PROCESSING TIMES (seconds):", times_for_diff_processing) @@ -999,11 +1098,11 @@ class TestRunner: Args: write: A callback to write info to (e.g., a LoggingWriter) or sys.stdout.write. - individual_test_timings: List of test_shell_thread.TestStats for all - tests. + individual_test_timings: List of dump_render_tree_thread.TestStats + for all tests. result_summary: summary object for test run """ - # Reverse-sort by the time spent in test_shell. + # Reverse-sort by the time spent in DumpRenderTree. individual_test_timings.sort(lambda a, b: cmp(b.test_run_time, a.test_run_time)) @@ -1098,6 +1197,8 @@ class TestRunner: timings.sort() num_tests = len(timings) + if not num_tests: + return percentile90 = timings[int(.9 * num_tests)] percentile99 = timings[int(.99 * num_tests)] @@ -1269,12 +1370,6 @@ class TestRunner: if len(unexpected_results['tests']) and self._options.verbose: print "-" * 78 - def _print_unexpected_test_result(self, test, result): - """Prints one unexpected test result line.""" - desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result][0] - self._meter.write(" %s -> unexpected %s\n" % - (self._port.relative_test_filename(test), desc)) - def _write_results_html_file(self, result_summary): """Write results.html which is a summary of tests that failed. @@ -1324,7 +1419,7 @@ class TestRunner: return True def _show_results_html_file(self): - """Launches the test shell open to the results.html page.""" + """Shows the results.html page.""" results_filename = os.path.join(self._options.results_directory, "results.html") self._port.show_results_html_file(results_filename) @@ -1345,7 +1440,7 @@ def read_test_files(files): def create_logging_writer(options, log_option): - """Returns a write() function that will write the string to logging.info() + """Returns a write() function that will write the string to _log.info() if comp was specified in --log or if --verbose is true. Otherwise the message is dropped. @@ -1355,16 +1450,21 @@ def create_logging_writer(options, log_option): to be logged (e.g., 'actual' or 'expected') """ if options.verbose or log_option in options.log.split(","): - return logging.info + return _log.info return lambda str: 1 -def main(options, args): - """Run the tests. Will call sys.exit when complete. +def main(options, args, print_results=True): + """Run the tests. Args: options: a dictionary of command line options args: a list of sub directories or files to test + print_results: whether or not to log anything to stdout. + Set to false by the unit tests + Returns: + the number of unexpected results that occurred, or -1 if there is an + error. """ if options.sources: @@ -1382,13 +1482,14 @@ def main(options, args): logging.basicConfig(level=log_level, format=log_fmt, datefmt=log_datefmt, stream=meter) - if not options.target: - if options.debug: - options.target = "Debug" - else: - options.target = "Release" - port_obj = port.get(options.platform, options) + executive = Executive() + + if not options.configuration: + options.configuration = port_obj.default_configuration() + + if options.pixel_tests is None: + options.pixel_tests = True if not options.use_apache: options.use_apache = sys.platform in ('darwin', 'linux2') @@ -1402,23 +1503,44 @@ def main(options, args): # Debug or Release. options.results_directory = port_obj.results_directory() + last_unexpected_results = [] + if options.print_unexpected_results or options.retry_unexpected_results: + unexpected_results_filename = os.path.join( + options.results_directory, "unexpected_results.json") + f = file(unexpected_results_filename) + results = simplejson.load(f) + f.close() + last_unexpected_results = results['tests'].keys() + if options.print_unexpected_results: + print "\n".join(last_unexpected_results) + "\n" + return 0 + if options.clobber_old_results: # Just clobber the actual test results directories since the other # files in the results directory are explicitly used for cross-run # tracking. - path = os.path.join(options.results_directory, 'LayoutTests') - if os.path.exists(path): - shutil.rmtree(path) - - if not options.num_test_shells: - # TODO(ojan): Investigate perf/flakiness impact of using numcores + 1. - options.num_test_shells = port_obj.num_cores() + meter.update("Clobbering old results in %s" % + options.results_directory) + layout_tests_dir = port_obj.layout_tests_dir() + possible_dirs = os.listdir(layout_tests_dir) + for dirname in possible_dirs: + if os.path.isdir(os.path.join(layout_tests_dir, dirname)): + shutil.rmtree(os.path.join(options.results_directory, dirname), + ignore_errors=True) + + if not options.child_processes: + # FIXME: Investigate perf/flakiness impact of using cpu_count + 1. + options.child_processes = port_obj.default_child_processes() write = create_logging_writer(options, 'config') - write("Running %s test_shells in parallel" % options.num_test_shells) + if options.child_processes == 1: + write("Running one %s" % port_obj.driver_name) + else: + write("Running %s %ss in parallel" % ( + options.child_processes, port_obj.driver_name())) if not options.time_out_ms: - if options.target == "Debug": + if options.configuration == "Debug": options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS) else: options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS) @@ -1436,44 +1558,57 @@ def main(options, args): paths = new_args if not paths: paths = [] + paths += last_unexpected_results if options.test_list: paths += read_test_files(options.test_list) # Create the output directory if it doesn't already exist. port_obj.maybe_make_directory(options.results_directory) - meter.update("Gathering files ...") + meter.update("Collecting tests ...") test_runner = TestRunner(port_obj, options, meter) test_runner.gather_file_paths(paths) if options.lint_test_files: - # Creating the expecations for each platform/target pair does all the - # test list parsing and ensures it's correct syntax (e.g. no dupes). - for platform in port_obj.test_platform_names(): - test_runner.parse_expectations(platform, is_debug_mode=True) - test_runner.parse_expectations(platform, is_debug_mode=False) + # Creating the expecations for each platform/configuration pair does + # all the test list parsing and ensures it's correct syntax (e.g. no + # dupes). + for platform_name in port_obj.test_platform_names(): + test_runner.parse_expectations(platform_name, is_debug_mode=True) + test_runner.parse_expectations(platform_name, is_debug_mode=False) + meter.update("") print ("If there are no fail messages, errors or exceptions, then the " "lint succeeded.") - sys.exit(0) - - # Check that the system dependencies (themes, fonts, ...) are correct. - if not options.nocheck_sys_deps: - if not port_obj.check_sys_deps(): - sys.exit(1) + return 0 write = create_logging_writer(options, "config") write("Using port '%s'" % port_obj.name()) write("Placing test results in %s" % options.results_directory) if options.new_baseline: write("Placing new baselines in %s" % port_obj.baseline_path()) - write("Using %s build" % options.target) - if options.no_pixel_tests: - write("Not running pixel tests") + write("Using %s build" % options.configuration) + if options.pixel_tests: + write("Pixel tests enabled") + else: + write("Pixel tests disabled") write("") meter.update("Parsing expectations ...") test_runner.parse_expectations(port_obj.test_platform_name(), - options.target == 'Debug') + options.configuration == 'Debug') + + meter.update("Checking build ...") + if not port_obj.check_build(test_runner.needs_http()): + return -1 + + meter.update("Starting helper ...") + port_obj.start_helper() + + # Check that the system dependencies (themes, fonts, ...) are correct. + if not options.nocheck_sys_deps: + meter.update("Checking system dependencies ...") + if not port_obj.check_sys_deps(test_runner.needs_http()): + return -1 meter.update("Preparing tests ...") write = create_logging_writer(options, "expected") @@ -1482,143 +1617,237 @@ def main(options, args): port_obj.setup_test_run() test_runner.add_test_type(text_diff.TestTextDiff) - if not options.no_pixel_tests: + if options.pixel_tests: test_runner.add_test_type(image_diff.ImageDiff) if options.fuzzy_pixel_tests: test_runner.add_test_type(fuzzy_image_diff.FuzzyImageDiff) - meter.update("Starting ...") - has_new_failures = test_runner.run(result_summary) + num_unexpected_results = test_runner.run(result_summary, print_results) - logging.debug("Exit status: %d" % has_new_failures) - sys.exit(has_new_failures) + port_obj.stop_helper() + + _log.debug("Exit status: %d" % num_unexpected_results) + return num_unexpected_results + + +def _compat_shim_callback(option, opt_str, value, parser): + print "Ignoring unsupported option: %s" % opt_str + + +def _compat_shim_option(option_name, nargs=0): + return optparse.make_option(option_name, action="callback", callback=_compat_shim_callback, nargs=nargs, help="Ignored, for old-run-webkit-tests compat only.") def parse_args(args=None): """Provides a default set of command line args. Returns a tuple of options, args from optparse""" - option_parser = optparse.OptionParser() - option_parser.add_option("", "--no-pixel-tests", action="store_true", - default=False, - help="disable pixel-to-pixel PNG comparisons") - option_parser.add_option("", "--fuzzy-pixel-tests", action="store_true", - default=False, - help="Also use fuzzy matching to compare pixel " - "test outputs.") - option_parser.add_option("", "--results-directory", - default="layout-test-results", - help="Output results directory source dir," - " relative to Debug or Release") - option_parser.add_option("", "--new-baseline", action="store_true", - default=False, - help="save all generated results as new baselines" - " into the platform directory, overwriting " - "whatever's already there.") - option_parser.add_option("", "--noshow-results", action="store_true", - default=False, help="don't launch the test_shell" - " with results after the tests are done") - option_parser.add_option("", "--full-results-html", action="store_true", - default=False, help="show all failures in " - "results.html, rather than only regressions") - option_parser.add_option("", "--clobber-old-results", action="store_true", - default=False, help="Clobbers test results from " - "previous runs.") - option_parser.add_option("", "--lint-test-files", action="store_true", - default=False, help="Makes sure the test files " - "parse for all configurations. Does not run any " - "tests.") - option_parser.add_option("", "--force", action="store_true", - default=False, - help="Run all tests, even those marked SKIP " - "in the test list") - option_parser.add_option("", "--num-test-shells", - help="Number of testshells to run in parallel.") - option_parser.add_option("", "--use-apache", action="store_true", - default=False, - help="Whether to use apache instead of lighttpd.") - option_parser.add_option("", "--time-out-ms", default=None, - help="Set the timeout for each test") - option_parser.add_option("", "--run-singly", action="store_true", - default=False, - help="run a separate test_shell for each test") - option_parser.add_option("", "--debug", action="store_true", default=False, - help="use the debug binary instead of the release" - " binary") - option_parser.add_option("", "--num-slow-tests-to-log", default=50, - help="Number of slow tests whose timings " - "to print.") - option_parser.add_option("", "--platform", - help="Override the platform for expected results") - option_parser.add_option("", "--target", default="", - help="Set the build target configuration " - "(overrides --debug)") - option_parser.add_option("", "--log", action="store", - default="detailed-progress,unexpected", - help="log various types of data. The param should" - " be a comma-separated list of values from: " - "actual,config," + LOG_DETAILED_PROGRESS + - ",expected,timing," + LOG_UNEXPECTED + " " - "(defaults to " + - "--log detailed-progress,unexpected)") - option_parser.add_option("-v", "--verbose", action="store_true", - default=False, help="include debug-level logging") - option_parser.add_option("", "--sources", action="store_true", - help="show expected result file path for each " - "test (implies --verbose)") - option_parser.add_option("", "--startup-dialog", action="store_true", - default=False, - help="create a dialog on test_shell.exe startup") - option_parser.add_option("", "--gp-fault-error-box", action="store_true", - default=False, - help="enable Windows GP fault error box") - option_parser.add_option("", "--wrapper", - help="wrapper command to insert before " - "invocations of test_shell; option is split " - "on whitespace before running. (Example: " - "--wrapper='valgrind --smc-check=all')") - option_parser.add_option("", "--test-list", action="append", - help="read list of tests to run from file", - metavar="FILE") - option_parser.add_option("", "--nocheck-sys-deps", action="store_true", - default=False, - help="Don't check the system dependencies " - "(themes)") - option_parser.add_option("", "--randomize-order", action="store_true", - default=False, - help=("Run tests in random order (useful for " - "tracking down corruption)")) - option_parser.add_option("", "--run-chunk", - default=None, - help=("Run a specified chunk (n:l), the " - "nth of len l, of the layout tests")) - option_parser.add_option("", "--run-part", - default=None, - help=("Run a specified part (n:m), the nth of m" - " parts, of the layout tests")) - option_parser.add_option("", "--batch-size", - default=None, - help=("Run a the tests in batches (n), after " - "every n tests, the test shell is " - "relaunched.")) - option_parser.add_option("", "--builder-name", - default="DUMMY_BUILDER_NAME", - help=("The name of the builder shown on the " - "waterfall running this script e.g. " - "WebKit.")) - option_parser.add_option("", "--build-name", - default="DUMMY_BUILD_NAME", - help=("The name of the builder used in its path, " - "e.g. webkit-rel.")) - option_parser.add_option("", "--build-number", - default="DUMMY_BUILD_NUMBER", - help=("The build number of the builder running" - "this script.")) - option_parser.add_option("", "--experimental-fully-parallel", - action="store_true", default=False, - help="run all tests in parallel") + + # FIXME: All of these options should be stored closer to the code which + # FIXME: actually uses them. configuration_options should move + # FIXME: to WebKitPort and be shared across all scripts. + configuration_options = [ + optparse.make_option("-t", "--target", dest="configuration", + help="(DEPRECATED)"), + # FIXME: --help should display which configuration is default. + optparse.make_option('--debug', action='store_const', const='Debug', + dest="configuration", + help='Set the configuration to Debug'), + optparse.make_option('--release', action='store_const', + const='Release', dest="configuration", + help='Set the configuration to Release'), + # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION. + ] + + logging_options = [ + optparse.make_option("--log", action="store", + default=LOG_DEFAULT_VALUE, + help=("log various types of data. The argument value should be a " + "comma-separated list of values from: %s (defaults to " + "--log %s)" % (LOG_VALUES, LOG_DEFAULT_VALUE))), + optparse.make_option("-v", "--verbose", action="store_true", + default=False, help="include debug-level logging"), + optparse.make_option("--sources", action="store_true", + help="show expected result file path for each test " + + "(implies --verbose)"), + # old-run-webkit-tests has a --slowest option which just prints + # the slowest 10. + optparse.make_option("--num-slow-tests-to-log", default=50, + help="Number of slow tests whose timings to print."), + ] + + # FIXME: These options should move onto the ChromiumPort. + chromium_options = [ + optparse.make_option("--chromium", action="store_true", default=False, + help="use the Chromium port"), + optparse.make_option("--startup-dialog", action="store_true", + default=False, help="create a dialog on DumpRenderTree startup"), + optparse.make_option("--gp-fault-error-box", action="store_true", + default=False, help="enable Windows GP fault error box"), + optparse.make_option("--nocheck-sys-deps", action="store_true", + default=False, + help="Don't check the system dependencies (themes)"), + optparse.make_option("--use-drt", action="store_true", + default=False, + help="Use DumpRenderTree instead of test_shell"), + ] + + # Missing Mac-specific old-run-webkit-tests options: + # FIXME: Need: -g, --guard for guard malloc support on Mac. + # FIXME: Need: -l --leaks Enable leaks checking. + # FIXME: Need: --sample-on-timeout Run sample on timeout + + old_run_webkit_tests_compat = [ + # NRWT doesn't generate results by default anyway. + _compat_shim_option("--no-new-test-results"), + # NRWT doesn't sample on timeout yet anyway. + _compat_shim_option("--no-sample-on-timeout"), + # FIXME: NRWT needs to support remote links eventually. + _compat_shim_option("--use-remote-links-to-tests"), + # FIXME: NRWT doesn't need this option as much since failures are + # designed to be cheap. We eventually plan to add this support. + _compat_shim_option("--exit-after-n-failures", nargs=1), + ] + + results_options = [ + # NEED for bots: --use-remote-links-to-tests Link to test files + # within the SVN repository in the results. + optparse.make_option("-p", "--pixel-tests", action="store_true", + dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), + optparse.make_option("--no-pixel-tests", action="store_false", + dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), + optparse.make_option("--fuzzy-pixel-tests", action="store_true", + default=False, + help="Also use fuzzy matching to compare pixel test outputs."), + # old-run-webkit-tests allows a specific tolerance: --tolerance t + # Ignore image differences less than this percentage (default: 0.1) + optparse.make_option("--results-directory", + default="layout-test-results", + help="Output results directory source dir, relative to Debug or " + "Release"), + optparse.make_option("--new-baseline", action="store_true", + default=False, help="Save all generated results as new baselines " + "into the platform directory, overwriting whatever's " + "already there."), + optparse.make_option("--no-show-results", action="store_false", + default=True, dest="show_results", + help="Don't launch a browser with results after the tests " + "are done"), + # FIXME: We should have a helper function to do this sort of + # deprectated mapping and automatically log, etc. + optparse.make_option("--noshow-results", action="store_false", + dest="show_results", + help="Deprecated, same as --no-show-results."), + optparse.make_option("--no-launch-safari", action="store_false", + dest="show_results", + help="old-run-webkit-tests compat, same as --noshow-results."), + # old-run-webkit-tests: + # --[no-]launch-safari Launch (or do not launch) Safari to display + # test results (default: launch) + optparse.make_option("--full-results-html", action="store_true", + default=False, + help="Show all failures in results.html, rather than only " + "regressions"), + optparse.make_option("--clobber-old-results", action="store_true", + default=False, help="Clobbers test results from previous runs."), + optparse.make_option("--platform", + help="Override the platform for expected results"), + # old-run-webkit-tests also has HTTP toggle options: + # --[no-]http Run (or do not run) http tests + # (default: run) + # --[no-]wait-for-httpd Wait for httpd if some other test + # session is using it already (same + # as WEBKIT_WAIT_FOR_HTTPD=1). + # (default: 0) + ] + + test_options = [ + optparse.make_option("--build", dest="build", + action="store_true", default=True, + help="Check to ensure the DumpRenderTree build is up-to-date " + "(default)."), + optparse.make_option("--no-build", dest="build", + action="store_false", help="Don't check to see if the " + "DumpRenderTree build is up-to-date."), + # old-run-webkit-tests has --valgrind instead of wrapper. + optparse.make_option("--wrapper", + help="wrapper command to insert before invocations of " + "DumpRenderTree; option is split on whitespace before " + "running. (Example: --wrapper='valgrind --smc-check=all')"), + # old-run-webkit-tests: + # -i|--ignore-tests Comma-separated list of directories + # or tests to ignore + optparse.make_option("--test-list", action="append", + help="read list of tests to run from file", metavar="FILE"), + # old-run-webkit-tests uses --skipped==[default|ignore|only] + # instead of --force: + optparse.make_option("--force", action="store_true", default=False, + help="Run all tests, even those marked SKIP in the test list"), + optparse.make_option("--use-apache", action="store_true", + default=False, help="Whether to use apache instead of lighttpd."), + optparse.make_option("--time-out-ms", + help="Set the timeout for each test"), + # old-run-webkit-tests calls --randomize-order --random: + optparse.make_option("--randomize-order", action="store_true", + default=False, help=("Run tests in random order (useful " + "for tracking down corruption)")), + optparse.make_option("--run-chunk", + help=("Run a specified chunk (n:l), the nth of len l, " + "of the layout tests")), + optparse.make_option("--run-part", help=("Run a specified part (n:m), " + "the nth of m parts, of the layout tests")), + # old-run-webkit-tests calls --batch-size: --nthly n + # Restart DumpRenderTree every n tests (default: 1000) + optparse.make_option("--batch-size", + help=("Run a the tests in batches (n), after every n tests, " + "DumpRenderTree is relaunched.")), + # old-run-webkit-tests calls --run-singly: -1|--singly + # Isolate each test case run (implies --nthly 1 --verbose) + optparse.make_option("--run-singly", action="store_true", + default=False, help="run a separate DumpRenderTree for each test"), + optparse.make_option("--child-processes", + help="Number of DumpRenderTrees to run in parallel."), + # FIXME: Display default number of child processes that will run. + optparse.make_option("--experimental-fully-parallel", + action="store_true", default=False, + help="run all tests in parallel"), + # FIXME: Need --exit-after-n-failures N + # Exit after the first N failures instead of running all tests + # FIXME: consider: --iterations n + # Number of times to run the set of tests (e.g. ABCABCABC) + optparse.make_option("--print-unexpected-results", action="store_true", + default=False, help="print the tests in the last run that " + "had unexpected results."), + optparse.make_option("--retry-unexpected-results", action="store_true", + default=False, help="re-try the tests in the last run that " + "had unexpected results."), + ] + + misc_options = [ + optparse.make_option("--lint-test-files", action="store_true", + default=False, help=("Makes sure the test files parse for all " + "configurations. Does not run any tests.")), + ] + + # FIXME: Move these into json_results_generator.py + results_json_options = [ + optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME", + help=("The name of the builder shown on the waterfall running " + "this script e.g. WebKit.")), + optparse.make_option("--build-name", default="DUMMY_BUILD_NAME", + help=("The name of the builder used in its path, e.g. " + "webkit-rel.")), + optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER", + help=("The build number of the builder running this script.")), + ] + + option_list = (configuration_options + logging_options + + chromium_options + results_options + test_options + + misc_options + results_json_options + + old_run_webkit_tests_compat) + option_parser = optparse.OptionParser(option_list=option_list) return option_parser.parse_args(args) if '__main__' == __name__: options, args = parse_args() - main(options, args) + sys.exit(main(options, args)) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py new file mode 100644 index 0000000..9fe0e74 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py @@ -0,0 +1,74 @@ +#!/usr/bin/python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for run_webkit_tests.""" + +import os +import sys +import unittest + +import webkitpy.layout_tests.run_webkit_tests as run_webkit_tests + + +def passing_run(args): + options, args = run_webkit_tests.parse_args(args) + res = run_webkit_tests.main(options, args, False) + return res == 0 + + +class MainTest(unittest.TestCase): + def test_fast(self): + self.assertTrue(passing_run(['--platform', 'test', + 'fast/html'])) + self.assertTrue(passing_run(['--platform', 'test', + '--run-singly', + 'fast/html'])) + self.assertTrue(passing_run(['--platform', 'test', + 'fast/html/article-element.html'])) + self.assertTrue(passing_run(['--platform', 'test', + '--child-processes', '1', + '--log', 'unexpected', + 'fast/html'])) + + +class DryrunTest(unittest.TestCase): + def test_basics(self): + self.assertTrue(passing_run(['--platform', 'dryrun', + 'fast/html'])) + #self.assertTrue(passing_run(['--platform', 'dryrun-mac', + # 'fast/html'])) + #self.assertTrue(passing_run(['--platform', 'dryrun-chromium-mac', + # 'fast/html'])) + #self.assertTrue(passing_run(['--platform', 'dryrun-chromium-win', + # 'fast/html'])) + #self.assertTrue(passing_run(['--platform', 'dryrun-chromium-linux', + # 'fast/html'])) + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/fuzzy_image_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/fuzzy_image_diff.py index 89dd192..64dfb20 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/fuzzy_image_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/fuzzy_image_diff.py @@ -36,13 +36,15 @@ import logging import os import shutil -from layout_package import test_failures -from test_types import test_type_base +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.test_types import test_type_base + +_log = logging.getLogger("webkitpy.layout_tests.test_types.fuzzy_image_diff") class FuzzyImageDiff(test_type_base.TestTypeBase): - def compare_output(self, filename, output, test_args, target): + def compare_output(self, filename, output, test_args, configuration): """Implementation of CompareOutput that checks the output image and checksum against the expected files from the LayoutTest directory. """ @@ -55,14 +57,14 @@ class FuzzyImageDiff(test_type_base.TestTypeBase): expected_png_file = self._port.expected_filename(filename, '.png') if test_args.show_sources: - logging.debug('Using %s' % expected_png_file) + _log.debug('Using %s' % expected_png_file) # Also report a missing expected PNG file. if not os.path.isfile(expected_png_file): failures.append(test_failures.FailureMissingImage(self)) # Run the fuzzymatcher - r = port.fuzzy_diff(test_args.png_path, expected_png_file) + r = self._port.fuzzy_diff(test_args.png_path, expected_png_file) if r != 0: failures.append(test_failures.FailureFuzzyFailure(self)) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py index 1df7ca3..b414358 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py @@ -39,13 +39,15 @@ import logging import os import shutil -from layout_package import test_failures -from test_types import test_type_base +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.test_types import test_type_base # Cache whether we have the image_diff executable available. _compare_available = True _compare_msg_printed = False +_log = logging.getLogger("webkitpy.layout_tests.test_types.image_diff") + class ImageDiff(test_type_base.TestTypeBase): @@ -82,12 +84,13 @@ class ImageDiff(test_type_base.TestTypeBase): self._save_baseline_data(filename, png_data, ".png") self._save_baseline_data(filename, checksum, ".checksum") - def _create_image_diff(self, port, filename, target): + def _create_image_diff(self, port, filename, configuration): """Creates the visual diff of the expected/actual PNGs. Args: filename: the name of the test - target: Debug or Release + configuration: Debug or Release + Returns True if the files are different, False if they match """ diff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_COMPARE) @@ -96,9 +99,10 @@ class ImageDiff(test_type_base.TestTypeBase): expected_filename = self.output_filename(filename, self.FILENAME_SUFFIX_EXPECTED + '.png') + result = True try: _compare_available = True - result = port.diff_image(actual_filename, expected_filename, + result = port.diff_image(expected_filename, actual_filename, diff_filename) except ValueError: _compare_available = False @@ -106,12 +110,12 @@ class ImageDiff(test_type_base.TestTypeBase): global _compare_msg_printed if not _compare_available and not _compare_msg_printed: _compare_msg_printed = True - print('image_diff not found. Make sure you have a ' + target + - ' build of the image_diff executable.') + print('image_diff not found. Make sure you have a ' + + configuration + ' build of the image_diff executable.') return result - def compare_output(self, port, filename, output, test_args, target): + def compare_output(self, port, filename, output, test_args, configuration): """Implementation of CompareOutput that checks the output image and checksum against the expected files from the LayoutTest directory. """ @@ -133,8 +137,8 @@ class ImageDiff(test_type_base.TestTypeBase): expected_png_file = self._port.expected_filename(filename, '.png') if test_args.show_sources: - logging.debug('Using %s' % expected_hash_file) - logging.debug('Using %s' % expected_png_file) + _log.debug('Using %s' % expected_hash_file) + _log.debug('Using %s' % expected_png_file) try: expected_hash = open(expected_hash_file, "r").read() @@ -146,9 +150,9 @@ class ImageDiff(test_type_base.TestTypeBase): if not os.path.isfile(expected_png_file): # Report a missing expected PNG file. - self.write_output_files(port, filename, '', '.checksum', + self.write_output_files(port, filename, '.checksum', test_args.hash, expected_hash, - diff=False, wdiff=False) + print_text_diffs=False) self._copy_output_png(filename, test_args.png_path, '-actual.png') failures.append(test_failures.FailureMissingImage(self)) return failures @@ -156,25 +160,22 @@ class ImageDiff(test_type_base.TestTypeBase): # Hash matched (no diff needed, okay to return). return failures - - self.write_output_files(port, filename, '', '.checksum', + self.write_output_files(port, filename, '.checksum', test_args.hash, expected_hash, - diff=False, wdiff=False) + print_text_diffs=False) self._copy_output_png(filename, test_args.png_path, '-actual.png') self._copy_output_png(filename, expected_png_file, '-expected.png') - # Even though we only use result in one codepath below but we + # Even though we only use the result in one codepath below but we # still need to call CreateImageDiff for other codepaths. - result = self._create_image_diff(port, filename, target) + images_are_different = self._create_image_diff(port, filename, configuration) if expected_hash == '': failures.append(test_failures.FailureMissingImageHash(self)) elif test_args.hash != expected_hash: - # Hashes don't match, so see if the images match. If they do, then - # the hash is wrong. - if result == 0: - failures.append(test_failures.FailureImageHashIncorrect(self)) - else: + if images_are_different: failures.append(test_failures.FailureImageHashMismatch(self)) + else: + failures.append(test_failures.FailureImageHashIncorrect(self)) return failures @@ -188,10 +189,7 @@ class ImageDiff(test_type_base.TestTypeBase): True if two files are different. False otherwise. """ - try: - result = port.diff_image(file1, file2) + return port.diff_image(file1, file2) except ValueError, e: return True - - return result == 1 diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py index efa2e8c..4c99be0 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py @@ -37,6 +37,8 @@ import errno import logging import os.path +_log = logging.getLogger("webkitpy.layout_tests.test_types.test_type_base") + class TestArguments(object): """Struct-like wrapper for additional arguments needed by @@ -68,19 +70,18 @@ class TestTypeBase(object): FILENAME_SUFFIX_EXPECTED = "-expected" FILENAME_SUFFIX_DIFF = "-diff" FILENAME_SUFFIX_WDIFF = "-wdiff.html" + FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html" FILENAME_SUFFIX_COMPARE = "-diff.png" - def __init__(self, port, platform, root_output_dir): + def __init__(self, port, root_output_dir): """Initialize a TestTypeBase object. Args: - platform: the platform (e.g., 'chromium-mac-leopard') - identifying the platform-specific results to be used. + port: object implementing port-specific information and methods root_output_dir: The unix style path to the output dir. """ self._root_output_dir = root_output_dir self._port = port - self._platform = platform def _make_output_directory(self, filename): """Creates the output directory (if needed) for a given test @@ -90,7 +91,7 @@ class TestTypeBase(object): self._port.maybe_make_directory(os.path.split(output_filename)[0]) def _save_baseline_data(self, filename, data, modifier): - """Saves a new baseline file into the platform directory. + """Saves a new baseline file into the port's baseline directory. The file will be named simply "<test>-expected<modifier>", suitable for use as the expected results in a later run. @@ -102,15 +103,16 @@ class TestTypeBase(object): """ relative_dir = os.path.dirname( self._port.relative_test_filename(filename)) - output_dir = os.path.join( - self._port.chromium_baseline_path(self._platform), relative_dir) + + baseline_path = self._port.baseline_path() + output_dir = os.path.join(baseline_path, relative_dir) output_file = os.path.basename(os.path.splitext(filename)[0] + self.FILENAME_SUFFIX_EXPECTED + modifier) self._port.maybe_make_directory(output_dir) output_path = os.path.join(output_dir, output_file) - logging.debug('writing new baseline to "%s"' % (output_path)) - open(output_path, "wb").write(data) + _log.debug('writing new baseline to "%s"' % (output_path)) + self._write_into_file_at_path(output_path, data) def output_filename(self, filename, modifier): """Returns a filename inside the output dir that contains modifier. @@ -130,7 +132,7 @@ class TestTypeBase(object): self._port.relative_test_filename(filename)) return os.path.splitext(output_filename)[0] + modifier - def compare_output(self, port, filename, output, test_args, target): + def compare_output(self, port, filename, output, test_args, configuration): """Method that compares the output from the test with the expected value. @@ -141,56 +143,59 @@ class TestTypeBase(object): output: a string containing the output of the test test_args: a TestArguments object holding optional additional arguments - target: Debug or Release + configuration: Debug or Release Return: a list of TestFailure objects, empty if the test passes """ raise NotImplemented - def write_output_files(self, port, filename, test_type, file_type, - output, expected, diff=True, wdiff=False): + def _write_into_file_at_path(self, file_path, contents): + file = open(file_path, "wb") + file.write(contents) + file.close() + + def write_output_files(self, port, filename, file_type, + output, expected, print_text_diffs=False): """Writes the test output, the expected output and optionally the diff between the two to files in the results directory. The full output filename of the actual, for example, will be - <filename><test_type>-actual<file_type> + <filename>-actual<file_type> For instance, - my_test-simp-actual.txt + my_test-actual.txt Args: filename: The test filename - test_type: A string describing the test type, e.g. "simp" file_type: A string describing the test output file type, e.g. ".txt" output: A string containing the test output expected: A string containing the expected test output - diff: if True, write a file containing the diffs too. This should be - False for results that are not text - wdiff: if True, write an HTML file containing word-by-word diffs + print_text_diffs: True for text diffs. (FIXME: We should be able to get this from the file type?) """ self._make_output_directory(filename) - actual_filename = self.output_filename(filename, - test_type + self.FILENAME_SUFFIX_ACTUAL + file_type) - expected_filename = self.output_filename(filename, - test_type + self.FILENAME_SUFFIX_EXPECTED + file_type) + actual_filename = self.output_filename(filename, self.FILENAME_SUFFIX_ACTUAL + file_type) + expected_filename = self.output_filename(filename, self.FILENAME_SUFFIX_EXPECTED + file_type) if output: - open(actual_filename, "wb").write(output) + self._write_into_file_at_path(actual_filename, output) if expected: - open(expected_filename, "wb").write(expected) + self._write_into_file_at_path(expected_filename, expected) if not output or not expected: return - if diff: - diff = port.diff_text(expected, output, expected_filename, - actual_filename) - diff_filename = self.output_filename(filename, - test_type + self.FILENAME_SUFFIX_DIFF + file_type) - open(diff_filename, "wb").write(diff) - - if wdiff: - # Shell out to wdiff to get colored inline diffs. - wdiff = port.wdiff_text(expected_filename, actual_filename) - filename = self.output_filename(filename, test_type + - self.FILENAME_SUFFIX_WDIFF) - out = open(filename, 'wb').write(wdiff) + if not print_text_diffs: + return + + diff = port.diff_text(expected, output, expected_filename, actual_filename) + diff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_DIFF + file_type) + self._write_into_file_at_path(diff_filename, diff) + + # Shell out to wdiff to get colored inline diffs. + wdiff = port.wdiff_text(expected_filename, actual_filename) + wdiff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_WDIFF) + self._write_into_file_at_path(wdiff_filename, wdiff) + + # Use WebKit's PrettyPatch.rb to get an HTML diff. + pretty_patch = port.pretty_patch_text(diff_filename) + pretty_patch_filename = self.output_filename(filename, self.FILENAME_SUFFIX_PRETTY_PATCH) + self._write_into_file_at_path(pretty_patch_filename, pretty_patch) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py index 54b332b..8f7907c 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py @@ -37,8 +37,10 @@ import errno import logging import os.path -from layout_package import test_failures -from test_types import test_type_base +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.test_types import test_type_base + +_log = logging.getLogger("webkitpy.layout_tests.test_types.text_diff") def is_render_tree_dump(data): @@ -63,7 +65,7 @@ class TestTextDiff(test_type_base.TestTypeBase): # Read the port-specific expected text. expected_filename = self._port.expected_filename(filename, '.txt') if show_sources: - logging.debug('Using %s' % expected_filename) + _log.debug('Using %s' % expected_filename) return self.get_normalized_text(expected_filename) @@ -78,7 +80,7 @@ class TestTextDiff(test_type_base.TestTypeBase): # Normalize line endings return text.strip("\r\n").replace("\r\n", "\n") + "\n" - def compare_output(self, port, filename, output, test_args, target): + def compare_output(self, port, filename, output, test_args, configuration): """Implementation of CompareOutput that checks the output text against the expected text from the LayoutTest directory.""" failures = [] @@ -96,8 +98,8 @@ class TestTextDiff(test_type_base.TestTypeBase): # Write output files for new tests, too. if port.compare_text(output, expected): # Text doesn't match, write output files. - self.write_output_files(port, filename, "", ".txt", output, - expected, diff=True, wdiff=True) + self.write_output_files(port, filename, ".txt", output, + expected, print_text_diffs=True) if expected == '': failures.append(test_failures.FailureMissingResult(self)) diff --git a/WebKitTools/Scripts/webkitpy/python24/__init__.py b/WebKitTools/Scripts/webkitpy/python24/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/python24/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/python24/versioning.py b/WebKitTools/Scripts/webkitpy/python24/versioning.py new file mode 100644 index 0000000..8b1f21b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/python24/versioning.py @@ -0,0 +1,133 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports Python version checking.""" + +import logging +import sys + +_log = logging.getLogger("webkitpy.python24.versioning") + +# The minimum Python version the webkitpy package supports. +_MINIMUM_SUPPORTED_PYTHON_VERSION = "2.5" + + +def compare_version(sysmodule=None, target_version=None): + """Compare the current Python version with a target version. + + Args: + sysmodule: An object with version and version_info data attributes + used to detect the current Python version. The attributes + should have the same semantics as sys.version and + sys.version_info. This parameter should only be used + for unit testing. Defaults to sys. + target_version: A string representing the Python version to compare + the current version against. The string should have + one of the following three forms: 2, 2.5, or 2.5.3. + Defaults to the minimum version that the webkitpy + package supports. + + Returns: + A triple of (comparison, current_version, target_version). + + comparison: An integer representing the result of comparing the + current version with the target version. A positive + number means the current version is greater than the + target, 0 means they are the same, and a negative number + means the current version is less than the target. + This method compares version information only up + to the precision of the given target version. For + example, if the target version is 2.6 and the current + version is 2.5.3, this method uses 2.5 for the purposes + of comparing with the target. + current_version: A string representing the current Python version, for + example 2.5.3. + target_version: A string representing the version that the current + version was compared against, for example 2.5. + + """ + if sysmodule is None: + sysmodule = sys + if target_version is None: + target_version = _MINIMUM_SUPPORTED_PYTHON_VERSION + + # The number of version parts to compare. + precision = len(target_version.split(".")) + + # We use sys.version_info rather than sys.version since its first + # three elements are guaranteed to be integers. + current_version_info_to_compare = sysmodule.version_info[:precision] + # Convert integers to strings. + current_version_info_to_compare = map(str, current_version_info_to_compare) + current_version_to_compare = ".".join(current_version_info_to_compare) + + # Compare version strings lexicographically. + if current_version_to_compare > target_version: + comparison = 1 + elif current_version_to_compare == target_version: + comparison = 0 + else: + comparison = -1 + + # The version number portion of the current version string, for + # example "2.6.4". + current_version = sysmodule.version.split()[0] + + return (comparison, current_version, target_version) + + +# FIXME: Add a logging level parameter to allow the version message +# to be logged at levels other than WARNING, for example CRITICAL. +def check_version(log=None, sysmodule=None, target_version=None): + """Check the current Python version against a target version. + + Logs a warning message if the current version is less than the + target version. + + Args: + log: A logging.logger instance to use when logging the version warning. + Defaults to the logger of this module. + sysmodule: See the compare_version() docstring. + target_version: See the compare_version() docstring. + + Returns: + A boolean value of whether the current version is greater than + or equal to the target version. + + """ + if log is None: + log = _log + + (comparison, current_version, target_version) = \ + compare_version(sysmodule, target_version) + + if comparison >= 0: + # Then the current version is at least the minimum version. + return True + + message = ("WebKit Python scripts do not support your current Python " + "version (%s). The minimum supported version is %s.\n" + " See the following page to upgrade your Python version:\n\n" + " http://trac.webkit.org/wiki/PythonGuidelines\n" + % (current_version, target_version)) + log.warn(message) + return False diff --git a/WebKitTools/Scripts/webkitpy/python24/versioning_unittest.py b/WebKitTools/Scripts/webkitpy/python24/versioning_unittest.py new file mode 100644 index 0000000..6939e2d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/python24/versioning_unittest.py @@ -0,0 +1,134 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains unit tests for versioning.py.""" + +import logging +import unittest + +from webkitpy.common.system.logtesting import LogTesting +from webkitpy.python24.versioning import check_version +from webkitpy.python24.versioning import compare_version + +class MockSys(object): + + """A mock sys module for passing to version-checking methods.""" + + def __init__(self, current_version): + """Create an instance. + + current_version: A version string with major, minor, and micro + version parts. + + """ + version_info = current_version.split(".") + version_info = map(int, version_info) + + self.version = current_version + " Version details." + self.version_info = version_info + + +class CompareVersionTest(unittest.TestCase): + + """Tests compare_version().""" + + def _mock_sys(self, current_version): + return MockSys(current_version) + + def test_default_minimum_version(self): + """Test the configured minimum version that webkitpy supports.""" + (comparison, current_version, min_version) = compare_version() + self.assertEquals(min_version, "2.5") + + def compare_version(self, target_version, current_version=None): + """Call compare_version().""" + if current_version is None: + current_version = "2.5.3" + mock_sys = self._mock_sys(current_version) + return compare_version(mock_sys, target_version) + + def compare(self, target_version, current_version=None): + """Call compare_version(), and return the comparison.""" + return self.compare_version(target_version, current_version)[0] + + def test_returned_current_version(self): + """Test the current_version return value.""" + current_version = self.compare_version("2.5")[1] + self.assertEquals(current_version, "2.5.3") + + def test_returned_target_version(self): + """Test the current_version return value.""" + target_version = self.compare_version("2.5")[2] + self.assertEquals(target_version, "2.5") + + def test_target_version_major(self): + """Test major version for target.""" + self.assertEquals(-1, self.compare("3")) + self.assertEquals(0, self.compare("2")) + self.assertEquals(1, self.compare("2", "3.0.0")) + + def test_target_version_minor(self): + """Test minor version for target.""" + self.assertEquals(-1, self.compare("2.6")) + self.assertEquals(0, self.compare("2.5")) + self.assertEquals(1, self.compare("2.4")) + + def test_target_version_micro(self): + """Test minor version for target.""" + self.assertEquals(-1, self.compare("2.5.4")) + self.assertEquals(0, self.compare("2.5.3")) + self.assertEquals(1, self.compare("2.5.2")) + + +class CheckVersionTest(unittest.TestCase): + + """Tests check_version().""" + + def setUp(self): + self._log = LogTesting.setUp(self) + + def tearDown(self): + self._log.tearDown() + + def _check_version(self, minimum_version): + """Call check_version().""" + mock_sys = MockSys("2.5.3") + return check_version(sysmodule=mock_sys, target_version=minimum_version) + + def test_true_return_value(self): + """Test the configured minimum version that webkitpy supports.""" + is_current = self._check_version("2.4") + self.assertEquals(True, is_current) + self._log.assertMessages([]) # No warning was logged. + + def test_false_return_value(self): + """Test the configured minimum version that webkitpy supports.""" + is_current = self._check_version("2.6") + self.assertEquals(False, is_current) + expected_message = ('WARNING: WebKit Python scripts do not support ' + 'your current Python version (2.5.3). ' + 'The minimum supported version is 2.6.\n ' + 'See the following page to upgrade your Python ' + 'version:\n\n ' + 'http://trac.webkit.org/wiki/PythonGuidelines\n\n') + self._log.assertMessages([expected_message]) + diff --git a/WebKitTools/Scripts/webkitpy/steps/__init__.py b/WebKitTools/Scripts/webkitpy/steps/__init__.py deleted file mode 100644 index 5ae4bea..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/__init__.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# FIXME: Is this the right way to do this? -from webkitpy.steps.applypatch import ApplyPatch -from webkitpy.steps.applypatchwithlocalcommit import ApplyPatchWithLocalCommit -from webkitpy.steps.build import Build -from webkitpy.steps.checkstyle import CheckStyle -from webkitpy.steps.cleanworkingdirectory import CleanWorkingDirectory -from webkitpy.steps.cleanworkingdirectorywithlocalcommits import CleanWorkingDirectoryWithLocalCommits -from webkitpy.steps.closebug import CloseBug -from webkitpy.steps.closebugforlanddiff import CloseBugForLandDiff -from webkitpy.steps.closepatch import ClosePatch -from webkitpy.steps.commit import Commit -from webkitpy.steps.completerollout import CompleteRollout -from webkitpy.steps.confirmdiff import ConfirmDiff -from webkitpy.steps.createbug import CreateBug -from webkitpy.steps.editchangelog import EditChangeLog -from webkitpy.steps.ensurebuildersaregreen import EnsureBuildersAreGreen -from webkitpy.steps.ensurelocalcommitifneeded import EnsureLocalCommitIfNeeded -from webkitpy.steps.obsoletepatches import ObsoletePatches -from webkitpy.steps.options import Options -from webkitpy.steps.postdiff import PostDiff -from webkitpy.steps.postdiffforcommit import PostDiffForCommit -from webkitpy.steps.preparechangelogforrevert import PrepareChangeLogForRevert -from webkitpy.steps.preparechangelog import PrepareChangeLog -from webkitpy.steps.promptforbugortitle import PromptForBugOrTitle -from webkitpy.steps.revertrevision import RevertRevision -from webkitpy.steps.runtests import RunTests -from webkitpy.steps.updatechangelogswithreviewer import UpdateChangeLogsWithReviewer -from webkitpy.steps.update import Update diff --git a/WebKitTools/Scripts/webkitpy/steps/completerollout.py b/WebKitTools/Scripts/webkitpy/steps/completerollout.py deleted file mode 100644 index 8534956..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/completerollout.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from webkitpy.comments import bug_comment_from_commit_text -from webkitpy.steps.build import Build -from webkitpy.steps.commit import Commit -from webkitpy.steps.metastep import MetaStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log - - -class CompleteRollout(MetaStep): - substeps = [ - Build, - Commit, - ] - - @classmethod - def options(cls): - collected_options = cls._collect_options_from_steps(cls.substeps) - collected_options.append(Options.complete_rollout) - return collected_options - - def run(self, state): - bug_id = state["bug_id"] - # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout. - # Once we trust rollout we will remove this option. - if not self._options.complete_rollout: - log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"webkit-patch land %s\" to commit the rollout." % bug_id) - return - - MetaStep.run(self, state) - - commit_comment = bug_comment_from_commit_text(self._tool.scm(), state["commit_text"]) - comment_text = "Reverted r%s for reason:\n\n%s\n\n%s" % (state["revision"], state["reason"], commit_comment) - - if not bug_id: - log(comment_text) - log("No bugs were updated.") - return - self._tool.bugs.reopen_bug(bug_id, comment_text) diff --git a/WebKitTools/Scripts/webkitpy/style/checker.py b/WebKitTools/Scripts/webkitpy/style/checker.py index 9beda9e..84ae3da 100644 --- a/WebKitTools/Scripts/webkitpy/style/checker.py +++ b/WebKitTools/Scripts/webkitpy/style/checker.py @@ -1,5 +1,6 @@ # Copyright (C) 2009 Google Inc. All rights reserved. # Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# Copyright (C) 2010 ProFUSION embedded systems # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -30,23 +31,26 @@ """Front end of some style-checker modules.""" import codecs +import logging import os.path import sys -from .. style_references import parse_patch from error_handlers import DefaultStyleErrorHandler -from error_handlers import PatchStyleErrorHandler from filter import FilterConfiguration from optparser import ArgumentParser from optparser import DefaultCommandOptionValues -from processors.common import check_no_carriage_return from processors.common import categories as CommonCategories +from processors.common import CarriageReturnProcessor from processors.cpp import CppProcessor +from processors.python import PythonProcessor from processors.text import TextProcessor +from webkitpy.style_references import parse_patch +from webkitpy.style_references import configure_logging as _configure_logging +_log = logging.getLogger("webkitpy.style.checker") # These are default option values for the command-line option parser. -_DEFAULT_VERBOSITY = 1 +_DEFAULT_MIN_CONFIDENCE = 1 _DEFAULT_OUTPUT_FORMAT = 'emacs' @@ -85,6 +89,16 @@ _BASE_FILTER_RULES = [ '-whitespace/blank_line', '-whitespace/end_of_line', '-whitespace/labels', + # List Python pep8 categories last. + # + # Because much of WebKit's Python code base does not abide by the + # PEP8 79 character limit, we ignore the 79-character-limit category + # pep8/E501 for now. + # + # FIXME: Consider bringing WebKit's Python code base into conformance + # with the 79 character limit, or some higher limit that is + # agreeable to the WebKit project. + '-pep8/E501', ] @@ -106,9 +120,10 @@ _PATH_RULES_SPECIFIER = [ "WebKit/qt/QGVLauncher/"], ["-build/include", "-readability/streams"]), - ([# The GTK+ APIs use GTK+ naming style, which includes - # lower-cased, underscore-separated values. - "WebKit/gtk/webkit/", + ([# The EFL APIs use EFL naming style, which includes + # both lower-cased and camel-cased, underscore-sparated + # values. + "WebKit/efl/ewk/", # There is no clean way to avoid "yy_*" names used by flex. "WebCore/css/CSSParser.cpp", # There is no clean way to avoid "xxx_data" methods inside @@ -117,6 +132,29 @@ _PATH_RULES_SPECIFIER = [ "WebKit/qt/tests/", "JavaScriptCore/qt/tests"], ["-readability/naming"]), + ([# The GTK+ APIs use GTK+ naming style, which includes + # lower-cased, underscore-separated values. + # Also, GTK+ allows the use of NULL. + "WebKit/gtk/webkit/", + "WebKitTools/DumpRenderTree/gtk/"], + ["-readability/naming", + "-readability/null"]), + ([# Header files in ForwardingHeaders have no header guards or + # exceptional header guards (e.g., WebCore_FWD_Debugger_h). + "/ForwardingHeaders/"], + ["-build/header_guard"]), + + # For third-party Python code, keep only the following checks-- + # + # No tabs: to avoid having to set the SVN allow-tabs property. + # No trailing white space: since this is easy to correct. + # No carriage-return line endings: since this is easy to correct. + # + (["webkitpy/thirdparty/"], + ["-", + "+pep8/W191", # Tabs + "+pep8/W291", # Trailing white space + "+whitespace/carriage_return"]), ] @@ -140,7 +178,8 @@ _SKIPPED_FILES_WITH_WARNING = [ # Don't include a warning for skipped files that are more common # and more obvious. _SKIPPED_FILES_WITHOUT_WARNING = [ - "LayoutTests/" + "LayoutTests/", + ".pyc", ] @@ -154,13 +193,22 @@ _MAX_REPORTS_PER_CATEGORY = { def _all_categories(): """Return the set of all categories used by check-webkit-style.""" # Take the union across all processors. - return CommonCategories.union(CppProcessor.categories) + categories = CommonCategories.union(CppProcessor.categories) + + # FIXME: Consider adding all of the pep8 categories. Since they + # are not too meaningful for documentation purposes, for + # now we add only the categories needed for the unit tests + # (which validate the consistency of the configuration + # settings against the known categories, etc). + categories = categories.union(["pep8/W191", "pep8/W291", "pep8/E501"]) + + return categories def _check_webkit_style_defaults(): """Return the default command-line options for check-webkit-style.""" - return DefaultCommandOptionValues(output_format=_DEFAULT_OUTPUT_FORMAT, - verbosity=_DEFAULT_VERBOSITY) + return DefaultCommandOptionValues(min_confidence=_DEFAULT_MIN_CONFIDENCE, + output_format=_DEFAULT_OUTPUT_FORMAT) # This function assists in optparser not having to import from checker. @@ -186,9 +234,97 @@ def check_webkit_style_configuration(options): return StyleCheckerConfiguration(filter_configuration=filter_configuration, max_reports_per_category=_MAX_REPORTS_PER_CATEGORY, + min_confidence=options.min_confidence, output_format=options.output_format, - stderr_write=sys.stderr.write, - verbosity=options.verbosity) + stderr_write=sys.stderr.write) + + +def _create_log_handlers(stream): + """Create and return a default list of logging.Handler instances. + + Format WARNING messages and above to display the logging level, and + messages strictly below WARNING not to display it. + + Args: + stream: See the configure_logging() docstring. + + """ + # Handles logging.WARNING and above. + error_handler = logging.StreamHandler(stream) + error_handler.setLevel(logging.WARNING) + formatter = logging.Formatter("%(levelname)s: %(message)s") + error_handler.setFormatter(formatter) + + # Create a logging.Filter instance that only accepts messages + # below WARNING (i.e. filters out anything WARNING or above). + non_error_filter = logging.Filter() + # The filter method accepts a logging.LogRecord instance. + non_error_filter.filter = lambda record: record.levelno < logging.WARNING + + non_error_handler = logging.StreamHandler(stream) + non_error_handler.addFilter(non_error_filter) + formatter = logging.Formatter("%(message)s") + non_error_handler.setFormatter(formatter) + + return [error_handler, non_error_handler] + + +def _create_debug_log_handlers(stream): + """Create and return a list of logging.Handler instances for debugging. + + Args: + stream: See the configure_logging() docstring. + + """ + handler = logging.StreamHandler(stream) + formatter = logging.Formatter("%(name)s: %(levelname)-8s %(message)s") + handler.setFormatter(formatter) + + return [handler] + + +def configure_logging(stream, logger=None, is_verbose=False): + """Configure logging, and return the list of handlers added. + + Returns: + A list of references to the logging handlers added to the root + logger. This allows the caller to later remove the handlers + using logger.removeHandler. This is useful primarily during unit + testing where the caller may want to configure logging temporarily + and then undo the configuring. + + Args: + stream: A file-like object to which to log. The stream must + define an "encoding" data attribute, or else logging + raises an error. + logger: A logging.logger instance to configure. This parameter + should be used only in unit tests. Defaults to the + root logger. + is_verbose: A boolean value of whether logging should be verbose. + + """ + # If the stream does not define an "encoding" data attribute, the + # logging module can throw an error like the following: + # + # Traceback (most recent call last): + # File "/System/Library/Frameworks/Python.framework/Versions/2.6/... + # lib/python2.6/logging/__init__.py", line 761, in emit + # self.stream.write(fs % msg.encode(self.stream.encoding)) + # LookupError: unknown encoding: unknown + if logger is None: + logger = logging.getLogger() + + if is_verbose: + logging_level = logging.DEBUG + handlers = _create_debug_log_handlers(stream) + else: + logging_level = logging.INFO + handlers = _create_log_handlers(stream) + + handlers = _configure_logging(logging_level=logging_level, logger=logger, + handlers=handlers) + + return handlers # Enum-like idiom @@ -197,7 +333,8 @@ class FileType: NONE = 1 # Alphabetize remaining types CPP = 2 - TEXT = 3 + PYTHON = 3 + TEXT = 4 class ProcessorDispatcher(object): @@ -218,7 +355,6 @@ class ProcessorDispatcher(object): 'mm', 'php', 'pm', - 'py', 'txt', ) @@ -252,20 +388,26 @@ class ProcessorDispatcher(object): # reading from stdin, cpp_style tests should not rely on # the extension. return FileType.CPP - elif ("ChangeLog" in file_path - or "WebKitTools/Scripts/" in file_path - or file_extension in self.text_file_extensions): + elif file_extension == "py": + return FileType.PYTHON + elif ("ChangeLog" in file_path or + (not file_extension and "WebKitTools/Scripts/" in file_path) or + file_extension in self.text_file_extensions): return FileType.TEXT else: return FileType.NONE - def _create_processor(self, file_type, file_path, handle_style_error, verbosity): + def _create_processor(self, file_type, file_path, handle_style_error, + min_confidence): """Instantiate and return a style processor based on file type.""" if file_type == FileType.NONE: processor = None elif file_type == FileType.CPP: file_extension = self._file_extension(file_path) - processor = CppProcessor(file_path, file_extension, handle_style_error, verbosity) + processor = CppProcessor(file_path, file_extension, + handle_style_error, min_confidence) + elif file_type == FileType.PYTHON: + processor = PythonProcessor(file_path, handle_style_error) elif file_type == FileType.TEXT: processor = TextProcessor(file_path, handle_style_error) else: @@ -278,39 +420,41 @@ class ProcessorDispatcher(object): return processor - def dispatch_processor(self, file_path, handle_style_error, verbosity): + def dispatch_processor(self, file_path, handle_style_error, min_confidence): """Instantiate and return a style processor based on file path.""" file_type = self._file_type(file_path) processor = self._create_processor(file_type, file_path, handle_style_error, - verbosity) + min_confidence) return processor +# FIXME: Remove the stderr_write attribute from this class and replace +# its use with calls to a logging module logger. class StyleCheckerConfiguration(object): """Stores configuration values for the StyleChecker class. Attributes: + min_confidence: An integer between 1 and 5 inclusive that is the + minimum confidence level of style errors to report. + max_reports_per_category: The maximum number of errors to report per category, per file. stderr_write: A function that takes a string as a parameter and serves as stderr.write. - verbosity: An integer between 1-5 inclusive that restricts output - to errors with a confidence score at or above this value. - """ def __init__(self, filter_configuration, max_reports_per_category, + min_confidence, output_format, - stderr_write, - verbosity): + stderr_write): """Create a StyleCheckerConfiguration instance. Args: @@ -321,6 +465,10 @@ class StyleCheckerConfiguration(object): max_reports_per_category: The maximum number of errors to report per category, per file. + min_confidence: An integer between 1 and 5 inclusive that is the + minimum confidence level of style errors to report. + The default is 1, which reports all style errors. + output_format: A string that is the output format. The supported output formats are "emacs" which emacs can parse and "vs7" which Microsoft Visual Studio 7 can parse. @@ -328,42 +476,37 @@ class StyleCheckerConfiguration(object): stderr_write: A function that takes a string as a parameter and serves as stderr.write. - verbosity: An integer between 1-5 inclusive that restricts output - to errors with a confidence score at or above this value. - The default is 1, which reports all errors. - """ self._filter_configuration = filter_configuration self._output_format = output_format self.max_reports_per_category = max_reports_per_category + self.min_confidence = min_confidence self.stderr_write = stderr_write - self.verbosity = verbosity def is_reportable(self, category, confidence_in_error, file_path): """Return whether an error is reportable. An error is reportable if both the confidence in the error is - at least the current verbosity level and the current filter + at least the minimum confidence level and the current filter says the category should be checked for the given path. Args: category: A string that is a style category. - confidence_in_error: An integer between 1 and 5, inclusive, that - represents the application's confidence in - the error. A higher number signifies greater - confidence. + confidence_in_error: An integer between 1 and 5 inclusive that is + the application's confidence in the error. + A higher number means greater confidence. file_path: The path of the file being checked """ - if confidence_in_error < self.verbosity: + if confidence_in_error < self.min_confidence: return False return self._filter_configuration.should_check(category, file_path) def write_style_error(self, category, - confidence, + confidence_in_error, file_path, line_number, message): @@ -377,9 +520,38 @@ class StyleCheckerConfiguration(object): line_number, message, category, - confidence)) + confidence_in_error)) +class ProcessorBase(object): + + """The base class for processors of lists of lines.""" + + def should_process(self, file_path): + """Return whether the file at file_path should be processed.""" + raise NotImplementedError('Subclasses should implement.') + + def process(self, lines, file_path, **kwargs): + """Process lines of text read from a file. + + Args: + lines: A list of lines of text to process. + file_path: The path from which the lines were read. + **kwargs: This argument signifies that the process() method of + subclasses of ProcessorBase may support additional + keyword arguments. + For example, a style processor's process() method + may support a "reportable_lines" parameter that represents + the line numbers of the lines for which style errors + should be reported. + + """ + raise NotImplementedError('Subclasses should implement.') + + +# FIXME: Modify this class to start using the TextFileReader class in +# webkitpy/style/filereader.py. This probably means creating +# a StyleProcessor class that inherits from ProcessorBase. class StyleChecker(object): """Supports checking style in files and patches. @@ -406,79 +578,142 @@ class StyleChecker(object): self.error_count = 0 self.file_count = 0 - def _stderr_write(self, message): - self._configuration.stderr_write(message) - def _increment_error_count(self): """Increment the total count of reported errors.""" self.error_count += 1 - def _process_file(self, processor, file_path, handle_style_error): - """Process the file using the given processor.""" - try: - # Support the UNIX convention of using "-" for stdin. Note that - # we are not opening the file with universal newline support - # (which codecs doesn't support anyway), so the resulting lines do - # contain trailing '\r' characters if we are reading a file that - # has CRLF endings. - # If after the split a trailing '\r' is present, it is removed - # below. If it is not expected to be present (i.e. os.linesep != - # '\r\n' as in Windows), a warning is issued below if this file - # is processed. - if file_path == '-': - file = codecs.StreamReaderWriter(sys.stdin, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace') - else: - file = codecs.open(file_path, 'r', 'utf8', 'replace') + def _read_lines(self, file_path): + """Read the file at a path, and return its lines. - contents = file.read() + Raises: + IOError: if the file does not exist or cannot be read. - except IOError: - self._stderr_write("Skipping input '%s': Can't open for reading\n" % file_path) - return + """ + # Support the UNIX convention of using "-" for stdin. + if file_path == '-': + file = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace') + else: + # We do not open the file with universal newline support + # (codecs does not support it anyway), so the resulting + # lines contain trailing "\r" characters if we are reading + # a file with CRLF endings. + file = codecs.open(file_path, 'r', 'utf8', 'replace') + + contents = file.read() lines = contents.split("\n") + return lines - for line_number in range(len(lines)): - # FIXME: We should probably use the SVN "eol-style" property - # or a white list to decide whether or not to do - # the carriage-return check. Originally, we did the - # check only if (os.linesep != '\r\n'). - # - # FIXME: As a minor optimization, we can have - # check_no_carriage_return() return whether - # the line ends with "\r". - check_no_carriage_return(lines[line_number], line_number, - handle_style_error) - if lines[line_number].endswith("\r"): - lines[line_number] = lines[line_number].rstrip("\r") + def _process_file(self, processor, file_path, handle_style_error): + """Process the file using the given style processor.""" + try: + lines = self._read_lines(file_path) + except IOError: + message = 'Could not read file. Skipping: "%s"' % file_path + _log.warn(message) + return + + # Check for and remove trailing carriage returns ("\r"). + # + # FIXME: We should probably use the SVN "eol-style" property + # or a white list to decide whether or not to do + # the carriage-return check. Originally, we did the + # check only if (os.linesep != '\r\n'). + carriage_return_processor = CarriageReturnProcessor(handle_style_error) + lines = carriage_return_processor.process(lines) processor.process(lines) - def check_file(self, file_path, handle_style_error=None, process_file=None): + def check_paths(self, paths, mock_check_file=None, mock_os=None): + """Check style in the given files or directories. + + Args: + paths: A list of file paths and directory paths. + mock_check_file: A mock of self.check_file for unit testing. + mock_os: A mock os for unit testing. + + """ + check_file = self.check_file if mock_check_file is None else \ + mock_check_file + os_module = os if mock_os is None else mock_os + + for path in paths: + if os_module.path.isdir(path): + self._check_directory(directory=path, + check_file=check_file, + mock_os_walk=os_module.walk) + else: + check_file(path) + + def _check_directory(self, directory, check_file, mock_os_walk=None): + """Check style in all files in a directory, recursively. + + Args: + directory: A path to a directory. + check_file: The function to use in place of self.check_file(). + mock_os_walk: A mock os.walk for unit testing. + + """ + os_walk = os.walk if mock_os_walk is None else mock_os_walk + + for dir_path, dir_names, file_names in os_walk(directory): + for file_name in file_names: + file_path = os.path.join(dir_path, file_name) + check_file(file_path) + + def check_file(self, file_path, line_numbers=None, + mock_handle_style_error=None, + mock_os_path_exists=None, + mock_process_file=None): """Check style in the given file. Args: - file_path: A string that is the path of the file to process. - handle_style_error: The function to call when a style error - occurs. This parameter is meant for internal - use within this class. Defaults to a - DefaultStyleErrorHandler instance. - process_file: The function to call to process the file. This - parameter should be used only for unit tests. - Defaults to the file processing method of this class. + file_path: The path of the file to process. If possible, the path + should be relative to the source root. Otherwise, + path-specific logic may not behave as expected. + line_numbers: An array of line numbers of the lines for which + style errors should be reported, or None if errors + for all lines should be reported. Normally, this + array contains the line numbers corresponding to the + modified lines of a patch. + mock_handle_style_error: A unit-testing replacement for the function + to call when a style error occurs. Defaults + to a DefaultStyleErrorHandler instance. + mock_os_path_exists: A unit-test replacement for os.path.exists. + This parameter should only be used for unit + tests. + mock_process_file: The function to call to process the file. This + parameter should be used only for unit tests. + Defaults to the file processing method of this + class. + + Raises: + SystemExit: if the file does not exist. """ - if handle_style_error is None: + if mock_handle_style_error is None: + increment = self._increment_error_count handle_style_error = DefaultStyleErrorHandler( configuration=self._configuration, file_path=file_path, - increment_error_count= - self._increment_error_count) - if process_file is None: - process_file = self._process_file + increment_error_count=increment, + line_numbers=line_numbers) + else: + handle_style_error = mock_handle_style_error + + os_path_exists = (os.path.exists if mock_os_path_exists is None else + mock_os_path_exists) + process_file = (self._process_file if mock_process_file is None else + mock_process_file) + + if not os_path_exists(file_path) and file_path != "-": + _log.error("File does not exist: %s" % file_path) + sys.exit(1) + + _log.debug("Checking: " + file_path) self.file_count += 1 @@ -487,31 +722,58 @@ class StyleChecker(object): if dispatcher.should_skip_without_warning(file_path): return if dispatcher.should_skip_with_warning(file_path): - self._stderr_write('Ignoring "%s": this file is exempt from the ' - "style guide.\n" % file_path) + _log.warn('File exempt from style guide. Skipping: "%s"' + % file_path) return - verbosity = self._configuration.verbosity + min_confidence = self._configuration.min_confidence processor = dispatcher.dispatch_processor(file_path, handle_style_error, - verbosity) + min_confidence) if processor is None: + _log.debug('File not a recognized type to check. Skipping: "%s"' + % file_path) return + _log.debug("Using class: " + processor.__class__.__name__) + process_file(processor, file_path, handle_style_error) - def check_patch(self, patch_string): - """Check style in the given patch. + +class PatchChecker(object): + + """Supports checking style in patches.""" + + def __init__(self, style_checker): + """Create a PatchChecker instance. Args: - patch_string: A string that is a patch string. + style_checker: A StyleChecker instance. """ + self._file_checker = style_checker + + def check(self, patch_string): + """Check style in the given patch.""" patch_files = parse_patch(patch_string) - for file_path, diff in patch_files.iteritems(): - style_error_handler = PatchStyleErrorHandler(diff, - file_path, - self._configuration, - self._increment_error_count) - self.check_file(file_path, style_error_handler) + # The diff variable is a DiffFile instance. + for path, diff in patch_files.iteritems(): + line_numbers = set() + for line in diff.lines: + # When deleted line is not set, it means that + # the line is newly added (or modified). + if not line[0]: + line_numbers.add(line[1]) + + _log.debug('Found %s new or modified lines in: %s' + % (len(line_numbers), path)) + + # If line_numbers is empty, the file has no new or + # modified lines. In this case, we don't check the file + # because we'll never output errors for the file. + # This optimization also prevents the program from exiting + # due to a deleted file. + if line_numbers: + self._file_checker.check_file(file_path=path, + line_numbers=line_numbers) diff --git a/WebKitTools/Scripts/webkitpy/style/checker_unittest.py b/WebKitTools/Scripts/webkitpy/style/checker_unittest.py index fe12512..401a7b3 100755 --- a/WebKitTools/Scripts/webkitpy/style/checker_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/checker_unittest.py @@ -34,16 +34,23 @@ """Unit tests for style.py.""" +import logging +import os import unittest import checker as style +from webkitpy.style_references import parse_patch +from webkitpy.style_references import LogTesting +from webkitpy.style_references import TestLogStream from checker import _BASE_FILTER_RULES from checker import _MAX_REPORTS_PER_CATEGORY from checker import _PATH_RULES_SPECIFIER as PATH_RULES_SPECIFIER from checker import _all_categories from checker import check_webkit_style_configuration from checker import check_webkit_style_parser +from checker import configure_logging from checker import ProcessorDispatcher +from checker import PatchChecker from checker import StyleChecker from checker import StyleCheckerConfiguration from filter import validate_filter_rules @@ -51,7 +58,94 @@ from filter import FilterConfiguration from optparser import ArgumentParser from optparser import CommandOptionValues from processors.cpp import CppProcessor +from processors.python import PythonProcessor from processors.text import TextProcessor +from webkitpy.common.system.logtesting import LoggingTestCase + + +class ConfigureLoggingTestBase(unittest.TestCase): + + """Base class for testing configure_logging(). + + Sub-classes should implement: + + is_verbose: The is_verbose value to pass to configure_logging(). + + """ + + def setUp(self): + is_verbose = self.is_verbose + + log_stream = TestLogStream(self) + + # Use a logger other than the root logger or one prefixed with + # webkit so as not to conflict with test-webkitpy logging. + logger = logging.getLogger("unittest") + + # Configure the test logger not to pass messages along to the + # root logger. This prevents test messages from being + # propagated to loggers used by test-webkitpy logging (e.g. + # the root logger). + logger.propagate = False + + self._handlers = configure_logging(stream=log_stream, logger=logger, + is_verbose=is_verbose) + self._log = logger + self._log_stream = log_stream + + def tearDown(self): + """Reset logging to its original state. + + This method ensures that the logging configuration set up + for a unit test does not affect logging in other unit tests. + + """ + logger = self._log + for handler in self._handlers: + logger.removeHandler(handler) + + def assert_log_messages(self, messages): + """Assert that the logged messages equal the given messages.""" + self._log_stream.assertMessages(messages) + + +class ConfigureLoggingTest(ConfigureLoggingTestBase): + + """Tests the configure_logging() function.""" + + is_verbose = False + + def test_warning_message(self): + self._log.warn("test message") + self.assert_log_messages(["WARNING: test message\n"]) + + def test_below_warning_message(self): + # We test the boundary case of a logging level equal to 29. + # In practice, we will probably only be calling log.info(), + # which corresponds to a logging level of 20. + level = logging.WARNING - 1 # Equals 29. + self._log.log(level, "test message") + self.assert_log_messages(["test message\n"]) + + def test_debug_message(self): + self._log.debug("test message") + self.assert_log_messages([]) + + def test_two_messages(self): + self._log.info("message1") + self._log.info("message2") + self.assert_log_messages(["message1\n", "message2\n"]) + + +class ConfigureLoggingVerboseTest(ConfigureLoggingTestBase): + + """Tests the configure_logging() function with is_verbose True.""" + + is_verbose = True + + def test_debug_message(self): + self._log.debug("test message") + self.assert_log_messages(["unittest: DEBUG test message\n"]) class GlobalVariablesTest(unittest.TestCase): @@ -91,7 +185,9 @@ class GlobalVariablesTest(unittest.TestCase): default_options=default_options) # No need to test the return value here since we test parse() # on valid arguments elsewhere. - parser.parse([]) # arguments valid: no error or SystemExit + # + # The default options are valid: no error or SystemExit. + parser.parse(args=[]) def test_path_rules_specifier(self): all_categories = self._all_categories() @@ -125,6 +221,10 @@ class GlobalVariablesTest(unittest.TestCase): "readability/naming") assertNoCheck("WebKit/gtk/webkit/webkit.h", "readability/naming") + assertNoCheck("WebKitTools/DumpRenderTree/gtk/DumpRenderTree.cpp", + "readability/null") + assertNoCheck("WebKit/efl/ewk/ewk_view.h", + "readability/naming") assertNoCheck("WebCore/css/CSSParser.cpp", "readability/naming") assertNoCheck("WebKit/qt/tests/qwebelement/tst_qwebelement.cpp", @@ -132,6 +232,16 @@ class GlobalVariablesTest(unittest.TestCase): assertNoCheck( "JavaScriptCore/qt/tests/qscriptengine/tst_qscriptengine.cpp", "readability/naming") + assertNoCheck("WebCore/ForwardingHeaders/debugger/Debugger.h", + "build/header_guard") + + # Third-party Python code: webkitpy/thirdparty + path = "WebKitTools/Scripts/webkitpy/thirdparty/mock.py" + assertNoCheck(path, "build/include") + assertNoCheck(path, "pep8/E401") # A random pep8 category. + assertCheck(path, "pep8/W191") + assertCheck(path, "pep8/W291") + assertCheck(path, "whitespace/carriage_return") def test_max_reports_per_category(self): """Check that _MAX_REPORTS_PER_CATEGORY is valid.""" @@ -212,7 +322,7 @@ class ProcessorDispatcherDispatchTest(unittest.TestCase): dispatcher = ProcessorDispatcher() processor = dispatcher.dispatch_processor(file_path, self.mock_handle_style_error, - verbosity=3) + min_confidence=3) return processor def assert_processor_none(self, file_path): @@ -235,6 +345,10 @@ class ProcessorDispatcherDispatchTest(unittest.TestCase): """Assert that the dispatched processor is a CppProcessor.""" self.assert_processor(file_path, CppProcessor) + def assert_processor_python(self, file_path): + """Assert that the dispatched processor is a PythonProcessor.""" + self.assert_processor(file_path, PythonProcessor) + def assert_processor_text(self, file_path): """Assert that the dispatched processor is a TextProcessor.""" self.assert_processor(file_path, TextProcessor) @@ -260,7 +374,7 @@ class ProcessorDispatcherDispatchTest(unittest.TestCase): self.assertEquals(processor.file_extension, file_extension) self.assertEquals(processor.file_path, file_path) self.assertEquals(processor.handle_style_error, self.mock_handle_style_error) - self.assertEquals(processor.verbosity, 3) + self.assertEquals(processor.min_confidence, 3) # Check "-" for good measure. file_base = "-" file_extension = "" @@ -270,6 +384,26 @@ class ProcessorDispatcherDispatchTest(unittest.TestCase): self.assertEquals(processor.file_extension, file_extension) self.assertEquals(processor.file_path, file_path) + def test_python_paths(self): + """Test paths that should be checked as Python.""" + paths = [ + "foo.py", + "WebKitTools/Scripts/modules/text_style.py", + ] + + for path in paths: + self.assert_processor_python(path) + + # Check processor attributes on a typical input. + file_base = "foo" + file_extension = "css" + file_path = file_base + "." + file_extension + self.assert_processor_text(file_path) + processor = self.dispatch_processor(file_path) + self.assertEquals(processor.file_path, file_path) + self.assertEquals(processor.handle_style_error, + self.mock_handle_style_error) + def test_text_paths(self): """Test paths that should be checked as text.""" paths = [ @@ -281,14 +415,12 @@ class ProcessorDispatcherDispatchTest(unittest.TestCase): "foo.mm", "foo.php", "foo.pm", - "foo.py", "foo.txt", "FooChangeLog.bak", "WebCore/ChangeLog", "WebCore/inspector/front-end/inspector.js", - "WebKitTools/Scripts/check-webkit=style", - "WebKitTools/Scripts/modules/text_style.py", - ] + "WebKitTools/Scripts/check-webkit-style", + ] for path in paths: self.assert_processor_text(path) @@ -333,9 +465,9 @@ class StyleCheckerConfigurationTest(unittest.TestCase): return StyleCheckerConfiguration( filter_configuration=filter_configuration, max_reports_per_category={"whitespace/newline": 1}, + min_confidence=3, output_format=output_format, - stderr_write=self._mock_stderr_write, - verbosity=3) + stderr_write=self._mock_stderr_write) def test_init(self): """Test the __init__() method.""" @@ -345,7 +477,7 @@ class StyleCheckerConfigurationTest(unittest.TestCase): self.assertEquals(configuration.max_reports_per_category, {"whitespace/newline": 1}) self.assertEquals(configuration.stderr_write, self._mock_stderr_write) - self.assertEquals(configuration.verbosity, 3) + self.assertEquals(configuration.min_confidence, 3) def test_is_reportable(self): """Test the is_reportable() method.""" @@ -362,7 +494,7 @@ class StyleCheckerConfigurationTest(unittest.TestCase): def _call_write_style_error(self, output_format): config = self._style_checker_configuration(output_format=output_format) config.write_style_error(category="whitespace/tab", - confidence=5, + confidence_in_error=5, file_path="foo.h", line_number=100, message="message") @@ -395,9 +527,9 @@ class StyleCheckerTest(unittest.TestCase): configuration = StyleCheckerConfiguration( filter_configuration=FilterConfiguration(), max_reports_per_category={}, + min_confidence=3, output_format="vs7", - stderr_write=self._mock_stderr_write, - verbosity=3) + stderr_write=self._mock_stderr_write) style_checker = self._style_checker(configuration) @@ -406,7 +538,25 @@ class StyleCheckerTest(unittest.TestCase): self.assertEquals(style_checker.file_count, 0) -class StyleCheckerCheckFileTest(unittest.TestCase): +class StyleCheckerCheckFileBase(LoggingTestCase): + + def setUp(self): + LoggingTestCase.setUp(self) + self.warning_messages = "" + + def mock_stderr_write(self, warning_message): + self.warning_messages += warning_message + + def _style_checker_configuration(self): + return StyleCheckerConfiguration( + filter_configuration=FilterConfiguration(), + max_reports_per_category={"whitespace/newline": 1}, + min_confidence=3, + output_format="vs7", + stderr_write=self.mock_stderr_write) + + +class StyleCheckerCheckFileTest(StyleCheckerCheckFileBase): """Test the check_file() method of the StyleChecker class. @@ -432,17 +582,19 @@ class StyleCheckerCheckFileTest(unittest.TestCase): """ def setUp(self): + StyleCheckerCheckFileBase.setUp(self) self.got_file_path = None self.got_handle_style_error = None self.got_processor = None - self.warning_messages = "" - - def mock_stderr_write(self, warning_message): - self.warning_messages += warning_message def mock_handle_style_error(self): pass + def mock_os_path_exists(self, path): + # We deliberately make it so that this method returns False unless + # the caller has made an effort to put "does_exist" in the path. + return path.find("does_exist") > -1 + def mock_process_file(self, processor, file_path, handle_style_error): """A mock _process_file(). @@ -470,25 +622,48 @@ class StyleCheckerCheckFileTest(unittest.TestCase): # Confirm that the attributes are reset. self.assert_attributes(None, None, None, "") - configuration = StyleCheckerConfiguration( - filter_configuration=FilterConfiguration(), - max_reports_per_category={"whitespace/newline": 1}, - output_format="vs7", - stderr_write=self.mock_stderr_write, - verbosity=3) + configuration = self._style_checker_configuration() style_checker = StyleChecker(configuration) - style_checker.check_file(file_path, - self.mock_handle_style_error, - self.mock_process_file) + style_checker.check_file(file_path=file_path, + mock_handle_style_error=self.mock_handle_style_error, + mock_os_path_exists=self.mock_os_path_exists, + mock_process_file=self.mock_process_file) + + self.assertEquals(style_checker.file_count, 1) + + def test_check_file_does_not_exist(self): + file_path = "file_does_not_exist.txt" - self.assertEquals(1, style_checker.file_count) + # Confirm that the file does not exist. + self.assertFalse(self.mock_os_path_exists(file_path)) + + # Check the outcome. + self.assertRaises(SystemExit, self.call_check_file, file_path) + self.assertLog(["ERROR: File does not exist: " + "file_does_not_exist.txt\n"]) + + def test_check_file_stdin(self): + file_path = "-" + + # Confirm that the file does not exist. + self.assertFalse(self.mock_os_path_exists(file_path)) + + # Check the outcome. + self.call_check_file(file_path) + expected_processor = CppProcessor(file_path, + "", + self.mock_handle_style_error, 3) + self.assert_attributes(file_path, + self.mock_handle_style_error, + expected_processor, + "") def test_check_file_on_skip_without_warning(self): """Test check_file() for a skipped-without-warning file.""" - file_path = "LayoutTests/foo.txt" + file_path = "LayoutTests/does_exist/foo.txt" dispatcher = ProcessorDispatcher() # Confirm that the input file is truly a skipped-without-warning file. @@ -501,7 +676,7 @@ class StyleCheckerCheckFileTest(unittest.TestCase): def test_check_file_on_skip_with_warning(self): """Test check_file() for a skipped-with-warning file.""" - file_path = "gtk2drawing.c" + file_path = "does_exist/gtk2drawing.c" dispatcher = ProcessorDispatcher() # Check that the input file is truly a skipped-with-warning file. @@ -509,15 +684,16 @@ class StyleCheckerCheckFileTest(unittest.TestCase): # Check the outcome. self.call_check_file(file_path) - self.assert_attributes(None, None, None, - 'Ignoring "gtk2drawing.c": this file is exempt from the style guide.\n') + self.assert_attributes(None, None, None, "") + self.assertLog(["WARNING: File exempt from style guide. " + 'Skipping: "does_exist/gtk2drawing.c"\n']) def test_check_file_on_non_skipped(self): # We use a C++ file since by using a CppProcessor, we can check # that all of the possible information is getting passed to - # process_file (in particular, the verbosity). - file_base = "foo" + # process_file (in particular, the min_confidence parameter). + file_base = "foo_does_exist" file_extension = "cpp" file_path = file_base + "." + file_extension @@ -536,7 +712,97 @@ class StyleCheckerCheckFileTest(unittest.TestCase): "") -if __name__ == '__main__': - import sys +class StyleCheckerCheckPathsTest(unittest.TestCase): + + """Test the check_paths() method of the StyleChecker class.""" + + class MockOs(object): + + class MockPath(object): + + """A mock os.path.""" + + def isdir(self, path): + return path == "directory" + + def __init__(self): + self.path = self.MockPath() - unittest.main() + def walk(self, directory): + """A mock of os.walk.""" + if directory == "directory": + dirs = [("dir_path1", [], ["file1", "file2"]), + ("dir_path2", [], ["file3"])] + return dirs + return None + + def setUp(self): + self._checked_files = [] + + def _mock_check_file(self, file): + self._checked_files.append(file) + + def test_check_paths(self): + """Test StyleChecker.check_paths().""" + checker = StyleChecker(configuration=None) + mock_check_file = self._mock_check_file + mock_os = self.MockOs() + + # Confirm that checked files is empty at the outset. + self.assertEquals(self._checked_files, []) + checker.check_paths(["path1", "directory"], + mock_check_file=mock_check_file, + mock_os=mock_os) + self.assertEquals(self._checked_files, + ["path1", + os.path.join("dir_path1", "file1"), + os.path.join("dir_path1", "file2"), + os.path.join("dir_path2", "file3")]) + + +class PatchCheckerTest(unittest.TestCase): + + """Test the PatchChecker class.""" + + class MockStyleChecker(object): + + def __init__(self): + self.checked_files = [] + """A list of (file_path, line_numbers) pairs.""" + + def check_file(self, file_path, line_numbers): + self.checked_files.append((file_path, line_numbers)) + + def setUp(self): + style_checker = self.MockStyleChecker() + self._style_checker = style_checker + self._patch_checker = PatchChecker(style_checker) + + def _call_check_patch(self, patch_string): + self._patch_checker.check(patch_string) + + def _assert_checked(self, checked_files): + self.assertEquals(self._style_checker.checked_files, checked_files) + + def test_check_patch(self): + # The modified line_numbers array for this patch is: [2]. + self._call_check_patch("""diff --git a/__init__.py b/__init__.py +index ef65bee..e3db70e 100644 +--- a/__init__.py ++++ b/__init__.py +@@ -1,1 +1,2 @@ + # Required for Python to search this directory for module files ++# New line +""") + self._assert_checked([("__init__.py", set([2]))]) + + def test_check_patch_with_deletion(self): + self._call_check_patch("""Index: __init__.py +=================================================================== +--- __init__.py (revision 3593) ++++ __init__.py (working copy) +@@ -1 +0,0 @@ +-foobar +""") + # _mock_check_file should not be called for the deletion patch. + self._assert_checked([]) diff --git a/WebKitTools/Scripts/webkitpy/style/error_handlers.py b/WebKitTools/Scripts/webkitpy/style/error_handlers.py index 6bc3f15..5666bfb 100644 --- a/WebKitTools/Scripts/webkitpy/style/error_handlers.py +++ b/WebKitTools/Scripts/webkitpy/style/error_handlers.py @@ -40,10 +40,10 @@ Methods: line_number: The integer line number of the line containing the error. category: The name of the category of the error, for example "whitespace/newline". - confidence: An integer between 1-5 that represents the level of - confidence in the error. The value 5 means that we are - certain of the problem, and the value 1 means that it - could be a legitimate construct. + confidence: An integer between 1 and 5 inclusive that represents the + application's level of confidence in the error. The value + 5 means that we are certain of the problem, and the + value 1 means that it could be a legitimate construct. message: The error message to report. """ @@ -56,7 +56,8 @@ class DefaultStyleErrorHandler(object): """The default style error handler.""" - def __init__(self, file_path, configuration, increment_error_count): + def __init__(self, file_path, configuration, increment_error_count, + line_numbers=None): """Create a default style error handler. Args: @@ -66,16 +67,44 @@ class DefaultStyleErrorHandler(object): increment_error_count: A function that takes no arguments and increments the total count of reportable errors. + line_numbers: An array of line numbers of the lines for which + style errors should be reported, or None if errors + for all lines should be reported. When it is not + None, this array normally contains the line numbers + corresponding to the modified lines of a patch. """ + if line_numbers is not None: + line_numbers = set(line_numbers) + self._file_path = file_path self._configuration = configuration self._increment_error_count = increment_error_count + self._line_numbers = line_numbers # A string to integer dictionary cache of the number of reportable # errors per category passed to this instance. self._category_totals = {} + # Useful for unit testing. + def __eq__(self, other): + """Return whether this instance is equal to another.""" + if self._configuration != other._configuration: + return False + if self._file_path != other._file_path: + return False + if self._increment_error_count != other._increment_error_count: + return False + if self._line_numbers != other._line_numbers: + return False + + return True + + # Useful for unit testing. + def __ne__(self, other): + # Python does not automatically deduce __ne__ from __eq__. + return not self.__eq__(other) + def _add_reportable_error(self, category): """Increment the error count and return the new category total.""" self._increment_error_count() # Increment the total. @@ -100,6 +129,12 @@ class DefaultStyleErrorHandler(object): See the docstring of this module for more information. """ + if (self._line_numbers is not None and + line_number not in self._line_numbers): + # Then the error occurred in a line that was not modified, so + # the error is not reportable. + return + if not self._configuration.is_reportable(category=category, confidence_in_error=confidence, file_path=self._file_path): @@ -114,7 +149,7 @@ class DefaultStyleErrorHandler(object): return self._configuration.write_style_error(category=category, - confidence=confidence, + confidence_in_error=confidence, file_path=self._file_path, line_number=line_number, message=message) @@ -122,56 +157,3 @@ class DefaultStyleErrorHandler(object): if category_total == max_reports: self._configuration.stderr_write("Suppressing further [%s] reports " "for this file.\n" % category) - - -class PatchStyleErrorHandler(object): - - """The style error function for patch files.""" - - def __init__(self, diff, file_path, configuration, increment_error_count): - """Create a patch style error handler for the given path. - - Args: - diff: A DiffFile instance. - Other arguments: see the DefaultStyleErrorHandler.__init__() - documentation for the other arguments. - - """ - self._diff = diff - - self._default_error_handler = DefaultStyleErrorHandler( - configuration=configuration, - file_path=file_path, - increment_error_count= - increment_error_count) - - # The line numbers of the modified lines. This is set lazily. - self._line_numbers = set() - - def _get_line_numbers(self): - """Return the line numbers of the modified lines.""" - if not self._line_numbers: - for line in self._diff.lines: - # When deleted line is not set, it means that - # the line is newly added (or modified). - if not line[0]: - self._line_numbers.add(line[1]) - - return self._line_numbers - - def __call__(self, line_number, category, confidence, message): - """Handle the occurrence of a style error. - - This function does not report errors occurring in lines not - marked as modified or added in the patch. - - See the docstring of this module for more information. - - """ - if line_number not in self._get_line_numbers(): - # Then the error is not reportable. - return - - self._default_error_handler(line_number, category, confidence, - message) - diff --git a/WebKitTools/Scripts/webkitpy/style/error_handlers_unittest.py b/WebKitTools/Scripts/webkitpy/style/error_handlers_unittest.py index a39ba2a..05e725a 100644 --- a/WebKitTools/Scripts/webkitpy/style/error_handlers_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/error_handlers_unittest.py @@ -25,18 +25,25 @@ import unittest -from .. style_references import parse_patch from checker import StyleCheckerConfiguration from error_handlers import DefaultStyleErrorHandler -from error_handlers import PatchStyleErrorHandler from filter import FilterConfiguration -class StyleErrorHandlerTestBase(unittest.TestCase): + +class DefaultStyleErrorHandlerTest(unittest.TestCase): + + """Tests the DefaultStyleErrorHandler class.""" def setUp(self): self._error_messages = [] self._error_count = 0 + _category = "whitespace/tab" + """The category name for the tests in this class.""" + + _file_path = "foo.h" + """The file path for the tests in this class.""" + def _mock_increment_error_count(self): self._error_count += 1 @@ -51,38 +58,66 @@ class StyleErrorHandlerTestBase(unittest.TestCase): return StyleCheckerConfiguration( filter_configuration=filter_configuration, max_reports_per_category={"whitespace/tab": 2}, + min_confidence=3, output_format="vs7", - stderr_write=self._mock_stderr_write, - verbosity=3) - + stderr_write=self._mock_stderr_write) -class DefaultStyleErrorHandlerTest(StyleErrorHandlerTestBase): - - """Tests DefaultStyleErrorHandler class.""" - - _category = "whitespace/tab" - """The category name for the tests in this class.""" - - _file_path = "foo.h" - """The file path for the tests in this class.""" + def _error_handler(self, configuration, line_numbers=None): + return DefaultStyleErrorHandler(configuration=configuration, + file_path=self._file_path, + increment_error_count=self._mock_increment_error_count, + line_numbers=line_numbers) def _check_initialized(self): """Check that count and error messages are initialized.""" self.assertEquals(0, self._error_count) self.assertEquals(0, len(self._error_messages)) - def _error_handler(self, configuration): - return DefaultStyleErrorHandler(configuration=configuration, - file_path=self._file_path, - increment_error_count=self._mock_increment_error_count) - - def _call_error_handler(self, handle_error, confidence): + def _call_error_handler(self, handle_error, confidence, line_number=100): """Call the given error handler with a test error.""" - handle_error(line_number=100, + handle_error(line_number=line_number, category=self._category, confidence=confidence, message="message") + def test_eq__true_return_value(self): + """Test the __eq__() method for the return value of True.""" + handler1 = self._error_handler(configuration=None) + handler2 = self._error_handler(configuration=None) + + self.assertTrue(handler1.__eq__(handler2)) + + def test_eq__false_return_value(self): + """Test the __eq__() method for the return value of False.""" + def make_handler(configuration=self._style_checker_configuration(), + file_path='foo.txt', increment_error_count=lambda: True, + line_numbers=[100]): + return DefaultStyleErrorHandler(configuration=configuration, + file_path=file_path, + increment_error_count=increment_error_count, + line_numbers=line_numbers) + + handler = make_handler() + + # Establish a baseline for our comparisons below. + self.assertTrue(handler.__eq__(make_handler())) + + # Verify that a difference in any argument causes equality to fail. + self.assertFalse(handler.__eq__(make_handler(configuration=None))) + self.assertFalse(handler.__eq__(make_handler(file_path='bar.txt'))) + self.assertFalse(handler.__eq__(make_handler(increment_error_count=None))) + self.assertFalse(handler.__eq__(make_handler(line_numbers=[50]))) + + def test_ne(self): + """Test the __ne__() method.""" + # By default, __ne__ always returns true on different objects. + # Thus, check just the distinguishing case to verify that the + # code defines __ne__. + handler1 = self._error_handler(configuration=None) + handler2 = self._error_handler(configuration=None) + + self.assertFalse(handler1.__ne__(handler2)) + def test_non_reportable_error(self): """Test __call__() with a non-reportable error.""" self._check_initialized() @@ -132,52 +167,21 @@ class DefaultStyleErrorHandlerTest(StyleErrorHandlerTestBase): self.assertEquals(3, self._error_count) self.assertEquals(3, len(self._error_messages)) - -class PatchStyleErrorHandlerTest(StyleErrorHandlerTestBase): - - """Tests PatchStyleErrorHandler class.""" - - _file_path = "__init__.py" - - _patch_string = """diff --git a/__init__.py b/__init__.py -index ef65bee..e3db70e 100644 ---- a/__init__.py -+++ b/__init__.py -@@ -1 +1,2 @@ - # Required for Python to search this directory for module files -+# New line - -""" - - def test_call(self): - patch_files = parse_patch(self._patch_string) - diff = patch_files[self._file_path] - + def test_line_numbers(self): + """Test the line_numbers parameter.""" + self._check_initialized() configuration = self._style_checker_configuration() - - handle_error = PatchStyleErrorHandler(diff=diff, - file_path=self._file_path, - configuration=configuration, - increment_error_count= - self._mock_increment_error_count) - - category = "whitespace/tab" + error_handler = self._error_handler(configuration, + line_numbers=[50]) confidence = 5 - message = "message" - - # Confirm error is reportable. - self.assertTrue(configuration.is_reportable(category, - confidence, - self._file_path)) - # Confirm error count initialized to zero. - self.assertEquals(0, self._error_count) - - # Test error in unmodified line (error count does not increment). - handle_error(1, category, confidence, message) + # Error on non-modified line: no error. + self._call_error_handler(error_handler, confidence, line_number=60) self.assertEquals(0, self._error_count) + self.assertEquals([], self._error_messages) - # Test error in modified line (error count increments). - handle_error(2, category, confidence, message) + # Error on modified line: error. + self._call_error_handler(error_handler, confidence, line_number=50) self.assertEquals(1, self._error_count) - + self.assertEquals(self._error_messages, + ["foo.h(50): message [whitespace/tab] [5]\n"]) diff --git a/WebKitTools/Scripts/webkitpy/style/filereader.py b/WebKitTools/Scripts/webkitpy/style/filereader.py new file mode 100644 index 0000000..081e6dc --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/filereader.py @@ -0,0 +1,148 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# Copyright (C) 2010 ProFUSION embedded systems +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports reading and processing text files.""" + +import codecs +import logging +import os +import sys + + +_log = logging.getLogger(__name__) + + +class TextFileReader(object): + + """Supports reading and processing text files. + + Attributes: + file_count: The total number of files passed to this instance + for processing, including non-text files and files + that should be skipped. + + """ + + def __init__(self, processor): + """Create an instance. + + Arguments: + processor: A ProcessorBase instance. + + """ + self._processor = processor + self.file_count = 0 + + def _read_lines(self, file_path): + """Read the file at a path, and return its lines. + + Raises: + IOError: If the file does not exist or cannot be read. + + """ + # Support the UNIX convention of using "-" for stdin. + if file_path == '-': + file = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace') + else: + # We do not open the file with universal newline support + # (codecs does not support it anyway), so the resulting + # lines contain trailing "\r" characters if we are reading + # a file with CRLF endings. + file = codecs.open(file_path, 'r', 'utf8', 'replace') + + try: + contents = file.read() + finally: + file.close() + + lines = contents.split('\n') + return lines + + def process_file(self, file_path, **kwargs): + """Process the given file by calling the processor's process() method. + + Args: + file_path: The path of the file to process. + **kwargs: Any additional keyword parameters that should be passed + to the processor's process() method. The process() + method should support these keyword arguments. + + Raises: + SystemExit: If no file at file_path exists. + + """ + self.file_count += 1 + + if not self._processor.should_process(file_path): + _log.debug("Skipping file: '%s'" % file_path) + return + _log.debug("Processing file: '%s'" % file_path) + + try: + lines = self._read_lines(file_path) + except IOError, err: + if not os.path.exists(file_path): + _log.error("File does not exist: '%s'" % file_path) + sys.exit(1) + + message = ("Could not read file. Skipping: '%s'\n %s" + % (file_path, err)) + _log.warn(message) + return + + self._processor.process(lines, file_path, **kwargs) + + def _process_directory(self, directory): + """Process all files in the given directory, recursively. + + Args: + directory: A directory path. + + """ + for dir_path, dir_names, file_names in os.walk(directory): + for file_name in file_names: + file_path = os.path.join(dir_path, file_name) + self.process_file(file_path) + + def process_paths(self, paths): + """Process the given file and directory paths. + + Args: + paths: A list of file and directory paths. + + """ + for path in paths: + if os.path.isdir(path): + self._process_directory(directory=path) + else: + self.process_file(path) diff --git a/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py b/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py new file mode 100644 index 0000000..8d1a159 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py @@ -0,0 +1,151 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains unit tests for filereader.py.""" + +import os +import shutil +import tempfile +import unittest + +from webkitpy.common.system.logtesting import LoggingTestCase +from webkitpy.style.checker import ProcessorBase +from webkitpy.style.filereader import TextFileReader + + +class TextFileReaderTest(LoggingTestCase): + + class MockProcessor(ProcessorBase): + + """A processor for test purposes. + + This processor simply records the parameters passed to its process() + method for later checking by the unittest test methods. + + """ + + def __init__(self): + self.processed = [] + """The parameters passed for all calls to the process() method.""" + + def should_process(self, file_path): + return not file_path.endswith('should_not_process.txt') + + def process(self, lines, file_path, test_kwarg=None): + self.processed.append((lines, file_path, test_kwarg)) + + def setUp(self): + LoggingTestCase.setUp(self) + processor = TextFileReaderTest.MockProcessor() + + temp_dir = tempfile.mkdtemp() + + self._file_reader = TextFileReader(processor) + self._processor = processor + self._temp_dir = temp_dir + + def tearDown(self): + LoggingTestCase.tearDown(self) + shutil.rmtree(self._temp_dir) + + def _create_file(self, rel_path, text): + """Create a file with given text and return the path to the file.""" + file_path = os.path.join(self._temp_dir, rel_path) + + file = open(file_path, 'w') + file.write(text) + file.close() + + return file_path + + def _passed_to_processor(self): + """Return the parameters passed to MockProcessor.process().""" + return self._processor.processed + + def _assert_file_reader(self, passed_to_processor, file_count): + """Assert the state of the file reader.""" + self.assertEquals(passed_to_processor, self._passed_to_processor()) + self.assertEquals(file_count, self._file_reader.file_count) + + def test_process_file__should_not_process(self): + self._file_reader.process_file('should_not_process.txt') + self._assert_file_reader([], 1) + + def test_process_file__does_not_exist(self): + try: + self._file_reader.process_file('does_not_exist.txt') + except SystemExit, err: + self.assertEquals(str(err), '1') + else: + self.fail('No Exception raised.') + self._assert_file_reader([], 1) + self.assertLog(["ERROR: File does not exist: 'does_not_exist.txt'\n"]) + + def test_process_file__is_dir(self): + temp_dir = os.path.join(self._temp_dir, 'test_dir') + os.mkdir(temp_dir) + + self._file_reader.process_file(temp_dir) + + # Because the log message below contains exception text, it is + # possible that the text varies across platforms. For this reason, + # we check only the portion of the log message that we control, + # namely the text at the beginning. + log_messages = self.logMessages() + # We remove the message we are looking at to prevent the tearDown() + # from raising an exception when it asserts that no log messages + # remain. + message = log_messages.pop() + + self.assertTrue(message.startswith('WARNING: Could not read file. ' + "Skipping: '%s'\n " % temp_dir)) + + self._assert_file_reader([], 1) + + def test_process_file__multiple_lines(self): + file_path = self._create_file('foo.txt', 'line one\r\nline two\n') + + self._file_reader.process_file(file_path) + processed = [(['line one\r', 'line two', ''], file_path, None)] + self._assert_file_reader(processed, 1) + + def test_process_file__with_kwarg(self): + file_path = self._create_file('foo.txt', 'file contents') + + self._file_reader.process_file(file_path=file_path, test_kwarg='foo') + processed = [(['file contents'], file_path, 'foo')] + self._assert_file_reader(processed, 1) + + def test_process_paths(self): + # We test a list of paths that contains both a file and a directory. + dir = os.path.join(self._temp_dir, 'foo_dir') + os.mkdir(dir) + + file_path1 = self._create_file('file1.txt', 'foo') + + rel_path = os.path.join('foo_dir', 'file2.txt') + file_path2 = self._create_file(rel_path, 'bar') + + self._file_reader.process_paths([dir, file_path1]) + processed = [(['bar'], file_path2, None), + (['foo'], file_path1, None)] + self._assert_file_reader(processed, 2) diff --git a/WebKitTools/Scripts/webkitpy/style/main.py b/WebKitTools/Scripts/webkitpy/style/main.py new file mode 100644 index 0000000..c933c6d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/main.py @@ -0,0 +1,130 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +import os +import sys + +from webkitpy.common.system.ospath import relpath as _relpath + + +_log = logging.getLogger(__name__) + + +def change_directory(checkout_root, paths, mock_os=None): + """Change the working directory to the WebKit checkout root, if possible. + + If every path in the paths parameter is below the checkout root (or if + the paths parameter is empty or None), this method changes the current + working directory to the checkout root and converts the paths parameter + as described below. + This allows the paths being checked to be displayed relative to the + checkout root, and for path-specific style checks to work as expected. + Path-specific checks include whether files should be skipped, whether + custom style rules should apply to certain files, etc. + If the checkout root is None or the empty string, this method returns + the paths parameter unchanged. + + Returns: + paths: A copy of the paths parameter -- possibly converted, as follows. + If this method changed the current working directory to the + checkout root, then the list is the paths parameter converted to + normalized paths relative to the checkout root. Otherwise, the + paths are not converted. + + Args: + paths: A list of paths to the files that should be checked for style. + This argument can be None or the empty list if a git commit + or all changes under the checkout root should be checked. + checkout_root: The path to the root of the WebKit checkout, or None or + the empty string if no checkout could be detected. + mock_os: A replacement module for unit testing. Defaults to os. + + """ + os_module = os if mock_os is None else mock_os + + if paths is not None: + paths = list(paths) + + if not checkout_root: + if not paths: + raise Exception("The paths parameter must be non-empty if " + "there is no checkout root.") + + # FIXME: Consider trying to detect the checkout root for each file + # being checked rather than only trying to detect the checkout + # root for the current working directory. This would allow + # files to be checked correctly even if the script is being + # run from outside any WebKit checkout. + # + # Moreover, try to find the "source root" for each file + # using path-based heuristics rather than using only the + # presence of a WebKit checkout. For example, we could + # examine parent directories until a directory is found + # containing JavaScriptCore, WebCore, WebKit, WebKitSite, + # and WebKitTools. + # Then log an INFO message saying that a source root not + # in a WebKit checkout was found. This will allow us to check + # the style of non-scm copies of the source tree (e.g. + # nightlies). + _log.warn("WebKit checkout root not found:\n" + " Path-dependent style checks may not work correctly.\n" + " See the help documentation for more info.") + + return paths + + if paths: + # Then try converting all of the paths to paths relative to + # the checkout root. + rel_paths = [] + for path in paths: + rel_path = _relpath(path, checkout_root) + if rel_path is None: + # Then the path is not below the checkout root. Since all + # paths should be interpreted relative to the same root, + # do not interpret any of the paths as relative to the + # checkout root. Interpret all of them relative to the + # current working directory, and do not change the current + # working directory. + _log.warn( +"""Path-dependent style checks may not work correctly: + + One of the given paths is outside the WebKit checkout of the current + working directory: + + Path: %s + Checkout root: %s + + Pass only files below the checkout root to ensure correct results. + See the help documentation for more info. +""" + % (path, checkout_root)) + + return paths + rel_paths.append(rel_path) + # If we got here, the conversion was successful. + paths = rel_paths + + _log.debug("Changing to checkout root: " + checkout_root) + os_module.chdir(checkout_root) + + return paths diff --git a/WebKitTools/Scripts/webkitpy/style/main_unittest.py b/WebKitTools/Scripts/webkitpy/style/main_unittest.py new file mode 100644 index 0000000..fe448f5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/main_unittest.py @@ -0,0 +1,124 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for main.py.""" + +import os +import unittest + +from main import change_directory +from webkitpy.style_references import LogTesting + + +class ChangeDirectoryTest(unittest.TestCase): + + """Tests change_directory().""" + + _original_directory = "/original" + _checkout_root = "/WebKit" + + class _MockOs(object): + + """A mock os module for unit testing.""" + + def __init__(self, test_case): + self._test_case = test_case + self._current_directory = \ + ChangeDirectoryTest._original_directory + + def chdir(self, current_directory): + self._current_directory = current_directory + + def assertCurrentDirectory(self, expected_directory): + self._test_case.assertEquals(expected_directory, + self._current_directory) + + def setUp(self): + self._log = LogTesting.setUp(self) + self._mock_os = self._MockOs(self) + + def tearDown(self): + self._log.tearDown() + + # This method is a convenient wrapper for change_working_directory() that + # passes the mock_os for this unit testing class. + def _change_directory(self, paths, checkout_root): + return change_directory(paths=paths, + checkout_root=checkout_root, + mock_os=self._mock_os) + + def _assert_result(self, actual_return_value, expected_return_value, + expected_log_messages, expected_current_directory): + self.assertEquals(actual_return_value, expected_return_value) + self._log.assertMessages(expected_log_messages) + self._mock_os.assertCurrentDirectory(expected_current_directory) + + def test_checkout_root_none_paths_none(self): + self.assertRaises(Exception, self._change_directory, + checkout_root=None, paths=None) + self._log.assertMessages([]) + self._mock_os.assertCurrentDirectory(self._original_directory) + + def test_checkout_root_none(self): + paths = self._change_directory(checkout_root=None, + paths=["path1"]) + log_messages = [ +"""WARNING: WebKit checkout root not found: + Path-dependent style checks may not work correctly. + See the help documentation for more info. +"""] + self._assert_result(paths, ["path1"], log_messages, + self._original_directory) + + def test_paths_none(self): + paths = self._change_directory(checkout_root=self._checkout_root, + paths=None) + self._assert_result(paths, None, [], self._checkout_root) + + def test_paths_convertible(self): + paths=["/WebKit/foo1.txt", + "/WebKit/foo2.txt"] + paths = self._change_directory(checkout_root=self._checkout_root, + paths=paths) + self._assert_result(paths, ["foo1.txt", "foo2.txt"], [], + self._checkout_root) + + def test_with_scm_paths_unconvertible(self): + paths=["/WebKit/foo1.txt", + "/outside/foo2.txt"] + paths = self._change_directory(checkout_root=self._checkout_root, + paths=paths) + log_messages = [ +"""WARNING: Path-dependent style checks may not work correctly: + + One of the given paths is outside the WebKit checkout of the current + working directory: + + Path: /outside/foo2.txt + Checkout root: /WebKit + + Pass only files below the checkout root to ensure correct results. + See the help documentation for more info. + +"""] + self._assert_result(paths, paths, log_messages, + self._original_directory) diff --git a/WebKitTools/Scripts/webkitpy/style/optparser.py b/WebKitTools/Scripts/webkitpy/style/optparser.py index 4137c8b..576c16a 100644 --- a/WebKitTools/Scripts/webkitpy/style/optparser.py +++ b/WebKitTools/Scripts/webkitpy/style/optparser.py @@ -22,84 +22,83 @@ """Supports the parsing of command-line options for check-webkit-style.""" -import getopt +import logging +from optparse import OptionParser import os.path import sys from filter import validate_filter_rules # This module should not import anything from checker.py. +_log = logging.getLogger(__name__) -def _create_usage(default_options): - """Return the usage string to display for command help. +_USAGE = """usage: %prog [--help] [options] [path1] [path2] ... - Args: - default_options: A DefaultCommandOptionValues instance. +Overview: + Check coding style according to WebKit style guidelines: - """ - usage = """ -Syntax: %(program_name)s [--verbose=#] [--git-commit=<SingleCommit>] [--output=vs7] - [--filter=-x,+y,...] [file] ... - - The style guidelines this tries to follow are here: - http://webkit.org/coding/coding-style.html + http://webkit.org/coding/coding-style.html - Every style error is given a confidence score from 1-5, with 5 meaning - we are certain of the problem, and 1 meaning it could be a legitimate - construct. This can miss some errors and does not substitute for - code review. + Path arguments can be files and directories. If neither a git commit nor + paths are passed, then all changes in your source control working directory + are checked. - To prevent specific lines from being linted, add a '// NOLINT' comment to the - end of the line. +Style errors: + This script assigns to every style error a confidence score from 1-5 and + a category name. A confidence score of 5 means the error is certainly + a problem, and 1 means it could be fine. - Linted extensions are .cpp, .c and .h. Other file types are ignored. + Category names appear in error messages in brackets, for example + [whitespace/indent]. See the options section below for an option that + displays all available categories and which are reported by default. - The file parameter is optional and accepts multiple files. Leaving - out the file parameter applies the check to all files considered changed - by your source control management system. +Filters: + Use filters to configure what errors to report. Filters are specified using + a comma-separated list of boolean filter rules. The script reports errors + in a category if the category passes the filter, as described below. - Flags: + All categories start out passing. Boolean filter rules are then evaluated + from left to right, with later rules taking precedence. For example, the + rule "+foo" passes any category that starts with "foo", and "-foo" fails + any such category. The filter input "-whitespace,+whitespace/braces" fails + the category "whitespace/tab" and passes "whitespace/braces". - verbose=# - A number 1-5 that restricts output to errors with a confidence - score at or above this value. In particular, the value 1 displays - all errors. The default is %(default_verbosity)s. + Examples: --filter=-whitespace,+whitespace/braces + --filter=-whitespace,-runtime/printf,+runtime/printf_format + --filter=-,+build/include_what_you_use - git-commit=<SingleCommit> - Checks the style of everything from the given commit to the local tree. +Paths: + Certain style-checking behavior depends on the paths relative to + the WebKit source root of the files being checked. For example, + certain types of errors may be handled differently for files in + WebKit/gtk/webkit/ (e.g. by suppressing "readability/naming" errors + for files in this directory). - output=vs7 - The output format, which may be one of - emacs : to ease emacs parsing - vs7 : compatible with Visual Studio - Defaults to "%(default_output_format)s". Other formats are unsupported. + Consequently, if the path relative to the source root cannot be + determined for a file being checked, then style checking may not + work correctly for that file. This can occur, for example, if no + WebKit checkout can be found, or if the source root can be detected, + but one of the files being checked lies outside the source tree. - filter=-x,+y,... - A comma-separated list of boolean filter rules used to filter - which categories of style guidelines to check. The script checks - a category if the category passes the filter rules, as follows. + If a WebKit checkout can be detected and all files being checked + are in the source tree, then all paths will automatically be + converted to paths relative to the source root prior to checking. + This is also useful for display purposes. - Any webkit category starts out passing. All filter rules are then - evaluated left to right, with later rules taking precedence. For - example, the rule "+foo" passes any category that starts with "foo", - and "-foo" fails any such category. The filter input "-whitespace, - +whitespace/braces" fails the category "whitespace/tab" and passes - "whitespace/braces". + Currently, this command can detect the source root only if the + command is run from within a WebKit checkout (i.e. if the current + working directory is below the root of a checkout). In particular, + it is not recommended to run this script from a directory outside + a checkout. - Examples: --filter=-whitespace,+whitespace/braces - --filter=-whitespace,-runtime/printf,+runtime/printf_format - --filter=-,+build/include_what_you_use + Running this script from a top-level WebKit source directory and + checking only files in the source tree will ensure that all style + checking behaves correctly -- whether or not a checkout can be + detected. This is because all file paths will already be relative + to the source root and so will not need to be converted.""" - Category names appear in error messages in brackets, for example - [whitespace/indent]. To see a list of all categories available to - %(program_name)s, along with which are enabled by default, pass - the empty filter as follows: - --filter= -""" % {'program_name': os.path.basename(sys.argv[0]), - 'default_verbosity': default_options.verbosity, - 'default_output_format': default_options.output_format} - - return usage +_EPILOG = ("This script can miss errors and does not substitute for " + "code review.") # This class should not have knowledge of the flag key names. @@ -109,26 +108,22 @@ class DefaultCommandOptionValues(object): Attributes: output_format: A string that is the default output format. - verbosity: An integer that is the default verbosity level. + min_confidence: An integer that is the default minimum confidence level. """ - def __init__(self, output_format, verbosity): + def __init__(self, min_confidence, output_format): + self.min_confidence = min_confidence self.output_format = output_format - self.verbosity = verbosity -# FIXME: Eliminate support for "extra_flag_values". -# # This class should not have knowledge of the flag key names. class CommandOptionValues(object): """Stores the option values passed by the user via the command line. Attributes: - extra_flag_values: A string-string dictionary of all flag key-value - pairs that are not otherwise represented by this - class. The default is the empty dictionary. + is_verbose: A boolean value of whether verbose logging is enabled. filter_rules: The list of filter rules provided by the user. These rules are appended to the base rules and @@ -138,54 +133,52 @@ class CommandOptionValues(object): git_commit: A string representing the git commit to check. The default is None. + min_confidence: An integer between 1 and 5 inclusive that is the + minimum confidence level of style errors to report. + The default is 1, which reports all errors. + output_format: A string that is the output format. The supported output formats are "emacs" which emacs can parse and "vs7" which Microsoft Visual Studio 7 can parse. - verbosity: An integer between 1-5 inclusive that restricts output - to errors with a confidence score at or above this value. - The default is 1, which reports all errors. - """ def __init__(self, - extra_flag_values=None, filter_rules=None, git_commit=None, - output_format="emacs", - verbosity=1): - if extra_flag_values is None: - extra_flag_values = {} + is_verbose=False, + min_confidence=1, + output_format="emacs"): if filter_rules is None: filter_rules = [] + if (min_confidence < 1) or (min_confidence > 5): + raise ValueError('Invalid "min_confidence" parameter: value ' + "must be an integer between 1 and 5 inclusive. " + 'Value given: "%s".' % min_confidence) + if output_format not in ("emacs", "vs7"): raise ValueError('Invalid "output_format" parameter: ' 'value must be "emacs" or "vs7". ' 'Value given: "%s".' % output_format) - if (verbosity < 1) or (verbosity > 5): - raise ValueError('Invalid "verbosity" parameter: ' - "value must be an integer between 1-5 inclusive. " - 'Value given: "%s".' % verbosity) - - self.extra_flag_values = extra_flag_values self.filter_rules = filter_rules self.git_commit = git_commit + self.is_verbose = is_verbose + self.min_confidence = min_confidence self.output_format = output_format - self.verbosity = verbosity # Useful for unit testing. def __eq__(self, other): """Return whether this instance is equal to another.""" - if self.extra_flag_values != other.extra_flag_values: - return False if self.filter_rules != other.filter_rules: return False if self.git_commit != other.git_commit: return False - if self.output_format != other.output_format: + if self.is_verbose != other.is_verbose: return False - if self.verbosity != other.verbosity: + if self.min_confidence != other.min_confidence: + return False + if self.output_format != other.output_format: return False return True @@ -212,10 +205,9 @@ class ArgumentPrinter(object): options: A CommandOptionValues instance. """ - flags = options.extra_flag_values.copy() - + flags = {} + flags['min-confidence'] = options.min_confidence flags['output'] = options.output_format - flags['verbose'] = options.verbosity # Only include the filter flag if user-provided rules are present. filter_rules = options.filter_rules if filter_rules: @@ -231,7 +223,6 @@ class ArgumentPrinter(object): return flag_string.strip() -# FIXME: Replace the use of getopt.getopt() with optparse.OptionParser. class ArgumentParser(object): # FIXME: Move the documentation of the attributes to the __init__ @@ -256,8 +247,8 @@ class ArgumentParser(object): all_categories, default_options, base_filter_rules=None, - create_usage=None, - stderr_write=None): + mock_stderr=None, + usage=None): """Create an ArgumentParser instance. Args: @@ -279,31 +270,96 @@ class ArgumentParser(object): """ if base_filter_rules is None: base_filter_rules = [] - if create_usage is None: - create_usage = _create_usage - if stderr_write is None: - stderr_write = sys.stderr.write + stderr = sys.stderr if mock_stderr is None else mock_stderr + if usage is None: + usage = _USAGE self._all_categories = all_categories self._base_filter_rules = base_filter_rules + # FIXME: Rename these to reflect that they are internal. - self.create_usage = create_usage self.default_options = default_options - self.stderr_write = stderr_write - - def _exit_with_usage(self, error_message=''): - """Exit and print a usage string with an optional error message. - - Args: - error_message: A string that is an error message to print. - - """ - usage = self.create_usage(self.default_options) - self.stderr_write(usage) + self.stderr_write = stderr.write + + self._parser = self._create_option_parser(stderr=stderr, + usage=usage, + default_min_confidence=self.default_options.min_confidence, + default_output_format=self.default_options.output_format) + + def _create_option_parser(self, stderr, usage, + default_min_confidence, default_output_format): + # Since the epilog string is short, it is not necessary to replace + # the epilog string with a mock epilog string when testing. + # For this reason, we use _EPILOG directly rather than passing it + # as an argument like we do for the usage string. + parser = OptionParser(usage=usage, epilog=_EPILOG) + + filter_help = ('set a filter to control what categories of style ' + 'errors to report. Specify a filter using a comma-' + 'delimited list of boolean filter rules, for example ' + '"--filter -whitespace,+whitespace/braces". To display ' + 'all categories and which are enabled by default, pass ' + """no value (e.g. '-f ""' or '--filter=').""") + parser.add_option("-f", "--filter-rules", metavar="RULES", + dest="filter_value", help=filter_help) + + git_help = "check all changes after the given git commit." + parser.add_option("-g", "--git-commit", "--git-diff", "--git-since", + metavar="COMMIT", dest="git_since", help=git_help,) + + min_confidence_help = ("set the minimum confidence of style errors " + "to report. Can be an integer 1-5, with 1 " + "displaying all errors. Defaults to %default.") + parser.add_option("-m", "--min-confidence", metavar="INT", + type="int", dest="min_confidence", + default=default_min_confidence, + help=min_confidence_help) + + output_format_help = ('set the output format, which can be "emacs" ' + 'or "vs7" (for Visual Studio). ' + 'Defaults to "%default".') + parser.add_option("-o", "--output-format", metavar="FORMAT", + choices=["emacs", "vs7"], + dest="output_format", default=default_output_format, + help=output_format_help) + + verbose_help = "enable verbose logging." + parser.add_option("-v", "--verbose", dest="is_verbose", default=False, + action="store_true", help=verbose_help) + + # Override OptionParser's error() method so that option help will + # also display when an error occurs. Normally, just the usage + # string displays and not option help. + parser.error = self._parse_error + + # Override OptionParser's print_help() method so that help output + # does not render to the screen while running unit tests. + print_help = parser.print_help + parser.print_help = lambda: print_help(file=stderr) + + return parser + + def _parse_error(self, error_message): + """Print the help string and an error message, and exit.""" + # The method format_help() includes both the usage string and + # the flag options. + help = self._parser.format_help() + # Separate help from the error message with a single blank line. + self.stderr_write(help + "\n") if error_message: - sys.exit('\nFATAL ERROR: ' + error_message) - else: - sys.exit(1) + _log.error(error_message) + + # Since we are using this method to replace/override the Python + # module optparse's OptionParser.error() method, we match its + # behavior and exit with status code 2. + # + # As additional background, Python documentation says-- + # + # "Unix programs generally use 2 for command line syntax errors + # and 1 for all other kind of errors." + # + # (from http://docs.python.org/library/sys.html#sys.exit ) + sys.exit(2) def _exit_with_categories(self): """Exit and print the style categories and default filter rules.""" @@ -335,90 +391,67 @@ class ArgumentParser(object): filters.append(filter) return filters - def parse(self, args, extra_flags=None): + def parse(self, args): """Parse the command line arguments to check-webkit-style. Args: args: A list of command-line arguments as returned by sys.argv[1:]. - extra_flags: A list of flags whose values we want to extract, but - are not supported by the CommandOptionValues class. - An example flag "new_flag=". This defaults to the - empty list. Returns: - A tuple of (filenames, options) + A tuple of (paths, options) - filenames: The list of filenames to check. + paths: The list of paths to check. options: A CommandOptionValues instance. """ - if extra_flags is None: - extra_flags = [] - - output_format = self.default_options.output_format - verbosity = self.default_options.verbosity + (options, paths) = self._parser.parse_args(args=args) + + filter_value = options.filter_value + git_commit = options.git_since + is_verbose = options.is_verbose + min_confidence = options.min_confidence + output_format = options.output_format + + if filter_value is not None and not filter_value: + # Then the user explicitly passed no filter, for + # example "-f ''" or "--filter=". + self._exit_with_categories() + + # Validate user-provided values. + + if paths and git_commit: + self._parse_error('You cannot provide both paths and a git ' + 'commit at the same time.') + + # FIXME: Add unit tests. + if git_commit and '..' in git_commit: + # FIXME: If the range is a "...", the code should find the common + # ancestor and start there. See git diff --help for how + # "..." usually works. + self._parse_error('invalid --git-commit option: option does ' + 'not support ranges "..": %s' % git_commit) + + min_confidence = int(min_confidence) + if (min_confidence < 1) or (min_confidence > 5): + self._parse_error('option --min-confidence: invalid integer: ' + '%s: value must be between 1 and 5' + % min_confidence) + + if filter_value: + filter_rules = self._parse_filter_flag(filter_value) + else: + filter_rules = [] - # The flags already supported by the CommandOptionValues class. - flags = ['help', 'output=', 'verbose=', 'filter=', 'git-commit='] + try: + validate_filter_rules(filter_rules, self._all_categories) + except ValueError, err: + self._parse_error(err) - for extra_flag in extra_flags: - if extra_flag in flags: - raise ValueError('Flag \'%(extra_flag)s is duplicated ' - 'or already supported.' % - {'extra_flag': extra_flag}) - flags.append(extra_flag) + options = CommandOptionValues(filter_rules=filter_rules, + git_commit=git_commit, + is_verbose=is_verbose, + min_confidence=min_confidence, + output_format=output_format) - try: - (opts, filenames) = getopt.getopt(args, '', flags) - except getopt.GetoptError: - # FIXME: Settle on an error handling approach: come up - # with a consistent guideline as to when and whether - # a ValueError should be raised versus calling - # sys.exit when needing to interrupt execution. - self._exit_with_usage('Invalid arguments.') - - extra_flag_values = {} - git_commit = None - filter_rules = [] - - for (opt, val) in opts: - if opt == '--help': - self._exit_with_usage() - elif opt == '--output': - output_format = val - elif opt == '--verbose': - verbosity = val - elif opt == '--git-commit': - git_commit = val - elif opt == '--filter': - if not val: - self._exit_with_categories() - filter_rules = self._parse_filter_flag(val) - else: - extra_flag_values[opt] = val - - # Check validity of resulting values. - if filenames and (git_commit != None): - self._exit_with_usage('It is not possible to check files and a ' - 'specific commit at the same time.') - - if output_format not in ('emacs', 'vs7'): - raise ValueError('Invalid --output value "%s": The only ' - 'allowed output formats are emacs and vs7.' % - output_format) - - validate_filter_rules(filter_rules, self._all_categories) - - verbosity = int(verbosity) - if (verbosity < 1) or (verbosity > 5): - raise ValueError('Invalid --verbose value %s: value must ' - 'be between 1-5.' % verbosity) - - options = CommandOptionValues(extra_flag_values=extra_flag_values, - filter_rules=filter_rules, - git_commit=git_commit, - output_format=output_format, - verbosity=verbosity) - - return (filenames, options) + return (paths, options) diff --git a/WebKitTools/Scripts/webkitpy/style/optparser_unittest.py b/WebKitTools/Scripts/webkitpy/style/optparser_unittest.py index f23c5d1..1c525c6 100644 --- a/WebKitTools/Scripts/webkitpy/style/optparser_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/optparser_unittest.py @@ -24,22 +24,11 @@ import unittest -from optparser import _create_usage -from optparser import ArgumentParser -from optparser import ArgumentPrinter -from optparser import CommandOptionValues as ProcessorOptions -from optparser import DefaultCommandOptionValues - - -class CreateUsageTest(unittest.TestCase): - - """Tests the _create_usage() function.""" - - def test_create_usage(self): - default_options = DefaultCommandOptionValues(output_format="vs7", - verbosity=3) - # Exercise the code path to make sure the function does not error out. - _create_usage(default_options) +from webkitpy.common.system.logtesting import LoggingTestCase +from webkitpy.style.optparser import ArgumentParser +from webkitpy.style.optparser import ArgumentPrinter +from webkitpy.style.optparser import CommandOptionValues as ProcessorOptions +from webkitpy.style.optparser import DefaultCommandOptionValues class ArgumentPrinterTest(unittest.TestCase): @@ -50,61 +39,65 @@ class ArgumentPrinterTest(unittest.TestCase): def _create_options(self, output_format='emacs', - verbosity=3, + min_confidence=3, filter_rules=[], - git_commit=None, - extra_flag_values={}): - return ProcessorOptions(extra_flag_values=extra_flag_values, - filter_rules=filter_rules, + git_commit=None): + return ProcessorOptions(filter_rules=filter_rules, git_commit=git_commit, - output_format=output_format, - verbosity=verbosity) + min_confidence=min_confidence, + output_format=output_format) def test_to_flag_string(self): - options = self._create_options('vs7', 5, ['+foo', '-bar'], 'git', - {'a': 0, 'z': 1}) - self.assertEquals('--a=0 --filter=+foo,-bar --git-commit=git ' - '--output=vs7 --verbose=5 --z=1', + options = self._create_options('vs7', 5, ['+foo', '-bar'], 'git') + self.assertEquals('--filter=+foo,-bar --git-commit=git ' + '--min-confidence=5 --output=vs7', self._printer.to_flag_string(options)) # This is to check that --filter and --git-commit do not # show up when not user-specified. options = self._create_options() - self.assertEquals('--output=emacs --verbose=3', + self.assertEquals('--min-confidence=3 --output=emacs', self._printer.to_flag_string(options)) -class ArgumentParserTest(unittest.TestCase): +class ArgumentParserTest(LoggingTestCase): """Test the ArgumentParser class.""" - def _parse(self): - """Return a default parse() function for testing.""" - return self._create_parser().parse + class _MockStdErr(object): + + def write(self, message): + # We do not want the usage string or style categories + # to print during unit tests, so print nothing. + return + + def _parse(self, args): + """Call a test parser.parse().""" + parser = self._create_parser() + return parser.parse(args) def _create_defaults(self): """Return a DefaultCommandOptionValues instance for testing.""" base_filter_rules = ["-", "+whitespace"] - return DefaultCommandOptionValues(output_format="vs7", - verbosity=3) + return DefaultCommandOptionValues(min_confidence=3, + output_format="vs7") def _create_parser(self): """Return an ArgumentParser instance for testing.""" - def stderr_write(message): - # We do not want the usage string or style categories - # to print during unit tests, so print nothing. - return - default_options = self._create_defaults() all_categories = ["build" ,"whitespace"] + + mock_stderr = self._MockStdErr() + return ArgumentParser(all_categories=all_categories, base_filter_rules=[], default_options=default_options, - stderr_write=stderr_write) + mock_stderr=mock_stderr, + usage="test usage") def test_parse_documentation(self): - parse = self._parse() + parse = self._parse # FIXME: Test both the printing of the usage string and the # filter categories help. @@ -115,56 +108,75 @@ class ArgumentParserTest(unittest.TestCase): self.assertRaises(SystemExit, parse, ['--filter=']) def test_parse_bad_values(self): - parse = self._parse() + parse = self._parse # Pass an unsupported argument. self.assertRaises(SystemExit, parse, ['--bad']) - - self.assertRaises(ValueError, parse, ['--verbose=bad']) - self.assertRaises(ValueError, parse, ['--verbose=0']) - self.assertRaises(ValueError, parse, ['--verbose=6']) - parse(['--verbose=1']) # works - parse(['--verbose=5']) # works - - self.assertRaises(ValueError, parse, ['--output=bad']) + self.assertLog(['ERROR: no such option: --bad\n']) + + self.assertRaises(SystemExit, parse, ['--git-diff=aa..bb']) + self.assertLog(['ERROR: invalid --git-commit option: ' + 'option does not support ranges "..": aa..bb\n']) + + self.assertRaises(SystemExit, parse, ['--min-confidence=bad']) + self.assertLog(['ERROR: option --min-confidence: ' + "invalid integer value: 'bad'\n"]) + self.assertRaises(SystemExit, parse, ['--min-confidence=0']) + self.assertLog(['ERROR: option --min-confidence: invalid integer: 0: ' + 'value must be between 1 and 5\n']) + self.assertRaises(SystemExit, parse, ['--min-confidence=6']) + self.assertLog(['ERROR: option --min-confidence: invalid integer: 6: ' + 'value must be between 1 and 5\n']) + parse(['--min-confidence=1']) # works + parse(['--min-confidence=5']) # works + + self.assertRaises(SystemExit, parse, ['--output=bad']) + self.assertLog(['ERROR: option --output-format: invalid choice: ' + "'bad' (choose from 'emacs', 'vs7')\n"]) parse(['--output=vs7']) # works # Pass a filter rule not beginning with + or -. - self.assertRaises(ValueError, parse, ['--filter=build']) + self.assertRaises(SystemExit, parse, ['--filter=build']) + self.assertLog(['ERROR: Invalid filter rule "build": ' + 'every rule must start with + or -.\n']) parse(['--filter=+build']) # works # Pass files and git-commit at the same time. - self.assertRaises(SystemExit, parse, ['--git-commit=', 'file.txt']) - # Pass an extra flag already supported. - self.assertRaises(ValueError, parse, [], ['filter=']) - parse([], ['extra=']) # works - # Pass an extra flag with typo. - self.assertRaises(SystemExit, parse, ['--extratypo='], ['extra=']) - parse(['--extra='], ['extra=']) # works - self.assertRaises(ValueError, parse, [], ['extra=', 'extra=']) - + self.assertRaises(SystemExit, parse, ['--git-commit=committish', + 'file.txt']) + self.assertLog(['ERROR: You cannot provide both paths and ' + 'a git commit at the same time.\n']) def test_parse_default_arguments(self): - parse = self._parse() + parse = self._parse (files, options) = parse([]) self.assertEquals(files, []) - self.assertEquals(options.output_format, 'vs7') - self.assertEquals(options.verbosity, 3) self.assertEquals(options.filter_rules, []) self.assertEquals(options.git_commit, None) + self.assertEquals(options.is_verbose, False) + self.assertEquals(options.min_confidence, 3) + self.assertEquals(options.output_format, 'vs7') def test_parse_explicit_arguments(self): - parse = self._parse() + parse = self._parse # Pass non-default explicit values. + (files, options) = parse(['--min-confidence=4']) + self.assertEquals(options.min_confidence, 4) (files, options) = parse(['--output=emacs']) self.assertEquals(options.output_format, 'emacs') - (files, options) = parse(['--verbose=4']) - self.assertEquals(options.verbosity, 4) + (files, options) = parse(['-g', 'commit']) + self.assertEquals(options.git_commit, 'commit') (files, options) = parse(['--git-commit=commit']) self.assertEquals(options.git_commit, 'commit') + (files, options) = parse(['--git-diff=commit']) + self.assertEquals(options.git_commit, 'commit') + (files, options) = parse(['--git-since=commit']) + self.assertEquals(options.git_commit, 'commit') + (files, options) = parse(['--verbose']) + self.assertEquals(options.is_verbose, True) # Pass user_rules. (files, options) = parse(['--filter=+build,-whitespace']) @@ -176,16 +188,8 @@ class ArgumentParserTest(unittest.TestCase): self.assertEquals(options.filter_rules, ["+build", "-whitespace"]) - # Pass extra flag values. - (files, options) = parse(['--extra'], ['extra']) - self.assertEquals(options.extra_flag_values, {'--extra': ''}) - (files, options) = parse(['--extra='], ['extra=']) - self.assertEquals(options.extra_flag_values, {'--extra': ''}) - (files, options) = parse(['--extra=x'], ['extra=']) - self.assertEquals(options.extra_flag_values, {'--extra': 'x'}) - def test_parse_files(self): - parse = self._parse() + parse = self._parse (files, options) = parse(['foo.cpp']) self.assertEquals(files, ['foo.cpp']) @@ -203,56 +207,60 @@ class CommandOptionValuesTest(unittest.TestCase): """Test __init__ constructor.""" # Check default parameters. options = ProcessorOptions() - self.assertEquals(options.extra_flag_values, {}) self.assertEquals(options.filter_rules, []) self.assertEquals(options.git_commit, None) + self.assertEquals(options.is_verbose, False) + self.assertEquals(options.min_confidence, 1) self.assertEquals(options.output_format, "emacs") - self.assertEquals(options.verbosity, 1) # Check argument validation. self.assertRaises(ValueError, ProcessorOptions, output_format="bad") ProcessorOptions(output_format="emacs") # No ValueError: works ProcessorOptions(output_format="vs7") # works - self.assertRaises(ValueError, ProcessorOptions, verbosity=0) - self.assertRaises(ValueError, ProcessorOptions, verbosity=6) - ProcessorOptions(verbosity=1) # works - ProcessorOptions(verbosity=5) # works + self.assertRaises(ValueError, ProcessorOptions, min_confidence=0) + self.assertRaises(ValueError, ProcessorOptions, min_confidence=6) + ProcessorOptions(min_confidence=1) # works + ProcessorOptions(min_confidence=5) # works # Check attributes. - options = ProcessorOptions(extra_flag_values={"extra_value" : 2}, - filter_rules=["+"], + options = ProcessorOptions(filter_rules=["+"], git_commit="commit", - output_format="vs7", - verbosity=3) - self.assertEquals(options.extra_flag_values, {"extra_value" : 2}) + is_verbose=True, + min_confidence=3, + output_format="vs7") self.assertEquals(options.filter_rules, ["+"]) self.assertEquals(options.git_commit, "commit") + self.assertEquals(options.is_verbose, True) + self.assertEquals(options.min_confidence, 3) self.assertEquals(options.output_format, "vs7") - self.assertEquals(options.verbosity, 3) def test_eq(self): """Test __eq__ equality function.""" - # == calls __eq__. - self.assertTrue(ProcessorOptions() == ProcessorOptions()) - - # Verify that a difference in any argument causes equality to fail. - options = ProcessorOptions(extra_flag_values={"extra_value" : 1}, - filter_rules=["+"], - git_commit="commit", - output_format="vs7", - verbosity=1) - self.assertFalse(options == ProcessorOptions(extra_flag_values= - {"extra_value" : 2})) - self.assertFalse(options == ProcessorOptions(filter_rules=["-"])) - self.assertFalse(options == ProcessorOptions(git_commit="commit2")) - self.assertFalse(options == ProcessorOptions(output_format="emacs")) - self.assertFalse(options == ProcessorOptions(verbosity=2)) + self.assertTrue(ProcessorOptions().__eq__(ProcessorOptions())) + + # Also verify that a difference in any argument causes equality to fail. + + # Explicitly create a ProcessorOptions instance with all default + # values. We do this to be sure we are assuming the right default + # values in our self.assertFalse() calls below. + options = ProcessorOptions(filter_rules=[], + git_commit=None, + is_verbose=False, + min_confidence=1, + output_format="emacs") + # Verify that we created options correctly. + self.assertTrue(options.__eq__(ProcessorOptions())) + + self.assertFalse(options.__eq__(ProcessorOptions(filter_rules=["+"]))) + self.assertFalse(options.__eq__(ProcessorOptions(git_commit="commit"))) + self.assertFalse(options.__eq__(ProcessorOptions(is_verbose=True))) + self.assertFalse(options.__eq__(ProcessorOptions(min_confidence=2))) + self.assertFalse(options.__eq__(ProcessorOptions(output_format="vs7"))) def test_ne(self): """Test __ne__ inequality function.""" - # != calls __ne__. # By default, __ne__ always returns true on different objects. # Thus, just check the distinguishing case to verify that the # code defines __ne__. - self.assertFalse(ProcessorOptions() != ProcessorOptions()) + self.assertFalse(ProcessorOptions().__ne__(ProcessorOptions())) diff --git a/WebKitTools/Scripts/webkitpy/style/processors/common.py b/WebKitTools/Scripts/webkitpy/style/processors/common.py index dbf4bea..30b8fed 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/common.py +++ b/WebKitTools/Scripts/webkitpy/style/processors/common.py @@ -20,7 +20,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Supports style checking not specific to any one processor.""" +"""Supports style checking not specific to any one file type.""" # FIXME: Test this list in the same way that the list of CppProcessor @@ -33,27 +33,25 @@ categories = set([ ]) -def check_no_carriage_return(line, line_number, error): - """Check that a line does not end with a carriage return. +class CarriageReturnProcessor(object): - Returns true if the check is successful (i.e. if the line does not - end with a carriage return), and false otherwise. + """Supports checking for and handling carriage returns.""" - Args: - line: A string that is the line to check. - line_number: The line number. - error: The function to call with any errors found. + def __init__(self, handle_style_error): + self._handle_style_error = handle_style_error - """ + def process(self, lines): + """Check for and strip trailing carriage returns from lines.""" + for line_number in range(len(lines)): + if not lines[line_number].endswith("\r"): + continue - if line.endswith("\r"): - error(line_number, - "whitespace/carriage_return", - 1, - "One or more unexpected \\r (^M) found; " - "better to use only a \\n") - return False - - return True + self._handle_style_error(line_number + 1, # Correct for offset. + "whitespace/carriage_return", + 1, + "One or more unexpected \\r (^M) found; " + "better to use only a \\n") + lines[line_number] = lines[line_number].rstrip("\r") + return lines diff --git a/WebKitTools/Scripts/webkitpy/style/processors/common_unittest.py b/WebKitTools/Scripts/webkitpy/style/processors/common_unittest.py index 9362b65..3dde7b9 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/common_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/processors/common_unittest.py @@ -22,10 +22,9 @@ """Unit tests for common.py.""" - import unittest -from common import check_no_carriage_return +from common import CarriageReturnProcessor # FIXME: The unit tests for the cpp, text, and common processors should @@ -33,13 +32,15 @@ from common import check_no_carriage_return # mock style error handling code and the code to check that all # of a processor's categories are covered by the unit tests. # Such shared code can be located in a shared test file, perhaps -# ilke this one. -class CarriageReturnTest(unittest.TestCase): +# even this file. +class CarriageReturnProcessorTest(unittest.TestCase): """Tests check_no_carriage_return().""" _category = "whitespace/carriage_return" _confidence = 1 + _expected_message = ("One or more unexpected \\r (^M) found; " + "better to use only a \\n") def setUp(self): self._style_errors = [] # The list of accumulated style errors. @@ -50,33 +51,44 @@ class CarriageReturnTest(unittest.TestCase): error = (line_number, category, confidence, message) self._style_errors.append(error) - def assert_carriage_return(self, line, is_error): - """Call check_no_carriage_return() and assert the result.""" - line_number = 100 + def assert_carriage_return(self, input_lines, expected_lines, error_lines): + """Process the given line and assert that the result is correct.""" handle_style_error = self._mock_style_error_handler - check_no_carriage_return(line, line_number, handle_style_error) + processor = CarriageReturnProcessor(handle_style_error) + output_lines = processor.process(input_lines) - expected_message = ("One or more unexpected \\r (^M) found; " - "better to use only a \\n") + # Check both the return value and error messages. + self.assertEquals(output_lines, expected_lines) - if is_error: - expected_errors = [(line_number, self._category, self._confidence, - expected_message)] - self.assertEquals(self._style_errors, expected_errors) - else: - self.assertEquals(self._style_errors, []) + expected_errors = [(line_number, self._category, self._confidence, + self._expected_message) + for line_number in error_lines] + self.assertEquals(self._style_errors, expected_errors) def test_ends_with_carriage(self): - self.assert_carriage_return("carriage return\r", is_error=True) + self.assert_carriage_return(["carriage return\r"], + ["carriage return"], + [1]) def test_ends_with_nothing(self): - self.assert_carriage_return("no carriage return", is_error=False) + self.assert_carriage_return(["no carriage return"], + ["no carriage return"], + []) def test_ends_with_newline(self): - self.assert_carriage_return("no carriage return\n", is_error=False) - - def test_ends_with_carriage_newline(self): - # Check_no_carriage_return only() checks the final character. - self.assert_carriage_return("carriage\r in a string", is_error=False) - + self.assert_carriage_return(["no carriage return\n"], + ["no carriage return\n"], + []) + + def test_carriage_in_middle(self): + # The CarriageReturnProcessor checks only the final character + # of each line. + self.assert_carriage_return(["carriage\r in a string"], + ["carriage\r in a string"], + []) + + def test_multiple_errors(self): + self.assert_carriage_return(["line1", "line2\r", "line3\r"], + ["line1", "line2", "line3"], + [2, 3]) diff --git a/WebKitTools/Scripts/webkitpy/style/processors/cpp.py b/WebKitTools/Scripts/webkitpy/style/processors/cpp.py index f83ae6a..23be9f9 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/cpp.py +++ b/WebKitTools/Scripts/webkitpy/style/processors/cpp.py @@ -272,18 +272,18 @@ class _FunctionState(object): """Tracks current function name and the number of lines in its body. Attributes: - verbosity: The verbosity level to use while checking style. + min_confidence: The minimum confidence level to use while checking style. """ _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. - def __init__(self, verbosity): - self.verbosity = verbosity + def __init__(self, min_confidence): + self.min_confidence = min_confidence + self.current_function = '' self.in_a_function = False self.lines_in_function = 0 - self.current_function = '' def begin(self, function_name): """Start analyzing function body. @@ -311,7 +311,7 @@ class _FunctionState(object): base_trigger = self._TEST_TRIGGER else: base_trigger = self._NORMAL_TRIGGER - trigger = base_trigger * 2 ** self.verbosity + trigger = base_trigger * 2 ** self.min_confidence if self.lines_in_function > trigger: error_level = int(math.log(self.lines_in_function / base_trigger, 2)) @@ -642,6 +642,10 @@ def get_header_guard_cpp_variable(filename): """ + # Restores original filename in case that style checker is invoked from Emacs's + # flymake. + filename = re.sub(r'_flymake\.h$', '.h', filename) + return sub(r'[-.\s]', '_', os.path.basename(filename)) @@ -1367,7 +1371,7 @@ def check_spacing(file_extension, clean_lines, line_number, error): if file_extension == 'cpp': # C++ should have the & or * beside the type not the variable name. - matched = match(r'\s*\w+(?<!\breturn)\s+(?P<pointer_operator>\*|\&)\w+', line) + matched = match(r'\s*\w+(?<!\breturn|\bdelete)\s+(?P<pointer_operator>\*|\&)\w+', line) if matched: error(line_number, 'whitespace/declaration', 3, 'Declaration has space between type name and %s in %s' % (matched.group('pointer_operator'), matched.group(0).strip())) @@ -1652,7 +1656,7 @@ def check_braces(clean_lines, line_number, error): # We check if a closed brace has started a line to see if a # one line control statement was previous. previous_line = clean_lines.elided[line_number - 2] - if (previous_line.find('{') > 0 + if (previous_line.find('{') > 0 and previous_line.find('}') < 0 and search(r'\b(if|for|foreach|while|else)\b', previous_line)): error(line_number, 'whitespace/braces', 4, 'One line control clauses should not use braces.') @@ -2769,9 +2773,7 @@ def check_for_include_what_you_use(filename, clean_lines, include_state, error, # found. # e.g. If the file name is 'foo_flymake.cpp', we should search for 'foo.h' # instead of 'foo_flymake.h' - emacs_flymake_suffix = '_flymake.cpp' - if abs_filename.endswith(emacs_flymake_suffix): - abs_filename = abs_filename[:-len(emacs_flymake_suffix)] + '.cpp' + abs_filename = re.sub(r'_flymake\.cpp$', '.cpp', abs_filename) # include_state is modified during iteration, so we iterate over a copy of # the keys. @@ -2836,7 +2838,7 @@ def process_line(filename, file_extension, check_invalid_increment(clean_lines, line, error) -def _process_lines(filename, file_extension, lines, error, verbosity): +def _process_lines(filename, file_extension, lines, error, min_confidence): """Performs lint checks and reports any errors to the given error function. Args: @@ -2850,7 +2852,7 @@ def _process_lines(filename, file_extension, lines, error, verbosity): ['// marker so line numbers end in a known way']) include_state = _IncludeState() - function_state = _FunctionState(verbosity) + function_state = _FunctionState(min_confidence) class_state = _ClassState() file_state = _FileState() @@ -2948,7 +2950,8 @@ class CppProcessor(object): 'whitespace/todo', ]) - def __init__(self, file_path, file_extension, handle_style_error, verbosity): + def __init__(self, file_path, file_extension, handle_style_error, + min_confidence): """Create a CppProcessor instance. Args: @@ -2959,7 +2962,7 @@ class CppProcessor(object): self.file_extension = file_extension self.file_path = file_path self.handle_style_error = handle_style_error - self.verbosity = verbosity + self.min_confidence = min_confidence # Useful for unit testing. def __eq__(self, other): @@ -2970,7 +2973,7 @@ class CppProcessor(object): return False if self.handle_style_error != other.handle_style_error: return False - if self.verbosity != other.verbosity: + if self.min_confidence != other.min_confidence: return False return True @@ -2982,10 +2985,10 @@ class CppProcessor(object): def process(self, lines): _process_lines(self.file_path, self.file_extension, lines, - self.handle_style_error, self.verbosity) + self.handle_style_error, self.min_confidence) # FIXME: Remove this function (requires refactoring unit tests). -def process_file_data(filename, file_extension, lines, error, verbosity): - processor = CppProcessor(filename, file_extension, error, verbosity) +def process_file_data(filename, file_extension, lines, error, min_confidence): + processor = CppProcessor(filename, file_extension, error, min_confidence) processor.process(lines) diff --git a/WebKitTools/Scripts/webkitpy/style/processors/cpp_unittest.py b/WebKitTools/Scripts/webkitpy/style/processors/cpp_unittest.py index c786b8e..0a3fe08 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/cpp_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/processors/cpp_unittest.py @@ -119,20 +119,21 @@ class CppStyleTestBase(unittest.TestCase): """Provides some useful helper functions for cpp_style tests. Attributes: - verbosity: An integer that is the current verbosity level for - the tests. + min_confidence: An integer that is the current minimum confidence + level for the tests. """ - # FIXME: Refactor the unit tests so the verbosity level is passed + # FIXME: Refactor the unit tests so the confidence level is passed # explicitly, just like it is in the real code. - verbosity = 1; + min_confidence = 1; - # Helper function to avoid needing to explicitly pass verbosity + # Helper function to avoid needing to explicitly pass confidence # in all the unit test calls to cpp_style.process_file_data(). def process_file_data(self, filename, file_extension, lines, error): - """Call cpp_style.process_file_data() with the current verbosity.""" - return cpp_style.process_file_data(filename, file_extension, lines, error, self.verbosity) + """Call cpp_style.process_file_data() with the min_confidence.""" + return cpp_style.process_file_data(filename, file_extension, lines, + error, self.min_confidence) # Perform lint on single line of input and return the error message. def perform_single_line_lint(self, code, file_name): @@ -141,7 +142,7 @@ class CppStyleTestBase(unittest.TestCase): cpp_style.remove_multi_line_comments(lines, error_collector) clean_lines = cpp_style.CleansedLines(lines) include_state = cpp_style._IncludeState() - function_state = cpp_style._FunctionState(self.verbosity) + function_state = cpp_style._FunctionState(self.min_confidence) ext = file_name[file_name.rfind('.') + 1:] class_state = cpp_style._ClassState() file_state = cpp_style._FileState() @@ -199,7 +200,7 @@ class CppStyleTestBase(unittest.TestCase): The accumulated errors. """ error_collector = ErrorCollector(self.assert_) - function_state = cpp_style._FunctionState(self.verbosity) + function_state = cpp_style._FunctionState(self.min_confidence) lines = code.split('\n') cpp_style.remove_multi_line_comments(lines, error_collector) lines = cpp_style.CleansedLines(lines) @@ -238,7 +239,7 @@ class CppStyleTestBase(unittest.TestCase): if re.search(expected_message_re, message): return - self.assertEquals(expected_message, messages) + self.assertEquals(expected_message_re, messages) def assert_multi_line_lint(self, code, expected_message, file_name='foo.h'): file_extension = file_name[file_name.rfind('.') + 1:] @@ -1547,6 +1548,7 @@ class CppStyleTest(CppStyleTestBase): 'Declaration has space between type name and * in int *b [whitespace/declaration] [3]', 'foo.cpp') self.assert_lint('return *b;', '', 'foo.cpp') + self.assert_lint('delete *b;', '', 'foo.cpp') self.assert_lint('int *b;', '', 'foo.c') self.assert_lint('int* b;', 'Declaration has space between * and variable name in int* b [whitespace/declaration] [3]', @@ -1734,6 +1736,26 @@ class CppStyleTest(CppStyleTestBase): ' [build/header_guard] [5]' % expected_guard), error_collector.result_list()) + # Special case for flymake + error_collector = ErrorCollector(self.assert_) + self.process_file_data('mydir/Foo_flymake.h', 'h', + ['#ifndef %s' % expected_guard, + '#define %s' % expected_guard, + '#endif // %s' % expected_guard], + error_collector) + for line in error_collector.result_list(): + if line.find('build/header_guard') != -1: + self.fail('Unexpected error: %s' % line) + + error_collector = ErrorCollector(self.assert_) + self.process_file_data('mydir/Foo_flymake.h', 'h', [], error_collector) + self.assertEquals( + 1, + error_collector.result_list().count( + 'No #ifndef header guard found, suggested CPP variable is: %s' + ' [build/header_guard] [5]' % expected_guard), + error_collector.result_list()) + def test_build_printf_format(self): self.assert_lint( r'printf("\%%d", value);', @@ -2227,11 +2249,11 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): cpp_style._FunctionState._TEST_TRIGGER = self.old_test_trigger # FIXME: Eliminate the need for this function. - def set_verbosity(self, verbosity): - """Set new test verbosity and return old test verbosity.""" - old_verbosity = self.verbosity - self.verbosity = verbosity - return old_verbosity + def set_min_confidence(self, min_confidence): + """Set new test confidence and return old test confidence.""" + old_min_confidence = self.min_confidence + self.min_confidence = min_confidence + return old_min_confidence def assert_function_lengths_check(self, code, expected_message): """Check warnings for long function bodies are as expected. @@ -2272,7 +2294,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): lines: Number of lines to generate. error_level: --v setting for cpp_style. """ - trigger_level = self.trigger_lines(self.verbosity) + trigger_level = self.trigger_lines(self.min_confidence) self.assert_function_lengths_check( 'void test(int x)' + self.function_body(lines), ('Small and focused functions are preferred: ' @@ -2355,29 +2377,29 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): '') def test_function_length_check_definition_below_severity0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_definition_ok(self.trigger_lines(0) - 1) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_at_severity0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_definition_ok(self.trigger_lines(0)) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_above_severity0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_above_error_level(0) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_below_severity1v0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_below_error_level(1) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_at_severity1v0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_at_error_level(1) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_below_severity1(self): self.assert_function_length_check_definition_ok(self.trigger_lines(1) - 1) @@ -2391,7 +2413,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_plus_blanks(self): error_level = 1 error_lines = self.trigger_lines(error_level) + 1 - trigger_level = self.trigger_lines(self.verbosity) + trigger_level = self.trigger_lines(self.min_confidence) self.assert_function_lengths_check( 'void test_blanks(int x)' + self.function_body(error_lines), ('Small and focused functions are preferred: ' @@ -2403,7 +2425,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_complex_definition_severity1(self): error_level = 1 error_lines = self.trigger_lines(error_level) + 1 - trigger_level = self.trigger_lines(self.verbosity) + trigger_level = self.trigger_lines(self.min_confidence) self.assert_function_lengths_check( ('my_namespace::my_other_namespace::MyVeryLongTypeName*\n' 'my_namespace::my_other_namespace::MyFunction(int arg1, char* arg2)' @@ -2418,7 +2440,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_for_test(self): error_level = 1 error_lines = self.trigger_test_lines(error_level) + 1 - trigger_level = self.trigger_test_lines(self.verbosity) + trigger_level = self.trigger_test_lines(self.min_confidence) self.assert_function_lengths_check( 'TEST_F(Test, Mutator)' + self.function_body(error_lines), ('Small and focused functions are preferred: ' @@ -2430,7 +2452,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_for_split_line_test(self): error_level = 1 error_lines = self.trigger_test_lines(error_level) + 1 - trigger_level = self.trigger_test_lines(self.verbosity) + trigger_level = self.trigger_test_lines(self.min_confidence) self.assert_function_lengths_check( ('TEST_F(GoogleUpdateRecoveryRegistryProtectedTest,\n' ' FixGoogleUpdate_AllValues_MachineApp)' # note: 4 spaces @@ -2445,7 +2467,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_for_bad_test_doesnt_break(self): error_level = 1 error_lines = self.trigger_test_lines(error_level) + 1 - trigger_level = self.trigger_test_lines(self.verbosity) + trigger_level = self.trigger_test_lines(self.min_confidence) self.assert_function_lengths_check( ('TEST_F(' + self.function_body(error_lines)), @@ -2458,7 +2480,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_with_embedded_no_lints(self): error_level = 1 error_lines = self.trigger_lines(error_level) + 1 - trigger_level = self.trigger_lines(self.verbosity) + trigger_level = self.trigger_lines(self.min_confidence) self.assert_function_lengths_check( 'void test(int x)' + self.function_body_with_no_lints(error_lines), ('Small and focused functions are preferred: ' @@ -3063,6 +3085,20 @@ class WebKitStyleTest(CppStyleTestBase): '}\n', ['More than one command on the same line in if [whitespace/parens] [4]', 'One line control clauses should not use braces. [whitespace/braces] [4]']) + self.assert_multi_line_lint( + 'void func()\n' + '{\n' + ' while (condition) { }\n' + ' return 0;\n' + '}\n', + '') + self.assert_multi_line_lint( + 'void func()\n' + '{\n' + ' for (i = 0; i < 42; i++) { foobar(); }\n' + ' return 0;\n' + '}\n', + 'More than one command on the same line in for [whitespace/parens] [4]') # 3. An else if statement should be written as an if statement # when the prior if concludes with a return statement. @@ -3653,7 +3689,7 @@ class CppProcessorTest(unittest.TestCase): self.assertEquals(processor.file_extension, "h") self.assertEquals(processor.file_path, "foo") self.assertEquals(processor.handle_style_error, self.mock_handle_style_error) - self.assertEquals(processor.verbosity, 3) + self.assertEquals(processor.min_confidence, 3) def test_eq(self): """Test __eq__ equality function.""" diff --git a/WebKitTools/Scripts/webkitpy/style/processors/python.py b/WebKitTools/Scripts/webkitpy/style/processors/python.py new file mode 100644 index 0000000..8ab936d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/processors/python.py @@ -0,0 +1,56 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports checking WebKit style in Python files.""" + +from ...style_references import pep8 + + +class PythonProcessor(object): + + """Processes text lines for checking style.""" + + def __init__(self, file_path, handle_style_error): + self._file_path = file_path + self._handle_style_error = handle_style_error + + def process(self, lines): + # Initialize pep8.options, which is necessary for + # Checker.check_all() to execute. + pep8.process_options(arglist=[self._file_path]) + + checker = pep8.Checker(self._file_path) + + def _pep8_handle_error(line_number, offset, text, check): + # FIXME: Incorporate the character offset into the error output. + # This will require updating the error handler __call__ + # signature to include an optional "offset" parameter. + pep8_code = text[:4] + pep8_message = text[5:] + + category = "pep8/" + pep8_code + + self._handle_style_error(line_number, category, 5, pep8_message) + + checker.report_error = _pep8_handle_error + + errors = checker.check_all() diff --git a/WebKitTools/Scripts/webkitpy/style/processors/python_unittest.py b/WebKitTools/Scripts/webkitpy/style/processors/python_unittest.py new file mode 100644 index 0000000..3ce3311 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/processors/python_unittest.py @@ -0,0 +1,62 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for python.py.""" + +import os +import unittest + +from python import PythonProcessor + + +class PythonProcessorTest(unittest.TestCase): + + """Tests the PythonProcessor class.""" + + def test_init(self): + """Test __init__() method.""" + def _mock_handle_style_error(self): + pass + + processor = PythonProcessor("foo.txt", _mock_handle_style_error) + self.assertEquals(processor._file_path, "foo.txt") + self.assertEquals(processor._handle_style_error, + _mock_handle_style_error) + + def test_process(self): + """Test process() method.""" + errors = [] + + def _mock_handle_style_error(line_number, category, confidence, + message): + error = (line_number, category, confidence, message) + errors.append(error) + + current_dir = os.path.dirname(__file__) + file_path = os.path.join(current_dir, "python_unittest_input.py") + + processor = PythonProcessor(file_path, _mock_handle_style_error) + processor.process(lines=[]) + + self.assertEquals(len(errors), 1) + self.assertEquals(errors[0], + (2, "pep8/W291", 5, "trailing whitespace")) diff --git a/WebKitTools/Scripts/webkitpy/style/processors/python_unittest_input.py b/WebKitTools/Scripts/webkitpy/style/processors/python_unittest_input.py new file mode 100644 index 0000000..9f1d118 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/processors/python_unittest_input.py @@ -0,0 +1,2 @@ +# This file is sample input for python_unittest.py and includes a single +# error which is an extra space at the end of this line. diff --git a/WebKitTools/Scripts/webkitpy/style/unittests.py b/WebKitTools/Scripts/webkitpy/style/unittests.py deleted file mode 100644 index 62615ab..0000000 --- a/WebKitTools/Scripts/webkitpy/style/unittests.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Apple Computer, Inc. ("Apple") nor the names of -# its contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Runs style package unit tests.""" - -# This module is imported by test-webkitpy. - -import sys -import unittest - -from checker_unittest import * -from error_handlers_unittest import * -from filter_unittest import * -from optparser_unittest import * -from processors.common_unittest import * -from processors.cpp_unittest import * -from processors.text_unittest import * diff --git a/WebKitTools/Scripts/webkitpy/style_references.py b/WebKitTools/Scripts/webkitpy/style_references.py index 2528c4d..ba2806e 100644 --- a/WebKitTools/Scripts/webkitpy/style_references.py +++ b/WebKitTools/Scripts/webkitpy/style_references.py @@ -40,8 +40,20 @@ import os -from diff_parser import DiffParser -from scm import detect_scm_system +from webkitpy.common.checkout.diff_parser import DiffParser +from webkitpy.common.system.logtesting import LogTesting +from webkitpy.common.system.logtesting import TestLogStream +from webkitpy.common.system.logutils import configure_logging +from webkitpy.common.checkout.scm import detect_scm_system +from webkitpy.thirdparty.autoinstalled import pep8 + + +def detect_checkout(): + """Return a WebKitCheckout instance, or None if it cannot be found.""" + cwd = os.path.abspath(os.curdir) + scm = detect_scm_system(cwd) + + return None if scm is None else WebKitCheckout(scm) def parse_patch(patch_string): @@ -52,16 +64,15 @@ def parse_patch(patch_string): return patch.files -class SimpleScm(object): +class WebKitCheckout(object): - """Simple facade to SCM for use by style package.""" + """Simple facade to the SCM class for use by style package.""" - def __init__(self): - cwd = os.path.abspath('.') - self._scm = detect_scm_system(cwd) + def __init__(self, scm): + self._scm = scm - def checkout_root(self): - """Return the source control root as an absolute path.""" + def root_path(self): + """Return the checkout root as an absolute path.""" return self._scm.checkout_root def create_patch(self): diff --git a/WebKitTools/Scripts/webkitpy/test/__init__.py b/WebKitTools/Scripts/webkitpy/test/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/test/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/test/main.py b/WebKitTools/Scripts/webkitpy/test/main.py new file mode 100644 index 0000000..daf255f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/test/main.py @@ -0,0 +1,129 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains the entry method for test-webkitpy.""" + +import logging +import os +import sys +import unittest + +import webkitpy + + +_log = logging.getLogger(__name__) + + +class Tester(object): + + """Discovers and runs webkitpy unit tests.""" + + def _find_unittest_files(self, webkitpy_dir): + """Return a list of paths to all unit-test files.""" + unittest_paths = [] # Return value. + + for dir_path, dir_names, file_names in os.walk(webkitpy_dir): + for file_name in file_names: + if not file_name.endswith("_unittest.py"): + continue + unittest_path = os.path.join(dir_path, file_name) + unittest_paths.append(unittest_path) + + return unittest_paths + + def _modules_from_paths(self, webkitpy_dir, paths): + """Return a list of fully-qualified module names given paths.""" + webkitpy_dir = os.path.abspath(webkitpy_dir) + webkitpy_name = os.path.split(webkitpy_dir)[1] # Equals "webkitpy". + + prefix_length = len(webkitpy_dir) + + modules = [] + for path in paths: + path = os.path.abspath(path) + # This gives us, for example: /common/config/ports_unittest.py + rel_path = path[prefix_length:] + # This gives us, for example: /common/config/ports_unittest + rel_path = os.path.splitext(rel_path)[0] + + parts = [] + while True: + (rel_path, tail) = os.path.split(rel_path) + if not tail: + break + parts.insert(0, tail) + # We now have, for example: common.config.ports_unittest + parts.insert(0, webkitpy_name) # Put "webkitpy" at the beginning. + module = ".".join(parts) + modules.append(module) + + return modules + + def run_tests(self, sys_argv): + """Run the unit tests in all *_unittest.py modules in webkitpy. + + This method excludes "webkitpy.common.checkout.scm_unittest" unless + the --all option is the second element of sys_argv. + + Args: + sys_argv: A reference to sys.argv. + + """ + if len(sys_argv) > 1 and not sys_argv[-1].startswith("-"): + # Then explicit modules or test names were provided, which + # the unittest module is equipped to handle. + unittest.main(argv=sys_argv, module=None) + # No need to return since unitttest.main() exits. + + # Otherwise, auto-detect all unit tests. + + webkitpy_dir = os.path.dirname(webkitpy.__file__) + unittest_paths = self._find_unittest_files(webkitpy_dir) + + modules = self._modules_from_paths(webkitpy_dir, unittest_paths) + modules.sort() + + # This is a sanity check to ensure that the unit-test discovery + # methods are working. + if len(modules) < 1: + raise Exception("No unit-test modules found.") + + for module in modules: + _log.debug("Found: %s" % module) + + # FIXME: This is a hack, but I'm tired of commenting out the test. + # See https://bugs.webkit.org/show_bug.cgi?id=31818 + if len(sys_argv) > 1 and sys.argv[1] == "--all": + sys.argv.remove("--all") + else: + excluded_module = "webkitpy.common.checkout.scm_unittest" + _log.info("Excluding: %s (use --all to include)" % excluded_module) + modules.remove(excluded_module) + + sys_argv.extend(modules) + + # We pass None for the module because we do not want the unittest + # module to resolve module names relative to a given module. + # (This would require importing all of the unittest modules from + # this module.) See the loadTestsFromName() method of the + # unittest.TestLoader class for more details on this parameter. + unittest.main(argv=sys_argv, module=None) diff --git a/WebKitTools/Scripts/webkitpy/BeautifulSoup.py b/WebKitTools/Scripts/webkitpy/thirdparty/BeautifulSoup.py index 34204e7..34204e7 100644 --- a/WebKitTools/Scripts/webkitpy/BeautifulSoup.py +++ b/WebKitTools/Scripts/webkitpy/thirdparty/BeautifulSoup.py diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py b/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py new file mode 100644 index 0000000..f1e5334 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py @@ -0,0 +1,97 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This module is required for Python to treat this directory as a package. + +"""Autoinstalls third-party code required by WebKit.""" + +import os + +from webkitpy.common.system.autoinstall import AutoInstaller + +# Putting the autoinstall code into webkitpy/thirdparty/__init__.py +# ensures that no autoinstalling occurs until a caller imports from +# webkitpy.thirdparty. This is useful if the caller wants to configure +# logging prior to executing autoinstall code. + +# FIXME: Ideally, a package should be autoinstalled only if the caller +# attempts to import from that individual package. This would +# make autoinstalling lazier than it is currently. This can +# perhaps be done using Python's import hooks as the original +# autoinstall implementation did. + +# We put auto-installed third-party modules in this directory-- +# +# webkitpy/thirdparty/autoinstalled +thirdparty_dir = os.path.dirname(__file__) +autoinstalled_dir = os.path.join(thirdparty_dir, "autoinstalled") + +# We need to download ClientForm since the mechanize package that we download +# below requires it. The mechanize package uses ClientForm, for example, +# in _html.py. Since mechanize imports ClientForm in the following way, +# +# > import sgmllib, ClientForm +# +# the search path needs to include ClientForm. We put ClientForm in +# its own directory so that we can include it in the search path without +# including other modules as a side effect. +clientform_dir = os.path.join(autoinstalled_dir, "clientform") +installer = AutoInstaller(append_to_search_path=True, + target_dir=clientform_dir) +installer.install(url="http://pypi.python.org/packages/source/C/ClientForm/ClientForm-0.2.10.zip", + url_subpath="ClientForm.py") + +# The remaining packages do not need to be in the search path, so we create +# a new AutoInstaller instance that does not append to the search path. +installer = AutoInstaller(target_dir=autoinstalled_dir) + +installer.install(url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", + url_subpath="mechanize") +installer.install(url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", + url_subpath="pep8-0.5.0/pep8.py") + + +rietveld_dir = os.path.join(autoinstalled_dir, "rietveld") +installer = AutoInstaller(target_dir=rietveld_dir) +installer.install(url="http://webkit-rietveld.googlecode.com/svn/trunk/static/upload.py", + target_name="upload.py") + + +# Since irclib and ircbot are two top-level packages, we need to import +# them separately. We group them into an irc package for better +# organization purposes. +irc_dir = os.path.join(autoinstalled_dir, "irc") +installer = AutoInstaller(target_dir=irc_dir) +installer.install(url="http://iweb.dl.sourceforge.net/project/python-irclib/python-irclib/0.4.8/python-irclib-0.4.8.zip", + url_subpath="irclib.py") +installer.install(url="http://iweb.dl.sourceforge.net/project/python-irclib/python-irclib/0.4.8/python-irclib-0.4.8.zip", + url_subpath="ircbot.py") + +readme_path = os.path.join(autoinstalled_dir, "README") +if not os.path.exists(readme_path): + file = open(readme_path, "w") + try: + file.write("This directory is auto-generated by WebKit and is " + "safe to delete.\nIt contains needed third-party Python " + "packages automatically downloaded from the web.") + finally: + file.close() diff --git a/WebKitTools/Scripts/webkitpy/mock.py b/WebKitTools/Scripts/webkitpy/thirdparty/mock.py index f6f328e..015c19e 100644 --- a/WebKitTools/Scripts/webkitpy/mock.py +++ b/WebKitTools/Scripts/webkitpy/thirdparty/mock.py @@ -1,309 +1,309 @@ -# mock.py
-# Test tools for mocking and patching.
-# Copyright (C) 2007-2009 Michael Foord
-# E-mail: fuzzyman AT voidspace DOT org DOT uk
-
-# mock 0.6.0
-# http://www.voidspace.org.uk/python/mock/
-
-# Released subject to the BSD License
-# Please see http://www.voidspace.org.uk/python/license.shtml
-
-# 2009-11-25: Licence downloaded from above URL.
-# BEGIN DOWNLOADED LICENSE
-#
-# Copyright (c) 2003-2009, Michael Foord
-# All rights reserved.
-# E-mail : fuzzyman AT voidspace DOT org DOT uk
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following
-# disclaimer in the documentation and/or other materials provided
-# with the distribution.
-#
-# * Neither the name of Michael Foord nor the name of Voidspace
-# may be used to endorse or promote products derived from this
-# software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# END DOWNLOADED LICENSE
-
-# Scripts maintained at http://www.voidspace.org.uk/python/index.shtml
-# Comments, suggestions and bug reports welcome.
-
-
-__all__ = (
- 'Mock',
- 'patch',
- 'patch_object',
- 'sentinel',
- 'DEFAULT'
-)
-
-__version__ = '0.6.0'
-
-class SentinelObject(object):
- def __init__(self, name):
- self.name = name
-
- def __repr__(self):
- return '<SentinelObject "%s">' % self.name
-
-
-class Sentinel(object):
- def __init__(self):
- self._sentinels = {}
-
- def __getattr__(self, name):
- return self._sentinels.setdefault(name, SentinelObject(name))
-
-
-sentinel = Sentinel()
-
-DEFAULT = sentinel.DEFAULT
-
-class OldStyleClass:
- pass
-ClassType = type(OldStyleClass)
-
-def _is_magic(name):
- return '__%s__' % name[2:-2] == name
-
-def _copy(value):
- if type(value) in (dict, list, tuple, set):
- return type(value)(value)
- return value
-
-
-class Mock(object):
-
- def __init__(self, spec=None, side_effect=None, return_value=DEFAULT,
- name=None, parent=None, wraps=None):
- self._parent = parent
- self._name = name
- if spec is not None and not isinstance(spec, list):
- spec = [member for member in dir(spec) if not _is_magic(member)]
-
- self._methods = spec
- self._children = {}
- self._return_value = return_value
- self.side_effect = side_effect
- self._wraps = wraps
-
- self.reset_mock()
-
-
- def reset_mock(self):
- self.called = False
- self.call_args = None
- self.call_count = 0
- self.call_args_list = []
- self.method_calls = []
- for child in self._children.itervalues():
- child.reset_mock()
- if isinstance(self._return_value, Mock):
- self._return_value.reset_mock()
-
-
- def __get_return_value(self):
- if self._return_value is DEFAULT:
- self._return_value = Mock()
- return self._return_value
-
- def __set_return_value(self, value):
- self._return_value = value
-
- return_value = property(__get_return_value, __set_return_value)
-
-
- def __call__(self, *args, **kwargs):
- self.called = True
- self.call_count += 1
- self.call_args = (args, kwargs)
- self.call_args_list.append((args, kwargs))
-
- parent = self._parent
- name = self._name
- while parent is not None:
- parent.method_calls.append((name, args, kwargs))
- if parent._parent is None:
- break
- name = parent._name + '.' + name
- parent = parent._parent
-
- ret_val = DEFAULT
- if self.side_effect is not None:
- if (isinstance(self.side_effect, Exception) or
- isinstance(self.side_effect, (type, ClassType)) and
- issubclass(self.side_effect, Exception)):
- raise self.side_effect
-
- ret_val = self.side_effect(*args, **kwargs)
- if ret_val is DEFAULT:
- ret_val = self.return_value
-
- if self._wraps is not None and self._return_value is DEFAULT:
- return self._wraps(*args, **kwargs)
- if ret_val is DEFAULT:
- ret_val = self.return_value
- return ret_val
-
-
- def __getattr__(self, name):
- if self._methods is not None:
- if name not in self._methods:
- raise AttributeError("Mock object has no attribute '%s'" % name)
- elif _is_magic(name):
- raise AttributeError(name)
-
- if name not in self._children:
- wraps = None
- if self._wraps is not None:
- wraps = getattr(self._wraps, name)
- self._children[name] = Mock(parent=self, name=name, wraps=wraps)
-
- return self._children[name]
-
-
- def assert_called_with(self, *args, **kwargs):
- assert self.call_args == (args, kwargs), 'Expected: %s\nCalled with: %s' % ((args, kwargs), self.call_args)
-
-
-def _dot_lookup(thing, comp, import_path):
- try:
- return getattr(thing, comp)
- except AttributeError:
- __import__(import_path)
- return getattr(thing, comp)
-
-
-def _importer(target):
- components = target.split('.')
- import_path = components.pop(0)
- thing = __import__(import_path)
-
- for comp in components:
- import_path += ".%s" % comp
- thing = _dot_lookup(thing, comp, import_path)
- return thing
-
-
-class _patch(object):
- def __init__(self, target, attribute, new, spec, create):
- self.target = target
- self.attribute = attribute
- self.new = new
- self.spec = spec
- self.create = create
- self.has_local = False
-
-
- def __call__(self, func):
- if hasattr(func, 'patchings'):
- func.patchings.append(self)
- return func
-
- def patched(*args, **keywargs):
- # don't use a with here (backwards compatability with 2.5)
- extra_args = []
- for patching in patched.patchings:
- arg = patching.__enter__()
- if patching.new is DEFAULT:
- extra_args.append(arg)
- args += tuple(extra_args)
- try:
- return func(*args, **keywargs)
- finally:
- for patching in getattr(patched, 'patchings', []):
- patching.__exit__()
-
- patched.patchings = [self]
- patched.__name__ = func.__name__
- patched.compat_co_firstlineno = getattr(func, "compat_co_firstlineno",
- func.func_code.co_firstlineno)
- return patched
-
-
- def get_original(self):
- target = self.target
- name = self.attribute
- create = self.create
-
- original = DEFAULT
- if _has_local_attr(target, name):
- try:
- original = target.__dict__[name]
- except AttributeError:
- # for instances of classes with slots, they have no __dict__
- original = getattr(target, name)
- elif not create and not hasattr(target, name):
- raise AttributeError("%s does not have the attribute %r" % (target, name))
- return original
-
-
- def __enter__(self):
- new, spec, = self.new, self.spec
- original = self.get_original()
- if new is DEFAULT:
- # XXXX what if original is DEFAULT - shouldn't use it as a spec
- inherit = False
- if spec == True:
- # set spec to the object we are replacing
- spec = original
- if isinstance(spec, (type, ClassType)):
- inherit = True
- new = Mock(spec=spec)
- if inherit:
- new.return_value = Mock(spec=spec)
- self.temp_original = original
- setattr(self.target, self.attribute, new)
- return new
-
-
- def __exit__(self, *_):
- if self.temp_original is not DEFAULT:
- setattr(self.target, self.attribute, self.temp_original)
- else:
- delattr(self.target, self.attribute)
- del self.temp_original
-
-
-def patch_object(target, attribute, new=DEFAULT, spec=None, create=False):
- return _patch(target, attribute, new, spec, create)
-
-
-def patch(target, new=DEFAULT, spec=None, create=False):
- try:
- target, attribute = target.rsplit('.', 1)
- except (TypeError, ValueError):
- raise TypeError("Need a valid target to patch. You supplied: %r" % (target,))
- target = _importer(target)
- return _patch(target, attribute, new, spec, create)
-
-
-
-def _has_local_attr(obj, name):
- try:
- return name in vars(obj)
- except TypeError:
- # objects without a __dict__
- return hasattr(obj, name)
+# mock.py +# Test tools for mocking and patching. +# Copyright (C) 2007-2009 Michael Foord +# E-mail: fuzzyman AT voidspace DOT org DOT uk + +# mock 0.6.0 +# http://www.voidspace.org.uk/python/mock/ + +# Released subject to the BSD License +# Please see http://www.voidspace.org.uk/python/license.shtml + +# 2009-11-25: Licence downloaded from above URL. +# BEGIN DOWNLOADED LICENSE +# +# Copyright (c) 2003-2009, Michael Foord +# All rights reserved. +# E-mail : fuzzyman AT voidspace DOT org DOT uk +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of Michael Foord nor the name of Voidspace +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# END DOWNLOADED LICENSE + +# Scripts maintained at http://www.voidspace.org.uk/python/index.shtml +# Comments, suggestions and bug reports welcome. + + +__all__ = ( + 'Mock', + 'patch', + 'patch_object', + 'sentinel', + 'DEFAULT' +) + +__version__ = '0.6.0' + +class SentinelObject(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return '<SentinelObject "%s">' % self.name + + +class Sentinel(object): + def __init__(self): + self._sentinels = {} + + def __getattr__(self, name): + return self._sentinels.setdefault(name, SentinelObject(name)) + + +sentinel = Sentinel() + +DEFAULT = sentinel.DEFAULT + +class OldStyleClass: + pass +ClassType = type(OldStyleClass) + +def _is_magic(name): + return '__%s__' % name[2:-2] == name + +def _copy(value): + if type(value) in (dict, list, tuple, set): + return type(value)(value) + return value + + +class Mock(object): + + def __init__(self, spec=None, side_effect=None, return_value=DEFAULT, + name=None, parent=None, wraps=None): + self._parent = parent + self._name = name + if spec is not None and not isinstance(spec, list): + spec = [member for member in dir(spec) if not _is_magic(member)] + + self._methods = spec + self._children = {} + self._return_value = return_value + self.side_effect = side_effect + self._wraps = wraps + + self.reset_mock() + + + def reset_mock(self): + self.called = False + self.call_args = None + self.call_count = 0 + self.call_args_list = [] + self.method_calls = [] + for child in self._children.itervalues(): + child.reset_mock() + if isinstance(self._return_value, Mock): + self._return_value.reset_mock() + + + def __get_return_value(self): + if self._return_value is DEFAULT: + self._return_value = Mock() + return self._return_value + + def __set_return_value(self, value): + self._return_value = value + + return_value = property(__get_return_value, __set_return_value) + + + def __call__(self, *args, **kwargs): + self.called = True + self.call_count += 1 + self.call_args = (args, kwargs) + self.call_args_list.append((args, kwargs)) + + parent = self._parent + name = self._name + while parent is not None: + parent.method_calls.append((name, args, kwargs)) + if parent._parent is None: + break + name = parent._name + '.' + name + parent = parent._parent + + ret_val = DEFAULT + if self.side_effect is not None: + if (isinstance(self.side_effect, Exception) or + isinstance(self.side_effect, (type, ClassType)) and + issubclass(self.side_effect, Exception)): + raise self.side_effect + + ret_val = self.side_effect(*args, **kwargs) + if ret_val is DEFAULT: + ret_val = self.return_value + + if self._wraps is not None and self._return_value is DEFAULT: + return self._wraps(*args, **kwargs) + if ret_val is DEFAULT: + ret_val = self.return_value + return ret_val + + + def __getattr__(self, name): + if self._methods is not None: + if name not in self._methods: + raise AttributeError("Mock object has no attribute '%s'" % name) + elif _is_magic(name): + raise AttributeError(name) + + if name not in self._children: + wraps = None + if self._wraps is not None: + wraps = getattr(self._wraps, name) + self._children[name] = Mock(parent=self, name=name, wraps=wraps) + + return self._children[name] + + + def assert_called_with(self, *args, **kwargs): + assert self.call_args == (args, kwargs), 'Expected: %s\nCalled with: %s' % ((args, kwargs), self.call_args) + + +def _dot_lookup(thing, comp, import_path): + try: + return getattr(thing, comp) + except AttributeError: + __import__(import_path) + return getattr(thing, comp) + + +def _importer(target): + components = target.split('.') + import_path = components.pop(0) + thing = __import__(import_path) + + for comp in components: + import_path += ".%s" % comp + thing = _dot_lookup(thing, comp, import_path) + return thing + + +class _patch(object): + def __init__(self, target, attribute, new, spec, create): + self.target = target + self.attribute = attribute + self.new = new + self.spec = spec + self.create = create + self.has_local = False + + + def __call__(self, func): + if hasattr(func, 'patchings'): + func.patchings.append(self) + return func + + def patched(*args, **keywargs): + # don't use a with here (backwards compatability with 2.5) + extra_args = [] + for patching in patched.patchings: + arg = patching.__enter__() + if patching.new is DEFAULT: + extra_args.append(arg) + args += tuple(extra_args) + try: + return func(*args, **keywargs) + finally: + for patching in getattr(patched, 'patchings', []): + patching.__exit__() + + patched.patchings = [self] + patched.__name__ = func.__name__ + patched.compat_co_firstlineno = getattr(func, "compat_co_firstlineno", + func.func_code.co_firstlineno) + return patched + + + def get_original(self): + target = self.target + name = self.attribute + create = self.create + + original = DEFAULT + if _has_local_attr(target, name): + try: + original = target.__dict__[name] + except AttributeError: + # for instances of classes with slots, they have no __dict__ + original = getattr(target, name) + elif not create and not hasattr(target, name): + raise AttributeError("%s does not have the attribute %r" % (target, name)) + return original + + + def __enter__(self): + new, spec, = self.new, self.spec + original = self.get_original() + if new is DEFAULT: + # XXXX what if original is DEFAULT - shouldn't use it as a spec + inherit = False + if spec == True: + # set spec to the object we are replacing + spec = original + if isinstance(spec, (type, ClassType)): + inherit = True + new = Mock(spec=spec) + if inherit: + new.return_value = Mock(spec=spec) + self.temp_original = original + setattr(self.target, self.attribute, new) + return new + + + def __exit__(self, *_): + if self.temp_original is not DEFAULT: + setattr(self.target, self.attribute, self.temp_original) + else: + delattr(self.target, self.attribute) + del self.temp_original + + +def patch_object(target, attribute, new=DEFAULT, spec=None, create=False): + return _patch(target, attribute, new, spec, create) + + +def patch(target, new=DEFAULT, spec=None, create=False): + try: + target, attribute = target.rsplit('.', 1) + except (TypeError, ValueError): + raise TypeError("Need a valid target to patch. You supplied: %r" % (target,)) + target = _importer(target) + return _patch(target, attribute, new, spec, create) + + + +def _has_local_attr(obj, name): + try: + return name in vars(obj) + except TypeError: + # objects without a __dict__ + return hasattr(obj, name) diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/COPYING b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/COPYING new file mode 100644 index 0000000..ab9d52d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/COPYING @@ -0,0 +1,28 @@ +Copyright 2009, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/MANIFEST.in b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/MANIFEST.in new file mode 100644 index 0000000..1925688 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/MANIFEST.in @@ -0,0 +1,6 @@ +include COPYING +include MANIFEST.in +include README +recursive-include example *.py +recursive-include mod_pywebsocket *.py +recursive-include test *.py diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/README b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/README new file mode 100644 index 0000000..1f9f05f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/README @@ -0,0 +1,6 @@ +Install this package by: +$ python setup.py build +$ sudo python setup.py install + +Then read document by: +$ pydoc mod_pywebsocket diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/README.webkit b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/README.webkit new file mode 100644 index 0000000..83e3cee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/README.webkit @@ -0,0 +1,14 @@ +This directory contains a copy of the pywebsocket Python module obtained +from the following location: + +http://code.google.com/p/pywebsocket/ + +This directory currently contains the following version: +0.4.9.2 + +The following modifications have been made to this local version: +minor updates in WebSocketRequestHandler.is_cgi + +More information on these local modifications can be found here: + +http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/example/echo_client.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/example/echo_client.py new file mode 100644 index 0000000..2b976e1 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/example/echo_client.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Web Socket Echo client. + +This is an example Web Socket client that talks with echo_wsh.py. +This may be useful for checking mod_pywebsocket installation. + +Note: +This code is far from robust, e.g., we cut corners in handshake. +""" + + +import codecs +from optparse import OptionParser +import socket +import sys + + +_TIMEOUT_SEC = 10 + +_DEFAULT_PORT = 80 +_DEFAULT_SECURE_PORT = 443 +_UNDEFINED_PORT = -1 + +_UPGRADE_HEADER = 'Upgrade: WebSocket\r\n' +_CONNECTION_HEADER = 'Connection: Upgrade\r\n' +_EXPECTED_RESPONSE = ( + 'HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + + _UPGRADE_HEADER + + _CONNECTION_HEADER) + +_GOODBYE_MESSAGE = 'Goodbye' + + +def _method_line(resource): + return 'GET %s HTTP/1.1\r\n' % resource + + +def _origin_header(origin): + return 'Origin: %s\r\n' % origin + + +class _TLSSocket(object): + """Wrapper for a TLS connection.""" + + def __init__(self, raw_socket): + self._ssl = socket.ssl(raw_socket) + + def send(self, bytes): + return self._ssl.write(bytes) + + def recv(self, size=-1): + return self._ssl.read(size) + + def close(self): + # Nothing to do. + pass + + +class EchoClient(object): + """Web Socket echo client.""" + + def __init__(self, options): + self._options = options + self._socket = None + + def run(self): + """Run the client. + + Shake hands and then repeat sending message and receiving its echo. + """ + self._socket = socket.socket() + self._socket.settimeout(self._options.socket_timeout) + try: + self._socket.connect((self._options.server_host, + self._options.server_port)) + if self._options.use_tls: + self._socket = _TLSSocket(self._socket) + self._handshake() + for line in self._options.message.split(',') + [_GOODBYE_MESSAGE]: + frame = '\x00' + line.encode('utf-8') + '\xff' + self._socket.send(frame) + if self._options.verbose: + print 'Send: %s' % line + received = self._socket.recv(len(frame)) + if received != frame: + raise Exception('Incorrect echo: %r' % received) + if self._options.verbose: + print 'Recv: %s' % received[1:-1].decode('utf-8', + 'replace') + finally: + self._socket.close() + + def _handshake(self): + self._socket.send(_method_line(self._options.resource)) + self._socket.send(_UPGRADE_HEADER) + self._socket.send(_CONNECTION_HEADER) + self._socket.send(self._format_host_header()) + self._socket.send(_origin_header(self._options.origin)) + self._socket.send('\r\n') + + for expected_char in _EXPECTED_RESPONSE: + received = self._socket.recv(1)[0] + if expected_char != received: + raise Exception('Handshake failure') + # We cut corners and skip other headers. + self._skip_headers() + + def _skip_headers(self): + terminator = '\r\n\r\n' + pos = 0 + while pos < len(terminator): + received = self._socket.recv(1)[0] + if received == terminator[pos]: + pos += 1 + elif received == terminator[0]: + pos = 1 + else: + pos = 0 + + def _format_host_header(self): + host = 'Host: ' + self._options.server_host + if ((not self._options.use_tls and + self._options.server_port != _DEFAULT_PORT) or + (self._options.use_tls and + self._options.server_port != _DEFAULT_SECURE_PORT)): + host += ':' + str(self._options.server_port) + host += '\r\n' + return host + + +def main(): + sys.stdout = codecs.getwriter('utf-8')(sys.stdout) + + parser = OptionParser() + parser.add_option('-s', '--server-host', '--server_host', + dest='server_host', type='string', + default='localhost', help='server host') + parser.add_option('-p', '--server-port', '--server_port', + dest='server_port', type='int', + default=_UNDEFINED_PORT, help='server port') + parser.add_option('-o', '--origin', dest='origin', type='string', + default='http://localhost/', help='origin') + parser.add_option('-r', '--resource', dest='resource', type='string', + default='/echo', help='resource path') + parser.add_option('-m', '--message', dest='message', type='string', + help=('comma-separated messages to send excluding "%s" ' + 'that is always sent at the end' % + _GOODBYE_MESSAGE)) + parser.add_option('-q', '--quiet', dest='verbose', action='store_false', + default=True, help='suppress messages') + parser.add_option('-t', '--tls', dest='use_tls', action='store_true', + default=False, help='use TLS (wss://)') + parser.add_option('-k', '--socket-timeout', '--socket_timeout', + dest='socket_timeout', type='int', default=_TIMEOUT_SEC, + help='Timeout(sec) for sockets') + + (options, unused_args) = parser.parse_args() + + # Default port number depends on whether TLS is used. + if options.server_port == _UNDEFINED_PORT: + if options.use_tls: + options.server_port = _DEFAULT_SECURE_PORT + else: + options.server_port = _DEFAULT_PORT + + # optparse doesn't seem to handle non-ascii default values. + # Set default message here. + if not options.message: + options.message = u'Hello,\u65e5\u672c' # "Japan" in Japanese + + EchoClient(options).run() + + +if __name__ == '__main__': + main() + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/example/echo_wsh.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/example/echo_wsh.py new file mode 100644 index 0000000..50cad31 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/example/echo_wsh.py @@ -0,0 +1,49 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from mod_pywebsocket import msgutil + + +_GOODBYE_MESSAGE = 'Goodbye' + + +def web_socket_do_extra_handshake(request): + pass # Always accept. + + +def web_socket_transfer_data(request): + while True: + line = msgutil.receive_message(request) + msgutil.send_message(request, line) + if line == _GOODBYE_MESSAGE: + return + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/example/handler_map.txt b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/example/handler_map.txt new file mode 100644 index 0000000..21c4c09 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/example/handler_map.txt @@ -0,0 +1,11 @@ +# websocket handler map file, used by standalone.py -m option. +# A line starting with '#' is a comment line. +# Each line consists of 'alias_resource_path' and 'existing_resource_path' +# separated by spaces. +# Aliasing is processed from the top to the bottom of the line, and +# 'existing_resource_path' must exist before it is aliased. +# For example, +# / /echo +# means that a request to '/' will be handled by handlers for '/echo'. +/ /echo + diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/__init__.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/__init__.py new file mode 100644 index 0000000..05e80e8 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/__init__.py @@ -0,0 +1,105 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Web Socket extension for Apache HTTP Server. + +mod_pywebsocket is a Web Socket extension for Apache HTTP Server +intended for testing or experimental purposes. mod_python is required. + +Installation: + +0. Prepare an Apache HTTP Server for which mod_python is enabled. + +1. Specify the following Apache HTTP Server directives to suit your + configuration. + + If mod_pywebsocket is not in the Python path, specify the following. + <websock_lib> is the directory where mod_pywebsocket is installed. + + PythonPath "sys.path+['<websock_lib>']" + + Always specify the following. <websock_handlers> is the directory where + user-written Web Socket handlers are placed. + + PythonOption mod_pywebsocket.handler_root <websock_handlers> + PythonHeaderParserHandler mod_pywebsocket.headerparserhandler + + To limit the search for Web Socket handlers to a directory <scan_dir> + under <websock_handlers>, configure as follows: + + PythonOption mod_pywebsocket.handler_scan <scan_dir> + + <scan_dir> is useful in saving scan time when <websock_handlers> + contains many non-Web Socket handler files. + + Example snippet of httpd.conf: + (mod_pywebsocket is in /websock_lib, Web Socket handlers are in + /websock_handlers, port is 80 for ws, 443 for wss.) + + <IfModule python_module> + PythonPath "sys.path+['/websock_lib']" + PythonOption mod_pywebsocket.handler_root /websock_handlers + PythonHeaderParserHandler mod_pywebsocket.headerparserhandler + </IfModule> + +Writing Web Socket handlers: + +When a Web Socket request comes in, the resource name +specified in the handshake is considered as if it is a file path under +<websock_handlers> and the handler defined in +<websock_handlers>/<resource_name>_wsh.py is invoked. + +For example, if the resource name is /example/chat, the handler defined in +<websock_handlers>/example/chat_wsh.py is invoked. + +A Web Socket handler is composed of the following two functions: + + web_socket_do_extra_handshake(request) + web_socket_transfer_data(request) + +where: + request: mod_python request. + +web_socket_do_extra_handshake is called during the handshake after the +headers are successfully parsed and Web Socket properties (ws_location, +ws_origin, ws_protocol, and ws_resource) are added to request. A handler +can reject the request by raising an exception. + +web_socket_transfer_data is called after the handshake completed +successfully. A handler can receive/send messages from/to the client +using request. mod_pywebsocket.msgutil module provides utilities +for data transfer. + +A Web Socket handler must be thread-safe if the server (Apache or +standalone.py) is configured to use threads. +""" + + +# vi:sts=4 sw=4 et tw=72 diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/dispatch.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/dispatch.py new file mode 100644 index 0000000..c52e9eb --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/dispatch.py @@ -0,0 +1,231 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Dispatch Web Socket request. +""" + + +import os +import re + +import util + + +_SOURCE_PATH_PATTERN = re.compile(r'(?i)_wsh\.py$') +_SOURCE_SUFFIX = '_wsh.py' +_DO_EXTRA_HANDSHAKE_HANDLER_NAME = 'web_socket_do_extra_handshake' +_TRANSFER_DATA_HANDLER_NAME = 'web_socket_transfer_data' + + +class DispatchError(Exception): + """Exception in dispatching Web Socket request.""" + + pass + + +def _normalize_path(path): + """Normalize path. + + Args: + path: the path to normalize. + + Path is converted to the absolute path. + The input path can use either '\\' or '/' as the separator. + The normalized path always uses '/' regardless of the platform. + """ + + path = path.replace('\\', os.path.sep) + path = os.path.realpath(path) + path = path.replace('\\', '/') + return path + + +def _path_to_resource_converter(base_dir): + base_dir = _normalize_path(base_dir) + base_len = len(base_dir) + suffix_len = len(_SOURCE_SUFFIX) + def converter(path): + if not path.endswith(_SOURCE_SUFFIX): + return None + path = _normalize_path(path) + if not path.startswith(base_dir): + return None + return path[base_len:-suffix_len] + return converter + + +def _source_file_paths(directory): + """Yield Web Socket Handler source file names in the given directory.""" + + for root, unused_dirs, files in os.walk(directory): + for base in files: + path = os.path.join(root, base) + if _SOURCE_PATH_PATTERN.search(path): + yield path + + +def _source(source_str): + """Source a handler definition string.""" + + global_dic = {} + try: + exec source_str in global_dic + except Exception: + raise DispatchError('Error in sourcing handler:' + + util.get_stack_trace()) + return (_extract_handler(global_dic, _DO_EXTRA_HANDSHAKE_HANDLER_NAME), + _extract_handler(global_dic, _TRANSFER_DATA_HANDLER_NAME)) + + +def _extract_handler(dic, name): + if name not in dic: + raise DispatchError('%s is not defined.' % name) + handler = dic[name] + if not callable(handler): + raise DispatchError('%s is not callable.' % name) + return handler + + +class Dispatcher(object): + """Dispatches Web Socket requests. + + This class maintains a map from resource name to handlers. + """ + + def __init__(self, root_dir, scan_dir=None): + """Construct an instance. + + Args: + root_dir: The directory where handler definition files are + placed. + scan_dir: The directory where handler definition files are + searched. scan_dir must be a directory under root_dir, + including root_dir itself. If scan_dir is None, root_dir + is used as scan_dir. scan_dir can be useful in saving + scan time when root_dir contains many subdirectories. + """ + + self._handlers = {} + self._source_warnings = [] + if scan_dir is None: + scan_dir = root_dir + if not os.path.realpath(scan_dir).startswith( + os.path.realpath(root_dir)): + raise DispatchError('scan_dir:%s must be a directory under ' + 'root_dir:%s.' % (scan_dir, root_dir)) + self._source_files_in_dir(root_dir, scan_dir) + + def add_resource_path_alias(self, + alias_resource_path, existing_resource_path): + """Add resource path alias. + + Once added, request to alias_resource_path would be handled by + handler registered for existing_resource_path. + + Args: + alias_resource_path: alias resource path + existing_resource_path: existing resource path + """ + try: + handler = self._handlers[existing_resource_path] + self._handlers[alias_resource_path] = handler + except KeyError: + raise DispatchError('No handler for: %r' % existing_resource_path) + + def source_warnings(self): + """Return warnings in sourcing handlers.""" + + return self._source_warnings + + def do_extra_handshake(self, request): + """Do extra checking in Web Socket handshake. + + Select a handler based on request.uri and call its + web_socket_do_extra_handshake function. + + Args: + request: mod_python request. + """ + + do_extra_handshake_, unused_transfer_data = self._handler(request) + try: + do_extra_handshake_(request) + except Exception, e: + util.prepend_message_to_exception( + '%s raised exception for %s: ' % ( + _DO_EXTRA_HANDSHAKE_HANDLER_NAME, + request.ws_resource), + e) + raise + + def transfer_data(self, request): + """Let a handler transfer_data with a Web Socket client. + + Select a handler based on request.ws_resource and call its + web_socket_transfer_data function. + + Args: + request: mod_python request. + """ + + unused_do_extra_handshake, transfer_data_ = self._handler(request) + try: + transfer_data_(request) + except Exception, e: + util.prepend_message_to_exception( + '%s raised exception for %s: ' % ( + _TRANSFER_DATA_HANDLER_NAME, request.ws_resource), + e) + raise + + def _handler(self, request): + try: + ws_resource_path = request.ws_resource.split('?', 1)[0] + return self._handlers[ws_resource_path] + except KeyError: + raise DispatchError('No handler for: %r' % request.ws_resource) + + def _source_files_in_dir(self, root_dir, scan_dir): + """Source all the handler source files in the scan_dir directory. + + The resource path is determined relative to root_dir. + """ + + to_resource = _path_to_resource_converter(root_dir) + for path in _source_file_paths(scan_dir): + try: + handlers = _source(open(path).read()) + except DispatchError, e: + self._source_warnings.append('%s: %s' % (path, e)) + continue + self._handlers[to_resource(path)] = handlers + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/handshake.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/handshake.py new file mode 100644 index 0000000..b86278e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/handshake.py @@ -0,0 +1,220 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Web Socket handshaking. + +Note: request.connection.write/read are used in this module, even though +mod_python document says that they should be used only in connection handlers. +Unfortunately, we have no other options. For example, request.write/read are +not suitable because they don't allow direct raw bytes writing/reading. +""" + + +import re + +import util + + +_DEFAULT_WEB_SOCKET_PORT = 80 +_DEFAULT_WEB_SOCKET_SECURE_PORT = 443 +_WEB_SOCKET_SCHEME = 'ws' +_WEB_SOCKET_SECURE_SCHEME = 'wss' + +_MANDATORY_HEADERS = [ + # key, expected value or None + ['Upgrade', 'WebSocket'], + ['Connection', 'Upgrade'], + ['Host', None], + ['Origin', None], +] + +_FIRST_FIVE_LINES = map(re.compile, [ + r'^GET /[\S]* HTTP/1.1\r\n$', + r'^Upgrade: WebSocket\r\n$', + r'^Connection: Upgrade\r\n$', + r'^Host: [\S]+\r\n$', + r'^Origin: [\S]+\r\n$', +]) + +_SIXTH_AND_LATER = re.compile( + r'^' + r'(WebSocket-Protocol: [\x20-\x7e]+\r\n)?' + r'(Cookie: [^\r]*\r\n)*' + r'(Cookie2: [^\r]*\r\n)?' + r'(Cookie: [^\r]*\r\n)*' + r'\r\n') + + +def _default_port(is_secure): + if is_secure: + return _DEFAULT_WEB_SOCKET_SECURE_PORT + else: + return _DEFAULT_WEB_SOCKET_PORT + + +class HandshakeError(Exception): + """Exception in Web Socket Handshake.""" + + pass + + +def _validate_protocol(protocol): + """Validate WebSocket-Protocol string.""" + + if not protocol: + raise HandshakeError('Invalid WebSocket-Protocol: empty') + for c in protocol: + if not 0x20 <= ord(c) <= 0x7e: + raise HandshakeError('Illegal character in protocol: %r' % c) + + +class Handshaker(object): + """This class performs Web Socket handshake.""" + + def __init__(self, request, dispatcher, strict=False): + """Construct an instance. + + Args: + request: mod_python request. + dispatcher: Dispatcher (dispatch.Dispatcher). + strict: Strictly check handshake request. Default: False. + If True, request.connection must provide get_memorized_lines + method. + + Handshaker will add attributes such as ws_resource in performing + handshake. + """ + + self._request = request + self._dispatcher = dispatcher + self._strict = strict + + def do_handshake(self): + """Perform Web Socket Handshake.""" + + self._check_header_lines() + self._set_resource() + self._set_origin() + self._set_location() + self._set_protocol() + self._dispatcher.do_extra_handshake(self._request) + self._send_handshake() + + def _set_resource(self): + self._request.ws_resource = self._request.uri + + def _set_origin(self): + self._request.ws_origin = self._request.headers_in['Origin'] + + def _set_location(self): + location_parts = [] + if self._request.is_https(): + location_parts.append(_WEB_SOCKET_SECURE_SCHEME) + else: + location_parts.append(_WEB_SOCKET_SCHEME) + location_parts.append('://') + host, port = self._parse_host_header() + connection_port = self._request.connection.local_addr[1] + if port != connection_port: + raise HandshakeError('Header/connection port mismatch: %d/%d' % + (port, connection_port)) + location_parts.append(host) + if (port != _default_port(self._request.is_https())): + location_parts.append(':') + location_parts.append(str(port)) + location_parts.append(self._request.uri) + self._request.ws_location = ''.join(location_parts) + + def _parse_host_header(self): + fields = self._request.headers_in['Host'].split(':', 1) + if len(fields) == 1: + return fields[0], _default_port(self._request.is_https()) + try: + return fields[0], int(fields[1]) + except ValueError, e: + raise HandshakeError('Invalid port number format: %r' % e) + + def _set_protocol(self): + protocol = self._request.headers_in.get('WebSocket-Protocol') + if protocol is not None: + _validate_protocol(protocol) + self._request.ws_protocol = protocol + + def _send_handshake(self): + self._request.connection.write( + 'HTTP/1.1 101 Web Socket Protocol Handshake\r\n') + self._request.connection.write('Upgrade: WebSocket\r\n') + self._request.connection.write('Connection: Upgrade\r\n') + self._request.connection.write('WebSocket-Origin: ') + self._request.connection.write(self._request.ws_origin) + self._request.connection.write('\r\n') + self._request.connection.write('WebSocket-Location: ') + self._request.connection.write(self._request.ws_location) + self._request.connection.write('\r\n') + if self._request.ws_protocol: + self._request.connection.write('WebSocket-Protocol: ') + self._request.connection.write(self._request.ws_protocol) + self._request.connection.write('\r\n') + self._request.connection.write('\r\n') + + def _check_header_lines(self): + for key, expected_value in _MANDATORY_HEADERS: + actual_value = self._request.headers_in.get(key) + if not actual_value: + raise HandshakeError('Header %s is not defined' % key) + if expected_value: + if actual_value != expected_value: + raise HandshakeError('Illegal value for header %s: %s' % + (key, actual_value)) + if self._strict: + try: + lines = self._request.connection.get_memorized_lines() + except AttributeError, e: + util.prepend_message_to_exception( + 'Strict handshake is specified but the connection ' + 'doesn\'t provide get_memorized_lines()', e) + raise + self._check_first_lines(lines) + + def _check_first_lines(self, lines): + if len(lines) < len(_FIRST_FIVE_LINES): + raise HandshakeError('Too few header lines: %d' % len(lines)) + for line, regexp in zip(lines, _FIRST_FIVE_LINES): + if not regexp.search(line): + raise HandshakeError('Unexpected header: %r doesn\'t match %r' + % (line, regexp.pattern)) + sixth_and_later = ''.join(lines[5:]) + if not _SIXTH_AND_LATER.search(sixth_and_later): + raise HandshakeError('Unexpected header: %r doesn\'t match %r' + % (sixth_and_later, + _SIXTH_AND_LATER.pattern)) + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/headerparserhandler.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/headerparserhandler.py new file mode 100644 index 0000000..124b9f1 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/headerparserhandler.py @@ -0,0 +1,99 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""PythonHeaderParserHandler for mod_pywebsocket. + +Apache HTTP Server and mod_python must be configured such that this +function is called to handle Web Socket request. +""" + + +from mod_python import apache + +import dispatch +import handshake +import util + + +# PythonOption to specify the handler root directory. +_PYOPT_HANDLER_ROOT = 'mod_pywebsocket.handler_root' + +# PythonOption to specify the handler scan directory. +# This must be a directory under the root directory. +# The default is the root directory. +_PYOPT_HANDLER_SCAN = 'mod_pywebsocket.handler_scan' + + +def _create_dispatcher(): + _HANDLER_ROOT = apache.main_server.get_options().get( + _PYOPT_HANDLER_ROOT, None) + if not _HANDLER_ROOT: + raise Exception('PythonOption %s is not defined' % _PYOPT_HANDLER_ROOT, + apache.APLOG_ERR) + _HANDLER_SCAN = apache.main_server.get_options().get( + _PYOPT_HANDLER_SCAN, _HANDLER_ROOT) + dispatcher = dispatch.Dispatcher(_HANDLER_ROOT, _HANDLER_SCAN) + for warning in dispatcher.source_warnings(): + apache.log_error('mod_pywebsocket: %s' % warning, apache.APLOG_WARNING) + return dispatcher + + +# Initialize +_dispatcher = _create_dispatcher() + + +def headerparserhandler(request): + """Handle request. + + Args: + request: mod_python request. + + This function is named headerparserhandler because it is the default name + for a PythonHeaderParserHandler. + """ + + try: + handshaker = handshake.Handshaker(request, _dispatcher) + handshaker.do_handshake() + request.log_error('mod_pywebsocket: resource: %r' % request.ws_resource, + apache.APLOG_DEBUG) + _dispatcher.transfer_data(request) + except handshake.HandshakeError, e: + # Handshake for ws/wss failed. + # But the request can be valid http/https request. + request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) + return apache.DECLINED + except dispatch.DispatchError, e: + request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_WARNING) + return apache.DECLINED + return apache.DONE # Return DONE such that no other handlers are invoked. + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/memorizingfile.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/memorizingfile.py new file mode 100644 index 0000000..2f8a54e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/memorizingfile.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Memorizing file. + +A memorizing file wraps a file and memorizes lines read by readline. +""" + + +import sys + + +class MemorizingFile(object): + """MemorizingFile wraps a file and memorizes lines read by readline. + + Note that data read by other methods are not memorized. This behavior + is good enough for memorizing lines SimpleHTTPServer reads before + the control reaches WebSocketRequestHandler. + """ + def __init__(self, file_, max_memorized_lines=sys.maxint): + """Construct an instance. + + Args: + file_: the file object to wrap. + max_memorized_lines: the maximum number of lines to memorize. + Only the first max_memorized_lines are memorized. + Default: sys.maxint. + """ + self._file = file_ + self._memorized_lines = [] + self._max_memorized_lines = max_memorized_lines + + def __getattribute__(self, name): + if name in ('_file', '_memorized_lines', '_max_memorized_lines', + 'readline', 'get_memorized_lines'): + return object.__getattribute__(self, name) + return self._file.__getattribute__(name) + + def readline(self): + """Override file.readline and memorize the line read.""" + + line = self._file.readline() + if line and len(self._memorized_lines) < self._max_memorized_lines: + self._memorized_lines.append(line) + return line + + def get_memorized_lines(self): + """Get lines memorized so far.""" + return self._memorized_lines + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/msgutil.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/msgutil.py new file mode 100644 index 0000000..90ae715 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/msgutil.py @@ -0,0 +1,250 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Message related utilities. + +Note: request.connection.write/read are used in this module, even though +mod_python document says that they should be used only in connection handlers. +Unfortunately, we have no other options. For example, request.write/read are +not suitable because they don't allow direct raw bytes writing/reading. +""" + + +import Queue +import threading +import util + + +class MsgUtilException(Exception): + pass + + +def _read(request, length): + bytes = request.connection.read(length) + if not bytes: + raise MsgUtilException( + 'Failed to receive message from %r' % + (request.connection.remote_addr,)) + return bytes + + +def _write(request, bytes): + try: + request.connection.write(bytes) + except Exception, e: + util.prepend_message_to_exception( + 'Failed to send message to %r: ' % + (request.connection.remote_addr,), + e) + raise + + +def send_message(request, message): + """Send message. + + Args: + request: mod_python request. + message: unicode string to send. + """ + + _write(request, '\x00' + message.encode('utf-8') + '\xff') + + +def receive_message(request): + """Receive a Web Socket frame and return its payload as unicode string. + + Args: + request: mod_python request. + """ + + while True: + # Read 1 byte. + # mp_conn.read will block if no bytes are available. + # Timeout is controlled by TimeOut directive of Apache. + frame_type_str = _read(request, 1) + frame_type = ord(frame_type_str[0]) + if (frame_type & 0x80) == 0x80: + # The payload length is specified in the frame. + # Read and discard. + length = _payload_length(request) + _receive_bytes(request, length) + else: + # The payload is delimited with \xff. + bytes = _read_until(request, '\xff') + # The Web Socket protocol section 4.4 specifies that invalid + # characters must be replaced with U+fffd REPLACEMENT CHARACTER. + message = bytes.decode('utf-8', 'replace') + if frame_type == 0x00: + return message + # Discard data of other types. + + +def _payload_length(request): + length = 0 + while True: + b_str = _read(request, 1) + b = ord(b_str[0]) + length = length * 128 + (b & 0x7f) + if (b & 0x80) == 0: + break + return length + + +def _receive_bytes(request, length): + bytes = [] + while length > 0: + new_bytes = _read(request, length) + bytes.append(new_bytes) + length -= len(new_bytes) + return ''.join(bytes) + + +def _read_until(request, delim_char): + bytes = [] + while True: + ch = _read(request, 1) + if ch == delim_char: + break + bytes.append(ch) + return ''.join(bytes) + + +class MessageReceiver(threading.Thread): + """This class receives messages from the client. + + This class provides three ways to receive messages: blocking, non-blocking, + and via callback. Callback has the highest precedence. + + Note: This class should not be used with the standalone server for wss + because pyOpenSSL used by the server raises a fatal error if the socket + is accessed from multiple threads. + """ + def __init__(self, request, onmessage=None): + """Construct an instance. + + Args: + request: mod_python request. + onmessage: a function to be called when a message is received. + May be None. If not None, the function is called on + another thread. In that case, MessageReceiver.receive + and MessageReceiver.receive_nowait are useless because + they will never return any messages. + """ + threading.Thread.__init__(self) + self._request = request + self._queue = Queue.Queue() + self._onmessage = onmessage + self._stop_requested = False + self.setDaemon(True) + self.start() + + def run(self): + while not self._stop_requested: + message = receive_message(self._request) + if self._onmessage: + self._onmessage(message) + else: + self._queue.put(message) + + def receive(self): + """ Receive a message from the channel, blocking. + + Returns: + message as a unicode string. + """ + return self._queue.get() + + def receive_nowait(self): + """ Receive a message from the channel, non-blocking. + + Returns: + message as a unicode string if available. None otherwise. + """ + try: + message = self._queue.get_nowait() + except Queue.Empty: + message = None + return message + + def stop(self): + """Request to stop this instance. + + The instance will be stopped after receiving the next message. + This method may not be very useful, but there is no clean way + in Python to forcefully stop a running thread. + """ + self._stop_requested = True + + +class MessageSender(threading.Thread): + """This class sends messages to the client. + + This class provides both synchronous and asynchronous ways to send + messages. + + Note: This class should not be used with the standalone server for wss + because pyOpenSSL used by the server raises a fatal error if the socket + is accessed from multiple threads. + """ + def __init__(self, request): + """Construct an instance. + + Args: + request: mod_python request. + """ + threading.Thread.__init__(self) + self._request = request + self._queue = Queue.Queue() + self.setDaemon(True) + self.start() + + def run(self): + while True: + message, condition = self._queue.get() + condition.acquire() + send_message(self._request, message) + condition.notify() + condition.release() + + def send(self, message): + """Send a message, blocking.""" + + condition = threading.Condition() + condition.acquire() + self._queue.put((message, condition)) + condition.wait() + + def send_nowait(self, message): + """Send a message, non-blocking.""" + + self._queue.put((message, threading.Condition())) + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/standalone.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/standalone.py new file mode 100644 index 0000000..f411910 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/standalone.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Standalone Web Socket server. + +Use this server to run mod_pywebsocket without Apache HTTP Server. + +Usage: + python standalone.py [-p <ws_port>] [-w <websock_handlers>] + [-s <scan_dir>] + [-d <document_root>] + [-m <websock_handlers_map_file>] + ... for other options, see _main below ... + +<ws_port> is the port number to use for ws:// connection. + +<document_root> is the path to the root directory of HTML files. + +<websock_handlers> is the path to the root directory of Web Socket handlers. +See __init__.py for details of <websock_handlers> and how to write Web Socket +handlers. If this path is relative, <document_root> is used as the base. + +<scan_dir> is a path under the root directory. If specified, only the handlers +under scan_dir are scanned. This is useful in saving scan time. + +Note: +This server is derived from SocketServer.ThreadingMixIn. Hence a thread is +used for each request. +""" + +import BaseHTTPServer +import CGIHTTPServer +import SimpleHTTPServer +import SocketServer +import logging +import logging.handlers +import optparse +import os +import re +import socket +import sys + +_HAS_OPEN_SSL = False +try: + import OpenSSL.SSL + _HAS_OPEN_SSL = True +except ImportError: + pass + +import dispatch +import handshake +import memorizingfile +import util + + +_LOG_LEVELS = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARN, + 'error': logging.ERROR, + 'critical': logging.CRITICAL}; + +_DEFAULT_LOG_MAX_BYTES = 1024 * 256 +_DEFAULT_LOG_BACKUP_COUNT = 5 + +_DEFAULT_REQUEST_QUEUE_SIZE = 128 + +# 1024 is practically large enough to contain WebSocket handshake lines. +_MAX_MEMORIZED_LINES = 1024 + +def _print_warnings_if_any(dispatcher): + warnings = dispatcher.source_warnings() + if warnings: + for warning in warnings: + logging.warning('mod_pywebsocket: %s' % warning) + + +class _StandaloneConnection(object): + """Mimic mod_python mp_conn.""" + + def __init__(self, request_handler): + """Construct an instance. + + Args: + request_handler: A WebSocketRequestHandler instance. + """ + self._request_handler = request_handler + + def get_local_addr(self): + """Getter to mimic mp_conn.local_addr.""" + return (self._request_handler.server.server_name, + self._request_handler.server.server_port) + local_addr = property(get_local_addr) + + def get_remote_addr(self): + """Getter to mimic mp_conn.remote_addr. + + Setting the property in __init__ won't work because the request + handler is not initialized yet there.""" + return self._request_handler.client_address + remote_addr = property(get_remote_addr) + + def write(self, data): + """Mimic mp_conn.write().""" + return self._request_handler.wfile.write(data) + + def read(self, length): + """Mimic mp_conn.read().""" + return self._request_handler.rfile.read(length) + + def get_memorized_lines(self): + """Get memorized lines.""" + return self._request_handler.rfile.get_memorized_lines() + + +class _StandaloneRequest(object): + """Mimic mod_python request.""" + + def __init__(self, request_handler, use_tls): + """Construct an instance. + + Args: + request_handler: A WebSocketRequestHandler instance. + """ + self._request_handler = request_handler + self.connection = _StandaloneConnection(request_handler) + self._use_tls = use_tls + + def get_uri(self): + """Getter to mimic request.uri.""" + return self._request_handler.path + uri = property(get_uri) + + def get_headers_in(self): + """Getter to mimic request.headers_in.""" + return self._request_handler.headers + headers_in = property(get_headers_in) + + def is_https(self): + """Mimic request.is_https().""" + return self._use_tls + + +class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): + """HTTPServer specialized for Web Socket.""" + + SocketServer.ThreadingMixIn.daemon_threads = True + + def __init__(self, server_address, RequestHandlerClass): + """Override SocketServer.BaseServer.__init__.""" + + SocketServer.BaseServer.__init__( + self, server_address, RequestHandlerClass) + self.socket = self._create_socket() + self.server_bind() + self.server_activate() + + def _create_socket(self): + socket_ = socket.socket(self.address_family, self.socket_type) + if WebSocketServer.options.use_tls: + ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) + ctx.use_privatekey_file(WebSocketServer.options.private_key) + ctx.use_certificate_file(WebSocketServer.options.certificate) + socket_ = OpenSSL.SSL.Connection(ctx, socket_) + return socket_ + + def handle_error(self, rquest, client_address): + """Override SocketServer.handle_error.""" + + logging.error( + ('Exception in processing request from: %r' % (client_address,)) + + '\n' + util.get_stack_trace()) + # Note: client_address is a tuple. To match it against %r, we need the + # trailing comma. + + +class WebSocketRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler): + """CGIHTTPRequestHandler specialized for Web Socket.""" + + def setup(self): + """Override SocketServer.StreamRequestHandler.setup.""" + + self.connection = self.request + self.rfile = memorizingfile.MemorizingFile( + socket._fileobject(self.request, 'rb', self.rbufsize), + max_memorized_lines=_MAX_MEMORIZED_LINES) + self.wfile = socket._fileobject(self.request, 'wb', self.wbufsize) + + def __init__(self, *args, **keywords): + self._request = _StandaloneRequest( + self, WebSocketRequestHandler.options.use_tls) + self._dispatcher = WebSocketRequestHandler.options.dispatcher + self._print_warnings_if_any() + self._handshaker = handshake.Handshaker( + self._request, self._dispatcher, + WebSocketRequestHandler.options.strict) + CGIHTTPServer.CGIHTTPRequestHandler.__init__( + self, *args, **keywords) + + def _print_warnings_if_any(self): + warnings = self._dispatcher.source_warnings() + if warnings: + for warning in warnings: + logging.warning('mod_pywebsocket: %s' % warning) + + def parse_request(self): + """Override BaseHTTPServer.BaseHTTPRequestHandler.parse_request. + + Return True to continue processing for HTTP(S), False otherwise. + """ + result = CGIHTTPServer.CGIHTTPRequestHandler.parse_request(self) + if result: + try: + self._handshaker.do_handshake() + self._dispatcher.transfer_data(self._request) + return False + except handshake.HandshakeError, e: + # Handshake for ws(s) failed. Assume http(s). + logging.info('mod_pywebsocket: %s' % e) + return True + except dispatch.DispatchError, e: + logging.warning('mod_pywebsocket: %s' % e) + return False + except Exception, e: + logging.warning('mod_pywebsocket: %s' % e) + logging.info('mod_pywebsocket: %s' % util.get_stack_trace()) + return False + return result + + def log_request(self, code='-', size='-'): + """Override BaseHTTPServer.log_request.""" + + logging.info('"%s" %s %s', + self.requestline, str(code), str(size)) + + def log_error(self, *args): + """Override BaseHTTPServer.log_error.""" + + # Despite the name, this method is for warnings than for errors. + # For example, HTTP status code is logged by this method. + logging.warn('%s - %s' % (self.address_string(), (args[0] % args[1:]))) + + def is_cgi(self): + """Test whether self.path corresponds to a CGI script. + + Add extra check that self.path doesn't contains .. + Also check if the file is a executable file or not. + If the file is not executable, it is handled as static file or dir + rather than a CGI script. + """ + if CGIHTTPServer.CGIHTTPRequestHandler.is_cgi(self): + if '..' in self.path: + return False + # strip query parameter from request path + resource_name = self.path.split('?', 2)[0] + # convert resource_name into real path name in filesystem. + scriptfile = self.translate_path(resource_name) + if not os.path.isfile(scriptfile): + return False + if not self.is_executable(scriptfile): + return False + return True + return False + + +def _configure_logging(options): + logger = logging.getLogger() + logger.setLevel(_LOG_LEVELS[options.log_level]) + if options.log_file: + handler = logging.handlers.RotatingFileHandler( + options.log_file, 'a', options.log_max, options.log_count) + else: + handler = logging.StreamHandler() + formatter = logging.Formatter( + "[%(asctime)s] [%(levelname)s] %(name)s: %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) + +def _alias_handlers(dispatcher, websock_handlers_map_file): + """Set aliases specified in websock_handler_map_file in dispatcher. + + Args: + dispatcher: dispatch.Dispatcher instance + websock_handler_map_file: alias map file + """ + fp = open(websock_handlers_map_file) + try: + for line in fp: + if line[0] == '#' or line.isspace(): + continue + m = re.match('(\S+)\s+(\S+)', line) + if not m: + logging.warning('Wrong format in map file:' + line) + continue + try: + dispatcher.add_resource_path_alias( + m.group(1), m.group(2)) + except dispatch.DispatchError, e: + logging.error(str(e)) + finally: + fp.close() + + + +def _main(): + parser = optparse.OptionParser() + parser.add_option('-H', '--server-host', '--server_host', + dest='server_host', + default='', + help='server hostname to listen to') + parser.add_option('-p', '--port', dest='port', type='int', + default=handshake._DEFAULT_WEB_SOCKET_PORT, + help='port to listen to') + parser.add_option('-w', '--websock-handlers', '--websock_handlers', + dest='websock_handlers', + default='.', + help='Web Socket handlers root directory.') + parser.add_option('-m', '--websock-handlers-map-file', + '--websock_handlers_map_file', + dest='websock_handlers_map_file', + default=None, + help=('Web Socket handlers map file. ' + 'Each line consists of alias_resource_path and ' + 'existing_resource_path, separated by spaces.')) + parser.add_option('-s', '--scan-dir', '--scan_dir', dest='scan_dir', + default=None, + help=('Web Socket handlers scan directory. ' + 'Must be a directory under websock_handlers.')) + parser.add_option('-d', '--document-root', '--document_root', + dest='document_root', default='.', + help='Document root directory.') + parser.add_option('-x', '--cgi-paths', '--cgi_paths', dest='cgi_paths', + default=None, + help=('CGI paths relative to document_root.' + 'Comma-separated. (e.g -x /cgi,/htbin) ' + 'Files under document_root/cgi_path are handled ' + 'as CGI programs. Must be executable.')) + parser.add_option('-t', '--tls', dest='use_tls', action='store_true', + default=False, help='use TLS (wss://)') + parser.add_option('-k', '--private-key', '--private_key', + dest='private_key', + default='', help='TLS private key file.') + parser.add_option('-c', '--certificate', dest='certificate', + default='', help='TLS certificate file.') + parser.add_option('-l', '--log-file', '--log_file', dest='log_file', + default='', help='Log file.') + parser.add_option('--log-level', '--log_level', type='choice', + dest='log_level', default='warn', + choices=['debug', 'info', 'warn', 'error', 'critical'], + help='Log level.') + parser.add_option('--log-max', '--log_max', dest='log_max', type='int', + default=_DEFAULT_LOG_MAX_BYTES, + help='Log maximum bytes') + parser.add_option('--log-count', '--log_count', dest='log_count', + type='int', default=_DEFAULT_LOG_BACKUP_COUNT, + help='Log backup count') + parser.add_option('--strict', dest='strict', action='store_true', + default=False, help='Strictly check handshake request') + parser.add_option('-q', '--queue', dest='request_queue_size', type='int', + default=_DEFAULT_REQUEST_QUEUE_SIZE, + help='request queue size') + options = parser.parse_args()[0] + + os.chdir(options.document_root) + + _configure_logging(options) + + SocketServer.TCPServer.request_queue_size = options.request_queue_size + CGIHTTPServer.CGIHTTPRequestHandler.cgi_directories = [] + + if options.cgi_paths: + CGIHTTPServer.CGIHTTPRequestHandler.cgi_directories = \ + options.cgi_paths.split(',') + if sys.platform in ('cygwin', 'win32'): + cygwin_path = None + # For Win32 Python, it is expected that CYGWIN_PATH + # is set to a directory of cygwin binaries. + # For example, websocket_server.py in Chromium sets CYGWIN_PATH to + # full path of third_party/cygwin/bin. + if 'CYGWIN_PATH' in os.environ: + cygwin_path = os.environ['CYGWIN_PATH'] + util.wrap_popen3_for_win(cygwin_path) + def __check_script(scriptpath): + return util.get_script_interp(scriptpath, cygwin_path) + CGIHTTPServer.executable = __check_script + + if options.use_tls: + if not _HAS_OPEN_SSL: + logging.critical('To use TLS, install pyOpenSSL.') + sys.exit(1) + if not options.private_key or not options.certificate: + logging.critical( + 'To use TLS, specify private_key and certificate.') + sys.exit(1) + + if not options.scan_dir: + options.scan_dir = options.websock_handlers + + try: + # Share a Dispatcher among request handlers to save time for + # instantiation. Dispatcher can be shared because it is thread-safe. + options.dispatcher = dispatch.Dispatcher(options.websock_handlers, + options.scan_dir) + if options.websock_handlers_map_file: + _alias_handlers(options.dispatcher, + options.websock_handlers_map_file) + _print_warnings_if_any(options.dispatcher) + + WebSocketRequestHandler.options = options + WebSocketServer.options = options + + server = WebSocketServer((options.server_host, options.port), + WebSocketRequestHandler) + server.serve_forever() + except Exception, e: + logging.critical(str(e)) + sys.exit(1) + + +if __name__ == '__main__': + _main() + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/util.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/util.py new file mode 100644 index 0000000..8ec9dca --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/mod_pywebsocket/util.py @@ -0,0 +1,121 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Web Sockets utilities. +""" + + +import StringIO +import os +import re +import traceback + + +def get_stack_trace(): + """Get the current stack trace as string. + + This is needed to support Python 2.3. + TODO: Remove this when we only support Python 2.4 and above. + Use traceback.format_exc instead. + """ + + out = StringIO.StringIO() + traceback.print_exc(file=out) + return out.getvalue() + + +def prepend_message_to_exception(message, exc): + """Prepend message to the exception.""" + + exc.args = (message + str(exc),) + return + + +def __translate_interp(interp, cygwin_path): + """Translate interp program path for Win32 python to run cygwin program + (e.g. perl). Note that it doesn't support path that contains space, + which is typically true for Unix, where #!-script is written. + For Win32 python, cygwin_path is a directory of cygwin binaries. + + Args: + interp: interp command line + cygwin_path: directory name of cygwin binary, or None + Returns: + translated interp command line. + """ + if not cygwin_path: + return interp + m = re.match("^[^ ]*/([^ ]+)( .*)?", interp) + if m: + cmd = os.path.join(cygwin_path, m.group(1)) + return cmd + m.group(2) + return interp + + +def get_script_interp(script_path, cygwin_path=None): + """Gets #!-interpreter command line from the script. + + It also fixes command path. When Cygwin Python is used, e.g. in WebKit, + it could run "/usr/bin/perl -wT hello.pl". + When Win32 Python is used, e.g. in Chromium, it couldn't. So, fix + "/usr/bin/perl" to "<cygwin_path>\perl.exe". + + Args: + script_path: pathname of the script + cygwin_path: directory name of cygwin binary, or None + Returns: + #!-interpreter command line, or None if it is not #!-script. + """ + fp = open(script_path) + line = fp.readline() + fp.close() + m = re.match("^#!(.*)", line) + if m: + return __translate_interp(m.group(1), cygwin_path) + return None + +def wrap_popen3_for_win(cygwin_path): + """Wrap popen3 to support #!-script on Windows. + + Args: + cygwin_path: path for cygwin binary if command path is needed to be + translated. None if no translation required. + """ + __orig_popen3 = os.popen3 + def __wrap_popen3(cmd, mode='t', bufsize=-1): + cmdline = cmd.split(' ') + interp = get_script_interp(cmdline[0], cygwin_path) + if interp: + cmd = interp + " " + cmd + return __orig_popen3(cmd, mode, bufsize) + os.popen3 = __wrap_popen3 + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/setup.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/setup.py new file mode 100644 index 0000000..a34a83b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/setup.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Set up script for mod_pywebsocket. +""" + + +from distutils.core import setup +import sys + + +_PACKAGE_NAME = 'mod_pywebsocket' + +if sys.version < '2.3': + print >>sys.stderr, '%s requires Python 2.3 or later.' % _PACKAGE_NAME + sys.exit(1) + +setup(author='Yuzo Fujishima', + author_email='yuzo@chromium.org', + description='Web Socket extension for Apache HTTP Server.', + long_description=( + 'mod_pywebsocket is an Apache HTTP Server extension for ' + 'Web Socket (http://tools.ietf.org/html/' + 'draft-hixie-thewebsocketprotocol). ' + 'See mod_pywebsocket/__init__.py for more detail.'), + license='See COPYING', + name=_PACKAGE_NAME, + packages=[_PACKAGE_NAME], + url='http://code.google.com/p/pywebsocket/', + version='0.4.9.2', + ) + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/config.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/config.py new file mode 100644 index 0000000..5aaab8c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/config.py @@ -0,0 +1,45 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Configuration for testing. + +Test files should import this module before mod_pywebsocket. +""" + + +import os +import sys + + +# Add the parent directory to sys.path to enable importing mod_pywebsocket. +sys.path += [os.path.join(os.path.split(__file__)[0], '..')] + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/mock.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/mock.py new file mode 100644 index 0000000..3b85d64 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/mock.py @@ -0,0 +1,205 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Mocks for testing. +""" + + +import Queue +import threading + + +class _MockConnBase(object): + """Base class of mocks for mod_python.apache.mp_conn. + + This enables tests to check what is written to a (mock) mp_conn. + """ + + def __init__(self): + self._write_data = [] + + def write(self, data): + """Override mod_python.apache.mp_conn.write.""" + + self._write_data.append(data) + + def written_data(self): + """Get bytes written to this mock.""" + + return ''.join(self._write_data) + + +class MockConn(_MockConnBase): + """Mock for mod_python.apache.mp_conn. + + This enables tests to specify what should be read from a (mock) mp_conn as + well as to check what is written to it. + """ + + def __init__(self, read_data): + """Constructs an instance. + + Args: + read_data: bytes that should be returned when read* methods are + called. + """ + + _MockConnBase.__init__(self) + self._read_data = read_data + self._read_pos = 0 + + def readline(self): + """Override mod_python.apache.mp_conn.readline.""" + + if self._read_pos >= len(self._read_data): + return '' + end_index = self._read_data.find('\n', self._read_pos) + 1 + if not end_index: + end_index = len(self._read_data) + return self._read_up_to(end_index) + + def read(self, length): + """Override mod_python.apache.mp_conn.read.""" + + if self._read_pos >= len(self._read_data): + return '' + end_index = min(len(self._read_data), self._read_pos + length) + return self._read_up_to(end_index) + + def _read_up_to(self, end_index): + line = self._read_data[self._read_pos:end_index] + self._read_pos = end_index + return line + + +class MockBlockingConn(_MockConnBase): + """Blocking mock for mod_python.apache.mp_conn. + + This enables tests to specify what should be read from a (mock) mp_conn as + well as to check what is written to it. + Callers of read* methods will block if there is no bytes available. + """ + + def __init__(self): + _MockConnBase.__init__(self) + self._queue = Queue.Queue() + + def readline(self): + """Override mod_python.apache.mp_conn.readline.""" + line = '' + while True: + c = self._queue.get() + line += c + if c == '\n': + return line + + def read(self, length): + """Override mod_python.apache.mp_conn.read.""" + + data = '' + for unused in range(length): + data += self._queue.get() + return data + + def put_bytes(self, bytes): + """Put bytes to be read from this mock. + + Args: + bytes: bytes to be read. + """ + + for byte in bytes: + self._queue.put(byte) + + +class MockTable(dict): + """Mock table. + + This mimics mod_python mp_table. Note that only the methods used by + tests are overridden. + """ + + def __init__(self, copy_from={}): + if isinstance(copy_from, dict): + copy_from = copy_from.items() + for key, value in copy_from: + self.__setitem__(key, value) + + def __getitem__(self, key): + return super(MockTable, self).__getitem__(key.lower()) + + def __setitem__(self, key, value): + super(MockTable, self).__setitem__(key.lower(), value) + + def get(self, key, def_value=None): + return super(MockTable, self).get(key.lower(), def_value) + + +class MockRequest(object): + """Mock request. + + This mimics mod_python request. + """ + + def __init__(self, uri=None, headers_in={}, connection=None, + is_https=False): + """Construct an instance. + + Arguments: + uri: URI of the request. + headers_in: Request headers. + connection: Connection used for the request. + is_https: Whether this request is over SSL. + + See the document of mod_python Request for details. + """ + self.uri = uri + self.connection = connection + self.headers_in = MockTable(headers_in) + # self.is_https_ needs to be accessible from tests. To avoid name + # conflict with self.is_https(), it is named as such. + self.is_https_ = is_https + + def is_https(self): + """Return whether this request is over SSL.""" + return self.is_https_ + + +class MockDispatcher(object): + """Mock for dispatch.Dispatcher.""" + + def do_extra_handshake(self, conn_context): + pass + + def transfer_data(self, conn_context): + pass + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/run-chromium-webkit-tests b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/run_all.py index 221b5aa..3885618 100755..100644 --- a/WebKitTools/Scripts/run-chromium-webkit-tests +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/run_all.py @@ -1,5 +1,7 @@ #!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Copyright 2009, Google Inc. +# All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -27,15 +29,36 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Wrapper around webkitpy/layout_tests/run-chromium-webkit-tests.py""" + +"""Run all tests in the same directory. +""" + + import os -import sys +import re +import unittest + + +_TEST_MODULE_PATTERN = re.compile(r'^(test_.+)\.py$') + + +def _list_test_modules(directory): + module_names = [] + for filename in os.listdir(directory): + match = _TEST_MODULE_PATTERN.search(filename) + if match: + module_names.append(match.group(1)) + return module_names + + +def _suite(): + loader = unittest.TestLoader() + return loader.loadTestsFromNames( + _list_test_modules(os.path.join(os.path.split(__file__)[0], '.'))) -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), - "webkitpy", "layout_tests")) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) -import run_chromium_webkit_tests if __name__ == '__main__': - options, args = run_chromium_webkit_tests.parse_args() - run_chromium_webkit_tests.main(options, args) + unittest.main(defaultTest='_suite') + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_dispatch.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_dispatch.py new file mode 100644 index 0000000..5403228 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_dispatch.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Tests for dispatch module.""" + + + +import os +import unittest + +import config # This must be imported before mod_pywebsocket. +from mod_pywebsocket import dispatch + +import mock + + +_TEST_HANDLERS_DIR = os.path.join( + os.path.split(__file__)[0], 'testdata', 'handlers') + +_TEST_HANDLERS_SUB_DIR = os.path.join(_TEST_HANDLERS_DIR, 'sub') + +class DispatcherTest(unittest.TestCase): + def test_normalize_path(self): + self.assertEqual(os.path.abspath('/a/b').replace('\\', '/'), + dispatch._normalize_path('/a/b')) + self.assertEqual(os.path.abspath('/a/b').replace('\\', '/'), + dispatch._normalize_path('\\a\\b')) + self.assertEqual(os.path.abspath('/a/b').replace('\\', '/'), + dispatch._normalize_path('/a/c/../b')) + self.assertEqual(os.path.abspath('abc').replace('\\', '/'), + dispatch._normalize_path('abc')) + + def test_converter(self): + converter = dispatch._path_to_resource_converter('/a/b') + self.assertEqual('/h', converter('/a/b/h_wsh.py')) + self.assertEqual('/c/h', converter('/a/b/c/h_wsh.py')) + self.assertEqual(None, converter('/a/b/h.py')) + self.assertEqual(None, converter('a/b/h_wsh.py')) + + converter = dispatch._path_to_resource_converter('a/b') + self.assertEqual('/h', converter('a/b/h_wsh.py')) + + converter = dispatch._path_to_resource_converter('/a/b///') + self.assertEqual('/h', converter('/a/b/h_wsh.py')) + self.assertEqual('/h', converter('/a/b/../b/h_wsh.py')) + + converter = dispatch._path_to_resource_converter('/a/../a/b/../b/') + self.assertEqual('/h', converter('/a/b/h_wsh.py')) + + converter = dispatch._path_to_resource_converter(r'\a\b') + self.assertEqual('/h', converter(r'\a\b\h_wsh.py')) + self.assertEqual('/h', converter(r'/a/b/h_wsh.py')) + + def test_source_file_paths(self): + paths = list(dispatch._source_file_paths(_TEST_HANDLERS_DIR)) + paths.sort() + self.assertEqual(7, len(paths)) + expected_paths = [ + os.path.join(_TEST_HANDLERS_DIR, 'blank_wsh.py'), + os.path.join(_TEST_HANDLERS_DIR, 'origin_check_wsh.py'), + os.path.join(_TEST_HANDLERS_DIR, 'sub', + 'exception_in_transfer_wsh.py'), + os.path.join(_TEST_HANDLERS_DIR, 'sub', 'non_callable_wsh.py'), + os.path.join(_TEST_HANDLERS_DIR, 'sub', 'plain_wsh.py'), + os.path.join(_TEST_HANDLERS_DIR, 'sub', + 'wrong_handshake_sig_wsh.py'), + os.path.join(_TEST_HANDLERS_DIR, 'sub', + 'wrong_transfer_sig_wsh.py'), + ] + for expected, actual in zip(expected_paths, paths): + self.assertEqual(expected, actual) + + def test_source(self): + self.assertRaises(dispatch.DispatchError, dispatch._source, '') + self.assertRaises(dispatch.DispatchError, dispatch._source, 'def') + self.assertRaises(dispatch.DispatchError, dispatch._source, '1/0') + self.failUnless(dispatch._source( + 'def web_socket_do_extra_handshake(request):pass\n' + 'def web_socket_transfer_data(request):pass\n')) + + def test_source_warnings(self): + dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None) + warnings = dispatcher.source_warnings() + warnings.sort() + expected_warnings = [ + (os.path.join(_TEST_HANDLERS_DIR, 'blank_wsh.py') + + ': web_socket_do_extra_handshake is not defined.'), + (os.path.join(_TEST_HANDLERS_DIR, 'sub', + 'non_callable_wsh.py') + + ': web_socket_do_extra_handshake is not callable.'), + (os.path.join(_TEST_HANDLERS_DIR, 'sub', + 'wrong_handshake_sig_wsh.py') + + ': web_socket_do_extra_handshake is not defined.'), + (os.path.join(_TEST_HANDLERS_DIR, 'sub', + 'wrong_transfer_sig_wsh.py') + + ': web_socket_transfer_data is not defined.'), + ] + self.assertEquals(4, len(warnings)) + for expected, actual in zip(expected_warnings, warnings): + self.assertEquals(expected, actual) + + def test_do_extra_handshake(self): + dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None) + request = mock.MockRequest() + request.ws_resource = '/origin_check' + request.ws_origin = 'http://example.com' + dispatcher.do_extra_handshake(request) # Must not raise exception. + + request.ws_origin = 'http://bad.example.com' + self.assertRaises(Exception, dispatcher.do_extra_handshake, request) + + def test_transfer_data(self): + dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None) + request = mock.MockRequest(connection=mock.MockConn('')) + request.ws_resource = '/origin_check' + request.ws_protocol = 'p1' + + dispatcher.transfer_data(request) + self.assertEqual('origin_check_wsh.py is called for /origin_check, p1', + request.connection.written_data()) + + request = mock.MockRequest(connection=mock.MockConn('')) + request.ws_resource = '/sub/plain' + request.ws_protocol = None + dispatcher.transfer_data(request) + self.assertEqual('sub/plain_wsh.py is called for /sub/plain, None', + request.connection.written_data()) + + request = mock.MockRequest(connection=mock.MockConn('')) + request.ws_resource = '/sub/plain?' + request.ws_protocol = None + dispatcher.transfer_data(request) + self.assertEqual('sub/plain_wsh.py is called for /sub/plain?, None', + request.connection.written_data()) + + request = mock.MockRequest(connection=mock.MockConn('')) + request.ws_resource = '/sub/plain?q=v' + request.ws_protocol = None + dispatcher.transfer_data(request) + self.assertEqual('sub/plain_wsh.py is called for /sub/plain?q=v, None', + request.connection.written_data()) + + def test_transfer_data_no_handler(self): + dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None) + for resource in ['/blank', '/sub/non_callable', + '/sub/no_wsh_at_the_end', '/does/not/exist']: + request = mock.MockRequest(connection=mock.MockConn('')) + request.ws_resource = resource + request.ws_protocol = 'p2' + try: + dispatcher.transfer_data(request) + self.fail() + except dispatch.DispatchError, e: + self.failUnless(str(e).find('No handler') != -1) + except Exception: + self.fail() + + def test_transfer_data_handler_exception(self): + dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None) + request = mock.MockRequest(connection=mock.MockConn('')) + request.ws_resource = '/sub/exception_in_transfer' + request.ws_protocol = 'p3' + try: + dispatcher.transfer_data(request) + self.fail() + except Exception, e: + self.failUnless(str(e).find('Intentional') != -1) + + def test_scan_dir(self): + disp = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None) + self.assertEqual(3, len(disp._handlers)) + self.failUnless(disp._handlers.has_key('/origin_check')) + self.failUnless(disp._handlers.has_key('/sub/exception_in_transfer')) + self.failUnless(disp._handlers.has_key('/sub/plain')) + + def test_scan_sub_dir(self): + disp = dispatch.Dispatcher(_TEST_HANDLERS_DIR, _TEST_HANDLERS_SUB_DIR) + self.assertEqual(2, len(disp._handlers)) + self.failIf(disp._handlers.has_key('/origin_check')) + self.failUnless(disp._handlers.has_key('/sub/exception_in_transfer')) + self.failUnless(disp._handlers.has_key('/sub/plain')) + + def test_scan_sub_dir_as_root(self): + disp = dispatch.Dispatcher(_TEST_HANDLERS_SUB_DIR, + _TEST_HANDLERS_SUB_DIR) + self.assertEqual(2, len(disp._handlers)) + self.failIf(disp._handlers.has_key('/origin_check')) + self.failIf(disp._handlers.has_key('/sub/exception_in_transfer')) + self.failIf(disp._handlers.has_key('/sub/plain')) + self.failUnless(disp._handlers.has_key('/exception_in_transfer')) + self.failUnless(disp._handlers.has_key('/plain')) + + def test_scan_dir_must_under_root(self): + dispatch.Dispatcher('a/b', 'a/b/c') # OK + dispatch.Dispatcher('a/b///', 'a/b') # OK + self.assertRaises(dispatch.DispatchError, + dispatch.Dispatcher, 'a/b/c', 'a/b') + + def test_resource_path_alias(self): + disp = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None) + disp.add_resource_path_alias('/', '/origin_check') + self.assertEqual(4, len(disp._handlers)) + self.failUnless(disp._handlers.has_key('/origin_check')) + self.failUnless(disp._handlers.has_key('/sub/exception_in_transfer')) + self.failUnless(disp._handlers.has_key('/sub/plain')) + self.failUnless(disp._handlers.has_key('/')) + self.assertRaises(dispatch.DispatchError, + disp.add_resource_path_alias, '/alias', '/not-exist') + + +if __name__ == '__main__': + unittest.main() + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_handshake.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_handshake.py new file mode 100644 index 0000000..8bf07be --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_handshake.py @@ -0,0 +1,513 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Tests for handshake module.""" + + +import unittest + +import config # This must be imported before mod_pywebsocket. +from mod_pywebsocket import handshake + +import mock + + +_GOOD_REQUEST = ( + 80, + '/demo', + { + 'Upgrade':'WebSocket', + 'Connection':'Upgrade', + 'Host':'example.com', + 'Origin':'http://example.com', + 'WebSocket-Protocol':'sample', + } +) + +_GOOD_RESPONSE_DEFAULT_PORT = ( + 'HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + 'Upgrade: WebSocket\r\n' + 'Connection: Upgrade\r\n' + 'WebSocket-Origin: http://example.com\r\n' + 'WebSocket-Location: ws://example.com/demo\r\n' + 'WebSocket-Protocol: sample\r\n' + '\r\n') + +_GOOD_RESPONSE_SECURE = ( + 'HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + 'Upgrade: WebSocket\r\n' + 'Connection: Upgrade\r\n' + 'WebSocket-Origin: http://example.com\r\n' + 'WebSocket-Location: wss://example.com/demo\r\n' + 'WebSocket-Protocol: sample\r\n' + '\r\n') + +_GOOD_REQUEST_NONDEFAULT_PORT = ( + 8081, + '/demo', + { + 'Upgrade':'WebSocket', + 'Connection':'Upgrade', + 'Host':'example.com:8081', + 'Origin':'http://example.com', + 'WebSocket-Protocol':'sample', + } +) + +_GOOD_RESPONSE_NONDEFAULT_PORT = ( + 'HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + 'Upgrade: WebSocket\r\n' + 'Connection: Upgrade\r\n' + 'WebSocket-Origin: http://example.com\r\n' + 'WebSocket-Location: ws://example.com:8081/demo\r\n' + 'WebSocket-Protocol: sample\r\n' + '\r\n') + +_GOOD_RESPONSE_SECURE_NONDEF = ( + 'HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + 'Upgrade: WebSocket\r\n' + 'Connection: Upgrade\r\n' + 'WebSocket-Origin: http://example.com\r\n' + 'WebSocket-Location: wss://example.com:8081/demo\r\n' + 'WebSocket-Protocol: sample\r\n' + '\r\n') + +_GOOD_REQUEST_NO_PROTOCOL = ( + 80, + '/demo', + { + 'Upgrade':'WebSocket', + 'Connection':'Upgrade', + 'Host':'example.com', + 'Origin':'http://example.com', + } +) + +_GOOD_RESPONSE_NO_PROTOCOL = ( + 'HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + 'Upgrade: WebSocket\r\n' + 'Connection: Upgrade\r\n' + 'WebSocket-Origin: http://example.com\r\n' + 'WebSocket-Location: ws://example.com/demo\r\n' + '\r\n') + +_GOOD_REQUEST_WITH_OPTIONAL_HEADERS = ( + 80, + '/demo', + { + 'Upgrade':'WebSocket', + 'Connection':'Upgrade', + 'Host':'example.com', + 'Origin':'http://example.com', + 'WebSocket-Protocol':'sample', + 'AKey':'AValue', + 'EmptyValue':'', + } +) + +_BAD_REQUESTS = ( + ( # HTTP request + 80, + '/demo', + { + 'Host':'www.google.com', + 'User-Agent':'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5;' + ' en-US; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3' + ' GTB6 GTBA', + 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,' + '*/*;q=0.8', + 'Accept-Language':'en-us,en;q=0.5', + 'Accept-Encoding':'gzip,deflate', + 'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.7', + 'Keep-Alive':'300', + 'Connection':'keep-alive', + } + ), + ( # Missing Upgrade + 80, + '/demo', + { + 'Connection':'Upgrade', + 'Host':'example.com', + 'Origin':'http://example.com', + 'WebSocket-Protocol':'sample', + } + ), + ( # Wrong Upgrade + 80, + '/demo', + { + 'Upgrade':'NonWebSocket', + 'Connection':'Upgrade', + 'Host':'example.com', + 'Origin':'http://example.com', + 'WebSocket-Protocol':'sample', + } + ), + ( # Empty WebSocket-Protocol + 80, + '/demo', + { + 'Upgrade':'WebSocket', + 'Connection':'Upgrade', + 'Host':'example.com', + 'Origin':'http://example.com', + 'WebSocket-Protocol':'', + } + ), + ( # Wrong port number format + 80, + '/demo', + { + 'Upgrade':'WebSocket', + 'Connection':'Upgrade', + 'Host':'example.com:0x50', + 'Origin':'http://example.com', + 'WebSocket-Protocol':'sample', + } + ), + ( # Header/connection port mismatch + 8080, + '/demo', + { + 'Upgrade':'WebSocket', + 'Connection':'Upgrade', + 'Host':'example.com', + 'Origin':'http://example.com', + 'WebSocket-Protocol':'sample', + } + ), + ( # Illegal WebSocket-Protocol + 80, + '/demo', + { + 'Upgrade':'WebSocket', + 'Connection':'Upgrade', + 'Host':'example.com', + 'Origin':'http://example.com', + 'WebSocket-Protocol':'illegal\x09protocol', + } + ), +) + +_STRICTLY_GOOD_REQUESTS = ( + ( + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + '\r\n', + ), + ( # WebSocket-Protocol + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + 'WebSocket-Protocol: sample\r\n', + '\r\n', + ), + ( # WebSocket-Protocol and Cookie + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + 'WebSocket-Protocol: sample\r\n', + 'Cookie: xyz\r\n' + '\r\n', + ), + ( # Cookie + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + 'Cookie: abc/xyz\r\n' + 'Cookie2: $Version=1\r\n' + 'Cookie: abc\r\n' + '\r\n', + ), + ( + 'GET / HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + '\r\n', + ), +) + +_NOT_STRICTLY_GOOD_REQUESTS = ( + ( # Extra space after GET + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + '\r\n', + ), + ( # Resource name doesn't stat with '/' + 'GET demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + '\r\n', + ), + ( # No space after : + 'GET /demo HTTP/1.1\r\n', + 'Upgrade:WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + '\r\n', + ), + ( # Lower case Upgrade header + 'GET /demo HTTP/1.1\r\n', + 'upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + '\r\n', + ), + ( # Connection comes before Upgrade + 'GET /demo HTTP/1.1\r\n', + 'Connection: Upgrade\r\n', + 'Upgrade: WebSocket\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + '\r\n', + ), + ( # Origin comes before Host + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Origin: http://example.com\r\n', + 'Host: example.com\r\n', + '\r\n', + ), + ( # Host continued to the next line + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example\r\n', + ' .com\r\n', + 'Origin: http://example.com\r\n', + '\r\n', + ), + ( # Cookie comes before WebSocket-Protocol + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + 'Cookie: xyz\r\n' + 'WebSocket-Protocol: sample\r\n', + '\r\n', + ), + ( # Unknown header + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + 'Content-Type: text/html\r\n' + '\r\n', + ), + ( # Cookie with continuation lines + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + 'Cookie: xyz\r\n', + ' abc\r\n', + ' defg\r\n', + '\r\n', + ), + ( # Wrong-case cookie + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + 'cookie: abc/xyz\r\n' + '\r\n', + ), + ( # Cookie, no space after colon + 'GET /demo HTTP/1.1\r\n', + 'Upgrade: WebSocket\r\n', + 'Connection: Upgrade\r\n', + 'Host: example.com\r\n', + 'Origin: http://example.com\r\n', + 'Cookie:abc/xyz\r\n' + '\r\n', + ), +) + + +def _create_request(request_def): + conn = mock.MockConn('') + conn.local_addr = ('0.0.0.0', request_def[0]) + return mock.MockRequest( + uri=request_def[1], + headers_in=request_def[2], + connection=conn) + + +def _create_get_memorized_lines(lines): + def get_memorized_lines(): + return lines + return get_memorized_lines + + +def _create_requests_with_lines(request_lines_set): + requests = [] + for lines in request_lines_set: + request = _create_request(_GOOD_REQUEST) + request.connection.get_memorized_lines = _create_get_memorized_lines( + lines) + requests.append(request) + return requests + + +class HandshakerTest(unittest.TestCase): + def test_validate_protocol(self): + handshake._validate_protocol('sample') # should succeed. + handshake._validate_protocol('Sample') # should succeed. + handshake._validate_protocol('sample\x20protocol') # should succeed. + handshake._validate_protocol('sample\x7eprotocol') # should succeed. + self.assertRaises(handshake.HandshakeError, + handshake._validate_protocol, + '') + self.assertRaises(handshake.HandshakeError, + handshake._validate_protocol, + 'sample\x19protocol') + self.assertRaises(handshake.HandshakeError, + handshake._validate_protocol, + 'sample\x7fprotocol') + self.assertRaises(handshake.HandshakeError, + handshake._validate_protocol, + # "Japan" in Japanese + u'\u65e5\u672c') + + def test_good_request_default_port(self): + request = _create_request(_GOOD_REQUEST) + handshaker = handshake.Handshaker(request, + mock.MockDispatcher()) + handshaker.do_handshake() + self.assertEqual(_GOOD_RESPONSE_DEFAULT_PORT, + request.connection.written_data()) + self.assertEqual('/demo', request.ws_resource) + self.assertEqual('http://example.com', request.ws_origin) + self.assertEqual('ws://example.com/demo', request.ws_location) + self.assertEqual('sample', request.ws_protocol) + + def test_good_request_secure_default_port(self): + request = _create_request(_GOOD_REQUEST) + request.connection.local_addr = ('0.0.0.0', 443) + request.is_https_ = True + handshaker = handshake.Handshaker(request, + mock.MockDispatcher()) + handshaker.do_handshake() + self.assertEqual(_GOOD_RESPONSE_SECURE, + request.connection.written_data()) + self.assertEqual('sample', request.ws_protocol) + + def test_good_request_nondefault_port(self): + request = _create_request(_GOOD_REQUEST_NONDEFAULT_PORT) + handshaker = handshake.Handshaker(request, + mock.MockDispatcher()) + handshaker.do_handshake() + self.assertEqual(_GOOD_RESPONSE_NONDEFAULT_PORT, + request.connection.written_data()) + self.assertEqual('sample', request.ws_protocol) + + def test_good_request_secure_non_default_port(self): + request = _create_request(_GOOD_REQUEST_NONDEFAULT_PORT) + request.is_https_ = True + handshaker = handshake.Handshaker(request, + mock.MockDispatcher()) + handshaker.do_handshake() + self.assertEqual(_GOOD_RESPONSE_SECURE_NONDEF, + request.connection.written_data()) + self.assertEqual('sample', request.ws_protocol) + + def test_good_request_default_no_protocol(self): + request = _create_request(_GOOD_REQUEST_NO_PROTOCOL) + handshaker = handshake.Handshaker(request, + mock.MockDispatcher()) + handshaker.do_handshake() + self.assertEqual(_GOOD_RESPONSE_NO_PROTOCOL, + request.connection.written_data()) + self.assertEqual(None, request.ws_protocol) + + def test_good_request_optional_headers(self): + request = _create_request(_GOOD_REQUEST_WITH_OPTIONAL_HEADERS) + handshaker = handshake.Handshaker(request, + mock.MockDispatcher()) + handshaker.do_handshake() + self.assertEqual('AValue', + request.headers_in['AKey']) + self.assertEqual('', + request.headers_in['EmptyValue']) + + def test_bad_requests(self): + for request in map(_create_request, _BAD_REQUESTS): + handshaker = handshake.Handshaker(request, + mock.MockDispatcher()) + self.assertRaises(handshake.HandshakeError, handshaker.do_handshake) + + def test_strictly_good_requests(self): + for request in _create_requests_with_lines(_STRICTLY_GOOD_REQUESTS): + strict_handshaker = handshake.Handshaker(request, + mock.MockDispatcher(), + True) + strict_handshaker.do_handshake() + + def test_not_strictly_good_requests(self): + for request in _create_requests_with_lines(_NOT_STRICTLY_GOOD_REQUESTS): + strict_handshaker = handshake.Handshaker(request, + mock.MockDispatcher(), + True) + self.assertRaises(handshake.HandshakeError, + strict_handshaker.do_handshake) + + + +if __name__ == '__main__': + unittest.main() + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_memorizingfile.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_memorizingfile.py new file mode 100644 index 0000000..2de77ba --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_memorizingfile.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Tests for memorizingfile module.""" + + +import StringIO +import unittest + +import config # This must be imported before mod_pywebsocket. +from mod_pywebsocket import memorizingfile + + +class UtilTest(unittest.TestCase): + def check(self, memorizing_file, num_read, expected_list): + for unused in range(num_read): + memorizing_file.readline() + actual_list = memorizing_file.get_memorized_lines() + self.assertEqual(len(expected_list), len(actual_list)) + for expected, actual in zip(expected_list, actual_list): + self.assertEqual(expected, actual) + + def test_get_memorized_lines(self): + memorizing_file = memorizingfile.MemorizingFile(StringIO.StringIO( + 'Hello\nWorld\nWelcome')) + self.check(memorizing_file, 3, ['Hello\n', 'World\n', 'Welcome']) + + def test_get_memorized_lines_limit_memorized_lines(self): + memorizing_file = memorizingfile.MemorizingFile(StringIO.StringIO( + 'Hello\nWorld\nWelcome'), 2) + self.check(memorizing_file, 3, ['Hello\n', 'World\n']) + + def test_get_memorized_lines_empty_file(self): + memorizing_file = memorizingfile.MemorizingFile(StringIO.StringIO( + '')) + self.check(memorizing_file, 10, []) + + +if __name__ == '__main__': + unittest.main() + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_mock.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_mock.py new file mode 100644 index 0000000..8b137d1 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_mock.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Tests for mock module.""" + + +import Queue +import threading +import unittest + +import mock + + +class MockConnTest(unittest.TestCase): + def setUp(self): + self._conn = mock.MockConn('ABC\r\nDEFG\r\n\r\nHIJK') + + def test_readline(self): + self.assertEqual('ABC\r\n', self._conn.readline()) + self.assertEqual('DEFG\r\n', self._conn.readline()) + self.assertEqual('\r\n', self._conn.readline()) + self.assertEqual('HIJK', self._conn.readline()) + self.assertEqual('', self._conn.readline()) + + def test_read(self): + self.assertEqual('ABC\r\nD', self._conn.read(6)) + self.assertEqual('EFG\r\n\r\nHI', self._conn.read(9)) + self.assertEqual('JK', self._conn.read(10)) + self.assertEqual('', self._conn.read(10)) + + def test_read_and_readline(self): + self.assertEqual('ABC\r\nD', self._conn.read(6)) + self.assertEqual('EFG\r\n', self._conn.readline()) + self.assertEqual('\r\nHIJK', self._conn.read(9)) + self.assertEqual('', self._conn.readline()) + + def test_write(self): + self._conn.write('Hello\r\n') + self._conn.write('World\r\n') + self.assertEqual('Hello\r\nWorld\r\n', self._conn.written_data()) + + +class MockBlockingConnTest(unittest.TestCase): + def test_read(self): + class LineReader(threading.Thread): + def __init__(self, conn, queue): + threading.Thread.__init__(self) + self._queue = queue + self._conn = conn + self.setDaemon(True) + self.start() + def run(self): + while True: + data = self._conn.readline() + self._queue.put(data) + conn = mock.MockBlockingConn() + queue = Queue.Queue() + reader = LineReader(conn, queue) + self.failUnless(queue.empty()) + conn.put_bytes('Foo bar\r\n') + read = queue.get() + self.assertEqual('Foo bar\r\n', read) + + +class MockTableTest(unittest.TestCase): + def test_create_from_dict(self): + table = mock.MockTable({'Key':'Value'}) + self.assertEqual('Value', table.get('KEY')) + self.assertEqual('Value', table['key']) + + def test_create_from_list(self): + table = mock.MockTable([('Key', 'Value')]) + self.assertEqual('Value', table.get('KEY')) + self.assertEqual('Value', table['key']) + + def test_create_from_tuple(self): + table = mock.MockTable((('Key', 'Value'),)) + self.assertEqual('Value', table.get('KEY')) + self.assertEqual('Value', table['key']) + + def test_set_and_get(self): + table = mock.MockTable() + self.assertEqual(None, table.get('Key')) + table['Key'] = 'Value' + self.assertEqual('Value', table.get('Key')) + self.assertEqual('Value', table.get('key')) + self.assertEqual('Value', table.get('KEY')) + self.assertEqual('Value', table['Key']) + self.assertEqual('Value', table['key']) + self.assertEqual('Value', table['KEY']) + + +if __name__ == '__main__': + unittest.main() + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_msgutil.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_msgutil.py new file mode 100644 index 0000000..16b88e0 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_msgutil.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Tests for msgutil module.""" + + +import Queue +import unittest + +import config # This must be imported before mod_pywebsocket. +from mod_pywebsocket import msgutil + +import mock + + +def _create_request(read_data): + return mock.MockRequest(connection=mock.MockConn(read_data)) + +def _create_blocking_request(): + return mock.MockRequest(connection=mock.MockBlockingConn()) + +class MessageTest(unittest.TestCase): + def test_send_message(self): + request = _create_request('') + msgutil.send_message(request, 'Hello') + self.assertEqual('\x00Hello\xff', request.connection.written_data()) + + def test_send_message_unicode(self): + request = _create_request('') + msgutil.send_message(request, u'\u65e5') + # U+65e5 is encoded as e6,97,a5 in UTF-8 + self.assertEqual('\x00\xe6\x97\xa5\xff', + request.connection.written_data()) + + def test_receive_message(self): + request = _create_request('\x00Hello\xff\x00World!\xff') + self.assertEqual('Hello', msgutil.receive_message(request)) + self.assertEqual('World!', msgutil.receive_message(request)) + + def test_receive_message_unicode(self): + request = _create_request('\x00\xe6\x9c\xac\xff') + # U+672c is encoded as e6,9c,ac in UTF-8 + self.assertEqual(u'\u672c', msgutil.receive_message(request)) + + def test_receive_message_erroneous_unicode(self): + # \x80 and \x81 are invalid as UTF-8. + request = _create_request('\x00\x80\x81\xff') + # Invalid characters should be replaced with + # U+fffd REPLACEMENT CHARACTER + self.assertEqual(u'\ufffd\ufffd', msgutil.receive_message(request)) + + def test_receive_message_discard(self): + request = _create_request('\x80\x06IGNORE\x00Hello\xff' + '\x01DISREGARD\xff\x00World!\xff') + self.assertEqual('Hello', msgutil.receive_message(request)) + self.assertEqual('World!', msgutil.receive_message(request)) + + def test_payload_length(self): + for length, bytes in ((0, '\x00'), (0x7f, '\x7f'), (0x80, '\x81\x00'), + (0x1234, '\x80\xa4\x34')): + self.assertEqual(length, + msgutil._payload_length(_create_request(bytes))) + + def test_receive_bytes(self): + request = _create_request('abcdefg') + self.assertEqual('abc', msgutil._receive_bytes(request, 3)) + self.assertEqual('defg', msgutil._receive_bytes(request, 4)) + + def test_read_until(self): + request = _create_request('abcXdefgX') + self.assertEqual('abc', msgutil._read_until(request, 'X')) + self.assertEqual('defg', msgutil._read_until(request, 'X')) + + +class MessageReceiverTest(unittest.TestCase): + def test_queue(self): + request = _create_blocking_request() + receiver = msgutil.MessageReceiver(request) + + self.assertEqual(None, receiver.receive_nowait()) + + request.connection.put_bytes('\x00Hello!\xff') + self.assertEqual('Hello!', receiver.receive()) + + def test_onmessage(self): + onmessage_queue = Queue.Queue() + def onmessage_handler(message): + onmessage_queue.put(message) + + request = _create_blocking_request() + receiver = msgutil.MessageReceiver(request, onmessage_handler) + + request.connection.put_bytes('\x00Hello!\xff') + self.assertEqual('Hello!', onmessage_queue.get()) + + +class MessageSenderTest(unittest.TestCase): + def test_send(self): + request = _create_blocking_request() + sender = msgutil.MessageSender(request) + + sender.send('World') + self.assertEqual('\x00World\xff', request.connection.written_data()) + + def test_send_nowait(self): + # Use a queue to check the bytes written by MessageSender. + # request.connection.written_data() cannot be used here because + # MessageSender runs in a separate thread. + send_queue = Queue.Queue() + def write(bytes): + send_queue.put(bytes) + request = _create_blocking_request() + request.connection.write = write + + sender = msgutil.MessageSender(request) + + sender.send_nowait('Hello') + sender.send_nowait('World') + self.assertEqual('\x00Hello\xff', send_queue.get()) + self.assertEqual('\x00World\xff', send_queue.get()) + + +if __name__ == '__main__': + unittest.main() + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_util.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_util.py new file mode 100644 index 0000000..61f0db5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/test_util.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Tests for util module.""" + + +import os +import sys +import unittest + +import config # This must be imported before mod_pywebsocket. +from mod_pywebsocket import util + +_TEST_DATA_DIR = os.path.join(os.path.split(__file__)[0], 'testdata') + +class UtilTest(unittest.TestCase): + def test_get_stack_trace(self): + self.assertEqual('None\n', util.get_stack_trace()) + try: + a = 1 / 0 # Intentionally raise exception. + except Exception: + trace = util.get_stack_trace() + self.failUnless(trace.startswith('Traceback')) + self.failUnless(trace.find('ZeroDivisionError') != -1) + + def test_prepend_message_to_exception(self): + exc = Exception('World') + self.assertEqual('World', str(exc)) + util.prepend_message_to_exception('Hello ', exc) + self.assertEqual('Hello World', str(exc)) + + def test_get_script_interp(self): + cygwin_path = 'c:\\cygwin\\bin' + cygwin_perl = os.path.join(cygwin_path, 'perl') + self.assertEqual(None, util.get_script_interp( + os.path.join(_TEST_DATA_DIR, 'README'))) + self.assertEqual(None, util.get_script_interp( + os.path.join(_TEST_DATA_DIR, 'README'), cygwin_path)) + self.assertEqual('/usr/bin/perl -wT', util.get_script_interp( + os.path.join(_TEST_DATA_DIR, 'hello.pl'))) + self.assertEqual(cygwin_perl + ' -wT', util.get_script_interp( + os.path.join(_TEST_DATA_DIR, 'hello.pl'), cygwin_path)) + + +if __name__ == '__main__': + unittest.main() + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/README b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/README new file mode 100644 index 0000000..c001aa5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/README @@ -0,0 +1 @@ +Test data directory diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/blank_wsh.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/blank_wsh.py new file mode 100644 index 0000000..7f87c6a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/blank_wsh.py @@ -0,0 +1,31 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# intentionally left blank diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/origin_check_wsh.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/origin_check_wsh.py new file mode 100644 index 0000000..2c139fa --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/origin_check_wsh.py @@ -0,0 +1,42 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +def web_socket_do_extra_handshake(request): + if request.ws_origin == 'http://example.com': + return + raise ValueError('Unacceptable origin: %r' % request.ws_origin) + + +def web_socket_transfer_data(request): + request.connection.write('origin_check_wsh.py is called for %s, %s' % + (request.ws_resource, request.ws_protocol)) + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/exception_in_transfer_wsh.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/exception_in_transfer_wsh.py new file mode 100644 index 0000000..b982d02 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/exception_in_transfer_wsh.py @@ -0,0 +1,44 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Exception in web_socket_transfer_data(). +""" + + +def web_socket_do_extra_handshake(request): + pass + + +def web_socket_transfer_data(request): + raise Exception('Intentional Exception for %s, %s' % + (request.ws_resource, request.ws_protocol)) + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/no_wsh_at_the_end.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/no_wsh_at_the_end.py new file mode 100644 index 0000000..17e7be1 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/no_wsh_at_the_end.py @@ -0,0 +1,45 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Correct signatures, wrong file name. +""" + + +def web_socket_do_extra_handshake(request): + pass + + +def web_socket_transfer_data(request): + request.connection.write( + 'sub/no_wsh_at_the_end.py is called for %s, %s' % + (request.ws_resource, request.ws_protocol)) + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/non_callable_wsh.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/non_callable_wsh.py new file mode 100644 index 0000000..26352eb --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/non_callable_wsh.py @@ -0,0 +1,39 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Non-callable handlers. +""" + + +web_socket_do_extra_handshake = True +web_socket_transfer_data = 1 + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/plain_wsh.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/plain_wsh.py new file mode 100644 index 0000000..db3ff69 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/plain_wsh.py @@ -0,0 +1,40 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +def web_socket_do_extra_handshake(request): + pass + + +def web_socket_transfer_data(request): + request.connection.write('sub/plain_wsh.py is called for %s, %s' % + (request.ws_resource, request.ws_protocol)) + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/wrong_handshake_sig_wsh.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/wrong_handshake_sig_wsh.py new file mode 100644 index 0000000..6bf659b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/wrong_handshake_sig_wsh.py @@ -0,0 +1,45 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Wrong web_socket_do_extra_handshake signature. +""" + + +def no_web_socket_do_extra_handshake(request): + pass + + +def web_socket_transfer_data(request): + request.connection.write( + 'sub/wrong_handshake_sig_wsh.py is called for %s, %s' % + (request.ws_resource, request.ws_protocol)) + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/wrong_transfer_sig_wsh.py b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/wrong_transfer_sig_wsh.py new file mode 100644 index 0000000..e0e2e55 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/handlers/sub/wrong_transfer_sig_wsh.py @@ -0,0 +1,45 @@ +# Copyright 2009, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""Wrong web_socket_transfer_data() signature. +""" + + +def web_socket_do_extra_handshake(request): + pass + + +def no_web_socket_transfer_data(request): + request.connection.write( + 'sub/wrong_transfer_sig_wsh.py is called for %s, %s' % + (request.ws_resource, request.ws_protocol)) + + +# vi:sts=4 sw=4 et diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/hello.pl b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/hello.pl new file mode 100644 index 0000000..9dd01b4 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/pywebsocket/test/testdata/hello.pl @@ -0,0 +1,2 @@ +#!/usr/bin/perl -wT +print "Hello\n"; diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/LICENSE.txt b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/LICENSE.txt new file mode 100644 index 0000000..ad95f29 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2006 Bob Ippolito + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/README.txt b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/README.txt new file mode 100644 index 0000000..7f726ce --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/README.txt @@ -0,0 +1,11 @@ +URL: http://undefined.org/python/#simplejson +Version: 1.7.3 +License: MIT +License File: LICENSE.txt + +Description: +simplejson is a JSON encoder and decoder for Python. + + +Local Modifications: +Removed unit tests from current distribution. diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/__init__.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/__init__.py new file mode 100644 index 0000000..38d6229 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/__init__.py @@ -0,0 +1,287 @@ +r""" +A simple, fast, extensible JSON encoder and decoder + +JSON (JavaScript Object Notation) <http://json.org> is a subset of +JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data +interchange format. + +simplejson exposes an API familiar to uses of the standard library +marshal and pickle modules. + +Encoding basic Python object hierarchies:: + + >>> import simplejson + >>> simplejson.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) + '["foo", {"bar": ["baz", null, 1.0, 2]}]' + >>> print simplejson.dumps("\"foo\bar") + "\"foo\bar" + >>> print simplejson.dumps(u'\u1234') + "\u1234" + >>> print simplejson.dumps('\\') + "\\" + >>> print simplejson.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) + {"a": 0, "b": 0, "c": 0} + >>> from StringIO import StringIO + >>> io = StringIO() + >>> simplejson.dump(['streaming API'], io) + >>> io.getvalue() + '["streaming API"]' + +Compact encoding:: + + >>> import simplejson + >>> simplejson.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + '[1,2,3,{"4":5,"6":7}]' + +Pretty printing:: + + >>> import simplejson + >>> print simplejson.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4) + { + "4": 5, + "6": 7 + } + +Decoding JSON:: + + >>> import simplejson + >>> simplejson.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') + [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] + >>> simplejson.loads('"\\"foo\\bar"') + u'"foo\x08ar' + >>> from StringIO import StringIO + >>> io = StringIO('["streaming API"]') + >>> simplejson.load(io) + [u'streaming API'] + +Specializing JSON object decoding:: + + >>> import simplejson + >>> def as_complex(dct): + ... if '__complex__' in dct: + ... return complex(dct['real'], dct['imag']) + ... return dct + ... + >>> simplejson.loads('{"__complex__": true, "real": 1, "imag": 2}', + ... object_hook=as_complex) + (1+2j) + +Extending JSONEncoder:: + + >>> import simplejson + >>> class ComplexEncoder(simplejson.JSONEncoder): + ... def default(self, obj): + ... if isinstance(obj, complex): + ... return [obj.real, obj.imag] + ... return simplejson.JSONEncoder.default(self, obj) + ... + >>> dumps(2 + 1j, cls=ComplexEncoder) + '[2.0, 1.0]' + >>> ComplexEncoder().encode(2 + 1j) + '[2.0, 1.0]' + >>> list(ComplexEncoder().iterencode(2 + 1j)) + ['[', '2.0', ', ', '1.0', ']'] + + +Note that the JSON produced by this module's default settings +is a subset of YAML, so it may be used as a serializer for that as well. +""" +__version__ = '1.7.3' +__all__ = [ + 'dump', 'dumps', 'load', 'loads', + 'JSONDecoder', 'JSONEncoder', +] + +from decoder import JSONDecoder +from encoder import JSONEncoder + +_default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8' +) + +def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', **kw): + """ + Serialize ``obj`` as a JSON formatted stream to ``fp`` (a + ``.write()``-supporting file-like object). + + If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is ``False``, then the some chunks written to ``fp`` + may be ``unicode`` instances, subject to normal Python ``str`` to + ``unicode`` coercion rules. Unless ``fp.write()`` explicitly + understands ``unicode`` (as in ``codecs.getwriter()``) this is likely + to cause an error. + + If ``check_circular`` is ``False``, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is ``False``, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) + in strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a non-negative integer, then JSON array elements and object + members will be pretty-printed with that indent level. An indent level + of 0 will only insert newlines. ``None`` is the most compact representation. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + """ + # cached encoder + if (skipkeys is False and ensure_ascii is True and + check_circular is True and allow_nan is True and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and not kw): + iterable = _default_encoder.iterencode(obj) + else: + if cls is None: + cls = JSONEncoder + iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, **kw).iterencode(obj) + # could accelerate with writelines in some versions of Python, at + # a debuggability cost + for chunk in iterable: + fp.write(chunk) + + +def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', **kw): + """ + Serialize ``obj`` to a JSON formatted ``str``. + + If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is ``False``, then the return value will be a + ``unicode`` instance subject to normal Python ``str`` to ``unicode`` + coercion rules instead of being escaped to an ASCII ``str``. + + If ``check_circular`` is ``False``, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is ``False``, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in + strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a non-negative integer, then JSON array elements and + object members will be pretty-printed with that indent level. An indent + level of 0 will only insert newlines. ``None`` is the most compact + representation. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + """ + # cached encoder + if (skipkeys is False and ensure_ascii is True and + check_circular is True and allow_nan is True and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and not kw): + return _default_encoder.encode(obj) + if cls is None: + cls = JSONEncoder + return cls( + skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, + **kw).encode(obj) + +_default_decoder = JSONDecoder(encoding=None, object_hook=None) + +def load(fp, encoding=None, cls=None, object_hook=None, **kw): + """ + Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + a JSON document) to a Python object. + + If the contents of ``fp`` is encoded with an ASCII based encoding other + than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must + be specified. Encodings that are not ASCII based (such as UCS-2) are + not allowed, and should be wrapped with + ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode`` + object and passed to ``loads()`` + + ``object_hook`` is an optional function that will be called with the + result of any object literal decode (a ``dict``). The return value of + ``object_hook`` will be used instead of the ``dict``. This feature + can be used to implement custom decoders (e.g. JSON-RPC class hinting). + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + """ + return loads(fp.read(), + encoding=encoding, cls=cls, object_hook=object_hook, **kw) + +def loads(s, encoding=None, cls=None, object_hook=None, **kw): + """ + Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + document) to a Python object. + + If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding + other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name + must be specified. Encodings that are not ASCII based (such as UCS-2) + are not allowed and should be decoded to ``unicode`` first. + + ``object_hook`` is an optional function that will be called with the + result of any object literal decode (a ``dict``). The return value of + ``object_hook`` will be used instead of the ``dict``. This feature + can be used to implement custom decoders (e.g. JSON-RPC class hinting). + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + """ + if cls is None and encoding is None and object_hook is None and not kw: + return _default_decoder.decode(s) + if cls is None: + cls = JSONDecoder + if object_hook is not None: + kw['object_hook'] = object_hook + return cls(encoding=encoding, **kw).decode(s) + +def read(s): + """ + json-py API compatibility hook. Use loads(s) instead. + """ + import warnings + warnings.warn("simplejson.loads(s) should be used instead of read(s)", + DeprecationWarning) + return loads(s) + +def write(obj): + """ + json-py API compatibility hook. Use dumps(s) instead. + """ + import warnings + warnings.warn("simplejson.dumps(s) should be used instead of write(s)", + DeprecationWarning) + return dumps(obj) + + diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/_speedups.c b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/_speedups.c new file mode 100644 index 0000000..8f290bb --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/_speedups.c @@ -0,0 +1,215 @@ +#include "Python.h" +#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) +typedef int Py_ssize_t; +#define PY_SSIZE_T_MAX INT_MAX +#define PY_SSIZE_T_MIN INT_MIN +#endif + +static Py_ssize_t +ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars); +static PyObject * +ascii_escape_unicode(PyObject *pystr); +static PyObject * +ascii_escape_str(PyObject *pystr); +static PyObject * +py_encode_basestring_ascii(PyObject* self __attribute__((__unused__)), PyObject *pystr); +void init_speedups(void); + +#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '/' && c != '"') + +#define MIN_EXPANSION 6 +#ifdef Py_UNICODE_WIDE +#define MAX_EXPANSION (2 * MIN_EXPANSION) +#else +#define MAX_EXPANSION MIN_EXPANSION +#endif + +static Py_ssize_t +ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars) { + Py_UNICODE x; + output[chars++] = '\\'; + switch (c) { + case '/': output[chars++] = (char)c; break; + case '\\': output[chars++] = (char)c; break; + case '"': output[chars++] = (char)c; break; + case '\b': output[chars++] = 'b'; break; + case '\f': output[chars++] = 'f'; break; + case '\n': output[chars++] = 'n'; break; + case '\r': output[chars++] = 'r'; break; + case '\t': output[chars++] = 't'; break; + default: +#ifdef Py_UNICODE_WIDE + if (c >= 0x10000) { + /* UTF-16 surrogate pair */ + Py_UNICODE v = c - 0x10000; + c = 0xd800 | ((v >> 10) & 0x3ff); + output[chars++] = 'u'; + x = (c & 0xf000) >> 12; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x0f00) >> 8; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x00f0) >> 4; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x000f); + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + c = 0xdc00 | (v & 0x3ff); + output[chars++] = '\\'; + } +#endif + output[chars++] = 'u'; + x = (c & 0xf000) >> 12; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x0f00) >> 8; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x00f0) >> 4; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x000f); + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + } + return chars; +} + +static PyObject * +ascii_escape_unicode(PyObject *pystr) { + Py_ssize_t i; + Py_ssize_t input_chars; + Py_ssize_t output_size; + Py_ssize_t chars; + PyObject *rval; + char *output; + Py_UNICODE *input_unicode; + + input_chars = PyUnicode_GET_SIZE(pystr); + input_unicode = PyUnicode_AS_UNICODE(pystr); + /* One char input can be up to 6 chars output, estimate 4 of these */ + output_size = 2 + (MIN_EXPANSION * 4) + input_chars; + rval = PyString_FromStringAndSize(NULL, output_size); + if (rval == NULL) { + return NULL; + } + output = PyString_AS_STRING(rval); + chars = 0; + output[chars++] = '"'; + for (i = 0; i < input_chars; i++) { + Py_UNICODE c = input_unicode[i]; + if (S_CHAR(c)) { + output[chars++] = (char)c; + } else { + chars = ascii_escape_char(c, output, chars); + } + if (output_size - chars < (1 + MAX_EXPANSION)) { + /* There's more than four, so let's resize by a lot */ + output_size *= 2; + /* This is an upper bound */ + if (output_size > 2 + (input_chars * MAX_EXPANSION)) { + output_size = 2 + (input_chars * MAX_EXPANSION); + } + if (_PyString_Resize(&rval, output_size) == -1) { + return NULL; + } + output = PyString_AS_STRING(rval); + } + } + output[chars++] = '"'; + if (_PyString_Resize(&rval, chars) == -1) { + return NULL; + } + return rval; +} + +static PyObject * +ascii_escape_str(PyObject *pystr) { + Py_ssize_t i; + Py_ssize_t input_chars; + Py_ssize_t output_size; + Py_ssize_t chars; + PyObject *rval; + char *output; + char *input_str; + + input_chars = PyString_GET_SIZE(pystr); + input_str = PyString_AS_STRING(pystr); + /* One char input can be up to 6 chars output, estimate 4 of these */ + output_size = 2 + (MIN_EXPANSION * 4) + input_chars; + rval = PyString_FromStringAndSize(NULL, output_size); + if (rval == NULL) { + return NULL; + } + output = PyString_AS_STRING(rval); + chars = 0; + output[chars++] = '"'; + for (i = 0; i < input_chars; i++) { + Py_UNICODE c = (Py_UNICODE)input_str[i]; + if (S_CHAR(c)) { + output[chars++] = (char)c; + } else if (c > 0x7F) { + /* We hit a non-ASCII character, bail to unicode mode */ + PyObject *uni; + Py_DECREF(rval); + uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); + if (uni == NULL) { + return NULL; + } + rval = ascii_escape_unicode(uni); + Py_DECREF(uni); + return rval; + } else { + chars = ascii_escape_char(c, output, chars); + } + /* An ASCII char can't possibly expand to a surrogate! */ + if (output_size - chars < (1 + MIN_EXPANSION)) { + /* There's more than four, so let's resize by a lot */ + output_size *= 2; + if (output_size > 2 + (input_chars * MIN_EXPANSION)) { + output_size = 2 + (input_chars * MIN_EXPANSION); + } + if (_PyString_Resize(&rval, output_size) == -1) { + return NULL; + } + output = PyString_AS_STRING(rval); + } + } + output[chars++] = '"'; + if (_PyString_Resize(&rval, chars) == -1) { + return NULL; + } + return rval; +} + +PyDoc_STRVAR(pydoc_encode_basestring_ascii, + "encode_basestring_ascii(basestring) -> str\n" + "\n" + "..." +); + +static PyObject * +py_encode_basestring_ascii(PyObject* self __attribute__((__unused__)), PyObject *pystr) { + /* METH_O */ + if (PyString_Check(pystr)) { + return ascii_escape_str(pystr); + } else if (PyUnicode_Check(pystr)) { + return ascii_escape_unicode(pystr); + } + PyErr_SetString(PyExc_TypeError, "first argument must be a string"); + return NULL; +} + +#define DEFN(n, k) \ + { \ + #n, \ + (PyCFunction)py_ ##n, \ + k, \ + pydoc_ ##n \ + } +static PyMethodDef speedups_methods[] = { + DEFN(encode_basestring_ascii, METH_O), + {} +}; +#undef DEFN + +void +init_speedups(void) +{ + PyObject *m; + m = Py_InitModule4("_speedups", speedups_methods, NULL, NULL, PYTHON_API_VERSION); +} diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/decoder.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/decoder.py new file mode 100644 index 0000000..b887b58 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/decoder.py @@ -0,0 +1,273 @@ +""" +Implementation of JSONDecoder +""" +import re + +from .scanner import Scanner, pattern + +FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL + +def _floatconstants(): + import struct + import sys + _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') + if sys.byteorder != 'big': + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] + nan, inf = struct.unpack('dd', _BYTES) + return nan, inf, -inf + +NaN, PosInf, NegInf = _floatconstants() + +def linecol(doc, pos): + lineno = doc.count('\n', 0, pos) + 1 + if lineno == 1: + colno = pos + else: + colno = pos - doc.rindex('\n', 0, pos) + return lineno, colno + +def errmsg(msg, doc, pos, end=None): + lineno, colno = linecol(doc, pos) + if end is None: + return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos) + endlineno, endcolno = linecol(doc, end) + return '%s: line %d column %d - line %d column %d (char %d - %d)' % ( + msg, lineno, colno, endlineno, endcolno, pos, end) + +_CONSTANTS = { + '-Infinity': NegInf, + 'Infinity': PosInf, + 'NaN': NaN, + 'true': True, + 'false': False, + 'null': None, +} + +def JSONConstant(match, context, c=_CONSTANTS): + return c[match.group(0)], None +pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant) + +def JSONNumber(match, context): + match = JSONNumber.regex.match(match.string, *match.span()) + integer, frac, exp = match.groups() + if frac or exp: + res = float(integer + (frac or '') + (exp or '')) + else: + res = int(integer) + return res, None +pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber) + +STRINGCHUNK = re.compile(r'(.*?)(["\\])', FLAGS) +BACKSLASH = { + '"': u'"', '\\': u'\\', '/': u'/', + 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', +} + +DEFAULT_ENCODING = "utf-8" + +def scanstring(s, end, encoding=None, _b=BACKSLASH, _m=STRINGCHUNK.match): + if encoding is None: + encoding = DEFAULT_ENCODING + chunks = [] + _append = chunks.append + begin = end - 1 + while 1: + chunk = _m(s, end) + if chunk is None: + raise ValueError( + errmsg("Unterminated string starting at", s, begin)) + end = chunk.end() + content, terminator = chunk.groups() + if content: + if not isinstance(content, unicode): + content = unicode(content, encoding) + _append(content) + if terminator == '"': + break + try: + esc = s[end] + except IndexError: + raise ValueError( + errmsg("Unterminated string starting at", s, begin)) + if esc != 'u': + try: + m = _b[esc] + except KeyError: + raise ValueError( + errmsg("Invalid \\escape: %r" % (esc,), s, end)) + end += 1 + else: + esc = s[end + 1:end + 5] + try: + m = unichr(int(esc, 16)) + if len(esc) != 4 or not esc.isalnum(): + raise ValueError + except ValueError: + raise ValueError(errmsg("Invalid \\uXXXX escape", s, end)) + end += 5 + _append(m) + return u''.join(chunks), end + +def JSONString(match, context): + encoding = getattr(context, 'encoding', None) + return scanstring(match.string, match.end(), encoding) +pattern(r'"')(JSONString) + +WHITESPACE = re.compile(r'\s*', FLAGS) + +def JSONObject(match, context, _w=WHITESPACE.match): + pairs = {} + s = match.string + end = _w(s, match.end()).end() + nextchar = s[end:end + 1] + # trivial empty object + if nextchar == '}': + return pairs, end + 1 + if nextchar != '"': + raise ValueError(errmsg("Expecting property name", s, end)) + end += 1 + encoding = getattr(context, 'encoding', None) + iterscan = JSONScanner.iterscan + while True: + key, end = scanstring(s, end, encoding) + end = _w(s, end).end() + if s[end:end + 1] != ':': + raise ValueError(errmsg("Expecting : delimiter", s, end)) + end = _w(s, end + 1).end() + try: + value, end = iterscan(s, idx=end, context=context).next() + except StopIteration: + raise ValueError(errmsg("Expecting object", s, end)) + pairs[key] = value + end = _w(s, end).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == '}': + break + if nextchar != ',': + raise ValueError(errmsg("Expecting , delimiter", s, end - 1)) + end = _w(s, end).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar != '"': + raise ValueError(errmsg("Expecting property name", s, end - 1)) + object_hook = getattr(context, 'object_hook', None) + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end +pattern(r'{')(JSONObject) + +def JSONArray(match, context, _w=WHITESPACE.match): + values = [] + s = match.string + end = _w(s, match.end()).end() + # look-ahead for trivial empty array + nextchar = s[end:end + 1] + if nextchar == ']': + return values, end + 1 + iterscan = JSONScanner.iterscan + while True: + try: + value, end = iterscan(s, idx=end, context=context).next() + except StopIteration: + raise ValueError(errmsg("Expecting object", s, end)) + values.append(value) + end = _w(s, end).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == ']': + break + if nextchar != ',': + raise ValueError(errmsg("Expecting , delimiter", s, end)) + end = _w(s, end).end() + return values, end +pattern(r'\[')(JSONArray) + +ANYTHING = [ + JSONObject, + JSONArray, + JSONString, + JSONConstant, + JSONNumber, +] + +JSONScanner = Scanner(ANYTHING) + +class JSONDecoder(object): + """ + Simple JSON <http://json.org> decoder + + Performs the following translations in decoding: + + +---------------+-------------------+ + | JSON | Python | + +===============+===================+ + | object | dict | + +---------------+-------------------+ + | array | list | + +---------------+-------------------+ + | string | unicode | + +---------------+-------------------+ + | number (int) | int, long | + +---------------+-------------------+ + | number (real) | float | + +---------------+-------------------+ + | true | True | + +---------------+-------------------+ + | false | False | + +---------------+-------------------+ + | null | None | + +---------------+-------------------+ + + It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as + their corresponding ``float`` values, which is outside the JSON spec. + """ + + _scanner = Scanner(ANYTHING) + __all__ = ['__init__', 'decode', 'raw_decode'] + + def __init__(self, encoding=None, object_hook=None): + """ + ``encoding`` determines the encoding used to interpret any ``str`` + objects decoded by this instance (utf-8 by default). It has no + effect when decoding ``unicode`` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as ``unicode``. + + ``object_hook``, if specified, will be called with the result + of every JSON object decoded and its return value will be used in + place of the given ``dict``. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + """ + self.encoding = encoding + self.object_hook = object_hook + + def decode(self, s, _w=WHITESPACE.match): + """ + Return the Python representation of ``s`` (a ``str`` or ``unicode`` + instance containing a JSON document) + """ + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + end = _w(s, end).end() + if end != len(s): + raise ValueError(errmsg("Extra data", s, end, len(s))) + return obj + + def raw_decode(self, s, **kw): + """ + Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning + with a JSON document) and return a 2-tuple of the Python + representation and the index in ``s`` where the document ended. + + This can be used to decode a JSON document from a string that may + have extraneous data at the end. + """ + kw.setdefault('context', self) + try: + obj, end = self._scanner.iterscan(s, **kw).next() + except StopIteration: + raise ValueError("No JSON object could be decoded") + return obj, end + +__all__ = ['JSONDecoder'] diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/encoder.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/encoder.py new file mode 100644 index 0000000..d29919a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/encoder.py @@ -0,0 +1,371 @@ +""" +Implementation of JSONEncoder +""" +import re +try: + from simplejson import _speedups +except ImportError: + _speedups = None + +ESCAPE = re.compile(r'[\x00-\x19\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"/]|[^\ -~])') +ESCAPE_DCT = { + # escape all forward slashes to prevent </script> attack + '/': '\\/', + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +# assume this produces an infinity on all machines (probably not guaranteed) +INFINITY = float('1e66666') + +def floatstr(o, allow_nan=True): + # Check for specials. Note that this type of test is processor- and/or + # platform-specific, so do tests which don't depend on the internals. + + if o != o: + text = 'NaN' + elif o == INFINITY: + text = 'Infinity' + elif o == -INFINITY: + text = '-Infinity' + else: + return repr(o) + + if not allow_nan: + raise ValueError("Out of range float values are not JSON compliant: %r" + % (o,)) + + return text + + +def encode_basestring(s): + """ + Return a JSON representation of a Python string + """ + def replace(match): + return ESCAPE_DCT[match.group(0)] + return '"' + ESCAPE.sub(replace, s) + '"' + +def encode_basestring_ascii(s): + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + +try: + encode_basestring_ascii = _speedups.encode_basestring_ascii + _need_utf8 = True +except AttributeError: + _need_utf8 = False + +class JSONEncoder(object): + """ + Extensible JSON <http://json.org> encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str, unicode | string | + +-------------------+---------------+ + | int, long, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + """ + __all__ = ['__init__', 'default', 'encode', 'iterencode'] + item_separator = ', ' + key_separator = ': ' + def __init__(self, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, sort_keys=False, + indent=None, separators=None, encoding='utf-8'): + """ + Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is False, then it is a TypeError to attempt + encoding of keys that are not str, int, long, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is True, the output is guaranteed to be str + objects with all incoming unicode characters escaped. If + ensure_ascii is false, the output will be unicode object. + + If check_circular is True, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is True, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is True, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a non-negative integer, then JSON array + elements and object members will be pretty-printed with that + indent level. An indent level of 0 will only insert newlines. + None is the most compact representation. + + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. + + If encoding is not None, then all input strings will be + transformed into unicode using that encoding prior to JSON-encoding. + The default is UTF-8. + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.indent = indent + self.current_indent_level = 0 + if separators is not None: + self.item_separator, self.key_separator = separators + self.encoding = encoding + + def _newline_indent(self): + return '\n' + (' ' * (self.indent * self.current_indent_level)) + + def _iterencode_list(self, lst, markers=None): + if not lst: + yield '[]' + return + if markers is not None: + markerid = id(lst) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = lst + yield '[' + if self.indent is not None: + self.current_indent_level += 1 + newline_indent = self._newline_indent() + separator = self.item_separator + newline_indent + yield newline_indent + else: + newline_indent = None + separator = self.item_separator + first = True + for value in lst: + if first: + first = False + else: + yield separator + for chunk in self._iterencode(value, markers): + yield chunk + if newline_indent is not None: + self.current_indent_level -= 1 + yield self._newline_indent() + yield ']' + if markers is not None: + del markers[markerid] + + def _iterencode_dict(self, dct, markers=None): + if not dct: + yield '{}' + return + if markers is not None: + markerid = id(dct) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = dct + yield '{' + key_separator = self.key_separator + if self.indent is not None: + self.current_indent_level += 1 + newline_indent = self._newline_indent() + item_separator = self.item_separator + newline_indent + yield newline_indent + else: + newline_indent = None + item_separator = self.item_separator + first = True + if self.ensure_ascii: + encoder = encode_basestring_ascii + else: + encoder = encode_basestring + allow_nan = self.allow_nan + if self.sort_keys: + keys = dct.keys() + keys.sort() + items = [(k, dct[k]) for k in keys] + else: + items = dct.iteritems() + _encoding = self.encoding + _do_decode = (_encoding is not None + and not (_need_utf8 and _encoding == 'utf-8')) + for key, value in items: + if isinstance(key, str): + if _do_decode: + key = key.decode(_encoding) + elif isinstance(key, basestring): + pass + # JavaScript is weakly typed for these, so it makes sense to + # also allow them. Many encoders seem to do something like this. + elif isinstance(key, float): + key = floatstr(key, allow_nan) + elif isinstance(key, (int, long)): + key = str(key) + elif key is True: + key = 'true' + elif key is False: + key = 'false' + elif key is None: + key = 'null' + elif self.skipkeys: + continue + else: + raise TypeError("key %r is not a string" % (key,)) + if first: + first = False + else: + yield item_separator + yield encoder(key) + yield key_separator + for chunk in self._iterencode(value, markers): + yield chunk + if newline_indent is not None: + self.current_indent_level -= 1 + yield self._newline_indent() + yield '}' + if markers is not None: + del markers[markerid] + + def _iterencode(self, o, markers=None): + if isinstance(o, basestring): + if self.ensure_ascii: + encoder = encode_basestring_ascii + else: + encoder = encode_basestring + _encoding = self.encoding + if (_encoding is not None and isinstance(o, str) + and not (_need_utf8 and _encoding == 'utf-8')): + o = o.decode(_encoding) + yield encoder(o) + elif o is None: + yield 'null' + elif o is True: + yield 'true' + elif o is False: + yield 'false' + elif isinstance(o, (int, long)): + yield str(o) + elif isinstance(o, float): + yield floatstr(o, self.allow_nan) + elif isinstance(o, (list, tuple)): + for chunk in self._iterencode_list(o, markers): + yield chunk + elif isinstance(o, dict): + for chunk in self._iterencode_dict(o, markers): + yield chunk + else: + if markers is not None: + markerid = id(o) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = o + for chunk in self._iterencode_default(o, markers): + yield chunk + if markers is not None: + del markers[markerid] + + def _iterencode_default(self, o, markers=None): + newobj = self.default(o) + return self._iterencode(newobj, markers) + + def default(self, o): + """ + Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + """ + raise TypeError("%r is not JSON serializable" % (o,)) + + def encode(self, o): + """ + Return a JSON string representation of a Python data structure. + + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo":["bar", "baz"]}' + """ + # This is for extremely simple cases and benchmarks... + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8' and _need_utf8)): + o = o.decode(_encoding) + return encode_basestring_ascii(o) + # This doesn't pass the iterator directly to ''.join() because it + # sucks at reporting exceptions. It's going to do this internally + # anyway because it uses PySequence_Fast or similar. + chunks = list(self.iterencode(o)) + return ''.join(chunks) + + def iterencode(self, o): + """ + Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + """ + if self.check_circular: + markers = {} + else: + markers = None + return self._iterencode(o, markers) + +__all__ = ['JSONEncoder'] diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/jsonfilter.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/jsonfilter.py new file mode 100644 index 0000000..01ca21d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/jsonfilter.py @@ -0,0 +1,40 @@ +import simplejson +import cgi + +class JSONFilter(object): + def __init__(self, app, mime_type='text/x-json'): + self.app = app + self.mime_type = mime_type + + def __call__(self, environ, start_response): + # Read JSON POST input to jsonfilter.json if matching mime type + response = {'status': '200 OK', 'headers': []} + def json_start_response(status, headers): + response['status'] = status + response['headers'].extend(headers) + environ['jsonfilter.mime_type'] = self.mime_type + if environ.get('REQUEST_METHOD', '') == 'POST': + if environ.get('CONTENT_TYPE', '') == self.mime_type: + args = [_ for _ in [environ.get('CONTENT_LENGTH')] if _] + data = environ['wsgi.input'].read(*map(int, args)) + environ['jsonfilter.json'] = simplejson.loads(data) + res = simplejson.dumps(self.app(environ, json_start_response)) + jsonp = cgi.parse_qs(environ.get('QUERY_STRING', '')).get('jsonp') + if jsonp: + content_type = 'text/javascript' + res = ''.join(jsonp + ['(', res, ')']) + elif 'Opera' in environ.get('HTTP_USER_AGENT', ''): + # Opera has bunk XMLHttpRequest support for most mime types + content_type = 'text/plain' + else: + content_type = self.mime_type + headers = [ + ('Content-type', content_type), + ('Content-length', len(res)), + ] + headers.extend(response['headers']) + start_response(response['status'], headers) + return [res] + +def factory(app, global_conf, **kw): + return JSONFilter(app, **kw) diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/scanner.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/scanner.py new file mode 100644 index 0000000..64f4999 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/scanner.py @@ -0,0 +1,63 @@ +""" +Iterator based sre token scanner +""" +import sre_parse, sre_compile, sre_constants +from sre_constants import BRANCH, SUBPATTERN +from re import VERBOSE, MULTILINE, DOTALL +import re + +__all__ = ['Scanner', 'pattern'] + +FLAGS = (VERBOSE | MULTILINE | DOTALL) +class Scanner(object): + def __init__(self, lexicon, flags=FLAGS): + self.actions = [None] + # combine phrases into a compound pattern + s = sre_parse.Pattern() + s.flags = flags + p = [] + for idx, token in enumerate(lexicon): + phrase = token.pattern + try: + subpattern = sre_parse.SubPattern(s, + [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))]) + except sre_constants.error: + raise + p.append(subpattern) + self.actions.append(token) + + p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) + self.scanner = sre_compile.compile(p) + + + def iterscan(self, string, idx=0, context=None): + """ + Yield match, end_idx for each match + """ + match = self.scanner.scanner(string, idx).match + actions = self.actions + lastend = idx + end = len(string) + while True: + m = match() + if m is None: + break + matchbegin, matchend = m.span() + if lastend == matchend: + break + action = actions[m.lastindex] + if action is not None: + rval, next_pos = action(m, context) + if next_pos is not None and next_pos != matchend: + # "fast forward" the scanner + matchend = next_pos + match = self.scanner.scanner(string, matchend).match + yield rval, matchend + lastend = matchend + +def pattern(pattern, flags=FLAGS): + def decorator(fn): + fn.pattern = pattern + fn.regex = re.compile(pattern, flags) + return fn + return decorator diff --git a/WebKitTools/Scripts/webkitpy/tool/__init__.py b/WebKitTools/Scripts/webkitpy/tool/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/__init__.py b/WebKitTools/Scripts/webkitpy/tool/bot/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/irc_command.py b/WebKitTools/Scripts/webkitpy/tool/bot/irc_command.py new file mode 100644 index 0000000..c21fdc6 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/irc_command.py @@ -0,0 +1,82 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import random +import webkitpy.common.config.irc as config_irc + +from webkitpy.common.checkout.changelog import view_source_url +from webkitpy.tool.bot.queueengine import TerminateQueue +from webkitpy.common.net.bugzilla import parse_bug_id +from webkitpy.common.system.executive import ScriptError + +# FIXME: Merge with Command? +class IRCCommand(object): + def execute(self, nick, args, tool, sheriff): + raise NotImplementedError, "subclasses must implement" + + +class LastGreenRevision(IRCCommand): + def execute(self, nick, args, tool, sheriff): + return "%s: %s" % (nick, + view_source_url(tool.buildbot.last_green_revision())) + + +class Restart(IRCCommand): + def execute(self, nick, args, tool, sheriff): + tool.irc().post("Restarting...") + raise TerminateQueue() + + +class Rollout(IRCCommand): + def execute(self, nick, args, tool, sheriff): + if len(args) < 2: + tool.irc().post("%s: Usage: SVN_REVISION REASON" % nick) + return + svn_revision = args[0] + rollout_reason = " ".join(args[1:]) + tool.irc().post("Preparing rollout for r%s..." % svn_revision) + try: + complete_reason = "%s (Requested by %s on %s)." % ( + rollout_reason, nick, config_irc.channel) + bug_id = sheriff.post_rollout_patch(svn_revision, complete_reason) + bug_url = tool.bugs.bug_url_for_bug_id(bug_id) + tool.irc().post("%s: Created rollout: %s" % (nick, bug_url)) + except ScriptError, e: + tool.irc().post("%s: Failed to create rollout patch:" % nick) + tool.irc().post("%s" % e) + bug_id = parse_bug_id(e.output) + if bug_id: + tool.irc().post("Ugg... Might have created %s" % + tool.bugs.bug_url_for_bug_id(bug_id)) + + +class Hi(IRCCommand): + def execute(self, nick, args, tool, sheriff): + quips = tool.bugs.quips() + quips.append('"Only you can prevent forest fires." -- Smokey the Bear') + return random.choice(quips) diff --git a/WebKitTools/Scripts/webkitpy/patchcollection.py b/WebKitTools/Scripts/webkitpy/tool/bot/patchcollection.py index 7e8603c..9a2cdfa 100644 --- a/WebKitTools/Scripts/webkitpy/patchcollection.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/patchcollection.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/WebKitTools/Scripts/webkitpy/patchcollection_unittest.py b/WebKitTools/Scripts/webkitpy/tool/bot/patchcollection_unittest.py index 811fed9..4ec6e25 100644 --- a/WebKitTools/Scripts/webkitpy/patchcollection_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/patchcollection_unittest.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -29,8 +28,8 @@ import unittest -from webkitpy.mock import Mock -from webkitpy.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate +from webkitpy.tool.bot.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate +from webkitpy.thirdparty.mock import Mock class TestPersistentPatchCollectionDelegate(PersistentPatchCollectionDelegate): diff --git a/WebKitTools/Scripts/webkitpy/queueengine.py b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine.py index d14177d..2c45ab6 100644 --- a/WebKitTools/Scripts/webkitpy/queueengine.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -34,9 +33,14 @@ import traceback from datetime import datetime, timedelta -from webkitpy.executive import ScriptError -from webkitpy.webkit_logging import log, OutputTee -from webkitpy.statusserver import StatusServer +from webkitpy.common.net.statusserver import StatusServer +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.system.deprecated_logging import log, OutputTee + + +class TerminateQueue(Exception): + pass + class QueueEngineDelegate: def queue_log_path(self): @@ -66,9 +70,10 @@ class QueueEngineDelegate: class QueueEngine: - def __init__(self, name, delegate): + def __init__(self, name, delegate, wakeup_event): self._name = name self._delegate = delegate + self._wakeup_event = wakeup_event self._output_tee = OutputTee() log_date_format = "%Y-%m-%d %H:%M:%S" @@ -101,7 +106,8 @@ class QueueEngine: # This looks fixed, no? self._open_work_log(work_item) try: - self._delegate.process_work_item(work_item) + if not self._delegate.process_work_item(work_item): + self._sleep("Unable to process work item.") except ScriptError, e: # Use a special exit code to indicate that the error was already # handled in the child process and we should just keep looping. @@ -109,6 +115,9 @@ class QueueEngine: continue message = "Unexpected failure when landing patch! Please file a bug against webkit-patch.\n%s" % e.message_with_output() self._delegate.handle_unexpected_error(work_item, message) + except TerminateQueue, e: + log("\nTerminateQueue exception received.") + return 0 except KeyboardInterrupt, e: log("\nUser terminated queue.") return 1 @@ -133,12 +142,11 @@ class QueueEngine: self._output_tee.remove_log(self._work_log) self._work_log = None - @classmethod - def _sleep_message(cls, message): - wake_time = datetime.now() + timedelta(seconds=cls.seconds_to_sleep) - return "%s Sleeping until %s (%s)." % (message, wake_time.strftime(cls.log_date_format), cls.sleep_duration_text) + def _sleep_message(self, message): + wake_time = datetime.now() + timedelta(seconds=self.seconds_to_sleep) + return "%s Sleeping until %s (%s)." % (message, wake_time.strftime(self.log_date_format), self.sleep_duration_text) - @classmethod - def _sleep(cls, message): - log(cls._sleep_message(message)) - time.sleep(cls.seconds_to_sleep) + def _sleep(self, message): + log(self._sleep_message(message)) + self._wakeup_event.wait(self.seconds_to_sleep) + self._wakeup_event.clear() diff --git a/WebKitTools/Scripts/webkitpy/queueengine_unittest.py b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine_unittest.py index a4036ea..626181d 100644 --- a/WebKitTools/Scripts/webkitpy/queueengine_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine_unittest.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -30,10 +29,11 @@ import os import shutil import tempfile +import threading import unittest -from webkitpy.executive import ScriptError -from webkitpy.queueengine import QueueEngine, QueueEngineDelegate +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate class LoggingDelegate(QueueEngineDelegate): def __init__(self, test): @@ -86,6 +86,7 @@ class LoggingDelegate(QueueEngineDelegate): def process_work_item(self, work_item): self.record("process_work_item") self._test.assertEquals(work_item, "work_item") + return True def handle_unexpected_error(self, work_item, message): self.record("handle_unexpected_error") @@ -111,7 +112,7 @@ class NotSafeToProceedDelegate(LoggingDelegate): class FastQueueEngine(QueueEngine): def __init__(self, delegate): - QueueEngine.__init__(self, "fast-queue", delegate) + QueueEngine.__init__(self, "fast-queue", delegate, threading.Event()) # No sleep for the wicked. seconds_to_sleep = 0 @@ -123,7 +124,7 @@ class FastQueueEngine(QueueEngine): class QueueEngineTest(unittest.TestCase): def test_trivial(self): delegate = LoggingDelegate(self) - work_queue = QueueEngine("trivial-queue", delegate) + work_queue = QueueEngine("trivial-queue", delegate, threading.Event()) work_queue.run() self.assertEquals(delegate._callbacks, LoggingDelegate.expected_callbacks) self.assertTrue(os.path.exists(os.path.join(self.temp_dir, "queue_log_path"))) @@ -131,7 +132,7 @@ class QueueEngineTest(unittest.TestCase): def test_unexpected_error(self): delegate = ThrowErrorDelegate(self, 3) - work_queue = QueueEngine("error-queue", delegate) + work_queue = QueueEngine("error-queue", delegate, threading.Event()) work_queue.run() expected_callbacks = LoggingDelegate.expected_callbacks[:] work_item_index = expected_callbacks.index('process_work_item') @@ -142,7 +143,7 @@ class QueueEngineTest(unittest.TestCase): def test_handled_error(self): delegate = ThrowErrorDelegate(self, QueueEngine.handled_error_code) - work_queue = QueueEngine("handled-error-queue", delegate) + work_queue = QueueEngine("handled-error-queue", delegate, threading.Event()) work_queue.run() self.assertEquals(delegate._callbacks, LoggingDelegate.expected_callbacks) diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/sheriff.py b/WebKitTools/Scripts/webkitpy/tool/bot/sheriff.py new file mode 100644 index 0000000..a38c3cf --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/sheriff.py @@ -0,0 +1,131 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.common.checkout.changelog import view_source_url +from webkitpy.common.net.bugzilla import parse_bug_id +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.grammar import join_with_separators + + +class Sheriff(object): + def __init__(self, tool, sheriffbot): + self._tool = tool + self._sheriffbot = sheriffbot + + def post_irc_warning(self, commit_info, builders): + irc_nicknames = sorted([party.irc_nickname for + party in commit_info.responsible_parties() + if party.irc_nickname]) + irc_prefix = ": " if irc_nicknames else "" + irc_message = "%s%s%s might have broken %s" % ( + ", ".join(irc_nicknames), + irc_prefix, + view_source_url(commit_info.revision()), + join_with_separators([builder.name() for builder in builders])) + + self._tool.irc().post(irc_message) + + def post_rollout_patch(self, svn_revision, rollout_reason): + # Ensure that svn_revision is a number (and not an option to + # create-rollout). + try: + svn_revision = int(svn_revision) + except: + raise ScriptError(message="Invalid svn revision number \"%s\"." + % svn_revision) + + if rollout_reason.startswith("-"): + raise ScriptError(message="The rollout reason may not begin " + "with - (\"%s\")." % rollout_reason) + + output = self._sheriffbot.run_webkit_patch([ + "create-rollout", + "--force-clean", + # In principle, we should pass --non-interactive here, but it + # turns out that create-rollout doesn't need it yet. We can't + # pass it prophylactically because we reject unrecognized command + # line switches. + "--parent-command=sheriff-bot", + svn_revision, + rollout_reason, + ]) + return parse_bug_id(output) + + def _rollout_reason(self, builders): + # FIXME: This should explain which layout tests failed + # however, that would require Build objects here, either passed + # in through failure_info, or through Builder.latest_build. + names = [builder.name() for builder in builders] + return "Caused builders %s to fail." % join_with_separators(names) + + def post_automatic_rollout_patch(self, commit_info, builders): + # For now we're only posting rollout patches for commit-queue patches. + commit_bot_email = "eseidel@chromium.org" + if commit_bot_email == commit_info.committer_email(): + try: + self.post_rollout_patch(commit_info.revision(), + self._rollout_reason(builders)) + except ScriptError, e: + log("Failed to create-rollout.") + + def post_blame_comment_on_bug(self, commit_info, builders, blame_list): + if not commit_info.bug_id(): + return + comment = "%s might have broken %s" % ( + view_source_url(commit_info.revision()), + join_with_separators([builder.name() for builder in builders])) + if len(blame_list) > 1: + comment += "\nThe following changes are on the blame list:\n" + comment += "\n".join(map(view_source_url, blame_list)) + self._tool.bugs.post_comment_to_bug(commit_info.bug_id(), + comment, + cc=self._sheriffbot.watchers) + + # FIXME: Should some of this logic be on BuildBot? + def provoke_flaky_builders(self, revisions_causing_failures): + # We force_build builders that are red but have not "failed" (i.e., + # been red twice). We do this to avoid a deadlock situation where a + # flaky test blocks the commit-queue and there aren't any other + # patches being landed to re-spin the builder. + failed_builders = sum([revisions_causing_failures[key] for + key in revisions_causing_failures.keys()], []) + failed_builder_names = \ + set([builder.name() for builder in failed_builders]) + idle_red_builder_names = \ + set([builder["name"] + for builder in self._tool.buildbot.idle_red_core_builders()]) + + # We only want to provoke these builders if they are idle and have not + # yet "failed" (i.e., been red twice) to avoid overloading the bots. + flaky_builder_names = idle_red_builder_names - failed_builder_names + + for name in flaky_builder_names: + flaky_builder = self._tool.buildbot.builder_with_name(name) + flaky_builder.force_build(username=self._sheriffbot.name, + comments="Probe for flakiness.") diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/sheriff_unittest.py b/WebKitTools/Scripts/webkitpy/tool/bot/sheriff_unittest.py new file mode 100644 index 0000000..dd048a1 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/sheriff_unittest.py @@ -0,0 +1,105 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import unittest + +from webkitpy.common.net.buildbot import Builder +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.bot.sheriff import Sheriff +from webkitpy.tool.mocktool import MockTool, mock_builder + + +class MockSheriffBot(object): + name = "mock-sheriff-bot" + watchers = [ + "watcher@example.com", + ] + + def run_webkit_patch(self, args): + return "Created bug https://bugs.webkit.org/show_bug.cgi?id=36936\n" + + +class SheriffTest(unittest.TestCase): + def test_rollout_reason(self): + sheriff = Sheriff(MockTool(), MockSheriffBot()) + builders = [ + Builder("Foo", None), + Builder("Bar", None), + ] + reason = "Caused builders Foo and Bar to fail." + self.assertEquals(sheriff._rollout_reason(builders), reason) + + def test_post_blame_comment_on_bug(self): + def run(): + sheriff = Sheriff(MockTool(), MockSheriffBot()) + builders = [ + Builder("Foo", None), + Builder("Bar", None), + ] + commit_info = Mock() + commit_info.bug_id = lambda: None + commit_info.revision = lambda: 4321 + # Should do nothing with no bug_id + sheriff.post_blame_comment_on_bug(commit_info, builders, []) + sheriff.post_blame_comment_on_bug(commit_info, builders, [2468, 5646]) + # Should try to post a comment to the bug, but MockTool.bugs does nothing. + commit_info.bug_id = lambda: 1234 + sheriff.post_blame_comment_on_bug(commit_info, builders, []) + sheriff.post_blame_comment_on_bug(commit_info, builders, [3432]) + sheriff.post_blame_comment_on_bug(commit_info, builders, [841, 5646]) + + expected_stderr = u"MOCK bug comment: bug_id=1234, cc=['watcher@example.com']\n--- Begin comment ---\\http://trac.webkit.org/changeset/4321 might have broken Foo and Bar\n--- End comment ---\n\nMOCK bug comment: bug_id=1234, cc=['watcher@example.com']\n--- Begin comment ---\\http://trac.webkit.org/changeset/4321 might have broken Foo and Bar\n--- End comment ---\n\nMOCK bug comment: bug_id=1234, cc=['watcher@example.com']\n--- Begin comment ---\\http://trac.webkit.org/changeset/4321 might have broken Foo and Bar\nThe following changes are on the blame list:\nhttp://trac.webkit.org/changeset/841\nhttp://trac.webkit.org/changeset/5646\n--- End comment ---\n\n" + OutputCapture().assert_outputs(self, run, expected_stderr=expected_stderr) + + def test_provoke_flaky_builders(self): + def run(): + tool = MockTool() + tool.buildbot.light_tree_on_fire() + sheriff = Sheriff(tool, MockSheriffBot()) + revisions_causing_failures = {} + sheriff.provoke_flaky_builders(revisions_causing_failures) + expected_stderr = "MOCK: force_build: name=Builder2, username=mock-sheriff-bot, comments=Probe for flakiness.\n" + OutputCapture().assert_outputs(self, run, expected_stderr=expected_stderr) + + def test_post_blame_comment_on_bug(self): + sheriff = Sheriff(MockTool(), MockSheriffBot()) + builders = [ + Builder("Foo", None), + Builder("Bar", None), + ] + commit_info = Mock() + commit_info.bug_id = lambda: None + commit_info.revision = lambda: 4321 + commit_info.committer = lambda: None + commit_info.committer_email = lambda: "foo@example.com" + commit_info.reviewer = lambda: None + commit_info.author = lambda: None + sheriff.post_automatic_rollout_patch(commit_info, builders) + diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot.py b/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot.py new file mode 100644 index 0000000..43aa9c3 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot.py @@ -0,0 +1,93 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import webkitpy.tool.bot.irc_command as irc_command + +from webkitpy.common.net.irc.ircbot import IRCBotDelegate +from webkitpy.common.thread.threadedmessagequeue import ThreadedMessageQueue + + +class _IRCThreadTearoff(IRCBotDelegate): + def __init__(self, password, message_queue, wakeup_event): + self._password = password + self._message_queue = message_queue + self._wakeup_event = wakeup_event + + # IRCBotDelegate methods + + def irc_message_received(self, nick, message): + self._message_queue.post([nick, message]) + self._wakeup_event.set() + + def irc_nickname(self): + return "sheriffbot" + + def irc_password(self): + return self._password + + +class SheriffIRCBot(object): + # FIXME: Lame. We should have an auto-registering CommandCenter. + commands = { + "last-green-revision": irc_command.LastGreenRevision, + "restart": irc_command.Restart, + "rollout": irc_command.Rollout, + "hi": irc_command.Hi, + } + + def __init__(self, tool, sheriff): + self._tool = tool + self._sheriff = sheriff + self._message_queue = ThreadedMessageQueue() + + def irc_delegate(self): + return _IRCThreadTearoff(self._tool.irc_password, + self._message_queue, + self._tool.wakeup_event) + + def process_message(self, message): + (nick, request) = message + tokenized_request = request.strip().split(" ") + if not tokenized_request: + return + command = self.commands.get(tokenized_request[0]) + if not command: + self._tool.irc().post("%s: Available commands: %s" % ( + nick, ", ".join(self.commands.keys()))) + return + response = command().execute(nick, + tokenized_request[1:], + self._tool, + self._sheriff) + if response: + self._tool.irc().post(response) + + def process_pending_messages(self): + (messages, is_running) = self._message_queue.take_all() + for message in messages: + self.process_message(message) diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot_unittest.py b/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot_unittest.py new file mode 100644 index 0000000..d5116e4 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot_unittest.py @@ -0,0 +1,91 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import random + +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.bot.sheriff import Sheriff +from webkitpy.tool.bot.sheriffircbot import SheriffIRCBot +from webkitpy.tool.bot.sheriff_unittest import MockSheriffBot +from webkitpy.tool.mocktool import MockTool + + +def run(message): + tool = MockTool() + tool.ensure_irc_connected(None) + bot = SheriffIRCBot(tool, Sheriff(tool, MockSheriffBot())) + bot._message_queue.post(["mock_nick", message]) + bot.process_pending_messages() + + +class SheriffIRCBotTest(unittest.TestCase): + def test_hi(self): + random.seed(23324) + expected_stderr = 'MOCK: irc.post: "Only you can prevent forest fires." -- Smokey the Bear\n' + OutputCapture().assert_outputs(self, run, args=["hi"], expected_stderr=expected_stderr) + + def test_bogus(self): + expected_stderr = "MOCK: irc.post: mock_nick: Available commands: rollout, hi, restart, last-green-revision\n" + OutputCapture().assert_outputs(self, run, args=["bogus"], expected_stderr=expected_stderr) + + def test_lgr(self): + expected_stderr = "MOCK: irc.post: mock_nick: http://trac.webkit.org/changeset/9479\n" + OutputCapture().assert_outputs(self, run, args=["last-green-revision"], expected_stderr=expected_stderr) + + def test_rollout(self): + expected_stderr = "MOCK: irc.post: Preparing rollout for r21654...\nMOCK: irc.post: mock_nick: Created rollout: http://example.com/36936\n" + OutputCapture().assert_outputs(self, run, args=["rollout 21654 This patch broke the world"], expected_stderr=expected_stderr) + + def test_rollout_bananas(self): + expected_stderr = "MOCK: irc.post: mock_nick: Usage: SVN_REVISION REASON\n" + OutputCapture().assert_outputs(self, run, args=["rollout bananas"], expected_stderr=expected_stderr) + + def test_rollout_invalidate_revision(self): + expected_stderr = ("MOCK: irc.post: Preparing rollout for r--component=Tools...\n" + "MOCK: irc.post: mock_nick: Failed to create rollout patch:\n" + "MOCK: irc.post: Invalid svn revision number \"--component=Tools\".\n") + OutputCapture().assert_outputs(self, run, + args=["rollout " + "--component=Tools 21654"], + expected_stderr=expected_stderr) + + def test_rollout_invalidate_reason(self): + expected_stderr = ("MOCK: irc.post: Preparing rollout for " + "r21654...\nMOCK: irc.post: mock_nick: Failed to " + "create rollout patch:\nMOCK: irc.post: The rollout" + " reason may not begin with - (\"-bad (Requested " + "by mock_nick on #webkit).\").\n") + OutputCapture().assert_outputs(self, run, + args=["rollout " + "21654 -bad"], + expected_stderr=expected_stderr) + + def test_rollout_no_reason(self): + expected_stderr = "MOCK: irc.post: mock_nick: Usage: SVN_REVISION REASON\n" + OutputCapture().assert_outputs(self, run, args=["rollout 21654"], expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/__init__.py b/WebKitTools/Scripts/webkitpy/tool/commands/__init__.py new file mode 100644 index 0000000..71c3719 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/__init__.py @@ -0,0 +1,4 @@ +# Required for Python to search this directory for module files + +from webkitpy.tool.commands.prettydiff import PrettyDiff +# FIXME: Add the rest of the commands here. diff --git a/WebKitTools/Scripts/webkitpy/commands/abstractsequencedcommand.py b/WebKitTools/Scripts/webkitpy/tool/commands/abstractsequencedcommand.py index 53af5b1..fc5a794 100644 --- a/WebKitTools/Scripts/webkitpy/commands/abstractsequencedcommand.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/abstractsequencedcommand.py @@ -26,8 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.multicommandtool import AbstractDeclarativeCommand -from webkitpy.stepsequence import StepSequence +from webkitpy.tool.commands.stepsequence import StepSequence +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand class AbstractSequencedCommand(AbstractDeclarativeCommand): diff --git a/WebKitTools/Scripts/webkitpy/commands/commandtest.py b/WebKitTools/Scripts/webkitpy/tool/commands/commandtest.py index a56cb05..887802c 100644 --- a/WebKitTools/Scripts/webkitpy/commands/commandtest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/commandtest.py @@ -28,11 +28,11 @@ import unittest -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.mocktool import MockTool +from webkitpy.thirdparty.mock import Mock class CommandsTest(unittest.TestCase): - def assert_execute_outputs(self, command, args, expected_stdout="", expected_stderr="", options=Mock(), tool=MockBugzillaTool()): + def assert_execute_outputs(self, command, args, expected_stdout="", expected_stderr="", options=Mock(), tool=MockTool()): command.bind_to_tool(tool) OutputCapture().assert_outputs(self, command.execute, [options, args, tool], expected_stdout=expected_stdout, expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/commands/download.py b/WebKitTools/Scripts/webkitpy/tool/commands/download.py index 49a6862..d960bbe 100644 --- a/WebKitTools/Scripts/webkitpy/commands/download.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/download.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -31,18 +31,25 @@ import os from optparse import make_option -import webkitpy.steps as steps +import webkitpy.tool.steps as steps -from webkitpy.bugzilla import parse_bug_id -# We could instead use from modules import buildsteps and then prefix every buildstep with "buildsteps." -from webkitpy.changelogs import ChangeLog -from webkitpy.commands.abstractsequencedcommand import AbstractSequencedCommand -from webkitpy.comments import bug_comment_from_commit_text -from webkitpy.executive import ScriptError -from webkitpy.grammar import pluralize -from webkitpy.webkit_logging import error, log -from webkitpy.multicommandtool import AbstractDeclarativeCommand -from webkitpy.stepsequence import StepSequence +from webkitpy.common.checkout.changelog import ChangeLog, view_source_url +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand +from webkitpy.tool.commands.stepsequence import StepSequence +from webkitpy.tool.comments import bug_comment_from_commit_text +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand +from webkitpy.common.system.deprecated_logging import error, log + + +class Update(AbstractSequencedCommand): + name = "update" + help_text = "Update working copy (used internally)" + steps = [ + steps.CleanWorkingDirectory, + steps.Update, + ] class Build(AbstractSequencedCommand): @@ -74,6 +81,7 @@ class Land(AbstractSequencedCommand): steps = [ steps.EnsureBuildersAreGreen, steps.UpdateChangeLogsWithReviewer, + steps.ValidateReviewer, steps.EnsureBuildersAreGreen, steps.Build, steps.RunTests, @@ -86,7 +94,7 @@ If a bug id is provided, or one can be found in the ChangeLog land will update t def _prepare_state(self, options, args, tool): return { - "bug_id" : (args and args[0]) or parse_bug_id(tool.scm().create_patch()), + "bug_id" : (args and args[0]) or tool.checkout().bug_id_for_this_commit() } @@ -209,6 +217,7 @@ class AbstractPatchLandingCommand(AbstractPatchSequencingCommand): steps.CleanWorkingDirectory, steps.Update, steps.ApplyPatch, + steps.ValidateReviewer, steps.EnsureBuildersAreGreen, steps.Build, steps.RunTests, @@ -240,11 +249,93 @@ class LandFromBug(AbstractPatchLandingCommand, ProcessBugsMixin): show_in_main_help = True -class Rollout(AbstractSequencedCommand): +class AbstractRolloutPrepCommand(AbstractSequencedCommand): + argument_names = "REVISION REASON" + + def _commit_info(self, revision): + commit_info = self.tool.checkout().commit_info_for_revision(revision) + if commit_info and commit_info.bug_id(): + # Note: Don't print a bug URL here because it will confuse the + # SheriffBot because the SheriffBot just greps the output + # of create-rollout for bug URLs. It should do better + # parsing instead. + log("Preparing rollout for bug %s." % commit_info.bug_id()) + return commit_info + log("Unable to parse bug number from diff.") + + def _prepare_state(self, options, args, tool): + revision = args[0] + commit_info = self._commit_info(revision) + cc_list = sorted([party.bugzilla_email() + for party in commit_info.responsible_parties() + if party.bugzilla_email()]) + return { + "revision": revision, + "bug_id": commit_info.bug_id(), + # FIXME: We should used the list as the canonical representation. + "bug_cc": ",".join(cc_list), + "reason": args[1], + } + + +class PrepareRollout(AbstractRolloutPrepCommand): + name = "prepare-rollout" + help_text = "Revert the given revision in the working copy and prepare ChangeLogs with revert reason" + long_help = """Updates the working copy. +Applies the inverse diff for the provided revision. +Creates an appropriate rollout ChangeLog, including a trac link and bug link. +""" + steps = [ + steps.CleanWorkingDirectory, + steps.Update, + steps.RevertRevision, + steps.PrepareChangeLogForRevert, + ] + + +class CreateRollout(AbstractRolloutPrepCommand): + name = "create-rollout" + help_text = "Creates a bug to track a broken SVN revision and uploads a rollout patch." + steps = [ + steps.CleanWorkingDirectory, + steps.Update, + steps.RevertRevision, + steps.CreateBug, + steps.PrepareChangeLogForRevert, + steps.PostDiffForRevert, + ] + + def _prepare_state(self, options, args, tool): + state = AbstractRolloutPrepCommand._prepare_state(self, options, args, tool) + # Currently, state["bug_id"] points to the bug that caused the + # regression. We want to create a new bug that blocks the old bug + # so we move state["bug_id"] to state["bug_blocked"] and delete the + # old state["bug_id"] so that steps.CreateBug will actually create + # the new bug that we want (and subsequently store its bug id into + # state["bug_id"]) + state["bug_blocked"] = state["bug_id"] + del state["bug_id"] + state["bug_title"] = "REGRESSION(r%s): %s" % (state["revision"], state["reason"]) + state["bug_description"] = "%s broke the build:\n%s" % (view_source_url(state["revision"]), state["reason"]) + # FIXME: If we had more context here, we could link to other open bugs + # that mention the test that regressed. + if options.parent_command == "sheriff-bot": + state["bug_description"] += """ + +This is an automatic bug report generated by the sheriff-bot. If this bug +report was created because of a flaky test, please file a bug for the flaky +test (if we don't already have one on file) and dup this bug against that bug +so that we can track how often these flaky tests case pain. + +"Only you can prevent forest fires." -- Smokey the Bear +""" + return state + + +class Rollout(AbstractRolloutPrepCommand): name = "rollout" show_in_main_help = True help_text = "Revert the given revision in the working copy and optionally commit the revert and re-open the original bug" - argument_names = "REVISION REASON" long_help = """Updates the working copy. Applies the inverse diff for the provided revision. Creates an appropriate rollout ChangeLog, including a trac link and bug link. @@ -258,27 +349,7 @@ Commits the revert and updates the bug (including re-opening the bug if necessar steps.PrepareChangeLogForRevert, steps.EditChangeLog, steps.ConfirmDiff, - steps.CompleteRollout, + steps.Build, + steps.Commit, + steps.ReopenBugAfterRollout, ] - - @staticmethod - def _parse_bug_id_from_revision_diff(tool, revision): - original_diff = tool.scm().diff_for_revision(revision) - return parse_bug_id(original_diff) - - def execute(self, options, args, tool): - revision = args[0] - reason = args[1] - bug_id = self._parse_bug_id_from_revision_diff(tool, revision) - if options.complete_rollout: - if bug_id: - log("Will re-open bug %s after rollout." % bug_id) - else: - log("Failed to parse bug number from diff. No bugs will be updated/reopened after the rollout.") - - state = { - "revision" : revision, - "bug_id" : bug_id, - "reason" : reason, - } - self._sequence.run_and_handle_errors(tool, options, state) diff --git a/WebKitTools/Scripts/webkitpy/commands/download_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/download_unittest.py index f60c5b8..926037c 100644 --- a/WebKitTools/Scripts/webkitpy/commands/download_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/download_unittest.py @@ -26,9 +26,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.download import * -from webkitpy.mock import Mock +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.download import * + class DownloadCommandsTest(CommandsTest): def _default_options(self): @@ -42,7 +43,6 @@ class DownloadCommandsTest(CommandsTest): options.build = True options.test = True options.close_bug = True - options.complete_rollout = False return options def test_build(self): @@ -116,12 +116,32 @@ Not closing bug 42 as attachment 197 has review=+. Assuming there are more patc """ self.assert_execute_outputs(LandFromBug(), [42], options=self._default_options(), expected_stderr=expected_stderr) + def test_prepare_rollout(self): + expected_stderr = "Preparing rollout for bug 42.\nUpdating working directory\nRunning prepare-ChangeLog\n" + self.assert_execute_outputs(PrepareRollout(), [852, "Reason"], options=self._default_options(), expected_stderr=expected_stderr) + + def test_create_rollout(self): + expected_stderr = """Preparing rollout for bug 42. +Updating working directory +MOCK create_bug +bug_title: REGRESSION(r852): Reason +bug_description: http://trac.webkit.org/changeset/852 broke the build: +Reason +Running prepare-ChangeLog +MOCK add_patch_to_bug: bug_id=None, description=ROLLOUT of r852, mark_for_review=False, mark_for_commit_queue=True, mark_for_landing=False +-- Begin comment -- +Any committer can land this patch automatically by marking it commit-queue+. The commit-queue will build and test the patch before landing to ensure that the rollout will be successful. This process takes approximately 15 minutes. + +If you would like to land the rollout faster, you can use the following command: + + webkit-patch land-attachment ATTACHMENT_ID --ignore-builders + +where ATTACHMENT_ID is the ID of this attachment. +-- End comment -- +""" + self.assert_execute_outputs(CreateRollout(), [852, "Reason"], options=self._default_options(), expected_stderr=expected_stderr) + def test_rollout(self): - expected_stderr = "Updating working directory\nRunning prepare-ChangeLog\n\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"webkit-patch land 12345\" to commit the rollout.\n" + expected_stderr = "Preparing rollout for bug 42.\nUpdating working directory\nRunning prepare-ChangeLog\nMOCK: user.open_url: file://...\nBuilding WebKit\n" self.assert_execute_outputs(Rollout(), [852, "Reason"], options=self._default_options(), expected_stderr=expected_stderr) - def test_complete_rollout(self): - options = self._default_options() - options.complete_rollout = True - expected_stderr = "Will re-open bug 12345 after rollout.\nUpdating working directory\nRunning prepare-ChangeLog\nBuilding WebKit\n" - self.assert_execute_outputs(Rollout(), [852, "Reason"], options=options, expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/commands/early_warning_system.py b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem.py index e3e14dd..9ea34c0 100644 --- a/WebKitTools/Scripts/webkitpy/commands/early_warning_system.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -29,11 +28,11 @@ from StringIO import StringIO -from webkitpy.commands.queues import AbstractReviewQueue -from webkitpy.committers import CommitterList -from webkitpy.executive import ScriptError -from webkitpy.webkitport import WebKitPort -from webkitpy.queueengine import QueueEngine +from webkitpy.tool.commands.queues import AbstractReviewQueue +from webkitpy.common.config.committers import CommitterList +from webkitpy.common.config.ports import WebKitPort +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.bot.queueengine import QueueEngine class AbstractEarlyWarningSystem(AbstractReviewQueue): @@ -44,30 +43,51 @@ class AbstractEarlyWarningSystem(AbstractReviewQueue): self.port = WebKitPort.port(self.port_name) def should_proceed_with_work_item(self, patch): + return True + + def _can_build(self): try: self.run_webkit_patch([ "build", self.port.flag(), + "--build", "--build-style=%s" % self._build_style, "--force-clean", + "--no-update", "--quiet"]) - self._update_status("Building", patch) + return True except ScriptError, e: self._update_status("Unable to perform a build") return False - return True - def _review_patch(self, patch): - self.run_webkit_patch([ - "build-attachment", - self.port.flag(), - "--build-style=%s" % self._build_style, - "--force-clean", - "--quiet", - "--non-interactive", - "--parent-command=%s" % self.name, - "--no-update", - patch.id()]) + def _build(self, patch, first_run=False): + try: + args = [ + "build-attachment", + self.port.flag(), + "--build", + "--build-style=%s" % self._build_style, + "--force-clean", + "--quiet", + "--non-interactive", + patch.id()] + if not first_run: + # See commit-queue for an explanation of what we're doing here. + args.append("--no-update") + args.append("--parent-command=%s" % self.name) + self.run_webkit_patch(args) + return True + except ScriptError, e: + if first_run: + return False + raise + + def review_patch(self, patch): + if not self._build(patch, first_run=True): + if not self._can_build(): + return False + self._build(patch) + return True @classmethod def handle_script_error(cls, tool, state, script_error): @@ -95,14 +115,32 @@ class QtEWS(AbstractEarlyWarningSystem): port_name = "qt" -class ChromiumEWS(AbstractEarlyWarningSystem): - name = "chromium-ews" +class WinEWS(AbstractEarlyWarningSystem): + name = "win-ews" + port_name = "win" + + +class AbstractChromiumEWS(AbstractEarlyWarningSystem): port_name = "chromium" watchers = AbstractEarlyWarningSystem.watchers + [ "dglazkov@chromium.org", ] +class ChromiumLinuxEWS(AbstractChromiumEWS): + # FIXME: We should rename this command to cr-linux-ews, but that requires + # a database migration. :( + name = "chromium-ews" + + +class ChromiumWindowsEWS(AbstractChromiumEWS): + name = "cr-win-ews" + + +class ChromiumMacEWS(AbstractChromiumEWS): + name = "cr-mac-ews" + + # For platforms that we can't run inside a VM (like Mac OS X), we require # patches to be uploaded by committers, who are generally trustworthy folk. :) class AbstractCommitterOnlyEWS(AbstractEarlyWarningSystem): diff --git a/WebKitTools/Scripts/webkitpy/commands/early_warning_system_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py index d516b84..4d23a4c 100644 --- a/WebKitTools/Scripts/webkitpy/commands/early_warning_system_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py @@ -28,35 +28,48 @@ import os -from webkitpy.commands.early_warning_system import * -from webkitpy.commands.queuestest import QueuesTest -from webkitpy.mock import Mock +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.earlywarningsystem import * +from webkitpy.tool.commands.queuestest import QueuesTest class EarlyWarningSytemTest(QueuesTest): - def test_chromium_ews(self): + def test_failed_builds(self): + ews = ChromiumLinuxEWS() + ews._build = lambda patch, first_run=False: False + ews._can_build = lambda: True + ews.review_patch(Mock()) + + def test_chromium_linux_ews(self): + expected_stderr = { + "begin_work_queue": "CAUTION: chromium-ews will discard all local changes in \"%s\"\nRunning WebKit chromium-ews.\n" % os.getcwd(), + "handle_unexpected_error": "Mock error message\n", + } + self.assert_queue_outputs(ChromiumLinuxEWS(), expected_stderr=expected_stderr) + + def test_chromium_windows_ews(self): expected_stderr = { - "begin_work_queue" : "CAUTION: chromium-ews will discard all local changes in \"%s\"\nRunning WebKit chromium-ews.\n" % os.getcwd(), - "handle_unexpected_error" : "Mock error message\n", + "begin_work_queue": "CAUTION: cr-win-ews will discard all local changes in \"%s\"\nRunning WebKit cr-win-ews.\n" % os.getcwd(), + "handle_unexpected_error": "Mock error message\n", } - self.assert_queue_outputs(ChromiumEWS(), expected_stderr=expected_stderr) + self.assert_queue_outputs(ChromiumWindowsEWS(), expected_stderr=expected_stderr) def test_qt_ews(self): expected_stderr = { - "begin_work_queue" : "CAUTION: qt-ews will discard all local changes in \"%s\"\nRunning WebKit qt-ews.\n" % os.getcwd(), - "handle_unexpected_error" : "Mock error message\n", + "begin_work_queue": "CAUTION: qt-ews will discard all local changes in \"%s\"\nRunning WebKit qt-ews.\n" % os.getcwd(), + "handle_unexpected_error": "Mock error message\n", } self.assert_queue_outputs(QtEWS(), expected_stderr=expected_stderr) def test_gtk_ews(self): expected_stderr = { - "begin_work_queue" : "CAUTION: gtk-ews will discard all local changes in \"%s\"\nRunning WebKit gtk-ews.\n" % os.getcwd(), - "handle_unexpected_error" : "Mock error message\n", + "begin_work_queue": "CAUTION: gtk-ews will discard all local changes in \"%s\"\nRunning WebKit gtk-ews.\n" % os.getcwd(), + "handle_unexpected_error": "Mock error message\n", } self.assert_queue_outputs(GtkEWS(), expected_stderr=expected_stderr) def test_mac_ews(self): expected_stderr = { - "begin_work_queue" : "CAUTION: mac-ews will discard all local changes in \"%s\"\nRunning WebKit mac-ews.\n" % os.getcwd(), - "handle_unexpected_error" : "Mock error message\n", + "begin_work_queue": "CAUTION: mac-ews will discard all local changes in \"%s\"\nRunning WebKit mac-ews.\n" % os.getcwd(), + "handle_unexpected_error": "Mock error message\n", } self.assert_queue_outputs(MacEWS(), expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/commands/openbugs.py b/WebKitTools/Scripts/webkitpy/tool/commands/openbugs.py index 25bdefc..5da5bbb 100644 --- a/WebKitTools/Scripts/webkitpy/commands/openbugs.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/openbugs.py @@ -29,8 +29,8 @@ import re import sys -from webkitpy.multicommandtool import AbstractDeclarativeCommand -from webkitpy.webkit_logging import log +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand +from webkitpy.common.system.deprecated_logging import log class OpenBugs(AbstractDeclarativeCommand): diff --git a/WebKitTools/Scripts/webkitpy/commands/openbugs_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/openbugs_unittest.py index 71fefd2..40a6e1b 100644 --- a/WebKitTools/Scripts/webkitpy/commands/openbugs_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/openbugs_unittest.py @@ -26,8 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.openbugs import OpenBugs +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.openbugs import OpenBugs class OpenBugsTest(CommandsTest): diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/prettydiff.py b/WebKitTools/Scripts/webkitpy/tool/commands/prettydiff.py new file mode 100644 index 0000000..e3fc00c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/prettydiff.py @@ -0,0 +1,38 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand +import webkitpy.tool.steps as steps + + +class PrettyDiff(AbstractSequencedCommand): + name = "pretty-diff" + help_text = "Shows the pretty diff in the default browser" + steps = [ + steps.ConfirmDiff, + ] diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/queries.py b/WebKitTools/Scripts/webkitpy/tool/commands/queries.py new file mode 100644 index 0000000..645060c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queries.py @@ -0,0 +1,285 @@ +# Copyright (c) 2009 Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from optparse import make_option + +from webkitpy.common.checkout.commitinfo import CommitInfo +from webkitpy.common.config.committers import CommitterList +from webkitpy.common.net.buildbot import BuildBot +from webkitpy.common.system.user import User +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand +from webkitpy.common.system.deprecated_logging import log + + +class BugsToCommit(AbstractDeclarativeCommand): + name = "bugs-to-commit" + help_text = "List bugs in the commit-queue" + + def execute(self, options, args, tool): + # FIXME: This command is poorly named. It's fetching the commit-queue list here. The name implies it's fetching pending-commit (all r+'d patches). + bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue() + for bug_id in bug_ids: + print "%s" % bug_id + + +class PatchesInCommitQueue(AbstractDeclarativeCommand): + name = "patches-in-commit-queue" + help_text = "List patches in the commit-queue" + + def execute(self, options, args, tool): + patches = tool.bugs.queries.fetch_patches_from_commit_queue() + log("Patches in commit queue:") + for patch in patches: + print patch.url() + + +class PatchesToCommitQueue(AbstractDeclarativeCommand): + name = "patches-to-commit-queue" + help_text = "Patches which should be added to the commit queue" + def __init__(self): + options = [ + make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"), + ] + AbstractDeclarativeCommand.__init__(self, options=options) + + @staticmethod + def _needs_commit_queue(patch): + if patch.commit_queue() == "+": # If it's already cq+, ignore the patch. + log("%s already has cq=%s" % (patch.id(), patch.commit_queue())) + return False + + # We only need to worry about patches from contributers who are not yet committers. + committer_record = CommitterList().committer_by_email(patch.attacher_email()) + if committer_record: + log("%s committer = %s" % (patch.id(), committer_record)) + return not committer_record + + def execute(self, options, args, tool): + patches = tool.bugs.queries.fetch_patches_from_pending_commit_list() + patches_needing_cq = filter(self._needs_commit_queue, patches) + if options.bugs: + bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq) + bugs_needing_cq = sorted(set(bugs_needing_cq)) + for bug_id in bugs_needing_cq: + print "%s" % tool.bugs.bug_url_for_bug_id(bug_id) + else: + for patch in patches_needing_cq: + print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit") + + +class PatchesToReview(AbstractDeclarativeCommand): + name = "patches-to-review" + help_text = "List patches that are pending review" + + def execute(self, options, args, tool): + patch_ids = tool.bugs.queries.fetch_attachment_ids_from_review_queue() + log("Patches pending review:") + for patch_id in patch_ids: + print patch_id + + +class LastGreenRevision(AbstractDeclarativeCommand): + name = "last-green-revision" + help_text = "Prints the last known good revision" + + def execute(self, options, args, tool): + print self.tool.buildbot.last_green_revision() + + +class WhatBroke(AbstractDeclarativeCommand): + name = "what-broke" + help_text = "Print failing buildbots (%s) and what revisions broke them" % BuildBot.default_host + + def _print_builder_line(self, builder_name, max_name_width, status_message): + print "%s : %s" % (builder_name.ljust(max_name_width), status_message) + + # FIXME: This is slightly different from Builder.suspect_revisions_for_green_to_red_transition + # due to needing to detect the "hit the limit" case an print a special message. + def _print_blame_information_for_builder(self, builder_status, name_width, avoid_flakey_tests=True): + builder = self.tool.buildbot.builder_with_name(builder_status["name"]) + red_build = builder.build(builder_status["build_number"]) + (last_green_build, first_red_build) = builder.find_failure_transition(red_build) + if not first_red_build: + self._print_builder_line(builder.name(), name_width, "FAIL (error loading build information)") + return + if not last_green_build: + self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: sometime before %s?)" % first_red_build.revision()) + return + + suspect_revisions = range(first_red_build.revision(), last_green_build.revision(), -1) + suspect_revisions.reverse() + first_failure_message = "" + if (first_red_build == builder.build(builder_status["build_number"])): + first_failure_message = " FIRST FAILURE, possibly a flaky test" + self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s%s)" % (suspect_revisions, first_failure_message)) + for revision in suspect_revisions: + commit_info = self.tool.checkout().commit_info_for_revision(revision) + if commit_info: + print commit_info.blame_string(self.tool.bugs) + else: + print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision + + def execute(self, options, args, tool): + builder_statuses = tool.buildbot.builder_statuses() + longest_builder_name = max(map(len, map(lambda builder: builder["name"], builder_statuses))) + failing_builders = 0 + for builder_status in builder_statuses: + # If the builder is green, print OK, exit. + if builder_status["is_green"]: + continue + self._print_blame_information_for_builder(builder_status, name_width=longest_builder_name) + failing_builders += 1 + if failing_builders: + print "%s of %s are failing" % (failing_builders, pluralize("builder", len(builder_statuses))) + else: + print "All builders are passing!" + + +class WhoBrokeIt(AbstractDeclarativeCommand): + name = "who-broke-it" + help_text = "Print a list of revisions causing failures on %s" % BuildBot.default_host + + def execute(self, options, args, tool): + for revision, builders in self.tool.buildbot.revisions_causing_failures(False).items(): + print "r%s appears to have broken %s" % (revision, [builder.name() for builder in builders]) + + +class ResultsFor(AbstractDeclarativeCommand): + name = "results-for" + help_text = "Print a list of failures for the passed revision from bots on %s" % BuildBot.default_host + argument_names = "REVISION" + + def _print_layout_test_results(self, results): + if not results: + print " No results." + return + for title, files in results.parsed_results().items(): + print " %s" % title + for filename in files: + print " %s" % filename + + def execute(self, options, args, tool): + builders = self.tool.buildbot.builders() + for builder in builders: + print "%s:" % builder.name() + build = builder.build_for_revision(args[0], allow_failed_lookups=True) + self._print_layout_test_results(build.layout_test_results()) + + +class FailureReason(AbstractDeclarativeCommand): + name = "failure-reason" + help_text = "Lists revisions where individual test failures started at %s" % BuildBot.default_host + + def _print_blame_information_for_transition(self, green_build, red_build, failing_tests): + suspect_revisions = green_build.builder().suspect_revisions_for_transition(green_build, red_build) + print "SUCCESS: Build %s (r%s) was the first to show failures: %s" % (red_build._number, red_build.revision(), failing_tests) + print "Suspect revisions:" + for revision in suspect_revisions: + commit_info = self.tool.checkout().commit_info_for_revision(revision) + if commit_info: + print commit_info.blame_string(self.tool.bugs) + else: + print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision + + def _explain_failures_for_builder(self, builder, start_revision): + print "Examining failures for \"%s\", starting at r%s" % (builder.name(), start_revision) + revision_to_test = start_revision + build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True) + layout_test_results = build.layout_test_results() + if not layout_test_results: + # FIXME: This could be made more user friendly. + print "Failed to load layout test results; can't continue. (start revision = r%s)" % start_revision + return 1 + + results_to_explain = set(layout_test_results.failing_tests()) + last_build_with_results = build + print "Starting at %s" % revision_to_test + while results_to_explain: + revision_to_test -= 1 + new_build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True) + if not new_build: + print "No build for %s" % revision_to_test + continue + build = new_build + latest_results = build.layout_test_results() + if not latest_results: + print "No results build %s (r%s)" % (build._number, build.revision()) + continue + failures = set(latest_results.failing_tests()) + if len(failures) >= 20: + # FIXME: We may need to move this logic into the LayoutTestResults class. + # The buildbot stops runs after 20 failures so we don't have full results to work with here. + print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision()) + continue + fixed_results = results_to_explain - failures + if not fixed_results: + print "No change in build %s (r%s), %s unexplained failures (%s in this build)" % (build._number, build.revision(), len(results_to_explain), len(failures)) + last_build_with_results = build + continue + self._print_blame_information_for_transition(build, last_build_with_results, fixed_results) + last_build_with_results = build + results_to_explain -= fixed_results + if results_to_explain: + print "Failed to explain failures: %s" % results_to_explain + return 1 + print "Explained all results for %s" % builder.name() + return 0 + + def _builder_to_explain(self): + builder_statuses = self.tool.buildbot.builder_statuses() + red_statuses = [status for status in builder_statuses if not status["is_green"]] + print "%s failing" % (pluralize("builder", len(red_statuses))) + builder_choices = [status["name"] for status in red_statuses] + # We could offer an "All" choice here. + chosen_name = User.prompt_with_list("Which builder to diagnose:", builder_choices) + # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object. + for status in red_statuses: + if status["name"] == chosen_name: + return (self.tool.buildbot.builder_with_name(chosen_name), status["built_revision"]) + + def execute(self, options, args, tool): + (builder, latest_revision) = self._builder_to_explain() + start_revision = self.tool.user.prompt("Revision to walk backwards from? [%s] " % latest_revision) or latest_revision + if not start_revision: + print "Revision required." + return 1 + return self._explain_failures_for_builder(builder, start_revision=int(start_revision)) + +class TreeStatus(AbstractDeclarativeCommand): + name = "tree-status" + help_text = "Print the status of the %s buildbots" % BuildBot.default_host + long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder +and displayes the status of each builder.""" + + def execute(self, options, args, tool): + for builder in tool.buildbot.builder_statuses(): + status_string = "ok" if builder["is_green"] else "FAIL" + print "%s : %s" % (status_string.ljust(4), builder["name"]) diff --git a/WebKitTools/Scripts/webkitpy/commands/queries_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/queries_unittest.py index b858777..98ed545 100644 --- a/WebKitTools/Scripts/webkitpy/commands/queries_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queries_unittest.py @@ -26,11 +26,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.bugzilla import Bugzilla -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.queries import * -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool +from webkitpy.common.net.bugzilla import Bugzilla +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.queries import * +from webkitpy.tool.mocktool import MockTool class QueryCommandsTest(CommandsTest): def test_bugs_to_commit(self): diff --git a/WebKitTools/Scripts/webkitpy/commands/queues.py b/WebKitTools/Scripts/webkitpy/tool/commands/queues.py index 6ea1c48..f0da379 100644 --- a/WebKitTools/Scripts/webkitpy/commands/queues.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queues.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -35,15 +34,15 @@ from datetime import datetime from optparse import make_option from StringIO import StringIO -from webkitpy.bugzilla import CommitterValidator -from webkitpy.executive import ScriptError -from webkitpy.grammar import pluralize -from webkitpy.webkit_logging import error, log -from webkitpy.multicommandtool import Command -from webkitpy.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate -from webkitpy.statusserver import StatusServer -from webkitpy.stepsequence import StepSequenceErrorHandler -from webkitpy.queueengine import QueueEngine, QueueEngineDelegate +from webkitpy.common.net.bugzilla import CommitterValidator +from webkitpy.common.net.statusserver import StatusServer +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.system.deprecated_logging import error, log +from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler +from webkitpy.tool.bot.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate +from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.multicommandtool import Command class AbstractQueue(Command, QueueEngineDelegate): watchers = [ @@ -57,8 +56,10 @@ class AbstractQueue(Command, QueueEngineDelegate): def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations options_list = (options or []) + [ make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue. Dangerous!"), + make_option("--exit-after-iteration", action="store", type="int", dest="iterations", default=None, help="Stop running the queue after iterating this number of times."), ] Command.__init__(self, "Run the %s" % self.name, options=options_list) + self._iteration_count = 0 def _cc_watchers(self, bug_id): try: @@ -67,24 +68,23 @@ class AbstractQueue(Command, QueueEngineDelegate): traceback.print_exc() log("Failed to CC watchers.") - def _update_status(self, message, patch=None, results_file=None): - self.tool.status_server.update_status(self.name, message, patch, results_file) - - def _did_pass(self, patch): - self._update_status(self._pass_status, patch) + def run_webkit_patch(self, args): + webkit_patch_args = [self.tool.path()] + # FIXME: This is a hack, we should have a more general way to pass global options. + webkit_patch_args += ["--status-host=%s" % self.tool.status_server.host] + webkit_patch_args += map(str, args) + return self.tool.executive.run_and_throw_if_fail(webkit_patch_args) - def _did_fail(self, patch): - self._update_status(self._fail_status, patch) + def _log_directory(self): + return "%s-logs" % self.name - def _did_error(self, patch, reason): - message = "%s: %s" % (self._error_status, reason) - self._update_status(message, patch) + # QueueEngineDelegate methods def queue_log_path(self): - return "%s.log" % self.name + return os.path.join(self._log_directory(), "%s.log" % self.name) - def work_item_log_path(self, patch): - return os.path.join("%s-logs" % self.name, "%s.log" % patch.bug_id()) + def work_item_log_path(self, work_item): + raise NotImplementedError, "subclasses must implement" def begin_work_queue(self): log("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self.tool.scm().checkout_root)) @@ -95,7 +95,8 @@ class AbstractQueue(Command, QueueEngineDelegate): log("Running WebKit %s." % self.name) def should_continue_work_queue(self): - return True + self._iteration_count += 1 + return not self.options.iterations or self._iteration_count <= self.options.iterations def next_work_item(self): raise NotImplementedError, "subclasses must implement" @@ -109,46 +110,62 @@ class AbstractQueue(Command, QueueEngineDelegate): def handle_unexpected_error(self, work_item, message): raise NotImplementedError, "subclasses must implement" - def run_webkit_patch(self, args): - webkit_patch_args = [self.tool.path()] - # FIXME: This is a hack, we should have a more general way to pass global options. - webkit_patch_args += ["--status-host=%s" % self.tool.status_server.host] - webkit_patch_args += map(str, args) - self.tool.executive.run_and_throw_if_fail(webkit_patch_args) - - def log_progress(self, patch_ids): - log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self.name, ", ".join(map(str, patch_ids)))) + # Command methods def execute(self, options, args, tool, engine=QueueEngine): self.options = options self.tool = tool - return engine(self.name, self).run() + return engine(self.name, self, self.tool.wakeup_event).run() @classmethod def _update_status_for_script_error(cls, tool, state, script_error, is_error=False): message = script_error.message if is_error: message = "Error: %s" % message - output = script_error.message_with_output(output_limit=5*1024*1024) # 5MB + output = script_error.message_with_output(output_limit=1024*1024) # 1MB return tool.status_server.update_status(cls.name, message, state["patch"], StringIO(output)) -class CommitQueue(AbstractQueue, StepSequenceErrorHandler): +class AbstractPatchQueue(AbstractQueue): + def _update_status(self, message, patch=None, results_file=None): + self.tool.status_server.update_status(self.name, message, patch, results_file) + + def _did_pass(self, patch): + self._update_status(self._pass_status, patch) + + def _did_fail(self, patch): + self._update_status(self._fail_status, patch) + + def _did_error(self, patch, reason): + message = "%s: %s" % (self._error_status, reason) + self._update_status(message, patch) + + def work_item_log_path(self, patch): + return os.path.join(self._log_directory(), "%s.log" % patch.bug_id()) + + def log_progress(self, patch_ids): + log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self.name, ", ".join(map(str, patch_ids)))) + + +class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler): name = "commit-queue" def __init__(self): - AbstractQueue.__init__(self) + AbstractPatchQueue.__init__(self) - # AbstractQueue methods + # AbstractPatchQueue methods def begin_work_queue(self): - AbstractQueue.begin_work_queue(self) + AbstractPatchQueue.begin_work_queue(self) self.committer_validator = CommitterValidator(self.tool.bugs) def _validate_patches_in_commit_queue(self): # Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers. bug_ids = self.tool.bugs.queries.fetch_bug_ids_from_commit_queue() all_patches = sum([self.tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True) for bug_id in bug_ids], []) - return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches) + valid_patches = self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches) + if not self._builders_are_green(): + return filter(lambda patch: patch.is_rollout(), valid_patches) + return valid_patches def next_work_item(self): patches = self._validate_patches_in_commit_queue() @@ -162,9 +179,17 @@ class CommitQueue(AbstractQueue, StepSequenceErrorHandler): def _can_build_and_test(self): try: - self.run_webkit_patch(["build-and-test", "--force-clean", "--non-interactive", "--build-style=both", "--quiet"]) + self.run_webkit_patch([ + "build-and-test", + "--force-clean", + "--build", + "--test", + "--non-interactive", + "--no-update", + "--build-style=both", + "--quiet"]) except ScriptError, e: - self._update_status("Unabled to successfully build and test", None) + self._update_status("Unable to successfully build and test", None) return False return True @@ -177,26 +202,67 @@ class CommitQueue(AbstractQueue, StepSequenceErrorHandler): return True def should_proceed_with_work_item(self, patch): - if not self._builders_are_green(): - return False - if not self._can_build_and_test(): - return False - if not self._builders_are_green(): - return False + if not patch.is_rollout(): + if not self._builders_are_green(): + return False self._update_status("Landing patch", patch) return True - def process_work_item(self, patch): + def _land(self, patch, first_run=False): try: - self._cc_watchers(patch.bug_id()) - # We pass --no-update here because we've already validated - # that the current revision actually builds and passes the tests. - # If we update, we risk moving to a revision that doesn't! - self.run_webkit_patch(["land-attachment", "--force-clean", "--non-interactive", "--no-update", "--parent-command=commit-queue", "--build-style=both", "--quiet", patch.id()]) + # We need to check the builders, unless we're trying to land a + # rollout (in which case the builders are probably red.) + if not patch.is_rollout() and not self._builders_are_green(): + # We return true here because we want to return to the main + # QueueEngine loop as quickly as possible. + return True + args = [ + "land-attachment", + "--force-clean", + "--build", + "--test", + "--non-interactive", + # The master process is responsible for checking the status + # of the builders (see above call to _builders_are_green). + "--ignore-builders", + "--build-style=both", + "--quiet", + patch.id() + ] + if not first_run: + # The first time through, we don't reject the patch from the + # commit queue because we want to make sure we can build and + # test ourselves. However, the second time through, we + # register ourselves as the parent-command so we can reject + # the patch on failure. + args.append("--parent-command=commit-queue") + # The second time through, we also don't want to update so we + # know we're testing the same revision that we successfully + # built and tested. + args.append("--no-update") + self.run_webkit_patch(args) self._did_pass(patch) + return True except ScriptError, e: + if first_run: + return False self._did_fail(patch) - raise e + raise + + def process_work_item(self, patch): + self._cc_watchers(patch.bug_id()) + if not self._land(patch, first_run=True): + # The patch failed to land, but the bots were green. It's possible + # that the bots were behind. To check that case, we try to build and + # test ourselves. + if not self._can_build_and_test(): + return False + # Hum, looks like the patch is actually bad. Of course, we could + # have been bitten by a flaky test the first time around. We try + # to land again. If it fails a second time, we're pretty sure its + # a bad test and re can reject it outright. + self._land(patch) + return True def handle_unexpected_error(self, patch, message): self.committer_validator.reject_patch_from_commit_queue(patch.id(), message) @@ -217,11 +283,11 @@ class CommitQueue(AbstractQueue, StepSequenceErrorHandler): validator.reject_patch_from_commit_queue(state["patch"].id(), cls._error_message_for_bug(tool, status_id, script_error)) -class AbstractReviewQueue(AbstractQueue, PersistentPatchCollectionDelegate, StepSequenceErrorHandler): +class AbstractReviewQueue(AbstractPatchQueue, PersistentPatchCollectionDelegate, StepSequenceErrorHandler): def __init__(self, options=None): - AbstractQueue.__init__(self, options) + AbstractPatchQueue.__init__(self, options) - def _review_patch(self, patch): + def review_patch(self, patch): raise NotImplementedError, "subclasses must implement" # PersistentPatchCollectionDelegate methods @@ -238,10 +304,10 @@ class AbstractReviewQueue(AbstractQueue, PersistentPatchCollectionDelegate, Step def is_terminal_status(self, status): return status == "Pass" or status == "Fail" or status.startswith("Error:") - # AbstractQueue methods + # AbstractPatchQueue methods def begin_work_queue(self): - AbstractQueue.begin_work_queue(self) + AbstractPatchQueue.begin_work_queue(self) self._patches = PersistentPatchCollection(self) def next_work_item(self): @@ -255,8 +321,10 @@ class AbstractReviewQueue(AbstractQueue, PersistentPatchCollectionDelegate, Step def process_work_item(self, patch): try: - self._review_patch(patch) + if not self.review_patch(patch): + return False self._did_pass(patch) + return True except ScriptError, e: if e.exit_code != QueueEngine.handled_error_code: self._did_fail(patch) @@ -281,8 +349,9 @@ class StyleQueue(AbstractReviewQueue): self._update_status("Checking style", patch) return True - def _review_patch(self, patch): + def review_patch(self, patch): self.run_webkit_patch(["check-style", "--force-clean", "--non-interactive", "--parent-command=style-queue", patch.id()]) + return True @classmethod def handle_script_error(cls, tool, state, script_error): diff --git a/WebKitTools/Scripts/webkitpy/commands/queues_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/queues_unittest.py index 87cd645..f0f7c86 100644 --- a/WebKitTools/Scripts/webkitpy/commands/queues_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queues_unittest.py @@ -28,14 +28,16 @@ import os -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.queues import * -from webkitpy.commands.queuestest import QueuesTest -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture +from webkitpy.common.net.bugzilla import Attachment +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.queues import * +from webkitpy.tool.commands.queuestest import QueuesTest +from webkitpy.tool.mocktool import MockTool, MockSCM -class TestQueue(AbstractQueue): +class TestQueue(AbstractPatchQueue): name = "test-queue" @@ -43,6 +45,17 @@ class TestReviewQueue(AbstractReviewQueue): name = "test-review-queue" +class MockPatch(object): + def is_rollout(self): + return True + + def bug_id(self): + return 12345 + + def id(self): + return 76543 + + class AbstractQueueTest(CommandsTest): def _assert_log_progress_output(self, patch_ids, progress_output): OutputCapture().assert_outputs(self, TestQueue().log_progress, [patch_ids], expected_stderr=progress_output) @@ -52,9 +65,13 @@ class AbstractQueueTest(CommandsTest): self._assert_log_progress_output(["1","2","3"], "3 patches in test-queue [1, 2, 3]\n") self._assert_log_progress_output([1], "1 patch in test-queue [1]\n") + def test_log_directory(self): + self.assertEquals(TestQueue()._log_directory(), "test-queue-logs") + def _assert_run_webkit_patch(self, run_args): queue = TestQueue() - tool = MockBugzillaTool() + tool = MockTool() + tool.executive = Mock() queue.bind_to_tool(tool) queue.run_webkit_patch(run_args) @@ -65,11 +82,28 @@ class AbstractQueueTest(CommandsTest): self._assert_run_webkit_patch([1]) self._assert_run_webkit_patch(["one", 2]) + def test_iteration_count(self): + queue = TestQueue() + queue.options = Mock() + queue.options.iterations = 3 + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + self.assertFalse(queue.should_continue_work_queue()) + + def test_no_iteration_count(self): + queue = TestQueue() + queue.options = Mock() + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + class AbstractReviewQueueTest(CommandsTest): def test_patch_collection_delegate_methods(self): queue = TestReviewQueue() - tool = MockBugzillaTool() + tool = MockTool() queue.bind_to_tool(tool) self.assertEquals(queue.collection_name(), "test-review-queue") self.assertEquals(queue.fetch_potential_patch_ids(), [103]) @@ -83,7 +117,7 @@ class AbstractReviewQueueTest(CommandsTest): class CommitQueueTest(QueuesTest): def test_commit_queue(self): expected_stderr = { - "begin_work_queue" : "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % os.getcwd(), + "begin_work_queue" : "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % MockSCM.fake_checkout_root, # FIXME: The commit-queue warns about bad committers twice. This is due to the fact that we access Attachment.reviewer() twice and it logs each time. "next_work_item" : """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) @@ -92,11 +126,39 @@ Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.c } self.assert_queue_outputs(CommitQueue(), expected_stderr=expected_stderr) + def test_rollout(self): + tool = MockTool(log_executive=True) + tool.buildbot.light_tree_on_fire() + expected_stderr = { + "begin_work_queue" : "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % MockSCM.fake_checkout_root, + # FIXME: The commit-queue warns about bad committers twice. This is due to the fact that we access Attachment.reviewer() twice and it logs each time. + "next_work_item" : """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +1 patch in commit-queue [106] +""", + } + self.assert_queue_outputs(CommitQueue(), tool=tool, expected_stderr=expected_stderr) + + def test_rollout_lands(self): + tool = MockTool(log_executive=True) + tool.buildbot.light_tree_on_fire() + rollout_patch = MockPatch() + expected_stderr = { + "begin_work_queue": "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % MockSCM.fake_checkout_root, + # FIXME: The commit-queue warns about bad committers twice. This is due to the fact that we access Attachment.reviewer() twice and it logs each time. + "next_work_item": """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +1 patch in commit-queue [106] +""", + "process_work_item": "MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--build', '--test', '--non-interactive', '--ignore-builders', '--build-style=both', '--quiet', '76543']\n", + } + self.assert_queue_outputs(CommitQueue(), tool=tool, work_item=rollout_patch, expected_stderr=expected_stderr) + class StyleQueueTest(QueuesTest): def test_style_queue(self): expected_stderr = { - "begin_work_queue" : "CAUTION: style-queue will discard all local changes in \"%s\"\nRunning WebKit style-queue.\n" % os.getcwd(), + "begin_work_queue" : "CAUTION: style-queue will discard all local changes in \"%s\"\nRunning WebKit style-queue.\n" % MockSCM.fake_checkout_root, "handle_unexpected_error" : "Mock error message\n", } self.assert_queue_outputs(StyleQueue(), expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/commands/queuestest.py b/WebKitTools/Scripts/webkitpy/tool/commands/queuestest.py index 09d1c26..bf7e32a 100644 --- a/WebKitTools/Scripts/webkitpy/commands/queuestest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queuestest.py @@ -28,14 +28,14 @@ import unittest -from webkitpy.bugzilla import Attachment -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture +from webkitpy.common.net.bugzilla import Attachment +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.mocktool import MockTool class MockQueueEngine(object): - def __init__(self, name, queue): + def __init__(self, name, queue, wakeup_event): pass def run(self): @@ -44,12 +44,13 @@ class MockQueueEngine(object): class QueuesTest(unittest.TestCase): mock_work_item = Attachment({ - "id" : 1234, - "bug_id" : 345, + "id": 1234, + "bug_id": 345, + "name": "Patch", "attacher_email": "adam@example.com", }, None) - def assert_queue_outputs(self, queue, args=None, work_item=None, expected_stdout=None, expected_stderr=None, options=Mock(), tool=MockBugzillaTool()): + def assert_queue_outputs(self, queue, args=None, work_item=None, expected_stdout=None, expected_stderr=None, options=Mock(), tool=MockTool()): if not expected_stdout: expected_stdout = {} if not expected_stderr: diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot.py b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot.py new file mode 100644 index 0000000..eb80d8f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot.py @@ -0,0 +1,107 @@ +# Copyright (c) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.config.ports import WebKitPort +from webkitpy.tool.bot.sheriff import Sheriff +from webkitpy.tool.bot.sheriffircbot import SheriffIRCBot +from webkitpy.tool.commands.queues import AbstractQueue +from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler + + +class SheriffBot(AbstractQueue, StepSequenceErrorHandler): + name = "sheriff-bot" + watchers = AbstractQueue.watchers + [ + "abarth@webkit.org", + "eric@webkit.org", + ] + + def _update(self): + self.run_webkit_patch(["update", "--force-clean", "--quiet"]) + + # AbstractQueue methods + + def begin_work_queue(self): + AbstractQueue.begin_work_queue(self) + self._sheriff = Sheriff(self.tool, self) + self._irc_bot = SheriffIRCBot(self.tool, self._sheriff) + self.tool.ensure_irc_connected(self._irc_bot.irc_delegate()) + + def work_item_log_path(self, new_failures): + return os.path.join("%s-logs" % self.name, "%s.log" % new_failures.keys()[0]) + + def next_work_item(self): + self._irc_bot.process_pending_messages() + self._update() + new_failures = {} + revisions_causing_failures = self.tool.buildbot.revisions_causing_failures() + for svn_revision, builders in revisions_causing_failures.items(): + if self.tool.status_server.svn_revision(svn_revision): + # FIXME: We should re-process the work item after some time delay. + # https://bugs.webkit.org/show_bug.cgi?id=36581 + continue + new_failures[svn_revision] = builders + self._sheriff.provoke_flaky_builders(revisions_causing_failures) + return new_failures + + def should_proceed_with_work_item(self, new_failures): + # Currently, we don't have any reasons not to proceed with work items. + return True + + def process_work_item(self, new_failures): + blame_list = new_failures.keys() + for svn_revision, builders in new_failures.items(): + try: + commit_info = self.tool.checkout().commit_info_for_revision(svn_revision) + if not commit_info: + print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision + continue + self._sheriff.post_irc_warning(commit_info, builders) + self._sheriff.post_blame_comment_on_bug(commit_info, + builders, + blame_list) + self._sheriff.post_automatic_rollout_patch(commit_info, + builders) + finally: + for builder in builders: + self.tool.status_server.update_svn_revision(svn_revision, + builder.name()) + return True + + def handle_unexpected_error(self, new_failures, message): + log(message) + + # StepSequenceErrorHandler methods + + @classmethod + def handle_script_error(cls, tool, state, script_error): + # Ideally we would post some information to IRC about what went wrong + # here, but we don't have the IRC password in the child process. + pass diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py new file mode 100644 index 0000000..f121eda --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py @@ -0,0 +1,47 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + +from webkitpy.tool.commands.queuestest import QueuesTest +from webkitpy.tool.commands.sheriffbot import SheriffBot +from webkitpy.tool.mocktool import mock_builder + + +class SheriffBotTest(QueuesTest): + def test_sheriff_bot(self): + mock_work_item = { + 29837: [mock_builder], + } + expected_stderr = { + "begin_work_queue": "CAUTION: sheriff-bot will discard all local changes in \"%s\"\nRunning WebKit sheriff-bot.\n" % os.getcwd(), + "next_work_item": "", + "process_work_item": "MOCK: irc.post: abarth, darin, eseidel: http://trac.webkit.org/changeset/29837 might have broken Mock builder name (Tests)\nMOCK bug comment: bug_id=42, cc=['webkit-bot-watchers@googlegroups.com', 'abarth@webkit.org', 'eric@webkit.org']\n--- Begin comment ---\\http://trac.webkit.org/changeset/29837 might have broken Mock builder name (Tests)\n--- End comment ---\n\n", + "handle_unexpected_error": "Mock error message\n" + } + self.assert_queue_outputs(SheriffBot(), work_item=mock_work_item, expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/stepsequence.py b/WebKitTools/Scripts/webkitpy/tool/commands/stepsequence.py index 008b366..c6de79f 100644 --- a/WebKitTools/Scripts/webkitpy/stepsequence.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/stepsequence.py @@ -26,12 +26,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import webkitpy.steps as steps +import webkitpy.tool.steps as steps -from webkitpy.executive import ScriptError -from webkitpy.webkit_logging import log -from webkitpy.scm import CheckoutNeedsUpdate -from webkitpy.queueengine import QueueEngine +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.checkout.scm import CheckoutNeedsUpdate +from webkitpy.tool.bot.queueengine import QueueEngine +from webkitpy.common.system.deprecated_logging import log class StepSequenceErrorHandler(): @@ -66,7 +66,6 @@ class StepSequence(object): self._run(tool, options, state) except CheckoutNeedsUpdate, e: log("Commit failed because the checkout is out of date. Please update and try again.") - log("You can pass --no-build to skip building/testing after update if you believe the new commits did not affect the results.") QueueEngine.exit_after_handled_error(e) except ScriptError, e: if not options.quiet: diff --git a/WebKitTools/Scripts/webkitpy/commands/upload.py b/WebKitTools/Scripts/webkitpy/tool/commands/upload.py index 15bdfbb..bdf060a 100644 --- a/WebKitTools/Scripts/webkitpy/commands/upload.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/upload.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009, 2010 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -35,25 +35,26 @@ import sys from optparse import make_option -import webkitpy.steps as steps +import webkitpy.tool.steps as steps -from webkitpy.bugzilla import parse_bug_id -from webkitpy.commands.abstractsequencedcommand import AbstractSequencedCommand -from webkitpy.comments import bug_comment_from_svn_revision -from webkitpy.committers import CommitterList -from webkitpy.grammar import pluralize, join_with_separators -from webkitpy.webkit_logging import error, log -from webkitpy.mock import Mock -from webkitpy.multicommandtool import AbstractDeclarativeCommand -from webkitpy.user import User +from webkitpy.common.config.committers import CommitterList +from webkitpy.common.net.bugzilla import parse_bug_id +from webkitpy.common.system.user import User +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand +from webkitpy.tool.grammar import pluralize, join_with_separators +from webkitpy.tool.comments import bug_comment_from_svn_revision +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand +from webkitpy.common.system.deprecated_logging import error, log class CommitMessageForCurrentDiff(AbstractDeclarativeCommand): name = "commit-message" help_text = "Print a commit message suitable for the uncommitted changes" def execute(self, options, args, tool): - os.chdir(tool.scm().checkout_root) - print "%s" % tool.scm().commit_message_for_this_commit().message() + # This command is a useful test to make sure commit_message_for_this_commit + # always returns the right value regardless of the current working directory. + print "%s" % tool.checkout().commit_message_for_this_commit().message() class CleanPendingCommit(AbstractDeclarativeCommand): name = "clean-pending-commit" @@ -97,8 +98,8 @@ class AssignToCommitter(AbstractDeclarativeCommand): def _assign_bug_to_last_patch_attacher(self, bug_id): committers = CommitterList() bug = self.tool.bugs.fetch_bug(bug_id) - assigned_to_email = bug.assigned_to_email() - if assigned_to_email != self.tool.bugs.unassigned_email: + if not bug.is_unassigned(): + assigned_to_email = bug.assigned_to_email() log("Bug %s is already assigned to %s (%s)." % (bug_id, assigned_to_email, committers.committer_by_email(assigned_to_email))) return @@ -144,15 +145,14 @@ class AbstractPatchUploadingCommand(AbstractSequencedCommand): # Perfer a bug id passed as an argument over a bug url in the diff (i.e. ChangeLogs). bug_id = args and args[0] if not bug_id: - state["diff"] = tool.scm().create_patch() - bug_id = parse_bug_id(state["diff"]) + bug_id = tool.checkout().bug_id_for_this_commit() return bug_id def _prepare_state(self, options, args, tool): state = {} state["bug_id"] = self._bug_id(args, tool, state) if not state["bug_id"]: - error("No bug id passed and no bug url found in diff.") + error("No bug id passed and no bug url found in ChangeLogs.") return state @@ -164,6 +164,7 @@ class Post(AbstractPatchUploadingCommand): steps = [ steps.CheckStyle, steps.ConfirmDiff, + steps.PostCodeReview, steps.ObsoletePatches, steps.PostDiff, ] @@ -208,6 +209,7 @@ class Upload(AbstractPatchUploadingCommand): steps.PrepareChangeLog, steps.EditChangeLog, steps.ConfirmDiff, + steps.PostCodeReview, steps.ObsoletePatches, steps.PostDiff, ] @@ -288,6 +290,7 @@ class PostCommits(AbstractDeclarativeCommand): tool.bugs.add_patch_to_bug(bug_id, diff_file, description, comment_text, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) +# FIXME: This command needs to be brought into the modern age with steps and CommitInfo. class MarkBugFixed(AbstractDeclarativeCommand): name = "mark-bug-fixed" help_text = "Mark the specified bug as fixed" @@ -301,6 +304,7 @@ class MarkBugFixed(AbstractDeclarativeCommand): ] AbstractDeclarativeCommand.__init__(self, options=options) + # FIXME: We should be using checkout().changelog_entries_for_revision(...) instead here. def _fetch_commit_log(self, tool, svn_revision): if not svn_revision: return tool.scm().last_svn_commit_log() @@ -415,7 +419,7 @@ class CreateBug(AbstractDeclarativeCommand): if options.prompt: (bug_title, comment_text) = self.prompt_for_bug_title_and_comment() else: - commit_message = tool.scm().commit_message_for_this_commit() + commit_message = tool.checkout().commit_message_for_this_commit() bug_title = commit_message.description(lstrip=True, strip_url=True) comment_text = commit_message.body(lstrip=True) diff --git a/WebKitTools/Scripts/webkitpy/commands/upload_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/upload_unittest.py index 7fa8797..271df01 100644 --- a/WebKitTools/Scripts/webkitpy/commands/upload_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/upload_unittest.py @@ -26,17 +26,17 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.upload import * -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.upload import * +from webkitpy.tool.mocktool import MockTool class UploadCommandsTest(CommandsTest): def test_commit_message_for_current_diff(self): - tool = MockBugzillaTool() + tool = MockTool() mock_commit_message_for_this_commit = Mock() mock_commit_message_for_this_commit.message = lambda: "Mock message" - tool._scm.commit_message_for_this_commit = lambda: mock_commit_message_for_this_commit + tool._checkout.commit_message_for_this_commit = lambda: mock_commit_message_for_this_commit expected_stdout = "Mock message\n" self.assert_execute_outputs(CommitMessageForCurrentDiff(), [], expected_stdout=expected_stdout, tool=tool) @@ -44,7 +44,7 @@ class UploadCommandsTest(CommandsTest): self.assert_execute_outputs(CleanPendingCommit(), []) def test_assign_to_committer(self): - tool = MockBugzillaTool() + tool = MockTool() expected_stderr = "Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)\nBug 77 is already assigned to foo@foo.com (None).\nBug 76 has no non-obsolete patches, ignoring.\n" self.assert_execute_outputs(AssignToCommitter(), [], expected_stderr=expected_stderr, tool=tool) tool.bugs.reassign_bug.assert_called_with(42, "eric@webkit.org", "Attachment 128 was posted by a committer and has review+, assigning to Eric Seidel for commit.") @@ -54,33 +54,57 @@ class UploadCommandsTest(CommandsTest): self.assert_execute_outputs(ObsoleteAttachments(), [42], expected_stderr=expected_stderr) def test_post(self): - expected_stderr = "Running check-webkit-style\nObsoleting 2 old patches on bug 42\n" - self.assert_execute_outputs(Post(), [42], expected_stderr=expected_stderr) + options = Mock() + options.description = "MOCK description" + options.request_commit = False + options.review = True + options.cc = None + expected_stderr = """Running check-webkit-style +MOCK: user.open_url: file://... +Obsoleting 2 old patches on bug 42 +MOCK add_patch_to_bug: bug_id=42, description=MOCK description, mark_for_review=True, mark_for_commit_queue=False, mark_for_landing=False +-- Begin comment -- +None +-- End comment -- +MOCK: user.open_url: http://example.com/42 +""" + self.assert_execute_outputs(Post(), [42], options=options, expected_stderr=expected_stderr) - def test_post(self): - expected_stderr = "Obsoleting 2 old patches on bug 42\n" + def test_land_safely(self): + expected_stderr = "Obsoleting 2 old patches on bug 42\nMOCK add_patch_to_bug: bug_id=42, description=Patch for landing, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=True\n-- Begin comment --\nNone\n-- End comment --\n" self.assert_execute_outputs(LandSafely(), [42], expected_stderr=expected_stderr) def test_prepare_diff_with_arg(self): self.assert_execute_outputs(Prepare(), [42]) def test_prepare(self): - self.assert_execute_outputs(Prepare(), []) + expected_stderr = "MOCK create_bug\nbug_title: Mock user response\nbug_description: Mock user response\n" + self.assert_execute_outputs(Prepare(), [], expected_stderr=expected_stderr) def test_upload(self): - expected_stderr = "Running check-webkit-style\nObsoleting 2 old patches on bug 42\nMOCK: user.open_url: http://example.com/42\n" - self.assert_execute_outputs(Upload(), [42], expected_stderr=expected_stderr) + options = Mock() + options.description = "MOCK description" + options.request_commit = False + options.review = True + options.cc = None + expected_stderr = """Running check-webkit-style +MOCK: user.open_url: file://... +Obsoleting 2 old patches on bug 42 +MOCK add_patch_to_bug: bug_id=42, description=MOCK description, mark_for_review=True, mark_for_commit_queue=False, mark_for_landing=False +-- Begin comment -- +None +-- End comment -- +MOCK: user.open_url: http://example.com/42 +""" + self.assert_execute_outputs(Upload(), [42], options=options, expected_stderr=expected_stderr) def test_mark_bug_fixed(self): - tool = MockBugzillaTool() + tool = MockTool() tool._scm.last_svn_commit_log = lambda: "r9876 |" options = Mock() options.bug_id = 42 - expected_stderr = """Bug: <http://example.com/42> Bug with two r+'d and cq+'d patches, one of which has an invalid commit-queue setter. -Revision: 9876 -MOCK: user.open_url: http://example.com/42 -Adding comment to Bug 42. -""" + options.comment = "MOCK comment" + expected_stderr = "Bug: <http://example.com/42> Bug with two r+'d and cq+'d patches, one of which has an invalid commit-queue setter.\nRevision: 9876\nMOCK: user.open_url: http://example.com/42\nAdding comment to Bug 42.\nMOCK bug comment: bug_id=42, cc=None\n--- Begin comment ---\\MOCK comment\n\nCommitted r9876: <http://trac.webkit.org/changeset/9876>\n--- End comment ---\n\n" self.assert_execute_outputs(MarkBugFixed(), [], expected_stderr=expected_stderr, tool=tool, options=options) def test_edit_changelog(self): diff --git a/WebKitTools/Scripts/webkitpy/comments.py b/WebKitTools/Scripts/webkitpy/tool/comments.py index 77ad239..83f2be8 100755 --- a/WebKitTools/Scripts/webkitpy/comments.py +++ b/WebKitTools/Scripts/webkitpy/tool/comments.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -30,7 +30,7 @@ # A tool for automating dealing with bugzilla, posting patches, committing # patches, etc. -from webkitpy.changelogs import view_source_url +from webkitpy.common.checkout.changelog import view_source_url def bug_comment_from_svn_revision(svn_revision): diff --git a/WebKitTools/Scripts/webkitpy/grammar.py b/WebKitTools/Scripts/webkitpy/tool/grammar.py index 651bbc9..8db9826 100644 --- a/WebKitTools/Scripts/webkitpy/grammar.py +++ b/WebKitTools/Scripts/webkitpy/tool/grammar.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -45,9 +44,11 @@ def pluralize(noun, count): return "%d %s" % (count, noun) -def join_with_separators(list_of_strings, separator=', ', last_separator=', and '): +def join_with_separators(list_of_strings, separator=', ', only_two_separator=" and ", last_separator=', and '): if not list_of_strings: return "" if len(list_of_strings) == 1: return list_of_strings[0] + if len(list_of_strings) == 2: + return only_two_separator.join(list_of_strings) return "%s%s%s" % (separator.join(list_of_strings[:-1]), last_separator, list_of_strings[-1]) diff --git a/WebKitTools/Scripts/webkitpy/grammar_unittest.py b/WebKitTools/Scripts/webkitpy/tool/grammar_unittest.py index 3d8b179..cab71db 100644 --- a/WebKitTools/Scripts/webkitpy/grammar_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/grammar_unittest.py @@ -27,11 +27,14 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from webkitpy.grammar import join_with_separators + +from webkitpy.tool.grammar import join_with_separators class GrammarTest(unittest.TestCase): def test_join_with_separators(self): + self.assertEqual(join_with_separators(["one"]), "one") + self.assertEqual(join_with_separators(["one", "two"]), "one and two") self.assertEqual(join_with_separators(["one", "two", "three"]), "one, two, and three") if __name__ == '__main__': diff --git a/WebKitTools/Scripts/webkitpy/tool/main.py b/WebKitTools/Scripts/webkitpy/tool/main.py new file mode 100755 index 0000000..06cde74 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/main.py @@ -0,0 +1,139 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# A tool for automating dealing with bugzilla, posting patches, committing patches, etc. + +import os +import threading + +from webkitpy.common.checkout.api import Checkout +from webkitpy.common.checkout.scm import detect_scm_system +from webkitpy.common.net.bugzilla import Bugzilla +from webkitpy.common.net.buildbot import BuildBot +from webkitpy.common.net.rietveld import Rietveld +from webkitpy.common.net.irc.ircproxy import IRCProxy +from webkitpy.common.system.executive import Executive +from webkitpy.common.system.user import User +import webkitpy.tool.commands as commands +# FIXME: Remove these imports once all the commands are in the root of the +# command package. +from webkitpy.tool.commands.download import * +from webkitpy.tool.commands.earlywarningsystem import * +from webkitpy.tool.commands.openbugs import OpenBugs +from webkitpy.tool.commands.queries import * +from webkitpy.tool.commands.queues import * +from webkitpy.tool.commands.sheriffbot import * +from webkitpy.tool.commands.upload import * +from webkitpy.tool.multicommandtool import MultiCommandTool +from webkitpy.common.system.deprecated_logging import log + + +class WebKitPatch(MultiCommandTool): + global_options = [ + make_option("--dry-run", action="store_true", dest="dry_run", default=False, help="do not touch remote servers"), + make_option("--status-host", action="store", dest="status_host", type="string", nargs=1, help="Hostname (e.g. localhost or commit.webkit.org) where status updates should be posted."), + make_option("--irc-password", action="store", dest="irc_password", type="string", nargs=1, help="Password to use when communicating via IRC."), + ] + + def __init__(self, path): + MultiCommandTool.__init__(self) + + self._path = path + self.wakeup_event = threading.Event() + self.bugs = Bugzilla() + self.buildbot = BuildBot() + self.executive = Executive() + self._irc = None + self.user = User() + self._scm = None + self._checkout = None + self.status_server = StatusServer() + self.codereview = Rietveld(self.executive) + + def scm(self): + # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands). + original_cwd = os.path.abspath(".") + if not self._scm: + self._scm = detect_scm_system(original_cwd) + + if not self._scm: + script_directory = os.path.abspath(sys.path[0]) + self._scm = detect_scm_system(script_directory) + if self._scm: + log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, self._scm.checkout_root)) + else: + error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, script_directory)) + + return self._scm + + def checkout(self): + if not self._checkout: + self._checkout = Checkout(self.scm()) + return self._checkout + + def ensure_irc_connected(self, irc_delegate): + if not self._irc: + self._irc = IRCProxy(irc_delegate) + + def irc(self): + # We don't automatically construct IRCProxy here because constructing + # IRCProxy actually connects to IRC. We want clients to explicitly + # connect to IRC. + return self._irc + + def path(self): + return self._path + + def command_completed(self): + if self._irc: + self._irc.disconnect() + + def should_show_in_main_help(self, command): + if not command.show_in_main_help: + return False + if command.requires_local_commits: + return self.scm().supports_local_commits() + return True + + # FIXME: This may be unnecessary since we pass global options to all commands during execute() as well. + def handle_global_options(self, options): + if options.dry_run: + self.scm().dryrun = True + self.bugs.dryrun = True + self.codereview.dryrun = True + if options.status_host: + self.status_server.set_host(options.status_host) + if options.irc_password: + self.irc_password = options.irc_password + + def should_execute_command(self, command): + if command.requires_local_commits and not self.scm().supports_local_commits(): + failure_reason = "%s requires local commits using %s in %s." % (command.name, self.scm().display_name(), self.scm().checkout_root) + return (False, failure_reason) + return (True, None) diff --git a/WebKitTools/Scripts/webkitpy/mock_bugzillatool.py b/WebKitTools/Scripts/webkitpy/tool/mocktool.py index f522e40..cc361ff 100644 --- a/WebKitTools/Scripts/webkitpy/mock_bugzillatool.py +++ b/WebKitTools/Scripts/webkitpy/tool/mocktool.py @@ -27,12 +27,15 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os +import threading -from webkitpy.bugzilla import Bug, Attachment -from webkitpy.committers import CommitterList, Reviewer -from webkitpy.mock import Mock -from webkitpy.scm import CommitMessage -from webkitpy.webkit_logging import log +from webkitpy.common.config.committers import CommitterList, Reviewer +from webkitpy.common.checkout.commitinfo import CommitInfo +from webkitpy.common.checkout.scm import CommitMessage +from webkitpy.common.net.bugzilla import Bug, Attachment +from webkitpy.common.net.rietveld import Rietveld +from webkitpy.thirdparty.mock import Mock +from webkitpy.common.system.deprecated_logging import log def _id_to_object_dictionary(*objects): @@ -41,6 +44,7 @@ def _id_to_object_dictionary(*objects): dictionary[thing["id"]] = thing return dictionary +# Testing # FIXME: The ids should be 1, 2, 3 instead of crazy numbers. @@ -49,6 +53,7 @@ _patch1 = { "id": 197, "bug_id": 42, "url": "http://example.com/197", + "name": "Patch1", "is_obsolete": False, "is_patch": True, "review": "+", @@ -63,6 +68,7 @@ _patch2 = { "id": 128, "bug_id": 42, "url": "http://example.com/128", + "name": "Patch2", "is_obsolete": False, "is_patch": True, "review": "+", @@ -77,6 +83,7 @@ _patch3 = { "id": 103, "bug_id": 75, "url": "http://example.com/103", + "name": "Patch3", "is_obsolete": False, "is_patch": True, "review": "?", @@ -88,6 +95,7 @@ _patch4 = { "id": 104, "bug_id": 77, "url": "http://example.com/103", + "name": "Patch3", "is_obsolete": False, "is_patch": True, "review": "+", @@ -101,6 +109,7 @@ _patch5 = { "id": 105, "bug_id": 77, "url": "http://example.com/103", + "name": "Patch5", "is_obsolete": False, "is_patch": True, "review": "+", @@ -113,6 +122,7 @@ _patch6 = { # Valid committer, but no reviewer. "id": 106, "bug_id": 77, "url": "http://example.com/103", + "name": "ROLLOUT of r3489", "is_obsolete": False, "is_patch": True, "commit-queue": "+", @@ -125,6 +135,7 @@ _patch7 = { # Valid review, patch is marked obsolete. "id": 107, "bug_id": 76, "url": "http://example.com/103", + "name": "Patch7", "is_obsolete": True, "is_patch": True, "review": "+", @@ -133,9 +144,8 @@ _patch7 = { # Valid review, patch is marked obsolete. } -# This must be defined before we define the bugs, thus we don't use -# MockBugzilla.unassigned_email directly. -_unassigned_email = "unassigned@example.com" +# This matches one of Bug.unassigned_emails +_unassigned_email = "webkit-unassigned@lists.webkit.org" # FIXME: The ids should be 1, 2, 3 instead of crazy numbers. @@ -174,6 +184,15 @@ _bug4 = { } +class MockBuilder(object): + + def name(self): + return "Mock builder name (Tests)" + + +mock_builder = MockBuilder() + + class MockBugzillaQueries(Mock): def __init__(self, bugzilla): @@ -221,8 +240,6 @@ class MockBugzilla(Mock): bug_server_url = "http://example.com" - unassigned_email = _unassigned_email - bug_cache = _id_to_object_dictionary(_bug1, _bug2, _bug3, _bug4) attachment_cache = _id_to_object_dictionary(_patch1, @@ -239,6 +256,23 @@ class MockBugzilla(Mock): self.committers = CommitterList(reviewers=[Reviewer("Foo Bar", "foo@bar.com")]) + def create_bug(self, + bug_title, + bug_description, + component=None, + patch_file_object=None, + patch_description=None, + cc=None, + blocked=None, + mark_for_review=False, + mark_for_commit_queue=False): + log("MOCK create_bug") + log("bug_title: %s" % bug_title) + log("bug_description: %s" % bug_description) + + def quips(self): + return ["Good artists copy. Great artists steal. - Pablo Picasso"] + def fetch_bug(self, bug_id): return Bug(self.bug_cache.get(bug_id), self) @@ -262,27 +296,93 @@ class MockBugzilla(Mock): action_param = "&action=%s" % action return "%s/%s%s" % (self.bug_server_url, attachment_id, action_param) + def post_comment_to_bug(self, bug_id, comment_text, cc=None): + log("MOCK bug comment: bug_id=%s, cc=%s\n--- Begin comment ---\%s\n--- End comment ---\n" % ( + bug_id, cc, comment_text)) -class MockBuildBot(Mock): + def add_patch_to_bug(self, + bug_id, + patch_file_object, + description, + comment_text=None, + mark_for_review=False, + mark_for_commit_queue=False, + mark_for_landing=False): + log("MOCK add_patch_to_bug: bug_id=%s, description=%s, mark_for_review=%s, mark_for_commit_queue=%s, mark_for_landing=%s" % + (bug_id, description, mark_for_review, mark_for_commit_queue, mark_for_landing)) + log("-- Begin comment --") + log(comment_text) + log("-- End comment --") - def builder_statuses(self): - return [{ + +class MockBuilder(object): + def __init__(self, name): + self._name = name + + def force_build(self, username, comments): + log("MOCK: force_build: name=%s, username=%s, comments=%s" % ( + self._name, username, comments)) + + +class MockBuildBot(object): + def __init__(self): + self._mock_builder1_status = { "name": "Builder1", "is_green": True, - }, { + "activity": "building", + } + self._mock_builder2_status = { "name": "Builder2", "is_green": True, - }] + "activity": "idle", + } + + def builder_with_name(self, name): + return MockBuilder(name) + + def builder_statuses(self): + return [ + self._mock_builder1_status, + self._mock_builder2_status, + ] def red_core_builders_names(self): + if not self._mock_builder2_status["is_green"]: + return [self._mock_builder2_status["name"]] + return [] + + def red_core_builders(self): + if not self._mock_builder2_status["is_green"]: + return [self._mock_builder2_status] + return [] + + def idle_red_core_builders(self): + if not self._mock_builder2_status["is_green"]: + return [self._mock_builder2_status] return [] + def last_green_revision(self): + return 9479 + + def light_tree_on_fire(self): + self._mock_builder2_status["is_green"] = False + + def revisions_causing_failures(self): + return { + "29837": [mock_builder] + } + class MockSCM(Mock): + fake_checkout_root = os.path.realpath("/tmp") # realpath is needed to allow for Mac OS X's /private/tmp + def __init__(self): Mock.__init__(self) - self.checkout_root = os.getcwd() + # FIXME: We should probably use real checkout-root detection logic here. + # os.getcwd() can't work here because other parts of the code assume that "checkout_root" + # will actually be the root. Since getcwd() is wrong, use a globally fake root for now. + self.checkout_root = self.fake_checkout_root def create_patch(self): return "Patch1" @@ -313,11 +413,40 @@ class MockSCM(Mock): def svn_revision_from_commit_text(self, commit_text): return "49824" + +class MockCheckout(object): + + _committer_list = CommitterList() + + def commit_info_for_revision(self, svn_revision): + return CommitInfo(svn_revision, "eric@webkit.org", { + "bug_id": 42, + "author_name": "Adam Barth", + "author_email": "abarth@webkit.org", + "author": self._committer_list.committer_by_email("abarth@webkit.org"), + "reviewer_text": "Darin Adler", + "reviewer": self._committer_list.committer_by_name("Darin Adler"), + }) + + def bug_id_for_revision(self, svn_revision): + return 12345 + def modified_changelogs(self): # Ideally we'd return something more interesting here. The problem is - # that LandDiff will try to actually read the path from disk! + # that LandDiff will try to actually read the patch from disk! return [] + def commit_message_for_this_commit(self): + commit_message = Mock() + commit_message.message = lambda:"This is a fake commit message that is at least 50 characters." + return commit_message + + def apply_patch(self, patch, force=False): + pass + + def apply_reverse_diff(self, revision): + pass + class MockUser(object): @@ -335,8 +464,19 @@ class MockUser(object): return True def open_url(self, url): + if url.startswith("file://"): + log("MOCK: user.open_url: file://...") + return log("MOCK: user.open_url: %s" % url) - pass + + +class MockIRC(object): + + def post(self, message): + log("MOCK: irc.post: %s" % message) + + def disconnect(self): + log("MOCK: irc.disconnect") class MockStatusServer(object): @@ -347,22 +487,59 @@ class MockStatusServer(object): def patch_status(self, queue_name, patch_id): return None + def svn_revision(self, svn_revision): + return None + def update_status(self, queue_name, status, patch=None, results_file=None): return 187 + def update_svn_revision(self, svn_revision, broken_bot): + return 191 -class MockBugzillaTool(): - def __init__(self): +class MockExecute(Mock): + def run_and_throw_if_fail(self, args, quiet=False): + return "MOCK output of child process" + + def run_command(self, + args, + cwd=None, + input=None, + error_handler=None, + return_exit_code=False, + return_stderr=True): + return "MOCK output of child process" + + +class MockTool(): + + def __init__(self, log_executive=False): + self.wakeup_event = threading.Event() self.bugs = MockBugzilla() self.buildbot = MockBuildBot() - self.executive = Mock() + self.executive = MockExecute() + if log_executive: + self.executive.run_and_throw_if_fail = lambda args: log("MOCK run_and_throw_if_fail: %s" % args) + self._irc = None self.user = MockUser() self._scm = MockSCM() + self._checkout = MockCheckout() self.status_server = MockStatusServer() + self.irc_password = "MOCK irc password" + self.codereview = Rietveld(self.executive) def scm(self): return self._scm + def checkout(self): + return self._checkout + + def ensure_irc_connected(self, delegate): + if not self._irc: + self._irc = MockIRC() + + def irc(self): + return self._irc + def path(self): return "echo" diff --git a/WebKitTools/Scripts/webkitpy/multicommandtool.py b/WebKitTools/Scripts/webkitpy/tool/multicommandtool.py index 10cf426..7940c06 100644 --- a/WebKitTools/Scripts/webkitpy/multicommandtool.py +++ b/WebKitTools/Scripts/webkitpy/tool/multicommandtool.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -35,8 +35,8 @@ import sys from optparse import OptionParser, IndentedHelpFormatter, SUPPRESS_USAGE, make_option -from webkitpy.grammar import pluralize -from webkitpy.webkit_logging import log +from webkitpy.tool.grammar import pluralize +from webkitpy.common.system.deprecated_logging import log class Command(object): @@ -263,6 +263,9 @@ class MultiCommandTool(object): def path(self): raise NotImplementedError, "subclasses must implement" + def command_completed(self): + pass + def should_show_in_main_help(self, command): return command.show_in_main_help @@ -296,4 +299,6 @@ class MultiCommandTool(object): log(failure_reason) return 0 # FIXME: Should this really be 0? - return command.check_arguments_and_execute(options, args, self) + result = command.check_arguments_and_execute(options, args, self) + self.command_completed() + return result diff --git a/WebKitTools/Scripts/webkitpy/multicommandtool_unittest.py b/WebKitTools/Scripts/webkitpy/tool/multicommandtool_unittest.py index ae77e73..268ebf0 100644 --- a/WebKitTools/Scripts/webkitpy/multicommandtool_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/multicommandtool_unittest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -28,11 +28,13 @@ import sys import unittest -from multicommandtool import MultiCommandTool, Command -from webkitpy.outputcapture import OutputCapture from optparse import make_option +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.multicommandtool import MultiCommandTool, Command + + class TrivialCommand(Command): name = "trivial" show_in_main_help = True @@ -72,7 +74,7 @@ class TrivialTool(MultiCommandTool): def __init__(self, commands=None): MultiCommandTool.__init__(self, name="trivial-tool", commands=commands) - def path(): + def path(self): return __file__ def should_execute_command(self, command): diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/__init__.py b/WebKitTools/Scripts/webkitpy/tool/steps/__init__.py new file mode 100644 index 0000000..d59cdc5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/__init__.py @@ -0,0 +1,59 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# FIXME: Is this the right way to do this? +from webkitpy.tool.steps.applypatch import ApplyPatch +from webkitpy.tool.steps.applypatchwithlocalcommit import ApplyPatchWithLocalCommit +from webkitpy.tool.steps.build import Build +from webkitpy.tool.steps.checkstyle import CheckStyle +from webkitpy.tool.steps.cleanworkingdirectory import CleanWorkingDirectory +from webkitpy.tool.steps.cleanworkingdirectorywithlocalcommits import CleanWorkingDirectoryWithLocalCommits +from webkitpy.tool.steps.closebug import CloseBug +from webkitpy.tool.steps.closebugforlanddiff import CloseBugForLandDiff +from webkitpy.tool.steps.closepatch import ClosePatch +from webkitpy.tool.steps.commit import Commit +from webkitpy.tool.steps.confirmdiff import ConfirmDiff +from webkitpy.tool.steps.createbug import CreateBug +from webkitpy.tool.steps.editchangelog import EditChangeLog +from webkitpy.tool.steps.ensurebuildersaregreen import EnsureBuildersAreGreen +from webkitpy.tool.steps.ensurelocalcommitifneeded import EnsureLocalCommitIfNeeded +from webkitpy.tool.steps.obsoletepatches import ObsoletePatches +from webkitpy.tool.steps.options import Options +from webkitpy.tool.steps.postcodereview import PostCodeReview +from webkitpy.tool.steps.postdiff import PostDiff +from webkitpy.tool.steps.postdiffforcommit import PostDiffForCommit +from webkitpy.tool.steps.postdiffforrevert import PostDiffForRevert +from webkitpy.tool.steps.preparechangelogforrevert import PrepareChangeLogForRevert +from webkitpy.tool.steps.preparechangelog import PrepareChangeLog +from webkitpy.tool.steps.promptforbugortitle import PromptForBugOrTitle +from webkitpy.tool.steps.reopenbugafterrollout import ReopenBugAfterRollout +from webkitpy.tool.steps.revertrevision import RevertRevision +from webkitpy.tool.steps.runtests import RunTests +from webkitpy.tool.steps.updatechangelogswithreviewer import UpdateChangeLogsWithReviewer +from webkitpy.tool.steps.update import Update +from webkitpy.tool.steps.validatereviewer import ValidateReviewer diff --git a/WebKitTools/Scripts/webkitpy/steps/abstractstep.py b/WebKitTools/Scripts/webkitpy/tool/steps/abstractstep.py index 639cf55..1ad343d 100644 --- a/WebKitTools/Scripts/webkitpy/steps/abstractstep.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/abstractstep.py @@ -26,8 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.webkit_logging import log -from webkitpy.webkitport import WebKitPort +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.config.ports import WebKitPort class AbstractStep(object): @@ -50,7 +50,7 @@ class AbstractStep(object): _well_known_keys = { "diff" : lambda self: self._tool.scm().create_patch(), - "changelogs" : lambda self: self._tool.scm().modified_changelogs(), + "changelogs" : lambda self: self._tool.checkout().modified_changelogs(), } def cached_lookup(self, state, key, promise=None): diff --git a/WebKitTools/Scripts/webkitpy/steps/applypatch.py b/WebKitTools/Scripts/webkitpy/tool/steps/applypatch.py index aba81ae..66d0a03 100644 --- a/WebKitTools/Scripts/webkitpy/steps/applypatch.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/applypatch.py @@ -26,9 +26,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class ApplyPatch(AbstractStep): @classmethod @@ -39,4 +39,4 @@ class ApplyPatch(AbstractStep): def run(self, state): log("Processing patch %s from bug %s." % (state["patch"].id(), state["patch"].bug_id())) - self._tool.scm().apply_patch(state["patch"], force=self._options.non_interactive) + self._tool.checkout().apply_patch(state["patch"], force=self._options.non_interactive) diff --git a/WebKitTools/Scripts/webkitpy/steps/applypatchwithlocalcommit.py b/WebKitTools/Scripts/webkitpy/tool/steps/applypatchwithlocalcommit.py index bfaf52a..70ddfe5 100644 --- a/WebKitTools/Scripts/webkitpy/steps/applypatchwithlocalcommit.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/applypatchwithlocalcommit.py @@ -26,8 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.applypatch import ApplyPatch -from webkitpy.steps.options import Options +from webkitpy.tool.steps.applypatch import ApplyPatch +from webkitpy.tool.steps.options import Options class ApplyPatchWithLocalCommit(ApplyPatch): @classmethod @@ -39,5 +39,5 @@ class ApplyPatchWithLocalCommit(ApplyPatch): def run(self, state): ApplyPatch.run(self, state) if self._options.local_commit: - commit_message = self._tool.scm().commit_message_for_this_commit() + commit_message = self._tool.checkout().commit_message_for_this_commit() self._tool.scm().commit_locally_with_message(commit_message.message() or state["patch"].name()) diff --git a/WebKitTools/Scripts/webkitpy/steps/build.py b/WebKitTools/Scripts/webkitpy/tool/steps/build.py index 1823cff..f0570f9 100644 --- a/WebKitTools/Scripts/webkitpy/steps/build.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/build.py @@ -26,9 +26,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class Build(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/checkstyle.py b/WebKitTools/Scripts/webkitpy/tool/steps/checkstyle.py index c8e20f8..63f0114 100644 --- a/WebKitTools/Scripts/webkitpy/steps/checkstyle.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/checkstyle.py @@ -28,10 +28,10 @@ import os -from webkitpy.executive import ScriptError -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import error class CheckStyle(AbstractStep): @classmethod diff --git a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectory.py b/WebKitTools/Scripts/webkitpy/tool/steps/cleanworkingdirectory.py index 88e38f5..3768297 100644 --- a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectory.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/cleanworkingdirectory.py @@ -28,8 +28,8 @@ import os -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options class CleanWorkingDirectory(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectorywithlocalcommits.py b/WebKitTools/Scripts/webkitpy/tool/steps/cleanworkingdirectorywithlocalcommits.py index cabeba2..f06f94e 100644 --- a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectorywithlocalcommits.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/cleanworkingdirectorywithlocalcommits.py @@ -26,7 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.cleanworkingdirectory import CleanWorkingDirectory +from webkitpy.tool.steps.cleanworkingdirectory import CleanWorkingDirectory class CleanWorkingDirectoryWithLocalCommits(CleanWorkingDirectory): def __init__(self, tool, options): diff --git a/WebKitTools/Scripts/webkitpy/steps/closebug.py b/WebKitTools/Scripts/webkitpy/tool/steps/closebug.py index 2640ee3..d5059ea 100644 --- a/WebKitTools/Scripts/webkitpy/steps/closebug.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/closebug.py @@ -26,9 +26,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class CloseBug(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff.py b/WebKitTools/Scripts/webkitpy/tool/steps/closebugforlanddiff.py index 43a0c66..476d3af 100644 --- a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/closebugforlanddiff.py @@ -26,10 +26,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.comments import bug_comment_from_commit_text -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.comments import bug_comment_from_commit_text +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class CloseBugForLandDiff(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff_unittest.py b/WebKitTools/Scripts/webkitpy/tool/steps/closebugforlanddiff_unittest.py index 73561ab..4e7f9e6 100644 --- a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/closebugforlanddiff_unittest.py @@ -28,14 +28,14 @@ import unittest -from webkitpy.steps.closebugforlanddiff import CloseBugForLandDiff -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.mocktool import MockTool +from webkitpy.tool.steps.closebugforlanddiff import CloseBugForLandDiff class CloseBugForLandDiffTest(unittest.TestCase): def test_empty_state(self): capture = OutputCapture() - step = CloseBugForLandDiff(MockBugzillaTool(), Mock()) + step = CloseBugForLandDiff(MockTool(), Mock()) expected_stderr = "Committed r49824: <http://trac.webkit.org/changeset/49824>\nNo bug id provided.\n" capture.assert_outputs(self, step.run, [{"commit_text" : "Mock commit text"}], expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/steps/closepatch.py b/WebKitTools/Scripts/webkitpy/tool/steps/closepatch.py index f20fe7e..ff94df8 100644 --- a/WebKitTools/Scripts/webkitpy/steps/closepatch.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/closepatch.py @@ -26,8 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.comments import bug_comment_from_commit_text -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.comments import bug_comment_from_commit_text +from webkitpy.tool.steps.abstractstep import AbstractStep class ClosePatch(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/commit.py b/WebKitTools/Scripts/webkitpy/tool/steps/commit.py new file mode 100644 index 0000000..294b41e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/commit.py @@ -0,0 +1,37 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.tool.steps.abstractstep import AbstractStep + + +class Commit(AbstractStep): + def run(self, state): + commit_message = self._tool.checkout().commit_message_for_this_commit() + if len(commit_message.message()) < 50: + raise Exception("Attempted to commit with a commit message shorter than 50 characters. Either your patch is missing a ChangeLog or webkit-patch may have a bug.") + state["commit_text"] = self._tool.scm().commit_with_message(commit_message.message()) diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/confirmdiff.py b/WebKitTools/Scripts/webkitpy/tool/steps/confirmdiff.py new file mode 100644 index 0000000..d08e477 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/confirmdiff.py @@ -0,0 +1,74 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import urllib + +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.prettypatch import PrettyPatch +from webkitpy.common.system import logutils +from webkitpy.common.system.executive import ScriptError + + +_log = logutils.get_logger(__file__) + + +class ConfirmDiff(AbstractStep): + @classmethod + def options(cls): + return [ + Options.confirm, + ] + + def _show_pretty_diff(self, diff): + try: + pretty_patch = PrettyPatch(self._tool.executive, + self._tool.scm().checkout_root) + pretty_diff_file = pretty_patch.pretty_diff_file(diff) + url = "file://%s" % urllib.quote(pretty_diff_file.name) + self._tool.user.open_url(url) + # We return the pretty_diff_file here because we need to keep the + # file alive until the user has had a chance to confirm the diff. + return pretty_diff_file + except ScriptError, e: + _log.warning("PrettyPatch failed. :(") + except OSError, e: + _log.warning("PrettyPatch unavailable.") + + def run(self, state): + if not self._options.confirm: + return + diff = self.cached_lookup(state, "diff") + pretty_diff_file = self._show_pretty_diff(diff) + if not pretty_diff_file: + self._tool.user.page(diff) + diff_correct = self._tool.user.confirm("Was that diff correct?") + if pretty_diff_file: + pretty_diff_file.close() + if not diff_correct: + exit(1) diff --git a/WebKitTools/Scripts/webkitpy/steps/createbug.py b/WebKitTools/Scripts/webkitpy/tool/steps/createbug.py index 75bf17f..2f3d42c 100644 --- a/WebKitTools/Scripts/webkitpy/steps/createbug.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/createbug.py @@ -26,8 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options class CreateBug(AbstractStep): @@ -42,4 +42,7 @@ class CreateBug(AbstractStep): # No need to create a bug if we already have one. if state.get("bug_id"): return - state["bug_id"] = self._tool.bugs.create_bug(state["bug_title"], state["bug_description"], component=self._options.component, cc=self._options.cc) + cc = self._options.cc + if not cc: + cc = state.get("bug_cc") + state["bug_id"] = self._tool.bugs.create_bug(state["bug_title"], state["bug_description"], blocked=state.get("bug_blocked"), component=self._options.component, cc=cc) diff --git a/WebKitTools/Scripts/webkitpy/steps/editchangelog.py b/WebKitTools/Scripts/webkitpy/tool/steps/editchangelog.py index d545c72..69c8732 100644 --- a/WebKitTools/Scripts/webkitpy/steps/editchangelog.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/editchangelog.py @@ -28,7 +28,7 @@ import os -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep class EditChangeLog(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/ensurebuildersaregreen.py b/WebKitTools/Scripts/webkitpy/tool/steps/ensurebuildersaregreen.py index 96f265a..fd44564 100644 --- a/WebKitTools/Scripts/webkitpy/steps/ensurebuildersaregreen.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/ensurebuildersaregreen.py @@ -26,9 +26,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import error class EnsureBuildersAreGreen(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/ensurelocalcommitifneeded.py b/WebKitTools/Scripts/webkitpy/tool/steps/ensurelocalcommitifneeded.py index cecf891..4f799f2 100644 --- a/WebKitTools/Scripts/webkitpy/steps/ensurelocalcommitifneeded.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/ensurelocalcommitifneeded.py @@ -26,9 +26,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import error class EnsureLocalCommitIfNeeded(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/metastep.py b/WebKitTools/Scripts/webkitpy/tool/steps/metastep.py index 9f368de..7cbd1c5 100644 --- a/WebKitTools/Scripts/webkitpy/steps/metastep.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/metastep.py @@ -26,7 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep # FIXME: Unify with StepSequence? I'm not sure yet which is the better design. diff --git a/WebKitTools/Scripts/webkitpy/steps/obsoletepatches.py b/WebKitTools/Scripts/webkitpy/tool/steps/obsoletepatches.py index dbdbabd..9f65d41 100644 --- a/WebKitTools/Scripts/webkitpy/steps/obsoletepatches.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/obsoletepatches.py @@ -26,10 +26,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.grammar import pluralize -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class ObsoletePatches(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/options.py b/WebKitTools/Scripts/webkitpy/tool/steps/options.py index 8b28f27..7f76f55 100644 --- a/WebKitTools/Scripts/webkitpy/steps/options.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/options.py @@ -29,18 +29,18 @@ from optparse import make_option class Options(object): - build = make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test.") + build = make_option("--build", action="store_true", dest="build", default=False, help="Build and run run-webkit-tests before committing.") build_style = make_option("--build-style", action="store", dest="build_style", default=None, help="Whether to build debug, release, or both.") cc = make_option("--cc", action="store", type="string", dest="cc", help="Comma-separated list of email addresses to carbon-copy.") check_builders = make_option("--ignore-builders", action="store_false", dest="check_builders", default=True, help="Don't check to see if the build.webkit.org builders are green before landing.") check_style = make_option("--ignore-style", action="store_false", dest="check_style", default=True, help="Don't check to see if the patch has proper style before uploading.") clean = make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches") close_bug = make_option("--no-close", action="store_false", dest="close_bug", default=True, help="Leave bug open after landing.") - complete_rollout = make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Commit the revert and re-open the original bug.") component = make_option("--component", action="store", type="string", dest="component", help="Component for the new bug.") confirm = make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Skip confirmation steps.") description = make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: \"patch\")") email = make_option("--email", action="store", type="string", dest="email", help="Email address to use in ChangeLogs.") + fancy_review = make_option("--fancy-review", action="store_true", dest="fancy_review", default=False, help="(Experimental) Upload the patch to Rietveld code review tool.") force_clean = make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)") local_commit = make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch") non_interactive = make_option("--non-interactive", action="store_true", dest="non_interactive", default=False, help="Never prompt the user, fail as fast as possible.") @@ -52,5 +52,5 @@ class Options(object): request_commit = make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review.") review = make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review.") reviewer = make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER.") - test = make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without running run-webkit-tests.") + test = make_option("--test", action="store_true", dest="test", default=False, help="Commit without running run-webkit-tests") update = make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory.") diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/postcodereview.py b/WebKitTools/Scripts/webkitpy/tool/steps/postcodereview.py new file mode 100644 index 0000000..3e7ed76 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/postcodereview.py @@ -0,0 +1,74 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options + + +class PostCodeReview(AbstractStep): + @classmethod + def options(cls): + return [ + Options.cc, + Options.description, + Options.fancy_review, + Options.review, + ] + + def run(self, state): + if not self._options.fancy_review: + return + # FIXME: This will always be None because we don't retrieve the issue + # number from the ChangeLog yet. + codereview_issue = state.get("codereview_issue") + message = self._options.description + if not message: + # If we have an issue number, then the message becomes the label + # of the new patch. Otherwise, it becomes the title of the whole + # issue. + if codereview_issue: + message = "Updated patch" + elif state.get("bug_title"): + # This is the common case for the the first "upload" command. + message = state.get("bug_title") + elif state.get("bug_id"): + # This is the common case for the "post" command and + # subsequent runs of the "upload" command. In this case, + # I'd rather add the new patch to the existing issue, but + # that's not implemented yet. + message = "Code review for %s" % self._tool.bugs.bug_url_for_bug_id(state["bug_id"]) + else: + # Unreachable with our current commands, but we might hit + # this case if we support bug-less code reviews. + message = "Code review" + created_issue = self._tool.codereview.post(message=message, + codereview_issue=codereview_issue, + cc=self._options.cc) + if created_issue: + # FIXME: Record the issue number in the ChangeLog. + state["codereview_issue"] = created_issue diff --git a/WebKitTools/Scripts/webkitpy/steps/postdiff.py b/WebKitTools/Scripts/webkitpy/tool/steps/postdiff.py index a5ba2a4..6a3dee4 100644 --- a/WebKitTools/Scripts/webkitpy/steps/postdiff.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/postdiff.py @@ -28,8 +28,8 @@ import StringIO -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options class PostDiff(AbstractStep): @@ -46,6 +46,12 @@ class PostDiff(AbstractStep): diff = self.cached_lookup(state, "diff") diff_file = StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object description = self._options.description or "Patch" - self._tool.bugs.add_patch_to_bug(state["bug_id"], diff_file, description, mark_for_review=self._options.review, mark_for_commit_queue=self._options.request_commit) + comment_text = None + codereview_issue = state.get("codereview_issue") + # Include codereview issue number in patch name. This is a bit of a hack, + # but it makes doing the rietveld integration a lot easier. + if codereview_issue: + description += "-%s" % state["codereview_issue"] + self._tool.bugs.add_patch_to_bug(state["bug_id"], diff_file, description, comment_text=comment_text, mark_for_review=self._options.review, mark_for_commit_queue=self._options.request_commit) if self._options.open_bug: self._tool.user.open_url(self._tool.bugs.bug_url_for_bug_id(state["bug_id"])) diff --git a/WebKitTools/Scripts/webkitpy/steps/postdiffforcommit.py b/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforcommit.py index 449381c..03b9e78 100644 --- a/WebKitTools/Scripts/webkitpy/steps/postdiffforcommit.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforcommit.py @@ -28,7 +28,7 @@ import StringIO -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep class PostDiffForCommit(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforrevert.py b/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforrevert.py new file mode 100644 index 0000000..3b9da04 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforrevert.py @@ -0,0 +1,51 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import StringIO + +from webkitpy.common.net.bugzilla import Attachment +from webkitpy.tool.steps.abstractstep import AbstractStep + + +class PostDiffForRevert(AbstractStep): + def run(self, state): + comment_text = "Any committer can land this patch automatically by \ +marking it commit-queue+. The commit-queue will build and test \ +the patch before landing to ensure that the rollout will be \ +successful. This process takes approximately 15 minutes.\n\n\ +If you would like to land the rollout faster, you can use the \ +following command:\n\n\ + webkit-patch land-attachment ATTACHMENT_ID --ignore-builders\n\n\ +where ATTACHMENT_ID is the ID of this attachment." + self._tool.bugs.add_patch_to_bug( + state["bug_id"], + StringIO.StringIO(self.cached_lookup(state, "diff")), + "%s%s" % (Attachment.rollout_preamble, state["revision"]), + comment_text=comment_text, + mark_for_review=False, + mark_for_commit_queue=True) diff --git a/WebKitTools/Scripts/webkitpy/steps/preparechangelog.py b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelog.py index bd41f0b..fcb40be 100644 --- a/WebKitTools/Scripts/webkitpy/steps/preparechangelog.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelog.py @@ -28,10 +28,10 @@ import os -from webkitpy.executive import ScriptError -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import error class PrepareChangeLog(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/preparechangelogforrevert.py b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelogforrevert.py index 88e5134..f7d9cd3 100644 --- a/WebKitTools/Scripts/webkitpy/steps/preparechangelogforrevert.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelogforrevert.py @@ -28,20 +28,15 @@ import os -from webkitpy.changelogs import ChangeLog -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.common.checkout.changelog import ChangeLog +from webkitpy.tool.steps.abstractstep import AbstractStep class PrepareChangeLogForRevert(AbstractStep): def run(self, state): - # First, discard the ChangeLog changes from the rollout. - os.chdir(self._tool.scm().checkout_root) - changelog_paths = self._tool.scm().modified_changelogs() - self._tool.scm().revert_files(changelog_paths) - - # Second, make new ChangeLog entries for this rollout. # This could move to prepare-ChangeLog by adding a --revert= option. self._run_script("prepare-ChangeLog") + changelog_paths = self._tool.checkout().modified_changelogs() bug_url = self._tool.bugs.bug_url_for_bug_id(state["bug_id"]) if state["bug_id"] else None for changelog_path in changelog_paths: # FIXME: Seems we should prepare the message outside of changelogs.py and then just pass in diff --git a/WebKitTools/Scripts/webkitpy/steps/promptforbugortitle.py b/WebKitTools/Scripts/webkitpy/tool/steps/promptforbugortitle.py index fb2f877..31c913c 100644 --- a/WebKitTools/Scripts/webkitpy/steps/promptforbugortitle.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/promptforbugortitle.py @@ -26,7 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep class PromptForBugOrTitle(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/confirmdiff.py b/WebKitTools/Scripts/webkitpy/tool/steps/reopenbugafterrollout.py index fc28f8f..f369ca9 100644 --- a/WebKitTools/Scripts/webkitpy/steps/confirmdiff.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/reopenbugafterrollout.py @@ -26,22 +26,19 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.tool.comments import bug_comment_from_commit_text +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.common.system.deprecated_logging import log -class ConfirmDiff(AbstractStep): - @classmethod - def options(cls): - return [ - Options.confirm, - ] - +class ReopenBugAfterRollout(AbstractStep): def run(self, state): - if not self._options.confirm: + commit_comment = bug_comment_from_commit_text(self._tool.scm(), state["commit_text"]) + comment_text = "Reverted r%s for reason:\n\n%s\n\n%s" % (state["revision"], state["reason"], commit_comment) + + bug_id = state["bug_id"] + if not bug_id: + log(comment_text) + log("No bugs were updated.") return - diff = self.cached_lookup(state, "diff") - self._tool.user.page(diff) - if not self._tool.user.confirm("Was that diff correct?"): - exit(1) + self._tool.bugs.reopen_bug(bug_id, comment_text) diff --git a/WebKitTools/Scripts/webkitpy/steps/revertrevision.py b/WebKitTools/Scripts/webkitpy/tool/steps/revertrevision.py index ce6c263..81b6bcb 100644 --- a/WebKitTools/Scripts/webkitpy/steps/revertrevision.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/revertrevision.py @@ -26,9 +26,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep class RevertRevision(AbstractStep): def run(self, state): - self._tool.scm().apply_reverse_diff(state["revision"]) + self._tool.checkout().apply_reverse_diff(state["revision"]) diff --git a/WebKitTools/Scripts/webkitpy/steps/runtests.py b/WebKitTools/Scripts/webkitpy/tool/steps/runtests.py index ebe809f..55d8c62 100644 --- a/WebKitTools/Scripts/webkitpy/steps/runtests.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/runtests.py @@ -26,15 +26,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class RunTests(AbstractStep): @classmethod def options(cls): return [ - Options.build, Options.test, Options.non_interactive, Options.quiet, @@ -42,8 +41,6 @@ class RunTests(AbstractStep): ] def run(self, state): - if not self._options.build: - return if not self._options.test: return diff --git a/WebKitTools/Scripts/webkitpy/steps/steps_unittest.py b/WebKitTools/Scripts/webkitpy/tool/steps/steps_unittest.py index 3e6a032..40bee90 100644 --- a/WebKitTools/Scripts/webkitpy/steps/steps_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/steps_unittest.py @@ -28,17 +28,17 @@ import unittest -from webkitpy.steps.update import Update -from webkitpy.steps.promptforbugortitle import PromptForBugOrTitle -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture -from webkitpy.mock import Mock +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.mocktool import MockTool +from webkitpy.tool.steps.update import Update +from webkitpy.tool.steps.promptforbugortitle import PromptForBugOrTitle class StepsTest(unittest.TestCase): def _run_step(self, step, tool=None, options=None, state=None): if not tool: - tool = MockBugzillaTool() + tool = MockTool() if not options: options = Mock() if not state: @@ -51,6 +51,6 @@ class StepsTest(unittest.TestCase): self._run_step(Update, options) def test_prompt_for_bug_or_title_step(self): - tool = MockBugzillaTool() + tool = MockTool() tool.user.prompt = lambda message: 42 self._run_step(PromptForBugOrTitle, tool=tool) diff --git a/WebKitTools/Scripts/webkitpy/steps/update.py b/WebKitTools/Scripts/webkitpy/tool/steps/update.py index 0f45671..c98eba7 100644 --- a/WebKitTools/Scripts/webkitpy/steps/update.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/update.py @@ -26,9 +26,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class Update(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreview_unittests.py b/WebKitTools/Scripts/webkitpy/tool/steps/updatechangelogswithreview_unittest.py index 102a454..0534718 100644 --- a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreview_unittests.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/updatechangelogswithreview_unittest.py @@ -28,19 +28,19 @@ import unittest -from webkitpy.steps.updatechangelogswithreviewer import UpdateChangeLogsWithReviewer -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.mocktool import MockTool +from webkitpy.tool.steps.updatechangelogswithreviewer import UpdateChangeLogsWithReviewer class UpdateChangeLogsWithReviewerTest(unittest.TestCase): def test_guess_reviewer_from_bug(self): capture = OutputCapture() - step = UpdateChangeLogsWithReviewer(MockBugzillaTool(), Mock()) + step = UpdateChangeLogsWithReviewer(MockTool(), Mock()) expected_stderr = "0 reviewed patches on bug 75, cannot infer reviewer.\n" capture.assert_outputs(self, step._guess_reviewer_from_bug, [75], expected_stderr=expected_stderr) def test_empty_state(self): capture = OutputCapture() - step = UpdateChangeLogsWithReviewer(MockBugzillaTool(), Mock()) + step = UpdateChangeLogsWithReviewer(MockTool(), Mock()) capture.assert_outputs(self, step.run, [{}]) diff --git a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreviewer.py b/WebKitTools/Scripts/webkitpy/tool/steps/updatechangelogswithreviewer.py index 90fdc35..a35ed8c 100644 --- a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreviewer.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/updatechangelogswithreviewer.py @@ -28,11 +28,11 @@ import os -from webkitpy.changelogs import ChangeLog -from webkitpy.grammar import pluralize -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log, error +from webkitpy.common.checkout.changelog import ChangeLog +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log, error class UpdateChangeLogsWithReviewer(AbstractStep): @classmethod @@ -67,5 +67,5 @@ class UpdateChangeLogsWithReviewer(AbstractStep): return os.chdir(self._tool.scm().checkout_root) - for changelog_path in self._tool.scm().modified_changelogs(): + for changelog_path in self._tool.checkout().modified_changelogs(): ChangeLog(changelog_path).set_reviewer(reviewer) diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer.py b/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer.py new file mode 100644 index 0000000..80b2c5d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer.py @@ -0,0 +1,64 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import re + +from webkitpy.common.checkout.changelog import ChangeLog +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.common.system.deprecated_logging import error, log + + +# FIXME: Some of this logic should probably be unified with CommitterValidator? +class ValidateReviewer(AbstractStep): + # FIXME: This should probably move onto ChangeLogEntry + def _has_valid_reviewer(self, changelog_entry): + if changelog_entry.reviewer(): + return True + if re.search("unreviewed", changelog_entry.contents(), re.IGNORECASE): + return True + if re.search("rubber[ -]stamp", changelog_entry.contents(), re.IGNORECASE): + return True + return False + + def run(self, state): + # FIXME: For now we disable this check when a user is driving the script + # this check is too draconian (and too poorly tested) to foist upon users. + if not self._options.non_interactive: + return + # FIXME: We should figure out how to handle the current working + # directory issue more globally. + os.chdir(self._tool.scm().checkout_root) + for changelog_path in self._tool.checkout().modified_changelogs(): + changelog_entry = ChangeLog(changelog_path).latest_entry() + if self._has_valid_reviewer(changelog_entry): + continue + reviewer_text = changelog_entry.reviewer_text() + if reviewer_text: + log("%s found in %s does not appear to be a valid reviewer according to committers.py." % (reviewer_text, changelog_path)) + error('%s neither lists a valid reviewer nor contains the string "Unreviewed" or "Rubber stamp" (case insensitive).' % changelog_path) diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer_unittest.py b/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer_unittest.py new file mode 100644 index 0000000..9105102 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer_unittest.py @@ -0,0 +1,58 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.checkout.changelog import ChangeLogEntry +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.mocktool import MockTool +from webkitpy.tool.steps.validatereviewer import ValidateReviewer + +class ValidateReviewerTest(unittest.TestCase): + _boilerplate_entry = '''2009-08-19 Eric Seidel <eric@webkit.org> + + REVIEW_LINE + + * Scripts/bugzilla-tool: +''' + + def _test_review_text(self, step, text, expected): + contents = self._boilerplate_entry.replace("REVIEW_LINE", text) + entry = ChangeLogEntry(contents) + self.assertEqual(step._has_valid_reviewer(entry), expected) + + def test_has_valid_reviewer(self): + step = ValidateReviewer(MockTool(), Mock()) + self._test_review_text(step, "Reviewed by Eric Seidel.", True) + self._test_review_text(step, "Reviewed by Eric Seidel", True) # Not picky about the '.' + self._test_review_text(step, "Reviewed by Eric.", False) + self._test_review_text(step, "Reviewed by Eric C Seidel.", False) + self._test_review_text(step, "Rubber-stamped by Eric.", True) + self._test_review_text(step, "Rubber stamped by Eric.", True) + self._test_review_text(step, "Unreviewed build fix.", True) |