From 6b70adc33054f8aee8c54d0f460458a9df11b8a5 Mon Sep 17 00:00:00 2001 From: Russell Brenner Date: Thu, 18 Nov 2010 17:33:13 -0800 Subject: Merge WebKit at r72274: Initial merge by git. Change-Id: Ie51f0b4a16da82942bd516dce59cfb79ebbe25fb --- WebKitTools/Scripts/build-webkit | 4 - WebKitTools/Scripts/do-file-rename | 1 + WebKitTools/Scripts/old-run-webkit-tests | 19 +- WebKitTools/Scripts/print-vse-failure-logs | 51 +- .../Scripts/webkitpy/common/checkout/scm.py | 3 +- .../Scripts/webkitpy/common/config/build.py | 138 ++++ .../webkitpy/common/config/build_unittest.py | 64 ++ .../Scripts/webkitpy/common/config/committers.py | 2 +- .../webkitpy/common/config/committervalidator.py | 120 +++ .../common/config/committervalidator_unittest.py | 43 + .../Scripts/webkitpy/common/net/bugzilla.py | 914 --------------------- .../webkitpy/common/net/bugzilla/__init__.py | 8 + .../webkitpy/common/net/bugzilla/attachment.py | 114 +++ .../Scripts/webkitpy/common/net/bugzilla/bug.py | 105 +++ .../webkitpy/common/net/bugzilla/bug_unittest.py | 40 + .../webkitpy/common/net/bugzilla/bugzilla.py | 674 +++++++++++++++ .../common/net/bugzilla/bugzilla_unittest.py | 342 ++++++++ .../webkitpy/common/net/bugzilla_unittest.py | 364 -------- .../Scripts/webkitpy/common/net/buildbot.py | 1 + .../webkitpy/common/net/buildbot_unittest.py | 8 + .../webkitpy/common/net/layouttestresults.py | 2 + .../common/net/layouttestresults_unittest.py | 1 + WebKitTools/Scripts/webkitpy/common/newstringio.py | 40 + .../webkitpy/common/newstringio_unittest.py | 46 ++ .../Scripts/webkitpy/common/system/executive.py | 4 +- .../webkitpy/common/system/executive_mock.py | 55 ++ .../Scripts/webkitpy/common/system/file_lock.py | 83 ++ .../webkitpy/common/system/file_lock_unittest.py | 61 ++ .../webkitpy/common/system/filesystem_mock.py | 75 ++ .../layout_package/dump_render_tree_thread.py | 147 ++-- .../json_layout_results_generator.py | 8 +- .../layout_package/json_results_generator.py | 122 ++- .../json_results_generator_unittest.py | 135 +-- .../layout_tests/layout_package/printing.py | 4 +- .../layout_tests/layout_package/test_output.py | 56 ++ .../Scripts/webkitpy/layout_tests/port/base.py | 197 ++--- .../webkitpy/layout_tests/port/base_unittest.py | 66 +- .../Scripts/webkitpy/layout_tests/port/chromium.py | 18 +- .../webkitpy/layout_tests/port/chromium_gpu.py | 7 +- .../Scripts/webkitpy/layout_tests/port/config.py | 68 +- .../webkitpy/layout_tests/port/config_mock.py | 50 ++ .../layout_tests/port/config_standalone.py | 70 ++ .../webkitpy/layout_tests/port/config_unittest.py | 167 ++-- .../Scripts/webkitpy/layout_tests/port/dryrun.py | 11 +- .../layout_tests/port/google_chrome_unittest.py | 5 +- .../webkitpy/layout_tests/port/http_lock.py | 40 +- .../Scripts/webkitpy/layout_tests/port/test.py | 14 +- .../Scripts/webkitpy/layout_tests/port/webkit.py | 30 +- .../webkitpy/layout_tests/run_webkit_tests.py | 95 +-- .../layout_tests/run_webkit_tests_unittest.py | 50 +- .../webkitpy/layout_tests/test_types/image_diff.py | 123 ++- .../layout_tests/test_types/test_type_base.py | 10 +- .../webkitpy/layout_tests/test_types/text_diff.py | 22 +- WebKitTools/Scripts/webkitpy/style/checker.py | 16 +- .../Scripts/webkitpy/style/checker_unittest.py | 27 +- WebKitTools/Scripts/webkitpy/style/checkers/cpp.py | 205 +++-- .../webkitpy/style/checkers/cpp_unittest.py | 98 ++- WebKitTools/Scripts/webkitpy/tool/bot/feeders.py | 2 +- .../Scripts/webkitpy/tool/commands/__init__.py | 8 +- .../tool/commands/data/rebaselineserver/index.html | 105 ++- .../tool/commands/data/rebaselineserver/loupe.js | 144 ++++ .../tool/commands/data/rebaselineserver/main.css | 159 +++- .../tool/commands/data/rebaselineserver/main.js | 363 ++++++++ .../tool/commands/data/rebaselineserver/util.js | 57 ++ .../Scripts/webkitpy/tool/commands/download.py | 2 - .../webkitpy/tool/commands/download_unittest.py | 9 +- .../Scripts/webkitpy/tool/commands/queues.py | 7 +- .../webkitpy/tool/commands/rebaselineserver.py | 49 +- WebKitTools/Scripts/webkitpy/tool/main.py | 13 +- WebKitTools/Scripts/webkitpy/tool/mocktool.py | 1 + WebKitTools/Scripts/webkitpy/tool/steps/commit.py | 8 +- 71 files changed, 4068 insertions(+), 2102 deletions(-) create mode 100644 WebKitTools/Scripts/webkitpy/common/config/build.py create mode 100644 WebKitTools/Scripts/webkitpy/common/config/build_unittest.py create mode 100644 WebKitTools/Scripts/webkitpy/common/config/committervalidator.py create mode 100644 WebKitTools/Scripts/webkitpy/common/config/committervalidator_unittest.py delete mode 100644 WebKitTools/Scripts/webkitpy/common/net/bugzilla.py create mode 100644 WebKitTools/Scripts/webkitpy/common/net/bugzilla/__init__.py create mode 100644 WebKitTools/Scripts/webkitpy/common/net/bugzilla/attachment.py create mode 100644 WebKitTools/Scripts/webkitpy/common/net/bugzilla/bug.py create mode 100644 WebKitTools/Scripts/webkitpy/common/net/bugzilla/bug_unittest.py create mode 100644 WebKitTools/Scripts/webkitpy/common/net/bugzilla/bugzilla.py create mode 100644 WebKitTools/Scripts/webkitpy/common/net/bugzilla/bugzilla_unittest.py delete mode 100644 WebKitTools/Scripts/webkitpy/common/net/bugzilla_unittest.py create mode 100644 WebKitTools/Scripts/webkitpy/common/newstringio.py create mode 100644 WebKitTools/Scripts/webkitpy/common/newstringio_unittest.py create mode 100644 WebKitTools/Scripts/webkitpy/common/system/executive_mock.py create mode 100644 WebKitTools/Scripts/webkitpy/common/system/file_lock.py create mode 100644 WebKitTools/Scripts/webkitpy/common/system/file_lock_unittest.py create mode 100644 WebKitTools/Scripts/webkitpy/common/system/filesystem_mock.py create mode 100644 WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_output.py create mode 100644 WebKitTools/Scripts/webkitpy/layout_tests/port/config_mock.py create mode 100644 WebKitTools/Scripts/webkitpy/layout_tests/port/config_standalone.py create mode 100644 WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/loupe.js create mode 100644 WebKitTools/Scripts/webkitpy/tool/commands/data/rebaselineserver/util.js (limited to 'WebKitTools/Scripts') diff --git a/WebKitTools/Scripts/build-webkit b/WebKitTools/Scripts/build-webkit index da336fe..d9534f8 100755 --- a/WebKitTools/Scripts/build-webkit +++ b/WebKitTools/Scripts/build-webkit @@ -87,7 +87,6 @@ my ( $notificationsSupport, $offlineWebApplicationSupport, $progressTagSupport, - $rubySupport, $sharedWorkersSupport, $svgSupport, $svgAnimationSupport, @@ -198,9 +197,6 @@ my @features = ( { option => "progress-tag", desc => "Progress Tag support", define => "ENABLE_PROGRESS_TAG", default => 1, value => \$progressTagSupport }, - { option => "ruby", desc => "Toggle HTML5 Ruby support", - define => "ENABLE_RUBY", default => 1, value => \$rubySupport }, - { option => "system-malloc", desc => "Toggle system allocator instead of TCmalloc", define => "USE_SYSTEM_MALLOC", default => 0, value => \$systemMallocSupport }, diff --git a/WebKitTools/Scripts/do-file-rename b/WebKitTools/Scripts/do-file-rename index b81b9dc..9c4c898 100755 --- a/WebKitTools/Scripts/do-file-rename +++ b/WebKitTools/Scripts/do-file-rename @@ -47,6 +47,7 @@ find(\&wanted, "JavaScriptCore"); find(\&wanted, "JavaScriptGlue"); find(\&wanted, "WebCore"); find(\&wanted, "WebKit"); +find(\&wanted, "WebKit2"); sub wanted { diff --git a/WebKitTools/Scripts/old-run-webkit-tests b/WebKitTools/Scripts/old-run-webkit-tests index 2cef18b..eeaaab3 100755 --- a/WebKitTools/Scripts/old-run-webkit-tests +++ b/WebKitTools/Scripts/old-run-webkit-tests @@ -917,8 +917,7 @@ for my $test (@tests) { testCrashedOrTimedOut($test, $base, 1, $actual, $error); } elsif (!defined $expected) { if ($verbose) { - print "new " . ($resetResults ? "result" : "test") ."\n"; - $atLineStart = 1; + print "new " . ($resetResults ? "result" : "test"); } $result = "new"; @@ -936,11 +935,13 @@ for my $test (@tests) { print "$test -> "; } my $resultsDir = catdir($expectedDir, dirname($base)); + if (!$verbose) { + print "new"; + } if ($generateNewResults) { - print "new (results generated in $resultsDir)\n"; - } else { - print "new\n"; + print " (results generated in $resultsDir)"; } + print "\n" unless $atLineStart; $atLineStart = 1; } } elsif ($actual eq $expected && $diffResult eq "passed") { @@ -1420,6 +1421,10 @@ sub openDumpTool() { return if $isDumpToolOpen; + if ($verbose && $testsPerDumpTool != 1) { + print "| Opening DumpTool |\n"; + } + my %CLEAN_ENV; # Generic environment variables @@ -1502,6 +1507,10 @@ sub closeDumpTool() { return if !$isDumpToolOpen; + if ($verbose && $testsPerDumpTool != 1) { + print "| Closing DumpTool |\n"; + } + close IN; close OUT; waitpid $dumpToolPID, 0; diff --git a/WebKitTools/Scripts/print-vse-failure-logs b/WebKitTools/Scripts/print-vse-failure-logs index cec806e..7580465 100755 --- a/WebKitTools/Scripts/print-vse-failure-logs +++ b/WebKitTools/Scripts/print-vse-failure-logs @@ -33,15 +33,14 @@ from __future__ import with_statement import codecs -import os.path import os +import re +from webkitpy.common.checkout import scm from webkitpy.common.system.executive import Executive +from webkitpy.thirdparty import BeautifulSoup -# This is just enough to make the win-ews bot useable. Others who -# actually use Windows should feel encouraged to improve this class, -# including just printing the text instead of the raw html. class PrintVisualStudioExpressLogs(object): def __init__(self): self._executive = Executive() @@ -55,25 +54,59 @@ class PrintVisualStudioExpressLogs(object): build_log_paths.append(file_path) return build_log_paths + def _build_order(self): + """Returns a list of project names in the order in which they are built.""" + script_path = os.path.join(self._scripts_directory(), "print-msvc-project-dependencies") + sln_path = os.path.join(scm.find_checkout_root(), "WebKit", "win", "WebKit.vcproj", "WebKit.sln") + lines = self._executive.run_command([script_path, sln_path]).splitlines() + order = [line.strip() for line in lines if line.find("Folder") == -1] + order.reverse() + return order + + def _sort_buildlogs(self, log_paths): + build_order = self._build_order() + def sort_key(log_path): + project_name = os.path.basename(os.path.dirname(os.path.dirname(log_path))) + try: + index = build_order.index(project_name) + except ValueError: + # If the project isn't in the list, sort it after all items that + # are in the list. + index = len(build_order) + # Sort first by build order, then by project name + return (index, project_name) + return sorted(log_paths, key=sort_key) + def _obj_directory(self): - scripts_directory = os.path.dirname(__file__) - build_directory_script_path = os.path.join(scripts_directory, "webkit-build-directory") + build_directory_script_path = os.path.join(self._scripts_directory(), "webkit-build-directory") # FIXME: ports/webkit.py should provide the build directory in a nice API. # NOTE: The windows VSE build does not seem to use different directories # for Debug and Release. build_directory = self._executive.run_command([build_directory_script_path, "--top-level"]).rstrip() return os.path.join(build_directory, "obj") + def _scripts_directory(self): + return os.path.dirname(__file__) + + def _relevant_text(self, log): + soup = BeautifulSoup.BeautifulSoup(log) + # The Output Window table is where the useful output starts in the build log. + output_window_table = soup.find(text=re.compile("Output Window")).findParent("table") + result = [] + for table in [output_window_table] + output_window_table.findNextSiblings("table"): + result.extend([text.replace(" ", "") for text in table.findAll(text=True)]) + result.append("\n") + return "".join(result) + def main(self): - obj_directory = self._obj_directory() - build_log_paths = self._find_buildlogs(obj_directory) + build_log_paths = self._sort_buildlogs(self._find_buildlogs(self._obj_directory())) print "Found %s Visual Studio Express Build Logs:\n%s" % (len(build_log_paths), "\n".join(build_log_paths)) for build_log_path in build_log_paths: print "%s:\n" % build_log_path with codecs.open(build_log_path, "r", "utf-16") as build_log: - print build_log.read() + print self._relevant_text(build_log) if __name__ == '__main__': diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/scm.py b/WebKitTools/Scripts/webkitpy/common/checkout/scm.py index 8aadcb8..11e82ac 100644 --- a/WebKitTools/Scripts/webkitpy/common/checkout/scm.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/scm.py @@ -700,8 +700,7 @@ class Git(SCM): """Returns a byte array (str()) representing the patch file. Patch files are effectively binary since they may contain files of multiple different encodings.""" - # FIXME: This should probably use cwd=self.checkout_root - return self.run(['git', 'diff', '--binary', "--no-ext-diff", "--full-index", "-M", self.merge_base(git_commit), "--"] + changed_files, decode_output=False) + return self.run(['git', 'diff', '--binary', "--no-ext-diff", "--full-index", "-M", self.merge_base(git_commit), "--"] + changed_files, decode_output=False, cwd=self.checkout_root) def _run_git_svn_find_rev(self, arg): # git svn find-rev always exits 0, even when the revision or commit is not found. diff --git a/WebKitTools/Scripts/webkitpy/common/config/build.py b/WebKitTools/Scripts/webkitpy/common/config/build.py new file mode 100644 index 0000000..c45f122 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/config/build.py @@ -0,0 +1,138 @@ +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Functions relating to building WebKit""" + +import re + + +def _should_file_trigger_build(target_platform, file): + # The directories and patterns lists below map directory names or + # regexp patterns to the bot platforms for which they should trigger a + # build. Mapping to the empty list means that no builds should be + # triggered on any platforms. Earlier directories/patterns take + # precendence over later ones. + + # FIXME: The patterns below have only been verified to be correct on + # Windows. We should implement this for other platforms and start using + # it for their bots. Someone familiar with each platform will have to + # figure out what the right set of directories/patterns is for that + # platform. + assert(target_platform == "win") + + directories = [ + # Directories that shouldn't trigger builds on any bots. + ("BugsSite", []), + ("PageLoadTests", []), + ("PlanetWebKit", []), + ("WebCore/manual-tests", []), + ("WebKitExamplePlugins", []), + ("WebKitSite", []), + ("android", []), + ("brew", []), + ("efl", []), + ("haiku", []), + ("iphone", []), + ("opengl", []), + ("opentype", []), + ("openvg", []), + ("wx", []), + ("wince", []), + + # Directories that should trigger builds on only some bots. + ("JavaScriptGlue", ["mac"]), + ("LayoutTests/platform/mac", ["mac", "win"]), + ("LayoutTests/platform/mac-snowleopard", ["mac-snowleopard", "win"]), + ("WebCore/image-decoders", ["chromium"]), + ("cairo", ["gtk", "wincairo"]), + ("cf", ["chromium-mac", "mac", "qt", "win"]), + ("chromium", ["chromium"]), + ("cocoa", ["chromium-mac", "mac"]), + ("curl", ["gtk", "wincairo"]), + ("gobject", ["gtk"]), + ("gpu", ["chromium", "mac"]), + ("gstreamer", ["gtk"]), + ("gtk", ["gtk"]), + ("mac", ["chromium-mac", "mac"]), + ("mac-leopard", ["mac-leopard"]), + ("mac-snowleopard", ["mac-snowleopard"]), + ("mac-wk2", ["mac-snowleopard", "win"]), + ("objc", ["mac"]), + ("qt", ["qt"]), + ("skia", ["chromium"]), + ("soup", ["gtk"]), + ("v8", ["chromium"]), + ("win", ["chromium-win", "win"]), + ] + patterns = [ + # Patterns that shouldn't trigger builds on any bots. + (r"(?:^|/)Makefile$", []), + (r"/ARM", []), + (r"/CMake.*", []), + (r"/ChangeLog.*$", []), + (r"/LICENSE[^/]+$", []), + (r"ARM(?:v7)?\.(?:cpp|h)$", []), + (r"MIPS\.(?:cpp|h)$", []), + (r"WinCE\.(?:cpp|h|mm)$", []), + (r"\.(?:bkl|mk)$", []), + + # Patterns that should trigger builds on only some bots. + (r"/GNUmakefile\.am$", ["gtk"]), + (r"/\w+Chromium\w*\.(?:cpp|h|mm)$", ["chromium"]), + (r"Mac\.(?:cpp|h|mm)$", ["mac"]), + (r"\.exp$", ["mac"]), + (r"\.gypi?", ["chromium"]), + (r"\.order$", ["mac"]), + (r"\.pr[io]$", ["qt"]), + (r"\.xcconfig$", ["mac"]), + (r"\.xcodeproj/", ["mac"]), + ] + + base_platform = target_platform.split("-")[0] + + # See if the file is in one of the known directories. + for directory, platforms in directories: + if re.search(r"(?:^|/)%s/" % directory, file): + return target_platform in platforms or base_platform in platforms + + # See if the file matches a known pattern. + for pattern, platforms in patterns: + if re.search(pattern, file): + return target_platform in platforms or base_platform in platforms + + # See if the file is a platform-specific test result. + match = re.match("LayoutTests/platform/(?P[^/]+)/", file) + if match: + # See if the file is a test result for this platform, our base + # platform, or one of our sub-platforms. + return match.group("platform") in (target_platform, base_platform) or match.group("platform").startswith("%s-" % target_platform) + + # The file isn't one we know about specifically, so we should assume we + # have to build. + return True + + +def should_build(target_platform, changed_files): + """Returns true if the changed files affect the given platform, and + thus a build should be performed. target_platform should be one of the + platforms used in the build.webkit.org master's config.json file.""" + return any(_should_file_trigger_build(target_platform, file) for file in changed_files) diff --git a/WebKitTools/Scripts/webkitpy/common/config/build_unittest.py b/WebKitTools/Scripts/webkitpy/common/config/build_unittest.py new file mode 100644 index 0000000..3e70ff0 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/config/build_unittest.py @@ -0,0 +1,64 @@ +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.config import build + + +class ShouldBuildTest(unittest.TestCase): + _should_build_tests = [ + (["BugsSite/foo", "WebCore/bar"], ["*"]), + (["BugsSite/foo"], []), + (["JavaScriptCore/JavaScriptCore.xcodeproj/foo"], ["mac-leopard", "mac-snowleopard"]), + (["JavaScriptGlue/foo", "WebCore/bar"], ["*"]), + (["JavaScriptGlue/foo"], ["mac-leopard", "mac-snowleopard"]), + (["LayoutTests/foo"], ["*"]), + (["LayoutTests/platform/chromium-linux/foo"], ["chromium-linux"]), + (["LayoutTests/platform/chromium-win/fast/compact/001-expected.txt"], ["chromium-win"]), + (["LayoutTests/platform/mac-leopard/foo"], ["mac-leopard"]), + (["LayoutTests/platform/mac-snowleopard/foo"], ["mac-snowleopard", "win"]), + (["LayoutTests/platform/mac-wk2/Skipped"], ["mac-snowleopard", "win"]), + (["LayoutTests/platform/mac/foo"], ["mac-leopard", "mac-snowleopard", "win"]), + (["LayoutTests/platform/win-xp/foo"], ["win"]), + (["LayoutTests/platform/win-wk2/foo"], ["win"]), + (["LayoutTests/platform/win/foo"], ["win"]), + (["WebCore/mac/foo"], ["chromium-mac", "mac-leopard", "mac-snowleopard"]), + (["WebCore/win/foo"], ["chromium-win", "win"]), + (["WebCore/platform/graphics/gpu/foo"], ["mac-leopard", "mac-snowleopard"]), + (["WebCore/platform/wx/wxcode/win/foo"], []), + (["WebCore/rendering/RenderThemeMac.mm", "WebCore/rendering/RenderThemeMac.h"], ["mac-leopard", "mac-snowleopard"]), + (["WebCore/rendering/RenderThemeChromiumLinux.h"], ["chromium-linux"]), + (["WebCore/rendering/RenderThemeWinCE.h"], []), + ] + + def test_should_build(self): + for files, platforms in self._should_build_tests: + # FIXME: We should test more platforms here once + # build._should_file_trigger_build is implemented for them. + for platform in ["win"]: + should_build = platform in platforms or "*" in platforms + self.assertEqual(build.should_build(platform, files), should_build, "%s should%s have built but did%s (files: %s)" % (platform, "" if should_build else "n't", "n't" if should_build else "", str(files))) + + +if __name__ == "__main__": + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/common/config/committers.py b/WebKitTools/Scripts/webkitpy/common/config/committers.py index 446c2b1..0967340 100644 --- a/WebKitTools/Scripts/webkitpy/common/config/committers.py +++ b/WebKitTools/Scripts/webkitpy/common/config/committers.py @@ -69,7 +69,7 @@ committers_unable_to_review = [ Committer("Albert J. Wong", "ajwong@chromium.org"), Committer("Alejandro G. Castro", ["alex@igalia.com", "alex@webkit.org"]), Committer("Alexander Kellett", ["lypanov@mac.com", "a-lists001@lypanov.net", "lypanov@kde.org"], "lypanov"), - Committer("Alexander Pavlov", "apavlov@chromium.org"), + Committer("Alexander Pavlov", "apavlov@chromium.org", "apavlov"), Committer("Andre Boule", "aboule@apple.com"), Committer("Andrei Popescu", "andreip@google.com", "andreip"), Committer("Andrew Wellington", ["andrew@webkit.org", "proton@wiretapped.net"], "proton"), diff --git a/WebKitTools/Scripts/webkitpy/common/config/committervalidator.py b/WebKitTools/Scripts/webkitpy/common/config/committervalidator.py new file mode 100644 index 0000000..b7b2990 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/config/committervalidator.py @@ -0,0 +1,120 @@ +# Copyright (c) 2009 Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# Copyright (c) 2010 Research In Motion Limited. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + +from webkitpy.common.system.ospath import relpath +from webkitpy.common.config import committers + + +class CommitterValidator(object): + + def __init__(self, bugzilla): + self._bugzilla = bugzilla + + # _view_source_url belongs in some sort of webkit_config.py module. + def _view_source_url(self, local_path): + return "http://trac.webkit.org/browser/trunk/%s" % local_path + + def _checkout_root(self): + # FIXME: This is a hack, we would have this from scm.checkout_root + # if we had any way to get to an scm object here. + components = __file__.split(os.sep) + tools_index = components.index("WebKitTools") + return os.sep.join(components[:tools_index]) + + def _committers_py_path(self): + # extension can sometimes be .pyc, we always want .py + (path, extension) = os.path.splitext(committers.__file__) + # FIXME: When we're allowed to use python 2.6 we can use the real + # os.path.relpath + path = relpath(path, self._checkout_root()) + return ".".join([path, "py"]) + + def _flag_permission_rejection_message(self, setter_email, flag_name): + # Should come from some webkit_config.py + contribution_guidlines = "http://webkit.org/coding/contributing.html" + # This could be queried from the status_server. + queue_administrator = "eseidel@chromium.org" + # This could be queried from the tool. + queue_name = "commit-queue" + committers_list = self._committers_py_path() + message = "%s does not have %s permissions according to %s." % ( + setter_email, + flag_name, + self._view_source_url(committers_list)) + message += "\n\n- If you do not have %s rights please read %s for instructions on how to use bugzilla flags." % ( + flag_name, contribution_guidlines) + message += "\n\n- If you have %s rights please correct the error in %s by adding yourself to the file (no review needed). " % ( + flag_name, committers_list) + message += "The %s restarts itself every 2 hours. After restart the %s will correctly respect your %s rights." % ( + queue_name, queue_name, flag_name) + return message + + def _validate_setter_email(self, patch, result_key, rejection_function): + committer = getattr(patch, result_key)() + # If the flag is set, and we don't recognize the setter, reject the + # flag! + setter_email = patch._attachment_dictionary.get("%s_email" % result_key) + if setter_email and not committer: + rejection_function(patch.id(), + self._flag_permission_rejection_message(setter_email, + result_key)) + return False + return True + + def _reject_patch_if_flags_are_invalid(self, patch): + return (self._validate_setter_email( + patch, "reviewer", self.reject_patch_from_review_queue) + and self._validate_setter_email( + patch, "committer", self.reject_patch_from_commit_queue)) + + def patches_after_rejecting_invalid_commiters_and_reviewers(self, patches): + return [patch for patch in patches if self._reject_patch_if_flags_are_invalid(patch)] + + def reject_patch_from_commit_queue(self, + attachment_id, + additional_comment_text=None): + comment_text = "Rejecting patch %s from commit-queue." % attachment_id + self._bugzilla.set_flag_on_attachment(attachment_id, + "commit-queue", + "-", + comment_text, + additional_comment_text) + + def reject_patch_from_review_queue(self, + attachment_id, + additional_comment_text=None): + comment_text = "Rejecting patch %s from review queue." % attachment_id + self._bugzilla.set_flag_on_attachment(attachment_id, + 'review', + '-', + comment_text, + additional_comment_text) diff --git a/WebKitTools/Scripts/webkitpy/common/config/committervalidator_unittest.py b/WebKitTools/Scripts/webkitpy/common/config/committervalidator_unittest.py new file mode 100644 index 0000000..61fa3bf --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/config/committervalidator_unittest.py @@ -0,0 +1,43 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from .committervalidator import CommitterValidator + + +class CommitterValidatorTest(unittest.TestCase): + def test_flag_permission_rejection_message(self): + validator = CommitterValidator(bugzilla=None) + self.assertEqual(validator._committers_py_path(), "WebKitTools/Scripts/webkitpy/common/config/committers.py") + expected_messsage = """foo@foo.com does not have review permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/common/config/committers.py. + +- If you do not have review rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags. + +- If you have review rights please correct the error in WebKitTools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed). The commit-queue restarts itself every 2 hours. After restart the commit-queue will correctly respect your review rights.""" + self.assertEqual(validator._flag_permission_rejection_message("foo@foo.com", "review"), expected_messsage) diff --git a/WebKitTools/Scripts/webkitpy/common/net/bugzilla.py b/WebKitTools/Scripts/webkitpy/common/net/bugzilla.py deleted file mode 100644 index a7dc1b7..0000000 --- a/WebKitTools/Scripts/webkitpy/common/net/bugzilla.py +++ /dev/null @@ -1,914 +0,0 @@ -# Copyright (c) 2009 Google Inc. All rights reserved. -# Copyright (c) 2009 Apple Inc. All rights reserved. -# Copyright (c) 2010 Research In Motion Limited. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# WebKit's Python module for interacting with Bugzilla - -import os.path -import re -import StringIO - -from datetime import datetime # used in timestamp() - -from webkitpy.common.system.deprecated_logging import error, log -from webkitpy.common.config import committers -from webkitpy.common.net.credentials import Credentials -from webkitpy.common.system.ospath import relpath -from webkitpy.common.system.user import User -from webkitpy.thirdparty.autoinstalled.mechanize import Browser -from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, SoupStrainer - - -def parse_bug_id(message): - if not message: - return None - match = re.search("http\://webkit\.org/b/(?P\d+)", message) - if match: - return int(match.group('bug_id')) - match = re.search( - Bugzilla.bug_server_regex + "show_bug\.cgi\?id=(?P\d+)", - message) - if match: - return int(match.group('bug_id')) - return None - - -def timestamp(): - return datetime.now().strftime("%Y%m%d%H%M%S") - - -class Attachment(object): - - rollout_preamble = "ROLLOUT of r" - - def __init__(self, attachment_dictionary, bug): - self._attachment_dictionary = attachment_dictionary - self._bug = bug - self._reviewer = None - self._committer = None - - def _bugzilla(self): - return self._bug._bugzilla - - def id(self): - return int(self._attachment_dictionary.get("id")) - - def attacher_is_committer(self): - return self._bugzilla.committers.committer_by_email( - patch.attacher_email()) - - def attacher_email(self): - return self._attachment_dictionary.get("attacher_email") - - def bug(self): - return self._bug - - def bug_id(self): - return int(self._attachment_dictionary.get("bug_id")) - - def is_patch(self): - return not not self._attachment_dictionary.get("is_patch") - - def is_obsolete(self): - return not not self._attachment_dictionary.get("is_obsolete") - - def is_rollout(self): - return self.name().startswith(self.rollout_preamble) - - def name(self): - return self._attachment_dictionary.get("name") - - def attach_date(self): - return self._attachment_dictionary.get("attach_date") - - def review(self): - return self._attachment_dictionary.get("review") - - def commit_queue(self): - return self._attachment_dictionary.get("commit-queue") - - def url(self): - # FIXME: This should just return - # self._bugzilla().attachment_url_for_id(self.id()). scm_unittest.py - # depends on the current behavior. - return self._attachment_dictionary.get("url") - - def contents(self): - # FIXME: We shouldn't be grabbing at _bugzilla. - return self._bug._bugzilla.fetch_attachment_contents(self.id()) - - def _validate_flag_value(self, flag): - email = self._attachment_dictionary.get("%s_email" % flag) - if not email: - return None - committer = getattr(self._bugzilla().committers, - "%s_by_email" % flag)(email) - if committer: - return committer - log("Warning, attachment %s on bug %s has invalid %s (%s)" % ( - self._attachment_dictionary['id'], - self._attachment_dictionary['bug_id'], flag, email)) - - def reviewer(self): - if not self._reviewer: - self._reviewer = self._validate_flag_value("reviewer") - return self._reviewer - - def committer(self): - if not self._committer: - self._committer = self._validate_flag_value("committer") - return self._committer - - -class Bug(object): - # FIXME: This class is kinda a hack for now. It exists so we have one - # place to hold bug logic, even if much of the code deals with - # dictionaries still. - - def __init__(self, bug_dictionary, bugzilla): - self.bug_dictionary = bug_dictionary - self._bugzilla = bugzilla - - def id(self): - return self.bug_dictionary["id"] - - def title(self): - return self.bug_dictionary["title"] - - def assigned_to_email(self): - return self.bug_dictionary["assigned_to_email"] - - # FIXME: This information should be stored in some sort of webkit_config.py instead of here. - unassigned_emails = frozenset([ - "webkit-unassigned@lists.webkit.org", - "webkit-qt-unassigned@trolltech.com", - ]) - def is_unassigned(self): - return self.assigned_to_email() in self.unassigned_emails - - def status(self): - return self.bug_dictionary["bug_status"] - - # Bugzilla has many status states we don't really use in WebKit: - # https://bugs.webkit.org/page.cgi?id=fields.html#status - _open_states = ["UNCONFIRMED", "NEW", "ASSIGNED", "REOPENED"] - _closed_states = ["RESOLVED", "VERIFIED", "CLOSED"] - - def is_open(self): - return self.status() in self._open_states - - def is_closed(self): - return not self.is_open() - - # Rarely do we actually want obsolete attachments - def attachments(self, include_obsolete=False): - attachments = self.bug_dictionary["attachments"] - if not include_obsolete: - attachments = filter(lambda attachment: - not attachment["is_obsolete"], attachments) - return [Attachment(attachment, self) for attachment in attachments] - - def patches(self, include_obsolete=False): - return [patch for patch in self.attachments(include_obsolete) - if patch.is_patch()] - - def unreviewed_patches(self): - return [patch for patch in self.patches() if patch.review() == "?"] - - def reviewed_patches(self, include_invalid=False): - patches = [patch for patch in self.patches() if patch.review() == "+"] - if include_invalid: - return patches - # Checking reviewer() ensures that it was both reviewed and has a valid - # reviewer. - return filter(lambda patch: patch.reviewer(), patches) - - def commit_queued_patches(self, include_invalid=False): - patches = [patch for patch in self.patches() - if patch.commit_queue() == "+"] - if include_invalid: - return patches - # Checking committer() ensures that it was both commit-queue+'d and has - # a valid committer. - return filter(lambda patch: patch.committer(), patches) - - -# A container for all of the logic for making and parsing buzilla queries. -class BugzillaQueries(object): - - def __init__(self, bugzilla): - self._bugzilla = bugzilla - - # Note: _load_query and _fetch_bug are the only two methods which access - # self._bugzilla. - - def _load_query(self, query): - self._bugzilla.authenticate() - - full_url = "%s%s" % (self._bugzilla.bug_server_url, query) - return self._bugzilla.browser.open(full_url) - - def _fetch_bug(self, bug_id): - return self._bugzilla.fetch_bug(bug_id) - - def _fetch_bug_ids_advanced_query(self, query): - soup = BeautifulSoup(self._load_query(query)) - # The contents of the inside the cells in the first column happen - # to be the bug id. - return [int(bug_link_cell.find("a").string) - for bug_link_cell in soup('td', "first-child")] - - def _parse_attachment_ids_request_query(self, page): - digits = re.compile("\d+") - attachment_href = re.compile("attachment.cgi\?id=\d+&action=review") - attachment_links = SoupStrainer("a", href=attachment_href) - return [int(digits.search(tag["href"]).group(0)) - for tag in BeautifulSoup(page, parseOnlyThese=attachment_links)] - - def _fetch_attachment_ids_request_query(self, query): - return self._parse_attachment_ids_request_query(self._load_query(query)) - - def _parse_quips(self, page): - soup = BeautifulSoup(page, convertEntities=BeautifulSoup.HTML_ENTITIES) - quips = soup.find(text=re.compile(r"Existing quips:")).findNext("ul").findAll("li") - return [unicode(quip_entry.string) for quip_entry in quips] - - def fetch_quips(self): - return self._parse_quips(self._load_query("/quips.cgi?action=show")) - - # List of all r+'d bugs. - def fetch_bug_ids_from_pending_commit_list(self): - needs_commit_query_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B" - return self._fetch_bug_ids_advanced_query(needs_commit_query_url) - - def fetch_patches_from_pending_commit_list(self): - return sum([self._fetch_bug(bug_id).reviewed_patches() - for bug_id in self.fetch_bug_ids_from_pending_commit_list()], []) - - def fetch_bug_ids_from_commit_queue(self): - commit_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B&order=Last+Changed" - return self._fetch_bug_ids_advanced_query(commit_queue_url) - - def fetch_patches_from_commit_queue(self): - # This function will only return patches which have valid committers - # set. It won't reject patches with invalid committers/reviewers. - return sum([self._fetch_bug(bug_id).commit_queued_patches() - for bug_id in self.fetch_bug_ids_from_commit_queue()], []) - - def fetch_bug_ids_from_review_queue(self): - review_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?" - return self._fetch_bug_ids_advanced_query(review_queue_url) - - # This method will make several requests to bugzilla. - def fetch_patches_from_review_queue(self, limit=None): - # [:None] returns the whole array. - return sum([self._fetch_bug(bug_id).unreviewed_patches() - for bug_id in self.fetch_bug_ids_from_review_queue()[:limit]], []) - - # NOTE: This is the only client of _fetch_attachment_ids_request_query - # This method only makes one request to bugzilla. - def fetch_attachment_ids_from_review_queue(self): - review_queue_url = "request.cgi?action=queue&type=review&group=type" - return self._fetch_attachment_ids_request_query(review_queue_url) - - -class CommitterValidator(object): - - def __init__(self, bugzilla): - self._bugzilla = bugzilla - - # _view_source_url belongs in some sort of webkit_config.py module. - def _view_source_url(self, local_path): - return "http://trac.webkit.org/browser/trunk/%s" % local_path - - def _checkout_root(self): - # FIXME: This is a hack, we would have this from scm.checkout_root - # if we had any way to get to an scm object here. - components = __file__.split(os.sep) - tools_index = components.index("WebKitTools") - return os.sep.join(components[:tools_index]) - - def _committers_py_path(self): - # extension can sometimes be .pyc, we always want .py - (path, extension) = os.path.splitext(committers.__file__) - # FIXME: When we're allowed to use python 2.6 we can use the real - # os.path.relpath - path = relpath(path, self._checkout_root()) - return ".".join([path, "py"]) - - def _flag_permission_rejection_message(self, setter_email, flag_name): - # Should come from some webkit_config.py - contribution_guidlines = "http://webkit.org/coding/contributing.html" - # This could be queried from the status_server. - queue_administrator = "eseidel@chromium.org" - # This could be queried from the tool. - queue_name = "commit-queue" - committers_list = self._committers_py_path() - message = "%s does not have %s permissions according to %s." % ( - setter_email, - flag_name, - self._view_source_url(committers_list)) - message += "\n\n- If you do not have %s rights please read %s for instructions on how to use bugzilla flags." % ( - flag_name, contribution_guidlines) - message += "\n\n- If you have %s rights please correct the error in %s by adding yourself to the file (no review needed). " % ( - flag_name, committers_list) - message += "The %s restarts itself every 2 hours. After restart the %s will correctly respect your %s rights." % ( - queue_name, queue_name, flag_name) - return message - - def _validate_setter_email(self, patch, result_key, rejection_function): - committer = getattr(patch, result_key)() - # If the flag is set, and we don't recognize the setter, reject the - # flag! - setter_email = patch._attachment_dictionary.get("%s_email" % result_key) - if setter_email and not committer: - rejection_function(patch.id(), - self._flag_permission_rejection_message(setter_email, - result_key)) - return False - return True - - def _reject_patch_if_flags_are_invalid(self, patch): - return (self._validate_setter_email( - patch, "reviewer", self.reject_patch_from_review_queue) - and self._validate_setter_email( - patch, "committer", self.reject_patch_from_commit_queue)) - - def patches_after_rejecting_invalid_commiters_and_reviewers(self, patches): - return [patch for patch in patches if self._reject_patch_if_flags_are_invalid(patch)] - - def reject_patch_from_commit_queue(self, - attachment_id, - additional_comment_text=None): - comment_text = "Rejecting patch %s from commit-queue." % attachment_id - self._bugzilla.set_flag_on_attachment(attachment_id, - "commit-queue", - "-", - comment_text, - additional_comment_text) - - def reject_patch_from_review_queue(self, - attachment_id, - additional_comment_text=None): - comment_text = "Rejecting patch %s from review queue." % attachment_id - self._bugzilla.set_flag_on_attachment(attachment_id, - 'review', - '-', - comment_text, - additional_comment_text) - - -class Bugzilla(object): - - def __init__(self, dryrun=False, committers=committers.CommitterList()): - self.dryrun = dryrun - self.authenticated = False - self.queries = BugzillaQueries(self) - self.committers = committers - self.cached_quips = [] - - # FIXME: We should use some sort of Browser mock object when in dryrun - # mode (to prevent any mistakes). - self.browser = Browser() - # Ignore bugs.webkit.org/robots.txt until we fix it to allow this - # script. - self.browser.set_handle_robots(False) - - # FIXME: Much of this should go into some sort of config module: - bug_server_host = "bugs.webkit.org" - bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) - bug_server_url = "https://%s/" % bug_server_host - - def quips(self): - # We only fetch and parse the list of quips once per instantiation - # so that we do not burden bugs.webkit.org. - if not self.cached_quips and not self.dryrun: - self.cached_quips = self.queries.fetch_quips() - return self.cached_quips - - def bug_url_for_bug_id(self, bug_id, xml=False): - if not bug_id: - return None - content_type = "&ctype=xml" if xml else "" - return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, - bug_id, - content_type) - - def short_bug_url_for_bug_id(self, bug_id): - if not bug_id: - return None - return "http://webkit.org/b/%s" % bug_id - - def attachment_url_for_id(self, attachment_id, action="view"): - if not attachment_id: - return None - action_param = "" - if action and action != "view": - action_param = "&action=%s" % action - return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, - attachment_id, - action_param) - - def _parse_attachment_flag(self, - element, - flag_name, - attachment, - result_key): - flag = element.find('flag', attrs={'name': flag_name}) - if flag: - attachment[flag_name] = flag['status'] - if flag['status'] == '+': - attachment[result_key] = flag['setter'] - # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date. - - def _string_contents(self, soup): - # WebKit's bugzilla instance uses UTF-8. - # BeautifulSoup always returns Unicode strings, however - # the .string method returns a (unicode) NavigableString. - # NavigableString can confuse other parts of the code, so we - # convert from NavigableString to a real unicode() object using unicode(). - return unicode(soup.string) - - # Example: 2010-01-20 14:31 PST - # FIXME: Some bugzilla dates seem to have seconds in them? - # Python does not support timezones out of the box. - # Assume that bugzilla always uses PST (which is true for bugs.webkit.org) - _bugzilla_date_format = "%Y-%m-%d %H:%M" - - @classmethod - def _parse_date(cls, date_string): - (date, time, time_zone) = date_string.split(" ") - # Ignore the timezone because python doesn't understand timezones out of the box. - date_string = "%s %s" % (date, time) - return datetime.strptime(date_string, cls._bugzilla_date_format) - - def _date_contents(self, soup): - return self._parse_date(self._string_contents(soup)) - - def _parse_attachment_element(self, element, bug_id): - attachment = {} - attachment['bug_id'] = bug_id - attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1") - attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1") - attachment['id'] = int(element.find('attachid').string) - # FIXME: No need to parse out the url here. - attachment['url'] = self.attachment_url_for_id(attachment['id']) - attachment["attach_date"] = self._date_contents(element.find("date")) - attachment['name'] = self._string_contents(element.find('desc')) - attachment['attacher_email'] = self._string_contents(element.find('attacher')) - attachment['type'] = self._string_contents(element.find('type')) - self._parse_attachment_flag( - element, 'review', attachment, 'reviewer_email') - self._parse_attachment_flag( - element, 'commit-queue', attachment, 'committer_email') - return attachment - - def _parse_bug_page(self, page): - soup = BeautifulSoup(page) - bug = {} - bug["id"] = int(soup.find("bug_id").string) - bug["title"] = self._string_contents(soup.find("short_desc")) - bug["bug_status"] = self._string_contents(soup.find("bug_status")) - bug["reporter_email"] = self._string_contents(soup.find("reporter")) - bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to")) - bug["cc_emails"] = [self._string_contents(element) - for element in soup.findAll('cc')] - bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')] - return bug - - # Makes testing fetch_*_from_bug() possible until we have a better - # BugzillaNetwork abstration. - - def _fetch_bug_page(self, bug_id): - bug_url = self.bug_url_for_bug_id(bug_id, xml=True) - log("Fetching: %s" % bug_url) - return self.browser.open(bug_url) - - def fetch_bug_dictionary(self, bug_id): - try: - return self._parse_bug_page(self._fetch_bug_page(bug_id)) - except KeyboardInterrupt: - raise - except: - self.authenticate() - return self._parse_bug_page(self._fetch_bug_page(bug_id)) - - # FIXME: A BugzillaCache object should provide all these fetch_ methods. - - def fetch_bug(self, bug_id): - return Bug(self.fetch_bug_dictionary(bug_id), self) - - def fetch_attachment_contents(self, attachment_id): - attachment_url = self.attachment_url_for_id(attachment_id) - # We need to authenticate to download patches from security bugs. - self.authenticate() - return self.browser.open(attachment_url).read() - - def _parse_bug_id_from_attachment_page(self, page): - # The "Up" relation happens to point to the bug. - up_link = BeautifulSoup(page).find('link', rel='Up') - if not up_link: - # This attachment does not exist (or you don't have permissions to - # view it). - return None - match = re.search("show_bug.cgi\?id=(?P\d+)", up_link['href']) - return int(match.group('bug_id')) - - def bug_id_for_attachment_id(self, attachment_id): - self.authenticate() - - attachment_url = self.attachment_url_for_id(attachment_id, 'edit') - log("Fetching: %s" % attachment_url) - page = self.browser.open(attachment_url) - return self._parse_bug_id_from_attachment_page(page) - - # FIXME: This should just return Attachment(id), which should be able to - # lazily fetch needed data. - - def fetch_attachment(self, attachment_id): - # We could grab all the attachment details off of the attachment edit - # page but we already have working code to do so off of the bugs page, - # so re-use that. - bug_id = self.bug_id_for_attachment_id(attachment_id) - if not bug_id: - return None - attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True) - for attachment in attachments: - if attachment.id() == int(attachment_id): - return attachment - return None # This should never be hit. - - def authenticate(self): - if self.authenticated: - return - - if self.dryrun: - log("Skipping log in for dry run...") - self.authenticated = True - return - - credentials = Credentials(self.bug_server_host, git_prefix="bugzilla") - - attempts = 0 - while not self.authenticated: - attempts += 1 - username, password = credentials.read_credentials() - - log("Logging in as %s..." % username) - self.browser.open(self.bug_server_url + - "index.cgi?GoAheadAndLogIn=1") - self.browser.select_form(name="login") - self.browser['Bugzilla_login'] = username - self.browser['Bugzilla_password'] = password - response = self.browser.submit() - - match = re.search("(.+?)", response.read()) - # If the resulting page has a title, and it contains the word - # "invalid" assume it's the login failure page. - if match and re.search("Invalid", match.group(1), re.IGNORECASE): - errorMessage = "Bugzilla login failed: %s" % match.group(1) - # raise an exception only if this was the last attempt - if attempts < 5: - log(errorMessage) - else: - raise Exception(errorMessage) - else: - self.authenticated = True - self.username = username - - def _fill_attachment_form(self, - description, - patch_file_object, - comment_text=None, - mark_for_review=False, - mark_for_commit_queue=False, - mark_for_landing=False, - bug_id=None): - self.browser['description'] = description - self.browser['ispatch'] = ("1",) - self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) - - if mark_for_landing: - self.browser['flag_type-3'] = ('+',) - elif mark_for_commit_queue: - self.browser['flag_type-3'] = ('?',) - else: - self.browser['flag_type-3'] = ('X',) - - if bug_id: - patch_name = "bug-%s-%s.patch" % (bug_id, timestamp()) - else: - patch_name ="%s.patch" % timestamp() - - self.browser.add_file(patch_file_object, - "text/plain", - patch_name, - 'data') - - def add_patch_to_bug(self, - bug_id, - diff, - description, - comment_text=None, - mark_for_review=False, - mark_for_commit_queue=False, - mark_for_landing=False): - self.authenticate() - - log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description, - self.bug_server_url, - bug_id)) - - if self.dryrun: - log(comment_text) - return - - self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % ( - self.bug_server_url, bug_id)) - self.browser.select_form(name="entryform") - - # _fill_attachment_form expects a file-like object - # Patch files are already binary, so no encoding needed. - assert(isinstance(diff, str)) - patch_file_object = StringIO.StringIO(diff) - self._fill_attachment_form(description, - patch_file_object, - mark_for_review=mark_for_review, - mark_for_commit_queue=mark_for_commit_queue, - mark_for_landing=mark_for_landing, - bug_id=bug_id) - if comment_text: - log(comment_text) - self.browser['comment'] = comment_text - self.browser.submit() - - def _check_create_bug_response(self, response_html): - match = re.search("Bug (?P<bug_id>\d+) Submitted", - response_html) - if match: - return match.group('bug_id') - - match = re.search( - '
(?P.+)