diff options
| author | Steve Block <steveblock@google.com> | 2011-05-06 11:45:16 +0100 |
|---|---|---|
| committer | Steve Block <steveblock@google.com> | 2011-05-12 13:44:10 +0100 |
| commit | cad810f21b803229eb11403f9209855525a25d57 (patch) | |
| tree | 29a6fd0279be608e0fe9ffe9841f722f0f4e4269 /Tools/Scripts/webkitpy/common | |
| parent | 121b0cf4517156d0ac5111caf9830c51b69bae8f (diff) | |
| download | external_webkit-cad810f21b803229eb11403f9209855525a25d57.zip external_webkit-cad810f21b803229eb11403f9209855525a25d57.tar.gz external_webkit-cad810f21b803229eb11403f9209855525a25d57.tar.bz2 | |
Merge WebKit at r75315: Initial merge by git.
Change-Id: I570314b346ce101c935ed22a626b48c2af266b84
Diffstat (limited to 'Tools/Scripts/webkitpy/common')
21 files changed, 546 insertions, 72 deletions
diff --git a/Tools/Scripts/webkitpy/common/checkout/__init__.py b/Tools/Scripts/webkitpy/common/checkout/__init__.py index 597dcbd..ef65bee 100644 --- a/Tools/Scripts/webkitpy/common/checkout/__init__.py +++ b/Tools/Scripts/webkitpy/common/checkout/__init__.py @@ -1,3 +1 @@ # Required for Python to search this directory for module files - -from api import Checkout diff --git a/Tools/Scripts/webkitpy/common/checkout/api.py b/Tools/Scripts/webkitpy/common/checkout/api.py index 6357982..29e43d3 100644 --- a/Tools/Scripts/webkitpy/common/checkout/api.py +++ b/Tools/Scripts/webkitpy/common/checkout/api.py @@ -39,14 +39,14 @@ from webkitpy.common.system.executive import Executive, run_command, ScriptError from webkitpy.common.system.deprecated_logging import log -# This class represents the WebKit-specific parts of the checkout (like -# ChangeLogs). +# This class represents the WebKit-specific parts of the checkout (like ChangeLogs). # FIXME: Move a bunch of ChangeLog-specific processing from SCM to this object. +# NOTE: All paths returned from this class should be absolute. class Checkout(object): def __init__(self, scm): self._scm = scm - def _is_path_to_changelog(self, path): + def is_path_to_changelog(self, path): return os.path.basename(path) == "ChangeLog" def _latest_entry_for_changelog_at_revision(self, changelog_path, revision): @@ -59,7 +59,7 @@ class Checkout(object): def changelog_entries_for_revision(self, revision): changed_files = self._scm.changed_files_for_revision(revision) - return [self._latest_entry_for_changelog_at_revision(path, revision) for path in changed_files if self._is_path_to_changelog(path)] + return [self._latest_entry_for_changelog_at_revision(path, revision) for path in changed_files if self.is_path_to_changelog(path)] @memoized def commit_info_for_revision(self, revision): @@ -96,10 +96,10 @@ class Checkout(object): return [path for path in absolute_paths if predicate(path)] def modified_changelogs(self, git_commit, changed_files=None): - return self._modified_files_matching_predicate(git_commit, self._is_path_to_changelog, changed_files=changed_files) + return self._modified_files_matching_predicate(git_commit, self.is_path_to_changelog, changed_files=changed_files) def modified_non_changelogs(self, git_commit, changed_files=None): - return self._modified_files_matching_predicate(git_commit, lambda path: not self._is_path_to_changelog(path), changed_files=changed_files) + return self._modified_files_matching_predicate(git_commit, lambda path: not self.is_path_to_changelog(path), changed_files=changed_files) def commit_message_for_this_commit(self, git_commit, changed_files=None): changelog_paths = self.modified_changelogs(git_commit, changed_files) diff --git a/Tools/Scripts/webkitpy/common/checkout/diff_parser.py b/Tools/Scripts/webkitpy/common/checkout/diff_parser.py index a6ea756..5a5546c 100644 --- a/Tools/Scripts/webkitpy/common/checkout/diff_parser.py +++ b/Tools/Scripts/webkitpy/common/checkout/diff_parser.py @@ -118,6 +118,8 @@ class DiffFile(object): self.lines.append((deleted_line_number, new_line_number, line)) +# If this is going to be called DiffParser, it should be a re-useable parser. +# Otherwise we should rename it to ParsedDiff or just Diff. class DiffParser(object): """A parser for a patch file. @@ -125,16 +127,18 @@ class DiffParser(object): a DiffFile object. """ - # FIXME: This function is way too long and needs to be broken up. def __init__(self, diff_input): """Parses a diff. Args: diff_input: An iterable object. """ - state = _INITIAL_STATE + self.files = self._parse_into_diff_files(diff_input) - self.files = {} + # FIXME: This function is way too long and needs to be broken up. + def _parse_into_diff_files(self, diff_input): + files = {} + state = _INITIAL_STATE current_file = None old_diff_line = None new_diff_line = None @@ -148,7 +152,7 @@ class DiffParser(object): if file_declaration: filename = file_declaration.group('FilePath') current_file = DiffFile(filename) - self.files[filename] = current_file + files[filename] = current_file state = _DECLARED_FILE_PATH continue @@ -179,3 +183,4 @@ class DiffParser(object): else: _log.error('Unexpected diff format when parsing a ' 'chunk: %r' % line) + return files diff --git a/Tools/Scripts/webkitpy/common/checkout/scm.py b/Tools/Scripts/webkitpy/common/checkout/scm.py index c54fb42..3f77043 100644 --- a/Tools/Scripts/webkitpy/common/checkout/scm.py +++ b/Tools/Scripts/webkitpy/common/checkout/scm.py @@ -172,14 +172,15 @@ class SCM: return os.path.join(self.scripts_directory(), script_name) def ensure_clean_working_directory(self, force_clean): - if not force_clean and not self.working_directory_is_clean(): + if self.working_directory_is_clean(): + return + if not force_clean: # FIXME: Shouldn't this use cwd=self.checkout_root? print self.run(self.status_command(), error_handler=Executive.ignore_error) raise ScriptError(message="Working directory has modifications, pass --force-clean or --no-clean to continue.") - log("Cleaning working directory") self.clean_working_directory() - + def ensure_no_local_commits(self, force): if not self.supports_local_commits(): return diff --git a/Tools/Scripts/webkitpy/common/config/build.py b/Tools/Scripts/webkitpy/common/config/build.py index 2a432ce..2b27c85 100644 --- a/Tools/Scripts/webkitpy/common/config/build.py +++ b/Tools/Scripts/webkitpy/common/config/build.py @@ -41,8 +41,8 @@ def _should_file_trigger_build(target_platform, file): directories = [ # Directories that shouldn't trigger builds on any bots. - ("PageLoadTests", []), - ("WebCore/manual-tests", []), + ("PerformanceTests", []), + ("Source/WebCore/manual-tests", []), ("Examples", []), ("Websites", []), ("android", []), @@ -57,10 +57,10 @@ def _should_file_trigger_build(target_platform, file): ("wince", []), # Directories that should trigger builds on only some bots. - ("JavaScriptGlue", ["mac"]), + ("Source/JavaScriptGlue", ["mac"]), ("LayoutTests/platform/mac", ["mac", "win"]), ("LayoutTests/platform/mac-snowleopard", ["mac-snowleopard", "win"]), - ("WebCore/image-decoders", ["chromium"]), + ("Source/WebCore/image-decoders", ["chromium"]), ("cairo", ["gtk", "wincairo"]), ("cf", ["chromium-mac", "mac", "qt", "win"]), ("chromium", ["chromium"]), diff --git a/Tools/Scripts/webkitpy/common/config/build_unittest.py b/Tools/Scripts/webkitpy/common/config/build_unittest.py index d833464..e07d42f 100644 --- a/Tools/Scripts/webkitpy/common/config/build_unittest.py +++ b/Tools/Scripts/webkitpy/common/config/build_unittest.py @@ -27,11 +27,11 @@ from webkitpy.common.config import build class ShouldBuildTest(unittest.TestCase): _should_build_tests = [ - (["Websites/bugs.webkit.org/foo", "WebCore/bar"], ["*"]), + (["Websites/bugs.webkit.org/foo", "Source/WebCore/bar"], ["*"]), (["Websites/bugs.webkit.org/foo"], []), - (["JavaScriptCore/JavaScriptCore.xcodeproj/foo"], ["mac-leopard", "mac-snowleopard"]), - (["JavaScriptGlue/foo", "WebCore/bar"], ["*"]), - (["JavaScriptGlue/foo"], ["mac-leopard", "mac-snowleopard"]), + (["Source/JavaScriptCore/JavaScriptCore.xcodeproj/foo"], ["mac-leopard", "mac-snowleopard"]), + (["Source/JavaScriptGlue/foo", "Source/WebCore/bar"], ["*"]), + (["Source/JavaScriptGlue/foo"], ["mac-leopard", "mac-snowleopard"]), (["LayoutTests/foo"], ["*"]), (["LayoutTests/platform/chromium-linux/foo"], ["chromium-linux"]), (["LayoutTests/platform/chromium-win/fast/compact/001-expected.txt"], ["chromium-win"]), @@ -42,13 +42,13 @@ class ShouldBuildTest(unittest.TestCase): (["LayoutTests/platform/win-xp/foo"], ["win"]), (["LayoutTests/platform/win-wk2/foo"], ["win"]), (["LayoutTests/platform/win/foo"], ["win"]), - (["WebCore/mac/foo"], ["chromium-mac", "mac-leopard", "mac-snowleopard"]), - (["WebCore/win/foo"], ["chromium-win", "win"]), - (["WebCore/platform/graphics/gpu/foo"], ["mac-leopard", "mac-snowleopard"]), - (["WebCore/platform/wx/wxcode/win/foo"], []), - (["WebCore/rendering/RenderThemeMac.mm", "WebCore/rendering/RenderThemeMac.h"], ["mac-leopard", "mac-snowleopard"]), - (["WebCore/rendering/RenderThemeChromiumLinux.h"], ["chromium-linux"]), - (["WebCore/rendering/RenderThemeWinCE.h"], []), + (["Source/WebCore/mac/foo"], ["chromium-mac", "mac-leopard", "mac-snowleopard"]), + (["Source/WebCore/win/foo"], ["chromium-win", "win"]), + (["Source/WebCore/platform/graphics/gpu/foo"], ["mac-leopard", "mac-snowleopard"]), + (["Source/WebCore/platform/wx/wxcode/win/foo"], []), + (["Source/WebCore/rendering/RenderThemeMac.mm", "Source/WebCore/rendering/RenderThemeMac.h"], ["mac-leopard", "mac-snowleopard"]), + (["Source/WebCore/rendering/RenderThemeChromiumLinux.h"], ["chromium-linux"]), + (["Source/WebCore/rendering/RenderThemeWinCE.h"], []), ] def test_should_build(self): diff --git a/Tools/Scripts/webkitpy/common/config/committers.py b/Tools/Scripts/webkitpy/common/config/committers.py index 7c5bf8b..c7c741b 100644 --- a/Tools/Scripts/webkitpy/common/config/committers.py +++ b/Tools/Scripts/webkitpy/common/config/committers.py @@ -89,6 +89,7 @@ committers_unable_to_review = [ Committer("Brett Wilson", "brettw@chromium.org", "brettx"), Committer("Brian Weinstein", "bweinstein@apple.com", "bweinstein"), Committer("Cameron McCormack", "cam@webkit.org", "heycam"), + Committer("Carlos Garcia Campos", ["cgarcia@igalia.com", "carlosgc@gnome.org", "carlosgc@webkit.org"], "KaL"), Committer("Carol Szabo", "carol.szabo@nokia.com"), Committer("Chang Shu", "Chang.Shu@nokia.com"), Committer("Chris Evans", "cevans@google.com"), @@ -122,6 +123,7 @@ committers_unable_to_review = [ Committer("Jakob Petsovits", ["jpetsovits@rim.com", "jpetso@gmx.at"], "jpetso"), Committer("Jakub Wieczorek", "jwieczorek@webkit.org", "fawek"), Committer("James Hawkins", ["jhawkins@chromium.org", "jhawkins@google.com"], "jhawkins"), + Committer("James Simonsen", "simonjam@chromium.org", "simonjam"), Committer("Jay Civelli", "jcivelli@chromium.org", "jcivelli"), Committer("Jens Alfke", ["snej@chromium.org", "jens@apple.com"]), Committer("Jer Noble", "jer.noble@apple.com", "jernoble"), @@ -160,7 +162,6 @@ committers_unable_to_review = [ Committer("Michael Nordman", "michaeln@google.com", "michaeln"), Committer("Michael Saboff", "msaboff@apple.com"), Committer("Michelangelo De Simone", "michelangelo@webkit.org", "michelangelo"), - Committer("Mihai Parparita", "mihaip@chromium.org", "mihaip"), Committer("Mike Belshe", ["mbelshe@chromium.org", "mike@belshe.com"]), Committer("Mike Fenton", ["mifenton@rim.com", "mike.fenton@torchmobile.com"], "mfenton"), Committer("Mike Thole", ["mthole@mikethole.com", "mthole@apple.com"]), @@ -175,6 +176,7 @@ committers_unable_to_review = [ Committer("Philippe Normand", ["pnormand@igalia.com", "philn@webkit.org"], "philn-tp"), Committer("Pierre d'Herbemont", ["pdherbemont@free.fr", "pdherbemont@apple.com"], "pdherbemont"), Committer("Pierre-Olivier Latour", "pol@apple.com", "pol"), + Committer("Pratik Solanki", "psolanki@apple.com", "psolanki"), Committer("Renata Hodovan", "reni@webkit.org", "reni"), Committer("Robert Hogan", ["robert@webkit.org", "robert@roberthogan.net", "lists@roberthogan.net"], "mwenge"), Committer("Roland Steiner", "rolandsteiner@chromium.org"), @@ -269,6 +271,7 @@ reviewers_list = [ Reviewer("Maciej Stachowiak", "mjs@apple.com", "othermaciej"), Reviewer("Mark Rowe", "mrowe@apple.com", "bdash"), Reviewer("Martin Robinson", ["mrobinson@webkit.org", "mrobinson@igalia.com", "martin.james.robinson@gmail.com"], "mrobinson"), + Reviewer("Mihai Parparita", "mihaip@chromium.org", "mihaip"), Reviewer("Nate Chapin", "japhet@chromium.org", "japhet"), Reviewer("Nikolas Zimmermann", ["zimmermann@kde.org", "zimmermann@physik.rwth-aachen.de", "zimmermann@webkit.org"], "wildfox"), Reviewer("Ojan Vafai", "ojan@chromium.org", "ojan"), diff --git a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py index 3cb6da5..98e2fae 100644 --- a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py +++ b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py @@ -270,6 +270,7 @@ class BuildBot(object): "Leopard", "Tiger", "Windows.*Build", + "EFL", "GTK.*32", "GTK.*64.*Debug", # Disallow the 64-bit Release bot which is broken. "Qt", diff --git a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py index a10e432..ba898ec 100644 --- a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py +++ b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py @@ -30,10 +30,15 @@ import unittest from webkitpy.common.net.layouttestresults import LayoutTestResults from webkitpy.common.net.buildbot import BuildBot, Builder, Build +from webkitpy.layout_tests.layout_package import test_results +from webkitpy.layout_tests.layout_package import test_failures from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup class BuilderTest(unittest.TestCase): + def _mock_test_result(self, testname): + return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) + def _install_fetch_build(self, failure): def _mock_fetch_build(build_number): build = Build( @@ -42,8 +47,8 @@ class BuilderTest(unittest.TestCase): revision=build_number + 1000, is_green=build_number < 4 ) - parsed_results = {LayoutTestResults.fail_key: failure(build_number)} - build._layout_test_results = LayoutTestResults(parsed_results) + results = [self._mock_test_result(testname) for testname in failure(build_number)] + build._layout_test_results = LayoutTestResults(results) return build self.builder._fetch_build = _mock_fetch_build @@ -254,6 +259,7 @@ class BuildBotTest(unittest.TestCase): "Leopard", "Tiger", "Windows.*Build", + "EFL", "GTK.*32", "GTK.*64.*Debug", # Disallow the 64-bit Release bot which is broken. "Qt", diff --git a/Tools/Scripts/webkitpy/common/net/layouttestresults.py b/Tools/Scripts/webkitpy/common/net/layouttestresults.py index 15e95ce..28caad4 100644 --- a/Tools/Scripts/webkitpy/common/net/layouttestresults.py +++ b/Tools/Scripts/webkitpy/common/net/layouttestresults.py @@ -27,8 +27,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # A module for parsing results.html files generated by old-run-webkit-tests +# This class is one big hack and only needs to exist until we transition to new-run-webkit-tests. +from webkitpy.common.system.deprecated_logging import log from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, SoupStrainer +from webkitpy.layout_tests.layout_package import test_results +from webkitpy.layout_tests.layout_package import test_failures # FIXME: This should be unified with all the layout test results code in the layout_tests package @@ -57,36 +61,89 @@ class LayoutTestResults(object): ] @classmethod + def _failures_from_fail_row(self, row): + # Look at all anchors in this row, and guess what type + # of new-run-webkit-test failures they equate to. + failures = set() + test_name = None + for anchor in row.findAll("a"): + anchor_text = unicode(anchor.string) + if not test_name: + test_name = anchor_text + continue + if anchor_text in ["expected image", "image diffs"] or '%' in anchor_text: + failures.add(test_failures.FailureImageHashMismatch()) + elif anchor_text in ["expected", "actual", "diff", "pretty diff"]: + failures.add(test_failures.FailureTextMismatch()) + else: + log("Unhandled link text in results.html parsing: %s. Please file a bug against webkitpy." % anchor_text) + # FIXME: Its possible the row contained no links due to ORWT brokeness. + # We should probably assume some type of failure anyway. + return failures + + @classmethod + def _failures_from_row(cls, row, table_title): + if table_title == cls.fail_key: + return cls._failures_from_fail_row(row) + if table_title == cls.crash_key: + return [test_failures.FailureCrash()] + if table_title == cls.timeout_key: + return [test_failures.FailureTimeout()] + if table_title == cls.missing_key: + return [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()] + return None + + @classmethod + def _test_result_from_row(cls, row, table_title): + test_name = unicode(row.find("a").string) + failures = cls._failures_from_row(row, table_title) + # TestResult is a class designed to work with new-run-webkit-tests. + # old-run-webkit-tests does not save quite enough information in results.html for us to parse. + # FIXME: It's unclear if test_name should include LayoutTests or not. + return test_results.TestResult(test_name, failures) + + @classmethod + def _parse_results_table(cls, table): + table_title = unicode(table.findPreviousSibling("p").string) + if table_title not in cls.expected_keys: + # This Exception should only ever be hit if run-webkit-tests changes its results.html format. + raise Exception("Unhandled title: %s" % table_title) + # Ignore stderr failures. Everyone ignores them anyway. + if table_title == cls.stderr_key: + return [] + # FIXME: We might end with two TestResults object for the same test if it appears in more than one row. + return [cls._test_result_from_row(row, table_title) for row in table.findAll("tr")] + + @classmethod def _parse_results_html(cls, page): - if not page: - return None - parsed_results = {} tables = BeautifulSoup(page).findAll("table") - for table in tables: - table_title = unicode(table.findPreviousSibling("p").string) - if table_title not in cls.expected_keys: - # This Exception should only ever be hit if run-webkit-tests changes its results.html format. - raise Exception("Unhandled title: %s" % table_title) - # We might want to translate table titles into identifiers before storing. - parsed_results[table_title] = [unicode(row.find("a").string) for row in table.findAll("tr")] - - return parsed_results + return sum([cls._parse_results_table(table) for table in tables], []) @classmethod def results_from_string(cls, string): - parsed_results = cls._parse_results_html(string) - if not parsed_results: + if not string: return None - return cls(parsed_results) + test_results = cls._parse_results_html(string) + if not test_results: + return None + return cls(test_results) + + def __init__(self, test_results): + self._test_results = test_results + + def test_results(self): + return self._test_results - def __init__(self, parsed_results): - self._parsed_results = parsed_results + def results_matching_failure_types(self, failure_types): + return [result for result in self._test_results if result.has_failure_matching_types(failure_types)] - def parsed_results(self): - return self._parsed_results + def tests_matching_failure_types(self, failure_types): + return [result.filename for result in self.results_matching_failure_types(failure_types)] - def results_matching_keys(self, result_keys): - return sorted(sum([tests for key, tests in self._parsed_results.items() if key in result_keys], [])) + def failing_test_results(self): + # These should match the "fail", "crash", and "timeout" keys. + failure_types = [test_failures.FailureTextMismatch, test_failures.FailureImageHashMismatch, test_failures.FailureCrash, test_failures.FailureTimeout] + return self.results_matching_failure_types(failure_types) def failing_tests(self): - return self.results_matching_keys([self.fail_key, self.crash_key, self.timeout_key]) + return [result.filename for result in self.failing_test_results()] diff --git a/Tools/Scripts/webkitpy/common/net/layouttestresults_unittest.py b/Tools/Scripts/webkitpy/common/net/layouttestresults_unittest.py index 8490eae..01b91b8 100644 --- a/Tools/Scripts/webkitpy/common/net/layouttestresults_unittest.py +++ b/Tools/Scripts/webkitpy/common/net/layouttestresults_unittest.py @@ -29,6 +29,10 @@ import unittest from webkitpy.common.net.layouttestresults import LayoutTestResults +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.layout_tests.layout_package import test_results +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup class LayoutTestResultsTest(unittest.TestCase): @@ -57,21 +61,28 @@ class LayoutTestResultsTest(unittest.TestCase): </html> """ - _expected_layout_test_results = { - 'Tests that had stderr output:': [ - 'accessibility/aria-activedescendant-crash.html', - ], - 'Tests that had no expected results (probably new):': [ - 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html', - ], - } - def test_parse_layout_test_results(self): + failures = [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()] + testname = 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html' + expected_results = [test_results.TestResult(testname, failures)] + results = LayoutTestResults._parse_results_html(self._example_results_html) - self.assertEqual(self._expected_layout_test_results, results) + self.assertEqual(expected_results, results) def test_results_from_string(self): self.assertEqual(LayoutTestResults.results_from_string(None), None) self.assertEqual(LayoutTestResults.results_from_string(""), None) results = LayoutTestResults.results_from_string(self._example_results_html) self.assertEqual(len(results.failing_tests()), 0) + + def test_failures_from_fail_row(self): + row = BeautifulSoup("<tr><td><a>test.hml</a></td><td><a>expected image</a></td><td><a>25%</a></td></tr>") + test_name = unicode(row.find("a").string) + # Even if the caller has already found the test name, findAll inside _failures_from_fail_row will see it again. + failures = OutputCapture().assert_outputs(self, LayoutTestResults._failures_from_fail_row, [row]) + self.assertEqual(len(failures), 1) + self.assertEqual(type(sorted(failures)[0]), test_failures.FailureImageHashMismatch) + + row = BeautifulSoup("<tr><td><a>test.hml</a><a>foo</a></td></tr>") + expected_stderr = "Unhandled link text in results.html parsing: foo. Please file a bug against webkitpy.\n" + OutputCapture().assert_outputs(self, LayoutTestResults._failures_from_fail_row, [row], expected_stderr=expected_stderr) diff --git a/Tools/Scripts/webkitpy/common/net/networktransaction.py b/Tools/Scripts/webkitpy/common/net/networktransaction.py index de19e94..21cc71f 100644 --- a/Tools/Scripts/webkitpy/common/net/networktransaction.py +++ b/Tools/Scripts/webkitpy/common/net/networktransaction.py @@ -58,8 +58,7 @@ class NetworkTransaction(object): if self._convert_404_to_None and e.code == 404: return None self._check_for_timeout() - _log.warn("Received HTTP status %s from server. Retrying in " - "%s seconds..." % (e.code, self._backoff_seconds)) + _log.warn("Received HTTP status %s loading \"%s\". Retrying in %s seconds..." % (e.code, e.filename, self._backoff_seconds)) self._sleep() def _check_for_timeout(self): diff --git a/Tools/Scripts/webkitpy/common/net/networktransaction_unittest.py b/Tools/Scripts/webkitpy/common/net/networktransaction_unittest.py index 49aaeed..c4cd4e0 100644 --- a/Tools/Scripts/webkitpy/common/net/networktransaction_unittest.py +++ b/Tools/Scripts/webkitpy/common/net/networktransaction_unittest.py @@ -70,9 +70,9 @@ class NetworkTransactionTest(LoggingTestCase): transaction = NetworkTransaction(initial_backoff_seconds=0) self.assertEqual(transaction.run(lambda: self._raise_500_error()), 42) self.assertEqual(self._run_count, 3) - self.assertLog(['WARNING: Received HTTP status 500 from server. ' + self.assertLog(['WARNING: Received HTTP status 500 loading "http://example.com/". ' 'Retrying in 0 seconds...\n', - 'WARNING: Received HTTP status 500 from server. ' + 'WARNING: Received HTTP status 500 loading "http://example.com/". ' 'Retrying in 0.0 seconds...\n']) def test_convert_404_to_None(self): diff --git a/Tools/Scripts/webkitpy/common/net/statusserver.py b/Tools/Scripts/webkitpy/common/net/statusserver.py index 64dd77b..abd298a 100644 --- a/Tools/Scripts/webkitpy/common/net/statusserver.py +++ b/Tools/Scripts/webkitpy/common/net/statusserver.py @@ -69,6 +69,13 @@ class StatusServer: return self._browser.add_file(results_file, "text/plain", "results.txt", 'results_file') + # 500 is the AppEngine limit for TEXT fields (which most of our fields are). + # Exceeding the limit will result in a 500 error from the server. + def _set_field(self, field_name, value, limit=500): + if len(value) > limit: + _log.warn("Attempted to set %s to value exceeding %s characters, truncating." % (field_name, limit)) + self._browser[field_name] = value[:limit] + def _post_status_to_server(self, queue_name, status, patch, results_file): if results_file: # We might need to re-wind the file if we've already tried to post it. @@ -81,7 +88,7 @@ class StatusServer: if self.bot_id: self._browser["bot_id"] = self.bot_id self._add_patch(patch) - self._browser["status"] = status + self._set_field("status", status, limit=500) self._add_results_file(results_file) return self._browser.submit().read() # This is the id of the newly created status object. diff --git a/Tools/Scripts/webkitpy/common/system/directoryfileset.py b/Tools/Scripts/webkitpy/common/system/directoryfileset.py new file mode 100644 index 0000000..11acaf4 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/directoryfileset.py @@ -0,0 +1,77 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import with_statement + +import os +import shutil + +from webkitpy.common.system.fileset import FileSetFileHandle +from webkitpy.common.system.filesystem import FileSystem +import webkitpy.common.system.ospath as ospath + + +class DirectoryFileSet(object): + """The set of files under a local directory.""" + def __init__(self, path, filesystem=None): + self._path = path + self._filesystem = filesystem or FileSystem() + if not self._path.endswith(os.path.sep): + self._path += os.path.sep + + def _full_path(self, filename): + assert self._is_under(self._path, filename) + return self._filesystem.join(self._path, filename) + + def _drop_directory_prefix(self, path): + return path[len(self._path):] + + def _files_in_directory(self): + """Returns a list of all the files in the directory, including the path + to the directory""" + return self._filesystem.files_under(self._path) + + def _is_under(self, dir, filename): + return bool(ospath.relpath(self._filesystem.join(dir, filename), dir)) + + def open(self, filename): + return FileSetFileHandle(self, filename, self._filesystem) + + def namelist(self): + return map(self._drop_directory_prefix, self._files_in_directory()) + + def read(self, filename): + return self._filesystem.read_text_file(self._full_path(filename)) + + def extract(self, filename, path): + """Extracts a file from this file set to the specified directory.""" + src = self._full_path(filename) + dest = self._filesystem.join(path, filename) + # As filename may have slashes in it, we must ensure that the same + # directory hierarchy exists at the output path. + self._filesystem.maybe_make_directory(os.path.split(dest)[0]) + self._filesystem.copyfile(src, dest) + + def delete(self, filename): + filename = self._full_path(filename) + self._filesystem.remove(filename) diff --git a/Tools/Scripts/webkitpy/common/system/directoryfileset_unittest.py b/Tools/Scripts/webkitpy/common/system/directoryfileset_unittest.py new file mode 100644 index 0000000..a3850ee --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/directoryfileset_unittest.py @@ -0,0 +1,70 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import with_statement + +import unittest + +from webkitpy.common.system.directoryfileset import DirectoryFileSet +from webkitpy.common.system.filesystem_mock import MockFileSystem + + +class DirectoryFileSetTest(unittest.TestCase): + def setUp(self): + files = {} + files['/test/some-file'] = 'contents' + files['/test/some-other-file'] = 'other contents' + files['/test/b/c'] = 'c' + self._filesystem = MockFileSystem(files) + self._fileset = DirectoryFileSet('/test', self._filesystem) + + def test_files_in_namelist(self): + self.assertTrue('some-file' in self._fileset.namelist()) + self.assertTrue('some-other-file' in self._fileset.namelist()) + self.assertTrue('b/c' in self._fileset.namelist()) + + def test_read(self): + self.assertEquals('contents', self._fileset.read('some-file')) + + def test_open(self): + file = self._fileset.open('some-file') + self.assertEquals('some-file', file.name()) + self.assertEquals('contents', file.contents()) + + def test_extract(self): + self._fileset.extract('some-file', '/test-directory') + contents = self._filesystem.read_text_file('/test-directory/some-file') + self.assertEquals('contents', contents) + + def test_extract_deep_file(self): + self._fileset.extract('b/c', '/test-directory') + self.assertTrue(self._filesystem.exists('/test-directory/b/c')) + + def test_delete(self): + self.assertTrue(self._filesystem.exists('/test/some-file')) + self._fileset.delete('some-file') + self.assertFalse(self._filesystem.exists('/test/some-file')) + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/common/system/fileset.py b/Tools/Scripts/webkitpy/common/system/fileset.py new file mode 100644 index 0000000..22f7c4d --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/fileset.py @@ -0,0 +1,64 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import with_statement +import os + +from webkitpy.common.system.filesystem import FileSystem + + +class FileSetFileHandle(object): + """Points to a file that resides in a file set""" + def __init__(self, fileset, filename, filesystem=None): + self._filename = filename + self._fileset = fileset + self._contents = None + self._filesystem = filesystem or FileSystem() + + def __str__(self): + return "%s:%s" % (self._fileset, self._filename) + + def contents(self): + if self._contents is None: + self._contents = self._fileset.read(self._filename) + return self._contents + + def save_to(self, path, filename=None): + if filename is None: + self._fileset.extract(self._filename, path) + return + with self._filesystem.mkdtemp() as temp_dir: + self._fileset.extract(self._filename, temp_dir) + + src = self._filesystem.join(temp_dir, self._filename) + dest = self._filesystem.join(path, filename) + self._filesystem.copyfile(src, dest) + + def delete(self): + self._fileset.delete(self._filename) + + def name(self): + return self._filename + + def splitext(self): + return os.path.splitext(self.name()) diff --git a/Tools/Scripts/webkitpy/common/system/filesystem.py b/Tools/Scripts/webkitpy/common/system/filesystem.py index f0b5e44..8830ea8 100644 --- a/Tools/Scripts/webkitpy/common/system/filesystem.py +++ b/Tools/Scripts/webkitpy/common/system/filesystem.py @@ -93,7 +93,7 @@ class FileSystem(object): def maybe_make_directory(self, *path): """Create the specified directory if it doesn't already exist.""" try: - os.makedirs(os.path.join(*path)) + os.makedirs(self.join(*path)) except OSError, e: if e.errno != errno.EEXIST: raise @@ -152,3 +152,9 @@ class FileSystem(object): """Copies the contents of the file at the given path to the destination path.""" shutil.copyfile(source, destination) + + def files_under(self, path): + """Return the list of all files under the given path.""" + return [self.join(path_to_file, filename) + for (path_to_file, _, filenames) in os.walk(path) + for filename in filenames] diff --git a/Tools/Scripts/webkitpy/common/system/filesystem_mock.py b/Tools/Scripts/webkitpy/common/system/filesystem_mock.py index ea0f3f9..c605cb2 100644 --- a/Tools/Scripts/webkitpy/common/system/filesystem_mock.py +++ b/Tools/Scripts/webkitpy/common/system/filesystem_mock.py @@ -29,6 +29,7 @@ import errno import os import path +import re class MockFileSystem(object): @@ -57,7 +58,7 @@ class MockFileSystem(object): return any(f.startswith(path) for f in self.files) def join(self, *comps): - return '/'.join(comps) + return re.sub(re.escape(os.path.sep), '/', os.path.join(*comps)) def listdir(self, path): if not self.isdir(path): @@ -107,3 +108,11 @@ class MockFileSystem(object): raise IOError(errno.EISDIR, destination, os.strerror(errno.ISDIR)) self.files[destination] = self.files[source] + + def files_under(self, path): + if not path.endswith('/'): + path += '/' + return [file for file in self.files if file.startswith(path)] + + def remove(self, path): + del self.files[path] diff --git a/Tools/Scripts/webkitpy/common/system/zipfileset.py b/Tools/Scripts/webkitpy/common/system/zipfileset.py new file mode 100644 index 0000000..fa2b762 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/zipfileset.py @@ -0,0 +1,65 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import urllib +import zipfile + +from webkitpy.common.net.networktransaction import NetworkTransaction +from webkitpy.common.system.fileset import FileSetFileHandle +from webkitpy.common.system.filesystem import FileSystem + + +class ZipFileSet(object): + """The set of files in a zip file that resides at a URL (local or remote)""" + def __init__(self, zip_url, filesystem=None, zip_factory=None): + self._zip_url = zip_url + self._zip_file = None + self._filesystem = filesystem or FileSystem() + self._zip_factory = zip_factory or self._retrieve_zip_file + + def _retrieve_zip_file(self, zip_url): + temp_file = NetworkTransaction().run(lambda: urllib.urlretrieve(zip_url)[0]) + return zipfile.ZipFile(temp_file) + + def _load(self): + if self._zip_file is None: + self._zip_file = self._zip_factory(self._zip_url) + + def open(self, filename): + self._load() + return FileSetFileHandle(self, filename, self._filesystem) + + def namelist(self): + self._load() + return self._zip_file.namelist() + + def read(self, filename): + self._load() + return self._zip_file.read(filename) + + def extract(self, filename, path): + self._load() + self._zip_file.extract(filename, path) + + def delete(self, filename): + raise Exception("Can't delete from a ZipFileSet.") diff --git a/Tools/Scripts/webkitpy/common/system/zipfileset_unittest.py b/Tools/Scripts/webkitpy/common/system/zipfileset_unittest.py new file mode 100644 index 0000000..a9ba5ad --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/zipfileset_unittest.py @@ -0,0 +1,95 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import shutil +import tempfile +import unittest +import zipfile + +from webkitpy.common.system.filesystem_mock import MockFileSystem +from webkitpy.common.system.zipfileset import ZipFileSet + + +class FakeZip(object): + def __init__(self, filesystem): + self._filesystem = filesystem + self._files = {} + + def add_file(self, filename, contents): + self._files[filename] = contents + + def open(self, filename): + return FileSetFileHandle(self, filename, self._filesystem) + + def namelist(self): + return self._files.keys() + + def read(self, filename): + return self._files[filename] + + def extract(self, filename, path): + self._filesystem.write_text_file(self._filesystem.join(path, filename), self.read(filename)) + + def delete(self, filename): + raise Exception("Can't delete from a ZipFileSet.") + + +class ZipFileSetTest(unittest.TestCase): + def setUp(self): + self._filesystem = MockFileSystem() + self._zip = ZipFileSet('blah', self._filesystem, self.make_fake_zip) + + def make_fake_zip(self, zip_url): + result = FakeZip(self._filesystem) + result.add_file('some-file', 'contents') + result.add_file('a/b/some-other-file', 'other contents') + return result + + def test_open(self): + file = self._zip.open('a/b/some-other-file') + self.assertEquals('a/b/some-other-file', file.name()) + self.assertEquals('other contents', file.contents()) + + def test_read(self): + self.assertEquals('contents', self._zip.read('some-file')) + + def test_extract(self): + self._filesystem.maybe_make_directory('/some-dir') + self._zip.extract('some-file', '/some-dir') + self.assertTrue(self._filesystem.isfile('/some-dir/some-file')) + + def test_deep_extract(self): + self._filesystem.maybe_make_directory('/some-dir') + self._zip.extract('a/b/some-other-file', '/some-dir') + self.assertTrue(self._filesystem.isfile('/some-dir/a/b/some-other-file')) + + def test_cant_delete(self): + self.assertRaises(Exception, self._zip.delete, 'some-file') + + def test_namelist(self): + self.assertTrue('some-file' in self._zip.namelist()) + + +if __name__ == '__main__': + unittest.main() |
