diff options
Diffstat (limited to 'WebKitTools/Scripts/webkitpy')
15 files changed, 193 insertions, 54 deletions
diff --git a/WebKitTools/Scripts/webkitpy/common/config/committers.py b/WebKitTools/Scripts/webkitpy/common/config/committers.py index 25b1725..5ebf18a 100644 --- a/WebKitTools/Scripts/webkitpy/common/config/committers.py +++ b/WebKitTools/Scripts/webkitpy/common/config/committers.py @@ -123,6 +123,7 @@ committers_unable_to_review = [ Committer("Jessie Berlin", ["jberlin@webkit.org", "jberlin@apple.com"]), Committer("Jesus Sanchez-Palencia", ["jesus@webkit.org", "jesus.palencia@openbossa.org"], "jeez_"), Committer("Jocelyn Turcotte", "jocelyn.turcotte@nokia.com", "jturcotte"), + Committer("Jochen Eisinger", "jochen@chromium.org", "jochen__"), Committer("John Abd-El-Malek", "jam@chromium.org", "jam"), Committer("John Gregg", ["johnnyg@google.com", "johnnyg@chromium.org"], "johnnyg"), Committer("Joost de Valk", ["joost@webkit.org", "webkit-dev@joostdevalk.nl"], "Altha"), diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py index c0525ea..6c36c93 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py @@ -56,7 +56,8 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase def __init__(self, port, builder_name, build_name, build_number, results_file_base_path, builder_base_url, - test_timings, expectations, result_summary, all_tests): + test_timings, expectations, result_summary, all_tests, + generate_incremental_results=False): """Modifies the results.json file. Grabs it off the archive directory if it is not found locally. @@ -66,7 +67,8 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase """ super(JSONLayoutResultsGenerator, self).__init__( builder_name, build_name, build_number, results_file_base_path, - builder_base_url, {}, port.test_repository_paths()) + builder_base_url, {}, port.test_repository_paths(), + generate_incremental_results) self._port = port self._expectations = expectations diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py index 595fc2b..e746bc0 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py @@ -60,7 +60,7 @@ class TestResult(object): class JSONResultsGeneratorBase(object): """A JSON results generator for generic tests.""" - MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 + MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 1500 # Min time (seconds) that will be added to the JSON. MIN_TIME = 1 JSON_PREFIX = "ADD_RESULTS(" @@ -82,10 +82,12 @@ class JSONResultsGeneratorBase(object): ALL_FIXABLE_COUNT = "allFixableCount" RESULTS_FILENAME = "results.json" + INCREMENTAL_RESULTS_FILENAME = "incremental_results.json" def __init__(self, builder_name, build_name, build_number, results_file_base_path, builder_base_url, - test_results_map, svn_repositories=None): + test_results_map, svn_repositories=None, + generate_incremental_results=False): """Modifies the results.json file. Grabs it off the archive directory if it is not found locally. @@ -108,41 +110,59 @@ class JSONResultsGeneratorBase(object): self._builder_base_url = builder_base_url self._results_file_path = os.path.join(results_file_base_path, self.RESULTS_FILENAME) + self._incremental_results_file_path = os.path.join( + results_file_base_path, self.INCREMENTAL_RESULTS_FILENAME) self._test_results_map = test_results_map self._test_results = test_results_map.values() + self._generate_incremental_results = generate_incremental_results self._svn_repositories = svn_repositories if not self._svn_repositories: self._svn_repositories = {} self._json = None + self._archived_results = None def generate_json_output(self): """Generates the JSON output file.""" + + # Generate the JSON output file that has full results. + # FIXME: stop writing out the full results file once all bots use + # incremental results. if not self._json: self._json = self.get_json() if self._json: - # Specify separators in order to get compact encoding. - json_data = simplejson.dumps(self._json, separators=(',', ':')) - json_string = self.JSON_PREFIX + json_data + self.JSON_SUFFIX + self._generate_json_file(self._json, self._results_file_path) - results_file = codecs.open(self._results_file_path, "w", "utf-8") - results_file.write(json_string) - results_file.close() + # Generate the JSON output file that only has incremental results. + if self._generate_incremental_results: + json = self.get_json(incremental=True) + if json: + self._generate_json_file( + json, self._incremental_results_file_path) - def get_json(self): + def get_json(self, incremental=False): """Gets the results for the results.json file.""" - if self._json: - return self._json - - results_json, error = self._get_archived_json_results() - if error: - # If there was an error don't write a results.json - # file at all as it would lose all the information on the bot. - _log.error("Archive directory is inaccessible. Not modifying " - "or clobbering the results.json file: " + str(error)) - return None + if incremental: + results_json = {} + else: + if self._json: + return self._json + + if not self._archived_results: + self._archived_results, error = \ + self._get_archived_json_results() + if error: + # If there was an error don't write a results.json + # file at all as it would lose all the information on the + # bot. + _log.error("Archive directory is inaccessible. Not " + "modifying or clobbering the results.json " + "file: " + str(error)) + return None + + results_json = self._archived_results builder_name = self._builder_name if results_json and builder_name not in results_json: @@ -168,8 +188,19 @@ class JSONResultsGeneratorBase(object): for test in all_failing_tests: self._insert_test_time_and_result(test, tests) - self._json = results_json - return self._json + return results_json + + def set_archived_results(self, archived_results): + self._archived_results = archived_results + + def _generate_json_file(self, json, file_path): + # Specify separators in order to get compact encoding. + json_data = simplejson.dumps(json, separators=(',', ':')) + json_string = self.JSON_PREFIX + json_data + self.JSON_SUFFIX + + results_file = codecs.open(file_path, "w", "utf-8") + results_file.write(json_string) + results_file.close() def _get_test_timing(self, test_name): """Returns test timing data (elapsed time) in second diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py index 0a60cc7..785cc1c 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py @@ -45,6 +45,13 @@ class JSONGeneratorTest(unittest.TestCase): self.builder_name = 'DUMMY_BUILDER_NAME' self.build_name = 'DUMMY_BUILD_NAME' self.build_number = 'DUMMY_BUILDER_NUMBER' + self._json = None + self._num_runs = 0 + self._tests_list = set([]) + self._test_timings = {} + self._failed_tests = {} + self._passed_tests = set([]) + self._skipped_tests = set([]) def _test_json_generation(self, passed_tests, failed_tests, skipped_tests): # Make sure we have sets (rather than lists). @@ -52,12 +59,13 @@ class JSONGeneratorTest(unittest.TestCase): skipped_tests = set(skipped_tests) tests_list = passed_tests | set(failed_tests.keys()) test_timings = {} + i = 0 for test in tests_list: - test_timings[test] = float(random.randint(1, 10)) + test_timings[test] = float(self._num_runs * 100 + i) + i += 1 port_obj = port.get(None) - # Generate a JSON file. generator = json_results_generator.JSONResultsGenerator(port_obj, self.builder_name, self.build_name, self.build_number, '', @@ -68,8 +76,39 @@ class JSONGeneratorTest(unittest.TestCase): skipped_tests, tests_list) - json = generator.get_json() - + # Test incremental json results + incremental_json = generator.get_json(incremental=True) + self._verify_json_results( + tests_list, + test_timings, + passed_tests, + failed_tests, + skipped_tests, + incremental_json, + 1) + + # Test aggregated json results + generator.set_archived_results(self._json) + json = generator.get_json(incremental=False) + self._json = json + self._num_runs += 1 + self._tests_list |= tests_list + self._test_timings.update(test_timings) + self._failed_tests.update(failed_tests) + self._passed_tests |= passed_tests + self._skipped_tests |= skipped_tests + self._verify_json_results( + self._tests_list, + self._test_timings, + self._passed_tests, + self._failed_tests, + self._skipped_tests, + self._json, + self._num_runs) + + def _verify_json_results(self, tests_list, test_timings, + passed_tests, failed_tests, + skipped_tests, json, num_runs): # Aliasing to a short name for better access to its constants. JRG = json_results_generator.JSONResultsGenerator @@ -79,11 +118,18 @@ class JSONGeneratorTest(unittest.TestCase): buildinfo = json[self.builder_name] self.assertTrue(JRG.FIXABLE in buildinfo) self.assertTrue(JRG.TESTS in buildinfo) - self.assertTrue(len(buildinfo[JRG.BUILD_NUMBERS]) == 1) + self.assertTrue(len(buildinfo[JRG.BUILD_NUMBERS]) == num_runs) self.assertTrue(buildinfo[JRG.BUILD_NUMBERS][0] == self.build_number) - if tests_list or skipped_tests: - fixable = buildinfo[JRG.FIXABLE][0] + if tests_list or skipped_tests: + fixable = {} + for fixable_items in buildinfo[JRG.FIXABLE]: + for (type, count) in fixable_items.iteritems(): + if type in fixable: + fixable[type] = fixable[type] + count + else: + fixable[type] = count + if passed_tests: self.assertTrue(fixable[JRG.PASS_RESULT] == len(passed_tests)) else: @@ -100,26 +146,35 @@ class JSONGeneratorTest(unittest.TestCase): for test_name, failure in failed_tests.iteritems(): self.assertTrue(test_name in tests) test = tests[test_name] - self.assertTrue(test[JRG.RESULTS][0][0] == 1) - self.assertTrue(test[JRG.RESULTS][0][1] == JRG.FAIL_RESULT) - self.assertTrue(test[JRG.TIMES][0][0] == 1) - self.assertTrue(test[JRG.TIMES][0][1] == - int(test_timings[test_name])) + + failed = 0 + for result in test[JRG.RESULTS]: + if result[1] == JRG.FAIL_RESULT: + failed = result[0] + self.assertTrue(failed == 1) + + timing_count = 0 + for timings in test[JRG.TIMES]: + if timings[1] == test_timings[test_name]: + timing_count = timings[0] + self.assertTrue(timing_count == 1) fixable_count = len(skipped_tests) + len(failed_tests.keys()) if skipped_tests or failed_tests: - self.assertTrue(buildinfo[JRG.FIXABLE_COUNT][0] == fixable_count) + self.assertTrue(sum(buildinfo[JRG.FIXABLE_COUNT]) == fixable_count) def test_json_generation(self): reason = test_expectations.TEXT self._test_json_generation([], {}, []) - self._test_json_generation(['A', 'B'], {}, []) - self._test_json_generation([], {'A': reason, 'B': reason}, []) - self._test_json_generation([], {}, ['A', 'B']) - self._test_json_generation(['A'], {'B': reason, 'C': reason}, []) - self._test_json_generation([], {'A': reason, 'B': reason}, ['C', 'D']) - self._test_json_generation(['A', 'B', 'C'], {'D': reason}, ['E', 'F']) + self._test_json_generation(['A1', 'B1'], {}, []) + self._test_json_generation([], {'A2': reason, 'B2': reason}, []) + self._test_json_generation([], {}, ['A3', 'B3']) + self._test_json_generation(['A4'], {'B4': reason, 'C4': reason}, []) + self._test_json_generation( + [], {'A5': reason, 'B5': reason}, ['C5', 'D5']) + self._test_json_generation( + ['A6', 'B6', 'C6'], {'D6': reason}, ['E6', 'F6']) if __name__ == '__main__': diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py index 81bf39e..7346671 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py @@ -60,7 +60,7 @@ _DEFAULT_WSS_PORT = 9323 def url_is_alive(url): """Checks to see if we get an http response from |url|. - We poll the url 5 times with a 1 second delay. If we don't + We poll the url 20 times with a 0.5 second delay. If we don't get a reply in that time, we give up and assume the httpd didn't start properly. @@ -70,7 +70,7 @@ def url_is_alive(url): True if the url is alive. """ sleep_time = 0.5 - wait_time = 5 + wait_time = 10 while wait_time > 0: try: response = urllib.urlopen(url) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index 41aab62..490ac3c 100755 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -889,7 +889,8 @@ class TestRunner: self._port, self._options.builder_name, self._options.build_name, self._options.build_number, self._options.results_directory, BUILDER_BASE_URL, individual_test_timings, - self._expectations, result_summary, self._test_files_list) + self._expectations, result_summary, self._test_files_list, + not self._options.upload_full_results) _log.debug("Finished writing JSON files.") @@ -900,8 +901,12 @@ class TestRunner: _log.info("Uploading JSON files for builder: %s", self._options.builder_name) - attrs = [('builder', self._options.builder_name)] - json_files = ["expectations.json", "results.json"] + attrs = [("builder", self._options.builder_name)] + json_files = ["expectations.json"] + if self._options.upload_full_results: + json_files.append("results.json") + else: + json_files.append("incremental_results.json") files = [(file, os.path.join(self._options.results_directory, file)) for file in json_files] @@ -1656,6 +1661,10 @@ def parse_args(args=None): optparse.make_option("--test-results-server", default="", help=("If specified, upload results json files to this appengine " "server.")), + optparse.make_option("--upload-full-results", + action="store_true", + default=False, + help="If true, upload full json results to server."), ] option_list = (configuration_options + print_options + diff --git a/WebKitTools/Scripts/webkitpy/style/checkers/cpp.py b/WebKitTools/Scripts/webkitpy/style/checkers/cpp.py index 9e4240c..62f40ea 100644 --- a/WebKitTools/Scripts/webkitpy/style/checkers/cpp.py +++ b/WebKitTools/Scripts/webkitpy/style/checkers/cpp.py @@ -2451,6 +2451,9 @@ def check_identifier_name_in_declaration(filename, line_number, line, error): line = sub(r'(unsigned|signed) (?=char|short|int|long)', '', line) line = sub(r'\b(inline|using|static|const|volatile|auto|register|extern|typedef|restrict|struct|class|virtual)(?=\W)', '', line) + # Remove "new" and "new (expr)" to simplify, too. + line = sub(r'new\s*(\([^)]*\))?', '', line) + # Remove all template parameters by removing matching < and >. # Loop until no templates are removed to remove nested templates. while True: diff --git a/WebKitTools/Scripts/webkitpy/style/checkers/cpp_unittest.py b/WebKitTools/Scripts/webkitpy/style/checkers/cpp_unittest.py index c927db6..16b1a3c 100644 --- a/WebKitTools/Scripts/webkitpy/style/checkers/cpp_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/checkers/cpp_unittest.py @@ -3715,6 +3715,12 @@ class WebKitStyleTest(CppStyleTestBase): self.assert_lint('unsigned _fillRule : 1;', '_fillRule' + name_underscore_error_message) + # new operators in initialization. + self.assert_lint('OwnPtr<uint32_t> variable(new uint32_t);', '') + self.assert_lint('OwnPtr<uint32_t> variable(new (expr) uint32_t);', '') + self.assert_lint('OwnPtr<uint32_t> under_score(new uint32_t);', + 'under_score' + name_underscore_error_message) + def test_comments(self): # A comment at the beginning of a line is ok. diff --git a/WebKitTools/Scripts/webkitpy/style/filereader.py b/WebKitTools/Scripts/webkitpy/style/filereader.py index 48455b3..1a24cb5 100644 --- a/WebKitTools/Scripts/webkitpy/style/filereader.py +++ b/WebKitTools/Scripts/webkitpy/style/filereader.py @@ -47,6 +47,10 @@ class TextFileReader(object): file_count: The total number of files passed to this instance for processing, including non-text files and files that should be skipped. + delete_only_file_count: The total number of files that are not + processed this instance actually because + the files don't have any modified lines + but should be treated as processed. """ @@ -59,6 +63,7 @@ class TextFileReader(object): """ self._processor = processor self.file_count = 0 + self.delete_only_file_count = 0 def _read_lines(self, file_path): """Read the file at a path, and return its lines. @@ -146,3 +151,12 @@ class TextFileReader(object): self._process_directory(directory=path) else: self.process_file(path) + + def count_delete_only_file(self): + """Count up files that contains only deleted lines. + + Files which has no modified or newly-added lines don't need + to check style, but should be treated as checked. For that + purpose, we just count up the number of such files. + """ + self.delete_only_file_count += 1 diff --git a/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py b/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py index 558ec5a..6328337 100644 --- a/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py @@ -159,3 +159,8 @@ class TextFileReaderTest(LoggingTestCase): processed = [(['bar'], file_path2, None), (['foo'], file_path1, None)] self._assert_file_reader(processed, 2) + + def test_count_delete_only_file(self): + self._file_reader.count_delete_only_file() + delete_only_file_count = self._file_reader.delete_only_file_count + self.assertEquals(delete_only_file_count, 1) diff --git a/WebKitTools/Scripts/webkitpy/style/patchreader.py b/WebKitTools/Scripts/webkitpy/style/patchreader.py index 7ba2b66..576504a 100644 --- a/WebKitTools/Scripts/webkitpy/style/patchreader.py +++ b/WebKitTools/Scripts/webkitpy/style/patchreader.py @@ -73,3 +73,8 @@ class PatchReader(object): if line_numbers: self._text_file_reader.process_file(file_path=path, line_numbers=line_numbers) + else: + # We don't check the file which contains deleted lines only + # but would like to treat it as to be processed so that + # we count up number of such files. + self._text_file_reader.count_delete_only_file() diff --git a/WebKitTools/Scripts/webkitpy/style/patchreader_unittest.py b/WebKitTools/Scripts/webkitpy/style/patchreader_unittest.py index 10791e4..2453c6b 100644 --- a/WebKitTools/Scripts/webkitpy/style/patchreader_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/patchreader_unittest.py @@ -45,10 +45,15 @@ class PatchReaderTest(unittest.TestCase): def __init__(self): self.passed_to_process_file = [] """A list of (file_path, line_numbers) pairs.""" + self.delete_only_file_count = 0 + """A number of times count_delete_only_file() called""" def process_file(self, file_path, line_numbers): self.passed_to_process_file.append((file_path, line_numbers)) + def count_delete_only_file(self): + self.delete_only_file_count += 1 + def setUp(self): file_reader = self.MockTextFileReader() self._file_reader = file_reader @@ -57,9 +62,11 @@ class PatchReaderTest(unittest.TestCase): def _call_check_patch(self, patch_string): self._patch_checker.check(patch_string) - def _assert_checked(self, passed_to_process_file): + def _assert_checked(self, passed_to_process_file, delete_only_file_count): self.assertEquals(self._file_reader.passed_to_process_file, passed_to_process_file) + self.assertEquals(self._file_reader.delete_only_file_count, + delete_only_file_count) def test_check_patch(self): # The modified line_numbers array for this patch is: [2]. @@ -71,7 +78,7 @@ index ef65bee..e3db70e 100644 # Required for Python to search this directory for module files +# New line """) - self._assert_checked([("__init__.py", set([2]))]) + self._assert_checked([("__init__.py", set([2]))], 0) def test_check_patch_with_deletion(self): self._call_check_patch("""Index: __init__.py @@ -82,4 +89,4 @@ index ef65bee..e3db70e 100644 -foobar """) # _mock_check_file should not be called for the deletion patch. - self._assert_checked([]) + self._assert_checked([], 1) diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py b/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py index 704970d..3a7aff3 100644 --- a/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py +++ b/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py @@ -79,7 +79,7 @@ installer.install(url="http://www.adambarth.com/webkit/eliza", rietveld_dir = os.path.join(autoinstalled_dir, "rietveld") installer = AutoInstaller(target_dir=rietveld_dir) -installer.install(url="http://webkit-rietveld.googlecode.com/svn/trunk/static/upload.py", +installer.install(url="http://webkit-rietveld.googlecode.com/svn/trunk/upload.py", target_name="upload.py") diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/queueengine.py b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine.py index a1a66a1..36cbc5f 100644 --- a/WebKitTools/Scripts/webkitpy/tool/bot/queueengine.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine.py @@ -107,7 +107,8 @@ class QueueEngine: self._open_work_log(work_item) try: if not self._delegate.process_work_item(work_item): - self._sleep("Unable to process work item.") + log("Unable to process work item.") + continue except ScriptError, e: # Use a special exit code to indicate that the error was already # handled in the child process and we should just keep looping. diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem.py b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem.py index 9fbfda6..750bbfd 100644 --- a/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem.py @@ -152,8 +152,8 @@ class AbstractCommitterOnlyEWS(AbstractEarlyWarningSystem): def process_work_item(self, patch): if not self._committers.committer_by_email(patch.attacher_email()): self._did_error(patch, "%s cannot process patches from non-committers :(" % self.name) - return - AbstractEarlyWarningSystem.process_work_item(self, patch) + return False + return AbstractEarlyWarningSystem.process_work_item(self, patch) class MacEWS(AbstractCommitterOnlyEWS): |