diff options
Diffstat (limited to 'tests/DumpRenderTree/run_layout_tests.py')
| -rwxr-xr-x | tests/DumpRenderTree/run_layout_tests.py | 274 |
1 files changed, 0 insertions, 274 deletions
diff --git a/tests/DumpRenderTree/run_layout_tests.py b/tests/DumpRenderTree/run_layout_tests.py deleted file mode 100755 index 433271e..0000000 --- a/tests/DumpRenderTree/run_layout_tests.py +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/python - -"""Run layout tests using Android emulator and instrumentation. - - First, you need to get an SD card or sdcard image that has layout tests on it. - Layout tests are in following directory: - /sdcard/android/layout_tests - For example, /sdcard/android/layout_tests/fast - - Usage: - Run all tests under fast/ directory: - run_layout_tests.py, or - run_layout_tests.py fast - - Run all tests under a sub directory: - run_layout_tests.py fast/dom - - Run a single test: - run_layout_tests.py fast/dom/ - - After a merge, if there are changes of layout tests in SD card, you need to - use --refresh-test-list option *once* to re-generate test list on the card. - - Some other options are: - --time-out-ms (default is 8000 millis) for each test - --adb-options="-e" passes option string to adb - --results-directory=..., (default is ./layout-test-results) directory name under which results are stored. -""" - -import logging -import optparse -import os -import subprocess -import sys -import time - -def CountLineNumber(filename): - """Compute the number of lines in a given file. - - Args: - filename: a file name related to the current directory. - """ - - fp = open(os.path.abspath(filename), "r"); - lines = 0 - for line in fp.readlines(): - lines = lines + 1 - fp.close() - return lines - -def DumpRenderTreeFinished(adb_cmd): - """ Check if DumpRenderTree finished running tests - - Args: - output: adb_cmd string - """ - - # pull /sdcard/running_test.txt, if the content is "#DONE", it's done - shell_cmd_str = adb_cmd + " shell cat /sdcard/running_test.txt" - adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] - return adb_output.strip() == "#DONE" - -def DiffResults(marker, new_results, old_results, diff_results, strip_reason): - """ Given two result files, generate diff and - write to diff_results file. All arguments are absolute paths - to files. - """ - old_file = open(old_results, "r") - new_file = open(new_results, "r") - diff_file = open(diff_results, "a") - - # Read lines from each file - ndict = new_file.readlines() - cdict = old_file.readlines() - - # Write marker to diff file - diff_file.writelines(marker + "\n") - diff_file.writelines("###############\n") - - # Strip reason from result lines - if strip_reason is True: - for i in range(0, len(ndict)): - ndict[i] = ndict[i].split(' ')[0] + "\n" - for i in range(0, len(cdict)): - cdict[i] = cdict[i].split(' ')[0] + "\n" - - # Find results in new_results missing in old_results - new_count=0 - for line in ndict: - if line not in cdict: - diff_file.writelines("+ " + line) - new_count += 1 - - # Find results in old_results missing in new_results - missing_count=0 - for line in cdict: - if line not in ndict: - diff_file.writelines("- " + line) - missing_count += 1 - - logging.info(marker + " >>> " + str(new_count) + " new, " + str(missing_count) + " misses") - - diff_file.writelines("\n\n") - - old_file.close() - new_file.close() - diff_file.close() - return - -def CompareResults(ref_dir, results_dir): - """Compare results in two directories - - Args: - ref_dir: the reference directory having layout results as references - results_dir: the results directory - """ - logging.info("Comparing results to " + ref_dir) - - diff_result = os.path.join(results_dir, "layout_tests_diff.txt") - if os.path.exists(diff_result): - os.remove(diff_result) - - files=["passed", "failed", "nontext", "crashed"] - for f in files: - result_file_name = "layout_tests_" + f + ".txt" - DiffResults(f, os.path.join(results_dir, result_file_name), - os.path.join(ref_dir, result_file_name), diff_result, - f == "failed") - logging.info("Detailed diffs are in " + diff_result) - -def main(options, args): - """Run the tests. Will call sys.exit when complete. - - Args: - options: a dictionary of command line options - args: a list of sub directories or files to test - """ - - # Set up logging format. - log_level = logging.INFO - if options.verbose: - log_level = logging.DEBUG - logging.basicConfig(level=log_level, - format='%(message)s') - - # Include all tests if none are specified. - if not args: - path = 'fast'; - else: - path = ' '.join(args); - - adb_cmd = "adb "; - if options.adb_options: - adb_cmd += options.adb_options - - # Re-generate the test list if --refresh-test-list is on - if options.refresh_test_list: - logging.info("Generating test list."); - shell_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path fast -w com.android.dumprendertree/.LayoutTestsAutoRunner" - adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] - - if adb_output.find('Process crashed') != -1: - logging.info("Aborting because cannot generate test list.\n" + adb_output) - sys.exit(1) - - - logging.info("Running tests") - - # Count crashed tests. - crashed_tests = [] - - timeout_ms = '8000' - if options.time_out_ms: - timeout_ms = options.time_out_ms - - # Run test until it's done - - # Call LayoutTestsAutoTest::startLayoutTests. - shell_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests -e path \"" + path + "\" -e timeout " + timeout_ms + " -w com.android.dumprendertree/.LayoutTestsAutoRunner" - adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] - while not DumpRenderTreeFinished(adb_cmd): - # Get the running_test.txt - logging.error("DumpRenderTree crashed, output:\n" + adb_output) - - shell_cmd_str = adb_cmd + " shell cat /sdcard/running_test.txt" - crashed_test = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE).communicate()[0] - - logging.info(crashed_test + " CRASHED"); - crashed_tests.append(crashed_test); - - logging.info("Resuming layout test runner..."); - # Call LayoutTestsAutoTest::resumeLayoutTests - shell_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests -e path \"" + path + "\" -e timeout " + timeout_ms + " -w com.android.dumprendertree/.LayoutTestsAutoRunner" - - adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] - - if adb_output.find('INSTRUMENTATION_FAILED') != -1: - logging.error("Error happened : " + adb_output) - sys.exit(1) - - logging.debug(adb_output); - logging.info("Done\n"); - - # Pull results from /sdcard - results_dir = options.results_directory - if not os.path.exists(results_dir): - os.makedirs(results_dir) - if not os.path.isdir(results_dir): - logging.error("Cannot create results dir: " + results_dir); - sys.exit(1); - - result_files = ["/sdcard/layout_tests_passed.txt", - "/sdcard/layout_tests_failed.txt", - "/sdcard/layout_tests_nontext.txt"] - for file in result_files: - shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir - adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] - logging.debug(adb_output) - - # Create the crash list. - fp = open(results_dir + "/layout_tests_crashed.txt", "w"); - fp.writelines(crashed_tests) - fp.close() - - # Count the number of tests in each category. - passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt") - logging.info(str(passed_tests) + " passed") - failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt") - logging.info(str(failed_tests) + " failed") - crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt") - logging.info(str(crashed_tests) + " crashed") - nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt") - logging.info(str(nontext_tests) + " no dumpAsText") - - logging.info("Results are stored under: " + results_dir + "\n") - - # Comparing results to references to find new fixes and regressions. - results_dir = os.path.abspath(options.results_directory) - ref_dir = options.ref_directory - - # if ref_dir is null, cannonify ref_dir to the script dir. - if not ref_dir: - script_self = sys.argv[0] - script_dir = os.path.dirname(script_self) - ref_dir = os.path.join(script_dir, "results") - - ref_dir = os.path.abspath(ref_dir) - - CompareResults(ref_dir, results_dir) - -if '__main__' == __name__: - option_parser = optparse.OptionParser() - option_parser.add_option("", "--time-out-ms", - default=None, - help="set the timeout for each test") - option_parser.add_option("", "--verbose", action="store_true", - default=False, - help="include debug-level logging") - option_parser.add_option("", "--refresh-test-list", action="store_true", - default=False, - help="re-generate test list, it may take some time.") - option_parser.add_option("", "--adb-options", - default=None, - help="pass options to adb, such as -d -e, etc"); - option_parser.add_option("", "--results-directory", - default="layout-test-results", - help="directory which results are stored.") - option_parser.add_option("", "--ref-directory", - default=None, - dest="ref_directory", - help="directory where reference results are stored.") - - options, args = option_parser.parse_args(); - main(options, args) |