diff options
Diffstat (limited to 'tests/DumpRenderTree/assets/run_layout_tests.py')
| -rwxr-xr-x | tests/DumpRenderTree/assets/run_layout_tests.py | 286 |
1 files changed, 286 insertions, 0 deletions
diff --git a/tests/DumpRenderTree/assets/run_layout_tests.py b/tests/DumpRenderTree/assets/run_layout_tests.py new file mode 100755 index 0000000..5409a0c --- /dev/null +++ b/tests/DumpRenderTree/assets/run_layout_tests.py @@ -0,0 +1,286 @@ +#!/usr/bin/python + +"""Run layout tests using Android emulator and instrumentation. + + First, you need to get an SD card or sdcard image that has layout tests on it. + Layout tests are in following directory: + /sdcard/android/layout_tests + For example, /sdcard/android/layout_tests/fast + + Usage: + Run all tests under fast/ directory: + run_layout_tests.py, or + run_layout_tests.py fast + + Run all tests under a sub directory: + run_layout_tests.py fast/dom + + Run a single test: + run_layout_tests.py fast/dom/ + + After a merge, if there are changes of layout tests in SD card, you need to + use --refresh-test-list option *once* to re-generate test list on the card. + + Some other options are: + --rebaseline generates expected layout tests results under /sdcard/android/expected_result/ + --time-out-ms (default is 8000 millis) for each test + --adb-options="-e" passes option string to adb + --results-directory=..., (default is ./layout-test-results) directory name under which results are stored. +""" + +import logging +import optparse +import os +import subprocess +import sys +import time + +def CountLineNumber(filename): + """Compute the number of lines in a given file. + + Args: + filename: a file name related to the current directory. + """ + + fp = open(os.path.abspath(filename), "r"); + lines = 0 + for line in fp.readlines(): + lines = lines + 1 + fp.close() + return lines + +def DumpRenderTreeFinished(adb_cmd): + """ Check if DumpRenderTree finished running tests + + Args: + output: adb_cmd string + """ + + # pull /sdcard/android/running_test.txt, if the content is "#DONE", it's done + shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt" + adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] + return adb_output.strip() == "#DONE" + +def DiffResults(marker, new_results, old_results, diff_results, strip_reason): + """ Given two result files, generate diff and + write to diff_results file. All arguments are absolute paths + to files. + """ + old_file = open(old_results, "r") + new_file = open(new_results, "r") + diff_file = open(diff_results, "a") + + # Read lines from each file + ndict = new_file.readlines() + cdict = old_file.readlines() + + # Write marker to diff file + diff_file.writelines(marker + "\n") + diff_file.writelines("###############\n") + + # Strip reason from result lines + if strip_reason is True: + for i in range(0, len(ndict)): + ndict[i] = ndict[i].split(' ')[0] + "\n" + for i in range(0, len(cdict)): + cdict[i] = cdict[i].split(' ')[0] + "\n" + + # Find results in new_results missing in old_results + new_count=0 + for line in ndict: + if line not in cdict: + diff_file.writelines("+ " + line) + new_count += 1 + + # Find results in old_results missing in new_results + missing_count=0 + for line in cdict: + if line not in ndict: + diff_file.writelines("- " + line) + missing_count += 1 + + logging.info(marker + " >>> " + str(new_count) + " new, " + str(missing_count) + " misses") + + diff_file.writelines("\n\n") + + old_file.close() + new_file.close() + diff_file.close() + return + +def CompareResults(ref_dir, results_dir): + """Compare results in two directories + + Args: + ref_dir: the reference directory having layout results as references + results_dir: the results directory + """ + logging.info("Comparing results to " + ref_dir) + + diff_result = os.path.join(results_dir, "layout_tests_diff.txt") + if os.path.exists(diff_result): + os.remove(diff_result) + + files=["passed", "failed", "nontext", "crashed"] + for f in files: + result_file_name = "layout_tests_" + f + ".txt" + DiffResults(f, os.path.join(results_dir, result_file_name), + os.path.join(ref_dir, result_file_name), diff_result, + f == "failed") + logging.info("Detailed diffs are in " + diff_result) + +def main(options, args): + """Run the tests. Will call sys.exit when complete. + + Args: + options: a dictionary of command line options + args: a list of sub directories or files to test + """ + + # Set up logging format. + log_level = logging.INFO + if options.verbose: + log_level = logging.DEBUG + logging.basicConfig(level=log_level, + format='%(message)s') + + # Include all tests if none are specified. + if not args: + path = '/'; + else: + path = ' '.join(args); + + adb_cmd = "adb "; + if options.adb_options: + adb_cmd += options.adb_options + + # Re-generate the test list if --refresh-test-list is on + if options.refresh_test_list: + logging.info("Generating test list."); + generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner" + adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] + + if adb_output.find('Process crashed') != -1: + logging.info("Aborting because cannot generate test list.\n" + adb_output) + sys.exit(1) + + + logging.info("Running tests") + + # Count crashed tests. + crashed_tests = [] + + timeout_ms = '5000' + if options.time_out_ms: + timeout_ms = options.time_out_ms + + # Run test until it's done + + run_layout_test_cmd_prefix = adb_cmd + " shell am instrument" + + run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms + if options.rebaseline: + run_layout_test_cmd_postfix += " -e rebaseline true" + run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner" + + # Call LayoutTestsAutoTest::startLayoutTests. + run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix + + adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] + while not DumpRenderTreeFinished(adb_cmd): + # Get the running_test.txt + logging.error("DumpRenderTree crashed, output:\n" + adb_output) + + shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt" + crashed_test = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE).communicate()[0] + + logging.info(crashed_test + " CRASHED"); + crashed_tests.append(crashed_test); + + logging.info("Resuming layout test runner..."); + # Call LayoutTestsAutoTest::resumeLayoutTests + run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix + + adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] + + if adb_output.find('INSTRUMENTATION_FAILED') != -1: + logging.error("Error happened : " + adb_output) + sys.exit(1) + + logging.debug(adb_output); + logging.info("Done\n"); + + # Pull results from /sdcard + results_dir = options.results_directory + if not os.path.exists(results_dir): + os.makedirs(results_dir) + if not os.path.isdir(results_dir): + logging.error("Cannot create results dir: " + results_dir); + sys.exit(1); + + result_files = ["/sdcard/layout_tests_passed.txt", + "/sdcard/layout_tests_failed.txt", + "/sdcard/layout_tests_nontext.txt"] + for file in result_files: + shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir + adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] + logging.debug(adb_output) + + # Create the crash list. + fp = open(results_dir + "/layout_tests_crashed.txt", "w"); + fp.writelines('\n'.join(crashed_tests)) + fp.close() + + # Count the number of tests in each category. + passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt") + logging.info(str(passed_tests) + " passed") + failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt") + logging.info(str(failed_tests) + " failed") + crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt") + logging.info(str(crashed_tests) + " crashed") + nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt") + logging.info(str(nontext_tests) + " no dumpAsText") + + logging.info("Results are stored under: " + results_dir + "\n") + + # Comparing results to references to find new fixes and regressions. + results_dir = os.path.abspath(options.results_directory) + ref_dir = options.ref_directory + + # if ref_dir is null, cannonify ref_dir to the script dir. + if not ref_dir: + script_self = sys.argv[0] + script_dir = os.path.dirname(script_self) + ref_dir = os.path.join(script_dir, "results") + + ref_dir = os.path.abspath(ref_dir) + + CompareResults(ref_dir, results_dir) + +if '__main__' == __name__: + option_parser = optparse.OptionParser() + option_parser.add_option("", "--rebaseline", action="store_true", + default=False, + help="generate expected results for those tests not having one") + option_parser.add_option("", "--time-out-ms", + default=None, + help="set the timeout for each test") + option_parser.add_option("", "--verbose", action="store_true", + default=False, + help="include debug-level logging") + option_parser.add_option("", "--refresh-test-list", action="store_true", + default=False, + help="re-generate test list, it may take some time.") + option_parser.add_option("", "--adb-options", + default=None, + help="pass options to adb, such as -d -e, etc"); + option_parser.add_option("", "--results-directory", + default="layout-test-results", + help="directory which results are stored.") + option_parser.add_option("", "--ref-directory", + default=None, + dest="ref_directory", + help="directory where reference results are stored.") + + options, args = option_parser.parse_args(); + main(options, args) |