1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
#!/usr/bin/python
"""Run layout tests using Android emulator and instrumentation.
First, you need to get an SD card or sdcard image that has layout tests on it.
Layout tests are in following directory:
/sdcard/android/layout_tests
For example, /sdcard/android/layout_tests/fast
Usage:
Run all tests under fast/ directory:
run_layout_tests.py, or
run_layout_tests.py fast
Run all tests under a sub directory:
run_layout_tests.py fast/dom
Run a single test:
run_layout_tests.py fast/dom/
After a merge, if there are changes of layout tests in SD card, you need to
use --refresh-test-list option *once* to re-generate test list on the card.
Some other options are:
--time-out-ms (default is 8000 millis) for each test
--adb-options="-e" passes option string to adb
--results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
"""
import logging
import optparse
import os
import subprocess
import sys
import time
def CountLineNumber(filename):
"""Compute the number of lines in a given file.
Args:
filename: a file name related to the current directory.
"""
fp = open(os.path.abspath(filename), "r");
lines = 0
for line in fp.readlines():
lines = lines + 1
fp.close()
return lines
def main(options, args):
"""Run the tests. Will call sys.exit when complete.
Args:
options: a dictionary of command line options
args: a list of sub directories or files to test
"""
# Set up logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format='%(message)s')
# Include all tests if none are specified.
if not args:
path = 'fast';
else:
path = ' '.join(args);
adb_cmd = "adb ";
if options.adb_options:
adb_cmd += options.adb_options
# Re-generate the test list if --refresh-test-list is on
if options.refresh_test_list:
logging.info("Generating test list.");
shell_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path fast -w com.android.dumprendertree/.LayoutTestsAutoRunner"
adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
if adb_output.find('Process crashed') != -1:
logging.info("Aborting because cannot generate test list.\n" + adb_output)
sys.exit(1)
logging.info("Starting tests")
# Count crashed tests.
crashed_tests = []
timeout_ms = '8000'
if options.time_out_ms:
timeout_ms = options.time_out_ms
# Run test until it's done
# Call LayoutTestsAutoTest::startLayoutTests.
shell_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests -e path \"" + path + "\" -e timeout " + timeout_ms + " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
while adb_output.find('Process crashed') != -1:
# Get the running_test.txt
logging.error("DumpRenderTree crashed, output:\n" + adb_output)
shell_cmd_str = adb_cmd + " shell cat /sdcard/running_test.txt"
crashed_test = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE).communicate()[0]
logging.info(crashed_test + " CRASHED");
crashed_tests.append(crashed_test);
logging.info("Resuming layout test runner...");
# Call LayoutTestsAutoTest::resumeLayoutTests
shell_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests -e path \"" + path + "\" -e timeout " + timeout_ms + " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
if adb_output.find('INSTRUMENTATION_FAILED') != -1:
logging.error("Error happened : " + adb_output)
sys.exit(1)
logging.info("Done");
logging.debug(adb_output);
# Pull results from /sdcard
results_dir = options.results_directory
if not os.path.exists(results_dir):
os.makedirs(results_dir)
if not os.path.isdir(results_dir):
logging.error("Cannot create results dir: " + results_dir);
sys.exit(1);
result_files = ["/sdcard/layout_tests_passed.txt",
"/sdcard/layout_tests_failed.txt",
"/sdcard/layout_tests_nontext.txt"]
for file in result_files:
shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
logging.debug(adb_output)
# Create the crash list.
fp = open(results_dir + "/layout_tests_crashed.txt", "w");
fp.writelines(crashed_tests)
fp.close()
# Count the number of tests in each category.
passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
logging.info(str(passed_tests) + " passed")
failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
logging.info(str(failed_tests) + " failed")
crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
logging.info(str(crashed_tests) + " crashed")
nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
logging.info(str(nontext_tests) + " no dumpAsText")
logging.info("Results are stored under: " + results_dir)
if '__main__' == __name__:
option_parser = optparse.OptionParser()
option_parser.add_option("", "--time-out-ms",
default=None,
help="set the timeout for each test")
option_parser.add_option("", "--verbose", action="store_true",
default=False,
help="include debug-level logging")
option_parser.add_option("", "--refresh-test-list", action="store_true",
default=False,
help="re-generate test list, it may take some time.")
option_parser.add_option("", "--adb-options",
default=None,
help="pass options to adb, such as -d -e, etc");
option_parser.add_option("", "--results-directory",
default="layout-test-results",
help="directory name under which results are stored.")
options, args = option_parser.parse_args();
main(options, args)
|
