summaryrefslogtreecommitdiffstats
path: root/tests/DumpRenderTree/run_layout_tests.py
blob: b4eb685068b6d04e9dd05a1539dad25b4c541d31 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
#!/usr/bin/python

"""Run layout tests using Android emulator and instrumentation.

  First, you need to get an SD card or sdcard image that has layout tests on it.
  Layout tests are in following directory:
    /sdcard/android/layout_tests
  For example, /sdcard/android/layout_tests/fast

  Usage:
    Run all tests under fast/ directory:
      run_layout_tests.py, or
      run_layout_tests.py fast

    Run all tests under a sub directory:
      run_layout_tests.py fast/dom

    Run a single test:
      run_layout_tests.py fast/dom/

  After a merge, if there are changes of layout tests in SD card, you need to
  use --refresh-test-list option *once* to re-generate test list on the card.

  Some other options are:
    --time-out-ms (default is 8000 millis) for each test
    --adb-options="-e" passes option string to adb
    --results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
"""

import logging
import optparse
import os
import subprocess
import sys
import time

def CountLineNumber(filename):
  """Compute the number of lines in a given file.

  Args:
    filename: a file name related to the current directory.
  """

  fp = open(os.path.abspath(filename), "r");
  lines = 0
  for line in fp.readlines():
    lines = lines + 1
  fp.close()
  return lines

def main(options, args):
  """Run the tests. Will call sys.exit when complete.
  
  Args:
    options: a dictionary of command line options
    args: a list of sub directories or files to test
  """

  # Set up logging format.
  log_level = logging.INFO
  if options.verbose:
    log_level = logging.DEBUG
  logging.basicConfig(level=log_level,
                      format='%(message)s')

  # Include all tests if none are specified.
  if not args:
    path = 'fast';
  else:
    path = ' '.join(args);

  adb_cmd = "adb ";
  if options.adb_options:
    adb_cmd += options.adb_options

  # Re-generate the test list if --refresh-test-list is on
  if options.refresh_test_list:
    logging.info("Generating test list.");
    shell_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path fast -w com.android.dumprendertree/.LayoutTestsAutoRunner"
    adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]

    if adb_output.find('Process crashed') != -1:
       logging.info("Aborting because cannot generate test list.\n" + adb_output)
       sys.exit(1)


  logging.info("Starting tests")

  # Count crashed tests.
  crashed_tests = []

  timeout_ms = '8000'
  if options.time_out_ms:
    timeout_ms = options.time_out_ms

  # Run test until it's done

  # Call LayoutTestsAutoTest::startLayoutTests.
  shell_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests -e path \"" + path + "\" -e timeout " + timeout_ms + " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
  adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
  while adb_output.find('Process crashed') != -1:
    # Get the running_test.txt
    logging.error("DumpRenderTree crashed, output:\n" + adb_output)

    shell_cmd_str = adb_cmd + " shell cat /sdcard/running_test.txt"
    crashed_test = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE).communicate()[0]
    
    logging.info(crashed_test + " CRASHED");
    crashed_tests.append(crashed_test);

    logging.info("Resuming layout test runner...");
    # Call LayoutTestsAutoTest::resumeLayoutTests
    shell_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests -e path \"" + path + "\" -e timeout " + timeout_ms + " -w com.android.dumprendertree/.LayoutTestsAutoRunner"

    adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]

  if adb_output.find('INSTRUMENTATION_FAILED') != -1:
    logging.error("Error happened : " + adb_output)
    sys.exit(1)

  logging.info("Done");
  logging.debug(adb_output);

  # Pull results from /sdcard
  results_dir = options.results_directory
  if not os.path.exists(results_dir):
    os.makedirs(results_dir)
  if not os.path.isdir(results_dir):
    logging.error("Cannot create results dir: " + results_dir);
    sys.exit(1);

  result_files = ["/sdcard/layout_tests_passed.txt",
                  "/sdcard/layout_tests_failed.txt",
                  "/sdcard/layout_tests_nontext.txt"]
  for file in result_files: 
    shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
    adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
    logging.debug(adb_output)
    
  # Create the crash list.
  fp = open(results_dir + "/layout_tests_crashed.txt", "w");
  fp.writelines(crashed_tests)
  fp.close()

  # Count the number of tests in each category.
  passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
  logging.info(str(passed_tests) + " passed")
  failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
  logging.info(str(failed_tests) + " failed")
  crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
  logging.info(str(crashed_tests) + " crashed")
  nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
  logging.info(str(nontext_tests) + " no dumpAsText")

  logging.info("Results are stored under: " + results_dir)


if '__main__' == __name__:
  option_parser = optparse.OptionParser()
  option_parser.add_option("", "--time-out-ms",
                           default=None,
                           help="set the timeout for each test")
  option_parser.add_option("", "--verbose", action="store_true",
                           default=False,
                           help="include debug-level logging")
  option_parser.add_option("", "--refresh-test-list", action="store_true",
                           default=False,
                           help="re-generate test list, it may take some time.")
  option_parser.add_option("", "--adb-options",
                           default=None,
                           help="pass options to adb, such as -d -e, etc");
  option_parser.add_option("", "--results-directory",
                           default="layout-test-results",
                           help="directory name under which results are stored.")
  options, args = option_parser.parse_args();
  main(options, args)