added reliability test usding DumpRenderTree

1 files changed, 214 insertions, 0 deletions

diff --git a/tests/DumpRenderTree/assets/run_reliability_tests.py b/tests/DumpRenderTree/assets/run_reliability_tests.py
new file mode 100755
index 0000000..a242293
--- /dev/null
+++ b/tests/DumpRenderTree/assets/run_reliability_tests.py
@@ -0,0 +1,214 @@
+#!/usr/bin/python2.4
+
+"""Run reliability tests using Android instrumentation.
+
+  A test file consists of list web sites to test is needed as a parameter
+
+  Usage:
+    run_reliability_tests.py path/to/url/list
+"""
+
+import logging
+import optparse
+import random
+import subprocess
+import sys
+import time
+
+TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
+TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
+TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
+HTTP_URL_FILE = "urllist_http"
+HTTPS_URL_FILE = "urllist_https"
+NUM_URLS = 25
+
+
+def DumpRenderTreeFinished(adb_cmd):
+  """Check if DumpRenderTree finished running.
+
+  Args:
+    adb_cmd: adb command string
+
+  Returns:
+    True if DumpRenderTree has finished, False otherwise
+  """
+
+  # pull test status file and look for "#DONE"
+  shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
+  adb_output = subprocess.Popen(shell_cmd_str,
+                                shell=True, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE).communicate()[0]
+  return adb_output.strip() == "#DONE"
+
+
+def RandomPick(file_name, approx_size, num_needed):
+  """Randomly pick lines from the text file specifed.
+
+  Args:
+    file_name: the text file where lines should be picked from
+    approx_size: an approximate size of the text file
+    num_needed: how many lines are needed from the file
+
+  Returns:
+    an array of string
+  """
+  p = float(num_needed) / approx_size
+  num_picked = 0
+  lines = []
+  random.seed()
+
+  while num_picked < num_needed:
+    file_handle = open(file_name, "r")
+    for line in file_handle:
+      line = line.strip()
+      if float(random.randint(0, approx_size)) / approx_size < p:
+        lines.append(line)
+        num_picked += 1
+        if num_picked == num_needed:
+          break
+    file_handle.close()
+  return lines
+
+
+def main(options, args):
+  """Send the url list to device and start testing, restart if crashed."""
+
+  generate_url = False
+
+  # Set up logging format.
+  log_level = logging.INFO
+  if options.verbose:
+    log_level = logging.DEBUG
+  logging.basicConfig(level=log_level,
+                      format="%(message)s")
+
+  # Include all tests if none are specified.
+  if not args:
+    path = "/tmp/url_list_%d.txt" % time.time()
+    generate_url = True
+    logging.info("A URL list is not provided, will be automatically generated.")
+  else:
+    path = args[0]
+
+  if not options.crash_file:
+    print "missing crash file name, use --crash-file to specify"
+    sys.exit(1)
+  else:
+    crashed_file = options.crash_file
+
+  if not options.timeout_file:
+    print "missing timeout file, use --timeout-file to specify"
+    sys.exit(1)
+  else:
+    timedout_file = options.timeout_file
+
+  http = RandomPick(HTTP_URL_FILE, 500000, NUM_URLS)
+  https = RandomPick(HTTPS_URL_FILE, 45000, NUM_URLS)
+
+  if generate_url:
+    file_handle = open(path, "w")
+    for i in range(0, NUM_URLS):
+      file_handle.write(http[i] + "\n")
+      file_handle.write(https[i] + "\n")
+    file_handle.close()
+
+  adb_cmd = "adb "
+  if options.adb_options:
+    adb_cmd += options.adb_options + " "
+
+  # push url list to device
+  test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
+  proc = subprocess.Popen(test_cmd, shell=True,
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+  (adb_output, adb_error) = proc.communicate()
+  if proc.returncode != 0:
+    logging.error("failed to push url list to device.")
+    logging.error(adb_output)
+    logging.error(adb_error)
+    sys.exit(1)
+
+  logging.info("Running the test ...")
+
+  # Count crashed tests.
+  crashed_tests = []
+
+  if options.time_out_ms:
+    timeout_ms = options.time_out_ms
+
+  # Run test until it's done
+  test_cmd_prefix = adb_cmd + " shell am instrument"
+  test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
+
+  # Call ReliabilityTestsAutoTest#startReliabilityTests
+  test_cmd = (test_cmd_prefix + " -e class "
+              "com.android.dumprendertree.ReliabilityTestsAutoTest#"
+              "startReliabilityTests -e timeout " + timeout_ms
+              + test_cmd_postfix)
+
+  time_start = time.time()
+  adb_output = subprocess.Popen(test_cmd, shell=True,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE).communicate()[0]
+  while not DumpRenderTreeFinished(adb_cmd):
+    logging.error("DumpRenderTree exited before all URLs are visited.")
+    shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
+    crashed_test = subprocess.Popen(shell_cmd_str, shell=True,
+                                    stdout=subprocess.PIPE).communicate()[0]
+    logging.info(crashed_test + " CRASHED")
+    crashed_tests.append(crashed_test)
+    logging.info("Resuming reliability test runner...")
+
+    test_cmd = (test_cmd_prefix + " -e class "
+                "com.android.dumprendertree.ReliabilityTestsAutoTest#"
+                "resumeReliabilityTests -e timeout " + timeout_ms
+                + test_cmd_postfix)
+    adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE).communicate()[0]
+
+  time_end = time.time()
+  fp = open("time_stat", "a")
+  fp.writelines("%.2f\n" % ((time_end - time_start) / NUM_URLS / 2))
+  fp.close()
+  if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
+      adb_output.find("Process crashed.") != -1):
+    logging.error("Error happened : " + adb_output)
+    sys.exit(1)
+
+  logging.info(adb_output)
+  logging.info("Done\n")
+
+  if crashed_tests:
+    file_handle = open(crashed_file, "w")
+    file_handle.writelines("\n".join(crashed_tests))
+    logging.info("Crashed URL list stored in: " + crashed_file)
+    file_handle.close()
+  else:
+    logging.info("No crash found.")
+
+  test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
+              + timedout_file +  "\"")
+
+  subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE).communicate()
+
+
+if "__main__" == __name__:
+  option_parser = optparse.OptionParser()
+  option_parser.add_option("", "--time-out-ms",
+                           default=60000,
+                           help="set the timeout for each test")
+  option_parser.add_option("", "--verbose", action="store_true",
+                           default=False,
+                           help="include debug-level logging")
+  option_parser.add_option("", "--adb-options",
+                           default=None,
+                           help="pass options to adb, such as -d -e, etc")
+  option_parser.add_option("", "--crash-file",
+                           default="reliability_crashed_sites.txt",
+                           help="the list of sites that cause browser to crash")
+  option_parser.add_option("", "--timeout-file",
+                           default="reliability_timedout_sites.txt",
+                           help="the list of sites that timedout during test.")
+  opts, arguments = option_parser.parse_args()
+  main(opts, arguments)