#!/usr/bin/env python
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Rebaselining tool that automatically produces baselines for all platforms.
The script does the following for each platform specified:
1. Compile a list of tests that need rebaselining.
2. Download test result archive from buildbot for the platform.
3. Extract baselines from the archive file for all identified files.
4. Add new baselines to SVN repository.
5. For each test that has been rebaselined, remove this platform option from
the test in test_expectation.txt. If no other platforms remain after
removal, delete the rebaselined test from the file.
At the end, the script generates a html that compares old and new baselines.
"""
import logging
import optparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
import urllib
import webbrowser
import zipfile
from port import path_utils
from layout_package import test_expectations
from test_types import image_diff
from test_types import text_diff
# Repository type constants.
REPO_SVN, REPO_UNKNOWN = range(2)
BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux']
ARCHIVE_DIR_NAME_DICT = {'win': 'webkit-rel',
'win-vista': 'webkit-dbg-vista',
'win-xp': 'webkit-rel',
'mac': 'webkit-rel-mac5',
'linux': 'webkit-rel-linux',
'win-canary': 'webkit-rel-webkit-org',
'win-vista-canary': 'webkit-dbg-vista',
'win-xp-canary': 'webkit-rel-webkit-org',
'mac-canary': 'webkit-rel-mac-webkit-org',
'linux-canary': 'webkit-rel-linux-webkit-org'}
def run_shell_with_return_code(command, print_output=False):
"""Executes a command and returns the output and process return code.
Args:
command: program and arguments.
print_output: if true, print the command results to standard output.
Returns:
command output, return code
"""
# Use a shell for subcommands on Windows to get a PATH search.
use_shell = sys.platform.startswith('win')
p = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, shell=use_shell)
if print_output:
output_array = []
while True:
line = p.stdout.readline()
if not line:
break
if print_output:
print line.strip('\n')
output_array.append(line)
output = ''.join(output_array)
else:
output = p.stdout.read()
p.wait()
p.stdout.close()
return output, p.returncode
def run_shell(command, print_output=False):
"""Executes a command and returns the output.
Args:
command: program and arguments.
print_output: if true, print the command results to standard output.
Returns:
command output
"""
output, return_code = run_shell_with_return_code(command, print_output)
return output
def log_dashed_string(text, platform, logging_level=logging.INFO):
"""Log text message with dashes on both sides."""
msg = text
if platform:
msg += ': ' + platform
if len(msg) < 78:
dashes = '-' * ((78 - len(msg)) / 2)
msg = '%s %s %s' % (dashes, msg, dashes)
if logging_level == logging.ERROR:
logging.error(msg)
elif logging_level == logging.WARNING:
logging.warn(msg)
else:
logging.info(msg)
def setup_html_directory(html_directory):
"""Setup the directory to store html results.
All html related files are stored in the "rebaseline_html" subdirectory.
Args:
html_directory: parent directory that stores the rebaselining results.
If None, a temp directory is created.
Returns:
the directory that stores the html related rebaselining results.
"""
if not html_directory:
html_directory = tempfile.mkdtemp()
elif not os.path.exists(html_directory):
os.mkdir(html_directory)
html_directory = os.path.join(html_directory, 'rebaseline_html')
logging.info('Html directory: "%s"', html_directory)
if os.path.exists(html_directory):
shutil.rmtree(html_directory, True)
logging.info('Deleted file at html directory: "%s"', html_directory)
if not os.path.exists(html_directory):
os.mkdir(html_directory)
return html_directory
def get_result_file_fullpath(html_directory, baseline_filename, platform,
result_type):
"""Get full path of the baseline result file.
Args:
html_directory: directory that stores the html related files.
baseline_filename: name of the baseline file.
platform: win, linux or mac
result_type: type of the baseline result: '.txt', '.png'.
Returns:
Full path of the baseline file for rebaselining result comparison.
"""
base, ext = os.path.splitext(baseline_filename)
result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext)
fullpath = os.path.join(html_directory, result_filename)
logging.debug(' Result file full path: "%s".', fullpath)
return fullpath
class Rebaseliner(object):
"""Class to produce new baselines for a given platform."""
REVISION_REGEX = r''
def __init__(self, platform, options):
self._file_dir = path_utils.path_from_base('webkit', 'tools',
'layout_tests')
self._platform = platform
self._options = options
self._rebaselining_tests = []
self._rebaselined_tests = []
# Create tests and expectations helper which is used to:
# -. compile list of tests that need rebaselining.
# -. update the tests in test_expectations file after rebaseline
# is done.
self._test_expectations = \
test_expectations.TestExpectations(None,
self._file_dir,
platform,
False,
False)
self._repo_type = self._get_repo_type()
def run(self, backup):
"""Run rebaseline process."""
log_dashed_string('Compiling rebaselining tests', self._platform)
if not self._compile_rebaselining_tests():
return True
log_dashed_string('Downloading archive', self._platform)
archive_file = self._download_buildbot_archive()
logging.info('')
if not archive_file:
logging.error('No archive found.')
return False
log_dashed_string('Extracting and adding new baselines',
self._platform)
if not self._extract_and_add_new_baselines(archive_file):
return False
log_dashed_string('Updating rebaselined tests in file',
self._platform)
self._update_rebaselined_tests_in_file(backup)
logging.info('')
if len(self._rebaselining_tests) != len(self._rebaselined_tests):
logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '
'REBASELINED.')
logging.warning(' Total tests needing rebaselining: %d',
len(self._rebaselining_tests))
logging.warning(' Total tests rebaselined: %d',
len(self._rebaselined_tests))
return False
logging.warning('All tests needing rebaselining were successfully '
'rebaselined.')
return True
def get_rebaselining_tests(self):
return self._rebaselining_tests
def _get_repo_type(self):
"""Get the repository type that client is using."""
output, return_code = run_shell_with_return_code(['svn', 'info'],
False)
if return_code == 0:
return REPO_SVN
return REPO_UNKNOWN
def _compile_rebaselining_tests(self):
"""Compile list of tests that need rebaselining for the platform.
Returns:
List of tests that need rebaselining or
None if there is no such test.
"""
self._rebaselining_tests = \
self._test_expectations.get_rebaselining_failures()
if not self._rebaselining_tests:
logging.warn('No tests found that need rebaselining.')
return None
logging.info('Total number of tests needing rebaselining '
'for "%s": "%d"', self._platform,
len(self._rebaselining_tests))
test_no = 1
for test in self._rebaselining_tests:
logging.info(' %d: %s', test_no, test)
test_no += 1
return self._rebaselining_tests
def _get_latest_revision(self, url):
"""Get the latest layout test revision number from buildbot.
Args:
url: Url to retrieve layout test revision numbers.
Returns:
latest revision or
None on failure.
"""
logging.debug('Url to retrieve revision: "%s"', url)
f = urllib.urlopen(url)
content = f.read()
f.close()
revisions = re.findall(self.REVISION_REGEX, content)
if not revisions:
logging.error('Failed to find revision, content: "%s"', content)
return None
revisions.sort(key=int)
logging.info('Latest revision: "%s"', revisions[len(revisions) - 1])
return revisions[len(revisions) - 1]
def _get_archive_dir_name(self, platform, webkit_canary):
"""Get name of the layout test archive directory.
Returns:
Directory name or
None on failure
"""
if webkit_canary:
platform += '-canary'
if platform in ARCHIVE_DIR_NAME_DICT:
return ARCHIVE_DIR_NAME_DICT[platform]
else:
logging.error('Cannot find platform key %s in archive '
'directory name dictionary', platform)
return None
def _get_archive_url(self):
"""Generate the url to download latest layout test archive.
Returns:
Url to download archive or
None on failure
"""
dir_name = self._get_archive_dir_name(self._platform,
self._options.webkit_canary)
if not dir_name:
return None
logging.debug('Buildbot platform dir name: "%s"', dir_name)
url_base = '%s/%s/' % (self._options.archive_url, dir_name)
latest_revision = self._get_latest_revision(url_base)
if latest_revision is None or latest_revision <= 0:
return None
archive_url = ('%s%s/layout-test-results.zip' % (url_base,
latest_revision))
logging.info('Archive url: "%s"', archive_url)
return archive_url
def _download_buildbot_archive(self):
"""Download layout test archive file from buildbot.
Returns:
True if download succeeded or
False otherwise.
"""
url = self._get_archive_url()
if url is None:
return None
fn = urllib.urlretrieve(url)[0]
logging.info('Archive downloaded and saved to file: "%s"', fn)
return fn
def _extract_and_add_new_baselines(self, archive_file):
"""Extract new baselines from archive and add them to SVN repository.
Args:
archive_file: full path to the archive file.
Returns:
List of tests that have been rebaselined or
None on failure.
"""
zip_file = zipfile.ZipFile(archive_file, 'r')
zip_namelist = zip_file.namelist()
logging.debug('zip file namelist:')
for name in zip_namelist:
logging.debug(' ' + name)
platform = path_utils.platform_name(self._platform)
logging.debug('Platform dir: "%s"', platform)
test_no = 1
self._rebaselined_tests = []
for test in self._rebaselining_tests:
logging.info('Test %d: %s', test_no, test)
found = False
svn_error = False
test_basename = os.path.splitext(test)[0]
for suffix in BASELINE_SUFFIXES:
archive_test_name = ('layout-test-results/%s-actual%s' %
(test_basename, suffix))
logging.debug(' Archive test file name: "%s"',
archive_test_name)
if not archive_test_name in zip_namelist:
logging.info(' %s file not in archive.', suffix)
continue
found = True
logging.info(' %s file found in archive.', suffix)
# Extract new baseline from archive and save it to a temp file.
data = zip_file.read(archive_test_name)
temp_fd, temp_name = tempfile.mkstemp(suffix)
f = os.fdopen(temp_fd, 'wb')
f.write(data)
f.close()
expected_filename = '%s-expected%s' % (test_basename, suffix)
expected_fullpath = os.path.join(
path_utils.chromium_baseline_path(platform),
expected_filename)
expected_fullpath = os.path.normpath(expected_fullpath)
logging.debug(' Expected file full path: "%s"',
expected_fullpath)
# TODO(victorw): for now, the rebaselining tool checks whether
# or not THIS baseline is duplicate and should be skipped.
# We could improve the tool to check all baselines in upper
# and lower
# levels and remove all duplicated baselines.
if self._is_dup_baseline(temp_name,
expected_fullpath,
test,
suffix,
self._platform):
os.remove(temp_name)
self._delete_baseline(expected_fullpath)
continue
# Create the new baseline directory if it doesn't already
# exist.
path_utils.maybe_make_directory(
os.path.dirname(expected_fullpath))
shutil.move(temp_name, expected_fullpath)
if not self._svn_add(expected_fullpath):
svn_error = True
elif suffix != '.checksum':
self._create_html_baseline_files(expected_fullpath)
if not found:
logging.warn(' No new baselines found in archive.')
else:
if svn_error:
logging.warn(' Failed to add baselines to SVN.')
else:
logging.info(' Rebaseline succeeded.')
self._rebaselined_tests.append(test)
test_no += 1
zip_file.close()
os.remove(archive_file)
return self._rebaselined_tests
def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix,
platform):
"""Check whether a baseline is duplicate and can fallback to same
baseline for another platform. For example, if a test has same
baseline on linux and windows, then we only store windows
baseline and linux baseline will fallback to the windows version.
Args:
expected_filename: baseline expectation file name.
test: test name.
suffix: file suffix of the expected results, including dot;
e.g. '.txt' or '.png'.
platform: baseline platform 'mac', 'win' or 'linux'.
Returns:
True if the baseline is unnecessary.
False otherwise.
"""
test_filepath = os.path.join(path_utils.layout_tests_dir(), test)
all_baselines = path_utils.expected_baselines(test_filepath,
suffix, platform, True)
for (fallback_dir, fallback_file) in all_baselines:
if fallback_dir and fallback_file:
fallback_fullpath = os.path.normpath(
os.path.join(fallback_dir, fallback_file))
if fallback_fullpath.lower() != baseline_path.lower():
if not self._diff_baselines(new_baseline,
fallback_fullpath):
logging.info(' Found same baseline at %s',
fallback_fullpath)
return True
else:
return False
return False
def _diff_baselines(self, file1, file2):
"""Check whether two baselines are different.
Args:
file1, file2: full paths of the baselines to compare.
Returns:
True if two files are different or have different extensions.
False otherwise.
"""
ext1 = os.path.splitext(file1)[1].upper()
ext2 = os.path.splitext(file2)[1].upper()
if ext1 != ext2:
logging.warn('Files to compare have different ext. '
'File1: %s; File2: %s', file1, file2)
return True
if ext1 == '.PNG':
return image_diff.ImageDiff(self._platform, '').diff_files(file1,
file2)
else:
return text_diff.TestTextDiff(self._platform, '').diff_files(file1,
file2)
def _delete_baseline(self, filename):
"""Remove the file from repository and delete it from disk.
Args:
filename: full path of the file to delete.
"""
if not filename or not os.path.isfile(filename):
return
if self._repo_type == REPO_SVN:
parent_dir, basename = os.path.split(filename)
original_dir = os.getcwd()
os.chdir(parent_dir)
run_shell(['svn', 'delete', '--force', basename], False)
os.chdir(original_dir)
else:
os.remove(filename)
def _update_rebaselined_tests_in_file(self, backup):
"""Update the rebaselined tests in test expectations file.
Args:
backup: if True, backup the original test expectations file.
Returns:
no
"""
if self._rebaselined_tests:
self._test_expectations.remove_platform_from_file(
self._rebaselined_tests, self._platform, backup)
else:
logging.info('No test was rebaselined so nothing to remove.')
def _svn_add(self, filename):
"""Add the file to SVN repository.
Args:
filename: full path of the file to add.
Returns:
True if the file already exists in SVN or is sucessfully added
to SVN.
False otherwise.
"""
if not filename:
return False
parent_dir, basename = os.path.split(filename)
if self._repo_type != REPO_SVN or parent_dir == filename:
logging.info("No svn checkout found, skip svn add.")
return True
original_dir = os.getcwd()
os.chdir(parent_dir)
status_output = run_shell(['svn', 'status', basename], False)
os.chdir(original_dir)
output = status_output.upper()
if output.startswith('A') or output.startswith('M'):
logging.info(' File already added to SVN: "%s"', filename)
return True
if output.find('IS NOT A WORKING COPY') >= 0:
logging.info(' File is not a working copy, add its parent: "%s"',
parent_dir)
return self._svn_add(parent_dir)
os.chdir(parent_dir)
add_output = run_shell(['svn', 'add', basename], True)
os.chdir(original_dir)
output = add_output.upper().rstrip()
if output.startswith('A') and output.find(basename.upper()) >= 0:
logging.info(' Added new file: "%s"', filename)
self._svn_prop_set(filename)
return True
if (not status_output) and (add_output.upper().find(
'ALREADY UNDER VERSION CONTROL') >= 0):
logging.info(' File already under SVN and has no change: "%s"',
filename)
return True
logging.warn(' Failed to add file to SVN: "%s"', filename)
logging.warn(' Svn status output: "%s"', status_output)
logging.warn(' Svn add output: "%s"', add_output)
return False
def _svn_prop_set(self, filename):
"""Set the baseline property
Args:
filename: full path of the file to add.
Returns:
True if the file already exists in SVN or is sucessfully added
to SVN.
False otherwise.
"""
ext = os.path.splitext(filename)[1].upper()
if ext != '.TXT' and ext != '.PNG' and ext != '.CHECKSUM':
return
parent_dir, basename = os.path.split(filename)
original_dir = os.getcwd()
os.chdir(parent_dir)
if ext == '.PNG':
cmd = ['svn', 'pset', 'svn:mime-type', 'image/png', basename]
else:
cmd = ['svn', 'pset', 'svn:eol-style', 'LF', basename]
logging.debug(' Set svn prop: %s', ' '.join(cmd))
run_shell(cmd, False)
os.chdir(original_dir)
def _create_html_baseline_files(self, baseline_fullpath):
"""Create baseline files (old, new and diff) in html directory.
The files are used to compare the rebaselining results.
Args:
baseline_fullpath: full path of the expected baseline file.
"""
if not baseline_fullpath or not os.path.exists(baseline_fullpath):
return
# Copy the new baseline to html directory for result comparison.
baseline_filename = os.path.basename(baseline_fullpath)
new_file = get_result_file_fullpath(self._options.html_directory,
baseline_filename, self._platform,
'new')
shutil.copyfile(baseline_fullpath, new_file)
logging.info(' Html: copied new baseline file from "%s" to "%s".',
baseline_fullpath, new_file)
# Get the old baseline from SVN and save to the html directory.
output = run_shell(['svn', 'cat', '-r', 'BASE', baseline_fullpath])
if (not output) or (output.upper().rstrip().endswith(
'NO SUCH FILE OR DIRECTORY')):
logging.info(' No base file: "%s"', baseline_fullpath)
return
base_file = get_result_file_fullpath(self._options.html_directory,
baseline_filename, self._platform,
'old')
f = open(base_file, 'wb')
f.write(output)
f.close()
logging.info(' Html: created old baseline file: "%s".',
base_file)
# Get the diff between old and new baselines and save to the html dir.
if baseline_filename.upper().endswith('.TXT'):
# If the user specified a custom diff command in their svn config
# file, then it'll be used when we do svn diff, which we don't want
# to happen since we want the unified diff. Using --diff-cmd=diff
# doesn't always work, since they can have another diff executable
# in their path that gives different line endings. So we use a
# bogus temp directory as the config directory, which gets
# around these problems.
if sys.platform.startswith("win"):
parent_dir = tempfile.gettempdir()
else:
parent_dir = sys.path[0] # tempdir is not secure.
bogus_dir = os.path.join(parent_dir, "temp_svn_config")
logging.debug(' Html: temp config dir: "%s".', bogus_dir)
if not os.path.exists(bogus_dir):
os.mkdir(bogus_dir)
delete_bogus_dir = True
else:
delete_bogus_dir = False
output = run_shell(["svn", "diff", "--config-dir", bogus_dir,
baseline_fullpath])
if output:
diff_file = get_result_file_fullpath(
self._options.html_directory, baseline_filename,
self._platform, 'diff')
f = open(diff_file, 'wb')
f.write(output)
f.close()
logging.info(' Html: created baseline diff file: "%s".',
diff_file)
if delete_bogus_dir:
shutil.rmtree(bogus_dir, True)
logging.debug(' Html: removed temp config dir: "%s".',
bogus_dir)
class HtmlGenerator(object):
"""Class to generate rebaselining result comparison html."""
HTML_REBASELINE = (''
''
''
'Rebaselining Result Comparison (%(time)s)'
''
''
''
'Rebaselining Result Comparison (%(time)s)
'
'%(body)s'
''
'')
HTML_NO_REBASELINING_TESTS = (
'No tests found that need rebaselining.
')
HTML_TABLE_TEST = ('
')
HTML_TR_TEST = (''
'| '
'%s'
' | '
'
')
HTML_TEST_DETAIL = (''
'
'
'| Baseline | '
'Platform | '
'Old | '
'New | '
'Difference | '
'
'
'%s'
'')
HTML_TD_NOLINK = '%s | '
HTML_TD_LINK = '%(name)s | '
HTML_TD_LINK_IMG = (''
's) | ')
HTML_TR = '%s
'
def __init__(self, options, platforms, rebaselining_tests):
self._html_directory = options.html_directory
self._platforms = platforms
self._rebaselining_tests = rebaselining_tests
self._html_file = os.path.join(options.html_directory,
'rebaseline.html')
def generate_html(self):
"""Generate html file for rebaselining result comparison."""
logging.info('Generating html file')
html_body = ''
if not self._rebaselining_tests:
html_body += self.HTML_NO_REBASELINING_TESTS
else:
tests = list(self._rebaselining_tests)
tests.sort()
test_no = 1
for test in tests:
logging.info('Test %d: %s', test_no, test)
html_body += self._generate_html_for_one_test(test)
html = self.HTML_REBASELINE % ({'time': time.asctime(),
'body': html_body})
logging.debug(html)
f = open(self._html_file, 'w')
f.write(html)
f.close()
logging.info('Baseline comparison html generated at "%s"',
self._html_file)
def show_html(self):
"""Launch the rebaselining html in brwoser."""
logging.info('Launching html: "%s"', self._html_file)
html_uri = path_utils.filename_to_uri(self._html_file)
webbrowser.open(html_uri, 1)
logging.info('Html launched.')
def _generate_baseline_links(self, test_basename, suffix, platform):
"""Generate links for baseline results (old, new and diff).
Args:
test_basename: base filename of the test
suffix: baseline file suffixes: '.txt', '.png'
platform: win, linux or mac
Returns:
html links for showing baseline results (old, new and diff)
"""
baseline_filename = '%s-expected%s' % (test_basename, suffix)
logging.debug(' baseline filename: "%s"', baseline_filename)
new_file = get_result_file_fullpath(self._html_directory,
baseline_filename, platform, 'new')
logging.info(' New baseline file: "%s"', new_file)
if not os.path.exists(new_file):
logging.info(' No new baseline file: "%s"', new_file)
return ''
old_file = get_result_file_fullpath(self._html_directory,
baseline_filename, platform, 'old')
logging.info(' Old baseline file: "%s"', old_file)
if suffix == '.png':
html_td_link = self.HTML_TD_LINK_IMG
else:
html_td_link = self.HTML_TD_LINK
links = ''
if os.path.exists(old_file):
links += html_td_link % {
'uri': path_utils.filename_to_uri(old_file),
'name': baseline_filename}
else:
logging.info(' No old baseline file: "%s"', old_file)
links += self.HTML_TD_NOLINK % ''
links += html_td_link % {'uri': path_utils.filename_to_uri(new_file),
'name': baseline_filename}
diff_file = get_result_file_fullpath(self._html_directory,
baseline_filename, platform,
'diff')
logging.info(' Baseline diff file: "%s"', diff_file)
if os.path.exists(diff_file):
links += html_td_link % {'uri': path_utils.filename_to_uri(
diff_file), 'name': 'Diff'}
else:
logging.info(' No baseline diff file: "%s"', diff_file)
links += self.HTML_TD_NOLINK % ''
return links
def _generate_html_for_one_test(self, test):
"""Generate html for one rebaselining test.
Args:
test: layout test name
Returns:
html that compares baseline results for the test.
"""
test_basename = os.path.basename(os.path.splitext(test)[0])
logging.info(' basename: "%s"', test_basename)
rows = []
for suffix in BASELINE_SUFFIXES:
if suffix == '.checksum':
continue
logging.info(' Checking %s files', suffix)
for platform in self._platforms:
links = self._generate_baseline_links(test_basename, suffix,
platform)
if links:
row = self.HTML_TD_NOLINK % self._get_baseline_result_type(
suffix)
row += self.HTML_TD_NOLINK % platform
row += links
logging.debug(' html row: %s', row)
rows.append(self.HTML_TR % row)
if rows:
test_path = os.path.join(path_utils.layout_tests_dir(), test)
html = self.HTML_TR_TEST % (path_utils.filename_to_uri(test_path),
test)
html += self.HTML_TEST_DETAIL % ' '.join(rows)
logging.debug(' html for test: %s', html)
return self.HTML_TABLE_TEST % html
return ''
def _get_baseline_result_type(self, suffix):
"""Name of the baseline result type."""
if suffix == '.png':
return 'Pixel'
elif suffix == '.txt':
return 'Render Tree'
else:
return 'Other'
def main():
"""Main function to produce new baselines."""
option_parser = optparse.OptionParser()
option_parser.add_option('-v', '--verbose',
action='store_true',
default=False,
help='include debug-level logging.')
option_parser.add_option('-p', '--platforms',
default='mac,win,win-xp,win-vista,linux',
help=('Comma delimited list of platforms '
'that need rebaselining.'))
option_parser.add_option('-u', '--archive_url',
default=('http://build.chromium.org/buildbot/'
'layout_test_results'),
help=('Url to find the layout test result archive'
' file.'))
option_parser.add_option('-w', '--webkit_canary',
action='store_true',
default=False,
help=('If True, pull baselines from webkit.org '
'canary bot.'))
option_parser.add_option('-b', '--backup',
action='store_true',
default=False,
help=('Whether or not to backup the original test'
' expectations file after rebaseline.'))
option_parser.add_option('-d', '--html_directory',
default='',
help=('The directory that stores the results for'
' rebaselining comparison.'))
options = option_parser.parse_args()[0]
# Set up our logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format=('%(asctime)s %(filename)s:%(lineno)-3d '
'%(levelname)s %(message)s'),
datefmt='%y%m%d %H:%M:%S')
# Verify 'platforms' option is valid
if not options.platforms:
logging.error('Invalid "platforms" option. --platforms must be '
'specified in order to rebaseline.')
sys.exit(1)
platforms = [p.strip().lower() for p in options.platforms.split(',')]
for platform in platforms:
if not platform in REBASELINE_PLATFORM_ORDER:
logging.error('Invalid platform: "%s"' % (platform))
sys.exit(1)
# Adjust the platform order so rebaseline tool is running at the order of
# 'mac', 'win' and 'linux'. This is in same order with layout test baseline
# search paths. It simplifies how the rebaseline tool detects duplicate
# baselines. Check _IsDupBaseline method for details.
rebaseline_platforms = []
for platform in REBASELINE_PLATFORM_ORDER:
if platform in platforms:
rebaseline_platforms.append(platform)
options.html_directory = setup_html_directory(options.html_directory)
rebaselining_tests = set()
backup = options.backup
for platform in rebaseline_platforms:
rebaseliner = Rebaseliner(platform, options)
logging.info('')
log_dashed_string('Rebaseline started', platform)
if rebaseliner.run(backup):
# Only need to backup one original copy of test expectation file.
backup = False
log_dashed_string('Rebaseline done', platform)
else:
log_dashed_string('Rebaseline failed', platform, logging.ERROR)
rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
logging.info('')
log_dashed_string('Rebaselining result comparison started', None)
html_generator = HtmlGenerator(options,
rebaseline_platforms,
rebaselining_tests)
html_generator.generate_html()
html_generator.show_html()
log_dashed_string('Rebaselining result comparison done', None)
sys.exit(0)
if '__main__' == __name__:
main()