path: root/Tools/Scripts/webkitpy/tool/commands/rebaselineserver.py

# Copyright (c) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Starts a local HTTP server which displays layout test failures (given a test
results directory), provides comparisons of expected and actual results (both
images and text) and allows one-click rebaselining of tests."""
from __future__ import with_statement

import codecs
import datetime
import fnmatch
import mimetypes
import os
import os.path
import shutil
import threading
import time
import urlparse
import BaseHTTPServer

from optparse import make_option
from wsgiref.handlers import format_date_time

from webkitpy.common import system
from webkitpy.layout_tests.layout_package import json_results_generator
from webkitpy.layout_tests.port import factory
from webkitpy.layout_tests.port.webkit import WebKitPort
from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
from webkitpy.thirdparty import simplejson

STATE_NEEDS_REBASELINE = 'needs_rebaseline'
STATE_REBASELINE_FAILED = 'rebaseline_failed'
STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded'

class RebaselineHTTPServer(BaseHTTPServer.HTTPServer):
    def __init__(self, httpd_port, test_config, results_json, platforms_json):
        BaseHTTPServer.HTTPServer.__init__(self, ("", httpd_port), RebaselineHTTPRequestHandler)
        self.test_config = test_config
        self.results_json = results_json
        self.platforms_json = platforms_json


class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    STATIC_FILE_NAMES = frozenset([
        "index.html",
        "loupe.js",
        "main.js",
        "main.css",
        "queue.js",
        "util.js",
    ])

    STATIC_FILE_DIRECTORY = os.path.join(
        os.path.dirname(__file__), "data", "rebaselineserver")

    def do_GET(self):
        self._handle_request()

    def do_POST(self):
        self._handle_request()

    def _handle_request(self):
        # Parse input.
        if "?" in self.path:
            path, query_string = self.path.split("?", 1)
            self.query = urlparse.parse_qs(query_string)
        else:
            path = self.path
            self.query = {}
        function_or_file_name = path[1:] or "index.html"

        # See if a static file matches.
        if function_or_file_name in RebaselineHTTPRequestHandler.STATIC_FILE_NAMES:
            self._serve_static_file(function_or_file_name)
            return

        # See if a class method matches.
        function_name = function_or_file_name.replace(".", "_")
        if not hasattr(self, function_name):
            self.send_error(404, "Unknown function %s" % function_name)
            return
        if function_name[0] == "_":
            self.send_error(
                401, "Not allowed to invoke private or protected methods")
            return
        function = getattr(self, function_name)
        function()

    def _serve_static_file(self, static_path):
        self._serve_file(os.path.join(
            RebaselineHTTPRequestHandler.STATIC_FILE_DIRECTORY, static_path))

    def rebaseline(self):
        test = self.query['test'][0]
        baseline_target = self.query['baseline-target'][0]
        baseline_move_to = self.query['baseline-move-to'][0]
        test_json = self.server.results_json['tests'][test]

        if test_json['state'] != STATE_NEEDS_REBASELINE:
            self.send_error(400, "Test %s is in unexpected state: %s" %
                (test, test_json["state"]))
            return

        log = []
        success = _rebaseline_test(
            test,
            baseline_target,
            baseline_move_to,
            self.server.test_config,
            log=lambda l: log.append(l))

        if success:
            test_json['state'] = STATE_REBASELINE_SUCCEEDED
            self.send_response(200)
        else:
            test_json['state'] = STATE_REBASELINE_FAILED
            self.send_response(500)

        self.send_header('Content-type', 'text/plain')
        self.end_headers()
        self.wfile.write('\n'.join(log))

    def quitquitquit(self):
        self.send_response(200)
        self.send_header("Content-type", "text/plain")
        self.end_headers()
        self.wfile.write("Quit.\n")

        # Shutdown has to happen on another thread from the server's thread,
        # otherwise there's a deadlock
        threading.Thread(target=lambda: self.server.shutdown()).start()

    def test_result(self):
        test_name, _ = os.path.splitext(self.query['test'][0])
        mode = self.query['mode'][0]
        if mode == 'expected-image':
            file_name = test_name + '-expected.png'
        elif mode == 'actual-image':
            file_name = test_name + '-actual.png'
        if mode == 'expected-checksum':
            file_name = test_name + '-expected.checksum'
        elif mode == 'actual-checksum':
            file_name = test_name + '-actual.checksum'
        elif mode == 'diff-image':
            file_name = test_name + '-diff.png'
        if mode == 'expected-text':
            file_name = test_name + '-expected.txt'
        elif mode == 'actual-text':
            file_name = test_name + '-actual.txt'
        elif mode == 'diff-text':
            file_name = test_name + '-diff.txt'
        elif mode == 'diff-text-pretty':
            file_name = test_name + '-pretty-diff.html'

        file_path = os.path.join(self.server.test_config.results_directory, file_name)

        # Let results be cached for 60 seconds, so that they can be pre-fetched
        # by the UI
        self._serve_file(file_path, cacheable_seconds=60)

    def results_json(self):
        self._serve_json(self.server.results_json)

    def platforms_json(self):
        self._serve_json(self.server.platforms_json)

    def _serve_json(self, json):
        self.send_response(200)
        self.send_header('Content-type', 'application/json')
        self.end_headers()
        simplejson.dump(json, self.wfile)

    def _serve_file(self, file_path, cacheable_seconds=0):
        if not os.path.exists(file_path):
            self.send_error(404, "File not found")
            return
        with codecs.open(file_path, "rb") as static_file:
            self.send_response(200)
            self.send_header("Content-Length", os.path.getsize(file_path))
            mime_type, encoding = mimetypes.guess_type(file_path)
            if mime_type:
                self.send_header("Content-type", mime_type)

            if cacheable_seconds:
                expires_time = (datetime.datetime.now() +
                    datetime.timedelta(0, cacheable_seconds))
                expires_formatted = format_date_time(
                    time.mktime(expires_time.timetuple()))
                self.send_header("Expires", expires_formatted)
            self.end_headers()

            shutil.copyfileobj(static_file, self.wfile)


class TestConfig(object):
    def __init__(self, test_port, layout_tests_directory, results_directory, platforms, filesystem, scm):
        self.test_port = test_port
        self.layout_tests_directory = layout_tests_directory
        self.results_directory = results_directory
        self.platforms = platforms
        self.filesystem = filesystem
        self.scm = scm


def _get_actual_result_files(test_file, test_config):
    test_name, _ = os.path.splitext(test_file)
    test_directory = os.path.dirname(test_file)

    test_results_directory = test_config.filesystem.join(
        test_config.results_directory, test_directory)
    actual_pattern = os.path.basename(test_name) + '-actual.*'
    actual_files = []
    for filename in test_config.filesystem.listdir(test_results_directory):
        if fnmatch.fnmatch(filename, actual_pattern):
            actual_files.append(filename)
    actual_files.sort()
    return tuple(actual_files)


def _rebaseline_test(test_file, baseline_target, baseline_move_to, test_config, log):
    test_name, _ = os.path.splitext(test_file)
    test_directory = os.path.dirname(test_name)

    log('Rebaselining %s...' % test_name)

    actual_result_files = _get_actual_result_files(test_file, test_config)
    filesystem = test_config.filesystem
    scm = test_config.scm
    layout_tests_directory = test_config.layout_tests_directory
    results_directory = test_config.results_directory
    target_expectations_directory = filesystem.join(
        layout_tests_directory, 'platform', baseline_target, test_directory)
    test_results_directory = test_config.filesystem.join(
        test_config.results_directory, test_directory)

    # If requested, move current baselines out
    current_baselines = _get_test_baselines(test_file, test_config)
    if baseline_target in current_baselines and baseline_move_to != 'none':
        log('  Moving current %s baselines to %s' %
            (baseline_target, baseline_move_to))

        # See which ones we need to move (only those that are about to be
        # updated), and make sure we're not clobbering any files in the
        # destination.
        current_extensions = set(current_baselines[baseline_target].keys())
        actual_result_extensions = [
            os.path.splitext(f)[1] for f in actual_result_files]
        extensions_to_move = current_extensions.intersection(
            actual_result_extensions)

        if extensions_to_move.intersection(
            current_baselines.get(baseline_move_to, {}).keys()):
            log('    Already had baselines in %s, could not move existing '
                '%s ones' % (baseline_move_to, baseline_target))
            return False

        # Do the actual move.
        if extensions_to_move:
            if not _move_test_baselines(
                test_file,
                list(extensions_to_move),
                baseline_target,
                baseline_move_to,
                test_config,
                log):
                return False
        else:
            log('    No current baselines to move')

    log('  Updating baselines for %s' % baseline_target)
    filesystem.maybe_make_directory(target_expectations_directory)
    for source_file in actual_result_files:
        source_path = filesystem.join(test_results_directory, source_file)
        destination_file = source_file.replace('-actual', '-expected')
        destination_path = filesystem.join(
            target_expectations_directory, destination_file)
        filesystem.copyfile(source_path, destination_path)
        exit_code = scm.add(destination_path, return_exit_code=True)
        if exit_code:
            log('    Could not update %s in SCM, exit code %d' %
                (destination_file, exit_code))
            return False
        else:
            log('    Updated %s' % destination_file)

    return True


def _move_test_baselines(test_file, extensions_to_move, source_platform, destination_platform, test_config, log):
    test_file_name = os.path.splitext(os.path.basename(test_file))[0]
    test_directory = os.path.dirname(test_file)
    filesystem = test_config.filesystem

    # Want predictable output order for unit tests.
    extensions_to_move.sort()

    source_directory = os.path.join(
        test_config.layout_tests_directory,
        'platform',
        source_platform,
        test_directory)
    destination_directory = os.path.join(
        test_config.layout_tests_directory,
        'platform',
        destination_platform,
        test_directory)
    filesystem.maybe_make_directory(destination_directory)

    for extension in extensions_to_move:
        file_name = test_file_name + '-expected' + extension
        source_path = filesystem.join(source_directory, file_name)
        destination_path = filesystem.join(destination_directory, file_name)
        filesystem.copyfile(source_path, destination_path)
        exit_code = test_config.scm.add(destination_path, return_exit_code=True)
        if exit_code:
            log('    Could not update %s in SCM, exit code %d' %
                (file_name, exit_code))
            return False
        else:
            log('    Moved %s' % file_name)

    return True

def _get_test_baselines(test_file, test_config):
    class AllPlatformsPort(WebKitPort):
        def __init__(self):
            WebKitPort.__init__(self, filesystem=test_config.filesystem)
            self._platforms_by_directory = dict(
                [(self._webkit_baseline_path(p), p) for p in test_config.platforms])

        def baseline_search_path(self):
            return self._platforms_by_directory.keys()

        def platform_from_directory(self, directory):
            return self._platforms_by_directory[directory]

    test_path = test_config.filesystem.join(
        test_config.layout_tests_directory, test_file)

    all_platforms_port = AllPlatformsPort()

    all_test_baselines = {}
    for baseline_extension in ('.txt', '.checksum', '.png'):
        test_baselines = test_config.test_port.expected_baselines(
            test_path, baseline_extension)
        baselines = all_platforms_port.expected_baselines(
            test_path, baseline_extension, all_baselines=True)
        for platform_directory, expected_filename in baselines:
            if not platform_directory:
                continue
            if platform_directory == test_config.layout_tests_directory:
                platform = 'base'
            else:
                platform = all_platforms_port.platform_from_directory(
                    platform_directory)
            platform_baselines = all_test_baselines.setdefault(platform, {})
            was_used_for_test = (
                platform_directory, expected_filename) in test_baselines
            platform_baselines[baseline_extension] = was_used_for_test

    return all_test_baselines


class RebaselineServer(AbstractDeclarativeCommand):
    name = "rebaseline-server"
    help_text = __doc__
    argument_names = "/path/to/results/directory"

    def __init__(self):
        options = [
            make_option("--httpd-port", action="store", type="int", default=8127, help="Port to use for the the rebaseline HTTP server"),
        ]
        AbstractDeclarativeCommand.__init__(self, options=options)

    def execute(self, options, args, tool):
        results_directory = args[0]
        filesystem = system.filesystem.FileSystem()
        scm = self._tool.scm()

        if options.dry_run:

            def no_op_copyfile(src, dest):
                pass

            def no_op_add(path, return_exit_code=False):
                if return_exit_code:
                    return 0

            filesystem.copyfile = no_op_copyfile
            scm.add = no_op_add

        print 'Parsing unexpected_results.json...'
        results_json_path = filesystem.join(results_directory, 'unexpected_results.json')
        results_json = json_results_generator.load_json(filesystem, results_json_path)

        port = factory.get()
        layout_tests_directory = port.layout_tests_dir()
        platforms = filesystem.listdir(
            filesystem.join(layout_tests_directory, 'platform'))
        test_config = TestConfig(
            port,
            layout_tests_directory,
            results_directory,
            platforms,
            filesystem,
            scm)

        print 'Gathering current baselines...'
        for test_file, test_json in results_json['tests'].items():
            test_json['state'] = STATE_NEEDS_REBASELINE
            test_path = filesystem.join(layout_tests_directory, test_file)
            test_json['baselines'] = _get_test_baselines(test_file, test_config)

        server_url = "http://localhost:%d/" % options.httpd_port
        print "Starting server at %s" % server_url
        print ("Use the 'Exit' link in the UI, %squitquitquit "
            "or Ctrl-C to stop") % server_url

        threading.Timer(
            .1, lambda: self._tool.user.open_url(server_url)).start()

        httpd = RebaselineHTTPServer(
            httpd_port=options.httpd_port,
            test_config=test_config,
            results_json=results_json,
            platforms_json={
                'platforms': platforms,
                'defaultPlatform': port.name(),
            })
        httpd.serve_forever()