path: root/Tools/Scripts/test-webkitpy

#!/usr/bin/env python
# Copyright (c) 2009 Google Inc. All rights reserved.
# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org)
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# 
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import logging
import os
import sys

# Do not import anything from webkitpy prior to cleaning webkitpy of
# orphaned *.pyc files.  This ensures that no orphaned *.pyc files are
# accidentally imported during the course of this script.
#
# Also, do not import or execute any Python code incompatible with
# Python 2.4 until after execution of the init() method below.


_log = logging.getLogger("test-webkitpy")


# Verbose logging is useful for debugging test-webkitpy code that runs
# before the actual unit tests -- things like autoinstall downloading and
# unit-test auto-detection logic.  This is different from verbose logging
# of the unit tests themselves (i.e. the unittest module's --verbose flag).
def configure_logging(is_verbose_logging):
    """Configure the root logger.

    Configure the root logger not to log any messages from webkitpy --
    except for messages from the autoinstall module.  Also set the
    logging level as described below.

    Args:
      is_verbose_logging: A boolean value of whether logging should be
                          verbose.  If this parameter is true, the logging
                          level for the handler on the root logger is set to
                          logging.DEBUG.  Otherwise, it is set to logging.INFO.

    """
    # Don't use the Python ternary operator here so that this method will
    # work with Python 2.4.
    if is_verbose_logging:
        logging_level = logging.DEBUG
    else:
        logging_level = logging.INFO

    handler = logging.StreamHandler(sys.stderr)
    # We constrain the level on the handler rather than on the root
    # logger itself.  This is probably better because the handler is
    # configured and known only to this module, whereas the root logger
    # is an object shared (and potentially modified) by many modules.
    # Modifying the handler, then, is less intrusive and less likely to
    # interfere with modifications made by other modules (e.g. in unit
    # tests).
    handler.setLevel(logging_level)
    formatter = logging.Formatter("%(name)s: %(levelname)-8s %(message)s")
    handler.setFormatter(formatter)

    logger = logging.getLogger()
    logger.addHandler(handler)
    logger.setLevel(logging.NOTSET)

    # Filter out most webkitpy messages.
    #
    # Messages can be selectively re-enabled for this script by updating
    # this method accordingly.
    def filter(record):
        """Filter out autoinstall and non-third-party webkitpy messages."""
        # FIXME: Figure out a way not to use strings here, for example by
        #        using syntax like webkitpy.test.__name__.  We want to be
        #        sure not to import any non-Python 2.4 code, though, until
        #        after the version-checking code has executed.
        if (record.name.startswith("webkitpy.common.system.autoinstall") or
            record.name.startswith("webkitpy.test")):
            return True
        if record.name.startswith("webkitpy"):
            return False
        return True

    testing_filter = logging.Filter()
    testing_filter.filter = filter

    # Display a message so developers are not mystified as to why
    # logging does not work in the unit tests.
    _log.info("Suppressing most webkitpy logging while running unit tests.")
    handler.addFilter(testing_filter)


def _clean_pyc_files(dir_to_clean, paths_not_to_log):
    """Delete from a directory all .pyc files that have no .py file.

    Args:
      dir_to_clean: The path to the directory to clean.
      paths_not_to_log: A list of paths to .pyc files whose deletions should
                        not be logged.  This list should normally include
                        only test .pyc files.

    """
    _log.debug("Cleaning orphaned *.pyc files from: %s" % dir_to_clean)

    # Normalize paths not to log.
    paths_not_to_log = [os.path.abspath(path) for path in paths_not_to_log]

    for dir_path, dir_names, file_names in os.walk(dir_to_clean):
        for file_name in file_names:
            if file_name.endswith(".pyc") and file_name[:-1] not in file_names:
                file_path = os.path.join(dir_path, file_name)
                if os.path.abspath(file_path) not in paths_not_to_log:
                    _log.info("Deleting orphan *.pyc file: %s" % file_path)
                os.remove(file_path)


# As a substitute for a unit test, this method tests _clean_pyc_files()
# in addition to calling it.  We chose not to use the unittest module
# because _clean_pyc_files() is called only once and is not used elsewhere.
def _clean_packages_with_test(external_package_paths):
    webkitpy_dir = os.path.join(os.path.dirname(__file__), "webkitpy")
    package_paths = [webkitpy_dir] + external_package_paths

    # The test .pyc file is--
    # webkitpy/python24/TEMP_test-webkitpy_test_pyc_file.pyc.
    test_path = os.path.join(webkitpy_dir, "python24",
                             "TEMP_test-webkitpy_test_pyc_file.pyc")

    test_file = open(test_path, "w")
    try:
        test_file.write("Test .pyc file generated by test-webkitpy.")
    finally:
        test_file.close()

    # Confirm that the test file exists so that when we check that it does
    # not exist, the result is meaningful.
    if not os.path.exists(test_path):
        raise Exception("Test .pyc file not created: %s" % test_path)

    for path in package_paths:
        _clean_pyc_files(path, [test_path])

    if os.path.exists(test_path):
        raise Exception("Test .pyc file not deleted: %s" % test_path)


def init(command_args, external_package_paths):
    """Execute code prior to importing from webkitpy.unittests.

    Args:
        command_args: The list of command-line arguments -- usually
                      sys.argv[1:].

    """
    verbose_logging_flag = "--verbose-logging"
    is_verbose_logging = verbose_logging_flag in command_args
    if is_verbose_logging:
        # Remove the flag so it doesn't cause unittest.main() to error out.
        #
        # FIXME: Get documentation for the --verbose-logging flag to show
        #        up in the usage instructions, which are currently generated
        #        by unittest.main().  It's possible that this will require
        #        re-implementing the option parser for unittest.main()
        #        since there may not be an easy way to modify its existing
        #        option parser.
        sys.argv.remove(verbose_logging_flag)

    configure_logging(is_verbose_logging)
    _log.debug("Verbose WebKit logging enabled.")

    # We clean orphaned *.pyc files from the packages prior to importing from
    # them to make sure that no import statements falsely succeed.
    # This helps to check that import statements have been updated correctly
    # after any file moves.  Otherwise, incorrect import statements can
    # be masked.
    #
    # For example, if webkitpy/python24/versioning.py were moved to a
    # different location without changing any import statements, and if
    # the corresponding .pyc file were left behind without deleting it,
    # then "import webkitpy.python24.versioning" would continue to succeed
    # even though it would fail for someone checking out a fresh copy
    # of the source tree.  This is because of a Python feature:
    #
    # "It is possible to have a file called spam.pyc (or spam.pyo when -O
    # is used) without a file spam.py for the same module. This can be used
    # to distribute a library of Python code in a form that is moderately
    # hard to reverse engineer."
    #
    # ( http://docs.python.org/tutorial/modules.html#compiled-python-files )
    #
    # Deleting the orphaned .pyc file prior to importing, however, would
    # cause an ImportError to occur on import as desired.
    _clean_packages_with_test(external_package_paths)

    import webkitpy.python24.versioning as versioning

    versioning.check_version(log=_log)

    (comparison, current_version, minimum_version) = \
        versioning.compare_version()

    if comparison > 0:
        # Then the current version is later than the minimum version.
        message = ("You are testing webkitpy with a Python version (%s) "
                   "higher than the minimum version (%s) it was meant "
                   "to support." % (current_version, minimum_version))
        _log.warn(message)


def _path_from_webkit_root(*components):
    webkit_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
    return os.path.join(webkit_root, *components)


def _test_import(module_path):
    try:
        sys.path.append(os.path.dirname(module_path))
        module_name = os.path.basename(module_path)
        __import__(module_name)
        return True
    except Exception, e:
        message = "Skipping tests in %s due to failure (%s)." % (module_path, e)
        if module_name.endswith("QueueStatusServer"):
            message += "  This module is optional.  The failure is likely due to a missing Google AppEngine install.  (http://code.google.com/appengine/downloads.html)"
        _log.warn(message)
        return False

if __name__ == "__main__":
    # FIXME: We should probably test each package separately to avoid naming conflicts.
    external_package_paths = [
        _path_from_webkit_root('WebKit2', 'Scripts', 'webkit2'),
        _path_from_webkit_root('Tools', 'QueueStatusServer'),
    ]
    init(sys.argv[1:], external_package_paths)

    # We import the unit test code after init() to ensure that any
    # Python version warnings are displayed in case an error occurs
    # while interpreting webkitpy.unittests.  This also allows
    # logging to be configured prior to importing -- for example to
    # enable the display of autoinstall logging.log messages while
    # running the unit tests.
    from webkitpy.test.main import Tester

    external_package_paths = filter(_test_import, external_package_paths)

    Tester().run_tests(sys.argv, external_package_paths)