diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/common/system')
31 files changed, 4055 insertions, 0 deletions
diff --git a/Tools/Scripts/webkitpy/common/system/__init__.py b/Tools/Scripts/webkitpy/common/system/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/Tools/Scripts/webkitpy/common/system/autoinstall.py b/Tools/Scripts/webkitpy/common/system/autoinstall.py new file mode 100755 index 0000000..9adab29 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/autoinstall.py @@ -0,0 +1,517 @@ +# Copyright (c) 2009, Daniel Krech All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of the Daniel Krech nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Support for automatically downloading Python packages from an URL.""" + + +from __future__ import with_statement + +import codecs +import logging +import new +import os +import shutil +import sys +import tarfile +import tempfile +import urllib +import urlparse +import zipfile +import zipimport + +_log = logging.getLogger(__name__) + + +class AutoInstaller(object): + + """Supports automatically installing Python packages from an URL. + + Supports uncompressed files, .tar.gz, and .zip formats. + + Basic usage: + + installer = AutoInstaller() + + installer.install(url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", + url_subpath="pep8-0.5.0/pep8.py") + installer.install(url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", + url_subpath="mechanize") + + """ + + def __init__(self, append_to_search_path=False, make_package=True, + target_dir=None, temp_dir=None): + """Create an AutoInstaller instance, and set up the target directory. + + Args: + append_to_search_path: A boolean value of whether to append the + target directory to the sys.path search path. + make_package: A boolean value of whether to make the target + directory a package. This adds an __init__.py file + to the target directory -- allowing packages and + modules within the target directory to be imported + explicitly using dotted module names. + target_dir: The directory path to which packages should be installed. + Defaults to a subdirectory of the folder containing + this module called "autoinstalled". + temp_dir: The directory path to use for any temporary files + generated while downloading, unzipping, and extracting + packages to install. Defaults to a standard temporary + location generated by the tempfile module. This + parameter should normally be used only for development + testing. + + """ + if target_dir is None: + this_dir = os.path.dirname(__file__) + target_dir = os.path.join(this_dir, "autoinstalled") + + # Ensure that the target directory exists. + self._set_up_target_dir(target_dir, append_to_search_path, make_package) + + self._target_dir = target_dir + self._temp_dir = temp_dir + + def _log_transfer(self, message, source, target, log_method=None): + """Log a debug message that involves a source and target.""" + if log_method is None: + log_method = _log.debug + + log_method("%s" % message) + log_method(' From: "%s"' % source) + log_method(' To: "%s"' % target) + + def _create_directory(self, path, name=None): + """Create a directory.""" + log = _log.debug + + name = name + " " if name is not None else "" + log('Creating %sdirectory...' % name) + log(' "%s"' % path) + + os.makedirs(path) + + def _write_file(self, path, text, encoding): + """Create a file at the given path with given text. + + This method overwrites any existing file. + + """ + _log.debug("Creating file...") + _log.debug(' "%s"' % path) + with codecs.open(path, "w", encoding) as file: + file.write(text) + + def _set_up_target_dir(self, target_dir, append_to_search_path, + make_package): + """Set up a target directory. + + Args: + target_dir: The path to the target directory to set up. + append_to_search_path: A boolean value of whether to append the + target directory to the sys.path search path. + make_package: A boolean value of whether to make the target + directory a package. This adds an __init__.py file + to the target directory -- allowing packages and + modules within the target directory to be imported + explicitly using dotted module names. + + """ + if not os.path.exists(target_dir): + self._create_directory(target_dir, "autoinstall target") + + if append_to_search_path: + sys.path.append(target_dir) + + if make_package: + init_path = os.path.join(target_dir, "__init__.py") + if not os.path.exists(init_path): + text = ("# This file is required for Python to search this " + "directory for modules.\n") + self._write_file(init_path, text, "ascii") + + def _create_scratch_directory_inner(self, prefix): + """Create a scratch directory without exception handling. + + Creates a scratch directory inside the AutoInstaller temp + directory self._temp_dir, or inside a platform-dependent temp + directory if self._temp_dir is None. Returns the path to the + created scratch directory. + + Raises: + OSError: [Errno 2] if the containing temp directory self._temp_dir + is not None and does not exist. + + """ + # The tempfile.mkdtemp() method function requires that the + # directory corresponding to the "dir" parameter already exist + # if it is not None. + scratch_dir = tempfile.mkdtemp(prefix=prefix, dir=self._temp_dir) + return scratch_dir + + def _create_scratch_directory(self, target_name): + """Create a temporary scratch directory, and return its path. + + The scratch directory is generated inside the temp directory + of this AutoInstaller instance. This method also creates the + temp directory if it does not already exist. + + """ + prefix = target_name + "_" + try: + scratch_dir = self._create_scratch_directory_inner(prefix) + except OSError: + # Handle case of containing temp directory not existing-- + # OSError: [Errno 2] No such file or directory:... + temp_dir = self._temp_dir + if temp_dir is None or os.path.exists(temp_dir): + raise + # Else try again after creating the temp directory. + self._create_directory(temp_dir, "autoinstall temp") + scratch_dir = self._create_scratch_directory_inner(prefix) + + return scratch_dir + + def _url_downloaded_path(self, target_name): + """Return the path to the file containing the URL downloaded.""" + filename = ".%s.url" % target_name + path = os.path.join(self._target_dir, filename) + return path + + def _is_downloaded(self, target_name, url): + """Return whether a package version has been downloaded.""" + version_path = self._url_downloaded_path(target_name) + + _log.debug('Checking %s URL downloaded...' % target_name) + _log.debug(' "%s"' % version_path) + + if not os.path.exists(version_path): + # Then no package version has been downloaded. + _log.debug("No URL file found.") + return False + + with codecs.open(version_path, "r", "utf-8") as file: + version = file.read() + + return version.strip() == url.strip() + + def _record_url_downloaded(self, target_name, url): + """Record the URL downloaded to a file.""" + version_path = self._url_downloaded_path(target_name) + _log.debug("Recording URL downloaded...") + _log.debug(' URL: "%s"' % url) + _log.debug(' To: "%s"' % version_path) + + self._write_file(version_path, url, "utf-8") + + def _extract_targz(self, path, scratch_dir): + # tarfile.extractall() extracts to a path without the + # trailing ".tar.gz". + target_basename = os.path.basename(path[:-len(".tar.gz")]) + target_path = os.path.join(scratch_dir, target_basename) + + self._log_transfer("Starting gunzip/extract...", path, target_path) + + try: + tar_file = tarfile.open(path) + except tarfile.ReadError, err: + # Append existing Error message to new Error. + message = ("Could not open tar file: %s\n" + " The file probably does not have the correct format.\n" + " --> Inner message: %s" + % (path, err)) + raise Exception(message) + + try: + # This is helpful for debugging purposes. + _log.debug("Listing tar file contents...") + for name in tar_file.getnames(): + _log.debug(' * "%s"' % name) + _log.debug("Extracting gzipped tar file...") + tar_file.extractall(target_path) + finally: + tar_file.close() + + return target_path + + # This is a replacement for ZipFile.extractall(), which is + # available in Python 2.6 but not in earlier versions. + def _extract_all(self, zip_file, target_dir): + self._log_transfer("Extracting zip file...", zip_file, target_dir) + + # This is helpful for debugging purposes. + _log.debug("Listing zip file contents...") + for name in zip_file.namelist(): + _log.debug(' * "%s"' % name) + + for name in zip_file.namelist(): + path = os.path.join(target_dir, name) + self._log_transfer("Extracting...", name, path) + + if not os.path.basename(path): + # Then the path ends in a slash, so it is a directory. + self._create_directory(path) + continue + # Otherwise, it is a file. + + try: + # We open this file w/o encoding, as we're reading/writing + # the raw byte-stream from the zip file. + outfile = open(path, 'wb') + except IOError, err: + # Not all zip files seem to list the directories explicitly, + # so try again after creating the containing directory. + _log.debug("Got IOError: retrying after creating directory...") + dir = os.path.dirname(path) + self._create_directory(dir) + outfile = open(path, 'wb') + + try: + outfile.write(zip_file.read(name)) + finally: + outfile.close() + + def _unzip(self, path, scratch_dir): + # zipfile.extractall() extracts to a path without the + # trailing ".zip". + target_basename = os.path.basename(path[:-len(".zip")]) + target_path = os.path.join(scratch_dir, target_basename) + + self._log_transfer("Starting unzip...", path, target_path) + + try: + zip_file = zipfile.ZipFile(path, "r") + except zipfile.BadZipfile, err: + message = ("Could not open zip file: %s\n" + " --> Inner message: %s" + % (path, err)) + raise Exception(message) + + try: + self._extract_all(zip_file, scratch_dir) + finally: + zip_file.close() + + return target_path + + def _prepare_package(self, path, scratch_dir): + """Prepare a package for use, if necessary, and return the new path. + + For example, this method unzips zipped files and extracts + tar files. + + Args: + path: The path to the downloaded URL contents. + scratch_dir: The scratch directory. Note that the scratch + directory contains the file designated by the + path parameter. + + """ + # FIXME: Add other natural extensions. + if path.endswith(".zip"): + new_path = self._unzip(path, scratch_dir) + elif path.endswith(".tar.gz"): + new_path = self._extract_targz(path, scratch_dir) + else: + # No preparation is needed. + new_path = path + + return new_path + + def _download_to_stream(self, url, stream): + """Download an URL to a stream, and return the number of bytes.""" + try: + netstream = urllib.urlopen(url) + except IOError, err: + # Append existing Error message to new Error. + message = ('Could not download Python modules from URL "%s".\n' + " Make sure you are connected to the internet.\n" + " You must be connected to the internet when " + "downloading needed modules for the first time.\n" + " --> Inner message: %s" + % (url, err)) + raise IOError(message) + code = 200 + if hasattr(netstream, "getcode"): + code = netstream.getcode() + if not 200 <= code < 300: + raise ValueError("HTTP Error code %s" % code) + + BUFSIZE = 2**13 # 8KB + bytes = 0 + while True: + data = netstream.read(BUFSIZE) + if not data: + break + stream.write(data) + bytes += len(data) + netstream.close() + return bytes + + def _download(self, url, scratch_dir): + """Download URL contents, and return the download path.""" + url_path = urlparse.urlsplit(url)[2] + url_path = os.path.normpath(url_path) # Removes trailing slash. + target_filename = os.path.basename(url_path) + target_path = os.path.join(scratch_dir, target_filename) + + self._log_transfer("Starting download...", url, target_path) + + with open(target_path, "wb") as stream: + bytes = self._download_to_stream(url, stream) + + _log.debug("Downloaded %s bytes." % bytes) + + return target_path + + def _install(self, scratch_dir, package_name, target_path, url, + url_subpath): + """Install a python package from an URL. + + This internal method overwrites the target path if the target + path already exists. + + """ + path = self._download(url=url, scratch_dir=scratch_dir) + path = self._prepare_package(path, scratch_dir) + + if url_subpath is None: + source_path = path + else: + source_path = os.path.join(path, url_subpath) + + if os.path.exists(target_path): + _log.debug('Refreshing install: deleting "%s".' % target_path) + if os.path.isdir(target_path): + shutil.rmtree(target_path) + else: + os.remove(target_path) + + self._log_transfer("Moving files into place...", source_path, target_path) + + # The shutil.move() command creates intermediate directories if they + # do not exist, but we do not rely on this behavior since we + # need to create the __init__.py file anyway. + shutil.move(source_path, target_path) + + self._record_url_downloaded(package_name, url) + + def install(self, url, should_refresh=False, target_name=None, + url_subpath=None): + """Install a python package from an URL. + + Args: + url: The URL from which to download the package. + + Optional Args: + should_refresh: A boolean value of whether the package should be + downloaded again if the package is already present. + target_name: The name of the folder or file in the autoinstaller + target directory at which the package should be + installed. Defaults to the base name of the + URL sub-path. This parameter must be provided if + the URL sub-path is not specified. + url_subpath: The relative path of the URL directory that should + be installed. Defaults to the full directory, or + the entire URL contents. + + """ + if target_name is None: + if not url_subpath: + raise ValueError('The "target_name" parameter must be ' + 'provided if the "url_subpath" parameter ' + "is not provided.") + # Remove any trailing slashes. + url_subpath = os.path.normpath(url_subpath) + target_name = os.path.basename(url_subpath) + + target_path = os.path.join(self._target_dir, target_name) + if not should_refresh and self._is_downloaded(target_name, url): + _log.debug('URL for %s already downloaded. Skipping...' + % target_name) + _log.debug(' "%s"' % url) + return + + self._log_transfer("Auto-installing package: %s" % target_name, + url, target_path, log_method=_log.info) + + # The scratch directory is where we will download and prepare + # files specific to this install until they are ready to move + # into place. + scratch_dir = self._create_scratch_directory(target_name) + + try: + self._install(package_name=target_name, + target_path=target_path, + scratch_dir=scratch_dir, + url=url, + url_subpath=url_subpath) + except Exception, err: + # Append existing Error message to new Error. + message = ("Error auto-installing the %s package to:\n" + ' "%s"\n' + " --> Inner message: %s" + % (target_name, target_path, err)) + raise Exception(message) + finally: + _log.debug('Cleaning up: deleting "%s".' % scratch_dir) + shutil.rmtree(scratch_dir) + _log.debug('Auto-installed %s to:' % target_name) + _log.debug(' "%s"' % target_path) + + +if __name__=="__main__": + + # Configure the autoinstall logger to log DEBUG messages for + # development testing purposes. + console = logging.StreamHandler() + + formatter = logging.Formatter('%(name)s: %(levelname)-8s %(message)s') + console.setFormatter(formatter) + _log.addHandler(console) + _log.setLevel(logging.DEBUG) + + # Use a more visible temp directory for debug purposes. + this_dir = os.path.dirname(__file__) + target_dir = os.path.join(this_dir, "autoinstalled") + temp_dir = os.path.join(target_dir, "Temp") + + installer = AutoInstaller(target_dir=target_dir, + temp_dir=temp_dir) + + installer.install(should_refresh=False, + target_name="pep8.py", + url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", + url_subpath="pep8-0.5.0/pep8.py") + installer.install(should_refresh=False, + target_name="mechanize", + url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", + url_subpath="mechanize") + diff --git a/Tools/Scripts/webkitpy/common/system/deprecated_logging.py b/Tools/Scripts/webkitpy/common/system/deprecated_logging.py new file mode 100644 index 0000000..9e6b529 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/deprecated_logging.py @@ -0,0 +1,91 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# WebKit's Python module for logging +# This module is now deprecated in favor of python's built-in logging.py. + +import codecs +import os +import sys + + +def log(string): + print >> sys.stderr, string + + +def error(string): + log("ERROR: %s" % string) + exit(1) + + +# Simple class to split output between multiple destinations +class tee: + def __init__(self, *files): + self.files = files + + # Callers should pass an already encoded string for writing. + def write(self, bytes): + for file in self.files: + file.write(bytes) + + +class OutputTee: + def __init__(self): + self._original_stdout = None + self._original_stderr = None + self._files_for_output = [] + + def add_log(self, path): + log_file = self._open_log_file(path) + self._files_for_output.append(log_file) + self._tee_outputs_to_files(self._files_for_output) + return log_file + + def remove_log(self, log_file): + self._files_for_output.remove(log_file) + self._tee_outputs_to_files(self._files_for_output) + log_file.close() + + @staticmethod + def _open_log_file(log_path): + (log_directory, log_name) = os.path.split(log_path) + if log_directory and not os.path.exists(log_directory): + os.makedirs(log_directory) + return codecs.open(log_path, "a+", "utf-8") + + def _tee_outputs_to_files(self, files): + if not self._original_stdout: + self._original_stdout = sys.stdout + self._original_stderr = sys.stderr + if files and len(files): + sys.stdout = tee(self._original_stdout, *files) + sys.stderr = tee(self._original_stderr, *files) + else: + sys.stdout = self._original_stdout + sys.stderr = self._original_stderr diff --git a/Tools/Scripts/webkitpy/common/system/deprecated_logging_unittest.py b/Tools/Scripts/webkitpy/common/system/deprecated_logging_unittest.py new file mode 100644 index 0000000..3778162 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/deprecated_logging_unittest.py @@ -0,0 +1,60 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import StringIO +import tempfile +import unittest + +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.system.deprecated_logging import * + +class LoggingTest(unittest.TestCase): + + def assert_log_equals(self, log_input, expected_output): + original_stderr = sys.stderr + test_stderr = StringIO.StringIO() + sys.stderr = test_stderr + + try: + log(log_input) + actual_output = test_stderr.getvalue() + finally: + sys.stderr = original_stderr + + self.assertEquals(actual_output, expected_output, "log(\"%s\") expected: %s actual: %s" % (log_input, expected_output, actual_output)) + + def test_log(self): + self.assert_log_equals("test", "test\n") + + # Test that log() does not throw an exception when passed an object instead of a string. + self.assert_log_equals(ScriptError(message="ScriptError"), "ScriptError\n") + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/common/system/directoryfileset.py b/Tools/Scripts/webkitpy/common/system/directoryfileset.py new file mode 100644 index 0000000..11acaf4 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/directoryfileset.py @@ -0,0 +1,77 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import with_statement + +import os +import shutil + +from webkitpy.common.system.fileset import FileSetFileHandle +from webkitpy.common.system.filesystem import FileSystem +import webkitpy.common.system.ospath as ospath + + +class DirectoryFileSet(object): + """The set of files under a local directory.""" + def __init__(self, path, filesystem=None): + self._path = path + self._filesystem = filesystem or FileSystem() + if not self._path.endswith(os.path.sep): + self._path += os.path.sep + + def _full_path(self, filename): + assert self._is_under(self._path, filename) + return self._filesystem.join(self._path, filename) + + def _drop_directory_prefix(self, path): + return path[len(self._path):] + + def _files_in_directory(self): + """Returns a list of all the files in the directory, including the path + to the directory""" + return self._filesystem.files_under(self._path) + + def _is_under(self, dir, filename): + return bool(ospath.relpath(self._filesystem.join(dir, filename), dir)) + + def open(self, filename): + return FileSetFileHandle(self, filename, self._filesystem) + + def namelist(self): + return map(self._drop_directory_prefix, self._files_in_directory()) + + def read(self, filename): + return self._filesystem.read_text_file(self._full_path(filename)) + + def extract(self, filename, path): + """Extracts a file from this file set to the specified directory.""" + src = self._full_path(filename) + dest = self._filesystem.join(path, filename) + # As filename may have slashes in it, we must ensure that the same + # directory hierarchy exists at the output path. + self._filesystem.maybe_make_directory(os.path.split(dest)[0]) + self._filesystem.copyfile(src, dest) + + def delete(self, filename): + filename = self._full_path(filename) + self._filesystem.remove(filename) diff --git a/Tools/Scripts/webkitpy/common/system/directoryfileset_unittest.py b/Tools/Scripts/webkitpy/common/system/directoryfileset_unittest.py new file mode 100644 index 0000000..a3850ee --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/directoryfileset_unittest.py @@ -0,0 +1,70 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import with_statement + +import unittest + +from webkitpy.common.system.directoryfileset import DirectoryFileSet +from webkitpy.common.system.filesystem_mock import MockFileSystem + + +class DirectoryFileSetTest(unittest.TestCase): + def setUp(self): + files = {} + files['/test/some-file'] = 'contents' + files['/test/some-other-file'] = 'other contents' + files['/test/b/c'] = 'c' + self._filesystem = MockFileSystem(files) + self._fileset = DirectoryFileSet('/test', self._filesystem) + + def test_files_in_namelist(self): + self.assertTrue('some-file' in self._fileset.namelist()) + self.assertTrue('some-other-file' in self._fileset.namelist()) + self.assertTrue('b/c' in self._fileset.namelist()) + + def test_read(self): + self.assertEquals('contents', self._fileset.read('some-file')) + + def test_open(self): + file = self._fileset.open('some-file') + self.assertEquals('some-file', file.name()) + self.assertEquals('contents', file.contents()) + + def test_extract(self): + self._fileset.extract('some-file', '/test-directory') + contents = self._filesystem.read_text_file('/test-directory/some-file') + self.assertEquals('contents', contents) + + def test_extract_deep_file(self): + self._fileset.extract('b/c', '/test-directory') + self.assertTrue(self._filesystem.exists('/test-directory/b/c')) + + def test_delete(self): + self.assertTrue(self._filesystem.exists('/test/some-file')) + self._fileset.delete('some-file') + self.assertFalse(self._filesystem.exists('/test/some-file')) + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/common/system/executive.py b/Tools/Scripts/webkitpy/common/system/executive.py new file mode 100644 index 0000000..02619db --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/executive.py @@ -0,0 +1,407 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +try: + # This API exists only in Python 2.6 and higher. :( + import multiprocessing +except ImportError: + multiprocessing = None + +import ctypes +import errno +import logging +import os +import platform +import StringIO +import signal +import subprocess +import sys +import time + +from webkitpy.common.system.deprecated_logging import tee +from webkitpy.python24 import versioning + + +_log = logging.getLogger("webkitpy.common.system") + + +class ScriptError(Exception): + + # This is a custom List.__str__ implementation to allow size limiting. + def _string_from_args(self, args, limit=100): + args_string = unicode(args) + # We could make this much fancier, but for now this is OK. + if len(args_string) > limit: + return args_string[:limit - 3] + "..." + return args_string + + def __init__(self, + message=None, + script_args=None, + exit_code=None, + output=None, + cwd=None): + if not message: + message = 'Failed to run "%s"' % self._string_from_args(script_args) + if exit_code: + message += " exit_code: %d" % exit_code + if cwd: + message += " cwd: %s" % cwd + + Exception.__init__(self, message) + self.script_args = script_args # 'args' is already used by Exception + self.exit_code = exit_code + self.output = output + self.cwd = cwd + + def message_with_output(self, output_limit=500): + if self.output: + if output_limit and len(self.output) > output_limit: + return u"%s\n\nLast %s characters of output:\n%s" % \ + (self, output_limit, self.output[-output_limit:]) + return u"%s\n\n%s" % (self, self.output) + return unicode(self) + + def command_name(self): + command_path = self.script_args + if type(command_path) is list: + command_path = command_path[0] + return os.path.basename(command_path) + + +def run_command(*args, **kwargs): + # FIXME: This should not be a global static. + # New code should use Executive.run_command directly instead + return Executive().run_command(*args, **kwargs) + + +class Executive(object): + + def _should_close_fds(self): + # We need to pass close_fds=True to work around Python bug #2320 + # (otherwise we can hang when we kill DumpRenderTree when we are running + # multiple threads). See http://bugs.python.org/issue2320 . + # Note that close_fds isn't supported on Windows, but this bug only + # shows up on Mac and Linux. + return sys.platform not in ('win32', 'cygwin') + + def _run_command_with_teed_output(self, args, teed_output): + args = map(unicode, args) # Popen will throw an exception if args are non-strings (like int()) + args = map(self._encode_argument_if_needed, args) + + child_process = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + close_fds=self._should_close_fds()) + + # Use our own custom wait loop because Popen ignores a tee'd + # stderr/stdout. + # FIXME: This could be improved not to flatten output to stdout. + while True: + output_line = child_process.stdout.readline() + if output_line == "" and child_process.poll() != None: + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + return child_process.poll() + # We assume that the child process wrote to us in utf-8, + # so no re-encoding is necessary before writing here. + teed_output.write(output_line) + + # FIXME: Remove this deprecated method and move callers to run_command. + # FIXME: This method is a hack to allow running command which both + # capture their output and print out to stdin. Useful for things + # like "build-webkit" where we want to display to the user that we're building + # but still have the output to stuff into a log file. + def run_and_throw_if_fail(self, args, quiet=False, decode_output=True): + # Cache the child's output locally so it can be used for error reports. + child_out_file = StringIO.StringIO() + tee_stdout = sys.stdout + if quiet: + dev_null = open(os.devnull, "w") # FIXME: Does this need an encoding? + tee_stdout = dev_null + child_stdout = tee(child_out_file, tee_stdout) + exit_code = self._run_command_with_teed_output(args, child_stdout) + if quiet: + dev_null.close() + + child_output = child_out_file.getvalue() + child_out_file.close() + + if decode_output: + child_output = child_output.decode(self._child_process_encoding()) + + if exit_code: + raise ScriptError(script_args=args, + exit_code=exit_code, + output=child_output) + return child_output + + def cpu_count(self): + if multiprocessing: + return multiprocessing.cpu_count() + # Darn. We don't have the multiprocessing package. + system_name = platform.system() + if system_name == "Darwin": + return int(self.run_command(["sysctl", "-n", "hw.ncpu"])) + elif system_name == "Windows": + return int(os.environ.get('NUMBER_OF_PROCESSORS', 1)) + elif system_name == "Linux": + num_cores = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(num_cores, int) and num_cores > 0: + return num_cores + # This quantity is a lie but probably a reasonable guess for modern + # machines. + return 2 + + def kill_process(self, pid): + """Attempts to kill the given pid. + Will fail silently if pid does not exist or insufficient permisssions.""" + if sys.platform == "win32": + # We only use taskkill.exe on windows (not cygwin) because subprocess.pid + # is a CYGWIN pid and taskkill.exe expects a windows pid. + # Thankfully os.kill on CYGWIN handles either pid type. + command = ["taskkill.exe", "/f", "/pid", pid] + # taskkill will exit 128 if the process is not found. We should log. + self.run_command(command, error_handler=self.ignore_error) + return + + # According to http://docs.python.org/library/os.html + # os.kill isn't available on Windows. python 2.5.5 os.kill appears + # to work in cygwin, however it occasionally raises EAGAIN. + retries_left = 10 if sys.platform == "cygwin" else 1 + while retries_left > 0: + try: + retries_left -= 1 + os.kill(pid, signal.SIGKILL) + except OSError, e: + if e.errno == errno.EAGAIN: + if retries_left <= 0: + _log.warn("Failed to kill pid %s. Too many EAGAIN errors." % pid) + continue + if e.errno == errno.ESRCH: # The process does not exist. + _log.warn("Called kill_process with a non-existant pid %s" % pid) + return + raise + + def _win32_check_running_pid(self, pid): + + class PROCESSENTRY32(ctypes.Structure): + _fields_ = [("dwSize", ctypes.c_ulong), + ("cntUsage", ctypes.c_ulong), + ("th32ProcessID", ctypes.c_ulong), + ("th32DefaultHeapID", ctypes.c_ulong), + ("th32ModuleID", ctypes.c_ulong), + ("cntThreads", ctypes.c_ulong), + ("th32ParentProcessID", ctypes.c_ulong), + ("pcPriClassBase", ctypes.c_ulong), + ("dwFlags", ctypes.c_ulong), + ("szExeFile", ctypes.c_char * 260)] + + CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot + Process32First = ctypes.windll.kernel32.Process32First + Process32Next = ctypes.windll.kernel32.Process32Next + CloseHandle = ctypes.windll.kernel32.CloseHandle + TH32CS_SNAPPROCESS = 0x00000002 # win32 magic number + hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) + pe32 = PROCESSENTRY32() + pe32.dwSize = ctypes.sizeof(PROCESSENTRY32) + result = False + if not Process32First(hProcessSnap, ctypes.byref(pe32)): + _log.debug("Failed getting first process.") + CloseHandle(hProcessSnap) + return result + while True: + if pe32.th32ProcessID == pid: + result = True + break + if not Process32Next(hProcessSnap, ctypes.byref(pe32)): + break + CloseHandle(hProcessSnap) + return result + + def check_running_pid(self, pid): + """Return True if pid is alive, otherwise return False.""" + if sys.platform in ('darwin', 'linux2', 'cygwin'): + try: + os.kill(pid, 0) + return True + except OSError: + return False + elif sys.platform == 'win32': + return self._win32_check_running_pid(pid) + + assert(False) + + def _windows_image_name(self, process_name): + name, extension = os.path.splitext(process_name) + if not extension: + # taskkill expects processes to end in .exe + # If necessary we could add a flag to disable appending .exe. + process_name = "%s.exe" % name + return process_name + + def kill_all(self, process_name): + """Attempts to kill processes matching process_name. + Will fail silently if no process are found.""" + if sys.platform in ("win32", "cygwin"): + image_name = self._windows_image_name(process_name) + command = ["taskkill.exe", "/f", "/im", image_name] + # taskkill will exit 128 if the process is not found. We should log. + self.run_command(command, error_handler=self.ignore_error) + return + + # FIXME: This is inconsistent that kill_all uses TERM and kill_process + # uses KILL. Windows is always using /f (which seems like -KILL). + # We should pick one mode, or add support for switching between them. + # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER + command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name] + # killall returns 1 if no process can be found and 2 on command error. + # FIXME: We should pass a custom error_handler to allow only exit_code 1. + # We should log in exit_code == 1 + self.run_command(command, error_handler=self.ignore_error) + + # Error handlers do not need to be static methods once all callers are + # updated to use an Executive object. + + @staticmethod + def default_error_handler(error): + raise error + + @staticmethod + def ignore_error(error): + pass + + def _compute_stdin(self, input): + """Returns (stdin, string_to_communicate)""" + # FIXME: We should be returning /dev/null for stdin + # or closing stdin after process creation to prevent + # child processes from getting input from the user. + if not input: + return (None, None) + if hasattr(input, "read"): # Check if the input is a file. + return (input, None) # Assume the file is in the right encoding. + + # Popen in Python 2.5 and before does not automatically encode unicode objects. + # http://bugs.python.org/issue5290 + # See https://bugs.webkit.org/show_bug.cgi?id=37528 + # for an example of a regresion caused by passing a unicode string directly. + # FIXME: We may need to encode differently on different platforms. + if isinstance(input, unicode): + input = input.encode(self._child_process_encoding()) + return (subprocess.PIPE, input) + + def _command_for_printing(self, args): + """Returns a print-ready string representing command args. + The string should be copy/paste ready for execution in a shell.""" + escaped_args = [] + for arg in args: + if isinstance(arg, unicode): + # Escape any non-ascii characters for easy copy/paste + arg = arg.encode("unicode_escape") + # FIXME: Do we need to fix quotes here? + escaped_args.append(arg) + return " ".join(escaped_args) + + # FIXME: run_and_throw_if_fail should be merged into this method. + def run_command(self, + args, + cwd=None, + input=None, + error_handler=None, + return_exit_code=False, + return_stderr=True, + decode_output=True): + """Popen wrapper for convenience and to work around python bugs.""" + assert(isinstance(args, list) or isinstance(args, tuple)) + start_time = time.time() + args = map(unicode, args) # Popen will throw an exception if args are non-strings (like int()) + args = map(self._encode_argument_if_needed, args) + + stdin, string_to_communicate = self._compute_stdin(input) + stderr = subprocess.STDOUT if return_stderr else None + + process = subprocess.Popen(args, + stdin=stdin, + stdout=subprocess.PIPE, + stderr=stderr, + cwd=cwd, + close_fds=self._should_close_fds()) + output = process.communicate(string_to_communicate)[0] + + # run_command automatically decodes to unicode() unless explicitly told not to. + if decode_output: + output = output.decode(self._child_process_encoding()) + + # wait() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + exit_code = process.wait() + + _log.debug('"%s" took %.2fs' % (self._command_for_printing(args), time.time() - start_time)) + + if return_exit_code: + return exit_code + + if exit_code: + script_error = ScriptError(script_args=args, + exit_code=exit_code, + output=output, + cwd=cwd) + (error_handler or self.default_error_handler)(script_error) + return output + + def _child_process_encoding(self): + # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW + # to launch subprocesses, so we have to encode arguments using the + # current code page. + if sys.platform == 'win32' and versioning.compare_version(sys, '3.0')[0] < 0: + return 'mbcs' + # All other platforms use UTF-8. + # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands + # which will expect arguments to be encoded using the current code + # page. + return 'utf-8' + + def _should_encode_child_process_arguments(self): + # Cygwin's Python's os.execv doesn't support unicode command + # arguments, and neither does Cygwin's execv itself. + if sys.platform == 'cygwin': + return True + + # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW + # to launch subprocesses, so we have to encode arguments using the + # current code page. + if sys.platform == 'win32' and versioning.compare_version(sys, '3.0')[0] < 0: + return True + + return False + + def _encode_argument_if_needed(self, argument): + if not self._should_encode_child_process_arguments(): + return argument + return argument.encode(self._child_process_encoding()) diff --git a/Tools/Scripts/webkitpy/common/system/executive_mock.py b/Tools/Scripts/webkitpy/common/system/executive_mock.py new file mode 100644 index 0000000..943b70c --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/executive_mock.py @@ -0,0 +1,67 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# FIXME: Implement the rest of the interface as needed for testing :). + +# FIXME: Unify with tool/mocktool.MockExecutive. + +from webkitpy.common.system import executive + + +class MockExecutive2(object): + def __init__(self, output='', exit_code=0, exception=None, + run_command_fn=None): + self._output = output + self._exit_code = exit_code + self._exception = exception + self._run_command_fn = run_command_fn + + def cpu_count(self): + return 2 + + def kill_all(self, process_name): + pass + + def kill_process(self, pid): + pass + + def run_command(self, arg_list, error_handler=None, return_exit_code=False, + decode_output=False): + if self._exception: + raise self._exception + if return_exit_code: + return self._exit_code + if self._run_command_fn: + return self._run_command_fn(arg_list) + if self._exit_code and error_handler: + script_error = executive.ScriptError(script_args=arg_list, + exit_code=self._exit_code, + output=self._output) + error_handler(script_error) + + return self._output diff --git a/Tools/Scripts/webkitpy/common/system/executive_unittest.py b/Tools/Scripts/webkitpy/common/system/executive_unittest.py new file mode 100644 index 0000000..a43b4dc --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/executive_unittest.py @@ -0,0 +1,168 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# Copyright (C) 2009 Daniel Bates (dbates@intudata.com). All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import signal +import subprocess +import sys +import unittest + +from webkitpy.common.system.executive import Executive, run_command, ScriptError +from webkitpy.test import cat, echo + + +class ScriptErrorTest(unittest.TestCase): + def test_string_from_args(self): + error = ScriptError() + self.assertEquals(error._string_from_args(None), 'None') + self.assertEquals(error._string_from_args([]), '[]') + self.assertEquals(error._string_from_args(map(str, range(30))), "['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17'...") + + def test_message_with_output(self): + error = ScriptError('My custom message!', '', -1) + self.assertEquals(error.message_with_output(), 'My custom message!') + error = ScriptError('My custom message!', '', -1, 'My output.') + self.assertEquals(error.message_with_output(), 'My custom message!\n\nMy output.') + error = ScriptError('', 'my_command!', -1, 'My output.', '/Users/username/blah') + self.assertEquals(error.message_with_output(), 'Failed to run "my_command!" exit_code: -1 cwd: /Users/username/blah\n\nMy output.') + error = ScriptError('', 'my_command!', -1, 'ab' + '1' * 499) + self.assertEquals(error.message_with_output(), 'Failed to run "my_command!" exit_code: -1\n\nLast 500 characters of output:\nb' + '1' * 499) + +def never_ending_command(): + """Arguments for a command that will never end (useful for testing process + killing). It should be a process that is unlikely to already be running + because all instances will be killed.""" + if sys.platform == 'win32': + return ['wmic'] + return ['yes'] + + +class ExecutiveTest(unittest.TestCase): + + def test_run_command_with_bad_command(self): + def run_bad_command(): + run_command(["foo_bar_command_blah"], error_handler=Executive.ignore_error, return_exit_code=True) + self.failUnlessRaises(OSError, run_bad_command) + + def test_run_command_args_type(self): + executive = Executive() + self.assertRaises(AssertionError, executive.run_command, "echo") + self.assertRaises(AssertionError, executive.run_command, u"echo") + executive.run_command(echo.command_arguments('foo')) + executive.run_command(tuple(echo.command_arguments('foo'))) + + def test_run_command_with_unicode(self): + """Validate that it is safe to pass unicode() objects + to Executive.run* methods, and they will return unicode() + objects by default unless decode_output=False""" + unicode_tor_input = u"WebKit \u2661 Tor Arne Vestb\u00F8!" + if sys.platform == 'win32': + encoding = 'mbcs' + else: + encoding = 'utf-8' + encoded_tor = unicode_tor_input.encode(encoding) + # On Windows, we expect the unicode->mbcs->unicode roundtrip to be + # lossy. On other platforms, we expect a lossless roundtrip. + if sys.platform == 'win32': + unicode_tor_output = encoded_tor.decode(encoding) + else: + unicode_tor_output = unicode_tor_input + + executive = Executive() + + output = executive.run_command(cat.command_arguments(), input=unicode_tor_input) + self.assertEquals(output, unicode_tor_output) + + output = executive.run_command(echo.command_arguments("-n", unicode_tor_input)) + self.assertEquals(output, unicode_tor_output) + + output = executive.run_command(echo.command_arguments("-n", unicode_tor_input), decode_output=False) + self.assertEquals(output, encoded_tor) + + # Make sure that str() input also works. + output = executive.run_command(cat.command_arguments(), input=encoded_tor, decode_output=False) + self.assertEquals(output, encoded_tor) + + # FIXME: We should only have one run* method to test + output = executive.run_and_throw_if_fail(echo.command_arguments("-n", unicode_tor_input), quiet=True) + self.assertEquals(output, unicode_tor_output) + + output = executive.run_and_throw_if_fail(echo.command_arguments("-n", unicode_tor_input), quiet=True, decode_output=False) + self.assertEquals(output, encoded_tor) + + def test_kill_process(self): + executive = Executive() + process = subprocess.Popen(never_ending_command(), stdout=subprocess.PIPE) + self.assertEqual(process.poll(), None) # Process is running + executive.kill_process(process.pid) + # Note: Can't use a ternary since signal.SIGKILL is undefined for sys.platform == "win32" + if sys.platform == "win32": + expected_exit_code = 1 + else: + expected_exit_code = -signal.SIGKILL + self.assertEqual(process.wait(), expected_exit_code) + # Killing again should fail silently. + executive.kill_process(process.pid) + + def _assert_windows_image_name(self, name, expected_windows_name): + executive = Executive() + windows_name = executive._windows_image_name(name) + self.assertEqual(windows_name, expected_windows_name) + + def test_windows_image_name(self): + self._assert_windows_image_name("foo", "foo.exe") + self._assert_windows_image_name("foo.exe", "foo.exe") + self._assert_windows_image_name("foo.com", "foo.com") + # If the name looks like an extension, even if it isn't + # supposed to, we have no choice but to return the original name. + self._assert_windows_image_name("foo.baz", "foo.baz") + self._assert_windows_image_name("foo.baz.exe", "foo.baz.exe") + + def test_kill_all(self): + executive = Executive() + # We use "yes" because it loops forever. + process = subprocess.Popen(never_ending_command(), stdout=subprocess.PIPE) + self.assertEqual(process.poll(), None) # Process is running + executive.kill_all(never_ending_command()[0]) + # Note: Can't use a ternary since signal.SIGTERM is undefined for sys.platform == "win32" + if sys.platform == "cygwin": + expected_exit_code = 0 # os.kill results in exit(0) for this process. + elif sys.platform == "win32": + expected_exit_code = 1 + else: + expected_exit_code = -signal.SIGTERM + self.assertEqual(process.wait(), expected_exit_code) + # Killing again should fail silently. + executive.kill_all(never_ending_command()[0]) + + def test_check_running_pid(self): + executive = Executive() + self.assertTrue(executive.check_running_pid(os.getpid())) + # Maximum pid number on Linux is 32768 by default + self.assertFalse(executive.check_running_pid(100000)) diff --git a/Tools/Scripts/webkitpy/common/system/file_lock.py b/Tools/Scripts/webkitpy/common/system/file_lock.py new file mode 100644 index 0000000..7296958 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/file_lock.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""This class helps to lock files exclusively across processes.""" + +import logging +import os +import sys +import time + + +_log = logging.getLogger("webkitpy.common.system.file_lock") + + +class FileLock(object): + + def __init__(self, lock_file_path, max_wait_time_sec=20): + self._lock_file_path = lock_file_path + self._lock_file_descriptor = None + self._max_wait_time_sec = max_wait_time_sec + + def _create_lock(self): + if sys.platform in ('darwin', 'linux2', 'cygwin'): + import fcntl + fcntl.flock(self._lock_file_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + elif sys.platform == 'win32': + import msvcrt + msvcrt.locking(self._lock_file_descriptor, msvcrt.LK_NBLCK, 32) + + def _remove_lock(self): + if sys.platform in ('darwin', 'linux2', 'cygwin'): + import fcntl + fcntl.flock(self._lock_file_descriptor, fcntl.LOCK_UN) + elif sys.platform == 'win32': + import msvcrt + msvcrt.locking(self._lock_file_descriptor, msvcrt.LK_UNLCK, 32) + + def acquire_lock(self): + self._lock_file_descriptor = os.open(self._lock_file_path, os.O_TRUNC | os.O_CREAT) + start_time = time.time() + while True: + try: + self._create_lock() + return True + except IOError: + if time.time() - start_time > self._max_wait_time_sec: + _log.debug("File locking failed: %s" % str(sys.exc_info())) + os.close(self._lock_file_descriptor) + self._lock_file_descriptor = None + return False + + def release_lock(self): + try: + if self._lock_file_descriptor: + self._remove_lock() + os.close(self._lock_file_descriptor) + self._lock_file_descriptor = None + os.unlink(self._lock_file_path) + except (IOError, OSError): + _log.debug("Warning in release lock: %s" % str(sys.exc_info())) diff --git a/Tools/Scripts/webkitpy/common/system/file_lock_unittest.py b/Tools/Scripts/webkitpy/common/system/file_lock_unittest.py new file mode 100644 index 0000000..c5c1db3 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/file_lock_unittest.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import tempfile +import unittest + +from webkitpy.common.system.file_lock import FileLock + + +class FileLockTest(unittest.TestCase): + + def setUp(self): + self._lock_name = "TestWebKit" + str(os.getpid()) + ".lock" + self._lock_path = os.path.join(tempfile.gettempdir(), self._lock_name) + self._file_lock1 = FileLock(self._lock_path, 1) + self._file_lock2 = FileLock(self._lock_path, 1) + + def tearDown(self): + self._file_lock1.release_lock() + self._file_lock2.release_lock() + + def test_lock_lifecycle(self): + # Create the lock. + self._file_lock1.acquire_lock() + self.assertTrue(os.path.exists(self._lock_path)) + + # Try to lock again. + self.assertFalse(self._file_lock2.acquire_lock()) + + # Release the lock. + self._file_lock1.release_lock() + self.assertFalse(os.path.exists(self._lock_path)) + + def test_stuck_lock(self): + open(self._lock_path, 'w').close() + self._file_lock1.acquire_lock() + self._file_lock1.release_lock() diff --git a/Tools/Scripts/webkitpy/common/system/fileset.py b/Tools/Scripts/webkitpy/common/system/fileset.py new file mode 100644 index 0000000..22f7c4d --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/fileset.py @@ -0,0 +1,64 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import with_statement +import os + +from webkitpy.common.system.filesystem import FileSystem + + +class FileSetFileHandle(object): + """Points to a file that resides in a file set""" + def __init__(self, fileset, filename, filesystem=None): + self._filename = filename + self._fileset = fileset + self._contents = None + self._filesystem = filesystem or FileSystem() + + def __str__(self): + return "%s:%s" % (self._fileset, self._filename) + + def contents(self): + if self._contents is None: + self._contents = self._fileset.read(self._filename) + return self._contents + + def save_to(self, path, filename=None): + if filename is None: + self._fileset.extract(self._filename, path) + return + with self._filesystem.mkdtemp() as temp_dir: + self._fileset.extract(self._filename, temp_dir) + + src = self._filesystem.join(temp_dir, self._filename) + dest = self._filesystem.join(path, filename) + self._filesystem.copyfile(src, dest) + + def delete(self): + self._fileset.delete(self._filename) + + def name(self): + return self._filename + + def splitext(self): + return os.path.splitext(self.name()) diff --git a/Tools/Scripts/webkitpy/common/system/filesystem.py b/Tools/Scripts/webkitpy/common/system/filesystem.py new file mode 100644 index 0000000..05513a9 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/filesystem.py @@ -0,0 +1,259 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Wrapper object for the file system / source tree.""" + +from __future__ import with_statement + +import codecs +import errno +import exceptions +import glob +import os +import shutil +import tempfile +import time + +class FileSystem(object): + """FileSystem interface for webkitpy. + + Unless otherwise noted, all paths are allowed to be either absolute + or relative.""" + def __init__(self): + self.sep = os.sep + + def abspath(self, path): + return os.path.abspath(path) + + def basename(self, path): + """Wraps os.path.basename().""" + return os.path.basename(path) + + def copyfile(self, source, destination): + """Copies the contents of the file at the given path to the destination + path.""" + shutil.copyfile(source, destination) + + def dirname(self, path): + """Wraps os.path.dirname().""" + return os.path.dirname(path) + + def exists(self, path): + """Return whether the path exists in the filesystem.""" + return os.path.exists(path) + + def files_under(self, path, dirs_to_skip=[], file_filter=None): + """Return the list of all files under the given path in topdown order. + + Args: + dirs_to_skip: a list of directories to skip over during the + traversal (e.g., .svn, resources, etc.) + file_filter: if not None, the filter will be invoked + with the filesystem object and the dirname and basename of + each file found. The file is included in the result if the + callback returns True. + """ + def filter_all(fs, dirpath, basename): + return True + + file_filter = file_filter or filter_all + files = [] + if self.isfile(path): + if file_filter(self, self.dirname(path), self.basename(path)): + files.append(path) + return files + + if self.basename(path) in dirs_to_skip: + return [] + + for (dirpath, dirnames, filenames) in os.walk(path): + for d in dirs_to_skip: + if d in dirnames: + dirnames.remove(d) + + for filename in filenames: + if file_filter(self, dirpath, filename): + files.append(self.join(dirpath, filename)) + return files + + def glob(self, path): + """Wraps glob.glob().""" + return glob.glob(path) + + def isabs(self, path): + """Return whether the path is an absolute path.""" + return os.path.isabs(path) + + def isfile(self, path): + """Return whether the path refers to a file.""" + return os.path.isfile(path) + + def isdir(self, path): + """Return whether the path refers to a directory.""" + return os.path.isdir(path) + + def join(self, *comps): + """Return the path formed by joining the components.""" + return os.path.join(*comps) + + def listdir(self, path): + """Return the contents of the directory pointed to by path.""" + return os.listdir(path) + + def mkdtemp(self, **kwargs): + """Create and return a uniquely named directory. + + This is like tempfile.mkdtemp, but if used in a with statement + the directory will self-delete at the end of the block (if the + directory is empty; non-empty directories raise errors). The + directory can be safely deleted inside the block as well, if so + desired. + + Note that the object returned is not a string and does not support all of the string + methods. If you need a string, coerce the object to a string and go from there. + """ + class TemporaryDirectory(object): + def __init__(self, **kwargs): + self._kwargs = kwargs + self._directory_path = tempfile.mkdtemp(**self._kwargs) + + def __str__(self): + return self._directory_path + + def __enter__(self): + return self._directory_path + + def __exit__(self, type, value, traceback): + # Only self-delete if necessary. + + # FIXME: Should we delete non-empty directories? + if os.path.exists(self._directory_path): + os.rmdir(self._directory_path) + + return TemporaryDirectory(**kwargs) + + def maybe_make_directory(self, *path): + """Create the specified directory if it doesn't already exist.""" + try: + os.makedirs(self.join(*path)) + except OSError, e: + if e.errno != errno.EEXIST: + raise + + def move(self, src, dest): + shutil.move(src, dest) + + def mtime(self, path): + return os.stat(path).st_mtime + + def normpath(self, path): + """Wraps os.path.normpath().""" + return os.path.normpath(path) + + def open_binary_tempfile(self, suffix): + """Create, open, and return a binary temp file. Returns a tuple of the file and the name.""" + temp_fd, temp_name = tempfile.mkstemp(suffix) + f = os.fdopen(temp_fd, 'wb') + return f, temp_name + + def open_text_file_for_writing(self, path, append=False): + """Returns a file handle suitable for writing to.""" + mode = 'w' + if append: + mode = 'a' + return codecs.open(path, mode, 'utf8') + + def read_binary_file(self, path): + """Return the contents of the file at the given path as a byte string.""" + with file(path, 'rb') as f: + return f.read() + + def read_text_file(self, path): + """Return the contents of the file at the given path as a Unicode string. + + The file is read assuming it is a UTF-8 encoded file with no BOM.""" + with codecs.open(path, 'r', 'utf8') as f: + return f.read() + + class _WindowsError(exceptions.OSError): + """Fake exception for Linux and Mac.""" + pass + + def remove(self, path, osremove=os.remove): + """On Windows, if a process was recently killed and it held on to a + file, the OS will hold on to the file for a short while. This makes + attempts to delete the file fail. To work around that, this method + will retry for a few seconds until Windows is done with the file.""" + try: + exceptions.WindowsError + except AttributeError: + exceptions.WindowsError = FileSystem._WindowsError + + retry_timeout_sec = 3.0 + sleep_interval = 0.1 + while True: + try: + osremove(path) + return True + except exceptions.WindowsError, e: + time.sleep(sleep_interval) + retry_timeout_sec -= sleep_interval + if retry_timeout_sec < 0: + raise e + + def rmtree(self, path): + """Delete the directory rooted at path, empty or no.""" + shutil.rmtree(path, ignore_errors=True) + + def read_binary_file(self, path): + """Return the contents of the file at the given path as a byte string.""" + with file(path, 'rb') as f: + return f.read() + + def read_text_file(self, path): + """Return the contents of the file at the given path as a Unicode string. + + The file is read assuming it is a UTF-8 encoded file with no BOM.""" + with codecs.open(path, 'r', 'utf8') as f: + return f.read() + + def splitext(self, path): + """Return (dirname + os.sep + basename, '.' + ext)""" + return os.path.splitext(path) + + def write_binary_file(self, path, contents): + """Write the contents to the file at the given location.""" + with file(path, 'wb') as f: + f.write(contents) + + def write_text_file(self, path, contents): + """Write the contents to the file at the given location. + + The file is written encoded as UTF-8 with no BOM.""" + with codecs.open(path, 'w', 'utf8') as f: + f.write(contents) diff --git a/Tools/Scripts/webkitpy/common/system/filesystem_mock.py b/Tools/Scripts/webkitpy/common/system/filesystem_mock.py new file mode 100644 index 0000000..0004944 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/filesystem_mock.py @@ -0,0 +1,270 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import errno +import os +import path +import re + + +class MockFileSystem(object): + def __init__(self, files=None): + """Initializes a "mock" filesystem that can be used to completely + stub out a filesystem. + + Args: + files: a dict of filenames -> file contents. A file contents + value of None is used to indicate that the file should + not exist. + """ + self.files = files or {} + self.written_files = {} + self.sep = '/' + self.current_tmpno = 0 + + def _raise_not_found(self, path): + raise IOError(errno.ENOENT, path, os.strerror(errno.ENOENT)) + + def _split(self, path): + idx = path.rfind('/') + return (path[0:idx], path[idx + 1:]) + + def abspath(self, path): + return path + + def basename(self, path): + return self._split(path)[1] + + def copyfile(self, source, destination): + if not self.exists(source): + self._raise_not_found(source) + if self.isdir(source): + raise IOError(errno.EISDIR, source, os.strerror(errno.ISDIR)) + if self.isdir(destination): + raise IOError(errno.EISDIR, destination, os.strerror(errno.ISDIR)) + + self.files[destination] = self.files[source] + + def dirname(self, path): + return self._split(path)[0] + + def exists(self, path): + return self.isfile(path) or self.isdir(path) + + def files_under(self, path, dirs_to_skip=[], file_filter=None): + def filter_all(fs, dirpath, basename): + return True + + file_filter = file_filter or filter_all + files = [] + if self.isfile(path): + if file_filter(self, self.dirname(path), self.basename(path)): + files.append(path) + return files + + if self.basename(path) in dirs_to_skip: + return [] + + if not path.endswith('/'): + path += '/' + + dir_substrings = ['/' + d + '/' for d in dirs_to_skip] + for filename in self.files: + if not filename.startswith(path): + continue + + suffix = filename[len(path) - 1:] + if any(dir_substring in suffix for dir_substring in dir_substrings): + continue + + dirpath, basename = self._split(filename) + if file_filter(self, dirpath, basename): + files.append(filename) + + return files + + def glob(self, path): + # FIXME: This only handles a wildcard '*' at the end of the path. + # Maybe it should handle more? + if path[-1] == '*': + return [f for f in self.files if f.startswith(path[:-1])] + else: + return [f for f in self.files if f == path] + + def isabs(self, path): + return path.startswith('/') + + def isfile(self, path): + return path in self.files and self.files[path] is not None + + def isdir(self, path): + if path in self.files: + return False + if not path.endswith('/'): + path += '/' + + # We need to use a copy of the keys here in order to avoid switching + # to a different thread and potentially modifying the dict in + # mid-iteration. + files = self.files.keys()[:] + return any(f.startswith(path) for f in files) + + def join(self, *comps): + return re.sub(re.escape(os.path.sep), '/', os.path.join(*comps)) + + def listdir(self, path): + if not self.isdir(path): + raise OSError("%s is not a directory" % path) + + if not path.endswith('/'): + path += '/' + + dirs = [] + files = [] + for f in self.files: + if self.exists(f) and f.startswith(path): + remaining = f[len(path):] + if '/' in remaining: + dir = remaining[:remaining.index('/')] + if not dir in dirs: + dirs.append(dir) + else: + files.append(remaining) + return dirs + files + + def mtime(self, path): + if self.exists(path): + return 0 + self._raise_not_found(path) + + def _mktemp(self, suffix='', prefix='tmp', dir=None, **kwargs): + if dir is None: + dir = '/__im_tmp' + curno = self.current_tmpno + self.current_tmpno += 1 + return self.join(dir, "%s_%u_%s" % (prefix, curno, suffix)) + + def mkdtemp(self, **kwargs): + class TemporaryDirectory(object): + def __init__(self, fs, **kwargs): + self._kwargs = kwargs + self._filesystem = fs + self._directory_path = fs._mktemp(**kwargs) + fs.maybe_make_directory(self._directory_path) + + def __str__(self): + return self._directory_path + + def __enter__(self): + return self._directory_path + + def __exit__(self, type, value, traceback): + # Only self-delete if necessary. + + # FIXME: Should we delete non-empty directories? + if self._filesystem.exists(self._directory_path): + self._filesystem.rmtree(self._directory_path) + + return TemporaryDirectory(fs=self, **kwargs) + + def maybe_make_directory(self, *path): + # FIXME: Implement such that subsequent calls to isdir() work? + pass + + def move(self, src, dst): + if self.files[src] is None: + self._raise_not_found(src) + self.files[dst] = self.files[src] + self.files[src] = None + + def normpath(self, path): + return path + + def open_binary_tempfile(self, suffix): + path = self._mktemp(suffix) + return WritableFileObject(self, path), path + + def open_text_file_for_writing(self, path, append=False): + return WritableFileObject(self, path, append) + + def read_text_file(self, path): + return self.read_binary_file(path) + + def read_binary_file(self, path): + # Intentionally raises KeyError if we don't recognize the path. + if self.files[path] is None: + self._raise_not_found(path) + return self.files[path] + + def remove(self, path): + if self.files[path] is None: + self._raise_not_found(path) + self.files[path] = None + + def rmtree(self, path): + if not path.endswith('/'): + path += '/' + + for f in self.files: + if f.startswith(path): + self.files[f] = None + + def splitext(self, path): + idx = path.rfind('.') + if idx == -1: + idx = 0 + return (path[0:idx], path[idx:]) + + def write_text_file(self, path, contents): + return self.write_binary_file(path, contents) + + def write_binary_file(self, path, contents): + self.files[path] = contents + self.written_files[path] = contents + + +class WritableFileObject(object): + def __init__(self, fs, path, append=False, encoding=None): + self.fs = fs + self.path = path + self.closed = False + if path not in self.fs.files or not append: + self.fs.files[path] = "" + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def close(self): + self.closed = True + + def write(self, str): + self.fs.files[self.path] += str + self.fs.written_files[self.path] = self.fs.files[self.path] diff --git a/Tools/Scripts/webkitpy/common/system/filesystem_unittest.py b/Tools/Scripts/webkitpy/common/system/filesystem_unittest.py new file mode 100644 index 0000000..267ca13 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/filesystem_unittest.py @@ -0,0 +1,172 @@ +# vim: set fileencoding=utf-8 : +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE: The fileencoding comment on the first line of the file is +# important; without it, Python will choke while trying to parse the file, +# since it includes non-ASCII characters. + +from __future__ import with_statement + +import os +import stat +import sys +import tempfile +import unittest + +from filesystem import FileSystem + + +class FileSystemTest(unittest.TestCase): + def setUp(self): + self._this_dir = os.path.dirname(os.path.abspath(__file__)) + self._missing_file = os.path.join(self._this_dir, 'missing_file.py') + self._this_file = os.path.join(self._this_dir, 'filesystem_unittest.py') + + def test_exists__true(self): + fs = FileSystem() + self.assertTrue(fs.exists(self._this_file)) + + def test_exists__false(self): + fs = FileSystem() + self.assertFalse(fs.exists(self._missing_file)) + + def test_isdir__true(self): + fs = FileSystem() + self.assertTrue(fs.isdir(self._this_dir)) + + def test_isdir__false(self): + fs = FileSystem() + self.assertFalse(fs.isdir(self._this_file)) + + def test_join(self): + fs = FileSystem() + self.assertEqual(fs.join('foo', 'bar'), + os.path.join('foo', 'bar')) + + def test_listdir(self): + fs = FileSystem() + with fs.mkdtemp(prefix='filesystem_unittest_') as d: + self.assertEqual(fs.listdir(d), []) + new_file = os.path.join(d, 'foo') + fs.write_text_file(new_file, u'foo') + self.assertEqual(fs.listdir(d), ['foo']) + os.remove(new_file) + + def test_maybe_make_directory__success(self): + fs = FileSystem() + + with fs.mkdtemp(prefix='filesystem_unittest_') as base_path: + sub_path = os.path.join(base_path, "newdir") + self.assertFalse(os.path.exists(sub_path)) + self.assertFalse(fs.isdir(sub_path)) + + fs.maybe_make_directory(sub_path) + self.assertTrue(os.path.exists(sub_path)) + self.assertTrue(fs.isdir(sub_path)) + + # Make sure we can re-create it. + fs.maybe_make_directory(sub_path) + self.assertTrue(os.path.exists(sub_path)) + self.assertTrue(fs.isdir(sub_path)) + + # Clean up. + os.rmdir(sub_path) + + self.assertFalse(os.path.exists(base_path)) + self.assertFalse(fs.isdir(base_path)) + + def test_maybe_make_directory__failure(self): + # FIXME: os.chmod() doesn't work on Windows to set directories + # as readonly, so we skip this test for now. + if sys.platform in ('win32', 'cygwin'): + return + + fs = FileSystem() + with fs.mkdtemp(prefix='filesystem_unittest_') as d: + # Remove write permissions on the parent directory. + os.chmod(d, stat.S_IRUSR) + + # Now try to create a sub directory - should fail. + sub_dir = fs.join(d, 'subdir') + self.assertRaises(OSError, fs.maybe_make_directory, sub_dir) + + # Clean up in case the test failed and we did create the + # directory. + if os.path.exists(sub_dir): + os.rmdir(sub_dir) + + def test_read_and_write_file(self): + fs = FileSystem() + text_path = None + binary_path = None + + unicode_text_string = u'Ūnĭcōde̽' + hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD' + try: + text_path = tempfile.mktemp(prefix='tree_unittest_') + binary_path = tempfile.mktemp(prefix='tree_unittest_') + fs.write_text_file(text_path, unicode_text_string) + contents = fs.read_binary_file(text_path) + self.assertEqual(contents, hex_equivalent) + + fs.write_text_file(binary_path, hex_equivalent) + text_contents = fs.read_text_file(binary_path) + self.assertEqual(text_contents, unicode_text_string) + except: + if text_path: + os.remove(text_path) + if binary_path: + os.remove(binary_path) + + def test_read_binary_file__missing(self): + fs = FileSystem() + self.assertRaises(IOError, fs.read_binary_file, self._missing_file) + + def test_read_text_file__missing(self): + fs = FileSystem() + self.assertRaises(IOError, fs.read_text_file, self._missing_file) + + def test_remove_file_with_retry(self): + FileSystemTest._remove_failures = 2 + + def remove_with_exception(filename): + FileSystemTest._remove_failures -= 1 + if FileSystemTest._remove_failures >= 0: + try: + raise WindowsError + except NameError: + raise FileSystem._WindowsError + + fs = FileSystem() + self.assertTrue(fs.remove('filename', remove_with_exception)) + self.assertEquals(-1, FileSystemTest._remove_failures) + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/common/system/fileutils.py b/Tools/Scripts/webkitpy/common/system/fileutils.py new file mode 100644 index 0000000..55821f8 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/fileutils.py @@ -0,0 +1,33 @@ +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys + + +def make_stdout_binary(): + """Puts sys.stdout into binary mode (on platforms that have a distinction + between text and binary mode).""" + if sys.platform != 'win32' or not hasattr(sys.stdout, 'fileno'): + return + import msvcrt + import os + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) diff --git a/Tools/Scripts/webkitpy/common/system/logtesting.py b/Tools/Scripts/webkitpy/common/system/logtesting.py new file mode 100644 index 0000000..e361cb5 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/logtesting.py @@ -0,0 +1,258 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports the unit-testing of logging code. + +Provides support for unit-testing messages logged using the built-in +logging module. + +Inherit from the LoggingTestCase class for basic testing needs. For +more advanced needs (e.g. unit-testing methods that configure logging), +see the TestLogStream class, and perhaps also the LogTesting class. + +""" + +import logging +import unittest + + +class TestLogStream(object): + + """Represents a file-like object for unit-testing logging. + + This is meant for passing to the logging.StreamHandler constructor. + Log messages captured by instances of this object can be tested + using self.assertMessages() below. + + """ + + def __init__(self, test_case): + """Create an instance. + + Args: + test_case: A unittest.TestCase instance. + + """ + self._test_case = test_case + self.messages = [] + """A list of log messages written to the stream.""" + + # Python documentation says that any object passed to the StreamHandler + # constructor should support write() and flush(): + # + # http://docs.python.org/library/logging.html#module-logging.handlers + def write(self, message): + self.messages.append(message) + + def flush(self): + pass + + def assertMessages(self, messages): + """Assert that the given messages match the logged messages. + + messages: A list of log message strings. + + """ + self._test_case.assertEquals(messages, self.messages) + + +class LogTesting(object): + + """Supports end-to-end unit-testing of log messages. + + Sample usage: + + class SampleTest(unittest.TestCase): + + def setUp(self): + self._log = LogTesting.setUp(self) # Turn logging on. + + def tearDown(self): + self._log.tearDown() # Turn off and reset logging. + + def test_logging_in_some_method(self): + call_some_method() # Contains calls to _log.info(), etc. + + # Check the resulting log messages. + self._log.assertMessages(["INFO: expected message #1", + "WARNING: expected message #2"]) + + """ + + def __init__(self, test_stream, handler): + """Create an instance. + + This method should never be called directly. Instances should + instead be created using the static setUp() method. + + Args: + test_stream: A TestLogStream instance. + handler: The handler added to the logger. + + """ + self._test_stream = test_stream + self._handler = handler + + @staticmethod + def _getLogger(): + """Return the logger being tested.""" + # It is possible we might want to return something other than + # the root logger in some special situation. For now, the + # root logger seems to suffice. + return logging.getLogger() + + @staticmethod + def setUp(test_case, logging_level=logging.INFO): + """Configure logging for unit testing. + + Configures the root logger to log to a testing log stream. + Only messages logged at or above the given level are logged + to the stream. Messages logged to the stream are formatted + in the following way, for example-- + + "INFO: This is a test log message." + + This method should normally be called in the setUp() method + of a unittest.TestCase. See the docstring of this class + for more details. + + Returns: + A LogTesting instance. + + Args: + test_case: A unittest.TestCase instance. + logging_level: An integer logging level that is the minimum level + of log messages you would like to test. + + """ + stream = TestLogStream(test_case) + handler = logging.StreamHandler(stream) + handler.setLevel(logging_level) + formatter = logging.Formatter("%(levelname)s: %(message)s") + handler.setFormatter(formatter) + + # Notice that we only change the root logger by adding a handler + # to it. In particular, we do not reset its level using + # logger.setLevel(). This ensures that we have not interfered + # with how the code being tested may have configured the root + # logger. + logger = LogTesting._getLogger() + logger.addHandler(handler) + + return LogTesting(stream, handler) + + def tearDown(self): + """Assert there are no remaining log messages, and reset logging. + + This method asserts that there are no more messages in the array of + log messages, and then restores logging to its original state. + This method should normally be called in the tearDown() method of a + unittest.TestCase. See the docstring of this class for more details. + + """ + self.assertMessages([]) + logger = LogTesting._getLogger() + logger.removeHandler(self._handler) + + def messages(self): + """Return the current list of log messages.""" + return self._test_stream.messages + + # FIXME: Add a clearMessages() method for cases where the caller + # deliberately doesn't want to assert every message. + + # We clear the log messages after asserting since they are no longer + # needed after asserting. This serves two purposes: (1) it simplifies + # the calling code when we want to check multiple logging calls in a + # single test method, and (2) it lets us check in the tearDown() method + # that there are no remaining log messages to be asserted. + # + # The latter ensures that no extra log messages are getting logged that + # the caller might not be aware of or may have forgotten to check for. + # This gets us a bit more mileage out of our tests without writing any + # additional code. + def assertMessages(self, messages): + """Assert the current array of log messages, and clear its contents. + + Args: + messages: A list of log message strings. + + """ + try: + self._test_stream.assertMessages(messages) + finally: + # We want to clear the array of messages even in the case of + # an Exception (e.g. an AssertionError). Otherwise, another + # AssertionError can occur in the tearDown() because the + # array might not have gotten emptied. + self._test_stream.messages = [] + + +# This class needs to inherit from unittest.TestCase. Otherwise, the +# setUp() and tearDown() methods will not get fired for test case classes +# that inherit from this class -- even if the class inherits from *both* +# unittest.TestCase and LoggingTestCase. +# +# FIXME: Rename this class to LoggingTestCaseBase to be sure that +# the unittest module does not interpret this class as a unittest +# test case itself. +class LoggingTestCase(unittest.TestCase): + + """Supports end-to-end unit-testing of log messages. + + Sample usage: + + class SampleTest(LoggingTestCase): + + def test_logging_in_some_method(self): + call_some_method() # Contains calls to _log.info(), etc. + + # Check the resulting log messages. + self.assertLog(["INFO: expected message #1", + "WARNING: expected message #2"]) + + """ + + def setUp(self): + self._log = LogTesting.setUp(self) + + def tearDown(self): + self._log.tearDown() + + def logMessages(self): + """Return the current list of log messages.""" + return self._log.messages() + + # FIXME: Add a clearMessages() method for cases where the caller + # deliberately doesn't want to assert every message. + + # See the code comments preceding LogTesting.assertMessages() for + # an explanation of why we clear the array of messages after + # asserting its contents. + def assertLog(self, messages): + """Assert the current array of log messages, and clear its contents. + + Args: + messages: A list of log message strings. + + """ + self._log.assertMessages(messages) diff --git a/Tools/Scripts/webkitpy/common/system/logutils.py b/Tools/Scripts/webkitpy/common/system/logutils.py new file mode 100644 index 0000000..cd4e60f --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/logutils.py @@ -0,0 +1,207 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports webkitpy logging.""" + +# FIXME: Move this file to webkitpy/python24 since logging needs to +# be configured prior to running version-checking code. + +import logging +import os +import sys + +import webkitpy + + +_log = logging.getLogger(__name__) + +# We set these directory paths lazily in get_logger() below. +_scripts_dir = "" +"""The normalized, absolute path to the ...Scripts directory.""" + +_webkitpy_dir = "" +"""The normalized, absolute path to the ...Scripts/webkitpy directory.""" + + +def _normalize_path(path): + """Return the given path normalized. + + Converts a path to an absolute path, removes any trailing slashes, + removes any extension, and lower-cases it. + + """ + path = os.path.abspath(path) + path = os.path.normpath(path) + path = os.path.splitext(path)[0] # Remove the extension, if any. + path = path.lower() + + return path + + +# Observe that the implementation of this function does not require +# the use of any hard-coded strings like "webkitpy", etc. +# +# The main benefit this function has over using-- +# +# _log = logging.getLogger(__name__) +# +# is that get_logger() returns the same value even if __name__ is +# "__main__" -- i.e. even if the module is the script being executed +# from the command-line. +def get_logger(path): + """Return a logging.logger for the given path. + + Returns: + A logger whose name is the name of the module corresponding to + the given path. If the module is in webkitpy, the name is + the fully-qualified dotted module name beginning with webkitpy.... + Otherwise, the name is the base name of the module (i.e. without + any dotted module name prefix). + + Args: + path: The path of the module. Normally, this parameter should be + the __file__ variable of the module. + + Sample usage: + + import webkitpy.common.system.logutils as logutils + + _log = logutils.get_logger(__file__) + + """ + # Since we assign to _scripts_dir and _webkitpy_dir in this function, + # we need to declare them global. + global _scripts_dir + global _webkitpy_dir + + path = _normalize_path(path) + + # Lazily evaluate _webkitpy_dir and _scripts_dir. + if not _scripts_dir: + # The normalized, absolute path to ...Scripts/webkitpy/__init__. + webkitpy_path = _normalize_path(webkitpy.__file__) + + _webkitpy_dir = os.path.split(webkitpy_path)[0] + _scripts_dir = os.path.split(_webkitpy_dir)[0] + + if path.startswith(_webkitpy_dir): + # Remove the initial Scripts directory portion, so the path + # starts with /webkitpy, for example "/webkitpy/init/logutils". + path = path[len(_scripts_dir):] + + parts = [] + while True: + (path, tail) = os.path.split(path) + if not tail: + break + parts.insert(0, tail) + + logger_name = ".".join(parts) # For example, webkitpy.common.system.logutils. + else: + # The path is outside of webkitpy. Default to the basename + # without the extension. + basename = os.path.basename(path) + logger_name = os.path.splitext(basename)[0] + + return logging.getLogger(logger_name) + + +def _default_handlers(stream): + """Return a list of the default logging handlers to use. + + Args: + stream: See the configure_logging() docstring. + + """ + # Create the filter. + def should_log(record): + """Return whether a logging.LogRecord should be logged.""" + # FIXME: Enable the logging of autoinstall messages once + # autoinstall is adjusted. Currently, autoinstall logs + # INFO messages when importing already-downloaded packages, + # which is too verbose. + if record.name.startswith("webkitpy.thirdparty.autoinstall"): + return False + return True + + logging_filter = logging.Filter() + logging_filter.filter = should_log + + # Create the handler. + handler = logging.StreamHandler(stream) + formatter = logging.Formatter("%(name)s: [%(levelname)s] %(message)s") + handler.setFormatter(formatter) + handler.addFilter(logging_filter) + + return [handler] + + +def configure_logging(logging_level=None, logger=None, stream=None, + handlers=None): + """Configure logging for standard purposes. + + Returns: + A list of references to the logging handlers added to the root + logger. This allows the caller to later remove the handlers + using logger.removeHandler. This is useful primarily during unit + testing where the caller may want to configure logging temporarily + and then undo the configuring. + + Args: + logging_level: The minimum logging level to log. Defaults to + logging.INFO. + logger: A logging.logger instance to configure. This parameter + should be used only in unit tests. Defaults to the + root logger. + stream: A file-like object to which to log used in creating the default + handlers. The stream must define an "encoding" data attribute, + or else logging raises an error. Defaults to sys.stderr. + handlers: A list of logging.Handler instances to add to the logger + being configured. If this parameter is provided, then the + stream parameter is not used. + + """ + # If the stream does not define an "encoding" data attribute, the + # logging module can throw an error like the following: + # + # Traceback (most recent call last): + # File "/System/Library/Frameworks/Python.framework/Versions/2.6/... + # lib/python2.6/logging/__init__.py", line 761, in emit + # self.stream.write(fs % msg.encode(self.stream.encoding)) + # LookupError: unknown encoding: unknown + if logging_level is None: + logging_level = logging.INFO + if logger is None: + logger = logging.getLogger() + if stream is None: + stream = sys.stderr + if handlers is None: + handlers = _default_handlers(stream) + + logger.setLevel(logging_level) + + for handler in handlers: + logger.addHandler(handler) + + _log.debug("Debug logging enabled.") + + return handlers diff --git a/Tools/Scripts/webkitpy/common/system/logutils_unittest.py b/Tools/Scripts/webkitpy/common/system/logutils_unittest.py new file mode 100644 index 0000000..b77c284 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/logutils_unittest.py @@ -0,0 +1,142 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for logutils.py.""" + +import logging +import os +import unittest + +from webkitpy.common.system.logtesting import LogTesting +from webkitpy.common.system.logtesting import TestLogStream +import webkitpy.common.system.logutils as logutils + + +class GetLoggerTest(unittest.TestCase): + + """Tests get_logger().""" + + def test_get_logger_in_webkitpy(self): + logger = logutils.get_logger(__file__) + self.assertEquals(logger.name, "webkitpy.common.system.logutils_unittest") + + def test_get_logger_not_in_webkitpy(self): + # Temporarily change the working directory so that we + # can test get_logger() for a path outside of webkitpy. + working_directory = os.getcwd() + root_dir = "/" + os.chdir(root_dir) + + logger = logutils.get_logger("/Tools/Scripts/test-webkitpy") + self.assertEquals(logger.name, "test-webkitpy") + + logger = logutils.get_logger("/Tools/Scripts/test-webkitpy.py") + self.assertEquals(logger.name, "test-webkitpy") + + os.chdir(working_directory) + + +class ConfigureLoggingTestBase(unittest.TestCase): + + """Base class for configure_logging() unit tests.""" + + def _logging_level(self): + raise Exception("Not implemented.") + + def setUp(self): + log_stream = TestLogStream(self) + + # Use a logger other than the root logger or one prefixed with + # "webkitpy." so as not to conflict with test-webkitpy logging. + logger = logging.getLogger("unittest") + + # Configure the test logger not to pass messages along to the + # root logger. This prevents test messages from being + # propagated to loggers used by test-webkitpy logging (e.g. + # the root logger). + logger.propagate = False + + logging_level = self._logging_level() + self._handlers = logutils.configure_logging(logging_level=logging_level, + logger=logger, + stream=log_stream) + self._log = logger + self._log_stream = log_stream + + def tearDown(self): + """Reset logging to its original state. + + This method ensures that the logging configuration set up + for a unit test does not affect logging in other unit tests. + + """ + logger = self._log + for handler in self._handlers: + logger.removeHandler(handler) + + def _assert_log_messages(self, messages): + """Assert that the logged messages equal the given messages.""" + self._log_stream.assertMessages(messages) + + +class ConfigureLoggingTest(ConfigureLoggingTestBase): + + """Tests configure_logging() with the default logging level.""" + + def _logging_level(self): + return None + + def test_info_message(self): + self._log.info("test message") + self._assert_log_messages(["unittest: [INFO] test message\n"]) + + def test_below_threshold_message(self): + # We test the boundary case of a logging level equal to 19. + # In practice, we will probably only be calling log.debug(), + # which corresponds to a logging level of 10. + level = logging.INFO - 1 # Equals 19. + self._log.log(level, "test message") + self._assert_log_messages([]) + + def test_two_messages(self): + self._log.info("message1") + self._log.info("message2") + self._assert_log_messages(["unittest: [INFO] message1\n", + "unittest: [INFO] message2\n"]) + + +class ConfigureLoggingCustomLevelTest(ConfigureLoggingTestBase): + + """Tests configure_logging() with a custom logging level.""" + + _level = 36 + + def _logging_level(self): + return self._level + + def test_logged_message(self): + self._log.log(self._level, "test message") + self._assert_log_messages(["unittest: [Level 36] test message\n"]) + + def test_below_threshold_message(self): + self._log.log(self._level - 1, "test message") + self._assert_log_messages([]) diff --git a/Tools/Scripts/webkitpy/common/system/ospath.py b/Tools/Scripts/webkitpy/common/system/ospath.py new file mode 100644 index 0000000..aed7a3d --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/ospath.py @@ -0,0 +1,83 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains a substitute for Python 2.6's os.path.relpath().""" + +import os + + +# This function is a replacement for os.path.relpath(), which is only +# available in Python 2.6: +# +# http://docs.python.org/library/os.path.html#os.path.relpath +# +# It should behave essentially the same as os.path.relpath(), except for +# returning None on paths not contained in abs_start_path. +def relpath(path, start_path, os_path_abspath=None): + """Return a path relative to the given start path, or None. + + Returns None if the path is not contained in the directory start_path. + + Args: + path: An absolute or relative path to convert to a relative path. + start_path: The path relative to which the given path should be + converted. + os_path_abspath: A replacement function for unit testing. This + function should strip trailing slashes just like + os.path.abspath(). Defaults to os.path.abspath. + + """ + if os_path_abspath is None: + os_path_abspath = os.path.abspath + + # Since os_path_abspath() calls os.path.normpath()-- + # + # (see http://docs.python.org/library/os.path.html#os.path.abspath ) + # + # it also removes trailing slashes and converts forward and backward + # slashes to the preferred slash os.sep. + start_path = os_path_abspath(start_path) + path = os_path_abspath(path) + + if not path.lower().startswith(start_path.lower()): + # Then path is outside the directory given by start_path. + return None + + rel_path = path[len(start_path):] + + if not rel_path: + # Then the paths are the same. + pass + elif rel_path[0] == os.sep: + # It is probably sufficient to remove just the first character + # since os.path.normpath() collapses separators, but we use + # lstrip() just to be sure. + rel_path = rel_path.lstrip(os.sep) + else: + # We are in the case typified by the following example: + # + # start_path = "/tmp/foo" + # path = "/tmp/foobar" + # rel_path = "bar" + return None + + return rel_path diff --git a/Tools/Scripts/webkitpy/common/system/ospath_unittest.py b/Tools/Scripts/webkitpy/common/system/ospath_unittest.py new file mode 100644 index 0000000..d84c2c6 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/ospath_unittest.py @@ -0,0 +1,62 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for ospath.py.""" + +import os +import unittest + +from webkitpy.common.system.ospath import relpath + + +# Make sure the tests in this class are platform independent. +class RelPathTest(unittest.TestCase): + + """Tests relpath().""" + + os_path_abspath = lambda self, path: path + + def _rel_path(self, path, abs_start_path): + return relpath(path, abs_start_path, self.os_path_abspath) + + def test_same_path(self): + rel_path = self._rel_path("WebKit", "WebKit") + self.assertEquals(rel_path, "") + + def test_long_rel_path(self): + start_path = "WebKit" + expected_rel_path = os.path.join("test", "Foo.txt") + path = os.path.join(start_path, expected_rel_path) + + rel_path = self._rel_path(path, start_path) + self.assertEquals(expected_rel_path, rel_path) + + def test_none_rel_path(self): + """Test _rel_path() with None return value.""" + start_path = "WebKit" + path = os.path.join("other_dir", "foo.txt") + + rel_path = self._rel_path(path, start_path) + self.assertTrue(rel_path is None) + + rel_path = self._rel_path("Tools", "WebKit") + self.assertTrue(rel_path is None) diff --git a/Tools/Scripts/webkitpy/common/system/outputcapture.py b/Tools/Scripts/webkitpy/common/system/outputcapture.py new file mode 100644 index 0000000..45e0e3f --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/outputcapture.py @@ -0,0 +1,86 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Class for unittest support. Used for capturing stderr/stdout. + +import sys +import unittest +from StringIO import StringIO + +class OutputCapture(object): + def __init__(self): + self.saved_outputs = dict() + + def _capture_output_with_name(self, output_name): + self.saved_outputs[output_name] = getattr(sys, output_name) + captured_output = StringIO() + setattr(sys, output_name, captured_output) + return captured_output + + def _restore_output_with_name(self, output_name): + captured_output = getattr(sys, output_name).getvalue() + setattr(sys, output_name, self.saved_outputs[output_name]) + del self.saved_outputs[output_name] + return captured_output + + def capture_output(self): + return (self._capture_output_with_name("stdout"), self._capture_output_with_name("stderr")) + + def restore_output(self): + return (self._restore_output_with_name("stdout"), self._restore_output_with_name("stderr")) + + def assert_outputs(self, testcase, function, args=[], kwargs={}, expected_stdout="", expected_stderr="", expected_exception=None): + self.capture_output() + if expected_exception: + return_value = testcase.assertRaises(expected_exception, function, *args, **kwargs) + else: + return_value = function(*args, **kwargs) + (stdout_string, stderr_string) = self.restore_output() + testcase.assertEqual(stdout_string, expected_stdout) + testcase.assertEqual(stderr_string, expected_stderr) + # This is a little strange, but I don't know where else to return this information. + return return_value + + +class OutputCaptureTestCaseBase(unittest.TestCase): + def setUp(self): + unittest.TestCase.setUp(self) + self.output_capture = OutputCapture() + (self.__captured_stdout, self.__captured_stderr) = self.output_capture.capture_output() + + def tearDown(self): + del self.__captured_stdout + del self.__captured_stderr + self.output_capture.restore_output() + unittest.TestCase.tearDown(self) + + def assertStdout(self, expected_stdout): + self.assertEquals(expected_stdout, self.__captured_stdout.getvalue()) + + def assertStderr(self, expected_stderr): + self.assertEquals(expected_stderr, self.__captured_stderr.getvalue()) diff --git a/Tools/Scripts/webkitpy/common/system/path.py b/Tools/Scripts/webkitpy/common/system/path.py new file mode 100644 index 0000000..09787d7 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/path.py @@ -0,0 +1,138 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""generic routines to convert platform-specific paths to URIs.""" +from __future__ import with_statement + +import atexit +import subprocess +import sys +import threading +import urllib + + +def abspath_to_uri(path, platform=None): + """Converts a platform-specific absolute path to a file: URL.""" + if platform is None: + platform = sys.platform + return "file:" + _escape(_convert_path(path, platform)) + + +def cygpath(path): + """Converts an absolute cygwin path to an absolute Windows path.""" + return _CygPath.convert_using_singleton(path) + + +# Note that this object is not threadsafe and must only be called +# from multiple threads under protection of a lock (as is done in cygpath()) +class _CygPath(object): + """Manages a long-running 'cygpath' process for file conversion.""" + _lock = None + _singleton = None + + @staticmethod + def stop_cygpath_subprocess(): + if not _CygPath._lock: + return + + with _CygPath._lock: + if _CygPath._singleton: + _CygPath._singleton.stop() + + @staticmethod + def convert_using_singleton(path): + if not _CygPath._lock: + _CygPath._lock = threading.Lock() + + with _CygPath._lock: + if not _CygPath._singleton: + _CygPath._singleton = _CygPath() + # Make sure the cygpath subprocess always gets shutdown cleanly. + atexit.register(_CygPath.stop_cygpath_subprocess) + + return _CygPath._singleton.convert(path) + + def __init__(self): + self._child_process = None + + def start(self): + assert(self._child_process is None) + args = ['cygpath', '-f', '-', '-wa'] + self._child_process = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + + def is_running(self): + if not self._child_process: + return False + return self._child_process.returncode is None + + def stop(self): + if self._child_process: + self._child_process.stdin.close() + self._child_process.wait() + self._child_process = None + + def convert(self, path): + if not self.is_running(): + self.start() + self._child_process.stdin.write("%s\r\n" % path) + self._child_process.stdin.flush() + windows_path = self._child_process.stdout.readline().rstrip() + # Some versions of cygpath use lowercase drive letters while others + # use uppercase. We always convert to uppercase for consistency. + windows_path = '%s%s' % (windows_path[0].upper(), windows_path[1:]) + return windows_path + + +def _escape(path): + """Handle any characters in the path that should be escaped.""" + # FIXME: web browsers don't appear to blindly quote every character + # when converting filenames to files. Instead of using urllib's default + # rules, we allow a small list of other characters through un-escaped. + # It's unclear if this is the best possible solution. + return urllib.quote(path, safe='/+:') + + +def _convert_path(path, platform): + """Handles any os-specific path separators, mappings, etc.""" + if platform == 'win32': + return _winpath_to_uri(path) + if platform == 'cygwin': + return _winpath_to_uri(cygpath(path)) + return _unixypath_to_uri(path) + + +def _winpath_to_uri(path): + """Converts a window absolute path to a file: URL.""" + return "///" + path.replace("\\", "/") + + +def _unixypath_to_uri(path): + """Converts a unix-style path to a file: URL.""" + return "//" + path diff --git a/Tools/Scripts/webkitpy/common/system/path_unittest.py b/Tools/Scripts/webkitpy/common/system/path_unittest.py new file mode 100644 index 0000000..4dbd38a --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/path_unittest.py @@ -0,0 +1,105 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import sys + +import path + +class AbspathTest(unittest.TestCase): + def assertMatch(self, test_path, expected_uri, + platform=None): + if platform == 'cygwin' and sys.platform != 'cygwin': + return + self.assertEqual(path.abspath_to_uri(test_path, platform=platform), + expected_uri) + + def test_abspath_to_uri_cygwin(self): + if sys.platform != 'cygwin': + return + + self.assertMatch('/cygdrive/c/foo/bar.html', + 'file:///C:/foo/bar.html', + platform='cygwin') + self.assertEqual(path.abspath_to_uri('/cygdrive/c/foo/bar.html', + platform='cygwin'), + 'file:///C:/foo/bar.html') + + def test_abspath_to_uri_darwin(self): + self.assertMatch('/foo/bar.html', + 'file:///foo/bar.html', + platform='darwin') + self.assertEqual(path.abspath_to_uri("/foo/bar.html", + platform='darwin'), + "file:///foo/bar.html") + + def test_abspath_to_uri_linux2(self): + self.assertMatch('/foo/bar.html', + 'file:///foo/bar.html', + platform='darwin') + self.assertEqual(path.abspath_to_uri("/foo/bar.html", + platform='linux2'), + "file:///foo/bar.html") + + def test_abspath_to_uri_win(self): + self.assertMatch('c:\\foo\\bar.html', + 'file:///c:/foo/bar.html', + platform='win32') + self.assertEqual(path.abspath_to_uri("c:\\foo\\bar.html", + platform='win32'), + "file:///c:/foo/bar.html") + + def test_abspath_to_uri_escaping(self): + self.assertMatch('/foo/bar + baz%?.html', + 'file:///foo/bar%20+%20baz%25%3F.html', + platform='darwin') + self.assertMatch('/foo/bar + baz%?.html', + 'file:///foo/bar%20+%20baz%25%3F.html', + platform='linux2') + + # Note that you can't have '?' in a filename on windows. + self.assertMatch('/cygdrive/c/foo/bar + baz%.html', + 'file:///C:/foo/bar%20+%20baz%25.html', + platform='cygwin') + + def test_stop_cygpath_subprocess(self): + if sys.platform != 'cygwin': + return + + # Call cygpath to ensure the subprocess is running. + path.cygpath("/cygdrive/c/foo.txt") + self.assertTrue(path._CygPath._singleton.is_running()) + + # Stop it. + path._CygPath.stop_cygpath_subprocess() + + # Ensure that it is stopped. + self.assertFalse(path._CygPath._singleton.is_running()) + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/common/system/platforminfo.py b/Tools/Scripts/webkitpy/common/system/platforminfo.py new file mode 100644 index 0000000..cc370ba --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/platforminfo.py @@ -0,0 +1,43 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import platform + + +# We use this instead of calls to platform directly to allow mocking. +class PlatformInfo(object): + + def display_name(self): + # platform.platform() returns Darwin information for Mac, which is just confusing. + if platform.system() == "Darwin": + return "Mac OS X %s" % platform.mac_ver()[0] + + # Returns strings like: + # Linux-2.6.18-194.3.1.el5-i686-with-redhat-5.5-Final + # Windows-2008ServerR2-6.1.7600 + return platform.platform() diff --git a/Tools/Scripts/webkitpy/common/system/user.py b/Tools/Scripts/webkitpy/common/system/user.py new file mode 100644 index 0000000..b79536c --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/user.py @@ -0,0 +1,143 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +import os +import re +import shlex +import subprocess +import sys +import webbrowser + + +_log = logging.getLogger("webkitpy.common.system.user") + + +try: + import readline +except ImportError: + if sys.platform != "win32": + # There is no readline module for win32, not much to do except cry. + _log.warn("Unable to import readline.") + # FIXME: We could give instructions for non-mac platforms. + # Lack of readline results in a very bad user experiance. + if sys.platform == "darwin": + _log.warn("If you're using MacPorts, try running:") + _log.warn(" sudo port install py25-readline") + + +class User(object): + DEFAULT_NO = 'n' + DEFAULT_YES = 'y' + + # FIXME: These are @classmethods because bugzilla.py doesn't have a Tool object (thus no User instance). + @classmethod + def prompt(cls, message, repeat=1, raw_input=raw_input): + response = None + while (repeat and not response): + repeat -= 1 + response = raw_input(message) + return response + + @classmethod + def prompt_with_list(cls, list_title, list_items, can_choose_multiple=False, raw_input=raw_input): + print list_title + i = 0 + for item in list_items: + i += 1 + print "%2d. %s" % (i, item) + + # Loop until we get valid input + while True: + if can_choose_multiple: + response = cls.prompt("Enter one or more numbers (comma-separated), or \"all\": ", raw_input=raw_input) + if not response.strip() or response == "all": + return list_items + try: + indices = [int(r) - 1 for r in re.split("\s*,\s*", response)] + except ValueError, err: + continue + return [list_items[i] for i in indices] + else: + try: + result = int(cls.prompt("Enter a number: ", raw_input=raw_input)) - 1 + except ValueError, err: + continue + return list_items[result] + + def edit(self, files): + editor = os.environ.get("EDITOR") or "vi" + args = shlex.split(editor) + # Note: Not thread safe: http://bugs.python.org/issue2320 + subprocess.call(args + files) + + def _warn_if_application_is_xcode(self, edit_application): + if "Xcode" in edit_application: + print "Instead of using Xcode.app, consider using EDITOR=\"xed --wait\"." + + def edit_changelog(self, files): + edit_application = os.environ.get("CHANGE_LOG_EDIT_APPLICATION") + if edit_application and sys.platform == "darwin": + # On Mac we support editing ChangeLogs using an application. + args = shlex.split(edit_application) + print "Using editor in the CHANGE_LOG_EDIT_APPLICATION environment variable." + print "Please quit the editor application when done editing." + self._warn_if_application_is_xcode(edit_application) + subprocess.call(["open", "-W", "-n", "-a"] + args + files) + return + self.edit(files) + + def page(self, message): + pager = os.environ.get("PAGER") or "less" + try: + # Note: Not thread safe: http://bugs.python.org/issue2320 + child_process = subprocess.Popen([pager], stdin=subprocess.PIPE) + child_process.communicate(input=message) + except IOError, e: + pass + + def confirm(self, message=None, default=DEFAULT_YES, raw_input=raw_input): + if not message: + message = "Continue?" + choice = {'y': 'Y/n', 'n': 'y/N'}[default] + response = raw_input("%s [%s]: " % (message, choice)) + if not response: + response = default + return response.lower() == 'y' + + def can_open_url(self): + try: + webbrowser.get() + return True + except webbrowser.Error, e: + return False + + def open_url(self, url): + if not self.can_open_url(): + _log.warn("Failed to open %s" % url) + webbrowser.open(url) diff --git a/Tools/Scripts/webkitpy/common/system/user_unittest.py b/Tools/Scripts/webkitpy/common/system/user_unittest.py new file mode 100644 index 0000000..7ec9b34 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/user_unittest.py @@ -0,0 +1,109 @@ +# Copyright (C) 2010 Research in Motion Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Research in Motion Ltd. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.common.system.user import User + +class UserTest(unittest.TestCase): + + example_user_response = "example user response" + + def test_prompt_repeat(self): + self.repeatsRemaining = 2 + def mock_raw_input(message): + self.repeatsRemaining -= 1 + if not self.repeatsRemaining: + return UserTest.example_user_response + return None + self.assertEqual(User.prompt("input", repeat=self.repeatsRemaining, raw_input=mock_raw_input), UserTest.example_user_response) + + def test_prompt_when_exceeded_repeats(self): + self.repeatsRemaining = 2 + def mock_raw_input(message): + self.repeatsRemaining -= 1 + return None + self.assertEqual(User.prompt("input", repeat=self.repeatsRemaining, raw_input=mock_raw_input), None) + + def test_prompt_with_list(self): + def run_prompt_test(inputs, expected_result, can_choose_multiple=False): + def mock_raw_input(message): + return inputs.pop(0) + output_capture = OutputCapture() + actual_result = output_capture.assert_outputs( + self, + User.prompt_with_list, + args=["title", ["foo", "bar"]], + kwargs={"can_choose_multiple": can_choose_multiple, "raw_input": mock_raw_input}, + expected_stdout="title\n 1. foo\n 2. bar\n") + self.assertEqual(actual_result, expected_result) + self.assertEqual(len(inputs), 0) + + run_prompt_test(["1"], "foo") + run_prompt_test(["badinput", "2"], "bar") + + run_prompt_test(["1,2"], ["foo", "bar"], can_choose_multiple=True) + run_prompt_test([" 1, 2 "], ["foo", "bar"], can_choose_multiple=True) + run_prompt_test(["all"], ["foo", "bar"], can_choose_multiple=True) + run_prompt_test([""], ["foo", "bar"], can_choose_multiple=True) + run_prompt_test([" "], ["foo", "bar"], can_choose_multiple=True) + run_prompt_test(["badinput", "all"], ["foo", "bar"], can_choose_multiple=True) + + def test_confirm(self): + test_cases = ( + (("Continue? [Y/n]: ", True), (User.DEFAULT_YES, 'y')), + (("Continue? [Y/n]: ", False), (User.DEFAULT_YES, 'n')), + (("Continue? [Y/n]: ", True), (User.DEFAULT_YES, '')), + (("Continue? [Y/n]: ", False), (User.DEFAULT_YES, 'q')), + (("Continue? [y/N]: ", True), (User.DEFAULT_NO, 'y')), + (("Continue? [y/N]: ", False), (User.DEFAULT_NO, 'n')), + (("Continue? [y/N]: ", False), (User.DEFAULT_NO, '')), + (("Continue? [y/N]: ", False), (User.DEFAULT_NO, 'q')), + ) + for test_case in test_cases: + expected, inputs = test_case + + def mock_raw_input(message): + self.assertEquals(expected[0], message) + return inputs[1] + + result = User().confirm(default=inputs[0], + raw_input=mock_raw_input) + self.assertEquals(expected[1], result) + + def test_warn_if_application_is_xcode(self): + output = OutputCapture() + user = User() + output.assert_outputs(self, user._warn_if_application_is_xcode, ["TextMate"]) + output.assert_outputs(self, user._warn_if_application_is_xcode, ["/Applications/TextMate.app"]) + output.assert_outputs(self, user._warn_if_application_is_xcode, ["XCode"]) # case sensitive matching + + xcode_warning = "Instead of using Xcode.app, consider using EDITOR=\"xed --wait\".\n" + output.assert_outputs(self, user._warn_if_application_is_xcode, ["Xcode"], expected_stdout=xcode_warning) + output.assert_outputs(self, user._warn_if_application_is_xcode, ["/Developer/Applications/Xcode.app"], expected_stdout=xcode_warning) diff --git a/Tools/Scripts/webkitpy/common/system/workspace.py b/Tools/Scripts/webkitpy/common/system/workspace.py new file mode 100644 index 0000000..afb0009 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/workspace.py @@ -0,0 +1,61 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# A home for file logic which should sit above FileSystem, but +# below more complicated objects. + +import zipfile + +class Workspace(object): + def __init__(self, filesystem, executive): + self._filesystem = filesystem + self._executive = executive # FIXME: Remove if create_zip is moved to python. + + def find_unused_filename(self, directory, name, extension, search_limit=100): + for count in range(search_limit): + if count: + target_name = "%s-%s.%s" % (name, count, extension) + else: + target_name = "%s.%s" % (name, extension) + target_path = self._filesystem.join(directory, target_name) + if not self._filesystem.exists(target_path): + return target_path + # If we can't find an unused name in search_limit tries, just give up. + return None + + def create_zip(self, zip_path, source_path, zip_class=zipfile.ZipFile): + # It's possible to create zips with Python: + # zip_file = ZipFile(zip_path, 'w') + # for root, dirs, files in os.walk(source_path): + # for path in files: + # absolute_path = os.path.join(root, path) + # zip_file.write(os.path.relpath(path, source_path)) + # However, getting the paths, encoding and compression correct could be non-trivial. + # So, for now we depend on the environment having "zip" installed (likely fails on Win32) + self._executive.run_command(['zip', '-r', zip_path, source_path]) + return zip_class(zip_path) diff --git a/Tools/Scripts/webkitpy/common/system/workspace_unittest.py b/Tools/Scripts/webkitpy/common/system/workspace_unittest.py new file mode 100644 index 0000000..6be7664 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/workspace_unittest.py @@ -0,0 +1,58 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.system.filesystem_mock import MockFileSystem +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.common.system.workspace import Workspace +from webkitpy.tool.mocktool import MockExecutive + + +class WorkspaceTest(unittest.TestCase): + + def test_find_unused_filename(self): + filesystem = MockFileSystem({ + "dir/foo.jpg": "", + "dir/foo-1.jpg": "", + "dir/foo-2.jpg": "", + }) + workspace = Workspace(filesystem, None) + self.assertEqual(workspace.find_unused_filename("bar", "bar", "bar"), "bar/bar.bar") + self.assertEqual(workspace.find_unused_filename("dir", "foo", "jpg", search_limit=1), None) + self.assertEqual(workspace.find_unused_filename("dir", "foo", "jpg", search_limit=2), None) + self.assertEqual(workspace.find_unused_filename("dir", "foo", "jpg"), "dir/foo-3.jpg") + + def test_create_zip(self): + workspace = Workspace(None, MockExecutive(should_log=True)) + expected_stderr = "MOCK run_command: ['zip', '-r', '/zip/path', '/source/path']\n" + class MockZipFile(object): + def __init__(self, path): + self.filename = path + archive = OutputCapture().assert_outputs(self, workspace.create_zip, ["/zip/path", "/source/path", MockZipFile], expected_stderr=expected_stderr) + self.assertEqual(archive.filename, "/zip/path") diff --git a/Tools/Scripts/webkitpy/common/system/zipfileset.py b/Tools/Scripts/webkitpy/common/system/zipfileset.py new file mode 100644 index 0000000..fa2b762 --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/zipfileset.py @@ -0,0 +1,65 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import urllib +import zipfile + +from webkitpy.common.net.networktransaction import NetworkTransaction +from webkitpy.common.system.fileset import FileSetFileHandle +from webkitpy.common.system.filesystem import FileSystem + + +class ZipFileSet(object): + """The set of files in a zip file that resides at a URL (local or remote)""" + def __init__(self, zip_url, filesystem=None, zip_factory=None): + self._zip_url = zip_url + self._zip_file = None + self._filesystem = filesystem or FileSystem() + self._zip_factory = zip_factory or self._retrieve_zip_file + + def _retrieve_zip_file(self, zip_url): + temp_file = NetworkTransaction().run(lambda: urllib.urlretrieve(zip_url)[0]) + return zipfile.ZipFile(temp_file) + + def _load(self): + if self._zip_file is None: + self._zip_file = self._zip_factory(self._zip_url) + + def open(self, filename): + self._load() + return FileSetFileHandle(self, filename, self._filesystem) + + def namelist(self): + self._load() + return self._zip_file.namelist() + + def read(self, filename): + self._load() + return self._zip_file.read(filename) + + def extract(self, filename, path): + self._load() + self._zip_file.extract(filename, path) + + def delete(self, filename): + raise Exception("Can't delete from a ZipFileSet.") diff --git a/Tools/Scripts/webkitpy/common/system/zipfileset_unittest.py b/Tools/Scripts/webkitpy/common/system/zipfileset_unittest.py new file mode 100644 index 0000000..a9ba5ad --- /dev/null +++ b/Tools/Scripts/webkitpy/common/system/zipfileset_unittest.py @@ -0,0 +1,95 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import shutil +import tempfile +import unittest +import zipfile + +from webkitpy.common.system.filesystem_mock import MockFileSystem +from webkitpy.common.system.zipfileset import ZipFileSet + + +class FakeZip(object): + def __init__(self, filesystem): + self._filesystem = filesystem + self._files = {} + + def add_file(self, filename, contents): + self._files[filename] = contents + + def open(self, filename): + return FileSetFileHandle(self, filename, self._filesystem) + + def namelist(self): + return self._files.keys() + + def read(self, filename): + return self._files[filename] + + def extract(self, filename, path): + self._filesystem.write_text_file(self._filesystem.join(path, filename), self.read(filename)) + + def delete(self, filename): + raise Exception("Can't delete from a ZipFileSet.") + + +class ZipFileSetTest(unittest.TestCase): + def setUp(self): + self._filesystem = MockFileSystem() + self._zip = ZipFileSet('blah', self._filesystem, self.make_fake_zip) + + def make_fake_zip(self, zip_url): + result = FakeZip(self._filesystem) + result.add_file('some-file', 'contents') + result.add_file('a/b/some-other-file', 'other contents') + return result + + def test_open(self): + file = self._zip.open('a/b/some-other-file') + self.assertEquals('a/b/some-other-file', file.name()) + self.assertEquals('other contents', file.contents()) + + def test_read(self): + self.assertEquals('contents', self._zip.read('some-file')) + + def test_extract(self): + self._filesystem.maybe_make_directory('/some-dir') + self._zip.extract('some-file', '/some-dir') + self.assertTrue(self._filesystem.isfile('/some-dir/some-file')) + + def test_deep_extract(self): + self._filesystem.maybe_make_directory('/some-dir') + self._zip.extract('a/b/some-other-file', '/some-dir') + self.assertTrue(self._filesystem.isfile('/some-dir/a/b/some-other-file')) + + def test_cant_delete(self): + self.assertRaises(Exception, self._zip.delete, 'some-file') + + def test_namelist(self): + self.assertTrue('some-file' in self._zip.namelist()) + + +if __name__ == '__main__': + unittest.main() |