diff options
Diffstat (limited to 'WebKitTools/Scripts')
379 files changed, 31843 insertions, 11870 deletions
diff --git a/WebKitTools/Scripts/VCSUtils.pm b/WebKitTools/Scripts/VCSUtils.pm index 022c72a..4516984 100644 --- a/WebKitTools/Scripts/VCSUtils.pm +++ b/WebKitTools/Scripts/VCSUtils.pm @@ -43,6 +43,7 @@ BEGIN { $VERSION = 1.00; @ISA = qw(Exporter); @EXPORT = qw( + &callSilently &canonicalizePath &changeLogEmailAddress &changeLogName @@ -61,10 +62,14 @@ BEGIN { &isSVNDirectory &isSVNVersion16OrNewer &makeFilePathRelative + &mergeChangeLogs &normalizePath &parsePatch &pathRelativeToSVNRepositoryRootForPath + &prepareParsedPatch &runPatchCommand + &scmToggleExecutableBit + &setChangeLogDateAndReviewer &svnRevisionForDirectory &svnStatus ); @@ -81,6 +86,15 @@ my $isGitBranchBuild; my $isSVN; my $svnVersion; +# Project time zone for Cupertino, CA, US +my $changeLogTimeZone = "PST8PDT"; + +my $gitDiffStartRegEx = qr#^diff --git (\w/)?(.+) (\w/)?([^\r\n]+)#; +my $svnDiffStartRegEx = qr#^Index: ([^\r\n]+)#; +my $svnPropertiesStartRegEx = qr#^Property changes on: ([^\r\n]+)#; # $1 is normally the same as the index path. +my $svnPropertyStartRegEx = qr#^(Modified|Name|Added|Deleted): ([^\r\n]+)#; # $2 is the name of the property. +my $svnPropertyValueStartRegEx = qr#^ (\+|-) ([^\r\n]+)#; # $2 is the start of the property's value (which may span multiple lines). + # This method is for portability. Return the system-appropriate exit # status of a child process. # @@ -95,6 +109,59 @@ sub exitStatus($) return WEXITSTATUS($returnvalue); } +# Call a function while suppressing STDERR, and return the return values +# as an array. +sub callSilently($@) { + my ($func, @args) = @_; + + # The following pattern was taken from here: + # http://www.sdsc.edu/~moreland/courses/IntroPerl/docs/manual/pod/perlfunc/open.html + # + # Also see this Perl documentation (search for "open OLDERR"): + # http://perldoc.perl.org/functions/open.html + open(OLDERR, ">&STDERR"); + close(STDERR); + my @returnValue = &$func(@args); + open(STDERR, ">&OLDERR"); + close(OLDERR); + + return @returnValue; +} + +# Note, this method will not error if the file corresponding to the path does not exist. +sub scmToggleExecutableBit +{ + my ($path, $executableBitDelta) = @_; + return if ! -e $path; + if ($executableBitDelta == 1) { + scmAddExecutableBit($path); + } elsif ($executableBitDelta == -1) { + scmRemoveExecutableBit($path); + } +} + +sub scmAddExecutableBit($) +{ + my ($path) = @_; + + if (isSVN()) { + system("svn", "propset", "svn:executable", "on", $path) == 0 or die "Failed to run 'svn propset svn:executable on $path'."; + } elsif (isGit()) { + chmod(0755, $path); + } +} + +sub scmRemoveExecutableBit($) +{ + my ($path) = @_; + + if (isSVN()) { + system("svn", "propdel", "svn:executable", $path) == 0 or die "Failed to run 'svn propdel svn:executable $path'."; + } elsif (isGit()) { + chmod(0664, $path); + } +} + sub isGitDirectory($) { my ($dir) = @_; @@ -361,164 +428,384 @@ sub svnStatus($) return $svnStatus; } -# Convert a line of a git-formatted patch to SVN format, while -# preserving any end-of-line characters. -sub gitdiff2svndiff($) +# Return whether the given file mode is executable in the source control +# sense. We make this determination based on whether the executable bit +# is set for "others" rather than the stronger condition that it be set +# for the user, group, and others. This is sufficient for distinguishing +# the default behavior in Git and SVN. +# +# Args: +# $fileMode: A number or string representing a file mode in octal notation. +sub isExecutable($) { - $_ = shift @_; + my $fileMode = shift; - if (m#^diff --git \w/(.+) \w/([^\r\n]+)#) { - return "Index: $1$POSTMATCH"; - } - if (m#^index [0-9a-f]{7}\.\.[0-9a-f]{7} [0-9]{6}#) { - # FIXME: No need to return dividing line once parseDiffHeader() is used. - return "===================================================================$POSTMATCH"; - } - if (m#^--- \w/([^\r\n]+)#) { - return "--- $1$POSTMATCH"; + return $fileMode % 2; +} + +# Parse the next Git diff header from the given file handle, and advance +# the handle so the last line read is the first line after the header. +# +# This subroutine dies if given leading junk. +# +# Args: +# $fileHandle: advanced so the last line read from the handle is the first +# line of the header to parse. This should be a line +# beginning with "diff --git". +# $line: the line last read from $fileHandle +# +# Returns ($headerHashRef, $lastReadLine): +# $headerHashRef: a hash reference representing a diff header, as follows-- +# copiedFromPath: the path from which the file was copied or moved if +# the diff is a copy or move. +# executableBitDelta: the value 1 or -1 if the executable bit was added or +# removed, respectively. New and deleted files have +# this value only if the file is executable, in which +# case the value is 1 and -1, respectively. +# indexPath: the path of the target file. +# isBinary: the value 1 if the diff is for a binary file. +# isDeletion: the value 1 if the diff is a file deletion. +# isCopyWithChanges: the value 1 if the file was copied or moved and +# the target file was changed in some way after being +# copied or moved (e.g. if its contents or executable +# bit were changed). +# isNew: the value 1 if the diff is for a new file. +# shouldDeleteSource: the value 1 if the file was copied or moved and +# the source file was deleted -- i.e. if the copy +# was actually a move. +# svnConvertedText: the header text with some lines converted to SVN +# format. Git-specific lines are preserved. +# $lastReadLine: the line last read from $fileHandle. +sub parseGitDiffHeader($$) +{ + my ($fileHandle, $line) = @_; + + $_ = $line; + + my $indexPath; + if (/$gitDiffStartRegEx/) { + # The first and second paths can differ in the case of copies + # and renames. We use the second file path because it is the + # destination path. + $indexPath = $4; + # Use $POSTMATCH to preserve the end-of-line character. + $_ = "Index: $indexPath$POSTMATCH"; # Convert to SVN format. + } else { + die("Could not parse leading \"diff --git\" line: \"$line\"."); } - if (m#^\+\+\+ \w/([^\r\n]+)#) { - return "+++ $1$POSTMATCH"; + + my $copiedFromPath; + my $foundHeaderEnding; + my $isBinary; + my $isDeletion; + my $isNew; + my $newExecutableBit = 0; + my $oldExecutableBit = 0; + my $shouldDeleteSource = 0; + my $similarityIndex = 0; + my $svnConvertedText; + while (1) { + # Temporarily strip off any end-of-line characters to simplify + # regex matching below. + s/([\n\r]+)$//; + my $eol = $1; + + if (/^(deleted file|old) mode (\d+)/) { + $oldExecutableBit = (isExecutable($2) ? 1 : 0); + $isDeletion = 1 if $1 eq "deleted file"; + } elsif (/^new( file)? mode (\d+)/) { + $newExecutableBit = (isExecutable($2) ? 1 : 0); + $isNew = 1 if $1; + } elsif (/^similarity index (\d+)%/) { + $similarityIndex = $1; + } elsif (/^copy from (\S+)/) { + $copiedFromPath = $1; + } elsif (/^rename from (\S+)/) { + # FIXME: Record this as a move rather than as a copy-and-delete. + # This will simplify adding rename support to svn-unapply. + # Otherwise, the hash for a deletion would have to know + # everything about the file being deleted in order to + # support undoing itself. Recording as a move will also + # permit us to use "svn move" and "git move". + $copiedFromPath = $1; + $shouldDeleteSource = 1; + } elsif (/^--- \S+/) { + $_ = "--- $indexPath"; # Convert to SVN format. + } elsif (/^\+\+\+ \S+/) { + $_ = "+++ $indexPath"; # Convert to SVN format. + $foundHeaderEnding = 1; + } elsif (/^GIT binary patch$/ ) { + $isBinary = 1; + $foundHeaderEnding = 1; + # The "git diff" command includes a line of the form "Binary files + # <path1> and <path2> differ" if the --binary flag is not used. + } elsif (/^Binary files / ) { + die("Error: the Git diff contains a binary file without the binary data in ". + "line: \"$_\". Be sure to use the --binary flag when invoking \"git diff\" ". + "with diffs containing binary files."); + } + + $svnConvertedText .= "$_$eol"; # Also restore end-of-line characters. + + $_ = <$fileHandle>; # Not defined if end-of-file reached. + + last if (!defined($_) || /$gitDiffStartRegEx/ || $foundHeaderEnding); } - return $_; + + my $executableBitDelta = $newExecutableBit - $oldExecutableBit; + + my %header; + + $header{copiedFromPath} = $copiedFromPath if $copiedFromPath; + $header{executableBitDelta} = $executableBitDelta if $executableBitDelta; + $header{indexPath} = $indexPath; + $header{isBinary} = $isBinary if $isBinary; + $header{isCopyWithChanges} = 1 if ($copiedFromPath && ($similarityIndex != 100 || $executableBitDelta)); + $header{isDeletion} = $isDeletion if $isDeletion; + $header{isNew} = $isNew if $isNew; + $header{shouldDeleteSource} = $shouldDeleteSource if $shouldDeleteSource; + $header{svnConvertedText} = $svnConvertedText; + + return (\%header, $_); } -# Parse the next diff header from the given file handle, and advance -# the file handle so the last line read is the first line after the -# parsed header block. +# Parse the next SVN diff header from the given file handle, and advance +# the handle so the last line read is the first line after the header. # -# This subroutine dies if given leading junk or if the end of the header -# block could not be detected. The last line of a header block is a -# line beginning with "+++". +# This subroutine dies if given leading junk or if it could not detect +# the end of the header block. # # Args: -# $fileHandle: advanced so the last line read is the first line of the -# next diff header. For SVN-formatted diffs, this is the -# "Index:" line. +# $fileHandle: advanced so the last line read from the handle is the first +# line of the header to parse. This should be a line +# beginning with "Index:". # $line: the line last read from $fileHandle # # Returns ($headerHashRef, $lastReadLine): -# $headerHashRef: a hash reference representing a diff header -# copiedFromPath: if a file copy, the path from which the file was -# copied. Otherwise, undefined. -# indexPath: the path in the "Index:" line. -# sourceRevision: the revision number of the source. This is the same -# as the revision number the file was copied from, in -# the case of a file copy. -# svnConvertedText: the header text converted to SVN format. -# Unrecognized lines are discarded. -# $lastReadLine: the line last read from $fileHandle. This is the first -# line after the header ending. -sub parseDiffHeader($$) +# $headerHashRef: a hash reference representing a diff header, as follows-- +# copiedFromPath: the path from which the file was copied if the diff +# is a copy. +# indexPath: the path of the target file, which is the path found in +# the "Index:" line. +# isBinary: the value 1 if the diff is for a binary file. +# isNew: the value 1 if the diff is for a new file. +# sourceRevision: the revision number of the source, if it exists. This +# is the same as the revision number the file was copied +# from, in the case of a file copy. +# svnConvertedText: the header text converted to a header with the paths +# in some lines corrected. +# $lastReadLine: the line last read from $fileHandle. +sub parseSvnDiffHeader($$) { my ($fileHandle, $line) = @_; - my $filter; - if ($line =~ m#^diff --git #) { - $filter = \&gitdiff2svndiff; - } - $line = &$filter($line) if $filter; + $_ = $line; my $indexPath; - if ($line =~ /^Index: ([^\r\n]+)/) { + if (/$svnDiffStartRegEx/) { $indexPath = $1; } else { - die("Could not parse first line of diff header: \"$line\"."); + die("First line of SVN diff does not begin with \"Index \": \"$_\""); } - my %header; - + my $copiedFromPath; my $foundHeaderEnding; - my $lastReadLine; + my $isBinary; + my $isNew; my $sourceRevision; - my $svnConvertedText = $line; - while (<$fileHandle>) { + my $svnConvertedText; + while (1) { # Temporarily strip off any end-of-line characters to simplify # regex matching below. s/([\n\r]+)$//; my $eol = $1; - $_ = &$filter($_) if $filter; - # Fix paths on ""---" and "+++" lines to match the leading # index line. if (s/^--- \S+/--- $indexPath/) { # --- if (/^--- .+\(revision (\d+)\)/) { - $sourceRevision = $1 if ($1 != 0); + $sourceRevision = $1; + $isNew = 1 if !$sourceRevision; # if revision 0. if (/\(from (\S+):(\d+)\)$/) { # The "from" clause is created by svn-create-patch, in # which case there is always also a "revision" clause. - $header{copiedFromPath} = $1; + $copiedFromPath = $1; die("Revision number \"$2\" in \"from\" clause does not match " . "source revision number \"$sourceRevision\".") if ($2 != $sourceRevision); } } - $_ = "=" x 67 . "$eol$_"; # Prepend dividing line ===.... } elsif (s/^\+\+\+ \S+/+++ $indexPath/) { - # +++ $foundHeaderEnding = 1; - } else { - # Skip unrecognized lines. - next; + } elsif (/^Cannot display: file marked as a binary type.$/) { + $isBinary = 1; + $foundHeaderEnding = 1; } $svnConvertedText .= "$_$eol"; # Also restore end-of-line characters. - if ($foundHeaderEnding) { - $lastReadLine = <$fileHandle>; - last; - } - } # $lastReadLine is undef if while loop ran out. + + $_ = <$fileHandle>; # Not defined if end-of-file reached. + + last if (!defined($_) || /$svnDiffStartRegEx/ || $foundHeaderEnding); + } if (!$foundHeaderEnding) { die("Did not find end of header block corresponding to index path \"$indexPath\"."); } + my %header; + + $header{copiedFromPath} = $copiedFromPath if $copiedFromPath; $header{indexPath} = $indexPath; - $header{sourceRevision} = $sourceRevision; + $header{isBinary} = $isBinary if $isBinary; + $header{isNew} = $isNew if $isNew; + $header{sourceRevision} = $sourceRevision if $sourceRevision; $header{svnConvertedText} = $svnConvertedText; - return (\%header, $lastReadLine); + return (\%header, $_); } +# Parse the next diff header from the given file handle, and advance +# the handle so the last line read is the first line after the header. +# +# This subroutine dies if given leading junk or if it could not detect +# the end of the header block. +# +# Args: +# $fileHandle: advanced so the last line read from the handle is the first +# line of the header to parse. For SVN-formatted diffs, this +# is a line beginning with "Index:". For Git, this is a line +# beginning with "diff --git". +# $line: the line last read from $fileHandle +# +# Returns ($headerHashRef, $lastReadLine): +# $headerHashRef: a hash reference representing a diff header +# copiedFromPath: the path from which the file was copied if the diff +# is a copy. +# executableBitDelta: the value 1 or -1 if the executable bit was added or +# removed, respectively. New and deleted files have +# this value only if the file is executable, in which +# case the value is 1 and -1, respectively. +# indexPath: the path of the target file. +# isBinary: the value 1 if the diff is for a binary file. +# isGit: the value 1 if the diff is Git-formatted. +# isSvn: the value 1 if the diff is SVN-formatted. +# sourceRevision: the revision number of the source, if it exists. This +# is the same as the revision number the file was copied +# from, in the case of a file copy. +# svnConvertedText: the header text with some lines converted to SVN +# format. Git-specific lines are preserved. +# $lastReadLine: the line last read from $fileHandle. +sub parseDiffHeader($$) +{ + my ($fileHandle, $line) = @_; + + my $header; # This is a hash ref. + my $isGit; + my $isSvn; + my $lastReadLine; + + if ($line =~ $svnDiffStartRegEx) { + $isSvn = 1; + ($header, $lastReadLine) = parseSvnDiffHeader($fileHandle, $line); + } elsif ($line =~ $gitDiffStartRegEx) { + $isGit = 1; + ($header, $lastReadLine) = parseGitDiffHeader($fileHandle, $line); + } else { + die("First line of diff does not begin with \"Index:\" or \"diff --git\": \"$line\""); + } + + $header->{isGit} = $isGit if $isGit; + $header->{isSvn} = $isSvn if $isSvn; + + return ($header, $lastReadLine); +} + +# FIXME: The %diffHash "object" should not have an svnConvertedText property. +# Instead, the hash object should store its information in a +# structured way as properties. This should be done in a way so +# that, if necessary, the text of an SVN or Git patch can be +# reconstructed from the information in those hash properties. +# +# A %diffHash is a hash representing a source control diff of a single +# file operation (e.g. a file modification, copy, or delete). +# +# These hashes appear, for example, in the parseDiff(), parsePatch(), +# and prepareParsedPatch() subroutines of this package. +# +# The corresponding values are-- +# +# copiedFromPath: the path from which the file was copied if the diff +# is a copy. +# executableBitDelta: the value 1 or -1 if the executable bit was added or +# removed from the target file, respectively. +# indexPath: the path of the target file. For SVN-formatted diffs, +# this is the same as the path in the "Index:" line. +# isBinary: the value 1 if the diff is for a binary file. +# isDeletion: the value 1 if the diff is known from the header to be a deletion. +# isGit: the value 1 if the diff is Git-formatted. +# isNew: the value 1 if the dif is known from the header to be a new file. +# isSvn: the value 1 if the diff is SVN-formatted. +# sourceRevision: the revision number of the source, if it exists. This +# is the same as the revision number the file was copied +# from, in the case of a file copy. +# svnConvertedText: the diff with some lines converted to SVN format. +# Git-specific lines are preserved. + # Parse one diff from a patch file created by svn-create-patch, and # advance the file handle so the last line read is the first line # of the next header block. # # This subroutine preserves any leading junk encountered before the header. # +# Composition of an SVN diff +# +# There are three parts to an SVN diff: the header, the property change, and +# the binary contents, in that order. Either the header or the property change +# may be ommitted, but not both. If there are binary changes, then you always +# have all three. +# # Args: # $fileHandle: a file handle advanced to the first line of the next # header block. Leading junk is okay. # $line: the line last read from $fileHandle. # -# Returns ($diffHashRef, $lastReadLine): -# $diffHashRef: -# copiedFromPath: if a file copy, the path from which the file was -# copied. Otherwise, undefined. -# indexPath: the path in the "Index:" line. -# sourceRevision: the revision number of the source. This is the same -# as the revision number the file was copied from, in -# the case of a file copy. -# svnConvertedText: the diff converted to SVN format. +# Returns ($diffHashRefs, $lastReadLine): +# $diffHashRefs: A reference to an array of references to %diffHash hashes. +# See the %diffHash documentation above. # $lastReadLine: the line last read from $fileHandle sub parseDiff($$) { + # FIXME: Adjust this method so that it dies if the first line does not + # match the start of a diff. This will require a change to + # parsePatch() so that parsePatch() skips over leading junk. my ($fileHandle, $line) = @_; - my $headerStartRegEx = qr#^Index: #; # SVN-style header for the default - my $gitHeaderStartRegEx = qr#^diff --git \w/#; + my $headerStartRegEx = $svnDiffStartRegEx; # SVN-style header for the default my $headerHashRef; # Last header found, as returned by parseDiffHeader(). + my $svnPropertiesHashRef; # Last SVN properties diff found, as returned by parseSvnDiffProperties(). my $svnText; while (defined($line)) { - if (!$headerHashRef && ($line =~ $gitHeaderStartRegEx)) { + if (!$headerHashRef && ($line =~ $gitDiffStartRegEx)) { # Then assume all diffs in the patch are Git-formatted. This # block was made to be enterable at most once since we assume # all diffs in the patch are formatted the same (SVN or Git). - $headerStartRegEx = $gitHeaderStartRegEx; + $headerStartRegEx = $gitDiffStartRegEx; } + if ($line =~ $svnPropertiesStartRegEx) { + my $propertyPath = $1; + if ($svnPropertiesHashRef || $headerHashRef && ($propertyPath ne $headerHashRef->{indexPath})) { + # This is the start of the second diff in the while loop, which happens to + # be a property diff. If $svnPropertiesHasRef is defined, then this is the + # second consecutive property diff, otherwise it's the start of a property + # diff for a file that only has property changes. + last; + } + ($svnPropertiesHashRef, $line) = parseSvnDiffProperties($fileHandle, $line); + next; + } if ($line !~ $headerStartRegEx) { # Then we are in the body of the diff. $svnText .= $line; @@ -526,8 +813,9 @@ sub parseDiff($$) next; } # Otherwise, we found a diff header. - if ($headerHashRef) { - # Then this is the second diff header of this while loop. + if ($svnPropertiesHashRef || $headerHashRef) { + # Then either we just processed an SVN property change or this + # is the start of the second diff header of this while loop. last; } @@ -536,13 +824,289 @@ sub parseDiff($$) $svnText .= $headerHashRef->{svnConvertedText}; } - my %diffHashRef; - $diffHashRef{copiedFromPath} = $headerHashRef->{copiedFromPath}; - $diffHashRef{indexPath} = $headerHashRef->{indexPath}; - $diffHashRef{sourceRevision} = $headerHashRef->{sourceRevision}; - $diffHashRef{svnConvertedText} = $svnText; + my @diffHashRefs; + + if ($headerHashRef->{shouldDeleteSource}) { + my %deletionHash; + $deletionHash{indexPath} = $headerHashRef->{copiedFromPath}; + $deletionHash{isDeletion} = 1; + push @diffHashRefs, \%deletionHash; + } + if ($headerHashRef->{copiedFromPath}) { + my %copyHash; + $copyHash{copiedFromPath} = $headerHashRef->{copiedFromPath}; + $copyHash{indexPath} = $headerHashRef->{indexPath}; + $copyHash{sourceRevision} = $headerHashRef->{sourceRevision} if $headerHashRef->{sourceRevision}; + if ($headerHashRef->{isSvn}) { + $copyHash{executableBitDelta} = $svnPropertiesHashRef->{executableBitDelta} if $svnPropertiesHashRef->{executableBitDelta}; + } + push @diffHashRefs, \%copyHash; + } - return (\%diffHashRef, $line); + # Note, the order of evaluation for the following if conditional has been explicitly chosen so that + # it evaluates to false when there is no headerHashRef (e.g. a property change diff for a file that + # only has property changes). + if ($headerHashRef->{isCopyWithChanges} || (%$headerHashRef && !$headerHashRef->{copiedFromPath})) { + # Then add the usual file modification. + my %diffHash; + # FIXME: We should expand this code to support other properties. In the future, + # parseSvnDiffProperties may return a hash whose keys are the properties. + if ($headerHashRef->{isSvn}) { + # SVN records the change to the executable bit in a separate property change diff + # that follows the contents of the diff, except for binary diffs. For binary + # diffs, the property change diff follows the diff header. + $diffHash{executableBitDelta} = $svnPropertiesHashRef->{executableBitDelta} if $svnPropertiesHashRef->{executableBitDelta}; + } elsif ($headerHashRef->{isGit}) { + # Git records the change to the executable bit in the header of a diff. + $diffHash{executableBitDelta} = $headerHashRef->{executableBitDelta} if $headerHashRef->{executableBitDelta}; + } + $diffHash{indexPath} = $headerHashRef->{indexPath}; + $diffHash{isBinary} = $headerHashRef->{isBinary} if $headerHashRef->{isBinary}; + $diffHash{isDeletion} = $headerHashRef->{isDeletion} if $headerHashRef->{isDeletion}; + $diffHash{isGit} = $headerHashRef->{isGit} if $headerHashRef->{isGit}; + $diffHash{isNew} = $headerHashRef->{isNew} if $headerHashRef->{isNew}; + $diffHash{isSvn} = $headerHashRef->{isSvn} if $headerHashRef->{isSvn}; + if (!$headerHashRef->{copiedFromPath}) { + # If the file was copied, then we have already incorporated the + # sourceRevision information into the change. + $diffHash{sourceRevision} = $headerHashRef->{sourceRevision} if $headerHashRef->{sourceRevision}; + } + # FIXME: Remove the need for svnConvertedText. See the %diffHash + # code comments above for more information. + # + # Note, we may not always have SVN converted text since we intend + # to deprecate it in the future. For example, a property change + # diff for a file that only has property changes will not return + # any SVN converted text. + $diffHash{svnConvertedText} = $svnText if $svnText; + push @diffHashRefs, \%diffHash; + } + + if (!%$headerHashRef && $svnPropertiesHashRef) { + # A property change diff for a file that only has property changes. + my %propertyChangeHash; + $propertyChangeHash{executableBitDelta} = $svnPropertiesHashRef->{executableBitDelta} if $svnPropertiesHashRef->{executableBitDelta}; + $propertyChangeHash{indexPath} = $svnPropertiesHashRef->{propertyPath}; + $propertyChangeHash{isSvn} = 1; + push @diffHashRefs, \%propertyChangeHash; + } + + return (\@diffHashRefs, $line); +} + +# Parse an SVN property change diff from the given file handle, and advance +# the handle so the last line read is the first line after this diff. +# +# For the case of an SVN binary diff, the binary contents will follow the +# the property changes. +# +# This subroutine dies if the first line does not begin with "Property changes on" +# or if the separator line that follows this line is missing. +# +# Args: +# $fileHandle: advanced so the last line read from the handle is the first +# line of the footer to parse. This line begins with +# "Property changes on". +# $line: the line last read from $fileHandle. +# +# Returns ($propertyHashRef, $lastReadLine): +# $propertyHashRef: a hash reference representing an SVN diff footer. +# propertyPath: the path of the target file. +# executableBitDelta: the value 1 or -1 if the executable bit was added or +# removed from the target file, respectively. +# $lastReadLine: the line last read from $fileHandle. +sub parseSvnDiffProperties($$) +{ + my ($fileHandle, $line) = @_; + + $_ = $line; + + my %footer; + if (/$svnPropertiesStartRegEx/) { + $footer{propertyPath} = $1; + } else { + die("Failed to find start of SVN property change, \"Property changes on \": \"$_\""); + } + + # We advance $fileHandle two lines so that the next line that + # we process is $svnPropertyStartRegEx in a well-formed footer. + # A well-formed footer has the form: + # Property changes on: FileA + # ___________________________________________________________________ + # Added: svn:executable + # + * + $_ = <$fileHandle>; # Not defined if end-of-file reached. + my $separator = "_" x 67; + if (defined($_) && /^$separator[\r\n]+$/) { + $_ = <$fileHandle>; + } else { + die("Failed to find separator line: \"$_\"."); + } + + # FIXME: We should expand this to support other SVN properties + # (e.g. return a hash of property key-values that represents + # all properties). + # + # Notice, we keep processing until we hit end-of-file or some + # line that does not resemble $svnPropertyStartRegEx, such as + # the empty line that precedes the start of the binary contents + # of a patch, or the start of the next diff (e.g. "Index:"). + my $propertyHashRef; + while (defined($_) && /$svnPropertyStartRegEx/) { + ($propertyHashRef, $_) = parseSvnProperty($fileHandle, $_); + if ($propertyHashRef->{name} eq "svn:executable") { + # Notice, for SVN properties, propertyChangeDelta is always non-zero + # because a property can only be added or removed. + $footer{executableBitDelta} = $propertyHashRef->{propertyChangeDelta}; + } + } + + return(\%footer, $_); +} + +# Parse the next SVN property from the given file handle, and advance the handle so the last +# line read is the first line after the property. +# +# This subroutine dies if the first line is not a valid start of an SVN property, +# or the property is missing a value, or the property change type (e.g. "Added") +# does not correspond to the property value type (e.g. "+"). +# +# Args: +# $fileHandle: advanced so the last line read from the handle is the first +# line of the property to parse. This should be a line +# that matches $svnPropertyStartRegEx. +# $line: the line last read from $fileHandle. +# +# Returns ($propertyHashRef, $lastReadLine): +# $propertyHashRef: a hash reference representing a SVN property. +# name: the name of the property. +# value: the last property value. For instance, suppose the property is "Modified". +# Then it has both a '-' and '+' property value in that order. Therefore, +# the value of this key is the value of the '+' property by ordering (since +# it is the last value). +# propertyChangeDelta: the value 1 or -1 if the property was added or +# removed, respectively. +# $lastReadLine: the line last read from $fileHandle. +sub parseSvnProperty($$) +{ + my ($fileHandle, $line) = @_; + + $_ = $line; + + my $propertyName; + my $propertyChangeType; + if (/$svnPropertyStartRegEx/) { + $propertyChangeType = $1; + $propertyName = $2; + } else { + die("Failed to find SVN property: \"$_\"."); + } + + $_ = <$fileHandle>; # Not defined if end-of-file reached. + + # The "svn diff" command neither inserts newline characters between property values + # nor between successive properties. + # + # FIXME: We do not support property values that contain tailing newline characters + # as it is difficult to disambiguate these trailing newlines from the empty + # line that precedes the contents of a binary patch. + my $propertyValue; + my $propertyValueType; + while (defined($_) && /$svnPropertyValueStartRegEx/) { + # Note, a '-' property may be followed by a '+' property in the case of a "Modified" + # or "Name" property. We only care about the ending value (i.e. the '+' property) + # in such circumstances. So, we take the property value for the property to be its + # last parsed property value. + # + # FIXME: We may want to consider strictly enforcing a '-', '+' property ordering or + # add error checking to prevent '+', '+', ..., '+' and other invalid combinations. + $propertyValueType = $1; + ($propertyValue, $_) = parseSvnPropertyValue($fileHandle, $_); + } + + if (!$propertyValue) { + die("Failed to find the property value for the SVN property \"$propertyName\": \"$_\"."); + } + + my $propertyChangeDelta; + if ($propertyValueType eq '+') { + $propertyChangeDelta = 1; + } elsif ($propertyValueType eq '-') { + $propertyChangeDelta = -1; + } else { + die("Not reached."); + } + + # We perform a simple validation that an "Added" or "Deleted" property + # change type corresponds with a "+" and "-" value type, respectively. + my $expectedChangeDelta; + if ($propertyChangeType eq "Added") { + $expectedChangeDelta = 1; + } elsif ($propertyChangeType eq "Deleted") { + $expectedChangeDelta = -1; + } + + if ($expectedChangeDelta && $propertyChangeDelta != $expectedChangeDelta) { + die("The final property value type found \"$propertyValueType\" does not " . + "correspond to the property change type found \"$propertyChangeType\"."); + } + + my %propertyHash; + $propertyHash{name} = $propertyName; + $propertyHash{propertyChangeDelta} = $propertyChangeDelta; + $propertyHash{value} = $propertyValue; + return (\%propertyHash, $_); +} + +# Parse the value of an SVN property from the given file handle, and advance +# the handle so the last line read is the first line after the property value. +# +# This subroutine dies if the first line is an invalid SVN property value line +# (i.e. a line that does not begin with " +" or " -"). +# +# Args: +# $fileHandle: advanced so the last line read from the handle is the first +# line of the property value to parse. This should be a line +# beginning with " +" or " -". +# $line: the line last read from $fileHandle. +# +# Returns ($propertyValue, $lastReadLine): +# $propertyValue: the value of the property. +# $lastReadLine: the line last read from $fileHandle. +sub parseSvnPropertyValue($$) +{ + my ($fileHandle, $line) = @_; + + $_ = $line; + + my $propertyValue; + my $eol; + if (/$svnPropertyValueStartRegEx/) { + $propertyValue = $2; # Does not include the end-of-line character(s). + $eol = $POSTMATCH; + } else { + die("Failed to find property value beginning with '+' or '-': \"$_\"."); + } + + while (<$fileHandle>) { + if (/^$/ || /$svnPropertyValueStartRegEx/ || /$svnPropertyStartRegEx/) { + # Note, we may encounter an empty line before the contents of a binary patch. + # Also, we check for $svnPropertyValueStartRegEx because a '-' property may be + # followed by a '+' property in the case of a "Modified" or "Name" property. + # We check for $svnPropertyStartRegEx because it indicates the start of the + # next property to parse. + last; + } + + # Temporarily strip off any end-of-line characters. We add the end-of-line characters + # from the previously processed line to the start of this line so that the last line + # of the property value does not end in end-of-line characters. + s/([\n\r]+)$//; + $propertyValue .= "$eol$_"; + $eol = $1; + } + + return ($propertyValue, $_); } # Parse a patch file created by svn-create-patch. @@ -552,27 +1116,144 @@ sub parseDiff($$) # read from. # # Returns: -# @diffHashRefs: an array of diff hash references. See parseDiff() for -# a description of each $diffHashRef. +# @diffHashRefs: an array of diff hash references. +# See the %diffHash documentation above. sub parsePatch($) { my ($fileHandle) = @_; + my $newDiffHashRefs; my @diffHashRefs; # return value my $line = <$fileHandle>; while (defined($line)) { # Otherwise, at EOF. - my $diffHashRef; - ($diffHashRef, $line) = parseDiff($fileHandle, $line); + ($newDiffHashRefs, $line) = parseDiff($fileHandle, $line); - push @diffHashRefs, $diffHashRef; + push @diffHashRefs, @$newDiffHashRefs; } return @diffHashRefs; } +# Prepare the results of parsePatch() for use in svn-apply and svn-unapply. +# +# Args: +# $shouldForce: Whether to continue processing if an unexpected +# state occurs. +# @diffHashRefs: An array of references to %diffHashes. +# See the %diffHash documentation above. +# +# Returns $preparedPatchHashRef: +# copyDiffHashRefs: A reference to an array of the $diffHashRefs in +# @diffHashRefs that represent file copies. The original +# ordering is preserved. +# nonCopyDiffHashRefs: A reference to an array of the $diffHashRefs in +# @diffHashRefs that do not represent file copies. +# The original ordering is preserved. +# sourceRevisionHash: A reference to a hash of source path to source +# revision number. +sub prepareParsedPatch($@) +{ + my ($shouldForce, @diffHashRefs) = @_; + + my %copiedFiles; + + # Return values + my @copyDiffHashRefs = (); + my @nonCopyDiffHashRefs = (); + my %sourceRevisionHash = (); + for my $diffHashRef (@diffHashRefs) { + my $copiedFromPath = $diffHashRef->{copiedFromPath}; + my $indexPath = $diffHashRef->{indexPath}; + my $sourceRevision = $diffHashRef->{sourceRevision}; + my $sourcePath; + + if (defined($copiedFromPath)) { + # Then the diff is a copy operation. + $sourcePath = $copiedFromPath; + + # FIXME: Consider printing a warning or exiting if + # exists($copiedFiles{$indexPath}) is true -- i.e. if + # $indexPath appears twice as a copy target. + $copiedFiles{$indexPath} = $sourcePath; + + push @copyDiffHashRefs, $diffHashRef; + } else { + # Then the diff is not a copy operation. + $sourcePath = $indexPath; + + push @nonCopyDiffHashRefs, $diffHashRef; + } + + if (defined($sourceRevision)) { + if (exists($sourceRevisionHash{$sourcePath}) && + ($sourceRevisionHash{$sourcePath} != $sourceRevision)) { + if (!$shouldForce) { + die "Two revisions of the same file required as a source:\n". + " $sourcePath:$sourceRevisionHash{$sourcePath}\n". + " $sourcePath:$sourceRevision"; + } + } + $sourceRevisionHash{$sourcePath} = $sourceRevision; + } + } + + my %preparedPatchHash; + + $preparedPatchHash{copyDiffHashRefs} = \@copyDiffHashRefs; + $preparedPatchHash{nonCopyDiffHashRefs} = \@nonCopyDiffHashRefs; + $preparedPatchHash{sourceRevisionHash} = \%sourceRevisionHash; + + return \%preparedPatchHash; +} + +# Return localtime() for the project's time zone, given an integer time as +# returned by Perl's time() function. +sub localTimeInProjectTimeZone($) +{ + my $epochTime = shift; + + # Change the time zone temporarily for the localtime() call. + my $savedTimeZone = $ENV{'TZ'}; + $ENV{'TZ'} = $changeLogTimeZone; + my @localTime = localtime($epochTime); + if (defined $savedTimeZone) { + $ENV{'TZ'} = $savedTimeZone; + } else { + delete $ENV{'TZ'}; + } + + return @localTime; +} + +# Set the reviewer and date in a ChangeLog patch, and return the new patch. +# +# Args: +# $patch: a ChangeLog patch as a string. +# $reviewer: the name of the reviewer, or undef if the reviewer should not be set. +# $epochTime: an integer time as returned by Perl's time() function. +sub setChangeLogDateAndReviewer($$$) +{ + my ($patch, $reviewer, $epochTime) = @_; + + my @localTime = localTimeInProjectTimeZone($epochTime); + my $newDate = strftime("%Y-%m-%d", @localTime); + + my $firstChangeLogLineRegEx = qr#(\n\+)\d{4}-[^-]{2}-[^-]{2}( )#; + $patch =~ s/$firstChangeLogLineRegEx/$1$newDate$2/; + + if (defined($reviewer)) { + # We include a leading plus ("+") in the regular expression to make + # the regular expression less likely to match text in the leading junk + # for the patch, if the patch has leading junk. + $patch =~ s/(\n\+.*)NOBODY \(OOPS!\)/$1$reviewer/; + } + + return $patch; +} + # If possible, returns a ChangeLog patch equivalent to the given one, # but with the newest ChangeLog entry inserted at the top of the # file -- i.e. no leading context and all lines starting with "+". @@ -670,7 +1351,7 @@ sub fixChangeLogPatch($) # Work backwards, shifting overlapping lines towards front # while checking that patch stays equivalent. - for ($i = $dateStartIndex - 1; $i >= $chunkStartIndex; --$i) { + for ($i = $dateStartIndex - 1; @overlappingLines && $i >= $chunkStartIndex; --$i) { my $line = $lines[$i]; if (substr($line, 0, 1) ne " ") { next; @@ -815,6 +1496,76 @@ sub runPatchCommand($$$;$) return $exitStatus; } +# Merge ChangeLog patches using a three-file approach. +# +# This is used by resolve-ChangeLogs when it's operated as a merge driver +# and when it's used to merge conflicts after a patch is applied or after +# an svn update. +# +# It's also used for traditional rejected patches. +# +# Args: +# $fileMine: The merged version of the file. Also known in git as the +# other branch's version (%B) or "ours". +# For traditional patch rejects, this is the *.rej file. +# $fileOlder: The base version of the file. Also known in git as the +# ancestor version (%O) or "base". +# For traditional patch rejects, this is the *.orig file. +# $fileNewer: The current version of the file. Also known in git as the +# current version (%A) or "theirs". +# For traditional patch rejects, this is the original-named +# file. +# +# Returns 1 if merge was successful, else 0. +sub mergeChangeLogs($$$) +{ + my ($fileMine, $fileOlder, $fileNewer) = @_; + + my $traditionalReject = $fileMine =~ /\.rej$/ ? 1 : 0; + + local $/ = undef; + + my $patch; + if ($traditionalReject) { + open(DIFF, "<", $fileMine) or die $!; + $patch = <DIFF>; + close(DIFF); + rename($fileMine, "$fileMine.save"); + rename($fileOlder, "$fileOlder.save"); + } else { + open(DIFF, "-|", qw(diff -u -a --binary), $fileOlder, $fileMine) or die $!; + $patch = <DIFF>; + close(DIFF); + } + + unlink("${fileNewer}.orig"); + unlink("${fileNewer}.rej"); + + open(PATCH, "| patch --force --fuzz=3 --binary $fileNewer > " . File::Spec->devnull()) or die $!; + print PATCH ($traditionalReject ? $patch : fixChangeLogPatch($patch)); + close(PATCH); + + my $result = !exitStatus($?); + + # Refuse to merge the patch if it did not apply cleanly + if (-e "${fileNewer}.rej") { + unlink("${fileNewer}.rej"); + if (-f "${fileNewer}.orig") { + unlink($fileNewer); + rename("${fileNewer}.orig", $fileNewer); + } + } else { + unlink("${fileNewer}.orig"); + } + + if ($traditionalReject) { + rename("$fileMine.save", $fileMine); + rename("$fileOlder.save", $fileOlder); + } + + return $result; +} + sub gitConfig($) { return unless $isGit; diff --git a/WebKitTools/Scripts/build-dumprendertree b/WebKitTools/Scripts/build-dumprendertree index 72e81b0..87c720f 100755 --- a/WebKitTools/Scripts/build-dumprendertree +++ b/WebKitTools/Scripts/build-dumprendertree @@ -45,6 +45,7 @@ Usage: $programName [options] [options to pass to build system] --gtk Build the GTK+ port --qt Build the Qt port --wx Build the wxWindows port + --chromium Build the Chromium port EOF GetOptions( @@ -69,8 +70,8 @@ if (isAppleMacWebKit()) { $result = buildXCodeProject("DumpRenderTree", $clean, XcodeOptions(), @ARGV); } elsif (isAppleWinWebKit()) { $result = buildVisualStudioProject("DumpRenderTree.sln", $clean); -} elsif (isQt() || isGtk() || isWx()) { - # Qt, Gtk and wxWindows build everything in one shot. No need to build anything here. +} elsif (isQt() || isGtk() || isWx() || isChromium()) { + # Qt, Gtk wxWindows, and Chromium build everything in one shot. No need to build anything here. $result = 0; } else { die "Building not defined for this platform!\n"; diff --git a/WebKitTools/Scripts/build-webkit b/WebKitTools/Scripts/build-webkit index 5ae1aae..21214cc 100755 --- a/WebKitTools/Scripts/build-webkit +++ b/WebKitTools/Scripts/build-webkit @@ -31,6 +31,7 @@ use strict; use File::Basename; +use File::Find; use File::Spec; use FindBin; use Getopt::Long qw(:config pass_through); @@ -48,23 +49,33 @@ chdirWebKit(); my $showHelp = 0; my $clean = 0; my $minimal = 0; +my $installHeaders; +my $installLibs; +my $prefixPath; my $makeArgs; my $startTime = time(); -my ($threeDCanvasSupport, $threeDRenderingSupport, $channelMessagingSupport, $clientBasedGeolocationSupport, $databaseSupport, $datagridSupport, $datalistSupport, - $domStorageSupport, $eventsourceSupport, $filtersSupport, $geolocationSupport, $iconDatabaseSupport, $indexedDatabaseSupport, - $javaScriptDebuggerSupport, $mathmlSupport, $offlineWebApplicationSupport, $rubySupport, $sharedWorkersSupport, +my ($linkPrefetchSupport, $threeDCanvasSupport, $threeDRenderingSupport, $channelMessagingSupport, $clientBasedGeolocationSupport, $databaseSupport, $datagridSupport, $datalistSupport, + $domStorageSupport, $eventsourceSupport, $filtersSupport, $geolocationSupport, $iconDatabaseSupport, $imageResizerSupport, $indexedDatabaseSupport, $inputSpeechSupport, + $javaScriptDebuggerSupport, $mathmlSupport, $offlineWebApplicationSupport, $rubySupport, $systemMallocSupport, $sandboxSupport, $sharedWorkersSupport, $svgSupport, $svgAnimationSupport, $svgAsImageSupport, $svgDOMObjCBindingsSupport, $svgFontsSupport, - $svgForeignObjectSupport, $svgUseSupport, $videoSupport, $webSocketsSupport, $wmlSupport, $wcssSupport, $xhtmlmpSupport, $workersSupport, - $xpathSupport, $xsltSupport, $coverageSupport, $notificationsSupport); + $svgForeignObjectSupport, $svgUseSupport, $videoSupport, $webSocketsSupport, $webTimingSupport, $wmlSupport, $wcssSupport, $xhtmlmpSupport, $workersSupport, + $xpathSupport, $xsltSupport, $coverageSupport, $notificationsSupport, $blobSliceSupport, $tiledBackingStoreSupport, + $fileReaderSupport, $fileWriterSupport, $fileSystemSupport, $directoryUploadSupport); my @features = ( + { option => "link-prefetch", desc => "Toggle pre fetching support", + define => "ENABLE_LINK_PREFETCH", default => 0, value => \$linkPrefetchSupport }, + { option => "3d-canvas", desc => "Toggle 3D canvas support", - define => "ENABLE_3D_CANVAS", default => (isAppleMacWebKit() && !isTiger()), value => \$threeDCanvasSupport }, + define => "ENABLE_3D_CANVAS", default => (isAppleMacWebKit() && !isTiger() && !isLeopard()), value => \$threeDCanvasSupport }, { option => "3d-rendering", desc => "Toggle 3D rendering support", define => "ENABLE_3D_RENDERING", default => (isAppleMacWebKit() && !isTiger()), value => \$threeDRenderingSupport }, + { option => "blob-slice", desc => "Toggle Blob.slice support", + define => "ENABLE_BLOB_SLICE", default => (isAppleMacWebKit()), value => \$blobSliceSupport }, + { option => "channel-messaging", desc => "Toggle MessageChannel and MessagePort support", define => "ENABLE_CHANNEL_MESSAGING", default => 1, value => \$channelMessagingSupport }, @@ -83,6 +94,9 @@ my @features = ( { option => "datalist", desc => "Toggle HTML5 datalist support", define => "ENABLE_DATALIST", default => 1, value => \$datalistSupport }, + { option => "directory-upload", desc => "Toogle Directory upload support", + define => "ENABLE_DIRECTORY_UPLOAD", default => 0, value => \$directoryUploadSupport }, + { option => "dom-storage", desc => "Toggle DOM Storage Support", define => "ENABLE_DOM_STORAGE", default => 1, value => \$domStorageSupport }, @@ -97,10 +111,16 @@ my @features = ( { option => "icon-database", desc => "Toggle Icon database support", define => "ENABLE_ICONDATABASE", default => 1, value => \$iconDatabaseSupport }, + + { option => "image-resizer", desc => "Toggle Image Resizer API support", + define => "ENABLE_IMAGE_RESIZER", default => 0, value => \$imageResizerSupport }, { option => "indexed-database", desc => "Toggle Indexed Database API support", define => "ENABLE_INDEXED_DATABASE", default => 0, value => \$indexedDatabaseSupport }, + { option => "input-speech", desc => "Speech Input API support", + define => "ENABLE_INPUT_SPEECH", default => 0, value => \$inputSpeechSupport }, + { option => "javascript-debugger", desc => "Toggle JavaScript Debugger/Profiler support", define => "ENABLE_JAVASCRIPT_DEBUGGER", default => 1, value => \$javaScriptDebuggerSupport }, @@ -116,6 +136,12 @@ my @features = ( { option => "ruby", desc => "Toggle HTML5 Ruby support", define => "ENABLE_RUBY", default => 1, value => \$rubySupport }, + { option => "system-malloc", desc => "Toggle system allocator instead of TCmalloc", + define => "USE_SYSTEM_MALLOC", default => 0, value => \$systemMallocSupport }, + + { option => "sandbox", desc => "Toggle HTML5 Sandboxed iframe support", + define => "ENABLE_SANDBOX", default => 1, value => \$sandboxSupport }, + { option => "shared-workers", desc => "Toggle SharedWorkers support", define => "ENABLE_SHARED_WORKERS", default => (isAppleWebKit() || isGtk()), value => \$sharedWorkersSupport }, @@ -140,12 +166,18 @@ my @features = ( { option => "svg-use", desc => "Toggle SVG use element support (implies SVG support)", define => "ENABLE_SVG_USE", default => 1, value => \$svgUseSupport }, + { option => "tiled-backing-store", desc => "Toggle Tiled Backing Store support", + define => "ENABLE_TILED_BACKING_STORE", default => isQt(), value => \$tiledBackingStoreSupport }, + { option => "video", desc => "Toggle Video support", define => "ENABLE_VIDEO", default => (isAppleWebKit() || isGtk()), value => \$videoSupport }, { option => "web-sockets", desc => "Toggle Web Sockets support", define => "ENABLE_WEB_SOCKETS", default => 1, value=> \$webSocketsSupport }, + { option => "web-timing", desc => "Toggle Web Timing support", + define => "ENABLE_WEB_TIMING", default => 0, value=> \$webTimingSupport }, + { option => "wml", desc => "Toggle WML support", define => "ENABLE_WML", default => 0, value => \$wmlSupport }, @@ -163,6 +195,15 @@ my @features = ( { option => "xslt", desc => "Toggle XSLT support", define => "ENABLE_XSLT", default => 1, value => \$xsltSupport }, + + { option => "file-reader", desc => "Toggle FileReader support", + define => "ENABLE_FILE_READER", default => isAppleWebKit(), value => \$fileReaderSupport }, + + { option => "file-writer", desc => "Toggle FileWriter support", + define => "ENABLE_FILE_WRITER", default => 0, value => \$fileWriterSupport }, + + { option => "file-system", desc => "Toggle FileSystem support", + define => "ENABLE_FILE_SYSTEM", default => 0, value => \$fileSystemSupport }, ); # Update defaults from Qt's project file @@ -206,6 +247,10 @@ Usage: $programName [options] [options to pass to build system] --qt Build the Qt port --inspector-frontend Copy changes to the inspector front-end files to the build directory + --install-headers=<path> Set installation path for the headers (Qt only) + --install-libs=<path> Set installation path for the libraries (Qt only) + + --prefix=<path> Set installation prefix to the given path (Gtk only) --makeargs=<arguments> Optional Makefile flags --minimal No optional features, unless explicitly enabled. @@ -215,6 +260,9 @@ EOF my %options = ( 'help' => \$showHelp, 'clean' => \$clean, + 'install-headers=s' => \$installHeaders, + 'install-libs=s' => \$installLibs, + 'prefix=s' => \$prefixPath, 'makeargs=s' => \$makeArgs, 'minimal' => \$minimal, ); @@ -238,10 +286,20 @@ setConfiguration(); my $productDir = productDir(); +# Remove 0 byte sized files from productDir after slave lost for Qt buildbots. +File::Find::find(\&unlinkZeroFiles, $productDir) if isQt(); + +sub unlinkZeroFiles () +{ + my $file = $File::Find::name; + if (! -s $file) { + unlink $file; + print "0 byte sized file removed from build directory: $file\n"; + } +} + # Check that all the project directories are there. my @projects = ("JavaScriptCore", "WebCore", "WebKit"); -# Only Apple builds JavaScriptGlue, and only on the Mac -splice @projects, 1, 0, "JavaScriptGlue" if isAppleMacWebKit(); my @otherDirs = ("WebKitLibraries"); for my $dir (@projects, @otherDirs) { @@ -254,17 +312,20 @@ my @options = (); # enable autotool options accordingly if (isGtk()) { + @options = @ARGV; foreach (@features) { push @options, autotoolsFlag(${$_->{value}}, $_->{option}); } + push @options, "--prefix=" . $prefixPath if defined($prefixPath); push @options, "--makeargs=" . $makeArgs if defined($makeArgs); } elsif (isAppleMacWebKit()) { push @options, XcodeOptions(); - sub option($$) + sub option($$$) { - my ($feature, $isEnabled) = @_; + my ($feature, $isEnabled, $defaultValue) = @_; + return "" if $defaultValue == $isEnabled; return $feature . "=" . ($isEnabled ? $feature : " "); } @@ -272,12 +333,18 @@ if (isGtk()) { if ($_->{option} eq "coverage") { push @options, XcodeCoverageSupportOptions() if $coverageSupport; } else { - push @options, option($_->{define}, ${$_->{value}}); + my $option = option($_->{define}, ${$_->{value}}, $_->{default}); + push @options, $option unless $option eq ""; } } - # Copy library and header from WebKitLibraries to a findable place in the product directory. + # Apple builds JavaScriptGlue, and only on the Mac. + splice @projects, 1, 0, "JavaScriptGlue"; + + # WebKit2 is only supported in SnowLeopard and later at present. + push @projects, ("WebKit2", "WebKitTools/MiniBrowser") if osXVersion()->{"minor"} >= 6; + # Copy library and header from WebKitLibraries to a findable place in the product directory. my @librariesToCopy = ( "libWebKitSystemInterfaceTiger.a", "libWebKitSystemInterfaceLeopard.a", @@ -315,11 +382,17 @@ if (isGtk()) { (system("perl WebKitTools/Scripts/update-webkit-support-libs") == 0) or die; } elsif (isQt()) { @options = @ARGV; + push @options, "--install-headers=" . $installHeaders if defined($installHeaders); + push @options, "--install-libs=" . $installLibs if defined($installLibs); push @options, "--makeargs=" . $makeArgs if defined($makeArgs); foreach (@features) { push @options, "DEFINES+=$_->{define}=${$_->{value}}" if ${$_->{value}} != $_->{default}; } + + if ($minimal) { + push @options, "CONFIG+=minimal"; + } } # Force re-link of existing libraries if different than expected @@ -337,6 +410,7 @@ if (isWx()) { } if (isChromium()) { + @options = @ARGV; # Chromium doesn't build by project directories. @projects = (); my $result = buildChromium($clean, @options); @@ -359,6 +433,7 @@ for my $dir (@projects) { } elsif (isQt()) { $result = buildQMakeQtProject($dir, $clean, @options); } elsif (isAppleMacWebKit()) { + $dir = "MiniBrowser" if $dir eq "WebKitTools/MiniBrowser"; $result = buildXCodeProject($dir, $clean, @options, @ARGV); } elsif (isAppleWinWebKit()) { if ($dir eq "WebKit") { diff --git a/WebKitTools/Scripts/build-webkittestrunner b/WebKitTools/Scripts/build-webkittestrunner new file mode 100755 index 0000000..dbc36d1 --- /dev/null +++ b/WebKitTools/Scripts/build-webkittestrunner @@ -0,0 +1,70 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +use strict; +use File::Basename; +use FindBin; +use Getopt::Long qw(:config pass_through); +use lib $FindBin::Bin; +use webkitdirs; +use POSIX; + +my $showHelp = 0; +my $clean = 0; + +my $programName = basename($0); +my $usage = <<EOF; +Usage: $programName [options] [options to pass to build system] + --help Show this help message + --clean Clean up the build directory +EOF + +GetOptions( + 'help' => \$showHelp, + 'clean' => \$clean, +); + +if ($showHelp) { + print STDERR $usage; + exit 1; +} + +checkRequiredSystemConfig(); +setConfiguration(); +chdirWebKit(); + +# Build +chdir "WebKitTools/WebKitTestRunner" or die; + +my $result; +if (isAppleMacWebKit()) { + $result = buildXCodeProject("WebKitTestRunner", $clean, XcodeOptions(), @ARGV); +} elsif (isAppleWinWebKit()) { + $result = buildVisualStudioProject("WebKitTestRunner.sln", $clean); +} else { + die "WebKitTestRunner is not supported on this platform.\n"; +} + +exit exitStatus($result); diff --git a/WebKitTools/Scripts/check-for-global-initializers b/WebKitTools/Scripts/check-for-global-initializers index cf83048..0472901 100755 --- a/WebKitTools/Scripts/check-for-global-initializers +++ b/WebKitTools/Scripts/check-for-global-initializers @@ -37,6 +37,7 @@ use strict; use File::Basename; sub touch($); +sub demangle($); my $arch = $ENV{'CURRENT_ARCH'}; my $configuration = $ENV{'CONFIGURATION'}; @@ -78,9 +79,14 @@ for my $file (sort @files) { next; } my $sawGlobal = 0; + my @globals; while (<NM>) { if (/^STDOUT:/) { - $sawGlobal = 1 if /__GLOBAL__I/; + my $line = $_; + if ($line =~ /__GLOBAL__I(.+)$/) { + $sawGlobal = 1; + push(@globals, demangle($1)); + } } else { print STDERR if $_ ne "nm: no name list\n"; } @@ -119,7 +125,7 @@ for my $file (sort @files) { } } - print "ERROR: $shortName has a global initializer in it! ($file)\n"; + print "ERROR: $shortName has one or more global initializers in it! ($file), near @globals\n"; $sawError = 1; } } @@ -138,3 +144,17 @@ sub touch($) open(TOUCH, ">", $path) or die "$!"; close(TOUCH); } + +sub demangle($) +{ + my ($symbol) = @_; + if (!open FILT, "c++filt $symbol |") { + print "ERROR: Could not open c++filt\n"; + return; + } + my $result = <FILT>; + close FILT; + chomp $result; + return $result; +} + diff --git a/WebKitTools/Scripts/check-for-inappropriate-files-in-framework b/WebKitTools/Scripts/check-for-inappropriate-files-in-framework new file mode 100755 index 0000000..a323bc4 --- /dev/null +++ b/WebKitTools/Scripts/check-for-inappropriate-files-in-framework @@ -0,0 +1,68 @@ +#!/usr/bin/env ruby + +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +base_directory = ENV['TARGET_BUILD_DIR'] or throw "Unable to find TARGET_BUILD_DIR in the environment!" +project_name = ENV['PROJECT_NAME'] or throw "Unable to find PROJECT_NAME in the environment!" +is_shallow_bundle = (ENV['SHALLOW_BUNDLE'] || "NO").upcase == "YES" + +$INAPPROPRIATE_FILES = { "WebCore" => { "Resources" => ["*.css", "*.in", "*.idl"] } } + +Dir.chdir base_directory + +$error_printed = false + +def print_error msg + $error_printed = true + STDERR.puts "ERROR: #{msg}" +end + +def print_inappropriate_file_error framework, relative_path + print_error "#{framework}.framework/#{relative_path} should not be present in the framework." +end + +def check_framework framework, is_shallow_bundle + $INAPPROPRIATE_FILES[framework].each do |directory, patterns| + framework_bundle_path = is_shallow_bundle ? "#{framework}.framework" : "#{framework}.framework/Versions/A/#{directory}" + Dir.chdir framework_bundle_path do + patterns.each do |pattern| + Dir.glob(pattern).each do |inappropriate_file| + print_inappropriate_file_error framework, is_shallow_bundle ? inappropriate_file : "#{directory}/#{inappropriate_file}" + File.unlink inappropriate_file + end + end + end + end +end + +check_framework project_name, is_shallow_bundle + +if $error_printed + STDERR.puts + STDERR.puts " Inappropriate files were detected and have been removed from the framework." + STDERR.puts " If this error continues to appear after building again then the build system needs" + STDERR.puts " to be modified so that the inappropriate files are no longer copied in to the framework." + STDERR.puts + exit 1 +end diff --git a/WebKitTools/Scripts/check-for-webkit-framework-include-consistency b/WebKitTools/Scripts/check-for-webkit-framework-include-consistency new file mode 100755 index 0000000..339fa7e --- /dev/null +++ b/WebKitTools/Scripts/check-for-webkit-framework-include-consistency @@ -0,0 +1,111 @@ +#!/usr/bin/env ruby + +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + + +base_directory = ENV['TARGET_BUILD_DIR'] or throw "Unable to find TARGET_BUILD_DIR in the environment!" +is_shallow_bundle = (ENV['SHALLOW_BUNDLE'] || "NO").upcase == "YES" + +unless base_directory + throw "Unable to find TARGET_BUILD_DIR in the environment!" +end + +Dir.chdir base_directory + +$PERMITTED_INCLUDE_TYPES = { :public => [ :public ], :private => [ :public, :private ] } + +$HEADER_NAMES_TO_TYPE = { } +$HEADERS_BY_TYPE = { :public => [], :private => [] } + +$error_printed = false + +def print_error msg + $error_printed = true + STDERR.puts "ERROR: #{msg}" +end + +def build_header_maps is_shallow_bundle + current_version_path = is_shallow_bundle ? "" : "Versions/A/" + all_headers = `find WebKit.framework/#{current_version_path}{,Private}Headers -type f -name '*.h'`.split + + all_headers.each do |header| + if /\/Headers\/(.*)/.match(header) + $HEADER_NAMES_TO_TYPE[$1] = :public + $HEADERS_BY_TYPE[:public] << header + elsif /\/PrivateHeaders\/(.*)/.match(header) + $HEADER_NAMES_TO_TYPE[$1] = :private + $HEADERS_BY_TYPE[:private] << header + else + print_error "Unknown header type: #{header}" + end + end +end + +def resolve_include(header, included_header, permitted_types) + # Ignore includes that aren't in the typical framework style. + return unless /<([^\/]+)\/(.*)>/.match(included_header) + + framework, included_header_name = [$1, $2] + + # Ignore includes that aren't related to other WebKit headers. + return unless framework =~ /^Web/ + + # A header of any type including a WebCore header is a recipe for disaster. + if framework == "WebCore" + # <rdar://problem/7718826> WebKeyGenerator.h should not include a WebCore header + return if header =~ /\/WebKeyGenerator.h$/ and included_header_name == "WebCoreKeyGenerator.h" + + print_error "#{header} included #{included_header}!" + return + end + + header_type = $HEADER_NAMES_TO_TYPE[included_header_name] + + if not header_type + print_error "#{header} included #{included_header} but I could not find a header of that name!" + elsif not permitted_types.member?(header_type) + print_error "#{header} included #{included_header} which is #{header_type}!" + end +end + +def verify_includes(header, permitted_types) + File.open(header) do |file| + file.each_line do |line| + if /#(include|import) (.*)/.match(line) + resolve_include(header, $2, permitted_types) + end + end + end +end + +build_header_maps is_shallow_bundle + +$HEADERS_BY_TYPE.each do |header_type, headers| + permitted_types = $PERMITTED_INCLUDE_TYPES[header_type] + headers.each do |header| + verify_includes header, permitted_types + end +end + +exit 1 if $error_printed diff --git a/WebKitTools/Scripts/check-webkit-style b/WebKitTools/Scripts/check-webkit-style index 501264b..9a1ae1e 100755 --- a/WebKitTools/Scripts/check-webkit-style +++ b/WebKitTools/Scripts/check-webkit-style @@ -43,52 +43,89 @@ same line, but it is far from perfect (in either direction). """ import codecs +import logging import os import os.path import sys +from webkitpy.style_references import detect_checkout import webkitpy.style.checker as checker -from webkitpy.style_references import SimpleScm +from webkitpy.style.patchreader import PatchReader +from webkitpy.style.checker import StyleProcessor +from webkitpy.style.filereader import TextFileReader +from webkitpy.style.main import change_directory +_log = logging.getLogger("check-webkit-style") + + +# FIXME: Move this code to style.main. def main(): # Change stderr to write with replacement characters so we don't die # if we try to print something containing non-ASCII characters. - sys.stderr = codecs.StreamReaderWriter(sys.stderr, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace') + stderr = codecs.StreamReaderWriter(sys.stderr, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace') + # Setting an "encoding" attribute on the stream is necessary to + # prevent the logging module from raising an error. See + # the checker.configure_logging() function for more information. + stderr.encoding = "UTF-8" + + # FIXME: Change webkitpy.style so that we do not need to overwrite + # the global sys.stderr. This involves updating the code to + # accept a stream parameter where necessary, and not calling + # sys.stderr explicitly anywhere. + sys.stderr = stderr + + args = sys.argv[1:] + + # Checking for the verbose flag before calling check_webkit_style_parser() + # lets us enable verbose logging earlier. + is_verbose = "-v" in args or "--verbose" in args + + checker.configure_logging(stream=stderr, is_verbose=is_verbose) + _log.debug("Verbose logging enabled.") + + parser = checker.check_webkit_style_parser() + (paths, options) = parser.parse(args) + + checkout = detect_checkout() + + if checkout is None: + if not paths: + _log.error("WebKit checkout not found: You must run this script " + "from within a WebKit checkout if you are not passing " + "specific paths to check.") + sys.exit(1) + + checkout_root = None + _log.debug("WebKit checkout not found for current directory.") + else: + checkout_root = checkout.root_path() + _log.debug("WebKit checkout found with root: %s" % checkout_root) - defaults = checker.webkit_argument_defaults() + configuration = checker.check_webkit_style_configuration(options) - parser = checker.ArgumentParser(defaults) - (files, options) = parser.parse(sys.argv[1:]) + paths = change_directory(checkout_root=checkout_root, paths=paths) - style_checker = checker.StyleChecker(options) + style_processor = StyleProcessor(configuration) - if files: - for filename in files: - style_checker.check_file(filename) + file_reader = TextFileReader(style_processor) + if paths: + file_reader.process_paths(paths) else: - scm = SimpleScm() - - os.chdir(scm.checkout_root()) - - if options.git_commit: - commit = options.git_commit - if '..' in commit: - # FIXME: If the range is a "...", the code should find the common ancestor and - # start there (see git diff --help for information about how ... usually works). - commit = commit[:commit.find('..')] - print >> sys.stderr, "Warning: Ranges are not supported for --git-commit. Checking all changes since %s.\n" % commit - patch = scm.create_patch_since_local_commit(commit) - else: - patch = scm.create_patch() - style_checker.check_patch(patch) - - error_count = style_checker.error_count - sys.stderr.write('Total errors found: %d\n' % error_count) - sys.exit(error_count > 0) + patch = checkout.create_patch(options.git_commit) + patch_checker = PatchReader(file_reader) + patch_checker.check(patch) + + error_count = style_processor.error_count + file_count = file_reader.file_count + + _log.info("Total errors found: %d in %d files" + % (error_count, file_count)) + # We fail when style errors are found or there are no checked files. + sys.exit(error_count > 0 or file_count == 0) if __name__ == "__main__": diff --git a/WebKitTools/Scripts/commit-log-editor b/WebKitTools/Scripts/commit-log-editor index 75017e3..a642731 100755 --- a/WebKitTools/Scripts/commit-log-editor +++ b/WebKitTools/Scripts/commit-log-editor @@ -1,6 +1,6 @@ #!/usr/bin/perl -w -# Copyright (C) 2006, 2007, 2008, 2009 Apple Inc. All rights reserved. +# Copyright (C) 2006, 2007, 2008, 2009, 2010 Apple Inc. All rights reserved. # Copyright (C) 2009 Torch Mobile Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -40,6 +40,7 @@ use webkitdirs; sub normalizeLineEndings($$); sub removeLongestCommonPrefixEndingInDoubleNewline(\%); +sub isCommitLogEditor($); sub usage { @@ -61,30 +62,33 @@ if (!$log) { my $baseDir = baseProductDir(); my $editor = $ENV{SVN_LOG_EDITOR}; -if (!$editor) { +if (!$editor || isCommitLogEditor($editor)) { $editor = $ENV{CVS_LOG_EDITOR}; } -if (!$editor) { +if (!$editor || isCommitLogEditor($editor)) { my $builtEditorApplication = "$baseDir/Release/Commit Log Editor.app/Contents/MacOS/Commit Log Editor"; $editor = $builtEditorApplication if -x $builtEditorApplication; } -if (!$editor) { +if (!$editor || isCommitLogEditor($editor)) { my $builtEditorApplication = "$baseDir/Debug/Commit Log Editor.app/Contents/MacOS/Commit Log Editor"; $editor = $builtEditorApplication if -x $builtEditorApplication; } -if (!$editor) { +if (!$editor || isCommitLogEditor($editor)) { my $installedEditorApplication = "$ENV{HOME}/Applications/Commit Log Editor.app/Contents/MacOS/Commit Log Editor"; $editor = $installedEditorApplication if -x $installedEditorApplication; } -if (!$editor) { - $editor = $ENV{EDITOR} || "/usr/bin/vi"; +if (!$editor || isCommitLogEditor($editor)) { + $editor = $ENV{EDITOR}; +} +if (!$editor || isCommitLogEditor($editor)) { + $editor = "/usr/bin/vi"; } my $inChangesToBeCommitted = !isGit(); my @changeLogs = (); my $logContents = ""; my $existingLog = 0; -open LOG, $log or die; +open LOG, $log or die "Could not open the log file."; while (<LOG>) { if (isGit()) { if (/^# Changes to be committed:$/) { @@ -102,7 +106,7 @@ while (<LOG>) { } $existingLog = isGit() && !(/^#/ || /^\s*$/) unless $existingLog; - push @changeLogs, makeFilePathRelative($1) if $inChangesToBeCommitted && (/^M....(.*ChangeLog)\r?\n?$/ || /^#\tmodified: (.*ChangeLog)/) && !/-ChangeLog/; + push @changeLogs, makeFilePathRelative($1) if $inChangesToBeCommitted && (/^(?:M|A)....(.*ChangeLog)\r?\n?$/ || /^#\t(?:modified|new file): (.*ChangeLog)$/) && !/-ChangeLog$/; } close LOG; @@ -151,8 +155,8 @@ for my $changeLog (@changeLogs) { # Remove indentation spaces $line =~ s/^ {8}//; - # Save the reviewed by line - if ($line =~ m/^Reviewed by .*/) { + # Save the reviewed / rubber stamped by line. + if ($line =~ m/^Reviewed by .*/ || $line =~ m/^Rubber[ \-]?stamped by .*/) { $reviewedByLine = $line; next; } @@ -184,7 +188,6 @@ for my $changeLog (@changeLogs) { $reviewedByLine = ""; } - $lineCount++; $contents .= $line; } else { @@ -204,12 +207,6 @@ for my $changeLog (@changeLogs) { my $sortKey = lc $label; if ($label eq "top level") { $sortKey = ""; - } elsif ($label eq "Tools") { - $sortKey = "-, just after top level"; - } elsif ($label eq "WebBrowser") { - $sortKey = lc "WebKit, WebBrowser after"; - } elsif ($label eq "WebCore") { - $sortKey = lc "WebFoundation, WebCore after"; } elsif ($label eq "LayoutTests") { $sortKey = lc "~, LayoutTests last"; } @@ -307,3 +304,9 @@ sub removeLongestCommonPrefixEndingInDoubleNewline(\%) } return substr($prefix, 0, $lastDoubleNewline + 2); } + +sub isCommitLogEditor($) +{ + my $editor = shift; + return $editor =~ m/commit-log-editor/; +} diff --git a/WebKitTools/Scripts/debug-minibrowser b/WebKitTools/Scripts/debug-minibrowser new file mode 100755 index 0000000..06685b4 --- /dev/null +++ b/WebKitTools/Scripts/debug-minibrowser @@ -0,0 +1,38 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2005, 2007 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Simplified "debug" script for debugging the WebKit2 MiniBrowser. + +use strict; +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; + +setConfiguration(); + +exit exitStatus(debugMiniBrowser()); diff --git a/WebKitTools/Scripts/debug-test-runner b/WebKitTools/Scripts/debug-test-runner new file mode 100755 index 0000000..5a9b7f9 --- /dev/null +++ b/WebKitTools/Scripts/debug-test-runner @@ -0,0 +1,35 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +# Simplified "debug" script for debugging the WebKitTestRunner. + +use strict; +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; + +setConfiguration(); + +exit exitStatus(debugWebKitTestRunner()); diff --git a/WebKitTools/Scripts/do-webcore-rename b/WebKitTools/Scripts/do-webcore-rename index 56d8bed..54fb0af 100755 --- a/WebKitTools/Scripts/do-webcore-rename +++ b/WebKitTools/Scripts/do-webcore-rename @@ -29,16 +29,42 @@ # Script to do a rename in JavaScriptCore, WebCore, and WebKit. use strict; + +use File::Find; use FindBin; +use Getopt::Long qw(:config pass_through); + use lib $FindBin::Bin; use webkitdirs; -use File::Find; use VCSUtils; setConfiguration(); chdirWebKit(); -my %words; +my $showHelp; +my $verbose; + +my $programName = basename($0); +my $usage = <<EOF; +Usage: $programName [options] + -h|--help Show this help message + -v|--verbose More verbose output +EOF + +my $getOptionsResult = GetOptions( + 'help|h' => \$showHelp, + 'verbose|v' => \$verbose, +); + +if (!$getOptionsResult || $showHelp) { + print STDERR $usage; + exit 1; +} + +my @directoriesToIgnoreList = ( + "icu", +); +my %directoriesToIgnore = map { $_ => 1 } @directoriesToIgnoreList; # find all files we want to process @@ -47,29 +73,30 @@ find(\&wanted, "JavaScriptCore"); find(\&wanted, "JavaScriptGlue"); find(\&wanted, "WebCore"); find(\&wanted, "WebKit"); +find(\&wanted, "WebKit2"); sub wanted { my $file = $_; - if ($file eq "icu") { + # Ignore excluded and hidden files/directories. + if ($directoriesToIgnore{$file} or $file =~ /^\../ or $file =~ /^ChangeLog/) { + print "Ignoring $File::Find::name\n" if $verbose; $File::Find::prune = 1; return; } - if ($file =~ /^\../) { - $File::Find::prune = 1; - return; - } - - return if $file =~ /^ChangeLog/; return if -d $file; push @paths, $File::Find::name; } + +# Setting isDOMTypeRename to 1 rather than 0 expands the regexps used +# below to handle custom JavaScript bindings. my $isDOMTypeRename = 0; my %renames = ( - "m_sel" => "m_selection", + # Renames go here in the form of: + # "HTMLDocumentParser" => "LegacyHTMLDocumentParser", ); my %renamesContemplatedForTheFuture = ( @@ -142,12 +169,39 @@ my %renamesContemplatedForTheFuture = ( "NativeFunction" => "HostFunction", ); +# Sort the keys of the renames hash in order of decreasing length. This +# handles the case where some of the renames are substrings of others; +# i.e., "Foo" => "Bar" and "FooBuffer" => "BarBuffer". +my @sortedRenameKeys = sort { length($b) - length($a) } keys %renames; + # rename files +sub renameFile +{ + my $file = shift; + + if ($isDOMTypeRename) { + # Find the longest key in %renames which matches this more permissive regexp. + # (The old regexp would match ".../Foo.cpp" but not ".../JSFooCustom.cpp".) + # This handles renaming of custom JavaScript bindings even when some of the + # renames are substrings of others. The only reason we don't do this all the + # time is to avoid accidental file renamings for short, non-DOM renames. + for my $key (@sortedRenameKeys) { + my $newFile = ""; + $newFile = "$1$renames{$2}$3" if $file =~ /^(.*\/\w*)($key)(\w*\.\w+)$/; + if ($newFile ne "") { + return $newFile; + } + } + } else { + $file = "$1$renames{$2}$3" if $file =~ /^(.*\/)(\w+)(\.\w+)$/ && $renames{$2}; + } + return $file; +} + my %newFile; for my $file (sort @paths) { - my $f = $file; - $f = "$1$renames{$2}$3" if $f =~ /^(.*\/)(\w+)(\.\w+)$/ && $renames{$2}; + my $f = renameFile($file); if ($f ne $file) { $newFile{$file} = $f; } @@ -175,24 +229,25 @@ for my $file (sort @paths) { my $contents; { local $/; - open FILE, $file or die; + open FILE, $file or die "Failed to open $file"; $contents = <FILE>; close FILE; } my $newContents = $contents; if ($isDOMTypeRename) { - for my $from (keys %renames) { - $newContents =~ s/\b$from/$renames{$from}/g; + for my $from (@sortedRenameKeys) { + # Handle JavaScript custom bindings. + $newContents =~ s/\b(JS|V8|to|)$from/$1$renames{$from}/g; } } else { - for my $from (keys %renames) { + for my $from (@sortedRenameKeys) { $newContents =~ s/\b$from(?!["\w])/$renames{$from}/g; # this " unconfuses Xcode syntax highlighting } } if ($newContents ne $contents) { - open FILE, ">", $file or die; + open FILE, ">", $file or die "Failed to open $file"; print FILE $newContents; close FILE; } diff --git a/WebKitTools/Scripts/ensure-valid-python b/WebKitTools/Scripts/ensure-valid-python new file mode 100755 index 0000000..aede812 --- /dev/null +++ b/WebKitTools/Scripts/ensure-valid-python @@ -0,0 +1,152 @@ +#!/usr/bin/perl -w +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use strict; + +use File::Basename; +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Getopt::Long; + +use lib $FindBin::Bin; +use webkitdirs; +use VCSUtils; + +my $macPythonURL = "http://www.python.org/ftp/python/2.6.5/python-2.6.5-macosx10.3-2010-03-24.dmg"; +my $macPythonMD5 = "84489bba813fdbb6041b69d4310a86da"; +my $macPythonInstallerName = "Python.mpkg"; + +# We could use a consistent download location, like the source or build directory. +my $tempDirectory = File::Temp::tempdir("WebKitPythonXXXX", TMPDIR => 1, CLEANUP => 1); +my $downloadDirectory = $tempDirectory; +my $mountPoint = File::Spec->join($tempDirectory, "mount"); + +sub checkPythonVersion() +{ + # Will exit 0 if Python is 2.5 or greater, non-zero otherwise. + `python -c "import sys;sys.exit(sys.version_info[:2] < (2,5))"`; + return exitStatus($?) == 0; +} + +sub downloadFileToPath($$) +{ + my ($remoteURL, $localPath) = @_; + print "Downloading $remoteURL to $localPath\n"; + my $exitCode = system("curl", "-o", $localPath, $remoteURL); + return exitStatus($exitCode) == 0; +} + +sub checkMD5($$) +{ + my ($path, $expectedMD5) = @_; + my $md5Output = `md5 -q "$path"`; + chomp($md5Output); + my $isValid = $md5Output eq $expectedMD5; + print "'$md5Output' does not match expected: '$expectedMD5'\n" unless $isValid; + return $isValid; +} + +sub mountDMG($$) +{ + my ($dmgPath, $mountPoint) = @_; + print "Mounting $dmgPath at $mountPoint\n"; + return system("hdiutil", "attach", "-mountpoint", $mountPoint, "-nobrowse", $dmgPath) == 0; +} + +sub unmountDMG($) +{ + my ($mountPoint) = @_; + print "Unmounting disk image from $mountPoint\n"; + my $exitCode = system("hdiutil", "detach", $mountPoint); + return exitStatus($exitCode) == 0; +} + +sub runInstaller($) +{ + my ($installerPackage) = @_; + print "sudo will now ask for your password to run the Python installer.\n"; + print "The installer will install Python in /Library/Frameworks/Python.framework\n"; + print "and add symlinks from /usr/local/bin.\n"; + return system("sudo", "installer", "-verbose", "-pkg", $installerPackage, "-target", "/") == 0; +} + +sub downloadAndMountMacPythonDMG($$) +{ + my ($pythonURL, $pythonMD5) = @_; + my $localFilename = basename($pythonURL); + my $localPath = File::Spec->join($downloadDirectory, $localFilename); + + downloadFileToPath($pythonURL, $localPath) or die "Failed to download $pythonURL"; + checkMD5($localPath, $pythonMD5) or die "MD5 check failed on $localPath"; + return mountDMG($localPath, $mountPoint); +} + +sub installMacPython() +{ + downloadAndMountMacPythonDMG($macPythonURL, $macPythonMD5) or die "Failed to download and mount disk image."; + print "Mounted python install image at: $mountPoint\n"; + my $installerPackage = File::Spec->join($mountPoint, $macPythonInstallerName); + my $installSuccess = runInstaller($installerPackage); + unmountDMG($mountPoint) or die "Failed to unmount disk image from $mountPoint"; + return $installSuccess; +} + +sub main() +{ + my $checkOnly = 0; + my $showHelp = 0; + my $getOptionsResult = GetOptions( + 'check-only!' => \$checkOnly, + 'help|h' => \$showHelp, + ); + if (!$getOptionsResult || $showHelp) { + print STDERR <<HELP; +Usage: $0 [options] + --check-only Check python version only. + -h|--help Show this help message. +HELP + return 1; + } + # Congrats, your Python is fine. + return 0 if checkPythonVersion(); + + return 1 if $checkOnly; + + if (!isTiger()) { + print "Your Python version is insufficient to run WebKit's Python code. Please update.\n"; + print "See http://trac.webkit.org/wiki/PythonGuidelines for more info.\n"; + return 1; + } + + installMacPython() or die "Failed to install Python."; + + checkPythonVersion() or die "Final version check failed, must have failed to update Python"; + print "Successfully updated python.\n"; +} + +exit(main()); diff --git a/WebKitTools/Scripts/extract-localizable-strings b/WebKitTools/Scripts/extract-localizable-strings index cf4f8f0..b31550a 100755 --- a/WebKitTools/Scripts/extract-localizable-strings +++ b/WebKitTools/Scripts/extract-localizable-strings @@ -1,6 +1,6 @@ #!/usr/bin/perl -w -# Copyright (C) 2006, 2007, 2009 Apple Inc. All rights reserved. +# Copyright (C) 2006, 2007, 2009, 2010 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -48,7 +48,7 @@ sub UnescapeHexSequence($); my %isDebugMacro = ( ASSERT_WITH_MESSAGE => 1, LOG_ERROR => 1, ERROR => 1, NSURL_ERROR => 1, FATAL => 1, LOG => 1, LOG_WARNING => 1, UI_STRING_LOCALIZE_LATER => 1, LPCTSTR_UI_STRING_LOCALIZE_LATER => 1, UNLOCALIZED_STRING => 1, UNLOCALIZED_LPCTSTR => 1, dprintf => 1, NSException => 1, NSLog => 1, printf => 1 ); -@ARGV >= 1 or die "Usage: extract-localizable-strings <exceptions file> [ directory... ]\nDid you mean to run update-webkit-localizable-strings instead?\n"; +@ARGV >= 2 or die "Usage: extract-localizable-strings <exceptions file> <file to update> [ directory... ]\nDid you mean to run update-webkit-localizable-strings instead?\n"; my $exceptionsFile = shift @ARGV; -f $exceptionsFile or die "Couldn't find exceptions file $exceptionsFile\n"; @@ -99,14 +99,13 @@ if (open EXCEPTIONS, $exceptionsFile) { my $quotedDirectoriesString = '"' . join('" "', @directories) . '"'; for my $dir (@directoriesToSkip) { - $quotedDirectoriesString .= ' -path "' . $dir . '" -prune'; + $quotedDirectoriesString .= ' -path "' . $dir . '" -prune -o'; } -my @files = ( split "\n", `find $quotedDirectoriesString -name "*.h" -o -name "*.m" -o -name "*.mm" -o -name "*.c" -o -name "*.cpp"` ); +my @files = ( split "\n", `find $quotedDirectoriesString \\( -name "*.h" -o -name "*.m" -o -name "*.mm" -o -name "*.c" -o -name "*.cpp" \\)` ); for my $file (sort @files) { - next if $file =~ /\/WebLocalizableStrings\.h$/; - next if $file =~ /\/icu\//; + next if $file =~ /\/\w+LocalizableStrings\.h$/; $file =~ s-^./--; diff --git a/WebKitTools/Scripts/new-run-webkit-httpd b/WebKitTools/Scripts/new-run-webkit-httpd new file mode 100755 index 0000000..f6ec164 --- /dev/null +++ b/WebKitTools/Scripts/new-run-webkit-httpd @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""A utility script for starting and stopping the HTTP server with the + same configuration as used in the layout tests.""" + +# +# FIXME: currently this code only works with the Chromium ports and LigHTTPd. +# It should be made to work on all ports. +# +# This script is also used by Chromium's ui_tests to run http layout tests +# in a browser. +# +import optparse +import os +import sys +import tempfile + +scripts_directory = os.path.dirname(os.path.abspath(sys.argv[0])) +webkitpy_directory = os.path.join(scripts_directory, "webkitpy") +sys.path.append(os.path.join(webkitpy_directory, "layout_tests")) + +import port +from port import http_server + +def run(options): + if not options.server: + print ('Usage: %s --server {start|stop} [--root=root_dir]' + ' [--port=port_number]' % sys.argv[0]) + else: + if (options.root is None) and (options.port is not None): + # specifying root but not port means we want httpd on default + # set of ports that LayoutTest use, but pointing to a different + # source of tests. Specifying port but no root does not seem + # meaningful. + raise 'Specifying port requires also a root.' + port_obj = port.get(None, options) + httpd = http_server.Lighttpd(port_obj, + tempfile.gettempdir(), + port=options.port, + root=options.root, + run_background=options.run_background) + if options.server == 'start': + httpd.start() + else: + httpd.stop(force=True) + + +def main(): + option_parser = optparse.OptionParser() + option_parser.add_option('-k', '--server', + help='Server action (start|stop)') + option_parser.add_option('-p', '--port', + help='Port to listen on (overrides layout test ports)') + option_parser.add_option('-r', '--root', + help='Absolute path to DocumentRoot (overrides layout test roots)') + option_parser.add_option('--register_cygwin', action="store_true", + dest="register_cygwin", help='Register Cygwin paths (on Win try bots)') + option_parser.add_option('--run_background', action="store_true", + dest="run_background", + help='Run on background (for running as UI test)') + options, args = option_parser.parse_args() + + # FIXME: Make this work with other ports as well. + options.chromium = True + + run(options) + + +if '__main__' == __name__: + main() diff --git a/WebKitTools/Scripts/new-run-webkit-tests b/WebKitTools/Scripts/new-run-webkit-tests new file mode 100755 index 0000000..9fcacaa --- /dev/null +++ b/WebKitTools/Scripts/new-run-webkit-tests @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Wrapper around webkitpy/layout_tests/run_webkit_tests.py""" +import signal +import sys + +import webkitpy.layout_tests.run_webkit_tests as run_webkit_tests + +if __name__ == '__main__': + try: + sys.exit(run_webkit_tests.main()) + except KeyboardInterrupt: + # this mirrors what the shell normally does + sys.exit(signal.SIGINT + 128) diff --git a/WebKitTools/Scripts/new-run-webkit-websocketserver b/WebKitTools/Scripts/new-run-webkit-websocketserver new file mode 100644 index 0000000..3350582 --- /dev/null +++ b/WebKitTools/Scripts/new-run-webkit-websocketserver @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""A utility script for starting and stopping the web socket server with the + same configuration as used in the layout tests.""" + +import logging +import optparse +import tempfile + +import webkitpy.layout_tests.port.factory as factory +import webkitpy.layout_tests.port.websocket_server as websocket_server + + +def main(): + option_parser = optparse.OptionParser() + option_parser.add_option('--server', type='choice', + choices=['start', 'stop'], default='start', + help='Server action (start|stop).') + option_parser.add_option('-p', '--port', dest='port', + default=None, help='Port to listen on.') + option_parser.add_option('-r', '--root', + help='Absolute path to DocumentRoot ' + '(overrides layout test roots).') + option_parser.add_option('-t', '--tls', dest='use_tls', + action='store_true', + default=False, help='use TLS (wss://).') + option_parser.add_option('-k', '--private_key', dest='private_key', + default='', help='TLS private key file.') + option_parser.add_option('-c', '--certificate', dest='certificate', + default='', help='TLS certificate file.') + option_parser.add_option('--chromium', action='store_true', + dest='chromium', + default=False, + help='Use the Chromium port.') + option_parser.add_option('--register_cygwin', action="store_true", + dest="register_cygwin", + help='Register Cygwin paths (on Win try bots).') + option_parser.add_option('--pidfile', help='path to pid file.') + option_parser.add_option('--output-dir', dest='output_dir', + default=None, help='output directory.') + option_parser.add_option('-v', '--verbose', action='store_true', + default=False, + help='Include debug-level logging.') + options, args = option_parser.parse_args() + + if not options.port: + if options.use_tls: + # FIXME: We shouldn't grab at this private variable. + options.port = websocket_server._DEFAULT_WSS_PORT + else: + # FIXME: We shouldn't grab at this private variable. + options.port = websocket_server._DEFAULT_WS_PORT + + if not options.output_dir: + options.output_dir = tempfile.gettempdir() + + kwds = {'port': options.port, 'use_tls': options.use_tls} + if options.root: + kwds['root'] = options.root + if options.private_key: + kwds['private_key'] = options.private_key + if options.certificate: + kwds['certificate'] = options.certificate + if options.pidfile: + kwds['pidfile'] = options.pidfile + + port_obj = factory.get(options=options) + pywebsocket = websocket_server.PyWebSocket(port_obj, options.output_dir, **kwds) + + log_level = logging.WARN + if options.verbose: + log_level = logging.DEBUG + logging.basicConfig(level=log_level) + + if 'start' == options.server: + pywebsocket.start() + else: + pywebsocket.stop(force=True) + +if '__main__' == __name__: + main() diff --git a/WebKitTools/Scripts/num-cpus b/WebKitTools/Scripts/num-cpus index ede9995..8a8c97f 100755 --- a/WebKitTools/Scripts/num-cpus +++ b/WebKitTools/Scripts/num-cpus @@ -1,3 +1,6 @@ -#!/bin/bash -# Assumes cygwin. -ls /proc/registry/HKEY_LOCAL_MACHINE/HARDWARE/DESCRIPTION/System/CentralProcessor | wc -w +#!/usr/bin/perl -w +use strict; +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; +print numberOfCPUs() . "\n"; diff --git a/WebKitTools/Scripts/old-run-webkit-tests b/WebKitTools/Scripts/old-run-webkit-tests new file mode 100755 index 0000000..af82545 --- /dev/null +++ b/WebKitTools/Scripts/old-run-webkit-tests @@ -0,0 +1,2337 @@ +#!/usr/bin/perl + +# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010 Apple Inc. All rights reserved. +# Copyright (C) 2006 Alexey Proskuryakov (ap@nypop.com) +# Copyright (C) 2007 Matt Lilek (pewtermoose@gmail.com) +# Copyright (C) 2007 Eric Seidel <eric@webkit.org> +# Copyright (C) 2009 Google Inc. All rights reserved. +# Copyright (C) 2009 Andras Becsi (becsi.andras@stud.u-szeged.hu), University of Szeged +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Script to run the WebKit Open Source Project layout tests. + +# Run all the tests passed in on the command line. +# If no tests are passed, find all the .html, .shtml, .xml, .xhtml, .xhtmlmp, .pl, .php (and svg) files in the test directory. + +# Run each text. +# Compare against the existing file xxx-expected.txt. +# If there is a mismatch, generate xxx-actual.txt and xxx-diffs.txt. + +# At the end, report: +# the number of tests that got the expected results +# the number of tests that ran, but did not get the expected results +# the number of tests that failed to run +# the number of tests that were run but had no expected results to compare against + +use strict; +use warnings; + +use Cwd; +use Data::Dumper; +use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK); +use File::Basename; +use File::Copy; +use File::Find; +use File::Path; +use File::Spec; +use File::Spec::Functions; +use File::Temp; +use FindBin; +use Getopt::Long; +use IPC::Open2; +use IPC::Open3; +use Time::HiRes qw(time usleep); + +use List::Util 'shuffle'; + +use lib $FindBin::Bin; +use webkitperl::features; +use webkitperl::httpd; +use webkitdirs; +use VCSUtils; +use POSIX; + +sub buildPlatformResultHierarchy(); +sub buildPlatformTestHierarchy(@); +sub checkPythonVersion(); +sub closeCygpaths(); +sub closeDumpTool(); +sub closeWebSocketServer(); +sub configureAndOpenHTTPDIfNeeded(); +sub countAndPrintLeaks($$$); +sub countFinishedTest($$$$); +sub deleteExpectedAndActualResults($); +sub dumpToolDidCrash(); +sub epiloguesAndPrologues($$); +sub expectedDirectoryForTest($;$;$); +sub fileNameWithNumber($$); +sub htmlForResultsSection(\@$&); +sub isTextOnlyTest($); +sub launchWithEnv(\@\%); +sub resolveAndMakeTestResultsDirectory(); +sub numericcmp($$); +sub openDiffTool(); +sub openDumpTool(); +sub parseLeaksandPrintUniqueLeaks(); +sub openWebSocketServerIfNeeded(); +sub pathcmp($$); +sub printFailureMessageForTest($$); +sub processIgnoreTests($$); +sub readFromDumpToolWithTimer(**); +sub readSkippedFiles($); +sub recordActualResultsAndDiff($$); +sub sampleDumpTool(); +sub setFileHandleNonBlocking(*$); +sub slowestcmp($$); +sub splitpath($); +sub stopRunningTestsEarlyIfNeeded(); +sub stripExtension($); +sub stripMetrics($$); +sub testCrashedOrTimedOut($$$$$); +sub toURL($); +sub toWindowsPath($); +sub validateSkippedArg($$;$); +sub writeToFile($$); + +# Argument handling +my $addPlatformExceptions = 0; +my $complexText = 0; +my $exitAfterNFailures = 0; +my $exitAfterNCrashesOrTimeouts = 0; +my $generateNewResults = isAppleMacWebKit() ? 1 : 0; +my $guardMalloc = ''; +# FIXME: Dynamic HTTP-port configuration in this file is wrong. The various +# apache config files in LayoutTests/http/config govern the port numbers. +# Dynamic configuration as-written will also cause random failures in +# an IPv6 environment. See https://bugs.webkit.org/show_bug.cgi?id=37104. +my $httpdPort = 8000; +my $httpdSSLPort = 8443; +my $ignoreMetrics = 0; +my $webSocketPort = 8880; +# wss is disabled until all platforms support pyOpenSSL. +# my $webSocketSecurePort = 9323; +my $ignoreTests = ''; +my $iterations = 1; +my $launchSafari = 1; +my $mergeDepth; +my $pixelTests = ''; +my $platform; +my $quiet = ''; +my $randomizeTests = 0; +my $repeatEach = 1; +my $report10Slowest = 0; +my $resetResults = 0; +my $reverseTests = 0; +my $root; +my $runSample = 1; +my $shouldCheckLeaks = 0; +my $showHelp = 0; +my $stripEditingCallbacks = isCygwin(); +my $testHTTP = 1; +my $testWebSocket = 1; +my $testMedia = 1; +my $tmpDir = "/tmp"; +my $testResultsDirectory = File::Spec->catfile($tmpDir, "layout-test-results"); +my $testsPerDumpTool = 1000; +my $threaded = 0; +my $html5treebuilder = 0; +# DumpRenderTree has an internal timeout of 30 seconds, so this must be > 30. +my $timeoutSeconds = 35; +my $tolerance = 0; +my $treatSkipped = "default"; +my $useRemoteLinksToTests = 0; +my $useValgrind = 0; +my $verbose = 0; +my $shouldWaitForHTTPD = 0; +my $useWebKitTestRunner = 0; + +my @leaksFilenames; + +if (isWindows() || isMsys()) { + print "This script has to be run under Cygwin to function correctly.\n"; + exit 1; +} + +# Default to --no-http for wx for now. +$testHTTP = 0 if (isWx()); + +my $expectedTag = "expected"; +my $actualTag = "actual"; +my $prettyDiffTag = "pretty-diff"; +my $diffsTag = "diffs"; +my $errorTag = "stderr"; + +my $realPlatform; + +my @macPlatforms = ("mac-tiger", "mac-leopard", "mac-snowleopard", "mac"); +my @winPlatforms = ("win-xp", "win-vista", "win-7", "win"); + +if (isAppleMacWebKit()) { + if (isTiger()) { + $platform = "mac-tiger"; + $tolerance = 1.0; + } elsif (isLeopard()) { + $platform = "mac-leopard"; + $tolerance = 0.1; + } elsif (isSnowLeopard()) { + $platform = "mac-snowleopard"; + $tolerance = 0.1; + } else { + $platform = "mac"; + } +} elsif (isQt()) { + if (isDarwin()) { + $platform = "qt-mac"; + } elsif (isLinux()) { + $platform = "qt-linux"; + } elsif (isWindows() || isCygwin()) { + $platform = "qt-win"; + } else { + $platform = "qt"; + } +} elsif (isGtk()) { + $platform = "gtk"; +} elsif (isWx()) { + $platform = "wx"; +} elsif (isCygwin()) { + if (isWindowsXP()) { + $platform = "win-xp"; + } elsif (isWindowsVista()) { + $platform = "win-vista"; + } elsif (isWindows7()) { + $platform = "win-7"; + } else { + $platform = "win"; + } +} + +if (isQt() || isGtk() || isCygwin()) { + my $testfontPath = $ENV{"WEBKIT_TESTFONTS"}; + if (!$testfontPath || !-d "$testfontPath") { + print "The WEBKIT_TESTFONTS environment variable is not defined or not set properly\n"; + print "You must set it before running the tests.\n"; + print "Use git to grab the actual fonts from http://gitorious.org/qtwebkit/testfonts\n"; + exit 1; + } +} + +if (!defined($platform)) { + print "WARNING: Your platform is not recognized. Any platform-specific results will be generated in platform/undefined.\n"; + $platform = "undefined"; +} + +if (!checkPythonVersion()) { + print "WARNING: Your platform does not have Python 2.5+, which is required to run websocket server, so disabling websocket/tests.\n"; + $testWebSocket = 0; +} + +my $programName = basename($0); +my $launchSafariDefault = $launchSafari ? "launch" : "do not launch"; +my $httpDefault = $testHTTP ? "run" : "do not run"; +my $sampleDefault = $runSample ? "run" : "do not run"; + +my $usage = <<EOF; +Usage: $programName [options] [testdir|testpath ...] + --add-platform-exceptions Put new results for non-platform-specific failing tests into the platform-specific results directory + --complex-text Use the complex text code path for all text (Mac OS X and Windows only) + -c|--configuration config Set DumpRenderTree build configuration + -g|--guard-malloc Enable malloc guard + --exit-after-n-failures N Exit after the first N failures (includes crashes) instead of running all tests + --exit-after-n-crashes-or-timeouts N + Exit after the first N crashes instead of running all tests + -h|--help Show this help message + --[no-]http Run (or do not run) http tests (default: $httpDefault) + --[no-]wait-for-httpd Wait for httpd if some other test session is using it already (same as WEBKIT_WAIT_FOR_HTTPD=1). (default: $shouldWaitForHTTPD) + -i|--ignore-tests Comma-separated list of directories or tests to ignore + --iterations n Number of times to run the set of tests (e.g. ABCABCABC) + --[no-]launch-safari Launch (or do not launch) Safari to display test results (default: $launchSafariDefault) + -l|--leaks Enable leaks checking + --[no-]new-test-results Generate results for new tests + --nthly n Restart DumpRenderTree every n tests (default: $testsPerDumpTool) + -p|--pixel-tests Enable pixel tests + --tolerance t Ignore image differences less than this percentage (default: $tolerance) + --platform Override the detected platform to use for tests and results (default: $platform) + --port Web server port to use with http tests + -q|--quiet Less verbose output + --reset-results Reset ALL results (including pixel tests if --pixel-tests is set) + -o|--results-directory Output results directory (default: $testResultsDirectory) + --random Run the tests in a random order + --repeat-each n Number of times to run each test (e.g. AAABBBCCC) + --reverse Run the tests in reverse alphabetical order + --root Path to root tools build + --[no-]sample-on-timeout Run sample on timeout (default: $sampleDefault) (Mac OS X only) + -1|--singly Isolate each test case run (implies --nthly 1 --verbose) + --skipped=[default|ignore|only] Specifies how to treat the Skipped file + default: Tests/directories listed in the Skipped file are not tested + ignore: The Skipped file is ignored + only: Only those tests/directories listed in the Skipped file will be run + --slowest Report the 10 slowest tests + --ignore-metrics Ignore metrics in tests + --[no-]strip-editing-callbacks Remove editing callbacks from expected results + -t|--threaded Run a concurrent JavaScript thead with each test + --html5-treebuilder Run the tests using the HTML5 tree builder + --timeout t Sets the number of seconds before a test times out (default: $timeoutSeconds) + --valgrind Run DumpRenderTree inside valgrind (Qt/Linux only) + -v|--verbose More verbose output (overrides --quiet) + -m|--merge-leak-depth arg Merges leak callStacks and prints the number of unique leaks beneath a callstack depth of arg. Defaults to 5. + --use-remote-links-to-tests Link to test files within the SVN repository in the results. + --webkit-test-runner Use WebKitTestRunner rather than DumpRenderTree. +EOF + +setConfiguration(); + +my $getOptionsResult = GetOptions( + 'add-platform-exceptions' => \$addPlatformExceptions, + 'complex-text' => \$complexText, + 'exit-after-n-failures=i' => \$exitAfterNFailures, + 'exit-after-n-crashes-or-timeouts=i' => \$exitAfterNCrashesOrTimeouts, + 'guard-malloc|g' => \$guardMalloc, + 'help|h' => \$showHelp, + 'http!' => \$testHTTP, + 'wait-for-httpd!' => \$shouldWaitForHTTPD, + 'ignore-metrics!' => \$ignoreMetrics, + 'ignore-tests|i=s' => \$ignoreTests, + 'iterations=i' => \$iterations, + 'launch-safari!' => \$launchSafari, + 'leaks|l' => \$shouldCheckLeaks, + 'merge-leak-depth|m:5' => \$mergeDepth, + 'new-test-results!' => \$generateNewResults, + 'nthly=i' => \$testsPerDumpTool, + 'pixel-tests|p' => \$pixelTests, + 'platform=s' => \$platform, + 'port=i' => \$httpdPort, + 'quiet|q' => \$quiet, + 'random' => \$randomizeTests, + 'repeat-each=i' => \$repeatEach, + 'reset-results' => \$resetResults, + 'results-directory|o=s' => \$testResultsDirectory, + 'reverse' => \$reverseTests, + 'root=s' => \$root, + 'sample-on-timeout!' => \$runSample, + 'singly|1' => sub { $testsPerDumpTool = 1; }, + 'skipped=s' => \&validateSkippedArg, + 'slowest' => \$report10Slowest, + 'strip-editing-callbacks!' => \$stripEditingCallbacks, + 'threaded|t' => \$threaded, + 'html5-treebuilder' => \$html5treebuilder, + 'timeout=i' => \$timeoutSeconds, + 'tolerance=f' => \$tolerance, + 'use-remote-links-to-tests' => \$useRemoteLinksToTests, + 'valgrind' => \$useValgrind, + 'verbose|v' => \$verbose, + 'webkit-test-runner' => \$useWebKitTestRunner, +); + +if (!$getOptionsResult || $showHelp) { + print STDERR $usage; + exit 1; +} + +if ($useWebKitTestRunner) { + if (isAppleMacWebKit()) { + $realPlatform = $platform; + $platform = "mac-wk2"; + } +} + + +my $ignoreSkipped = $treatSkipped eq "ignore"; +my $skippedOnly = $treatSkipped eq "only"; + +my $configuration = configuration(); + +# We need an environment variable to be able to enable the feature per-slave +$shouldWaitForHTTPD = $ENV{"WEBKIT_WAIT_FOR_HTTPD"} unless ($shouldWaitForHTTPD); +$verbose = 1 if $testsPerDumpTool == 1; + +if ($shouldCheckLeaks && $testsPerDumpTool > 1000) { + print STDERR "\nWARNING: Running more than 1000 tests at a time with MallocStackLogging enabled may cause a crash.\n\n"; +} + +# Stack logging does not play well with QuickTime on Tiger (rdar://problem/5537157) +$testMedia = 0 if $shouldCheckLeaks && isTiger(); + +# Generating remote links causes a lot of unnecessary spew on GTK build bot +$useRemoteLinksToTests = 0 if isGtk(); + +setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root)); +my $productDir = productDir(); +$productDir .= "/bin" if isQt(); +$productDir .= "/Programs" if isGtk(); + +chdirWebKit(); + +my $dumpToolName = $useWebKitTestRunner ? "WebKitTestRunner" : "DumpRenderTree"; + +if (!defined($root)) { + my $dumpToolBuildScript = "build-" . lc($dumpToolName); + print STDERR "Running $dumpToolBuildScript\n"; + + local *DEVNULL; + my ($childIn, $childOut, $childErr); + if ($quiet) { + open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null"; + $childOut = ">&DEVNULL"; + $childErr = ">&DEVNULL"; + } else { + # When not quiet, let the child use our stdout/stderr. + $childOut = ">&STDOUT"; + $childErr = ">&STDERR"; + } + + my @args = argumentsForConfiguration(); + my $buildProcess = open3($childIn, $childOut, $childErr, "WebKitTools/Scripts/$dumpToolBuildScript", @args) or die "Failed to run build-dumprendertree"; + close($childIn); + waitpid $buildProcess, 0; + my $buildResult = $?; + close($childOut); + close($childErr); + + close DEVNULL if ($quiet); + + if ($buildResult) { + print STDERR "Compiling $dumpToolName failed!\n"; + exit exitStatus($buildResult); + } +} + +$dumpToolName .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/; +my $dumpTool = "$productDir/$dumpToolName"; +die "can't find executable $dumpToolName (looked in $productDir)\n" unless -x $dumpTool; + +my $imageDiffTool = "$productDir/ImageDiff"; +$imageDiffTool .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/; +die "can't find executable $imageDiffTool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool; + +checkFrameworks() unless isCygwin(); + +if (isAppleMacWebKit()) { + push @INC, $productDir; + require DumpRenderTreeSupport; +} + +my $layoutTestsName = "LayoutTests"; +my $testDirectory = File::Spec->rel2abs($layoutTestsName); +my $expectedDirectory = $testDirectory; +my $platformBaseDirectory = catdir($testDirectory, "platform"); +my $platformTestDirectory = catdir($platformBaseDirectory, $platform); +my @platformResultHierarchy = buildPlatformResultHierarchy(); +my @platformTestHierarchy = buildPlatformTestHierarchy(@platformResultHierarchy); + +$expectedDirectory = $ENV{"WebKitExpectedTestResultsDirectory"} if $ENV{"WebKitExpectedTestResultsDirectory"}; + +$testResultsDirectory = File::Spec->rel2abs($testResultsDirectory); +my $testResults = File::Spec->catfile($testResultsDirectory, "results.html"); + +if (isAppleMacWebKit()) { + print STDERR "Compiling Java tests\n"; + my $javaTestsDirectory = catdir($testDirectory, "java"); + + if (system("/usr/bin/make", "-C", "$javaTestsDirectory")) { + exit 1; + } +} + + +print "Running tests from $testDirectory\n"; +if ($pixelTests) { + print "Enabling pixel tests with a tolerance of $tolerance%\n"; + if (isDarwin()) { + print "WARNING: Temporarily changing the main display color profile:\n"; + print "\tThe colors on your screen will change for the duration of the testing.\n"; + print "\tThis allows the pixel tests to have consistent color values across all machines.\n"; + + if (isPerianInstalled()) { + print "WARNING: Perian's QuickTime component is installed and this may affect pixel test results!\n"; + print "\tYou should avoid generating new pixel results in this environment.\n"; + print "\tSee https://bugs.webkit.org/show_bug.cgi?id=22615 for details.\n"; + } + } +} + +system "ln", "-s", $testDirectory, "/tmp/LayoutTests" unless -x "/tmp/LayoutTests"; + +my %ignoredFiles = ( "results.html" => 1 ); +my %ignoredDirectories = map { $_ => 1 } qw(platform); +my %ignoredLocalDirectories = map { $_ => 1 } qw(.svn _svn resources script-tests); +my %supportedFileExtensions = map { $_ => 1 } qw(html shtml xml xhtml xhtmlmp pl php); + +if (!checkWebCoreFeatureSupport("MathML", 0)) { + $ignoredDirectories{'mathml'} = 1; +} + +# FIXME: We should fix webkitperl/features.pm:hasFeature() to do the correct feature detection for Cygwin. +if (checkWebCoreFeatureSupport("SVG", 0)) { + $supportedFileExtensions{'svg'} = 1; +} elsif (isCygwin()) { + $supportedFileExtensions{'svg'} = 1; +} else { + $ignoredLocalDirectories{'svg'} = 1; +} + +if (!$testHTTP) { + $ignoredDirectories{'http'} = 1; + $ignoredDirectories{'websocket'} = 1; +} +if (!$testWebSocket) { + $ignoredDirectories{'websocket'} = 1; +} + +if (!$testMedia) { + $ignoredDirectories{'media'} = 1; + $ignoredDirectories{'http/tests/media'} = 1; +} + +my $supportedFeaturesResult = ""; + +if (isCygwin()) { + # Collect supported features list + setPathForRunningWebKitApp(\%ENV); + my $supportedFeaturesCommand = $dumpTool . " --print-supported-features 2>&1"; + $supportedFeaturesResult = `$supportedFeaturesCommand 2>&1`; +} + +my $hasAcceleratedCompositing = 0; +my $has3DRendering = 0; + +if (isCygwin()) { + $hasAcceleratedCompositing = $supportedFeaturesResult =~ /AcceleratedCompositing/; + $has3DRendering = $supportedFeaturesResult =~ /3DRendering/; +} else { + $hasAcceleratedCompositing = checkWebCoreFeatureSupport("Accelerated Compositing", 0); + $has3DRendering = checkWebCoreFeatureSupport("3D Rendering", 0); +} + +if (!$hasAcceleratedCompositing) { + $ignoredDirectories{'compositing'} = 1; +} + +if (!$has3DRendering) { + $ignoredDirectories{'animations/3d'} = 1; + $ignoredDirectories{'transforms/3d'} = 1; +} + +if (!checkWebCoreFeatureSupport("3D Canvas", 0)) { + $ignoredDirectories{'fast/canvas/webgl'} = 1; + $ignoredDirectories{'compositing/webgl'} = 1; +} + +if (checkWebCoreFeatureSupport("WML", 0)) { + $supportedFileExtensions{'wml'} = 1; +} else { + $ignoredDirectories{'http/tests/wml'} = 1; + $ignoredDirectories{'fast/wml'} = 1; + $ignoredDirectories{'wml'} = 1; +} + +if (!checkWebCoreFeatureSupport("WCSS", 0)) { + $ignoredDirectories{'fast/wcss'} = 1; +} + +if (!checkWebCoreFeatureSupport("XHTMLMP", 0)) { + $ignoredDirectories{'fast/xhtmlmp'} = 1; +} + +processIgnoreTests($ignoreTests, "ignore-tests") if $ignoreTests; +if (!$ignoreSkipped) { + if (!$skippedOnly || @ARGV == 0) { + readSkippedFiles(""); + } else { + # Since readSkippedFiles() appends to @ARGV, we must use a foreach + # loop so that we only iterate over the original argument list. + foreach my $argnum (0 .. $#ARGV) { + readSkippedFiles(shift @ARGV); + } + } +} + +my @tests = findTestsToRun(); + +die "no tests to run\n" if !@tests; + +my %counts; +my %tests; +my %imagesPresent; +my %imageDifferences; +my %durations; +my $count = 0; +my $leaksOutputFileNumber = 1; +my $totalLeaks = 0; + +my @toolArgs = (); +push @toolArgs, "--pixel-tests" if $pixelTests; +push @toolArgs, "--threaded" if $threaded; +push @toolArgs, "--html5-treebuilder" if $html5treebuilder; +push @toolArgs, "--complex-text" if $complexText; +push @toolArgs, "-"; + +my @diffToolArgs = (); +push @diffToolArgs, "--tolerance", $tolerance; + +$| = 1; + +my $dumpToolPID; +my $isDumpToolOpen = 0; +my $dumpToolCrashed = 0; +my $imageDiffToolPID; +my $isDiffToolOpen = 0; + +my $atLineStart = 1; +my $lastDirectory = ""; + +my $isHttpdOpen = 0; +my $isWebSocketServerOpen = 0; +my $webSocketServerPidFile = 0; +my $failedToStartWebSocketServer = 0; +# wss is disabled until all platforms support pyOpenSSL. +# my $webSocketSecureServerPID = 0; + +sub catch_pipe { $dumpToolCrashed = 1; } +$SIG{"PIPE"} = "catch_pipe"; + +print "Testing ", scalar @tests, " test cases"; +print " $iterations times" if ($iterations > 1); +print ", repeating each test $repeatEach times" if ($repeatEach > 1); +print ".\n"; + +my $overallStartTime = time; + +my %expectedResultPaths; + +my @originalTests = @tests; +# Add individual test repetitions +if ($repeatEach > 1) { + @tests = (); + foreach my $test (@originalTests) { + for (my $i = 0; $i < $repeatEach; $i++) { + push(@tests, $test); + } + } +} +# Add test set repetitions +for (my $i = 1; $i < $iterations; $i++) { + push(@tests, @originalTests); +} + +for my $test (@tests) { + my $newDumpTool = not $isDumpToolOpen; + openDumpTool(); + + my $base = stripExtension($test); + my $expectedExtension = ".txt"; + + my $dir = $base; + $dir =~ s|/[^/]+$||; + + if ($newDumpTool || $dir ne $lastDirectory) { + foreach my $logue (epiloguesAndPrologues($newDumpTool ? "" : $lastDirectory, $dir)) { + if (isCygwin()) { + $logue = toWindowsPath($logue); + } else { + $logue = canonpath($logue); + } + if ($verbose) { + print "running epilogue or prologue $logue\n"; + } + print OUT "$logue\n"; + # Throw away output from DumpRenderTree. + # Once for the test output and once for pixel results (empty) + while (<IN>) { + last if /#EOF/; + } + while (<IN>) { + last if /#EOF/; + } + } + } + + if ($verbose) { + print "running $test -> "; + $atLineStart = 0; + } elsif (!$quiet) { + if ($dir ne $lastDirectory) { + print "\n" unless $atLineStart; + print "$dir "; + } + print "."; + $atLineStart = 0; + } + + $lastDirectory = $dir; + + my $result; + + my $startTime = time if $report10Slowest; + + # Try to read expected hash file for pixel tests + my $suffixExpectedHash = ""; + if ($pixelTests && !$resetResults) { + my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); + if (open EXPECTEDHASH, "$expectedPixelDir/$base-$expectedTag.checksum") { + my $expectedHash = <EXPECTEDHASH>; + chomp($expectedHash); + close EXPECTEDHASH; + + # Format expected hash into a suffix string that is appended to the path / URL passed to DRT + $suffixExpectedHash = "'$expectedHash"; + } + } + + if ($test =~ /^http\//) { + configureAndOpenHTTPDIfNeeded(); + if ($test !~ /^http\/tests\/local\// && $test !~ /^http\/tests\/ssl\// && $test !~ /^http\/tests\/wml\// && $test !~ /^http\/tests\/media\//) { + my $path = canonpath($test); + $path =~ s/^http\/tests\///; + print OUT "http://127.0.0.1:$httpdPort/$path$suffixExpectedHash\n"; + } elsif ($test =~ /^http\/tests\/ssl\//) { + my $path = canonpath($test); + $path =~ s/^http\/tests\///; + print OUT "https://127.0.0.1:$httpdSSLPort/$path$suffixExpectedHash\n"; + } else { + my $testPath = "$testDirectory/$test"; + if (isCygwin()) { + $testPath = toWindowsPath($testPath); + } else { + $testPath = canonpath($testPath); + } + print OUT "$testPath$suffixExpectedHash\n"; + } + } elsif ($test =~ /^websocket\//) { + if ($test =~ /^websocket\/tests\/local\//) { + my $testPath = "$testDirectory/$test"; + if (isCygwin()) { + $testPath = toWindowsPath($testPath); + } else { + $testPath = canonpath($testPath); + } + print OUT "$testPath\n"; + } else { + if (openWebSocketServerIfNeeded()) { + my $path = canonpath($test); + if ($test =~ /^websocket\/tests\/ssl\//) { + # wss is disabled until all platforms support pyOpenSSL. + print STDERR "Error: wss is disabled until all platforms support pyOpenSSL."; + # print OUT "https://127.0.0.1:$webSocketSecurePort/$path\n"; + } else { + print OUT "http://127.0.0.1:$webSocketPort/$path\n"; + } + } else { + # We failed to launch the WebSocket server. Display a useful error message rather than attempting + # to run tests that expect the server to be available. + my $errorMessagePath = "$testDirectory/websocket/resources/server-failed-to-start.html"; + $errorMessagePath = isCygwin() ? toWindowsPath($errorMessagePath) : canonpath($errorMessagePath); + print OUT "$errorMessagePath\n"; + } + } + } else { + my $testPath = "$testDirectory/$test"; + if (isCygwin()) { + $testPath = toWindowsPath($testPath); + } else { + $testPath = canonpath($testPath); + } + print OUT "$testPath$suffixExpectedHash\n" if defined $testPath; + } + + # DumpRenderTree is expected to dump two "blocks" to stdout for each test. + # Each block is terminated by a #EOF on a line by itself. + # The first block is the output of the test (in text, RenderTree or other formats). + # The second block is for optional pixel data in PNG format, and may be empty if + # pixel tests are not being run, or the test does not dump pixels (e.g. text tests). + my $readResults = readFromDumpToolWithTimer(IN, ERROR); + + my $actual = $readResults->{output}; + my $error = $readResults->{error}; + + $expectedExtension = $readResults->{extension}; + my $expectedFileName = "$base-$expectedTag.$expectedExtension"; + + my $isText = isTextOnlyTest($actual); + + my $expectedDir = expectedDirectoryForTest($base, $isText, $expectedExtension); + $expectedResultPaths{$base} = "$expectedDir/$expectedFileName"; + + unless ($readResults->{status} eq "success") { + my $crashed = $readResults->{status} eq "crashed"; + testCrashedOrTimedOut($test, $base, $crashed, $actual, $error); + countFinishedTest($test, $base, $crashed ? "crash" : "timedout", 0); + last if stopRunningTestsEarlyIfNeeded(); + next; + } + + $durations{$test} = time - $startTime if $report10Slowest; + + my $expected; + + if (!$resetResults && open EXPECTED, "<", "$expectedDir/$expectedFileName") { + $expected = ""; + while (<EXPECTED>) { + next if $stripEditingCallbacks && $_ =~ /^EDITING DELEGATE:/; + $expected .= $_; + } + close EXPECTED; + } + + if ($ignoreMetrics && !$isText && defined $expected) { + ($actual, $expected) = stripMetrics($actual, $expected); + } + + if ($shouldCheckLeaks && $testsPerDumpTool == 1) { + print " $test -> "; + } + + my $actualPNG = ""; + my $diffPNG = ""; + my $diffPercentage = 0; + my $diffResult = "passed"; + + my $actualHash = ""; + my $expectedHash = ""; + my $actualPNGSize = 0; + + while (<IN>) { + last if /#EOF/; + if (/ActualHash: ([a-f0-9]{32})/) { + $actualHash = $1; + } elsif (/ExpectedHash: ([a-f0-9]{32})/) { + $expectedHash = $1; + } elsif (/Content-Length: (\d+)\s*/) { + $actualPNGSize = $1; + read(IN, $actualPNG, $actualPNGSize); + } + } + + if ($verbose && $pixelTests && !$resetResults && $actualPNGSize) { + if ($actualHash eq "" && $expectedHash eq "") { + printFailureMessageForTest($test, "WARNING: actual & expected pixel hashes are missing!"); + } elsif ($actualHash eq "") { + printFailureMessageForTest($test, "WARNING: actual pixel hash is missing!"); + } elsif ($expectedHash eq "") { + printFailureMessageForTest($test, "WARNING: expected pixel hash is missing!"); + } + } + + if ($actualPNGSize > 0) { + my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); + + if (!$resetResults && ($expectedHash ne $actualHash || ($actualHash eq "" && $expectedHash eq ""))) { + if (-f "$expectedPixelDir/$base-$expectedTag.png") { + my $expectedPNGSize = -s "$expectedPixelDir/$base-$expectedTag.png"; + my $expectedPNG = ""; + open EXPECTEDPNG, "$expectedPixelDir/$base-$expectedTag.png"; + read(EXPECTEDPNG, $expectedPNG, $expectedPNGSize); + + openDiffTool(); + print DIFFOUT "Content-Length: $actualPNGSize\n"; + print DIFFOUT $actualPNG; + + print DIFFOUT "Content-Length: $expectedPNGSize\n"; + print DIFFOUT $expectedPNG; + + while (<DIFFIN>) { + last if /^error/ || /^diff:/; + if (/Content-Length: (\d+)\s*/) { + read(DIFFIN, $diffPNG, $1); + } + } + + if (/^diff: (.+)% (passed|failed)/) { + $diffPercentage = $1 + 0; + $imageDifferences{$base} = $diffPercentage; + $diffResult = $2; + } + + if (!$diffPercentage) { + printFailureMessageForTest($test, "pixel hash failed (but pixel test still passes)"); + } + } elsif ($verbose) { + printFailureMessageForTest($test, "WARNING: expected image is missing!"); + } + } + + if ($resetResults || !-f "$expectedPixelDir/$base-$expectedTag.png") { + mkpath catfile($expectedPixelDir, dirname($base)) if $testDirectory ne $expectedPixelDir; + writeToFile("$expectedPixelDir/$base-$expectedTag.png", $actualPNG); + } + + if ($actualHash ne "" && ($resetResults || !-f "$expectedPixelDir/$base-$expectedTag.checksum")) { + writeToFile("$expectedPixelDir/$base-$expectedTag.checksum", $actualHash); + } + } + + if (dumpToolDidCrash()) { + $result = "crash"; + testCrashedOrTimedOut($test, $base, 1, $actual, $error); + } elsif (!defined $expected) { + if ($verbose) { + print "new " . ($resetResults ? "result" : "test") ."\n"; + $atLineStart = 1; + } + $result = "new"; + + if ($generateNewResults || $resetResults) { + mkpath catfile($expectedDir, dirname($base)) if $testDirectory ne $expectedDir; + writeToFile("$expectedDir/$expectedFileName", $actual); + } + deleteExpectedAndActualResults($base); + recordActualResultsAndDiff($base, $actual); + if (!$resetResults) { + # Always print the file name for new tests, as they will probably need some manual inspection. + # in verbose mode we already printed the test case, so no need to do it again. + unless ($verbose) { + print "\n" unless $atLineStart; + print "$test -> "; + } + my $resultsDir = catdir($expectedDir, dirname($base)); + if ($generateNewResults) { + print "new (results generated in $resultsDir)\n"; + } else { + print "new\n"; + } + $atLineStart = 1; + } + } elsif ($actual eq $expected && $diffResult eq "passed") { + if ($verbose) { + print "succeeded\n"; + $atLineStart = 1; + } + $result = "match"; + deleteExpectedAndActualResults($base); + } else { + $result = "mismatch"; + + my $pixelTestFailed = $pixelTests && $diffPNG && $diffPNG ne ""; + my $testFailed = $actual ne $expected; + + my $message = !$testFailed ? "pixel test failed" : "failed"; + + if (($testFailed || $pixelTestFailed) && $addPlatformExceptions) { + my $testBase = catfile($testDirectory, $base); + my $expectedBase = catfile($expectedDir, $base); + my $testIsMaximallyPlatformSpecific = $testBase =~ m|^\Q$platformTestDirectory\E/|; + my $expectedResultIsMaximallyPlatformSpecific = $expectedBase =~ m|^\Q$platformTestDirectory\E/|; + if (!$testIsMaximallyPlatformSpecific && !$expectedResultIsMaximallyPlatformSpecific) { + mkpath catfile($platformTestDirectory, dirname($base)); + if ($testFailed) { + my $expectedFile = catfile($platformTestDirectory, "$expectedFileName"); + writeToFile("$expectedFile", $actual); + } + if ($pixelTestFailed) { + my $expectedFile = catfile($platformTestDirectory, "$base-$expectedTag.checksum"); + writeToFile("$expectedFile", $actualHash); + + $expectedFile = catfile($platformTestDirectory, "$base-$expectedTag.png"); + writeToFile("$expectedFile", $actualPNG); + } + $message .= " (results generated in $platformTestDirectory)"; + } + } + + printFailureMessageForTest($test, $message); + + my $dir = "$testResultsDirectory/$base"; + $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; + my $testName = $1; + mkpath $dir; + + deleteExpectedAndActualResults($base); + recordActualResultsAndDiff($base, $actual); + + if ($pixelTestFailed) { + $imagesPresent{$base} = 1; + + writeToFile("$testResultsDirectory/$base-$actualTag.png", $actualPNG); + writeToFile("$testResultsDirectory/$base-$diffsTag.png", $diffPNG); + + my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); + copy("$expectedPixelDir/$base-$expectedTag.png", "$testResultsDirectory/$base-$expectedTag.png"); + + open DIFFHTML, ">$testResultsDirectory/$base-$diffsTag.html" or die; + print DIFFHTML "<html>\n"; + print DIFFHTML "<head>\n"; + print DIFFHTML "<title>$base Image Compare</title>\n"; + print DIFFHTML "<script language=\"Javascript\" type=\"text/javascript\">\n"; + print DIFFHTML "var currentImage = 0;\n"; + print DIFFHTML "var imageNames = new Array(\"Actual\", \"Expected\");\n"; + print DIFFHTML "var imagePaths = new Array(\"$testName-$actualTag.png\", \"$testName-$expectedTag.png\");\n"; + if (-f "$testDirectory/$base-w3c.png") { + copy("$testDirectory/$base-w3c.png", "$testResultsDirectory/$base-w3c.png"); + print DIFFHTML "imageNames.push(\"W3C\");\n"; + print DIFFHTML "imagePaths.push(\"$testName-w3c.png\");\n"; + } + print DIFFHTML "function animateImage() {\n"; + print DIFFHTML " var image = document.getElementById(\"animatedImage\");\n"; + print DIFFHTML " var imageText = document.getElementById(\"imageText\");\n"; + print DIFFHTML " image.src = imagePaths[currentImage];\n"; + print DIFFHTML " imageText.innerHTML = imageNames[currentImage] + \" Image\";\n"; + print DIFFHTML " currentImage = (currentImage + 1) % imageNames.length;\n"; + print DIFFHTML " setTimeout('animateImage()',2000);\n"; + print DIFFHTML "}\n"; + print DIFFHTML "</script>\n"; + print DIFFHTML "</head>\n"; + print DIFFHTML "<body onLoad=\"animateImage();\">\n"; + print DIFFHTML "<table>\n"; + if ($diffPercentage) { + print DIFFHTML "<tr>\n"; + print DIFFHTML "<td>Difference between images: <a href=\"$testName-$diffsTag.png\">$diffPercentage%</a></td>\n"; + print DIFFHTML "</tr>\n"; + } + print DIFFHTML "<tr>\n"; + print DIFFHTML "<td><a href=\"" . toURL("$testDirectory/$test") . "\">test file</a></td>\n"; + print DIFFHTML "</tr>\n"; + print DIFFHTML "<tr>\n"; + print DIFFHTML "<td id=\"imageText\" style=\"text-weight: bold;\">Actual Image</td>\n"; + print DIFFHTML "</tr>\n"; + print DIFFHTML "<tr>\n"; + print DIFFHTML "<td><img src=\"$testName-$actualTag.png\" id=\"animatedImage\"></td>\n"; + print DIFFHTML "</tr>\n"; + print DIFFHTML "</table>\n"; + print DIFFHTML "</body>\n"; + print DIFFHTML "</html>\n"; + } + } + + if ($error) { + my $dir = "$testResultsDirectory/$base"; + $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; + mkpath $dir; + + writeToFile("$testResultsDirectory/$base-$errorTag.txt", $error); + + $counts{error}++; + push @{$tests{error}}, $test; + } + + countFinishedTest($test, $base, $result, $isText); + last if stopRunningTestsEarlyIfNeeded(); +} + +my $totalTestingTime = time - $overallStartTime; +my $waitTime = getWaitTime(); +if ($waitTime > 0.1) { + my $normalizedTestingTime = $totalTestingTime - $waitTime; + printf "\n%0.2fs HTTPD waiting time\n", $waitTime . ""; + printf "%0.2fs normalized testing time", $normalizedTestingTime . ""; +} +printf "\n%0.2fs total testing time\n", $totalTestingTime . ""; + +!$isDumpToolOpen || die "Failed to close $dumpToolName.\n"; + +$isHttpdOpen = !closeHTTPD(); +closeWebSocketServer(); + +# Because multiple instances of this script are running concurrently we cannot +# safely delete this symlink. +# system "rm /tmp/LayoutTests"; + +# FIXME: Do we really want to check the image-comparison tool for leaks every time? +if ($isDiffToolOpen && $shouldCheckLeaks) { + $totalLeaks += countAndPrintLeaks("ImageDiff", $imageDiffToolPID, "$testResultsDirectory/ImageDiff-leaks.txt"); +} + +if ($totalLeaks) { + if ($mergeDepth) { + parseLeaksandPrintUniqueLeaks(); + } else { + print "\nWARNING: $totalLeaks total leaks found!\n"; + print "See above for individual leaks results.\n" if ($leaksOutputFileNumber > 2); + } +} + +close IN; +close OUT; +close ERROR; + +if ($report10Slowest) { + print "\n\nThe 10 slowest tests:\n\n"; + my $count = 0; + for my $test (sort slowestcmp keys %durations) { + printf "%0.2f secs: %s\n", $durations{$test}, $test; + last if ++$count == 10; + } +} + +print "\n"; + +if ($skippedOnly && $counts{"match"}) { + print "The following tests are in the Skipped file (" . File::Spec->abs2rel("$platformTestDirectory/Skipped", $testDirectory) . "), but succeeded:\n"; + foreach my $test (@{$tests{"match"}}) { + print " $test\n"; + } +} + +if ($resetResults || ($counts{match} && $counts{match} == $count)) { + print "all $count test cases succeeded\n"; + unlink $testResults; + exit; +} + +printResults(); + +mkpath $testResultsDirectory; + +open HTML, ">", $testResults or die "Failed to open $testResults. $!"; +print HTML "<html>\n"; +print HTML "<head>\n"; +print HTML "<title>Layout Test Results</title>\n"; +print HTML "</head>\n"; +print HTML "<body>\n"; + +if ($ignoreMetrics) { + print HTML "<h4>Tested with metrics ignored.</h4>"; +} + +print HTML htmlForResultsSection(@{$tests{mismatch}}, "Tests where results did not match expected results", \&linksForMismatchTest); +print HTML htmlForResultsSection(@{$tests{timedout}}, "Tests that timed out", \&linksForErrorTest); +print HTML htmlForResultsSection(@{$tests{crash}}, "Tests that caused the DumpRenderTree tool to crash", \&linksForErrorTest); +print HTML htmlForResultsSection(@{$tests{error}}, "Tests that had stderr output", \&linksForErrorTest); +print HTML htmlForResultsSection(@{$tests{new}}, "Tests that had no expected results (probably new)", \&linksForNewTest); + +print HTML "</body>\n"; +print HTML "</html>\n"; +close HTML; + +my @configurationArgs = argumentsForConfiguration(); + +if (isGtk()) { + system "WebKitTools/Scripts/run-launcher", @configurationArgs, "file://".$testResults if $launchSafari; +} elsif (isQt()) { + unshift @configurationArgs, qw(-graphicssystem raster -style windows); + if (isCygwin()) { + $testResults = "/" . toWindowsPath($testResults); + $testResults =~ s/\\/\//g; + } + system "WebKitTools/Scripts/run-launcher", @configurationArgs, "file://".$testResults if $launchSafari; +} elsif (isCygwin()) { + system "cygstart", $testResults if $launchSafari; +} else { + system "WebKitTools/Scripts/run-safari", @configurationArgs, "-NSOpen", $testResults if $launchSafari; +} + +closeCygpaths() if isCygwin(); + +exit 1; + +sub countAndPrintLeaks($$$) +{ + my ($dumpToolName, $dumpToolPID, $leaksFilePath) = @_; + + print "\n" unless $atLineStart; + $atLineStart = 1; + + # We are excluding the following reported leaks so they don't get in our way when looking for WebKit leaks: + # This allows us ignore known leaks and only be alerted when new leaks occur. Some leaks are in the old + # versions of the system frameworks that are being used by the leaks bots. Even though a leak has been + # fixed, it will be listed here until the bot has been updated with the newer frameworks. + + my @typesToExclude = ( + ); + + my @callStacksToExclude = ( + "Flash_EnforceLocalSecurity" # leaks in Flash plug-in code, rdar://problem/4449747 + ); + + if (isTiger()) { + # Leak list for the version of Tiger used on the build bot. + push @callStacksToExclude, ( + "CFRunLoopRunSpecific \\| malloc_zone_malloc", "CFRunLoopRunSpecific \\| CFAllocatorAllocate ", # leak in CFRunLoopRunSpecific, rdar://problem/4670839 + "CGImageSourceGetPropertiesAtIndex", # leak in ImageIO, rdar://problem/4628809 + "FOGetCoveredUnicodeChars", # leak in ATS, rdar://problem/3943604 + "GetLineDirectionPreference", "InitUnicodeUtilities", # leaks tool falsely reporting leak in CFNotificationCenterAddObserver, rdar://problem/4964790 + "ICCFPrefWrapper::GetPrefDictionary", # leaks in Internet Config. code, rdar://problem/4449794 + "NSHTTPURLProtocol setResponseHeader:", # leak in multipart/mixed-replace handling in Foundation, no Radar, but fixed in Leopard + "NSURLCache cachedResponseForRequest", # leak in CFURL cache, rdar://problem/4768430 + "PCFragPrepareClosureFromFile", # leak in Code Fragment Manager, rdar://problem/3426998 + "WebCore::Selection::toRange", # bug in 'leaks', rdar://problem/4967949 + "WebCore::SubresourceLoader::create", # bug in 'leaks', rdar://problem/4985806 + "_CFPreferencesDomainDeepCopyDictionary", # leak in CFPreferences, rdar://problem/4220786 + "_objc_msgForward", # leak in NSSpellChecker, rdar://problem/4965278 + "gldGetString", # leak in OpenGL, rdar://problem/5013699 + "_setDefaultUserInfoFromURL", # leak in NSHTTPAuthenticator, rdar://problem/5546453 + "SSLHandshake", # leak in SSL, rdar://problem/5546440 + "SecCertificateCreateFromData", # leak in SSL code, rdar://problem/4464397 + ); + push @typesToExclude, ( + "THRD", # bug in 'leaks', rdar://problem/3387783 + "DRHT", # ditto (endian little hate i) + ); + } + + if (isLeopard()) { + # Leak list for the version of Leopard used on the build bot. + push @callStacksToExclude, ( + "CFHTTPMessageAppendBytes", # leak in CFNetwork, rdar://problem/5435912 + "sendDidReceiveDataCallback", # leak in CFNetwork, rdar://problem/5441619 + "_CFHTTPReadStreamReadMark", # leak in CFNetwork, rdar://problem/5441468 + "httpProtocolStart", # leak in CFNetwork, rdar://problem/5468837 + "_CFURLConnectionSendCallbacks", # leak in CFNetwork, rdar://problem/5441600 + "DispatchQTMsg", # leak in QuickTime, PPC only, rdar://problem/5667132 + "QTMovieContentView createVisualContext", # leak in QuickTime, PPC only, rdar://problem/5667132 + "_CopyArchitecturesForJVMVersion", # leak in Java, rdar://problem/5910823 + ); + } + + if (isSnowLeopard()) { + push @callStacksToExclude, ( + "readMakerNoteProps", # <rdar://problem/7156432> leak in ImageIO + "QTKitMovieControllerView completeUISetup", # <rdar://problem/7155156> leak in QTKit + "getVMInitArgs", # <rdar://problem/7714444> leak in Java + "Java_java_lang_System_initProperties", # <rdar://problem/7714465> leak in Java + "glrCompExecuteKernel", # <rdar://problem/7815391> leak in graphics driver while using OpenGL + ); + } + + if (isDarwin() && !isTiger() && !isLeopard() && !isSnowLeopard()) { + push @callStacksToExclude, ( + "CGGradientCreateWithColorComponents", # leak in CoreGraphics, <rdar://problem/7888492> + ); + } + + my $leaksTool = sourceDir() . "/WebKitTools/Scripts/run-leaks"; + my $excludeString = "--exclude-callstack '" . (join "' --exclude-callstack '", @callStacksToExclude) . "'"; + $excludeString .= " --exclude-type '" . (join "' --exclude-type '", @typesToExclude) . "'" if @typesToExclude; + + print " ? checking for leaks in $dumpToolName\n"; + my $leaksOutput = `$leaksTool $excludeString $dumpToolPID`; + my ($count, $bytes) = $leaksOutput =~ /Process $dumpToolPID: (\d+) leaks? for (\d+) total/; + my ($excluded) = $leaksOutput =~ /(\d+) leaks? excluded/; + + my $adjustedCount = $count; + $adjustedCount -= $excluded if $excluded; + + if (!$adjustedCount) { + print " - no leaks found\n"; + unlink $leaksFilePath; + return 0; + } else { + my $dir = $leaksFilePath; + $dir =~ s|/[^/]+$|| or die; + mkpath $dir; + + if ($excluded) { + print " + $adjustedCount leaks ($bytes bytes including $excluded excluded leaks) were found, details in $leaksFilePath\n"; + } else { + print " + $count leaks ($bytes bytes) were found, details in $leaksFilePath\n"; + } + + writeToFile($leaksFilePath, $leaksOutput); + + push @leaksFilenames, $leaksFilePath; + } + + return $adjustedCount; +} + +sub writeToFile($$) +{ + my ($filePath, $contents) = @_; + open NEWFILE, ">", "$filePath" or die "Could not create $filePath. $!\n"; + print NEWFILE $contents; + close NEWFILE; +} + +# Break up a path into the directory (with slash) and base name. +sub splitpath($) +{ + my ($path) = @_; + + my $pathSeparator = "/"; + my $dirname = dirname($path) . $pathSeparator; + $dirname = "" if $dirname eq "." . $pathSeparator; + + return ($dirname, basename($path)); +} + +# Sort first by directory, then by file, so all paths in one directory are grouped +# rather than being interspersed with items from subdirectories. +# Use numericcmp to sort directory and filenames to make order logical. +sub pathcmp($$) +{ + my ($patha, $pathb) = @_; + + my ($dira, $namea) = splitpath($patha); + my ($dirb, $nameb) = splitpath($pathb); + + return numericcmp($dira, $dirb) if $dira ne $dirb; + return numericcmp($namea, $nameb); +} + +# Sort numeric parts of strings as numbers, other parts as strings. +# Makes 1.33 come after 1.3, which is cool. +sub numericcmp($$) +{ + my ($aa, $bb) = @_; + + my @a = split /(\d+)/, $aa; + my @b = split /(\d+)/, $bb; + + # Compare one chunk at a time. + # Each chunk is either all numeric digits, or all not numeric digits. + while (@a && @b) { + my $a = shift @a; + my $b = shift @b; + + # Use numeric comparison if chunks are non-equal numbers. + return $a <=> $b if $a =~ /^\d/ && $b =~ /^\d/ && $a != $b; + + # Use string comparison if chunks are any other kind of non-equal string. + return $a cmp $b if $a ne $b; + } + + # One of the two is now empty; compare lengths for result in this case. + return @a <=> @b; +} + +# Sort slowest tests first. +sub slowestcmp($$) +{ + my ($testa, $testb) = @_; + + my $dura = $durations{$testa}; + my $durb = $durations{$testb}; + return $durb <=> $dura if $dura != $durb; + return pathcmp($testa, $testb); +} + +sub launchWithEnv(\@\%) +{ + my ($args, $env) = @_; + + # Dump the current environment as perl code and then put it in quotes so it is one parameter. + my $environmentDumper = Data::Dumper->new([\%{$env}], [qw(*ENV)]); + $environmentDumper->Indent(0); + $environmentDumper->Purity(1); + my $allEnvVars = $environmentDumper->Dump(); + unshift @{$args}, "\"$allEnvVars\""; + + my $execScript = File::Spec->catfile(sourceDir(), qw(WebKitTools Scripts execAppWithEnv)); + unshift @{$args}, $execScript; + return @{$args}; +} + +sub resolveAndMakeTestResultsDirectory() +{ + my $absTestResultsDirectory = File::Spec->rel2abs(glob $testResultsDirectory); + mkpath $absTestResultsDirectory; + return $absTestResultsDirectory; +} + +sub openDiffTool() +{ + return if $isDiffToolOpen; + return if !$pixelTests; + + my %CLEAN_ENV; + $CLEAN_ENV{MallocStackLogging} = 1 if $shouldCheckLeaks; + $imageDiffToolPID = open2(\*DIFFIN, \*DIFFOUT, $imageDiffTool, launchWithEnv(@diffToolArgs, %CLEAN_ENV)) or die "unable to open $imageDiffTool\n"; + $isDiffToolOpen = 1; +} + +sub openDumpTool() +{ + return if $isDumpToolOpen; + + my %CLEAN_ENV; + + # Generic environment variables + if (defined $ENV{'WEBKIT_TESTFONTS'}) { + $CLEAN_ENV{WEBKIT_TESTFONTS} = $ENV{'WEBKIT_TESTFONTS'}; + } + + # unique temporary directory for each DumpRendertree - needed for running more DumpRenderTree in parallel + $CLEAN_ENV{DUMPRENDERTREE_TEMP} = File::Temp::tempdir('DumpRenderTree-XXXXXX', TMPDIR => 1, CLEANUP => 1); + $CLEAN_ENV{XML_CATALOG_FILES} = ""; # work around missing /etc/catalog <rdar://problem/4292995> + + # Platform spesifics + if (isLinux()) { + if (defined $ENV{'DISPLAY'}) { + $CLEAN_ENV{DISPLAY} = $ENV{'DISPLAY'}; + } else { + $CLEAN_ENV{DISPLAY} = ":1"; + } + if (defined $ENV{'XAUTHORITY'}) { + $CLEAN_ENV{XAUTHORITY} = $ENV{'XAUTHORITY'}; + } + + $CLEAN_ENV{HOME} = $ENV{'HOME'}; + + if (defined $ENV{'LD_LIBRARY_PATH'}) { + $CLEAN_ENV{LD_LIBRARY_PATH} = $ENV{'LD_LIBRARY_PATH'}; + } + if (defined $ENV{'DBUS_SESSION_BUS_ADDRESS'}) { + $CLEAN_ENV{DBUS_SESSION_BUS_ADDRESS} = $ENV{'DBUS_SESSION_BUS_ADDRESS'}; + } + } elsif (isDarwin()) { + if (defined $ENV{'DYLD_LIBRARY_PATH'}) { + $CLEAN_ENV{DYLD_LIBRARY_PATH} = $ENV{'DYLD_LIBRARY_PATH'}; + } + + $CLEAN_ENV{DYLD_FRAMEWORK_PATH} = $productDir; + $CLEAN_ENV{DYLD_INSERT_LIBRARIES} = "/usr/lib/libgmalloc.dylib" if $guardMalloc; + } elsif (isCygwin()) { + $CLEAN_ENV{HOMEDRIVE} = $ENV{'HOMEDRIVE'}; + $CLEAN_ENV{HOMEPATH} = $ENV{'HOMEPATH'}; + + setPathForRunningWebKitApp(\%CLEAN_ENV); + } + + # Port specifics + if (isGtk()) { + $CLEAN_ENV{GTK_MODULES} = "gail"; + } + + if (isQt()) { + $CLEAN_ENV{QTWEBKIT_PLUGIN_PATH} = productDir() . "/lib/plugins"; + $CLEAN_ENV{QT_DRT_WEBVIEW_MODE} = $ENV{"QT_DRT_WEBVIEW_MODE"}; + } + + my @args = ($dumpTool, @toolArgs); + if (isAppleMacWebKit() and !isTiger()) { + unshift @args, "arch", "-" . architecture(); + } + + if ($useValgrind) { + unshift @args, "valgrind", "--suppressions=$platformBaseDirectory/qt/SuppressedValgrindErrors"; + } + + $CLEAN_ENV{MallocStackLogging} = 1 if $shouldCheckLeaks; + + $dumpToolPID = open3(\*OUT, \*IN, \*ERROR, launchWithEnv(@args, %CLEAN_ENV)) or die "Failed to start tool: $dumpTool\n"; + $isDumpToolOpen = 1; + $dumpToolCrashed = 0; +} + +sub closeDumpTool() +{ + return if !$isDumpToolOpen; + + close IN; + close OUT; + waitpid $dumpToolPID, 0; + + # check for WebCore counter leaks. + if ($shouldCheckLeaks) { + while (<ERROR>) { + print; + } + } + close ERROR; + $isDumpToolOpen = 0; +} + +sub dumpToolDidCrash() +{ + return 1 if $dumpToolCrashed; + return 0 unless $isDumpToolOpen; + my $pid = waitpid(-1, WNOHANG); + return 1 if ($pid == $dumpToolPID); + + # On Mac OS X, crashing may be significantly delayed by crash reporter. + return 0 unless isAppleMacWebKit(); + + return DumpRenderTreeSupport::processIsCrashing($dumpToolPID); +} + +sub configureAndOpenHTTPDIfNeeded() +{ + return if $isHttpdOpen; + my $absTestResultsDirectory = resolveAndMakeTestResultsDirectory(); + my $listen = "127.0.0.1:$httpdPort"; + my @args = ( + "-c", "CustomLog \"$absTestResultsDirectory/access_log.txt\" common", + "-c", "ErrorLog \"$absTestResultsDirectory/error_log.txt\"", + "-C", "Listen $listen" + ); + + my @defaultArgs = getDefaultConfigForTestDirectory($testDirectory); + @args = (@defaultArgs, @args); + + waitForHTTPDLock() if $shouldWaitForHTTPD; + $isHttpdOpen = openHTTPD(@args); +} + +sub checkPythonVersion() +{ + # we have not chdir to sourceDir yet. + system sourceDir() . "/WebKitTools/Scripts/ensure-valid-python", "--check-only"; + return exitStatus($?) == 0; +} + +sub openWebSocketServerIfNeeded() +{ + return 1 if $isWebSocketServerOpen; + return 0 if $failedToStartWebSocketServer; + + my $webSocketHandlerDir = "$testDirectory"; + my $absTestResultsDirectory = resolveAndMakeTestResultsDirectory(); + $webSocketServerPidFile = "$absTestResultsDirectory/websocket.pid"; + + my @args = ( + "WebKitTools/Scripts/new-run-webkit-websocketserver", + "--server", "start", + "--port", "$webSocketPort", + "--root", "$webSocketHandlerDir", + "--output-dir", "$absTestResultsDirectory", + "--pidfile", "$webSocketServerPidFile" + ); + system "/usr/bin/python", @args; + + $isWebSocketServerOpen = 1; + return 1; +} + +sub closeWebSocketServer() +{ + return if !$isWebSocketServerOpen; + + my @args = ( + "WebKitTools/Scripts/new-run-webkit-websocketserver", + "--server", "stop", + "--pidfile", "$webSocketServerPidFile" + ); + system "/usr/bin/python", @args; + unlink "$webSocketServerPidFile"; + + # wss is disabled until all platforms support pyOpenSSL. + $isWebSocketServerOpen = 0; +} + +sub fileNameWithNumber($$) +{ + my ($base, $number) = @_; + return "$base$number" if ($number > 1); + return $base; +} + +sub processIgnoreTests($$) +{ + my @ignoreList = split(/\s*,\s*/, shift); + my $listName = shift; + + my $disabledSuffix = "-disabled"; + + my $addIgnoredDirectories = sub { + return () if exists $ignoredLocalDirectories{basename($File::Find::dir)}; + $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)} = 1; + return @_; + }; + foreach my $item (@ignoreList) { + my $path = catfile($testDirectory, $item); + if (-d $path) { + $ignoredDirectories{$item} = 1; + find({ preprocess => $addIgnoredDirectories, wanted => sub {} }, $path); + } + elsif (-f $path) { + $ignoredFiles{$item} = 1; + } elsif (-f $path . $disabledSuffix) { + # The test is disabled, so do nothing. + } else { + print "$listName list contained '$item', but no file of that name could be found\n"; + } + } +} + +sub stripExtension($) +{ + my ($test) = @_; + + $test =~ s/\.[a-zA-Z]+$//; + return $test; +} + +sub isTextOnlyTest($) +{ + my ($actual) = @_; + my $isText; + if ($actual =~ /^layer at/ms) { + $isText = 0; + } else { + $isText = 1; + } + return $isText; +} + +sub expectedDirectoryForTest($;$;$) +{ + my ($base, $isText, $expectedExtension) = @_; + + my @directories = @platformResultHierarchy; + push @directories, map { catdir($platformBaseDirectory, $_) } qw(mac-snowleopard mac) if isCygwin(); + push @directories, $expectedDirectory; + + # If we already have expected results, just return their location. + foreach my $directory (@directories) { + return $directory if (-f "$directory/$base-$expectedTag.$expectedExtension"); + } + + # For cross-platform tests, text-only results should go in the cross-platform directory, + # while render tree dumps should go in the least-specific platform directory. + return $isText ? $expectedDirectory : $platformResultHierarchy[$#platformResultHierarchy]; +} + +sub countFinishedTest($$$$) +{ + my ($test, $base, $result, $isText) = @_; + + if (($count + 1) % $testsPerDumpTool == 0 || $count == $#tests) { + if ($shouldCheckLeaks) { + my $fileName; + if ($testsPerDumpTool == 1) { + $fileName = "$testResultsDirectory/$base-leaks.txt"; + } else { + $fileName = "$testResultsDirectory/" . fileNameWithNumber($dumpToolName, $leaksOutputFileNumber) . "-leaks.txt"; + } + my $leakCount = countAndPrintLeaks($dumpToolName, $dumpToolPID, $fileName); + $totalLeaks += $leakCount; + $leaksOutputFileNumber++ if ($leakCount); + } + + closeDumpTool(); + } + + $count++; + $counts{$result}++; + push @{$tests{$result}}, $test; +} + +sub testCrashedOrTimedOut($$$$$) +{ + my ($test, $base, $didCrash, $actual, $error) = @_; + + printFailureMessageForTest($test, $didCrash ? "crashed" : "timed out"); + + sampleDumpTool() unless $didCrash; + + my $dir = "$testResultsDirectory/$base"; + $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; + mkpath $dir; + + deleteExpectedAndActualResults($base); + + if (defined($error) && length($error)) { + writeToFile("$testResultsDirectory/$base-$errorTag.txt", $error); + } + + recordActualResultsAndDiff($base, $actual); + + kill 9, $dumpToolPID unless $didCrash; + + closeDumpTool(); + + return unless isCygwin() && !$didCrash && $base =~ /^http/; + # On Cygwin, http tests timing out can be a symptom of a non-responsive httpd. + # If we timed out running an http test, try restarting httpd. + $isHttpdOpen = !closeHTTPD(); + configureAndOpenHTTPDIfNeeded(); +} + +sub printFailureMessageForTest($$) +{ + my ($test, $description) = @_; + + unless ($verbose) { + print "\n" unless $atLineStart; + print "$test -> "; + } + print "$description\n"; + $atLineStart = 1; +} + +my %cygpaths = (); + +sub openCygpathIfNeeded($) +{ + my ($options) = @_; + + return unless isCygwin(); + return $cygpaths{$options} if $cygpaths{$options} && $cygpaths{$options}->{"open"}; + + local (*CYGPATHIN, *CYGPATHOUT); + my $pid = open2(\*CYGPATHIN, \*CYGPATHOUT, "cygpath -f - $options"); + my $cygpath = { + "pid" => $pid, + "in" => *CYGPATHIN, + "out" => *CYGPATHOUT, + "open" => 1 + }; + + $cygpaths{$options} = $cygpath; + + return $cygpath; +} + +sub closeCygpaths() +{ + return unless isCygwin(); + + foreach my $cygpath (values(%cygpaths)) { + close $cygpath->{"in"}; + close $cygpath->{"out"}; + waitpid($cygpath->{"pid"}, 0); + $cygpath->{"open"} = 0; + + } +} + +sub convertPathUsingCygpath($$) +{ + my ($path, $options) = @_; + + # cygpath -f (at least in Cygwin 1.7) converts spaces into newlines. We remove spaces here and + # add them back in after conversion to work around this. + my $spaceSubstitute = "__NOTASPACE__"; + $path =~ s/ /\Q$spaceSubstitute\E/g; + + my $cygpath = openCygpathIfNeeded($options); + local *inFH = $cygpath->{"in"}; + local *outFH = $cygpath->{"out"}; + print outFH $path . "\n"; + my $convertedPath = <inFH>; + chomp($convertedPath) if defined $convertedPath; + + $convertedPath =~ s/\Q$spaceSubstitute\E/ /g; + return $convertedPath; +} + +sub toWindowsPath($) +{ + my ($path) = @_; + return unless isCygwin(); + + return convertPathUsingCygpath($path, "-w"); +} + +sub toURL($) +{ + my ($path) = @_; + + if ($useRemoteLinksToTests) { + my $relativePath = File::Spec->abs2rel($path, $testDirectory); + + # If the file is below the test directory then convert it into a link to the file in SVN + if ($relativePath !~ /^\.\.\//) { + my $revision = svnRevisionForDirectory($testDirectory); + my $svnPath = pathRelativeToSVNRepositoryRootForPath($path); + return "http://trac.webkit.org/export/$revision/$svnPath"; + } + } + + return $path unless isCygwin(); + + return "file:///" . convertPathUsingCygpath($path, "-m"); +} + +sub validateSkippedArg($$;$) +{ + my ($option, $value, $value2) = @_; + my %validSkippedValues = map { $_ => 1 } qw(default ignore only); + $value = lc($value); + die "Invalid argument '" . $value . "' for option $option" unless $validSkippedValues{$value}; + $treatSkipped = $value; +} + +sub htmlForResultsSection(\@$&) +{ + my ($tests, $description, $linkGetter) = @_; + + my @html = (); + return join("\n", @html) unless @{$tests}; + + push @html, "<p>$description:</p>"; + push @html, "<table>"; + foreach my $test (@{$tests}) { + push @html, "<tr>"; + push @html, "<td><a href=\"" . toURL("$testDirectory/$test") . "\">$test</a></td>"; + foreach my $link (@{&{$linkGetter}($test)}) { + push @html, "<td><a href=\"$link->{href}\">$link->{text}</a></td>"; + } + push @html, "</tr>"; + } + push @html, "</table>"; + + return join("\n", @html); +} + +sub linksForExpectedAndActualResults($) +{ + my ($base) = @_; + + my @links = (); + + return \@links unless -s "$testResultsDirectory/$base-$diffsTag.txt"; + + my $expectedResultPath = $expectedResultPaths{$base}; + my ($expectedResultFileName, $expectedResultsDirectory, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); + + push @links, { href => "$base-$expectedTag$expectedResultExtension", text => "expected" }; + push @links, { href => "$base-$actualTag$expectedResultExtension", text => "actual" }; + push @links, { href => "$base-$diffsTag.txt", text => "diff" }; + push @links, { href => "$base-$prettyDiffTag.html", text => "pretty diff" }; + + return \@links; +} + +sub linksForMismatchTest +{ + my ($test) = @_; + + my @links = (); + + my $base = stripExtension($test); + + push @links, @{linksForExpectedAndActualResults($base)}; + return \@links unless $pixelTests && $imagesPresent{$base}; + + push @links, { href => "$base-$expectedTag.png", text => "expected image" }; + push @links, { href => "$base-$diffsTag.html", text => "image diffs" }; + push @links, { href => "$base-$diffsTag.png", text => "$imageDifferences{$base}%" }; + + return \@links; +} + +sub linksForErrorTest +{ + my ($test) = @_; + + my @links = (); + + my $base = stripExtension($test); + + push @links, @{linksForExpectedAndActualResults($base)}; + push @links, { href => "$base-$errorTag.txt", text => "stderr" }; + + return \@links; +} + +sub linksForNewTest +{ + my ($test) = @_; + + my @links = (); + + my $base = stripExtension($test); + + my $expectedResultPath = $expectedResultPaths{$base}; + my ($expectedResultFileName, $expectedResultsDirectory, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); + + push @links, { href => "$base-$actualTag$expectedResultExtension", text => "result" }; + if ($pixelTests && $imagesPresent{$base}) { + push @links, { href => "$base-$expectedTag.png", text => "image" }; + } + + return \@links; +} + +sub deleteExpectedAndActualResults($) +{ + my ($base) = @_; + + unlink "$testResultsDirectory/$base-$actualTag.txt"; + unlink "$testResultsDirectory/$base-$diffsTag.txt"; + unlink "$testResultsDirectory/$base-$errorTag.txt"; +} + +sub recordActualResultsAndDiff($$) +{ + my ($base, $actualResults) = @_; + + return unless defined($actualResults) && length($actualResults); + + my $expectedResultPath = $expectedResultPaths{$base}; + my ($expectedResultFileNameMinusExtension, $expectedResultDirectoryPath, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); + my $actualResultsPath = "$testResultsDirectory/$base-$actualTag$expectedResultExtension"; + my $copiedExpectedResultsPath = "$testResultsDirectory/$base-$expectedTag$expectedResultExtension"; + + mkpath(dirname($actualResultsPath)); + writeToFile("$actualResultsPath", $actualResults); + + if (-f $expectedResultPath) { + copy("$expectedResultPath", "$copiedExpectedResultsPath"); + } else { + open EMPTY, ">$copiedExpectedResultsPath"; + close EMPTY; + } + + my $diffOuputBasePath = "$testResultsDirectory/$base"; + my $diffOutputPath = "$diffOuputBasePath-$diffsTag.txt"; + system "diff -u \"$copiedExpectedResultsPath\" \"$actualResultsPath\" > \"$diffOutputPath\""; + + my $prettyDiffOutputPath = "$diffOuputBasePath-$prettyDiffTag.html"; + my $prettyPatchPath = "BugsSite/PrettyPatch/"; + my $prettifyPath = "$prettyPatchPath/prettify.rb"; + system "ruby -I \"$prettyPatchPath\" \"$prettifyPath\" \"$diffOutputPath\" > \"$prettyDiffOutputPath\""; +} + +sub buildPlatformResultHierarchy() +{ + mkpath($platformTestDirectory) if ($platform eq "undefined" && !-d "$platformTestDirectory"); + + my @platforms; + + my $isMac = $platform =~ /^mac/; + my $isWin = $platform =~ /^win/; + if ($isMac || $isWin) { + my $effectivePlatform = $platform; + if ($platform eq "mac-wk2") { + push @platforms, "mac-wk2"; + $effectivePlatform = $realPlatform; + } + + my @platformList = $isMac ? @macPlatforms : @winPlatforms; + my $i; + for ($i = 0; $i < @platformList; $i++) { + last if $platformList[$i] eq $effectivePlatform; + } + for (; $i < @platformList; $i++) { + push @platforms, $platformList[$i]; + } + } elsif ($platform =~ /^qt-/) { + push @platforms, $platform; + push @platforms, "qt"; + } else { + @platforms = $platform; + } + + my @hierarchy; + for (my $i = 0; $i < @platforms; $i++) { + my $scoped = catdir($platformBaseDirectory, $platforms[$i]); + push(@hierarchy, $scoped) if (-d $scoped); + } + + return @hierarchy; +} + +sub buildPlatformTestHierarchy(@) +{ + my (@platformHierarchy) = @_; + return @platformHierarchy if (@platformHierarchy < 2); + if ($platformHierarchy[0] =~ /mac-wk2/) { + return ($platformHierarchy[0], $platformHierarchy[1], $platformHierarchy[$#platformHierarchy]); + } + return ($platformHierarchy[0], $platformHierarchy[$#platformHierarchy]); +} + +sub epiloguesAndPrologues($$) +{ + my ($lastDirectory, $directory) = @_; + my @lastComponents = split('/', $lastDirectory); + my @components = split('/', $directory); + + while (@lastComponents) { + if (!defined($components[0]) || $lastComponents[0] ne $components[0]) { + last; + } + shift @components; + shift @lastComponents; + } + + my @result; + my $leaving = $lastDirectory; + foreach (@lastComponents) { + my $epilogue = $leaving . "/resources/run-webkit-tests-epilogue.html"; + foreach (@platformResultHierarchy) { + push @result, catdir($_, $epilogue) if (stat(catdir($_, $epilogue))); + } + push @result, catdir($testDirectory, $epilogue) if (stat(catdir($testDirectory, $epilogue))); + $leaving =~ s|(^\|/)[^/]+$||; + } + + my $entering = $leaving; + foreach (@components) { + $entering .= '/' . $_; + my $prologue = $entering . "/resources/run-webkit-tests-prologue.html"; + push @result, catdir($testDirectory, $prologue) if (stat(catdir($testDirectory, $prologue))); + foreach (reverse @platformResultHierarchy) { + push @result, catdir($_, $prologue) if (stat(catdir($_, $prologue))); + } + } + return @result; +} + +sub parseLeaksandPrintUniqueLeaks() +{ + return unless @leaksFilenames; + + my $mergedFilenames = join " ", @leaksFilenames; + my $parseMallocHistoryTool = sourceDir() . "/WebKitTools/Scripts/parse-malloc-history"; + + open MERGED_LEAKS, "cat $mergedFilenames | $parseMallocHistoryTool --merge-depth $mergeDepth - |" ; + my @leakLines = <MERGED_LEAKS>; + close MERGED_LEAKS; + + my $uniqueLeakCount = 0; + my $totalBytes; + foreach my $line (@leakLines) { + ++$uniqueLeakCount if ($line =~ /^(\d*)\scalls/); + $totalBytes = $1 if $line =~ /^total\:\s(.*)\s\(/; + } + + print "\nWARNING: $totalLeaks total leaks found for a total of $totalBytes!\n"; + print "WARNING: $uniqueLeakCount unique leaks found!\n"; + print "See above for individual leaks results.\n" if ($leaksOutputFileNumber > 2); + +} + +sub extensionForMimeType($) +{ + my ($mimeType) = @_; + + if ($mimeType eq "application/x-webarchive") { + return "webarchive"; + } elsif ($mimeType eq "application/pdf") { + return "pdf"; + } + return "txt"; +} + +# Read up to the first #EOF (the content block of the test), or until detecting crashes or timeouts. +sub readFromDumpToolWithTimer(**) +{ + my ($fhIn, $fhError) = @_; + + setFileHandleNonBlocking($fhIn, 1); + setFileHandleNonBlocking($fhError, 1); + + my $maximumSecondsWithoutOutput = $timeoutSeconds; + $maximumSecondsWithoutOutput *= 10 if $guardMalloc; + my $microsecondsToWaitBeforeReadingAgain = 1000; + + my $timeOfLastSuccessfulRead = time; + + my @output = (); + my @error = (); + my $status = "success"; + my $mimeType = "text/plain"; + # We don't have a very good way to know when the "headers" stop + # and the content starts, so we use this as a hack: + my $haveSeenContentType = 0; + my $haveSeenEofIn = 0; + my $haveSeenEofError = 0; + + while (1) { + if (time - $timeOfLastSuccessfulRead > $maximumSecondsWithoutOutput) { + $status = dumpToolDidCrash() ? "crashed" : "timedOut"; + last; + } + + # Once we've seen the EOF, we must not read anymore. + my $lineIn = readline($fhIn) unless $haveSeenEofIn; + my $lineError = readline($fhError) unless $haveSeenEofError; + if (!defined($lineIn) && !defined($lineError)) { + last if ($haveSeenEofIn && $haveSeenEofError); + + if ($! != EAGAIN) { + $status = "crashed"; + last; + } + + # No data ready + usleep($microsecondsToWaitBeforeReadingAgain); + next; + } + + $timeOfLastSuccessfulRead = time; + + if (defined($lineIn)) { + if (!$haveSeenContentType && $lineIn =~ /^Content-Type: (\S+)$/) { + $mimeType = $1; + $haveSeenContentType = 1; + } elsif ($lineIn =~ /#EOF/) { + $haveSeenEofIn = 1; + } else { + push @output, $lineIn; + } + } + if (defined($lineError)) { + if ($lineError =~ /#EOF/) { + $haveSeenEofError = 1; + } else { + push @error, $lineError; + } + } + } + + setFileHandleNonBlocking($fhIn, 0); + setFileHandleNonBlocking($fhError, 0); + return { + output => join("", @output), + error => join("", @error), + status => $status, + mimeType => $mimeType, + extension => extensionForMimeType($mimeType) + }; +} + +sub setFileHandleNonBlocking(*$) +{ + my ($fh, $nonBlocking) = @_; + + my $flags = fcntl($fh, F_GETFL, 0) or die "Couldn't get filehandle flags"; + + if ($nonBlocking) { + $flags |= O_NONBLOCK; + } else { + $flags &= ~O_NONBLOCK; + } + + fcntl($fh, F_SETFL, $flags) or die "Couldn't set filehandle flags"; + + return 1; +} + +sub sampleDumpTool() +{ + return unless isAppleMacWebKit(); + return unless $runSample; + + my $outputDirectory = "$ENV{HOME}/Library/Logs/DumpRenderTree"; + -d $outputDirectory or mkdir $outputDirectory; + + my $outputFile = "$outputDirectory/HangReport.txt"; + system "/usr/bin/sample", $dumpToolPID, qw(10 10 -file), $outputFile; +} + +sub stripMetrics($$) +{ + my ($actual, $expected) = @_; + + foreach my $result ($actual, $expected) { + $result =~ s/at \(-?[0-9]+,-?[0-9]+\) *//g; + $result =~ s/size -?[0-9]+x-?[0-9]+ *//g; + $result =~ s/text run width -?[0-9]+: //g; + $result =~ s/text run width -?[0-9]+ [a-zA-Z ]+: //g; + $result =~ s/RenderButton {BUTTON} .*/RenderButton {BUTTON}/g; + $result =~ s/RenderImage {INPUT} .*/RenderImage {INPUT}/g; + $result =~ s/RenderBlock {INPUT} .*/RenderBlock {INPUT}/g; + $result =~ s/RenderTextControl {INPUT} .*/RenderTextControl {INPUT}/g; + $result =~ s/\([0-9]+px/px/g; + $result =~ s/ *" *\n +" */ /g; + $result =~ s/" +$/"/g; + + $result =~ s/- /-/g; + $result =~ s/\n( *)"\s+/\n$1"/g; + $result =~ s/\s+"\n/"\n/g; + $result =~ s/scrollWidth [0-9]+/scrollWidth/g; + $result =~ s/scrollHeight [0-9]+/scrollHeight/g; + } + + return ($actual, $expected); +} + +sub fileShouldBeIgnored +{ + my ($filePath) = @_; + foreach my $ignoredDir (keys %ignoredDirectories) { + if ($filePath =~ m/^$ignoredDir/) { + return 1; + } + } + return 0; +} + +sub readSkippedFiles($) +{ + my ($constraintPath) = @_; + + foreach my $level (@platformTestHierarchy) { + if (open SKIPPED, "<", "$level/Skipped") { + if ($verbose) { + my ($dir, $name) = splitpath($level); + print "Skipped tests in $name:\n"; + } + + while (<SKIPPED>) { + my $skipped = $_; + chomp $skipped; + $skipped =~ s/^[ \n\r]+//; + $skipped =~ s/[ \n\r]+$//; + if ($skipped && $skipped !~ /^#/) { + if ($skippedOnly) { + if (!fileShouldBeIgnored($skipped)) { + if (!$constraintPath) { + # Always add $skipped since no constraint path was specified on the command line. + push(@ARGV, $skipped); + } elsif ($skipped =~ /^($constraintPath)/) { + # Add $skipped only if it matches the current path constraint, e.g., + # "--skipped=only dir1" with "dir1/file1.html" on the skipped list. + push(@ARGV, $skipped); + } elsif ($constraintPath =~ /^($skipped)/) { + # Add current path constraint if it is more specific than the skip list entry, + # e.g., "--skipped=only dir1/dir2/dir3" with "dir1" on the skipped list. + push(@ARGV, $constraintPath); + } + } elsif ($verbose) { + print " $skipped\n"; + } + } else { + if ($verbose) { + print " $skipped\n"; + } + processIgnoreTests($skipped, "Skipped"); + } + } + } + close SKIPPED; + } + } +} + +my @testsFound; + +sub directoryFilter +{ + return () if exists $ignoredLocalDirectories{basename($File::Find::dir)}; + return () if exists $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)}; + return @_; +} + +sub fileFilter +{ + my $filename = $_; + if ($filename =~ /\.([^.]+)$/) { + if (exists $supportedFileExtensions{$1}) { + my $path = File::Spec->abs2rel(catfile($File::Find::dir, $filename), $testDirectory); + push @testsFound, $path if !exists $ignoredFiles{$path}; + } + } +} + +sub findTestsToRun +{ + my @testsToRun = (); + + for my $test (@ARGV) { + $test =~ s/^($layoutTestsName|$testDirectory)\///; + my $fullPath = catfile($testDirectory, $test); + if (file_name_is_absolute($test)) { + print "can't run test $test outside $testDirectory\n"; + } elsif (-f $fullPath) { + my ($filename, $pathname, $fileExtension) = fileparse($test, qr{\.[^.]+$}); + if (!exists $supportedFileExtensions{substr($fileExtension, 1)}) { + print "test $test does not have a supported extension\n"; + } elsif ($testHTTP || $pathname !~ /^http\//) { + push @testsToRun, $test; + } + } elsif (-d $fullPath) { + @testsFound = (); + find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $fullPath); + for my $level (@platformTestHierarchy) { + my $platformPath = catfile($level, $test); + find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $platformPath) if (-d $platformPath); + } + push @testsToRun, sort pathcmp @testsFound; + @testsFound = (); + } else { + print "test $test not found\n"; + } + } + + if (!scalar @ARGV) { + @testsFound = (); + find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $testDirectory); + for my $level (@platformTestHierarchy) { + find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $level); + } + push @testsToRun, sort pathcmp @testsFound; + @testsFound = (); + + # We need to minimize the time when Apache and WebSocketServer is locked by tests + # so run them last if no explicit order was specified in the argument list. + my @httpTests; + my @websocketTests; + my @otherTests; + foreach my $test (@testsToRun) { + if ($test =~ /^http\//) { + push(@httpTests, $test); + } elsif ($test =~ /^websocket\//) { + push(@websocketTests, $test); + } else { + push(@otherTests, $test); + } + } + @testsToRun = (@otherTests, @httpTests, @websocketTests); + } + + # Reverse the tests + @testsToRun = reverse @testsToRun if $reverseTests; + + # Shuffle the array + @testsToRun = shuffle(@testsToRun) if $randomizeTests; + + return @testsToRun; +} + +sub printResults +{ + my %text = ( + match => "succeeded", + mismatch => "had incorrect layout", + new => "were new", + timedout => "timed out", + crash => "crashed", + error => "had stderr output" + ); + + for my $type ("match", "mismatch", "new", "timedout", "crash", "error") { + my $typeCount = $counts{$type}; + next unless $typeCount; + my $typeText = $text{$type}; + my $message; + if ($typeCount == 1) { + $typeText =~ s/were/was/; + $message = sprintf "1 test case (%d%%) %s\n", 1 * 100 / $count, $typeText; + } else { + $message = sprintf "%d test cases (%d%%) %s\n", $typeCount, $typeCount * 100 / $count, $typeText; + } + $message =~ s-\(0%\)-(<1%)-; + print $message; + } +} + +sub stopRunningTestsEarlyIfNeeded() +{ + # --reset-results does not check pass vs. fail, so exitAfterNFailures makes no sense with --reset-results. + return 0 if $resetResults; + + my $passCount = $counts{match} || 0; # $counts{match} will be undefined if we've not yet passed a test (e.g. the first test fails). + my $newCount = $counts{new} || 0; + my $failureCount = $count - $passCount - $newCount; # "Failure" here includes timeouts, crashes, etc. + if ($exitAfterNFailures && $failureCount >= $exitAfterNFailures) { + print "\nExiting early after $failureCount failures. $count tests run."; + closeDumpTool(); + return 1; + } + + my $crashCount = $counts{crash} || 0; + my $timeoutCount = $counts{timedout} || 0; + if ($exitAfterNCrashesOrTimeouts && $crashCount + $timeoutCount >= $exitAfterNCrashesOrTimeouts) { + print "\nExiting early after $crashCount crashes and $timeoutCount timeouts. $count tests run."; + closeDumpTool(); + return 1; + } + + return 0; +} diff --git a/WebKitTools/Scripts/prepare-ChangeLog b/WebKitTools/Scripts/prepare-ChangeLog index 3350aa3..2ef1eb4 100755 --- a/WebKitTools/Scripts/prepare-ChangeLog +++ b/WebKitTools/Scripts/prepare-ChangeLog @@ -89,6 +89,7 @@ sub get_function_line_ranges($$); sub get_function_line_ranges_for_c($$); sub get_function_line_ranges_for_java($$); sub get_function_line_ranges_for_javascript($$); +sub get_selector_line_ranges_for_css($$); sub method_decl_to_selector($); sub processPaths(\@); sub reviewerAndDescriptionForGitCommit($); @@ -101,6 +102,7 @@ my $changeLogTimeZone = "PST8PDT"; my $bugNumber; my $name; my $emailAddress; +my $mergeBase = 0; my $gitCommit = 0; my $gitIndex = ""; my $gitReviewer = ""; @@ -114,6 +116,7 @@ my $parseOptionsResult = "bug:i" => \$bugNumber, "name:s" => \$name, "email:s" => \$emailAddress, + "merge-base:s" => \$mergeBase, "git-commit:s" => \$gitCommit, "git-index" => \$gitIndex, "git-reviewer:s" => \$gitReviewer, @@ -125,6 +128,7 @@ if (!$parseOptionsResult || $showHelp) { print STDERR basename($0) . " [--bug] [-d|--diff] [-h|--help] [-o|--open] [--git-commit=<committish>] [--git-reviewer=<name>] [svndir1 [svndir2 ...]]\n"; print STDERR " --bug Fill in the ChangeLog bug information from the given bug.\n"; print STDERR " -d|--diff Spew diff to stdout when running\n"; + print STDERR " --merge-base Populate the ChangeLogs with the diff to this branch\n"; print STDERR " --git-commit Populate the ChangeLogs from the specified git commit\n"; print STDERR " --git-index Populate the ChangeLogs from the git index only\n"; print STDERR " --git-reviewer When populating the ChangeLogs from a git commit claim that the spcified name reviewed the change.\n"; @@ -256,7 +260,9 @@ if ($bugNumber) { $bugURL = "https://bugs.webkit.org/show_bug.cgi?id=$bugNumber"; my $bugXMLURL = "$bugURL&ctype=xml"; # Perl has no built in XML processing, so we'll fetch and parse with curl and grep - my $descriptionLine = `curl --silent "$bugXMLURL" | grep short_desc`; + # Pass --insecure because some cygwin installs have no certs we don't + # care about validating that bugs.webkit.org is who it says it is here. + my $descriptionLine = `curl --insecure --silent "$bugXMLURL" | grep short_desc`; if ($descriptionLine !~ /<short_desc>(.*)<\/short_desc>/) { print STDERR " Bug $bugNumber has no bug description. Maybe you set wrong bug ID?\n"; print STDERR " The bug URL: $bugXMLURL\n"; @@ -474,6 +480,8 @@ sub get_function_line_ranges($$) return get_function_line_ranges_for_java ($file_handle, $file_name); } elsif ($file_name =~ /\.js$/) { return get_function_line_ranges_for_javascript ($file_handle, $file_name); + } elsif ($file_name =~ /\.css$/) { + return get_selector_line_ranges_for_css ($file_handle, $file_name); } return (); } @@ -1173,6 +1181,41 @@ sub get_function_line_ranges_for_javascript($$) return @ranges; } +# Read a file and get all the line ranges of the things that look like CSS selectors. A selector is +# anything before an opening brace on a line. A selector starts at the line containing the opening +# brace and ends at the closing brace. +# FIXME: Comments are parsed just like uncommented text. +# +# Result is a list of triples: [ start_line, end_line, selector ]. + +sub get_selector_line_ranges_for_css($$) +{ + my ($fileHandle, $fileName) = @_; + + my @ranges; + + my $currentSelector = ""; + my $start = 0; + + while (<$fileHandle>) { + if (/^[ \t]*(.*[^ \t])[ \t]*{/) { + $currentSelector = $1; + $start = $.; + } + if (index($_, "}") >= 0) { + unless ($start) { + warn "mismatched braces in $fileName\n"; + next; + } + push(@ranges, [$start, $., $currentSelector]); + $currentSelector = ""; + $start = 0; + next; + } + } + + return @ranges; +} sub processPaths(\@) { @@ -1216,6 +1259,7 @@ sub diffFromToString() return $gitCommit if $gitCommit =~ m/.+\.\..+/; return "\"$gitCommit^\" \"$gitCommit\"" if $gitCommit; return "--cached" if $gitIndex; + return $mergeBase if $mergeBase; return "HEAD" if $isGit; } @@ -1230,7 +1274,7 @@ sub diffCommand(@) $command = "$SVN diff --diff-cmd diff -x -N $pathsString"; } elsif ($isGit) { $command = "$GIT diff --no-ext-diff -U0 " . diffFromToString(); - $command .= " -- $pathsString" unless $gitCommit; + $command .= " -- $pathsString" unless $gitCommit or $mergeBase; } return $command; diff --git a/WebKitTools/Scripts/rebaseline-chromium-webkit-tests b/WebKitTools/Scripts/rebaseline-chromium-webkit-tests index aea0edc..8d14b86 100755 --- a/WebKitTools/Scripts/rebaseline-chromium-webkit-tests +++ b/WebKitTools/Scripts/rebaseline-chromium-webkit-tests @@ -31,8 +31,12 @@ import os import sys -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), - "webkitpy", "layout_tests")) +scripts_directory = os.path.dirname(os.path.abspath(sys.argv[0])) +webkitpy_directory = os.path.join(scripts_directory, "webkitpy") +sys.path.append(os.path.join(webkitpy_directory, "layout_tests")) + +# For simplejson +sys.path.append(os.path.join(webkitpy_directory, "thirdparty")) import rebaseline_chromium_webkit_tests diff --git a/WebKitTools/Scripts/resolve-ChangeLogs b/WebKitTools/Scripts/resolve-ChangeLogs index 1a2d2af..6635711 100755 --- a/WebKitTools/Scripts/resolve-ChangeLogs +++ b/WebKitTools/Scripts/resolve-ChangeLogs @@ -34,6 +34,7 @@ use FindBin; use lib $FindBin::Bin; use File::Basename; +use File::Copy; use File::Path; use File::Spec; use Getopt::Long; @@ -48,7 +49,6 @@ sub fixMergedChangeLogs($;@); sub fixOneMergedChangeLog($); sub hasGitUnmergedFiles(); sub isInGitFilterBranch(); -sub mergeChanges($$$); sub parseFixMerged($$;$); sub removeChangeLogArguments($); sub resolveChangeLog($); @@ -64,28 +64,30 @@ my $GIT = "git"; my $fixMerged; my $gitRebaseContinue = 0; +my $mergeDriver = 0; my $printWarnings = 1; my $showHelp; my $getOptionsResult = GetOptions( - 'c|continue!' => \$gitRebaseContinue, - 'f|fix-merged:s' => \&parseFixMerged, - 'h|help' => \$showHelp, - 'w|warnings!' => \$printWarnings, + 'c|continue!' => \$gitRebaseContinue, + 'f|fix-merged:s' => \&parseFixMerged, + 'm|merge-driver!' => \$mergeDriver, + 'h|help' => \$showHelp, + 'w|warnings!' => \$printWarnings, ); my $relativePath = isInGitFilterBranch() ? '.' : chdirReturningRelativePath(determineVCSRoot()); my @changeLogFiles = removeChangeLogArguments($relativePath); -if (!defined $fixMerged && scalar(@changeLogFiles) == 0) { +if (!defined $fixMerged && !$mergeDriver && scalar(@changeLogFiles) == 0) { @changeLogFiles = findUnmergedChangeLogs(); } -if (scalar(@ARGV) > 0) { +if (!$mergeDriver && scalar(@ARGV) > 0) { print STDERR "ERROR: Files listed on command-line that are not ChangeLogs.\n"; undef $getOptionsResult; -} elsif (!defined $fixMerged && scalar(@changeLogFiles) == 0) { +} elsif (!defined $fixMerged && !$mergeDriver && scalar(@changeLogFiles) == 0) { print STDERR "ERROR: No ChangeLog files listed on command-line or found unmerged.\n"; undef $getOptionsResult; } elsif ($gitRebaseContinue && !$isGit) { @@ -94,6 +96,12 @@ if (scalar(@ARGV) > 0) { } elsif (defined $fixMerged && !$isGit) { print STDERR "ERROR: --fix-merged may only be used with a git repository\n"; undef $getOptionsResult; +} elsif ($mergeDriver && !$isGit) { + print STDERR "ERROR: --merge-driver may only be used with a git repository\n"; + undef $getOptionsResult; +} elsif ($mergeDriver && scalar(@ARGV) < 3) { + print STDERR "ERROR: --merge-driver expects %O %A %B as arguments\n"; + undef $getOptionsResult; } sub usageAndExit() @@ -104,6 +112,7 @@ Usage: @{[ basename($0) ]} [options] [path/to/ChangeLog] [path/to/another/Change entries (default: --no-continue) -f|--fix-merged [revision-range] fix git-merged ChangeLog entries; if a revision-range is specified, run git filter-branch on the range + -m|--merge-driver %O %A %B act as a git merge-driver on files %O %A %B -h|--help show this help message -w|--[no-]warnings show or suppress warnings (default: show warnings) __END__ @@ -118,6 +127,14 @@ if (defined $fixMerged && length($fixMerged) > 0) { my $commitRange = $fixMerged; $commitRange = $commitRange . "..HEAD" if index($commitRange, "..") < 0; fixMergedChangeLogs($commitRange, @changeLogFiles); +} elsif ($mergeDriver) { + my ($base, $theirs, $ours) = @ARGV; + if (mergeChangeLogs($ours, $base, $theirs)) { + unlink($ours); + copy($theirs, $ours) or die $!; + } else { + exec qw(git merge-file -L THEIRS -L BASE -L OURS), $theirs, $base, $ours; + } } elsif (@changeLogFiles) { for my $file (@changeLogFiles) { if (defined $fixMerged) { @@ -383,55 +400,6 @@ sub isInGitFilterBranch() return exists $ENV{MAPPED_PREVIOUS_COMMIT} && $ENV{MAPPED_PREVIOUS_COMMIT}; } -sub mergeChanges($$$) -{ - my ($fileMine, $fileOlder, $fileNewer) = @_; - - my $traditionalReject = $fileMine =~ /\.rej$/ ? 1 : 0; - - local $/ = undef; - - my $patch; - if ($traditionalReject) { - open(DIFF, "<", $fileMine) or die $!; - $patch = <DIFF>; - close(DIFF); - rename($fileMine, "$fileMine.save"); - rename($fileOlder, "$fileOlder.save"); - } else { - open(DIFF, "-|", qw(diff -u -a --binary), $fileOlder, $fileMine) or die $!; - $patch = <DIFF>; - close(DIFF); - } - - unlink("${fileNewer}.orig"); - unlink("${fileNewer}.rej"); - - open(PATCH, "| patch --fuzz=3 --binary $fileNewer > " . File::Spec->devnull()) or die $!; - print PATCH fixChangeLogPatch($patch); - close(PATCH); - - my $result; - - # Refuse to merge the patch if it did not apply cleanly - if (-e "${fileNewer}.rej") { - unlink("${fileNewer}.rej"); - unlink($fileNewer); - rename("${fileNewer}.orig", $fileNewer); - $result = 0; - } else { - unlink("${fileNewer}.orig"); - $result = 1; - } - - if ($traditionalReject) { - rename("$fileMine.save", $fileMine); - rename("$fileOlder.save", $fileOlder); - } - - return $result; -} - sub parseFixMerged($$;$) { my ($switchName, $key, $value) = @_; @@ -473,7 +441,7 @@ sub resolveChangeLog($) return unless $fileMine && $fileOlder && $fileNewer; - if (mergeChanges($fileMine, $fileOlder, $fileNewer)) { + if (mergeChangeLogs($fileMine, $fileOlder, $fileNewer)) { if ($file ne $fileNewer) { unlink($file); rename($fileNewer, $file) or die $!; diff --git a/WebKitTools/Scripts/run-bindings-tests b/WebKitTools/Scripts/run-bindings-tests new file mode 100755 index 0000000..4a093d1 --- /dev/null +++ b/WebKitTools/Scripts/run-bindings-tests @@ -0,0 +1,137 @@ +#!/usr/bin/python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +# This script generates h and cpp file for TestObj.idl using the V8 code +# generator. Please execute the script whenever changes are made to +# CodeGeneratorV8.pm, and submit the changes in V8TestObj.h/cpp in the same +# patch. This makes it easier to track and review changes in generated code. +# To execute, invoke: 'python run_tests.py' + +import os +import os.path +import subprocess +import sys +import tempfile +from webkitpy.common.checkout import scm + + +def generate_from_idl(generator, idl_file, output_directory): + cmd = ['perl', '-w', + '-IWebCore/bindings/scripts', + 'WebCore/bindings/scripts/generate-bindings.pl', + # idl include directories (path relative to generate-bindings.pl) + '--include', '.', + '--defines', 'TESTING_%s' % generator, + '--generator', generator, + '--outputDir', output_directory, + idl_file] + return subprocess.call(cmd) == 0 + + +def detect_changes(work_directory, reference_directory): + changes_found = False + for output_file in os.listdir(work_directory): + print 'Detecting changes in %s...' % output_file + cmd = ['diff', + '-u', + os.path.join(reference_directory, output_file), + os.path.join(work_directory, output_file)] + if subprocess.call(cmd) != 0: + print 'Detected changes in %s (see above)' % output_file + changes_found = True + else: + print 'No changes found.' + + return changes_found + + +def run_tests(generator, input_directory, reference_directory, reset_results): + work_directory = reference_directory + + passed = True + for input_file in os.listdir(input_directory): + (name, extension) = os.path.splitext(input_file) + if extension != '.idl': + continue + print 'Testing the %s generator on %s' % (generator, input_file) + # Generate output into the work directory (either the given one or a + # temp one if not reset_results is performed) + if not reset_results: + work_directory = tempfile.mkdtemp() + if not generate_from_idl(generator, os.path.join(input_directory, + input_file), + work_directory): + passed = False + if reset_results: + print "Overwrote reference files" + continue + # Detect changes + if detect_changes(work_directory, reference_directory): + passed = False + + if not passed: + print '%s generator failed.' % generator + return passed + + +def main(argv): + """Runs WebCore bindings code generators on test IDL files and compares + the results with reference files. + + Options: + --reset-results: Overwrites the reference files with the generated results. + + """ + reset_results = "--reset-results" in argv + + current_scm = scm.detect_scm_system(os.curdir) + os.chdir(current_scm.checkout_root) + + all_tests_passed = True + + generators = [ + 'JS', + 'V8', + 'ObjC', + 'GObject', + 'CPP' + ] + + for generator in generators: + input_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test') + reference_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test', generator) + if not run_tests(generator, input_directory, reference_directory, reset_results): + all_tests_passed = False + + if all_tests_passed: + print 'All tests passed!' + return 0 + else: + print '(To update the reference files, execute "run-bindings-tests --reset-results")' + return -1 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/WebKitTools/Scripts/run-gtk-tests b/WebKitTools/Scripts/run-gtk-tests new file mode 100644 index 0000000..9a57319 --- /dev/null +++ b/WebKitTools/Scripts/run-gtk-tests @@ -0,0 +1,35 @@ +#!/usr/bin/perl +# +# Copyright (C) 2009 Gustavo Noronha Silva <gns@gnome.org> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Library General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Library General Public License for more details. +# +# You should have received a copy of the GNU Library General Public License +# along with this library; see the file COPYING.LIB. If not, write to +# the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + +use strict; +use warnings; + +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; + +# This initializes the correct configuration (Release/Debug) +setConfiguration(); + +my $productDir = productDir(); +my @unitTests = glob $productDir . "/Programs/unittests/*"; +if ($#unitTests < 1) { + die "ERROR: tests not found in $productDir.\n"; +} +system "gtester -k @unitTests" diff --git a/WebKitTools/Scripts/run-launcher b/WebKitTools/Scripts/run-launcher index e12a64a..414d4af 100755 --- a/WebKitTools/Scripts/run-launcher +++ b/WebKitTools/Scripts/run-launcher @@ -47,7 +47,7 @@ checkFrameworks(); # Set paths according to the build system used if (isQt()) { my $libDir = catdir(productDir(), 'lib'); - $launcherPath = catdir($launcherPath, "bin", "QtLauncher"); + $launcherPath = catdir($launcherPath, "bin", "QtTestBrowser"); $ENV{QTWEBKIT_PLUGIN_PATH} = catdir($libDir, 'plugins'); @@ -64,6 +64,10 @@ if (isQt()) { $launcherPath = catdir($launcherPath, "Programs", "GtkLauncher"); } + if (isEfl()) { + $launcherPath = catdir($launcherPath, "Programs", "EWebLauncher"); + } + if (isWx()) { if (isDarwin()) { $launcherPath = catdir($launcherPath, 'wxBrowser.app', 'Contents', 'MacOS', 'wxBrowser'); diff --git a/WebKitTools/Scripts/run-minibrowser b/WebKitTools/Scripts/run-minibrowser new file mode 100755 index 0000000..c2fd412 --- /dev/null +++ b/WebKitTools/Scripts/run-minibrowser @@ -0,0 +1,38 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2005, 2007 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Simplified "run" script for launching the WebKit2 MiniBrowser. + +use strict; +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; + +setConfiguration(); + +exit exitStatus(runMiniBrowser()); diff --git a/WebKitTools/Scripts/run-qtwebkit-tests b/WebKitTools/Scripts/run-qtwebkit-tests new file mode 100644 index 0000000..373de0a --- /dev/null +++ b/WebKitTools/Scripts/run-qtwebkit-tests @@ -0,0 +1,358 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +#Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies) + +#This library is free software; you can redistribute it and/or +#modify it under the terms of the GNU Library General Public +#License as published by the Free Software Foundation; either +#version 2 of the License, or (at your option) any later version. + +#This library is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +#Library General Public License for more details. + +#You should have received a copy of the GNU Library General Public License +#along with this library; see the file COPYING.LIB. If not, write to +#the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +#Boston, MA 02110-1301, USA. + +from __future__ import with_statement + +import sys +import os +import os.path +import re +import logging +from subprocess import Popen, PIPE, STDOUT +from optparse import OptionParser + + +class Log(object): + def __init__(self, name): + self._log = logging.getLogger(name) + self.debug = self._log.debug + self.warn = self._log.warn + self.error = self._log.error + self.exception = self._log.exception + self.info = self._log.info + + +class Options(Log): + """ Option manager. It parses and checks script's parameters, sets an internal variable. """ + + def __init__(self, args): + Log.__init__(self, "Options") + log = self._log + opt = OptionParser("%prog [options] PathToSearch.\nTry -h or --help.") + opt.add_option("-j", "--parallel-level", action="store", type="int", + dest="parallel_level", default=None, + help="Number of parallel processes executing the Qt's tests. Default: cpu count.") + opt.add_option("-v", "--verbose", action="store", type="int", + dest="verbose", default=2, + help="Verbose level (0 - quiet, 1 - errors only, 2 - infos and warnings, 3 - debug information). Default: %default.") + opt.add_option("", "--tests-options", action="store", type="string", + dest="tests_options", default="", + help="Parameters passed to Qt's tests (for example '-eventdelay 123').") + opt.add_option("-o", "--output-file", action="store", type="string", + dest="output_file", default="/tmp/qtwebkittests.html", + help="File where results will be stored. The file will be overwritten. Default: %default.") + opt.add_option("-b", "--browser", action="store", dest="browser", + default="xdg-open", + help="Browser in which results will be opened. Default %default.") + opt.add_option("", "--do-not-open-results", action="store_false", + dest="open_results", default=True, + help="The results shouldn't pop-up in a browser automatically") + opt.add_option("-d", "--developer-mode", action="store_true", + dest="developer", default=False, + help="Special mode for debugging. In general it simulates human behavior, running all autotests. In the mode everything is executed synchronously, no html output will be generated, no changes or transformation will be applied to stderr or stdout. In this mode options; parallel-level, output-file, browser and do-not-open-results will be ignored.") + + self._o, self._a = opt.parse_args(args) + verbose = self._o.verbose + if verbose == 0: + logging.basicConfig(level=logging.CRITICAL,) + elif verbose == 1: + logging.basicConfig(level=logging.ERROR,) + elif verbose == 2: + logging.basicConfig(level=logging.INFO,) + elif verbose == 3: + logging.basicConfig(level=logging.DEBUG,) + else: + logging.basicConfig(level=logging.INFO,) + log.warn("Bad verbose level, switching to default.") + try: + if not os.path.exists(self._a[0]): + raise Exception("Given path doesn't exist.") + if len(self._a) > 1: + raise IndexError("Only one directory could be provided.") + self._o.path = self._a[0] + except IndexError: + log.error("Bad usage. Please try -h or --help.") + sys.exit(1) + except Exception: + log.error("Path '%s' doesn't exist", self._a[0]) + sys.exit(2) + if self._o.developer: + if not self._o.parallel_level is None: + log.warn("Developer mode sets parallel-level option to one.") + self._o.parallel_level = 1 + self._o.open_results = False + + def __getattr__(self, attr): + """ Maps all options properties into this object (remove one level of indirection). """ + return getattr(self._o, attr) + + +def run_test(args): + """ Runs one given test. + args should contain a tuple with 3 elements; + TestSuiteResult containing full file name of an autotest executable. + str with options that should be passed to the autotest executable + bool if true then the stdout will be buffered and separated from the stderr, if it is false + then the stdout and the stderr will be merged together and left unbuffered (the TestSuiteResult output will be None). + """ + log = logging.getLogger("Exec") + test_suite, options, buffered = args + try: + log.info("Running... %s", test_suite.test_file_name()) + if buffered: + tst = Popen(test_suite.test_file_name() + options, stdout=PIPE, stderr=None, shell=True) + else: + tst = Popen(test_suite.test_file_name() + options, stdout=None, stderr=STDOUT, shell=True) + except OSError, e: + log.exception("Can't open an autotest file: '%s'. Skipping the test...", e.filename) + else: + test_suite.set_output(tst.communicate()[0]) # takes stdout only, in developer mode it would be None. + log.info("Finished %s", test_suite.test_file_name()) + return test_suite + + +class TestSuiteResult(object): + """ Keeps information about a test. """ + + def __init__(self): + self._output = None + self._test_file_name = None + + def set_output(self, xml): + if xml: + self._output = xml.strip() + + def output(self): + return self._output + + def set_test_file_name(self, file_name): + self._test_file_name = file_name + + def test_file_name(self): + return self._test_file_name + + +class Main(Log): + """ The main script. All real work is done in run() method. """ + + def __init__(self, options): + Log.__init__(self, "Main") + self._options = options + if options.parallel_level > 1 or options.parallel_level is None: + try: + from multiprocessing import Pool + except ImportError: + self.warn("Import Error: the multiprocessing module couldn't be loaded (may be lack of python-multiprocessing package?). The Qt autotests will be executed one by one.") + options.parallel_level = 1 + if options.parallel_level == 1: + + class Pool(object): + """ A hack, created to avoid problems with multiprocessing module, this class is single thread replacement for the multiprocessing.Pool class. """ + def __init__(self, processes): + pass + + def imap_unordered(self, func, files): + return map(func, files) + + def map(self, func, files): + return map(func, files) + + self._Pool = Pool + + def run(self): + """ Find && execute && publish results of all test. "All in one" function. """ + self.debug("Searching executables...") + tests_executables = self.find_tests_paths(self._options.path) + self.debug("Found: %s", len(tests_executables)) + self.debug("Executing tests...") + results = self.run_tests(tests_executables) + if not self._options.developer: + self.debug("Transforming...") + transformed_results = self.transform(results) + self.debug("Publishing...") + self.announce_results(transformed_results) + + def find_tests_paths(self, path): + """ Finds all tests executables inside the given path. """ + executables = [] + for root, dirs, files in os.walk(path): + # Check only for a file that name starts from 'tst_' and that we can execute. + filtered_path = filter(lambda w: w.startswith('tst_') and os.access(os.path.join(root, w), os.X_OK), files) + filtered_path = map(lambda w: os.path.join(root, w), filtered_path) + for file_name in filtered_path: + r = TestSuiteResult() + r.set_test_file_name(file_name) + executables.append(r) + return executables + + def run_tests(self, files): + """ Executes given files by using a pool of workers. """ + workers = self._Pool(processes=self._options.parallel_level) + # to each file add options. + self.debug("Using %s the workers pool, number of workers %i", repr(workers), self._options.parallel_level) + package = map(lambda w: [w, self._options.tests_options, not self._options.developer], files) + self.debug("Generated packages for workers: %s", repr(package)) + results = workers.map(run_test, package) # Collects results. + return results + + def transform(self, results): + """ Transforms list of the results to specialized versions. """ + stdout = self.convert_to_stdout(results) + html = self.convert_to_html(results) + return {"stdout": stdout, "html": html} + + def announce_results(self, results): + """ Shows the results. """ + self.announce_results_stdout(results['stdout']) + self.announce_results_html(results['html']) + + def announce_results_stdout(self, results): + """ Show the results by printing to the stdout.""" + print(results) + + def announce_results_html(self, results): + """ Shows the result by creating a html file and calling a web browser to render it. """ + with file(self._options.output_file, 'w') as f: + f.write(results) + if self._options.open_results: + Popen(self._options.browser + " " + self._options.output_file, stdout=None, stderr=None, shell=True) + + def convert_to_stdout(self, results): + """ Converts results, that they could be nicely presented in the stdout. """ + # Join all results into one piece. + txt = "\n\n".join(map(lambda w: w.output(), results)) + # Find total count of failed, skipped and passed tests. + totals = re.findall(r"([0-9]+) passed, ([0-9]+) failed, ([0-9]+) skipped", txt) + totals = reduce(lambda x, y: (int(x[0]) + int(y[0]), int(x[1]) + int(y[1]), int(x[2]) + int(y[2])), totals) + totals = map(str, totals) + totals = totals[0] + " passed, " + totals[1] + " failed, " + totals[2] + " skipped" + # Add a summary. + txt += '\n\n\n' + '*' * 70 + txt += "\n**" + ("TOTALS: " + totals).center(66) + '**' + txt += '\n' + '*' * 70 + '\n' + return txt + + def convert_to_html(self, results): + """ Converts results, that they could showed as a html page. """ + # Join results into one piece. + txt = "\n\n".join(map(lambda w: w.output(), results)) + txt = txt.replace('&', '&').replace('<', "<").replace('>', ">") + # Add a color and a style. + txt = re.sub(r"([* ]+(Finished)[ a-z_A-Z0-9]+[*]+)", + lambda w: r"", + txt) + txt = re.sub(r"([*]+[ a-z_A-Z0-9]+[*]+)", + lambda w: "<case class='good'><br><br><b>" + w.group(0) + r"</b></case>", + txt) + txt = re.sub(r"(Config: Using QTest library)((.)+)", + lambda w: "\n<case class='good'><br><i>" + w.group(0) + r"</i> ", + txt) + txt = re.sub(r"\n(PASS)((.)+)", + lambda w: "</case>\n<case class='good'><br><status class='pass'>" + w.group(1) + r"</status>" + w.group(2), + txt) + txt = re.sub(r"\n(FAIL!)((.)+)", + lambda w: "</case>\n<case class='bad'><br><status class='fail'>" + w.group(1) + r"</status>" + w.group(2), + txt) + txt = re.sub(r"\n(XPASS)((.)+)", + lambda w: "</case>\n<case class='bad'><br><status class='xpass'>" + w.group(1) + r"</status>" + w.group(2), + txt) + txt = re.sub(r"\n(XFAIL)((.)+)", + lambda w: "</case>\n<case class='good'><br><status class='xfail'>" + w.group(1) + r"</status>" + w.group(2), + txt) + txt = re.sub(r"\n(SKIP)((.)+)", + lambda w: "</case>\n<case class='good'><br><status class='xfail'>" + w.group(1) + r"</status>" + w.group(2), + txt) + txt = re.sub(r"\n(QWARN)((.)+)", + lambda w: "</case>\n<case class='bad'><br><status class='warn'>" + w.group(1) + r"</status>" + w.group(2), + txt) + txt = re.sub(r"\n(RESULT)((.)+)", + lambda w: "</case>\n<case class='good'><br><status class='benchmark'>" + w.group(1) + r"</status>" + w.group(2), + txt) + txt = re.sub(r"\n(QFATAL)((.)+)", + lambda w: "</case>\n<case class='bad'><br><status class='crash'>" + w.group(1) + r"</status>" + w.group(2), + txt) + txt = re.sub(r"\n(Totals:)([0-9', a-z]*)", + lambda w: "</case>\n<case class='good'><br><b>" + w.group(1) + r"</b>" + w.group(2) + "</case>", + txt) + # Find total count of failed, skipped and passed tests. + totals = re.findall(r"([0-9]+) passed, ([0-9]+) failed, ([0-9]+) skipped", txt) + totals = reduce(lambda x, y: (int(x[0]) + int(y[0]), int(x[1]) + int(y[1]), int(x[2]) + int(y[2])), totals) + totals = map(str, totals) + totals = totals[0] + " passed, " + totals[1] + " failed, " + totals[2] + " skipped." + # Create a header of the html source. + txt = """ + <html> + <head> + <script> + function init() { + // Try to find the right styleSheet (this document could be embedded in an other html doc) + for (i = document.styleSheets.length - 1; i >= 0; --i) { + if (document.styleSheets[i].cssRules[0].selectorText == "case.good") { + resultStyleSheet = i; + return; + } + } + // The styleSheet hasn't been found, but it should be the last one. + resultStyleSheet = document.styleSheets.length - 1; + } + + function hide() { + document.styleSheets[resultStyleSheet].cssRules[0].style.display='none'; + } + + function show() { + document.styleSheets[resultStyleSheet].cssRules[0].style.display=''; + } + + </script> + <style type="text/css"> + case.good {color:black} + case.bad {color:black} + status.pass {color:green} + status.crash {color:red} + status.fail {color:red} + status.xpass {color:663300} + status.xfail {color:004500} + status.benchmark {color:000088} + status.warn {color:orange} + status.crash {color:red; text-decoration:blink; background-color:black} + </style> + </head> + <body onload="init()"> + <center> + <h1>Qt's autotests results</h1>%(totals)s<br> + <hr> + <form> + <input type="button" value="Show failures only" onclick="hide()"/> + + <input type="button" value="Show all" onclick="show()"/> + </form> + </center> + <hr> + %(results)s + </body> + </html>""" % {"totals": totals, "results": txt} + return txt + + +if __name__ == '__main__': + options = Options(sys.argv[1:]) + main = Main(options) + main.run() diff --git a/WebKitTools/Scripts/run-test-runner b/WebKitTools/Scripts/run-test-runner new file mode 100755 index 0000000..98fa3b6 --- /dev/null +++ b/WebKitTools/Scripts/run-test-runner @@ -0,0 +1,35 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +# Simplified "run" script for launching the WebKit2 WebKitTestRunner. + +use strict; +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; + +setConfiguration(); + +exit exitStatus(runWebKitTestRunner()); diff --git a/WebKitTools/Scripts/run-webkit-httpd b/WebKitTools/Scripts/run-webkit-httpd index 018f64c..9ea2551 100755 --- a/WebKitTools/Scripts/run-webkit-httpd +++ b/WebKitTools/Scripts/run-webkit-httpd @@ -42,6 +42,10 @@ use lib $FindBin::Bin; use webkitperl::httpd; use webkitdirs; +# FIXME: Dynamic HTTP-port configuration in this file is wrong. The various +# apache config files in LayoutTests/http/config govern the port numbers. +# Dynamic configuration as-written will also cause random failures in +# an IPv6 environment. See https://bugs.webkit.org/show_bug.cgi?id=37104. # Argument handling my $httpdPort = 8000; my $allInterfaces = 0; diff --git a/WebKitTools/Scripts/run-webkit-tests b/WebKitTools/Scripts/run-webkit-tests index 809e078..8fe8360 100755 --- a/WebKitTools/Scripts/run-webkit-tests +++ b/WebKitTools/Scripts/run-webkit-tests @@ -1,2222 +1,83 @@ #!/usr/bin/perl - -# Copyright (C) 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved. -# Copyright (C) 2006 Alexey Proskuryakov (ap@nypop.com) -# Copyright (C) 2007 Matt Lilek (pewtermoose@gmail.com) -# Copyright (C) 2007 Eric Seidel <eric@webkit.org> -# Copyright (C) 2009 Google Inc. All rights reserved. -# Copyright (C) 2009 Andras Becsi (becsi.andras@stud.u-szeged.hu), University of Szeged +# Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: +# modification, are permitted provided that the following conditions are +# met: # -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of -# its contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. # -# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Script to run the WebKit Open Source Project layout tests. - -# Run all the tests passed in on the command line. -# If no tests are passed, find all the .html, .shtml, .xml, .xhtml, .pl, .php (and svg) files in the test directory. - -# Run each text. -# Compare against the existing file xxx-expected.txt. -# If there is a mismatch, generate xxx-actual.txt and xxx-diffs.txt. - -# At the end, report: -# the number of tests that got the expected results -# the number of tests that ran, but did not get the expected results -# the number of tests that failed to run -# the number of tests that were run but had no expected results to compare against +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This file is a temporary hack. +# It will be removed as soon as all platforms are are ready to move to +# new-run-webkit-tests and we can then update the buildbots to explicitly +# call old-run-webkit-tests for any platforms which will never support +# a Python run-webkit-tests. + +# This is intentionally written in Perl to guarantee support on +# the same set of platforms as old-run-webkit-tests currently supports. +# The buildbot master.cfg also currently passes run-webkit-tests to +# perl directly instead of executing it in a shell. use strict; use warnings; -use Cwd; -use Data::Dumper; -use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK); -use File::Basename; -use File::Copy; -use File::Find; -use File::Path; -use File::Spec; -use File::Spec::Functions; use FindBin; -use Getopt::Long; -use IPC::Open2; -use IPC::Open3; -use Time::HiRes qw(time usleep); - -use List::Util 'shuffle'; - use lib $FindBin::Bin; -use webkitperl::features; -use webkitperl::httpd; use webkitdirs; -use VCSUtils; -use POSIX; - -sub buildPlatformResultHierarchy(); -sub buildPlatformTestHierarchy(@); -sub closeCygpaths(); -sub closeDumpTool(); -sub closeWebSocketServer(); -sub configureAndOpenHTTPDIfNeeded(); -sub countAndPrintLeaks($$$); -sub countFinishedTest($$$$); -sub deleteExpectedAndActualResults($); -sub dumpToolDidCrash(); -sub epiloguesAndPrologues($$); -sub expectedDirectoryForTest($;$;$); -sub fileNameWithNumber($$); -sub htmlForResultsSection(\@$&); -sub isTextOnlyTest($); -sub launchWithEnv(\@\%); -sub resolveAndMakeTestResultsDirectory(); -sub numericcmp($$); -sub openDiffTool(); -sub openDumpTool(); -sub parseLeaksandPrintUniqueLeaks(); -sub openWebSocketServerIfNeeded(); -sub pathcmp($$); -sub printFailureMessageForTest($$); -sub processIgnoreTests($$); -sub readFromDumpToolWithTimer(**); -sub readSkippedFiles($); -sub recordActualResultsAndDiff($$); -sub sampleDumpTool(); -sub setFileHandleNonBlocking(*$); -sub slowestcmp($$); -sub splitpath($); -sub stripExtension($); -sub stripMetrics($$); -sub testCrashedOrTimedOut($$$$$); -sub toURL($); -sub toWindowsPath($); -sub validateSkippedArg($$;$); -sub writeToFile($$); - -# Argument handling -my $addPlatformExceptions = 0; -my $complexText = 0; -my $exitAfterNFailures = 0; -my $generateNewResults = isAppleMacWebKit() ? 1 : 0; -my $guardMalloc = ''; -my $httpdPort = 8000; -my $httpdSSLPort = 8443; -my $ignoreMetrics = 0; -my $webSocketPort = 8880; -# wss is disabled until all platforms support pyOpenSSL. -# my $webSocketSecurePort = 9323; -my $ignoreTests = ''; -my $iterations = 1; -my $launchSafari = 1; -my $mergeDepth; -my $pixelTests = ''; -my $platform; -my $quiet = ''; -my $randomizeTests = 0; -my $repeatEach = 1; -my $report10Slowest = 0; -my $resetResults = 0; -my $reverseTests = 0; -my $root; -my $runSample = 1; -my $shouldCheckLeaks = 0; -my $showHelp = 0; -my $stripEditingCallbacks = isCygwin(); -my $testHTTP = 1; -my $testMedia = 1; -my $tmpDir = "/tmp"; -my $testResultsDirectory = File::Spec->catfile($tmpDir, "layout-test-results"); -my $testsPerDumpTool = 1000; -my $threaded = 0; -# DumpRenderTree has an internal timeout of 15 seconds, so this must be > 15. -my $timeoutSeconds = 20; -my $tolerance = 0; -my $treatSkipped = "default"; -my $useRemoteLinksToTests = 0; -my $useValgrind = 0; -my $verbose = 0; -my $shouldWaitForHTTPD = 0; - -my @leaksFilenames; - -if (isWindows() || isMsys()) { - print "This script has to be run under Cygwin to function correctly.\n"; - exit 1; -} - -# Default to --no-http for wx for now. -$testHTTP = 0 if (isWx()); - -my $expectedTag = "expected"; -my $actualTag = "actual"; -my $prettyDiffTag = "pretty-diff"; -my $diffsTag = "diffs"; -my $errorTag = "stderr"; - -my @macPlatforms = ("mac-tiger", "mac-leopard", "mac-snowleopard", "mac"); - -if (isAppleMacWebKit()) { - if (isTiger()) { - $platform = "mac-tiger"; - $tolerance = 1.0; - } elsif (isLeopard()) { - $platform = "mac-leopard"; - $tolerance = 0.1; - } elsif (isSnowLeopard()) { - $platform = "mac-snowleopard"; - $tolerance = 0.1; - } else { - $platform = "mac"; - } -} elsif (isQt()) { - if (isDarwin()) { - $platform = "qt-mac"; - } elsif (isLinux()) { - $platform = "qt-linux"; - } elsif (isWindows() || isCygwin()) { - $platform = "qt-win"; - } else { - $platform = "qt"; - } -} elsif (isGtk()) { - $platform = "gtk"; - if (!$ENV{"WEBKIT_TESTFONTS"}) { - print "The WEBKIT_TESTFONTS environment variable is not defined.\n"; - print "You must set it before running the tests.\n"; - print "Use git to grab the actual fonts from http://gitorious.org/qtwebkit/testfonts\n"; - exit 1; - } -} elsif (isWx()) { - $platform = "wx"; -} elsif (isCygwin()) { - $platform = "win"; -} - -if (!defined($platform)) { - print "WARNING: Your platform is not recognized. Any platform-specific results will be generated in platform/undefined.\n"; - $platform = "undefined"; -} - -my $programName = basename($0); -my $launchSafariDefault = $launchSafari ? "launch" : "do not launch"; -my $httpDefault = $testHTTP ? "run" : "do not run"; -my $sampleDefault = $runSample ? "run" : "do not run"; - -my $usage = <<EOF; -Usage: $programName [options] [testdir|testpath ...] - --add-platform-exceptions Put new results for non-platform-specific failing tests into the platform-specific results directory - --complex-text Use the complex text code path for all text (Mac OS X and Windows only) - -c|--configuration config Set DumpRenderTree build configuration - -g|--guard-malloc Enable malloc guard - --exit-after-n-failures N Exit after the first N failures instead of running all tests - -h|--help Show this help message - --[no-]http Run (or do not run) http tests (default: $httpDefault) - --[no-]wait-for-httpd Wait for httpd if some other test session is using it already (same as WEBKIT_WAIT_FOR_HTTPD=1). (default: $shouldWaitForHTTPD) - -i|--ignore-tests Comma-separated list of directories or tests to ignore - --iterations n Number of times to run the set of tests (e.g. ABCABCABC) - --[no-]launch-safari Launch (or do not launch) Safari to display test results (default: $launchSafariDefault) - -l|--leaks Enable leaks checking - --[no-]new-test-results Generate results for new tests - --nthly n Restart DumpRenderTree every n tests (default: $testsPerDumpTool) - -p|--pixel-tests Enable pixel tests - --tolerance t Ignore image differences less than this percentage (default: $tolerance) - --platform Override the detected platform to use for tests and results (default: $platform) - --port Web server port to use with http tests - -q|--quiet Less verbose output - --reset-results Reset ALL results (including pixel tests if --pixel-tests is set) - -o|--results-directory Output results directory (default: $testResultsDirectory) - --random Run the tests in a random order - --repeat-each n Number of times to run each test (e.g. AAABBBCCC) - --reverse Run the tests in reverse alphabetical order - --root Path to root tools build - --[no-]sample-on-timeout Run sample on timeout (default: $sampleDefault) (Mac OS X only) - -1|--singly Isolate each test case run (implies --nthly 1 --verbose) - --skipped=[default|ignore|only] Specifies how to treat the Skipped file - default: Tests/directories listed in the Skipped file are not tested - ignore: The Skipped file is ignored - only: Only those tests/directories listed in the Skipped file will be run - --slowest Report the 10 slowest tests - --ignore-metrics Ignore metrics in tests - --[no-]strip-editing-callbacks Remove editing callbacks from expected results - -t|--threaded Run a concurrent JavaScript thead with each test - --timeout t Sets the number of seconds before a test times out (default: $timeoutSeconds) - --valgrind Run DumpRenderTree inside valgrind (Qt/Linux only) - -v|--verbose More verbose output (overrides --quiet) - -m|--merge-leak-depth arg Merges leak callStacks and prints the number of unique leaks beneath a callstack depth of arg. Defaults to 5. - --use-remote-links-to-tests Link to test files within the SVN repository in the results. -EOF - -setConfiguration(); - -my $getOptionsResult = GetOptions( - 'add-platform-exceptions' => \$addPlatformExceptions, - 'complex-text' => \$complexText, - 'exit-after-n-failures=i' => \$exitAfterNFailures, - 'guard-malloc|g' => \$guardMalloc, - 'help|h' => \$showHelp, - 'http!' => \$testHTTP, - 'wait-for-httpd!' => \$shouldWaitForHTTPD, - 'ignore-metrics!' => \$ignoreMetrics, - 'ignore-tests|i=s' => \$ignoreTests, - 'iterations=i' => \$iterations, - 'launch-safari!' => \$launchSafari, - 'leaks|l' => \$shouldCheckLeaks, - 'merge-leak-depth|m:5' => \$mergeDepth, - 'new-test-results!' => \$generateNewResults, - 'nthly=i' => \$testsPerDumpTool, - 'pixel-tests|p' => \$pixelTests, - 'platform=s' => \$platform, - 'port=i' => \$httpdPort, - 'quiet|q' => \$quiet, - 'random' => \$randomizeTests, - 'repeat-each=i' => \$repeatEach, - 'reset-results' => \$resetResults, - 'results-directory|o=s' => \$testResultsDirectory, - 'reverse' => \$reverseTests, - 'root=s' => \$root, - 'sample-on-timeout!' => \$runSample, - 'singly|1' => sub { $testsPerDumpTool = 1; }, - 'skipped=s' => \&validateSkippedArg, - 'slowest' => \$report10Slowest, - 'strip-editing-callbacks!' => \$stripEditingCallbacks, - 'threaded|t' => \$threaded, - 'timeout=i' => \$timeoutSeconds, - 'tolerance=f' => \$tolerance, - 'use-remote-links-to-tests' => \$useRemoteLinksToTests, - 'valgrind' => \$useValgrind, - 'verbose|v' => \$verbose, -); - -if (!$getOptionsResult || $showHelp) { - print STDERR $usage; - exit 1; -} - -my $ignoreSkipped = $treatSkipped eq "ignore"; -my $skippedOnly = $treatSkipped eq "only"; - -my $configuration = configuration(); - -# We need an environment variable to be able to enable the feature per-slave -$shouldWaitForHTTPD = $ENV{"WEBKIT_WAIT_FOR_HTTPD"} unless ($shouldWaitForHTTPD); -$verbose = 1 if $testsPerDumpTool == 1; - -if ($shouldCheckLeaks && $testsPerDumpTool > 1000) { - print STDERR "\nWARNING: Running more than 1000 tests at a time with MallocStackLogging enabled may cause a crash.\n\n"; -} - -# Stack logging does not play well with QuickTime on Tiger (rdar://problem/5537157) -$testMedia = 0 if $shouldCheckLeaks && isTiger(); - -# Generating remote links causes a lot of unnecessary spew on GTK build bot -$useRemoteLinksToTests = 0 if isGtk(); - -setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root)); -my $productDir = productDir(); -$productDir .= "/bin" if isQt(); -$productDir .= "/Programs" if isGtk(); - -chdirWebKit(); - -if (!defined($root)) { - print STDERR "Running build-dumprendertree\n"; - - local *DEVNULL; - my ($childIn, $childOut, $childErr); - if ($quiet) { - open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null"; - $childOut = ">&DEVNULL"; - $childErr = ">&DEVNULL"; - } else { - # When not quiet, let the child use our stdout/stderr. - $childOut = ">&STDOUT"; - $childErr = ">&STDERR"; - } - - my @args = argumentsForConfiguration(); - my $buildProcess = open3($childIn, $childOut, $childErr, "WebKitTools/Scripts/build-dumprendertree", @args) or die "Failed to run build-dumprendertree"; - close($childIn); - waitpid $buildProcess, 0; - my $buildResult = $?; - close($childOut); - close($childErr); - - close DEVNULL if ($quiet); - - if ($buildResult) { - print STDERR "Compiling DumpRenderTree failed!\n"; - exit exitStatus($buildResult); - } -} - -my $dumpToolName = "DumpRenderTree"; -$dumpToolName .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/; -my $dumpTool = "$productDir/$dumpToolName"; -die "can't find executable $dumpToolName (looked in $productDir)\n" unless -x $dumpTool; - -my $imageDiffTool = "$productDir/ImageDiff"; -$imageDiffTool .= "_debug" if isCygwin() && configurationForVisualStudio() !~ /^Release|Debug_Internal$/; -die "can't find executable $imageDiffTool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool; - -checkFrameworks() unless isCygwin(); - -if (isAppleMacWebKit()) { - push @INC, $productDir; - require DumpRenderTreeSupport; -} - -my $layoutTestsName = "LayoutTests"; -my $testDirectory = File::Spec->rel2abs($layoutTestsName); -my $expectedDirectory = $testDirectory; -my $platformBaseDirectory = catdir($testDirectory, "platform"); -my $platformTestDirectory = catdir($platformBaseDirectory, $platform); -my @platformResultHierarchy = buildPlatformResultHierarchy(); -my @platformTestHierarchy = buildPlatformTestHierarchy(@platformResultHierarchy); - -$expectedDirectory = $ENV{"WebKitExpectedTestResultsDirectory"} if $ENV{"WebKitExpectedTestResultsDirectory"}; - -$testResultsDirectory = File::Spec->rel2abs($testResultsDirectory); -my $testResults = File::Spec->catfile($testResultsDirectory, "results.html"); - -print "Running tests from $testDirectory\n"; -if ($pixelTests) { - print "Enabling pixel tests with a tolerance of $tolerance%\n"; - if (isDarwin()) { - print "WARNING: Temporarily changing the main display color profile:\n"; - print "\tThe colors on your screen will change for the duration of the testing.\n"; - print "\tThis allows the pixel tests to have consistent color values across all machines.\n"; - - if (isPerianInstalled()) { - print "WARNING: Perian's QuickTime component is installed and this may affect pixel test results!\n"; - print "\tYou should avoid generating new pixel results in this environment.\n"; - print "\tSee https://bugs.webkit.org/show_bug.cgi?id=22615 for details.\n"; - } - } -} - -system "ln", "-s", $testDirectory, "/tmp/LayoutTests" unless -x "/tmp/LayoutTests"; - -my %ignoredFiles = ( "results.html" => 1 ); -my %ignoredDirectories = map { $_ => 1 } qw(platform); -my %ignoredLocalDirectories = map { $_ => 1 } qw(.svn _svn resources script-tests); -my %supportedFileExtensions = map { $_ => 1 } qw(html shtml xml xhtml pl php); - -if (!checkWebCoreFeatureSupport("MathML", 0)) { - $ignoredDirectories{'mathml'} = 1; -} - -# FIXME: We should fix webkitperl/features.pm:hasFeature() to do the correct feature detection for Cygwin. -if (checkWebCoreFeatureSupport("SVG", 0)) { - $supportedFileExtensions{'svg'} = 1; -} elsif (isCygwin()) { - $supportedFileExtensions{'svg'} = 1; -} else { - $ignoredLocalDirectories{'svg'} = 1; -} - -if (!$testHTTP) { - $ignoredDirectories{'http'} = 1; - $ignoredDirectories{'websocket'} = 1; -} - -if (!$testMedia) { - $ignoredDirectories{'media'} = 1; - $ignoredDirectories{'http/tests/media'} = 1; -} - -if (!checkWebCoreFeatureSupport("Accelerated Compositing", 0)) { - $ignoredDirectories{'compositing'} = 1; -} - -if (!checkWebCoreFeatureSupport("3D Rendering", 0)) { - $ignoredDirectories{'animations/3d'} = 1; - $ignoredDirectories{'transforms/3d'} = 1; -} - -if (!checkWebCoreFeatureSupport("3D Canvas", 0)) { - $ignoredDirectories{'fast/canvas/webgl'} = 1; -} - -if (checkWebCoreFeatureSupport("WML", 0)) { - $supportedFileExtensions{'wml'} = 1; -} else { - $ignoredDirectories{'http/tests/wml'} = 1; - $ignoredDirectories{'fast/wml'} = 1; - $ignoredDirectories{'wml'} = 1; -} - -if (!checkWebCoreFeatureSupport("XHTMLMP", 0)) { - $ignoredDirectories{'fast/xhtmlmp'} = 1; -} - -processIgnoreTests($ignoreTests, "ignore-tests") if $ignoreTests; -if (!$ignoreSkipped) { - if (!$skippedOnly || @ARGV == 0) { - readSkippedFiles(""); - } else { - # Since readSkippedFiles() appends to @ARGV, we must use a foreach - # loop so that we only iterate over the original argument list. - foreach my $argnum (0 .. $#ARGV) { - readSkippedFiles(shift @ARGV); - } - } -} - -my @tests = findTestsToRun(); - -die "no tests to run\n" if !@tests; - -my %counts; -my %tests; -my %imagesPresent; -my %imageDifferences; -my %durations; -my $count = 0; -my $leaksOutputFileNumber = 1; -my $totalLeaks = 0; - -my @toolArgs = (); -push @toolArgs, "--pixel-tests" if $pixelTests; -push @toolArgs, "--threaded" if $threaded; -push @toolArgs, "--complex-text" if $complexText; -push @toolArgs, "-"; - -my @diffToolArgs = (); -push @diffToolArgs, "--tolerance", $tolerance; - -$| = 1; - -my $dumpToolPID; -my $isDumpToolOpen = 0; -my $dumpToolCrashed = 0; -my $imageDiffToolPID; -my $isDiffToolOpen = 0; - -my $atLineStart = 1; -my $lastDirectory = ""; - -my $isHttpdOpen = 0; -my $isWebSocketServerOpen = 0; -my $webSocketServerPID = 0; -my $failedToStartWebSocketServer = 0; -# wss is disabled until all platforms support pyOpenSSL. -# my $webSocketSecureServerPID = 0; - -sub catch_pipe { $dumpToolCrashed = 1; } -$SIG{"PIPE"} = "catch_pipe"; - -print "Testing ", scalar @tests, " test cases"; -print " $iterations times" if ($iterations > 1); -print ", repeating each test $repeatEach times" if ($repeatEach > 1); -print ".\n"; - -my $overallStartTime = time; - -my %expectedResultPaths; - -my @originalTests = @tests; -# Add individual test repetitions -if ($repeatEach > 1) { - @tests = (); - foreach my $test (@originalTests) { - for (my $i = 0; $i < $repeatEach; $i++) { - push(@tests, $test); - } - } -} -# Add test set repetitions -for (my $i = 1; $i < $iterations; $i++) { - push(@tests, @originalTests); -} - -for my $test (@tests) { - my $newDumpTool = not $isDumpToolOpen; - openDumpTool(); - - my $base = stripExtension($test); - my $expectedExtension = ".txt"; - - my $dir = $base; - $dir =~ s|/[^/]+$||; - - if ($newDumpTool || $dir ne $lastDirectory) { - foreach my $logue (epiloguesAndPrologues($newDumpTool ? "" : $lastDirectory, $dir)) { - if (isCygwin()) { - $logue = toWindowsPath($logue); - } else { - $logue = canonpath($logue); - } - if ($verbose) { - print "running epilogue or prologue $logue\n"; - } - print OUT "$logue\n"; - # Throw away output from DumpRenderTree. - # Once for the test output and once for pixel results (empty) - while (<IN>) { - last if /#EOF/; - } - while (<IN>) { - last if /#EOF/; - } - } - } - - if ($verbose) { - print "running $test -> "; - $atLineStart = 0; - } elsif (!$quiet) { - if ($dir ne $lastDirectory) { - print "\n" unless $atLineStart; - print "$dir "; - } - print "."; - $atLineStart = 0; - } - - $lastDirectory = $dir; - - my $result; - - my $startTime = time if $report10Slowest; - - # Try to read expected hash file for pixel tests - my $suffixExpectedHash = ""; - if ($pixelTests && !$resetResults) { - my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); - if (open EXPECTEDHASH, "$expectedPixelDir/$base-$expectedTag.checksum") { - my $expectedHash = <EXPECTEDHASH>; - chomp($expectedHash); - close EXPECTEDHASH; - - # Format expected hash into a suffix string that is appended to the path / URL passed to DRT - $suffixExpectedHash = "'$expectedHash"; - } - } - - if ($test =~ /^http\//) { - configureAndOpenHTTPDIfNeeded(); - if ($test !~ /^http\/tests\/local\// && $test !~ /^http\/tests\/ssl\// && $test !~ /^http\/tests\/wml\// && $test !~ /^http\/tests\/media\//) { - my $path = canonpath($test); - $path =~ s/^http\/tests\///; - print OUT "http://127.0.0.1:$httpdPort/$path$suffixExpectedHash\n"; - } elsif ($test =~ /^http\/tests\/ssl\//) { - my $path = canonpath($test); - $path =~ s/^http\/tests\///; - print OUT "https://127.0.0.1:$httpdSSLPort/$path$suffixExpectedHash\n"; - } else { - my $testPath = "$testDirectory/$test"; - if (isCygwin()) { - $testPath = toWindowsPath($testPath); - } else { - $testPath = canonpath($testPath); - } - print OUT "$testPath$suffixExpectedHash\n"; - } - } elsif ($test =~ /^websocket\//) { - if ($test =~ /^websocket\/tests\/local\//) { - my $testPath = "$testDirectory/$test"; - if (isCygwin()) { - $testPath = toWindowsPath($testPath); - } else { - $testPath = canonpath($testPath); - } - print OUT "$testPath\n"; - } else { - if (openWebSocketServerIfNeeded()) { - my $path = canonpath($test); - if ($test =~ /^websocket\/tests\/ssl\//) { - # wss is disabled until all platforms support pyOpenSSL. - print STDERR "Error: wss is disabled until all platforms support pyOpenSSL."; - # print OUT "https://127.0.0.1:$webSocketSecurePort/$path\n"; - } else { - print OUT "http://127.0.0.1:$webSocketPort/$path\n"; - } - } else { - # We failed to launch the WebSocket server. Display a useful error message rather than attempting - # to run tests that expect the server to be available. - my $errorMessagePath = "$testDirectory/websocket/resources/server-failed-to-start.html"; - $errorMessagePath = isCygwin() ? toWindowsPath($errorMessagePath) : canonpath($errorMessagePath); - print OUT "$errorMessagePath\n"; - } - } - } else { - my $testPath = "$testDirectory/$test"; - if (isCygwin()) { - $testPath = toWindowsPath($testPath); - } else { - $testPath = canonpath($testPath); - } - print OUT "$testPath$suffixExpectedHash\n" if defined $testPath; - } - - # DumpRenderTree is expected to dump two "blocks" to stdout for each test. - # Each block is terminated by a #EOF on a line by itself. - # The first block is the output of the test (in text, RenderTree or other formats). - # The second block is for optional pixel data in PNG format, and may be empty if - # pixel tests are not being run, or the test does not dump pixels (e.g. text tests). - my $readResults = readFromDumpToolWithTimer(IN, ERROR); - - my $actual = $readResults->{output}; - my $error = $readResults->{error}; - - $expectedExtension = $readResults->{extension}; - my $expectedFileName = "$base-$expectedTag.$expectedExtension"; - - my $isText = isTextOnlyTest($actual); - - my $expectedDir = expectedDirectoryForTest($base, $isText, $expectedExtension); - $expectedResultPaths{$base} = "$expectedDir/$expectedFileName"; - - unless ($readResults->{status} eq "success") { - my $crashed = $readResults->{status} eq "crashed"; - testCrashedOrTimedOut($test, $base, $crashed, $actual, $error); - countFinishedTest($test, $base, $crashed ? "crash" : "timedout", 0); - next; - } - - $durations{$test} = time - $startTime if $report10Slowest; - - my $expected; - - if (!$resetResults && open EXPECTED, "<", "$expectedDir/$expectedFileName") { - $expected = ""; - while (<EXPECTED>) { - next if $stripEditingCallbacks && $_ =~ /^EDITING DELEGATE:/; - $expected .= $_; - } - close EXPECTED; - } - - if ($ignoreMetrics && !$isText && defined $expected) { - ($actual, $expected) = stripMetrics($actual, $expected); - } - - if ($shouldCheckLeaks && $testsPerDumpTool == 1) { - print " $test -> "; - } - - my $actualPNG = ""; - my $diffPNG = ""; - my $diffPercentage = 0; - my $diffResult = "passed"; - - my $actualHash = ""; - my $expectedHash = ""; - my $actualPNGSize = 0; - - while (<IN>) { - last if /#EOF/; - if (/ActualHash: ([a-f0-9]{32})/) { - $actualHash = $1; - } elsif (/ExpectedHash: ([a-f0-9]{32})/) { - $expectedHash = $1; - } elsif (/Content-Length: (\d+)\s*/) { - $actualPNGSize = $1; - read(IN, $actualPNG, $actualPNGSize); - } - } - - if ($verbose && $pixelTests && !$resetResults && $actualPNGSize) { - if ($actualHash eq "" && $expectedHash eq "") { - printFailureMessageForTest($test, "WARNING: actual & expected pixel hashes are missing!"); - } elsif ($actualHash eq "") { - printFailureMessageForTest($test, "WARNING: actual pixel hash is missing!"); - } elsif ($expectedHash eq "") { - printFailureMessageForTest($test, "WARNING: expected pixel hash is missing!"); - } - } - if ($actualPNGSize > 0) { - my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); - - if (!$resetResults && ($expectedHash ne $actualHash || ($actualHash eq "" && $expectedHash eq ""))) { - if (-f "$expectedPixelDir/$base-$expectedTag.png") { - my $expectedPNGSize = -s "$expectedPixelDir/$base-$expectedTag.png"; - my $expectedPNG = ""; - open EXPECTEDPNG, "$expectedPixelDir/$base-$expectedTag.png"; - read(EXPECTEDPNG, $expectedPNG, $expectedPNGSize); - - openDiffTool(); - print DIFFOUT "Content-Length: $actualPNGSize\n"; - print DIFFOUT $actualPNG; - - print DIFFOUT "Content-Length: $expectedPNGSize\n"; - print DIFFOUT $expectedPNG; - - while (<DIFFIN>) { - last if /^error/ || /^diff:/; - if (/Content-Length: (\d+)\s*/) { - read(DIFFIN, $diffPNG, $1); - } - } - - if (/^diff: (.+)% (passed|failed)/) { - $diffPercentage = $1 + 0; - $imageDifferences{$base} = $diffPercentage; - $diffResult = $2; - } - - if (!$diffPercentage) { - printFailureMessageForTest($test, "pixel hash failed (but pixel test still passes)"); - } - } elsif ($verbose) { - printFailureMessageForTest($test, "WARNING: expected image is missing!"); - } - } - - if ($resetResults || !-f "$expectedPixelDir/$base-$expectedTag.png") { - mkpath catfile($expectedPixelDir, dirname($base)) if $testDirectory ne $expectedPixelDir; - writeToFile("$expectedPixelDir/$base-$expectedTag.png", $actualPNG); - } - - if ($actualHash ne "" && ($resetResults || !-f "$expectedPixelDir/$base-$expectedTag.checksum")) { - writeToFile("$expectedPixelDir/$base-$expectedTag.checksum", $actualHash); - } - } - - if (dumpToolDidCrash()) { - $result = "crash"; - testCrashedOrTimedOut($test, $base, 1, $actual, $error); - } elsif (!defined $expected) { - if ($verbose) { - print "new " . ($resetResults ? "result" : "test") ."\n"; - $atLineStart = 1; - } - $result = "new"; - - if ($generateNewResults || $resetResults) { - mkpath catfile($expectedDir, dirname($base)) if $testDirectory ne $expectedDir; - writeToFile("$expectedDir/$expectedFileName", $actual); - } - deleteExpectedAndActualResults($base); - recordActualResultsAndDiff($base, $actual); - if (!$resetResults) { - # Always print the file name for new tests, as they will probably need some manual inspection. - # in verbose mode we already printed the test case, so no need to do it again. - unless ($verbose) { - print "\n" unless $atLineStart; - print "$test -> "; - } - my $resultsDir = catdir($expectedDir, dirname($base)); - if ($generateNewResults) { - print "new (results generated in $resultsDir)\n"; - } else { - print "new\n"; - } - $atLineStart = 1; - } - } elsif ($actual eq $expected && $diffResult eq "passed") { - if ($verbose) { - print "succeeded\n"; - $atLineStart = 1; - } - $result = "match"; - deleteExpectedAndActualResults($base); - } else { - $result = "mismatch"; - - my $pixelTestFailed = $pixelTests && $diffPNG && $diffPNG ne ""; - my $testFailed = $actual ne $expected; - - my $message = !$testFailed ? "pixel test failed" : "failed"; - - if (($testFailed || $pixelTestFailed) && $addPlatformExceptions) { - my $testBase = catfile($testDirectory, $base); - my $expectedBase = catfile($expectedDir, $base); - my $testIsMaximallyPlatformSpecific = $testBase =~ m|^\Q$platformTestDirectory\E/|; - my $expectedResultIsMaximallyPlatformSpecific = $expectedBase =~ m|^\Q$platformTestDirectory\E/|; - if (!$testIsMaximallyPlatformSpecific && !$expectedResultIsMaximallyPlatformSpecific) { - mkpath catfile($platformTestDirectory, dirname($base)); - if ($testFailed) { - my $expectedFile = catfile($platformTestDirectory, "$expectedFileName"); - writeToFile("$expectedFile", $actual); - } - if ($pixelTestFailed) { - my $expectedFile = catfile($platformTestDirectory, "$base-$expectedTag.checksum"); - writeToFile("$expectedFile", $actualHash); - - $expectedFile = catfile($platformTestDirectory, "$base-$expectedTag.png"); - writeToFile("$expectedFile", $actualPNG); - } - $message .= " (results generated in $platformTestDirectory)"; - } - } - - printFailureMessageForTest($test, $message); - - my $dir = "$testResultsDirectory/$base"; - $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; - my $testName = $1; - mkpath $dir; - - deleteExpectedAndActualResults($base); - recordActualResultsAndDiff($base, $actual); - - if ($pixelTestFailed) { - $imagesPresent{$base} = 1; - - writeToFile("$testResultsDirectory/$base-$actualTag.png", $actualPNG); - writeToFile("$testResultsDirectory/$base-$diffsTag.png", $diffPNG); - - my $expectedPixelDir = expectedDirectoryForTest($base, 0, "png"); - copy("$expectedPixelDir/$base-$expectedTag.png", "$testResultsDirectory/$base-$expectedTag.png"); - - open DIFFHTML, ">$testResultsDirectory/$base-$diffsTag.html" or die; - print DIFFHTML "<html>\n"; - print DIFFHTML "<head>\n"; - print DIFFHTML "<title>$base Image Compare</title>\n"; - print DIFFHTML "<script language=\"Javascript\" type=\"text/javascript\">\n"; - print DIFFHTML "var currentImage = 0;\n"; - print DIFFHTML "var imageNames = new Array(\"Actual\", \"Expected\");\n"; - print DIFFHTML "var imagePaths = new Array(\"$testName-$actualTag.png\", \"$testName-$expectedTag.png\");\n"; - if (-f "$testDirectory/$base-w3c.png") { - copy("$testDirectory/$base-w3c.png", "$testResultsDirectory/$base-w3c.png"); - print DIFFHTML "imageNames.push(\"W3C\");\n"; - print DIFFHTML "imagePaths.push(\"$testName-w3c.png\");\n"; - } - print DIFFHTML "function animateImage() {\n"; - print DIFFHTML " var image = document.getElementById(\"animatedImage\");\n"; - print DIFFHTML " var imageText = document.getElementById(\"imageText\");\n"; - print DIFFHTML " image.src = imagePaths[currentImage];\n"; - print DIFFHTML " imageText.innerHTML = imageNames[currentImage] + \" Image\";\n"; - print DIFFHTML " currentImage = (currentImage + 1) % imageNames.length;\n"; - print DIFFHTML " setTimeout('animateImage()',2000);\n"; - print DIFFHTML "}\n"; - print DIFFHTML "</script>\n"; - print DIFFHTML "</head>\n"; - print DIFFHTML "<body onLoad=\"animateImage();\">\n"; - print DIFFHTML "<table>\n"; - if ($diffPercentage) { - print DIFFHTML "<tr>\n"; - print DIFFHTML "<td>Difference between images: <a href=\"$testName-$diffsTag.png\">$diffPercentage%</a></td>\n"; - print DIFFHTML "</tr>\n"; - } - print DIFFHTML "<tr>\n"; - print DIFFHTML "<td><a href=\"" . toURL("$testDirectory/$test") . "\">test file</a></td>\n"; - print DIFFHTML "</tr>\n"; - print DIFFHTML "<tr>\n"; - print DIFFHTML "<td id=\"imageText\" style=\"text-weight: bold;\">Actual Image</td>\n"; - print DIFFHTML "</tr>\n"; - print DIFFHTML "<tr>\n"; - print DIFFHTML "<td><img src=\"$testName-$actualTag.png\" id=\"animatedImage\"></td>\n"; - print DIFFHTML "</tr>\n"; - print DIFFHTML "</table>\n"; - print DIFFHTML "</body>\n"; - print DIFFHTML "</html>\n"; - } - } - - if ($error) { - my $dir = "$testResultsDirectory/$base"; - $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; - mkpath $dir; - - writeToFile("$testResultsDirectory/$base-$errorTag.txt", $error); - - $counts{error}++; - push @{$tests{error}}, $test; - } - - countFinishedTest($test, $base, $result, $isText); - - # --reset-results does not check pass vs. fail, so exitAfterNFailures makes no sense with --reset-results. - if ($exitAfterNFailures && !$resetResults) { - my $passCount = $counts{match} || 0; # $counts{match} will be undefined if we've not yet passed a test (e.g. the first test fails). - my $failureCount = $count - $passCount; # "Failure" here includes new tests, timeouts, crashes, etc. - if ($failureCount >= $exitAfterNFailures) { - print "\nExiting early after $failureCount failures. $count tests run."; - closeDumpTool(); - last; - } - } -} -my $totalTestingTime = time - $overallStartTime; -my $waitTime = getWaitTime(); -if ($waitTime > 0.1) { - my $normalizedTestingTime = $totalTestingTime - $waitTime; - printf "\n%0.2fs HTTPD waiting time\n", $waitTime . ""; - printf "%0.2fs normalized testing time", $normalizedTestingTime . ""; -} -printf "\n%0.2fs total testing time\n", $totalTestingTime . ""; - -!$isDumpToolOpen || die "Failed to close $dumpToolName.\n"; - -$isHttpdOpen = !closeHTTPD(); -closeWebSocketServer(); - -# Because multiple instances of this script are running concurrently we cannot -# safely delete this symlink. -# system "rm /tmp/LayoutTests"; - -# FIXME: Do we really want to check the image-comparison tool for leaks every time? -if ($isDiffToolOpen && $shouldCheckLeaks) { - $totalLeaks += countAndPrintLeaks("ImageDiff", $imageDiffToolPID, "$testResultsDirectory/ImageDiff-leaks.txt"); -} - -if ($totalLeaks) { - if ($mergeDepth) { - parseLeaksandPrintUniqueLeaks(); - } else { - print "\nWARNING: $totalLeaks total leaks found!\n"; - print "See above for individual leaks results.\n" if ($leaksOutputFileNumber > 2); - } -} - -close IN; -close OUT; -close ERROR; - -if ($report10Slowest) { - print "\n\nThe 10 slowest tests:\n\n"; - my $count = 0; - for my $test (sort slowestcmp keys %durations) { - printf "%0.2f secs: %s\n", $durations{$test}, $test; - last if ++$count == 10; - } -} - -print "\n"; - -if ($skippedOnly && $counts{"match"}) { - print "The following tests are in the Skipped file (" . File::Spec->abs2rel("$platformTestDirectory/Skipped", $testDirectory) . "), but succeeded:\n"; - foreach my $test (@{$tests{"match"}}) { - print " $test\n"; - } -} - -if ($resetResults || ($counts{match} && $counts{match} == $count)) { - print "all $count test cases succeeded\n"; - unlink $testResults; - exit; -} - -printResults(); - -mkpath $testResultsDirectory; - -open HTML, ">", $testResults or die "Failed to open $testResults. $!"; -print HTML "<html>\n"; -print HTML "<head>\n"; -print HTML "<title>Layout Test Results</title>\n"; -print HTML "</head>\n"; -print HTML "<body>\n"; - -if ($ignoreMetrics) { - print HTML "<h4>Tested with metrics ignored.</h4>"; -} - -print HTML htmlForResultsSection(@{$tests{mismatch}}, "Tests where results did not match expected results", \&linksForMismatchTest); -print HTML htmlForResultsSection(@{$tests{timedout}}, "Tests that timed out", \&linksForErrorTest); -print HTML htmlForResultsSection(@{$tests{crash}}, "Tests that caused the DumpRenderTree tool to crash", \&linksForErrorTest); -print HTML htmlForResultsSection(@{$tests{error}}, "Tests that had stderr output", \&linksForErrorTest); -print HTML htmlForResultsSection(@{$tests{new}}, "Tests that had no expected results (probably new)", \&linksForNewTest); - -print HTML "</body>\n"; -print HTML "</html>\n"; -close HTML; - -my @configurationArgs = argumentsForConfiguration(); - -if (isGtk()) { - system "WebKitTools/Scripts/run-launcher", @configurationArgs, "file://".$testResults if $launchSafari; -} elsif (isQt()) { - unshift @configurationArgs, qw(-graphicssystem raster -style windows); - if (isCygwin()) { - $testResults = "/" . toWindowsPath($testResults); - $testResults =~ s/\\/\//g; - } - system "WebKitTools/Scripts/run-launcher", @configurationArgs, "file://".$testResults if $launchSafari; -} elsif (isCygwin()) { - system "cygstart", $testResults if $launchSafari; -} else { - system "WebKitTools/Scripts/run-safari", @configurationArgs, "-NSOpen", $testResults if $launchSafari; -} - -closeCygpaths() if isCygwin(); - -exit 1; - -sub countAndPrintLeaks($$$) +sub runningOnBuildBot() { - my ($dumpToolName, $dumpToolPID, $leaksFilePath) = @_; - - print "\n" unless $atLineStart; - $atLineStart = 1; - - # We are excluding the following reported leaks so they don't get in our way when looking for WebKit leaks: - # This allows us ignore known leaks and only be alerted when new leaks occur. Some leaks are in the old - # versions of the system frameworks that are being used by the leaks bots. Even though a leak has been - # fixed, it will be listed here until the bot has been updated with the newer frameworks. - - my @typesToExclude = ( - ); - - my @callStacksToExclude = ( - "Flash_EnforceLocalSecurity" # leaks in Flash plug-in code, rdar://problem/4449747 - ); - - if (isTiger()) { - # Leak list for the version of Tiger used on the build bot. - push @callStacksToExclude, ( - "CFRunLoopRunSpecific \\| malloc_zone_malloc", "CFRunLoopRunSpecific \\| CFAllocatorAllocate ", # leak in CFRunLoopRunSpecific, rdar://problem/4670839 - "CGImageSourceGetPropertiesAtIndex", # leak in ImageIO, rdar://problem/4628809 - "FOGetCoveredUnicodeChars", # leak in ATS, rdar://problem/3943604 - "GetLineDirectionPreference", "InitUnicodeUtilities", # leaks tool falsely reporting leak in CFNotificationCenterAddObserver, rdar://problem/4964790 - "ICCFPrefWrapper::GetPrefDictionary", # leaks in Internet Config. code, rdar://problem/4449794 - "NSHTTPURLProtocol setResponseHeader:", # leak in multipart/mixed-replace handling in Foundation, no Radar, but fixed in Leopard - "NSURLCache cachedResponseForRequest", # leak in CFURL cache, rdar://problem/4768430 - "PCFragPrepareClosureFromFile", # leak in Code Fragment Manager, rdar://problem/3426998 - "WebCore::Selection::toRange", # bug in 'leaks', rdar://problem/4967949 - "WebCore::SubresourceLoader::create", # bug in 'leaks', rdar://problem/4985806 - "_CFPreferencesDomainDeepCopyDictionary", # leak in CFPreferences, rdar://problem/4220786 - "_objc_msgForward", # leak in NSSpellChecker, rdar://problem/4965278 - "gldGetString", # leak in OpenGL, rdar://problem/5013699 - "_setDefaultUserInfoFromURL", # leak in NSHTTPAuthenticator, rdar://problem/5546453 - "SSLHandshake", # leak in SSL, rdar://problem/5546440 - "SecCertificateCreateFromData", # leak in SSL code, rdar://problem/4464397 - ); - push @typesToExclude, ( - "THRD", # bug in 'leaks', rdar://problem/3387783 - "DRHT", # ditto (endian little hate i) - ); - } - - if (isLeopard()) { - # Leak list for the version of Leopard used on the build bot. - push @callStacksToExclude, ( - "CFHTTPMessageAppendBytes", # leak in CFNetwork, rdar://problem/5435912 - "sendDidReceiveDataCallback", # leak in CFNetwork, rdar://problem/5441619 - "_CFHTTPReadStreamReadMark", # leak in CFNetwork, rdar://problem/5441468 - "httpProtocolStart", # leak in CFNetwork, rdar://problem/5468837 - "_CFURLConnectionSendCallbacks", # leak in CFNetwork, rdar://problem/5441600 - "DispatchQTMsg", # leak in QuickTime, PPC only, rdar://problem/5667132 - "QTMovieContentView createVisualContext", # leak in QuickTime, PPC only, rdar://problem/5667132 - "_CopyArchitecturesForJVMVersion", # leak in Java, rdar://problem/5910823 - ); - } - - if (isSnowLeopard()) { - push @callStacksToExclude, ( - "readMakerNoteProps", # <rdar://problem/7156432> leak in ImageIO - "QTKitMovieControllerView completeUISetup", # <rdar://problem/7155156> leak in QTKit - ); - } - - my $leaksTool = sourceDir() . "/WebKitTools/Scripts/run-leaks"; - my $excludeString = "--exclude-callstack '" . (join "' --exclude-callstack '", @callStacksToExclude) . "'"; - $excludeString .= " --exclude-type '" . (join "' --exclude-type '", @typesToExclude) . "'" if @typesToExclude; - - print " ? checking for leaks in $dumpToolName\n"; - my $leaksOutput = `$leaksTool $excludeString $dumpToolPID`; - my ($count, $bytes) = $leaksOutput =~ /Process $dumpToolPID: (\d+) leaks? for (\d+) total/; - my ($excluded) = $leaksOutput =~ /(\d+) leaks? excluded/; - - my $adjustedCount = $count; - $adjustedCount -= $excluded if $excluded; - - if (!$adjustedCount) { - print " - no leaks found\n"; - unlink $leaksFilePath; - return 0; - } else { - my $dir = $leaksFilePath; - $dir =~ s|/[^/]+$|| or die; - mkpath $dir; - - if ($excluded) { - print " + $adjustedCount leaks ($bytes bytes including $excluded excluded leaks) were found, details in $leaksFilePath\n"; - } else { - print " + $count leaks ($bytes bytes) were found, details in $leaksFilePath\n"; - } - - writeToFile($leaksFilePath, $leaksOutput); - - push @leaksFilenames, $leaksFilePath; - } - - return $adjustedCount; -} - -sub writeToFile($$) -{ - my ($filePath, $contents) = @_; - open NEWFILE, ">", "$filePath" or die "Could not create $filePath. $!\n"; - print NEWFILE $contents; - close NEWFILE; -} - -# Break up a path into the directory (with slash) and base name. -sub splitpath($) -{ - my ($path) = @_; - - my $pathSeparator = "/"; - my $dirname = dirname($path) . $pathSeparator; - $dirname = "" if $dirname eq "." . $pathSeparator; - - return ($dirname, basename($path)); -} - -# Sort first by directory, then by file, so all paths in one directory are grouped -# rather than being interspersed with items from subdirectories. -# Use numericcmp to sort directory and filenames to make order logical. -sub pathcmp($$) -{ - my ($patha, $pathb) = @_; - - my ($dira, $namea) = splitpath($patha); - my ($dirb, $nameb) = splitpath($pathb); - - return numericcmp($dira, $dirb) if $dira ne $dirb; - return numericcmp($namea, $nameb); -} - -# Sort numeric parts of strings as numbers, other parts as strings. -# Makes 1.33 come after 1.3, which is cool. -sub numericcmp($$) -{ - my ($aa, $bb) = @_; - - my @a = split /(\d+)/, $aa; - my @b = split /(\d+)/, $bb; - - # Compare one chunk at a time. - # Each chunk is either all numeric digits, or all not numeric digits. - while (@a && @b) { - my $a = shift @a; - my $b = shift @b; - - # Use numeric comparison if chunks are non-equal numbers. - return $a <=> $b if $a =~ /^\d/ && $b =~ /^\d/ && $a != $b; - - # Use string comparison if chunks are any other kind of non-equal string. - return $a cmp $b if $a ne $b; - } - - # One of the two is now empty; compare lengths for result in this case. - return @a <=> @b; -} - -# Sort slowest tests first. -sub slowestcmp($$) -{ - my ($testa, $testb) = @_; - - my $dura = $durations{$testa}; - my $durb = $durations{$testb}; - return $durb <=> $dura if $dura != $durb; - return pathcmp($testa, $testb); -} - -sub launchWithEnv(\@\%) -{ - my ($args, $env) = @_; - - # Dump the current environment as perl code and then put it in quotes so it is one parameter. - my $environmentDumper = Data::Dumper->new([\%{$env}], [qw(*ENV)]); - $environmentDumper->Indent(0); - $environmentDumper->Purity(1); - my $allEnvVars = $environmentDumper->Dump(); - unshift @{$args}, "\"$allEnvVars\""; - - my $execScript = File::Spec->catfile(sourceDir(), qw(WebKitTools Scripts execAppWithEnv)); - unshift @{$args}, $execScript; - return @{$args}; -} - -sub resolveAndMakeTestResultsDirectory() -{ - my $absTestResultsDirectory = File::Spec->rel2abs(glob $testResultsDirectory); - mkpath $absTestResultsDirectory; - return $absTestResultsDirectory; -} - -sub openDiffTool() -{ - return if $isDiffToolOpen; - return if !$pixelTests; - - my %CLEAN_ENV; - $CLEAN_ENV{MallocStackLogging} = 1 if $shouldCheckLeaks; - $imageDiffToolPID = open2(\*DIFFIN, \*DIFFOUT, $imageDiffTool, launchWithEnv(@diffToolArgs, %CLEAN_ENV)) or die "unable to open $imageDiffTool\n"; - $isDiffToolOpen = 1; + # This is a hack to detect if we're running on the buildbot so we can + # pass --verbose to new-run-webkit-tests. This will be removed when we + # update the buildbot config to call new-run-webkit-tests explicitly. + my %isBuildBotUser = ("apple" => 1, "buildbot" => 1); + return $isBuildBotUser{$ENV{"USER"}}; } -sub openDumpTool() +sub useNewRunWebKitTests() { - return if $isDumpToolOpen; - - my %CLEAN_ENV; - - # Generic environment variables - if (defined $ENV{'WEBKIT_TESTFONTS'}) { - $CLEAN_ENV{WEBKIT_TESTFONTS} = $ENV{'WEBKIT_TESTFONTS'}; - } - - $CLEAN_ENV{XML_CATALOG_FILES} = ""; # work around missing /etc/catalog <rdar://problem/4292995> - - # Platform spesifics - if (isLinux()) { - if (defined $ENV{'DISPLAY'}) { - $CLEAN_ENV{DISPLAY} = $ENV{'DISPLAY'}; - } else { - $CLEAN_ENV{DISPLAY} = ":1"; - } - if (defined $ENV{'XAUTHORITY'}) { - $CLEAN_ENV{XAUTHORITY} = $ENV{'XAUTHORITY'}; - } - - $CLEAN_ENV{HOME} = $ENV{'HOME'}; - - if (defined $ENV{'LD_LIBRARY_PATH'}) { - $CLEAN_ENV{LD_LIBRARY_PATH} = $ENV{'LD_LIBRARY_PATH'}; - } - if (defined $ENV{'DBUS_SESSION_BUS_ADDRESS'}) { - $CLEAN_ENV{DBUS_SESSION_BUS_ADDRESS} = $ENV{'DBUS_SESSION_BUS_ADDRESS'}; - } - } elsif (isDarwin()) { - if (defined $ENV{'DYLD_LIBRARY_PATH'}) { - $CLEAN_ENV{DYLD_LIBRARY_PATH} = $ENV{'DYLD_LIBRARY_PATH'}; - } - - $CLEAN_ENV{DYLD_FRAMEWORK_PATH} = $productDir; - $CLEAN_ENV{DYLD_INSERT_LIBRARIES} = "/usr/lib/libgmalloc.dylib" if $guardMalloc; - } elsif (isCygwin()) { - $CLEAN_ENV{HOMEDRIVE} = $ENV{'HOMEDRIVE'}; - $CLEAN_ENV{HOMEPATH} = $ENV{'HOMEPATH'}; - - setPathForRunningWebKitApp(\%CLEAN_ENV); - } - - # Port spesifics - if (isQt()) { - $CLEAN_ENV{QTWEBKIT_PLUGIN_PATH} = productDir() . "/lib/plugins"; - } - - my @args = ($dumpTool, @toolArgs); - if (isAppleMacWebKit() and !isTiger()) { - unshift @args, "arch", "-" . architecture(); - } - - if ($useValgrind) { - unshift @args, "valgrind", "--suppressions=$platformBaseDirectory/qt/SuppressedValgrindErrors"; - } - - $CLEAN_ENV{MallocStackLogging} = 1 if $shouldCheckLeaks; - - $dumpToolPID = open3(\*OUT, \*IN, \*ERROR, launchWithEnv(@args, %CLEAN_ENV)) or die "Failed to start tool: $dumpTool\n"; - $isDumpToolOpen = 1; - $dumpToolCrashed = 0; -} - -sub closeDumpTool() -{ - return if !$isDumpToolOpen; - - close IN; - close OUT; - waitpid $dumpToolPID, 0; - - # check for WebCore counter leaks. - if ($shouldCheckLeaks) { - while (<ERROR>) { - print; - } - } - close ERROR; - $isDumpToolOpen = 0; -} - -sub dumpToolDidCrash() -{ - return 1 if $dumpToolCrashed; - return 0 unless $isDumpToolOpen; - my $pid = waitpid(-1, WNOHANG); - return 1 if ($pid == $dumpToolPID); - - # On Mac OS X, crashing may be significantly delayed by crash reporter. - return 0 unless isAppleMacWebKit(); - - return DumpRenderTreeSupport::processIsCrashing($dumpToolPID); -} - -sub configureAndOpenHTTPDIfNeeded() -{ - return if $isHttpdOpen; - my $absTestResultsDirectory = resolveAndMakeTestResultsDirectory(); - my $listen = "127.0.0.1:$httpdPort"; - my @args = ( - "-c", "CustomLog \"$absTestResultsDirectory/access_log.txt\" common", - "-c", "ErrorLog \"$absTestResultsDirectory/error_log.txt\"", - "-C", "Listen $listen" - ); - - my @defaultArgs = getDefaultConfigForTestDirectory($testDirectory); - @args = (@defaultArgs, @args); - - waitForHTTPDLock() if $shouldWaitForHTTPD; - $isHttpdOpen = openHTTPD(@args); -} - -sub openWebSocketServerIfNeeded() -{ - return 1 if $isWebSocketServerOpen; - return 0 if $failedToStartWebSocketServer; - - my $webSocketServerPath = "/usr/bin/python"; - my $webSocketPythonPath = "WebKitTools/pywebsocket"; - my $webSocketHandlerDir = "$testDirectory"; - my $webSocketHandlerScanDir = "$testDirectory/websocket/tests"; - my $webSocketHandlerMapFile = "$webSocketHandlerScanDir/handler_map.txt"; - my $sslCertificate = "$testDirectory/http/conf/webkit-httpd.pem"; - my $absTestResultsDirectory = resolveAndMakeTestResultsDirectory(); - my $logFile = "$absTestResultsDirectory/pywebsocket_log.txt"; - - my @args = ( - "WebKitTools/pywebsocket/mod_pywebsocket/standalone.py", - "-p", "$webSocketPort", - "-d", "$webSocketHandlerDir", - "-s", "$webSocketHandlerScanDir", - "-m", "$webSocketHandlerMapFile", - "-x", "/websocket/tests/cookies", - "-l", "$logFile", - "--strict", - ); - # wss is disabled until all platforms support pyOpenSSL. - # my @argsSecure = ( - # "WebKitTools/pywebsocket/mod_pywebsocket/standalone.py", - # "-p", "$webSocketSecurePort", - # "-d", "$webSocketHandlerDir", - # "-t", - # "-k", "$sslCertificate", - # "-c", "$sslCertificate", - # ); - - $ENV{"PYTHONPATH"} = $webSocketPythonPath; - $webSocketServerPID = open3(\*WEBSOCKETSERVER_IN, \*WEBSOCKETSERVER_OUT, \*WEBSOCKETSERVER_ERR, $webSocketServerPath, @args); - # wss is disabled until all platforms support pyOpenSSL. - # $webSocketSecureServerPID = open3(\*WEBSOCKETSECURESERVER_IN, \*WEBSOCKETSECURESERVER_OUT, \*WEBSOCKETSECURESERVER_ERR, $webSocketServerPath, @argsSecure); - # my @listen = ("http://127.0.0.1:$webSocketPort", "https://127.0.0.1:$webSocketSecurePort"); - my @listen = ("http://127.0.0.1:$webSocketPort"); - for (my $i = 0; $i < @listen; $i++) { - my $retryCount = 10; - while (system("/usr/bin/curl -k -q --silent --stderr - --output /dev/null $listen[$i]") && $retryCount) { - sleep 1; - --$retryCount; - } - unless ($retryCount) { - print STDERR "Timed out waiting for WebSocketServer to start.\n"; - $failedToStartWebSocketServer = 1; - return 0; - } - } - - $isWebSocketServerOpen = 1; - return 1; -} - -sub closeWebSocketServer() -{ - return if !$isWebSocketServerOpen; - - close WEBSOCKETSERVER_IN; - close WEBSOCKETSERVER_OUT; - close WEBSOCKETSERVER_ERR; - kill 15, $webSocketServerPID; - - # wss is disabled until all platforms support pyOpenSSL. - # close WEBSOCKETSECURESERVER_IN; - # close WEBSOCKETSECURESERVER_OUT; - # close WEBSOCKETSECURESERVER_ERR; - # kill 15, $webSocketSecureServerPID; - - $isWebSocketServerOpen = 0; -} - -sub fileNameWithNumber($$) -{ - my ($base, $number) = @_; - return "$base$number" if ($number > 1); - return $base; -} - -sub processIgnoreTests($$) -{ - my @ignoreList = split(/\s*,\s*/, shift); - my $listName = shift; - - my $disabledSuffix = "-disabled"; - - my $addIgnoredDirectories = sub { - return () if exists $ignoredLocalDirectories{basename($File::Find::dir)}; - $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)} = 1; - return @_; - }; - foreach my $item (@ignoreList) { - my $path = catfile($testDirectory, $item); - if (-d $path) { - $ignoredDirectories{$item} = 1; - find({ preprocess => $addIgnoredDirectories, wanted => sub {} }, $path); - } - elsif (-f $path) { - $ignoredFiles{$item} = 1; - } elsif (-f $path . $disabledSuffix) { - # The test is disabled, so do nothing. - } else { - print "$listName list contained '$item', but no file of that name could be found\n"; - } - } -} - -sub stripExtension($) -{ - my ($test) = @_; - - $test =~ s/\.[a-zA-Z]+$//; - return $test; -} - -sub isTextOnlyTest($) -{ - my ($actual) = @_; - my $isText; - if ($actual =~ /^layer at/ms) { - $isText = 0; - } else { - $isText = 1; - } - return $isText; -} - -sub expectedDirectoryForTest($;$;$) -{ - my ($base, $isText, $expectedExtension) = @_; - - my @directories = @platformResultHierarchy; - push @directories, map { catdir($platformBaseDirectory, $_) } qw(mac-snowleopard mac) if isCygwin(); - push @directories, $expectedDirectory; - - # If we already have expected results, just return their location. - foreach my $directory (@directories) { - return $directory if (-f "$directory/$base-$expectedTag.$expectedExtension"); - } - - # For cross-platform tests, text-only results should go in the cross-platform directory, - # while render tree dumps should go in the least-specific platform directory. - return $isText ? $expectedDirectory : $platformResultHierarchy[$#platformResultHierarchy]; -} - -sub countFinishedTest($$$$) -{ - my ($test, $base, $result, $isText) = @_; - - if (($count + 1) % $testsPerDumpTool == 0 || $count == $#tests) { - if ($shouldCheckLeaks) { - my $fileName; - if ($testsPerDumpTool == 1) { - $fileName = "$testResultsDirectory/$base-leaks.txt"; - } else { - $fileName = "$testResultsDirectory/" . fileNameWithNumber($dumpToolName, $leaksOutputFileNumber) . "-leaks.txt"; - } - my $leakCount = countAndPrintLeaks($dumpToolName, $dumpToolPID, $fileName); - $totalLeaks += $leakCount; - $leaksOutputFileNumber++ if ($leakCount); - } - - closeDumpTool(); - } - - $count++; - $counts{$result}++; - push @{$tests{$result}}, $test; -} - -sub testCrashedOrTimedOut($$$$$) -{ - my ($test, $base, $didCrash, $actual, $error) = @_; - - printFailureMessageForTest($test, $didCrash ? "crashed" : "timed out"); - - sampleDumpTool() unless $didCrash; - - my $dir = "$testResultsDirectory/$base"; - $dir =~ s|/([^/]+)$|| or die "Failed to find test name from base\n"; - mkpath $dir; - - deleteExpectedAndActualResults($base); - - if (defined($error) && length($error)) { - writeToFile("$testResultsDirectory/$base-$errorTag.txt", $error); - } - - recordActualResultsAndDiff($base, $actual); - - kill 9, $dumpToolPID unless $didCrash; - - closeDumpTool(); -} - -sub printFailureMessageForTest($$) -{ - my ($test, $description) = @_; - - unless ($verbose) { - print "\n" unless $atLineStart; - print "$test -> "; - } - print "$description\n"; - $atLineStart = 1; -} - -my %cygpaths = (); - -sub openCygpathIfNeeded($) -{ - my ($options) = @_; - - return unless isCygwin(); - return $cygpaths{$options} if $cygpaths{$options} && $cygpaths{$options}->{"open"}; - - local (*CYGPATHIN, *CYGPATHOUT); - my $pid = open2(\*CYGPATHIN, \*CYGPATHOUT, "cygpath -f - $options"); - my $cygpath = { - "pid" => $pid, - "in" => *CYGPATHIN, - "out" => *CYGPATHOUT, - "open" => 1 - }; - - $cygpaths{$options} = $cygpath; - - return $cygpath; -} - -sub closeCygpaths() -{ - return unless isCygwin(); - - foreach my $cygpath (values(%cygpaths)) { - close $cygpath->{"in"}; - close $cygpath->{"out"}; - waitpid($cygpath->{"pid"}, 0); - $cygpath->{"open"} = 0; - - } -} - -sub convertPathUsingCygpath($$) -{ - my ($path, $options) = @_; - - my $cygpath = openCygpathIfNeeded($options); - local *inFH = $cygpath->{"in"}; - local *outFH = $cygpath->{"out"}; - print outFH $path . "\n"; - my $convertedPath = <inFH>; - chomp($convertedPath) if defined $convertedPath; - return $convertedPath; -} - -sub toWindowsPath($) -{ - my ($path) = @_; - return unless isCygwin(); - - return convertPathUsingCygpath($path, "-w"); -} - -sub toURL($) -{ - my ($path) = @_; - - if ($useRemoteLinksToTests) { - my $relativePath = File::Spec->abs2rel($path, $testDirectory); - - # If the file is below the test directory then convert it into a link to the file in SVN - if ($relativePath !~ /^\.\.\//) { - my $revision = svnRevisionForDirectory($testDirectory); - my $svnPath = pathRelativeToSVNRepositoryRootForPath($path); - return "http://trac.webkit.org/export/$revision/$svnPath"; - } - } - - return $path unless isCygwin(); - - return "file:///" . convertPathUsingCygpath($path, "-m"); -} - -sub validateSkippedArg($$;$) -{ - my ($option, $value, $value2) = @_; - my %validSkippedValues = map { $_ => 1 } qw(default ignore only); - $value = lc($value); - die "Invalid argument '" . $value . "' for option $option" unless $validSkippedValues{$value}; - $treatSkipped = $value; -} - -sub htmlForResultsSection(\@$&) -{ - my ($tests, $description, $linkGetter) = @_; - - my @html = (); - return join("\n", @html) unless @{$tests}; - - push @html, "<p>$description:</p>"; - push @html, "<table>"; - foreach my $test (@{$tests}) { - push @html, "<tr>"; - push @html, "<td><a href=\"" . toURL("$testDirectory/$test") . "\">$test</a></td>"; - foreach my $link (@{&{$linkGetter}($test)}) { - push @html, "<td><a href=\"$link->{href}\">$link->{text}</a></td>"; - } - push @html, "</tr>"; - } - push @html, "</table>"; - - return join("\n", @html); -} - -sub linksForExpectedAndActualResults($) -{ - my ($base) = @_; - - my @links = (); - - return \@links unless -s "$testResultsDirectory/$base-$diffsTag.txt"; - - my $expectedResultPath = $expectedResultPaths{$base}; - my ($expectedResultFileName, $expectedResultsDirectory, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); - - push @links, { href => "$base-$expectedTag$expectedResultExtension", text => "expected" }; - push @links, { href => "$base-$actualTag$expectedResultExtension", text => "actual" }; - push @links, { href => "$base-$diffsTag.txt", text => "diff" }; - push @links, { href => "$base-$prettyDiffTag.html", text => "pretty diff" }; - - return \@links; -} - -sub linksForMismatchTest -{ - my ($test) = @_; - - my @links = (); - - my $base = stripExtension($test); - - push @links, @{linksForExpectedAndActualResults($base)}; - return \@links unless $pixelTests && $imagesPresent{$base}; - - push @links, { href => "$base-$expectedTag.png", text => "expected image" }; - push @links, { href => "$base-$diffsTag.html", text => "image diffs" }; - push @links, { href => "$base-$diffsTag.png", text => "$imageDifferences{$base}%" }; - - return \@links; -} - -sub linksForErrorTest -{ - my ($test) = @_; - - my @links = (); - - my $base = stripExtension($test); - - push @links, @{linksForExpectedAndActualResults($base)}; - push @links, { href => "$base-$errorTag.txt", text => "stderr" }; - - return \@links; -} - -sub linksForNewTest -{ - my ($test) = @_; - - my @links = (); - - my $base = stripExtension($test); - - my $expectedResultPath = $expectedResultPaths{$base}; - my ($expectedResultFileName, $expectedResultsDirectory, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); - - push @links, { href => "$base-$actualTag$expectedResultExtension", text => "result" }; - if ($pixelTests && $imagesPresent{$base}) { - push @links, { href => "$base-$expectedTag.png", text => "image" }; - } - - return \@links; -} - -sub deleteExpectedAndActualResults($) -{ - my ($base) = @_; - - unlink "$testResultsDirectory/$base-$actualTag.txt"; - unlink "$testResultsDirectory/$base-$diffsTag.txt"; - unlink "$testResultsDirectory/$base-$errorTag.txt"; -} - -sub recordActualResultsAndDiff($$) -{ - my ($base, $actualResults) = @_; - - return unless defined($actualResults) && length($actualResults); - - my $expectedResultPath = $expectedResultPaths{$base}; - my ($expectedResultFileNameMinusExtension, $expectedResultDirectoryPath, $expectedResultExtension) = fileparse($expectedResultPath, qr{\.[^.]+$}); - my $actualResultsPath = "$testResultsDirectory/$base-$actualTag$expectedResultExtension"; - my $copiedExpectedResultsPath = "$testResultsDirectory/$base-$expectedTag$expectedResultExtension"; - - mkpath(dirname($actualResultsPath)); - writeToFile("$actualResultsPath", $actualResults); - - if (-f $expectedResultPath) { - copy("$expectedResultPath", "$copiedExpectedResultsPath"); - } else { - open EMPTY, ">$copiedExpectedResultsPath"; - close EMPTY; - } - - my $diffOuputBasePath = "$testResultsDirectory/$base"; - my $diffOutputPath = "$diffOuputBasePath-$diffsTag.txt"; - system "diff -u \"$copiedExpectedResultsPath\" \"$actualResultsPath\" > \"$diffOutputPath\""; - - my $prettyDiffOutputPath = "$diffOuputBasePath-$prettyDiffTag.html"; - my $prettyPatchPath = "BugsSite/PrettyPatch/"; - my $prettifyPath = "$prettyPatchPath/prettify.rb"; - system "ruby -I \"$prettyPatchPath\" \"$prettifyPath\" \"$diffOutputPath\" > \"$prettyDiffOutputPath\""; -} - -sub buildPlatformResultHierarchy() -{ - mkpath($platformTestDirectory) if ($platform eq "undefined" && !-d "$platformTestDirectory"); - - my @platforms; - if ($platform =~ /^mac-/) { - my $i; - for ($i = 0; $i < @macPlatforms; $i++) { - last if $macPlatforms[$i] eq $platform; - } - for (; $i < @macPlatforms; $i++) { - push @platforms, $macPlatforms[$i]; - } - } elsif ($platform =~ /^qt-/) { - push @platforms, $platform; - push @platforms, "qt"; - } else { - @platforms = $platform; - } - - my @hierarchy; - for (my $i = 0; $i < @platforms; $i++) { - my $scoped = catdir($platformBaseDirectory, $platforms[$i]); - push(@hierarchy, $scoped) if (-d $scoped); - } - - return @hierarchy; -} - -sub buildPlatformTestHierarchy(@) -{ - my (@platformHierarchy) = @_; - return @platformHierarchy if (@platformHierarchy < 2); - - return ($platformHierarchy[0], $platformHierarchy[$#platformHierarchy]); -} - -sub epiloguesAndPrologues($$) -{ - my ($lastDirectory, $directory) = @_; - my @lastComponents = split('/', $lastDirectory); - my @components = split('/', $directory); - - while (@lastComponents) { - if (!defined($components[0]) || $lastComponents[0] ne $components[0]) { - last; - } - shift @components; - shift @lastComponents; - } - - my @result; - my $leaving = $lastDirectory; - foreach (@lastComponents) { - my $epilogue = $leaving . "/resources/run-webkit-tests-epilogue.html"; - foreach (@platformResultHierarchy) { - push @result, catdir($_, $epilogue) if (stat(catdir($_, $epilogue))); - } - push @result, catdir($testDirectory, $epilogue) if (stat(catdir($testDirectory, $epilogue))); - $leaving =~ s|(^\|/)[^/]+$||; - } - - my $entering = $leaving; - foreach (@components) { - $entering .= '/' . $_; - my $prologue = $entering . "/resources/run-webkit-tests-prologue.html"; - push @result, catdir($testDirectory, $prologue) if (stat(catdir($testDirectory, $prologue))); - foreach (reverse @platformResultHierarchy) { - push @result, catdir($_, $prologue) if (stat(catdir($_, $prologue))); - } - } - return @result; -} - -sub parseLeaksandPrintUniqueLeaks() -{ - return unless @leaksFilenames; - - my $mergedFilenames = join " ", @leaksFilenames; - my $parseMallocHistoryTool = sourceDir() . "/WebKitTools/Scripts/parse-malloc-history"; - - open MERGED_LEAKS, "cat $mergedFilenames | $parseMallocHistoryTool --merge-depth $mergeDepth - |" ; - my @leakLines = <MERGED_LEAKS>; - close MERGED_LEAKS; - - my $uniqueLeakCount = 0; - my $totalBytes; - foreach my $line (@leakLines) { - ++$uniqueLeakCount if ($line =~ /^(\d*)\scalls/); - $totalBytes = $1 if $line =~ /^total\:\s(.*)\s\(/; - } - - print "\nWARNING: $totalLeaks total leaks found for a total of $totalBytes!\n"; - print "WARNING: $uniqueLeakCount unique leaks found!\n"; - print "See above for individual leaks results.\n" if ($leaksOutputFileNumber > 2); - -} - -sub extensionForMimeType($) -{ - my ($mimeType) = @_; - - if ($mimeType eq "application/x-webarchive") { - return "webarchive"; - } elsif ($mimeType eq "application/pdf") { - return "pdf"; - } - return "txt"; -} - -# Read up to the first #EOF (the content block of the test), or until detecting crashes or timeouts. -sub readFromDumpToolWithTimer(**) -{ - my ($fhIn, $fhError) = @_; - - setFileHandleNonBlocking($fhIn, 1); - setFileHandleNonBlocking($fhError, 1); - - my $maximumSecondsWithoutOutput = $timeoutSeconds; - $maximumSecondsWithoutOutput *= 10 if $guardMalloc; - my $microsecondsToWaitBeforeReadingAgain = 1000; - - my $timeOfLastSuccessfulRead = time; - - my @output = (); - my @error = (); - my $status = "success"; - my $mimeType = "text/plain"; - # We don't have a very good way to know when the "headers" stop - # and the content starts, so we use this as a hack: - my $haveSeenContentType = 0; - my $haveSeenEofIn = 0; - my $haveSeenEofError = 0; - - while (1) { - if (time - $timeOfLastSuccessfulRead > $maximumSecondsWithoutOutput) { - $status = dumpToolDidCrash() ? "crashed" : "timedOut"; - last; - } - - # Once we've seen the EOF, we must not read anymore. - my $lineIn = readline($fhIn) unless $haveSeenEofIn; - my $lineError = readline($fhError) unless $haveSeenEofError; - if (!defined($lineIn) && !defined($lineError)) { - last if ($haveSeenEofIn && $haveSeenEofError); - - if ($! != EAGAIN) { - $status = "crashed"; - last; - } - - # No data ready - usleep($microsecondsToWaitBeforeReadingAgain); - next; - } - - $timeOfLastSuccessfulRead = time; - - if (defined($lineIn)) { - if (!$haveSeenContentType && $lineIn =~ /^Content-Type: (\S+)$/) { - $mimeType = $1; - $haveSeenContentType = 1; - } elsif ($lineIn =~ /#EOF/) { - $haveSeenEofIn = 1; - } else { - push @output, $lineIn; - } - } - if (defined($lineError)) { - if ($lineError =~ /#EOF/) { - $haveSeenEofError = 1; - } else { - push @error, $lineError; - } - } - } - - setFileHandleNonBlocking($fhIn, 0); - setFileHandleNonBlocking($fhError, 0); - return { - output => join("", @output), - error => join("", @error), - status => $status, - mimeType => $mimeType, - extension => extensionForMimeType($mimeType) - }; -} - -sub setFileHandleNonBlocking(*$) -{ - my ($fh, $nonBlocking) = @_; - - my $flags = fcntl($fh, F_GETFL, 0) or die "Couldn't get filehandle flags"; - - if ($nonBlocking) { - $flags |= O_NONBLOCK; - } else { - $flags &= ~O_NONBLOCK; - } - - fcntl($fh, F_SETFL, $flags) or die "Couldn't set filehandle flags"; - - return 1; -} - -sub sampleDumpTool() -{ - return unless isAppleMacWebKit(); - return unless $runSample; - - my $outputDirectory = "$ENV{HOME}/Library/Logs/DumpRenderTree"; - -d $outputDirectory or mkdir $outputDirectory; - - my $outputFile = "$outputDirectory/HangReport.txt"; - system "/usr/bin/sample", $dumpToolPID, qw(10 10 -file), $outputFile; -} - -sub stripMetrics($$) -{ - my ($actual, $expected) = @_; - - foreach my $result ($actual, $expected) { - $result =~ s/at \(-?[0-9]+,-?[0-9]+\) *//g; - $result =~ s/size -?[0-9]+x-?[0-9]+ *//g; - $result =~ s/text run width -?[0-9]+: //g; - $result =~ s/text run width -?[0-9]+ [a-zA-Z ]+: //g; - $result =~ s/RenderButton {BUTTON} .*/RenderButton {BUTTON}/g; - $result =~ s/RenderImage {INPUT} .*/RenderImage {INPUT}/g; - $result =~ s/RenderBlock {INPUT} .*/RenderBlock {INPUT}/g; - $result =~ s/RenderTextControl {INPUT} .*/RenderTextControl {INPUT}/g; - $result =~ s/\([0-9]+px/px/g; - $result =~ s/ *" *\n +" */ /g; - $result =~ s/" +$/"/g; - - $result =~ s/- /-/g; - $result =~ s/\n( *)"\s+/\n$1"/g; - $result =~ s/\s+"\n/"\n/g; - $result =~ s/scrollWidth [0-9]+/scrollWidth/g; - $result =~ s/scrollHeight [0-9]+/scrollHeight/g; - } - - return ($actual, $expected); -} - -sub fileShouldBeIgnored -{ - my ($filePath) = @_; - foreach my $ignoredDir (keys %ignoredDirectories) { - if ($filePath =~ m/^$ignoredDir/) { - return 1; - } - } + # Change this check to control which platforms use + # new-run-webkit-tests by default. + # Example: return runningOnBuildBot() && isLeopard(); + # would enable new-run-webkit-tests on only the leopard buildbots. return 0; } -sub readSkippedFiles($) -{ - my ($constraintPath) = @_; - - foreach my $level (@platformTestHierarchy) { - if (open SKIPPED, "<", "$level/Skipped") { - if ($verbose) { - my ($dir, $name) = splitpath($level); - print "Skipped tests in $name:\n"; - } - - while (<SKIPPED>) { - my $skipped = $_; - chomp $skipped; - $skipped =~ s/^[ \n\r]+//; - $skipped =~ s/[ \n\r]+$//; - if ($skipped && $skipped !~ /^#/) { - if ($skippedOnly) { - if (!fileShouldBeIgnored($skipped)) { - if (!$constraintPath) { - # Always add $skipped since no constraint path was specified on the command line. - push(@ARGV, $skipped); - } elsif ($skipped =~ /^($constraintPath)/) { - # Add $skipped only if it matches the current path constraint, e.g., - # "--skipped=only dir1" with "dir1/file1.html" on the skipped list. - push(@ARGV, $skipped); - } elsif ($constraintPath =~ /^($skipped)/) { - # Add current path constraint if it is more specific than the skip list entry, - # e.g., "--skipped=only dir1/dir2/dir3" with "dir1" on the skipped list. - push(@ARGV, $constraintPath); - } - } elsif ($verbose) { - print " $skipped\n"; - } - } else { - if ($verbose) { - print " $skipped\n"; - } - processIgnoreTests($skipped, "Skipped"); - } - } - } - close SKIPPED; - } - } -} - -my @testsToRun; - -sub directoryFilter -{ - return () if exists $ignoredLocalDirectories{basename($File::Find::dir)}; - return () if exists $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)}; - return @_; -} - -sub fileFilter -{ - my $filename = $_; - if ($filename =~ /\.([^.]+)$/) { - if (exists $supportedFileExtensions{$1}) { - my $path = File::Spec->abs2rel(catfile($File::Find::dir, $filename), $testDirectory); - push @testsToRun, $path if !exists $ignoredFiles{$path}; - } - } -} - -sub findTestsToRun -{ - @testsToRun = (); - - for my $test (@ARGV) { - $test =~ s/^($layoutTestsName|$testDirectory)\///; - my $fullPath = catfile($testDirectory, $test); - if (file_name_is_absolute($test)) { - print "can't run test $test outside $testDirectory\n"; - } elsif (-f $fullPath) { - my ($filename, $pathname, $fileExtension) = fileparse($test, qr{\.[^.]+$}); - if (!exists $supportedFileExtensions{substr($fileExtension, 1)}) { - print "test $test does not have a supported extension\n"; - } elsif ($testHTTP || $pathname !~ /^http\//) { - push @testsToRun, $test; - } - } elsif (-d $fullPath) { - find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $fullPath); - for my $level (@platformTestHierarchy) { - my $platformPath = catfile($level, $test); - find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $platformPath) if (-d $platformPath); - } - } else { - print "test $test not found\n"; - } - } - - if (!scalar @ARGV) { - find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $testDirectory); - for my $level (@platformTestHierarchy) { - find({ preprocess => \&directoryFilter, wanted => \&fileFilter }, $level); - } - } - - # Remove duplicate tests - @testsToRun = keys %{{ map { $_ => 1 } @testsToRun }}; +my $harnessName = "old-run-webkit-tests"; - @testsToRun = sort pathcmp @testsToRun; - - # We need to minimize the time when Apache and WebSocketServer is locked by tests - # so run them last if no explicit order was specified in the argument list. - if (!scalar @ARGV) { - my @httpTests; - my @websocketTests; - my @otherTests; - foreach my $test (@testsToRun) { - if ($test =~ /^http\//) { - push(@httpTests, $test); - } elsif ($test =~ /^websocket\//) { - push(@websocketTests, $test); - } else { - push(@otherTests, $test); - } - } - @testsToRun = (@otherTests, @httpTests, @websocketTests); +if (useNewRunWebKitTests()) { + $harnessName = "new-run-webkit-tests"; + if (runningOnBuildBot()) { + push(@ARGV, "--verbose"); + # old-run-webkit-tests treats --results-directory as $CWD relative. + # new-run-webkit-tests treats --results-directory as build directory relative. + # Override the passed in --results-directory by appending a new one + # (later arguments override earlier ones in Python's optparse). + push(@ARGV, "--results-directory"); + # The buildbot always uses $SRCDIR/layout-test-results, hardcode it: + push(@ARGV, sourceDir() . "/layout-test-results"); } - - # Reverse the tests - @testsToRun = reverse @testsToRun if $reverseTests; - - # Shuffle the array - @testsToRun = shuffle(@testsToRun) if $randomizeTests; - - return @testsToRun; } -sub printResults -{ - my %text = ( - match => "succeeded", - mismatch => "had incorrect layout", - new => "were new", - timedout => "timed out", - crash => "crashed", - error => "had stderr output" - ); - - for my $type ("match", "mismatch", "new", "timedout", "crash", "error") { - my $typeCount = $counts{$type}; - next unless $typeCount; - my $typeText = $text{$type}; - my $message; - if ($typeCount == 1) { - $typeText =~ s/were/was/; - $message = sprintf "1 test case (%d%%) %s\n", 1 * 100 / $count, $typeText; - } else { - $message = sprintf "%d test cases (%d%%) %s\n", $typeCount, $typeCount * 100 / $count, $typeText; - } - $message =~ s-\(0%\)-(<1%)-; - print $message; - } -} +my $harnessPath = sprintf("%s/%s", relativeScriptsDir(), $harnessName); +exec $harnessPath ($harnessPath, @ARGV) or die "Failed to execute $harnessPath"; diff --git a/WebKitTools/Scripts/run-webkit-websocketserver b/WebKitTools/Scripts/run-webkit-websocketserver index 64a724d..08d430b 100755 --- a/WebKitTools/Scripts/run-webkit-websocketserver +++ b/WebKitTools/Scripts/run-webkit-websocketserver @@ -47,7 +47,7 @@ my $webSocketPort = 8880; my $srcDir = sourceDir(); my $layoutTestsName = "$srcDir/LayoutTests"; my $testDirectory = File::Spec->rel2abs($layoutTestsName); -my $webSocketServerPID = 0; +my $webSocketServerPidFile = "$testDirectory/websocket.pid"; print "Starting Web Socket server...\n"; @@ -60,40 +60,29 @@ closeWebSocketServer(); print "Stopped.\n"; exit 0; - sub openWebSocketServer() { - my $webSocketServerPath = "/usr/bin/python"; - my $webSocketPythonPath = "$srcDir/WebKitTools/pywebsocket"; my $webSocketHandlerDir = "$testDirectory"; - my $webSocketHandlerScanDir = "$testDirectory/websocket/tests"; - my $webSocketHandlerMapFile = "$webSocketHandlerScanDir/handler_map.txt"; my @args = ( - "$srcDir/WebKitTools/pywebsocket/mod_pywebsocket/standalone.py", - "-p", "$webSocketPort", - "-d", "$webSocketHandlerDir", - "-s", "$webSocketHandlerScanDir", - "-m", "$webSocketHandlerMapFile", - "-x", "/websocket/tests/cookies", + "$srcDir/WebKitTools/Scripts/new-run-webkit-websocketserver", + "--server", "start", + "--port", "$webSocketPort", + "--root", "$webSocketHandlerDir", + "--pidfile", "$webSocketServerPidFile" ); - - $ENV{"PYTHONPATH"} = $webSocketPythonPath; - $webSocketServerPID = open2(\*WEBSOCKETSERVER_IN, \*WEBSOCKETSERVER_OUT, $webSocketServerPath, @args); - - my $listen = "http://127.0.0.1:$webSocketPort"; - my $retryCount = 10; - while (system("/usr/bin/curl -k -q --silent --stderr - --output /dev/null $listen") && $retryCount) { - sleep 1; - --$retryCount; - } - die "Timed out waiting for WebSocketServer to start" unless $retryCount; + system "/usr/bin/python", @args; } sub closeWebSocketServer() { - close WEBSOCKETSERVER_IN; - close WEBSOCKETSERVER_OUT; - kill 15, $webSocketServerPID; + my @args = ( + "$srcDir/WebKitTools/Scripts/new-run-webkit-websocketserver", + "--server", "stop", + "--pidfile", "$webSocketServerPidFile" + ); + system "/usr/bin/python", @args; + unlink "$webSocketServerPidFile"; } + diff --git a/WebKitTools/Scripts/sunspider-compare-results b/WebKitTools/Scripts/sunspider-compare-results index 3446cd8..8c3f7f5 100755 --- a/WebKitTools/Scripts/sunspider-compare-results +++ b/WebKitTools/Scripts/sunspider-compare-results @@ -55,7 +55,7 @@ Usage: $programName [options] FILE FILE --parse-only Use the parse-only benchmark suite. Same as --suite=parse-only EOF -GetOptions('root=s' => sub { my ($argName, $value); setConfigurationProductDir(Cwd::abs_path($value)); }, +GetOptions('root=s' => sub { my ($argName, $value) = @_; setConfigurationProductDir(Cwd::abs_path($value)); }, 'suite=s' => \$suite, 'ubench' => \$ubench, 'v8' => \$v8, diff --git a/WebKitTools/Scripts/svn-apply b/WebKitTools/Scripts/svn-apply index f586211..33b2279 100755 --- a/WebKitTools/Scripts/svn-apply +++ b/WebKitTools/Scripts/svn-apply @@ -42,8 +42,7 @@ # Paths from Index: lines are used rather than the paths on the patch lines, which # makes patches generated by "cvs diff" work (increasingly unimportant since we # use Subversion now). -# ChangeLog patches use --fuzz=3 to prevent rejects, and the entry date is set in -# the patch to today's date using $changeLogTimeZone. +# ChangeLog patches use --fuzz=3 to prevent rejects. # Handles binary files (requires patches made by svn-create-patch). # Handles copied and moved files (requires patches made by svn-create-patch). # Handles git-diff patches (without binary changes) created at the top-level directory @@ -80,7 +79,6 @@ sub handleGitBinaryChange($$); sub isDirectoryEmptyForRemoval($); sub patch($); sub removeDirectoriesIfNeeded(); -sub setChangeLogDateAndReviewer($$); # These should be replaced by an scm class/module: sub scmKnowsOfFile($); @@ -88,10 +86,6 @@ sub scmCopy($$); sub scmAdd($); sub scmRemove($); - -# Project time zone for Cupertino, CA, US -my $changeLogTimeZone = "PST8PDT"; - my $merge = 0; my $showHelp = 0; my $reviewer; @@ -117,78 +111,47 @@ my %removeDirectoryIgnoreList = ( '_svn' => 1, ); +my $epochTime = time(); # This is used to set the date in ChangeLog files. my $globalExitStatus = 0; my $repositoryRootPath = determineVCSRoot(); my %checkedDirectories; -my %copiedFiles; -my @patches; -my %versions; - -my $copiedFromPath; -my $filter; -my $indexPath; -my $patch; -while (<>) { - s/([\n\r]+)$//mg; - my $eol = $1; - if (!defined($indexPath) && m#^diff --git \w/#) { - $filter = \&gitdiff2svndiff; - } - $_ = &$filter($_) if $filter; - if (/^Index: (.+)/) { - $indexPath = $1; - if ($patch) { - if (!$copiedFromPath) { - push @patches, $patch; - } - $copiedFromPath = ""; - $patch = ""; - } - } - if ($indexPath) { - # Fix paths on diff, ---, and +++ lines to match preceding Index: line. - s/\S+$/$indexPath/ if /^diff/; - s/^--- \S+/--- $indexPath/; - if (/^--- .+\(from (\S+):(\d+)\)$/) { - $copiedFromPath = $1; - $copiedFiles{$indexPath} = $copiedFromPath; - $versions{$copiedFromPath} = $2 if ($2 != 0); - } - elsif (/^--- .+\(revision (\d+)\)$/) { - $versions{$indexPath} = $1 if ($1 != 0); - } - if (s/^\+\+\+ \S+/+++ $indexPath/) { - $indexPath = ""; - } - } - $patch .= $_; - $patch .= $eol; -} -if ($patch && !$copiedFromPath) { - push @patches, $patch; -} +# Need to use a typeglob to pass the file handle as a parameter, +# otherwise get a bareword error. +my @diffHashRefs = parsePatch(*ARGV); + +print "Parsed " . @diffHashRefs . " diffs from patch file(s).\n"; + +my $preparedPatchHash = prepareParsedPatch($force, @diffHashRefs); + +my @copyDiffHashRefs = @{$preparedPatchHash->{copyDiffHashRefs}}; +my @nonCopyDiffHashRefs = @{$preparedPatchHash->{nonCopyDiffHashRefs}}; +my %sourceRevisions = %{$preparedPatchHash->{sourceRevisionHash}}; if ($merge) { die "--merge is currently only supported for SVN" unless isSVN(); # How do we handle Git patches applied to an SVN checkout here? - for my $file (sort keys %versions) { - my $version = $versions{$file}; + for my $file (sort keys %sourceRevisions) { + my $version = $sourceRevisions{$file}; print "Getting version $version of $file\n"; system("svn", "update", "-r", $version, $file) == 0 or die "Failed to run svn update -r $version $file."; } } -# Handle copied and moved files first since moved files may have their source deleted before the move. -for my $file (keys %copiedFiles) { - addDirectoriesIfNeeded(dirname($file)); - scmCopy($copiedFiles{$file}, $file); +# Handle copied and moved files first since moved files may have their +# source deleted before the move. +for my $copyDiffHashRef (@copyDiffHashRefs) { + my $indexPath = $copyDiffHashRef->{indexPath}; + my $copiedFromPath = $copyDiffHashRef->{copiedFromPath}; + + addDirectoriesIfNeeded(dirname($indexPath)); + scmCopy($copiedFromPath, $indexPath); } -for $patch (@patches) { - patch($patch); +for my $diffHashRef (@nonCopyDiffHashRefs) { + patch($diffHashRef); } removeDirectoriesIfNeeded(); @@ -276,14 +239,16 @@ sub handleBinaryChange($$) sub handleGitBinaryChange($$) { - my ($fullPath, $contents) = @_; + my ($fullPath, $diffHashRef) = @_; + + my $contents = $diffHashRef->{svnConvertedText}; my ($binaryChunkType, $binaryChunk, $reverseBinaryChunkType, $reverseBinaryChunk) = decodeGitBinaryPatch($contents, $fullPath); # FIXME: support "delta" type. die "only literal type is supported now" if ($binaryChunkType ne "literal" || $reverseBinaryChunkType ne "literal"); - my $isFileAddition = $contents =~ /\nnew file mode \d+\n/; - my $isFileDeletion = $contents =~ /\ndeleted file mode \d+\n/; + my $isFileAddition = $diffHashRef->{isNew}; + my $isFileDeletion = $diffHashRef->{isDeletion}; my $originalContents = ""; if (open FILE, $fullPath) { @@ -310,6 +275,7 @@ sub handleGitBinaryChange($$) sub isDirectoryEmptyForRemoval($) { my ($dir) = @_; + return 1 unless -d $dir; my $directoryIsEmpty = 1; opendir DIR, $dir or die "Could not open '$dir' to list files: $?"; for (my $item = readdir DIR; $item && $directoryIsEmpty; $item = readdir DIR) { @@ -325,37 +291,36 @@ sub isDirectoryEmptyForRemoval($) return $directoryIsEmpty; } +# Args: +# $diffHashRef: a diff hash reference of the type returned by parsePatch(). sub patch($) { - my ($patch) = @_; - return if !$patch; - - unless ($patch =~ m|^Index: ([^\r\n]+)|) { - my $separator = '-' x 67; - warn "Failed to find 'Index:' in:\n$separator\n$patch\n$separator\n"; - die unless $force; - return; - } - my $fullPath = $1; + my ($diffHashRef) = @_; + + # Make sure $patch is initialized to some value. A deletion can have no + # svnConvertedText property in the case of a deletion resulting from a + # Git rename. + my $patch = $diffHashRef->{svnConvertedText} || ""; + + my $fullPath = $diffHashRef->{indexPath}; + my $isBinary = $diffHashRef->{isBinary}; + my $isGit = $diffHashRef->{isGit}; my $deletion = 0; my $addition = 0; - my $isBinary = 0; - my $isGitBinary = 0; - $addition = 1 if ($patch =~ /\n--- .+\(revision 0\)\r?\n/ || $patch =~ /\n@@ -0,0 .* @@/) && !exists($copiedFiles{$fullPath}); - $deletion = 1 if $patch =~ /\n@@ .* \+0,0 @@/; - $isBinary = 1 if $patch =~ /\nCannot display: file marked as a binary type\./; - $isGitBinary = 1 if $patch =~ /\nGIT binary patch\n/; + $addition = 1 if ($diffHashRef->{isNew} || $patch =~ /\n@@ -0,0 .* @@/); + $deletion = 1 if ($diffHashRef->{isDeletion} || $patch =~ /\n@@ .* \+0,0 @@/); - if (!$addition && !$deletion && !$isBinary && !$isGitBinary) { + if (!$addition && !$deletion && !$isBinary) { # Standard patch, patch tool can handle this. if (basename($fullPath) eq "ChangeLog") { my $changeLogDotOrigExisted = -f "${fullPath}.orig"; - applyPatch(setChangeLogDateAndReviewer(fixChangeLogPatch($patch), $reviewer), $fullPath, ["--fuzz=3"]); + my $newPatch = setChangeLogDateAndReviewer(fixChangeLogPatch($patch), $reviewer, $epochTime); + applyPatch($newPatch, $fullPath, ["--fuzz=3"]); unlink("${fullPath}.orig") if (! $changeLogDotOrigExisted); } else { - applyPatch($patch, $fullPath); + applyPatch($patch, $fullPath) if $patch; } } else { # Either a deletion, an addition or a binary change. @@ -363,25 +328,26 @@ sub patch($) addDirectoriesIfNeeded(dirname($fullPath)); if ($isBinary) { - # Binary change - handleBinaryChange($fullPath, $patch); - } elsif ($isGitBinary) { - # Git binary change - handleGitBinaryChange($fullPath, $patch); + if ($isGit) { + handleGitBinaryChange($fullPath, $diffHashRef); + } else { + handleBinaryChange($fullPath, $patch) if $patch; + } } elsif ($deletion) { - # Deletion - applyPatch($patch, $fullPath, ["--force"]); + applyPatch($patch, $fullPath, ["--force"]) if $patch; scmRemove($fullPath); } else { # Addition rename($fullPath, "$fullPath.orig") if -e $fullPath; - applyPatch($patch, $fullPath); + applyPatch($patch, $fullPath) if $patch; unlink("$fullPath.orig") if -e "$fullPath.orig" && checksum($fullPath) eq checksum("$fullPath.orig"); scmAdd($fullPath); # What is this for? system("svn", "stat", "$fullPath.orig") if isSVN() && -e "$fullPath.orig"; } } + + scmToggleExecutableBit($fullPath, $diffHashRef->{executableBitDelta}) if defined($diffHashRef->{executableBitDelta}); } sub removeDirectoriesIfNeeded() @@ -393,26 +359,6 @@ sub removeDirectoriesIfNeeded() } } -sub setChangeLogDateAndReviewer($$) -{ - my $patch = shift; - my $reviewer = shift; - my $savedTimeZone = $ENV{'TZ'}; - # Set TZ temporarily so that localtime() is in that time zone - $ENV{'TZ'} = $changeLogTimeZone; - my $newDate = strftime("%Y-%m-%d", localtime()); - if (defined $savedTimeZone) { - $ENV{'TZ'} = $savedTimeZone; - } else { - delete $ENV{'TZ'}; - } - $patch =~ s/(\n\+)\d{4}-[^-]{2}-[^-]{2}( )/$1$newDate$2/; - if (defined($reviewer)) { - $patch =~ s/NOBODY \(OOPS!\)/$reviewer/; - } - return $patch; -} - # This could be made into a more general "status" call, except svn and git # have different ideas about "moving" files which might get confusing. sub scmWillDeleteFile($) @@ -428,6 +374,22 @@ sub scmWillDeleteFile($) return 0; } +# Return whether the file at the given path is known to Git. +# +# This method outputs a message like the following to STDERR when +# returning false: +# +# "error: pathspec 'test.png' did not match any file(s) known to git. +# Did you forget to 'git add'?" +sub gitKnowsOfFile($) +{ + my $path = shift; + + `git ls-files --error-unmatch -- $path`; + my $exitStatus = exitStatus($?); + return $exitStatus == 0; +} + sub scmKnowsOfFile($) { my ($path) = @_; @@ -440,9 +402,8 @@ sub scmKnowsOfFile($) # This does not handle errors well. return 1; } elsif (isGit()) { - `git ls-files --error-unmatch -- $path`; - my $exitCode = $? >> 8; - return $exitCode == 0; + my @result = callSilently(\&gitKnowsOfFile, $path); + return $result[0]; } } @@ -481,6 +442,12 @@ sub scmRemove($) close SVN; print $svnOutput if $svnOutput; } elsif (isGit()) { - system("git", "rm", "--force", $path) == 0 or die "Failed to git rm --force $path."; + # Git removes a directory if it becomes empty when the last file it contains is + # removed by `git rm`. In svn-apply this can happen when a directory is being + # removed in a patch, and all of the files inside of the directory are removed + # before attemping to remove the directory itself. In this case, Git will have + # already deleted the directory and `git rm` would exit with an error claiming + # there was no file. The --ignore-unmatch switch gracefully handles this case. + system("git", "rm", "--force", "--ignore-unmatch", $path) == 0 or die "Failed to git rm --force --ignore-unmatch $path."; } } diff --git a/WebKitTools/Scripts/svn-create-patch b/WebKitTools/Scripts/svn-create-patch index 768a8ed..5aead2e 100755 --- a/WebKitTools/Scripts/svn-create-patch +++ b/WebKitTools/Scripts/svn-create-patch @@ -56,12 +56,14 @@ use Time::gmtime; use VCSUtils; sub binarycmp($$); +sub diffOptionsForFile($); sub findBaseUrl($); sub findMimeType($;$); sub findModificationType($); sub findSourceFileAndRevision($); sub generateDiff($$); sub generateFileList($\%); +sub hunkHeaderLineRegExForFile($); sub isBinaryMimeType($); sub manufacturePatchForAdditionWithHistory($); sub numericcmp($$); @@ -99,9 +101,16 @@ for my $path (keys %paths) { my $svnRoot = determineSVNRoot(); my $prefix = chdirReturningRelativePath($svnRoot); +my $patchSize = 0; + # Generate the diffs, in a order chosen for easy reviewing. for my $path (sort patchpathcmp values %diffFiles) { - generateDiff($path, $prefix); + $patchSize += generateDiff($path, $prefix); +} + +if ($patchSize > 20480) { + print STDERR "WARNING: Patch's size is " . int($patchSize/1024) . " kbytes.\n"; + print STDERR "Patches 20k or smaller are more likely to be reviewed. Larger patches may sit unreviewed for a long time.\n"; } exit 0; @@ -130,6 +139,19 @@ sub binarycmp($$) return $fileDataA->{isBinary} <=> $fileDataB->{isBinary}; } +sub diffOptionsForFile($) +{ + my ($file) = @_; + + my $options = "uaNp"; + + if (my $hunkHeaderLineRegEx = hunkHeaderLineRegExForFile($file)) { + $options .= "F'$hunkHeaderLineRegEx'"; + } + + return $options; +} + sub findBaseUrl($) { my ($infoPath) = @_; @@ -196,24 +218,27 @@ sub generateDiff($$) my $file = File::Spec->catdir($prefix, $fileData->{path}); if ($ignoreChangelogs && basename($file) eq "ChangeLog") { - return; + return 0; } - my $patch; + my $patch = ""; if ($fileData->{modificationType} eq "additionWithHistory") { manufacturePatchForAdditionWithHistory($fileData); } - open DIFF, "svn diff --diff-cmd diff -x -uaNp '$file' |" or die; + + my $diffOptions = diffOptionsForFile($file); + open DIFF, "svn diff --diff-cmd diff -x -$diffOptions '$file' |" or die; while (<DIFF>) { $patch .= $_; } close DIFF; $patch = fixChangeLogPatch($patch) if basename($file) eq "ChangeLog"; - print $patch if $patch; + print $patch; if ($fileData->{isBinary}) { print "\n" if ($patch && $patch =~ m/\n\S+$/m); outputBinaryContent($file); } + return length($patch); } sub generateFileList($\%) @@ -252,6 +277,15 @@ sub generateFileList($\%) close STAT; } +sub hunkHeaderLineRegExForFile($) +{ + my ($file) = @_; + + my $startOfObjCInterfaceRegEx = "@(implementation\\|interface\\|protocol)"; + return "^[-+]\\|$startOfObjCInterfaceRegEx" if $file =~ /\.mm?$/; + return "^$startOfObjCInterfaceRegEx" if $file =~ /^(.*\/)?(mac|objc)\// && $file =~ /\.h$/; +} + sub isBinaryMimeType($) { my ($file) = @_; diff --git a/WebKitTools/Scripts/svn-unapply b/WebKitTools/Scripts/svn-unapply index eb20ca0..53ab1b5 100755 --- a/WebKitTools/Scripts/svn-unapply +++ b/WebKitTools/Scripts/svn-unapply @@ -97,54 +97,24 @@ my $repositoryRootPath = determineVCSRoot(); my @copiedFiles; my %directoriesToCheck; -my $copiedFromPath; -my $filter; -my $indexPath; -my $patch; -while (<>) { - s/([\n\r]+)$//mg; - my $eol = $1; - if (!defined($indexPath) && m#^diff --git \w/#) { - $filter = \&gitdiff2svndiff; - } - $_ = &$filter($_) if $filter; - if (/^Index: (.+)/) { - $indexPath = $1; - if ($patch) { - if ($copiedFromPath) { - push @copiedFiles, $patch; - } else { - patch($patch); - } - $copiedFromPath = ""; - $patch = ""; - } - } - if ($indexPath) { - # Fix paths on diff, ---, and +++ lines to match preceding Index: line. - s/^--- \S+/--- $indexPath/; - if (/^--- .+\(from (\S+):\d+\)$/) { - $copiedFromPath = $1; - } - if (s/^\+\+\+ \S+/+++ $indexPath/) { - $indexPath = ""; - } - } - $patch .= $_; - $patch .= $eol; -} +# Need to use a typeglob to pass the file handle as a parameter, +# otherwise get a bareword error. +my @diffHashRefs = parsePatch(*ARGV); -if ($patch) { - if ($copiedFromPath) { - push @copiedFiles, $patch; - } else { - patch($patch); - } +print "Parsed " . @diffHashRefs . " diffs from patch file(s).\n"; + +my $preparedPatchHash = prepareParsedPatch($force, @diffHashRefs); + +my @copyDiffHashRefs = @{$preparedPatchHash->{copyDiffHashRefs}}; +my @nonCopyDiffHashRefs = @{$preparedPatchHash->{nonCopyDiffHashRefs}}; + +for my $diffHashRef (@nonCopyDiffHashRefs) { + patch($diffHashRef); } # Handle copied and moved files last since they may have had post-copy changes that have now been unapplied -for $patch (@copiedFiles) { - patch($patch); +for my $diffHashRef (@copyDiffHashRefs) { + patch($diffHashRef); } if (isSVN()) { @@ -163,28 +133,28 @@ sub checksum($) return $checksum; } +# Args: +# $diffHashRef: a diff hash reference of the type returned by parsePatch(). sub patch($) { - my ($patch) = @_; - return if !$patch; + my ($diffHashRef) = @_; + + # Make sure $patch is initialized to some value. There is no + # svnConvertedText when reversing an svn copy/move. + my $patch = $diffHashRef->{svnConvertedText} || ""; + + my $fullPath = $diffHashRef->{indexPath}; + my $isSvnBinary = $diffHashRef->{isBinary} && $diffHashRef->{isSvn}; - unless ($patch =~ m|^Index: ([^\r\n]+)|) { - my $separator = '-' x 67; - warn "Failed to find 'Index:' in:\n$separator\n$patch\n$separator\n"; - return; - } - my $fullPath = $1; $directoriesToCheck{dirname($fullPath)} = 1; my $deletion = 0; my $addition = 0; - my $isBinary = 0; - $addition = 1 if ($patch =~ /\n--- .+\(revision 0\)\n/ || $patch =~ /\n@@ -0,0 .* @@/); - $deletion = 1 if $patch =~ /\n@@ .* \+0,0 @@/; - $isBinary = 1 if $patch =~ /\nCannot display: file marked as a binary type\./; + $addition = 1 if ($diffHashRef->{isNew} || $diffHashRef->{copiedFromPath} || $patch =~ /\n@@ -0,0 .* @@/); + $deletion = 1 if ($diffHashRef->{isDeletion} || $patch =~ /\n@@ .* \+0,0 @@/); - if (!$addition && !$deletion && !$isBinary) { + if (!$addition && !$deletion && !$isSvnBinary) { # Standard patch, patch tool can handle this. if (basename($fullPath) eq "ChangeLog") { my $changeLogDotOrigExisted = -f "${fullPath}.orig"; @@ -196,7 +166,8 @@ sub patch($) } else { # Either a deletion, an addition or a binary change. - if ($isBinary) { + # FIXME: Add support for Git binary files. + if ($isSvnBinary) { # Reverse binary change unlink($fullPath) if (-e $fullPath); system "svn", "revert", $fullPath; @@ -228,11 +199,17 @@ sub patch($) system "svn", "stat", $fullPath; } else { # Reverse addition - unapplyPatch($patch, $fullPath, ["--force"]); + # + # FIXME: This should use the same logic as svn-apply's deletion + # code. In particular, svn-apply's scmRemove() subroutine + # should be used here. + unapplyPatch($patch, $fullPath, ["--force"]) if $patch; unlink($fullPath) if -z $fullPath; system "svn", "revert", $fullPath; } } + + scmToggleExecutableBit($fullPath, -1 * $diffHashRef->{executableBitDelta}) if defined($diffHashRef->{executableBitDelta}); } sub revertDirectories() diff --git a/WebKitTools/Scripts/test-html5-parser b/WebKitTools/Scripts/test-html5-parser new file mode 100755 index 0000000..eb9bab4 --- /dev/null +++ b/WebKitTools/Scripts/test-html5-parser @@ -0,0 +1,88 @@ +#!/usr/bin/perl +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of Google, Inc. nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY GOOGLE AND ITS CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL GOOGLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Temporary script to test the HTML5 parser until it is able to +# run enough LayoutTests so that we know what changes we'll need +# to make to run-webkit-tests to support testing with the new parser. + +# NOTE: This script is a total hack and should be rolled into +# run-webkit-tests instead of being improved further. + +use strict; +use warnings; + +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; +use VCSUtils; + +sub writeToFile($$) +{ + my ($filePath, $contents) = @_; + open NEWFILE, ">", "$filePath" or die "Could not create $filePath. $!\n"; + print NEWFILE $contents; + close NEWFILE; +} + +setConfiguration(); + +my $productDir = productDir(); +my $dumpTool = "$productDir/DumpRenderTree"; + +chdirWebKit(); + +my @args = argumentsForConfiguration(); +system("WebKitTools/Scripts/build-dumprendertree", @args) == 0 or die "Failed to build DumpRenderTree"; + +my @tests = ( + "html5lib/runner", +); + +foreach my $test (@tests) { + # This logic is super-dumb. Instead of making it smarter, we should + # roll this into run-webkit-tests once we can run enough of the layout tests. + my $testPath = "LayoutTests/$test.html"; + my $expectedPath = "LayoutTests/$test-expected-html5.txt"; + my $actualPath = "LayoutTests/$test-actual-html5.txt"; + my $command = "DYLD_FRAMEWORK_PATH=$productDir $dumpTool --html5-treebuilder $testPath"; + print $command, "\n"; + my $output = `$command`; + writeToFile($actualPath, $output); + if (-r $expectedPath) { + my $expectedOutput = `cat $expectedPath`; + if ($expectedOutput eq $output) { + print "$test -> PASS\n"; + } else { + print "$test -> FAIL, diff:\n"; + system("diff -u $expectedPath $actualPath"); + } + } else { + print "$test -> NEW, results:\n"; + writeToFile($expectedPath, $output); + print $output; + } +} diff --git a/WebKitTools/Scripts/test-webkitperl b/WebKitTools/Scripts/test-webkitperl index 2e31593..4e63b8a 100755 --- a/WebKitTools/Scripts/test-webkitperl +++ b/WebKitTools/Scripts/test-webkitperl @@ -39,10 +39,20 @@ use Test::Harness; use lib $FindBin::Bin; # so this script can be run from any directory. use VCSUtils; -# Use an absolute path so this script can be run from any directory. -my $scriptsDir = $FindBin::Bin; +# Change the working directory so that we can pass shorter, relative +# paths to runtests(), rather than longer, absolute paths. +# +# We change to the source root so the paths can be relative to the +# source root. These paths display on the screen, and their meaning +# will be clearer to the user if relative to the root, rather than to +# the Scripts directory, say. +# +# Source root is two levels up from the Scripts directory. +my $sourceRootDir = File::Spec->catfile($FindBin::Bin, "../.."); +chdir($sourceRootDir); -my $pattern = File::Spec->catfile($scriptsDir, "webkitperl/*_unittest/*.pl"); +# Relative to root +my $pattern = "WebKitTools/Scripts/webkitperl/*_unittest/*.pl"; my @files = <${pattern}>; # lists files alphabetically diff --git a/WebKitTools/Scripts/test-webkitpy b/WebKitTools/Scripts/test-webkitpy index cfd3434..e35c6e6 100755 --- a/WebKitTools/Scripts/test-webkitpy +++ b/WebKitTools/Scripts/test-webkitpy @@ -1,5 +1,6 @@ #!/usr/bin/env python # Copyright (c) 2009 Google Inc. All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -27,40 +28,213 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +import os import sys -import unittest - -from webkitpy.bugzilla_unittest import * -from webkitpy.buildbot_unittest import * -from webkitpy.changelogs_unittest import * -from webkitpy.commands.download_unittest import * -from webkitpy.commands.early_warning_system_unittest import * -from webkitpy.commands.openbugs_unittest import OpenBugsTest -from webkitpy.commands.upload_unittest import * -from webkitpy.commands.queries_unittest import * -from webkitpy.commands.queues_unittest import * -from webkitpy.committers_unittest import * -from webkitpy.credentials_unittest import * -from webkitpy.diff_parser_unittest import * -from webkitpy.executive_unittest import * -from webkitpy.grammar_unittest import * -from webkitpy.multicommandtool_unittest import * -from webkitpy.networktransaction_unittest import * -from webkitpy.patchcollection_unittest import * -from webkitpy.queueengine_unittest import * -from webkitpy.steps.steps_unittest import * -from webkitpy.steps.closebugforlanddiff_unittest import * -from webkitpy.steps.updatechangelogswithreview_unittests import * -from webkitpy.style.unittests import * # for check-webkit-style -from webkitpy.user_unittest import * -from webkitpy.webkit_logging_unittest import * -from webkitpy.webkitport_unittest import * + +# Do not import anything from webkitpy prior to cleaning webkitpy of +# orphaned *.pyc files. This ensures that no orphaned *.pyc files are +# accidentally imported during the course of this script. +# +# Also, do not import or execute any Python code incompatible with +# Python 2.4 until after execution of the init() method below. + + +_log = logging.getLogger("test-webkitpy") + + +# Verbose logging is useful for debugging test-webkitpy code that runs +# before the actual unit tests -- things like autoinstall downloading and +# unit-test auto-detection logic. This is different from verbose logging +# of the unit tests themselves (i.e. the unittest module's --verbose flag). +def configure_logging(is_verbose_logging): + """Configure the root logger. + + Configure the root logger not to log any messages from webkitpy -- + except for messages from the autoinstall module. Also set the + logging level as described below. + + Args: + is_verbose_logging: A boolean value of whether logging should be + verbose. If this parameter is true, the logging + level for the handler on the root logger is set to + logging.DEBUG. Otherwise, it is set to logging.INFO. + + """ + # Don't use the Python ternary operator here so that this method will + # work with Python 2.4. + if is_verbose_logging: + logging_level = logging.DEBUG + else: + logging_level = logging.INFO + + handler = logging.StreamHandler(sys.stderr) + # We constrain the level on the handler rather than on the root + # logger itself. This is probably better because the handler is + # configured and known only to this module, whereas the root logger + # is an object shared (and potentially modified) by many modules. + # Modifying the handler, then, is less intrusive and less likely to + # interfere with modifications made by other modules (e.g. in unit + # tests). + handler.setLevel(logging_level) + formatter = logging.Formatter("%(name)s: %(levelname)-8s %(message)s") + handler.setFormatter(formatter) + + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.NOTSET) + + # Filter out most webkitpy messages. + # + # Messages can be selectively re-enabled for this script by updating + # this method accordingly. + def filter(record): + """Filter out autoinstall and non-third-party webkitpy messages.""" + # FIXME: Figure out a way not to use strings here, for example by + # using syntax like webkitpy.test.__name__. We want to be + # sure not to import any non-Python 2.4 code, though, until + # after the version-checking code has executed. + if (record.name.startswith("webkitpy.common.system.autoinstall") or + record.name.startswith("webkitpy.test")): + return True + if record.name.startswith("webkitpy"): + return False + return True + + testing_filter = logging.Filter() + testing_filter.filter = filter + + # Display a message so developers are not mystified as to why + # logging does not work in the unit tests. + _log.info("Suppressing most webkitpy logging while running unit tests.") + handler.addFilter(testing_filter) + + +def _clean_pyc_files(dir_to_clean, paths_not_to_log): + """Delete from a directory all .pyc files that have no .py file. + + Args: + dir_to_clean: The path to the directory to clean. + paths_not_to_log: A list of paths to .pyc files whose deletions should + not be logged. This list should normally include + only test .pyc files. + + """ + _log.debug("Cleaning orphaned *.pyc files from: %s" % dir_to_clean) + + # Normalize paths not to log. + paths_not_to_log = [os.path.abspath(path) for path in paths_not_to_log] + + for dir_path, dir_names, file_names in os.walk(dir_to_clean): + for file_name in file_names: + if file_name.endswith(".pyc") and file_name[:-1] not in file_names: + file_path = os.path.join(dir_path, file_name) + if os.path.abspath(file_path) not in paths_not_to_log: + _log.info("Deleting orphan *.pyc file: %s" % file_path) + os.remove(file_path) + + +# As a substitute for a unit test, this method tests _clean_pyc_files() +# in addition to calling it. We chose not to use the unittest module +# because _clean_pyc_files() is called only once and is not used elsewhere. +def _clean_webkitpy_with_test(): + webkitpy_dir = os.path.join(os.path.dirname(__file__), "webkitpy") + + # The test .pyc file is-- + # webkitpy/python24/TEMP_test-webkitpy_test_pyc_file.pyc. + test_path = os.path.join(webkitpy_dir, "python24", + "TEMP_test-webkitpy_test_pyc_file.pyc") + + test_file = open(test_path, "w") + try: + test_file.write("Test .pyc file generated by test-webkitpy.") + finally: + test_file.close() + + # Confirm that the test file exists so that when we check that it does + # not exist, the result is meaningful. + if not os.path.exists(test_path): + raise Exception("Test .pyc file not created: %s" % test_path) + + _clean_pyc_files(webkitpy_dir, [test_path]) + + if os.path.exists(test_path): + raise Exception("Test .pyc file not deleted: %s" % test_path) + + +def init(command_args): + """Execute code prior to importing from webkitpy.unittests. + + Args: + command_args: The list of command-line arguments -- usually + sys.argv[1:]. + + """ + verbose_logging_flag = "--verbose-logging" + is_verbose_logging = verbose_logging_flag in command_args + if is_verbose_logging: + # Remove the flag so it doesn't cause unittest.main() to error out. + # + # FIXME: Get documentation for the --verbose-logging flag to show + # up in the usage instructions, which are currently generated + # by unittest.main(). It's possible that this will require + # re-implementing the option parser for unittest.main() + # since there may not be an easy way to modify its existing + # option parser. + sys.argv.remove(verbose_logging_flag) + + configure_logging(is_verbose_logging) + _log.debug("Verbose WebKit logging enabled.") + + # We clean orphaned *.pyc files from webkitpy prior to importing from + # webkitpy to make sure that no import statements falsely succeed. + # This helps to check that import statements have been updated correctly + # after any file moves. Otherwise, incorrect import statements can + # be masked. + # + # For example, if webkitpy/python24/versioning.py were moved to a + # different location without changing any import statements, and if + # the corresponding .pyc file were left behind without deleting it, + # then "import webkitpy.python24.versioning" would continue to succeed + # even though it would fail for someone checking out a fresh copy + # of the source tree. This is because of a Python feature: + # + # "It is possible to have a file called spam.pyc (or spam.pyo when -O + # is used) without a file spam.py for the same module. This can be used + # to distribute a library of Python code in a form that is moderately + # hard to reverse engineer." + # + # ( http://docs.python.org/tutorial/modules.html#compiled-python-files ) + # + # Deleting the orphaned .pyc file prior to importing, however, would + # cause an ImportError to occur on import as desired. + _clean_webkitpy_with_test() + + import webkitpy.python24.versioning as versioning + + versioning.check_version(log=_log) + + (comparison, current_version, minimum_version) = \ + versioning.compare_version() + + if comparison > 0: + # Then the current version is later than the minimum version. + message = ("You are testing webkitpy with a Python version (%s) " + "higher than the minimum version (%s) it was meant " + "to support." % (current_version, minimum_version)) + _log.warn(message) + if __name__ == "__main__": - # FIXME: This is a hack, but I'm tired of commenting out the test. - # See https://bugs.webkit.org/show_bug.cgi?id=31818 - if len(sys.argv) > 1 and sys.argv[1] == "--all": - sys.argv.remove("--all") - from webkitpy.scm_unittest import * - unittest.main() + init(sys.argv[1:]) + + # We import the unit test code after init() to ensure that any + # Python version warnings are displayed in case an error occurs + # while interpreting webkitpy.unittests. This also allows + # logging to be configured prior to importing -- for example to + # enable the display of autoinstall logging.log messages while + # running the unit tests. + from webkitpy.test.main import Tester + + Tester().run_tests(sys.argv) diff --git a/WebKitTools/Scripts/update-iexploder-cssproperties b/WebKitTools/Scripts/update-iexploder-cssproperties index b7ae6cb..3fbcf83 100755 --- a/WebKitTools/Scripts/update-iexploder-cssproperties +++ b/WebKitTools/Scripts/update-iexploder-cssproperties @@ -1,6 +1,7 @@ #!/usr/bin/perl # Copyright (C) 2007 Apple Inc. All rights reserved. +# Copyright (C) 2010 Holger Hans Peter Freyther # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,87 +27,103 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# This script updates WebKitTools/iExploder/htdocs/cssproperties.in based on -# WebCore/css/CSSPropertyNames.in. +# This script updates WebKitTools/iExploder/htdocs/*.in based on +# WebCore/css/CSSPropertyNames.in, WebCore/html/HTMLTagNames.in +# and WebCore/html/HTMLAttributeNames.in use warnings; use strict; use FindBin; use lib $FindBin::Bin; +use VCSUtils; use webkitdirs; use File::Spec; -sub generateSectionFromCSSPropertyNamesFile(); -sub readiExploderFile(); -sub svnRevision($); -sub writeiExploderFile(); +sub generateEntityListFromFile($); +sub readiExploderFile($); +sub update($$); +sub writeiExploderFile($@); -my $iExploderFile = File::Spec->catfile(sourceDir(), split("/", "WebKitTools/iExploder/htdocs/cssproperties.in")); -my $cssPropertyNamesFile = File::Spec->catfile(sourceDir(), split("/", "WebCore/css/CSSPropertyNames.in")); - -my @sections = readiExploderFile(); -$sections[0] = generateSectionFromCSSPropertyNamesFile(); -writeiExploderFile(); - -print `svn stat $iExploderFile`; +update("cssproperties.in", "css/CSSPropertyNames.in"); +update("htmlattrs.in", "html/HTMLAttributeNames.in"); +update("htmltags.in", "html/HTMLTagNames.in"); print "Successfully updated!\n"; exit 0; -sub generateSectionFromCSSPropertyNamesFile() +sub generateEntityListFromFile($) { - my $revision = svnRevision($cssPropertyNamesFile); - my $path = File::Spec->abs2rel($cssPropertyNamesFile, sourceDir()); + my ($filename) = @_; + + my $revision = svnRevisionForDirectory(dirname($filename)); + my $path = File::Spec->abs2rel($filename, sourceDir()); my $result = "# From WebKit svn r" . $revision . " (" . $path . ")\n"; - my @properties = (); + my @entities = (); + my $in_namespace = 0; - open(IN, $cssPropertyNamesFile) || die "$!"; + open(IN, $filename) || die "$!"; while (my $l = <IN>) { chomp $l; + if ($l =~ m/^namespace=\"/) { + $in_namespace = 1; + } elsif ($in_namespace && $l =~ m/^$/) { + $in_namespace = 0; + } + + next if $in_namespace; next if $l =~ m/^\s*#/ || $l =~ m/^\s*$/; - push(@properties, $l); + + # For HTML Tags that can have additional information + if ($l =~ m/ /) { + my @split = split / /, $l; + $l = $split[0] + } + + push(@entities, $l); } close(IN); - $result .= join("\n", sort { $a cmp $b } @properties) . "\n\n"; + $result .= join("\n", sort { $a cmp $b } @entities) . "\n\n"; return $result; } -sub readiExploderFile() +sub readiExploderFile($) { + my ($filename) = @_; + my @sections = (); local $/ = "\n\n"; - open(IN, $iExploderFile) || die "$!"; + open(IN, $filename) || die "$!"; @sections = <IN>; close(IN); return @sections; } -sub svnRevision($) +sub update($$) { - my ($file) = @_; - my $revision = ""; + my ($iexploderPath, $webcorePath) = @_; - open INFO, "svn info '$file' |" or die; - while (<INFO>) { - if (/^Revision: (.+)/) { - $revision = $1; - } - } - close INFO; + $iexploderPath = File::Spec->catfile(sourceDir(), "WebKitTools", "iExploder", "htdocs", split("/", $iexploderPath)); + $webcorePath = File::Spec->catfile(sourceDir(), "WebCore", split("/", $webcorePath)); - return $revision ? $revision : "UNKNOWN"; + my @sections = readiExploderFile($iexploderPath); + $sections[0] = generateEntityListFromFile($webcorePath); + writeiExploderFile($iexploderPath, @sections); } -sub writeiExploderFile() + +sub writeiExploderFile($@) { - open(OUT, "> $iExploderFile") || die "$!"; + my ($filename, @sections) = @_; + + open(OUT, "> $filename") || die "$!"; print OUT join("", @sections); close(OUT); } + diff --git a/WebKitTools/Scripts/update-webkit b/WebKitTools/Scripts/update-webkit index 7602c41..3fc2efd 100755 --- a/WebKitTools/Scripts/update-webkit +++ b/WebKitTools/Scripts/update-webkit @@ -33,6 +33,7 @@ use strict; use FindBin; use lib $FindBin::Bin; use File::Basename; +use File::Path; use File::Spec; use Getopt::Long; use VCSUtils; @@ -67,6 +68,8 @@ __END__ exit 1; } +my $startTime = time(); + my @svnOptions = (); push @svnOptions, '-q' if $quiet; @@ -83,6 +86,13 @@ if (-d "../Internal") { runSvnUpdate() if $isSVN; runGitUpdate() if $isGit; } elsif (isChromium()) { + # Workaround for https://bugs.webkit.org/show_bug.cgi?id=38926 + # We should remove the following "if" block when we find a right fix. + if ((isCygwin() || isWindows()) && (stat("WebKit/chromium/features.gypi"))[9] >= $startTime) { + print "features.gypi has been updated. Cleaning the build directories.\n"; + rmtree(["WebKit/chromium/Debug", "WebKit/chromium/Release"]); + } + system("perl", "WebKitTools/Scripts/update-webkit-chromium") == 0 or die $!; } elsif (isAppleWinWebKit()) { system("perl", "WebKitTools/Scripts/update-webkit-auxiliary-libs") == 0 or die; diff --git a/WebKitTools/Scripts/update-webkit-auxiliary-libs b/WebKitTools/Scripts/update-webkit-auxiliary-libs index 9c52449..19e4ad3 100755 --- a/WebKitTools/Scripts/update-webkit-auxiliary-libs +++ b/WebKitTools/Scripts/update-webkit-auxiliary-libs @@ -31,11 +31,12 @@ use strict; use warnings; -use HTTP::Date qw(str2time); use File::Find; -use File::Temp (); use File::Spec; +use File::Temp (); use FindBin; +use HTTP::Date qw(str2time); +use POSIX; use lib $FindBin::Bin; use webkitdirs; @@ -58,9 +59,21 @@ my $tmpDir = File::Spec->rel2abs(File::Temp::tempdir("webkitlibsXXXXXXX", TMPDIR print "Checking Last-Modified date of $zipFile...\n"; my $result = system "curl -s -I $auxiliaryLibsURL | grep Last-Modified > \"$tmpDir/$file.headers\""; -print STDERR "Couldn't check Last-Modified date of new $zipFile.\n" if $result; -if (!$result && open NEW, "$tmpDir/$file.headers") { +if (WEXITSTATUS($result)) { + print STDERR "Couldn't check Last-Modified date of new $zipFile.\n"; + print STDERR "Please ensure that $auxiliaryLibsURL is reachable.\n"; + + if (! -f "$webkitLibrariesDir/$file.headers") { + print STDERR "Unable to check Last-Modified date and no version of $file to fall back to.\n"; + exit 1; + } + + print STDERR "Falling back to existing version of $file.\n"; + exit 0; +} + +if (open NEW, "$tmpDir/$file.headers") { my $new = lastModifiedToUnixTime(<NEW>); close NEW; diff --git a/WebKitTools/Scripts/update-webkit-chromium b/WebKitTools/Scripts/update-webkit-chromium index fa94f8c..8458f83 100755 --- a/WebKitTools/Scripts/update-webkit-chromium +++ b/WebKitTools/Scripts/update-webkit-chromium @@ -28,6 +28,9 @@ # Update script for the WebKit Chromium Port. +use File::Path; +use Getopt::Long; + chdir("WebKit/chromium") or die $!; # Find gclient or install it. @@ -45,11 +48,18 @@ if (`gclient --version`) { if (! -e ".gclient") { # If .gclient configuration file doesn't exist, create it. print "Configuring gclient...\n"; - system($gclientPath, + system($gclientPath, "config", "--spec=solutions=[{'name':'./','url':None}]") == 0 or die $!; } +my $force = 0; +GetOptions( + 'force' => \$force, +); + # Execute gclient sync. print "Updating chromium port dependencies using gclient...\n"; -system($gclientPath, "sync", "--force") == 0 or die $!; +my @gclientArgs = ($gclientPath, "sync"); +push @gclientArgs, "--force" if $force; +system(@gclientArgs) == 0 or die $!; diff --git a/WebKitTools/Scripts/update-webkit-localizable-strings b/WebKitTools/Scripts/update-webkit-localizable-strings index 1d1f413..493a777 100755 --- a/WebKitTools/Scripts/update-webkit-localizable-strings +++ b/WebKitTools/Scripts/update-webkit-localizable-strings @@ -34,7 +34,7 @@ use FindBin; use lib $FindBin::Bin; use webkitdirs; -my @directoriesToScan = ("WebKit/mac", "WebKit/win"); +my @directoriesToScan = ("WebKit/mac", "WebKit/win", "-WebCore/icu", "-WebKit/mac/icu"); my $fileToUpdate = "WebKit/English.lproj/Localizable.strings"; my $exceptionsFile = "WebKit/StringsNotToBeLocalized.txt"; diff --git a/WebKitTools/Scripts/validate-committer-lists b/WebKitTools/Scripts/validate-committer-lists index 2f2dd32..ad3d358 100755 --- a/WebKitTools/Scripts/validate-committer-lists +++ b/WebKitTools/Scripts/validate-committer-lists @@ -36,13 +36,13 @@ import subprocess import re import urllib2 from datetime import date, datetime, timedelta -from webkitpy.committers import CommitterList -from webkitpy.webkit_logging import log, error +from webkitpy.common.config.committers import CommitterList +from webkitpy.common.system.deprecated_logging import log, error from webkitpy.scm import Git # WebKit includes a built copy of BeautifulSoup in Scripts/webkitpy # so this import should always succeed. -from webkitpy.BeautifulSoup import BeautifulSoup +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup def print_list_if_non_empty(title, list_to_print): if not list_to_print: diff --git a/WebKitTools/Scripts/webkit-build-directory b/WebKitTools/Scripts/webkit-build-directory index a85c587..bf7d66d 100755 --- a/WebKitTools/Scripts/webkit-build-directory +++ b/WebKitTools/Scripts/webkit-build-directory @@ -34,31 +34,35 @@ use Getopt::Long; use lib $FindBin::Bin; use webkitdirs; -my $showBaseProductDirectory = 0; +my $showConfigurationDirectory = 0; my $showHelp = 0; +my $showTopLevelDirectory = 0; + my $programName = basename($0); my $usage = <<EOF; Usage: $programName [options] - --base Show the root build directory instead of one corresponding to the current target (e.g. Debug, Release) - --debug Show build directory for the Debug target - -h|--help Show this help message - --release Show build directory for the Release target + --configuration Show the build directory for a specific configuration (e.g. Debug, Release. Defaults to the active configuration set by set-webkit-configuration) + -h|--help Show this help message + --top-level Show the top-level build directory + +Either --configuration or --top-level is required. EOF setConfiguration(); # Figure out from the command line if we're --debug or --release or the default. my $getOptionsResult = GetOptions( - 'base' => \$showBaseProductDirectory, + 'configuration' => \$showConfigurationDirectory, + 'top-level' => \$showTopLevelDirectory, 'help|h' => \$showHelp, ); -if (!$getOptionsResult || $showHelp) { +if (!$getOptionsResult || $showHelp || (!$showConfigurationDirectory && !$showTopLevelDirectory)) { print STDERR $usage; exit 1; } -if ($showBaseProductDirectory) { +if ($showTopLevelDirectory) { print baseProductDir() . "\n"; } else { print productDir() . "\n"; diff --git a/WebKitTools/Scripts/webkit-patch b/WebKitTools/Scripts/webkit-patch index b4bcc4c..8300b9f 100755 --- a/WebKitTools/Scripts/webkit-patch +++ b/WebKitTools/Scripts/webkit-patch @@ -1,6 +1,7 @@ #!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2010 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -30,80 +31,34 @@ # # A tool for automating dealing with bugzilla, posting patches, committing patches, etc. +import logging import os +import sys -from webkitpy.bugzilla import Bugzilla -from webkitpy.buildbot import BuildBot -from webkitpy.commands.download import * -from webkitpy.commands.early_warning_system import * -from webkitpy.commands.openbugs import OpenBugs -from webkitpy.commands.queries import * -from webkitpy.commands.queues import * -from webkitpy.commands.upload import * -from webkitpy.executive import Executive -from webkitpy.webkit_logging import log -from webkitpy.multicommandtool import MultiCommandTool -from webkitpy.scm import detect_scm_system -from webkitpy.user import User +from webkitpy.common.system.logutils import configure_logging +import webkitpy.python24.versioning as versioning -class WebKitPatch(MultiCommandTool): - global_options = [ - make_option("--dry-run", action="store_true", dest="dry_run", default=False, help="do not touch remote servers"), - make_option("--status-host", action="store", dest="status_host", type="string", nargs=1, help="Hostname (e.g. localhost or commit.webkit.org) where status updates should be posted."), - ] +def main(): + # This is a hack to let us enable DEBUG logging as early as possible. + # Note this can't be ternary as versioning.check_version() + # hasn't run yet and this python might be older than 2.5. + if set(["-v", "--verbose"]).intersection(set(sys.argv)): + logging_level = logging.DEBUG + else: + logging_level = logging.INFO + configure_logging(logging_level=logging_level) - def __init__(self): - MultiCommandTool.__init__(self) + versioning.check_version() - self.bugs = Bugzilla() - self.buildbot = BuildBot() - self.executive = Executive() - self.user = User() - self._scm = None - self.status_server = StatusServer() + # Import webkit-patch code only after version-checking so that + # script doesn't error out before having a chance to report the + # version warning. + from webkitpy.tool.main import WebKitPatch - def scm(self): - # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands). - original_cwd = os.path.abspath(".") - if not self._scm: - self._scm = detect_scm_system(original_cwd) - - if not self._scm: - script_directory = os.path.abspath(sys.path[0]) - webkit_directory = os.path.abspath(os.path.join(script_directory, "../..")) - self._scm = detect_scm_system(webkit_directory) - if self._scm: - log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory)) - else: - error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory)) - - return self._scm - - def path(self): - return __file__ - - def should_show_in_main_help(self, command): - if not command.show_in_main_help: - return False - if command.requires_local_commits: - return self.scm().supports_local_commits() - return True - - # FIXME: This may be unnecessary since we pass global options to all commands during execute() as well. - def handle_global_options(self, options): - if options.dry_run: - self.scm().dryrun = True - self.bugs.dryrun = True - if options.status_host: - self.status_server.set_host(options.status_host) - - def should_execute_command(self, command): - if command.requires_local_commits and not self.scm().supports_local_commits(): - failure_reason = "%s requires local commits using %s in %s." % (command.name, self.scm().display_name(), self.scm().checkout_root) - return (False, failure_reason) - return (True, None) + WebKitPatch(__file__).main() if __name__ == "__main__": - WebKitPatch().main() + + main() diff --git a/WebKitTools/Scripts/webkitdirs.pm b/WebKitTools/Scripts/webkitdirs.pm index a788b3d..745e808 100644 --- a/WebKitTools/Scripts/webkitdirs.pm +++ b/WebKitTools/Scripts/webkitdirs.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2005, 2006, 2007 Apple Inc. All rights reserved. +# Copyright (C) 2005, 2006, 2007, 2010 Apple Inc. All rights reserved. # Copyright (C) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -50,6 +50,7 @@ BEGIN { our @EXPORT_OK; my $architecture; +my $numberOfCPUs; my $baseProductDir; my @baseProductDirOption; my $configuration; @@ -63,6 +64,7 @@ my $isSymbian; my %qtFeatureDefaults; my $isGtk; my $isWx; +my $isEfl; my @wxArgs; my $isChromium; my $isInspectorFrontend; @@ -71,6 +73,8 @@ my $isInspectorFrontend; my $vcBuildPath; my $windowsTmpPath; my $windowsSourceDir; +my $winVersion; +my $willUseVCExpressWhenBuilding = 0; # Defined in VCSUtils. sub exitStatus($); @@ -113,7 +117,9 @@ sub determineBaseProductDir return if defined $baseProductDir; determineSourceDir(); - if (isAppleMacWebKit()) { + $baseProductDir = $ENV{"WEBKITOUTPUTDIR"}; + + if (!defined($baseProductDir) and isAppleMacWebKit()) { # Silently remove ~/Library/Preferences/xcodebuild.plist which can # cause build failure. The presence of # ~/Library/Preferences/xcodebuild.plist can prevent xcodebuild from @@ -146,7 +152,7 @@ sub determineBaseProductDir } if (!defined($baseProductDir)) { # Port-spesific checks failed, use default - $baseProductDir = $ENV{"WEBKITOUTPUTDIR"} || "$sourceDir/WebKitBuild"; + $baseProductDir = "$sourceDir/WebKitBuild"; } if (isGit() && isGitBranchBuild()) { @@ -225,12 +231,35 @@ sub determineArchitecture } } +sub determineNumberOfCPUs +{ + return if defined $numberOfCPUs; + if (isLinux()) { + # First try the nproc utility, if it exists. If we get no + # results fall back to just interpretting /proc directly. + chomp($numberOfCPUs = `nproc 2> /dev/null`); + if ($numberOfCPUs eq "") { + $numberOfCPUs = (grep /processor/, `cat /proc/cpuinfo`); + } + } elsif (isWindows() || isCygwin()) { + if (defined($ENV{NUMBER_OF_PROCESSORS})) { + $numberOfCPUs = $ENV{NUMBER_OF_PROCESSORS}; + } else { + # Assumes cygwin + $numberOfCPUs = `ls /proc/registry/HKEY_LOCAL_MACHINE/HARDWARE/DESCRIPTION/System/CentralProcessor | wc -w`; + } + } elsif (isDarwin()) { + $numberOfCPUs = `sysctl -n hw.ncpu`; + } +} + sub jscPath($) { my ($productDir) = @_; my $jscName = "jsc"; $jscName .= "_debug" if (isCygwin() && ($configuration eq "Debug")); - return "$productDir/$jscName"; + return "$productDir/$jscName" if -e "$productDir/$jscName"; + return "$productDir/JavaScriptCore.framework/Resources/$jscName"; } sub argumentsForConfiguration() @@ -245,6 +274,7 @@ sub argumentsForConfiguration() push(@args, '--qt') if isQt(); push(@args, '--symbian') if isSymbian(); push(@args, '--gtk') if isGtk(); + push(@args, '--efl') if isEfl(); push(@args, '--wx') if isWx(); push(@args, '--chromium') if isChromium(); push(@args, '--inspector-frontend') if isInspectorFrontend(); @@ -270,11 +300,11 @@ sub determineConfigurationProductDir if (isAppleWinWebKit() && !isWx()) { $configurationProductDir = "$baseProductDir/bin"; } else { - # [Gtk] We don't have Release/Debug configurations in straight + # [Gtk][Efl] We don't have Release/Debug configurations in straight # autotool builds (non build-webkit). In this case and if # WEBKITOUTPUTDIR exist, use that as our configuration dir. This will # allows us to run run-webkit-tests without using build-webkit. - if ($ENV{"WEBKITOUTPUTDIR"} && isGtk()) { + if ($ENV{"WEBKITOUTPUTDIR"} && (isGtk() || isEfl())) { $configurationProductDir = "$baseProductDir"; } else { $configurationProductDir = "$baseProductDir/$configuration"; @@ -325,7 +355,7 @@ sub jscProductDir my $productDir = productDir(); $productDir .= "/JavaScriptCore" if isQt(); $productDir .= "/$configuration" if (isQt() && isWindows()); - $productDir .= "/Programs" if isGtk(); + $productDir .= "/Programs" if (isGtk() || isEfl()); return $productDir; } @@ -463,6 +493,12 @@ sub architecture() return $architecture; } +sub numberOfCPUs() +{ + determineNumberOfCPUs(); + return $numberOfCPUs; +} + sub setArchitecture { if (my $arch = shift @_) { @@ -541,6 +577,11 @@ sub builtDylibPathForName if (isDarwin() and -d "$configurationProductDir/lib/$libraryName.framework") { return "$configurationProductDir/lib/$libraryName.framework/$libraryName"; } elsif (isWindows()) { + if (configuration() eq "Debug") { + # On Windows, there is a "d" suffix to the library name. See <http://trac.webkit.org/changeset/53924/>. + $libraryName .= "d"; + } + my $mkspec = `qmake -query QMAKE_MKSPECS`; $mkspec =~ s/[\n|\r]$//g; my $qtMajorVersion = retrieveQMakespecVar("$mkspec/qconfig.pri", "QT_MAJOR_VERSION"); @@ -556,7 +597,10 @@ sub builtDylibPathForName return "$configurationProductDir/libwxwebkit.dylib"; } if (isGtk()) { - return "$configurationProductDir/$libraryName/../.libs/libwebkit-1.0.so"; + return "$configurationProductDir/$libraryName/../.libs/libwebkitgtk-1.0.so"; + } + if (isEfl()) { + return "$configurationProductDir/$libraryName/../.libs/libewebkit.so"; } if (isAppleMacWebKit()) { return "$configurationProductDir/$libraryName.framework/Versions/A/$libraryName"; @@ -569,7 +613,7 @@ sub builtDylibPathForName } } - die "Unsupported platform, can't determine built library locations."; + die "Unsupported platform, can't determine built library locations.\nTry `build-webkit --help` for more information.\n"; } # Check to see that all the frameworks are built. @@ -614,15 +658,23 @@ sub qtFeatureDefaults() return %qtFeatureDefaults; } +sub commandExists($) +{ + my $command = shift; + my $devnull = File::Spec->devnull(); + return `$command --version 2> $devnull`; +} + sub determineQtFeatureDefaults() { return if %qtFeatureDefaults; + die "ERROR: qmake missing but required to build WebKit.\n" if not commandExists("qmake"); my $originalCwd = getcwd(); chdir File::Spec->catfile(sourceDir(), "WebCore"); my $defaults = `qmake CONFIG+=compute_defaults 2>&1`; chdir $originalCwd; - while ($defaults =~ m/(\S*?)=(.*?)( |$)/gi) { + while ($defaults =~ m/(\S+?)=(\S+?)/gi) { $qtFeatureDefaults{$1}=$2; } } @@ -649,8 +701,8 @@ sub determineIsQt() return; } - # The presence of QTDIR only means Qt if --gtk is not on the command-line - if (isGtk() || isWx()) { + # The presence of QTDIR only means Qt if --gtk or --wx or --efl are not on the command-line + if (isGtk() || isWx() || isEfl()) { $isQt = 0; return; } @@ -666,8 +718,18 @@ sub determineIsSymbian() $isSymbian = 1; return; } +} + +sub determineIsEfl() +{ + return if defined($isEfl); + $isEfl = checkForArgumentAndRemoveFromARGV("--efl"); +} - $isSymbian = defined($ENV{'EPOCROOT'}); +sub isEfl() +{ + determineIsEfl(); + return $isEfl; } sub isGtk() @@ -739,6 +801,41 @@ sub isCygwin() return ($^O eq "cygwin") || 0; } +sub determineWinVersion() +{ + return if $winVersion; + + if (!isCygwin()) { + $winVersion = -1; + return; + } + + my $versionString = `uname -s`; + $versionString =~ /(\d\.\d)/; + $winVersion = $1; +} + +sub winVersion() +{ + determineWinVersion(); + return $winVersion; +} + +sub isWindows7() +{ + return winVersion() eq "6.1"; +} + +sub isWindowsVista() +{ + return winVersion() eq "6.0"; +} + +sub isWindowsXP() +{ + return winVersion() eq "5.1"; +} + sub isDarwin() { return ($^O eq "darwin") || 0; @@ -761,7 +858,7 @@ sub isLinux() sub isAppleWebKit() { - return !(isQt() or isGtk() or isWx() or isChromium()); + return !(isQt() or isGtk() or isWx() or isChromium() or isEfl()); } sub isAppleMacWebKit() @@ -848,7 +945,7 @@ sub relativeScriptsDir() sub launcherPath() { my $relativeScriptsPath = relativeScriptsDir(); - if (isGtk() || isQt() || isWx()) { + if (isGtk() || isQt() || isWx() || isEfl()) { return "$relativeScriptsPath/run-launcher"; } elsif (isAppleWebKit()) { return "$relativeScriptsPath/run-safari"; @@ -860,11 +957,13 @@ sub launcherName() if (isGtk()) { return "GtkLauncher"; } elsif (isQt()) { - return "QtLauncher"; + return "QtTestBrowser"; } elsif (isWx()) { return "wxBrowser"; } elsif (isAppleWebKit()) { return "Safari"; + } elsif (isEfl()) { + return "EWebLauncher"; } } @@ -887,14 +986,13 @@ sub checkRequiredSystemConfig print "http://developer.apple.com/tools/xcode\n"; print "*************************************************************\n"; } - } elsif (isGtk() or isQt() or isWx()) { + } elsif (isGtk() or isQt() or isWx() or isEfl()) { my @cmds = qw(flex bison gperf); my @missing = (); foreach my $cmd (@cmds) { - if (not `$cmd --version`) { - push @missing, $cmd; - } + push @missing, $cmd if not commandExists($cmd); } + if (@missing) { my $list = join ", ", @missing; die "ERROR: $list missing but required to build WebKit.\n"; @@ -995,6 +1093,7 @@ sub setupCygwinEnv() print "*************************************************************\n"; die; } + $willUseVCExpressWhenBuilding = 1; } my $qtSDKPath = "$programFilesPath/QuickTime SDK"; @@ -1016,6 +1115,27 @@ sub setupCygwinEnv() print "WEBKITLIBRARIESDIR is set to: ", $ENV{"WEBKITLIBRARIESDIR"}, "\n"; } +sub dieIfWindowsPlatformSDKNotInstalled +{ + my $registry32Path = "/proc/registry/"; + my $registry64Path = "/proc/registry64/"; + my $windowsPlatformSDKRegistryEntry = "HKEY_LOCAL_MACHINE/SOFTWARE/Microsoft/MicrosoftSDK/InstalledSDKs/D2FF9F89-8AA2-4373-8A31-C838BF4DBBE1"; + + # FIXME: It would be better to detect whether we are using 32- or 64-bit Windows + # and only check the appropriate entry. But for now we just blindly check both. + return if (-e $registry32Path . $windowsPlatformSDKRegistryEntry) || (-e $registry64Path . $windowsPlatformSDKRegistryEntry); + + print "*************************************************************\n"; + print "Cannot find registry entry '$windowsPlatformSDKRegistryEntry'.\n"; + print "Please download and install the Microsoft Windows Server 2003 R2\n"; + print "Platform SDK from <http://www.microsoft.com/downloads/details.aspx?\n"; + print "familyid=0baf2b35-c656-4969-ace8-e4c0c0716adb&displaylang=en>.\n\n"; + print "Then follow step 2 in the Windows section of the \"Installing Developer\n"; + print "Tools\" instructions at <http://www.webkit.org/building/tools.html>.\n"; + print "*************************************************************\n"; + die; +} + sub copyInspectorFrontendFiles { my $productDir = productDir(); @@ -1033,6 +1153,9 @@ sub copyInspectorFrontendFiles } elsif (isQt() || isGtk()) { my $prefix = $ENV{"WebKitInstallationPrefix"}; $inspectorResourcesDirPath = (defined($prefix) ? $prefix : "/usr/share") . "/webkit-1.0/webinspector"; + } elsif (isEfl()) { + my $prefix = $ENV{"WebKitInstallationPrefix"}; + $inspectorResourcesDirPath = (defined($prefix) ? $prefix : "/usr/share") . "/ewebkit/webinspector"; } if (! -d $inspectorResourcesDirPath) { @@ -1045,7 +1168,7 @@ sub copyInspectorFrontendFiles print "*************************************************************\n"; die; } - return system "rsync", "-aut", "--exclude=/.DS_Store", "--exclude=.svn/", !isQt() ? "--exclude=/WebKit.qrc" : "", $sourceInspectorPath, $inspectorResourcesDirPath; + return system "rsync", "-aut", "--exclude=/.DS_Store", "--exclude=*.re2js", "--exclude=.svn/", !isQt() ? "--exclude=/WebKit.qrc" : "", $sourceInspectorPath, $inspectorResourcesDirPath; } sub buildXCodeProject($$@) @@ -1067,6 +1190,8 @@ sub buildVisualStudioProject my $config = configurationForVisualStudio(); + dieIfWindowsPlatformSDKNotInstalled() if $willUseVCExpressWhenBuilding; + chomp(my $winProjectPath = `cygpath -w "$project"`); my $action = "/build"; @@ -1162,7 +1287,7 @@ sub qtMakeCommand($) #print "default spec: " . $mkspec . "\n"; #print "compiler found: " . $compiler . "\n"; - if ($compiler eq "cl") { + if ($compiler && $compiler eq "cl") { return "nmake"; } @@ -1184,7 +1309,7 @@ sub buildAutotoolsProject($@) my $make = 'make'; my $dir = productDir(); my $config = passedConfiguration() || configuration(); - my $prefix = $ENV{"WebKitInstallationPrefix"}; + my $prefix; my @buildArgs = (); my $makeArgs = $ENV{"WebKitMakeArguments"} || ""; @@ -1192,11 +1317,20 @@ sub buildAutotoolsProject($@) my $opt = $buildParams[$i]; if ($opt =~ /^--makeargs=(.*)/i ) { $makeArgs = $makeArgs . " " . $1; + } elsif ($opt =~ /^--prefix=(.*)/i ) { + $prefix = $1; } else { push @buildArgs, $opt; } } + # Automatically determine the number of CPUs for make only + # if make arguments haven't already been specified. + if ($makeArgs eq "") { + $makeArgs = "-j" . numberOfCPUs(); + } + + $prefix = $ENV{"WebKitInstallationPrefix"} if !defined($prefix); push @buildArgs, "--prefix=" . $prefix if defined($prefix); # check if configuration is Debug @@ -1212,31 +1346,29 @@ sub buildAutotoolsProject($@) } if (! -d $dir) { - system "mkdir", "-p", "$dir"; - if (! -d $dir) { - die "Failed to create build directory " . $dir; - } + File::Path::mkpath($dir) or die "Failed to create build directory " . $dir } - chdir $dir or die "Failed to cd into " . $dir . "\n"; - my $result; if ($clean) { - #$result = system $make, "distclean"; return 0; } - print "Calling configure in " . $dir . "\n\n"; - print "Installation directory: $prefix\n" if(defined($prefix)); - - # Make the path relative since it will appear in all -I compiler flags. - # Long argument lists cause bizarre slowdowns in libtool. - my $relSourceDir = File::Spec->abs2rel($sourceDir); - $relSourceDir = "." if !$relSourceDir; - - $result = system "$relSourceDir/autogen.sh", @buildArgs; - if ($result ne 0) { - die "Failed to setup build environment using 'autotools'!\n"; + # If GNUmakefile exists, don't run autogen.sh. The makefile should be + # smart enough to track autotools dependencies and re-run autogen.sh + # when build files change. + my $result; + if (! -e "GNUmakefile") { + print "Calling configure in " . $dir . "\n\n"; + print "Installation prefix directory: $prefix\n" if(defined($prefix)); + + # Make the path relative since it will appear in all -I compiler flags. + # Long argument lists cause bizarre slowdowns in libtool. + my $relSourceDir = File::Spec->abs2rel($sourceDir) || "."; + $result = system "$relSourceDir/autogen.sh", @buildArgs; + if ($result ne 0) { + die "Failed to setup build environment using 'autotools'!\n"; + } } $result = system "$make $makeArgs"; @@ -1256,6 +1388,8 @@ sub buildQMakeProject($@) my $qmakebin = "qmake"; # Allow override of the qmake binary from $PATH my $makeargs = ""; + my $installHeaders; + my $installLibs; for my $i (0 .. $#buildParams) { my $opt = $buildParams[$i]; if ($opt =~ /^--qmake=(.*)/i ) { @@ -1264,6 +1398,10 @@ sub buildQMakeProject($@) push @buildArgs, $1; } elsif ($opt =~ /^--makeargs=(.*)/i ) { $makeargs = $1; + } elsif ($opt =~ /^--install-headers=(.*)/i ) { + $installHeaders = $1; + } elsif ($opt =~ /^--install-libs=(.*)/i ) { + $installLibs = $1; } else { push @buildArgs, $opt; } @@ -1271,7 +1409,8 @@ sub buildQMakeProject($@) my $make = qtMakeCommand($qmakebin); my $config = configuration(); - my $prefix = $ENV{"WebKitInstallationPrefix"}; + push @buildArgs, "INSTALL_HEADERS=" . $installHeaders if defined($installHeaders); + push @buildArgs, "INSTALL_LIBS=" . $installLibs if defined($installLibs); my $dir = File::Spec->canonpath(baseProductDir()); $dir = File::Spec->catfile($dir, $config) unless isSymbian(); File::Path::mkpath($dir); @@ -1323,7 +1462,8 @@ sub buildQMakeProject($@) } print "Calling '$qmakebin @buildArgs' in " . $dir . "\n\n"; - print "Installation directory: $prefix\n" if(defined($prefix)); + print "Installation headers directory: $installHeaders\n" if(defined($installHeaders)); + print "Installation libraries directory: $installLibs\n" if(defined($installLibs)); $result = system "$qmakebin @buildArgs"; if ($result ne 0) { @@ -1364,15 +1504,15 @@ sub buildGtkProject($$@) return buildAutotoolsProject($clean, @buildArgs); } -sub buildChromiumMakefile($$$) +sub buildChromiumMakefile($$) { - my ($dir, $target, $clean) = @_; - chdir $dir; + my ($target, $clean) = @_; if ($clean) { return system qw(rm -rf out); } my $config = configuration(); - my @command = ("make", "-j4", "BUILDTYPE=$config", $target); + my $numCpus = numberOfCPUs(); + my @command = ("make", "-fMakefile.chromium", "-j$numCpus", "BUILDTYPE=$config", $target); print join(" ", @command) . "\n"; return system @command; } @@ -1396,6 +1536,19 @@ sub buildChromiumVisualStudioProject($$) $vsInstallDir = `cygpath "$vsInstallDir"` if isCygwin(); chomp $vsInstallDir; $vcBuildPath = "$vsInstallDir/Common7/IDE/devenv.com"; + if (! -e $vcBuildPath) { + # Visual Studio not found, try VC++ Express + $vcBuildPath = "$vsInstallDir/Common7/IDE/VCExpress.exe"; + if (! -e $vcBuildPath) { + print "*************************************************************\n"; + print "Cannot find '$vcBuildPath'\n"; + print "Please execute the file 'vcvars32.bat' from\n"; + print "'$programFilesPath\\Microsoft Visual Studio 8\\VC\\bin\\'\n"; + print "to setup the necessary environment variables.\n"; + print "*************************************************************\n"; + die; + } + } # Create command line and execute it. my @command = ($vcBuildPath, $projectPath, $action, $config); @@ -1408,16 +1561,19 @@ sub buildChromium($@) { my ($clean, @options) = @_; + # We might need to update DEPS or re-run GYP if things have changed. + system("perl", "WebKitTools/Scripts/update-webkit-chromium") == 0 or die $!; + my $result = 1; if (isDarwin()) { # Mac build - builds the root xcode project. - $result = buildXCodeProject("WebKit/chromium/WebKit", $clean, (@options)); + $result = buildXCodeProject("WebKit/chromium/WebKit", $clean, "-configuration", configuration(), @options); } elsif (isCygwin() || isWindows()) { # Windows build - builds the root visual studio solution. $result = buildChromiumVisualStudioProject("WebKit/chromium/WebKit.sln", $clean); } elsif (isLinux()) { # Linux build - build using make. - $ result = buildChromiumMakefile("WebKit/chromium/", "all", $clean); + $ result = buildChromiumMakefile("all", $clean); } else { print STDERR "This platform is not supported by chromium.\n"; } @@ -1454,7 +1610,7 @@ sub runSafari my ($debugger) = @_; if (isAppleMacWebKit()) { - return system "$FindBin::Bin/gdb-safari", @ARGV if $debugger; + return system "$FindBin::Bin/gdb-safari", argumentsForConfiguration() if $debugger; my $productDir = productDir(); print "Starting Safari with DYLD_FRAMEWORK_PATH set to point to built WebKit in $productDir.\n"; @@ -1468,17 +1624,97 @@ sub runSafari } if (isAppleWinWebKit()) { - my $script = "run-webkit-nightly.cmd"; - my $result = system "cp", "$FindBin::Bin/$script", productDir(); + my $result; + my $productDir = productDir(); + if ($debugger) { + setupCygwinEnv(); + chomp($ENV{WEBKITNIGHTLY} = `cygpath -wa "$productDir"`); + my $safariPath = safariPath(); + chomp($safariPath = `cygpath -wa "$safariPath"`); + $result = system $vcBuildPath, "/debugexe", "\"$safariPath\"", @ARGV; + } else { + $result = system File::Spec->catfile(productDir(), "WebKit.exe"), @ARGV; + } return $result if $result; + } + + return 1; +} + +sub runMiniBrowser +{ + if (isAppleMacWebKit()) { + my $productDir = productDir(); + print "Starting MiniBrowser with DYLD_FRAMEWORK_PATH set to point to $productDir.\n"; + $ENV{DYLD_FRAMEWORK_PATH} = $productDir; + $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES"; + my $miniBrowserPath = "$productDir/MiniBrowser.app/Contents/MacOS/MiniBrowser"; + if (!isTiger() && architecture()) { + return system "arch", "-" . architecture(), $miniBrowserPath, @ARGV; + } else { + return system $miniBrowserPath, @ARGV; + } + } + + return 1; +} + +sub debugMiniBrowser +{ + if (isAppleMacWebKit()) { + my $gdbPath = "/usr/bin/gdb"; + die "Can't find gdb executable. Is gdb installed?\n" unless -x $gdbPath; + + my $productDir = productDir(); + + $ENV{DYLD_FRAMEWORK_PATH} = $productDir; + $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = 'YES'; + + my $miniBrowserPath = "$productDir/MiniBrowser.app/Contents/MacOS/MiniBrowser"; + + print "Starting MiniBrowser under gdb with DYLD_FRAMEWORK_PATH set to point to built WebKit2 in $productDir.\n"; + my @architectureFlags = ("-arch", architecture()) if !isTiger(); + exec $gdbPath, @architectureFlags, $miniBrowserPath or die; + return; + } + + return 1; +} + +sub runWebKitTestRunner +{ + if (isAppleMacWebKit()) { + my $productDir = productDir(); + print "Starting WebKitTestRunner with DYLD_FRAMEWORK_PATH set to point to $productDir.\n"; + $ENV{DYLD_FRAMEWORK_PATH} = $productDir; + $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES"; + my $webKitTestRunnerPath = "$productDir/WebKitTestRunner"; + if (!isTiger() && architecture()) { + return system "arch", "-" . architecture(), $webKitTestRunnerPath, @ARGV; + } else { + return system $webKitTestRunnerPath, @ARGV; + } + } + + return 1; +} + +sub debugWebKitTestRunner +{ + if (isAppleMacWebKit()) { + my $gdbPath = "/usr/bin/gdb"; + die "Can't find gdb executable. Is gdb installed?\n" unless -x $gdbPath; - my $cwd = getcwd(); - chdir productDir(); + my $productDir = productDir(); + $ENV{DYLD_FRAMEWORK_PATH} = $productDir; + $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = 'YES'; - my $debuggerFlag = $debugger ? "/debugger" : ""; - $result = system "cmd", "/c", "call $script $debuggerFlag"; - chdir $cwd; - return $result; + my $webKitTestRunnerPath = "$productDir/WebKitTestRunner"; + + print "Starting WebKitTestRunner under gdb with DYLD_FRAMEWORK_PATH set to point to $productDir.\n"; + my @architectureFlags = ("-arch", architecture()) if !isTiger(); + exec $gdbPath, @architectureFlags, $webKitTestRunnerPath or die; + return; } return 1; diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/fixChangeLogPatch.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/fixChangeLogPatch.pl index d21c706..2fd187b 100644 --- a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/fixChangeLogPatch.pl +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/fixChangeLogPatch.pl @@ -30,7 +30,7 @@ # Unit tests of VCSUtils::fixChangeLogPatch(). -use Test::Simple tests => 7; +use Test::Simple tests => 8; use VCSUtils; # The source ChangeLog for these tests is the following: @@ -123,6 +123,31 @@ END ok(fixChangeLogPatch($in) eq $in, $title); # New test +$title = "fixChangeLogPatch: [no change] New entry inserted earlier in the file, but after an entry with the same author and date."; + +$in = <<'END'; +--- ChangeLog ++++ ChangeLog +@@ -70,6 +70,14 @@ + + 2009-12-22 Alice <alice@email.address> + ++ Reviewed by Sue. ++ ++ Changed some more code on 2009-12-22. ++ ++ * File: ++ ++2009-12-22 Alice <alice@email.address> ++ + Reviewed by Ray. + + Changed some code on 2009-12-22. +END + +ok(fixChangeLogPatch($in) eq $in, $title); + +# New test $title = "fixChangeLogPatch: Leading context includes first line."; $in = <<'END'; diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/gitdiff2svndiff.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/gitdiff2svndiff.pl deleted file mode 100644 index 93708d6..0000000 --- a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/gitdiff2svndiff.pl +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/perl -w -# -# Copyright (C) 2009, 2010 Chris Jerdonek (chris.jerdonek@gmail.com) -# Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies) -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Unit tests of VCSUtils::gitdiff2svndiff() - -use strict; -use warnings; - -use Test::Simple tests => 20; -use VCSUtils; - -# We use this for display purposes, to keep each test title on one line. -sub excerptString($) -{ - my ($text) = @_; - - my $length = 25; - - my $shortened = substr($text, 0, $length); - $shortened .= "..." if (length($text) > $length); - - return $shortened; -} - -my $git_patch = <<END; -diff --git a/WebCore/rendering/style/StyleFlexibleBoxData.h b/WebCore/rendering/style/StyleFlexibleBoxData.h -index f5d5e74..3b6aa92 100644 ---- a/WebCore/rendering/style/StyleFlexibleBoxData.h -+++ b/WebCore/rendering/style/StyleFlexibleBoxData.h -@@ -47,7 +47,6 @@ public: -END - -my $svn_patch = <<END; -Index: WebCore/rendering/style/StyleFlexibleBoxData.h -=================================================================== ---- WebCore/rendering/style/StyleFlexibleBoxData.h -+++ WebCore/rendering/style/StyleFlexibleBoxData.h -@@ -47,7 +47,6 @@ public: -END - -my @gitLines = split("\n", $git_patch); -my @svnLines = split("\n", $svn_patch); - -# New test: check each git header line with different line endings -my $titleHeader = "gitdiff2svndiff: "; - -my @lineEndingPairs = ( # display name, value - ["", ""], - ["\\n", "\n"], - ["\\r\\n", "\r\n"], -); - -for (my $i = 0; $i < @gitLines; $i++) { - foreach my $pair (@lineEndingPairs) { - my $gitLine = $gitLines[$i] . $pair->[1]; - my $expected = $svnLines[$i] . $pair->[1]; - my $title = $titleHeader . excerptString($gitLine); - $title .= " [line-end: \"$pair->[0]\"]"; - - ok($expected eq gitdiff2svndiff($gitLine), $title); - } -} - -# New test -my $title = "gitdiff2svndiff: Convert mnemonic git diff to svn diff"; - -my @prefixes = ( - { 'a' => 'i', 'b' => 'w' }, # git-diff (compares the (i)ndex and the (w)ork tree) - { 'a' => 'c', 'b' => 'w' }, # git-diff HEAD (compares a (c)ommit and the (w)ork tree) - { 'a' => 'c', 'b' => 'i' }, # git diff --cached (compares a (c)ommit and the (i)ndex) - { 'a' => 'o', 'b' => 'w' }, # git-diff HEAD:file1 file2 (compares an (o)bject and a (w)ork tree entity) - { 'a' => '1', 'b' => '2' }, # git diff --no-index a b (compares two non-git things (1) and (2)) -); - -my $out = ""; - -foreach my $prefix (@prefixes) { - my $mnemonic_patch = $git_patch; - $mnemonic_patch =~ s/ a\// $prefix->{'a'}\//g; - $mnemonic_patch =~ s/ b\// $prefix->{'b'}\//g; - - $out = ""; - foreach my $line (split('\n', $mnemonic_patch)) { - $out .= gitdiff2svndiff($line) . "\n"; - } - - ok($svn_patch eq $out, $title . " (" . $prefix->{'a'} . "," . $prefix->{'b'} . ")"); -} - diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/mergeChangeLogs.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/mergeChangeLogs.pl new file mode 100644 index 0000000..a226e43 --- /dev/null +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/mergeChangeLogs.pl @@ -0,0 +1,336 @@ +#!/usr/bin/perl +# +# Copyright (C) 2010 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Unit tests of VCSUtils::mergeChangeLogs(). + +use strict; + +use Test::Simple tests => 16; +use File::Temp qw(tempfile); +use VCSUtils; + +# Read contents of a file and return it. +sub readFile($) +{ + my ($fileName) = @_; + + local $/; + open(FH, "<", $fileName); + my $content = <FH>; + close(FH); + + return $content; +} + +# Write a temporary file and return the filename. +sub writeTempFile($$$) +{ + my ($name, $extension, $content) = @_; + + my ($FH, $fileName) = tempfile( + $name . "-XXXXXXXX", + DIR => ($ENV{'TMPDIR'} || "/tmp"), + UNLINK => 0, + ); + print $FH $content; + close $FH; + + if ($extension) { + my $newFileName = $fileName . $extension; + rename($fileName, $newFileName); + $fileName = $newFileName; + } + + return $fileName; +} + +# -------------------------------------------------------------------------------- + +{ + # New test + my $title = "mergeChangeLogs: traditional rejected patch success"; + + my $fileNewerContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileNewer = writeTempFile("file", "", $fileNewerContent); + + my $fileMineContent = <<'EOF'; +*************** +*** 1,3 **** + 2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. +--- 1,9 ---- ++ 2010-01-29 Oliver Hunt <oliver@apple.com> ++ ++ Reviewed by Darin Adler. ++ ++ JSC is failing to propagate anonymous slot count on some transitions ++ + 2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. +EOF + my $fileMine = writeTempFile("file", ".rej", $fileMineContent); + rename($fileMine, $fileNewer . ".rej"); + $fileMine = $fileNewer . ".rej"; + + my $fileOlderContent = $fileNewerContent; + my $fileOlder = writeTempFile("file", ".orig", $fileOlderContent); + rename($fileOlder, $fileNewer . ".orig"); + $fileOlder = $fileNewer . ".orig"; + + my $exitStatus = mergeChangeLogs($fileMine, $fileOlder, $fileNewer); + + # mergeChangeLogs() should return 1 since the patch succeeded. + ok($exitStatus == 1, "$title: should return 1 for success"); + + ok(readFile($fileMine) eq $fileMineContent, "$title: \$fileMine should be unchanged"); + ok(readFile($fileOlder) eq $fileOlderContent, "$title: \$fileOlder should be unchanged"); + + my $expectedContent = <<'EOF'; +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +EOF + $expectedContent .= $fileNewerContent; + ok(readFile($fileNewer) eq $expectedContent, "$title: \$fileNewer should be updated to include patch"); + + unlink($fileMine, $fileOlder, $fileNewer); +} + +# -------------------------------------------------------------------------------- + +{ + # New test + my $title = "mergeChangeLogs: traditional rejected patch failure"; + + my $fileNewerContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileNewer = writeTempFile("file", "", $fileNewerContent); + + my $fileMineContent = <<'EOF'; +*************** +*** 1,9 **** +- 2010-01-29 Oliver Hunt <oliver@apple.com> +- +- Reviewed by Darin Adler. +- +- JSC is failing to propagate anonymous slot count on some transitions +- + 2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. +--- 1,3 ---- + 2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. +EOF + my $fileMine = writeTempFile("file", ".rej", $fileMineContent); + rename($fileMine, $fileNewer . ".rej"); + $fileMine = $fileNewer . ".rej"; + + my $fileOlderContent = $fileNewerContent; + my $fileOlder = writeTempFile("file", ".orig", $fileOlderContent); + rename($fileOlder, $fileNewer . ".orig"); + $fileOlder = $fileNewer . ".orig"; + + my $exitStatus = mergeChangeLogs($fileMine, $fileOlder, $fileNewer); + + # mergeChangeLogs() should return 0 since the patch failed. + ok($exitStatus == 0, "$title: should return 0 for failure"); + + ok(readFile($fileMine) eq $fileMineContent, "$title: \$fileMine should be unchanged"); + ok(readFile($fileOlder) eq $fileOlderContent, "$title: \$fileOlder should be unchanged"); + ok(readFile($fileNewer) eq $fileNewerContent, "$title: \$fileNewer should be unchanged"); + + unlink($fileMine, $fileOlder, $fileNewer); +} + +# -------------------------------------------------------------------------------- + +{ + # New test + my $title = "mergeChangeLogs: patch succeeds"; + + my $fileMineContent = <<'EOF'; +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileMine = writeTempFile("fileMine", "", $fileMineContent); + + my $fileOlderContent = <<'EOF'; +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileOlder = writeTempFile("fileOlder", "", $fileOlderContent); + + my $fileNewerContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileNewer = writeTempFile("fileNewer", "", $fileNewerContent); + + my $exitStatus = mergeChangeLogs($fileMine, $fileOlder, $fileNewer); + + # mergeChangeLogs() should return 1 since the patch succeeded. + ok($exitStatus == 1, "$title: should return 1 for success"); + + ok(readFile($fileMine) eq $fileMineContent, "$title: \$fileMine should be unchanged"); + ok(readFile($fileOlder) eq $fileOlderContent, "$title: \$fileOlder should be unchanged"); + + my $expectedContent = <<'EOF'; +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +EOF + $expectedContent .= $fileNewerContent; + + ok(readFile($fileNewer) eq $expectedContent, "$title: \$fileNewer should be patched"); + + unlink($fileMine, $fileOlder, $fileNewer); +} + +# -------------------------------------------------------------------------------- + +{ + # New test + my $title = "mergeChangeLogs: patch fails"; + + my $fileMineContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileMine = writeTempFile("fileMine", "", $fileMineContent); + + my $fileOlderContent = <<'EOF'; +2010-01-29 Mark Rowe <mrowe@apple.com> + + Fix the Mac build. + + Disable ENABLE_INDEXED_DATABASE since it is "completely non-functional". + +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileOlder = writeTempFile("fileOlder", "", $fileOlderContent); + + my $fileNewerContent = <<'EOF'; +2010-01-29 Oliver Hunt <oliver@apple.com> + + Reviewed by Darin Adler. + + JSC is failing to propagate anonymous slot count on some transitions + +2010-01-29 Simon Hausmann <simon.hausmann@nokia.com> + + Rubber-stamped by Maciej Stachowiak. + + Fix the ARM build. +EOF + my $fileNewer = writeTempFile("fileNewer", "", $fileNewerContent); + + my $exitStatus = mergeChangeLogs($fileMine, $fileOlder, $fileNewer); + + # mergeChangeLogs() should return a non-zero exit status since the patch failed. + ok($exitStatus == 0, "$title: return non-zero exit status for failure"); + + ok(readFile($fileMine) eq $fileMineContent, "$title: \$fileMine should be unchanged"); + ok(readFile($fileOlder) eq $fileOlderContent, "$title: \$fileOlder should be unchanged"); + + # $fileNewer should still exist unchanged because the patch failed + ok(readFile($fileNewer) eq $fileNewerContent, "$title: \$fileNewer should be unchanged"); + + unlink($fileMine, $fileOlder, $fileNewer); +} + +# -------------------------------------------------------------------------------- + diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiff.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiff.pl index 2507d2d..245916c 100644 --- a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiff.pl +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiff.pl @@ -30,13 +30,6 @@ use warnings; use Test::More; use VCSUtils; -my @diffHashRefKeys = ( # The $diffHashRef keys to check. - "copiedFromPath", - "indexPath", - "sourceRevision", - "svnConvertedText", -); - # The array of test cases. my @testCaseHashRefs = ( { @@ -53,7 +46,8 @@ Index: Makefile all: END - # Header keys to check + expectedReturn => [ +[{ svnConvertedText => <<'END', # Same as input text Index: Makefile =================================================================== @@ -65,12 +59,48 @@ Index: Makefile all: END - copiedFromPath => undef, indexPath => "Makefile", + isSvn => 1, sourceRevision => "53052", - # Other values to check - lastReadLine => undef, - nextLine => undef, +}], +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "SVN: binary file (isBinary true)", + inputText => <<'END', +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + +Property changes on: test_file.swf +___________________________________________________________________ +Name: svn:mime-type + + application/octet-stream + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', # Same as input text +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + indexPath => "test_file.swf", + isBinary => 1, + isSvn => 1, +}], +undef], + expectedNextLine => undef, }, { # New test @@ -89,7 +119,8 @@ Index: Makefile all: END - # Header keys to check + expectedReturn => [ +[{ svnConvertedText => <<'END', # Same as input text LEADING JUNK @@ -104,12 +135,12 @@ Index: Makefile all: END - copiedFromPath => undef, indexPath => "Makefile", + isSvn => 1, sourceRevision => "53052", - # Other values to check - lastReadLine => undef, - nextLine => undef, +}], +undef], + expectedNextLine => undef, }, { # New test @@ -122,21 +153,14 @@ Index: Makefile_new @@ -0,0 +1,1 @@ +MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKitTools END - # Header keys to check - svnConvertedText => <<'END', # Same as input text -Index: Makefile_new -=================================================================== ---- Makefile_new (revision 53131) (from Makefile:53131) -+++ Makefile_new (working copy) -@@ -0,0 +1,1 @@ -+MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKitTools -END + expectedReturn => [ +[{ copiedFromPath => "Makefile", indexPath => "Makefile_new", sourceRevision => "53131", - # Other values to check - lastReadLine => undef, - nextLine => undef, +}], +undef], + expectedNextLine => undef, }, { # New test @@ -152,7 +176,8 @@ Index: Makefile_new =================================================================== --- Makefile_new (revision 53131) (from Makefile:53131) END - # Header keys to check + expectedReturn => [ +[{ svnConvertedText => <<'END', Index: Makefile =================================================================== @@ -161,12 +186,12 @@ Index: Makefile @@ -1,1 +0,0 @@ -MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKitTools END - copiedFromPath => undef, indexPath => "Makefile", + isSvn => 1, sourceRevision => "53131", - # Other values to check - lastReadLine => "Index: Makefile_new\n", - nextLine => "===================================================================\n", +}], +"Index: Makefile_new\n"], + expectedNextLine => "===================================================================\n", }, { # New test @@ -184,7 +209,8 @@ index f5d5e74..3b6aa92 100644 +++ b/Makefile @@ -1,1 1,1 @@ public: END - # Header keys to check + expectedReturn => [ +[{ svnConvertedText => <<'END', # Same as input text Index: Makefile =================================================================== @@ -198,13 +224,411 @@ index f5d5e74..3b6aa92 100644 +++ b/Makefile @@ -1,1 1,1 @@ public: END - copiedFromPath => undef, indexPath => "Makefile", + isSvn => 1, sourceRevision => "53131", - # Other values to check - lastReadLine => undef, - nextLine => undef, +}], +undef], + expectedNextLine => undef, +}, +#### +# Property Changes: Simple +## +{ + # New test + diffName => "SVN: file change diff with property change diff", + inputText => <<'END', +Index: Makefile +=================================================================== +--- Makefile (revision 60021) ++++ Makefile (working copy) +@@ -1,3 +1,4 @@ ++ + MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + + all: + +Property changes on: Makefile +___________________________________________________________________ +Name: svn:executable + + * +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', # Same as input text +Index: Makefile +=================================================================== +--- Makefile (revision 60021) ++++ Makefile (working copy) +@@ -1,3 +1,4 @@ ++ + MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + + all: + +END + executableBitDelta => 1, + indexPath => "Makefile", + isSvn => 1, + sourceRevision => "60021", +}], +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "SVN: file change diff, followed by property change diff on different file", + inputText => <<'END', +Index: Makefile +=================================================================== +--- Makefile (revision 60021) ++++ Makefile (working copy) +@@ -1,3 +1,4 @@ ++ + MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + + all: + +Property changes on: Makefile.shared +___________________________________________________________________ +Name: svn:executable + + * +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', # Same as input text +Index: Makefile +=================================================================== +--- Makefile (revision 60021) ++++ Makefile (working copy) +@@ -1,3 +1,4 @@ ++ + MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + + all: + +END + indexPath => "Makefile", + isSvn => 1, + sourceRevision => "60021", +}], +"Property changes on: Makefile.shared\n"], + expectedNextLine => "___________________________________________________________________\n", +}, +{ + # New test + diffName => "SVN: property diff, followed by file change diff", + inputText => <<'END', +Property changes on: Makefile +___________________________________________________________________ +Deleted: svn:executable + - * + +Index: Makefile.shared +=================================================================== +--- Makefile.shared (revision 60021) ++++ Makefile.shared (working copy) +@@ -1,3 +1,4 @@ ++ +SCRIPTS_PATH ?= ../WebKitTools/Scripts +XCODE_OPTIONS = `perl -I$(SCRIPTS_PATH) -Mwebkitdirs -e 'print XcodeOptionString()'` $(ARGS) +END + expectedReturn => [ +[{ + executableBitDelta => -1, + indexPath => "Makefile", + isSvn => 1, +}], +"Index: Makefile.shared\n"], + expectedNextLine => "===================================================================\n", +}, +{ + # New test + diffName => "SVN: copied file with property change", + inputText => <<'END', +Index: NMakefile +=================================================================== +--- NMakefile (revision 60021) (from Makefile:60021) ++++ NMakefile (working copy) +@@ -0,0 +1,1 @@ ++MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + +Property changes on: NMakefile +___________________________________________________________________ +Added: svn:executable + + * +END + expectedReturn => [ +[{ + copiedFromPath => "Makefile", + executableBitDelta => 1, + indexPath => "NMakefile", + sourceRevision => "60021", +}], +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "SVN: two consecutive property diffs", + inputText => <<'END', +Property changes on: Makefile +___________________________________________________________________ +Added: svn:executable + + * + + +Property changes on: Makefile.shared +___________________________________________________________________ +Added: svn:executable + + * +END + expectedReturn => [ +[{ + executableBitDelta => 1, + indexPath => "Makefile", + isSvn => 1, +}], +"Property changes on: Makefile.shared\n"], + expectedNextLine => "___________________________________________________________________\n", +}, +#### +# Property Changes: Binary files +## +{ + # New test + diffName => "SVN: binary file with executable bit change", + inputText => <<'END', +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + +Property changes on: test_file.swf +___________________________________________________________________ +Name: svn:mime-type + + application/octet-stream +Name: svn:executable + + * + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', # Same as input text +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + executableBitDelta => 1, + indexPath => "test_file.swf", + isBinary => 1, + isSvn => 1, +}], +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "SVN: binary file followed by property change on different file", + inputText => <<'END', +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + +Property changes on: test_file.swf +___________________________________________________________________ +Name: svn:mime-type + + application/octet-stream + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== + +Property changes on: Makefile +___________________________________________________________________ +Added: svn:executable + + * +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', # Same as input text +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== + +END + indexPath => "test_file.swf", + isBinary => 1, + isSvn => 1, +}], +"Property changes on: Makefile\n"], + expectedNextLine => "___________________________________________________________________\n", +}, +{ + # New test + diffName => "SVN: binary file followed by file change on different file", + inputText => <<'END', +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + +Property changes on: test_file.swf +___________________________________________________________________ +Name: svn:mime-type + + application/octet-stream + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== + +Index: Makefile +=================================================================== +--- Makefile (revision 60021) ++++ Makefile (working copy) +@@ -1,3 +1,4 @@ ++ + MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + + all: +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', # Same as input text +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== + +END + indexPath => "test_file.swf", + isBinary => 1, + isSvn => 1, +}], +"Index: Makefile\n"], + expectedNextLine => "===================================================================\n", +}, +#### +# Property Changes: File change with property change +## +{ + # New test + diffName => "SVN: file change diff with property change, followed by property change diff", + inputText => <<'END', +Index: Makefile +=================================================================== +--- Makefile (revision 60021) ++++ Makefile (working copy) +@@ -1,3 +1,4 @@ ++ + MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + + all: + +Property changes on: Makefile +___________________________________________________________________ +Added: svn:executable + + * + + +Property changes on: Makefile.shared +___________________________________________________________________ +Deleted: svn:executable + - * +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', # Same as input text +Index: Makefile +=================================================================== +--- Makefile (revision 60021) ++++ Makefile (working copy) +@@ -1,3 +1,4 @@ ++ + MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + + all: + + + +END + executableBitDelta => 1, + indexPath => "Makefile", + isSvn => 1, + sourceRevision => "60021", +}], +"Property changes on: Makefile.shared\n"], + expectedNextLine => "___________________________________________________________________\n", +}, +{ + # New test + diffName => "SVN: file change diff with property change, followed by file change diff", + inputText => <<'END', +Index: Makefile +=================================================================== +--- Makefile (revision 60021) ++++ Makefile (working copy) +@@ -1,3 +1,4 @@ ++ + MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + + all: + +Property changes on: Makefile +___________________________________________________________________ +Name: svn:executable + - * + +Index: Makefile.shared +=================================================================== +--- Makefile.shared (revision 60021) ++++ Makefile.shared (working copy) +@@ -1,3 +1,4 @@ ++ +SCRIPTS_PATH ?= ../WebKitTools/Scripts +XCODE_OPTIONS = `perl -I$(SCRIPTS_PATH) -Mwebkitdirs -e 'print XcodeOptionString()'` $(ARGS) +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', # Same as input text +Index: Makefile +=================================================================== +--- Makefile (revision 60021) ++++ Makefile (working copy) +@@ -1,3 +1,4 @@ ++ + MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKit2 WebKitTools + + all: + + +END + executableBitDelta => -1, + indexPath => "Makefile", + isSvn => 1, + sourceRevision => "60021", +}], +"Index: Makefile.shared\n"], + expectedNextLine => "===================================================================\n", }, +#### +# Git test cases +## { # New test diffName => "Git: simple", @@ -215,20 +639,82 @@ index f5d5e74..3b6aa92 100644 +++ b/Makefile @@ -1,1 1,1 @@ public: END - # Header keys to check + expectedReturn => [ +[{ svnConvertedText => <<'END', Index: Makefile -=================================================================== +index f5d5e74..3b6aa92 100644 --- Makefile +++ Makefile @@ -1,1 1,1 @@ public: END - copiedFromPath => undef, indexPath => "Makefile", - sourceRevision => undef, - # Other values to check - lastReadLine => undef, - nextLine => undef, + isGit => 1, +}], +undef], + expectedNextLine => undef, +}, +{ # New test + diffName => "Git: new file", + inputText => <<'END', +diff --git a/foo.h b/foo.h +new file mode 100644 +index 0000000..3c9f114 +--- /dev/null ++++ b/foo.h +@@ -0,0 +1,34 @@ ++<html> +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', +Index: foo.h +new file mode 100644 +index 0000000..3c9f114 +--- foo.h ++++ foo.h +@@ -0,0 +1,34 @@ ++<html> +END + indexPath => "foo.h", + isGit => 1, + isNew => 1, +}], +"diff --git a/bar b/bar\n"], + expectedNextLine => "index d45dd40..3494526 100644\n", +}, +{ # New test + diffName => "Git: file deletion", + inputText => <<'END', +diff --git a/foo b/foo +deleted file mode 100644 +index 1e50d1d..0000000 +--- a/foo ++++ /dev/null +@@ -1,1 +0,0 @@ +-line1 +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +[{ + svnConvertedText => <<'END', +Index: foo +deleted file mode 100644 +index 1e50d1d..0000000 +--- foo ++++ foo +@@ -1,1 +0,0 @@ +-line1 +END + indexPath => "foo", + isDeletion => 1, + isGit => 1, +}], +"diff --git a/bar b/bar\n"], + expectedNextLine => "index d45dd40..3494526 100644\n", }, { # New test @@ -243,10 +729,11 @@ Index: Makefile_new =================================================================== --- Makefile_new (revision 53131) (from Makefile:53131) END - # Header keys to check + expectedReturn => [ +[{ svnConvertedText => <<'END', Index: Makefile -=================================================================== +index f5d5e74..3b6aa92 100644 --- Makefile +++ Makefile @@ -1,1 1,1 @@ public: @@ -254,75 +741,139 @@ Index: Makefile_new =================================================================== --- Makefile_new (revision 53131) (from Makefile:53131) END - copiedFromPath => undef, indexPath => "Makefile", - sourceRevision => undef, - # Other values to check - lastReadLine => undef, - nextLine => undef, + isGit => 1, +}], +undef], + expectedNextLine => undef, +}, +#### +# Git test cases: file moves (multiple return values) +## +{ + diffName => "Git: rename (with similarity index 100%)", + inputText => <<'END', +diff --git a/foo b/foo_new +similarity index 100% +rename from foo +rename to foo_new +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +[{ + indexPath => "foo", + isDeletion => 1, +}, +{ + copiedFromPath => "foo", + indexPath => "foo_new", +}], +"diff --git a/bar b/bar\n"], + expectedNextLine => "index d45dd40..3494526 100644\n", +}, +{ + diffName => "rename (with similarity index < 100%)", + inputText => <<'END', +diff --git a/foo b/foo_new +similarity index 99% +rename from foo +rename to foo_new +index 1e50d1d..1459d21 100644 +--- a/foo ++++ b/foo_new +@@ -15,3 +15,4 @@ release r deployment dep deploy: + line1 + line2 + line3 ++line4 +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +[{ + indexPath => "foo", + isDeletion => 1, +}, +{ + copiedFromPath => "foo", + indexPath => "foo_new", +}, +{ + indexPath => "foo_new", + isGit => 1, + svnConvertedText => <<'END', +Index: foo_new +similarity index 99% +rename from foo +rename to foo_new +index 1e50d1d..1459d21 100644 +--- foo_new ++++ foo_new +@@ -15,3 +15,4 @@ release r deployment dep deploy: + line1 + line2 + line3 ++line4 +END +}], +"diff --git a/bar b/bar\n"], + expectedNextLine => "index d45dd40..3494526 100644\n", +}, +{ + diffName => "rename (with executable bit change)", + inputText => <<'END', +diff --git a/foo b/foo_new +old mode 100644 +new mode 100755 +similarity index 100% +rename from foo +rename to foo_new +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +[{ + indexPath => "foo", + isDeletion => 1, +}, +{ + copiedFromPath => "foo", + indexPath => "foo_new", +}, +{ + executableBitDelta => 1, + indexPath => "foo_new", + isGit => 1, + svnConvertedText => <<'END', +Index: foo_new +old mode 100644 +new mode 100755 +similarity index 100% +rename from foo +rename to foo_new +END +}], +"diff --git a/bar b/bar\n"], + expectedNextLine => "index d45dd40..3494526 100644\n", }, ); -# Return the arguments for each assertion per test case. -# -# In particular, the number of assertions per test case is the length -# of the return value of this subroutine on a sample input. -# -# Returns @assertionArgsArrayRefs: -# $assertionArgsArrayRef: A reference to an array of parameters to pass -# to each call to is(). The parameters are-- -# $got: The value obtained -# $expected: The expected value -# $testName: The name of the test -sub testParseDiffAssertionArgs($) -{ - my ($testCaseHashRef) = @_; +my $testCasesCount = @testCaseHashRefs; +plan(tests => 2 * $testCasesCount); # Total number of assertions. - my $fileHandle; - open($fileHandle, "<", \$testCaseHashRef->{inputText}); +foreach my $testCase (@testCaseHashRefs) { + my $testNameStart = "parseDiff(): $testCase->{diffName}: comparing"; + my $fileHandle; + open($fileHandle, "<", \$testCase->{inputText}); my $line = <$fileHandle>; - my ($diffHashRef, $lastReadLine) = VCSUtils::parseDiff($fileHandle, $line); - - my $testNameStart = "parseDiff(): [$testCaseHashRef->{diffName}] "; - - my @assertionArgsArrayRefs; # Return value - my @assertionArgs; - foreach my $diffHashRefKey (@diffHashRefKeys) { - my $testName = "${testNameStart}key=\"$diffHashRefKey\""; - @assertionArgs = ($diffHashRef->{$diffHashRefKey}, $testCaseHashRef->{$diffHashRefKey}, $testName); - push(@assertionArgsArrayRefs, \@assertionArgs); - } - - @assertionArgs = ($lastReadLine, $testCaseHashRef->{lastReadLine}, "${testNameStart}lastReadLine"); - push(@assertionArgsArrayRefs, \@assertionArgs); - - my $nextLine = <$fileHandle>; - @assertionArgs = ($nextLine, $testCaseHashRef->{nextLine}, "${testNameStart}nextLine"); - push(@assertionArgsArrayRefs, \@assertionArgs); - - return @assertionArgsArrayRefs; -} - -# Test parseDiff() for the given test case. -sub testParseDiff($) -{ - my ($testCaseHashRef) = @_; - - my @assertionArgsArrayRefs = testParseDiffAssertionArgs($testCaseHashRef); - - foreach my $arrayRef (@assertionArgsArrayRefs) { - # The parameters are -- is($got, $expected, $testName). - is($arrayRef->[0], $arrayRef->[1], $arrayRef->[2]); - } -} - -# Count the number of assertions per test case, using a sample test case. -my $assertionCount = testParseDiffAssertionArgs($testCaseHashRefs[0]); + my @got = VCSUtils::parseDiff($fileHandle, $line); + my $expectedReturn = $testCase->{expectedReturn}; -plan(tests => @testCaseHashRefs * $assertionCount); # Total number of tests + is_deeply(\@got, $expectedReturn, "$testNameStart return value."); -foreach my $testCaseHashRef (@testCaseHashRefs) { - testParseDiff($testCaseHashRef); + my $gotNextLine = <$fileHandle>; + is($gotNextLine, $testCase->{expectedNextLine}, "$testNameStart next read line."); } diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl index a7a3c26..8c20f65 100644 --- a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl @@ -36,68 +36,21 @@ use warnings; use Test::More; use VCSUtils; -my @diffHeaderHashRefKeys = ( # The $diffHeaderHashRef keys to check. - "copiedFromPath", - "indexPath", - "sourceRevision", - "svnConvertedText", -); - -# The array of test cases. +# The unit tests for parseGitDiffHeader() and parseSvnDiffHeader() +# already thoroughly test parsing each format. +# +# For parseDiffHeader(), it should suffice to verify that -- (1) for each +# format, the method can return non-trivial values back for each key +# supported by that format (e.g. "sourceRevision" for SVN), (2) the method +# correctly sets default values when specific key-values are not set +# (e.g. undef for "sourceRevision" for Git), and (3) key-values unique to +# this method are set correctly (e.g. "scmFormat"). my @testCaseHashRefs = ( -{ - # New test - diffName => "SVN: simple", - inputText => <<'END', -Index: WebKitTools/Scripts/VCSUtils.pm -=================================================================== ---- WebKitTools/Scripts/VCSUtils.pm (revision 53004) -+++ WebKitTools/Scripts/VCSUtils.pm (working copy) -@@ -32,6 +32,7 @@ use strict; - use warnings; -END - # Header keys to check - svnConvertedText => <<'END', -Index: WebKitTools/Scripts/VCSUtils.pm -=================================================================== ---- WebKitTools/Scripts/VCSUtils.pm (revision 53004) -+++ WebKitTools/Scripts/VCSUtils.pm (working copy) -END - copiedFromPath => undef, - indexPath => "WebKitTools/Scripts/VCSUtils.pm", - sourceRevision => "53004", - # Other values to check - lastReadLine => "@@ -32,6 +32,7 @@ use strict;\n", - nextLine => " use warnings;\n", -}, -{ - # New test - diffName => "SVN: new file", - inputText => <<'END', -Index: WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl -=================================================================== ---- WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl (revision 0) -+++ WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl (revision 0) -@@ -0,0 +1,262 @@ -+#!/usr/bin/perl -w -END - # Header keys to check - svnConvertedText => <<'END', -Index: WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl -=================================================================== ---- WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl (revision 0) -+++ WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl (revision 0) -END - copiedFromPath => undef, - indexPath => "WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl", - sourceRevision => undef, - # Other values to check - lastReadLine => "@@ -0,0 +1,262 @@\n", - nextLine => "+#!/usr/bin/perl -w\n", -}, -{ - # New test - diffName => "SVN: copy", +#### +# SVN test cases +## +{ # New test + diffName => "SVN: non-trivial copiedFromPath and sourceRevision values", inputText => <<'END', Index: index_path.py =================================================================== @@ -106,7 +59,8 @@ Index: index_path.py @@ -0,0 +1,7 @@ +# Python file... END - # Header keys to check + expectedReturn => [ +{ svnConvertedText => <<'END', Index: index_path.py =================================================================== @@ -115,174 +69,53 @@ Index: index_path.py END copiedFromPath => "copied_from_path.py", indexPath => "index_path.py", + isSvn => 1, sourceRevision => 53048, - # Other values to check - lastReadLine => "@@ -0,0 +1,7 @@\n", - nextLine => "+# Python file...\n", }, -{ - # New test - diffName => "SVN: \\r\\n lines", - inputText => <<END, # No single quotes to allow interpolation of "\r" -Index: index_path.py\r -===================================================================\r ---- index_path.py (revision 53048) (from copied_from_path.py:53048)\r -+++ index_path.py (working copy)\r -@@ -0,0 +1,7 @@\r -+# Python file...\r -END - # Header keys to check - svnConvertedText => <<END, # No single quotes to allow interpolation of "\r" -Index: index_path.py\r -===================================================================\r ---- index_path.py (revision 53048) (from copied_from_path.py:53048)\r -+++ index_path.py (working copy)\r -END - copiedFromPath => "copied_from_path.py", - indexPath => "index_path.py", - sourceRevision => 53048, - # Other values to check - lastReadLine => "@@ -0,0 +1,7 @@\r\n", - nextLine => "+# Python file...\r\n", +"@@ -0,0 +1,7 @@\n"], + expectedNextLine => "+# Python file...\n", }, -{ - # New test - diffName => "SVN: path corrections", +#### +# Git test cases +## +{ # New test case + diffName => "Git: Non-zero executable bit", inputText => <<'END', -Index: index_path.py -=================================================================== ---- bad_path (revision 53048) (from copied_from_path.py:53048) -+++ bad_path (working copy) -@@ -0,0 +1,7 @@ -+# Python file... -END - # Header keys to check - svnConvertedText => <<'END', -Index: index_path.py -=================================================================== ---- index_path.py (revision 53048) (from copied_from_path.py:53048) -+++ index_path.py (working copy) +diff --git a/foo.exe b/foo.exe +old mode 100644 +new mode 100755 END - copiedFromPath => "copied_from_path.py", - indexPath => "index_path.py", - sourceRevision => 53048, - # Other values to check - lastReadLine => "@@ -0,0 +1,7 @@\n", - nextLine => "+# Python file...\n", -}, + expectedReturn => [ { - # New test - diffName => "Git: simple", - inputText => <<'END', -diff --git a/WebCore/rendering/style/StyleFlexibleBoxData.h b/WebCore/rendering/style/StyleFlexibleBoxData.h -index f5d5e74..3b6aa92 100644 ---- a/WebCore/rendering/style/StyleFlexibleBoxData.h -+++ b/WebCore/rendering/style/StyleFlexibleBoxData.h -@@ -47,7 +47,6 @@ public: -END - # Header keys to check svnConvertedText => <<'END', -Index: WebCore/rendering/style/StyleFlexibleBoxData.h -=================================================================== ---- WebCore/rendering/style/StyleFlexibleBoxData.h -+++ WebCore/rendering/style/StyleFlexibleBoxData.h +Index: foo.exe +old mode 100644 +new mode 100755 END - copiedFromPath => undef, - indexPath => "WebCore/rendering/style/StyleFlexibleBoxData.h", - sourceRevision => undef, - # Other values to check - lastReadLine => "@@ -47,7 +47,6 @@ public:\n", - nextLine => undef, + executableBitDelta => 1, + indexPath => "foo.exe", + isGit => 1, }, -{ - # New test - diffName => "Git: unrecognized lines", - inputText => <<'END', -diff --git a/LayoutTests/http/tests/security/listener/xss-inactive-closure.html b/LayoutTests/http/tests/security/listener/xss-inactive-closure.html -new file mode 100644 -index 0000000..3c9f114 ---- /dev/null -+++ b/LayoutTests/http/tests/security/listener/xss-inactive-closure.html -@@ -0,0 +1,34 @@ -+<html> -END - # Header keys to check - svnConvertedText => <<'END', -Index: LayoutTests/http/tests/security/listener/xss-inactive-closure.html -=================================================================== ---- LayoutTests/http/tests/security/listener/xss-inactive-closure.html -+++ LayoutTests/http/tests/security/listener/xss-inactive-closure.html -END - copiedFromPath => undef, - indexPath => "LayoutTests/http/tests/security/listener/xss-inactive-closure.html", - sourceRevision => undef, - # Other values to check - lastReadLine => "@@ -0,0 +1,34 @@\n", - nextLine => "+<html>\n", +undef], + expectedNextLine => undef, }, ); -# Return the arguments for each assertion per test case. -# -# In particular, the number of assertions per test case is the length -# of the return value of this subroutine on a sample input. -# -# Returns @assertionArgsArrayRefs: -# $assertionArgsArrayRef: A reference to an array of parameters to pass -# to each call to is(). The parameters are-- -# $got: The value obtained -# $expected: The expected value -# $testName: The name of the test -sub testParseDiffHeaderAssertionArgs($) -{ - my ($testCaseHashRef) = @_; +my $testCasesCount = @testCaseHashRefs; +plan(tests => 2 * $testCasesCount); # Total number of assertions. - my $fileHandle; - open($fileHandle, "<", \$testCaseHashRef->{inputText}); +foreach my $testCase (@testCaseHashRefs) { + my $testNameStart = "parseDiffHeader(): $testCase->{diffName}: comparing"; + my $fileHandle; + open($fileHandle, "<", \$testCase->{inputText}); my $line = <$fileHandle>; - my ($headerHashRef, $lastReadLine) = VCSUtils::parseDiffHeader($fileHandle, $line); - - my $testNameStart = "parseDiffHeader(): [$testCaseHashRef->{diffName}] "; - - my @assertionArgsArrayRefs; # Return value - my @assertionArgs; - foreach my $diffHeaderHashRefKey (@diffHeaderHashRefKeys) { - my $testName = "${testNameStart}key=\"$diffHeaderHashRefKey\""; - @assertionArgs = ($headerHashRef->{$diffHeaderHashRefKey}, $testCaseHashRef->{$diffHeaderHashRefKey}, $testName); - push(@assertionArgsArrayRefs, \@assertionArgs); - } - - @assertionArgs = ($lastReadLine, $testCaseHashRef->{lastReadLine}, "${testNameStart}lastReadLine"); - push(@assertionArgsArrayRefs, \@assertionArgs); - - my $nextLine = <$fileHandle>; - @assertionArgs = ($nextLine, $testCaseHashRef->{nextLine}, "${testNameStart}nextLine"); - push(@assertionArgsArrayRefs, \@assertionArgs); - - return @assertionArgsArrayRefs; -} - -# Test parseDiffHeader() for the given test case. -sub testParseDiffHeader($) -{ - my ($testCaseHashRef) = @_; - - my @assertionArgsArrayRefs = testParseDiffHeaderAssertionArgs($testCaseHashRef); - - foreach my $arrayRef (@assertionArgsArrayRefs) { - # The parameters are -- is($got, $expected, $testName). - is($arrayRef->[0], $arrayRef->[1], $arrayRef->[2]); - } -} - -# Count the number of assertions per test case to calculate the total number -# of Test::More tests. We could have used any test case for the count. -my $assertionCount = testParseDiffHeaderAssertionArgs($testCaseHashRefs[0]); + my @got = VCSUtils::parseDiffHeader($fileHandle, $line); + my $expectedReturn = $testCase->{expectedReturn}; -plan(tests => @testCaseHashRefs * $assertionCount); # Total number of tests + is_deeply(\@got, $expectedReturn, "$testNameStart return value."); -foreach my $testCaseHashRef (@testCaseHashRefs) { - testParseDiffHeader($testCaseHashRef); + my $gotNextLine = <$fileHandle>; + is($gotNextLine, $testCase->{expectedNextLine}, "$testNameStart next read line."); } diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseGitDiffHeader.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseGitDiffHeader.pl new file mode 100644 index 0000000..bc0d4d4 --- /dev/null +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseGitDiffHeader.pl @@ -0,0 +1,494 @@ +#!/usr/bin/perl -w +# +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Unit tests of parseGitDiffHeader(). + +use strict; +use warnings; + +use Test::More; +use VCSUtils; + +# The array of test cases. +my @testCaseHashRefs = ( +{ # New test + diffName => "Modified file", + inputText => <<'END', +diff --git a/foo.h b/foo.h +index f5d5e74..3b6aa92 100644 +--- a/foo.h ++++ b/foo.h +@@ -1 +1 @@ +-file contents ++new file contents +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo.h +index f5d5e74..3b6aa92 100644 +--- foo.h ++++ foo.h +END + indexPath => "foo.h", +}, +"@@ -1 +1 @@\n"], + expectedNextLine => "-file contents\n", +}, +{ # New test + diffName => "new file", + inputText => <<'END', +diff --git a/foo.h b/foo.h +new file mode 100644 +index 0000000..3c9f114 +--- /dev/null ++++ b/foo.h +@@ -0,0 +1,34 @@ ++<html> +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo.h +new file mode 100644 +index 0000000..3c9f114 +--- foo.h ++++ foo.h +END + indexPath => "foo.h", + isNew => 1, +}, +"@@ -0,0 +1,34 @@\n"], + expectedNextLine => "+<html>\n", +}, +{ # New test + diffName => "file deletion", + inputText => <<'END', +diff --git a/foo b/foo +deleted file mode 100644 +index 1e50d1d..0000000 +--- a/foo ++++ /dev/null +@@ -1,1 +0,0 @@ +-line1 +diff --git a/configure.ac b/configure.ac +index d45dd40..3494526 100644 +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo +deleted file mode 100644 +index 1e50d1d..0000000 +--- foo ++++ foo +END + indexPath => "foo", + isDeletion => 1, +}, +"@@ -1,1 +0,0 @@\n"], + expectedNextLine => "-line1\n", +}, +{ # New test + diffName => "using --no-prefix", + inputText => <<'END', +diff --git foo.h foo.h +index c925780..9e65c43 100644 +--- foo.h ++++ foo.h +@@ -1,3 +1,17 @@ ++contents +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo.h +index c925780..9e65c43 100644 +--- foo.h ++++ foo.h +END + indexPath => "foo.h", +}, +"@@ -1,3 +1,17 @@\n"], + expectedNextLine => "+contents\n", +}, +#### +# Copy operations +## +{ # New test + diffName => "copy (with similarity index 100%)", + inputText => <<'END', +diff --git a/foo b/foo_new +similarity index 100% +copy from foo +copy to foo_new +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo_new +similarity index 100% +copy from foo +copy to foo_new +END + copiedFromPath => "foo", + indexPath => "foo_new", +}, +"diff --git a/bar b/bar\n"], + expectedNextLine => "index d45dd40..3494526 100644\n", +}, +{ # New test + diffName => "copy (with similarity index < 100%)", + inputText => <<'END', +diff --git a/foo b/foo_new +similarity index 99% +copy from foo +copy to foo_new +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo_new +similarity index 99% +copy from foo +copy to foo_new +END + copiedFromPath => "foo", + indexPath => "foo_new", + isCopyWithChanges => 1, +}, +"diff --git a/bar b/bar\n"], + expectedNextLine => "index d45dd40..3494526 100644\n", +}, +{ # New test + diffName => "rename (with similarity index 100%)", + inputText => <<'END', +diff --git a/foo b/foo_new +similarity index 100% +rename from foo +rename to foo_new +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo_new +similarity index 100% +rename from foo +rename to foo_new +END + copiedFromPath => "foo", + indexPath => "foo_new", + shouldDeleteSource => 1, +}, +"diff --git a/bar b/bar\n"], + expectedNextLine => "index d45dd40..3494526 100644\n", +}, +{ # New test + diffName => "rename (with similarity index < 100%)", + inputText => <<'END', +diff --git a/foo b/foo_new +similarity index 99% +rename from foo +rename to foo_new +index 1e50d1d..1459d21 100644 +--- a/foo ++++ b/foo_new +@@ -15,3 +15,4 @@ release r deployment dep deploy: + line1 + line2 + line3 ++line4 +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo_new +similarity index 99% +rename from foo +rename to foo_new +index 1e50d1d..1459d21 100644 +--- foo_new ++++ foo_new +END + copiedFromPath => "foo", + indexPath => "foo_new", + isCopyWithChanges => 1, + shouldDeleteSource => 1, +}, +"@@ -15,3 +15,4 @@ release r deployment dep deploy:\n"], + expectedNextLine => " line1\n", +}, +{ # New test + diffName => "rename (with executable bit change)", + inputText => <<'END', +diff --git a/foo b/foo_new +old mode 100644 +new mode 100755 +similarity index 100% +rename from foo +rename to foo_new +diff --git a/bar b/bar +index d45dd40..3494526 100644 +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo_new +old mode 100644 +new mode 100755 +similarity index 100% +rename from foo +rename to foo_new +END + copiedFromPath => "foo", + executableBitDelta => 1, + indexPath => "foo_new", + isCopyWithChanges => 1, + shouldDeleteSource => 1, +}, +"diff --git a/bar b/bar\n"], + expectedNextLine => "index d45dd40..3494526 100644\n", +}, +#### +# Binary file test cases +## +{ + # New test case + diffName => "New binary file", + inputText => <<'END', +diff --git a/foo.gif b/foo.gif +new file mode 100644 +index 0000000000000000000000000000000000000000..64a9532e7794fcd791f6f12157406d9060151690 +GIT binary patch +literal 7 +OcmYex&reDa;sO8*F9L)B + +literal 0 +HcmV?d00001 + +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo.gif +new file mode 100644 +index 0000000000000000000000000000000000000000..64a9532e7794fcd791f6f12157406d9060151690 +GIT binary patch +END + indexPath => "foo.gif", + isBinary => 1, + isNew => 1, +}, +"literal 7\n"], + expectedNextLine => "OcmYex&reDa;sO8*F9L)B\n", +}, +{ + # New test case + diffName => "Deleted binary file", + inputText => <<'END', +diff --git a/foo.gif b/foo.gif +deleted file mode 100644 +index 323fae0..0000000 +GIT binary patch +literal 0 +HcmV?d00001 + +literal 7 +OcmYex&reDa;sO8*F9L)B + +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo.gif +deleted file mode 100644 +index 323fae0..0000000 +GIT binary patch +END + indexPath => "foo.gif", + isBinary => 1, + isDeletion => 1, +}, +"literal 0\n"], + expectedNextLine => "HcmV?d00001\n", +}, +#### +# Executable bit test cases +## +{ + # New test case + diffName => "Modified executable file", + inputText => <<'END', +diff --git a/foo b/foo +index d03e242..435ad3a 100755 +--- a/foo ++++ b/foo +@@ -1 +1 @@ +-file contents ++new file contents + +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo +index d03e242..435ad3a 100755 +--- foo ++++ foo +END + indexPath => "foo", +}, +"@@ -1 +1 @@\n"], + expectedNextLine => "-file contents\n", +}, +{ + # New test case + diffName => "Making file executable (last diff)", + inputText => <<'END', +diff --git a/foo.exe b/foo.exe +old mode 100644 +new mode 100755 +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo.exe +old mode 100644 +new mode 100755 +END + executableBitDelta => 1, + indexPath => "foo.exe", +}, +undef], + expectedNextLine => undef, +}, +{ + # New test case + diffName => "Making file executable (not last diff)", + inputText => <<'END', +diff --git a/foo.exe b/foo.exe +old mode 100644 +new mode 100755 +diff --git a/another_file.txt b/another_file.txt +index d03e242..435ad3a 100755 +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo.exe +old mode 100644 +new mode 100755 +END + executableBitDelta => 1, + indexPath => "foo.exe", +}, +"diff --git a/another_file.txt b/another_file.txt\n"], + expectedNextLine => "index d03e242..435ad3a 100755\n", +}, +{ + # New test case + diffName => "New executable file", + inputText => <<'END', +diff --git a/foo b/foo +new file mode 100755 +index 0000000..d03e242 +--- /dev/null ++++ b/foo +@@ -0,0 +1 @@ ++file contents + +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo +new file mode 100755 +index 0000000..d03e242 +--- foo ++++ foo +END + executableBitDelta => 1, + indexPath => "foo", + isNew => 1, +}, +"@@ -0,0 +1 @@\n"], + expectedNextLine => "+file contents\n", +}, +{ + # New test case + diffName => "Deleted executable file", + inputText => <<'END', +diff --git a/foo b/foo +deleted file mode 100755 +index d03e242..0000000 +--- a/foo ++++ /dev/null +@@ -1 +0,0 @@ +-file contents + +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: foo +deleted file mode 100755 +index d03e242..0000000 +--- foo ++++ foo +END + executableBitDelta => -1, + indexPath => "foo", + isDeletion => 1, +}, +"@@ -1 +0,0 @@\n"], + expectedNextLine => "-file contents\n", +}, +); + +my $testCasesCount = @testCaseHashRefs; +plan(tests => 2 * $testCasesCount); # Total number of assertions. + +foreach my $testCase (@testCaseHashRefs) { + my $testNameStart = "parseGitDiffHeader(): $testCase->{diffName}: comparing"; + + my $fileHandle; + open($fileHandle, "<", \$testCase->{inputText}); + my $line = <$fileHandle>; + + my @got = VCSUtils::parseGitDiffHeader($fileHandle, $line); + my $expectedReturn = $testCase->{expectedReturn}; + + is_deeply(\@got, $expectedReturn, "$testNameStart return value."); + + my $gotNextLine = <$fileHandle>; + is($gotNextLine, $testCase->{expectedNextLine}, "$testNameStart next read line."); +} diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parsePatch.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parsePatch.pl index e6f82ca..8aae3d4 100644 --- a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parsePatch.pl +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parsePatch.pl @@ -69,14 +69,6 @@ END sourceRevision => "53131", }, { - svnConvertedText => <<'END', -Index: Makefile_new -=================================================================== ---- Makefile_new (revision 53131) (from Makefile:53131) -+++ Makefile_new (working copy) -@@ -0,0 +1,1 @@ -+MODULES = JavaScriptCore JavaScriptGlue WebCore WebKit WebKitTools -END copiedFromPath => "Makefile", indexPath => "Makefile_new", sourceRevision => "53131", diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnDiffFooter.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnDiffFooter.pl new file mode 100644 index 0000000..e305484 --- /dev/null +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnDiffFooter.pl @@ -0,0 +1,348 @@ +#!/usr/bin/perl -w +# +# Copyright (C) Research in Motion Limited 2010. All Rights Reserved. +# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Unit tests of parseSvnDiffProperties(). + +use strict; +use warnings; + +use Test::More; +use VCSUtils; + +my @testCaseHashRefs = ( +#### +# Simple test cases +## +{ + # New test + diffName => "simple: add svn:executable", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: svn:executable + + * +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => 1, +}, +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "simple: delete svn:executable", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Deleted: svn:executable + - * +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => -1, +}, +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "simple: delete svn:executable using SVN 1.4 syntax", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Name: svn:executable + - * +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => -1, +}, +undef], + expectedNextLine => undef, +}, +#### +# Property value followed by empty line and start of next diff +## +{ + # New test + diffName => "add svn:executable, followed by empty line and start of next diff", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: svn:executable + + * + +Index: Makefile.shared +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => 1, +}, +"\n"], + expectedNextLine => "Index: Makefile.shared\n", +}, +{ + # New test + diffName => "add svn:executable, followed by empty line and start of next property diff", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: svn:executable + + * + +Property changes on: Makefile.shared +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => 1, +}, +"\n"], + expectedNextLine => "Property changes on: Makefile.shared\n", +}, +#### +# Property value followed by empty line and start of the binary contents +## +{ + # New test + diffName => "add svn:executable, followed by empty line and start of binary contents", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: svn:executable + + * + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => 1, +}, +"\n"], + expectedNextLine => "Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA==\n", +}, +{ + # New test + diffName => "custom property followed by svn:executable, empty line and start of binary contents", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: documentation + + This is an example sentence. +Added: svn:executable + + * + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => 1, +}, +"\n"], + expectedNextLine => "Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA==\n", +}, +#### +# Successive properties +## +{ + # New test + diffName => "svn:executable followed by custom property", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: svn:executable + + * +Added: documentation + + This is an example sentence. +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => 1, +}, +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "custom property followed by svn:executable", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: documentation + + This is an example sentence. +Added: svn:executable + + * +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => 1, +}, +undef], + expectedNextLine => undef, +}, +#### +# Successive properties followed by empty line and start of next diff +## +{ + # New test + diffName => "custom property followed by svn:executable, empty line and start of next property diff", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: documentation + + This is an example sentence. +Added: svn:executable + + * + +Property changes on: Makefile.shared +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => 1, +}, +"\n"], + expectedNextLine => "Property changes on: Makefile.shared\n", +}, +{ + # New test + diffName => "custom property followed by svn:executable, empty line and start of next index diff", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: documentation + + This is an example sentence. +Added: svn:executable + + * + +Index: Makefile.shared +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => 1, +}, +"\n"], + expectedNextLine => "Index: Makefile.shared\n", +}, +#### +# Custom properties +## +# FIXME: We do not support anything other than the svn:executable property. +# We should add support for handling other properties. +{ + # New test + diffName => "simple: custom property", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Name: documentation + + This is an example sentence. +END + expectedReturn => [ +{ + propertyPath => "FileA", +}, +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "custom property followed by custom property", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: copyright + + Copyright (C) Research in Motion Limited 2010. All Rights Reserved. +Added: documentation + + This is an example sentence. +END + expectedReturn => [ +{ + propertyPath => "FileA", +}, +undef], + expectedNextLine => undef, +}, +#### +# Malformed property diffs +## +# We shouldn't encounter such diffs in practice. +{ + # New test + diffName => "svn:executable followed by custom property and svn:executable", + inputText => <<'END', +Property changes on: FileA +___________________________________________________________________ +Added: svn:executable + + * +Added: documentation + + This is an example sentence. +Deleted: svn:executable + - * +END + expectedReturn => [ +{ + propertyPath => "FileA", + executableBitDelta => -1, +}, +undef], + expectedNextLine => undef, +}, +); + +my $testCasesCount = @testCaseHashRefs; +plan(tests => 2 * $testCasesCount); # Total number of assertions. + +foreach my $testCase (@testCaseHashRefs) { + my $testNameStart = "parseSvnDiffProperties(): $testCase->{diffName}: comparing"; + + my $fileHandle; + open($fileHandle, "<", \$testCase->{inputText}); + my $line = <$fileHandle>; + + my @got = VCSUtils::parseSvnDiffProperties($fileHandle, $line); + my $expectedReturn = $testCase->{expectedReturn}; + + is_deeply(\@got, $expectedReturn, "$testNameStart return value."); + + my $gotNextLine = <$fileHandle>; + is($gotNextLine, $testCase->{expectedNextLine}, "$testNameStart next read line."); +} diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnDiffHeader.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnDiffHeader.pl new file mode 100644 index 0000000..ed8550d --- /dev/null +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnDiffHeader.pl @@ -0,0 +1,220 @@ +#!/usr/bin/perl -w +# +# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Unit tests of parseSvnDiffHeader(). + +use strict; +use warnings; + +use Test::More; +use VCSUtils; + +# The array of test cases. +my @testCaseHashRefs = ( +{ + # New test + diffName => "simple diff", + inputText => <<'END', +Index: WebKitTools/Scripts/VCSUtils.pm +=================================================================== +--- WebKitTools/Scripts/VCSUtils.pm (revision 53004) ++++ WebKitTools/Scripts/VCSUtils.pm (working copy) +@@ -32,6 +32,7 @@ use strict; + use warnings; +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: WebKitTools/Scripts/VCSUtils.pm +=================================================================== +--- WebKitTools/Scripts/VCSUtils.pm (revision 53004) ++++ WebKitTools/Scripts/VCSUtils.pm (working copy) +END + indexPath => "WebKitTools/Scripts/VCSUtils.pm", + sourceRevision => "53004", +}, +"@@ -32,6 +32,7 @@ use strict;\n"], + expectedNextLine => " use warnings;\n", +}, +{ + # New test + diffName => "new file", + inputText => <<'END', +Index: WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl +=================================================================== +--- WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl (revision 0) ++++ WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl (revision 0) +@@ -0,0 +1,262 @@ ++#!/usr/bin/perl -w +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl +=================================================================== +--- WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl (revision 0) ++++ WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl (revision 0) +END + indexPath => "WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseDiffHeader.pl", + isNew => 1, +}, +"@@ -0,0 +1,262 @@\n"], + expectedNextLine => "+#!/usr/bin/perl -w\n", +}, +{ + # New test + diffName => "copied file", + inputText => <<'END', +Index: index_path.py +=================================================================== +--- index_path.py (revision 53048) (from copied_from_path.py:53048) ++++ index_path.py (working copy) +@@ -0,0 +1,7 @@ ++# Python file... +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: index_path.py +=================================================================== +--- index_path.py (revision 53048) (from copied_from_path.py:53048) ++++ index_path.py (working copy) +END + copiedFromPath => "copied_from_path.py", + indexPath => "index_path.py", + sourceRevision => 53048, +}, +"@@ -0,0 +1,7 @@\n"], + expectedNextLine => "+# Python file...\n", +}, +{ + # New test + diffName => "contains \\r\\n lines", + inputText => <<END, # No single quotes to allow interpolation of "\r" +Index: index_path.py\r +===================================================================\r +--- index_path.py (revision 53048)\r ++++ index_path.py (working copy)\r +@@ -0,0 +1,7 @@\r ++# Python file...\r +END + expectedReturn => [ +{ + svnConvertedText => <<END, # No single quotes to allow interpolation of "\r" +Index: index_path.py\r +===================================================================\r +--- index_path.py (revision 53048)\r ++++ index_path.py (working copy)\r +END + indexPath => "index_path.py", + sourceRevision => 53048, +}, +"@@ -0,0 +1,7 @@\r\n"], + expectedNextLine => "+# Python file...\r\n", +}, +{ + # New test + diffName => "contains path corrections", + inputText => <<'END', +Index: index_path.py +=================================================================== +--- bad_path (revision 53048) (from copied_from_path.py:53048) ++++ bad_path (working copy) +@@ -0,0 +1,7 @@ ++# Python file... +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: index_path.py +=================================================================== +--- index_path.py (revision 53048) (from copied_from_path.py:53048) ++++ index_path.py (working copy) +END + copiedFromPath => "copied_from_path.py", + indexPath => "index_path.py", + sourceRevision => 53048, +}, +"@@ -0,0 +1,7 @@\n"], + expectedNextLine => "+# Python file...\n", +}, +#### +# Binary test cases +## +{ + # New test + diffName => "binary file", + inputText => <<'END', +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + +Property changes on: test_file.swf +___________________________________________________________________ +Name: svn:mime-type + + application/octet-stream + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => [ +{ + svnConvertedText => <<'END', +Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +END + indexPath => "test_file.swf", + isBinary => 1, +}, +"svn:mime-type = application/octet-stream\n"], + expectedNextLine => "\n", +}, +); + +my $testCasesCount = @testCaseHashRefs; +plan(tests => 2 * $testCasesCount); # Total number of assertions. + +foreach my $testCase (@testCaseHashRefs) { + my $testNameStart = "parseSvnDiffHeader(): $testCase->{diffName}: comparing"; + + my $fileHandle; + open($fileHandle, "<", \$testCase->{inputText}); + my $line = <$fileHandle>; + + my @got = VCSUtils::parseSvnDiffHeader($fileHandle, $line); + my $expectedReturn = $testCase->{expectedReturn}; + + is_deeply(\@got, $expectedReturn, "$testNameStart return value."); + + my $gotNextLine = <$fileHandle>; + is($gotNextLine, $testCase->{expectedNextLine}, "$testNameStart next read line."); +} diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnProperty.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnProperty.pl new file mode 100644 index 0000000..cff7c2e --- /dev/null +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnProperty.pl @@ -0,0 +1,419 @@ +#!/usr/bin/perl -w +# +# Copyright (C) Research in Motion Limited 2010. All Rights Reserved. +# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Unit tests of parseSvnProperty(). + +use strict; +use warnings; + +use Test::More; +use VCSUtils; + +my @testCaseHashRefs = ( +#### +# Simple test cases +## +{ + # New test + diffName => "simple: add svn:executable", + inputText => <<'END', +Added: svn:executable + + * +END + expectedReturn => [ +{ + name => "svn:executable", + propertyChangeDelta => 1, + value => "*", +}, +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "simple: delete svn:executable", + inputText => <<'END', +Deleted: svn:executable + - * +END + expectedReturn => [ +{ + name => "svn:executable", + propertyChangeDelta => -1, + value => "*", +}, +undef], + expectedNextLine => undef, +}, +#### +# Using SVN 1.4 syntax +## +{ + # New test + diffName => "simple: delete svn:executable using SVN 1.4 syntax", + inputText => <<'END', +Name: svn:executable + - * +END + expectedReturn => [ +{ + name => "svn:executable", + propertyChangeDelta => -1, + value => "*", +}, +undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "simple: add svn:executable using SVN 1.4 syntax", + inputText => <<'END', +Name: svn:executable + + * +END + expectedReturn => [ +{ + name => "svn:executable", + propertyChangeDelta => 1, + value => "*", +}, +undef], + expectedNextLine => undef, +}, +#### +# Property value followed by empty line and start of next diff +## +{ + # New test + diffName => "add svn:executable, followed by empty line and start of next diff", + inputText => <<'END', +Added: svn:executable + + * + +Index: Makefile.shared +END + expectedReturn => [ +{ + name => "svn:executable", + propertyChangeDelta => 1, + value => "*", +}, +"\n"], + expectedNextLine => "Index: Makefile.shared\n", +}, +{ + # New test + diffName => "add svn:executable, followed by empty line and start of next property diff", + inputText => <<'END', +Added: svn:executable + + * + +Property changes on: Makefile.shared +END + expectedReturn => [ +{ + name => "svn:executable", + propertyChangeDelta => 1, + value => "*", +}, +"\n"], + expectedNextLine => "Property changes on: Makefile.shared\n", +}, +{ + # New test + diffName => "multi-line '+' change, followed by empty line and start of next diff", + inputText => <<'END', +Name: documentation + + A +long sentence that spans +multiple lines. + +Index: Makefile.shared +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => 1, + value => "A\nlong sentence that spans\nmultiple lines.", +}, +"\n"], + expectedNextLine => "Index: Makefile.shared\n", +}, +{ + # New test + diffName => "multi-line '+' change, followed by empty line and start of next property diff", + inputText => <<'END', +Name: documentation + + A +long sentence that spans +multiple lines. + +Property changes on: Makefile.shared +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => 1, + value => "A\nlong sentence that spans\nmultiple lines.", +}, +"\n"], + expectedNextLine => "Property changes on: Makefile.shared\n", +}, +#### +# Property value followed by empty line and start of binary patch +## +{ + # New test + diffName => "add svn:executable, followed by empty line and start of binary patch", + inputText => <<'END', +Added: svn:executable + + * + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => [ +{ + name => "svn:executable", + propertyChangeDelta => 1, + value => "*", +}, +"\n"], + expectedNextLine => "Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA==\n", +}, +{ + # New test + diffName => "multi-line '+' change, followed by empty line and start of binary patch", + inputText => <<'END', +Name: documentation + + A +long sentence that spans +multiple lines. + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => 1, + value => "A\nlong sentence that spans\nmultiple lines.", +}, +"\n"], + expectedNextLine => "Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA==\n", +}, +{ + # New test + diffName => "multi-line '-' change, followed by multi-line '+' change, empty line, and start of binary patch", + inputText => <<'END', +Modified: documentation + - A +long sentence that spans +multiple lines. + + Another +long sentence that spans +multiple lines. + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => 1, + value => "Another\nlong sentence that spans\nmultiple lines.", +}, +"\n"], + expectedNextLine => "Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA==\n", +}, +#### +# Successive properties +## +{ + # New test + diffName => "single-line '+' change followed by custom property with single-line '+' change", + inputText => <<'END', +Added: svn:executable + + * +Added: documentation + + A sentence. +END + expectedReturn => [ +{ + name => "svn:executable", + propertyChangeDelta => 1, + value => "*", +}, +"Added: documentation\n"], + expectedNextLine => " + A sentence.\n", +}, +{ + # New test + diffName => "multi-line '+' change, followed by svn:executable", + inputText => <<'END', +Name: documentation + + A +long sentence that spans +multiple lines. +Name: svn:executable + + * +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => 1, + value => "A\nlong sentence that spans\nmultiple lines.", +}, +"Name: svn:executable\n"], + expectedNextLine => " + *\n", +}, +{ + # New test + diffName => "multi-line '-' change, followed by multi-line '+' change and add svn:executable", + inputText => <<'END', +Modified: documentation + - A +long sentence that spans +multiple lines. + + Another +long sentence that spans +multiple lines. +Added: svn:executable + + * +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => 1, + value => "Another\nlong sentence that spans\nmultiple lines.", +}, +"Added: svn:executable\n"], + expectedNextLine => " + *\n", +}, +#### +# Property values with trailing new lines. +## +# FIXME: We do not support property values with trailing new lines, since it is difficult to +# disambiguate them from the empty line that preceeds the contents of a binary patch as +# in the test case (above): "multi-line '+' change, followed by empty line and start of binary patch". +{ + # New test + diffName => "single-line '+' with trailing new line", + inputText => <<'END', +Added: documentation + + A sentence. + +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => 1, + value => "A sentence.", +}, +"\n"], + expectedNextLine => undef, +}, +{ + # New test + diffName => "single-line '+' with trailing new line, followed by empty line and start of binary patch", + inputText => <<'END', +Added: documentation + + A sentence. + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => 1, + value => "A sentence.", +}, +"\n"], + expectedNextLine => "\n", +}, +{ + # New test + diffName => "single-line '-' change with trailing new line, and single-line '+' change", + inputText => <<'END', +Modified: documentation + - A long sentence. + + + A sentence. +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => -1, # Since we only interpret the '-' property. + value => "A long sentence.", +}, +"\n"], + expectedNextLine => " + A sentence.\n", +}, +{ + # New test + diffName => "multi-line '-' change with trailing new line, and multi-line '+' change", + inputText => <<'END', +Modified: documentation + - A +long sentence that spans +multiple lines. + + + Another +long sentence that spans +multiple lines. +END + expectedReturn => [ +{ + name => "documentation", + propertyChangeDelta => -1, # Since we only interpret the '-' property. + value => "A\nlong sentence that spans\nmultiple lines.", +}, +"\n"], + expectedNextLine => " + Another\n", +}, +); + +my $testCasesCount = @testCaseHashRefs; +plan(tests => 2 * $testCasesCount); # Total number of assertions. + +foreach my $testCase (@testCaseHashRefs) { + my $testNameStart = "parseSvnProperty(): $testCase->{diffName}: comparing"; + + my $fileHandle; + open($fileHandle, "<", \$testCase->{inputText}); + my $line = <$fileHandle>; + + my @got = VCSUtils::parseSvnProperty($fileHandle, $line); + my $expectedReturn = $testCase->{expectedReturn}; + + is_deeply(\@got, $expectedReturn, "$testNameStart return value."); + + my $gotNextLine = <$fileHandle>; + is($gotNextLine, $testCase->{expectedNextLine}, "$testNameStart next read line."); +} diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnPropertyValue.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnPropertyValue.pl new file mode 100644 index 0000000..5c79862 --- /dev/null +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/parseSvnPropertyValue.pl @@ -0,0 +1,149 @@ +#!/usr/bin/perl -w +# +# Copyright (C) Research in Motion Limited 2010. All Rights Reserved. +# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Apple Computer, Inc. ("Apple") nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Unit tests of parseSvnPropertyValue(). + +use strict; +use warnings; + +use Test::More; +use VCSUtils; + +my @testCaseHashRefs = ( +{ + # New test + diffName => "singe-line '+' change", + inputText => <<'END', + + * +END + expectedReturn => ["*", undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "single-line '-' change", + inputText => <<'END', + - * +END + expectedReturn => ["*", undef], + expectedNextLine => undef, +}, +{ + # New test + diffName => "single-line '-' change followed by empty line", + inputText => <<'END', + - * + +END + expectedReturn => ["*", "\n"], + expectedNextLine => undef, +}, +{ + # New test + diffName => "single-line '-' change followed by the next property", + inputText => <<'END', + - * +Deleted: svn:executable +END + expectedReturn => ["*", "Deleted: svn:executable\n"], + expectedNextLine => undef, +}, +{ + # New test + diffName => "multi-line '+' change and start of binary patch", + inputText => <<'END', + + A +long sentence that spans +multiple lines. + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +END + expectedReturn => ["A\nlong sentence that spans\nmultiple lines.", "\n"], + expectedNextLine => "Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA==\n", +}, +{ + # New test + diffName => "multi-line '-' change followed by '+' single-line change", + inputText => <<'END', + - A +long sentence that spans +multiple lines. + + A single-line. +END + expectedReturn => ["A\nlong sentence that spans\nmultiple lines.", " + A single-line.\n"], + expectedNextLine => undef, +}, +{ + # New test + diffName => "multi-line '-' change followed by the next property", + inputText => <<'END', + - A +long sentence that spans +multiple lines. +Added: svn:executable +END + expectedReturn => ["A\nlong sentence that spans\nmultiple lines.", "Added: svn:executable\n"], + expectedNextLine => undef, +}, +{ + # New test + diffName => "multi-line '-' change followed by '+' multi-line change", + inputText => <<'END', + - A +long sentence that spans +multiple lines. + + Another +long sentence that spans +multiple lines. +END + expectedReturn => ["A\nlong sentence that spans\nmultiple lines.", " + Another\n"], + expectedNextLine => "long sentence that spans\n", +}, +); + +my $testCasesCount = @testCaseHashRefs; +plan(tests => 2 * $testCasesCount); # Total number of assertions. + +foreach my $testCase (@testCaseHashRefs) { + my $testNameStart = "parseSvnPropertyValue(): $testCase->{diffName}: comparing"; + + my $fileHandle; + open($fileHandle, "<", \$testCase->{inputText}); + my $line = <$fileHandle>; + + my @got = VCSUtils::parseSvnPropertyValue($fileHandle, $line); + my $expectedReturn = $testCase->{expectedReturn}; + + is_deeply(\@got, $expectedReturn, "$testNameStart return value."); + + my $gotNextLine = <$fileHandle>; + is($gotNextLine, $testCase->{expectedNextLine}, "$testNameStart next read line."); +} diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/prepareParsedPatch.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/prepareParsedPatch.pl new file mode 100644 index 0000000..a7ae807 --- /dev/null +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/prepareParsedPatch.pl @@ -0,0 +1,136 @@ +#!/usr/bin/perl -w +# +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Unit tests of prepareParsedPatch(). + +use strict; +use warnings; + +use Test::More; +use VCSUtils; + +my $diffHashRef1 = { # not a copy, no source revision + copiedFromPath => undef, + indexPath => "indexPath1", + sourceRevision => undef, + svnConvertedText => "diff1", +}; +my $diffHashRef2 = { # not a copy, has source revision + copiedFromPath => undef, + indexPath => "indexPath2", + sourceRevision => 20, + svnConvertedText => "diff2", +}; +my $diffHashRef3 = { # a copy (copies always have source revision) + copiedFromPath => "sourcePath3", + indexPath => "indexPath2", # Deliberately choosing same as $diffHashRef2 + sourceRevision => 3, + svnConvertedText => "diff3", +}; + +my @testCases = ( +{ + # New test + testName => "zero diffs: empty array", + diffHashRefsInput => [], + expected => { + copyDiffHashRefs => [], + nonCopyDiffHashRefs => [], + sourceRevisionHash => {}, + }, +}, +{ + # New test + testName => "one diff: non-copy, no revision", + diffHashRefsInput => [$diffHashRef1], + expected => { + copyDiffHashRefs => [], + nonCopyDiffHashRefs => [$diffHashRef1], + sourceRevisionHash => {}, + }, +}, +{ + # New test + testName => "one diff: non-copy, has revision", + diffHashRefsInput => [$diffHashRef2], + expected => { + copyDiffHashRefs => [], + nonCopyDiffHashRefs => [$diffHashRef2], + sourceRevisionHash => { + "indexPath2" => 20, + } + }, +}, +{ + # New test + testName => "one diff: copy (has revision)", + diffHashRefsInput => [$diffHashRef3], + expected => { + copyDiffHashRefs => [$diffHashRef3], + nonCopyDiffHashRefs => [], + sourceRevisionHash => { + "sourcePath3" => 3, + } + }, +}, +{ + # New test + testName => "two diffs: two non-copies", + diffHashRefsInput => [$diffHashRef1, $diffHashRef2], + expected => { + copyDiffHashRefs => [], + nonCopyDiffHashRefs => [$diffHashRef1, $diffHashRef2], + sourceRevisionHash => { + "indexPath2" => 20, + } + }, +}, +{ + # New test + testName => "two diffs: non-copy and copy", + diffHashRefsInput => [$diffHashRef2, $diffHashRef3], + expected => { + copyDiffHashRefs => [$diffHashRef3], + nonCopyDiffHashRefs => [$diffHashRef2], + sourceRevisionHash => { + "sourcePath3" => 3, + "indexPath2" => 20, + } + }, +}, +); + +my $testCasesCount = @testCases; +plan(tests => $testCasesCount); + +foreach my $testCase (@testCases) { + my $testName = $testCase->{testName}; + my @diffHashRefs = @{$testCase->{diffHashRefsInput}}; + my $expected = $testCase->{expected}; + + my $got = prepareParsedPatch(0, @diffHashRefs); + + is_deeply($got, $expected, $testName); +} + diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/runPatchCommand.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/runPatchCommand.pl index 8111def..5acc517 100644 --- a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/runPatchCommand.pl +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/runPatchCommand.pl @@ -33,19 +33,6 @@ use Test::Simple tests => 4; use VCSUtils; -# Call a function while suppressing STDERR. -sub callSilently($@) { - my ($func, @args) = @_; - - open(OLDERR, ">&STDERR"); - close(STDERR); - my @returnValue = &$func(@args); - open(STDERR, ">&OLDERR"); - close(OLDERR); # FIXME: Is this necessary? - - return @returnValue; -} - # New test $title = "runPatchCommand: Unsuccessful patch, forcing."; diff --git a/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/setChangeLogDateAndReviewer.pl b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/setChangeLogDateAndReviewer.pl new file mode 100644 index 0000000..076d88c --- /dev/null +++ b/WebKitTools/Scripts/webkitperl/VCSUtils_unittest/setChangeLogDateAndReviewer.pl @@ -0,0 +1,128 @@ +#!/usr/bin/perl -w +# +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Unit tests of setChangeLogDateAndReviewer(). + +use strict; +use warnings; + +use Test::More; +use VCSUtils; + +my @testCaseHashRefs = ( +{ + testName => "reviewer defined and \"NOBODY (OOPS!)\" in leading junk", + reviewer => "John Doe", + epochTime => 1273414321, + patch => <<'END', +Subject: [PATCH] + +Reviewed by NOBODY (OOPS!). + +diff --git a/WebCore/ChangeLog b/WebCore/ChangeLog +--- a/WebCore/ChangeLog ++++ b/WebCore/ChangeLog +@@ -1,3 +1,15 @@ ++2010-05-08 Chris Jerdonek <cjerdonek@webkit.org> ++ ++ Reviewed by NOBODY (OOPS!). ++ + 2010-05-08 Chris Jerdonek <cjerdonek@webkit.org> + + Reviewed by Jane Doe. +END + expectedReturn => <<'END', +Subject: [PATCH] + +Reviewed by NOBODY (OOPS!). + +diff --git a/WebCore/ChangeLog b/WebCore/ChangeLog +--- a/WebCore/ChangeLog ++++ b/WebCore/ChangeLog +@@ -1,3 +1,15 @@ ++2010-05-09 Chris Jerdonek <cjerdonek@webkit.org> ++ ++ Reviewed by John Doe. ++ + 2010-05-08 Chris Jerdonek <cjerdonek@webkit.org> + + Reviewed by Jane Doe. +END +}, +{ + testName => "reviewer not defined and \"NOBODY (OOPS!)\" in leading junk", + reviewer => undef, + epochTime => 1273414321, + patch => <<'END', +Subject: [PATCH] + +Reviewed by NOBODY (OOPS!). + +diff --git a/WebCore/ChangeLog b/WebCore/ChangeLog +--- a/WebCore/ChangeLog ++++ b/WebCore/ChangeLog +@@ -1,3 +1,15 @@ ++2010-05-08 Chris Jerdonek <cjerdonek@webkit.org> ++ ++ Reviewed by NOBODY (OOPS!). ++ + 2010-05-08 Chris Jerdonek <cjerdonek@webkit.org> + + Reviewed by Jane Doe. +END + expectedReturn => <<'END', +Subject: [PATCH] + +Reviewed by NOBODY (OOPS!). + +diff --git a/WebCore/ChangeLog b/WebCore/ChangeLog +--- a/WebCore/ChangeLog ++++ b/WebCore/ChangeLog +@@ -1,3 +1,15 @@ ++2010-05-09 Chris Jerdonek <cjerdonek@webkit.org> ++ ++ Reviewed by NOBODY (OOPS!). ++ + 2010-05-08 Chris Jerdonek <cjerdonek@webkit.org> + + Reviewed by Jane Doe. +END +}, +); + +my $testCasesCount = @testCaseHashRefs; +plan(tests => 1 * $testCasesCount); # Total number of assertions. + +foreach my $testCase (@testCaseHashRefs) { + my $testNameStart = "setChangeLogDateAndReviewer(): $testCase->{testName}: comparing"; + + my $patch = $testCase->{patch}; + my $reviewer = $testCase->{reviewer}; + my $epochTime = $testCase->{epochTime}; + + my $got = VCSUtils::setChangeLogDateAndReviewer($patch, $reviewer, $epochTime); + my $expectedReturn = $testCase->{expectedReturn}; + + is($got, $expectedReturn, "$testNameStart return value."); +} diff --git a/WebKitTools/Scripts/webkitperl/features.pm b/WebKitTools/Scripts/webkitperl/features.pm index 1f88022..7ca924b 100644 --- a/WebKitTools/Scripts/webkitperl/features.pm +++ b/WebKitTools/Scripts/webkitperl/features.pm @@ -72,6 +72,7 @@ sub hasFeature($$) "3D Rendering" => "WebCoreHas3DRendering", "3D Canvas" => "WebGLShader", "WML" => "WMLElement", + "WCSS" => "parseWCSSInputProperty", "XHTMLMP" => "isXHTMLMPDocument", ); my $symbolName = $symbolForFeature{$featureName}; diff --git a/WebKitTools/Scripts/webkitperl/httpd.pm b/WebKitTools/Scripts/webkitperl/httpd.pm index 05eb21c..b415db6 100644 --- a/WebKitTools/Scripts/webkitperl/httpd.pm +++ b/WebKitTools/Scripts/webkitperl/httpd.pm @@ -31,6 +31,7 @@ use strict; use warnings; +use File::Copy; use File::Path; use File::Spec; use File::Spec::Functions; @@ -153,19 +154,12 @@ sub openHTTPD(@) close PIDFILE; if (0 != kill 0, $oldPid) { print "\nhttpd is already running: pid $oldPid, killing...\n"; - kill 15, $oldPid; - - my $retryCount = 20; - while ((kill(0, $oldPid) != 0) && $retryCount) { - sleep 1; - --$retryCount; - } - - if (!$retryCount) { + if (!killHTTPD($oldPid)) { cleanUp(); die "Timed out waiting for httpd to quit"; } } + unlink $httpdPidFile; } $httpdPath = "/usr/sbin/httpd" unless ($httpdPath); @@ -195,22 +189,31 @@ sub openHTTPD(@) sub closeHTTPD { close HTTPDIN; - my $retryCount = 20; - if ($httpdPid) { - kill 15, $httpdPid; - while (-f $httpdPidFile && $retryCount) { - sleep 1; - --$retryCount; - } - } + my $succeeded = killHTTPD($httpdPid); cleanUp(); - if (!$retryCount) { - print STDERR "Timed out waiting for httpd to terminate!\n"; + unless ($succeeded) { + print STDERR "Timed out waiting for httpd to terminate!\n" unless $succeeded; return 0; } return 1; } +sub killHTTPD +{ + my ($pid) = @_; + + return 1 unless $pid; + + kill 15, $pid; + + my $retryCount = 20; + while (kill(0, $pid) && $retryCount) { + sleep 1; + --$retryCount; + } + return $retryCount != 0; +} + sub setShouldWaitForUserInterrupt { $waitForUserInterrupt = 1; @@ -218,7 +221,16 @@ sub setShouldWaitForUserInterrupt sub handleInterrupt { - closeHTTPD(); + # On Cygwin, when we receive a signal Apache is still running, so we need + # to kill it. On other platforms (at least Mac OS X), Apache will have + # already been killed, and trying to kill it again will cause us to hang. + # All we need to do in this case is clean up our own files. + if (isCygwin()) { + closeHTTPD(); + } else { + cleanUp(); + } + print "\n"; exit(1); } diff --git a/WebKitTools/Scripts/webkitpy/BeautifulSoup.pyc b/WebKitTools/Scripts/webkitpy/BeautifulSoup.pyc Binary files differdeleted file mode 100644 index dffb144..0000000 --- a/WebKitTools/Scripts/webkitpy/BeautifulSoup.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/__init__.py b/WebKitTools/Scripts/webkitpy/__init__.py index 94ecc70..b376bf2 100644 --- a/WebKitTools/Scripts/webkitpy/__init__.py +++ b/WebKitTools/Scripts/webkitpy/__init__.py @@ -1,8 +1,13 @@ # Required for Python to search this directory for module files -import autoinstall - -# List our third-party library dependencies here and where they can be -# downloaded. -autoinstall.bind("ClientForm", "http://pypi.python.org/packages/source/C/ClientForm/ClientForm-0.2.10.zip", "ClientForm-0.2.10") -autoinstall.bind("mechanize", "http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", "mechanize-0.1.11") +# Keep this file free of any code or import statements that could +# cause either an error to occur or a log message to be logged. +# This ensures that calling code can import initialization code from +# webkitpy before any errors or log messages due to code in this file. +# Initialization code can include things like version-checking code and +# logging configuration code. +# +# We do not execute any version-checking code or logging configuration +# code in this file so that callers can opt-in as they want. This also +# allows different callers to choose different initialization code, +# as necessary. diff --git a/WebKitTools/Scripts/webkitpy/__init__.pyc b/WebKitTools/Scripts/webkitpy/__init__.pyc Binary files differdeleted file mode 100644 index d1ffa10..0000000 --- a/WebKitTools/Scripts/webkitpy/__init__.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/autoinstall.py b/WebKitTools/Scripts/webkitpy/autoinstall.py deleted file mode 100644 index 467e6b4..0000000 --- a/WebKitTools/Scripts/webkitpy/autoinstall.py +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright (c) 2009, Daniel Krech All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the Daniel Krech nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""\ -package loader for auto installing Python packages. - -A package loader in the spirit of Zero Install that can be used to -inject dependencies into the import process. - - -To install:: - - easy_install -U autoinstall - - or - - download, unpack, python setup.py install - - or - - try the bootstrap loader. See below. - - -To use:: - - # You can bind any package name to a URL pointing to something - # that can be imported using the zipimporter. - - autoinstall.bind("pymarc", "http://pypi.python.org/packages/2.5/p/pymarc/pymarc-2.1-py2.5.egg") - - import pymarc - - print pymarc.__version__, pymarc.__file__ - - -Changelog:: - -- added support for non top level packages. -- cache files now use filename part from URL. -- applied patch from Eric Seidel <eseidel@google.com> to add support -for loading modules where the module is not at the root of the .zip -file. - - -TODO:: - -- a description of the intended use case -- address other issues pointed out in: - - http://mail.python.org/pipermail/python-dev/2008-March/077926.html - -Scribbles:: - -pull vs. push -user vs. system -web vs. filesystem -auto vs. manual - -manage development sandboxes - -optional interfaces... - - def get_data(pathname) -> string with file data. - - Return the data associated with 'pathname'. Raise IOError if - the file wasn't found."); - - def is_package, - "is_package(fullname) -> bool. - - Return True if the module specified by fullname is a package. - Raise ZipImportError is the module couldn't be found."); - - def get_code, - "get_code(fullname) -> code object. - - Return the code object for the specified module. Raise ZipImportError - is the module couldn't be found."); - - def get_source, - "get_source(fullname) -> source string. - - Return the source code for the specified module. Raise ZipImportError - is the module couldn't be found, return None if the archive does - contain the module, but has no source for it."); - - -Autoinstall can also be bootstraped with the nascent package loader -bootstrap module. For example:: - - # or via the bootstrap - # loader. - - try: - _version = "0.2" - import autoinstall - if autoinstall.__version__ != _version: - raise ImportError("A different version than expected found.") - except ImportError, e: - # http://svn.python.org/projects/sandbox/trunk/bootstrap/bootstrap.py - import bootstrap - pypi = "http://pypi.python.org" - dir = "packages/source/a/autoinstall" - url = "%s/%s/autoinstall-%s.tar.gz" % (pypi, dir, _version) - bootstrap.main((url,)) - import autoinstall - -References:: - - http://0install.net/ - http://www.python.org/dev/peps/pep-0302/ - http://svn.python.org/projects/sandbox/trunk/import_in_py - http://0install.net/injector-find.html - http://roscidus.com/desktop/node/903 - -""" - -# To allow use of the "with" keyword for Python 2.5 users. -from __future__ import with_statement - -__version__ = "0.2" -__docformat__ = "restructuredtext en" - -import os -import new -import sys -import urllib -import logging -import tempfile -import zipimport - -_logger = logging.getLogger(__name__) - - -_importer = None - -def _getImporter(): - global _importer - if _importer is None: - _importer = Importer() - sys.meta_path.append(_importer) - return _importer - -def bind(package_name, url, zip_subpath=None): - """bind a top level package name to a URL. - - The package name should be a package name and the url should be a - url to something that can be imported using the zipimporter. - - Optional zip_subpath parameter allows searching for modules - below the root level of the zip file. - """ - _getImporter().bind(package_name, url, zip_subpath) - - -class Cache(object): - - def __init__(self, directory=None): - if directory is None: - # Default to putting the cache directory in the same directory - # as this file. - containing_directory = os.path.dirname(__file__) - directory = os.path.join(containing_directory, "autoinstall.cache.d"); - - self.directory = directory - try: - if not os.path.exists(self.directory): - self._create_cache_directory() - except Exception, err: - _logger.exception(err) - self.cache_directry = tempfile.mkdtemp() - _logger.info("Using cache directory '%s'." % self.directory) - - def _create_cache_directory(self): - _logger.debug("Creating cache directory '%s'." % self.directory) - os.mkdir(self.directory) - readme_path = os.path.join(self.directory, "README") - with open(readme_path, "w") as f: - f.write("This directory was auto-generated by '%s'.\n" - "It is safe to delete.\n" % __file__) - - def get(self, url): - _logger.info("Getting '%s' from cache." % url) - filename = url.rsplit("/")[-1] - - # so that source url is significant in determining cache hits - d = os.path.join(self.directory, "%s" % hash(url)) - if not os.path.exists(d): - os.mkdir(d) - - filename = os.path.join(d, filename) - - if os.path.exists(filename): - _logger.debug("... already cached in file '%s'." % filename) - else: - _logger.debug("... not in cache. Caching in '%s'." % filename) - stream = file(filename, "wb") - self.download(url, stream) - stream.close() - return filename - - def download(self, url, stream): - _logger.info("Downloading: %s" % url) - try: - netstream = urllib.urlopen(url) - code = 200 - if hasattr(netstream, "getcode"): - code = netstream.getcode() - if not 200 <= code < 300: - raise ValueError("HTTP Error code %s" % code) - except Exception, err: - _logger.exception(err) - - BUFSIZE = 2**13 # 8KB - size = 0 - while True: - data = netstream.read(BUFSIZE) - if not data: - break - stream.write(data) - size += len(data) - netstream.close() - _logger.info("Downloaded %d bytes." % size) - - -class Importer(object): - - def __init__(self): - self.packages = {} - self.__cache = None - - def __get_store(self): - return self.__store - store = property(__get_store) - - def _get_cache(self): - if self.__cache is None: - self.__cache = Cache() - return self.__cache - def _set_cache(self, cache): - self.__cache = cache - cache = property(_get_cache, _set_cache) - - def find_module(self, fullname, path=None): - """-> self or None. - - Search for a module specified by 'fullname'. 'fullname' must be - the fully qualified (dotted) module name. It returns the - zipimporter instance itself if the module was found, or None if - it wasn't. The optional 'path' argument is ignored -- it's - there for compatibility with the importer protocol."); - """ - _logger.debug("find_module(%s, path=%s)" % (fullname, path)) - - if fullname in self.packages: - (url, zip_subpath) = self.packages[fullname] - filename = self.cache.get(url) - zip_path = "%s/%s" % (filename, zip_subpath) if zip_subpath else filename - _logger.debug("fullname: %s url: %s path: %s zip_path: %s" % (fullname, url, path, zip_path)) - try: - loader = zipimport.zipimporter(zip_path) - _logger.debug("returning: %s" % loader) - except Exception, e: - _logger.exception(e) - return None - return loader - return None - - def bind(self, package_name, url, zip_subpath): - _logger.info("binding: %s -> %s subpath: %s" % (package_name, url, zip_subpath)) - self.packages[package_name] = (url, zip_subpath) - - -if __name__=="__main__": - import logging - #logging.basicConfig() - logger = logging.getLogger() - - console = logging.StreamHandler() - console.setLevel(logging.DEBUG) - # set a format which is simpler for console use - formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') - # tell the handler to use this format - console.setFormatter(formatter) - # add the handler to the root logger - logger.addHandler(console) - logger.setLevel(logging.INFO) - - bind("pymarc", "http://pypi.python.org/packages/2.5/p/pymarc/pymarc-2.1-py2.5.egg") - - import pymarc - - print pymarc.__version__, pymarc.__file__ - - assert pymarc.__version__=="2.1" - - d = _getImporter().cache.directory - assert d in pymarc.__file__, "'%s' not found in pymarc.__file__ (%s)" % (d, pymarc.__file__) - - # Can now also bind to non top level packages. The packages - # leading up to the package being bound will need to be defined - # however. - # - # bind("rdf.plugins.stores.memory", - # "http://pypi.python.org/packages/2.5/r/rdf.plugins.stores.memeory/rdf.plugins.stores.memory-0.9a-py2.5.egg") - # - # from rdf.plugins.stores.memory import Memory - - diff --git a/WebKitTools/Scripts/webkitpy/autoinstall.pyc b/WebKitTools/Scripts/webkitpy/autoinstall.pyc Binary files differdeleted file mode 100644 index 68e46fd..0000000 --- a/WebKitTools/Scripts/webkitpy/autoinstall.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/bugzilla.pyc b/WebKitTools/Scripts/webkitpy/bugzilla.pyc Binary files differdeleted file mode 100644 index dfde47c..0000000 --- a/WebKitTools/Scripts/webkitpy/bugzilla.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/buildbot.py b/WebKitTools/Scripts/webkitpy/buildbot.py deleted file mode 100644 index 38828fd..0000000 --- a/WebKitTools/Scripts/webkitpy/buildbot.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# WebKit's Python module for interacting with WebKit's buildbot - -import re -import urllib2 - -# Import WebKit-specific modules. -from webkitpy.webkit_logging import log - -# WebKit includes a built copy of BeautifulSoup in Scripts/webkitpy -# so this import should always succeed. -from .BeautifulSoup import BeautifulSoup - - -class BuildBot: - - default_host = "build.webkit.org" - - def __init__(self, host=default_host): - self.buildbot_host = host - self.buildbot_server_url = "http://%s/" % self.buildbot_host - - # If any Leopard builder/tester, Windows builder or Chromium builder is - # red we should not be landing patches. Other builders should be added - # to this list once they are known to be reliable. - # See https://bugs.webkit.org/show_bug.cgi?id=33296 and related bugs. - self.core_builder_names_regexps = [ - "Leopard", - "Windows.*Build", - "Chromium", - ] - - def _parse_builder_status_from_row(self, status_row): - # If WebKit's buildbot has an XMLRPC interface we could use, we could - # do something more sophisticated here. For now we just parse out the - # basics, enough to support basic questions like "is the tree green?" - status_cells = status_row.findAll('td') - builder = {} - - name_link = status_cells[0].find('a') - builder['name'] = name_link.string - # We could generate the builder_url from the name in a future version - # of this code. - builder['builder_url'] = self.buildbot_server_url + name_link['href'] - - status_link = status_cells[1].find('a') - if not status_link: - # We failed to find a link in the first cell, just give up. This - # can happen if a builder is just-added, the first cell will just - # be "no build" - # Other parts of the code depend on is_green being present. - builder['is_green'] = False - return builder - # Will be either a revision number or a build number - revision_string = status_link.string - # If revision_string has non-digits assume it's not a revision number. - builder['built_revision'] = int(revision_string) \ - if not re.match('\D', revision_string) \ - else None - builder['is_green'] = not re.search('fail', - status_cells[1].renderContents()) - # We could parse out the build number instead, but for now just store - # the URL. - builder['build_url'] = self.buildbot_server_url + status_link['href'] - - # We could parse out the current activity too. - - return builder - - def _builder_statuses_with_names_matching_regexps(self, - builder_statuses, - name_regexps): - builders = [] - for builder in builder_statuses: - for name_regexp in name_regexps: - if re.match(name_regexp, builder['name']): - builders.append(builder) - return builders - - def red_core_builders(self): - red_builders = [] - for builder in self._builder_statuses_with_names_matching_regexps( - self.builder_statuses(), - self.core_builder_names_regexps): - if not builder['is_green']: - red_builders.append(builder) - return red_builders - - def red_core_builders_names(self): - red_builders = self.red_core_builders() - return map(lambda builder: builder['name'], red_builders) - - def core_builders_are_green(self): - return not self.red_core_builders() - - def builder_statuses(self): - build_status_url = self.buildbot_server_url + 'one_box_per_builder' - page = urllib2.urlopen(build_status_url) - soup = BeautifulSoup(page) - - builders = [] - status_table = soup.find('table') - for status_row in status_table.findAll('tr'): - builder = self._parse_builder_status_from_row(status_row) - builders.append(builder) - return builders diff --git a/WebKitTools/Scripts/webkitpy/buildbot.pyc b/WebKitTools/Scripts/webkitpy/buildbot.pyc Binary files differdeleted file mode 100644 index 49b1e68..0000000 --- a/WebKitTools/Scripts/webkitpy/buildbot.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/buildbot_unittest.py b/WebKitTools/Scripts/webkitpy/buildbot_unittest.py deleted file mode 100644 index bde3e04..0000000 --- a/WebKitTools/Scripts/webkitpy/buildbot_unittest.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (C) 2009 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import unittest - -from webkitpy.buildbot import BuildBot - -from webkitpy.BeautifulSoup import BeautifulSoup - -class BuildBotTest(unittest.TestCase): - - _example_one_box_status = ''' - <table> - <tr> - <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td> - <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td> - <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td> - <tr> - <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td> - <td class="LastBuild box" >no build</td> - <td align="center" class="Activity building">building<br />< 1 min</td> - <tr> - <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td> - <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td> - <td align="center" class="Activity idle">idle</td> - </table> -''' - _expected_example_one_box_parsings = [ - { - 'builder_url': u'http://build.webkit.org/builders/Windows%20Debug%20%28Tests%29', - 'build_url': u'http://build.webkit.org/builders/Windows%20Debug%20%28Tests%29/builds/3693', - 'is_green': True, - 'name': u'Windows Debug (Tests)', - 'built_revision': 47380 - }, - { - 'builder_url': u'http://build.webkit.org/builders/SnowLeopard%20Intel%20Release', - 'is_green': False, - 'name': u'SnowLeopard Intel Release', - }, - { - 'builder_url': u'http://build.webkit.org/builders/Qt%20Linux%20Release', - 'build_url': u'http://build.webkit.org/builders/Qt%20Linux%20Release/builds/654', - 'is_green': False, - 'name': u'Qt Linux Release', - 'built_revision': 47383 - }, - ] - - def test_status_parsing(self): - buildbot = BuildBot() - - soup = BeautifulSoup(self._example_one_box_status) - status_table = soup.find("table") - input_rows = status_table.findAll('tr') - - for x in range(len(input_rows)): - status_row = input_rows[x] - expected_parsing = self._expected_example_one_box_parsings[x] - - builder = buildbot._parse_builder_status_from_row(status_row) - - # Make sure we aren't parsing more or less than we expect - self.assertEquals(builder.keys(), expected_parsing.keys()) - - for key, expected_value in expected_parsing.items(): - self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value))) - - def test_core_builder_methods(self): - buildbot = BuildBot() - - # Override builder_statuses function to not touch the network. - def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to. - return BuildBotTest._expected_example_one_box_parsings - buildbot.builder_statuses = example_builder_statuses - - buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ] - self.assertEquals(buildbot.red_core_builders_names(), []) - self.assertTrue(buildbot.core_builders_are_green()) - - buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ] - self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ]) - self.assertFalse(buildbot.core_builders_are_green()) - - def test_builder_name_regexps(self): - buildbot = BuildBot() - - # For complete testing, this list should match the list of builders at build.webkit.org: - example_builders = [ - { 'name': u'Tiger Intel Release', }, - { 'name': u'Leopard Intel Release (Build)', }, - { 'name': u'Leopard Intel Release (Tests)', }, - { 'name': u'Leopard Intel Debug (Build)', }, - { 'name': u'Leopard Intel Debug (Tests)', }, - { 'name': u'SnowLeopard Intel Release (Build)', }, - { 'name': u'SnowLeopard Intel Release (Tests)', }, - { 'name': u'SnowLeopard Intel Leaks', }, - { 'name': u'Windows Release (Build)', }, - { 'name': u'Windows Release (Tests)', }, - { 'name': u'Windows Debug (Build)', }, - { 'name': u'Windows Debug (Tests)', }, - { 'name': u'Qt Linux Release', }, - { 'name': u'Gtk Linux Release', }, - { 'name': u'Gtk Linux 32-bit Debug', }, - { 'name': u'Gtk Linux 64-bit Debug', }, - { 'name': u'Chromium Linux Release', }, - { 'name': u'Chromium Mac Release', }, - { 'name': u'Chromium Win Release', }, - ] - name_regexps = [ "Leopard", "Windows.*Build", "Chromium" ] - expected_builders = [ - { 'name': u'Leopard Intel Release (Build)', }, - { 'name': u'Leopard Intel Release (Tests)', }, - { 'name': u'Leopard Intel Debug (Build)', }, - { 'name': u'Leopard Intel Debug (Tests)', }, - { 'name': u'Windows Release (Build)', }, - { 'name': u'Windows Debug (Build)', }, - { 'name': u'Chromium Linux Release', }, - { 'name': u'Chromium Mac Release', }, - { 'name': u'Chromium Win Release', }, - ] - - # This test should probably be updated if the default regexp list changes - self.assertEquals(buildbot.core_builder_names_regexps, name_regexps) - - builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps) - self.assertEquals(builders, expected_builders) - -if __name__ == '__main__': - unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/changelogs.pyc b/WebKitTools/Scripts/webkitpy/changelogs.pyc Binary files differdeleted file mode 100644 index 2fca994..0000000 --- a/WebKitTools/Scripts/webkitpy/changelogs.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/commands/__init__.pyc b/WebKitTools/Scripts/webkitpy/commands/__init__.pyc Binary files differdeleted file mode 100644 index ac801ef..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/__init__.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/commands/abstractsequencedcommand.pyc b/WebKitTools/Scripts/webkitpy/commands/abstractsequencedcommand.pyc Binary files differdeleted file mode 100644 index 6d3afc4..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/abstractsequencedcommand.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/commands/download.pyc b/WebKitTools/Scripts/webkitpy/commands/download.pyc Binary files differdeleted file mode 100644 index 3a0046f..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/download.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/commands/early_warning_system.pyc b/WebKitTools/Scripts/webkitpy/commands/early_warning_system.pyc Binary files differdeleted file mode 100644 index d6e0800..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/early_warning_system.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/commands/early_warning_system_unittest.py b/WebKitTools/Scripts/webkitpy/commands/early_warning_system_unittest.py deleted file mode 100644 index d516b84..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/early_warning_system_unittest.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (C) 2009 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os - -from webkitpy.commands.early_warning_system import * -from webkitpy.commands.queuestest import QueuesTest -from webkitpy.mock import Mock - -class EarlyWarningSytemTest(QueuesTest): - def test_chromium_ews(self): - expected_stderr = { - "begin_work_queue" : "CAUTION: chromium-ews will discard all local changes in \"%s\"\nRunning WebKit chromium-ews.\n" % os.getcwd(), - "handle_unexpected_error" : "Mock error message\n", - } - self.assert_queue_outputs(ChromiumEWS(), expected_stderr=expected_stderr) - - def test_qt_ews(self): - expected_stderr = { - "begin_work_queue" : "CAUTION: qt-ews will discard all local changes in \"%s\"\nRunning WebKit qt-ews.\n" % os.getcwd(), - "handle_unexpected_error" : "Mock error message\n", - } - self.assert_queue_outputs(QtEWS(), expected_stderr=expected_stderr) - - def test_gtk_ews(self): - expected_stderr = { - "begin_work_queue" : "CAUTION: gtk-ews will discard all local changes in \"%s\"\nRunning WebKit gtk-ews.\n" % os.getcwd(), - "handle_unexpected_error" : "Mock error message\n", - } - self.assert_queue_outputs(GtkEWS(), expected_stderr=expected_stderr) - - def test_mac_ews(self): - expected_stderr = { - "begin_work_queue" : "CAUTION: mac-ews will discard all local changes in \"%s\"\nRunning WebKit mac-ews.\n" % os.getcwd(), - "handle_unexpected_error" : "Mock error message\n", - } - self.assert_queue_outputs(MacEWS(), expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/commands/queries.py b/WebKitTools/Scripts/webkitpy/commands/queries.py deleted file mode 100644 index 3ca4f42..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/queries.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. -# Copyright (c) 2009 Apple Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -from optparse import make_option - -from webkitpy.buildbot import BuildBot -from webkitpy.committers import CommitterList -from webkitpy.webkit_logging import log -from webkitpy.multicommandtool import AbstractDeclarativeCommand - - -class BugsToCommit(AbstractDeclarativeCommand): - name = "bugs-to-commit" - help_text = "List bugs in the commit-queue" - - def execute(self, options, args, tool): - # FIXME: This command is poorly named. It's fetching the commit-queue list here. The name implies it's fetching pending-commit (all r+'d patches). - bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue() - for bug_id in bug_ids: - print "%s" % bug_id - - -class PatchesInCommitQueue(AbstractDeclarativeCommand): - name = "patches-in-commit-queue" - help_text = "List patches in the commit-queue" - - def execute(self, options, args, tool): - patches = tool.bugs.queries.fetch_patches_from_commit_queue() - log("Patches in commit queue:") - for patch in patches: - print patch.url() - - -class PatchesToCommitQueue(AbstractDeclarativeCommand): - name = "patches-to-commit-queue" - help_text = "Patches which should be added to the commit queue" - def __init__(self): - options = [ - make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"), - ] - AbstractDeclarativeCommand.__init__(self, options=options) - - @staticmethod - def _needs_commit_queue(patch): - if patch.commit_queue() == "+": # If it's already cq+, ignore the patch. - log("%s already has cq=%s" % (patch.id(), patch.commit_queue())) - return False - - # We only need to worry about patches from contributers who are not yet committers. - committer_record = CommitterList().committer_by_email(patch.attacher_email()) - if committer_record: - log("%s committer = %s" % (patch.id(), committer_record)) - return not committer_record - - def execute(self, options, args, tool): - patches = tool.bugs.queries.fetch_patches_from_pending_commit_list() - patches_needing_cq = filter(self._needs_commit_queue, patches) - if options.bugs: - bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq) - bugs_needing_cq = sorted(set(bugs_needing_cq)) - for bug_id in bugs_needing_cq: - print "%s" % tool.bugs.bug_url_for_bug_id(bug_id) - else: - for patch in patches_needing_cq: - print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit") - - -class PatchesToReview(AbstractDeclarativeCommand): - name = "patches-to-review" - help_text = "List patches that are pending review" - - def execute(self, options, args, tool): - patch_ids = tool.bugs.queries.fetch_attachment_ids_from_review_queue() - log("Patches pending review:") - for patch_id in patch_ids: - print patch_id - - -class TreeStatus(AbstractDeclarativeCommand): - name = "tree-status" - help_text = "Print the status of the %s buildbots" % BuildBot.default_host - long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder -and displayes the status of each builder.""" - - def execute(self, options, args, tool): - for builder in tool.buildbot.builder_statuses(): - status_string = "ok" if builder["is_green"] else "FAIL" - print "%s : %s" % (status_string.ljust(4), builder["name"]) diff --git a/WebKitTools/Scripts/webkitpy/commands/queries.pyc b/WebKitTools/Scripts/webkitpy/commands/queries.pyc Binary files differdeleted file mode 100644 index 829b6e3..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/queries.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/commands/queues.pyc b/WebKitTools/Scripts/webkitpy/commands/queues.pyc Binary files differdeleted file mode 100644 index 8d52d05..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/queues.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/commands/queues_unittest.py b/WebKitTools/Scripts/webkitpy/commands/queues_unittest.py deleted file mode 100644 index 87cd645..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/queues_unittest.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (C) 2009 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os - -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.queues import * -from webkitpy.commands.queuestest import QueuesTest -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture - - -class TestQueue(AbstractQueue): - name = "test-queue" - - -class TestReviewQueue(AbstractReviewQueue): - name = "test-review-queue" - - -class AbstractQueueTest(CommandsTest): - def _assert_log_progress_output(self, patch_ids, progress_output): - OutputCapture().assert_outputs(self, TestQueue().log_progress, [patch_ids], expected_stderr=progress_output) - - def test_log_progress(self): - self._assert_log_progress_output([1,2,3], "3 patches in test-queue [1, 2, 3]\n") - self._assert_log_progress_output(["1","2","3"], "3 patches in test-queue [1, 2, 3]\n") - self._assert_log_progress_output([1], "1 patch in test-queue [1]\n") - - def _assert_run_webkit_patch(self, run_args): - queue = TestQueue() - tool = MockBugzillaTool() - queue.bind_to_tool(tool) - - queue.run_webkit_patch(run_args) - expected_run_args = ["echo", "--status-host=example.com"] + map(str, run_args) - tool.executive.run_and_throw_if_fail.assert_called_with(expected_run_args) - - def test_run_webkit_patch(self): - self._assert_run_webkit_patch([1]) - self._assert_run_webkit_patch(["one", 2]) - - -class AbstractReviewQueueTest(CommandsTest): - def test_patch_collection_delegate_methods(self): - queue = TestReviewQueue() - tool = MockBugzillaTool() - queue.bind_to_tool(tool) - self.assertEquals(queue.collection_name(), "test-review-queue") - self.assertEquals(queue.fetch_potential_patch_ids(), [103]) - queue.status_server() - self.assertTrue(queue.is_terminal_status("Pass")) - self.assertTrue(queue.is_terminal_status("Fail")) - self.assertTrue(queue.is_terminal_status("Error: Your patch exploded")) - self.assertFalse(queue.is_terminal_status("Foo")) - - -class CommitQueueTest(QueuesTest): - def test_commit_queue(self): - expected_stderr = { - "begin_work_queue" : "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % os.getcwd(), - # FIXME: The commit-queue warns about bad committers twice. This is due to the fact that we access Attachment.reviewer() twice and it logs each time. - "next_work_item" : """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) -Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) -2 patches in commit-queue [197, 106] -""", - } - self.assert_queue_outputs(CommitQueue(), expected_stderr=expected_stderr) - - -class StyleQueueTest(QueuesTest): - def test_style_queue(self): - expected_stderr = { - "begin_work_queue" : "CAUTION: style-queue will discard all local changes in \"%s\"\nRunning WebKit style-queue.\n" % os.getcwd(), - "handle_unexpected_error" : "Mock error message\n", - } - self.assert_queue_outputs(StyleQueue(), expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/commands/queuestest.py b/WebKitTools/Scripts/webkitpy/commands/queuestest.py deleted file mode 100644 index 09d1c26..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/queuestest.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (C) 2009 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import unittest - -from webkitpy.bugzilla import Attachment -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture - - -class MockQueueEngine(object): - def __init__(self, name, queue): - pass - - def run(self): - pass - - -class QueuesTest(unittest.TestCase): - mock_work_item = Attachment({ - "id" : 1234, - "bug_id" : 345, - "attacher_email": "adam@example.com", - }, None) - - def assert_queue_outputs(self, queue, args=None, work_item=None, expected_stdout=None, expected_stderr=None, options=Mock(), tool=MockBugzillaTool()): - if not expected_stdout: - expected_stdout = {} - if not expected_stderr: - expected_stderr = {} - if not args: - args = [] - if not work_item: - work_item = self.mock_work_item - tool.user.prompt = lambda message: "yes" - - queue.execute(options, args, tool, engine=MockQueueEngine) - - OutputCapture().assert_outputs(self, - queue.queue_log_path, - expected_stdout=expected_stdout.get("queue_log_path", ""), - expected_stderr=expected_stderr.get("queue_log_path", "")) - OutputCapture().assert_outputs(self, - queue.work_item_log_path, - args=[work_item], - expected_stdout=expected_stdout.get("work_item_log_path", ""), - expected_stderr=expected_stderr.get("work_item_log_path", "")) - OutputCapture().assert_outputs(self, - queue.begin_work_queue, - expected_stdout=expected_stdout.get("begin_work_queue", ""), - expected_stderr=expected_stderr.get("begin_work_queue", "")) - OutputCapture().assert_outputs(self, - queue.should_continue_work_queue, - expected_stdout=expected_stdout.get("should_continue_work_queue", ""), expected_stderr=expected_stderr.get("should_continue_work_queue", "")) - OutputCapture().assert_outputs(self, - queue.next_work_item, - expected_stdout=expected_stdout.get("next_work_item", ""), - expected_stderr=expected_stderr.get("next_work_item", "")) - OutputCapture().assert_outputs(self, - queue.should_proceed_with_work_item, - args=[work_item], - expected_stdout=expected_stdout.get("should_proceed_with_work_item", ""), - expected_stderr=expected_stderr.get("should_proceed_with_work_item", "")) - OutputCapture().assert_outputs(self, - queue.process_work_item, - args=[work_item], - expected_stdout=expected_stdout.get("process_work_item", ""), - expected_stderr=expected_stderr.get("process_work_item", "")) - OutputCapture().assert_outputs(self, - queue.handle_unexpected_error, - args=[work_item, "Mock error message"], - expected_stdout=expected_stdout.get("handle_unexpected_error", ""), - expected_stderr=expected_stderr.get("handle_unexpected_error", "")) diff --git a/WebKitTools/Scripts/webkitpy/commands/upload.pyc b/WebKitTools/Scripts/webkitpy/commands/upload.pyc Binary files differdeleted file mode 100644 index a4bd81b..0000000 --- a/WebKitTools/Scripts/webkitpy/commands/upload.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/comments.pyc b/WebKitTools/Scripts/webkitpy/comments.pyc Binary files differdeleted file mode 100644 index ead9e58..0000000 --- a/WebKitTools/Scripts/webkitpy/comments.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/committers.py b/WebKitTools/Scripts/webkitpy/committers.py deleted file mode 100644 index 7af0987..0000000 --- a/WebKitTools/Scripts/webkitpy/committers.py +++ /dev/null @@ -1,270 +0,0 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# WebKit's Python module for committer and reviewer validation - - -class Committer: - - def __init__(self, name, email_or_emails): - self.full_name = name - if isinstance(email_or_emails, str): - self.emails = [email_or_emails] - else: - self.emails = email_or_emails - self.can_review = False - - def bugzilla_email(self): - # FIXME: We're assuming the first email is a valid bugzilla email, - # which might not be right. - return self.emails[0] - - def __str__(self): - return '"%s" <%s>' % (self.full_name, self.emails[0]) - - -class Reviewer(Committer): - - def __init__(self, name, email_or_emails): - Committer.__init__(self, name, email_or_emails) - self.can_review = True - - -# This is intended as a canonical, machine-readable list of all non-reviewer -# committers for WebKit. If your name is missing here and you are a committer, -# please add it. No review needed. All reviewers are committers, so this list -# is only of committers who are not reviewers. - - -committers_unable_to_review = [ - Committer("Aaron Boodman", "aa@chromium.org"), - Committer("Adam Langley", "agl@chromium.org"), - Committer("Albert J. Wong", "ajwong@chromium.org"), - Committer("Alejandro G. Castro", ["alex@igalia.com", "alex@webkit.org"]), - Committer("Alexander Kellett", ["lypanov@mac.com", "a-lists001@lypanov.net", "lypanov@kde.org"]), - Committer("Alexander Pavlov", "apavlov@chromium.org"), - Committer("Andre Boule", "aboule@apple.com"), - Committer("Andrew Wellington", ["andrew@webkit.org", "proton@wiretapped.net"]), - Committer("Andras Becsi", "abecsi@webkit.org"), - Committer("Anthony Ricaud", "rik@webkit.org"), - Committer("Anton Muhin", "antonm@chromium.org"), - Committer("Antonio Gomes", "tonikitoo@webkit.org"), - Committer("Ben Murdoch", "benm@google.com"), - Committer("Benjamin C Meyer", ["ben@meyerhome.net", "ben@webkit.org"]), - Committer("Benjamin Otte", ["otte@gnome.org", "otte@webkit.org"]), - Committer("Brent Fulgham", "bfulgham@webkit.org"), - Committer("Brett Wilson", "brettw@chromium.org"), - Committer("Brian Weinstein", "bweinstein@apple.com"), - Committer("Cameron McCormack", "cam@webkit.org"), - Committer("Carol Szabo", "carol.szabo@nokia.com"), - Committer("Chang Shu", "chang.shu@nokia.com"), - Committer("Chris Fleizach", "cfleizach@apple.com"), - Committer("Chris Jerdonek", "cjerdonek@webkit.org"), - Committer("Chris Marrin", "cmarrin@apple.com"), - Committer("Chris Petersen", "cpetersen@apple.com"), - Committer("Christian Dywan", ["christian@twotoasts.de", "christian@webkit.org"]), - Committer("Collin Jackson", "collinj@webkit.org"), - Committer("Csaba Osztrogonac", "ossy@webkit.org"), - Committer("Daniel Bates", "dbates@webkit.org"), - Committer("David Smith", ["catfish.man@gmail.com", "dsmith@webkit.org"]), - Committer("Dean Jackson", "dino@apple.com"), - Committer("Dirk Pranke", "dpranke@chromium.org"), - Committer("Drew Wilson", "atwilson@chromium.org"), - Committer("Dumitru Daniliuc", "dumi@chromium.org"), - Committer("Eli Fidler", "eli@staikos.net"), - Committer("Enrica Casucci", "enrica@apple.com"), - Committer("Erik Arvidsson", "arv@chromium.org"), - Committer("Eric Roman", "eroman@chromium.org"), - Committer("Feng Qian", "feng@chromium.org"), - Committer("Fumitoshi Ukai", "ukai@chromium.org"), - Committer("Gabor Loki", "loki@webkit.org"), - Committer("Girish Ramakrishnan", ["girish@forwardbias.in", "ramakrishnan.girish@gmail.com"]), - Committer("Graham Dennis", ["Graham.Dennis@gmail.com", "gdennis@webkit.org"]), - Committer("Greg Bolsinga", "bolsinga@apple.com"), - Committer("Hin-Chung Lam", ["hclam@google.com", "hclam@chromium.org"]), - Committer("Jakob Petsovits", ["jpetsovits@rim.com", "jpetso@gmx.at"]), - Committer("Jens Alfke", ["snej@chromium.org", "jens@apple.com"]), - Committer("Jeremy Moskovich", ["playmobil@google.com", "jeremy@chromium.org"]), - Committer("Jessie Berlin", ["jberlin@webkit.org", "jberlin@apple.com"]), - Committer("Jian Li", "jianli@chromium.org"), - Committer("John Abd-El-Malek", "jam@chromium.org"), - Committer("Joost de Valk", ["joost@webkit.org", "webkit-dev@joostdevalk.nl"]), - Committer("Joseph Pecoraro", "joepeck@webkit.org"), - Committer("Julie Parent", ["jparent@google.com", "jparent@chromium.org"]), - Committer("Julien Chaffraix", ["jchaffraix@webkit.org", "julien.chaffraix@gmail.com"]), - Committer("Jungshik Shin", "jshin@chromium.org"), - Committer("Keishi Hattori", "keishi@webkit.org"), - Committer("Kelly Norton", "knorton@google.com"), - Committer("Kenneth Russell", "kbr@google.com"), - Committer("Kent Tamura", "tkent@chromium.org"), - Committer("Krzysztof Kowalczyk", "kkowalczyk@gmail.com"), - Committer("Levi Weintraub", "lweintraub@apple.com"), - Committer("Mads Ager", "ager@chromium.org"), - Committer("Matt Lilek", ["webkit@mattlilek.com", "pewtermoose@webkit.org"]), - Committer("Matt Perry", "mpcomplete@chromium.org"), - Committer("Maxime Britto", ["maxime.britto@gmail.com", "britto@apple.com"]), - Committer("Maxime Simon", ["simon.maxime@gmail.com", "maxime.simon@webkit.org"]), - Committer("Martin Robinson", ["mrobinson@webkit.org", "martin.james.robinson@gmail.com"]), - Committer("Michelangelo De Simone", "michelangelo@webkit.org"), - Committer("Mike Belshe", ["mbelshe@chromium.org", "mike@belshe.com"]), - Committer("Mike Fenton", ["mike.fenton@torchmobile.com", "mifenton@rim.com"]), - Committer("Mike Thole", ["mthole@mikethole.com", "mthole@apple.com"]), - Committer("Mikhail Naganov", "mnaganov@chromium.org"), - Committer("Ojan Vafai", "ojan@chromium.org"), - Committer("Pam Greene", "pam@chromium.org"), - Committer("Peter Kasting", ["pkasting@google.com", "pkasting@chromium.org"]), - Committer("Philippe Normand", ["pnormand@igalia.com", "philn@webkit.org"]), - Committer("Pierre d'Herbemont", ["pdherbemont@free.fr", "pdherbemont@apple.com"]), - Committer("Pierre-Olivier Latour", "pol@apple.com"), - Committer("Roland Steiner", "rolandsteiner@chromium.org"), - Committer("Ryosuke Niwa", "rniwa@webkit.org"), - Committer("Scott Violet", "sky@chromium.org"), - Committer("Stephen White", "senorblanco@chromium.org"), - Committer("Steve Block", "steveblock@google.com"), - Committer("Tony Chang", "tony@chromium.org"), - Committer("Trey Matteson", "trey@usa.net"), - Committer("Tristan O'Tierney", ["tristan@otierney.net", "tristan@apple.com"]), - Committer("Victor Wang", "victorw@chromium.org"), - Committer("William Siegrist", "wsiegrist@apple.com"), - Committer("Yael Aharon", "yael.aharon@nokia.com"), - Committer("Yaar Schnitman", ["yaar@chromium.org", "yaar@google.com"]), - Committer("Yong Li", ["yong.li@torchmobile.com", "yong.li.webkit@gmail.com"]), - Committer("Yongjun Zhang", "yongjun.zhang@nokia.com"), - Committer("Yury Semikhatsky", "yurys@chromium.org"), - Committer("Yuzo Fujishima", "yuzo@google.com"), - Committer("Zoltan Herczeg", "zherczeg@webkit.org"), - Committer("Zoltan Horvath", "zoltan@webkit.org"), -] - - -# This is intended as a canonical, machine-readable list of all reviewers for -# WebKit. If your name is missing here and you are a reviewer, please add it. -# No review needed. - - -reviewers_list = [ - Reviewer("Ada Chan", "adachan@apple.com"), - Reviewer("Adam Barth", "abarth@webkit.org"), - Reviewer("Adam Roben", "aroben@apple.com"), - Reviewer("Adam Treat", ["treat@kde.org", "treat@webkit.org"]), - Reviewer("Adele Peterson", "adele@apple.com"), - Reviewer("Alexey Proskuryakov", ["ap@webkit.org", "ap@apple.com"]), - Reviewer("Alice Liu", "alice.liu@apple.com"), - Reviewer("Alp Toker", ["alp@nuanti.com", "alp@atoker.com", "alp@webkit.org"]), - Reviewer("Anders Carlsson", ["andersca@apple.com", "acarlsson@apple.com"]), - Reviewer("Antti Koivisto", ["koivisto@iki.fi", "antti@apple.com"]), - Reviewer("Ariya Hidayat", ["ariya.hidayat@gmail.com", "ariya@webkit.org"]), - Reviewer("Beth Dakin", "bdakin@apple.com"), - Reviewer("Brady Eidson", "beidson@apple.com"), - Reviewer("Cameron Zwarich", ["zwarich@apple.com", "cwzwarich@apple.com", "cwzwarich@webkit.org"]), - Reviewer("Chris Blumenberg", "cblu@apple.com"), - Reviewer("Dan Bernstein", ["mitz@webkit.org", "mitz@apple.com"]), - Reviewer("Darin Adler", "darin@apple.com"), - Reviewer("Darin Fisher", ["fishd@chromium.org", "darin@chromium.org"]), - Reviewer("David Harrison", "harrison@apple.com"), - Reviewer("David Hyatt", "hyatt@apple.com"), - Reviewer("David Kilzer", ["ddkilzer@webkit.org", "ddkilzer@apple.com"]), - Reviewer("David Levin", "levin@chromium.org"), - Reviewer("Dimitri Glazkov", "dglazkov@chromium.org"), - Reviewer("Dirk Schulze", "krit@webkit.org"), - Reviewer("Dmitry Titov", "dimich@chromium.org"), - Reviewer("Don Melton", "gramps@apple.com"), - Reviewer("Eric Carlson", "eric.carlson@apple.com"), - Reviewer("Eric Seidel", "eric@webkit.org"), - Reviewer("Gavin Barraclough", "barraclough@apple.com"), - Reviewer("Geoffrey Garen", "ggaren@apple.com"), - Reviewer("George Staikos", ["staikos@kde.org", "staikos@webkit.org"]), - Reviewer("Gustavo Noronha Silva", ["gns@gnome.org", "kov@webkit.org"]), - Reviewer("Holger Freyther", ["zecke@selfish.org", "zecke@webkit.org"]), - Reviewer("Jan Alonzo", ["jmalonzo@gmail.com", "jmalonzo@webkit.org"]), - Reviewer("Jeremy Orlow", "jorlow@chromium.org"), - Reviewer("John Sullivan", "sullivan@apple.com"), - Reviewer("Jon Honeycutt", "jhoneycutt@apple.com"), - Reviewer("Justin Garcia", "justin.garcia@apple.com"), - Reviewer("Ken Kocienda", "kocienda@apple.com"), - Reviewer("Kenneth Rohde Christiansen", ["kenneth@webkit.org", "kenneth.christiansen@openbossa.org"]), - Reviewer("Kevin Decker", "kdecker@apple.com"), - Reviewer("Kevin McCullough", "kmccullough@apple.com"), - Reviewer("Kevin Ollivier", ["kevino@theolliviers.com", "kevino@webkit.org"]), - Reviewer("Lars Knoll", ["lars@trolltech.com", "lars@kde.org"]), - Reviewer("Laszlo Gombos", "laszlo.1.gombos@nokia.com"), - Reviewer("Maciej Stachowiak", "mjs@apple.com"), - Reviewer("Mark Rowe", "mrowe@apple.com"), - Reviewer("Nate Chapin", "japhet@chromium.org"), - Reviewer("Nikolas Zimmermann", ["zimmermann@kde.org", "zimmermann@physik.rwth-aachen.de", "zimmermann@webkit.org"]), - Reviewer("Oliver Hunt", "oliver@apple.com"), - Reviewer("Pavel Feldman", "pfeldman@chromium.org"), - Reviewer("Richard Williamson", "rjw@apple.com"), - Reviewer("Rob Buis", ["rwlbuis@gmail.com", "rwlbuis@webkit.org"]), - Reviewer("Sam Weinig", ["sam@webkit.org", "weinig@apple.com"]), - Reviewer("Shinichiro Hamaji", "hamaji@chromium.org"), - Reviewer("Simon Fraser", "simon.fraser@apple.com"), - Reviewer("Simon Hausmann", ["hausmann@webkit.org", "hausmann@kde.org", "simon.hausmann@nokia.com"]), - Reviewer("Stephanie Lewis", "slewis@apple.com"), - Reviewer("Steve Falkenburg", "sfalken@apple.com"), - Reviewer("Tim Omernick", "timo@apple.com"), - Reviewer("Timothy Hatcher", ["timothy@hatcher.name", "timothy@apple.com"]), - Reviewer(u'Tor Arne Vestb\xf8', "vestbo@webkit.org"), - Reviewer("Vicki Murley", "vicki@apple.com"), - Reviewer("Xan Lopez", ["xan.lopez@gmail.com", "xan@gnome.org", "xan@webkit.org"]), - Reviewer("Zack Rusin", "zack@kde.org"), -] - - -class CommitterList: - - # Committers and reviewers are passed in to allow easy testing - - def __init__(self, - committers=committers_unable_to_review, - reviewers=reviewers_list): - self._committers = committers + reviewers - self._reviewers = reviewers - self._committers_by_email = {} - - def committers(self): - return self._committers - - def reviewers(self): - return self._reviewers - - def _email_to_committer_map(self): - if not len(self._committers_by_email): - for committer in self._committers: - for email in committer.emails: - self._committers_by_email[email] = committer - return self._committers_by_email - - def committer_by_email(self, email): - return self._email_to_committer_map().get(email) - - def reviewer_by_email(self, email): - committer = self.committer_by_email(email) - if committer and not committer.can_review: - return None - return committer diff --git a/WebKitTools/Scripts/webkitpy/committers.pyc b/WebKitTools/Scripts/webkitpy/committers.pyc Binary files differdeleted file mode 100644 index bce8c17..0000000 --- a/WebKitTools/Scripts/webkitpy/committers.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/commands/__init__.py b/WebKitTools/Scripts/webkitpy/common/__init__.py index ef65bee..ef65bee 100644 --- a/WebKitTools/Scripts/webkitpy/commands/__init__.py +++ b/WebKitTools/Scripts/webkitpy/common/__init__.py diff --git a/WebKitTools/Scripts/webkitpy/common/array_stream.py b/WebKitTools/Scripts/webkitpy/common/array_stream.py new file mode 100644 index 0000000..e425d02 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/array_stream.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Package that private an array-based implementation of a stream.""" + + +class ArrayStream(object): + """Simple class that implmements a stream interface on top of an array. + + This is used primarily by unit test classes to mock output streams. It + performs a similar function to StringIO, but (a) it is write-only, and + (b) it can be used to retrieve each individual write(); StringIO + concatenates all of the writes together. + """ + + def __init__(self): + self._contents = [] + + def write(self, msg): + """Implement stream.write() by appending to the stream's contents.""" + self._contents.append(msg) + + def get(self): + """Return the contents of a stream (as an array).""" + return self._contents + + def reset(self): + """Empty the stream.""" + self._contents = [] + + def empty(self): + """Return whether the stream is empty.""" + return (len(self._contents) == 0) + + def flush(self): + """Flush the stream (a no-op implemented for compatibility).""" + pass + + def __repr__(self): + return '<ArrayStream: ' + str(self._contents) + '>' diff --git a/WebKitTools/Scripts/webkitpy/steps/steps_unittest.py b/WebKitTools/Scripts/webkitpy/common/array_stream_unittest.py index 3e6a032..1a9b34a 100644 --- a/WebKitTools/Scripts/webkitpy/steps/steps_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/array_stream_unittest.py @@ -1,9 +1,10 @@ +#!/usr/bin/python # Copyright (C) 2010 Google Inc. All rights reserved. -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above @@ -13,7 +14,7 @@ # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -26,31 +27,52 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Unit tests for array_stream.py.""" + +import pdb import unittest -from webkitpy.steps.update import Update -from webkitpy.steps.promptforbugortitle import PromptForBugOrTitle -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture -from webkitpy.mock import Mock - - -class StepsTest(unittest.TestCase): - def _run_step(self, step, tool=None, options=None, state=None): - if not tool: - tool = MockBugzillaTool() - if not options: - options = Mock() - if not state: - state = {} - step(tool, options).run(state) - - def test_update_step(self): - options = Mock() - options.update = True - self._run_step(Update, options) - - def test_prompt_for_bug_or_title_step(self): - tool = MockBugzillaTool() - tool.user.prompt = lambda message: 42 - self._run_step(PromptForBugOrTitle, tool=tool) +from webkitpy.common.array_stream import ArrayStream + + +class ArrayStreamTest(unittest.TestCase): + def assertEmpty(self, a_stream): + self.assertTrue(a_stream.empty()) + + def assertNotEmpty(self, a_stream): + self.assertFalse(a_stream.empty()) + + def assertContentsMatch(self, a_stream, contents): + self.assertEquals(a_stream.get(), contents) + + def test_basics(self): + a = ArrayStream() + self.assertEmpty(a) + self.assertContentsMatch(a, []) + + a.flush() + self.assertEmpty(a) + self.assertContentsMatch(a, []) + + a.write("foo") + a.write("bar") + self.assertNotEmpty(a) + self.assertContentsMatch(a, ["foo", "bar"]) + + a.flush() + self.assertNotEmpty(a) + self.assertContentsMatch(a, ["foo", "bar"]) + + a.reset() + self.assertEmpty(a) + self.assertContentsMatch(a, []) + + self.assertEquals(str(a), "<ArrayStream: []>") + + a.write("foo") + self.assertNotEmpty(a) + self.assertContentsMatch(a, ["foo"]) + self.assertEquals(str(a), "<ArrayStream: ['foo']>") + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/style/processors/__init__.py b/WebKitTools/Scripts/webkitpy/common/checkout/__init__.py index ef65bee..ef65bee 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/__init__.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/__init__.py diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/api.py b/WebKitTools/Scripts/webkitpy/common/checkout/api.py new file mode 100644 index 0000000..ca28e32 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/api.py @@ -0,0 +1,140 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import StringIO + +from webkitpy.common.checkout.changelog import ChangeLog +from webkitpy.common.checkout.commitinfo import CommitInfo +from webkitpy.common.checkout.scm import CommitMessage +from webkitpy.common.net.bugzilla import parse_bug_id +from webkitpy.common.system.executive import Executive, run_command, ScriptError +from webkitpy.common.system.deprecated_logging import log + + +# This class represents the WebKit-specific parts of the checkout (like +# ChangeLogs). +# FIXME: Move a bunch of ChangeLog-specific processing from SCM to this object. +class Checkout(object): + def __init__(self, scm): + self._scm = scm + + def _is_path_to_changelog(self, path): + return os.path.basename(path) == "ChangeLog" + + def _latest_entry_for_changelog_at_revision(self, changelog_path, revision): + changelog_contents = self._scm.contents_at_revision(changelog_path, revision) + # contents_at_revision returns a byte array (str()), but we know + # that ChangeLog files are utf-8. parse_latest_entry_from_file + # expects a file-like object which vends unicode(), so we decode here. + changelog_file = StringIO.StringIO(changelog_contents.decode("utf-8")) + return ChangeLog.parse_latest_entry_from_file(changelog_file) + + def changelog_entries_for_revision(self, revision): + changed_files = self._scm.changed_files_for_revision(revision) + return [self._latest_entry_for_changelog_at_revision(path, revision) for path in changed_files if self._is_path_to_changelog(path)] + + def commit_info_for_revision(self, revision): + committer_email = self._scm.committer_email_for_revision(revision) + changelog_entries = self.changelog_entries_for_revision(revision) + # Assume for now that the first entry has everything we need: + # FIXME: This will throw an exception if there were no ChangeLogs. + if not len(changelog_entries): + return None + changelog_entry = changelog_entries[0] + changelog_data = { + "bug_id": parse_bug_id(changelog_entry.contents()), + "author_name": changelog_entry.author_name(), + "author_email": changelog_entry.author_email(), + "author": changelog_entry.author(), + "reviewer_text": changelog_entry.reviewer_text(), + "reviewer": changelog_entry.reviewer(), + } + # We could pass the changelog_entry instead of a dictionary here, but that makes + # mocking slightly more involved, and would make aggregating data from multiple + # entries more difficult to wire in if we need to do that in the future. + return CommitInfo(revision, committer_email, changelog_data) + + def bug_id_for_revision(self, revision): + return self.commit_info_for_revision(revision).bug_id() + + def modified_changelogs(self, git_commit): + # SCM returns paths relative to scm.checkout_root + # Callers (especially those using the ChangeLog class) may + # expect absolute paths, so this method returns absolute paths. + changed_files = self._scm.changed_files(git_commit) + absolute_paths = [os.path.join(self._scm.checkout_root, path) for path in changed_files] + return [path for path in absolute_paths if self._is_path_to_changelog(path)] + + def commit_message_for_this_commit(self, git_commit): + changelog_paths = self.modified_changelogs(git_commit) + if not len(changelog_paths): + raise ScriptError(message="Found no modified ChangeLogs, cannot create a commit message.\n" + "All changes require a ChangeLog. See:\n" + "http://webkit.org/coding/contributing.html") + + changelog_messages = [] + for changelog_path in changelog_paths: + log("Parsing ChangeLog: %s" % changelog_path) + changelog_entry = ChangeLog(changelog_path).latest_entry() + if not changelog_entry: + raise ScriptError(message="Failed to parse ChangeLog: %s" % os.path.abspath(changelog_path)) + changelog_messages.append(changelog_entry.contents()) + + # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does. + return CommitMessage("".join(changelog_messages).splitlines()) + + def bug_id_for_this_commit(self, git_commit): + try: + return parse_bug_id(self.commit_message_for_this_commit(git_commit).message()) + except ScriptError, e: + pass # We might not have ChangeLogs. + + def apply_patch(self, patch, force=False): + # It's possible that the patch was not made from the root directory. + # We should detect and handle that case. + # FIXME: Move _scm.script_path here once we get rid of all the dependencies. + args = [self._scm.script_path('svn-apply')] + if patch.reviewer(): + args += ['--reviewer', patch.reviewer().full_name] + if force: + args.append('--force') + run_command(args, input=patch.contents()) + + def apply_reverse_diff(self, revision): + self._scm.apply_reverse_diff(revision) + + # We revert the ChangeLogs because removing lines from a ChangeLog + # doesn't make sense. ChangeLogs are append only. + changelog_paths = self.modified_changelogs(git_commit=None) + if len(changelog_paths): + self._scm.revert_files(changelog_paths) + + conflicts = self._scm.conflicted_files() + if len(conflicts): + raise ScriptError(message="Failed to apply reverse diff for revision %s because of the following conflicts:\n%s" % (revision, "\n".join(conflicts))) diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/api_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/api_unittest.py new file mode 100644 index 0000000..fdfd879 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/api_unittest.py @@ -0,0 +1,175 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import with_statement + +import codecs +import os +import shutil +import tempfile +import unittest + +from webkitpy.common.checkout.api import Checkout +from webkitpy.common.checkout.changelog import ChangeLogEntry +from webkitpy.common.checkout.scm import detect_scm_system, CommitMessage +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock + + +# FIXME: Copied from scm_unittest.py +def write_into_file_at_path(file_path, contents, encoding="utf-8"): + with codecs.open(file_path, "w", encoding) as file: + file.write(contents) + + +_changelog1entry1 = u"""2010-03-25 Tor Arne Vestb\u00f8 <vestbo@webkit.org> + + Unreviewed build fix to un-break webkit-patch land. + + Move commit_message_for_this_commit from scm to checkout + https://bugs.webkit.org/show_bug.cgi?id=36629 + + * Scripts/webkitpy/common/checkout/api.py: import scm.CommitMessage +""" +_changelog1entry2 = u"""2010-03-25 Adam Barth <abarth@webkit.org> + + Reviewed by Eric Seidel. + + Move commit_message_for_this_commit from scm to checkout + https://bugs.webkit.org/show_bug.cgi?id=36629 + + * Scripts/webkitpy/common/checkout/api.py: +""" +_changelog1 = u"\n".join([_changelog1entry1, _changelog1entry2]) +_changelog2 = u"""2010-03-25 Tor Arne Vestb\u00f8 <vestbo@webkit.org> + + Unreviewed build fix to un-break webkit-patch land. + + Second part of this complicated change. + + * Path/To/Complicated/File: Added. + +2010-03-25 Adam Barth <abarth@webkit.org> + + Reviewed by Eric Seidel. + + Filler change. +""" + +class CommitMessageForThisCommitTest(unittest.TestCase): + expected_commit_message = u"""2010-03-25 Tor Arne Vestb\u00f8 <vestbo@webkit.org> + + Unreviewed build fix to un-break webkit-patch land. + + Move commit_message_for_this_commit from scm to checkout + https://bugs.webkit.org/show_bug.cgi?id=36629 + + * Scripts/webkitpy/common/checkout/api.py: import scm.CommitMessage +2010-03-25 Tor Arne Vestb\u00f8 <vestbo@webkit.org> + + Unreviewed build fix to un-break webkit-patch land. + + Second part of this complicated change. + + * Path/To/Complicated/File: Added. +""" + + def setUp(self): + self.temp_dir = tempfile.mkdtemp(suffix="changelogs") + self.old_cwd = os.getcwd() + os.chdir(self.temp_dir) + write_into_file_at_path("ChangeLog1", _changelog1) + write_into_file_at_path("ChangeLog2", _changelog2) + + def tearDown(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + os.chdir(self.old_cwd) + + # FIXME: This should not need to touch the file system, however + # ChangeLog is difficult to mock at current. + def test_commit_message_for_this_commit(self): + checkout = Checkout(None) + checkout.modified_changelogs = lambda git_commit: ["ChangeLog1", "ChangeLog2"] + output = OutputCapture() + expected_stderr = "Parsing ChangeLog: ChangeLog1\nParsing ChangeLog: ChangeLog2\n" + commit_message = output.assert_outputs(self, checkout.commit_message_for_this_commit, + kwargs={"git_commit": None}, expected_stderr=expected_stderr) + self.assertEqual(commit_message.message(), self.expected_commit_message) + + +class CheckoutTest(unittest.TestCase): + def test_latest_entry_for_changelog_at_revision(self): + scm = Mock() + def mock_contents_at_revision(changelog_path, revision): + self.assertEqual(changelog_path, "foo") + self.assertEqual(revision, "bar") + # contents_at_revision is expected to return a byte array (str) + # so we encode our unicode ChangeLog down to a utf-8 stream. + return _changelog1.encode("utf-8") + scm.contents_at_revision = mock_contents_at_revision + checkout = Checkout(scm) + entry = checkout._latest_entry_for_changelog_at_revision("foo", "bar") + self.assertEqual(entry.contents(), _changelog1entry1) + + def test_commit_info_for_revision(self): + scm = Mock() + scm.committer_email_for_revision = lambda revision: "committer@example.com" + checkout = Checkout(scm) + checkout.changelog_entries_for_revision = lambda revision: [ChangeLogEntry(_changelog1entry1)] + commitinfo = checkout.commit_info_for_revision(4) + self.assertEqual(commitinfo.bug_id(), 36629) + self.assertEqual(commitinfo.author_name(), u"Tor Arne Vestb\u00f8") + self.assertEqual(commitinfo.author_email(), "vestbo@webkit.org") + self.assertEqual(commitinfo.reviewer_text(), None) + self.assertEqual(commitinfo.reviewer(), None) + self.assertEqual(commitinfo.committer_email(), "committer@example.com") + self.assertEqual(commitinfo.committer(), None) + + checkout.changelog_entries_for_revision = lambda revision: [] + self.assertEqual(checkout.commit_info_for_revision(1), None) + + def test_bug_id_for_revision(self): + scm = Mock() + scm.committer_email_for_revision = lambda revision: "committer@example.com" + checkout = Checkout(scm) + checkout.changelog_entries_for_revision = lambda revision: [ChangeLogEntry(_changelog1entry1)] + self.assertEqual(checkout.bug_id_for_revision(4), 36629) + + def test_bug_id_for_this_commit(self): + scm = Mock() + checkout = Checkout(scm) + checkout.commit_message_for_this_commit = lambda git_commit: CommitMessage(ChangeLogEntry(_changelog1entry1).contents().splitlines()) + self.assertEqual(checkout.bug_id_for_this_commit(git_commit=None), 36629) + + def test_modified_changelogs(self): + scm = Mock() + scm.checkout_root = "/foo/bar" + scm.changed_files = lambda git_commit: ["file1", "ChangeLog", "relative/path/ChangeLog"] + checkout = Checkout(scm) + expected_changlogs = ["/foo/bar/ChangeLog", "/foo/bar/relative/path/ChangeLog"] + self.assertEqual(checkout.modified_changelogs(git_commit=None), expected_changlogs) diff --git a/WebKitTools/Scripts/webkitpy/changelogs.py b/WebKitTools/Scripts/webkitpy/common/checkout/changelog.py index ebc89c4..40657eb 100644 --- a/WebKitTools/Scripts/webkitpy/changelogs.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/changelog.py @@ -28,10 +28,16 @@ # # WebKit's Python module for parsing and modifying ChangeLog files +import codecs import fileinput # inplace file editing for set_reviewer_in_changelog +import os.path import re import textwrap +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.config.committers import CommitterList +from webkitpy.common.net.bugzilla import parse_bug_id + def view_source_url(revision_number): # FIMXE: This doesn't really belong in this file, but we don't have a @@ -40,39 +46,89 @@ def view_source_url(revision_number): return "http://trac.webkit.org/changeset/%s" % revision_number -class ChangeLog: +class ChangeLogEntry(object): + # e.g. 2009-06-03 Eric Seidel <eric@webkit.org> + date_line_regexp = r'^(?P<date>\d{4}-\d{2}-\d{2})\s+(?P<name>.+?)\s+<(?P<email>[^<>]+)>$' + + def __init__(self, contents, committer_list=CommitterList()): + self._contents = contents + self._committer_list = committer_list + self._parse_entry() + + def _parse_entry(self): + match = re.match(self.date_line_regexp, self._contents, re.MULTILINE) + if not match: + log("WARNING: Creating invalid ChangeLogEntry:\n%s" % self._contents) + + # FIXME: group("name") does not seem to be Unicode? Probably due to self._contents not being unicode. + self._author_name = match.group("name") if match else None + self._author_email = match.group("email") if match else None + + match = re.search("^\s+Reviewed by (?P<reviewer>.*?)[\.,]?\s*$", self._contents, re.MULTILINE) # Discard everything after the first period + self._reviewer_text = match.group("reviewer") if match else None + + self._reviewer = self._committer_list.committer_by_name(self._reviewer_text) + self._author = self._committer_list.committer_by_email(self._author_email) or self._committer_list.committer_by_name(self._author_name) + + def author_name(self): + return self._author_name + + def author_email(self): + return self._author_email + + def author(self): + return self._author # Might be None + + # FIXME: Eventually we would like to map reviwer names to reviewer objects. + # See https://bugs.webkit.org/show_bug.cgi?id=26533 + def reviewer_text(self): + return self._reviewer_text + + def reviewer(self): + return self._reviewer # Might be None + + def contents(self): + return self._contents + + def bug_id(self): + return parse_bug_id(self._contents) + + +# FIXME: Various methods on ChangeLog should move into ChangeLogEntry instead. +class ChangeLog(object): def __init__(self, path): self.path = path _changelog_indent = " " * 8 - # e.g. 2009-06-03 Eric Seidel <eric@webkit.org> - date_line_regexp = re.compile('^(\d{4}-\d{2}-\d{2})' # Consume the date. - + '\s+(.+)\s+' # Consume the name. - + '<([^<>]+)>$') # And the email address. - @staticmethod - def _parse_latest_entry_from_file(changelog_file): + def parse_latest_entry_from_file(changelog_file): + """changelog_file must be a file-like object which returns + unicode strings. Use codecs.open or StringIO(unicode()) + to pass file objects to this class.""" + date_line_regexp = re.compile(ChangeLogEntry.date_line_regexp) entry_lines = [] # The first line should be a date line. first_line = changelog_file.readline() - if not ChangeLog.date_line_regexp.match(first_line): + assert(isinstance(first_line, unicode)) + if not date_line_regexp.match(first_line): return None entry_lines.append(first_line) for line in changelog_file: # If we've hit the next entry, return. - if ChangeLog.date_line_regexp.match(line): + if date_line_regexp.match(line): # Remove the extra newline at the end - return ''.join(entry_lines[:-1]) + return ChangeLogEntry(''.join(entry_lines[:-1])) entry_lines.append(line) return None # We never found a date line! def latest_entry(self): - changelog_file = open(self.path) + # ChangeLog files are always UTF-8, we read them in as such to support Reviewers with unicode in their names. + changelog_file = codecs.open(self.path, "r", "utf-8") try: - return self._parse_latest_entry_from_file(changelog_file) + return self.parse_latest_entry_from_file(changelog_file) finally: changelog_file.close() @@ -96,7 +152,7 @@ class ChangeLog: # This probably does not belong in changelogs.py def _message_for_revert(self, revision, reason, bug_url): - message = "No review, rolling out r%s.\n" % revision + message = "Unreviewed, rolling out r%s.\n" % revision message += "%s\n" % view_source_url(revision) if bug_url: message += "%s\n" % bug_url @@ -132,3 +188,8 @@ class ChangeLog: for line in fileinput.FileInput(self.path, inplace=1): # Trailing comma suppresses printing newline print line.replace("NOBODY (OOPS!)", reviewer.encode("utf-8")), + + def set_short_description_and_bug_url(self, short_description, bug_url): + message = "%s\n %s" % (short_description, bug_url) + for line in fileinput.FileInput(self.path, inplace=1): + print line.replace("Need a short description and bug URL (OOPS!)", message.encode("utf-8")), diff --git a/WebKitTools/Scripts/webkitpy/changelogs_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/changelog_unittest.py index de3e60c..6aeb1f8 100644 --- a/WebKitTools/Scripts/webkitpy/changelogs_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/changelog_unittest.py @@ -26,18 +26,23 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import unittest -from changelogs import * +from __future__ import with_statement +import codecs import os import tempfile +import unittest + from StringIO import StringIO -class ChangeLogsTest(unittest.TestCase): +from webkitpy.common.checkout.changelog import * - _example_entry = '''2009-08-17 Peter Kasting <pkasting@google.com> - Reviewed by Steve Falkenburg. +class ChangeLogTest(unittest.TestCase): + + _example_entry = u'''2009-08-17 Peter Kasting <pkasting@google.com> + + Reviewed by Tor Arne Vestb\xf8. https://bugs.webkit.org/show_bug.cgi?id=27323 Only add Cygwin to the path when it isn't already there. This avoids @@ -50,7 +55,7 @@ class ChangeLogsTest(unittest.TestCase): ''' # More example text than we need. Eventually we need to support parsing this all and write tests for the parsing. - _example_changelog = '''2009-08-17 David Kilzer <ddkilzer@apple.com> + _example_changelog = u"""2009-08-17 Tor Arne Vestb\xf8 <vestbo@webkit.org> <http://webkit.org/b/28393> check-webkit-style: add check for use of std::max()/std::min() instead of MAX()/MIN() @@ -82,28 +87,30 @@ class ChangeLogsTest(unittest.TestCase): so we can't assert here. == Rolled over to ChangeLog-2009-06-16 == -''' +""" def test_latest_entry_parse(self): - changelog_contents = "%s\n%s" % (self._example_entry, self._example_changelog) + changelog_contents = u"%s\n%s" % (self._example_entry, self._example_changelog) changelog_file = StringIO(changelog_contents) - latest_entry = ChangeLog._parse_latest_entry_from_file(changelog_file) - self.assertEquals(self._example_entry, latest_entry) + latest_entry = ChangeLog.parse_latest_entry_from_file(changelog_file) + self.assertEquals(latest_entry.contents(), self._example_entry) + self.assertEquals(latest_entry.author_name(), "Peter Kasting") + self.assertEquals(latest_entry.author_email(), "pkasting@google.com") + self.assertEquals(latest_entry.reviewer_text(), u"Tor Arne Vestb\xf8") + self.assertTrue(latest_entry.reviewer()) # Make sure that our UTF8-based lookup of Tor works. @staticmethod - def _write_tmp_file_with_contents(contents): + def _write_tmp_file_with_contents(byte_array): + assert(isinstance(byte_array, str)) (file_descriptor, file_path) = tempfile.mkstemp() # NamedTemporaryFile always deletes the file on close in python < 2.6 - file = os.fdopen(file_descriptor, 'w') - file.write(contents) - file.close() + with os.fdopen(file_descriptor, "w") as file: + file.write(byte_array) return file_path @staticmethod - def _read_file_contents(file_path): - file = open(file_path) - contents = file.read() - file.close() - return contents + def _read_file_contents(file_path, encoding): + with codecs.open(file_path, "r", encoding) as file: + return file.read() _new_entry_boilerplate = '''2009-08-19 Eric Seidel <eric@webkit.org> @@ -115,16 +122,28 @@ class ChangeLogsTest(unittest.TestCase): ''' def test_set_reviewer(self): - changelog_contents = "%s\n%s" % (self._new_entry_boilerplate, self._example_changelog) - changelog_path = self._write_tmp_file_with_contents(changelog_contents) + changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate, self._example_changelog) + changelog_path = self._write_tmp_file_with_contents(changelog_contents.encode("utf-8")) reviewer_name = 'Test Reviewer' ChangeLog(changelog_path).set_reviewer(reviewer_name) - actual_contents = self._read_file_contents(changelog_path) + actual_contents = self._read_file_contents(changelog_path, "utf-8") expected_contents = changelog_contents.replace('NOBODY (OOPS!)', reviewer_name) os.remove(changelog_path) self.assertEquals(actual_contents, expected_contents) - _revert_message = """ No review, rolling out r12345. + def test_set_short_description_and_bug_url(self): + changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate, self._example_changelog) + changelog_path = self._write_tmp_file_with_contents(changelog_contents.encode("utf-8")) + short_description = "A short description" + bug_url = "http://example.com/b/2344" + ChangeLog(changelog_path).set_short_description_and_bug_url(short_description, bug_url) + actual_contents = self._read_file_contents(changelog_path, "utf-8") + expected_message = "%s\n %s" % (short_description, bug_url) + expected_contents = changelog_contents.replace("Need a short description and bug URL (OOPS!)", expected_message) + os.remove(changelog_path) + self.assertEquals(actual_contents, expected_contents) + + _revert_message = """ Unreviewed, rolling out r12345. http://trac.webkit.org/changeset/12345 http://example.com/123 @@ -143,7 +162,7 @@ class ChangeLogsTest(unittest.TestCase): _revert_entry_with_bug_url = '''2009-08-19 Eric Seidel <eric@webkit.org> - No review, rolling out r12345. + Unreviewed, rolling out r12345. http://trac.webkit.org/changeset/12345 http://example.com/123 @@ -154,7 +173,7 @@ class ChangeLogsTest(unittest.TestCase): _revert_entry_without_bug_url = '''2009-08-19 Eric Seidel <eric@webkit.org> - No review, rolling out r12345. + Unreviewed, rolling out r12345. http://trac.webkit.org/changeset/12345 Reason @@ -163,17 +182,22 @@ class ChangeLogsTest(unittest.TestCase): ''' def _assert_update_for_revert_output(self, args, expected_entry): - changelog_contents = "%s\n%s" % (self._new_entry_boilerplate, self._example_changelog) - changelog_path = self._write_tmp_file_with_contents(changelog_contents) + changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate, self._example_changelog) + changelog_path = self._write_tmp_file_with_contents(changelog_contents.encode("utf-8")) changelog = ChangeLog(changelog_path) changelog.update_for_revert(*args) actual_entry = changelog.latest_entry() os.remove(changelog_path) - self.assertEquals(actual_entry, expected_entry) + self.assertEquals(actual_entry.contents(), expected_entry) + self.assertEquals(actual_entry.reviewer_text(), None) + # These checks could be removed to allow this to work on other entries: + self.assertEquals(actual_entry.author_name(), "Eric Seidel") + self.assertEquals(actual_entry.author_email(), "eric@webkit.org") def test_update_for_revert(self): self._assert_update_for_revert_output([12345, "Reason"], self._revert_entry_without_bug_url) self._assert_update_for_revert_output([12345, "Reason", "http://example.com/123"], self._revert_entry_with_bug_url) + if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo.py b/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo.py new file mode 100644 index 0000000..448d530 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo.py @@ -0,0 +1,93 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# WebKit's python module for holding information on a commit + +from webkitpy.common.checkout.changelog import view_source_url +from webkitpy.common.config.committers import CommitterList + + +class CommitInfo(object): + def __init__(self, revision, committer_email, changelog_data, committer_list=CommitterList()): + self._revision = revision + self._committer_email = committer_email + self._bug_id = changelog_data["bug_id"] + self._author_name = changelog_data["author_name"] + self._author_email = changelog_data["author_email"] + self._author = changelog_data["author"] + self._reviewer_text = changelog_data["reviewer_text"] + self._reviewer = changelog_data["reviewer"] + + # Derived values: + self._committer = committer_list.committer_by_email(committer_email) + + def revision(self): + return self._revision + + def committer(self): + return self._committer # None if committer isn't in committers.py + + def committer_email(self): + return self._committer_email + + def bug_id(self): + return self._bug_id # May be None + + def author(self): + return self._author # May be None + + def author_name(self): + return self._author_name + + def author_email(self): + return self._author_email + + def reviewer(self): + return self._reviewer # May be None + + def reviewer_text(self): + return self._reviewer_text # May be None + + def responsible_parties(self): + responsible_parties = [ + self.committer(), + self.author(), + self.reviewer(), + ] + return set([party for party in responsible_parties if party]) # Filter out None + + # FIXME: It is slightly lame that this "view" method is on this "model" class (in MVC terms) + def blame_string(self, bugs): + string = "r%s:\n" % self.revision() + string += " %s\n" % view_source_url(self.revision()) + string += " Bug: %s (%s)\n" % (self.bug_id(), bugs.bug_url_for_bug_id(self.bug_id())) + author_line = "\"%s\" <%s>" % (self.author_name(), self.author_email()) + string += " Author: %s\n" % (self.author() or author_line) + string += " Reviewer: %s\n" % (self.reviewer() or self.reviewer_text()) + string += " Committer: %s" % self.committer() + return string diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo_unittest.py new file mode 100644 index 0000000..f58e6f1 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/commitinfo_unittest.py @@ -0,0 +1,61 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.checkout.commitinfo import CommitInfo +from webkitpy.common.config.committers import CommitterList, Committer, Reviewer + +class CommitInfoTest(unittest.TestCase): + + def test_commit_info_creation(self): + author = Committer("Author", "author@example.com") + committer = Committer("Committer", "committer@example.com") + reviewer = Reviewer("Reviewer", "reviewer@example.com") + committer_list = CommitterList(committers=[author, committer], reviewers=[reviewer]) + + changelog_data = { + "bug_id": 1234, + "author_name": "Committer", + "author_email": "author@example.com", + "author": author, + "reviewer_text": "Reviewer", + "reviewer": reviewer, + } + commit = CommitInfo(123, "committer@example.com", changelog_data, committer_list) + + self.assertEqual(commit.revision(), 123) + self.assertEqual(commit.bug_id(), 1234) + self.assertEqual(commit.author_name(), "Committer") + self.assertEqual(commit.author_email(), "author@example.com") + self.assertEqual(commit.author(), author) + self.assertEqual(commit.reviewer_text(), "Reviewer") + self.assertEqual(commit.reviewer(), reviewer) + self.assertEqual(commit.committer(), committer) + self.assertEqual(commit.committer_email(), "committer@example.com") + self.assertEqual(commit.responsible_parties(), set([author, committer, reviewer])) diff --git a/WebKitTools/Scripts/webkitpy/diff_parser.py b/WebKitTools/Scripts/webkitpy/common/checkout/diff_parser.py index 7dce7e8..d8ebae6 100644 --- a/WebKitTools/Scripts/webkitpy/diff_parser.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/diff_parser.py @@ -31,6 +31,7 @@ import logging import re +_log = logging.getLogger("webkitpy.common.checkout.diff_parser") _regexp_compile_cache = {} @@ -138,7 +139,8 @@ class DiffParser: lines_changed = match(r"^@@ -(?P<OldStartLine>\d+)(,\d+)? \+(?P<NewStartLine>\d+)(,\d+)? @@", line) if lines_changed: if state != _DECLARED_FILE_PATH and state != _PROCESSING_CHUNK: - logging.error('Unexpected line change without file path declaration: %r' % line) + _log.error('Unexpected line change without file path ' + 'declaration: %r' % line) old_diff_line = int(lines_changed.group('OldStartLine')) new_diff_line = int(lines_changed.group('NewStartLine')) state = _PROCESSING_CHUNK @@ -159,4 +161,5 @@ class DiffParser: # Nothing to do. We may still have some added lines. pass else: - logging.error('Unexpected diff format when parsing a chunk: %r' % line) + _log.error('Unexpected diff format when parsing a ' + 'chunk: %r' % line) diff --git a/WebKitTools/Scripts/webkitpy/diff_parser_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/diff_parser_unittest.py index 7eb0eab..7eb0eab 100644 --- a/WebKitTools/Scripts/webkitpy/diff_parser_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/checkout/diff_parser_unittest.py diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/scm.py b/WebKitTools/Scripts/webkitpy/common/checkout/scm.py new file mode 100644 index 0000000..569558a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/scm.py @@ -0,0 +1,845 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Python module for interacting with an SCM system (like SVN or Git) + +import os +import re +import sys +import shutil + +from webkitpy.common.system.executive import Executive, run_command, ScriptError +from webkitpy.common.system.deprecated_logging import error, log + + +def detect_scm_system(path): + absolute_path = os.path.abspath(path) + + if SVN.in_working_directory(absolute_path): + return SVN(cwd=absolute_path) + + if Git.in_working_directory(absolute_path): + return Git(cwd=absolute_path) + + return None + + +def first_non_empty_line_after_index(lines, index=0): + first_non_empty_line = index + for line in lines[index:]: + if re.match("^\s*$", line): + first_non_empty_line += 1 + else: + break + return first_non_empty_line + + +class CommitMessage: + def __init__(self, message): + self.message_lines = message[first_non_empty_line_after_index(message, 0):] + + def body(self, lstrip=False): + lines = self.message_lines[first_non_empty_line_after_index(self.message_lines, 1):] + if lstrip: + lines = [line.lstrip() for line in lines] + return "\n".join(lines) + "\n" + + def description(self, lstrip=False, strip_url=False): + line = self.message_lines[0] + if lstrip: + line = line.lstrip() + if strip_url: + line = re.sub("^(\s*)<.+> ", "\1", line) + return line + + def message(self): + return "\n".join(self.message_lines) + "\n" + + +class CheckoutNeedsUpdate(ScriptError): + def __init__(self, script_args, exit_code, output, cwd): + ScriptError.__init__(self, script_args=script_args, exit_code=exit_code, output=output, cwd=cwd) + + +def commit_error_handler(error): + if re.search("resource out of date", error.output): + raise CheckoutNeedsUpdate(script_args=error.script_args, exit_code=error.exit_code, output=error.output, cwd=error.cwd) + Executive.default_error_handler(error) + + +class AuthenticationError(Exception): + def __init__(self, server_host): + self.server_host = server_host + + +class AmbiguousCommitError(Exception): + def __init__(self, num_local_commits, working_directory_is_clean): + self.num_local_commits = num_local_commits + self.working_directory_is_clean = working_directory_is_clean + + +# SCM methods are expected to return paths relative to self.checkout_root. +class SCM: + def __init__(self, cwd): + self.cwd = cwd + self.checkout_root = self.find_checkout_root(self.cwd) + self.dryrun = False + + # A wrapper used by subclasses to create processes. + def run(self, args, cwd=None, input=None, error_handler=None, return_exit_code=False, return_stderr=True, decode_output=True): + # FIXME: We should set cwd appropriately. + # FIXME: We should use Executive. + return run_command(args, + cwd=cwd, + input=input, + error_handler=error_handler, + return_exit_code=return_exit_code, + return_stderr=return_stderr, + decode_output=decode_output) + + # SCM always returns repository relative path, but sometimes we need + # absolute paths to pass to rm, etc. + def absolute_path(self, repository_relative_path): + return os.path.join(self.checkout_root, repository_relative_path) + + # FIXME: This belongs in Checkout, not SCM. + def scripts_directory(self): + return os.path.join(self.checkout_root, "WebKitTools", "Scripts") + + # FIXME: This belongs in Checkout, not SCM. + def script_path(self, script_name): + return os.path.join(self.scripts_directory(), script_name) + + def ensure_clean_working_directory(self, force_clean): + if not force_clean and not self.working_directory_is_clean(): + # FIXME: Shouldn't this use cwd=self.checkout_root? + print self.run(self.status_command(), error_handler=Executive.ignore_error) + raise ScriptError(message="Working directory has modifications, pass --force-clean or --no-clean to continue.") + + log("Cleaning working directory") + self.clean_working_directory() + + def ensure_no_local_commits(self, force): + if not self.supports_local_commits(): + return + commits = self.local_commits() + if not len(commits): + return + if not force: + error("Working directory has local commits, pass --force-clean to continue.") + self.discard_local_commits() + + def run_status_and_extract_filenames(self, status_command, status_regexp): + filenames = [] + # We run with cwd=self.checkout_root so that returned-paths are root-relative. + for line in self.run(status_command, cwd=self.checkout_root).splitlines(): + match = re.search(status_regexp, line) + if not match: + continue + # status = match.group('status') + filename = match.group('filename') + filenames.append(filename) + return filenames + + def strip_r_from_svn_revision(self, svn_revision): + match = re.match("^r(?P<svn_revision>\d+)", unicode(svn_revision)) + if (match): + return match.group('svn_revision') + return svn_revision + + def svn_revision_from_commit_text(self, commit_text): + match = re.search(self.commit_success_regexp(), commit_text, re.MULTILINE) + return match.group('svn_revision') + + @staticmethod + def _subclass_must_implement(): + raise NotImplementedError("subclasses must implement") + + @staticmethod + def in_working_directory(path): + SCM._subclass_must_implement() + + @staticmethod + def find_checkout_root(path): + SCM._subclass_must_implement() + + @staticmethod + def commit_success_regexp(): + SCM._subclass_must_implement() + + def working_directory_is_clean(self): + self._subclass_must_implement() + + def clean_working_directory(self): + self._subclass_must_implement() + + def status_command(self): + self._subclass_must_implement() + + def add(self, path, return_exit_code=False): + self._subclass_must_implement() + + def delete(self, path): + self._subclass_must_implement() + + def changed_files(self, git_commit=None): + self._subclass_must_implement() + + def changed_files_for_revision(self): + self._subclass_must_implement() + + def added_files(self): + self._subclass_must_implement() + + def conflicted_files(self): + self._subclass_must_implement() + + def display_name(self): + self._subclass_must_implement() + + def create_patch(self, git_commit=None): + self._subclass_must_implement() + + def committer_email_for_revision(self, revision): + self._subclass_must_implement() + + def contents_at_revision(self, path, revision): + self._subclass_must_implement() + + def diff_for_revision(self, revision): + self._subclass_must_implement() + + def diff_for_file(self, path, log=None): + self._subclass_must_implement() + + def show_head(self, path): + self._subclass_must_implement() + + def apply_reverse_diff(self, revision): + self._subclass_must_implement() + + def revert_files(self, file_paths): + self._subclass_must_implement() + + def commit_with_message(self, message, username=None, git_commit=None, force_squash=False): + self._subclass_must_implement() + + def svn_commit_log(self, svn_revision): + self._subclass_must_implement() + + def last_svn_commit_log(self): + self._subclass_must_implement() + + # Subclasses must indicate if they support local commits, + # but the SCM baseclass will only call local_commits methods when this is true. + @staticmethod + def supports_local_commits(): + SCM._subclass_must_implement() + + def remote_merge_base(): + SCM._subclass_must_implement() + + def commit_locally_with_message(self, message): + error("Your source control manager does not support local commits.") + + def discard_local_commits(self): + pass + + def local_commits(self): + return [] + + +class SVN(SCM): + # FIXME: We should move these values to a WebKit-specific config. file. + svn_server_host = "svn.webkit.org" + svn_server_realm = "<http://svn.webkit.org:80> Mac OS Forge" + + def __init__(self, cwd): + SCM.__init__(self, cwd) + self.cached_version = None + self._bogus_dir = None + + @staticmethod + def in_working_directory(path): + return os.path.isdir(os.path.join(path, '.svn')) + + @classmethod + def find_uuid(cls, path): + if not cls.in_working_directory(path): + return None + return cls.value_from_svn_info(path, 'Repository UUID') + + @classmethod + def value_from_svn_info(cls, path, field_name): + svn_info_args = ['svn', 'info', path] + info_output = run_command(svn_info_args).rstrip() + match = re.search("^%s: (?P<value>.+)$" % field_name, info_output, re.MULTILINE) + if not match: + raise ScriptError(script_args=svn_info_args, message='svn info did not contain a %s.' % field_name) + return match.group('value') + + @staticmethod + def find_checkout_root(path): + uuid = SVN.find_uuid(path) + # If |path| is not in a working directory, we're supposed to return |path|. + if not uuid: + return path + # Search up the directory hierarchy until we find a different UUID. + last_path = None + while True: + if uuid != SVN.find_uuid(path): + return last_path + last_path = path + (path, last_component) = os.path.split(path) + if last_path == path: + return None + + @staticmethod + def commit_success_regexp(): + return "^Committed revision (?P<svn_revision>\d+)\.$" + + def has_authorization_for_realm(self, realm=svn_server_realm, home_directory=os.getenv("HOME")): + # Assumes find and grep are installed. + if not os.path.isdir(os.path.join(home_directory, ".subversion")): + return False + find_args = ["find", ".subversion", "-type", "f", "-exec", "grep", "-q", realm, "{}", ";", "-print"]; + find_output = self.run(find_args, cwd=home_directory, error_handler=Executive.ignore_error).rstrip() + return find_output and os.path.isfile(os.path.join(home_directory, find_output)) + + def svn_version(self): + if not self.cached_version: + self.cached_version = self.run(['svn', '--version', '--quiet']) + + return self.cached_version + + def working_directory_is_clean(self): + return self.run(["svn", "diff"], cwd=self.checkout_root, decode_output=False) == "" + + def clean_working_directory(self): + # svn revert -R is not as awesome as git reset --hard. + # It will leave added files around, causing later svn update + # calls to fail on the bots. We make this mirror git reset --hard + # by deleting any added files as well. + added_files = reversed(sorted(self.added_files())) + # added_files() returns directories for SVN, we walk the files in reverse path + # length order so that we remove files before we try to remove the directories. + self.run(["svn", "revert", "-R", "."], cwd=self.checkout_root) + for path in added_files: + # This is robust against cwd != self.checkout_root + absolute_path = self.absolute_path(path) + # Completely lame that there is no easy way to remove both types with one call. + if os.path.isdir(path): + os.rmdir(absolute_path) + else: + os.remove(absolute_path) + + def status_command(self): + return ['svn', 'status'] + + def _status_regexp(self, expected_types): + field_count = 6 if self.svn_version() > "1.6" else 5 + return "^(?P<status>[%s]).{%s} (?P<filename>.+)$" % (expected_types, field_count) + + def _add_parent_directories(self, path): + """Does 'svn add' to the path and its parents.""" + if self.in_working_directory(path): + return + dirname = os.path.dirname(path) + # We have dirname directry - ensure it added. + if dirname != path: + self._add_parent_directories(dirname) + self.add(path) + + def add(self, path, return_exit_code=False): + self._add_parent_directories(os.path.dirname(os.path.abspath(path))) + return self.run(["svn", "add", path], return_exit_code=return_exit_code) + + def delete(self, path): + parent, base = os.path.split(os.path.abspath(path)) + return self.run(["svn", "delete", "--force", base], cwd=parent) + + def changed_files(self, git_commit=None): + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("ACDMR")) + + def changed_files_for_revision(self, revision): + # As far as I can tell svn diff --summarize output looks just like svn status output. + # No file contents printed, thus utf-8 auto-decoding in self.run is fine. + status_command = ["svn", "diff", "--summarize", "-c", revision] + return self.run_status_and_extract_filenames(status_command, self._status_regexp("ACDMR")) + + def conflicted_files(self): + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("C")) + + def added_files(self): + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("A")) + + def deleted_files(self): + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("D")) + + @staticmethod + def supports_local_commits(): + return False + + def display_name(self): + return "svn" + + # FIXME: This method should be on Checkout. + def create_patch(self, git_commit=None): + """Returns a byte array (str()) representing the patch file. + Patch files are effectively binary since they may contain + files of multiple different encodings.""" + return self.run([self.script_path("svn-create-patch")], + cwd=self.checkout_root, return_stderr=False, + decode_output=False) + + def committer_email_for_revision(self, revision): + return self.run(["svn", "propget", "svn:author", "--revprop", "-r", revision]).rstrip() + + def contents_at_revision(self, path, revision): + """Returns a byte array (str()) containing the contents + of path @ revision in the repository.""" + remote_path = "%s/%s" % (self._repository_url(), path) + return self.run(["svn", "cat", "-r", revision, remote_path], decode_output=False) + + def diff_for_revision(self, revision): + # FIXME: This should probably use cwd=self.checkout_root + return self.run(['svn', 'diff', '-c', revision]) + + def _bogus_dir_name(self): + if sys.platform.startswith("win"): + parent_dir = tempfile.gettempdir() + else: + parent_dir = sys.path[0] # tempdir is not secure. + return os.path.join(parent_dir, "temp_svn_config") + + def _setup_bogus_dir(self, log): + self._bogus_dir = self._bogus_dir_name() + if not os.path.exists(self._bogus_dir): + os.mkdir(self._bogus_dir) + self._delete_bogus_dir = True + else: + self._delete_bogus_dir = False + if log: + log.debug(' Html: temp config dir: "%s".', self._bogus_dir) + + def _teardown_bogus_dir(self, log): + if self._delete_bogus_dir: + shutil.rmtree(self._bogus_dir, True) + if log: + log.debug(' Html: removed temp config dir: "%s".', self._bogus_dir) + self._bogus_dir = None + + def diff_for_file(self, path, log=None): + self._setup_bogus_dir(log) + try: + args = ['svn', 'diff'] + if self._bogus_dir: + args += ['--config-dir', self._bogus_dir] + args.append(path) + return self.run(args) + finally: + self._teardown_bogus_dir(log) + + def show_head(self, path): + return self.run(['svn', 'cat', '-r', 'BASE', path], decode_output=False) + + def _repository_url(self): + return self.value_from_svn_info(self.checkout_root, 'URL') + + def apply_reverse_diff(self, revision): + # '-c -revision' applies the inverse diff of 'revision' + svn_merge_args = ['svn', 'merge', '--non-interactive', '-c', '-%s' % revision, self._repository_url()] + log("WARNING: svn merge has been known to take more than 10 minutes to complete. It is recommended you use git for rollouts.") + log("Running '%s'" % " ".join(svn_merge_args)) + # FIXME: Should this use cwd=self.checkout_root? + self.run(svn_merge_args) + + def revert_files(self, file_paths): + # FIXME: This should probably use cwd=self.checkout_root. + self.run(['svn', 'revert'] + file_paths) + + def commit_with_message(self, message, username=None, git_commit=None, force_squash=False): + # git-commit and force are not used by SVN. + if self.dryrun: + # Return a string which looks like a commit so that things which parse this output will succeed. + return "Dry run, no commit.\nCommitted revision 0." + + svn_commit_args = ["svn", "commit"] + + if not username and not self.has_authorization_for_realm(): + raise AuthenticationError(self.svn_server_host) + if username: + svn_commit_args.extend(["--username", username]) + + svn_commit_args.extend(["-m", message]) + # FIXME: Should this use cwd=self.checkout_root? + return self.run(svn_commit_args, error_handler=commit_error_handler) + + def svn_commit_log(self, svn_revision): + svn_revision = self.strip_r_from_svn_revision(svn_revision) + return self.run(['svn', 'log', '--non-interactive', '--revision', svn_revision]) + + def last_svn_commit_log(self): + # BASE is the checkout revision, HEAD is the remote repository revision + # http://svnbook.red-bean.com/en/1.0/ch03s03.html + return self.svn_commit_log('BASE') + + def propset(self, pname, pvalue, path): + dir, base = os.path.split(path) + return self.run(['svn', 'pset', pname, pvalue, base], cwd=dir) + + def propget(self, pname, path): + dir, base = os.path.split(path) + return self.run(['svn', 'pget', pname, base], cwd=dir).encode('utf-8').rstrip("\n") + +# All git-specific logic should go here. +class Git(SCM): + def __init__(self, cwd): + SCM.__init__(self, cwd) + + @classmethod + def in_working_directory(cls, path): + return run_command(['git', 'rev-parse', '--is-inside-work-tree'], cwd=path, error_handler=Executive.ignore_error).rstrip() == "true" + + @classmethod + def find_checkout_root(cls, path): + # "git rev-parse --show-cdup" would be another way to get to the root + (checkout_root, dot_git) = os.path.split(run_command(['git', 'rev-parse', '--git-dir'], cwd=(path or "./"))) + # If we were using 2.6 # checkout_root = os.path.relpath(checkout_root, path) + if not os.path.isabs(checkout_root): # Sometimes git returns relative paths + checkout_root = os.path.join(path, checkout_root) + return checkout_root + + @classmethod + def to_object_name(cls, filepath): + root_end_with_slash = os.path.join(cls.find_checkout_root(os.path.dirname(filepath)), '') + return filepath.replace(root_end_with_slash, '') + + @classmethod + def read_git_config(cls, key): + # FIXME: This should probably use cwd=self.checkout_root. + return run_command(["git", "config", key], + error_handler=Executive.ignore_error).rstrip('\n') + + @staticmethod + def commit_success_regexp(): + return "^Committed r(?P<svn_revision>\d+)$" + + def discard_local_commits(self): + # FIXME: This should probably use cwd=self.checkout_root + self.run(['git', 'reset', '--hard', self.remote_branch_ref()]) + + def local_commits(self): + # FIXME: This should probably use cwd=self.checkout_root + return self.run(['git', 'log', '--pretty=oneline', 'HEAD...' + self.remote_branch_ref()]).splitlines() + + def rebase_in_progress(self): + return os.path.exists(os.path.join(self.checkout_root, '.git/rebase-apply')) + + def working_directory_is_clean(self): + # FIXME: This should probably use cwd=self.checkout_root + return self.run(['git', 'diff', 'HEAD', '--name-only']) == "" + + def clean_working_directory(self): + # FIXME: These should probably use cwd=self.checkout_root. + # Could run git clean here too, but that wouldn't match working_directory_is_clean + self.run(['git', 'reset', '--hard', 'HEAD']) + # Aborting rebase even though this does not match working_directory_is_clean + if self.rebase_in_progress(): + self.run(['git', 'rebase', '--abort']) + + def status_command(self): + # git status returns non-zero when there are changes, so we use git diff name --name-status HEAD instead. + # No file contents printed, thus utf-8 autodecoding in self.run is fine. + return ["git", "diff", "--name-status", "HEAD"] + + def _status_regexp(self, expected_types): + return '^(?P<status>[%s])\t(?P<filename>.+)$' % expected_types + + def add(self, path, return_exit_code=False): + return self.run(["git", "add", path], return_exit_code=return_exit_code) + + def delete(self, path): + return self.run(["git", "rm", "-f", path]) + + def _assert_synced(self): + if len(run_command(['git', 'rev-list', '--max-count=1', self.remote_branch_ref(), '^HEAD'])): + raise ScriptError(message="Not fully merged/rebased to %s. This branch needs to be synced first." % self.remote_branch_ref()) + + def merge_base(self, git_commit): + if git_commit: + # Special-case HEAD.. to mean working-copy changes only. + if git_commit.upper() == 'HEAD..': + return 'HEAD' + + if '..' not in git_commit: + git_commit = git_commit + "^.." + git_commit + return git_commit + + self._assert_synced() + return self.remote_merge_base() + + def changed_files(self, git_commit=None): + status_command = ['git', 'diff', '-r', '--name-status', '-C', '-M', "--no-ext-diff", "--full-index", self.merge_base(git_commit)] + return self.run_status_and_extract_filenames(status_command, self._status_regexp("ADM")) + + def _changes_files_for_commit(self, git_commit): + # --pretty="format:" makes git show not print the commit log header, + changed_files = self.run(["git", "show", "--pretty=format:", "--name-only", git_commit]).splitlines() + # instead it just prints a blank line at the top, so we skip the blank line: + return changed_files[1:] + + def changed_files_for_revision(self, revision): + commit_id = self.git_commit_from_svn_revision(revision) + return self._changes_files_for_commit(commit_id) + + def conflicted_files(self): + # We do not need to pass decode_output for this diff command + # as we're passing --name-status which does not output any data. + status_command = ['git', 'diff', '--name-status', '-C', '-M', '--diff-filter=U'] + return self.run_status_and_extract_filenames(status_command, self._status_regexp("U")) + + def added_files(self): + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("A")) + + def deleted_files(self): + return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("D")) + + @staticmethod + def supports_local_commits(): + return True + + def display_name(self): + return "git" + + def create_patch(self, git_commit=None): + """Returns a byte array (str()) representing the patch file. + Patch files are effectively binary since they may contain + files of multiple different encodings.""" + # FIXME: This should probably use cwd=self.checkout_root + return self.run(['git', 'diff', '--binary', "--no-ext-diff", "--full-index", "-M", self.merge_base(git_commit)], decode_output=False) + + @classmethod + def git_commit_from_svn_revision(cls, revision): + # FIXME: This should probably use cwd=self.checkout_root + git_commit = run_command(['git', 'svn', 'find-rev', 'r%s' % revision]).rstrip() + # git svn find-rev always exits 0, even when the revision is not found. + if not git_commit: + raise ScriptError(message='Failed to find git commit for revision %s, your checkout likely needs an update.' % revision) + return git_commit + + def contents_at_revision(self, path, revision): + """Returns a byte array (str()) containing the contents + of path @ revision in the repository.""" + return self.run(["git", "show", "%s:%s" % (self.git_commit_from_svn_revision(revision), path)], decode_output=False) + + def diff_for_revision(self, revision): + git_commit = self.git_commit_from_svn_revision(revision) + return self.create_patch(git_commit) + + def diff_for_file(self, path, log=None): + return self.run(['git', 'diff', 'HEAD', '--', path]) + + def show_head(self, path): + return self.run(['git', 'show', 'HEAD:' + self.to_object_name(path)], decode_output=False) + + def committer_email_for_revision(self, revision): + git_commit = self.git_commit_from_svn_revision(revision) + committer_email = self.run(["git", "log", "-1", "--pretty=format:%ce", git_commit]) + # Git adds an extra @repository_hash to the end of every committer email, remove it: + return committer_email.rsplit("@", 1)[0] + + def apply_reverse_diff(self, revision): + # Assume the revision is an svn revision. + git_commit = self.git_commit_from_svn_revision(revision) + # I think this will always fail due to ChangeLogs. + self.run(['git', 'revert', '--no-commit', git_commit], error_handler=Executive.ignore_error) + + def revert_files(self, file_paths): + self.run(['git', 'checkout', 'HEAD'] + file_paths) + + def _assert_can_squash(self, working_directory_is_clean): + squash = Git.read_git_config('webkit-patch.commit_should_always_squash') + should_squash = squash and squash.lower() == "true" + + if not should_squash: + # Only warn if there are actually multiple commits to squash. + num_local_commits = len(self.local_commits()) + if num_local_commits > 1 or (num_local_commits > 0 and not working_directory_is_clean): + raise AmbiguousCommitError(num_local_commits, working_directory_is_clean) + + def commit_with_message(self, message, username=None, git_commit=None, force_squash=False): + # Username is ignored during Git commits. + working_directory_is_clean = self.working_directory_is_clean() + + if git_commit: + # Special-case HEAD.. to mean working-copy changes only. + if git_commit.upper() == 'HEAD..': + if working_directory_is_clean: + raise ScriptError(message="The working copy is not modified. --git-commit=HEAD.. only commits working copy changes.") + self.commit_locally_with_message(message) + return self._commit_on_branch(message, 'HEAD') + + # Need working directory changes to be committed so we can checkout the merge branch. + if not working_directory_is_clean: + # FIXME: webkit-patch land will modify the ChangeLogs to correct the reviewer. + # That will modify the working-copy and cause us to hit this error. + # The ChangeLog modification could be made to modify the existing local commit. + raise ScriptError(message="Working copy is modified. Cannot commit individual git_commits.") + return self._commit_on_branch(message, git_commit) + + if not force_squash: + self._assert_can_squash(working_directory_is_clean) + self._assert_synced() + self.run(['git', 'reset', '--soft', self.remote_branch_ref()]) + self.commit_locally_with_message(message) + return self.push_local_commits_to_server() + + def _commit_on_branch(self, message, git_commit): + branch_ref = self.run(['git', 'symbolic-ref', 'HEAD']).strip() + branch_name = branch_ref.replace('refs/heads/', '') + commit_ids = self.commit_ids_from_commitish_arguments([git_commit]) + + # We want to squash all this branch's commits into one commit with the proper description. + # We do this by doing a "merge --squash" into a new commit branch, then dcommitting that. + MERGE_BRANCH_NAME = 'webkit-patch-land' + self.delete_branch(MERGE_BRANCH_NAME) + + # We might be in a directory that's present in this branch but not in the + # trunk. Move up to the top of the tree so that git commands that expect a + # valid CWD won't fail after we check out the merge branch. + os.chdir(self.checkout_root) + + # Stuff our change into the merge branch. + # We wrap in a try...finally block so if anything goes wrong, we clean up the branches. + commit_succeeded = True + try: + self.run(['git', 'checkout', '-q', '-b', MERGE_BRANCH_NAME, self.remote_branch_ref()]) + + for commit in commit_ids: + # We're on a different branch now, so convert "head" to the branch name. + commit = re.sub(r'(?i)head', branch_name, commit) + # FIXME: Once changed_files and create_patch are modified to separately handle each + # commit in a commit range, commit each cherry pick so they'll get dcommitted separately. + self.run(['git', 'cherry-pick', '--no-commit', commit]) + + self.run(['git', 'commit', '-m', message]) + output = self.push_local_commits_to_server() + except Exception, e: + log("COMMIT FAILED: " + str(e)) + output = "Commit failed." + commit_succeeded = False + finally: + # And then swap back to the original branch and clean up. + self.clean_working_directory() + self.run(['git', 'checkout', '-q', branch_name]) + self.delete_branch(MERGE_BRANCH_NAME) + + return output + + def svn_commit_log(self, svn_revision): + svn_revision = self.strip_r_from_svn_revision(svn_revision) + return self.run(['git', 'svn', 'log', '-r', svn_revision]) + + def last_svn_commit_log(self): + return self.run(['git', 'svn', 'log', '--limit=1']) + + # Git-specific methods: + def _branch_ref_exists(self, branch_ref): + return self.run(['git', 'show-ref', '--quiet', '--verify', branch_ref], return_exit_code=True) == 0 + + def delete_branch(self, branch_name): + if self._branch_ref_exists('refs/heads/' + branch_name): + self.run(['git', 'branch', '-D', branch_name]) + + def remote_merge_base(self): + return self.run(['git', 'merge-base', self.remote_branch_ref(), 'HEAD']).strip() + + def remote_branch_ref(self): + # Use references so that we can avoid collisions, e.g. we don't want to operate on refs/heads/trunk if it exists. + + # FIXME: This should so something like: Git.read_git_config('svn-remote.svn.fetch').split(':')[1] + # but that doesn't work if the git repo is tracking multiple svn branches. + remote_branch_refs = [ + 'refs/remotes/trunk', # A git-svn checkout as per http://trac.webkit.org/wiki/UsingGitWithWebKit. + 'refs/remotes/origin/master', # A git clone of git://git.webkit.org/WebKit.git that is not tracking svn. + ] + + for ref in remote_branch_refs: + if self._branch_ref_exists(ref): + return ref + + raise ScriptError(message="Can't find a branch to diff against. %s branches do not exist." % " and ".join(remote_branch_refs)) + + def commit_locally_with_message(self, message): + self.run(['git', 'commit', '--all', '-F', '-'], input=message) + + def push_local_commits_to_server(self): + dcommit_command = ['git', 'svn', 'dcommit'] + if self.dryrun: + dcommit_command.append('--dry-run') + output = self.run(dcommit_command, error_handler=commit_error_handler) + # Return a string which looks like a commit so that things which parse this output will succeed. + if self.dryrun: + output += "\nCommitted r0" + return output + + # This function supports the following argument formats: + # no args : rev-list trunk..HEAD + # A..B : rev-list A..B + # A...B : error! + # A B : [A, B] (different from git diff, which would use "rev-list A..B") + def commit_ids_from_commitish_arguments(self, args): + if not len(args): + args.append('%s..HEAD' % self.remote_branch_ref()) + + commit_ids = [] + for commitish in args: + if '...' in commitish: + raise ScriptError(message="'...' is not supported (found in '%s'). Did you mean '..'?" % commitish) + elif '..' in commitish: + commit_ids += reversed(self.run(['git', 'rev-list', commitish]).splitlines()) + else: + # Turn single commits or branch or tag names into commit ids. + commit_ids += self.run(['git', 'rev-parse', '--revs-only', commitish]).splitlines() + return commit_ids + + def commit_message_for_local_commit(self, commit_id): + commit_lines = self.run(['git', 'cat-file', 'commit', commit_id]).splitlines() + + # Skip the git headers. + first_line_after_headers = 0 + for line in commit_lines: + first_line_after_headers += 1 + if line == "": + break + return CommitMessage(commit_lines[first_line_after_headers:]) + + def files_changed_summary_for_commit(self, commit_id): + return self.run(['git', 'diff-tree', '--shortstat', '--no-commit-id', commit_id]) diff --git a/WebKitTools/Scripts/webkitpy/common/checkout/scm_unittest.py b/WebKitTools/Scripts/webkitpy/common/checkout/scm_unittest.py new file mode 100644 index 0000000..852f838 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/checkout/scm_unittest.py @@ -0,0 +1,1197 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# Copyright (C) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import with_statement + +import base64 +import codecs +import getpass +import os +import os.path +import re +import stat +import subprocess +import tempfile +import unittest +import urllib +import shutil + +from datetime import date +from webkitpy.common.checkout.api import Checkout +from webkitpy.common.checkout.scm import detect_scm_system, SCM, SVN, CheckoutNeedsUpdate, commit_error_handler, AuthenticationError, AmbiguousCommitError +from webkitpy.common.config.committers import Committer # FIXME: This should not be needed +from webkitpy.common.net.bugzilla import Attachment # FIXME: This should not be needed +from webkitpy.common.system.executive import Executive, run_command, ScriptError + +# Eventually we will want to write tests which work for both scms. (like update_webkit, changed_files, etc.) +# Perhaps through some SCMTest base-class which both SVNTest and GitTest inherit from. + +# FIXME: This should be unified into one of the executive.py commands! +# Callers could use run_and_throw_if_fail(args, cwd=cwd, quiet=True) +def run_silent(args, cwd=None): + # Note: Not thread safe: http://bugs.python.org/issue2320 + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) + process.communicate() # ignore output + exit_code = process.wait() + if exit_code: + raise ScriptError('Failed to run "%s" exit_code: %d cwd: %s' % (args, exit_code, cwd)) + + +def write_into_file_at_path(file_path, contents, encoding="utf-8"): + if encoding: + with codecs.open(file_path, "w", encoding) as file: + file.write(contents) + else: + with open(file_path, "w") as file: + file.write(contents) + + +def read_from_path(file_path, encoding="utf-8"): + with codecs.open(file_path, "r", encoding) as file: + return file.read() + + +def _make_diff(command, *args): + # We use this wrapper to disable output decoding. diffs should be treated as + # binary files since they may include text files of multiple differnet encodings. + return run_command([command, "diff"] + list(args), decode_output=False) + + +def _svn_diff(*args): + return _make_diff("svn", *args) + + +def _git_diff(*args): + return _make_diff("git", *args) + + +# Exists to share svn repository creation code between the git and svn tests +class SVNTestRepository: + @classmethod + def _svn_add(cls, path): + run_command(["svn", "add", path]) + + @classmethod + def _svn_commit(cls, message): + run_command(["svn", "commit", "--quiet", "--message", message]) + + @classmethod + def _setup_test_commits(cls, test_object): + # Add some test commits + os.chdir(test_object.svn_checkout_path) + + write_into_file_at_path("test_file", "test1") + cls._svn_add("test_file") + cls._svn_commit("initial commit") + + write_into_file_at_path("test_file", "test1test2") + # This used to be the last commit, but doing so broke + # GitTest.test_apply_git_patch which use the inverse diff of the last commit. + # svn-apply fails to remove directories in Git, see: + # https://bugs.webkit.org/show_bug.cgi?id=34871 + os.mkdir("test_dir") + # Slash should always be the right path separator since we use cygwin on Windows. + test_file3_path = "test_dir/test_file3" + write_into_file_at_path(test_file3_path, "third file") + cls._svn_add("test_dir") + cls._svn_commit("second commit") + + write_into_file_at_path("test_file", "test1test2test3\n") + write_into_file_at_path("test_file2", "second file") + cls._svn_add("test_file2") + cls._svn_commit("third commit") + + # This 4th commit is used to make sure that our patch file handling + # code correctly treats patches as binary and does not attempt to + # decode them assuming they're utf-8. + write_into_file_at_path("test_file", u"latin1 test: \u00A0\n", "latin1") + write_into_file_at_path("test_file2", u"utf-8 test: \u00A0\n", "utf-8") + cls._svn_commit("fourth commit") + + # svn does not seem to update after commit as I would expect. + run_command(['svn', 'update']) + + @classmethod + def setup(cls, test_object): + # Create an test SVN repository + test_object.svn_repo_path = tempfile.mkdtemp(suffix="svn_test_repo") + test_object.svn_repo_url = "file://%s" % test_object.svn_repo_path # Not sure this will work on windows + # git svn complains if we don't pass --pre-1.5-compatible, not sure why: + # Expected FS format '2'; found format '3' at /usr/local/libexec/git-core//git-svn line 1477 + run_command(['svnadmin', 'create', '--pre-1.5-compatible', test_object.svn_repo_path]) + + # Create a test svn checkout + test_object.svn_checkout_path = tempfile.mkdtemp(suffix="svn_test_checkout") + run_command(['svn', 'checkout', '--quiet', test_object.svn_repo_url, test_object.svn_checkout_path]) + + # Create and checkout a trunk dir to match the standard svn configuration to match git-svn's expectations + os.chdir(test_object.svn_checkout_path) + os.mkdir('trunk') + cls._svn_add('trunk') + # We can add tags and branches as well if we ever need to test those. + cls._svn_commit('add trunk') + + # Change directory out of the svn checkout so we can delete the checkout directory. + # _setup_test_commits will CD back to the svn checkout directory. + os.chdir('/') + run_command(['rm', '-rf', test_object.svn_checkout_path]) + run_command(['svn', 'checkout', '--quiet', test_object.svn_repo_url + '/trunk', test_object.svn_checkout_path]) + + cls._setup_test_commits(test_object) + + @classmethod + def tear_down(cls, test_object): + run_command(['rm', '-rf', test_object.svn_repo_path]) + run_command(['rm', '-rf', test_object.svn_checkout_path]) + + # Now that we've deleted the checkout paths, cwddir may be invalid + # Change back to a valid directory so that later calls to os.getcwd() do not fail. + os.chdir(detect_scm_system(os.path.dirname(__file__)).checkout_root) + +# For testing the SCM baseclass directly. +class SCMClassTests(unittest.TestCase): + def setUp(self): + self.dev_null = open(os.devnull, "w") # Used to make our Popen calls quiet. + + def tearDown(self): + self.dev_null.close() + + def test_run_command_with_pipe(self): + input_process = subprocess.Popen(['echo', 'foo\nbar'], stdout=subprocess.PIPE, stderr=self.dev_null) + self.assertEqual(run_command(['grep', 'bar'], input=input_process.stdout), "bar\n") + + # Test the non-pipe case too: + self.assertEqual(run_command(['grep', 'bar'], input="foo\nbar"), "bar\n") + + command_returns_non_zero = ['/bin/sh', '--invalid-option'] + # Test when the input pipe process fails. + input_process = subprocess.Popen(command_returns_non_zero, stdout=subprocess.PIPE, stderr=self.dev_null) + self.assertTrue(input_process.poll() != 0) + self.assertRaises(ScriptError, run_command, ['grep', 'bar'], input=input_process.stdout) + + # Test when the run_command process fails. + input_process = subprocess.Popen(['echo', 'foo\nbar'], stdout=subprocess.PIPE, stderr=self.dev_null) # grep shows usage and calls exit(2) when called w/o arguments. + self.assertRaises(ScriptError, run_command, command_returns_non_zero, input=input_process.stdout) + + def test_error_handlers(self): + git_failure_message="Merge conflict during commit: Your file or directory 'WebCore/ChangeLog' is probably out-of-date: resource out of date; try updating at /usr/local/libexec/git-core//git-svn line 469" + svn_failure_message="""svn: Commit failed (details follow): +svn: File or directory 'ChangeLog' is out of date; try updating +svn: resource out of date; try updating +""" + command_does_not_exist = ['does_not_exist', 'invalid_option'] + self.assertRaises(OSError, run_command, command_does_not_exist) + self.assertRaises(OSError, run_command, command_does_not_exist, error_handler=Executive.ignore_error) + + command_returns_non_zero = ['/bin/sh', '--invalid-option'] + self.assertRaises(ScriptError, run_command, command_returns_non_zero) + # Check if returns error text: + self.assertTrue(run_command(command_returns_non_zero, error_handler=Executive.ignore_error)) + + self.assertRaises(CheckoutNeedsUpdate, commit_error_handler, ScriptError(output=git_failure_message)) + self.assertRaises(CheckoutNeedsUpdate, commit_error_handler, ScriptError(output=svn_failure_message)) + self.assertRaises(ScriptError, commit_error_handler, ScriptError(output='blah blah blah')) + + +# GitTest and SVNTest inherit from this so any test_ methods here will be run once for this class and then once for each subclass. +class SCMTest(unittest.TestCase): + def _create_patch(self, patch_contents): + # FIXME: This code is brittle if the Attachment API changes. + attachment = Attachment({"bug_id": 12345}, None) + attachment.contents = lambda: patch_contents + + joe_cool = Committer(name="Joe Cool", email_or_emails=None) + attachment.reviewer = lambda: joe_cool + + return attachment + + def _setup_webkittools_scripts_symlink(self, local_scm): + webkit_scm = detect_scm_system(os.path.dirname(os.path.abspath(__file__))) + webkit_scripts_directory = webkit_scm.scripts_directory() + local_scripts_directory = local_scm.scripts_directory() + os.mkdir(os.path.dirname(local_scripts_directory)) + os.symlink(webkit_scripts_directory, local_scripts_directory) + + # Tests which both GitTest and SVNTest should run. + # FIXME: There must be a simpler way to add these w/o adding a wrapper method to both subclasses + + def _shared_test_changed_files(self): + write_into_file_at_path("test_file", "changed content") + self.assertEqual(self.scm.changed_files(), ["test_file"]) + write_into_file_at_path("test_dir/test_file3", "new stuff") + self.assertEqual(self.scm.changed_files(), ["test_dir/test_file3", "test_file"]) + old_cwd = os.getcwd() + os.chdir("test_dir") + # Validate that changed_files does not change with our cwd, see bug 37015. + self.assertEqual(self.scm.changed_files(), ["test_dir/test_file3", "test_file"]) + os.chdir(old_cwd) + + def _shared_test_added_files(self): + write_into_file_at_path("test_file", "changed content") + self.assertEqual(self.scm.added_files(), []) + + write_into_file_at_path("added_file", "new stuff") + self.scm.add("added_file") + + os.mkdir("added_dir") + write_into_file_at_path("added_dir/added_file2", "new stuff") + self.scm.add("added_dir") + + # SVN reports directory changes, Git does not. + added_files = self.scm.added_files() + if "added_dir" in added_files: + added_files.remove("added_dir") + self.assertEqual(added_files, ["added_dir/added_file2", "added_file"]) + + # Test also to make sure clean_working_directory removes added files + self.scm.clean_working_directory() + self.assertEqual(self.scm.added_files(), []) + self.assertFalse(os.path.exists("added_file")) + self.assertFalse(os.path.exists("added_dir")) + + def _shared_test_changed_files_for_revision(self): + # SVN reports directory changes, Git does not. + changed_files = self.scm.changed_files_for_revision(3) + if "test_dir" in changed_files: + changed_files.remove("test_dir") + self.assertEqual(changed_files, ["test_dir/test_file3", "test_file"]) + self.assertEqual(sorted(self.scm.changed_files_for_revision(4)), sorted(["test_file", "test_file2"])) # Git and SVN return different orders. + self.assertEqual(self.scm.changed_files_for_revision(2), ["test_file"]) + + def _shared_test_contents_at_revision(self): + self.assertEqual(self.scm.contents_at_revision("test_file", 3), "test1test2") + self.assertEqual(self.scm.contents_at_revision("test_file", 4), "test1test2test3\n") + + # Verify that contents_at_revision returns a byte array, aka str(): + self.assertEqual(self.scm.contents_at_revision("test_file", 5), u"latin1 test: \u00A0\n".encode("latin1")) + self.assertEqual(self.scm.contents_at_revision("test_file2", 5), u"utf-8 test: \u00A0\n".encode("utf-8")) + + self.assertEqual(self.scm.contents_at_revision("test_file2", 4), "second file") + # Files which don't exist: + # Currently we raise instead of returning None because detecting the difference between + # "file not found" and any other error seems impossible with svn (git seems to expose such through the return code). + self.assertRaises(ScriptError, self.scm.contents_at_revision, "test_file2", 2) + self.assertRaises(ScriptError, self.scm.contents_at_revision, "does_not_exist", 2) + + def _shared_test_committer_email_for_revision(self): + self.assertEqual(self.scm.committer_email_for_revision(3), getpass.getuser()) # Committer "email" will be the current user + + def _shared_test_reverse_diff(self): + self._setup_webkittools_scripts_symlink(self.scm) # Git's apply_reverse_diff uses resolve-ChangeLogs + # Only test the simple case, as any other will end up with conflict markers. + self.scm.apply_reverse_diff('5') + self.assertEqual(read_from_path('test_file'), "test1test2test3\n") + + def _shared_test_diff_for_revision(self): + # Patch formats are slightly different between svn and git, so just regexp for things we know should be there. + r3_patch = self.scm.diff_for_revision(4) + self.assertTrue(re.search('test3', r3_patch)) + self.assertFalse(re.search('test4', r3_patch)) + self.assertTrue(re.search('test2', r3_patch)) + self.assertTrue(re.search('test2', self.scm.diff_for_revision(3))) + + def _shared_test_svn_apply_git_patch(self): + self._setup_webkittools_scripts_symlink(self.scm) + git_binary_addition = """diff --git a/fizzbuzz7.gif b/fizzbuzz7.gif +new file mode 100644 +index 0000000000000000000000000000000000000000..64a9532e7794fcd791f6f12157406d90 +60151690 +GIT binary patch +literal 512 +zcmZ?wbhEHbRAx|MU|?iW{Kxc~?KofD;ckY;H+&5HnHl!!GQMD7h+sU{_)e9f^V3c? +zhJP##HdZC#4K}7F68@!1jfWQg2daCm-gs#3|JREDT>c+pG4L<_2;w##WMO#ysPPap +zLqpAf1OE938xAsSp4!5f-o><?VKe(#0jEcwfHGF4%M1^kRs14oVBp2ZEL{E1N<-zJ +zsfLmOtKta;2_;2c#^S1-8cf<nb!QnGl>c!Xe6RXvrEtAWBvSDTgTO1j3vA31Puw!A +zs(87q)j_mVDTqBo-P+03-P5mHCEnJ+x}YdCuS7#bCCyePUe(ynK+|4b-3qK)T?Z&) +zYG+`tl4h?GZv_$t82}X4*DTE|$;{DEiPyF@)U-1+FaX++T9H{&%cag`W1|zVP@`%b +zqiSkp6{BTpWTkCr!=<C6Q=?#~R8^JfrliAF6Q^gV9Iup8RqCXqqhqC`qsyhk<-nlB +z00f{QZvfK&|Nm#oZ0TQl`Yr$BIa6A@16O26ud7H<QM=xl`toLKnz-3h@9c9q&wm|X +z{89I|WPyD!*M?gv?q`;L=2YFeXrJQNti4?}s!zFo=5CzeBxC69xA<zrjP<wUcCRh4 +ptUl-ZG<%a~#LwkIWv&q!KSCH7tQ8cJDiw+|GV?MN)RjY50RTb-xvT&H + +literal 0 +HcmV?d00001 + +""" + self.checkout.apply_patch(self._create_patch(git_binary_addition)) + added = read_from_path('fizzbuzz7.gif', encoding=None) + self.assertEqual(512, len(added)) + self.assertTrue(added.startswith('GIF89a')) + self.assertTrue('fizzbuzz7.gif' in self.scm.changed_files()) + + # The file already exists. + self.assertRaises(ScriptError, self.checkout.apply_patch, self._create_patch(git_binary_addition)) + + git_binary_modification = """diff --git a/fizzbuzz7.gif b/fizzbuzz7.gif +index 64a9532e7794fcd791f6f12157406d9060151690..323fae03f4606ea9991df8befbb2fca7 +GIT binary patch +literal 7 +OcmYex&reD$;sO8*F9L)B + +literal 512 +zcmZ?wbhEHbRAx|MU|?iW{Kxc~?KofD;ckY;H+&5HnHl!!GQMD7h+sU{_)e9f^V3c? +zhJP##HdZC#4K}7F68@!1jfWQg2daCm-gs#3|JREDT>c+pG4L<_2;w##WMO#ysPPap +zLqpAf1OE938xAsSp4!5f-o><?VKe(#0jEcwfHGF4%M1^kRs14oVBp2ZEL{E1N<-zJ +zsfLmOtKta;2_;2c#^S1-8cf<nb!QnGl>c!Xe6RXvrEtAWBvSDTgTO1j3vA31Puw!A +zs(87q)j_mVDTqBo-P+03-P5mHCEnJ+x}YdCuS7#bCCyePUe(ynK+|4b-3qK)T?Z&) +zYG+`tl4h?GZv_$t82}X4*DTE|$;{DEiPyF@)U-1+FaX++T9H{&%cag`W1|zVP@`%b +zqiSkp6{BTpWTkCr!=<C6Q=?#~R8^JfrliAF6Q^gV9Iup8RqCXqqhqC`qsyhk<-nlB +z00f{QZvfK&|Nm#oZ0TQl`Yr$BIa6A@16O26ud7H<QM=xl`toLKnz-3h@9c9q&wm|X +z{89I|WPyD!*M?gv?q`;L=2YFeXrJQNti4?}s!zFo=5CzeBxC69xA<zrjP<wUcCRh4 +ptUl-ZG<%a~#LwkIWv&q!KSCH7tQ8cJDiw+|GV?MN)RjY50RTb-xvT&H + +""" + self.checkout.apply_patch(self._create_patch(git_binary_modification)) + modified = read_from_path('fizzbuzz7.gif', encoding=None) + self.assertEqual('foobar\n', modified) + self.assertTrue('fizzbuzz7.gif' in self.scm.changed_files()) + + # Applying the same modification should fail. + self.assertRaises(ScriptError, self.checkout.apply_patch, self._create_patch(git_binary_modification)) + + git_binary_deletion = """diff --git a/fizzbuzz7.gif b/fizzbuzz7.gif +deleted file mode 100644 +index 323fae0..0000000 +GIT binary patch +literal 0 +HcmV?d00001 + +literal 7 +OcmYex&reD$;sO8*F9L)B + +""" + self.checkout.apply_patch(self._create_patch(git_binary_deletion)) + self.assertFalse(os.path.exists('fizzbuzz7.gif')) + self.assertFalse('fizzbuzz7.gif' in self.scm.changed_files()) + + # Cannot delete again. + self.assertRaises(ScriptError, self.checkout.apply_patch, self._create_patch(git_binary_deletion)) + + def _shared_test_add_recursively(self): + os.mkdir("added_dir") + write_into_file_at_path("added_dir/added_file", "new stuff") + self.scm.add("added_dir/added_file") + self.assertTrue("added_dir/added_file" in self.scm.added_files()) + +class SVNTest(SCMTest): + + @staticmethod + def _set_date_and_reviewer(changelog_entry): + # Joe Cool matches the reviewer set in SCMTest._create_patch + changelog_entry = changelog_entry.replace('REVIEWER_HERE', 'Joe Cool') + # svn-apply will update ChangeLog entries with today's date. + return changelog_entry.replace('DATE_HERE', date.today().isoformat()) + + def test_svn_apply(self): + first_entry = """2009-10-26 Eric Seidel <eric@webkit.org> + + Reviewed by Foo Bar. + + Most awesome change ever. + + * scm_unittest.py: +""" + intermediate_entry = """2009-10-27 Eric Seidel <eric@webkit.org> + + Reviewed by Baz Bar. + + A more awesomer change yet! + + * scm_unittest.py: +""" + one_line_overlap_patch = """Index: ChangeLog +=================================================================== +--- ChangeLog (revision 5) ++++ ChangeLog (working copy) +@@ -1,5 +1,13 @@ + 2009-10-26 Eric Seidel <eric@webkit.org> + ++ Reviewed by NOBODY (OOPS!). ++ ++ Second most awesome change ever. ++ ++ * scm_unittest.py: ++ ++2009-10-26 Eric Seidel <eric@webkit.org> ++ + Reviewed by Foo Bar. + + Most awesome change ever. +""" + one_line_overlap_entry = """DATE_HERE Eric Seidel <eric@webkit.org> + + Reviewed by REVIEWER_HERE. + + Second most awesome change ever. + + * scm_unittest.py: +""" + two_line_overlap_patch = """Index: ChangeLog +=================================================================== +--- ChangeLog (revision 5) ++++ ChangeLog (working copy) +@@ -2,6 +2,14 @@ + + Reviewed by Foo Bar. + ++ Second most awesome change ever. ++ ++ * scm_unittest.py: ++ ++2009-10-26 Eric Seidel <eric@webkit.org> ++ ++ Reviewed by Foo Bar. ++ + Most awesome change ever. + + * scm_unittest.py: +""" + two_line_overlap_entry = """DATE_HERE Eric Seidel <eric@webkit.org> + + Reviewed by Foo Bar. + + Second most awesome change ever. + + * scm_unittest.py: +""" + write_into_file_at_path('ChangeLog', first_entry) + run_command(['svn', 'add', 'ChangeLog']) + run_command(['svn', 'commit', '--quiet', '--message', 'ChangeLog commit']) + + # Patch files were created against just 'first_entry'. + # Add a second commit to make svn-apply have to apply the patches with fuzz. + changelog_contents = "%s\n%s" % (intermediate_entry, first_entry) + write_into_file_at_path('ChangeLog', changelog_contents) + run_command(['svn', 'commit', '--quiet', '--message', 'Intermediate commit']) + + self._setup_webkittools_scripts_symlink(self.scm) + self.checkout.apply_patch(self._create_patch(one_line_overlap_patch)) + expected_changelog_contents = "%s\n%s" % (self._set_date_and_reviewer(one_line_overlap_entry), changelog_contents) + self.assertEquals(read_from_path('ChangeLog'), expected_changelog_contents) + + self.scm.revert_files(['ChangeLog']) + self.checkout.apply_patch(self._create_patch(two_line_overlap_patch)) + expected_changelog_contents = "%s\n%s" % (self._set_date_and_reviewer(two_line_overlap_entry), changelog_contents) + self.assertEquals(read_from_path('ChangeLog'), expected_changelog_contents) + + def setUp(self): + SVNTestRepository.setup(self) + os.chdir(self.svn_checkout_path) + self.scm = detect_scm_system(self.svn_checkout_path) + # For historical reasons, we test some checkout code here too. + self.checkout = Checkout(self.scm) + + def tearDown(self): + SVNTestRepository.tear_down(self) + + def test_detect_scm_system_relative_url(self): + scm = detect_scm_system(".") + # I wanted to assert that we got the right path, but there was some + # crazy magic with temp folder names that I couldn't figure out. + self.assertTrue(scm.checkout_root) + + def test_create_patch_is_full_patch(self): + test_dir_path = os.path.join(self.svn_checkout_path, "test_dir2") + os.mkdir(test_dir_path) + test_file_path = os.path.join(test_dir_path, 'test_file2') + write_into_file_at_path(test_file_path, 'test content') + run_command(['svn', 'add', 'test_dir2']) + + # create_patch depends on 'svn-create-patch', so make a dummy version. + scripts_path = os.path.join(self.svn_checkout_path, 'WebKitTools', 'Scripts') + os.makedirs(scripts_path) + create_patch_path = os.path.join(scripts_path, 'svn-create-patch') + write_into_file_at_path(create_patch_path, '#!/bin/sh\necho $PWD') # We could pass -n to prevent the \n, but not all echo accept -n. + os.chmod(create_patch_path, stat.S_IXUSR | stat.S_IRUSR) + + # Change into our test directory and run the create_patch command. + os.chdir(test_dir_path) + scm = detect_scm_system(test_dir_path) + self.assertEqual(scm.checkout_root, self.svn_checkout_path) # Sanity check that detection worked right. + patch_contents = scm.create_patch() + # Our fake 'svn-create-patch' returns $PWD instead of a patch, check that it was executed from the root of the repo. + self.assertEqual("%s\n" % os.path.realpath(scm.checkout_root), patch_contents) # Add a \n because echo adds a \n. + + def test_detection(self): + scm = detect_scm_system(self.svn_checkout_path) + self.assertEqual(scm.display_name(), "svn") + self.assertEqual(scm.supports_local_commits(), False) + + def test_apply_small_binary_patch(self): + patch_contents = """Index: test_file.swf +=================================================================== +Cannot display: file marked as a binary type. +svn:mime-type = application/octet-stream + +Property changes on: test_file.swf +___________________________________________________________________ +Name: svn:mime-type + + application/octet-stream + + +Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== +""" + expected_contents = base64.b64decode("Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA==") + self._setup_webkittools_scripts_symlink(self.scm) + patch_file = self._create_patch(patch_contents) + self.checkout.apply_patch(patch_file) + actual_contents = read_from_path("test_file.swf", encoding=None) + self.assertEqual(actual_contents, expected_contents) + + def test_apply_svn_patch(self): + scm = detect_scm_system(self.svn_checkout_path) + patch = self._create_patch(_svn_diff("-r5:4")) + self._setup_webkittools_scripts_symlink(scm) + Checkout(scm).apply_patch(patch) + + def test_apply_svn_patch_force(self): + scm = detect_scm_system(self.svn_checkout_path) + patch = self._create_patch(_svn_diff("-r3:5")) + self._setup_webkittools_scripts_symlink(scm) + self.assertRaises(ScriptError, Checkout(scm).apply_patch, patch, force=True) + + def test_commit_logs(self): + # Commits have dates and usernames in them, so we can't just direct compare. + self.assertTrue(re.search('fourth commit', self.scm.last_svn_commit_log())) + self.assertTrue(re.search('second commit', self.scm.svn_commit_log(3))) + + def _shared_test_commit_with_message(self, username=None): + write_into_file_at_path('test_file', 'more test content') + commit_text = self.scm.commit_with_message("another test commit", username) + self.assertEqual(self.scm.svn_revision_from_commit_text(commit_text), '6') + + self.scm.dryrun = True + write_into_file_at_path('test_file', 'still more test content') + commit_text = self.scm.commit_with_message("yet another test commit", username) + self.assertEqual(self.scm.svn_revision_from_commit_text(commit_text), '0') + + def test_commit_text_parsing(self): + self._shared_test_commit_with_message() + + def test_commit_with_username(self): + self._shared_test_commit_with_message("dbates@webkit.org") + + def test_commit_without_authorization(self): + self.scm.has_authorization_for_realm = lambda: False + self.assertRaises(AuthenticationError, self._shared_test_commit_with_message) + + def test_has_authorization_for_realm(self): + scm = detect_scm_system(self.svn_checkout_path) + fake_home_dir = tempfile.mkdtemp(suffix="fake_home_dir") + svn_config_dir_path = os.path.join(fake_home_dir, ".subversion") + os.mkdir(svn_config_dir_path) + fake_webkit_auth_file = os.path.join(svn_config_dir_path, "fake_webkit_auth_file") + write_into_file_at_path(fake_webkit_auth_file, SVN.svn_server_realm) + self.assertTrue(scm.has_authorization_for_realm(home_directory=fake_home_dir)) + os.remove(fake_webkit_auth_file) + os.rmdir(svn_config_dir_path) + os.rmdir(fake_home_dir) + + def test_not_have_authorization_for_realm(self): + scm = detect_scm_system(self.svn_checkout_path) + fake_home_dir = tempfile.mkdtemp(suffix="fake_home_dir") + svn_config_dir_path = os.path.join(fake_home_dir, ".subversion") + os.mkdir(svn_config_dir_path) + self.assertFalse(scm.has_authorization_for_realm(home_directory=fake_home_dir)) + os.rmdir(svn_config_dir_path) + os.rmdir(fake_home_dir) + + def test_reverse_diff(self): + self._shared_test_reverse_diff() + + def test_diff_for_revision(self): + self._shared_test_diff_for_revision() + + def test_svn_apply_git_patch(self): + self._shared_test_svn_apply_git_patch() + + def test_changed_files(self): + self._shared_test_changed_files() + + def test_changed_files_for_revision(self): + self._shared_test_changed_files_for_revision() + + def test_added_files(self): + self._shared_test_added_files() + + def test_contents_at_revision(self): + self._shared_test_contents_at_revision() + + def test_committer_email_for_revision(self): + self._shared_test_committer_email_for_revision() + + def test_add_recursively(self): + self._shared_test_add_recursively() + + def test_delete(self): + os.chdir(self.svn_checkout_path) + self.scm.delete("test_file") + self.assertTrue("test_file" in self.scm.deleted_files()) + + def test_propset_propget(self): + filepath = os.path.join(self.svn_checkout_path, "test_file") + expected_mime_type = "x-application/foo-bar" + self.scm.propset("svn:mime-type", expected_mime_type, filepath) + self.assertEqual(expected_mime_type, self.scm.propget("svn:mime-type", filepath)) + + def test_show_head(self): + write_into_file_at_path("test_file", u"Hello!", "utf-8") + SVNTestRepository._svn_commit("fourth commit") + self.assertEqual("Hello!", self.scm.show_head('test_file')) + + def test_show_head_binary(self): + data = "\244" + write_into_file_at_path("binary_file", data, encoding=None) + self.scm.add("binary_file") + self.scm.commit_with_message("a test commit") + self.assertEqual(data, self.scm.show_head('binary_file')) + + def do_test_diff_for_file(self): + write_into_file_at_path('test_file', 'some content') + self.scm.commit_with_message("a test commit") + diff = self.scm.diff_for_file('test_file') + self.assertEqual(diff, "") + + write_into_file_at_path("test_file", "changed content") + diff = self.scm.diff_for_file('test_file') + self.assertTrue("-some content" in diff) + self.assertTrue("+changed content" in diff) + + def clean_bogus_dir(self): + self.bogus_dir = self.scm._bogus_dir_name() + if os.path.exists(self.bogus_dir): + shutil.rmtree(self.bogus_dir) + + def test_diff_for_file_with_existing_bogus_dir(self): + self.clean_bogus_dir() + os.mkdir(self.bogus_dir) + self.do_test_diff_for_file() + self.assertTrue(os.path.exists(self.bogus_dir)) + shutil.rmtree(self.bogus_dir) + + def test_diff_for_file_with_missing_bogus_dir(self): + self.clean_bogus_dir() + self.do_test_diff_for_file() + self.assertFalse(os.path.exists(self.bogus_dir)) + + +class GitTest(SCMTest): + + def setUp(self): + """Sets up fresh git repository with one commit. Then setups a second git + repo that tracks the first one.""" + self.original_dir = os.getcwd() + + self.untracking_checkout_path = tempfile.mkdtemp(suffix="git_test_checkout2") + run_command(['git', 'init', self.untracking_checkout_path]) + + os.chdir(self.untracking_checkout_path) + write_into_file_at_path('foo_file', 'foo') + run_command(['git', 'add', 'foo_file']) + run_command(['git', 'commit', '-am', 'dummy commit']) + self.untracking_scm = detect_scm_system(self.untracking_checkout_path) + + self.tracking_git_checkout_path = tempfile.mkdtemp(suffix="git_test_checkout") + run_command(['git', 'clone', '--quiet', self.untracking_checkout_path, self.tracking_git_checkout_path]) + os.chdir(self.tracking_git_checkout_path) + self.tracking_scm = detect_scm_system(self.tracking_git_checkout_path) + + def tearDown(self): + # Change back to a valid directory so that later calls to os.getcwd() do not fail. + os.chdir(self.original_dir) + run_command(['rm', '-rf', self.tracking_git_checkout_path]) + run_command(['rm', '-rf', self.untracking_checkout_path]) + + def test_remote_branch_ref(self): + self.assertEqual(self.tracking_scm.remote_branch_ref(), 'refs/remotes/origin/master') + + os.chdir(self.untracking_checkout_path) + self.assertRaises(ScriptError, self.untracking_scm.remote_branch_ref) + + +class GitSVNTest(SCMTest): + + def _setup_git_checkout(self): + self.git_checkout_path = tempfile.mkdtemp(suffix="git_test_checkout") + # --quiet doesn't make git svn silent, so we use run_silent to redirect output + run_silent(['git', 'svn', 'clone', '-T', 'trunk', self.svn_repo_url, self.git_checkout_path]) + os.chdir(self.git_checkout_path) + + def _tear_down_git_checkout(self): + # Change back to a valid directory so that later calls to os.getcwd() do not fail. + os.chdir(self.original_dir) + run_command(['rm', '-rf', self.git_checkout_path]) + + def setUp(self): + self.original_dir = os.getcwd() + + SVNTestRepository.setup(self) + self._setup_git_checkout() + self.scm = detect_scm_system(self.git_checkout_path) + # For historical reasons, we test some checkout code here too. + self.checkout = Checkout(self.scm) + + def tearDown(self): + SVNTestRepository.tear_down(self) + self._tear_down_git_checkout() + + def test_detection(self): + scm = detect_scm_system(self.git_checkout_path) + self.assertEqual(scm.display_name(), "git") + self.assertEqual(scm.supports_local_commits(), True) + + def test_read_git_config(self): + key = 'test.git-config' + value = 'git-config value' + run_command(['git', 'config', key, value]) + self.assertEqual(self.scm.read_git_config(key), value) + + def test_local_commits(self): + test_file = os.path.join(self.git_checkout_path, 'test_file') + write_into_file_at_path(test_file, 'foo') + run_command(['git', 'commit', '-a', '-m', 'local commit']) + + self.assertEqual(len(self.scm.local_commits()), 1) + + def test_discard_local_commits(self): + test_file = os.path.join(self.git_checkout_path, 'test_file') + write_into_file_at_path(test_file, 'foo') + run_command(['git', 'commit', '-a', '-m', 'local commit']) + + self.assertEqual(len(self.scm.local_commits()), 1) + self.scm.discard_local_commits() + self.assertEqual(len(self.scm.local_commits()), 0) + + def test_delete_branch(self): + new_branch = 'foo' + + run_command(['git', 'checkout', '-b', new_branch]) + self.assertEqual(run_command(['git', 'symbolic-ref', 'HEAD']).strip(), 'refs/heads/' + new_branch) + + run_command(['git', 'checkout', '-b', 'bar']) + self.scm.delete_branch(new_branch) + + self.assertFalse(re.search(r'foo', run_command(['git', 'branch']))) + + def test_remote_merge_base(self): + # Diff to merge-base should include working-copy changes, + # which the diff to svn_branch.. doesn't. + test_file = os.path.join(self.git_checkout_path, 'test_file') + write_into_file_at_path(test_file, 'foo') + + diff_to_common_base = _git_diff(self.scm.remote_branch_ref() + '..') + diff_to_merge_base = _git_diff(self.scm.remote_merge_base()) + + self.assertFalse(re.search(r'foo', diff_to_common_base)) + self.assertTrue(re.search(r'foo', diff_to_merge_base)) + + def test_rebase_in_progress(self): + svn_test_file = os.path.join(self.svn_checkout_path, 'test_file') + write_into_file_at_path(svn_test_file, "svn_checkout") + run_command(['svn', 'commit', '--message', 'commit to conflict with git commit'], cwd=self.svn_checkout_path) + + git_test_file = os.path.join(self.git_checkout_path, 'test_file') + write_into_file_at_path(git_test_file, "git_checkout") + run_command(['git', 'commit', '-a', '-m', 'commit to be thrown away by rebase abort']) + + # --quiet doesn't make git svn silent, so use run_silent to redirect output + self.assertRaises(ScriptError, run_silent, ['git', 'svn', '--quiet', 'rebase']) # Will fail due to a conflict leaving us mid-rebase. + + scm = detect_scm_system(self.git_checkout_path) + self.assertTrue(scm.rebase_in_progress()) + + # Make sure our cleanup works. + scm.clean_working_directory() + self.assertFalse(scm.rebase_in_progress()) + + # Make sure cleanup doesn't throw when no rebase is in progress. + scm.clean_working_directory() + + def test_commitish_parsing(self): + scm = detect_scm_system(self.git_checkout_path) + + # Multiple revisions are cherry-picked. + self.assertEqual(len(scm.commit_ids_from_commitish_arguments(['HEAD~2'])), 1) + self.assertEqual(len(scm.commit_ids_from_commitish_arguments(['HEAD', 'HEAD~2'])), 2) + + # ... is an invalid range specifier + self.assertRaises(ScriptError, scm.commit_ids_from_commitish_arguments, ['trunk...HEAD']) + + def test_commitish_order(self): + scm = detect_scm_system(self.git_checkout_path) + + commit_range = 'HEAD~3..HEAD' + + actual_commits = scm.commit_ids_from_commitish_arguments([commit_range]) + expected_commits = [] + expected_commits += reversed(run_command(['git', 'rev-list', commit_range]).splitlines()) + + self.assertEqual(actual_commits, expected_commits) + + def test_apply_git_patch(self): + scm = detect_scm_system(self.git_checkout_path) + # We carefullly pick a diff which does not have a directory addition + # as currently svn-apply will error out when trying to remove directories + # in Git: https://bugs.webkit.org/show_bug.cgi?id=34871 + patch = self._create_patch(_git_diff('HEAD..HEAD^')) + self._setup_webkittools_scripts_symlink(scm) + Checkout(scm).apply_patch(patch) + + def test_apply_git_patch_force(self): + scm = detect_scm_system(self.git_checkout_path) + patch = self._create_patch(_git_diff('HEAD~2..HEAD')) + self._setup_webkittools_scripts_symlink(scm) + self.assertRaises(ScriptError, Checkout(scm).apply_patch, patch, force=True) + + def test_commit_text_parsing(self): + write_into_file_at_path('test_file', 'more test content') + commit_text = self.scm.commit_with_message("another test commit") + self.assertEqual(self.scm.svn_revision_from_commit_text(commit_text), '6') + + self.scm.dryrun = True + write_into_file_at_path('test_file', 'still more test content') + commit_text = self.scm.commit_with_message("yet another test commit") + self.assertEqual(self.scm.svn_revision_from_commit_text(commit_text), '0') + + def test_commit_with_message_working_copy_only(self): + write_into_file_at_path('test_file_commit1', 'more test content') + run_command(['git', 'add', 'test_file_commit1']) + scm = detect_scm_system(self.git_checkout_path) + commit_text = scm.commit_with_message("yet another test commit") + + self.assertEqual(scm.svn_revision_from_commit_text(commit_text), '6') + svn_log = run_command(['git', 'svn', 'log', '--limit=1', '--verbose']) + self.assertTrue(re.search(r'test_file_commit1', svn_log)) + + def _one_local_commit(self): + write_into_file_at_path('test_file_commit1', 'more test content') + run_command(['git', 'add', 'test_file_commit1']) + self.scm.commit_locally_with_message("another test commit") + + def _one_local_commit_plus_working_copy_changes(self): + self._one_local_commit() + write_into_file_at_path('test_file_commit2', 'still more test content') + run_command(['git', 'add', 'test_file_commit2']) + + def _two_local_commits(self): + self._one_local_commit() + write_into_file_at_path('test_file_commit2', 'still more test content') + run_command(['git', 'add', 'test_file_commit2']) + self.scm.commit_locally_with_message("yet another test commit") + + def _three_local_commits(self): + write_into_file_at_path('test_file_commit0', 'more test content') + run_command(['git', 'add', 'test_file_commit0']) + self.scm.commit_locally_with_message("another test commit") + self._two_local_commits() + + def test_commit_with_message(self): + self._one_local_commit_plus_working_copy_changes() + scm = detect_scm_system(self.git_checkout_path) + self.assertRaises(AmbiguousCommitError, scm.commit_with_message, "yet another test commit") + commit_text = scm.commit_with_message("yet another test commit", force_squash=True) + + self.assertEqual(scm.svn_revision_from_commit_text(commit_text), '6') + svn_log = run_command(['git', 'svn', 'log', '--limit=1', '--verbose']) + self.assertTrue(re.search(r'test_file_commit2', svn_log)) + self.assertTrue(re.search(r'test_file_commit1', svn_log)) + + def test_commit_with_message_git_commit(self): + self._two_local_commits() + + scm = detect_scm_system(self.git_checkout_path) + commit_text = scm.commit_with_message("another test commit", git_commit="HEAD^") + self.assertEqual(scm.svn_revision_from_commit_text(commit_text), '6') + + svn_log = run_command(['git', 'svn', 'log', '--limit=1', '--verbose']) + self.assertTrue(re.search(r'test_file_commit1', svn_log)) + self.assertFalse(re.search(r'test_file_commit2', svn_log)) + + def test_commit_with_message_git_commit_range(self): + self._three_local_commits() + + scm = detect_scm_system(self.git_checkout_path) + commit_text = scm.commit_with_message("another test commit", git_commit="HEAD~2..HEAD") + self.assertEqual(scm.svn_revision_from_commit_text(commit_text), '6') + + svn_log = run_command(['git', 'svn', 'log', '--limit=1', '--verbose']) + self.assertFalse(re.search(r'test_file_commit0', svn_log)) + self.assertTrue(re.search(r'test_file_commit1', svn_log)) + self.assertTrue(re.search(r'test_file_commit2', svn_log)) + + def test_changed_files_working_copy_only(self): + self._one_local_commit_plus_working_copy_changes() + scm = detect_scm_system(self.git_checkout_path) + commit_text = scm.commit_with_message("another test commit", git_commit="HEAD..") + self.assertFalse(re.search(r'test_file_commit1', svn_log)) + self.assertTrue(re.search(r'test_file_commit2', svn_log)) + + def test_commit_with_message_only_local_commit(self): + self._one_local_commit() + scm = detect_scm_system(self.git_checkout_path) + commit_text = scm.commit_with_message("another test commit") + svn_log = run_command(['git', 'svn', 'log', '--limit=1', '--verbose']) + self.assertTrue(re.search(r'test_file_commit1', svn_log)) + + def test_commit_with_message_multiple_local_commits_and_working_copy(self): + self._two_local_commits() + write_into_file_at_path('test_file_commit1', 'working copy change') + scm = detect_scm_system(self.git_checkout_path) + + self.assertRaises(AmbiguousCommitError, scm.commit_with_message, "another test commit") + commit_text = scm.commit_with_message("another test commit", force_squash=True) + + self.assertEqual(scm.svn_revision_from_commit_text(commit_text), '6') + svn_log = run_command(['git', 'svn', 'log', '--limit=1', '--verbose']) + self.assertTrue(re.search(r'test_file_commit2', svn_log)) + self.assertTrue(re.search(r'test_file_commit1', svn_log)) + + def test_commit_with_message_git_commit_and_working_copy(self): + self._two_local_commits() + write_into_file_at_path('test_file_commit1', 'working copy change') + scm = detect_scm_system(self.git_checkout_path) + self.assertRaises(ScriptError, scm.commit_with_message, "another test commit", git_commit="HEAD^") + + def test_commit_with_message_multiple_local_commits_always_squash(self): + self._two_local_commits() + scm = detect_scm_system(self.git_checkout_path) + scm._assert_can_squash = lambda working_directory_is_clean: True + commit_text = scm.commit_with_message("yet another test commit") + self.assertEqual(scm.svn_revision_from_commit_text(commit_text), '6') + + svn_log = run_command(['git', 'svn', 'log', '--limit=1', '--verbose']) + self.assertTrue(re.search(r'test_file_commit2', svn_log)) + self.assertTrue(re.search(r'test_file_commit1', svn_log)) + + def test_commit_with_message_multiple_local_commits(self): + self._two_local_commits() + scm = detect_scm_system(self.git_checkout_path) + self.assertRaises(AmbiguousCommitError, scm.commit_with_message, "yet another test commit") + commit_text = scm.commit_with_message("yet another test commit", force_squash=True) + + self.assertEqual(scm.svn_revision_from_commit_text(commit_text), '6') + + svn_log = run_command(['git', 'svn', 'log', '--limit=1', '--verbose']) + self.assertTrue(re.search(r'test_file_commit2', svn_log)) + self.assertTrue(re.search(r'test_file_commit1', svn_log)) + + def test_commit_with_message_not_synced(self): + run_command(['git', 'checkout', '-b', 'my-branch', 'trunk~3']) + self._two_local_commits() + scm = detect_scm_system(self.git_checkout_path) + self.assertRaises(AmbiguousCommitError, scm.commit_with_message, "another test commit") + self.assertRaises(ScriptError, scm.commit_with_message, "another test commit", force_squash=True) + + def test_remote_branch_ref(self): + self.assertEqual(self.scm.remote_branch_ref(), 'refs/remotes/trunk') + + def test_reverse_diff(self): + self._shared_test_reverse_diff() + + def test_diff_for_revision(self): + self._shared_test_diff_for_revision() + + def test_svn_apply_git_patch(self): + self._shared_test_svn_apply_git_patch() + + def test_create_patch_local_plus_working_copy(self): + self._one_local_commit_plus_working_copy_changes() + scm = detect_scm_system(self.git_checkout_path) + patch = scm.create_patch() + self.assertTrue(re.search(r'test_file_commit1', patch)) + self.assertTrue(re.search(r'test_file_commit2', patch)) + + def test_create_patch(self): + self._one_local_commit_plus_working_copy_changes() + scm = detect_scm_system(self.git_checkout_path) + patch = scm.create_patch() + self.assertTrue(re.search(r'test_file_commit2', patch)) + self.assertTrue(re.search(r'test_file_commit1', patch)) + + def test_create_patch_git_commit(self): + self._two_local_commits() + scm = detect_scm_system(self.git_checkout_path) + patch = scm.create_patch(git_commit="HEAD^") + self.assertTrue(re.search(r'test_file_commit1', patch)) + self.assertFalse(re.search(r'test_file_commit2', patch)) + + def test_create_patch_git_commit_range(self): + self._three_local_commits() + scm = detect_scm_system(self.git_checkout_path) + patch = scm.create_patch(git_commit="HEAD~2..HEAD") + self.assertFalse(re.search(r'test_file_commit0', patch)) + self.assertTrue(re.search(r'test_file_commit2', patch)) + self.assertTrue(re.search(r'test_file_commit1', patch)) + + def test_create_patch_working_copy_only(self): + self._one_local_commit_plus_working_copy_changes() + scm = detect_scm_system(self.git_checkout_path) + patch = scm.create_patch(git_commit="HEAD..") + self.assertFalse(re.search(r'test_file_commit1', patch)) + self.assertTrue(re.search(r'test_file_commit2', patch)) + + def test_create_patch_multiple_local_commits(self): + self._two_local_commits() + scm = detect_scm_system(self.git_checkout_path) + patch = scm.create_patch() + self.assertTrue(re.search(r'test_file_commit2', patch)) + self.assertTrue(re.search(r'test_file_commit1', patch)) + + def test_create_patch_not_synced(self): + run_command(['git', 'checkout', '-b', 'my-branch', 'trunk~3']) + self._two_local_commits() + scm = detect_scm_system(self.git_checkout_path) + self.assertRaises(ScriptError, scm.create_patch) + + def test_create_binary_patch(self): + # Create a git binary patch and check the contents. + scm = detect_scm_system(self.git_checkout_path) + test_file_name = 'binary_file' + test_file_path = os.path.join(self.git_checkout_path, test_file_name) + file_contents = ''.join(map(chr, range(256))) + write_into_file_at_path(test_file_path, file_contents, encoding=None) + run_command(['git', 'add', test_file_name]) + patch = scm.create_patch() + self.assertTrue(re.search(r'\nliteral 0\n', patch)) + self.assertTrue(re.search(r'\nliteral 256\n', patch)) + + # Check if we can apply the created patch. + run_command(['git', 'rm', '-f', test_file_name]) + self._setup_webkittools_scripts_symlink(scm) + self.checkout.apply_patch(self._create_patch(patch)) + self.assertEqual(file_contents, read_from_path(test_file_path, encoding=None)) + + # Check if we can create a patch from a local commit. + write_into_file_at_path(test_file_path, file_contents, encoding=None) + run_command(['git', 'add', test_file_name]) + run_command(['git', 'commit', '-m', 'binary diff']) + patch_from_local_commit = scm.create_patch('HEAD') + self.assertTrue(re.search(r'\nliteral 0\n', patch_from_local_commit)) + self.assertTrue(re.search(r'\nliteral 256\n', patch_from_local_commit)) + + def test_changed_files_local_plus_working_copy(self): + self._one_local_commit_plus_working_copy_changes() + scm = detect_scm_system(self.git_checkout_path) + files = scm.changed_files() + self.assertTrue('test_file_commit1' in files) + self.assertTrue('test_file_commit2' in files) + + def test_changed_files_git_commit(self): + self._two_local_commits() + scm = detect_scm_system(self.git_checkout_path) + files = scm.changed_files(git_commit="HEAD^") + self.assertTrue('test_file_commit1' in files) + self.assertFalse('test_file_commit2' in files) + + def test_changed_files_git_commit_range(self): + self._three_local_commits() + scm = detect_scm_system(self.git_checkout_path) + files = scm.changed_files(git_commit="HEAD~2..HEAD") + self.assertTrue('test_file_commit0' not in files) + self.assertTrue('test_file_commit1' in files) + self.assertTrue('test_file_commit2' in files) + + def test_changed_files_working_copy_only(self): + self._one_local_commit_plus_working_copy_changes() + scm = detect_scm_system(self.git_checkout_path) + files = scm.changed_files(git_commit="HEAD..") + self.assertFalse('test_file_commit1' in files) + self.assertTrue('test_file_commit2' in files) + + def test_changed_files_multiple_local_commits(self): + self._two_local_commits() + scm = detect_scm_system(self.git_checkout_path) + files = scm.changed_files() + self.assertTrue('test_file_commit2' in files) + self.assertTrue('test_file_commit1' in files) + + def test_changed_files_not_synced(self): + run_command(['git', 'checkout', '-b', 'my-branch', 'trunk~3']) + self._two_local_commits() + scm = detect_scm_system(self.git_checkout_path) + self.assertRaises(ScriptError, scm.changed_files) + + def test_changed_files(self): + self._shared_test_changed_files() + + def test_changed_files_for_revision(self): + self._shared_test_changed_files_for_revision() + + def test_contents_at_revision(self): + self._shared_test_contents_at_revision() + + def test_added_files(self): + self._shared_test_added_files() + + def test_committer_email_for_revision(self): + self._shared_test_committer_email_for_revision() + + def test_add_recursively(self): + self._shared_test_add_recursively() + + def test_delete(self): + self._two_local_commits() + self.scm.delete('test_file_commit1') + self.assertTrue("test_file_commit1" in self.scm.deleted_files()) + + def test_to_object_name(self): + relpath = 'test_file_commit1' + fullpath = os.path.join(self.git_checkout_path, relpath) + self._two_local_commits() + self.assertEqual(relpath, self.scm.to_object_name(fullpath)) + + def test_show_head(self): + self._two_local_commits() + self.assertEqual("more test content", self.scm.show_head('test_file_commit1')) + + def test_show_head_binary(self): + self._two_local_commits() + data = "\244" + write_into_file_at_path("binary_file", data, encoding=None) + self.scm.add("binary_file") + self.scm.commit_locally_with_message("a test commit") + self.assertEqual(data, self.scm.show_head('binary_file')) + + def test_diff_for_file(self): + self._two_local_commits() + write_into_file_at_path('test_file_commit1', "Updated", encoding=None) + + diff = self.scm.diff_for_file('test_file_commit1') + cached_diff = self.scm.diff_for_file('test_file_commit1') + self.assertTrue("+Updated" in diff) + self.assertTrue("-more test content" in diff) + + self.scm.add('test_file_commit1') + + cached_diff = self.scm.diff_for_file('test_file_commit1') + self.assertTrue("+Updated" in cached_diff) + self.assertTrue("-more test content" in cached_diff) + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/common/config/__init__.py b/WebKitTools/Scripts/webkitpy/common/config/__init__.py new file mode 100644 index 0000000..62d129e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/config/__init__.py @@ -0,0 +1,6 @@ +# Required for Python to search this directory for module files + +import re + +codereview_server_host = "wkrietveld.appspot.com" +codereview_server_url = "https://%s/" % codereview_server_host diff --git a/WebKitTools/Scripts/webkitpy/common/config/committers.py b/WebKitTools/Scripts/webkitpy/common/config/committers.py new file mode 100644 index 0000000..0354981 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/config/committers.py @@ -0,0 +1,307 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# WebKit's Python module for committer and reviewer validation + + +class Committer: + + def __init__(self, name, email_or_emails, irc_nickname=None): + self.full_name = name + if isinstance(email_or_emails, str): + self.emails = [email_or_emails] + else: + self.emails = email_or_emails + self.irc_nickname = irc_nickname + self.can_review = False + + def bugzilla_email(self): + # FIXME: We're assuming the first email is a valid bugzilla email, + # which might not be right. + return self.emails[0] + + def __str__(self): + return '"%s" <%s>' % (self.full_name, self.emails[0]) + + +class Reviewer(Committer): + + def __init__(self, name, email_or_emails, irc_nickname=None): + Committer.__init__(self, name, email_or_emails, irc_nickname) + self.can_review = True + + +# This is intended as a canonical, machine-readable list of all non-reviewer +# committers for WebKit. If your name is missing here and you are a committer, +# please add it. No review needed. All reviewers are committers, so this list +# is only of committers who are not reviewers. + + +committers_unable_to_review = [ + Committer("Aaron Boodman", "aa@chromium.org", "aboodman"), + Committer("Abhishek Arya", "inferno@chromium.org", "inferno-sec"), + Committer("Adam Langley", "agl@chromium.org", "agl"), + Committer("Albert J. Wong", "ajwong@chromium.org"), + Committer("Alejandro G. Castro", ["alex@igalia.com", "alex@webkit.org"]), + Committer("Alexander Kellett", ["lypanov@mac.com", "a-lists001@lypanov.net", "lypanov@kde.org"], "lypanov"), + Committer("Alexander Pavlov", "apavlov@chromium.org"), + Committer("Andre Boule", "aboule@apple.com"), + Committer("Andrei Popescu", "andreip@google.com", "andreip"), + Committer("Andrew Wellington", ["andrew@webkit.org", "proton@wiretapped.net"], "proton"), + Committer("Andras Becsi", "abecsi@webkit.org", "bbandix"), + Committer("Andreas Kling", "andreas.kling@nokia.com", "kling"), + Committer("Andy Estes", "aestes@apple.com", "estes"), + Committer("Anthony Ricaud", "rik@webkit.org", "rik"), + Committer("Anton Muhin", "antonm@chromium.org", "antonm"), + Committer("Ben Murdoch", "benm@google.com", "benm"), + Committer("Benjamin C Meyer", ["ben@meyerhome.net", "ben@webkit.org"], "icefox"), + Committer("Benjamin Otte", ["otte@gnome.org", "otte@webkit.org"], "otte"), + Committer("Brent Fulgham", "bfulgham@webkit.org", "bfulgham"), + Committer("Brett Wilson", "brettw@chromium.org", "brettx"), + Committer("Brian Weinstein", "bweinstein@apple.com", "bweinstein"), + Committer("Cameron McCormack", "cam@webkit.org", "heycam"), + Committer("Carol Szabo", "carol.szabo@nokia.com"), + Committer("Chang Shu", "Chang.Shu@nokia.com"), + Committer("Chris Evans", "cevans@google.com"), + Committer("Chris Marrin", "cmarrin@apple.com", "cmarrin"), + Committer("Chris Petersen", "cpetersen@apple.com", "cpetersen"), + Committer("Christian Dywan", ["christian@twotoasts.de", "christian@webkit.org"]), + Committer("Collin Jackson", "collinj@webkit.org"), + Committer("Csaba Osztrogonac", "ossy@webkit.org", "ossy"), + Committer("David Smith", ["catfish.man@gmail.com", "dsmith@webkit.org"], "catfishman"), + Committer("Dean Jackson", "dino@apple.com", "dino"), + Committer("Diego Gonzalez", ["diegohcg@webkit.org", "diego.gonzalez@openbossa.org"], "diegohcg"), + Committer("Dirk Pranke", "dpranke@chromium.org"), + Committer("Drew Wilson", "atwilson@chromium.org", "atwilson"), + Committer("Eli Fidler", "eli@staikos.net", "QBin"), + Committer("Enrica Casucci", "enrica@apple.com"), + Committer("Erik Arvidsson", "arv@chromium.org", "arv"), + Committer("Eric Roman", "eroman@chromium.org", "eroman"), + Committer("Evan Martin", "evan@chromium.org", "evmar"), + Committer("Evan Stade", "estade@chromium.org", "estade"), + Committer("Feng Qian", "feng@chromium.org"), + Committer("Fumitoshi Ukai", "ukai@chromium.org", "ukai"), + Committer("Gabor Loki", "loki@webkit.org", "loki04"), + Committer("Girish Ramakrishnan", ["girish@forwardbias.in", "ramakrishnan.girish@gmail.com"]), + Committer("Graham Dennis", ["Graham.Dennis@gmail.com", "gdennis@webkit.org"]), + Committer("Greg Bolsinga", "bolsinga@apple.com"), + Committer("Hin-Chung Lam", ["hclam@google.com", "hclam@chromium.org"]), + Committer("Ilya Tikhonovsky", "loislo@chromium.org", "loislo"), + Committer("Jakob Petsovits", ["jpetsovits@rim.com", "jpetso@gmx.at"], "jpetso"), + Committer("Jakub Wieczorek", "jwieczorek@webkit.org", "fawek"), + Committer("James Hawkins", ["jhawkins@chromium.org", "jhawkins@google.com"], "jhawkins"), + Committer("James Robinson", ["jamesr@chromium.org", "jamesr@google.com"], "jamesr"), + Committer("Jay Civelli", "jcivelli@chromium.org", "jcivelli"), + Committer("Jens Alfke", ["snej@chromium.org", "jens@apple.com"]), + Committer("Jer Noble", "jer.noble@apple.com", "jernoble"), + Committer("Jeremy Moskovich", ["playmobil@google.com", "jeremy@chromium.org"], "jeremymos"), + Committer("Jessie Berlin", ["jberlin@webkit.org", "jberlin@apple.com"]), + Committer("Jesus Sanchez-Palencia", ["jesus@webkit.org", "jesus.palencia@openbossa.org"], "jeez_"), + Committer("Jocelyn Turcotte", "jocelyn.turcotte@nokia.com", "jturcotte"), + Committer("John Abd-El-Malek", "jam@chromium.org", "jam"), + Committer("John Gregg", ["johnnyg@google.com", "johnnyg@chromium.org"], "johnnyg"), + Committer("Joost de Valk", ["joost@webkit.org", "webkit-dev@joostdevalk.nl"], "Altha"), + Committer("Julie Parent", ["jparent@google.com", "jparent@chromium.org"], "jparent"), + Committer("Julien Chaffraix", ["jchaffraix@webkit.org", "julien.chaffraix@gmail.com"]), + Committer("Jungshik Shin", "jshin@chromium.org"), + Committer("Justin Schuh", "jschuh@chromium.org", "jschuh"), + Committer("Keishi Hattori", "keishi@webkit.org", "keishi"), + Committer("Kelly Norton", "knorton@google.com"), + Committer("Kenneth Russell", "kbr@google.com"), + Committer("Kent Hansen", "kent.hansen@nokia.com", "khansen"), + Committer("Kinuko Yasuda", "kinuko@chromium.org", "kinuko"), + Committer("Krzysztof Kowalczyk", "kkowalczyk@gmail.com"), + Committer("Leandro Pereira", ["leandro@profusion.mobi", "leandro@webkit.org"], "acidx"), + Committer("Levi Weintraub", "lweintraub@apple.com"), + Committer("Luiz Agostini", ["luiz@webkit.org", "luiz.agostini@openbossa.org"], "lca"), + Committer("Mads Ager", "ager@chromium.org"), + Committer("Marcus Voltis Bulach", "bulach@chromium.org"), + Committer("Matt Lilek", ["webkit@mattlilek.com", "pewtermoose@webkit.org"]), + Committer("Matt Perry", "mpcomplete@chromium.org"), + Committer("Maxime Britto", ["maxime.britto@gmail.com", "britto@apple.com"]), + Committer("Maxime Simon", ["simon.maxime@gmail.com", "maxime.simon@webkit.org"], "maxime.simon"), + Committer("Martin Robinson", ["mrobinson@igalia.com", "mrobinson@webkit.org", "martin.james.robinson@gmail.com"], "mrobinson"), + Committer("Michelangelo De Simone", "michelangelo@webkit.org", "michelangelo"), + Committer("Mike Belshe", ["mbelshe@chromium.org", "mike@belshe.com"]), + Committer("Mike Fenton", ["mifenton@rim.com", "mike.fenton@torchmobile.com"], "mfenton"), + Committer("Mike Thole", ["mthole@mikethole.com", "mthole@apple.com"]), + Committer("Mikhail Naganov", "mnaganov@chromium.org"), + Committer("MORITA Hajime", "morrita@google.com", "morrita"), + Committer("Nico Weber", ["thakis@chromium.org", "thakis@google.com"], "thakis"), + Committer("Pam Greene", "pam@chromium.org", "pamg"), + Committer("Peter Kasting", ["pkasting@google.com", "pkasting@chromium.org"], "pkasting"), + Committer("Philippe Normand", ["pnormand@igalia.com", "philn@webkit.org"], "philn-tp"), + Committer("Pierre d'Herbemont", ["pdherbemont@free.fr", "pdherbemont@apple.com"], "pdherbemont"), + Committer("Pierre-Olivier Latour", "pol@apple.com", "pol"), + Committer("Robert Hogan", ["robert@webkit.org", "robert@roberthogan.net"], "mwenge"), + Committer("Roland Steiner", "rolandsteiner@chromium.org"), + Committer("Ryosuke Niwa", "rniwa@webkit.org", "rniwa"), + Committer("Scott Violet", "sky@chromium.org", "sky"), + Committer("Stephen White", "senorblanco@chromium.org", "senorblanco"), + Committer("Tony Chang", "tony@chromium.org", "tony^work"), + Committer("Tony Gentilcore", "tonyg@chromium.org", "tonyg-cr"), + Committer("Trey Matteson", "trey@usa.net", "trey"), + Committer("Tristan O'Tierney", ["tristan@otierney.net", "tristan@apple.com"]), + Committer("Victor Wang", "victorw@chromium.org", "victorw"), + Committer("Vitaly Repeshko", "vitalyr@chromium.org"), + Committer("William Siegrist", "wsiegrist@apple.com", "wms"), + Committer("Xiaomei Ji", "xji@chromium.org", "xji"), + Committer("Yael Aharon", "yael.aharon@nokia.com"), + Committer("Yaar Schnitman", ["yaar@chromium.org", "yaar@google.com"]), + Committer("Yong Li", ["yong.li.webkit@gmail.com", "yong.li@torchmobile.com"], "yong"), + Committer("Yongjun Zhang", "yongjun.zhang@nokia.com"), + Committer("Yuzo Fujishima", "yuzo@google.com", "yuzo"), + Committer("Zhenyao Mo", "zmo@google.com", "zhenyao"), + Committer("Zoltan Herczeg", "zherczeg@webkit.org", "zherczeg"), + Committer("Zoltan Horvath", "zoltan@webkit.org", "zoltan"), +] + + +# This is intended as a canonical, machine-readable list of all reviewers for +# WebKit. If your name is missing here and you are a reviewer, please add it. +# No review needed. + + +reviewers_list = [ + Reviewer("Ada Chan", "adachan@apple.com", "chanada"), + Reviewer("Adam Barth", "abarth@webkit.org", "abarth"), + Reviewer("Adam Roben", "aroben@apple.com", "aroben"), + Reviewer("Adam Treat", ["treat@kde.org", "treat@webkit.org"], "manyoso"), + Reviewer("Adele Peterson", "adele@apple.com", "adele"), + Reviewer("Alexey Proskuryakov", ["ap@webkit.org", "ap@apple.com"], "ap"), + Reviewer("Alice Liu", "alice.liu@apple.com", "aliu"), + Reviewer("Alp Toker", ["alp@nuanti.com", "alp@atoker.com", "alp@webkit.org"], "alp"), + Reviewer("Anders Carlsson", ["andersca@apple.com", "acarlsson@apple.com"], "andersca"), + Reviewer("Antonio Gomes", "tonikitoo@webkit.org", "tonikitoo"), + Reviewer("Antti Koivisto", ["koivisto@iki.fi", "antti@apple.com"], "anttik"), + Reviewer("Ariya Hidayat", ["ariya.hidayat@gmail.com", "ariya@webkit.org"], "ariya"), + Reviewer("Beth Dakin", "bdakin@apple.com", "dethbakin"), + Reviewer("Brady Eidson", "beidson@apple.com", "bradee-oh"), + Reviewer("Cameron Zwarich", ["zwarich@apple.com", "cwzwarich@apple.com", "cwzwarich@webkit.org"]), + Reviewer("Chris Blumenberg", "cblu@apple.com", "cblu"), + Reviewer("Chris Fleizach", "cfleizach@apple.com", "cfleizach"), + Reviewer("Chris Jerdonek", "cjerdonek@webkit.org", "cjerdonek"), + Reviewer("Dan Bernstein", ["mitz@webkit.org", "mitz@apple.com"], "mitzpettel"), + Reviewer("Daniel Bates", "dbates@webkit.org", "dydz"), + Reviewer("Darin Adler", "darin@apple.com", "darin"), + Reviewer("Darin Fisher", ["fishd@chromium.org", "darin@chromium.org"], "fishd"), + Reviewer("David Harrison", "harrison@apple.com", "harrison"), + Reviewer("David Hyatt", "hyatt@apple.com", "hyatt"), + Reviewer("David Kilzer", ["ddkilzer@webkit.org", "ddkilzer@apple.com"], "ddkilzer"), + Reviewer("David Levin", "levin@chromium.org", "dave_levin"), + Reviewer("Dimitri Glazkov", "dglazkov@chromium.org", "dglazkov"), + Reviewer("Dirk Schulze", "krit@webkit.org", "krit"), + Reviewer("Dmitry Titov", "dimich@chromium.org", "dimich"), + Reviewer("Don Melton", "gramps@apple.com", "gramps"), + Reviewer("Dumitru Daniliuc", "dumi@chromium.org", "dumi"), + Reviewer("Eric Carlson", "eric.carlson@apple.com"), + Reviewer("Eric Seidel", "eric@webkit.org", "eseidel"), + Reviewer("Gavin Barraclough", "barraclough@apple.com", "gbarra"), + Reviewer("Geoffrey Garen", "ggaren@apple.com", "ggaren"), + Reviewer("George Staikos", ["staikos@kde.org", "staikos@webkit.org"]), + Reviewer("Gustavo Noronha Silva", ["gns@gnome.org", "kov@webkit.org"], "kov"), + Reviewer("Holger Freyther", ["zecke@selfish.org", "zecke@webkit.org"], "zecke"), + Reviewer("Jan Alonzo", ["jmalonzo@gmail.com", "jmalonzo@webkit.org"], "janm"), + Reviewer("Jeremy Orlow", "jorlow@chromium.org", "jorlow"), + Reviewer("Jian Li", "jianli@chromium.org", "jianli"), + Reviewer("John Sullivan", "sullivan@apple.com", "sullivan"), + Reviewer("Jon Honeycutt", "jhoneycutt@apple.com", "jhoneycutt"), + Reviewer("Joseph Pecoraro", "joepeck@webkit.org", "JoePeck"), + Reviewer("Justin Garcia", "justin.garcia@apple.com", "justing"), + Reviewer("Ken Kocienda", "kocienda@apple.com"), + Reviewer("Kenneth Rohde Christiansen", ["kenneth@webkit.org", "kenneth.christiansen@openbossa.org"], "kenne"), + Reviewer("Kent Tamura", "tkent@chromium.org", "tkent"), + Reviewer("Kevin Decker", "kdecker@apple.com", "superkevin"), + Reviewer("Kevin McCullough", "kmccullough@apple.com", "maculloch"), + Reviewer("Kevin Ollivier", ["kevino@theolliviers.com", "kevino@webkit.org"], "kollivier"), + Reviewer("Lars Knoll", ["lars@trolltech.com", "lars@kde.org"], "lars"), + Reviewer("Laszlo Gombos", "laszlo.1.gombos@nokia.com", "lgombos"), + Reviewer("Maciej Stachowiak", "mjs@apple.com", "othermaciej"), + Reviewer("Mark Rowe", "mrowe@apple.com", "bdash"), + Reviewer("Nate Chapin", "japhet@chromium.org", "japhet"), + Reviewer("Nikolas Zimmermann", ["zimmermann@kde.org", "zimmermann@physik.rwth-aachen.de", "zimmermann@webkit.org"], "wildfox"), + Reviewer("Ojan Vafai", "ojan@chromium.org", "ojan"), + Reviewer("Oliver Hunt", "oliver@apple.com", "olliej"), + Reviewer("Pavel Feldman", "pfeldman@chromium.org", "pfeldman"), + Reviewer("Richard Williamson", "rjw@apple.com", "rjw"), + Reviewer("Rob Buis", ["rwlbuis@gmail.com", "rwlbuis@webkit.org"], "rwlbuis"), + Reviewer("Sam Weinig", ["sam@webkit.org", "weinig@apple.com"], "weinig"), + Reviewer("Shinichiro Hamaji", "hamaji@chromium.org", "hamaji"), + Reviewer("Simon Fraser", "simon.fraser@apple.com", "smfr"), + Reviewer("Simon Hausmann", ["hausmann@webkit.org", "hausmann@kde.org", "simon.hausmann@nokia.com"], "tronical"), + Reviewer("Stephanie Lewis", "slewis@apple.com", "sundiamonde"), + Reviewer("Steve Block", "steveblock@google.com", "steveblock"), + Reviewer("Steve Falkenburg", "sfalken@apple.com", "sfalken"), + Reviewer("Tim Omernick", "timo@apple.com"), + Reviewer("Timothy Hatcher", ["timothy@hatcher.name", "timothy@apple.com"], "xenon"), + Reviewer(u"Tor Arne Vestb\u00f8", ["vestbo@webkit.org", "tor.arne.vestbo@nokia.com"], "torarne"), + Reviewer("Vicki Murley", "vicki@apple.com"), + Reviewer("Xan Lopez", ["xan.lopez@gmail.com", "xan@gnome.org", "xan@webkit.org"], "xan"), + Reviewer("Yury Semikhatsky", "yurys@chromium.org", "yurys"), + Reviewer("Zack Rusin", "zack@kde.org", "zackr"), +] + + +class CommitterList: + + # Committers and reviewers are passed in to allow easy testing + + def __init__(self, + committers=committers_unable_to_review, + reviewers=reviewers_list): + self._committers = committers + reviewers + self._reviewers = reviewers + self._committers_by_email = {} + + def committers(self): + return self._committers + + def reviewers(self): + return self._reviewers + + def _email_to_committer_map(self): + if not len(self._committers_by_email): + for committer in self._committers: + for email in committer.emails: + self._committers_by_email[email] = committer + return self._committers_by_email + + def committer_by_name(self, name): + # This could be made into a hash lookup if callers need it to be fast. + for committer in self.committers(): + if committer.full_name == name: + return committer + + def committer_by_email(self, email): + return self._email_to_committer_map().get(email) + + def reviewer_by_email(self, email): + committer = self.committer_by_email(email) + if committer and not committer.can_review: + return None + return committer diff --git a/WebKitTools/Scripts/webkitpy/committers_unittest.py b/WebKitTools/Scripts/webkitpy/common/config/committers_unittest.py index f5dc539..068c0ee 100644 --- a/WebKitTools/Scripts/webkitpy/committers_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/config/committers_unittest.py @@ -27,12 +27,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from committers import CommitterList, Committer, Reviewer +from webkitpy.common.config.committers import CommitterList, Committer, Reviewer class CommittersTest(unittest.TestCase): def test_committer_lookup(self): - committer = Committer('Test One', 'one@test.com') + committer = Committer('Test One', 'one@test.com', 'one') reviewer = Reviewer('Test Two', ['two@test.com', 'two@rad.com', 'so_two@gmail.com']) committer_list = CommitterList(committers=[committer], reviewers=[reviewer]) @@ -43,6 +43,11 @@ class CommittersTest(unittest.TestCase): self.assertEqual(committer_list.committer_by_email('two@rad.com'), reviewer) self.assertEqual(committer_list.reviewer_by_email('so_two@gmail.com'), reviewer) + # Test valid committer and reviewer lookup + self.assertEqual(committer_list.committer_by_name("Test One"), committer) + self.assertEqual(committer_list.committer_by_name("Test Two"), reviewer) + self.assertEqual(committer_list.committer_by_name("Test Three"), None) + # Test that the first email is assumed to be the Bugzilla email address (for now) self.assertEqual(committer_list.committer_by_email('two@rad.com').bugzilla_email(), 'two@test.com') @@ -56,6 +61,8 @@ class CommittersTest(unittest.TestCase): # Test that emails returns a list. self.assertEqual(committer.emails, ['one@test.com']) + self.assertEqual(committer.irc_nickname, 'one') + # Test that committers returns committers and reviewers and reviewers() just reviewers. self.assertEqual(committer_list.committers(), [committer, reviewer]) self.assertEqual(committer_list.reviewers(), [reviewer]) diff --git a/WebKitTools/Scripts/webkitpy/common/config/irc.py b/WebKitTools/Scripts/webkitpy/common/config/irc.py new file mode 100644 index 0000000..950c573 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/config/irc.py @@ -0,0 +1,31 @@ +# Copyright (c) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +server="irc.freenode.net" +port=6667 +channel="#webkit" diff --git a/WebKitTools/Scripts/webkitpy/webkitport.py b/WebKitTools/Scripts/webkitpy/common/config/ports.py index cd60a54..9d4ac3f 100644 --- a/WebKitTools/Scripts/webkitpy/webkitport.py +++ b/WebKitTools/Scripts/webkitpy/common/config/ports.py @@ -29,9 +29,9 @@ # WebKit's Python module for understanding the various ports import os +import platform -from optparse import make_option -from webkitpy.executive import Executive +from webkitpy.common.system.executive import Executive class WebKitPort(object): @@ -47,10 +47,22 @@ class WebKitPort(object): "chromium": ChromiumPort, "gtk": GtkPort, "mac": MacPort, + "win": WinPort, "qt": QtPort, } - # FIXME: We should default to WinPort on Windows. - return ports.get(port_name, MacPort) + default_port = { + "Windows": WinPort, + "Darwin": MacPort, + } + # Do we really need MacPort as the ultimate default? + return ports.get(port_name, default_port.get(platform.system(), MacPort)) + + @staticmethod + def makeArgs(): + args = '--makeargs="-j%s"' % Executive().cpu_count() + if os.environ.has_key('MAKEFLAGS'): + args = '--makeargs="%s"' % os.environ['MAKEFLAGS'] + return args @classmethod def name(cls): @@ -100,6 +112,28 @@ class MacPort(WebKitPort): def flag(cls): return "--port=mac" + @classmethod + def _system_version(cls): + version_string = platform.mac_ver()[0] # e.g. "10.5.6" + version_tuple = version_string.split('.') + return map(int, version_tuple) + + @classmethod + def is_leopard(cls): + return tuple(cls._system_version()[:2]) == (10, 5) + + +class WinPort(WebKitPort): + + @classmethod + def name(cls): + return "Win" + + @classmethod + def flag(cls): + # FIXME: This is lame. We should autogenerate this from a codename or something. + return "--port=win" + class GtkPort(WebKitPort): @@ -115,7 +149,7 @@ class GtkPort(WebKitPort): def build_webkit_command(cls, build_style=None): command = WebKitPort.build_webkit_command(build_style=build_style) command.append("--gtk") - command.append('--makeargs="-j%s"' % Executive.cpu_count()) + command.append(WebKitPort.makeArgs()) return command @classmethod @@ -139,7 +173,7 @@ class QtPort(WebKitPort): def build_webkit_command(cls, build_style=None): command = WebKitPort.build_webkit_command(build_style=build_style) command.append("--qt") - command.append('--makeargs="-j%s"' % Executive.cpu_count()) + command.append(WebKitPort.makeArgs()) return command diff --git a/WebKitTools/Scripts/webkitpy/webkitport_unittest.py b/WebKitTools/Scripts/webkitpy/common/config/ports_unittest.py index 202234f..42c4f2d 100644 --- a/WebKitTools/Scripts/webkitpy/webkitport_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/config/ports_unittest.py @@ -29,8 +29,7 @@ import unittest -from webkitpy.executive import Executive -from webkitpy.webkitport import WebKitPort, MacPort, GtkPort, QtPort, ChromiumPort +from webkitpy.common.config.ports import WebKitPort, MacPort, GtkPort, QtPort, ChromiumPort class WebKitPortTest(unittest.TestCase): @@ -42,19 +41,25 @@ class WebKitPortTest(unittest.TestCase): self.assertEquals(MacPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug"]) self.assertEquals(MacPort.build_webkit_command(build_style="release"), [WebKitPort.script_path("build-webkit"), "--release"]) + class TestIsLeopard(MacPort): + @classmethod + def _system_version(cls): + return [10, 5] + self.assertTrue(TestIsLeopard.is_leopard()) + def test_gtk_port(self): self.assertEquals(GtkPort.name(), "Gtk") self.assertEquals(GtkPort.flag(), "--port=gtk") self.assertEquals(GtkPort.run_webkit_tests_command(), [WebKitPort.script_path("run-webkit-tests"), "--gtk"]) - self.assertEquals(GtkPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--gtk", '--makeargs="-j%s"' % Executive.cpu_count()]) - self.assertEquals(GtkPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--gtk", '--makeargs="-j%s"' % Executive.cpu_count()]) + self.assertEquals(GtkPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--gtk", WebKitPort.makeArgs()]) + self.assertEquals(GtkPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--gtk", WebKitPort.makeArgs()]) def test_qt_port(self): self.assertEquals(QtPort.name(), "Qt") self.assertEquals(QtPort.flag(), "--port=qt") self.assertEquals(QtPort.run_webkit_tests_command(), [WebKitPort.script_path("run-webkit-tests")]) - self.assertEquals(QtPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--qt", '--makeargs="-j%s"' % Executive.cpu_count()]) - self.assertEquals(QtPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--qt", '--makeargs="-j%s"' % Executive.cpu_count()]) + self.assertEquals(QtPort.build_webkit_command(), [WebKitPort.script_path("build-webkit"), "--qt", WebKitPort.makeArgs()]) + self.assertEquals(QtPort.build_webkit_command(build_style="debug"), [WebKitPort.script_path("build-webkit"), "--debug", "--qt", WebKitPort.makeArgs()]) def test_chromium_port(self): self.assertEquals(ChromiumPort.name(), "Chromium") diff --git a/WebKitTools/Scripts/webkitpy/common/net/__init__.py b/WebKitTools/Scripts/webkitpy/common/net/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/bugzilla.py b/WebKitTools/Scripts/webkitpy/common/net/bugzilla.py index 4506af2..40db32c 100644 --- a/WebKitTools/Scripts/webkitpy/bugzilla.py +++ b/WebKitTools/Scripts/webkitpy/common/net/bugzilla.py @@ -30,25 +30,24 @@ # # WebKit's Python module for interacting with Bugzilla +import os.path import re -import subprocess +import StringIO from datetime import datetime # used in timestamp() -# Import WebKit-specific modules. -from webkitpy.webkit_logging import error, log -from webkitpy.committers import CommitterList -from webkitpy.credentials import Credentials -from webkitpy.user import User - -# WebKit includes a built copy of BeautifulSoup in Scripts/webkitpy -# so this import should always succeed. -from .BeautifulSoup import BeautifulSoup, SoupStrainer - -from mechanize import Browser +from webkitpy.common.system.deprecated_logging import error, log +from webkitpy.common.config import committers +from webkitpy.common.net.credentials import Credentials +from webkitpy.common.system.ospath import relpath +from webkitpy.common.system.user import User +from webkitpy.thirdparty.autoinstalled.mechanize import Browser +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, SoupStrainer def parse_bug_id(message): + if not message: + return None match = re.search("http\://webkit\.org/b/(?P<bug_id>\d+)", message) if match: return int(match.group('bug_id')) @@ -66,6 +65,8 @@ def timestamp(): class Attachment(object): + rollout_preamble = "ROLLOUT of r" + def __init__(self, attachment_dictionary, bug): self._attachment_dictionary = attachment_dictionary self._bug = bug @@ -97,21 +98,34 @@ class Attachment(object): def is_obsolete(self): return not not self._attachment_dictionary.get("is_obsolete") + def is_rollout(self): + return self.name().startswith(self.rollout_preamble) + def name(self): return self._attachment_dictionary.get("name") + def attach_date(self): + return self._attachment_dictionary.get("attach_date") + def review(self): return self._attachment_dictionary.get("review") def commit_queue(self): return self._attachment_dictionary.get("commit-queue") + def in_rietveld(self): + return self._attachment_dictionary.get("in-rietveld") + def url(self): # FIXME: This should just return # self._bugzilla().attachment_url_for_id(self.id()). scm_unittest.py # depends on the current behavior. return self._attachment_dictionary.get("url") + def contents(self): + # FIXME: We shouldn't be grabbing at _bugzilla. + return self._bug._bugzilla.fetch_attachment_contents(self.id()) + def _validate_flag_value(self, flag): email = self._attachment_dictionary.get("%s_email" % flag) if not email: @@ -147,9 +161,20 @@ class Bug(object): def id(self): return self.bug_dictionary["id"] + def title(self): + return self.bug_dictionary["title"] + def assigned_to_email(self): return self.bug_dictionary["assigned_to_email"] + # FIXME: This information should be stored in some sort of webkit_config.py instead of here. + unassigned_emails = frozenset([ + "webkit-unassigned@lists.webkit.org", + "webkit-qt-unassigned@trolltech.com", + ]) + def is_unassigned(self): + return self.assigned_to_email() in self.unassigned_emails + # Rarely do we actually want obsolete attachments def attachments(self, include_obsolete=False): attachments = self.bug_dictionary["attachments"] @@ -182,6 +207,9 @@ class Bug(object): # a valid committer. return filter(lambda patch: patch.committer(), patches) + def in_rietveld_queue_patches(self): + return [patch for patch in self.patches() if patch.in_rietveld() == None] + # A container for all of the logic for making and parsing buzilla queries. class BugzillaQueries(object): @@ -218,6 +246,14 @@ class BugzillaQueries(object): def _fetch_attachment_ids_request_query(self, query): return self._parse_attachment_ids_request_query(self._load_query(query)) + def _parse_quips(self, page): + soup = BeautifulSoup(page, convertEntities=BeautifulSoup.HTML_ENTITIES) + quips = soup.find(text=re.compile(r"Existing quips:")).findNext("ul").findAll("li") + return [unicode(quip_entry.string) for quip_entry in quips] + + def fetch_quips(self): + return self._parse_quips(self._load_query("/quips.cgi?action=show")) + # List of all r+'d bugs. def fetch_bug_ids_from_pending_commit_list(self): needs_commit_query_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B" @@ -237,6 +273,16 @@ class BugzillaQueries(object): return sum([self._fetch_bug(bug_id).commit_queued_patches() for bug_id in self.fetch_bug_ids_from_commit_queue()], []) + def fetch_first_patch_from_rietveld_queue(self): + # rietveld-queue processes all patches that don't have in-rietveld set. + query_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=notsubstring&value0-0-0=in-rietveld&field0-1-0=attachments.ispatch&type0-1-0=equals&value0-1-0=1&order=Last+Changed&field0-2-0=attachments.isobsolete&type0-2-0=equals&value0-2-0=0" + bugs = self._fetch_bug_ids_advanced_query(query_url) + if not len(bugs): + return None + + patches = self._fetch_bug(bugs[0]).in_rietveld_queue_patches() + return patches[0] if len(patches) else None + def _fetch_bug_ids_from_review_queue(self): review_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?" return self._fetch_bug_ids_advanced_query(review_queue_url) @@ -264,23 +310,37 @@ class CommitterValidator(object): def _view_source_url(self, local_path): return "http://trac.webkit.org/browser/trunk/%s" % local_path + def _checkout_root(self): + # FIXME: This is a hack, we would have this from scm.checkout_root + # if we had any way to get to an scm object here. + components = __file__.split(os.sep) + tools_index = components.index("WebKitTools") + return os.sep.join(components[:tools_index]) + + def _committers_py_path(self): + # extension can sometimes be .pyc, we always want .py + (path, extension) = os.path.splitext(committers.__file__) + # FIXME: When we're allowed to use python 2.6 we can use the real + # os.path.relpath + path = relpath(path, self._checkout_root()) + return ".".join([path, "py"]) + def _flag_permission_rejection_message(self, setter_email, flag_name): - # This could be computed from CommitterList.__file__ - committer_list = "WebKitTools/Scripts/webkitpy/committers.py" # Should come from some webkit_config.py contribution_guidlines = "http://webkit.org/coding/contributing.html" # This could be queried from the status_server. queue_administrator = "eseidel@chromium.org" # This could be queried from the tool. queue_name = "commit-queue" + committers_list = self._committers_py_path() message = "%s does not have %s permissions according to %s." % ( setter_email, flag_name, - self._view_source_url(committer_list)) + self._view_source_url(committers_list)) message += "\n\n- If you do not have %s rights please read %s for instructions on how to use bugzilla flags." % ( flag_name, contribution_guidlines) message += "\n\n- If you have %s rights please correct the error in %s by adding yourself to the file (no review needed). " % ( - flag_name, committer_list) + flag_name, committers_list) message += "Due to bug 30084 the %s will require a restart after your change. " % queue_name message += "Please contact %s to request a %s restart. " % ( queue_administrator, queue_name) @@ -333,11 +393,12 @@ class CommitterValidator(object): class Bugzilla(object): - def __init__(self, dryrun=False, committers=CommitterList()): + def __init__(self, dryrun=False, committers=committers.CommitterList()): self.dryrun = dryrun self.authenticated = False self.queries = BugzillaQueries(self) self.committers = committers + self.cached_quips = [] # FIXME: We should use some sort of Browser mock object when in dryrun # mode (to prevent any mistakes). @@ -350,18 +411,30 @@ class Bugzilla(object): bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) bug_server_url = "https://%s/" % bug_server_host - unassigned_email = "webkit-unassigned@lists.webkit.org" + + def quips(self): + # We only fetch and parse the list of quips once per instantiation + # so that we do not burden bugs.webkit.org. + if not self.cached_quips and not self.dryrun: + self.cached_quips = self.queries.fetch_quips() + return self.cached_quips def bug_url_for_bug_id(self, bug_id, xml=False): + if not bug_id: + return None content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def short_bug_url_for_bug_id(self, bug_id): + if not bug_id: + return None return "http://webkit.org/b/%s" % bug_id def attachment_url_for_id(self, attachment_id, action="view"): + if not attachment_id: + return None action_param = "" if action and action != "view": action_param = "&action=%s" % action @@ -379,6 +452,31 @@ class Bugzilla(object): attachment[flag_name] = flag['status'] if flag['status'] == '+': attachment[result_key] = flag['setter'] + # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date. + + def _string_contents(self, soup): + # WebKit's bugzilla instance uses UTF-8. + # BeautifulSoup always returns Unicode strings, however + # the .string method returns a (unicode) NavigableString. + # NavigableString can confuse other parts of the code, so we + # convert from NavigableString to a real unicode() object using unicode(). + return unicode(soup.string) + + # Example: 2010-01-20 14:31 PST + # FIXME: Some bugzilla dates seem to have seconds in them? + # Python does not support timezones out of the box. + # Assume that bugzilla always uses PST (which is true for bugs.webkit.org) + _bugzilla_date_format = "%Y-%m-%d %H:%M" + + @classmethod + def _parse_date(cls, date_string): + (date, time, time_zone) = date_string.split(" ") + # Ignore the timezone because python doesn't understand timezones out of the box. + date_string = "%s %s" % (date, time) + return datetime.strptime(date_string, cls._bugzilla_date_format) + + def _date_contents(self, soup): + return self._parse_date(self._string_contents(soup)) def _parse_attachment_element(self, element, bug_id): attachment = {} @@ -388,12 +486,15 @@ class Bugzilla(object): attachment['id'] = int(element.find('attachid').string) # FIXME: No need to parse out the url here. attachment['url'] = self.attachment_url_for_id(attachment['id']) - attachment['name'] = unicode(element.find('desc').string) - attachment['attacher_email'] = str(element.find('attacher').string) - attachment['type'] = str(element.find('type').string) + attachment["attach_date"] = self._date_contents(element.find("date")) + attachment['name'] = self._string_contents(element.find('desc')) + attachment['attacher_email'] = self._string_contents(element.find('attacher')) + attachment['type'] = self._string_contents(element.find('type')) self._parse_attachment_flag( element, 'review', attachment, 'reviewer_email') self._parse_attachment_flag( + element, 'in-rietveld', attachment, 'rietveld_uploader_email') + self._parse_attachment_flag( element, 'commit-queue', attachment, 'committer_email') return attachment @@ -401,10 +502,10 @@ class Bugzilla(object): soup = BeautifulSoup(page) bug = {} bug["id"] = int(soup.find("bug_id").string) - bug["title"] = unicode(soup.find("short_desc").string) - bug["reporter_email"] = str(soup.find("reporter").string) - bug["assigned_to_email"] = str(soup.find("assigned_to").string) - bug["cc_emails"] = [str(element.string) + bug["title"] = self._string_contents(soup.find("short_desc")) + bug["reporter_email"] = self._string_contents(soup.find("reporter")) + bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to")) + bug["cc_emails"] = [self._string_contents(element) for element in soup.findAll('cc')] bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')] return bug @@ -418,13 +519,23 @@ class Bugzilla(object): return self.browser.open(bug_url) def fetch_bug_dictionary(self, bug_id): - return self._parse_bug_page(self._fetch_bug_page(bug_id)) + try: + return self._parse_bug_page(self._fetch_bug_page(bug_id)) + except: + self.authenticate() + return self._parse_bug_page(self._fetch_bug_page(bug_id)) # FIXME: A BugzillaCache object should provide all these fetch_ methods. def fetch_bug(self, bug_id): return Bug(self.fetch_bug_dictionary(bug_id), self) + def fetch_attachment_contents(self, attachment_id): + attachment_url = self.attachment_url_for_id(attachment_id) + # We need to authenticate to download patches from security bugs. + self.authenticate() + return self.browser.open(attachment_url).read() + def _parse_bug_id_from_attachment_page(self, page): # The "Up" relation happens to point to the bug. up_link = BeautifulSoup(page).find('link', rel='Up') @@ -494,6 +605,7 @@ class Bugzilla(object): raise Exception(errorMessage) else: self.authenticated = True + self.username = username def _fill_attachment_form(self, description, @@ -501,7 +613,8 @@ class Bugzilla(object): comment_text=None, mark_for_review=False, mark_for_commit_queue=False, - mark_for_landing=False, bug_id=None): + mark_for_landing=False, + bug_id=None): self.browser['description'] = description self.browser['ispatch'] = ("1",) self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) @@ -517,6 +630,7 @@ class Bugzilla(object): patch_name = "bug-%s-%s.patch" % (bug_id, timestamp()) else: patch_name ="%s.patch" % timestamp() + self.browser.add_file(patch_file_object, "text/plain", patch_name, @@ -524,7 +638,7 @@ class Bugzilla(object): def add_patch_to_bug(self, bug_id, - patch_file_object, + diff, description, comment_text=None, mark_for_review=False, @@ -543,6 +657,11 @@ class Bugzilla(object): self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % ( self.bug_server_url, bug_id)) self.browser.select_form(name="entryform") + + # _fill_attachment_form expects a file-like object + # Patch files are already binary, so no encoding needed. + assert(isinstance(diff, str)) + patch_file_object = StringIO.StringIO(diff) self._fill_attachment_form(description, patch_file_object, mark_for_review=mark_for_review, @@ -554,15 +673,6 @@ class Bugzilla(object): self.browser['comment'] = comment_text self.browser.submit() - def prompt_for_component(self, components): - log("Please pick a component:") - i = 0 - for name in components: - i += 1 - log("%2d. %s" % (i, name)) - result = int(User.prompt("Enter a number: ")) - 1 - return components[result] - def _check_create_bug_response(self, response_html): match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html) @@ -586,9 +696,11 @@ class Bugzilla(object): bug_title, bug_description, component=None, - patch_file_object=None, + diff=None, patch_description=None, cc=None, + blocked=None, + assignee=None, mark_for_review=False, mark_for_commit_queue=False): self.authenticate() @@ -605,14 +717,24 @@ class Bugzilla(object): if not component: component = "New Bugs" if component not in component_names: - component = self.prompt_for_component(component_names) - self.browser['component'] = [component] + component = User.prompt_with_list("Please pick a component:", component_names) + self.browser["component"] = [component] if cc: - self.browser['cc'] = cc - self.browser['short_desc'] = bug_title - self.browser['comment'] = bug_description - - if patch_file_object: + self.browser["cc"] = cc + if blocked: + self.browser["blocked"] = unicode(blocked) + if assignee == None: + assignee = self.username + if assignee and not self.browser.find_control("assigned_to").disabled: + self.browser["assigned_to"] = assignee + self.browser["short_desc"] = bug_title + self.browser["comment"] = bug_description + + if diff: + # _fill_attachment_form expects a file-like object + # Patch files are already binary, so no encoding needed. + assert(isinstance(diff, str)) + patch_file_object = StringIO.StringIO(diff) self._fill_attachment_form( patch_description, patch_file_object, @@ -630,8 +752,10 @@ class Bugzilla(object): # FIXME: This will break if we ever re-order attachment flags if flag_name == "review": return self.browser.find_control(type='select', nr=0) - if flag_name == "commit-queue": + elif flag_name == "commit-queue": return self.browser.find_control(type='select', nr=1) + elif flag_name == "in-rietveld": + return self.browser.find_control(type='select', nr=2) raise Exception("Don't know how to find flag named \"%s\"" % flag_name) def clear_attachment_flags(self, @@ -658,8 +782,8 @@ class Bugzilla(object): attachment_id, flag_name, flag_value, - comment_text, - additional_comment_text): + comment_text=None, + additional_comment_text=None): # FIXME: We need a way to test this function on a live bugzilla # instance. @@ -674,7 +798,10 @@ class Bugzilla(object): self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) - self.browser.set_value(comment_text, name='comment', nr=0) + + if comment_text: + self.browser.set_value(comment_text, name='comment', nr=0) + self._find_select_element_for_flag(flag_name).value = (flag_value,) self.browser.submit() @@ -782,9 +909,13 @@ class Bugzilla(object): possible_bug_statuses = map(lambda item: item.name, bug_status.items) if "REOPENED" in possible_bug_statuses: bug_status.value = ["REOPENED"] + # If the bug was never confirmed it will not have a "REOPENED" + # state, but only an "UNCONFIRMED" state. + elif "UNCONFIRMED" in possible_bug_statuses: + bug_status.value = ["UNCONFIRMED"] else: - log("Did not reopen bug %s. " + - "It appears to already be open with status %s." % ( - bug_id, bug_status.value)) + # FIXME: This logic is slightly backwards. We won't print this + # message if the bug is already open with state "UNCONFIRMED". + log("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value)) self.browser['comment'] = comment_text self.browser.submit() diff --git a/WebKitTools/Scripts/webkitpy/bugzilla_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/bugzilla_unittest.py index d555f78..3556121 100644 --- a/WebKitTools/Scripts/webkitpy/bugzilla_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/net/bugzilla_unittest.py @@ -28,12 +28,13 @@ import unittest -from webkitpy.committers import CommitterList, Reviewer, Committer -from webkitpy.bugzilla import Bugzilla, BugzillaQueries, parse_bug_id, CommitterValidator -from webkitpy.outputcapture import OutputCapture -from webkitpy.mock import Mock +import datetime -from webkitpy.BeautifulSoup import BeautifulSoup +from webkitpy.common.config.committers import CommitterList, Reviewer, Committer +from webkitpy.common.net.bugzilla import Bugzilla, BugzillaQueries, parse_bug_id, CommitterValidator, Bug +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup class MockBrowser(object): @@ -49,14 +50,25 @@ class MockBrowser(object): def submit(self): pass + +class BugTest(unittest.TestCase): + def test_is_unassigned(self): + for email in Bug.unassigned_emails: + bug = Bug({"assigned_to_email" : email}, bugzilla=None) + self.assertTrue(bug.is_unassigned()) + bug = Bug({"assigned_to_email" : "test@test.com"}, bugzilla=None) + self.assertFalse(bug.is_unassigned()) + + class CommitterValidatorTest(unittest.TestCase): def test_flag_permission_rejection_message(self): validator = CommitterValidator(bugzilla=None) - expected_messsage="""foo@foo.com does not have review permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/committers.py. + self.assertEqual(validator._committers_py_path(), "WebKitTools/Scripts/webkitpy/common/config/committers.py") + expected_messsage="""foo@foo.com does not have review permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/common/config/committers.py. - If you do not have review rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags. -- If you have review rights please correct the error in WebKitTools/Scripts/webkitpy/committers.py by adding yourself to the file (no review needed). Due to bug 30084 the commit-queue will require a restart after your change. Please contact eseidel@chromium.org to request a commit-queue restart. After restart the commit-queue will correctly respect your review rights.""" +- If you have review rights please correct the error in WebKitTools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed). Due to bug 30084 the commit-queue will require a restart after your change. Please contact eseidel@chromium.org to request a commit-queue restart. After restart the commit-queue will correctly respect your review rights.""" self.assertEqual(validator._flag_permission_rejection_message("foo@foo.com", "review"), expected_messsage) @@ -84,9 +96,15 @@ class BugzillaTest(unittest.TestCase): status="+" setter="two@test.com" /> + <flag name="in-rietveld" + id="17933" + status="+" + setter="three@test.com" + /> </attachment> ''' _expected_example_attachment_parsing = { + 'attach_date': datetime.datetime(2009, 07, 29, 10, 23), 'bug_id' : 100, 'is_obsolete' : True, 'is_patch' : True, @@ -98,9 +116,18 @@ class BugzillaTest(unittest.TestCase): 'reviewer_email' : 'one@test.com', 'commit-queue' : '+', 'committer_email' : 'two@test.com', + 'in-rietveld': '+', + 'rietveld_uploader_email': 'three@test.com', 'attacher_email' : 'christian.plesner.hansen@gmail.com', } + def test_url_creation(self): + # FIXME: These would be all better as doctests + bugs = Bugzilla() + self.assertEquals(None, bugs.bug_url_for_bug_id(None)) + self.assertEquals(None, bugs.short_bug_url_for_bug_id(None)) + self.assertEquals(None, bugs.attachment_url_for_id(None)) + def test_parse_bug_id(self): # FIXME: These would be all better as doctests bugs = Bugzilla() @@ -171,12 +198,12 @@ removed-because-it-was-really-long ZEZpbmlzaExvYWRXaXRoUmVhc29uOnJlYXNvbl07Cit9CisKIEBlbmQKIAogI2VuZGlmCg== </data> - <flag name="review" - id="27602" - status="?" - setter="mjs@apple.com" - /> - </attachment> + <flag name="review" + id="27602" + status="?" + setter="mjs@apple.com" + /> + </attachment> </bug> </bugzilla> """ @@ -187,6 +214,7 @@ ZEZpbmlzaExvYWRXaXRoUmVhc29uOnJlYXNvbl07Cit9CisKIEBlbmQKIAogI2VuZGlmCg== "reporter_email" : "eric@webkit.org", "assigned_to_email" : "webkit-unassigned@lists.webkit.org", "attachments" : [{ + "attach_date": datetime.datetime(2009, 12, 27, 23, 51), 'name': u'Patch', 'url' : "https://bugs.webkit.org/attachment.cgi?id=45548", 'is_obsolete': False, @@ -199,6 +227,7 @@ ZEZpbmlzaExvYWRXaXRoUmVhc29uOnJlYXNvbl07Cit9CisKIEBlbmQKIAogI2VuZGlmCg== }], } + # FIXME: This should move to a central location and be shared by more unit tests. def _assert_dictionaries_equal(self, actual, expected): # Make sure we aren't parsing more or less than we expect self.assertEquals(sorted(actual.keys()), sorted(expected.keys())) @@ -241,6 +270,35 @@ ZEZpbmlzaExvYWRXaXRoUmVhc29uOnJlYXNvbl07Cit9CisKIEBlbmQKIAogI2VuZGlmCg== expected_stderr = "Adding ['adam@example.com'] to the CC list for bug 42\n" OutputCapture().assert_outputs(self, bugzilla.add_cc_to_bug, [42, ["adam@example.com"]], expected_stderr=expected_stderr) + def _mock_control_item(self, name): + mock_item = Mock() + mock_item.name = name + return mock_item + + def _mock_find_control(self, item_names=[], selected_index=0): + mock_control = Mock() + mock_control.items = [self._mock_control_item(name) for name in item_names] + mock_control.value = [item_names[selected_index]] if item_names else None + return lambda name, type: mock_control + + def _assert_reopen(self, item_names=None, selected_index=None, extra_stderr=None): + bugzilla = Bugzilla() + bugzilla.browser = MockBrowser() + bugzilla.authenticate = lambda: None + + mock_find_control = self._mock_find_control(item_names, selected_index) + bugzilla.browser.find_control = mock_find_control + expected_stderr = "Re-opening bug 42\n['comment']\n" + if extra_stderr: + expected_stderr += extra_stderr + OutputCapture().assert_outputs(self, bugzilla.reopen_bug, [42, ["comment"]], expected_stderr=expected_stderr) + + def test_reopen_bug(self): + self._assert_reopen(item_names=["REOPENED", "RESOLVED", "CLOSED"], selected_index=1) + self._assert_reopen(item_names=["UNCONFIRMED", "RESOLVED", "CLOSED"], selected_index=1) + extra_stderr = "Did not reopen bug 42, it appears to already be open with status ['NEW'].\n" + self._assert_reopen(item_names=["NEW", "RESOLVED"], selected_index=0, extra_stderr=extra_stderr) + class BugzillaQueriesTest(unittest.TestCase): _sample_request_page = """ @@ -289,15 +347,37 @@ class BugzillaQueriesTest(unittest.TestCase): </body> </html> """ + _sample_quip_page = u""" +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" + "http://www.w3.org/TR/html4/loose.dtd"> +<html> + <head> + <title>Bugzilla Quip System</title> + </head> + <body> + <h2> + + Existing quips: + </h2> + <ul> + <li>Everything should be made as simple as possible, but not simpler. - Albert Einstein</li> + <li>Good artists copy. Great artists steal. - Pablo Picasso</li> + <li>\u00e7gua mole em pedra dura, tanto bate at\u008e que fura.</li> + + </ul> + </body> +</html> +""" def test_request_page_parsing(self): queries = BugzillaQueries(None) self.assertEquals([40511, 40722, 40723], queries._parse_attachment_ids_request_query(self._sample_request_page)) + def test_quip_page_parsing(self): + queries = BugzillaQueries(None) + expected_quips = ["Everything should be made as simple as possible, but not simpler. - Albert Einstein", "Good artists copy. Great artists steal. - Pablo Picasso", u"\u00e7gua mole em pedra dura, tanto bate at\u008e que fura."] + self.assertEquals(expected_quips, queries._parse_quips(self._sample_quip_page)) + def test_load_query(self): queries = BugzillaQueries(Mock()) queries._load_query("request.cgi?action=queue&type=review&group=type") - - -if __name__ == '__main__': - unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/common/net/buildbot.py b/WebKitTools/Scripts/webkitpy/common/net/buildbot.py new file mode 100644 index 0000000..5078c55 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/buildbot.py @@ -0,0 +1,500 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# WebKit's Python module for interacting with WebKit's buildbot + +import operator +import re +import urllib +import urllib2 +import xmlrpclib + +from webkitpy.common.system.logutils import get_logger +from webkitpy.thirdparty.autoinstalled.mechanize import Browser +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup + + +_log = get_logger(__file__) + + +class Builder(object): + def __init__(self, name, buildbot): + self._name = name + self._buildbot = buildbot + self._builds_cache = {} + self._revision_to_build_number = None + self._browser = Browser() + self._browser.set_handle_robots(False) # The builder pages are excluded by robots.txt + + def name(self): + return self._name + + def results_url(self): + return "http://%s/results/%s" % (self._buildbot.buildbot_host, self.url_encoded_name()) + + def url_encoded_name(self): + return urllib.quote(self._name) + + def url(self): + return "http://%s/builders/%s" % (self._buildbot.buildbot_host, self.url_encoded_name()) + + # This provides a single place to mock + def _fetch_build(self, build_number): + build_dictionary = self._buildbot._fetch_xmlrpc_build_dictionary(self, build_number) + if not build_dictionary: + return None + return Build(self, + build_number=int(build_dictionary['number']), + revision=int(build_dictionary['revision']), + is_green=(build_dictionary['results'] == 0) # Undocumented, buildbot XMLRPC, 0 seems to mean "pass" + ) + + def build(self, build_number): + if not build_number: + return None + cached_build = self._builds_cache.get(build_number) + if cached_build: + return cached_build + + build = self._fetch_build(build_number) + self._builds_cache[build_number] = build + return build + + def force_build(self, username="webkit-patch", comments=None): + def predicate(form): + try: + return form.find_control("username") + except Exception, e: + return False + self._browser.open(self.url()) + self._browser.select_form(predicate=predicate) + self._browser["username"] = username + if comments: + self._browser["comments"] = comments + return self._browser.submit() + + file_name_regexp = re.compile(r"r(?P<revision>\d+) \((?P<build_number>\d+)\)") + def _revision_and_build_for_filename(self, filename): + # Example: "r47483 (1)/" or "r47483 (1).zip" + match = self.file_name_regexp.match(filename) + return (int(match.group("revision")), int(match.group("build_number"))) + + def _fetch_revision_to_build_map(self): + # All _fetch requests go through _buildbot for easier mocking + try: + # FIXME: This method is horribly slow due to the huge network load. + # FIXME: This is a poor way to do revision -> build mapping. + # Better would be to ask buildbot through some sort of API. + print "Loading revision/build list from %s." % self.results_url() + print "This may take a while..." + result_files = self._buildbot._fetch_twisted_directory_listing(self.results_url()) + except urllib2.HTTPError, error: + if error.code != 404: + raise + result_files = [] + + # This assumes there was only one build per revision, which is false but we don't care for now. + return dict([self._revision_and_build_for_filename(file_info["filename"]) for file_info in result_files]) + + def _revision_to_build_map(self): + if not self._revision_to_build_number: + self._revision_to_build_number = self._fetch_revision_to_build_map() + return self._revision_to_build_number + + def revision_build_pairs_with_results(self): + return self._revision_to_build_map().items() + + # This assumes there can be only one build per revision, which is false, but we don't care for now. + def build_for_revision(self, revision, allow_failed_lookups=False): + # NOTE: This lookup will fail if that exact revision was never built. + build_number = self._revision_to_build_map().get(int(revision)) + if not build_number: + return None + build = self.build(build_number) + if not build and allow_failed_lookups: + # Builds for old revisions with fail to lookup via buildbot's xmlrpc api. + build = Build(self, + build_number=build_number, + revision=revision, + is_green=False, + ) + return build + + def find_failure_transition(self, red_build, look_back_limit=30): + if not red_build or red_build.is_green(): + return (None, None) + common_failures = None + current_build = red_build + build_after_current_build = None + look_back_count = 0 + while current_build: + if current_build.is_green(): + # current_build can't possibly have any failures in common + # with red_build because it's green. + break + results = current_build.layout_test_results() + # We treat a lack of results as if all the test failed. + # This occurs, for example, when we can't compile at all. + if results: + failures = set(results.failing_tests()) + if common_failures == None: + common_failures = failures + common_failures = common_failures.intersection(failures) + if not common_failures: + # current_build doesn't have any failures in common with + # the red build we're worried about. We assume that any + # failures in current_build were due to flakiness. + break + look_back_count += 1 + if look_back_count > look_back_limit: + return (None, current_build) + build_after_current_build = current_build + current_build = current_build.previous_build() + # We must iterate at least once because red_build is red. + assert(build_after_current_build) + # Current build must either be green or have no failures in common + # with red build, so we've found our failure transition. + return (current_build, build_after_current_build) + + # FIXME: This likely does not belong on Builder + def suspect_revisions_for_transition(self, last_good_build, first_bad_build): + suspect_revisions = range(first_bad_build.revision(), + last_good_build.revision(), + -1) + suspect_revisions.reverse() + return suspect_revisions + + def blameworthy_revisions(self, red_build_number, look_back_limit=30, avoid_flakey_tests=True): + red_build = self.build(red_build_number) + (last_good_build, first_bad_build) = \ + self.find_failure_transition(red_build, look_back_limit) + if not last_good_build: + return [] # We ran off the limit of our search + # If avoid_flakey_tests, require at least 2 bad builds before we + # suspect a real failure transition. + if avoid_flakey_tests and first_bad_build == red_build: + return [] + return self.suspect_revisions_for_transition(last_good_build, first_bad_build) + + +# FIXME: This should be unified with all the layout test results code in the layout_tests package +class LayoutTestResults(object): + stderr_key = u'Tests that had stderr output:' + fail_key = u'Tests where results did not match expected results:' + timeout_key = u'Tests that timed out:' + crash_key = u'Tests that caused the DumpRenderTree tool to crash:' + missing_key = u'Tests that had no expected results (probably new):' + + expected_keys = [ + stderr_key, + fail_key, + crash_key, + timeout_key, + missing_key, + ] + + @classmethod + def _parse_results_html(cls, page): + parsed_results = {} + tables = BeautifulSoup(page).findAll("table") + for table in tables: + table_title = unicode(table.findPreviousSibling("p").string) + if table_title not in cls.expected_keys: + # This Exception should only ever be hit if run-webkit-tests changes its results.html format. + raise Exception("Unhandled title: %s" % table_title) + # We might want to translate table titles into identifiers before storing. + parsed_results[table_title] = [unicode(row.find("a").string) for row in table.findAll("tr")] + + return parsed_results + + @classmethod + def _fetch_results_html(cls, base_url): + results_html = "%s/results.html" % base_url + # FIXME: We need to move this sort of 404 logic into NetworkTransaction or similar. + try: + page = urllib2.urlopen(results_html) + return cls._parse_results_html(page) + except urllib2.HTTPError, error: + if error.code != 404: + raise + + @classmethod + def results_from_url(cls, base_url): + parsed_results = cls._fetch_results_html(base_url) + if not parsed_results: + return None + return cls(base_url, parsed_results) + + def __init__(self, base_url, parsed_results): + self._base_url = base_url + self._parsed_results = parsed_results + + def parsed_results(self): + return self._parsed_results + + def failing_tests(self): + failing_keys = [self.fail_key, self.crash_key, self.timeout_key] + return sorted(sum([tests for key, tests in self._parsed_results.items() if key in failing_keys], [])) + + +class Build(object): + def __init__(self, builder, build_number, revision, is_green): + self._builder = builder + self._number = build_number + self._revision = revision + self._is_green = is_green + self._layout_test_results = None + + @staticmethod + def build_url(builder, build_number): + return "%s/builds/%s" % (builder.url(), build_number) + + def url(self): + return self.build_url(self.builder(), self._number) + + def results_url(self): + results_directory = "r%s (%s)" % (self.revision(), self._number) + return "%s/%s" % (self._builder.results_url(), urllib.quote(results_directory)) + + def layout_test_results(self): + if not self._layout_test_results: + self._layout_test_results = LayoutTestResults.results_from_url(self.results_url()) + return self._layout_test_results + + def builder(self): + return self._builder + + def revision(self): + return self._revision + + def is_green(self): + return self._is_green + + def previous_build(self): + # previous_build() allows callers to avoid assuming build numbers are sequential. + # They may not be sequential across all master changes, or when non-trunk builds are made. + return self._builder.build(self._number - 1) + + +class BuildBot(object): + # FIXME: This should move into some sort of webkit_config.py + default_host = "build.webkit.org" + + def __init__(self, host=default_host): + self.buildbot_host = host + self._builder_by_name = {} + + # If any core builder is red we should not be landing patches. Other + # builders should be added to this list once they are known to be + # reliable. + # See https://bugs.webkit.org/show_bug.cgi?id=33296 and related bugs. + self.core_builder_names_regexps = [ + "SnowLeopard.*Build", + "SnowLeopard.*Test", + "Leopard", + "Tiger", + "Windows.*Build", + "GTK.*32", + "GTK.*64.*Debug", # Disallow the 64-bit Release bot which is broken. + "Qt", + "Chromium.*Release$", + ] + + def _parse_last_build_cell(self, builder, cell): + status_link = cell.find('a') + if status_link: + # Will be either a revision number or a build number + revision_string = status_link.string + # If revision_string has non-digits assume it's not a revision number. + builder['built_revision'] = int(revision_string) \ + if not re.match('\D', revision_string) \ + else None + + # FIXME: We treat slave lost as green even though it is not to + # work around the Qts bot being on a broken internet connection. + # The real fix is https://bugs.webkit.org/show_bug.cgi?id=37099 + builder['is_green'] = not re.search('fail', cell.renderContents()) or \ + not not re.search('lost', cell.renderContents()) + + status_link_regexp = r"builders/(?P<builder_name>.*)/builds/(?P<build_number>\d+)" + link_match = re.match(status_link_regexp, status_link['href']) + builder['build_number'] = int(link_match.group("build_number")) + else: + # We failed to find a link in the first cell, just give up. This + # can happen if a builder is just-added, the first cell will just + # be "no build" + # Other parts of the code depend on is_green being present. + builder['is_green'] = False + builder['built_revision'] = None + builder['build_number'] = None + + def _parse_current_build_cell(self, builder, cell): + activity_lines = cell.renderContents().split("<br />") + builder["activity"] = activity_lines[0] # normally "building" or "idle" + # The middle lines document how long left for any current builds. + match = re.match("(?P<pending_builds>\d) pending", activity_lines[-1]) + builder["pending_builds"] = int(match.group("pending_builds")) if match else 0 + + def _parse_builder_status_from_row(self, status_row): + status_cells = status_row.findAll('td') + builder = {} + + # First cell is the name + name_link = status_cells[0].find('a') + builder["name"] = unicode(name_link.string) + + self._parse_last_build_cell(builder, status_cells[1]) + self._parse_current_build_cell(builder, status_cells[2]) + return builder + + def _matches_regexps(self, builder_name, name_regexps): + for name_regexp in name_regexps: + if re.match(name_regexp, builder_name): + return True + return False + + # FIXME: Should move onto Builder + def _is_core_builder(self, builder_name): + return self._matches_regexps(builder_name, self.core_builder_names_regexps) + + # FIXME: This method needs to die, but is used by a unit test at the moment. + def _builder_statuses_with_names_matching_regexps(self, builder_statuses, name_regexps): + return [builder for builder in builder_statuses if self._matches_regexps(builder["name"], name_regexps)] + + def red_core_builders(self): + return [builder for builder in self.core_builder_statuses() if not builder["is_green"]] + + def red_core_builders_names(self): + return [builder["name"] for builder in self.red_core_builders()] + + def idle_red_core_builders(self): + return [builder for builder in self.red_core_builders() if builder["activity"] == "idle"] + + def core_builders_are_green(self): + return not self.red_core_builders() + + # FIXME: These _fetch methods should move to a networking class. + def _fetch_xmlrpc_build_dictionary(self, builder, build_number): + # The buildbot XMLRPC API is super-limited. + # For one, you cannot fetch info on builds which are incomplete. + proxy = xmlrpclib.ServerProxy("http://%s/xmlrpc" % self.buildbot_host, allow_none=True) + try: + return proxy.getBuild(builder.name(), int(build_number)) + except xmlrpclib.Fault, err: + build_url = Build.build_url(builder, build_number) + _log.error("Error fetching data for %s build %s (%s): %s" % (builder.name(), build_number, build_url, err)) + return None + + def _fetch_one_box_per_builder(self): + build_status_url = "http://%s/one_box_per_builder" % self.buildbot_host + return urllib2.urlopen(build_status_url) + + def _parse_twisted_file_row(self, file_row): + string_or_empty = lambda soup: unicode(soup.string) if soup.string else u"" + file_cells = file_row.findAll('td') + return { + "filename": string_or_empty(file_cells[0].find("a")), + "size": string_or_empty(file_cells[1]), + "type": string_or_empty(file_cells[2]), + "encoding": string_or_empty(file_cells[3]), + } + + def _parse_twisted_directory_listing(self, page): + soup = BeautifulSoup(page) + # HACK: Match only table rows with a class to ignore twisted header/footer rows. + file_rows = soup.find('table').findAll('tr', { "class" : True }) + return [self._parse_twisted_file_row(file_row) for file_row in file_rows] + + # FIXME: There should be a better way to get this information directly from twisted. + def _fetch_twisted_directory_listing(self, url): + return self._parse_twisted_directory_listing(urllib2.urlopen(url)) + + def builders(self): + return [self.builder_with_name(status["name"]) for status in self.builder_statuses()] + + # This method pulls from /one_box_per_builder as an efficient way to get information about + def builder_statuses(self): + soup = BeautifulSoup(self._fetch_one_box_per_builder()) + return [self._parse_builder_status_from_row(status_row) for status_row in soup.find('table').findAll('tr')] + + def core_builder_statuses(self): + return [builder for builder in self.builder_statuses() if self._is_core_builder(builder["name"])] + + def builder_with_name(self, name): + builder = self._builder_by_name.get(name) + if not builder: + builder = Builder(name, self) + self._builder_by_name[name] = builder + return builder + + def revisions_causing_failures(self, only_core_builders=True): + builder_statuses = self.core_builder_statuses() if only_core_builders else self.builder_statuses() + revision_to_failing_bots = {} + for builder_status in builder_statuses: + if builder_status["is_green"]: + continue + builder = self.builder_with_name(builder_status["name"]) + revisions = builder.blameworthy_revisions(builder_status["build_number"]) + for revision in revisions: + failing_bots = revision_to_failing_bots.get(revision, []) + failing_bots.append(builder) + revision_to_failing_bots[revision] = failing_bots + return revision_to_failing_bots + + # This makes fewer requests than calling Builder.latest_build would. It grabs all builder + # statuses in one request using self.builder_statuses (fetching /one_box_per_builder instead of builder pages). + def _latest_builds_from_builders(self, only_core_builders=True): + builder_statuses = self.core_builder_statuses() if only_core_builders else self.builder_statuses() + return [self.builder_with_name(status["name"]).build(status["build_number"]) for status in builder_statuses] + + def _build_at_or_before_revision(self, build, revision): + while build: + if build.revision() <= revision: + return build + build = build.previous_build() + + def last_green_revision(self, only_core_builders=True): + builds = self._latest_builds_from_builders(only_core_builders) + target_revision = builds[0].revision() + # An alternate way to do this would be to start at one revision and walk backwards + # checking builder.build_for_revision, however build_for_revision is very slow on first load. + while True: + # Make builds agree on revision + builds = [self._build_at_or_before_revision(build, target_revision) for build in builds] + if None in builds: # One of the builds failed to load from the server. + return None + min_revision = min(map(lambda build: build.revision(), builds)) + if min_revision != target_revision: + target_revision = min_revision + continue # Builds don't all agree on revision, keep searching + # Check to make sure they're all green + all_are_green = reduce(operator.and_, map(lambda build: build.is_green(), builds)) + if not all_are_green: + target_revision -= 1 + continue + return min_revision diff --git a/WebKitTools/Scripts/webkitpy/common/net/buildbot_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/buildbot_unittest.py new file mode 100644 index 0000000..8fb4c2c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/buildbot_unittest.py @@ -0,0 +1,446 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.net.buildbot import BuildBot, Builder, Build, LayoutTestResults + +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup + + +class BuilderTest(unittest.TestCase): + def _install_fetch_build(self, failure): + def _mock_fetch_build(build_number): + build = Build( + builder=self.builder, + build_number=build_number, + revision=build_number + 1000, + is_green=build_number < 4 + ) + build._layout_test_results = LayoutTestResults( + "http://buildbot.example.com/foo", { + LayoutTestResults.fail_key: failure(build_number), + }) + return build + self.builder._fetch_build = _mock_fetch_build + + def setUp(self): + self.buildbot = BuildBot() + self.builder = Builder(u"Test Builder \u2661", self.buildbot) + self._install_fetch_build(lambda build_number: ["test1", "test2"]) + + def test_find_failure_transition(self): + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1003) + self.assertEqual(red_build.revision(), 1004) + + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10), look_back_limit=2) + self.assertEqual(green_build, None) + self.assertEqual(red_build.revision(), 1008) + + def test_none_build(self): + self.builder._fetch_build = lambda build_number: None + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build, None) + self.assertEqual(red_build, None) + + def test_flaky_tests(self): + self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1009) + self.assertEqual(red_build.revision(), 1010) + + def test_failure_and_flaky(self): + self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1003) + self.assertEqual(red_build.revision(), 1004) + + def test_no_results(self): + self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1003) + self.assertEqual(red_build.revision(), 1004) + + def test_failure_after_flaky(self): + self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) + (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) + self.assertEqual(green_build.revision(), 1006) + self.assertEqual(red_build.revision(), 1007) + + def test_blameworthy_revisions(self): + self.assertEqual(self.builder.blameworthy_revisions(10), [1004]) + self.assertEqual(self.builder.blameworthy_revisions(10, look_back_limit=2), []) + # Flakey test avoidance requires at least 2 red builds: + self.assertEqual(self.builder.blameworthy_revisions(4), []) + self.assertEqual(self.builder.blameworthy_revisions(4, avoid_flakey_tests=False), [1004]) + # Green builder: + self.assertEqual(self.builder.blameworthy_revisions(3), []) + + def test_build_caching(self): + self.assertEqual(self.builder.build(10), self.builder.build(10)) + + def test_build_and_revision_for_filename(self): + expectations = { + "r47483 (1)/" : (47483, 1), + "r47483 (1).zip" : (47483, 1), + } + for filename, revision_and_build in expectations.items(): + self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build) + + +class LayoutTestResultsTest(unittest.TestCase): + _example_results_html = """ +<html> +<head> +<title>Layout Test Results</title> +</head> +<body> +<p>Tests that had stderr output:</p> +<table> +<tr> +<td><a href="/var/lib/buildbot/build/gtk-linux-64-release/build/LayoutTests/accessibility/aria-activedescendant-crash.html">accessibility/aria-activedescendant-crash.html</a></td> +<td><a href="accessibility/aria-activedescendant-crash-stderr.txt">stderr</a></td> +</tr> +<td><a href="/var/lib/buildbot/build/gtk-linux-64-release/build/LayoutTests/http/tests/security/canvas-remote-read-svg-image.html">http/tests/security/canvas-remote-read-svg-image.html</a></td> +<td><a href="http/tests/security/canvas-remote-read-svg-image-stderr.txt">stderr</a></td> +</tr> +</table><p>Tests that had no expected results (probably new):</p> +<table> +<tr> +<td><a href="/var/lib/buildbot/build/gtk-linux-64-release/build/LayoutTests/fast/repaint/no-caret-repaint-in-non-content-editable-element.html">fast/repaint/no-caret-repaint-in-non-content-editable-element.html</a></td> +<td><a href="fast/repaint/no-caret-repaint-in-non-content-editable-element-actual.txt">result</a></td> +</tr> +</table></body> +</html> +""" + + _expected_layout_test_results = { + 'Tests that had stderr output:' : [ + 'accessibility/aria-activedescendant-crash.html' + ], + 'Tests that had no expected results (probably new):' : [ + 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html' + ] + } + def test_parse_layout_test_results(self): + results = LayoutTestResults._parse_results_html(self._example_results_html) + self.assertEqual(self._expected_layout_test_results, results) + + +class BuildBotTest(unittest.TestCase): + + _example_one_box_status = ''' + <table> + <tr> + <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td> + <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td> + <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td> + <tr> + <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td> + <td class="LastBuild box" >no build</td> + <td align="center" class="Activity building">building<br />< 1 min</td> + <tr> + <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td> + <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td> + <td align="center" class="Activity idle">idle<br />3 pending</td> + <tr> + <td class="box"><a href="builders/Qt%20Windows%2032-bit%20Debug">Qt Windows 32-bit Debug</a></td> + <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Windows%2032-bit%20Debug/builds/2090">60563</a><br />failed<br />failed<br />slave<br />lost</td> + <td align="center" class="Activity building">building<br />ETA in<br />~ 5 mins<br />at 08:25</td> + </table> +''' + _expected_example_one_box_parsings = [ + { + 'is_green': True, + 'build_number' : 3693, + 'name': u'Windows Debug (Tests)', + 'built_revision': 47380, + 'activity': 'building', + 'pending_builds': 0, + }, + { + 'is_green': False, + 'build_number' : None, + 'name': u'SnowLeopard Intel Release', + 'built_revision': None, + 'activity': 'building', + 'pending_builds': 0, + }, + { + 'is_green': False, + 'build_number' : 654, + 'name': u'Qt Linux Release', + 'built_revision': 47383, + 'activity': 'idle', + 'pending_builds': 3, + }, + { + 'is_green': True, + 'build_number' : 2090, + 'name': u'Qt Windows 32-bit Debug', + 'built_revision': 60563, + 'activity': 'building', + 'pending_builds': 0, + }, + ] + + def test_status_parsing(self): + buildbot = BuildBot() + + soup = BeautifulSoup(self._example_one_box_status) + status_table = soup.find("table") + input_rows = status_table.findAll('tr') + + for x in range(len(input_rows)): + status_row = input_rows[x] + expected_parsing = self._expected_example_one_box_parsings[x] + + builder = buildbot._parse_builder_status_from_row(status_row) + + # Make sure we aren't parsing more or less than we expect + self.assertEquals(builder.keys(), expected_parsing.keys()) + + for key, expected_value in expected_parsing.items(): + self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value))) + + def test_core_builder_methods(self): + buildbot = BuildBot() + + # Override builder_statuses function to not touch the network. + def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to. + return BuildBotTest._expected_example_one_box_parsings + buildbot.builder_statuses = example_builder_statuses + + buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ] + self.assertEquals(buildbot.red_core_builders_names(), []) + self.assertTrue(buildbot.core_builders_are_green()) + + buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ] + self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ]) + self.assertFalse(buildbot.core_builders_are_green()) + + def test_builder_name_regexps(self): + buildbot = BuildBot() + + # For complete testing, this list should match the list of builders at build.webkit.org: + example_builders = [ + {'name': u'Tiger Intel Release', }, + {'name': u'Leopard Intel Release (Build)', }, + {'name': u'Leopard Intel Release (Tests)', }, + {'name': u'Leopard Intel Debug (Build)', }, + {'name': u'Leopard Intel Debug (Tests)', }, + {'name': u'SnowLeopard Intel Release (Build)', }, + {'name': u'SnowLeopard Intel Release (Tests)', }, + {'name': u'SnowLeopard Intel Leaks', }, + {'name': u'Windows Release (Build)', }, + {'name': u'Windows Release (Tests)', }, + {'name': u'Windows Debug (Build)', }, + {'name': u'Windows Debug (Tests)', }, + {'name': u'GTK Linux 32-bit Release', }, + {'name': u'GTK Linux 32-bit Debug', }, + {'name': u'GTK Linux 64-bit Debug', }, + {'name': u'GTK Linux 64-bit Release', }, + {'name': u'Qt Linux Release', }, + {'name': u'Qt Linux Release minimal', }, + {'name': u'Qt Linux ARMv5 Release', }, + {'name': u'Qt Linux ARMv7 Release', }, + {'name': u'Qt Windows 32-bit Release', }, + {'name': u'Qt Windows 32-bit Debug', }, + {'name': u'Chromium Linux Release', }, + {'name': u'Chromium Mac Release', }, + {'name': u'Chromium Win Release', }, + {'name': u'Chromium Linux Release (Tests)', }, + {'name': u'Chromium Mac Release (Tests)', }, + {'name': u'Chromium Win Release (Tests)', }, + {'name': u'New run-webkit-tests', }, + ] + name_regexps = [ + "SnowLeopard.*Build", + "SnowLeopard.*Test", + "Leopard", + "Tiger", + "Windows.*Build", + "GTK.*32", + "GTK.*64.*Debug", # Disallow the 64-bit Release bot which is broken. + "Qt", + "Chromium.*Release$", + ] + expected_builders = [ + {'name': u'Tiger Intel Release', }, + {'name': u'Leopard Intel Release (Build)', }, + {'name': u'Leopard Intel Release (Tests)', }, + {'name': u'Leopard Intel Debug (Build)', }, + {'name': u'Leopard Intel Debug (Tests)', }, + {'name': u'SnowLeopard Intel Release (Build)', }, + {'name': u'SnowLeopard Intel Release (Tests)', }, + {'name': u'Windows Release (Build)', }, + {'name': u'Windows Debug (Build)', }, + {'name': u'GTK Linux 32-bit Release', }, + {'name': u'GTK Linux 32-bit Debug', }, + {'name': u'GTK Linux 64-bit Debug', }, + {'name': u'Qt Linux Release', }, + {'name': u'Qt Linux Release minimal', }, + {'name': u'Qt Linux ARMv5 Release', }, + {'name': u'Qt Linux ARMv7 Release', }, + {'name': u'Qt Windows 32-bit Release', }, + {'name': u'Qt Windows 32-bit Debug', }, + {'name': u'Chromium Linux Release', }, + {'name': u'Chromium Mac Release', }, + {'name': u'Chromium Win Release', }, + ] + + # This test should probably be updated if the default regexp list changes + self.assertEquals(buildbot.core_builder_names_regexps, name_regexps) + + builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps) + self.assertEquals(builders, expected_builders) + + def test_builder_with_name(self): + buildbot = BuildBot() + + builder = buildbot.builder_with_name("Test Builder") + self.assertEqual(builder.name(), "Test Builder") + self.assertEqual(builder.url(), "http://build.webkit.org/builders/Test%20Builder") + self.assertEqual(builder.url_encoded_name(), "Test%20Builder") + self.assertEqual(builder.results_url(), "http://build.webkit.org/results/Test%20Builder") + + # Override _fetch_xmlrpc_build_dictionary function to not touch the network. + def mock_fetch_xmlrpc_build_dictionary(self, build_number): + build_dictionary = { + "revision" : 2 * build_number, + "number" : int(build_number), + "results" : build_number % 2, # 0 means pass + } + return build_dictionary + buildbot._fetch_xmlrpc_build_dictionary = mock_fetch_xmlrpc_build_dictionary + + build = builder.build(10) + self.assertEqual(build.builder(), builder) + self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/10") + self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r20%20%2810%29") + self.assertEqual(build.revision(), 20) + self.assertEqual(build.is_green(), True) + + build = build.previous_build() + self.assertEqual(build.builder(), builder) + self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/9") + self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r18%20%289%29") + self.assertEqual(build.revision(), 18) + self.assertEqual(build.is_green(), False) + + self.assertEqual(builder.build(None), None) + + _example_directory_listing = ''' +<h1>Directory listing for /results/SnowLeopard Intel Leaks/</h1> + +<table> + <thead> + <tr> + <th>Filename</th> + <th>Size</th> + <th>Content type</th> + <th>Content encoding</th> + </tr> + </thead> + <tbody> +<tr class="odd"> + <td><a href="r47483%20%281%29/">r47483 (1)/</a></td> + <td></td> + <td>[Directory]</td> + <td></td> +</tr> +<tr class="odd"> + <td><a href="r47484%20%282%29.zip">r47484 (2).zip</a></td> + <td>89K</td> + <td>[application/zip]</td> + <td></td> +</tr> +''' + _expected_files = [ + { + "filename" : "r47483 (1)/", + "size" : "", + "type" : "[Directory]", + "encoding" : "", + }, + { + "filename" : "r47484 (2).zip", + "size" : "89K", + "type" : "[application/zip]", + "encoding" : "", + }, + ] + + def test_parse_build_to_revision_map(self): + buildbot = BuildBot() + files = buildbot._parse_twisted_directory_listing(self._example_directory_listing) + self.assertEqual(self._expected_files, files) + + # Revision, is_green + # Ordered from newest (highest number) to oldest. + fake_builder1 = [ + [2, False], + [1, True], + ] + fake_builder2 = [ + [2, False], + [1, True], + ] + fake_builders = [ + fake_builder1, + fake_builder2, + ] + def _build_from_fake(self, fake_builder, index): + if index >= len(fake_builder): + return None + fake_build = fake_builder[index] + build = Build( + builder=fake_builder, + build_number=index, + revision=fake_build[0], + is_green=fake_build[1], + ) + def mock_previous_build(): + return self._build_from_fake(fake_builder, index + 1) + build.previous_build = mock_previous_build + return build + + def _fake_builds_at_index(self, index): + return [self._build_from_fake(builder, index) for builder in self.fake_builders] + + def test_last_green_revision(self): + buildbot = BuildBot() + def mock_builds_from_builders(only_core_builders): + return self._fake_builds_at_index(0) + buildbot._latest_builds_from_builders = mock_builds_from_builders + self.assertEqual(buildbot.last_green_revision(), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/credentials.py b/WebKitTools/Scripts/webkitpy/common/net/credentials.py index 295c576..1d5f83d 100644 --- a/WebKitTools/Scripts/webkitpy/credentials.py +++ b/WebKitTools/Scripts/webkitpy/common/net/credentials.py @@ -34,30 +34,23 @@ import os import platform import re -from webkitpy.executive import Executive, ScriptError -from webkitpy.webkit_logging import log -from webkitpy.scm import Git -from webkitpy.user import User +from webkitpy.common.checkout.scm import Git +from webkitpy.common.system.executive import Executive, ScriptError +from webkitpy.common.system.user import User +from webkitpy.common.system.deprecated_logging import log class Credentials(object): def __init__(self, host, git_prefix=None, executive=None, cwd=os.getcwd()): self.host = host - self.git_prefix = git_prefix + self.git_prefix = "%s." % git_prefix if git_prefix else "" self.executive = executive or Executive() self.cwd = cwd def _credentials_from_git(self): - return [self._read_git_config("username"), - self._read_git_config("password")] - - def _read_git_config(self, key): - config_key = "%s.%s" % (self.git_prefix, key) if self.git_prefix \ - else key - return self.executive.run_command( - ["git", "config", "--get", config_key], - error_handler=Executive.ignore_error).rstrip('\n') + return [Git.read_git_config(self.git_prefix + "username"), + Git.read_git_config(self.git_prefix + "password")] def _keychain_value_with_label(self, label, source_text): match = re.search("%s\"(?P<value>.+)\"" % label, diff --git a/WebKitTools/Scripts/webkitpy/credentials_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/credentials_unittest.py index 0bd5340..9a42bdd 100644 --- a/WebKitTools/Scripts/webkitpy/credentials_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/net/credentials_unittest.py @@ -29,10 +29,10 @@ import os import tempfile import unittest -from webkitpy.credentials import Credentials -from webkitpy.executive import Executive -from webkitpy.outputcapture import OutputCapture -from webkitpy.mock import Mock +from webkitpy.common.net.credentials import Credentials +from webkitpy.common.system.executive import Executive +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock class CredentialsTest(unittest.TestCase): example_security_output = """keychain: "/Users/test/Library/Keychains/login.keychain" @@ -101,16 +101,6 @@ password: "SECRETSAUCE" self._assert_security_call() self._assert_security_call(username="foo") - def test_git_config_calls(self): - executive_mock = Mock() - credentials = Credentials("example.com", executive=executive_mock) - credentials._read_git_config("foo") - executive_mock.run_command.assert_called_with(["git", "config", "--get", "foo"], error_handler=Executive.ignore_error) - - credentials = Credentials("example.com", git_prefix="test_prefix", executive=executive_mock) - credentials._read_git_config("foo") - executive_mock.run_command.assert_called_with(["git", "config", "--get", "test_prefix.foo"], error_handler=Executive.ignore_error) - def test_read_credentials_without_git_repo(self): class FakeCredentials(Credentials): def _is_mac_os_x(self): diff --git a/WebKitTools/Scripts/webkitpy/common/net/irc/__init__.py b/WebKitTools/Scripts/webkitpy/common/net/irc/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/irc/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/common/net/irc/ircbot.py b/WebKitTools/Scripts/webkitpy/common/net/irc/ircbot.py new file mode 100644 index 0000000..f742867 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/irc/ircbot.py @@ -0,0 +1,91 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import webkitpy.common.config.irc as config_irc + +from webkitpy.common.thread.messagepump import MessagePump, MessagePumpDelegate +from webkitpy.thirdparty.autoinstalled.irc import ircbot +from webkitpy.thirdparty.autoinstalled.irc import irclib + + +class IRCBotDelegate(object): + def irc_message_received(self, nick, message): + raise NotImplementedError, "subclasses must implement" + + def irc_nickname(self): + raise NotImplementedError, "subclasses must implement" + + def irc_password(self): + raise NotImplementedError, "subclasses must implement" + + +class IRCBot(ircbot.SingleServerIRCBot, MessagePumpDelegate): + # FIXME: We should get this information from a config file. + def __init__(self, + message_queue, + delegate): + self._message_queue = message_queue + self._delegate = delegate + ircbot.SingleServerIRCBot.__init__( + self, + [( + config_irc.server, + config_irc.port, + self._delegate.irc_password() + )], + self._delegate.irc_nickname(), + self._delegate.irc_nickname()) + self._channel = config_irc.channel + + # ircbot.SingleServerIRCBot methods + + def on_nicknameinuse(self, connection, event): + connection.nick(connection.get_nickname() + "_") + + def on_welcome(self, connection, event): + connection.join(self._channel) + self._message_pump = MessagePump(self, self._message_queue) + + def on_pubmsg(self, connection, event): + nick = irclib.nm_to_n(event.source()) + request = event.arguments()[0].split(":", 1) + if len(request) > 1 and irclib.irc_lower(request[0]) == irclib.irc_lower(self.connection.get_nickname()): + response = self._delegate.irc_message_received(nick, request[1]) + if response: + connection.privmsg(self._channel, response) + + # MessagePumpDelegate methods + + def schedule(self, interval, callback): + self.connection.execute_delayed(interval, callback) + + def message_available(self, message): + self.connection.privmsg(self._channel, message) + + def final_message_delivered(self): + self.die() diff --git a/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy.py b/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy.py new file mode 100644 index 0000000..13348b4 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy.py @@ -0,0 +1,62 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import threading + +from webkitpy.common.net.irc.ircbot import IRCBot +from webkitpy.common.thread.threadedmessagequeue import ThreadedMessageQueue +from webkitpy.common.system.deprecated_logging import log + + +class _IRCThread(threading.Thread): + def __init__(self, message_queue, irc_delegate, irc_bot): + threading.Thread.__init__(self) + self.setDaemon(True) + self._message_queue = message_queue + self._irc_delegate = irc_delegate + self._irc_bot = irc_bot + + def run(self): + bot = self._irc_bot(self._message_queue, self._irc_delegate) + bot.start() + + +class IRCProxy(object): + def __init__(self, irc_delegate, irc_bot=IRCBot): + log("Connecting to IRC") + self._message_queue = ThreadedMessageQueue() + self._child_thread = _IRCThread(self._message_queue, irc_delegate, irc_bot) + self._child_thread.start() + + def post(self, message): + self._message_queue.post(message) + + def disconnect(self): + log("Disconnecting from IRC...") + self._message_queue.stop() + self._child_thread.join() diff --git a/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy_unittest.py new file mode 100644 index 0000000..b44ce40 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/irc/ircproxy_unittest.py @@ -0,0 +1,43 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.net.irc.ircproxy import IRCProxy +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock + +class IRCProxyTest(unittest.TestCase): + def test_trivial(self): + def fun(): + proxy = IRCProxy(Mock(), Mock()) + proxy.post("hello") + proxy.disconnect() + + expected_stderr = "Connecting to IRC\nDisconnecting from IRC...\n" + OutputCapture().assert_outputs(self, fun, expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/networktransaction.py b/WebKitTools/Scripts/webkitpy/common/net/networktransaction.py index 65ea27d..c82fc6f 100644 --- a/WebKitTools/Scripts/webkitpy/networktransaction.py +++ b/WebKitTools/Scripts/webkitpy/common/net/networktransaction.py @@ -26,10 +26,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging import time -from mechanize import HTTPError -from webkitpy.webkit_logging import log +from webkitpy.thirdparty.autoinstalled.mechanize import HTTPError +from webkitpy.common.system.deprecated_logging import log + + +_log = logging.getLogger(__name__) class NetworkTimeout(Exception): @@ -37,7 +41,7 @@ class NetworkTimeout(Exception): class NetworkTransaction(object): - def __init__(self, initial_backoff_seconds=10, grown_factor=1.1, timeout_seconds=5*60*60): + def __init__(self, initial_backoff_seconds=10, grown_factor=1.5, timeout_seconds=10*60): self._initial_backoff_seconds = initial_backoff_seconds self._grown_factor = grown_factor self._timeout_seconds = timeout_seconds @@ -50,7 +54,8 @@ class NetworkTransaction(object): return request() except HTTPError, e: self._check_for_timeout() - log("Received HTTP status %s from server. Retrying in %s seconds..." % (e.code, self._backoff_seconds)) + _log.warn("Received HTTP status %s from server. Retrying in " + "%s seconds..." % (e.code, self._backoff_seconds)) self._sleep() def _check_for_timeout(self): diff --git a/WebKitTools/Scripts/webkitpy/networktransaction_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/networktransaction_unittest.py index 3cffe02..cd0702b 100644 --- a/WebKitTools/Scripts/webkitpy/networktransaction_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/net/networktransaction_unittest.py @@ -28,10 +28,12 @@ import unittest -from mechanize import HTTPError -from webkitpy.networktransaction import NetworkTransaction, NetworkTimeout +from webkitpy.common.net.networktransaction import NetworkTransaction, NetworkTimeout +from webkitpy.common.system.logtesting import LoggingTestCase +from webkitpy.thirdparty.autoinstalled.mechanize import HTTPError -class NetworkTransactionTest(unittest.TestCase): + +class NetworkTransactionTest(LoggingTestCase): exception = Exception("Test exception") def test_success(self): @@ -65,6 +67,10 @@ class NetworkTransactionTest(unittest.TestCase): transaction = NetworkTransaction(initial_backoff_seconds=0) self.assertEqual(transaction.run(lambda: self._raise_http_error()), 42) self.assertEqual(self._run_count, 3) + self.assertLog(['WARNING: Received HTTP status 500 from server. ' + 'Retrying in 0 seconds...\n', + 'WARNING: Received HTTP status 500 from server. ' + 'Retrying in 0.0 seconds...\n']) def test_timeout(self): self._run_count = 0 diff --git a/WebKitTools/Scripts/webkitpy/common/net/rietveld.py b/WebKitTools/Scripts/webkitpy/common/net/rietveld.py new file mode 100644 index 0000000..0c6a313 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/rietveld.py @@ -0,0 +1,78 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +import os +import re +import stat + +import webkitpy.common.config as config +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.system.executive import ScriptError +import webkitpy.thirdparty.autoinstalled.rietveld.upload as upload + + +class Rietveld(object): + def __init__(self, executive, dryrun=False): + self.dryrun = dryrun + self._executive = executive + + def url_for_issue(self, codereview_issue): + if not codereview_issue: + return None + return "%s%s" % (config.codereview_server_url, codereview_issue) + + def post(self, diff, message=None, codereview_issue=None, cc=None): + if not message: + raise ScriptError("Rietveld requires a message.") + + # Rietveld has a 100 character limit on message length. + if len(message) > 100: + message = message[:100] + + args = [ + # First argument is empty string to mimic sys.argv. + "", + "--assume_yes", + "--server=%s" % config.codereview_server_host, + "--message=%s" % message, + ] + if codereview_issue: + args.append("--issue=%s" % codereview_issue) + if cc: + args.append("--cc=%s" % cc) + + if self.dryrun: + log("Would have run %s" % args) + return + + # Use RealMain instead of calling upload from the commandline so that + # we can pass in the diff ourselves. Otherwise, upload will just use + # git diff for git checkouts, which doesn't respect --git-commit. + issue, patchset = upload.RealMain(args, data=diff) + return issue diff --git a/WebKitTools/Scripts/webkitpy/common/net/rietveld_unittest.py b/WebKitTools/Scripts/webkitpy/common/net/rietveld_unittest.py new file mode 100644 index 0000000..9c5a29e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/net/rietveld_unittest.py @@ -0,0 +1,39 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.net.rietveld import Rietveld +from webkitpy.thirdparty.mock import Mock + + +class RietveldTest(unittest.TestCase): + def test_url_for_issue(self): + rietveld = Rietveld(Mock()) + self.assertEqual(rietveld.url_for_issue(34223), + "https://wkrietveld.appspot.com/34223") diff --git a/WebKitTools/Scripts/webkitpy/statusserver.py b/WebKitTools/Scripts/webkitpy/common/net/statusserver.py index ff0ddfa..c8fced6 100644 --- a/WebKitTools/Scripts/webkitpy/statusserver.py +++ b/WebKitTools/Scripts/webkitpy/common/net/statusserver.py @@ -26,19 +26,20 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.networktransaction import NetworkTransaction -from webkitpy.webkit_logging import log -from mechanize import Browser - -# WebKit includes a built copy of BeautifulSoup in Scripts/webkitpy -# so this import should always succeed. -from .BeautifulSoup import BeautifulSoup +from webkitpy.common.net.networktransaction import NetworkTransaction +from webkitpy.common.system.deprecated_logging import log +from webkitpy.thirdparty.autoinstalled.mechanize import Browser +from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup +import logging import urllib2 +_log = logging.getLogger("webkitpy.common.net.statusserver") + + class StatusServer: - default_host = "webkit-commit-queue.appspot.com" + default_host = "queues.webkit.org" def __init__(self, host=default_host): self.set_host(host) @@ -55,16 +56,16 @@ class StatusServer: if not patch: return if patch.bug_id(): - self.browser["bug_id"] = str(patch.bug_id()) + self.browser["bug_id"] = unicode(patch.bug_id()) if patch.id(): - self.browser["patch_id"] = str(patch.id()) + self.browser["patch_id"] = unicode(patch.id()) def _add_results_file(self, results_file): if not results_file: return self.browser.add_file(results_file, "text/plain", "results.txt", 'results_file') - def _post_to_server(self, queue_name, status, patch, results_file): + def _post_status_to_server(self, queue_name, status, patch, results_file): if results_file: # We might need to re-wind the file if we've already tried to post it. results_file.seek(0) @@ -72,25 +73,53 @@ class StatusServer: update_status_url = "%s/update-status" % self.url self.browser.open(update_status_url) self.browser.select_form(name="update_status") - self.browser['queue_name'] = queue_name + self.browser["queue_name"] = queue_name self._add_patch(patch) - self.browser['status'] = status + self.browser["status"] = status self._add_results_file(results_file) return self.browser.submit().read() # This is the id of the newly created status object. - def update_status(self, queue_name, status, patch=None, results_file=None): - # During unit testing, host is None - if not self.host: - return + def _post_svn_revision_to_server(self, svn_revision_number, broken_bot): + update_svn_revision_url = "%s/update-svn-revision" % self.url + self.browser.open(update_svn_revision_url) + self.browser.select_form(name="update_svn_revision") + self.browser["number"] = unicode(svn_revision_number) + self.browser["broken_bot"] = broken_bot + return self.browser.submit().read() + + def _post_work_items_to_server(self, queue_name, work_items): + update_work_items_url = "%s/update-work-items" % self.url + self.browser.open(update_work_items_url) + self.browser.select_form(name="update_work_items") + self.browser["queue_name"] = queue_name + work_items = map(unicode, work_items) # .join expects strings + self.browser["work_items"] = " ".join(work_items) + return self.browser.submit().read() + + def update_work_items(self, queue_name, work_items): + _log.debug("Recording work items: %s for %s" % (work_items, queue_name)) + return NetworkTransaction().run(lambda: self._post_work_items_to_server(queue_name, work_items)) + def update_status(self, queue_name, status, patch=None, results_file=None): log(status) - return NetworkTransaction().run(lambda: self._post_to_server(queue_name, status, patch, results_file)) + return NetworkTransaction().run(lambda: self._post_status_to_server(queue_name, status, patch, results_file)) - def patch_status(self, queue_name, patch_id): - update_status_url = "%s/patch-status/%s/%s" % (self.url, queue_name, patch_id) + def update_svn_revision(self, svn_revision_number, broken_bot): + log("SVN revision: %s broke %s" % (svn_revision_number, broken_bot)) + return NetworkTransaction().run(lambda: self._post_svn_revision_to_server(svn_revision_number, broken_bot)) + + def _fetch_url(self, url): try: - return urllib2.urlopen(update_status_url).read() + return urllib2.urlopen(url).read() except urllib2.HTTPError, e: if e.code == 404: return None raise e + + def patch_status(self, queue_name, patch_id): + patch_status_url = "%s/patch-status/%s/%s" % (self.url, queue_name, patch_id) + return self._fetch_url(patch_status_url) + + def svn_revision(self, svn_revision_number): + svn_revision_url = "%s/svn-revision/%s" % (self.url, svn_revision_number) + return self._fetch_url(svn_revision_url) diff --git a/WebKitTools/Scripts/webkitpy/common/prettypatch.py b/WebKitTools/Scripts/webkitpy/common/prettypatch.py new file mode 100644 index 0000000..4e92a53 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/prettypatch.py @@ -0,0 +1,66 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import tempfile + + +class PrettyPatch(object): + # FIXME: PrettyPatch should not require checkout_root. + def __init__(self, executive, checkout_root): + self._executive = executive + self._checkout_root = checkout_root + + def pretty_diff_file(self, diff): + # Diffs can contain multiple text files of different encodings + # so we always deal with them as byte arrays, not unicode strings. + assert(isinstance(diff, str)) + pretty_diff = self.pretty_diff(diff) + diff_file = tempfile.NamedTemporaryFile(suffix=".html") + diff_file.write(pretty_diff) + diff_file.flush() + return diff_file + + def pretty_diff(self, diff): + # pretify.rb will hang forever if given no input. + # Avoid the hang by returning an empty string. + if not diff: + return "" + + pretty_patch_path = os.path.join(self._checkout_root, + "BugsSite", "PrettyPatch") + prettify_path = os.path.join(pretty_patch_path, "prettify.rb") + args = [ + "ruby", + "-I", + pretty_patch_path, + prettify_path, + ] + # PrettyPatch does not modify the encoding of the diff output + # so we can't expect it to be utf-8. + return self._executive.run_command(args, input=diff, decode_output=False) diff --git a/WebKitTools/Scripts/webkitpy/common/prettypatch_unittest.py b/WebKitTools/Scripts/webkitpy/common/prettypatch_unittest.py new file mode 100644 index 0000000..1307856 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/prettypatch_unittest.py @@ -0,0 +1,70 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os.path +import unittest + +from webkitpy.common.system.executive import Executive +from webkitpy.common.prettypatch import PrettyPatch + + +class PrettyPatchTest(unittest.TestCase): + + _diff_with_multiple_encodings = """ +Index: utf8_test +=================================================================== +--- utf8_test\t(revision 0) ++++ utf8_test\t(revision 0) +@@ -0,0 +1 @@ ++utf-8 test: \xc2\xa0 +Index: latin1_test +=================================================================== +--- latin1_test\t(revision 0) ++++ latin1_test\t(revision 0) +@@ -0,0 +1 @@ ++latin1 test: \xa0 +""" + + def _webkit_root(self): + webkitpy_common = os.path.dirname(__file__) + webkitpy = os.path.dirname(webkitpy_common) + scripts = os.path.dirname(webkitpy) + webkit_tools = os.path.dirname(scripts) + webkit_root = os.path.dirname(webkit_tools) + return webkit_root + + def test_pretty_diff_encodings(self): + pretty_patch = PrettyPatch(Executive(), self._webkit_root()) + pretty = pretty_patch.pretty_diff(self._diff_with_multiple_encodings) + self.assertTrue(pretty) # We got some output + self.assertTrue(isinstance(pretty, str)) # It's a byte array, not unicode + + def test_pretty_print_empty_string(self): + # Make sure that an empty diff does not hang the process. + pretty_patch = PrettyPatch(Executive(), self._webkit_root()) + self.assertEqual(pretty_patch.pretty_diff(""), "") diff --git a/WebKitTools/Scripts/webkitpy/common/system/__init__.py b/WebKitTools/Scripts/webkitpy/common/system/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/common/system/autoinstall.py b/WebKitTools/Scripts/webkitpy/common/system/autoinstall.py new file mode 100755 index 0000000..9adab29 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/autoinstall.py @@ -0,0 +1,517 @@ +# Copyright (c) 2009, Daniel Krech All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of the Daniel Krech nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Support for automatically downloading Python packages from an URL.""" + + +from __future__ import with_statement + +import codecs +import logging +import new +import os +import shutil +import sys +import tarfile +import tempfile +import urllib +import urlparse +import zipfile +import zipimport + +_log = logging.getLogger(__name__) + + +class AutoInstaller(object): + + """Supports automatically installing Python packages from an URL. + + Supports uncompressed files, .tar.gz, and .zip formats. + + Basic usage: + + installer = AutoInstaller() + + installer.install(url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", + url_subpath="pep8-0.5.0/pep8.py") + installer.install(url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", + url_subpath="mechanize") + + """ + + def __init__(self, append_to_search_path=False, make_package=True, + target_dir=None, temp_dir=None): + """Create an AutoInstaller instance, and set up the target directory. + + Args: + append_to_search_path: A boolean value of whether to append the + target directory to the sys.path search path. + make_package: A boolean value of whether to make the target + directory a package. This adds an __init__.py file + to the target directory -- allowing packages and + modules within the target directory to be imported + explicitly using dotted module names. + target_dir: The directory path to which packages should be installed. + Defaults to a subdirectory of the folder containing + this module called "autoinstalled". + temp_dir: The directory path to use for any temporary files + generated while downloading, unzipping, and extracting + packages to install. Defaults to a standard temporary + location generated by the tempfile module. This + parameter should normally be used only for development + testing. + + """ + if target_dir is None: + this_dir = os.path.dirname(__file__) + target_dir = os.path.join(this_dir, "autoinstalled") + + # Ensure that the target directory exists. + self._set_up_target_dir(target_dir, append_to_search_path, make_package) + + self._target_dir = target_dir + self._temp_dir = temp_dir + + def _log_transfer(self, message, source, target, log_method=None): + """Log a debug message that involves a source and target.""" + if log_method is None: + log_method = _log.debug + + log_method("%s" % message) + log_method(' From: "%s"' % source) + log_method(' To: "%s"' % target) + + def _create_directory(self, path, name=None): + """Create a directory.""" + log = _log.debug + + name = name + " " if name is not None else "" + log('Creating %sdirectory...' % name) + log(' "%s"' % path) + + os.makedirs(path) + + def _write_file(self, path, text, encoding): + """Create a file at the given path with given text. + + This method overwrites any existing file. + + """ + _log.debug("Creating file...") + _log.debug(' "%s"' % path) + with codecs.open(path, "w", encoding) as file: + file.write(text) + + def _set_up_target_dir(self, target_dir, append_to_search_path, + make_package): + """Set up a target directory. + + Args: + target_dir: The path to the target directory to set up. + append_to_search_path: A boolean value of whether to append the + target directory to the sys.path search path. + make_package: A boolean value of whether to make the target + directory a package. This adds an __init__.py file + to the target directory -- allowing packages and + modules within the target directory to be imported + explicitly using dotted module names. + + """ + if not os.path.exists(target_dir): + self._create_directory(target_dir, "autoinstall target") + + if append_to_search_path: + sys.path.append(target_dir) + + if make_package: + init_path = os.path.join(target_dir, "__init__.py") + if not os.path.exists(init_path): + text = ("# This file is required for Python to search this " + "directory for modules.\n") + self._write_file(init_path, text, "ascii") + + def _create_scratch_directory_inner(self, prefix): + """Create a scratch directory without exception handling. + + Creates a scratch directory inside the AutoInstaller temp + directory self._temp_dir, or inside a platform-dependent temp + directory if self._temp_dir is None. Returns the path to the + created scratch directory. + + Raises: + OSError: [Errno 2] if the containing temp directory self._temp_dir + is not None and does not exist. + + """ + # The tempfile.mkdtemp() method function requires that the + # directory corresponding to the "dir" parameter already exist + # if it is not None. + scratch_dir = tempfile.mkdtemp(prefix=prefix, dir=self._temp_dir) + return scratch_dir + + def _create_scratch_directory(self, target_name): + """Create a temporary scratch directory, and return its path. + + The scratch directory is generated inside the temp directory + of this AutoInstaller instance. This method also creates the + temp directory if it does not already exist. + + """ + prefix = target_name + "_" + try: + scratch_dir = self._create_scratch_directory_inner(prefix) + except OSError: + # Handle case of containing temp directory not existing-- + # OSError: [Errno 2] No such file or directory:... + temp_dir = self._temp_dir + if temp_dir is None or os.path.exists(temp_dir): + raise + # Else try again after creating the temp directory. + self._create_directory(temp_dir, "autoinstall temp") + scratch_dir = self._create_scratch_directory_inner(prefix) + + return scratch_dir + + def _url_downloaded_path(self, target_name): + """Return the path to the file containing the URL downloaded.""" + filename = ".%s.url" % target_name + path = os.path.join(self._target_dir, filename) + return path + + def _is_downloaded(self, target_name, url): + """Return whether a package version has been downloaded.""" + version_path = self._url_downloaded_path(target_name) + + _log.debug('Checking %s URL downloaded...' % target_name) + _log.debug(' "%s"' % version_path) + + if not os.path.exists(version_path): + # Then no package version has been downloaded. + _log.debug("No URL file found.") + return False + + with codecs.open(version_path, "r", "utf-8") as file: + version = file.read() + + return version.strip() == url.strip() + + def _record_url_downloaded(self, target_name, url): + """Record the URL downloaded to a file.""" + version_path = self._url_downloaded_path(target_name) + _log.debug("Recording URL downloaded...") + _log.debug(' URL: "%s"' % url) + _log.debug(' To: "%s"' % version_path) + + self._write_file(version_path, url, "utf-8") + + def _extract_targz(self, path, scratch_dir): + # tarfile.extractall() extracts to a path without the + # trailing ".tar.gz". + target_basename = os.path.basename(path[:-len(".tar.gz")]) + target_path = os.path.join(scratch_dir, target_basename) + + self._log_transfer("Starting gunzip/extract...", path, target_path) + + try: + tar_file = tarfile.open(path) + except tarfile.ReadError, err: + # Append existing Error message to new Error. + message = ("Could not open tar file: %s\n" + " The file probably does not have the correct format.\n" + " --> Inner message: %s" + % (path, err)) + raise Exception(message) + + try: + # This is helpful for debugging purposes. + _log.debug("Listing tar file contents...") + for name in tar_file.getnames(): + _log.debug(' * "%s"' % name) + _log.debug("Extracting gzipped tar file...") + tar_file.extractall(target_path) + finally: + tar_file.close() + + return target_path + + # This is a replacement for ZipFile.extractall(), which is + # available in Python 2.6 but not in earlier versions. + def _extract_all(self, zip_file, target_dir): + self._log_transfer("Extracting zip file...", zip_file, target_dir) + + # This is helpful for debugging purposes. + _log.debug("Listing zip file contents...") + for name in zip_file.namelist(): + _log.debug(' * "%s"' % name) + + for name in zip_file.namelist(): + path = os.path.join(target_dir, name) + self._log_transfer("Extracting...", name, path) + + if not os.path.basename(path): + # Then the path ends in a slash, so it is a directory. + self._create_directory(path) + continue + # Otherwise, it is a file. + + try: + # We open this file w/o encoding, as we're reading/writing + # the raw byte-stream from the zip file. + outfile = open(path, 'wb') + except IOError, err: + # Not all zip files seem to list the directories explicitly, + # so try again after creating the containing directory. + _log.debug("Got IOError: retrying after creating directory...") + dir = os.path.dirname(path) + self._create_directory(dir) + outfile = open(path, 'wb') + + try: + outfile.write(zip_file.read(name)) + finally: + outfile.close() + + def _unzip(self, path, scratch_dir): + # zipfile.extractall() extracts to a path without the + # trailing ".zip". + target_basename = os.path.basename(path[:-len(".zip")]) + target_path = os.path.join(scratch_dir, target_basename) + + self._log_transfer("Starting unzip...", path, target_path) + + try: + zip_file = zipfile.ZipFile(path, "r") + except zipfile.BadZipfile, err: + message = ("Could not open zip file: %s\n" + " --> Inner message: %s" + % (path, err)) + raise Exception(message) + + try: + self._extract_all(zip_file, scratch_dir) + finally: + zip_file.close() + + return target_path + + def _prepare_package(self, path, scratch_dir): + """Prepare a package for use, if necessary, and return the new path. + + For example, this method unzips zipped files and extracts + tar files. + + Args: + path: The path to the downloaded URL contents. + scratch_dir: The scratch directory. Note that the scratch + directory contains the file designated by the + path parameter. + + """ + # FIXME: Add other natural extensions. + if path.endswith(".zip"): + new_path = self._unzip(path, scratch_dir) + elif path.endswith(".tar.gz"): + new_path = self._extract_targz(path, scratch_dir) + else: + # No preparation is needed. + new_path = path + + return new_path + + def _download_to_stream(self, url, stream): + """Download an URL to a stream, and return the number of bytes.""" + try: + netstream = urllib.urlopen(url) + except IOError, err: + # Append existing Error message to new Error. + message = ('Could not download Python modules from URL "%s".\n' + " Make sure you are connected to the internet.\n" + " You must be connected to the internet when " + "downloading needed modules for the first time.\n" + " --> Inner message: %s" + % (url, err)) + raise IOError(message) + code = 200 + if hasattr(netstream, "getcode"): + code = netstream.getcode() + if not 200 <= code < 300: + raise ValueError("HTTP Error code %s" % code) + + BUFSIZE = 2**13 # 8KB + bytes = 0 + while True: + data = netstream.read(BUFSIZE) + if not data: + break + stream.write(data) + bytes += len(data) + netstream.close() + return bytes + + def _download(self, url, scratch_dir): + """Download URL contents, and return the download path.""" + url_path = urlparse.urlsplit(url)[2] + url_path = os.path.normpath(url_path) # Removes trailing slash. + target_filename = os.path.basename(url_path) + target_path = os.path.join(scratch_dir, target_filename) + + self._log_transfer("Starting download...", url, target_path) + + with open(target_path, "wb") as stream: + bytes = self._download_to_stream(url, stream) + + _log.debug("Downloaded %s bytes." % bytes) + + return target_path + + def _install(self, scratch_dir, package_name, target_path, url, + url_subpath): + """Install a python package from an URL. + + This internal method overwrites the target path if the target + path already exists. + + """ + path = self._download(url=url, scratch_dir=scratch_dir) + path = self._prepare_package(path, scratch_dir) + + if url_subpath is None: + source_path = path + else: + source_path = os.path.join(path, url_subpath) + + if os.path.exists(target_path): + _log.debug('Refreshing install: deleting "%s".' % target_path) + if os.path.isdir(target_path): + shutil.rmtree(target_path) + else: + os.remove(target_path) + + self._log_transfer("Moving files into place...", source_path, target_path) + + # The shutil.move() command creates intermediate directories if they + # do not exist, but we do not rely on this behavior since we + # need to create the __init__.py file anyway. + shutil.move(source_path, target_path) + + self._record_url_downloaded(package_name, url) + + def install(self, url, should_refresh=False, target_name=None, + url_subpath=None): + """Install a python package from an URL. + + Args: + url: The URL from which to download the package. + + Optional Args: + should_refresh: A boolean value of whether the package should be + downloaded again if the package is already present. + target_name: The name of the folder or file in the autoinstaller + target directory at which the package should be + installed. Defaults to the base name of the + URL sub-path. This parameter must be provided if + the URL sub-path is not specified. + url_subpath: The relative path of the URL directory that should + be installed. Defaults to the full directory, or + the entire URL contents. + + """ + if target_name is None: + if not url_subpath: + raise ValueError('The "target_name" parameter must be ' + 'provided if the "url_subpath" parameter ' + "is not provided.") + # Remove any trailing slashes. + url_subpath = os.path.normpath(url_subpath) + target_name = os.path.basename(url_subpath) + + target_path = os.path.join(self._target_dir, target_name) + if not should_refresh and self._is_downloaded(target_name, url): + _log.debug('URL for %s already downloaded. Skipping...' + % target_name) + _log.debug(' "%s"' % url) + return + + self._log_transfer("Auto-installing package: %s" % target_name, + url, target_path, log_method=_log.info) + + # The scratch directory is where we will download and prepare + # files specific to this install until they are ready to move + # into place. + scratch_dir = self._create_scratch_directory(target_name) + + try: + self._install(package_name=target_name, + target_path=target_path, + scratch_dir=scratch_dir, + url=url, + url_subpath=url_subpath) + except Exception, err: + # Append existing Error message to new Error. + message = ("Error auto-installing the %s package to:\n" + ' "%s"\n' + " --> Inner message: %s" + % (target_name, target_path, err)) + raise Exception(message) + finally: + _log.debug('Cleaning up: deleting "%s".' % scratch_dir) + shutil.rmtree(scratch_dir) + _log.debug('Auto-installed %s to:' % target_name) + _log.debug(' "%s"' % target_path) + + +if __name__=="__main__": + + # Configure the autoinstall logger to log DEBUG messages for + # development testing purposes. + console = logging.StreamHandler() + + formatter = logging.Formatter('%(name)s: %(levelname)-8s %(message)s') + console.setFormatter(formatter) + _log.addHandler(console) + _log.setLevel(logging.DEBUG) + + # Use a more visible temp directory for debug purposes. + this_dir = os.path.dirname(__file__) + target_dir = os.path.join(this_dir, "autoinstalled") + temp_dir = os.path.join(target_dir, "Temp") + + installer = AutoInstaller(target_dir=target_dir, + temp_dir=temp_dir) + + installer.install(should_refresh=False, + target_name="pep8.py", + url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", + url_subpath="pep8-0.5.0/pep8.py") + installer.install(should_refresh=False, + target_name="mechanize", + url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", + url_subpath="mechanize") + diff --git a/WebKitTools/Scripts/webkitpy/webkit_logging.py b/WebKitTools/Scripts/webkitpy/common/system/deprecated_logging.py index ba1c5eb..9e6b529 100644 --- a/WebKitTools/Scripts/webkitpy/webkit_logging.py +++ b/WebKitTools/Scripts/webkitpy/common/system/deprecated_logging.py @@ -30,24 +30,30 @@ # WebKit's Python module for logging # This module is now deprecated in favor of python's built-in logging.py. +import codecs import os import sys + def log(string): print >> sys.stderr, string + def error(string): log("ERROR: %s" % string) exit(1) + # Simple class to split output between multiple destinations class tee: def __init__(self, *files): self.files = files - def write(self, string): + # Callers should pass an already encoded string for writing. + def write(self, bytes): for file in self.files: - file.write(string) + file.write(bytes) + class OutputTee: def __init__(self): @@ -71,7 +77,7 @@ class OutputTee: (log_directory, log_name) = os.path.split(log_path) if log_directory and not os.path.exists(log_directory): os.makedirs(log_directory) - return open(log_path, 'a+') + return codecs.open(log_path, "a+", "utf-8") def _tee_outputs_to_files(self, files): if not self._original_stdout: diff --git a/WebKitTools/Scripts/webkitpy/webkit_logging_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/deprecated_logging_unittest.py index b940a4d..3778162 100644 --- a/WebKitTools/Scripts/webkitpy/webkit_logging_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/system/deprecated_logging_unittest.py @@ -27,13 +27,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import subprocess import StringIO import tempfile import unittest -from webkitpy.executive import ScriptError -from webkitpy.webkit_logging import * +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.system.deprecated_logging import * class LoggingTest(unittest.TestCase): @@ -46,7 +45,7 @@ class LoggingTest(unittest.TestCase): log(log_input) actual_output = test_stderr.getvalue() finally: - original_stderr = original_stderr + sys.stderr = original_stderr self.assertEquals(actual_output, expected_output, "log(\"%s\") expected: %s actual: %s" % (log_input, expected_output, actual_output)) diff --git a/WebKitTools/Scripts/webkitpy/common/system/executive.py b/WebKitTools/Scripts/webkitpy/common/system/executive.py new file mode 100644 index 0000000..6088680 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/executive.py @@ -0,0 +1,312 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +try: + # This API exists only in Python 2.6 and higher. :( + import multiprocessing +except ImportError: + multiprocessing = None + +import errno +import logging +import os +import platform +import StringIO +import signal +import subprocess +import sys +import time + +from webkitpy.common.system.deprecated_logging import tee + + +_log = logging.getLogger("webkitpy.common.system") + + +class ScriptError(Exception): + + def __init__(self, + message=None, + script_args=None, + exit_code=None, + output=None, + cwd=None): + if not message: + message = 'Failed to run "%s"' % script_args + if exit_code: + message += " exit_code: %d" % exit_code + if cwd: + message += " cwd: %s" % cwd + + Exception.__init__(self, message) + self.script_args = script_args # 'args' is already used by Exception + self.exit_code = exit_code + self.output = output + self.cwd = cwd + + def message_with_output(self, output_limit=500): + if self.output: + if output_limit and len(self.output) > output_limit: + return "%s\nLast %s characters of output:\n%s" % \ + (self, output_limit, self.output[-output_limit:]) + return "%s\n%s" % (self, self.output) + return str(self) + + def command_name(self): + command_path = self.script_args + if type(command_path) is list: + command_path = command_path[0] + return os.path.basename(command_path) + + +def run_command(*args, **kwargs): + # FIXME: This should not be a global static. + # New code should use Executive.run_command directly instead + return Executive().run_command(*args, **kwargs) + + +class Executive(object): + + def _should_close_fds(self): + # We need to pass close_fds=True to work around Python bug #2320 + # (otherwise we can hang when we kill DumpRenderTree when we are running + # multiple threads). See http://bugs.python.org/issue2320 . + # Note that close_fds isn't supported on Windows, but this bug only + # shows up on Mac and Linux. + return sys.platform not in ('win32', 'cygwin') + + def _run_command_with_teed_output(self, args, teed_output): + args = map(unicode, args) # Popen will throw an exception if args are non-strings (like int()) + child_process = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + close_fds=self._should_close_fds()) + + # Use our own custom wait loop because Popen ignores a tee'd + # stderr/stdout. + # FIXME: This could be improved not to flatten output to stdout. + while True: + output_line = child_process.stdout.readline() + if output_line == "" and child_process.poll() != None: + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + return child_process.poll() + # We assume that the child process wrote to us in utf-8, + # so no re-encoding is necessary before writing here. + teed_output.write(output_line) + + # FIXME: Remove this deprecated method and move callers to run_command. + # FIXME: This method is a hack to allow running command which both + # capture their output and print out to stdin. Useful for things + # like "build-webkit" where we want to display to the user that we're building + # but still have the output to stuff into a log file. + def run_and_throw_if_fail(self, args, quiet=False, decode_output=True): + # Cache the child's output locally so it can be used for error reports. + child_out_file = StringIO.StringIO() + tee_stdout = sys.stdout + if quiet: + dev_null = open(os.devnull, "w") # FIXME: Does this need an encoding? + tee_stdout = dev_null + child_stdout = tee(child_out_file, tee_stdout) + exit_code = self._run_command_with_teed_output(args, child_stdout) + if quiet: + dev_null.close() + + child_output = child_out_file.getvalue() + child_out_file.close() + + # We assume the child process output utf-8 + if decode_output: + child_output = child_output.decode("utf-8") + + if exit_code: + raise ScriptError(script_args=args, + exit_code=exit_code, + output=child_output) + return child_output + + def cpu_count(self): + if multiprocessing: + return multiprocessing.cpu_count() + # Darn. We don't have the multiprocessing package. + system_name = platform.system() + if system_name == "Darwin": + return int(self.run_command(["sysctl", "-n", "hw.ncpu"])) + elif system_name == "Windows": + return int(os.environ.get('NUMBER_OF_PROCESSORS', 1)) + elif system_name == "Linux": + num_cores = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(num_cores, int) and num_cores > 0: + return num_cores + # This quantity is a lie but probably a reasonable guess for modern + # machines. + return 2 + + def kill_process(self, pid): + """Attempts to kill the given pid. + Will fail silently if pid does not exist or insufficient permisssions.""" + if sys.platform == "win32": + # We only use taskkill.exe on windows (not cygwin) because subprocess.pid + # is a CYGWIN pid and taskkill.exe expects a windows pid. + # Thankfully os.kill on CYGWIN handles either pid type. + command = ["taskkill.exe", "/f", "/pid", pid] + # taskkill will exit 128 if the process is not found. We should log. + self.run_command(command, error_handler=self.ignore_error) + return + + # According to http://docs.python.org/library/os.html + # os.kill isn't available on Windows. python 2.5.5 os.kill appears + # to work in cygwin, however it occasionally raises EAGAIN. + retries_left = 10 if sys.platform == "cygwin" else 1 + while retries_left > 0: + try: + retries_left -= 1 + os.kill(pid, signal.SIGKILL) + except OSError, e: + if e.errno == errno.EAGAIN: + if retries_left <= 0: + _log.warn("Failed to kill pid %s. Too many EAGAIN errors." % pid) + continue + if e.errno == errno.ESRCH: # The process does not exist. + _log.warn("Called kill_process with a non-existant pid %s" % pid) + return + raise + + def _windows_image_name(self, process_name): + name, extension = os.path.splitext(process_name) + if not extension: + # taskkill expects processes to end in .exe + # If necessary we could add a flag to disable appending .exe. + process_name = "%s.exe" % name + return process_name + + def kill_all(self, process_name): + """Attempts to kill processes matching process_name. + Will fail silently if no process are found.""" + if sys.platform in ("win32", "cygwin"): + image_name = self._windows_image_name(process_name) + command = ["taskkill.exe", "/f", "/im", image_name] + # taskkill will exit 128 if the process is not found. We should log. + self.run_command(command, error_handler=self.ignore_error) + return + + # FIXME: This is inconsistent that kill_all uses TERM and kill_process + # uses KILL. Windows is always using /f (which seems like -KILL). + # We should pick one mode, or add support for switching between them. + # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER + command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name] + # killall returns 1 if no process can be found and 2 on command error. + # FIXME: We should pass a custom error_handler to allow only exit_code 1. + # We should log in exit_code == 1 + self.run_command(command, error_handler=self.ignore_error) + + # Error handlers do not need to be static methods once all callers are + # updated to use an Executive object. + + @staticmethod + def default_error_handler(error): + raise error + + @staticmethod + def ignore_error(error): + pass + + def _compute_stdin(self, input): + """Returns (stdin, string_to_communicate)""" + # FIXME: We should be returning /dev/null for stdin + # or closing stdin after process creation to prevent + # child processes from getting input from the user. + if not input: + return (None, None) + if hasattr(input, "read"): # Check if the input is a file. + return (input, None) # Assume the file is in the right encoding. + + # Popen in Python 2.5 and before does not automatically encode unicode objects. + # http://bugs.python.org/issue5290 + # See https://bugs.webkit.org/show_bug.cgi?id=37528 + # for an example of a regresion caused by passing a unicode string directly. + # FIXME: We may need to encode differently on different platforms. + if isinstance(input, unicode): + input = input.encode("utf-8") + return (subprocess.PIPE, input) + + def _command_for_printing(self, args): + """Returns a print-ready string representing command args. + The string should be copy/paste ready for execution in a shell.""" + escaped_args = [] + for arg in args: + if isinstance(arg, unicode): + # Escape any non-ascii characters for easy copy/paste + arg = arg.encode("unicode_escape") + # FIXME: Do we need to fix quotes here? + escaped_args.append(arg) + return " ".join(escaped_args) + + # FIXME: run_and_throw_if_fail should be merged into this method. + def run_command(self, + args, + cwd=None, + input=None, + error_handler=None, + return_exit_code=False, + return_stderr=True, + decode_output=True): + """Popen wrapper for convenience and to work around python bugs.""" + assert(isinstance(args, list) or isinstance(args, tuple)) + start_time = time.time() + args = map(unicode, args) # Popen will throw an exception if args are non-strings (like int()) + stdin, string_to_communicate = self._compute_stdin(input) + stderr = subprocess.STDOUT if return_stderr else None + + process = subprocess.Popen(args, + stdin=stdin, + stdout=subprocess.PIPE, + stderr=stderr, + cwd=cwd, + close_fds=self._should_close_fds()) + output = process.communicate(string_to_communicate)[0] + # run_command automatically decodes to unicode() unless explicitly told not to. + if decode_output: + output = output.decode("utf-8") + # wait() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + exit_code = process.wait() + + _log.debug('"%s" took %.2fs' % (self._command_for_printing(args), time.time() - start_time)) + + if return_exit_code: + return exit_code + + if exit_code: + script_error = ScriptError(script_args=args, + exit_code=exit_code, + output=output, + cwd=cwd) + (error_handler or self.default_error_handler)(script_error) + return output diff --git a/WebKitTools/Scripts/webkitpy/common/system/executive_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/executive_unittest.py new file mode 100644 index 0000000..32f8f51 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/executive_unittest.py @@ -0,0 +1,122 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# Copyright (C) 2009 Daniel Bates (dbates@intudata.com). All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import signal +import subprocess +import sys +import unittest + +from webkitpy.common.system.executive import Executive, run_command, ScriptError + + +class ExecutiveTest(unittest.TestCase): + + def test_run_command_with_bad_command(self): + def run_bad_command(): + run_command(["foo_bar_command_blah"], error_handler=Executive.ignore_error, return_exit_code=True) + self.failUnlessRaises(OSError, run_bad_command) + + def test_run_command_args_type(self): + executive = Executive() + self.assertRaises(AssertionError, executive.run_command, "echo") + self.assertRaises(AssertionError, executive.run_command, u"echo") + executive.run_command(["echo", "foo"]) + executive.run_command(("echo", "foo")) + + def test_run_command_with_unicode(self): + """Validate that it is safe to pass unicode() objects + to Executive.run* methods, and they will return unicode() + objects by default unless decode_output=False""" + executive = Executive() + unicode_tor = u"WebKit \u2661 Tor Arne Vestb\u00F8!" + utf8_tor = unicode_tor.encode("utf-8") + + output = executive.run_command(["cat"], input=unicode_tor) + self.assertEquals(output, unicode_tor) + + output = executive.run_command(["echo", "-n", unicode_tor]) + self.assertEquals(output, unicode_tor) + + output = executive.run_command(["echo", "-n", unicode_tor], decode_output=False) + self.assertEquals(output, utf8_tor) + + # Make sure that str() input also works. + output = executive.run_command(["cat"], input=utf8_tor, decode_output=False) + self.assertEquals(output, utf8_tor) + + # FIXME: We should only have one run* method to test + output = executive.run_and_throw_if_fail(["echo", "-n", unicode_tor], quiet=True) + self.assertEquals(output, unicode_tor) + + output = executive.run_and_throw_if_fail(["echo", "-n", unicode_tor], quiet=True, decode_output=False) + self.assertEquals(output, utf8_tor) + + def test_kill_process(self): + executive = Executive() + # We use "yes" because it loops forever. + process = subprocess.Popen(["yes"], stdout=subprocess.PIPE) + self.assertEqual(process.poll(), None) # Process is running + executive.kill_process(process.pid) + # Note: Can't use a ternary since signal.SIGKILL is undefined for sys.platform == "win32" + if sys.platform == "win32": + expected_exit_code = 0 # taskkill.exe results in exit(0) + else: + expected_exit_code = -signal.SIGKILL + self.assertEqual(process.wait(), expected_exit_code) + # Killing again should fail silently. + executive.kill_process(process.pid) + + def _assert_windows_image_name(self, name, expected_windows_name): + executive = Executive() + windows_name = executive._windows_image_name(name) + self.assertEqual(windows_name, expected_windows_name) + + def test_windows_image_name(self): + self._assert_windows_image_name("foo", "foo.exe") + self._assert_windows_image_name("foo.exe", "foo.exe") + self._assert_windows_image_name("foo.com", "foo.com") + # If the name looks like an extension, even if it isn't + # supposed to, we have no choice but to return the original name. + self._assert_windows_image_name("foo.baz", "foo.baz") + self._assert_windows_image_name("foo.baz.exe", "foo.baz.exe") + + def test_kill_all(self): + executive = Executive() + # We use "yes" because it loops forever. + process = subprocess.Popen(["yes"], stdout=subprocess.PIPE) + self.assertEqual(process.poll(), None) # Process is running + executive.kill_all("yes") + # Note: Can't use a ternary since signal.SIGTERM is undefined for sys.platform == "win32" + if sys.platform in ("win32", "cygwin"): + expected_exit_code = 0 # taskkill.exe results in exit(0) + else: + expected_exit_code = -signal.SIGTERM + self.assertEqual(process.wait(), expected_exit_code) + # Killing again should fail silently. + executive.kill_all("yes") diff --git a/WebKitTools/Scripts/webkitpy/common/system/logtesting.py b/WebKitTools/Scripts/webkitpy/common/system/logtesting.py new file mode 100644 index 0000000..e361cb5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/logtesting.py @@ -0,0 +1,258 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports the unit-testing of logging code. + +Provides support for unit-testing messages logged using the built-in +logging module. + +Inherit from the LoggingTestCase class for basic testing needs. For +more advanced needs (e.g. unit-testing methods that configure logging), +see the TestLogStream class, and perhaps also the LogTesting class. + +""" + +import logging +import unittest + + +class TestLogStream(object): + + """Represents a file-like object for unit-testing logging. + + This is meant for passing to the logging.StreamHandler constructor. + Log messages captured by instances of this object can be tested + using self.assertMessages() below. + + """ + + def __init__(self, test_case): + """Create an instance. + + Args: + test_case: A unittest.TestCase instance. + + """ + self._test_case = test_case + self.messages = [] + """A list of log messages written to the stream.""" + + # Python documentation says that any object passed to the StreamHandler + # constructor should support write() and flush(): + # + # http://docs.python.org/library/logging.html#module-logging.handlers + def write(self, message): + self.messages.append(message) + + def flush(self): + pass + + def assertMessages(self, messages): + """Assert that the given messages match the logged messages. + + messages: A list of log message strings. + + """ + self._test_case.assertEquals(messages, self.messages) + + +class LogTesting(object): + + """Supports end-to-end unit-testing of log messages. + + Sample usage: + + class SampleTest(unittest.TestCase): + + def setUp(self): + self._log = LogTesting.setUp(self) # Turn logging on. + + def tearDown(self): + self._log.tearDown() # Turn off and reset logging. + + def test_logging_in_some_method(self): + call_some_method() # Contains calls to _log.info(), etc. + + # Check the resulting log messages. + self._log.assertMessages(["INFO: expected message #1", + "WARNING: expected message #2"]) + + """ + + def __init__(self, test_stream, handler): + """Create an instance. + + This method should never be called directly. Instances should + instead be created using the static setUp() method. + + Args: + test_stream: A TestLogStream instance. + handler: The handler added to the logger. + + """ + self._test_stream = test_stream + self._handler = handler + + @staticmethod + def _getLogger(): + """Return the logger being tested.""" + # It is possible we might want to return something other than + # the root logger in some special situation. For now, the + # root logger seems to suffice. + return logging.getLogger() + + @staticmethod + def setUp(test_case, logging_level=logging.INFO): + """Configure logging for unit testing. + + Configures the root logger to log to a testing log stream. + Only messages logged at or above the given level are logged + to the stream. Messages logged to the stream are formatted + in the following way, for example-- + + "INFO: This is a test log message." + + This method should normally be called in the setUp() method + of a unittest.TestCase. See the docstring of this class + for more details. + + Returns: + A LogTesting instance. + + Args: + test_case: A unittest.TestCase instance. + logging_level: An integer logging level that is the minimum level + of log messages you would like to test. + + """ + stream = TestLogStream(test_case) + handler = logging.StreamHandler(stream) + handler.setLevel(logging_level) + formatter = logging.Formatter("%(levelname)s: %(message)s") + handler.setFormatter(formatter) + + # Notice that we only change the root logger by adding a handler + # to it. In particular, we do not reset its level using + # logger.setLevel(). This ensures that we have not interfered + # with how the code being tested may have configured the root + # logger. + logger = LogTesting._getLogger() + logger.addHandler(handler) + + return LogTesting(stream, handler) + + def tearDown(self): + """Assert there are no remaining log messages, and reset logging. + + This method asserts that there are no more messages in the array of + log messages, and then restores logging to its original state. + This method should normally be called in the tearDown() method of a + unittest.TestCase. See the docstring of this class for more details. + + """ + self.assertMessages([]) + logger = LogTesting._getLogger() + logger.removeHandler(self._handler) + + def messages(self): + """Return the current list of log messages.""" + return self._test_stream.messages + + # FIXME: Add a clearMessages() method for cases where the caller + # deliberately doesn't want to assert every message. + + # We clear the log messages after asserting since they are no longer + # needed after asserting. This serves two purposes: (1) it simplifies + # the calling code when we want to check multiple logging calls in a + # single test method, and (2) it lets us check in the tearDown() method + # that there are no remaining log messages to be asserted. + # + # The latter ensures that no extra log messages are getting logged that + # the caller might not be aware of or may have forgotten to check for. + # This gets us a bit more mileage out of our tests without writing any + # additional code. + def assertMessages(self, messages): + """Assert the current array of log messages, and clear its contents. + + Args: + messages: A list of log message strings. + + """ + try: + self._test_stream.assertMessages(messages) + finally: + # We want to clear the array of messages even in the case of + # an Exception (e.g. an AssertionError). Otherwise, another + # AssertionError can occur in the tearDown() because the + # array might not have gotten emptied. + self._test_stream.messages = [] + + +# This class needs to inherit from unittest.TestCase. Otherwise, the +# setUp() and tearDown() methods will not get fired for test case classes +# that inherit from this class -- even if the class inherits from *both* +# unittest.TestCase and LoggingTestCase. +# +# FIXME: Rename this class to LoggingTestCaseBase to be sure that +# the unittest module does not interpret this class as a unittest +# test case itself. +class LoggingTestCase(unittest.TestCase): + + """Supports end-to-end unit-testing of log messages. + + Sample usage: + + class SampleTest(LoggingTestCase): + + def test_logging_in_some_method(self): + call_some_method() # Contains calls to _log.info(), etc. + + # Check the resulting log messages. + self.assertLog(["INFO: expected message #1", + "WARNING: expected message #2"]) + + """ + + def setUp(self): + self._log = LogTesting.setUp(self) + + def tearDown(self): + self._log.tearDown() + + def logMessages(self): + """Return the current list of log messages.""" + return self._log.messages() + + # FIXME: Add a clearMessages() method for cases where the caller + # deliberately doesn't want to assert every message. + + # See the code comments preceding LogTesting.assertMessages() for + # an explanation of why we clear the array of messages after + # asserting its contents. + def assertLog(self, messages): + """Assert the current array of log messages, and clear its contents. + + Args: + messages: A list of log message strings. + + """ + self._log.assertMessages(messages) diff --git a/WebKitTools/Scripts/webkitpy/common/system/logutils.py b/WebKitTools/Scripts/webkitpy/common/system/logutils.py new file mode 100644 index 0000000..cd4e60f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/logutils.py @@ -0,0 +1,207 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports webkitpy logging.""" + +# FIXME: Move this file to webkitpy/python24 since logging needs to +# be configured prior to running version-checking code. + +import logging +import os +import sys + +import webkitpy + + +_log = logging.getLogger(__name__) + +# We set these directory paths lazily in get_logger() below. +_scripts_dir = "" +"""The normalized, absolute path to the ...Scripts directory.""" + +_webkitpy_dir = "" +"""The normalized, absolute path to the ...Scripts/webkitpy directory.""" + + +def _normalize_path(path): + """Return the given path normalized. + + Converts a path to an absolute path, removes any trailing slashes, + removes any extension, and lower-cases it. + + """ + path = os.path.abspath(path) + path = os.path.normpath(path) + path = os.path.splitext(path)[0] # Remove the extension, if any. + path = path.lower() + + return path + + +# Observe that the implementation of this function does not require +# the use of any hard-coded strings like "webkitpy", etc. +# +# The main benefit this function has over using-- +# +# _log = logging.getLogger(__name__) +# +# is that get_logger() returns the same value even if __name__ is +# "__main__" -- i.e. even if the module is the script being executed +# from the command-line. +def get_logger(path): + """Return a logging.logger for the given path. + + Returns: + A logger whose name is the name of the module corresponding to + the given path. If the module is in webkitpy, the name is + the fully-qualified dotted module name beginning with webkitpy.... + Otherwise, the name is the base name of the module (i.e. without + any dotted module name prefix). + + Args: + path: The path of the module. Normally, this parameter should be + the __file__ variable of the module. + + Sample usage: + + import webkitpy.common.system.logutils as logutils + + _log = logutils.get_logger(__file__) + + """ + # Since we assign to _scripts_dir and _webkitpy_dir in this function, + # we need to declare them global. + global _scripts_dir + global _webkitpy_dir + + path = _normalize_path(path) + + # Lazily evaluate _webkitpy_dir and _scripts_dir. + if not _scripts_dir: + # The normalized, absolute path to ...Scripts/webkitpy/__init__. + webkitpy_path = _normalize_path(webkitpy.__file__) + + _webkitpy_dir = os.path.split(webkitpy_path)[0] + _scripts_dir = os.path.split(_webkitpy_dir)[0] + + if path.startswith(_webkitpy_dir): + # Remove the initial Scripts directory portion, so the path + # starts with /webkitpy, for example "/webkitpy/init/logutils". + path = path[len(_scripts_dir):] + + parts = [] + while True: + (path, tail) = os.path.split(path) + if not tail: + break + parts.insert(0, tail) + + logger_name = ".".join(parts) # For example, webkitpy.common.system.logutils. + else: + # The path is outside of webkitpy. Default to the basename + # without the extension. + basename = os.path.basename(path) + logger_name = os.path.splitext(basename)[0] + + return logging.getLogger(logger_name) + + +def _default_handlers(stream): + """Return a list of the default logging handlers to use. + + Args: + stream: See the configure_logging() docstring. + + """ + # Create the filter. + def should_log(record): + """Return whether a logging.LogRecord should be logged.""" + # FIXME: Enable the logging of autoinstall messages once + # autoinstall is adjusted. Currently, autoinstall logs + # INFO messages when importing already-downloaded packages, + # which is too verbose. + if record.name.startswith("webkitpy.thirdparty.autoinstall"): + return False + return True + + logging_filter = logging.Filter() + logging_filter.filter = should_log + + # Create the handler. + handler = logging.StreamHandler(stream) + formatter = logging.Formatter("%(name)s: [%(levelname)s] %(message)s") + handler.setFormatter(formatter) + handler.addFilter(logging_filter) + + return [handler] + + +def configure_logging(logging_level=None, logger=None, stream=None, + handlers=None): + """Configure logging for standard purposes. + + Returns: + A list of references to the logging handlers added to the root + logger. This allows the caller to later remove the handlers + using logger.removeHandler. This is useful primarily during unit + testing where the caller may want to configure logging temporarily + and then undo the configuring. + + Args: + logging_level: The minimum logging level to log. Defaults to + logging.INFO. + logger: A logging.logger instance to configure. This parameter + should be used only in unit tests. Defaults to the + root logger. + stream: A file-like object to which to log used in creating the default + handlers. The stream must define an "encoding" data attribute, + or else logging raises an error. Defaults to sys.stderr. + handlers: A list of logging.Handler instances to add to the logger + being configured. If this parameter is provided, then the + stream parameter is not used. + + """ + # If the stream does not define an "encoding" data attribute, the + # logging module can throw an error like the following: + # + # Traceback (most recent call last): + # File "/System/Library/Frameworks/Python.framework/Versions/2.6/... + # lib/python2.6/logging/__init__.py", line 761, in emit + # self.stream.write(fs % msg.encode(self.stream.encoding)) + # LookupError: unknown encoding: unknown + if logging_level is None: + logging_level = logging.INFO + if logger is None: + logger = logging.getLogger() + if stream is None: + stream = sys.stderr + if handlers is None: + handlers = _default_handlers(stream) + + logger.setLevel(logging_level) + + for handler in handlers: + logger.addHandler(handler) + + _log.debug("Debug logging enabled.") + + return handlers diff --git a/WebKitTools/Scripts/webkitpy/common/system/logutils_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/logutils_unittest.py new file mode 100644 index 0000000..a4a6496 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/logutils_unittest.py @@ -0,0 +1,142 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for logutils.py.""" + +import logging +import os +import unittest + +from webkitpy.common.system.logtesting import LogTesting +from webkitpy.common.system.logtesting import TestLogStream +import webkitpy.common.system.logutils as logutils + + +class GetLoggerTest(unittest.TestCase): + + """Tests get_logger().""" + + def test_get_logger_in_webkitpy(self): + logger = logutils.get_logger(__file__) + self.assertEquals(logger.name, "webkitpy.common.system.logutils_unittest") + + def test_get_logger_not_in_webkitpy(self): + # Temporarily change the working directory so that we + # can test get_logger() for a path outside of webkitpy. + working_directory = os.getcwd() + root_dir = "/" + os.chdir(root_dir) + + logger = logutils.get_logger("/WebKitTools/Scripts/test-webkitpy") + self.assertEquals(logger.name, "test-webkitpy") + + logger = logutils.get_logger("/WebKitTools/Scripts/test-webkitpy.py") + self.assertEquals(logger.name, "test-webkitpy") + + os.chdir(working_directory) + + +class ConfigureLoggingTestBase(unittest.TestCase): + + """Base class for configure_logging() unit tests.""" + + def _logging_level(self): + raise Exception("Not implemented.") + + def setUp(self): + log_stream = TestLogStream(self) + + # Use a logger other than the root logger or one prefixed with + # "webkitpy." so as not to conflict with test-webkitpy logging. + logger = logging.getLogger("unittest") + + # Configure the test logger not to pass messages along to the + # root logger. This prevents test messages from being + # propagated to loggers used by test-webkitpy logging (e.g. + # the root logger). + logger.propagate = False + + logging_level = self._logging_level() + self._handlers = logutils.configure_logging(logging_level=logging_level, + logger=logger, + stream=log_stream) + self._log = logger + self._log_stream = log_stream + + def tearDown(self): + """Reset logging to its original state. + + This method ensures that the logging configuration set up + for a unit test does not affect logging in other unit tests. + + """ + logger = self._log + for handler in self._handlers: + logger.removeHandler(handler) + + def _assert_log_messages(self, messages): + """Assert that the logged messages equal the given messages.""" + self._log_stream.assertMessages(messages) + + +class ConfigureLoggingTest(ConfigureLoggingTestBase): + + """Tests configure_logging() with the default logging level.""" + + def _logging_level(self): + return None + + def test_info_message(self): + self._log.info("test message") + self._assert_log_messages(["unittest: [INFO] test message\n"]) + + def test_below_threshold_message(self): + # We test the boundary case of a logging level equal to 19. + # In practice, we will probably only be calling log.debug(), + # which corresponds to a logging level of 10. + level = logging.INFO - 1 # Equals 19. + self._log.log(level, "test message") + self._assert_log_messages([]) + + def test_two_messages(self): + self._log.info("message1") + self._log.info("message2") + self._assert_log_messages(["unittest: [INFO] message1\n", + "unittest: [INFO] message2\n"]) + + +class ConfigureLoggingCustomLevelTest(ConfigureLoggingTestBase): + + """Tests configure_logging() with a custom logging level.""" + + _level = 36 + + def _logging_level(self): + return self._level + + def test_logged_message(self): + self._log.log(self._level, "test message") + self._assert_log_messages(["unittest: [Level 36] test message\n"]) + + def test_below_threshold_message(self): + self._log.log(self._level - 1, "test message") + self._assert_log_messages([]) diff --git a/WebKitTools/Scripts/webkitpy/common/system/ospath.py b/WebKitTools/Scripts/webkitpy/common/system/ospath.py new file mode 100644 index 0000000..aed7a3d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/ospath.py @@ -0,0 +1,83 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains a substitute for Python 2.6's os.path.relpath().""" + +import os + + +# This function is a replacement for os.path.relpath(), which is only +# available in Python 2.6: +# +# http://docs.python.org/library/os.path.html#os.path.relpath +# +# It should behave essentially the same as os.path.relpath(), except for +# returning None on paths not contained in abs_start_path. +def relpath(path, start_path, os_path_abspath=None): + """Return a path relative to the given start path, or None. + + Returns None if the path is not contained in the directory start_path. + + Args: + path: An absolute or relative path to convert to a relative path. + start_path: The path relative to which the given path should be + converted. + os_path_abspath: A replacement function for unit testing. This + function should strip trailing slashes just like + os.path.abspath(). Defaults to os.path.abspath. + + """ + if os_path_abspath is None: + os_path_abspath = os.path.abspath + + # Since os_path_abspath() calls os.path.normpath()-- + # + # (see http://docs.python.org/library/os.path.html#os.path.abspath ) + # + # it also removes trailing slashes and converts forward and backward + # slashes to the preferred slash os.sep. + start_path = os_path_abspath(start_path) + path = os_path_abspath(path) + + if not path.lower().startswith(start_path.lower()): + # Then path is outside the directory given by start_path. + return None + + rel_path = path[len(start_path):] + + if not rel_path: + # Then the paths are the same. + pass + elif rel_path[0] == os.sep: + # It is probably sufficient to remove just the first character + # since os.path.normpath() collapses separators, but we use + # lstrip() just to be sure. + rel_path = rel_path.lstrip(os.sep) + else: + # We are in the case typified by the following example: + # + # start_path = "/tmp/foo" + # path = "/tmp/foobar" + # rel_path = "bar" + return None + + return rel_path diff --git a/WebKitTools/Scripts/webkitpy/common/system/ospath_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/ospath_unittest.py new file mode 100644 index 0000000..0493c68 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/system/ospath_unittest.py @@ -0,0 +1,62 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for ospath.py.""" + +import os +import unittest + +from webkitpy.common.system.ospath import relpath + + +# Make sure the tests in this class are platform independent. +class RelPathTest(unittest.TestCase): + + """Tests relpath().""" + + os_path_abspath = lambda self, path: path + + def _rel_path(self, path, abs_start_path): + return relpath(path, abs_start_path, self.os_path_abspath) + + def test_same_path(self): + rel_path = self._rel_path("WebKit", "WebKit") + self.assertEquals(rel_path, "") + + def test_long_rel_path(self): + start_path = "WebKit" + expected_rel_path = os.path.join("test", "Foo.txt") + path = os.path.join(start_path, expected_rel_path) + + rel_path = self._rel_path(path, start_path) + self.assertEquals(expected_rel_path, rel_path) + + def test_none_rel_path(self): + """Test _rel_path() with None return value.""" + start_path = "WebKit" + path = os.path.join("other_dir", "foo.txt") + + rel_path = self._rel_path(path, start_path) + self.assertTrue(rel_path is None) + + rel_path = self._rel_path("WebKitTools", "WebKit") + self.assertTrue(rel_path is None) diff --git a/WebKitTools/Scripts/webkitpy/outputcapture.py b/WebKitTools/Scripts/webkitpy/common/system/outputcapture.py index 592a669..68a3919 100644 --- a/WebKitTools/Scripts/webkitpy/outputcapture.py +++ b/WebKitTools/Scripts/webkitpy/common/system/outputcapture.py @@ -52,9 +52,12 @@ class OutputCapture(object): def restore_output(self): return (self._restore_output_with_name("stdout"), self._restore_output_with_name("stderr")) - def assert_outputs(self, testcase, function, args=[], kwargs={}, expected_stdout="", expected_stderr=""): + def assert_outputs(self, testcase, function, args=[], kwargs={}, expected_stdout="", expected_stderr="", expected_exception=None): self.capture_output() - return_value = function(*args, **kwargs) + if expected_exception: + return_value = testcase.assertRaises(expected_exception, function, *args, **kwargs) + else: + return_value = function(*args, **kwargs) (stdout_string, stderr_string) = self.restore_output() testcase.assertEqual(stdout_string, expected_stdout) testcase.assertEqual(stderr_string, expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/user.py b/WebKitTools/Scripts/webkitpy/common/system/user.py index b2ec19e..9444c00 100644 --- a/WebKitTools/Scripts/webkitpy/user.py +++ b/WebKitTools/Scripts/webkitpy/common/system/user.py @@ -26,28 +26,73 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging import os import shlex import subprocess +import sys import webbrowser + +_log = logging.getLogger("webkitpy.common.system.user") + + +try: + import readline +except ImportError: + if sys.platform != "win32": + # There is no readline module for win32, not much to do except cry. + _log.warn("Unable to import readline.") + # FIXME: We could give instructions for non-mac platforms. + # Lack of readline results in a very bad user experiance. + if sys.platform == "mac": + _log.warn("If you're using MacPorts, try running:") + _log.warn(" sudo port install py25-readline") + + class User(object): - @staticmethod - def prompt(message, repeat=1, raw_input=raw_input): + # FIXME: These are @classmethods because bugzilla.py doesn't have a Tool object (thus no User instance). + @classmethod + def prompt(cls, message, repeat=1, raw_input=raw_input): response = None while (repeat and not response): repeat -= 1 response = raw_input(message) return response + @classmethod + def prompt_with_list(cls, list_title, list_items): + print list_title + i = 0 + for item in list_items: + i += 1 + print "%2d. %s" % (i, item) + result = int(cls.prompt("Enter a number: ")) - 1 + return list_items[result] + def edit(self, files): editor = os.environ.get("EDITOR") or "vi" args = shlex.split(editor) + # Note: Not thread safe: http://bugs.python.org/issue2320 subprocess.call(args + files) + def edit_changelog(self, files): + edit_application = os.environ.get("CHANGE_LOG_EDIT_APPLICATION") + if edit_application and sys.platform == "darwin": + # On Mac we support editing ChangeLogs using an application. + args = shlex.split(edit_application) + print "Using editor in the CHANGE_LOG_EDIT_APPLICATION environment variable." + print "Please quit the editor application when done editing." + if edit_application.find("Xcode.app"): + print "Instead of using Xcode.app, consider using EDITOR=\"xed --wait\"." + subprocess.call(["open", "-W", "-n", "-a"] + args + files) + return + self.edit(files) + def page(self, message): pager = os.environ.get("PAGER") or "less" try: + # Note: Not thread safe: http://bugs.python.org/issue2320 child_process = subprocess.Popen([pager], stdin=subprocess.PIPE) child_process.communicate(input=message) except IOError, e: @@ -59,5 +104,14 @@ class User(object): response = raw_input("%s [Y/n]: " % message) return not response or response.lower() == "y" + def can_open_url(self): + try: + webbrowser.get() + return True + except webbrowser.Error, e: + return False + def open_url(self, url): + if not self.can_open_url(): + _log.warn("Failed to open %s" % url) webbrowser.open(url) diff --git a/WebKitTools/Scripts/webkitpy/user_unittest.py b/WebKitTools/Scripts/webkitpy/common/system/user_unittest.py index 34d9983..dadead3 100644 --- a/WebKitTools/Scripts/webkitpy/user_unittest.py +++ b/WebKitTools/Scripts/webkitpy/common/system/user_unittest.py @@ -27,7 +27,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from webkitpy.user import User + +from webkitpy.common.system.user import User class UserTest(unittest.TestCase): diff --git a/WebKitTools/Scripts/webkitpy/common/thread/__init__.py b/WebKitTools/Scripts/webkitpy/common/thread/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/thread/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/common/thread/messagepump.py b/WebKitTools/Scripts/webkitpy/common/thread/messagepump.py new file mode 100644 index 0000000..0e39285 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/thread/messagepump.py @@ -0,0 +1,59 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class MessagePumpDelegate(object): + def schedule(self, interval, callback): + raise NotImplementedError, "subclasses must implement" + + def message_available(self, message): + raise NotImplementedError, "subclasses must implement" + + def final_message_delivered(self): + raise NotImplementedError, "subclasses must implement" + + +class MessagePump(object): + interval = 10 # seconds + + def __init__(self, delegate, message_queue): + self._delegate = delegate + self._message_queue = message_queue + self._schedule() + + def _schedule(self): + self._delegate.schedule(self.interval, self._callback) + + def _callback(self): + (messages, is_running) = self._message_queue.take_all() + for message in messages: + self._delegate.message_available(message) + if not is_running: + self._delegate.final_message_delivered() + return + self._schedule() diff --git a/WebKitTools/Scripts/webkitpy/common/thread/messagepump_unittest.py b/WebKitTools/Scripts/webkitpy/common/thread/messagepump_unittest.py new file mode 100644 index 0000000..f731db2 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/thread/messagepump_unittest.py @@ -0,0 +1,83 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.thread.messagepump import MessagePump, MessagePumpDelegate +from webkitpy.common.thread.threadedmessagequeue import ThreadedMessageQueue + + +class TestDelegate(MessagePumpDelegate): + def __init__(self): + self.log = [] + + def schedule(self, interval, callback): + self.callback = callback + self.log.append("schedule") + + def message_available(self, message): + self.log.append("message_available: %s" % message) + + def final_message_delivered(self): + self.log.append("final_message_delivered") + + +class MessagePumpTest(unittest.TestCase): + + def test_basic(self): + queue = ThreadedMessageQueue() + delegate = TestDelegate() + pump = MessagePump(delegate, queue) + self.assertEqual(delegate.log, [ + 'schedule' + ]) + delegate.callback() + queue.post("Hello") + queue.post("There") + delegate.callback() + self.assertEqual(delegate.log, [ + 'schedule', + 'schedule', + 'message_available: Hello', + 'message_available: There', + 'schedule' + ]) + queue.post("More") + queue.post("Messages") + queue.stop() + delegate.callback() + self.assertEqual(delegate.log, [ + 'schedule', + 'schedule', + 'message_available: Hello', + 'message_available: There', + 'schedule', + 'message_available: More', + 'message_available: Messages', + 'final_message_delivered' + ]) diff --git a/WebKitTools/Scripts/run-chromium-webkit-tests b/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue.py index 221b5aa..17b6277 100755..100644 --- a/WebKitTools/Scripts/run-chromium-webkit-tests +++ b/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. +# Copyright (c) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -27,15 +26,29 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Wrapper around webkitpy/layout_tests/run-chromium-webkit-tests.py""" -import os -import sys +from __future__ import with_statement -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), - "webkitpy", "layout_tests")) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) -import run_chromium_webkit_tests +import threading + + +class ThreadedMessageQueue(object): + def __init__(self): + self._messages = [] + self._is_running = True + self._lock = threading.Lock() + + def post(self, message): + with self._lock: + self._messages.append(message) + + def stop(self): + with self._lock: + self._is_running = False + + def take_all(self): + with self._lock: + messages = self._messages + is_running = self._is_running + self._messages = [] + return (messages, is_running) -if __name__ == '__main__': - options, args = run_chromium_webkit_tests.parse_args() - run_chromium_webkit_tests.main(options, args) diff --git a/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue_unittest.py b/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue_unittest.py new file mode 100644 index 0000000..cb67c1e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/common/thread/threadedmessagequeue_unittest.py @@ -0,0 +1,53 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.thread.threadedmessagequeue import ThreadedMessageQueue + +class ThreadedMessageQueueTest(unittest.TestCase): + + def test_basic(self): + queue = ThreadedMessageQueue() + queue.post("Hello") + queue.post("There") + (messages, is_running) = queue.take_all() + self.assertEqual(messages, ["Hello", "There"]) + self.assertTrue(is_running) + (messages, is_running) = queue.take_all() + self.assertEqual(messages, []) + self.assertTrue(is_running) + queue.post("More") + queue.stop() + queue.post("Messages") + (messages, is_running) = queue.take_all() + self.assertEqual(messages, ["More", "Messages"]) + self.assertFalse(is_running) + (messages, is_running) = queue.take_all() + self.assertEqual(messages, []) + self.assertFalse(is_running) diff --git a/WebKitTools/Scripts/webkitpy/credentials.pyc b/WebKitTools/Scripts/webkitpy/credentials.pyc Binary files differdeleted file mode 100644 index cd42568..0000000 --- a/WebKitTools/Scripts/webkitpy/credentials.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/executive.py b/WebKitTools/Scripts/webkitpy/executive.py deleted file mode 100644 index 50b119b..0000000 --- a/WebKitTools/Scripts/webkitpy/executive.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. -# Copyright (c) 2009 Apple Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import StringIO -import subprocess -import sys - -from webkitpy.webkit_logging import tee - - -class ScriptError(Exception): - - def __init__(self, - message=None, - script_args=None, - exit_code=None, - output=None, - cwd=None): - if not message: - message = 'Failed to run "%s"' % script_args - if exit_code: - message += " exit_code: %d" % exit_code - if cwd: - message += " cwd: %s" % cwd - - Exception.__init__(self, message) - self.script_args = script_args # 'args' is already used by Exception - self.exit_code = exit_code - self.output = output - self.cwd = cwd - - def message_with_output(self, output_limit=500): - if self.output: - if output_limit and len(self.output) > output_limit: - return "%s\nLast %s characters of output:\n%s" % \ - (self, output_limit, self.output[-output_limit:]) - return "%s\n%s" % (self, self.output) - return str(self) - - def command_name(self): - command_path = self.script_args - if type(command_path) is list: - command_path = command_path[0] - return os.path.basename(command_path) - - -def run_command(*args, **kwargs): - # FIXME: This should not be a global static. - # New code should use Executive.run_command directly instead - return Executive().run_command(*args, **kwargs) - - -class Executive(object): - - def _run_command_with_teed_output(self, args, teed_output): - child_process = subprocess.Popen(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - # Use our own custom wait loop because Popen ignores a tee'd - # stderr/stdout. - # FIXME: This could be improved not to flatten output to stdout. - while True: - output_line = child_process.stdout.readline() - if output_line == "" and child_process.poll() != None: - return child_process.poll() - teed_output.write(output_line) - - def run_and_throw_if_fail(self, args, quiet=False): - # Cache the child's output locally so it can be used for error reports. - child_out_file = StringIO.StringIO() - if quiet: - dev_null = open(os.devnull, "w") - child_stdout = tee(child_out_file, dev_null if quiet else sys.stdout) - exit_code = self._run_command_with_teed_output(args, child_stdout) - if quiet: - dev_null.close() - - child_output = child_out_file.getvalue() - child_out_file.close() - - if exit_code: - raise ScriptError(script_args=args, - exit_code=exit_code, - output=child_output) - - @staticmethod - def cpu_count(): - # This API exists only in Python 2.6 and higher. :( - try: - import multiprocessing - return multiprocessing.cpu_count() - except (ImportError, NotImplementedError): - # This quantity is a lie but probably a reasonable guess for modern - # machines. - return 2 - - # Error handlers do not need to be static methods once all callers are - # updated to use an Executive object. - - @staticmethod - def default_error_handler(error): - raise error - - @staticmethod - def ignore_error(error): - pass - - # FIXME: This should be merged with run_and_throw_if_fail - - def run_command(self, - args, - cwd=None, - input=None, - error_handler=None, - return_exit_code=False, - return_stderr=True): - if hasattr(input, 'read'): # Check if the input is a file. - stdin = input - string_to_communicate = None - else: - stdin = subprocess.PIPE if input else None - string_to_communicate = input - if return_stderr: - stderr = subprocess.STDOUT - else: - stderr = None - - process = subprocess.Popen(args, - stdin=stdin, - stdout=subprocess.PIPE, - stderr=stderr, - cwd=cwd) - output = process.communicate(string_to_communicate)[0] - exit_code = process.wait() - if exit_code: - script_error = ScriptError(script_args=args, - exit_code=exit_code, - output=output, - cwd=cwd) - (error_handler or self.default_error_handler)(script_error) - if return_exit_code: - return exit_code - return output diff --git a/WebKitTools/Scripts/webkitpy/executive.pyc b/WebKitTools/Scripts/webkitpy/executive.pyc Binary files differdeleted file mode 100644 index 190fabb..0000000 --- a/WebKitTools/Scripts/webkitpy/executive.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/grammar.pyc b/WebKitTools/Scripts/webkitpy/grammar.pyc Binary files differdeleted file mode 100644 index 50edeeb..0000000 --- a/WebKitTools/Scripts/webkitpy/grammar.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/__init__.py b/WebKitTools/Scripts/webkitpy/layout_tests/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-bg.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-bg.html new file mode 100644 index 0000000..2022676 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-bg.html @@ -0,0 +1,22 @@ +<html> + <head> + <style> + div { background: -webkit-canvas(squares); width:600px; height:600px; border:2px solid black } + </style> + + <script type="application/x-javascript"> +function draw(w, h) { + var ctx = document.getCSSCanvasContext("2d", "squares", w, h); + + ctx.fillStyle = "rgb(200,0,0)"; + ctx.fillRect (10, 10, 55, 50); + + ctx.fillStyle = "rgba(0, 0, 200, 0.5)"; + ctx.fillRect (30, 30, 55, 50); +} + </script> + </head> + <body onload="draw(300, 300)"> + <div></div> + </body> +</html>
\ No newline at end of file diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.checksum new file mode 100644 index 0000000..7373fe2 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.checksum @@ -0,0 +1 @@ +afa0f2d246120c180005d67d47636b92
\ No newline at end of file diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.png Binary files differnew file mode 100644 index 0000000..44952b4 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.txt new file mode 100644 index 0000000..288458d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.txt @@ -0,0 +1,22 @@ +layer at (0,0) size 800x600 + RenderView at (0,0) size 800x600 +layer at (0,0) size 800x600 + RenderBlock {HTML} at (0,0) size 800x600 + RenderBody {BODY} at (8,8) size 784x584 + RenderBlock {P} at (0,0) size 784x18 + RenderText {#text} at (0,0) size 624x18 + text run at (0,0) width 624: "These should be four green hollow boxes with dimensions 600x300, 100x300, 600x100, 100x100." + RenderBlock (anonymous) at (0,34) size 784x420 + RenderHTMLCanvas {CANVAS} at (0,0) size 606x306 [border: (3px solid #008000)] + RenderText {#text} at (606,292) size 4x18 + text run at (606,292) width 4: " " + RenderText {#text} at (0,0) size 0x0 + RenderHTMLCanvas {CANVAS} at (610,0) size 106x306 [border: (3px solid #008000)] + RenderText {#text} at (0,0) size 0x0 + RenderText {#text} at (0,0) size 0x0 + RenderHTMLCanvas {CANVAS} at (0,310) size 606x106 [border: (3px solid #008000)] + RenderText {#text} at (606,402) size 4x18 + text run at (606,402) width 4: " " + RenderText {#text} at (0,0) size 0x0 + RenderHTMLCanvas {CANVAS} at (610,310) size 106x106 [border: (3px solid #008000)] + RenderText {#text} at (0,0) size 0x0 diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom.html new file mode 100644 index 0000000..4dabce1 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom.html @@ -0,0 +1,15 @@ +<style> + canvas { border: solid green; + zoom: 2; } +</style> +<p> + These should be four green hollow boxes with dimensions 600x300, 100x300, 600x100, 100x100. +</p> +<!-- 300x150 --> +<canvas id="canvas"></canvas> +<!-- 50x150 --> +<canvas id="canvas" width="50"></canvas> +<!-- 300x50 --> +<canvas id="canvas" height="50"></canvas> +<!-- 50x50 --> +<canvas id="canvas" width="50" height="50"></canvas> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash-expected.txt new file mode 100644 index 0000000..521c3f5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash-expected.txt @@ -0,0 +1 @@ +This test is expected to crash. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash.html new file mode 100644 index 0000000..b9820d6 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash.html @@ -0,0 +1,5 @@ +<html> +<body> +This test is expected to crash. +</body> +</html> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/missing-expectation.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/missing-expectation.html new file mode 100644 index 0000000..1f00b50 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/missing-expectation.html @@ -0,0 +1,5 @@ +<html> +<body> +This test intentionally doesn't have an expected result checked in. +</body> +</html> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing-expected.txt new file mode 100644 index 0000000..26bd316 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing-expected.txt @@ -0,0 +1 @@ +This test is expected to pass. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing.html new file mode 100644 index 0000000..db7e3de --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing.html @@ -0,0 +1,5 @@ +<html> +<body> +This test is expected to pass. +</body> +</html> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.checksum new file mode 100644 index 0000000..4cd8dac --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.checksum @@ -0,0 +1 @@ +790b681a41697634fcf2a2587afb89c6
\ No newline at end of file diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.png Binary files differnew file mode 100644 index 0000000..3d00450 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.txt new file mode 100644 index 0000000..2411c0a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.txt @@ -0,0 +1,6 @@ +layer at (0,0) size 785x620 + RenderView at (0,0) size 785x600 +layer at (0,0) size 785x620 + RenderBlock {HTML} at (0,0) size 785x620 + RenderBody {BODY} at (8,8) size 769x604 + RenderBlock {DIV} at (0,0) size 604x604 [border: (2px solid #000000)] diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt new file mode 100644 index 0000000..b78a01c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt @@ -0,0 +1 @@ +WONTFIX : misc/missing-expectation.html = MISSING PASS diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element-expected.txt new file mode 100644 index 0000000..f60ac38 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element-expected.txt @@ -0,0 +1,20 @@ +Various tests for the article element. + +On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE". + + +<article> closes <p>: +PASS article1.parentNode.nodeName == "p" is false +<p> does not close <article>: +PASS p1.parentNode.nodeName is "ARTICLE" +<article> can be nested inside <article>: +PASS article3.parentNode.id is "article2" +Residual style: +PASS getWeight("article4") is "bold" +PASS getWeight("span1") is "bold" +FormatBlock: +PASS document.getElementById("span2").parentNode.nodeName is "ARTICLE" +PASS successfullyParsed is true + +TEST COMPLETE + diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element.html new file mode 100644 index 0000000..c0f4547 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element.html @@ -0,0 +1,13 @@ +<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN"> +<html> +<head> +<link rel="stylesheet" href="../../fast/js/resources/js-test-style.css"> +<script src="../../fast/js/resources/js-test-pre.js"></script> +</head> +<body> +<p id="description"></p> +<div id="console"></div> +<script src="script-tests/article-element.js"></script> +<script src="../../fast/js/resources/js-test-post.js"></script> +</body> +</html> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py b/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py index 6e4ba99..633dfe8 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py @@ -42,7 +42,8 @@ import port def run_tests(port, options, tests): # |image_path| is a path to the image capture from the driver. image_path = 'image_result.png' - driver = port.start_driver(image_path, None) + driver = port.create_driver(image_path, None) + driver.start() for t in tests: uri = port.filename_to_uri(os.path.join(port.layout_tests_dir(), t)) print "uri: " + uri @@ -58,20 +59,23 @@ def run_tests(port, options, tests): print ''.join(err) print '"""' print + driver.stop() if __name__ == '__main__': - optparser = optparse.OptionParser() - optparser.add_option('-p', '--platform', action='store', default='mac', - help='Platform to test (e.g., "mac", "chromium-mac", etc.') - optparser.add_option('-t', '--target', action='store', default='Release', - help='build type ("Debug" or "Release")') - optparser.add_option('', '--timeout', action='store', default='2000', - help='test timeout in milliseconds (2000 by default)') - optparser.add_option('', '--wrapper', action='store') - optparser.add_option('', '--no-pixel-tests', action='store_true', - default=False, - help='disable pixel-to-pixel PNG comparisons') + # FIXME: configuration_options belong in a shared location. + configuration_options = [ + optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), + optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), + ] + misc_options = [ + optparse.make_option('-p', '--platform', action='store', default='mac', help='Platform to test (e.g., "mac", "chromium-mac", etc.'), + optparse.make_option('--timeout', action='store', default='2000', help='test timeout in milliseconds (2000 by default)'), + optparse.make_option('--wrapper', action='store'), + optparse.make_option('--no-pixel-tests', action='store_true', default=False, help='disable pixel-to-pixel PNG comparisons'), + ] + option_list = configuration_options + misc_options + optparser = optparse.OptionParser(option_list=option_list) options, args = optparser.parse_args() p = port.get(options.platform, options) run_tests(p, options, args) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_shell_thread.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py index 3452035..6364511 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_shell_thread.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py @@ -27,14 +27,17 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""A Thread object for running the test shell and processing URLs from a +"""A Thread object for running DumpRenderTree and processing URLs from a shared queue. -Each thread runs a separate instance of the test_shell binary and validates +Each thread runs a separate instance of the DumpRenderTree binary and validates the output. When there are no more URLs to process in the shared queue, the thread exits. """ +from __future__ import with_statement + +import codecs import copy import logging import os @@ -47,23 +50,26 @@ import time import test_failures +_log = logging.getLogger("webkitpy.layout_tests.layout_package." + "dump_render_tree_thread") + -def process_output(port, test_info, test_types, test_args, target, output_dir, - crash, timeout, test_run_time, actual_checksum, +def process_output(port, test_info, test_types, test_args, configuration, + output_dir, crash, timeout, test_run_time, actual_checksum, output, error): - """Receives the output from a test_shell process, subjects it to a number - of tests, and returns a list of failure types the test produced. + """Receives the output from a DumpRenderTree process, subjects it to a + number of tests, and returns a list of failure types the test produced. Args: port: port-specific hooks - proc: an active test_shell process + proc: an active DumpRenderTree process test_info: Object containing the test filename, uri and timeout test_types: list of test types to subject the output to test_args: arguments to be passed to each test - target: Debug or Release + configuration: Debug or Release output_dir: directory to put crash stack traces into - Returns: a list of failure objects and times for the test being processed + Returns: a TestResult object """ failures = [] @@ -79,16 +85,17 @@ def process_output(port, test_info, test_types, test_args, target, output_dir, failures.append(test_failures.FailureTimeout()) if crash: - logging.debug("Stacktrace for %s:\n%s" % (test_info.filename, error)) + _log.debug("Stacktrace for %s:\n%s" % (test_info.filename, error)) # Strip off "file://" since RelativeTestFilename expects # filesystem paths. - filename = os.path.join(output_dir, test_info.filename) + filename = os.path.join(output_dir, port.relative_test_filename( + test_info.filename)) filename = os.path.splitext(filename)[0] + "-stack.txt" port.maybe_make_directory(os.path.split(filename)[0]) - open(filename, "wb").write(error) + with codecs.open(filename, "wb", "utf-8") as file: + file.write(error) elif error: - logging.debug("Previous test output extra lines after dump:\n%s" % - error) + _log.debug("Previous test output stderr lines:\n%s" % error) # Check the output and save the results. start_time = time.time() @@ -97,7 +104,7 @@ def process_output(port, test_info, test_types, test_args, target, output_dir, start_diff_time = time.time() new_failures = test_type.compare_output(port, test_info.filename, output, local_test_args, - target) + configuration) # Don't add any more failures if we already have a crash, so we don't # double-report those tests. We do double-report for timeouts since # we still want to see the text and image output. @@ -107,26 +114,27 @@ def process_output(port, test_info, test_types, test_args, target, output_dir, time.time() - start_diff_time) total_time_for_all_diffs = time.time() - start_diff_time - return TestStats(test_info.filename, failures, test_run_time, - total_time_for_all_diffs, time_for_diffs) + return TestResult(test_info.filename, failures, test_run_time, + total_time_for_all_diffs, time_for_diffs) -class TestStats: +class TestResult(object): def __init__(self, filename, failures, test_run_time, total_time_for_all_diffs, time_for_diffs): - self.filename = filename self.failures = failures + self.filename = filename self.test_run_time = test_run_time - self.total_time_for_all_diffs = total_time_for_all_diffs self.time_for_diffs = time_for_diffs + self.total_time_for_all_diffs = total_time_for_all_diffs + self.type = test_failures.determine_result_type(failures) class SingleTestThread(threading.Thread): """Thread wrapper for running a single test file.""" def __init__(self, port, image_path, shell_args, test_info, - test_types, test_args, target, output_dir): + test_types, test_args, configuration, output_dir): """ Args: port: object implementing port-specific hooks @@ -142,32 +150,33 @@ class SingleTestThread(threading.Thread): self._test_info = test_info self._test_types = test_types self._test_args = test_args - self._target = target + self._configuration = configuration self._output_dir = output_dir def run(self): - driver = self._port.start_test_driver(self._image_path, - self._shell_args) + test_info = self._test_info + driver = self._port.create_driver(self._image_path, self._shell_args) + driver.start() start = time.time() crash, timeout, actual_checksum, output, error = \ driver.run_test(test_info.uri.strip(), test_info.timeout, - test_info.image_hash) + test_info.image_hash()) end = time.time() - self._test_stats = process_output(self._port, - self._test_info, self._test_types, self._test_args, - self._target, self._output_dir, crash, timeout, end - start, + self._test_result = process_output(self._port, + test_info, self._test_types, self._test_args, + self._configuration, self._output_dir, crash, timeout, end - start, actual_checksum, output, error) driver.stop() - def get_test_stats(self): - return self._test_stats + def get_test_result(self): + return self._test_result class TestShellThread(threading.Thread): def __init__(self, port, filename_list_queue, result_queue, test_types, test_args, image_path, shell_args, options): - """Initialize all the local state for this test shell thread. + """Initialize all the local state for this DumpRenderTree thread. Args: port: interface to port-specific hooks @@ -178,7 +187,7 @@ class TestShellThread(threading.Thread): test_types: A list of TestType objects to run the test output against. test_args: A TestArguments object to pass to each TestType. - shell_args: Any extra arguments to be passed to test_shell.exe. + shell_args: Any extra arguments to be passed to DumpRenderTree. options: A property dictionary as produced by optparse. The command-line options should match those expected by run_webkit_tests; they are typically passed via the @@ -197,7 +206,7 @@ class TestShellThread(threading.Thread): self._canceled = False self._exception_info = None self._directory_timing_stats = {} - self._test_stats = [] + self._test_results = [] self._num_tests = 0 self._start_time = 0 self._stop_time = 0 @@ -214,10 +223,13 @@ class TestShellThread(threading.Thread): (number of tests in that directory, time to run the tests)""" return self._directory_timing_stats - def get_individual_test_stats(self): - """Returns a list of (test_filename, time_to_run_test, - total_time_for_all_diffs, time_for_diffs) tuples.""" - return self._test_stats + def get_test_results(self): + """Return the list of all tests run on this thread. + + This is used to calculate per-thread statistics. + + """ + return self._test_results def cancel(self): """Set a flag telling this thread to quit.""" @@ -242,17 +254,20 @@ class TestShellThread(threading.Thread): self._start_time = time.time() self._num_tests = 0 try: - logging.debug('%s starting' % (self.getName())) + _log.debug('%s starting' % (self.getName())) self._run(test_runner=None, result_summary=None) - logging.debug('%s done (%d tests)' % (self.getName(), - self.get_num_tests())) + _log.debug('%s done (%d tests)' % (self.getName(), + self.get_num_tests())) + except KeyboardInterrupt: + self._exception_info = sys.exc_info() + _log.debug("%s interrupted" % self.getName()) except: # Save the exception for our caller to see. self._exception_info = sys.exc_info() self._stop_time = time.time() # Re-raise it and die. - logging.error('%s dying: %s' % (self.getName(), - self._exception_info)) + _log.error('%s dying: %s' % (self.getName(), + self._exception_info)) raise self._stop_time = time.time() @@ -275,18 +290,18 @@ class TestShellThread(threading.Thread): try: batch_size = int(self._options.batch_size) except: - logging.info("Ignoring invalid batch size '%s'" % - self._options.batch_size) + _log.info("Ignoring invalid batch size '%s'" % + self._options.batch_size) # Append tests we're running to the existing tests_run.txt file. # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput. tests_run_filename = os.path.join(self._options.results_directory, "tests_run.txt") - tests_run_file = open(tests_run_filename, "a") + tests_run_file = codecs.open(tests_run_filename, "a", "utf-8") while True: if self._canceled: - logging.info('Testing canceled') + _log.debug('Testing cancelled') tests_run_file.close() return @@ -300,7 +315,7 @@ class TestShellThread(threading.Thread): self._current_dir, self._filename_list = \ self._filename_list_queue.get_nowait() except Queue.Empty: - self._kill_test_shell() + self._kill_dump_render_tree() tests_run_file.close() return @@ -313,31 +328,33 @@ class TestShellThread(threading.Thread): batch_count += 1 self._num_tests += 1 if self._options.run_singly: - failures = self._run_test_singly(test_info) + result = self._run_test_singly(test_info) else: - failures = self._run_test(test_info) + result = self._run_test(test_info) filename = test_info.filename tests_run_file.write(filename + "\n") - if failures: - # Check and kill test shell if we need too. - if len([1 for f in failures if f.should_kill_test_shell()]): - self._kill_test_shell() + if result.failures: + # Check and kill DumpRenderTree if we need to. + if len([1 for f in result.failures + if f.should_kill_dump_render_tree()]): + self._kill_dump_render_tree() # Reset the batch count since the shell just bounced. batch_count = 0 # Print the error message(s). - error_str = '\n'.join([' ' + f.message() for f in failures]) - logging.debug("%s %s failed:\n%s" % (self.getName(), - self._port.relative_test_filename(filename), - error_str)) + error_str = '\n'.join([' ' + f.message() for + f in result.failures]) + _log.debug("%s %s failed:\n%s" % (self.getName(), + self._port.relative_test_filename(filename), + error_str)) else: - logging.debug("%s %s passed" % (self.getName(), - self._port.relative_test_filename(filename))) - self._result_queue.put((filename, failures)) + _log.debug("%s %s passed" % (self.getName(), + self._port.relative_test_filename(filename))) + self._result_queue.put(result) if batch_size > 0 and batch_count > batch_size: # Bounce the shell and reset count. - self._kill_test_shell() + self._kill_dump_render_tree() batch_count = 0 if test_runner: @@ -353,88 +370,96 @@ class TestShellThread(threading.Thread): Args: test_info: Object containing the test filename, uri and timeout - Return: - A list of TestFailure objects describing the error. + Returns: + A TestResult + """ worker = SingleTestThread(self._port, self._image_path, self._shell_args, test_info, self._test_types, self._test_args, - self._options.target, + self._options.configuration, self._options.results_directory) worker.start() - # When we're running one test per test_shell process, we can enforce - # a hard timeout. the test_shell watchdog uses 2.5x the timeout - # We want to be larger than that. + # When we're running one test per DumpRenderTree process, we can + # enforce a hard timeout. The DumpRenderTree watchdog uses 2.5x + # the timeout; we want to be larger than that. worker.join(int(test_info.timeout) * 3.0 / 1000.0) if worker.isAlive(): # If join() returned with the thread still running, the - # test_shell.exe is completely hung and there's nothing + # DumpRenderTree is completely hung and there's nothing # more we can do with it. We have to kill all the - # test_shells to free it up. If we're running more than - # one test_shell thread, we'll end up killing the other - # test_shells too, introducing spurious crashes. We accept that - # tradeoff in order to avoid losing the rest of this thread's - # results. - logging.error('Test thread hung: killing all test_shells') + # DumpRenderTrees to free it up. If we're running more than + # one DumpRenderTree thread, we'll end up killing the other + # DumpRenderTrees too, introducing spurious crashes. We accept + # that tradeoff in order to avoid losing the rest of this + # thread's results. + _log.error('Test thread hung: killing all DumpRenderTrees') worker._driver.stop() try: - stats = worker.get_test_stats() - self._test_stats.append(stats) - failures = stats.failures + result = worker.get_test_result() except AttributeError, e: failures = [] - logging.error('Cannot get results of test: %s' % - test_info.filename) + _log.error('Cannot get results of test: %s' % + test_info.filename) + result = TestResult(test_info.filename, failures=[], + test_run_time=0, total_time_for_all_diffs=0, + time_for_diffs=0) - return failures + return result def _run_test(self, test_info): - """Run a single test file using a shared test_shell process. + """Run a single test file using a shared DumpRenderTree process. Args: test_info: Object containing the test filename, uri and timeout - Return: + Returns: A list of TestFailure objects describing the error. + """ - self._ensure_test_shell_is_running() + self._ensure_dump_render_tree_is_running() # The pixel_hash is used to avoid doing an image dump if the # checksums match, so it should be set to a blank value if we # are generating a new baseline. (Otherwise, an image from a # previous run will be copied into the baseline.) - image_hash = test_info.image_hash - if image_hash and self._test_args.new_baseline: + image_hash = test_info.image_hash() + if (image_hash and + (self._test_args.new_baseline or self._test_args.reset_results or + not self._options.pixel_tests)): image_hash = "" start = time.time() crash, timeout, actual_checksum, output, error = \ self._driver.run_test(test_info.uri, test_info.timeout, image_hash) end = time.time() - stats = process_output(self._port, test_info, self._test_types, - self._test_args, self._options.target, - self._options.results_directory, crash, - timeout, end - start, actual_checksum, - output, error) + result = process_output(self._port, test_info, self._test_types, + self._test_args, self._options.configuration, + self._options.results_directory, crash, + timeout, end - start, actual_checksum, + output, error) + self._test_results.append(result) + return result + + def _ensure_dump_render_tree_is_running(self): + """Start the shared DumpRenderTree, if it's not running. - self._test_stats.append(stats) - return stats.failures + This is not for use when running tests singly, since those each start + a separate DumpRenderTree in their own thread. - def _ensure_test_shell_is_running(self): - """Start the shared test shell, if it's not running. Not for use when - running tests singly, since those each start a separate test shell in - their own thread. """ + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 if (not self._driver or self._driver.poll() is not None): - self._driver = self._port.start_driver( - self._image_path, self._shell_args) + self._driver = self._port.create_driver(self._image_path, self._shell_args) + self._driver.start() - def _kill_test_shell(self): - """Kill the test shell process if it's running.""" + def _kill_dump_render_tree(self): + """Kill the DumpRenderTree process if it's running.""" if self._driver: self._driver.stop() self._driver = None diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py index 520ab1f..c0525ea 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -29,14 +28,14 @@ import logging import os -import simplejson -from layout_package import json_results_generator -from layout_package import test_expectations -from layout_package import test_failures +from webkitpy.layout_tests.layout_package import json_results_generator +from webkitpy.layout_tests.layout_package import test_expectations +from webkitpy.layout_tests.layout_package import test_failures +import webkitpy.thirdparty.simplejson as simplejson -class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator): +class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase): """A JSON results generator for layout tests.""" LAYOUT_TESTS_PATH = "LayoutTests" @@ -45,6 +44,16 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator): WONTFIX = "wontfixCounts" DEFERRED = "deferredCounts" + # Note that we omit test_expectations.FAIL from this list because + # it should never show up (it's a legacy input expectation, never + # an output expectation). + FAILURE_TO_CHAR = {test_expectations.CRASH: "C", + test_expectations.TIMEOUT: "T", + test_expectations.IMAGE: "I", + test_expectations.TEXT: "F", + test_expectations.MISSING: "O", + test_expectations.IMAGE_PLUS_TEXT: "Z"} + def __init__(self, port, builder_name, build_name, build_number, results_file_base_path, builder_base_url, test_timings, expectations, result_summary, all_tests): @@ -54,20 +63,14 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator): Args: result_summary: ResultsSummary object storing the summary of the test results. - (see the comment of JSONResultsGenerator.__init__ for other Args) """ + super(JSONLayoutResultsGenerator, self).__init__( + builder_name, build_name, build_number, results_file_base_path, + builder_base_url, {}, port.test_repository_paths()) + self._port = port - self._builder_name = builder_name - self._build_name = build_name - self._build_number = build_number - self._builder_base_url = builder_base_url - self._results_file_path = os.path.join(results_file_base_path, - self.RESULTS_FILENAME) self._expectations = expectations - # We don't use self._skipped_tests and self._passed_tests as we - # override _InsertFailureSummaries. - # We want relative paths to LayoutTest root for JSON output. path_to_name = self._get_path_relative_to_layout_test_root self._result_summary = result_summary @@ -79,7 +82,7 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator): (path_to_name(test_tuple.filename), test_tuple.test_run_time) for test_tuple in test_timings) - self._generate_json_output() + self.generate_json_output() def _get_path_relative_to_layout_test_root(self, test): """Returns the path of the test relative to the layout test root. @@ -102,6 +105,27 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator): return relativePath.replace('\\', '/') # override + def _get_test_timing(self, test_name): + if test_name in self._test_timings: + # Floor for now to get time in seconds. + return int(self._test_timings[test_name]) + return 0 + + # override + def _get_failed_test_names(self): + return set(self._failures.keys()) + + # override + def _get_result_type_char(self, test_name): + if test_name not in self._all_tests: + return self.NO_DATA_RESULT + + if test_name in self._failures: + return self.FAILURE_TO_CHAR[self._failures[test_name]] + + return self.PASS_RESULT + + # override def _convert_json_to_current_version(self, results_json): archive_version = None if self.VERSION_KEY in results_json: diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py index 84be0e1..595fc2b 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -27,19 +26,39 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import with_statement + +import codecs import logging import os -import simplejson import subprocess import sys import time import urllib2 import xml.dom.minidom -from layout_package import test_expectations +import webkitpy.thirdparty.simplejson as simplejson + +# A JSON results generator for generic tests. +# FIXME: move this code out of the layout_package directory. + +_log = logging.getLogger("webkitpy.layout_tests.layout_package.json_results_generator") + +class TestResult(object): + """A simple class that represents a single test result.""" + def __init__(self, name, failed=False, skipped=False, elapsed_time=0): + self.name = name + self.failed = failed + self.skipped = skipped + self.time = elapsed_time -class JSONResultsGenerator(object): + def fixable(self): + return self.failed or self.skipped + + +class JSONResultsGeneratorBase(object): + """A JSON results generator for generic tests.""" MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 # Min time (seconds) that will be added to the JSON. @@ -48,14 +67,13 @@ class JSONResultsGenerator(object): JSON_SUFFIX = ");" PASS_RESULT = "P" SKIP_RESULT = "X" + FAIL_RESULT = "F" NO_DATA_RESULT = "N" VERSION = 3 VERSION_KEY = "version" RESULTS = "results" TIMES = "times" BUILD_NUMBERS = "buildNumbers" - WEBKIT_SVN = "webkitRevision" - CHROME_SVN = "chromeRevision" TIME = "secondsSinceEpoch" TESTS = "tests" @@ -63,22 +81,11 @@ class JSONResultsGenerator(object): FIXABLE = "fixableCounts" ALL_FIXABLE_COUNT = "allFixableCount" - # Note that we omit test_expectations.FAIL from this list because - # it should never show up (it's a legacy input expectation, never - # an output expectation). - FAILURE_TO_CHAR = {test_expectations.CRASH: "C", - test_expectations.TIMEOUT: "T", - test_expectations.IMAGE: "I", - test_expectations.TEXT: "F", - test_expectations.MISSING: "O", - test_expectations.IMAGE_PLUS_TEXT: "Z"} - FAILURE_CHARS = FAILURE_TO_CHAR.values() - RESULTS_FILENAME = "results.json" - def __init__(self, port, builder_name, build_name, build_number, + def __init__(self, builder_name, build_name, build_number, results_file_base_path, builder_base_url, - test_timings, failures, passed_tests, skipped_tests, all_tests): + test_results_map, svn_repositories=None): """Modifies the results.json file. Grabs it off the archive directory if it is not found locally. @@ -89,36 +96,112 @@ class JSONResultsGenerator(object): results_file_base_path: Absolute path to the directory containing the results json file. builder_base_url: the URL where we have the archived test results. - test_timings: Map of test name to a test_run-time. - failures: Map of test name to a failure type (of test_expectations). - passed_tests: A set containing all the passed tests. - skipped_tests: A set containing all the skipped tests. - all_tests: List of all the tests that were run. This should not - include skipped tests. + If this is None no archived results will be retrieved. + test_results_map: A dictionary that maps test_name to TestResult. + svn_repositories: A (json_field_name, svn_path) pair for SVN + repositories that tests rely on. The SVN revision will be + included in the JSON with the given json_field_name. """ - self._port = port self._builder_name = builder_name self._build_name = build_name self._build_number = build_number self._builder_base_url = builder_base_url self._results_file_path = os.path.join(results_file_base_path, self.RESULTS_FILENAME) - self._test_timings = test_timings - self._failures = failures - self._passed_tests = passed_tests - self._skipped_tests = skipped_tests - self._all_tests = all_tests - self._generate_json_output() + self._test_results_map = test_results_map + self._test_results = test_results_map.values() + + self._svn_repositories = svn_repositories + if not self._svn_repositories: + self._svn_repositories = {} - def _generate_json_output(self): + self._json = None + + def generate_json_output(self): """Generates the JSON output file.""" - json = self._get_json() - if json: - results_file = open(self._results_file_path, "w") - results_file.write(json) + if not self._json: + self._json = self.get_json() + if self._json: + # Specify separators in order to get compact encoding. + json_data = simplejson.dumps(self._json, separators=(',', ':')) + json_string = self.JSON_PREFIX + json_data + self.JSON_SUFFIX + + results_file = codecs.open(self._results_file_path, "w", "utf-8") + results_file.write(json_string) results_file.close() + def get_json(self): + """Gets the results for the results.json file.""" + if self._json: + return self._json + + results_json, error = self._get_archived_json_results() + if error: + # If there was an error don't write a results.json + # file at all as it would lose all the information on the bot. + _log.error("Archive directory is inaccessible. Not modifying " + "or clobbering the results.json file: " + str(error)) + return None + + builder_name = self._builder_name + if results_json and builder_name not in results_json: + _log.debug("Builder name (%s) is not in the results.json file." + % builder_name) + + self._convert_json_to_current_version(results_json) + + if builder_name not in results_json: + results_json[builder_name] = ( + self._create_results_for_builder_json()) + + results_for_builder = results_json[builder_name] + + self._insert_generic_metadata(results_for_builder) + + self._insert_failure_summaries(results_for_builder) + + # Update the all failing tests with result type and time. + tests = results_for_builder[self.TESTS] + all_failing_tests = self._get_failed_test_names() + all_failing_tests.update(tests.iterkeys()) + for test in all_failing_tests: + self._insert_test_time_and_result(test, tests) + + self._json = results_json + return self._json + + def _get_test_timing(self, test_name): + """Returns test timing data (elapsed time) in second + for the given test_name.""" + if test_name in self._test_results_map: + # Floor for now to get time in seconds. + return int(self._test_results_map[test_name].time) + return 0 + + def _get_failed_test_names(self): + """Returns a set of failed test names.""" + return set([r.name for r in self._test_results if r.failed]) + + def _get_result_type_char(self, test_name): + """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, + PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result + for the given test_name. + """ + if test_name not in self._test_results_map: + return JSONResultsGenerator.NO_DATA_RESULT + + test_result = self._test_results_map[test_name] + if test_result.skipped: + return JSONResultsGenerator.SKIP_RESULT + if test_result.failed: + return JSONResultsGenerator.FAIL_RESULT + + return JSONResultsGenerator.PASS_RESULT + + # FIXME: Callers should use scm.py instead. + # FIXME: Identify and fix the run-time errors that were observed on Windows + # chromium buildbot when we had updated this code to use scm.py once before. def _get_svn_revision(self, in_directory): """Returns the svn revision for the given directory. @@ -126,6 +209,7 @@ class JSONResultsGenerator(object): in_directory: The directory where svn is to be run. """ if os.path.exists(os.path.join(in_directory, '.svn')): + # Note: Not thread safe: http://bugs.python.org/issue2320 output = subprocess.Popen(["svn", "info", "--xml"], cwd=in_directory, shell=(sys.platform == 'win32'), @@ -148,14 +232,14 @@ class JSONResultsGenerator(object): error = None if os.path.exists(self._results_file_path): - old_results_file = open(self._results_file_path, "r") - old_results = old_results_file.read() + with codecs.open(self._results_file_path, "r", "utf-8") as file: + old_results = file.read() elif self._builder_base_url: # Check if we have the archived JSON file on the buildbot server. results_file_url = (self._builder_base_url + self._build_name + "/" + self.RESULTS_FILENAME) - logging.error("Local results.json file does not exist. Grabbing " - "it off the archive at " + results_file_url) + _log.error("Local results.json file does not exist. Grabbing " + "it off the archive at " + results_file_url) try: results_file = urllib2.urlopen(results_file_url) @@ -177,85 +261,43 @@ class JSONResultsGenerator(object): try: results_json = simplejson.loads(old_results) except: - logging.debug("results.json was not valid JSON. Clobbering.") + _log.debug("results.json was not valid JSON. Clobbering.") # The JSON file is not valid JSON. Just clobber the results. results_json = {} else: - logging.debug('Old JSON results do not exist. Starting fresh.') + _log.debug('Old JSON results do not exist. Starting fresh.') results_json = {} return results_json, error - def _get_json(self): - """Gets the results for the results.json file.""" - results_json, error = self._get_archived_json_results() - if error: - # If there was an error don't write a results.json - # file at all as it would lose all the information on the bot. - logging.error("Archive directory is inaccessible. Not modifying " - "or clobbering the results.json file: " + str(error)) - return None - - builder_name = self._builder_name - if results_json and builder_name not in results_json: - logging.debug("Builder name (%s) is not in the results.json file." - % builder_name) - - self._convert_json_to_current_version(results_json) - - if builder_name not in results_json: - results_json[builder_name] = ( - self._create_results_for_builder_json()) - - results_for_builder = results_json[builder_name] - - self._insert_generic_metadata(results_for_builder) - - self._insert_failure_summaries(results_for_builder) - - # Update the all failing tests with result type and time. - tests = results_for_builder[self.TESTS] - all_failing_tests = set(self._failures.iterkeys()) - all_failing_tests.update(tests.iterkeys()) - for test in all_failing_tests: - self._insert_test_time_and_result(test, tests) - - # Specify separators in order to get compact encoding. - results_str = simplejson.dumps(results_json, separators=(',', ':')) - return self.JSON_PREFIX + results_str + self.JSON_SUFFIX - def _insert_failure_summaries(self, results_for_builder): """Inserts aggregate pass/failure statistics into the JSON. - This method reads self._skipped_tests, self._passed_tests and - self._failures and inserts FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT - entries. + This method reads self._test_results and generates + FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT entries. Args: results_for_builder: Dictionary containing the test results for a single builder. """ - # Insert the number of tests that failed. + # Insert the number of tests that failed or skipped. + fixable_count = len([r for r in self._test_results if r.fixable()]) self._insert_item_into_raw_list(results_for_builder, - len(set(self._failures.keys()) | self._skipped_tests), - self.FIXABLE_COUNT) + fixable_count, self.FIXABLE_COUNT) # Create a pass/skip/failure summary dictionary. entry = {} - entry[self.SKIP_RESULT] = len(self._skipped_tests) - entry[self.PASS_RESULT] = len(self._passed_tests) - get = entry.get - for failure_type in self._failures.values(): - failure_char = self.FAILURE_TO_CHAR[failure_type] - entry[failure_char] = get(failure_char, 0) + 1 + for test_name in self._test_results_map.iterkeys(): + result_char = self._get_result_type_char(test_name) + entry[result_char] = entry.get(result_char, 0) + 1 # Insert the pass/skip/failure summary dictionary. self._insert_item_into_raw_list(results_for_builder, entry, self.FIXABLE) # Insert the number of all the tests that are supposed to pass. + all_test_count = len(self._test_results) self._insert_item_into_raw_list(results_for_builder, - len(self._skipped_tests | self._all_tests), - self.ALL_FIXABLE_COUNT) + all_test_count, self.ALL_FIXABLE_COUNT) def _insert_item_into_raw_list(self, results_for_builder, item, key): """Inserts the item into the list with the given key in the results for @@ -304,19 +346,11 @@ class JSONResultsGenerator(object): self._insert_item_into_raw_list(results_for_builder, self._build_number, self.BUILD_NUMBERS) - # These next two branches test to see which source repos we can - # pull revisions from. - if hasattr(self._port, 'path_from_webkit_base'): - path_to_webkit = self._port.path_from_webkit_base() + # Include SVN revisions for the given repositories. + for (name, path) in self._svn_repositories: self._insert_item_into_raw_list(results_for_builder, - self._get_svn_revision(path_to_webkit), - self.WEBKIT_SVN) - - if hasattr(self._port, 'path_from_chromium_base'): - path_to_chrome = self._port.path_from_chromium_base() - self._insert_item_into_raw_list(results_for_builder, - self._get_svn_revision(path_to_chrome), - self.CHROME_SVN) + self._get_svn_revision(path), + name + 'Revision') self._insert_item_into_raw_list(results_for_builder, int(time.time()), @@ -329,18 +363,8 @@ class JSONResultsGenerator(object): tests: Dictionary containing test result entries. """ - result = JSONResultsGenerator.PASS_RESULT - time = 0 - - if test_name not in self._all_tests: - result = JSONResultsGenerator.NO_DATA_RESULT - - if test_name in self._failures: - result = self.FAILURE_TO_CHAR[self._failures[test_name]] - - if test_name in self._test_timings: - # Floor for now to get time in seconds. - time = int(self._test_timings[test_name]) + result = self._get_result_type_char(test_name) + time = self._get_test_timing(test_name) if test_name not in tests: tests[test_name] = self._create_results_and_times_json() @@ -418,3 +442,59 @@ class JSONResultsGenerator(object): """Returns whether all the results are of the given type (e.g. all passes).""" return len(results) == 1 and results[0][1] == type + + +# A wrapper class for JSONResultsGeneratorBase. +# Note: There's a script outside the WebKit codebase calling this script. +# FIXME: Please keep the interface until the other script is cleaned up. +# (http://src.chromium.org/viewvc/chrome/trunk/src/webkit/tools/layout_tests/webkitpy/layout_tests/test_output_xml_to_json.py?view=markup) +class JSONResultsGenerator(JSONResultsGeneratorBase): + # The flag is for backward compatibility. + output_json_in_init = True + + def __init__(self, port, builder_name, build_name, build_number, + results_file_base_path, builder_base_url, + test_timings, failures, passed_tests, skipped_tests, all_tests): + """Generates a JSON results file. + + Args + builder_name: the builder name (e.g. Webkit). + build_name: the build name (e.g. webkit-rel). + build_number: the build number. + results_file_base_path: Absolute path to the directory containing the + results json file. + builder_base_url: the URL where we have the archived test results. + test_timings: Map of test name to a test_run-time. + failures: Map of test name to a failure type (of test_expectations). + passed_tests: A set containing all the passed tests. + skipped_tests: A set containing all the skipped tests. + all_tests: List of all the tests that were run. This should not + include skipped tests. + """ + + # Create a map of (name, TestResult). + test_results_map = dict() + get = test_results_map.get + for (test, time) in test_timings.iteritems(): + test_results_map[test] = TestResult(test, elapsed_time=time) + for test in failures.iterkeys(): + test_results_map[test] = test_result = get(test, TestResult(test)) + test_result.failed = True + for test in skipped_tests: + test_results_map[test] = test_result = get(test, TestResult(test)) + test_result.skipped = True + for test in passed_tests: + test_results_map[test] = test_result = get(test, TestResult(test)) + test_result.failed = False + test_result.skipped = False + for test in all_tests: + if test not in test_results_map: + test_results_map[test] = TestResult(test) + + super(JSONResultsGenerator, self).__init__( + builder_name, build_name, build_number, + results_file_base_path, builder_base_url, test_results_map, + svn_repositories=port.test_repository_paths()) + + if self.__class__.output_json_in_init: + self.generate_json_output() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py new file mode 100644 index 0000000..0a60cc7 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py @@ -0,0 +1,126 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for json_results_generator.py.""" + +import unittest +import optparse +import random +import shutil +import tempfile + +from webkitpy.layout_tests.layout_package import json_results_generator +from webkitpy.layout_tests.layout_package import test_expectations +from webkitpy.layout_tests import port + + +class JSONGeneratorTest(unittest.TestCase): + def setUp(self): + json_results_generator.JSONResultsGenerator.output_json_in_init = False + self.builder_name = 'DUMMY_BUILDER_NAME' + self.build_name = 'DUMMY_BUILD_NAME' + self.build_number = 'DUMMY_BUILDER_NUMBER' + + def _test_json_generation(self, passed_tests, failed_tests, skipped_tests): + # Make sure we have sets (rather than lists). + passed_tests = set(passed_tests) + skipped_tests = set(skipped_tests) + tests_list = passed_tests | set(failed_tests.keys()) + test_timings = {} + for test in tests_list: + test_timings[test] = float(random.randint(1, 10)) + + port_obj = port.get(None) + + # Generate a JSON file. + generator = json_results_generator.JSONResultsGenerator(port_obj, + self.builder_name, self.build_name, self.build_number, + '', + None, # don't fetch past json results archive + test_timings, + failed_tests, + passed_tests, + skipped_tests, + tests_list) + + json = generator.get_json() + + # Aliasing to a short name for better access to its constants. + JRG = json_results_generator.JSONResultsGenerator + + self.assertTrue(JRG.VERSION_KEY in json) + self.assertTrue(self.builder_name in json) + + buildinfo = json[self.builder_name] + self.assertTrue(JRG.FIXABLE in buildinfo) + self.assertTrue(JRG.TESTS in buildinfo) + self.assertTrue(len(buildinfo[JRG.BUILD_NUMBERS]) == 1) + self.assertTrue(buildinfo[JRG.BUILD_NUMBERS][0] == self.build_number) + + if tests_list or skipped_tests: + fixable = buildinfo[JRG.FIXABLE][0] + if passed_tests: + self.assertTrue(fixable[JRG.PASS_RESULT] == len(passed_tests)) + else: + self.assertTrue(JRG.PASS_RESULT not in fixable or + fixable[JRG.PASS_RESULT] == 0) + if skipped_tests: + self.assertTrue(fixable[JRG.SKIP_RESULT] == len(skipped_tests)) + else: + self.assertTrue(JRG.SKIP_RESULT not in fixable or + fixable[JRG.SKIP_RESULT] == 0) + + if failed_tests: + tests = buildinfo[JRG.TESTS] + for test_name, failure in failed_tests.iteritems(): + self.assertTrue(test_name in tests) + test = tests[test_name] + self.assertTrue(test[JRG.RESULTS][0][0] == 1) + self.assertTrue(test[JRG.RESULTS][0][1] == JRG.FAIL_RESULT) + self.assertTrue(test[JRG.TIMES][0][0] == 1) + self.assertTrue(test[JRG.TIMES][0][1] == + int(test_timings[test_name])) + + fixable_count = len(skipped_tests) + len(failed_tests.keys()) + if skipped_tests or failed_tests: + self.assertTrue(buildinfo[JRG.FIXABLE_COUNT][0] == fixable_count) + + def test_json_generation(self): + reason = test_expectations.TEXT + + self._test_json_generation([], {}, []) + self._test_json_generation(['A', 'B'], {}, []) + self._test_json_generation([], {'A': reason, 'B': reason}, []) + self._test_json_generation([], {}, ['A', 'B']) + self._test_json_generation(['A'], {'B': reason, 'C': reason}, []) + self._test_json_generation([], {'A': reason, 'B': reason}, ['C', 'D']) + self._test_json_generation(['A', 'B', 'C'], {'D': reason}, ['E', 'F']) + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py index 72b30a1..20646a1 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py @@ -32,23 +32,50 @@ Package that implements a stream wrapper that has 'meters' as well as regular output. A 'meter' is a single line of text that can be erased and rewritten repeatedly, without producing multiple lines of output. It can be used to produce effects like progress bars. + +This package should only be called by the printing module in the layout_tests +package. """ +import logging + +_log = logging.getLogger("webkitpy.layout_tests.metered_stream") + class MeteredStream: """This class is a wrapper around a stream that allows you to implement - meters. + meters (progress bars, etc.). + + It can be used directly as a stream, by calling write(), but provides + two other methods for output, update(), and progress(). + + In normal usage, update() will overwrite the output of the immediately + preceding update() (write() also will overwrite update()). So, calling + multiple update()s in a row can provide an updating status bar (note that + if an update string contains newlines, only the text following the last + newline will be overwritten/erased). - It can be used like a stream, but calling update() will print - the string followed by only a carriage return (instead of a carriage - return and a line feed). This can be used to implement progress bars and - other sorts of meters. Note that anything written by update() will be - erased by a subsequent update(), write(), or flush().""" + If the MeteredStream is constructed in "verbose" mode (i.e., by passing + verbose=true), then update() no longer overwrite a previous update(), and + instead the call is equivalent to write(), although the text is + actually sent to the logger rather than to the stream passed + to the constructor. + + progress() is just like update(), except that if you are in verbose mode, + progress messages are not output at all (they are dropped). This is + used for things like progress bars which are presumed to be unwanted in + verbose mode. + + Note that the usual usage for this class is as a destination for + a logger that can also be written to directly (i.e., some messages go + through the logger, some don't). We thus have to dance around a + layering inversion in update() for things to work correctly. + """ def __init__(self, verbose, stream): """ Args: - verbose: whether update is a no-op + verbose: whether progress is a no-op and updates() aren't overwritten stream: output stream to write to """ self._dirty = False @@ -57,40 +84,63 @@ class MeteredStream: self._last_update = "" def write(self, txt): - """Write text directly to the stream, overwriting and resetting the - meter.""" + """Write to the stream, overwriting and resetting the meter.""" if self._dirty: - self.update("") + self._write(txt) self._dirty = False - self._stream.write(txt) + self._last_update = '' + else: + self._stream.write(txt) def flush(self): """Flush any buffered output.""" self._stream.flush() - def update(self, str): - """Write an update to the stream that will get overwritten by the next - update() or by a write(). + def progress(self, str): + """ + Write a message to the stream that will get overwritten. This is used for progress updates that don't need to be preserved in - the log. Note that verbose disables this routine; we have this in - case we are logging lots of output and the update()s will get lost - or won't work properly (typically because verbose streams are - redirected to files. - - TODO(dpranke): figure out if there is a way to detect if we're writing - to a stream that handles CRs correctly (e.g., terminals). That might - be a cleaner way of handling this. + the log. If the MeteredStream was initialized with verbose==True, + then this output is discarded. We have this in case we are logging + lots of output and the update()s will get lost or won't work + properly (typically because verbose streams are redirected to files). + """ if self._verbose: return + self._write(str) + + def update(self, str): + """ + Write a message that is also included when logging verbosely. + + This routine preserves the same console logging behavior as progress(), + but will also log the message if verbose() was true. + + """ + # Note this is a separate routine that calls either into the logger + # or the metering stream. We have to be careful to avoid a layering + # inversion (stream calling back into the logger). + if self._verbose: + _log.info(str) + else: + self._write(str) + + def _write(self, str): + """Actually write the message to the stream.""" + + # FIXME: Figure out if there is a way to detect if we're writing + # to a stream that handles CRs correctly (e.g., terminals). That might + # be a cleaner way of handling this. # Print the necessary number of backspaces to erase the previous # message. - self._stream.write("\b" * len(self._last_update)) + if len(self._last_update): + self._stream.write("\b" * len(self._last_update) + + " " * len(self._last_update) + + "\b" * len(self._last_update)) self._stream.write(str) - num_remaining = len(self._last_update) - len(str) - if num_remaining > 0: - self._stream.write(" " * num_remaining + "\b" * num_remaining) - self._last_update = str + last_newline = str.rfind("\n") + self._last_update = str[(last_newline + 1):] self._dirty = True diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream_unittest.py new file mode 100644 index 0000000..a9c6d5b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/metered_stream_unittest.py @@ -0,0 +1,110 @@ +#!/usr/bin/python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for metered_stream.py.""" + +import os +import optparse +import pdb +import sys +import unittest +import logging + +from webkitpy.common.array_stream import ArrayStream +from webkitpy.layout_tests.layout_package import metered_stream + + +class TestMeteredStream(unittest.TestCase): + def test_regular(self): + a = ArrayStream() + m = metered_stream.MeteredStream(verbose=False, stream=a) + self.assertTrue(a.empty()) + + # basic test - note that the flush() is a no-op, but we include it + # for coverage. + m.write("foo") + m.flush() + exp = ['foo'] + self.assertEquals(a.get(), exp) + + # now check that a second write() does not overwrite the first. + m.write("bar") + exp.append('bar') + self.assertEquals(a.get(), exp) + + m.update("batter") + exp.append('batter') + self.assertEquals(a.get(), exp) + + # The next update() should overwrite the laste update() but not the + # other text. Note that the cursor is effectively positioned at the + # end of 'foo', even though we had to erase three more characters. + m.update("foo") + exp.append('\b\b\b\b\b\b \b\b\b\b\b\b') + exp.append('foo') + self.assertEquals(a.get(), exp) + + m.progress("progress") + exp.append('\b\b\b \b\b\b') + exp.append('progress') + self.assertEquals(a.get(), exp) + + # now check that a write() does overwrite the progress bar + m.write("foo") + exp.append('\b\b\b\b\b\b\b\b \b\b\b\b\b\b\b\b') + exp.append('foo') + self.assertEquals(a.get(), exp) + + # Now test that we only back up to the most recent newline. + + # Note also that we do not back up to erase the most recent write(), + # i.e., write()s do not get erased. + a.reset() + m.update("foo\nbar") + m.update("baz") + self.assertEquals(a.get(), ['foo\nbar', '\b\b\b \b\b\b', 'baz']) + + def test_verbose(self): + a = ArrayStream() + m = metered_stream.MeteredStream(verbose=True, stream=a) + self.assertTrue(a.empty()) + m.write("foo") + self.assertEquals(a.get(), ['foo']) + + m.update("bar") + # FIXME: figure out how to test that this went to the logger. Is this + # good enough? + self.assertEquals(a.get(), ['foo']) + + m.progress("dropped") + self.assertEquals(a.get(), ['foo']) + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py new file mode 100644 index 0000000..f838a7b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py @@ -0,0 +1,519 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Package that handles non-debug, non-file output for run-webkit-tests.""" + +import logging +import optparse +import os +import pdb + +from webkitpy.layout_tests.layout_package import metered_stream +from webkitpy.layout_tests.layout_package import test_expectations + +_log = logging.getLogger("webkitpy.layout_tests.printer") + +TestExpectationsFile = test_expectations.TestExpectationsFile + +NUM_SLOW_TESTS_TO_LOG = 10 + +PRINT_DEFAULT = ("misc,one-line-progress,one-line-summary,unexpected," + "unexpected-results,updates") +PRINT_EVERYTHING = ("actual,config,expected,misc,one-line-progress," + "one-line-summary,slowest,timing,unexpected," + "unexpected-results,updates") + +HELP_PRINTING = """ +Output for run-webkit-tests is controlled by a comma-separated list of +values passed to --print. Values either influence the overall output, or +the output at the beginning of the run, during the run, or at the end: + +Overall options: + nothing don't print anything. This overrides every other option + default include the default options. This is useful for logging + the default options plus additional settings. + everything print everything (except the trace-* options and the + detailed-progress option, see below for the full list ) + misc print miscellaneous things like blank lines + +At the beginning of the run: + config print the test run configuration + expected print a summary of what is expected to happen + (# passes, # failures, etc.) + +During the run: + detailed-progress print one dot per test completed + one-line-progress print a one-line progress bar + unexpected print any unexpected results as they occur + updates print updates on which stage is executing + trace-everything print detailed info on every test's results + (baselines, expectation, time it took to run). If + this is specified it will override the '*-progress' + options, the 'trace-unexpected' option, and the + 'unexpected' option. + trace-unexpected like 'trace-everything', but only for tests with + unexpected results. If this option is specified, + it will override the 'unexpected' option. + +At the end of the run: + actual print a summary of the actual results + slowest print %(slowest)d slowest tests and the time they took + timing print timing statistics + unexpected-results print a list of the tests with unexpected results + one-line-summary print a one-line summary of the run + +Notes: + - 'detailed-progress' can only be used if running in a single thread + (using --child-processes=1) or a single queue of tests (using + --experimental-fully-parallel). If these conditions aren't true, + 'one-line-progress' will be used instead. + - If both 'detailed-progress' and 'one-line-progress' are specified (and + both are possible), 'detailed-progress' will be used. + - If 'nothing' is specified, it overrides all of the other options. + - Specifying --verbose is equivalent to --print everything plus it + changes the format of the log messages to add timestamps and other + information. If you specify --verbose and --print X, then X overrides + the --print everything implied by --verbose. + +--print 'everything' is equivalent to --print '%(everything)s'. + +The default (--print default) is equivalent to --print '%(default)s'. +""" % {'slowest': NUM_SLOW_TESTS_TO_LOG, 'everything': PRINT_EVERYTHING, + 'default': PRINT_DEFAULT} + + +def print_options(): + return [ + # Note: We use print_options rather than just 'print' because print + # is a reserved word. + # Note: Also, we don't specify a default value so we can detect when + # no flag is specified on the command line and use different defaults + # based on whether or not --verbose is specified (since --print + # overrides --verbose). + optparse.make_option("--print", dest="print_options", + help=("controls print output of test run. " + "Use --help-printing for more.")), + optparse.make_option("--help-printing", action="store_true", + help="show detailed help on controlling print output"), + optparse.make_option("-v", "--verbose", action="store_true", + default=False, help="include debug-level logging"), + + # FIXME: we should remove this; it's pretty much obsolete with the + # --print trace-everything option. + optparse.make_option("--sources", action="store_true", + help=("show expected result file path for each test " + "(implies --verbose)")), + ] + + +def configure_logging(options, meter): + """Configures the logging system.""" + log_fmt = '%(message)s' + log_datefmt = '%y%m%d %H:%M:%S' + log_level = logging.INFO + if options.verbose: + log_fmt = ('%(asctime)s %(filename)s:%(lineno)-4d %(levelname)s ' + '%(message)s') + log_level = logging.DEBUG + + logging.basicConfig(level=log_level, format=log_fmt, + datefmt=log_datefmt, stream=meter) + + +def parse_print_options(print_options, verbose, child_processes, + is_fully_parallel): + """Parse the options provided to --print and dedup and rank them. + + Returns + a set() of switches that govern how logging is done + + """ + if print_options: + switches = set(print_options.split(',')) + elif verbose: + switches = set(PRINT_EVERYTHING.split(',')) + else: + switches = set(PRINT_DEFAULT.split(',')) + + if 'nothing' in switches: + return set() + + if (child_processes != 1 and not is_fully_parallel and + 'detailed-progress' in switches): + _log.warn("Can only print 'detailed-progress' if running " + "with --child-processes=1 or " + "with --experimental-fully-parallel. " + "Using 'one-line-progress' instead.") + switches.discard('detailed-progress') + switches.add('one-line-progress') + + if 'everything' in switches: + switches.discard('everything') + switches.update(set(PRINT_EVERYTHING.split(','))) + + if 'default' in switches: + switches.discard('default') + switches.update(set(PRINT_DEFAULT.split(','))) + + if 'detailed-progress' in switches: + switches.discard('one-line-progress') + + if 'trace-everything' in switches: + switches.discard('detailed-progress') + switches.discard('one-line-progress') + switches.discard('trace-unexpected') + switches.discard('unexpected') + + if 'trace-unexpected' in switches: + switches.discard('unexpected') + + return switches + + +class Printer(object): + """Class handling all non-debug-logging printing done by run-webkit-tests. + + Printing from run-webkit-tests falls into two buckets: general or + regular output that is read only by humans and can be changed at any + time, and output that is parsed by buildbots (and humans) and hence + must be changed more carefully and in coordination with the buildbot + parsing code (in chromium.org's buildbot/master.chromium/scripts/master/ + log_parser/webkit_test_command.py script). + + By default the buildbot-parsed code gets logged to stdout, and regular + output gets logged to stderr.""" + def __init__(self, port, options, regular_output, buildbot_output, + child_processes, is_fully_parallel): + """ + Args + port interface to port-specific routines + options OptionParser object with command line settings + regular_output stream to which output intended only for humans + should be written + buildbot_output stream to which output intended to be read by + the buildbots (and humans) should be written + child_processes number of parallel threads running (usually + controlled by --child-processes) + is_fully_parallel are the tests running in a single queue, or + in shards (usually controlled by + --experimental-fully-parallel) + + Note that the last two args are separate rather than bundled into + the options structure so that this object does not assume any flags + set in options that weren't returned from logging_options(), above. + The two are used to determine whether or not we can sensibly use + the 'detailed-progress' option, or can only use 'one-line-progress'. + """ + self._buildbot_stream = buildbot_output + self._options = options + self._port = port + self._stream = regular_output + + # These are used for --print detailed-progress to track status by + # directory. + self._current_dir = None + self._current_progress_str = "" + self._current_test_number = 0 + + self._meter = metered_stream.MeteredStream(options.verbose, + regular_output) + configure_logging(self._options, self._meter) + + self.switches = parse_print_options(options.print_options, + options.verbose, child_processes, is_fully_parallel) + + # These two routines just hide the implmentation of the switches. + def disabled(self, option): + return not option in self.switches + + def enabled(self, option): + return option in self.switches + + def help_printing(self): + self._write(HELP_PRINTING) + + def print_actual(self, msg): + if self.disabled('actual'): + return + self._buildbot_stream.write("%s\n" % msg) + + def print_config(self, msg): + self.write(msg, 'config') + + def print_expected(self, msg): + self.write(msg, 'expected') + + def print_timing(self, msg): + self.write(msg, 'timing') + + def print_one_line_summary(self, total, expected, unexpected): + """Print a one-line summary of the test run to stdout. + + Args: + total: total number of tests run + expected: number of expected results + unexpected: number of unexpected results + """ + if self.disabled('one-line-summary'): + return + + incomplete = total - expected - unexpected + if incomplete: + self._write("") + incomplete_str = " (%d didn't run)" % incomplete + expected_str = str(expected) + else: + incomplete_str = "" + expected_str = "All %d" % expected + + if unexpected == 0: + self._write("%s tests ran as expected%s." % + (expected_str, incomplete_str)) + elif expected == 1: + self._write("1 test ran as expected, %d didn't%s:" % + (unexpected, incomplete_str)) + else: + self._write("%d tests ran as expected, %d didn't%s:" % + (expected, unexpected, incomplete_str)) + self._write("") + + def print_test_result(self, result, expected, exp_str, got_str): + """Print the result of the test as determined by --print.""" + if (self.enabled('trace-everything') or + self.enabled('trace-unexpected') and not expected): + self._print_test_trace(result, exp_str, got_str) + elif (not expected and self.enabled('unexpected') and + self.disabled('detailed-progress')): + # Note: 'detailed-progress' handles unexpected results internally, + # so we skip it here. + self._print_unexpected_test_result(result) + + def _print_test_trace(self, result, exp_str, got_str): + """Print detailed results of a test (triggered by --print trace-*). + For each test, print: + - location of the expected baselines + - expected results + - actual result + - timing info + """ + filename = result.filename + test_name = self._port.relative_test_filename(filename) + self._write('trace: %s' % test_name) + self._write(' txt: %s' % + self._port.relative_test_filename( + self._port.expected_filename(filename, '.txt'))) + png_file = self._port.expected_filename(filename, '.png') + if os.path.exists(png_file): + self._write(' png: %s' % + self._port.relative_test_filename(png_file)) + else: + self._write(' png: <none>') + self._write(' exp: %s' % exp_str) + self._write(' got: %s' % got_str) + self._write(' took: %-.3f' % result.test_run_time) + self._write('') + + def _print_unexpected_test_result(self, result): + """Prints one unexpected test result line.""" + desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result.type][0] + self.write(" %s -> unexpected %s" % + (self._port.relative_test_filename(result.filename), + desc), "unexpected") + + def print_progress(self, result_summary, retrying, test_list): + """Print progress through the tests as determined by --print.""" + if self.enabled('detailed-progress'): + self._print_detailed_progress(result_summary, test_list) + elif self.enabled('one-line-progress'): + self._print_one_line_progress(result_summary, retrying) + else: + return + + if result_summary.remaining == 0: + self._meter.update('') + + def _print_one_line_progress(self, result_summary, retrying): + """Displays the progress through the test run.""" + percent_complete = 100 * (result_summary.expected + + result_summary.unexpected) / result_summary.total + action = "Testing" + if retrying: + action = "Retrying" + self._meter.progress("%s (%d%%): %d ran as expected, %d didn't," + " %d left" % (action, percent_complete, result_summary.expected, + result_summary.unexpected, result_summary.remaining)) + + def _print_detailed_progress(self, result_summary, test_list): + """Display detailed progress output where we print the directory name + and one dot for each completed test. This is triggered by + "--log detailed-progress".""" + if self._current_test_number == len(test_list): + return + + next_test = test_list[self._current_test_number] + next_dir = os.path.dirname( + self._port.relative_test_filename(next_test)) + if self._current_progress_str == "": + self._current_progress_str = "%s: " % (next_dir) + self._current_dir = next_dir + + while next_test in result_summary.results: + if next_dir != self._current_dir: + self._meter.write("%s\n" % (self._current_progress_str)) + self._current_progress_str = "%s: ." % (next_dir) + self._current_dir = next_dir + else: + self._current_progress_str += "." + + if (next_test in result_summary.unexpected_results and + self.enabled('unexpected')): + self._meter.write("%s\n" % self._current_progress_str) + test_result = result_summary.results[next_test] + self._print_unexpected_test_result(test_result) + self._current_progress_str = "%s: " % self._current_dir + + self._current_test_number += 1 + if self._current_test_number == len(test_list): + break + + next_test = test_list[self._current_test_number] + next_dir = os.path.dirname( + self._port.relative_test_filename(next_test)) + + if result_summary.remaining: + remain_str = " (%d)" % (result_summary.remaining) + self._meter.progress("%s%s" % (self._current_progress_str, + remain_str)) + else: + self._meter.progress("%s" % (self._current_progress_str)) + + def print_unexpected_results(self, unexpected_results): + """Prints a list of the unexpected results to the buildbot stream.""" + if self.disabled('unexpected-results'): + return + + passes = {} + flaky = {} + regressions = {} + + for test, results in unexpected_results['tests'].iteritems(): + actual = results['actual'].split(" ") + expected = results['expected'].split(" ") + if actual == ['PASS']: + if 'CRASH' in expected: + _add_to_dict_of_lists(passes, + 'Expected to crash, but passed', + test) + elif 'TIMEOUT' in expected: + _add_to_dict_of_lists(passes, + 'Expected to timeout, but passed', + test) + else: + _add_to_dict_of_lists(passes, + 'Expected to fail, but passed', + test) + elif len(actual) > 1: + # We group flaky tests by the first actual result we got. + _add_to_dict_of_lists(flaky, actual[0], test) + else: + _add_to_dict_of_lists(regressions, results['actual'], test) + + if len(passes) or len(flaky) or len(regressions): + self._buildbot_stream.write("\n") + + if len(passes): + for key, tests in passes.iteritems(): + self._buildbot_stream.write("%s: (%d)\n" % (key, len(tests))) + tests.sort() + for test in tests: + self._buildbot_stream.write(" %s\n" % test) + self._buildbot_stream.write("\n") + self._buildbot_stream.write("\n") + + if len(flaky): + descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS + for key, tests in flaky.iteritems(): + result = TestExpectationsFile.EXPECTATIONS[key.lower()] + self._buildbot_stream.write("Unexpected flakiness: %s (%d)\n" + % (descriptions[result][1], len(tests))) + tests.sort() + + for test in tests: + result = unexpected_results['tests'][test] + actual = result['actual'].split(" ") + expected = result['expected'].split(" ") + result = TestExpectationsFile.EXPECTATIONS[key.lower()] + new_expectations_list = list(set(actual) | set(expected)) + self._buildbot_stream.write(" %s = %s\n" % + (test, " ".join(new_expectations_list))) + self._buildbot_stream.write("\n") + self._buildbot_stream.write("\n") + + if len(regressions): + descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS + for key, tests in regressions.iteritems(): + result = TestExpectationsFile.EXPECTATIONS[key.lower()] + self._buildbot_stream.write( + "Regressions: Unexpected %s : (%d)\n" % ( + descriptions[result][1], len(tests))) + tests.sort() + for test in tests: + self._buildbot_stream.write(" %s = %s\n" % (test, key)) + self._buildbot_stream.write("\n") + self._buildbot_stream.write("\n") + + if len(unexpected_results['tests']) and self._options.verbose: + self._buildbot_stream.write("%s\n" % ("-" * 78)) + + def print_update(self, msg): + if self.disabled('updates'): + return + self._meter.update(msg) + + def write(self, msg, option="misc"): + if self.disabled(option): + return + self._write(msg) + + def _write(self, msg): + # FIXME: we could probably get away with calling _log.info() all of + # the time, but there doesn't seem to be a good way to test the output + # from the logger :(. + if self._options.verbose: + _log.info(msg) + else: + self._meter.write("%s\n" % msg) + +# +# Utility routines used by the Controller class +# + + +def _add_to_dict_of_lists(dict, key, value): + dict.setdefault(key, []).append(value) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py new file mode 100644 index 0000000..c8648bc --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py @@ -0,0 +1,510 @@ +#!/usr/bin/python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for printing.py.""" + +import os +import optparse +import pdb +import sys +import unittest +import logging + +from webkitpy.common import array_stream +from webkitpy.layout_tests import port +from webkitpy.layout_tests.layout_package import printing +from webkitpy.layout_tests.layout_package import dump_render_tree_thread +from webkitpy.layout_tests.layout_package import test_expectations +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests import run_webkit_tests + + +def get_options(args): + print_options = printing.print_options() + option_parser = optparse.OptionParser(option_list=print_options) + return option_parser.parse_args(args) + + +def get_result(filename, result_type=test_expectations.PASS, run_time=0): + failures = [] + if result_type == test_expectations.TIMEOUT: + failures = [test_failures.FailureTimeout()] + elif result_type == test_expectations.CRASH: + failures = [test_failures.FailureCrash()] + return dump_render_tree_thread.TestResult(filename, failures, run_time, + total_time_for_all_diffs=0, + time_for_diffs=0) + + +def get_result_summary(port_obj, test_files, expectations_str): + expectations = test_expectations.TestExpectations( + port_obj, test_files, expectations_str, + port_obj.test_platform_name(), is_debug_mode=False, + is_lint_mode=False, tests_are_present=False) + + rs = run_webkit_tests.ResultSummary(expectations, test_files) + return rs, expectations + + +class TestUtilityFunctions(unittest.TestCase): + def test_configure_logging(self): + # FIXME: We need to figure out how to reset the basic logger. + # FIXME: If other testing classes call logging.basicConfig() then + # FIXME: these calls become no-ops and we can't control the + # FIXME: configuration to test things properly. + options, args = get_options([]) + stream = array_stream.ArrayStream() + printing.configure_logging(options, stream) + logging.info("this should be logged") + # self.assertFalse(stream.empty()) + + stream.reset() + logging.debug("this should not be logged") + # self.assertTrue(stream.empty()) + + stream.reset() + options, args = get_options(['--verbose']) + printing.configure_logging(options, stream) + logging.debug("this should be logged") + # self.assertFalse(stream.empty()) + + def test_print_options(self): + options, args = get_options([]) + self.assertTrue(options is not None) + + def test_parse_print_options(self): + def test_switches(args, verbose, child_processes, is_fully_parallel, + expected_switches_str): + options, args = get_options(args) + if expected_switches_str: + expected_switches = set(expected_switches_str.split(',')) + else: + expected_switches = set() + switches = printing.parse_print_options(options.print_options, + verbose, + child_processes, + is_fully_parallel) + self.assertEqual(expected_switches, switches) + + # test that we default to the default set of switches + test_switches([], False, 1, False, + printing.PRINT_DEFAULT) + + # test that verbose defaults to everything + test_switches([], True, 1, False, + printing.PRINT_EVERYTHING) + + # test that --print default does what it's supposed to + test_switches(['--print', 'default'], False, 1, False, + printing.PRINT_DEFAULT) + + # test that --print nothing does what it's supposed to + test_switches(['--print', 'nothing'], False, 1, False, + None) + + # test that --print everything does what it's supposed to + test_switches(['--print', 'everything'], False, 1, False, + printing.PRINT_EVERYTHING) + + # this tests that '--print X' overrides '--verbose' + test_switches(['--print', 'actual'], True, 1, False, + 'actual') + + +class Testprinter(unittest.TestCase): + def get_printer(self, args=None, single_threaded=False, + is_fully_parallel=False): + printing_options = printing.print_options() + option_parser = optparse.OptionParser(option_list=printing_options) + options, args = option_parser.parse_args(args) + self._port = port.get('test', options) + nproc = 2 + if single_threaded: + nproc = 1 + + regular_output = array_stream.ArrayStream() + buildbot_output = array_stream.ArrayStream() + printer = printing.Printer(self._port, options, regular_output, + buildbot_output, single_threaded, + is_fully_parallel) + return printer, regular_output, buildbot_output + + def test_help_printer(self): + # Here and below we'll call the "regular" printer err and the + # buildbot printer out; this corresponds to how things run on the + # bots with stderr and stdout. + printer, err, out = self.get_printer() + + # This routine should print something to stdout. testing what it is + # is kind of pointless. + printer.help_printing() + self.assertFalse(err.empty()) + self.assertTrue(out.empty()) + + def do_switch_tests(self, method_name, switch, to_buildbot, + message='hello', exp_err=None, exp_bot=None): + def do_helper(method_name, switch, message, exp_err, exp_bot): + printer, err, bot = self.get_printer(['--print', switch]) + getattr(printer, method_name)(message) + self.assertEqual(err.get(), exp_err) + self.assertEqual(bot.get(), exp_bot) + + if to_buildbot: + if exp_err is None: + exp_err = [] + if exp_bot is None: + exp_bot = [message + "\n"] + else: + if exp_err is None: + exp_err = [message + "\n"] + if exp_bot is None: + exp_bot = [] + do_helper(method_name, 'nothing', 'hello', [], []) + do_helper(method_name, switch, 'hello', exp_err, exp_bot) + do_helper(method_name, 'everything', 'hello', exp_err, exp_bot) + + def test_print_actual(self): + # Actual results need to be logged to the buildbot's stream. + self.do_switch_tests('print_actual', 'actual', to_buildbot=True) + + def test_print_actual_buildbot(self): + # FIXME: Test that the format of the actual results matches what the + # buildbot is expecting. + pass + + def test_print_config(self): + self.do_switch_tests('print_config', 'config', to_buildbot=False) + + def test_print_expected(self): + self.do_switch_tests('print_expected', 'expected', to_buildbot=False) + + def test_print_timing(self): + self.do_switch_tests('print_timing', 'timing', to_buildbot=False) + + def test_print_update(self): + # Note that there shouldn't be a carriage return here; updates() + # are meant to be overwritten. + self.do_switch_tests('print_update', 'updates', to_buildbot=False, + message='hello', exp_err=['hello']) + + def test_print_one_line_summary(self): + printer, err, out = self.get_printer(['--print', 'nothing']) + printer.print_one_line_summary(1, 1, 0) + self.assertTrue(err.empty()) + + printer, err, out = self.get_printer(['--print', 'one-line-summary']) + printer.print_one_line_summary(1, 1, 0) + self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) + + printer, err, out = self.get_printer(['--print', 'everything']) + printer.print_one_line_summary(1, 1, 0) + self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) + + err.reset() + printer.print_one_line_summary(2, 1, 1) + self.assertEquals(err.get(), + ["1 test ran as expected, 1 didn't:\n", "\n"]) + + err.reset() + printer.print_one_line_summary(3, 2, 1) + self.assertEquals(err.get(), + ["2 tests ran as expected, 1 didn't:\n", "\n"]) + + err.reset() + printer.print_one_line_summary(3, 2, 0) + self.assertEquals(err.get(), + ['\n', "2 tests ran as expected (1 didn't run).\n", + '\n']) + + + def test_print_test_result(self): + result = get_result('foo.html') + printer, err, out = self.get_printer(['--print', 'nothing']) + result = get_result(os.path.join(self._port.layout_tests_dir(), + 'foo.html')) + printer.print_test_result(result, expected=False, exp_str='', + got_str='') + self.assertTrue(err.empty()) + + printer, err, out = self.get_printer(['--print', 'unexpected']) + printer.print_test_result(result, expected=True, exp_str='', + got_str='') + self.assertTrue(err.empty()) + printer.print_test_result(result, expected=False, exp_str='', + got_str='') + self.assertEquals(err.get(), + [' foo.html -> unexpected pass\n']) + + printer, err, out = self.get_printer(['--print', 'everything']) + printer.print_test_result(result, expected=True, exp_str='', + got_str='') + self.assertTrue(err.empty()) + + printer.print_test_result(result, expected=False, exp_str='', + got_str='') + self.assertEquals(err.get(), + [' foo.html -> unexpected pass\n']) + + printer, err, out = self.get_printer(['--print', 'nothing']) + printer.print_test_result(result, expected=False, exp_str='', + got_str='') + self.assertTrue(err.empty()) + + printer, err, out = self.get_printer(['--print', + 'trace-unexpected']) + printer.print_test_result(result, expected=True, exp_str='', + got_str='') + self.assertTrue(err.empty()) + + err.reset() + printer.print_test_result(result, expected=False, exp_str='', + got_str='') + self.assertFalse(err.empty()) + + printer, err, out = self.get_printer(['--print', 'trace-everything']) + printer.print_test_result(result, expected=True, exp_str='', + got_str='') + self.assertFalse(err.empty()) + + err.reset() + printer.print_test_result(result, expected=False, exp_str='', + got_str='') + + def test_print_progress(self): + test_files = ['foo.html', 'bar.html'] + expectations = '' + + # test that we print nothing + printer, err, out = self.get_printer(['--print', 'nothing']) + rs, exp = get_result_summary(self._port, test_files, expectations) + + printer.print_progress(rs, False, test_files) + self.assertTrue(out.empty()) + self.assertTrue(err.empty()) + + printer.print_progress(rs, True, test_files) + self.assertTrue(out.empty()) + self.assertTrue(err.empty()) + + # test regular functionality + printer, err, out = self.get_printer(['--print', + 'one-line-progress']) + printer.print_progress(rs, False, test_files) + self.assertTrue(out.empty()) + self.assertFalse(err.empty()) + + err.reset() + out.reset() + printer.print_progress(rs, True, test_files) + self.assertFalse(err.empty()) + self.assertTrue(out.empty()) + + def test_print_progress__detailed(self): + test_files = ['pass/pass.html', 'pass/timeout.html', 'fail/crash.html'] + expectations = 'pass/timeout.html = TIMEOUT' + + # first, test that it is disabled properly + # should still print one-line-progress + printer, err, out = self.get_printer( + ['--print', 'detailed-progress'], single_threaded=False) + rs, exp = get_result_summary(self._port, test_files, expectations) + printer.print_progress(rs, False, test_files) + self.assertFalse(err.empty()) + self.assertTrue(out.empty()) + + # now test the enabled paths + printer, err, out = self.get_printer( + ['--print', 'detailed-progress'], single_threaded=True) + rs, exp = get_result_summary(self._port, test_files, expectations) + printer.print_progress(rs, False, test_files) + self.assertFalse(err.empty()) + self.assertTrue(out.empty()) + + err.reset() + out.reset() + printer.print_progress(rs, True, test_files) + self.assertFalse(err.empty()) + self.assertTrue(out.empty()) + + rs.add(get_result('pass/pass.html', test_expectations.TIMEOUT), False) + rs.add(get_result('pass/timeout.html'), True) + rs.add(get_result('fail/crash.html', test_expectations.CRASH), True) + err.reset() + out.reset() + printer.print_progress(rs, False, test_files) + self.assertFalse(err.empty()) + self.assertTrue(out.empty()) + + # We only clear the meter when retrying w/ detailed-progress. + err.reset() + out.reset() + printer.print_progress(rs, True, test_files) + self.assertEqual(err.get(), ['']) + self.assertTrue(out.empty()) + + printer, err, out = self.get_printer( + ['--print', 'detailed-progress,unexpected'], single_threaded=True) + rs, exp = get_result_summary(self._port, test_files, expectations) + printer.print_progress(rs, False, test_files) + self.assertFalse(err.empty()) + self.assertTrue(out.empty()) + + err.reset() + out.reset() + printer.print_progress(rs, True, test_files) + self.assertFalse(err.empty()) + self.assertTrue(out.empty()) + + rs.add(get_result('pass/pass.html', test_expectations.TIMEOUT), False) + rs.add(get_result('pass/timeout.html'), True) + rs.add(get_result('fail/crash.html', test_expectations.CRASH), True) + err.reset() + out.reset() + printer.print_progress(rs, False, test_files) + self.assertFalse(err.empty()) + self.assertTrue(out.empty()) + + # We only clear the meter when retrying w/ detailed-progress. + err.reset() + out.reset() + printer.print_progress(rs, True, test_files) + self.assertEqual(err.get(), ['']) + self.assertTrue(out.empty()) + + def test_write(self): + printer, err, out = self.get_printer(['--print', 'nothing']) + printer.write("foo") + self.assertTrue(err.empty()) + + printer, err, out = self.get_printer(['--print', 'misc']) + printer.write("foo") + self.assertFalse(err.empty()) + err.reset() + printer.write("foo", "config") + self.assertTrue(err.empty()) + + printer, err, out = self.get_printer(['--print', 'everything']) + printer.write("foo") + self.assertFalse(err.empty()) + err.reset() + printer.write("foo", "config") + self.assertFalse(err.empty()) + + def test_print_unexpected_results(self): + # This routine is the only one that prints stuff that the bots + # care about. + def get_unexpected_results(expected, passing, flaky): + rs, exp = get_result_summary(self._port, test_files, expectations) + if expected: + rs.add(get_result('pass/pass.html', test_expectations.PASS), + expected) + rs.add(get_result('pass/timeout.html', + test_expectations.TIMEOUT), expected) + rs.add(get_result('fail/crash.html', test_expectations.CRASH), + expected) + elif passing: + rs.add(get_result('pass/pass.html'), expected) + rs.add(get_result('pass/timeout.html'), expected) + rs.add(get_result('fail/crash.html'), expected) + else: + rs.add(get_result('pass/pass.html', test_expectations.TIMEOUT), + expected) + rs.add(get_result('pass/timeout.html', + test_expectations.CRASH), expected) + rs.add(get_result('fail/crash.html', + test_expectations.TIMEOUT), + expected) + retry = rs + if flaky: + retry, exp = get_result_summary(self._port, test_files, + expectations) + retry.add(get_result('pass/pass.html'), True) + retry.add(get_result('pass/timeout.html'), True) + retry.add(get_result('fail/crash.html'), True) + unexpected_results = run_webkit_tests.summarize_unexpected_results( + self._port, exp, rs, retry) + return unexpected_results + + test_files = ['pass/pass.html', 'pass/timeout.html', 'fail/crash.html'] + expectations = '' + + printer, err, out = self.get_printer(['--print', 'nothing']) + ur = get_unexpected_results(expected=False, passing=False, flaky=False) + printer.print_unexpected_results(ur) + self.assertTrue(err.empty()) + self.assertTrue(out.empty()) + + printer, err, out = self.get_printer(['--print', + 'unexpected-results']) + + # test everything running as expected + ur = get_unexpected_results(expected=True, passing=False, flaky=False) + printer.print_unexpected_results(ur) + self.assertTrue(err.empty()) + self.assertTrue(out.empty()) + + # test failures + err.reset() + out.reset() + ur = get_unexpected_results(expected=False, passing=False, flaky=False) + printer.print_unexpected_results(ur) + self.assertTrue(err.empty()) + self.assertFalse(out.empty()) + + # test unexpected flaky results + err.reset() + out.reset() + ur = get_unexpected_results(expected=False, passing=True, flaky=False) + printer.print_unexpected_results(ur) + self.assertTrue(err.empty()) + self.assertFalse(out.empty()) + + # test unexpected passes + err.reset() + out.reset() + ur = get_unexpected_results(expected=False, passing=False, flaky=True) + printer.print_unexpected_results(ur) + self.assertTrue(err.empty()) + self.assertFalse(out.empty()) + + err.reset() + out.reset() + printer, err, out = self.get_printer(['--print', 'everything']) + ur = get_unexpected_results(expected=False, passing=False, flaky=False) + printer.print_unexpected_results(ur) + self.assertTrue(err.empty()) + self.assertFalse(out.empty()) + + def test_print_unexpected_results_buildbot(self): + # FIXME: Test that print_unexpected_results() produces the printer the + # buildbot is expecting. + pass + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py index a3650ed..38223dd 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py @@ -35,9 +35,11 @@ import logging import os import re import sys -import time -import simplejson +import webkitpy.thirdparty.simplejson as simplejson + +_log = logging.getLogger("webkitpy.layout_tests.layout_package." + "test_expectations") # Test expectation and modifier constants. (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, TIMEOUT, CRASH, SKIP, WONTFIX, @@ -47,11 +49,46 @@ import simplejson (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4) +def result_was_expected(result, expected_results, test_needs_rebaselining, + test_is_skipped): + """Returns whether we got a result we were expecting. + Args: + result: actual result of a test execution + expected_results: set of results listed in test_expectations + test_needs_rebaselining: whether test was marked as REBASELINE + test_is_skipped: whether test was marked as SKIP""" + if result in expected_results: + return True + if result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and FAIL in expected_results: + return True + if result == MISSING and test_needs_rebaselining: + return True + if result == SKIP and test_is_skipped: + return True + return False + + +def remove_pixel_failures(expected_results): + """Returns a copy of the expected results for a test, except that we + drop any pixel failures and return the remaining expectations. For example, + if we're not running pixel tests, then tests expected to fail as IMAGE + will PASS.""" + expected_results = expected_results.copy() + if IMAGE in expected_results: + expected_results.remove(IMAGE) + expected_results.add(PASS) + if IMAGE_PLUS_TEXT in expected_results: + expected_results.remove(IMAGE_PLUS_TEXT) + expected_results.add(TEXT) + return expected_results + + class TestExpectations: TEST_LIST = "test_expectations.txt" def __init__(self, port, tests, expectations, test_platform_name, - is_debug_mode, is_lint_mode, tests_are_present=True): + is_debug_mode, is_lint_mode, tests_are_present=True, + overrides=None): """Loads and parses the test expectations given in the string. Args: port: handle to object containing platform-specific functionality @@ -68,10 +105,14 @@ class TestExpectations: system and can be probed for. This is useful for distinguishing test files from directories, and is needed by the LTTF dashboard, where the files aren't actually locally present. + overrides: test expectations that are allowed to override any + entries in |expectations|. This is used by callers + that need to manage two sets of expectations (e.g., upstream + and downstream expectations). """ self._expected_failures = TestExpectationsFile(port, expectations, tests, test_platform_name, is_debug_mode, is_lint_mode, - tests_are_present=tests_are_present) + tests_are_present=tests_are_present, overrides=overrides) # TODO(ojan): Allow for removing skipped tests when getting the list of # tests to run, but not when getting metrics. @@ -102,12 +143,16 @@ class TestExpectations: retval = [] for expectation in expectations: - for item in TestExpectationsFile.EXPECTATIONS.items(): - if item[1] == expectation: - retval.append(item[0]) - break + retval.append(self.expectation_to_string(expectation)) + + return " ".join(retval) - return " ".join(retval).upper() + def expectation_to_string(self, expectation): + """Return the uppercased string equivalent of a given expectation.""" + for item in TestExpectationsFile.EXPECTATIONS.items(): + if item[1] == expectation: + return item[0].upper() + return "" def get_timeline_for_test(self, test): return self._expected_failures.get_timeline_for_test(test) @@ -118,14 +163,13 @@ class TestExpectations: def get_tests_with_timeline(self, timeline): return self._expected_failures.get_tests_with_timeline(timeline) - def matches_an_expected_result(self, test, result): - """Returns whether we got one of the expected results for this test.""" - return (result in self._expected_failures.get_expectations(test) or - (result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and - FAIL in self._expected_failures.get_expectations(test)) or - result == MISSING and self.is_rebaselining(test) or - result == SKIP and self._expected_failures.has_modifier(test, - SKIP)) + def matches_an_expected_result(self, test, result, + pixel_tests_are_enabled): + expected_results = self._expected_failures.get_expectations(test) + if not pixel_tests_are_enabled: + expected_results = remove_pixel_failures(expected_results) + return result_was_expected(result, expected_results, + self.is_rebaselining(test), self.has_modifier(test, SKIP)) def is_rebaselining(self, test): return self._expected_failures.has_modifier(test, REBASELINE) @@ -133,10 +177,9 @@ class TestExpectations: def has_modifier(self, test, modifier): return self._expected_failures.has_modifier(test, modifier) - def remove_platform_from_file(self, tests, platform, backup=False): - return self._expected_failures.remove_platform_from_file(tests, - platform, - backup) + def remove_platform_from_expectations(self, tests, platform): + return self._expected_failures.remove_platform_from_expectations( + tests, platform) def strip_comments(line): @@ -234,8 +277,8 @@ class TestExpectationsFile: IMAGE: ('image mismatch', 'image mismatch'), IMAGE_PLUS_TEXT: ('image and text mismatch', 'image and text mismatch'), - CRASH: ('test shell crash', - 'test shell crashes'), + CRASH: ('DumpRenderTree crash', + 'DumpRenderTree crashes'), TIMEOUT: ('test timed out', 'tests timed out'), MISSING: ('no expected result found', 'no expected results found')} @@ -263,7 +306,7 @@ class TestExpectationsFile: def __init__(self, port, expectations, full_test_list, test_platform_name, is_debug_mode, is_lint_mode, suppress_errors=False, - tests_are_present=True): + tests_are_present=True, overrides=None): """ expectations: Contents of the expectations file full_test_list: The list of all tests to be run pending processing of @@ -277,6 +320,10 @@ class TestExpectationsFile: tests_are_present: Whether the test files are present in the local filesystem. The LTTF Dashboard uses False here to avoid having to keep a local copy of the tree. + overrides: test expectations that are allowed to override any + entries in |expectations|. This is used by callers + that need to manage two sets of expectations (e.g., upstream + and downstream expectations). """ self._port = port @@ -286,6 +333,7 @@ class TestExpectationsFile: self._is_debug_mode = is_debug_mode self._is_lint_mode = is_lint_mode self._tests_are_present = tests_are_present + self._overrides = overrides self._suppress_errors = suppress_errors self._errors = [] self._non_fatal_errors = [] @@ -313,7 +361,50 @@ class TestExpectationsFile: self._timeline_to_tests = self._dict_of_sets(self.TIMELINES) self._result_type_to_tests = self._dict_of_sets(self.RESULT_TYPES) - self._read(self._get_iterable_expectations()) + self._read(self._get_iterable_expectations(self._expectations), + overrides_allowed=False) + + # List of tests that are in the overrides file (used for checking for + # duplicates inside the overrides file itself). Note that just because + # a test is in this set doesn't mean it's necessarily overridding a + # expectation in the regular expectations; the test might not be + # mentioned in the regular expectations file at all. + self._overridding_tests = set() + + if overrides: + self._read(self._get_iterable_expectations(self._overrides), + overrides_allowed=True) + + self._handle_any_read_errors() + self._process_tests_without_expectations() + + def _handle_any_read_errors(self): + if not self._suppress_errors and ( + len(self._errors) or len(self._non_fatal_errors)): + if self._is_debug_mode: + build_type = 'DEBUG' + else: + build_type = 'RELEASE' + _log.error('') + _log.error("FAILURES FOR PLATFORM: %s, BUILD_TYPE: %s" % + (self._test_platform_name.upper(), build_type)) + + for error in self._non_fatal_errors: + _log.error(error) + _log.error('') + + if len(self._errors): + raise SyntaxError('\n'.join(map(str, self._errors))) + + def _process_tests_without_expectations(self): + expectations = set([PASS]) + options = [] + modifiers = [] + if self._full_test_list: + for test in self._full_test_list: + if not test in self._test_list_paths: + self._add_test(test, modifiers, expectations, options, + overrides_allowed=False) def _dict_of_sets(self, strings_to_constants): """Takes a dict of strings->constants and returns a dict mapping @@ -323,12 +414,11 @@ class TestExpectationsFile: d[c] = set() return d - def _get_iterable_expectations(self): + def _get_iterable_expectations(self, expectations_str): """Returns an object that can be iterated over. Allows for not caring about whether we're iterating over a file or a new-line separated string.""" - iterable = [x + "\n" for x in - self._expectations.split("\n")] + iterable = [x + "\n" for x in expectations_str.split("\n")] # Strip final entry if it's empty to avoid added in an extra # newline. if iterable[-1] == "\n": @@ -373,8 +463,9 @@ class TestExpectationsFile: def contains(self, test): return test in self._test_to_expectations - def remove_platform_from_file(self, tests, platform, backup=False): - """Remove the platform option from test expectations file. + def remove_platform_from_expectations(self, tests, platform): + """Returns a copy of the expectations with the tests matching the + platform remove. If a test is in the test list and has an option that matches the given platform, remove the matching platform and save the updated test back @@ -384,24 +475,13 @@ class TestExpectationsFile: Args: tests: list of tests that need to update.. platform: which platform option to remove. - backup: if true, the original test expectations file is saved as - [self.TEST_LIST].orig.YYYYMMDDHHMMSS Returns: - no + the updated string. """ - # FIXME - remove_platform_from file worked by writing a new - # test_expectations.txt file over the old one. Now that we're just - # parsing strings, we need to change this to return the new - # expectations string. - raise NotImplementedException('remove_platform_from_file') - - new_file = self._path + '.new' - logging.debug('Original file: "%s"', self._path) - logging.debug('New file: "%s"', new_file) - f_orig = self._get_iterable_expectations() - f_new = open(new_file, 'w') + f_orig = self._get_iterable_expectations(self._expectations) + f_new = [] tests_removed = 0 tests_updated = 0 @@ -412,20 +492,20 @@ class TestExpectationsFile: platform) if action == NO_CHANGE: # Save the original line back to the file - logging.debug('No change to test: %s', line) - f_new.write(line) + _log.debug('No change to test: %s', line) + f_new.append(line) elif action == REMOVE_TEST: tests_removed += 1 - logging.info('Test removed: %s', line) + _log.info('Test removed: %s', line) elif action == REMOVE_PLATFORM: parts = line.split(':') new_options = parts[0].replace(platform.upper() + ' ', '', 1) new_line = ('%s:%s' % (new_options, parts[1])) - f_new.write(new_line) + f_new.append(new_line) tests_updated += 1 - logging.info('Test updated: ') - logging.info(' old: %s', line) - logging.info(' new: %s', new_line) + _log.info('Test updated: ') + _log.info(' old: %s', line) + _log.info(' new: %s', new_line) elif action == ADD_PLATFORMS_EXCEPT_THIS: parts = line.split(':') new_options = parts[0] @@ -440,35 +520,19 @@ class TestExpectationsFile: if not p in (platform.upper(), 'WIN-VISTA', 'WIN-7'): new_options += p + ' ' new_line = ('%s:%s' % (new_options, parts[1])) - f_new.write(new_line) + f_new.append(new_line) tests_updated += 1 - logging.info('Test updated: ') - logging.info(' old: %s', line) - logging.info(' new: %s', new_line) + _log.info('Test updated: ') + _log.info(' old: %s', line) + _log.info(' new: %s', new_line) else: - logging.error('Unknown update action: %d; line: %s', - action, line) - - logging.info('Total tests removed: %d', tests_removed) - logging.info('Total tests updated: %d', tests_updated) - - f_orig.close() - f_new.close() - - if backup: - date_suffix = time.strftime('%Y%m%d%H%M%S', - time.localtime(time.time())) - backup_file = ('%s.orig.%s' % (self._path, date_suffix)) - if os.path.exists(backup_file): - os.remove(backup_file) - logging.info('Saving original file to "%s"', backup_file) - os.rename(self._path, backup_file) - else: - os.remove(self._path) + _log.error('Unknown update action: %d; line: %s', + action, line) - logging.debug('Saving new file to "%s"', self._path) - os.rename(new_file, self._path) - return True + _log.info('Total tests removed: %d', tests_removed) + _log.info('Total tests updated: %d', tests_updated) + + return "".join(f_new) def parse_expectations_line(self, line, lineno): """Parses a line from test_expectations.txt and returns a tuple @@ -602,7 +666,7 @@ class TestExpectationsFile: self._all_expectations[test].append( ModifiersAndExpectations(options, expectations)) - def _read(self, expectations): + def _read(self, expectations, overrides_allowed): """For each test in an expectations iterable, generate the expectations for it.""" lineno = 0 @@ -653,30 +717,7 @@ class TestExpectationsFile: tests = self._expand_tests(test_list_path) self._add_tests(tests, expectations, test_list_path, lineno, - modifiers, options) - - if not self._suppress_errors and ( - len(self._errors) or len(self._non_fatal_errors)): - if self._is_debug_mode: - build_type = 'DEBUG' - else: - build_type = 'RELEASE' - print "\nFAILURES FOR PLATFORM: %s, BUILD_TYPE: %s" \ - % (self._test_platform_name.upper(), build_type) - - for error in self._non_fatal_errors: - logging.error(error) - if len(self._errors): - raise SyntaxError('\n'.join(map(str, self._errors))) - - # Now add in the tests that weren't present in the expectations file - expectations = set([PASS]) - options = [] - modifiers = [] - if self._full_test_list: - for test in self._full_test_list: - if not test in self._test_list_paths: - self._add_test(test, modifiers, expectations, options) + modifiers, options, overrides_allowed) def _get_options_list(self, listString): return [part.strip().lower() for part in listString.strip().split(' ')] @@ -720,15 +761,18 @@ class TestExpectationsFile: return path def _add_tests(self, tests, expectations, test_list_path, lineno, - modifiers, options): + modifiers, options, overrides_allowed): for test in tests: - if self._already_seen_test(test, test_list_path, lineno): + if self._already_seen_test(test, test_list_path, lineno, + overrides_allowed): continue self._clear_expectations_for_test(test, test_list_path) - self._add_test(test, modifiers, expectations, options) + self._add_test(test, modifiers, expectations, options, + overrides_allowed) - def _add_test(self, test, modifiers, expectations, options): + def _add_test(self, test, modifiers, expectations, options, + overrides_allowed): """Sets the expected state for a given test. This routine assumes the test has not been added before. If it has, @@ -739,7 +783,9 @@ class TestExpectationsFile: test: test to add modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.) expectations: sequence of expectations (PASS, IMAGE, etc.) - options: sequence of keywords and bug identifiers.""" + options: sequence of keywords and bug identifiers. + overrides_allowed: whether we're parsing the regular expectations + or the overridding expectations""" self._test_to_expectations[test] = expectations for expectation in expectations: self._expectation_to_tests[expectation].add(test) @@ -767,6 +813,9 @@ class TestExpectationsFile: else: self._result_type_to_tests[FAIL].add(test) + if overrides_allowed: + self._overridding_tests.add(test) + def _clear_expectations_for_test(self, test, test_list_path): """Remove prexisting expectations for this test. This happens if we are seeing a more precise path @@ -791,7 +840,8 @@ class TestExpectationsFile: if test in set_of_tests: set_of_tests.remove(test) - def _already_seen_test(self, test, test_list_path, lineno): + def _already_seen_test(self, test, test_list_path, lineno, + allow_overrides): """Returns true if we've already seen a more precise path for this test than the test_list_path. """ @@ -800,8 +850,19 @@ class TestExpectationsFile: prev_base_path = self._test_list_paths[test] if (prev_base_path == os.path.normpath(test_list_path)): - self._add_error(lineno, 'Duplicate expectations.', test) - return True + if (not allow_overrides or test in self._overridding_tests): + if allow_overrides: + expectation_source = "override" + else: + expectation_source = "expectation" + self._add_error(lineno, 'Duplicate %s.' % expectation_source, + test) + return True + else: + # We have seen this path, but that's okay because its + # in the overrides and the earlier path was in the + # expectations. + return False # Check if we've already seen a more precise path. return prev_base_path.startswith(os.path.normpath(test_list_path)) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py new file mode 100644 index 0000000..cf3c560 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py @@ -0,0 +1,168 @@ +#!/usr/bin/python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for test_expectations.py.""" + +import os +import sys +import unittest + +try: + d = os.path.dirname(__file__) +except NameError: + d = os.path.dirname(sys.argv[0]) + +sys.path.append(os.path.abspath(os.path.join(d, '..'))) +sys.path.append(os.path.abspath(os.path.join(d, '../../thirdparty'))) + +import port +from test_expectations import * + +class FunctionsTest(unittest.TestCase): + def test_result_was_expected(self): + # test basics + self.assertEquals(result_was_expected(PASS, set([PASS]), + False, False), True) + self.assertEquals(result_was_expected(TEXT, set([PASS]), + False, False), False) + + # test handling of FAIL expectations + self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]), + False, False), True) + self.assertEquals(result_was_expected(IMAGE, set([FAIL]), + False, False), True) + self.assertEquals(result_was_expected(TEXT, set([FAIL]), + False, False), True) + self.assertEquals(result_was_expected(CRASH, set([FAIL]), + False, False), False) + + # test handling of SKIPped tests and results + self.assertEquals(result_was_expected(SKIP, set([CRASH]), + False, True), True) + self.assertEquals(result_was_expected(SKIP, set([CRASH]), + False, False), False) + + # test handling of MISSING results and the REBASELINE modifier + self.assertEquals(result_was_expected(MISSING, set([PASS]), + True, False), True) + self.assertEquals(result_was_expected(MISSING, set([PASS]), + False, False), False) + + def test_remove_pixel_failures(self): + self.assertEquals(remove_pixel_failures(set([TEXT])), + set([TEXT])) + self.assertEquals(remove_pixel_failures(set([PASS])), + set([PASS])) + self.assertEquals(remove_pixel_failures(set([IMAGE])), + set([PASS])) + self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])), + set([TEXT])) + self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])), + set([PASS, CRASH])) + + +class TestExpectationsTest(unittest.TestCase): + + def __init__(self, testFunc, setUp=None, tearDown=None, description=None): + self._port = port.get('test', None) + self._exp = None + unittest.TestCase.__init__(self, testFunc) + + def get_test(self, test_name): + return os.path.join(self._port.layout_tests_dir(), test_name) + + def get_basic_tests(self): + return [self.get_test('text/article-element.html'), + self.get_test('image/canvas-bg.html'), + self.get_test('image/canvas-zoom.html'), + self.get_test('misc/crash.html'), + self.get_test('misc/passing.html')] + + def get_basic_expectations(self): + return """ +BUG_TEST : text/article-element.html = TEXT +BUG_TEST SKIP : misc/crash.html = CRASH +BUG_TEST REBASELINE : misc/missing-expectation.html = MISSING +BUG_TEST : image = IMAGE +""" + + def parse_exp(self, expectations, overrides=None): + self._exp = TestExpectations(self._port, + tests=self.get_basic_tests(), + expectations=expectations, + test_platform_name=self._port.test_platform_name(), + is_debug_mode=False, + is_lint_mode=False, + tests_are_present=True, + overrides=overrides) + + def assert_exp(self, test, result): + self.assertEquals(self._exp.get_expectations(self.get_test(test)), + set([result])) + + def test_basic(self): + self.parse_exp(self.get_basic_expectations()) + self.assert_exp('text/article-element.html', TEXT) + self.assert_exp('image/canvas-zoom.html', IMAGE) + self.assert_exp('misc/passing.html', PASS) + + def test_duplicates(self): + self.assertRaises(SyntaxError, self.parse_exp, """ +BUG_TEST : text/article-element.html = TEXT +BUG_TEST : text/article-element.html = IMAGE""") + self.assertRaises(SyntaxError, self.parse_exp, + self.get_basic_expectations(), """ +BUG_TEST : text/article-element.html = TEXT +BUG_TEST : text/article-element.html = IMAGE""") + + def test_overrides(self): + self.parse_exp(self.get_basic_expectations(), """ +BUG_OVERRIDE : text/article-element.html = IMAGE""") + self.assert_exp('text/article-element.html', IMAGE) + + def test_matches_an_expected_result(self): + + def match(test, result, pixel_tests_enabled): + return self._exp.matches_an_expected_result( + self.get_test(test), result, pixel_tests_enabled) + + self.parse_exp(self.get_basic_expectations()) + self.assertTrue(match('text/article-element.html', TEXT, True)) + self.assertTrue(match('text/article-element.html', TEXT, False)) + self.assertFalse(match('text/article-element.html', CRASH, True)) + self.assertFalse(match('text/article-element.html', CRASH, False)) + + self.assertTrue(match('image/canvas-bg.html', IMAGE, True)) + self.assertTrue(match('image/canvas-bg.html', PASS, False)) + + self.assertTrue(match('misc/crash.html', SKIP, False)) + self.assertTrue(match('misc/passing.html', PASS, False)) + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py index 56d7b5a..60bdbca 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py @@ -79,8 +79,8 @@ class TestFailure(object): """Returns an HTML string to be included on the results.html page.""" raise NotImplemented - def should_kill_test_shell(self): - """Returns True if we should kill the test shell before the next + def should_kill_dump_render_tree(self): + """Returns True if we should kill DumpRenderTree before the next test.""" return False @@ -110,7 +110,7 @@ class FailureWithType(TestFailure): def __init__(self, test_type): TestFailure.__init__(self) - # TODO(ojan): This class no longer needs to know the test_type. + # FIXME: This class no longer needs to know the test_type. self._test_type = test_type # Filename suffixes used by ResultHtmlOutput. @@ -127,6 +127,9 @@ class FailureWithType(TestFailure): single item is the [actual] filename suffix. If out_names is empty, returns the empty string. """ + # FIXME: Seems like a bad idea to separate the display name data + # from the path data by hard-coding the display name here + # and passing in the path information via out_names. links = [''] uris = [self.relative_output_filename(filename, fn) for fn in out_names] @@ -138,6 +141,8 @@ class FailureWithType(TestFailure): links.append("<a href='%s'>diff</a>" % uris[2]) if len(uris) > 3: links.append("<a href='%s'>wdiff</a>" % uris[3]) + if len(uris) > 4: + links.append("<a href='%s'>pretty diff</a>" % uris[4]) return ' '.join(links) def result_html_output(self, filename): @@ -145,7 +150,7 @@ class FailureWithType(TestFailure): class FailureTimeout(TestFailure): - """Test timed out. We also want to restart the test shell if this + """Test timed out. We also want to restart DumpRenderTree if this happens.""" @staticmethod @@ -155,7 +160,7 @@ class FailureTimeout(TestFailure): def result_html_output(self, filename): return "<strong>%s</strong>" % self.message() - def should_kill_test_shell(self): + def should_kill_dump_render_tree(self): return True @@ -172,7 +177,7 @@ class FailureCrash(TestFailure): return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(), stack) - def should_kill_test_shell(self): + def should_kill_dump_render_tree(self): return True @@ -192,9 +197,10 @@ class FailureMissingResult(FailureWithType): class FailureTextMismatch(FailureWithType): """Text diff output failed.""" # Filename suffixes used by ResultHtmlOutput. + # FIXME: Why don't we use the constants from TestTypeBase here? OUT_FILENAMES = ["-actual.txt", "-expected.txt", "-diff.txt"] OUT_FILENAMES_WDIFF = ["-actual.txt", "-expected.txt", "-diff.txt", - "-wdiff.html"] + "-wdiff.html", "-pretty-diff.html"] def __init__(self, test_type, has_wdiff): FailureWithType.__init__(self, test_type) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_files.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_files.py index 3c087c0..8f79505 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_files.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_files.py @@ -36,9 +36,16 @@ under that directory.""" import glob import os +import time + +from webkitpy.common.system import logutils + + +_log = logutils.get_logger(__file__) + # When collecting test cases, we include any file with these extensions. -_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl', +_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.xhtmlmp', '.pl', '.php', '.svg']) # When collecting test cases, skip these directories _skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests']) @@ -51,9 +58,11 @@ def gather_test_files(port, paths): paths: a list of command line paths relative to the webkit/tests directory. glob patterns are ok. """ + gather_start_time = time.time() paths_to_walk = set() # if paths is empty, provide a pre-defined list. if paths: + _log.debug("Gathering tests from: %s relative to %s" % (paths, port.layout_tests_dir())) for path in paths: # If there's an * in the name, assume it's a glob pattern. path = os.path.join(port.layout_tests_dir(), path) @@ -63,6 +72,7 @@ def gather_test_files(port, paths): else: paths_to_walk.add(path) else: + _log.debug("Gathering tests from: %s" % port.layout_tests_dir()) paths_to_walk.add(port.layout_tests_dir()) # Now walk all the paths passed in on the command line and get filenames @@ -73,10 +83,16 @@ def gather_test_files(port, paths): continue for root, dirs, files in os.walk(path): - # don't walk skipped directories and sub directories + # Don't walk skipped directories or their sub-directories. if os.path.basename(root) in _skipped_directories: del dirs[:] continue + # This copy and for-in is slightly inefficient, but + # the extra walk avoidance consistently shaves .5 seconds + # off of total walk() time on my MacBook Pro. + for directory in dirs[:]: + if directory in _skipped_directories: + dirs.remove(directory) for filename in files: if _has_supported_extension(filename): @@ -84,6 +100,9 @@ def gather_test_files(port, paths): filename = os.path.normpath(filename) test_files.add(filename) + gather_time = time.time() - gather_start_time + _log.debug("Test gathering took %f seconds" % gather_time) + return test_files diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py new file mode 100644 index 0000000..680b848 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import mimetypes +import socket + +from webkitpy.common.net.networktransaction import NetworkTransaction +from webkitpy.thirdparty.autoinstalled.mechanize import Browser + + +def get_mime_type(filename): + return mimetypes.guess_type(filename)[0] or "text/plain" + + +class TestResultsUploader: + def __init__(self, host): + self._host = host + self._browser = Browser() + + def _upload_files(self, attrs, file_objs): + self._browser.open("http://%s/testfile/uploadform" % self._host) + self._browser.select_form("test_result_upload") + for (name, data) in attrs: + self._browser[name] = str(data) + + for (filename, handle) in file_objs: + self._browser.add_file(handle, get_mime_type(filename), filename, + "file") + + self._browser.submit() + + def upload(self, params, files, timeout_seconds): + orig_timeout = socket.getdefaulttimeout() + file_objs = [] + try: + file_objs = [(filename, open(path, "rb")) for (filename, path) + in files] + + socket.setdefaulttimeout(timeout_seconds) + NetworkTransaction(timeout_seconds=timeout_seconds).run( + lambda: self._upload_files(params, file_objs)) + finally: + socket.setdefaulttimeout(orig_timeout) + for (filename, handle) in file_objs: + handle.close() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/__init__.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/__init__.py index 3509675..e3ad6f4 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/__init__.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/__init__.py @@ -29,37 +29,4 @@ """Port-specific entrypoints for the layout tests test infrastructure.""" - -import sys - - -def get(port_name=None, options=None): - """Returns an object implementing the Port interface. If - port_name is None, this routine attempts to guess at the most - appropriate port on this platform.""" - port_to_use = port_name - if port_to_use is None: - if sys.platform == 'win32': - port_to_use = 'chromium-win' - elif sys.platform == 'linux2': - port_to_use = 'chromium-linux' - elif sys.platform == 'darwin': - port_to_use = 'chromium-mac' - - if port_to_use == 'test': - import test - return test.TestPort(port_name, options) - elif port_to_use.startswith('mac'): - import mac - return mac.MacPort(port_name, options) - elif port_to_use.startswith('chromium-mac'): - import chromium_mac - return chromium_mac.ChromiumMacPort(port_name, options) - elif port_to_use.startswith('chromium-linux'): - import chromium_linux - return chromium_linux.ChromiumLinuxPort(port_name, options) - elif port_to_use.startswith('chromium-win'): - import chromium_win - return chromium_win.ChromiumWinPort(port_name, options) - - raise NotImplementedError('unsupported port: %s' % port_to_use) +from factory import get diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/apache_http_server.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/apache_http_server.py index 9ff3671..46617f6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/apache_http_server.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/apache_http_server.py @@ -29,6 +29,10 @@ """A class to start/stop the apache http server used by layout tests.""" + +from __future__ import with_statement + +import codecs import logging import optparse import os @@ -38,6 +42,8 @@ import sys import http_server_base +_log = logging.getLogger("webkitpy.layout_tests.port.apache_http_server") + class LayoutTestApacheHttpd(http_server_base.HttpServerBase): @@ -77,14 +83,15 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): error_log = self._cygwin_safe_join(output_dir, "error_log.txt") document_root = self._cygwin_safe_join(test_dir, "http", "tests") + # FIXME: We shouldn't be calling a protected method of _port_obj! executable = self._port_obj._path_to_apache() if self._is_cygwin(): executable = self._get_cygwin_path(executable) cmd = [executable, - '-f', self._get_apache_config_file_path(test_dir, output_dir), - '-C', "\'DocumentRoot %s\'" % document_root, - '-c', "\'Alias /js-test-resources %s\'" % js_test_resources_dir, + '-f', "\"%s\"" % self._get_apache_config_file_path(test_dir, output_dir), + '-C', "\'DocumentRoot \"%s\"\'" % document_root, + '-c', "\'Alias /js-test-resources \"%s\"'" % js_test_resources_dir, '-C', "\'Listen %s\'" % "127.0.0.1:8000", '-C', "\'Listen %s\'" % "127.0.0.1:8081", '-c', "\'TypesConfig \"%s\"\'" % mime_types_path, @@ -148,7 +155,9 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): """ httpd_config = self._port_obj._path_to_apache_config_file() httpd_config_copy = os.path.join(output_dir, "httpd.conf") - httpd_conf = open(httpd_config).read() + # httpd.conf is always utf-8 according to http://archive.apache.org/gnats/10125 + with codecs.open(httpd_config, "r", "utf-8") as httpd_config_file: + httpd_conf = httpd_config_file.read() if self._is_cygwin(): # This is a gross hack, but it lets us use the upstream .conf file # and our checked in cygwin. This tells the server the root @@ -161,9 +170,8 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): httpd_conf = httpd_conf.replace('ServerRoot "/usr"', 'ServerRoot "%s"' % self._get_cygwin_path(cygusr)) - f = open(httpd_config_copy, 'wb') - f.write(httpd_conf) - f.close() + with codecs.open(httpd_config_copy, "w", "utf-8") as file: + file.write(httpd_conf) if self._is_cygwin(): return self._get_cygwin_path(httpd_config_copy) @@ -174,7 +182,7 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): It will listen to 127.0.0.1 on each of the given port. """ return '\n'.join(('<VirtualHost 127.0.0.1:%s>' % port, - 'DocumentRoot %s' % document_root, + 'DocumentRoot "%s"' % document_root, ssl and 'SSLEngine On' or '', '</VirtualHost>', '')) @@ -183,12 +191,15 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): # Use shell=True because we join the arguments into a string for # the sake of Window/Cygwin and it needs quoting that breaks # shell=False. + # FIXME: We should not need to be joining shell arguments into strings. + # shell=True is a trail of tears. + # Note: Not thread safe: http://bugs.python.org/issue2320 self._httpd_proc = subprocess.Popen(self._start_cmd, stderr=subprocess.PIPE, shell=True) err = self._httpd_proc.stderr.read() if len(err): - logging.debug(err) + _log.debug(err) return False return True @@ -197,22 +208,23 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): # Stop any currently running servers. self.stop() - logging.debug("Starting apache http server") + _log.debug("Starting apache http server") server_started = self.wait_for_action(self._start_httpd_process) if server_started: - logging.debug("Apache started. Testing ports") + _log.debug("Apache started. Testing ports") server_started = self.wait_for_action( self.is_server_running_on_all_ports) if server_started: - logging.debug("Server successfully started") + _log.debug("Server successfully started") else: raise Exception('Failed to start http server') def stop(self): """Stops the apache http server.""" - logging.debug("Shutting down any running http servers") + _log.debug("Shutting down any running http servers") httpd_pid = None if os.path.exists(self._pid_file): httpd_pid = int(open(self._pid_file).readline()) + # FIXME: We shouldn't be calling a protected method of _port_obj! self._port_obj._shut_down_http_server(httpd_pid) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py index ce06b44..d226e64 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py @@ -34,29 +34,63 @@ import cgi import difflib import errno import os -import subprocess +import shlex import sys +import time import apache_http_server import http_server import websocket_server -# Python bug workaround. See Port.wdiff_text() for an explanation. -_wdiff_available = True +from webkitpy.common.system import logutils +from webkitpy.common.system.executive import Executive, ScriptError + + +_log = logutils.get_logger(__file__) +# Python's Popen has a bug that causes any pipes opened to a +# process that can't be executed to be leaked. Since this +# code is specifically designed to tolerate exec failures +# to gracefully handle cases where wdiff is not installed, +# the bug results in a massive file descriptor leak. As a +# workaround, if an exec failure is ever experienced for +# wdiff, assume it's not available. This will leak one +# file descriptor but that's better than leaking each time +# wdiff would be run. +# +# http://mail.python.org/pipermail/python-list/ +# 2008-August/505753.html +# http://bugs.python.org/issue3210 +_wdiff_available = True +_pretty_patch_available = True + # FIXME: This class should merge with webkitpy.webkit_port at some point. class Port(object): """Abstract class for Port-specific hooks for the layout_test package. """ - def __init__(self, port_name=None, options=None): + @staticmethod + def flag_from_configuration(configuration): + flags_by_configuration = { + "Debug": "--debug", + "Release": "--release", + } + return flags_by_configuration[configuration] + + def __init__(self, port_name=None, options=None, executive=Executive()): self._name = port_name self._options = options self._helper = None self._http_server = None self._webkit_base_dir = None self._websocket_server = None + self._executive = executive + + def default_child_processes(self): + """Return the number of DumpRenderTree instances to use for this + port.""" + return self._executive.cpu_count() def baseline_path(self): """Return the absolute path to the directory to store new baselines @@ -68,49 +102,65 @@ class Port(object): baselines. The directories are searched in order.""" raise NotImplementedError('Port.baseline_search_path') - def check_sys_deps(self): + def check_build(self, needs_http): + """This routine is used to ensure that the build is up to date + and all the needed binaries are present.""" + raise NotImplementedError('Port.check_build') + + def check_sys_deps(self, needs_http): """If the port needs to do some runtime checks to ensure that the - tests can be run successfully, they should be done here. + tests can be run successfully, it should override this routine. + This step can be skipped with --nocheck-sys-deps. Returns whether the system is properly configured.""" - raise NotImplementedError('Port.check_sys_deps') + return True - def compare_text(self, actual_text, expected_text): + def check_image_diff(self, override_step=None, logging=True): + """This routine is used to check whether image_diff binary exists.""" + raise NotImplemented('Port.check_image_diff') + + def compare_text(self, expected_text, actual_text): """Return whether or not the two strings are *not* equal. This routine is used to diff text output. While this is a generic routine, we include it in the Port interface so that it can be overriden for testing purposes.""" - return actual_text != expected_text + return expected_text != actual_text - def diff_image(self, actual_filename, expected_filename, diff_filename): + def diff_image(self, expected_filename, actual_filename, + diff_filename=None, tolerance=0): """Compare two image files and produce a delta image file. - Return 1 if the two files are different, 0 if they are the same. + Return True if the two files are different, False if they are the same. Also produce a delta image of the two images and write that into - |diff_filename|. + |diff_filename| if it is not None. + + |tolerance| should be a percentage value (0.0 - 100.0). + If it is omitted, the port default tolerance value is used. While this is a generic routine, we include it in the Port interface so that it can be overriden for testing purposes.""" executable = self._path_to_image_diff() - cmd = [executable, '--diff', actual_filename, expected_filename, - diff_filename] - result = 1 + + if diff_filename: + cmd = [executable, '--diff', expected_filename, actual_filename, + diff_filename] + else: + cmd = [executable, expected_filename, actual_filename] + + result = True try: - result = subprocess.call(cmd) + if self._executive.run_command(cmd, return_exit_code=True) == 0: + return False except OSError, e: if e.errno == errno.ENOENT or e.errno == errno.EACCES: _compare_available = False else: raise e - except ValueError: - # work around a race condition in Python 2.4's implementation - # of subprocess.Popen. See http://bugs.python.org/issue1199282 . - pass return result - def diff_text(self, actual_text, expected_text, - actual_filename, expected_filename): + def diff_text(self, expected_text, actual_text, + expected_filename, actual_filename): """Returns a string containing the diff of the two text strings in 'unified diff' format. @@ -122,6 +172,13 @@ class Port(object): actual_filename) return ''.join(diff) + def driver_name(self): + """Returns the name of the actual binary that is performing the test, + so that it can be referred to in log messages. In most cases this + will be DumpRenderTree, but if a port uses a binary with a different + name, it can be overridden here.""" + return "DumpRenderTree" + def expected_baselines(self, filename, suffix, all_baselines=False): """Given a test name, finds where the baseline results are located. @@ -260,21 +317,28 @@ class Port(object): may be different (e.g., 'win-xp' instead of 'chromium-win-xp'.""" return self._name - def num_cores(self): - """Return the number of cores/cpus available on this machine. - - This routine is used to determine the default amount of parallelism - used by run-chromium-webkit-tests.""" - raise NotImplementedError('Port.num_cores') - + # FIXME: This could be replaced by functions in webkitpy.common.checkout.scm. def path_from_webkit_base(self, *comps): """Returns the full path to path made by joining the top of the WebKit source tree and the list of path components in |*comps|.""" if not self._webkit_base_dir: abspath = os.path.abspath(__file__) self._webkit_base_dir = abspath[0:abspath.find('WebKitTools')] + _log.debug("Using WebKit root: %s" % self._webkit_base_dir) + return os.path.join(self._webkit_base_dir, *comps) + # FIXME: Callers should eventually move to scm.script_path. + def script_path(self, script_name): + return self.path_from_webkit_base("WebKitTools", "Scripts", script_name) + + def path_to_test_expectations_file(self): + """Update the test expectations to the passed-in string. + + This is used by the rebaselining tool. Raises NotImplementedError + if the port does not use expectations files.""" + raise NotImplementedError('Port.path_to_test_expectations_file') + def remove_directory(self, *path): """Recursively removes a directory, even if it's marked read-only. @@ -307,7 +371,7 @@ class Port(object): win32 = False def remove_with_retry(rmfunc, path): - os.chmod(path, stat.S_IWRITE) + os.chmod(path, os.stat.S_IWRITE) if win32: win32api.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_NORMAL) @@ -346,6 +410,8 @@ class Port(object): """Relative unix-style path for a filename under the LayoutTests directory. Filenames outside the LayoutTests directory should raise an error.""" + # FIXME This should assert() here but cannot due to printing_unittest.Testprinter + # assert(filename.startswith(self.layout_tests_dir())) return filename[len(self.layout_tests_dir()) + 1:] def results_directory(self): @@ -353,24 +419,32 @@ class Port(object): raise NotImplemented('Port.results_directory') def setup_test_run(self): - """This routine can be overridden to perform any port-specific - work that shouuld be done at the beginning of a test run.""" + """Perform port-specific work at the beginning of a test run.""" pass + def setup_environ_for_server(self): + """Perform port-specific work at the beginning of a server launch. + + Returns: + Operating-system's environment. + """ + return os.environ.copy() + def show_html_results_file(self, results_filename): """This routine should display the HTML file pointed at by results_filename in a users' browser.""" raise NotImplementedError('Port.show_html_results_file') - def start_driver(self, png_path, options): - """Starts a new test Driver and returns a handle to the object.""" - raise NotImplementedError('Port.start_driver') + def create_driver(self, png_path, options): + """Return a newly created base.Driver subclass for starting/stopping + the test driver.""" + raise NotImplementedError('Port.create_driver') def start_helper(self): - """Start a layout test helper if needed on this port. The test helper - is used to reconfigure graphics settings and do other things that - may be necessary to ensure a known test configuration.""" - raise NotImplementedError('Port.start_helper') + """If a port needs to reconfigure graphics settings or do other + things to ensure a known test configuration, it should override this + method.""" + pass def start_http_server(self): """Start a web server if it is available. Do nothing if @@ -394,8 +468,9 @@ class Port(object): def stop_helper(self): """Shut down the test helper if it is running. Do nothing if - it isn't, or it isn't available.""" - raise NotImplementedError('Port.stop_helper') + it isn't, or it isn't available. If a port overrides start_helper() + it must override this routine as well.""" + pass def stop_http_server(self): """Shut down the http server if it is running. Do nothing if @@ -416,6 +491,15 @@ class Port(object): test_expectations file. See test_expectations.py for more details.""" raise NotImplementedError('Port.test_expectations') + def test_expectations_overrides(self): + """Returns an optional set of overrides for the test_expectations. + + This is used by ports that have code in two repositories, and where + it is possible that you might need "downstream" expectations that + temporarily override the "upstream" expectations until the port can + sync up the two repos.""" + return None + def test_base_platform_names(self): """Return a list of the 'base' platforms on your port. The base platforms represent different architectures, operating systems, @@ -444,6 +528,12 @@ class Port(object): might return 'mac' as a test_platform name'.""" raise NotImplementedError('Port.platforms') + def test_platform_name_to_name(self, test_platform_name): + """Returns the Port platform name that corresponds to the name as + referenced in the expectations file. E.g., "mac" returns + "chromium-mac" on the Chromium ports.""" + raise NotImplementedError('Port.test_platform_name_to_name') + def version(self): """Returns a string indicating the version of a given platform, e.g. '-leopard' or '-xp'. @@ -452,60 +542,99 @@ class Port(object): expectations, determining search paths, and logging information.""" raise NotImplementedError('Port.version') + def test_repository_paths(self): + """Returns a list of (repository_name, repository_path) tuples + of its depending code base. By default it returns a list that only + contains a ('webkit', <webkitRepossitoryPath>) tuple. + """ + return [('webkit', self.layout_tests_dir())] + + + _WDIFF_DEL = '##WDIFF_DEL##' + _WDIFF_ADD = '##WDIFF_ADD##' + _WDIFF_END = '##WDIFF_END##' + + def _format_wdiff_output_as_html(self, wdiff): + wdiff = cgi.escape(wdiff) + wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>") + wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>") + wdiff = wdiff.replace(self._WDIFF_END, "</span>") + html = "<head><style>.del { background: #faa; } " + html += ".add { background: #afa; }</style></head>" + html += "<pre>%s</pre>" % wdiff + return html + + def _wdiff_command(self, actual_filename, expected_filename): + executable = self._path_to_wdiff() + return [executable, + "--start-delete=%s" % self._WDIFF_DEL, + "--end-delete=%s" % self._WDIFF_END, + "--start-insert=%s" % self._WDIFF_ADD, + "--end-insert=%s" % self._WDIFF_END, + actual_filename, + expected_filename] + + @staticmethod + def _handle_wdiff_error(script_error): + # Exit 1 means the files differed, any other exit code is an error. + if script_error.exit_code != 1: + raise script_error + + def _run_wdiff(self, actual_filename, expected_filename): + """Runs wdiff and may throw exceptions. + This is mostly a hook for unit testing.""" + # Diffs are treated as binary as they may include multiple files + # with conflicting encodings. Thus we do not decode the output. + command = self._wdiff_command(actual_filename, expected_filename) + wdiff = self._executive.run_command(command, decode_output=False, + error_handler=self._handle_wdiff_error) + return self._format_wdiff_output_as_html(wdiff) + def wdiff_text(self, actual_filename, expected_filename): """Returns a string of HTML indicating the word-level diff of the contents of the two filenames. Returns an empty string if word-level diffing isn't available.""" - executable = self._path_to_wdiff() - cmd = [executable, - '--start-delete=##WDIFF_DEL##', - '--end-delete=##WDIFF_END##', - '--start-insert=##WDIFF_ADD##', - '--end-insert=##WDIFF_END##', - expected_filename, - actual_filename] - global _wdiff_available - result = '' + global _wdiff_available # See explaination at top of file. + if not _wdiff_available: + return "" try: - # Python's Popen has a bug that causes any pipes opened to a - # process that can't be executed to be leaked. Since this - # code is specifically designed to tolerate exec failures - # to gracefully handle cases where wdiff is not installed, - # the bug results in a massive file descriptor leak. As a - # workaround, if an exec failure is ever experienced for - # wdiff, assume it's not available. This will leak one - # file descriptor but that's better than leaking each time - # wdiff would be run. - # - # http://mail.python.org/pipermail/python-list/ - # 2008-August/505753.html - # http://bugs.python.org/issue3210 - # - # It also has a threading bug, so we don't output wdiff if - # the Popen raises a ValueError. - # http://bugs.python.org/issue1236 - if _wdiff_available: - try: - wdiff = subprocess.Popen(cmd, - stdout=subprocess.PIPE).communicate()[0] - except ValueError, e: - # Working around a race in Python 2.4's implementation - # of Popen(). - wdiff = '' - wdiff = cgi.escape(wdiff) - wdiff = wdiff.replace('##WDIFF_DEL##', '<span class=del>') - wdiff = wdiff.replace('##WDIFF_ADD##', '<span class=add>') - wdiff = wdiff.replace('##WDIFF_END##', '</span>') - result = '<head><style>.del { background: #faa; } ' - result += '.add { background: #afa; }</style></head>' - result += '<pre>' + wdiff + '</pre>' + # It's possible to raise a ScriptError we pass wdiff invalid paths. + return self._run_wdiff(actual_filename, expected_filename) except OSError, e: - if (e.errno == errno.ENOENT or e.errno == errno.EACCES or - e.errno == errno.ECHILD): + if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]: + # Silently ignore cases where wdiff is missing. _wdiff_available = False - else: - raise e - return result + return "" + raise + assert(False) # Should never be reached. + + _pretty_patch_error_html = "Failed to run PrettyPatch, see error console." + + def pretty_patch_text(self, diff_path): + # FIXME: Much of this function could move to prettypatch.rb + global _pretty_patch_available + if not _pretty_patch_available: + return self._pretty_patch_error_html + pretty_patch_path = self.path_from_webkit_base("BugsSite", "PrettyPatch") + prettify_path = os.path.join(pretty_patch_path, "prettify.rb") + command = ["ruby", "-I", pretty_patch_path, prettify_path, diff_path] + try: + # Diffs are treated as binary (we pass decode_output=False) as they + # may contain multiple files of conflicting encodings. + return self._executive.run_command(command, decode_output=False) + except OSError, e: + # If the system is missing ruby log the error and stop trying. + _pretty_patch_available = False + _log.error("Failed to run PrettyPatch (%s): %s" % (command, e)) + return self._pretty_patch_error_html + except ScriptError, e: + # If ruby failed to run for some reason, log the command output and stop trying. + _pretty_patch_available = False + _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output())) + return self._pretty_patch_error_html + + def default_configuration(self): + return "Release" # # PROTECTED ROUTINES @@ -514,13 +643,6 @@ class Port(object): # or any of its subclasses. # - def _kill_process(self, pid): - """Forcefully kill a process. - - This routine should not be used or needed generically, but can be - used in helper files like http_server.py.""" - raise NotImplementedError('Port.kill_process') - def _path_to_apache(self): """Returns the full path to the apache binary. @@ -533,7 +655,7 @@ class Port(object): This is needed only by ports that use the apache_http_server module.""" raise NotImplementedError('Port.path_to_apache_config_file') - def _path_to_driver(self): + def _path_to_driver(self, configuration=None): """Returns the full path to the test driver (DumpRenderTree).""" raise NotImplementedError('Port.path_to_driver') @@ -631,6 +753,24 @@ class Driver: specified in the __init__() call.""" raise NotImplementedError('Driver.run_test') + # FIXME: This is static so we can test it w/o creating a Base instance. + @classmethod + def _command_wrapper(cls, wrapper_option): + # Hook for injecting valgrind or other runtime instrumentation, + # used by e.g. tools/valgrind/valgrind_tests.py. + wrapper = [] + browser_wrapper = os.environ.get("BROWSER_WRAPPER", None) + if browser_wrapper: + # FIXME: There seems to be no reason to use BROWSER_WRAPPER over --wrapper. + # Remove this code any time after the date listed below. + _log.error("BROWSER_WRAPPER is deprecated, please use --wrapper instead.") + _log.error("BROWSER_WRAPPER will be removed any time after June 1st 2010 and your scripts will break.") + wrapper += [browser_wrapper] + + if wrapper_option: + wrapper += shlex.split(wrapper_option) + return wrapper + def poll(self): """Returns None if the Driver is still running. Returns the returncode if it has exited.""" diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py new file mode 100644 index 0000000..f821353 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py @@ -0,0 +1,126 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import base +import unittest +import tempfile + +from webkitpy.common.system.executive import Executive, ScriptError +from webkitpy.thirdparty.mock import Mock + + +class PortTest(unittest.TestCase): + + def test_format_wdiff_output_as_html(self): + output = "OUTPUT %s %s %s" % (base.Port._WDIFF_DEL, base.Port._WDIFF_ADD, base.Port._WDIFF_END) + html = base.Port()._format_wdiff_output_as_html(output) + expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>" + self.assertEqual(html, expected_html) + + def test_wdiff_command(self): + port = base.Port() + port._path_to_wdiff = lambda: "/path/to/wdiff" + command = port._wdiff_command("/actual/path", "/expected/path") + expected_command = [ + "/path/to/wdiff", + "--start-delete=##WDIFF_DEL##", + "--end-delete=##WDIFF_END##", + "--start-insert=##WDIFF_ADD##", + "--end-insert=##WDIFF_END##", + "/actual/path", + "/expected/path", + ] + self.assertEqual(command, expected_command) + + def _file_with_contents(self, contents, encoding="utf-8"): + new_file = tempfile.NamedTemporaryFile() + new_file.write(contents.encode(encoding)) + new_file.flush() + return new_file + + def test_run_wdiff(self): + executive = Executive() + # This may fail on some systems. We could ask the port + # object for the wdiff path, but since we don't know what + # port object to use, this is sufficient for now. + try: + wdiff_path = executive.run_command(["which", "wdiff"]).rstrip() + except Exception, e: + wdiff_path = None + + port = base.Port() + port._path_to_wdiff = lambda: wdiff_path + + if wdiff_path: + # "with tempfile.NamedTemporaryFile() as actual" does not seem to work in Python 2.5 + actual = self._file_with_contents(u"foo") + expected = self._file_with_contents(u"bar") + wdiff = port._run_wdiff(actual.name, expected.name) + expected_wdiff = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre><span class=del>foo</span><span class=add>bar</span></pre>" + self.assertEqual(wdiff, expected_wdiff) + # Running the full wdiff_text method should give the same result. + base._wdiff_available = True # In case it's somehow already disabled. + wdiff = port.wdiff_text(actual.name, expected.name) + self.assertEqual(wdiff, expected_wdiff) + # wdiff should still be available after running wdiff_text with a valid diff. + self.assertTrue(base._wdiff_available) + actual.close() + expected.close() + + # Bogus paths should raise a script error. + self.assertRaises(ScriptError, port._run_wdiff, "/does/not/exist", "/does/not/exist2") + self.assertRaises(ScriptError, port.wdiff_text, "/does/not/exist", "/does/not/exist2") + # wdiff will still be available after running wdiff_text with invalid paths. + self.assertTrue(base._wdiff_available) + base._wdiff_available = True + + # If wdiff does not exist _run_wdiff should throw an OSError. + port._path_to_wdiff = lambda: "/invalid/path/to/wdiff" + self.assertRaises(OSError, port._run_wdiff, "foo", "bar") + + # wdiff_text should not throw an error if wdiff does not exist. + self.assertEqual(port.wdiff_text("foo", "bar"), "") + # However wdiff should not be available after running wdiff_text if wdiff is missing. + self.assertFalse(base._wdiff_available) + base._wdiff_available = True + + +class DriverTest(unittest.TestCase): + + def _assert_wrapper(self, wrapper_string, expected_wrapper): + wrapper = base.Driver._command_wrapper(wrapper_string) + self.assertEqual(wrapper, expected_wrapper) + + def test_command_wrapper(self): + self._assert_wrapper(None, []) + self._assert_wrapper("valgrind", ["valgrind"]) + + # Validate that shlex works as expected. + command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo" + expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"] + self._assert_wrapper(command_with_spaces, expected_parse) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py index 70a8dea..f8b181c 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -29,118 +29,227 @@ """Chromium implementations of the Port interface.""" +from __future__ import with_statement + +import codecs import logging import os +import re import shutil import signal import subprocess import sys import time +import webbrowser import base import http_server + +from webkitpy.common.system.executive import Executive + +# Chromium DRT on non-Windows uses WebKitDriver. +if sys.platform not in ('win32', 'cygwin'): + import webkit + import websocket_server +_log = logging.getLogger("webkitpy.layout_tests.port.chromium") + + +# FIXME: This function doesn't belong in this package. +def check_file_exists(path_to_file, file_description, override_step=None, + logging=True): + """Verify the file is present where expected or log an error. + + Args: + file_name: The (human friendly) name or description of the file + you're looking for (e.g., "HTTP Server"). Used for error logging. + override_step: An optional string to be logged if the check fails. + logging: Whether or not log the error messages.""" + if not os.path.exists(path_to_file): + if logging: + _log.error('Unable to find %s' % file_description) + _log.error(' at %s' % path_to_file) + if override_step: + _log.error(' %s' % override_step) + _log.error('') + return False + return True + class ChromiumPort(base.Port): """Abstract base class for Chromium implementations of the Port class.""" - def __init__(self, port_name=None, options=None): - base.Port.__init__(self, port_name, options) + def __init__(self, port_name=None, options=None, **kwargs): + base.Port.__init__(self, port_name, options, **kwargs) self._chromium_base_dir = None def baseline_path(self): - return self._chromium_baseline_path(self._name) + return self._webkit_baseline_path(self._name) - def check_sys_deps(self): + def check_build(self, needs_http): result = True - test_shell_binary_path = self._path_to_driver() - if os.path.exists(test_shell_binary_path): - proc = subprocess.Popen([test_shell_binary_path, - '--check-layout-test-sys-deps']) - if proc.wait() != 0: - logging.error("Aborting because system dependencies check " - "failed.") - logging.error("To override, invoke with --nocheck-sys-deps") - result = False + + dump_render_tree_binary_path = self._path_to_driver() + result = check_file_exists(dump_render_tree_binary_path, + 'test driver') and result + if result and self._options.build: + result = self._check_driver_build_up_to_date( + self._options.configuration) else: - logging.error('test driver is not found at %s' % - test_shell_binary_path) - result = False + _log.error('') - image_diff_path = self._path_to_image_diff() - if (not os.path.exists(image_diff_path) and not - self._options.no_pixel_tests): - logging.error('image diff not found at %s' % image_diff_path) - logging.error("To override, invoke with --no-pixel-tests") - result = False + helper_path = self._path_to_helper() + if helper_path: + result = check_file_exists(helper_path, + 'layout test helper') and result + + if self._options.pixel_tests: + result = self.check_image_diff( + 'To override, invoke with --no-pixel-tests') and result return result - def compare_text(self, actual_text, expected_text): - return actual_text != expected_text + def check_sys_deps(self, needs_http): + cmd = [self._path_to_driver(), '--check-layout-test-sys-deps'] + if self._executive.run_command(cmd, return_exit_code=True): + _log.error('System dependencies check failed.') + _log.error('To override, invoke with --nocheck-sys-deps') + _log.error('') + return False + return True + + def check_image_diff(self, override_step=None, logging=True): + image_diff_path = self._path_to_image_diff() + return check_file_exists(image_diff_path, 'image diff exe', + override_step, logging) + + def driver_name(self): + return "test_shell" def path_from_chromium_base(self, *comps): """Returns the full path to path made by joining the top of the Chromium source tree and the list of path components in |*comps|.""" if not self._chromium_base_dir: abspath = os.path.abspath(__file__) - self._chromium_base_dir = abspath[0:abspath.find('third_party')] + offset = abspath.find('third_party') + if offset == -1: + self._chromium_base_dir = os.path.join( + abspath[0:abspath.find('WebKitTools')], + 'WebKit', 'chromium') + else: + self._chromium_base_dir = abspath[0:offset] return os.path.join(self._chromium_base_dir, *comps) + def path_to_test_expectations_file(self): + return self.path_from_webkit_base('LayoutTests', 'platform', + 'chromium', 'test_expectations.txt') + def results_directory(self): - return self.path_from_chromium_base('webkit', self._options.target, - self._options.results_directory) + try: + return self.path_from_chromium_base('webkit', + self._options.configuration, self._options.results_directory) + except AssertionError: + return self._build_path(self._options.configuration, + self._options.results_directory) def setup_test_run(self): # Delete the disk cache if any to ensure a clean test run. - test_shell_binary_path = self._path_to_driver() - cachedir = os.path.split(test_shell_binary_path)[0] + dump_render_tree_binary_path = self._path_to_driver() + cachedir = os.path.split(dump_render_tree_binary_path)[0] cachedir = os.path.join(cachedir, "cache") if os.path.exists(cachedir): shutil.rmtree(cachedir) def show_results_html_file(self, results_filename): - subprocess.Popen([self._path_to_driver(), - self.filename_to_uri(results_filename)]) + uri = self.filename_to_uri(results_filename) + if self._options.use_drt: + # FIXME: This should use User.open_url + webbrowser.open(uri, new=1) + else: + # Note: Not thread safe: http://bugs.python.org/issue2320 + subprocess.Popen([self._path_to_driver(), uri]) - def start_driver(self, image_path, options): + def create_driver(self, image_path, options): """Starts a new Driver and returns a handle to it.""" - return ChromiumDriver(self, image_path, options) + if self._options.use_drt and sys.platform not in ('win32', 'cygwin'): + return webkit.WebKitDriver(self, image_path, options, executive=self._executive) + if self._options.use_drt: + options += ['--test-shell'] + else: + options += ['--layout-tests'] + return ChromiumDriver(self, image_path, options, executive=self._executive) def start_helper(self): helper_path = self._path_to_helper() if helper_path: - logging.debug("Starting layout helper %s" % helper_path) + _log.debug("Starting layout helper %s" % helper_path) + # Note: Not thread safe: http://bugs.python.org/issue2320 self._helper = subprocess.Popen([helper_path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None) is_ready = self._helper.stdout.readline() if not is_ready.startswith('ready'): - logging.error("layout_test_helper failed to be ready") + _log.error("layout_test_helper failed to be ready") def stop_helper(self): if self._helper: - logging.debug("Stopping layout test helper") + _log.debug("Stopping layout test helper") self._helper.stdin.write("x\n") self._helper.stdin.close() + # wait() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 self._helper.wait() def test_base_platform_names(self): return ('linux', 'mac', 'win') - def test_expectations(self, options=None): + def test_expectations(self): """Returns the test expectations for this port. Basically this string should contain the equivalent of a test_expectations file. See test_expectations.py for more details.""" - expectations_file = self.path_from_chromium_base('webkit', 'tools', - 'layout_tests', 'test_expectations.txt') - return file(expectations_file, "r").read() + expectations_path = self.path_to_test_expectations_file() + with codecs.open(expectations_path, "r", "utf-8") as file: + return file.read() + + def test_expectations_overrides(self): + # FIXME: This drt_overrides handling should be removed when we switch + # from tes_shell to DRT. + drt_overrides = '' + if self._options.use_drt: + drt_overrides_path = self.path_from_webkit_base('LayoutTests', + 'platform', 'chromium', 'drt_expectations.txt') + if os.path.exists(drt_overrides_path): + with codecs.open(drt_overrides_path, "r", "utf-8") as file: + drt_overrides = file.read() + + try: + overrides_path = self.path_from_chromium_base('webkit', 'tools', + 'layout_tests', 'test_expectations.txt') + except AssertionError: + return None + if not os.path.exists(overrides_path): + return None + with codecs.open(overrides_path, "r", "utf-8") as file: + return file.read() + drt_overrides def test_platform_names(self): return self.test_base_platform_names() + ('win-xp', 'win-vista', 'win-7') + def test_platform_name_to_name(self, test_platform_name): + if test_platform_name in self.test_platform_names(): + return 'chromium-' + test_platform_name + raise ValueError('Unsupported test_platform_name: %s' % + test_platform_name) + + def test_repository_paths(self): + # Note: for JSON file's backward-compatibility we use 'chrome' rather + # than 'chromium' here. + repos = super(ChromiumPort, self).test_repository_paths() + repos.append(('chrome', self.path_from_chromium_base())) + return repos + # # PROTECTED METHODS # @@ -148,47 +257,109 @@ class ChromiumPort(base.Port): # or any subclasses. # + def _check_driver_build_up_to_date(self, configuration): + if configuration in ('Debug', 'Release'): + try: + debug_path = self._path_to_driver('Debug') + release_path = self._path_to_driver('Release') + + debug_mtime = os.stat(debug_path).st_mtime + release_mtime = os.stat(release_path).st_mtime + + if (debug_mtime > release_mtime and configuration == 'Release' or + release_mtime > debug_mtime and configuration == 'Debug'): + _log.warning('You are not running the most ' + 'recent DumpRenderTree binary. You need to ' + 'pass --debug or not to select between ' + 'Debug and Release.') + _log.warning('') + # This will fail if we don't have both a debug and release binary. + # That's fine because, in this case, we must already be running the + # most up-to-date one. + except OSError: + pass + return True + def _chromium_baseline_path(self, platform): if platform is None: platform = self.name() - return self.path_from_chromium_base('webkit', 'data', 'layout_tests', - 'platform', platform, 'LayoutTests') + return self.path_from_webkit_base('LayoutTests', 'platform', platform) + + def _path_to_image_diff(self): + binary_name = 'image_diff' + if self._options.use_drt: + binary_name = 'ImageDiff' + return self._build_path(self._options.configuration, binary_name) class ChromiumDriver(base.Driver): - """Abstract interface for the DumpRenderTree interface.""" + """Abstract interface for test_shell.""" - def __init__(self, port, image_path, options): + def __init__(self, port, image_path, options, executive=Executive()): self._port = port + self._configuration = port._options.configuration + # FIXME: _options is very confusing, because it's not an Options() element. + # FIXME: These don't need to be passed into the constructor, but could rather + # be passed into .start() self._options = options - self._target = port._options.target self._image_path = image_path + self._executive = executive + def start(self): + # FIXME: Should be an error to call this method twice. cmd = [] - # Hook for injecting valgrind or other runtime instrumentation, - # used by e.g. tools/valgrind/valgrind_tests.py. - wrapper = os.environ.get("BROWSER_WRAPPER", None) - if wrapper != None: - cmd += [wrapper] - if self._port._options.wrapper: - # This split() isn't really what we want -- it incorrectly will - # split quoted strings within the wrapper argument -- but in - # practice it shouldn't come up and the --help output warns - # about it anyway. - cmd += self._options.wrapper.split() - cmd += [port._path_to_driver(), '--layout-tests'] - if options: - cmd += options + # FIXME: We should not be grabbing at self._port._options.wrapper directly. + cmd += self._command_wrapper(self._port._options.wrapper) + cmd += [self._port._path_to_driver()] + if self._options: + cmd += self._options + + # We need to pass close_fds=True to work around Python bug #2320 + # (otherwise we can hang when we kill DumpRenderTree when we are running + # multiple threads). See http://bugs.python.org/issue2320 . + # Note that close_fds isn't supported on Windows, but this bug only + # shows up on Mac and Linux. + close_flag = sys.platform not in ('win32', 'cygwin') self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + close_fds=close_flag) def poll(self): + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 return self._proc.poll() def returncode(self): return self._proc.returncode + def _write_command_and_read_line(self, input=None): + """Returns a tuple: (line, did_crash)""" + try: + if input: + if isinstance(input, unicode): + # TestShell expects utf-8 + input = input.encode("utf-8") + self._proc.stdin.write(input) + # DumpRenderTree text output is always UTF-8. However some tests + # (e.g. webarchive) may spit out binary data instead of text so we + # don't bother to decode the output (for either DRT or test_shell). + line = self._proc.stdout.readline() + # We could assert() here that line correctly decodes as UTF-8. + return (line, False) + except IOError, e: + _log.error("IOError communicating w/ test_shell: " + str(e)) + return (None, True) + + def _test_shell_command(self, uri, timeoutms, checksum): + cmd = uri + if timeoutms: + cmd += ' ' + str(timeoutms) + if checksum: + cmd += ' ' + checksum + cmd += "\n" + return cmd + def run_test(self, uri, timeoutms, checksum): output = [] error = [] @@ -198,21 +369,16 @@ class ChromiumDriver(base.Driver): actual_checksum = None start_time = time.time() - cmd = uri - if timeoutms: - cmd += ' ' + str(timeoutms) - if checksum: - cmd += ' ' + checksum - cmd += "\n" - self._proc.stdin.write(cmd) - line = self._proc.stdout.readline() - while line.rstrip() != "#EOF": + cmd = self._test_shell_command(uri, timeoutms, checksum) + (line, crash) = self._write_command_and_read_line(input=cmd) + + while not crash and line.rstrip() != "#EOF": # Make sure we haven't crashed. if line == '' and self.poll() is not None: # This is hex code 0xc000001d, which is used for abrupt # termination. This happens if we hit ctrl+c from the prompt - # and we happen to be waiting on the test_shell. + # and we happen to be waiting on test_shell. # sdoyon: Not sure for which OS and in what circumstances the # above code is valid. What works for me under Linux to detect # ctrl+c is for the subprocess returncode to be negative @@ -227,9 +393,12 @@ class ChromiumDriver(base.Driver): if line.startswith("#URL:"): actual_uri = line.rstrip()[5:] if uri != actual_uri: - logging.fatal("Test got out of sync:\n|%s|\n|%s|" % - (uri, actual_uri)) - raise AssertionError("test out of sync") + # GURL capitalizes the drive letter of a file URL. + if (not re.search("^file:///[a-z]:", uri) or + uri.lower() != actual_uri.lower()): + _log.fatal("Test got out of sync:\n|%s|\n|%s|" % + (uri, actual_uri)) + raise AssertionError("test out of sync") elif line.startswith("#MD5:"): actual_checksum = line.rstrip()[5:] elif line.startswith("#TEST_TIMED_OUT"): @@ -240,7 +409,7 @@ class ChromiumDriver(base.Driver): else: error.append(line) - line = self._proc.stdout.readline() + (line, crash) = self._write_command_and_read_line(input=None) return (crash, timeout, actual_checksum, ''.join(output), ''.join(error)) @@ -251,10 +420,21 @@ class ChromiumDriver(base.Driver): self._proc.stdout.close() if self._proc.stderr: self._proc.stderr.close() - if (sys.platform not in ('win32', 'cygwin') and - not self._proc.poll()): - # Closing stdin/stdout/stderr hangs sometimes on OS X. - null = open(os.devnull, "w") - subprocess.Popen(["kill", "-9", - str(self._proc.pid)], stderr=null) - null.close() + if sys.platform not in ('win32', 'cygwin'): + # Closing stdin/stdout/stderr hangs sometimes on OS X, + # (see __init__(), above), and anyway we don't want to hang + # the harness if test_shell is buggy, so we wait a couple + # seconds to give test_shell a chance to clean up, but then + # force-kill the process if necessary. + KILL_TIMEOUT = 3.0 + timeout = time.time() + KILL_TIMEOUT + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + while self._proc.poll() is None and time.time() < timeout: + time.sleep(0.1) + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + if self._proc.poll() is None: + _log.warning('stopping test driver timed out, ' + 'killing it') + self._executive.kill_process(self._proc.pid) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py index 8fd5343..4df43e0 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py @@ -27,15 +27,16 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Chromium Mac implementation of the Port interface.""" +"""Chromium Linux implementation of the Port interface.""" +import logging import os -import platform import signal -import subprocess import chromium +_log = logging.getLogger("webkitpy.layout_tests.port.chromium_linux") + class ChromiumLinuxPort(chromium.ChromiumPort): """Chromium Linux implementation of the Port class.""" @@ -43,23 +44,29 @@ class ChromiumLinuxPort(chromium.ChromiumPort): def __init__(self, port_name=None, options=None): if port_name is None: port_name = 'chromium-linux' + if options and not hasattr(options, 'configuration'): + options.configuration = 'Release' chromium.ChromiumPort.__init__(self, port_name, options) def baseline_search_path(self): - return [self.baseline_path(), - self._chromium_baseline_path('chromium-win'), - self._webkit_baseline_path('win'), - self._webkit_baseline_path('mac')] - - def check_sys_deps(self): - # We have no platform-specific dependencies to check. - return True - - def num_cores(self): - num_cores = os.sysconf("SC_NPROCESSORS_ONLN") - if isinstance(num_cores, int) and num_cores > 0: - return num_cores - return 1 + port_names = ["chromium-linux", "chromium-win", "chromium", "win", "mac"] + return map(self._webkit_baseline_path, port_names) + + def check_build(self, needs_http): + result = chromium.ChromiumPort.check_build(self, needs_http) + if needs_http: + if self._options.use_apache: + result = self._check_apache_install() and result + else: + result = self._check_lighttpd_install() and result + result = self._check_wdiff_install() and result + + if not result: + _log.error('For complete Linux build requirements, please see:') + _log.error('') + _log.error(' http://code.google.com/p/chromium/wiki/' + 'LinuxBuildInstructions') + return result def test_platform_name(self): # We use 'linux' instead of 'chromium-linux' in test_expectations.txt. @@ -76,32 +83,61 @@ class ChromiumLinuxPort(chromium.ChromiumPort): def _build_path(self, *comps): base = self.path_from_chromium_base() if os.path.exists(os.path.join(base, 'sconsbuild')): - return self.path_from_chromium_base('sconsbuild', - self._options.target, *comps) - else: - return self.path_from_chromium_base('out', - self._options.target, *comps) - - def _kill_process(self, pid): - """Forcefully kill the process. - - Args: - pid: The id of the process to be killed. - """ - os.kill(pid, signal.SIGKILL) - - def _kill_all_process(self, process_name): - null = open(os.devnull) - subprocess.call(['killall', '-TERM', '-u', os.getenv('USER'), - process_name], stderr=null) - null.close() + return os.path.join(base, 'sconsbuild', *comps) + if os.path.exists(os.path.join(base, 'out', *comps)) or not self._options.use_drt: + return os.path.join(base, 'out', *comps) + base = self.path_from_webkit_base() + if os.path.exists(os.path.join(base, 'sconsbuild')): + return os.path.join(base, 'sconsbuild', *comps) + return os.path.join(base, 'out', *comps) + + def _check_apache_install(self): + result = chromium.check_file_exists(self._path_to_apache(), + "apache2") + result = chromium.check_file_exists(self._path_to_apache_config_file(), + "apache2 config file") and result + if not result: + _log.error(' Please install using: "sudo apt-get install ' + 'apache2 libapache2-mod-php5"') + _log.error('') + return result + + def _check_lighttpd_install(self): + result = chromium.check_file_exists( + self._path_to_lighttpd(), "LigHTTPd executable") + result = chromium.check_file_exists(self._path_to_lighttpd_php(), + "PHP CGI executable") and result + result = chromium.check_file_exists(self._path_to_lighttpd_modules(), + "LigHTTPd modules") and result + if not result: + _log.error(' Please install using: "sudo apt-get install ' + 'lighttpd php5-cgi"') + _log.error('') + return result + + def _check_wdiff_install(self): + result = chromium.check_file_exists(self._path_to_wdiff(), 'wdiff') + if not result: + _log.error(' Please install using: "sudo apt-get install ' + 'wdiff"') + _log.error('') + # FIXME: The ChromiumMac port always returns True. + return result def _path_to_apache(self): - return '/usr/sbin/apache2' + if self._is_redhat_based(): + return '/usr/sbin/httpd' + else: + return '/usr/sbin/apache2' def _path_to_apache_config_file(self): + if self._is_redhat_based(): + config_name = 'fedora-httpd.conf' + else: + config_name = 'apache2-debian-httpd.conf' + return os.path.join(self.layout_tests_dir(), 'http', 'conf', - 'apache2-debian-httpd.conf') + config_name) def _path_to_lighttpd(self): return "/usr/sbin/lighttpd" @@ -112,17 +148,25 @@ class ChromiumLinuxPort(chromium.ChromiumPort): def _path_to_lighttpd_php(self): return "/usr/bin/php-cgi" - def _path_to_driver(self): - return self._build_path('test_shell') + def _path_to_driver(self, configuration=None): + if not configuration: + configuration = self._options.configuration + binary_name = 'test_shell' + if self._options.use_drt: + binary_name = 'DumpRenderTree' + return self._build_path(configuration, binary_name) def _path_to_helper(self): return None - def _path_to_image_diff(self): - return self._build_path('image_diff') - def _path_to_wdiff(self): - return 'wdiff' + if self._is_redhat_based(): + return '/usr/bin/dwdiff' + else: + return '/usr/bin/wdiff' + + def _is_redhat_based(self): + return os.path.exists(os.path.join('/etc', 'redhat-release')) def _shut_down_http_server(self, server_pid): """Shut down the lighttpd web server. Blocks until it's fully @@ -136,8 +180,8 @@ class ChromiumLinuxPort(chromium.ChromiumPort): # TODO(mmoss) This isn't ideal, since it could conflict with # lighttpd processes not started by http_server.py, # but good enough for now. - self._kill_all_process('lighttpd') - self._kill_all_process('apache2') + self._executive.kill_all("lighttpd") + self._executive.kill_all("apache2") else: try: os.kill(server_pid, signal.SIGTERM) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py index 7e7b4ca..abd84ae 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py @@ -29,13 +29,17 @@ """Chromium Mac implementation of the Port interface.""" +import logging import os import platform import signal -import subprocess import chromium +from webkitpy.common.system.executive import Executive + +_log = logging.getLogger("webkitpy.layout_tests.port.chromium_mac") + class ChromiumMacPort(chromium.ChromiumPort): """Chromium Mac implementation of the Port class.""" @@ -43,26 +47,45 @@ class ChromiumMacPort(chromium.ChromiumPort): def __init__(self, port_name=None, options=None): if port_name is None: port_name = 'chromium-mac' + if options and not hasattr(options, 'configuration'): + options.configuration = 'Release' chromium.ChromiumPort.__init__(self, port_name, options) def baseline_search_path(self): - return [self.baseline_path(), - self._webkit_baseline_path('mac' + self.version()), - self._webkit_baseline_path('mac')] - - def check_sys_deps(self): - # We have no specific platform dependencies. - return True - - def num_cores(self): - return int(subprocess.Popen(['sysctl','-n','hw.ncpu'], - stdout=subprocess.PIPE).stdout.read()) + port_names = ["chromium-mac", "chromium", "mac" + self.version(), "mac"] + return map(self._webkit_baseline_path, port_names) + + def check_build(self, needs_http): + result = chromium.ChromiumPort.check_build(self, needs_http) + result = self._check_wdiff_install() and result + if not result: + _log.error('For complete Mac build requirements, please see:') + _log.error('') + _log.error(' http://code.google.com/p/chromium/wiki/' + 'MacBuildInstructions') + return result + + def default_child_processes(self): + # FIXME: we need to run single-threaded for now. See + # https://bugs.webkit.org/show_bug.cgi?id=38553. Unfortunately this + # routine is called right before the logger is configured, so if we + # try to _log.warning(), it gets thrown away. + import sys + sys.stderr.write("Defaulting to one child - see https://bugs.webkit.org/show_bug.cgi?id=38553\n") + return 1 + + def driver_name(self): + """name for this port's equivalent of DumpRenderTree.""" + if self._options.use_drt: + return "DumpRenderTree" + return "TestShell" def test_platform_name(self): # We use 'mac' instead of 'chromium-mac' return 'mac' def version(self): + # FIXME: It's strange that this string is -version, not just version. os_version_string = platform.mac_ver()[0] # e.g. "10.5.6" if not os_version_string: return '-leopard' @@ -79,33 +102,25 @@ class ChromiumMacPort(chromium.ChromiumPort): # def _build_path(self, *comps): - return self.path_from_chromium_base('xcodebuild', self._options.target, - *comps) + path = self.path_from_chromium_base('xcodebuild', *comps) + if os.path.exists(path) or not self._options.use_drt: + return path + return self.path_from_webkit_base('WebKit', 'chromium', 'xcodebuild', + *comps) + + def _check_wdiff_install(self): + try: + # We're ignoring the return and always returning True + self._executive.run_command([self._path_to_wdiff()], error_handler=Executive.ignore_error) + except OSError: + _log.warning('wdiff not found. Install using MacPorts or some ' + 'other means') + return True def _lighttpd_path(self, *comps): return self.path_from_chromium_base('third_party', 'lighttpd', 'mac', *comps) - def _kill_process(self, pid): - """Forcefully kill the process. - - Args: - pid: The id of the process to be killed. - """ - os.kill(pid, signal.SIGKILL) - - def _kill_all_process(self, process_name): - """Kill any processes running under this name.""" - # On Mac OS X 10.6, killall has a new constraint: -SIGNALNAME or - # -SIGNALNUMBER must come first. Example problem: - # $ killall -u $USER -TERM lighttpd - # killall: illegal option -- T - # Use of the earlier -TERM placement is just fine on 10.5. - null = open(os.devnull) - subprocess.call(['killall', '-TERM', '-u', os.getenv('USER'), - process_name], stderr=null) - null.close() - def _path_to_apache(self): return '/usr/sbin/httpd' @@ -114,25 +129,27 @@ class ChromiumMacPort(chromium.ChromiumPort): 'apache2-httpd.conf') def _path_to_lighttpd(self): - return self._lighttp_path('bin', 'lighttp') + return self._lighttpd_path('bin', 'lighttpd') def _path_to_lighttpd_modules(self): - return self._lighttp_path('lib') + return self._lighttpd_path('lib') def _path_to_lighttpd_php(self): return self._lighttpd_path('bin', 'php-cgi') - def _path_to_driver(self): - # TODO(pinkerton): make |target| happy with case-sensitive file + def _path_to_driver(self, configuration=None): + # FIXME: make |configuration| happy with case-sensitive file # systems. - return self._build_path('TestShell.app', 'Contents', 'MacOS', - 'TestShell') + if not configuration: + configuration = self._options.configuration + return self._build_path(configuration, self.driver_name() + '.app', + 'Contents', 'MacOS', self.driver_name()) def _path_to_helper(self): - return self._build_path('layout_test_helper') - - def _path_to_image_diff(self): - return self._build_path('image_diff') + binary_name = 'layout_test_helper' + if self._options.use_drt: + binary_name = 'LayoutTestHelper' + return self._build_path(self._options.configuration, binary_name) def _path_to_wdiff(self): return 'wdiff' @@ -149,8 +166,8 @@ class ChromiumMacPort(chromium.ChromiumPort): # TODO(mmoss) This isn't ideal, since it could conflict with # lighttpd processes not started by http_server.py, # but good enough for now. - self._kill_all_process('lighttpd') - self._kill_all_process('httpd') + self._executive.kill_all('lighttpd') + self._executive.kill_all('httpd') else: try: os.kill(server_pid, signal.SIGTERM) diff --git a/WebKitTools/Scripts/webkitpy/executive_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py index f78e301..d63faa0 100644 --- a/WebKitTools/Scripts/webkitpy/executive_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py @@ -1,5 +1,4 @@ -# Copyright (C) 2009 Google Inc. All rights reserved. -# Copyright (C) 2009 Daniel Bates (dbates@intudata.com). All rights reserved. +# Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -27,15 +26,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import chromium_mac import unittest -from webkitpy.executive import Executive, run_command -class ExecutiveTest(unittest.TestCase): +from webkitpy.thirdparty.mock import Mock - def test_run_command_with_bad_command(self): - def run_bad_command(): - run_command(["foo_bar_command_blah"], error_handler=Executive.ignore_error, return_exit_code=True) - self.failUnlessRaises(OSError, run_bad_command) -if __name__ == '__main__': - unittest.main() +class ChromiumMacPortTest(unittest.TestCase): + + def test_check_wdiff_install(self): + port = chromium_mac.ChromiumMacPort() + # Currently is always true, just logs if missing. + self.assertTrue(port._check_wdiff_install()) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py new file mode 100644 index 0000000..a32eafd --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py @@ -0,0 +1,97 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import chromium +import chromium_linux +import chromium_mac +import chromium_win +import unittest +import StringIO + +from webkitpy.thirdparty.mock import Mock + + +class ChromiumDriverTest(unittest.TestCase): + + def setUp(self): + mock_port = Mock() + # FIXME: The Driver should not be grabbing at port._options! + mock_port._options = Mock() + mock_port._options.wrapper = "" + self.driver = chromium.ChromiumDriver(mock_port, image_path=None, options=None) + + def test_test_shell_command(self): + expected_command = "test.html 2 checksum\n" + self.assertEqual(self.driver._test_shell_command("test.html", 2, "checksum"), expected_command) + + def _assert_write_command_and_read_line(self, input=None, expected_line=None, expected_stdin=None, expected_crash=False): + if not expected_stdin: + if input: + expected_stdin = input + else: + # We reset stdin, so we should expect stdin.getValue = "" + expected_stdin = "" + self.driver._proc.stdin = StringIO.StringIO() + line, did_crash = self.driver._write_command_and_read_line(input) + self.assertEqual(self.driver._proc.stdin.getvalue(), expected_stdin) + self.assertEqual(line, expected_line) + self.assertEqual(did_crash, expected_crash) + + def test_write_command_and_read_line(self): + self.driver._proc = Mock() + # Set up to read 3 lines before we get an IOError + self.driver._proc.stdout = StringIO.StringIO("first\nsecond\nthird\n") + + unicode_input = u"I \u2661 Unicode" + utf8_input = unicode_input.encode("utf-8") + # Test unicode input conversion to utf-8 + self._assert_write_command_and_read_line(input=unicode_input, expected_stdin=utf8_input, expected_line="first\n") + # Test str() input. + self._assert_write_command_and_read_line(input="foo", expected_line="second\n") + # Test input=None + self._assert_write_command_and_read_line(expected_line="third\n") + # Test reading from a closed/empty stream. + # reading from a StringIO does not raise IOError like a real file would, so raise IOError manually. + def mock_readline(): + raise IOError + self.driver._proc.stdout.readline = mock_readline + self._assert_write_command_and_read_line(expected_crash=True) + + def test_path_to_image_diff(self): + class MockOptions: + def __init__(self): + self.use_drt = True + + port = chromium_linux.ChromiumLinuxPort('test-port', options=MockOptions()) + self.assertTrue(port._path_to_image_diff().endswith( + '/out/Release/ImageDiff')) + port = chromium_mac.ChromiumMacPort('test-port', options=MockOptions()) + self.assertTrue(port._path_to_image_diff().endswith( + '/xcodebuild/Release/ImageDiff')) + # FIXME: Figure out how this is going to work on Windows. + #port = chromium_win.ChromiumWinPort('test-port', options=MockOptions()) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py index 352916c..8072bc0 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py @@ -29,46 +29,69 @@ """Chromium Win implementation of the Port interface.""" +import logging import os -import platform -import signal -import subprocess import sys import chromium +_log = logging.getLogger("webkitpy.layout_tests.port.chromium_win") + class ChromiumWinPort(chromium.ChromiumPort): """Chromium Win implementation of the Port class.""" def __init__(self, port_name=None, options=None): if port_name is None: - port_name = 'chromium-win' + self.version() + port_name = "chromium-win" + self.version() + if options and not hasattr(options, "configuration"): + options.configuration = "Release" chromium.ChromiumPort.__init__(self, port_name, options) + def setup_environ_for_server(self): + env = chromium.ChromiumPort.setup_environ_for_server(self) + # Put the cygwin directory first in the path to find cygwin1.dll. + env["PATH"] = "%s;%s" % ( + self.path_from_chromium_base("third_party", "cygwin", "bin"), + env["PATH"]) + # Configure the cygwin directory so that pywebsocket finds proper + # python executable to run cgi program. + env["CYGWIN_PATH"] = self.path_from_chromium_base( + "third_party", "cygwin", "bin") + if (sys.platform == "win32" and self._options and + hasattr(self._options, "register_cygwin") and + self._options.register_cygwin): + setup_mount = self.path_from_chromium_base("third_party", + "cygwin", + "setup_mount.bat") + self._executive.run_command([setup_mount]) + return env + def baseline_search_path(self): - dirs = [] + port_names = [] if self._name == 'chromium-win-xp': - dirs.append(self._chromium_baseline_path(self._name)) + port_names.append("chromium-win-xp") if self._name in ('chromium-win-xp', 'chromium-win-vista'): - dirs.append(self._chromium_baseline_path('chromium-win-vista')) - dirs.append(self._chromium_baseline_path('chromium-win')) - dirs.append(self._webkit_baseline_path('win')) - dirs.append(self._webkit_baseline_path('mac')) - return dirs - - def check_sys_deps(self): - # TODO(dpranke): implement this - return True + port_names.append("chromium-win-vista") + # FIXME: This may need to include mac-snowleopard like win.py. + port_names.extend(["chromium-win", "chromium", "win", "mac"]) + return map(self._webkit_baseline_path, port_names) + + def check_build(self, needs_http): + result = chromium.ChromiumPort.check_build(self, needs_http) + if not result: + _log.error('For complete Windows build requirements, please ' + 'see:') + _log.error('') + _log.error(' http://dev.chromium.org/developers/how-tos/' + 'build-instructions-windows') + return result def get_absolute_path(self, filename): """Return the absolute path in unix format for the given filename.""" abspath = os.path.abspath(filename) return abspath.replace('\\', '/') - def num_cores(self): - return int(os.environ.get('NUMBER_OF_PROCESSORS', 1)) - def relative_test_filename(self, filename): path = filename[len(self.layout_tests_dir()) + 1:] return path.replace('\\', '/') @@ -78,6 +101,8 @@ class ChromiumWinPort(chromium.ChromiumPort): return 'win' + self.version() def version(self): + if not hasattr(sys, 'getwindowsversion'): + return '' winver = sys.getwindowsversion() if winver[0] == 6 and (winver[1] == 1): return '-7' @@ -92,24 +117,19 @@ class ChromiumWinPort(chromium.ChromiumPort): # def _build_path(self, *comps): - # FIXME(dpranke): allow for builds under 'chrome' as well. - return self.path_from_chromium_base('webkit', self._options.target, - *comps) + p = self.path_from_chromium_base('webkit', *comps) + if os.path.exists(p): + return p + p = self.path_from_chromium_base('chrome', *comps) + if os.path.exists(p) or not self._options.use_drt: + return p + return os.path.join(self.path_from_webkit_base(), 'WebKit', 'chromium', + *comps) def _lighttpd_path(self, *comps): return self.path_from_chromium_base('third_party', 'lighttpd', 'win', *comps) - def _kill_process(self, pid): - """Forcefully kill the process. - - Args: - pid: The id of the process to be killed. - """ - subprocess.call(('taskkill.exe', '/f', '/pid', str(pid)), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - def _path_to_apache(self): return self.path_from_chromium_base('third_party', 'cygwin', 'usr', 'sbin', 'httpd') @@ -127,14 +147,25 @@ class ChromiumWinPort(chromium.ChromiumPort): def _path_to_lighttpd_php(self): return self._lighttpd_path('php5', 'php-cgi.exe') - def _path_to_driver(self): - return self._build_path('test_shell.exe') + def _path_to_driver(self, configuration=None): + if not configuration: + configuration = self._options.configuration + binary_name = 'test_shell.exe' + if self._options.use_drt: + binary_name = 'DumpRenderTree.exe' + return self._build_path(configuration, binary_name) def _path_to_helper(self): - return self._build_path('layout_test_helper.exe') + binary_name = 'layout_test_helper.exe' + if self._options.use_drt: + binary_name = 'LayoutTestHelper.exe' + return self._build_path(self._options.configuration, binary_name) def _path_to_image_diff(self): - return self._build_path('image_diff.exe') + binary_name = 'image_diff.exe' + if self._options.use_drt: + binary_name = 'ImageDiff.exe' + return self._build_path(self._options.configuration, binary_name) def _path_to_wdiff(self): return self.path_from_chromium_base('third_party', 'cygwin', 'bin', @@ -147,9 +178,7 @@ class ChromiumWinPort(chromium.ChromiumPort): Args: server_pid: The process ID of the running server. """ - subprocess.Popen(('taskkill.exe', '/f', '/im', 'LightTPD.exe'), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).wait() - subprocess.Popen(('taskkill.exe', '/f', '/im', 'httpd.exe'), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).wait() + # FIXME: Why are we ignoring server_pid and calling + # _kill_all instead of Executive.kill_process(pid)? + self._executive.kill_all("LightTPD.exe") + self._executive.kill_all("httpd.exe") diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py new file mode 100644 index 0000000..81db32c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py @@ -0,0 +1,74 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import sys +import unittest +import chromium_win +from webkitpy.common.system import outputcapture +from webkitpy.tool import mocktool + + +class ChromiumWinTest(unittest.TestCase): + + class RegisterCygwinOption(object): + def __init__(self): + self.register_cygwin = True + + def setUp(self): + self.orig_platform = sys.platform + + def tearDown(self): + sys.platform = self.orig_platform + + def _mock_path_from_chromium_base(self, *comps): + return os.path.join("/chromium/src", *comps) + + def test_setup_environ_for_server(self): + port = chromium_win.ChromiumWinPort() + port._executive = mocktool.MockExecute(True) + port.path_from_chromium_base = self._mock_path_from_chromium_base + output = outputcapture.OutputCapture() + orig_environ = os.environ.copy() + env = output.assert_outputs(self, port.setup_environ_for_server) + self.assertEqual(orig_environ["PATH"], os.environ["PATH"]) + self.assertNotEqual(env["PATH"], os.environ["PATH"]) + + def test_setup_environ_for_server_register_cygwin(self): + sys.platform = "win32" + port = chromium_win.ChromiumWinPort( + options=ChromiumWinTest.RegisterCygwinOption()) + port._executive = mocktool.MockExecute(True) + port.path_from_chromium_base = self._mock_path_from_chromium_base + setup_mount = self._mock_path_from_chromium_base("third_party", + "cygwin", + "setup_mount.bat") + expected_stderr = "MOCK run_command: %s\n" % [setup_mount] + output = outputcapture.OutputCapture() + output.assert_outputs(self, port.setup_environ_for_server, + expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py new file mode 100644 index 0000000..e01bd2f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""This is a test implementation of the Port interface that generates the + correct output for every test. It can be used for perf testing, because + it is pretty much a lower limit on how fast a port can possibly run. + + This implementation acts as a wrapper around a real port (the real port + is held as a delegate object). To specify which port, use the port name + 'dryrun-XXX' (e.g., 'dryrun-chromium-mac-leopard'). If you use just + 'dryrun', it uses the default port. + + Note that because this is really acting as a wrapper around the underlying + port, you must be able to run the underlying port as well + (check_build() and check_sys_deps() must pass and auxiliary binaries + like layout_test_helper and httpd must work). + + This implementation also modifies the test expectations so that all + tests are either SKIPPED or expected to PASS.""" + +from __future__ import with_statement + +import sys + +import base +import factory + + +def _read_file(path, mode='r'): + """Return the contents of a file as a string. + + Returns '' if anything goes wrong, instead of throwing an IOError. + + """ + contents = '' + try: + with open(path, mode) as f: + contents = f.read() + except IOError: + pass + return contents + + +def _write_file(path, contents, mode='w'): + """Write the string to the specified path. + + Returns nothing if the write fails, instead of raising an IOError. + + """ + try: + with open(path, mode) as f: + f.write(contents) + except IOError: + pass + + +class DryRunPort(object): + """DryRun implementation of the Port interface.""" + + def __init__(self, port_name=None, options=None): + pfx = 'dryrun-' + if port_name.startswith(pfx): + port_name = port_name[len(pfx):] + else: + port_name = None + self.__delegate = factory.get(port_name, options) + + def __getattr__(self, name): + return getattr(self.__delegate, name) + + def check_build(self, needs_http): + return True + + def check_sys_deps(self, needs_http): + return True + + def start_helper(self): + pass + + def start_http_server(self): + pass + + def start_websocket_server(self): + pass + + def stop_helper(self): + pass + + def stop_http_server(self): + pass + + def stop_websocket_server(self): + pass + + def create_driver(self, image_path, options): + return DryrunDriver(self, image_path, options) + + +class DryrunDriver(base.Driver): + """Dryrun implementation of the DumpRenderTree / Driver interface.""" + + def __init__(self, port, image_path, test_driver_options): + self._port = port + self._driver_options = test_driver_options + self._image_path = image_path + self._layout_tests_dir = None + + def poll(self): + return None + + def returncode(self): + return 0 + + def run_test(self, uri, timeoutms, image_hash): + test_name = self._uri_to_test(uri) + + text_filename = self._port.expected_filename(test_name, '.txt') + text_output = _read_file(text_filename) + + if image_hash is not None: + image_filename = self._port.expected_filename(test_name, '.png') + image = _read_file(image_filename, 'rb') + if self._image_path: + _write_file(self._image_path, image) + hash_filename = self._port.expected_filename(test_name, + '.checksum') + hash = _read_file(hash_filename) + else: + hash = None + return (False, False, hash, text_output, None) + + def start(self): + pass + + def stop(self): + pass + + def _uri_to_test(self, uri): + """Return the base layout test name for a given URI. + + This returns the test name for a given URI, e.g., if you passed in + "file:///src/LayoutTests/fast/html/keygen.html" it would return + "fast/html/keygen.html". + + """ + if not self._layout_tests_dir: + self._layout_tests_dir = self._port.layout_tests_dir() + test = uri + + if uri.startswith("file:///"): + if sys.platform == 'win32': + test = test.replace('file:///', '') + test = test.replace('/', '\\') + else: + test = test.replace('file://', '') + return test + elif uri.startswith("http://127.0.0.1:8880/"): + # websocket tests + test = test.replace('http://127.0.0.1:8880/', + self._layout_tests_dir + '/') + return test + elif uri.startswith("http://"): + # regular HTTP test + test = test.replace('http://127.0.0.1:8000/', + self._layout_tests_dir + '/http/tests/') + return test + elif uri.startswith("https://"): + test = test.replace('https://127.0.0.1:8443/', + self._layout_tests_dir + '/http/tests/') + return test + else: + raise NotImplementedError('unknown url type: %s' % uri) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py new file mode 100644 index 0000000..95b90da --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Factory method to retrieve the appropriate port implementation.""" + + +import sys + + +def get(port_name=None, options=None): + """Returns an object implementing the Port interface. If + port_name is None, this routine attempts to guess at the most + appropriate port on this platform.""" + port_to_use = port_name + if port_to_use is None: + if sys.platform == 'win32' or sys.platform == 'cygwin': + if options and hasattr(options, 'chromium') and options.chromium: + port_to_use = 'chromium-win' + else: + port_to_use = 'win' + elif sys.platform == 'linux2': + port_to_use = 'chromium-linux' + elif sys.platform == 'darwin': + if options and hasattr(options, 'chromium') and options.chromium: + port_to_use = 'chromium-mac' + else: + port_to_use = 'mac' + + if port_to_use is None: + raise NotImplementedError('unknown port; sys.platform = "%s"' % + sys.platform) + + if port_to_use == 'test': + import test + return test.TestPort(port_name, options) + elif port_to_use.startswith('dryrun'): + import dryrun + return dryrun.DryRunPort(port_name, options) + elif port_to_use.startswith('mac'): + import mac + return mac.MacPort(port_name, options) + elif port_to_use.startswith('win'): + import win + return win.WinPort(port_name, options) + elif port_to_use.startswith('gtk'): + import gtk + return gtk.GtkPort(port_name, options) + elif port_to_use.startswith('qt'): + import qt + return qt.QtPort(port_name, options) + elif port_to_use.startswith('chromium-mac'): + import chromium_mac + return chromium_mac.ChromiumMacPort(port_name, options) + elif port_to_use.startswith('chromium-linux'): + import chromium_linux + return chromium_linux.ChromiumLinuxPort(port_name, options) + elif port_to_use.startswith('chromium-win'): + import chromium_win + return chromium_win.ChromiumWinPort(port_name, options) + + raise NotImplementedError('unsupported port: %s' % port_to_use) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py new file mode 100644 index 0000000..d8dffdf --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py @@ -0,0 +1,138 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys +import unittest + +import chromium_linux +import chromium_mac +import chromium_win +import dryrun +import factory +import gtk +import mac +import qt +import test +import win + + +class FactoryTest(unittest.TestCase): + """Test factory creates proper port object for the target. + + Target is specified by port_name, sys.platform and options. + + """ + # FIXME: The ports themselves should expose what options they require, + # instead of passing generic "options". + + class WebKitOptions(object): + """Represents the minimum options for WebKit port.""" + def __init__(self): + self.pixel_tests = False + + class ChromiumOptions(WebKitOptions): + """Represents minimum options for Chromium port.""" + def __init__(self): + FactoryTest.WebKitOptions.__init__(self) + self.chromium = True + + def setUp(self): + self.real_sys_platform = sys.platform + self.webkit_options = FactoryTest.WebKitOptions() + self.chromium_options = FactoryTest.ChromiumOptions() + + def tearDown(self): + sys.platform = self.real_sys_platform + + def assert_port(self, port_name, expected_port): + """Helper assert for port_name. + + Args: + port_name: port name to get port object. + expected_port: class of expected port object. + + """ + self.assertTrue(isinstance(factory.get(port_name=port_name), + expected_port)) + + def assert_platform_port(self, platform, options, expected_port): + """Helper assert for platform and options. + + Args: + platform: sys.platform. + options: options to get port object. + expected_port: class of expected port object. + + """ + orig_platform = sys.platform + sys.platform = platform + self.assertTrue(isinstance(factory.get(options=options), + expected_port)) + sys.platform = orig_platform + + def test_test(self): + self.assert_port("test", test.TestPort) + + def test_dryrun(self): + self.assert_port("dryrun-test", dryrun.DryRunPort) + self.assert_port("dryrun-mac", dryrun.DryRunPort) + + def test_mac(self): + self.assert_port("mac", mac.MacPort) + self.assert_platform_port("darwin", None, mac.MacPort) + self.assert_platform_port("darwin", self.webkit_options, mac.MacPort) + + def test_win(self): + self.assert_port("win", win.WinPort) + self.assert_platform_port("win32", None, win.WinPort) + self.assert_platform_port("win32", self.webkit_options, win.WinPort) + self.assert_platform_port("cygwin", None, win.WinPort) + self.assert_platform_port("cygwin", self.webkit_options, win.WinPort) + + def test_gtk(self): + self.assert_port("gtk", gtk.GtkPort) + + def test_qt(self): + self.assert_port("qt", qt.QtPort) + + def test_chromium_mac(self): + self.assert_port("chromium-mac", chromium_mac.ChromiumMacPort) + self.assert_platform_port("darwin", self.chromium_options, + chromium_mac.ChromiumMacPort) + + def test_chromium_linux(self): + self.assert_port("chromium-linux", chromium_linux.ChromiumLinuxPort) + self.assert_platform_port("linux2", self.chromium_options, + chromium_linux.ChromiumLinuxPort) + + def test_chromium_win(self): + self.assert_port("chromium-win", chromium_win.ChromiumWinPort) + self.assert_platform_port("win32", self.chromium_options, + chromium_win.ChromiumWinPort) + self.assert_platform_port("cygwin", self.chromium_options, + chromium_win.ChromiumWinPort) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py new file mode 100644 index 0000000..59dc1d9 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py @@ -0,0 +1,84 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""WebKit Gtk implementation of the Port interface.""" + +import logging +import os + +from webkitpy.layout_tests.port.webkit import WebKitPort + +_log = logging.getLogger("webkitpy.layout_tests.port.gtk") + + +class GtkPort(WebKitPort): + """WebKit Gtk implementation of the Port class.""" + + def __init__(self, port_name=None, options=None): + if port_name is None: + port_name = 'gtk' + WebKitPort.__init__(self, port_name, options) + + def _tests_for_other_platforms(self): + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + # This really need to be automated. + return [ + "platform/chromium", + "platform/win", + "platform/qt", + "platform/mac", + ] + + def _path_to_apache_config_file(self): + # FIXME: This needs to detect the distribution and change config files. + return os.path.join(self.layout_tests_dir(), 'http', 'conf', + 'apache2-debian-httpd.conf') + + def _shut_down_http_server(self, server_pid): + """Shut down the httpd web server. Blocks until it's fully + shut down. + + Args: + server_pid: The process ID of the running server. + """ + # server_pid is not set when "http_server.py stop" is run manually. + if server_pid is None: + # FIXME: This isn't ideal, since it could conflict with + # lighttpd processes not started by http_server.py, + # but good enough for now. + self._executive.kill_all('apache2') + else: + try: + os.kill(server_pid, signal.SIGTERM) + # TODO(mmoss) Maybe throw in a SIGKILL just to be sure? + except OSError: + # Sometimes we get a bad PID (e.g. from a stale httpd.pid + # file), so if kill fails on the given PID, just try to + # 'killall' web servers. + self._shut_down_http_server(None) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server.py index 0315704..0f8a21e 100755 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server.py @@ -29,7 +29,9 @@ """A class to help start/stop the lighttpd server used by layout tests.""" +from __future__ import with_statement +import codecs import logging import optparse import os @@ -40,8 +42,11 @@ import tempfile import time import urllib +import factory import http_server_base +_log = logging.getLogger("webkitpy.layout_tests.port.http_server") + class HttpdNotStarted(Exception): pass @@ -50,7 +55,7 @@ class HttpdNotStarted(Exception): class Lighttpd(http_server_base.HttpServerBase): def __init__(self, port_obj, output_dir, background=False, port=None, - root=None, register_cygwin=None, run_background=None): + root=None, run_background=None): """Args: output_dir: the absolute path to the layout test result directory """ @@ -60,7 +65,6 @@ class Lighttpd(http_server_base.HttpServerBase): self._process = None self._port = port self._root = root - self._register_cygwin = register_cygwin self._run_background = run_background if self._port: self._port = int(self._port) @@ -111,11 +115,14 @@ class Lighttpd(http_server_base.HttpServerBase): self.remove_log_files(self._output_dir, "error.log-") # Write out the config - f = file(base_conf_file, 'rb') - base_conf = f.read() - f.close() - - f = file(out_conf_file, 'wb') + with codecs.open(base_conf_file, "r", "utf-8") as file: + base_conf = file.read() + + # FIXME: This should be re-worked so that this block can + # use with open() instead of a manual file.close() call. + # lighttpd.conf files seem to be UTF-8 without BOM: + # http://redmine.lighttpd.net/issues/992 + f = codecs.open(out_conf_file, "w", "utf-8") f.write(base_conf) # Write out our cgi handlers. Run perl through env so that it @@ -191,20 +198,9 @@ class Lighttpd(http_server_base.HttpServerBase): shutil.copyfile(os.path.join(module_path, lib_file), os.path.join(tmp_module_path, lib_file)) - # Put the cygwin directory first in the path to find cygwin1.dll - env = os.environ - if sys.platform in ('cygwin', 'win32'): - env['PATH'] = '%s;%s' % ( - self._port_obj.path_from_chromium_base('third_party', - 'cygwin', 'bin'), - env['PATH']) - - if sys.platform == 'win32' and self._register_cygwin: - setup_mount = port.path_from_chromium_base('third_party', - 'cygwin', 'setup_mount.bat') - subprocess.Popen(setup_mount).wait() - - logging.debug('Starting http server') + env = self._port_obj.setup_environ_for_server() + _log.debug('Starting http server') + # FIXME: Should use Executive.run_command self._process = subprocess.Popen(start_cmd, env=env) # Wait for server to start. @@ -216,7 +212,7 @@ class Lighttpd(http_server_base.HttpServerBase): if not server_started or self._process.returncode != None: raise google.httpd_utils.HttpdNotStarted('Failed to start httpd.') - logging.debug("Server successfully started") + _log.debug("Server successfully started") # TODO(deanm): Find a nicer way to shutdown cleanly. Our log files are # probably not being flushed, etc... why doesn't our python have os.kill ? @@ -231,42 +227,7 @@ class Lighttpd(http_server_base.HttpServerBase): self._port_obj._shut_down_http_server(httpd_pid) if self._process: + # wait() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 self._process.wait() self._process = None - -if '__main__' == __name__: - # Provide some command line params for starting/stopping the http server - # manually. Also used in ui_tests to run http layout tests in a browser. - option_parser = optparse.OptionParser() - option_parser.add_option('-k', '--server', - help='Server action (start|stop)') - option_parser.add_option('-p', '--port', - help='Port to listen on (overrides layout test ports)') - option_parser.add_option('-r', '--root', - help='Absolute path to DocumentRoot (overrides layout test roots)') - option_parser.add_option('--register_cygwin', action="store_true", - dest="register_cygwin", help='Register Cygwin paths (on Win try bots)') - option_parser.add_option('--run_background', action="store_true", - dest="run_background", - help='Run on background (for running as UI test)') - options, args = option_parser.parse_args() - - if not options.server: - print ('Usage: %s --server {start|stop} [--root=root_dir]' - ' [--port=port_number]' % sys.argv[0]) - else: - if (options.root is None) and (options.port is not None): - # specifying root but not port means we want httpd on default - # set of ports that LayoutTest use, but pointing to a different - # source of tests. Specifying port but no root does not seem - # meaningful. - raise 'Specifying port requires also a root.' - httpd = Lighttpd(tempfile.gettempdir(), - port=options.port, - root=options.root, - register_cygwin=options.register_cygwin, - run_background=options.run_background) - if 'start' == options.server: - httpd.start() - else: - httpd.stop(force=True) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server_base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server_base.py index e82943e..2745cce 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server_base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_server_base.py @@ -34,6 +34,8 @@ import os import time import urllib +_log = logging.getLogger("webkitpy.layout_tests.port.http_server_base") + class HttpServerBase(object): @@ -47,6 +49,7 @@ class HttpServerBase(object): while time.time() - start_time < 20: if action(): return True + _log.debug("Waiting for action: %s" % action) time.sleep(1) return False @@ -63,9 +66,9 @@ class HttpServerBase(object): try: response = urllib.urlopen(url) - logging.debug("Server running at %s" % url) - except IOError: - logging.debug("Server NOT running at %s" % url) + _log.debug("Server running at %s" % url) + except IOError, e: + _log.debug("Server NOT running at %s: %s" % (url, e)) return False return True diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf b/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf index d3150dd..2e9c82e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf @@ -21,6 +21,7 @@ mimetype.assign = ( ".html" => "text/html", ".htm" => "text/html", ".xhtml" => "application/xhtml+xml", + ".xhtmlmp" => "application/vnd.wap.xhtml+xml", ".js" => "text/javascript", ".log" => "text/plain", ".conf" => "text/plain", diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py index 4b73cec..413b5f2 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -29,137 +28,63 @@ """WebKit Mac implementation of the Port interface.""" -import fcntl import logging import os -import pdb import platform -import select import signal -import subprocess -import sys -import time -import webbrowser -import base +import webkitpy.common.system.ospath as ospath +import webkitpy.layout_tests.port.server_process as server_process +from webkitpy.layout_tests.port.webkit import WebKitPort, WebKitDriver -import webkitpy -from webkitpy import executive +_log = logging.getLogger("webkitpy.layout_tests.port.mac") -class MacPort(base.Port): + +class MacPort(WebKitPort): """WebKit Mac implementation of the Port class.""" def __init__(self, port_name=None, options=None): if port_name is None: port_name = 'mac' + self.version() - base.Port.__init__(self, port_name, options) - self._cached_build_root = None + WebKitPort.__init__(self, port_name, options) + + def default_child_processes(self): + # FIXME: new-run-webkit-tests is unstable on Mac running more than + # four threads in parallel. + # See https://bugs.webkit.org/show_bug.cgi?id=36622 + child_processes = WebKitPort.default_child_processes(self) + if child_processes > 4: + return 4 + return child_processes def baseline_search_path(self): - dirs = [] + port_names = [] if self._name == 'mac-tiger': - dirs.append(self._webkit_baseline_path(self._name)) + port_names.append("mac-tiger") if self._name in ('mac-tiger', 'mac-leopard'): - dirs.append(self._webkit_baseline_path('mac-leopard')) + port_names.append("mac-leopard") if self._name in ('mac-tiger', 'mac-leopard', 'mac-snowleopard'): - dirs.append(self._webkit_baseline_path('mac-snowleopard')) - dirs.append(self._webkit_baseline_path('mac')) - return dirs - - def check_sys_deps(self): - # FIXME: This should run build-dumprendertree. - # This should also validate that all of the tool paths are valid. - return True - - def num_cores(self): - return int(os.popen2("sysctl -n hw.ncpu")[1].read()) - - def results_directory(self): - return ('/tmp/run-chromium-webkit-tests-' + - self._options.results_directory) - - def setup_test_run(self): - # This port doesn't require any specific configuration. - pass + port_names.append("mac-snowleopard") + port_names.append("mac") + return map(self._webkit_baseline_path, port_names) - def show_results_html_file(self, results_filename): - uri = self.filename_to_uri(results_filename) - webbrowser.open(uri, new=1) + def path_to_test_expectations_file(self): + return self.path_from_webkit_base('LayoutTests', 'platform', + 'mac', 'test_expectations.txt') - def start_driver(self, image_path, options): - """Starts a new Driver and returns a handle to it.""" - return MacDriver(self, image_path, options) - - def start_helper(self): - # This port doesn't use a helper process. - pass - - def stop_helper(self): - # This port doesn't use a helper process. - pass - - def test_base_platform_names(self): - # At the moment we don't use test platform names, but we have - # to return something. - return ('mac',) - - def test_expectations(self): - # - # The WebKit mac port uses 'Skipped' files at the moment. Each - # file contains a list of files or directories to be skipped during - # the test run. The total list of tests to skipped is given by the - # contents of the generic Skipped file found in platform/X plus - # a version-specific file found in platform/X-version. Duplicate - # entries are allowed. This routine reads those files and turns - # contents into the format expected by test_expectations. - expectations = [] + def _skipped_file_paths(self): + # FIXME: This method will need to be made work for non-mac + # platforms and moved into base.Port. skipped_files = [] if self._name in ('mac-tiger', 'mac-leopard', 'mac-snowleopard'): skipped_files.append(os.path.join( self._webkit_baseline_path(self._name), 'Skipped')) skipped_files.append(os.path.join(self._webkit_baseline_path('mac'), 'Skipped')) - for filename in skipped_files: - if os.path.exists(filename): - f = file(filename) - for l in f.readlines(): - l = l.strip() - if not l.startswith('#') and len(l): - l = 'BUG_SKIPPED SKIP : ' + l + ' = FAIL' - if l not in expectations: - expectations.append(l) - f.close() - - # TODO - figure out how to check for these dynamically - expectations.append('BUG_SKIPPED SKIP : fast/wcss = FAIL') - expectations.append('BUG_SKIPPED SKIP : fast/xhtmlmp = FAIL') - expectations.append('BUG_SKIPPED SKIP : http/tests/wml = FAIL') - expectations.append('BUG_SKIPPED SKIP : mathml = FAIL') - expectations.append('BUG_SKIPPED SKIP : platform/chromium = FAIL') - expectations.append('BUG_SKIPPED SKIP : platform/gtk = FAIL') - expectations.append('BUG_SKIPPED SKIP : platform/qt = FAIL') - expectations.append('BUG_SKIPPED SKIP : platform/win = FAIL') - expectations.append('BUG_SKIPPED SKIP : wml = FAIL') - - # TODO - figure out how to handle webarchive tests - expectations.append('BUG_SKIPPED SKIP : webarchive = PASS') - expectations.append('BUG_SKIPPED SKIP : svg/webarchive = PASS') - expectations.append('BUG_SKIPPED SKIP : http/tests/webarchive = PASS') - expectations.append('BUG_SKIPPED SKIP : svg/custom/' - 'image-with-prefix-in-webarchive.svg = PASS') - - expectations_str = '\n'.join(expectations) - return expectations_str + return skipped_files def test_platform_name(self): - # At the moment we don't use test platform names, but we have - # to return something. - return 'mac' - - def test_platform_names(self): - # At the moment we don't use test platform names, but we have - # to return something. - return ('mac',) + return 'mac' + self.version() def version(self): os_version_string = platform.mac_ver()[0] # e.g. "10.5.6" @@ -174,53 +99,36 @@ class MacPort(base.Port): return '-snowleopard' return '' - # - # PROTECTED METHODS - # - - def _build_path(self, *comps): - if not self._cached_build_root: - self._cached_build_root = executive.run_command(["webkit-build-directory", "--base"]).rstrip() - return os.path.join(self._cached_build_root, self._options.target, *comps) - - def _kill_process(self, pid): - """Forcefully kill the process. - - Args: - pid: The id of the process to be killed. - """ - os.kill(pid, signal.SIGKILL) - - def _kill_all_process(self, process_name): - # On Mac OS X 10.6, killall has a new constraint: -SIGNALNAME or - # -SIGNALNUMBER must come first. Example problem: - # $ killall -u $USER -TERM lighttpd - # killall: illegal option -- T - # Use of the earlier -TERM placement is just fine on 10.5. - null = open(os.devnull) - subprocess.call(['killall', '-TERM', '-u', os.getenv('USER'), - process_name], stderr=null) - null.close() + def _build_java_test_support(self): + java_tests_path = os.path.join(self.layout_tests_dir(), "java") + build_java = ["/usr/bin/make", "-C", java_tests_path] + if self._executive.run_command(build_java, return_exit_code=True): + _log.error("Failed to build Java support files: %s" % build_java) + return False + return True - def _path_to_apache(self): - return '/usr/sbin/httpd' + def _check_port_build(self): + return self._build_java_test_support() + + def _tests_for_other_platforms(self): + # The original run-webkit-tests builds up a "whitelist" of tests to + # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes + # we run *all* tests and test_expectations.txt functions as a + # blacklist. + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + return [ + "platform/chromium", + "platform/gtk", + "platform/qt", + "platform/win", + ] def _path_to_apache_config_file(self): return os.path.join(self.layout_tests_dir(), 'http', 'conf', 'apache2-httpd.conf') - def _path_to_driver(self): - return self._build_path('DumpRenderTree') - - def _path_to_helper(self): - return None - - def _path_to_image_diff(self): - return self._build_path('image_diff') # FIXME: This is wrong and should be "ImageDiff", but having the correct path causes other parts of the script to hang. - - def _path_to_wdiff(self): - return 'wdiff' # FIXME: This does not exist on a default Mac OS X Leopard install. - + # FIXME: This doesn't have anything to do with WebKit. def _shut_down_http_server(self, server_pid): """Shut down the lighttpd web server. Blocks until it's fully shut down. @@ -230,210 +138,16 @@ class MacPort(base.Port): """ # server_pid is not set when "http_server.py stop" is run manually. if server_pid is None: - # TODO(mmoss) This isn't ideal, since it could conflict with + # FIXME: This isn't ideal, since it could conflict with # lighttpd processes not started by http_server.py, # but good enough for now. - self._kill_all_process('httpd') + self._executive.kill_all('httpd') else: try: os.kill(server_pid, signal.SIGTERM) - # TODO(mmoss) Maybe throw in a SIGKILL just to be sure? + # FIXME: Maybe throw in a SIGKILL just to be sure? except OSError: # Sometimes we get a bad PID (e.g. from a stale httpd.pid # file), so if kill fails on the given PID, just try to # 'killall' web servers. self._shut_down_http_server(None) - - -class MacDriver(base.Driver): - """implementation of the DumpRenderTree interface.""" - - def __init__(self, port, image_path, driver_options): - self._port = port - self._driver_options = driver_options - self._target = port._options.target - self._image_path = image_path - self._stdout_fd = None - self._cmd = None - self._env = None - self._proc = None - self._read_buffer = '' - - cmd = [] - # Hook for injecting valgrind or other runtime instrumentation, - # used by e.g. tools/valgrind/valgrind_tests.py. - wrapper = os.environ.get("BROWSER_WRAPPER", None) - if wrapper != None: - cmd += [wrapper] - if self._port._options.wrapper: - # This split() isn't really what we want -- it incorrectly will - # split quoted strings within the wrapper argument -- but in - # practice it shouldn't come up and the --help output warns - # about it anyway. - cmd += self._options.wrapper.split() - # FIXME: Using arch here masks any possible file-not-found errors from a non-existant driver executable. - cmd += ['arch', '-i386', port._path_to_driver(), '-'] - - # FIXME: This is a hack around our lack of ImageDiff support for now. - if not self._port._options.no_pixel_tests: - logging.warn("This port does not yet support pixel tests.") - self._port._options.no_pixel_tests = True - #cmd.append('--pixel-tests') - - #if driver_options: - # cmd += driver_options - env = os.environ - env['DYLD_FRAMEWORK_PATH'] = self._port._build_path() - self._cmd = cmd - self._env = env - self.restart() - - def poll(self): - return self._proc.poll() - - def restart(self): - self.stop() - self._proc = subprocess.Popen(self._cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self._env) - - def returncode(self): - return self._proc.returncode - - def run_test(self, uri, timeoutms, image_hash): - output = [] - error = [] - image = '' - crash = False - timeout = False - actual_uri = None - actual_image_hash = None - - if uri.startswith("file:///"): - cmd = uri[7:] - else: - cmd = uri - - if image_hash: - cmd += "'" + image_hash - cmd += "\n" - - self._proc.stdin.write(cmd) - self._stdout_fd = self._proc.stdout.fileno() - fl = fcntl.fcntl(self._stdout_fd, fcntl.F_GETFL) - fcntl.fcntl(self._stdout_fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) - - stop_time = time.time() + (int(timeoutms) / 1000.0) - resp = '' - (timeout, line) = self._read_line(timeout, stop_time) - resp += line - have_seen_content_type = False - while not timeout and line.rstrip() != "#EOF": - # Make sure we haven't crashed. - if line == '' and self.poll() is not None: - # This is hex code 0xc000001d, which is used for abrupt - # termination. This happens if we hit ctrl+c from the prompt - # and we happen to be waiting on the test_shell. - # sdoyon: Not sure for which OS and in what circumstances the - # above code is valid. What works for me under Linux to detect - # ctrl+c is for the subprocess returncode to be negative - # SIGINT. And that agrees with the subprocess documentation. - if (-1073741510 == self.returncode() or - - signal.SIGINT == self.returncode()): - raise KeyboardInterrupt - crash = True - break - - elif (line.startswith('Content-Type:') and not - have_seen_content_type): - have_seen_content_type = True - pass - else: - output.append(line) - - (timeout, line) = self._read_line(timeout, stop_time) - resp += line - - # Now read a second block of text for the optional image data - image_length = 0 - (timeout, line) = self._read_line(timeout, stop_time) - resp += line - HASH_HEADER = 'ActualHash: ' - LENGTH_HEADER = 'Content-Length: ' - while not timeout and not crash and line.rstrip() != "#EOF": - if line == '' and self.poll() is not None: - if (-1073741510 == self.returncode() or - - signal.SIGINT == self.returncode()): - raise KeyboardInterrupt - crash = True - break - elif line.startswith(HASH_HEADER): - actual_image_hash = line[len(HASH_HEADER):].strip() - elif line.startswith('Content-Type:'): - pass - elif line.startswith(LENGTH_HEADER): - image_length = int(line[len(LENGTH_HEADER):]) - elif image_length: - image += line - - (timeout, line) = self._read_line(timeout, stop_time, image_length) - resp += line - - if timeout: - self.restart() - - if self._image_path and len(self._image_path): - image_file = file(self._image_path, "wb") - image_file.write(image) - image_file.close() - - return (crash, timeout, actual_image_hash, - ''.join(output), ''.join(error)) - pass - - def stop(self): - if self._proc: - self._proc.stdin.close() - self._proc.stdout.close() - if self._proc.stderr: - self._proc.stderr.close() - if (sys.platform not in ('win32', 'cygwin') and - not self._proc.poll()): - # Closing stdin/stdout/stderr hangs sometimes on OS X. - null = open(os.devnull, "w") - subprocess.Popen(["kill", "-9", - str(self._proc.pid)], stderr=null) - null.close() - - def _read_line(self, timeout, stop_time, image_length=0): - now = time.time() - read_fds = [] - - # first check to see if we have a line already read or if we've - # read the entire image - if image_length and len(self._read_buffer) >= image_length: - out = self._read_buffer[0:image_length] - self._read_buffer = self._read_buffer[image_length:] - return (timeout, out) - - idx = self._read_buffer.find('\n') - if not image_length and idx != -1: - out = self._read_buffer[0:idx + 1] - self._read_buffer = self._read_buffer[idx + 1:] - return (timeout, out) - - # If we've timed out, return just what we have, if anything - if timeout or now >= stop_time: - out = self._read_buffer - self._read_buffer = '' - return (True, out) - - (read_fds, write_fds, err_fds) = select.select( - [self._stdout_fd], [], [], stop_time - now) - try: - if timeout or len(read_fds) == 1: - self._read_buffer += self._proc.stdout.read() - except IOError, e: - read = [] - return self._read_line(timeout, stop_time) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/mac_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac_unittest.py new file mode 100644 index 0000000..ae7d40c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac_unittest.py @@ -0,0 +1,66 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import mac +import StringIO + +class MacTest(unittest.TestCase): + + def test_skipped_file_paths(self): + port = mac.MacPort() + skipped_paths = port._skipped_file_paths() + # FIXME: _skipped_file_paths should return WebKit-relative paths. + # So to make it unit testable, we strip the WebKit directory from the path. + relative_paths = [path[len(port.path_from_webkit_base()):] for path in skipped_paths] + self.assertEqual(relative_paths, ['LayoutTests/platform/mac-leopard/Skipped', 'LayoutTests/platform/mac/Skipped']) + + example_skipped_file = u""" +# <rdar://problem/5647952> fast/events/mouseout-on-window.html needs mac DRT to issue mouse out events +fast/events/mouseout-on-window.html + +# <rdar://problem/5643675> window.scrollTo scrolls a window with no scrollbars +fast/events/attempt-scroll-with-no-scrollbars.html + +# see bug <rdar://problem/5646437> REGRESSION (r28015): svg/batik/text/smallFonts fails +svg/batik/text/smallFonts.svg +""" + example_skipped_tests = [ + "fast/events/mouseout-on-window.html", + "fast/events/attempt-scroll-with-no-scrollbars.html", + "svg/batik/text/smallFonts.svg", + ] + + def test_skipped_file_paths(self): + port = mac.MacPort() + skipped_file = StringIO.StringIO(self.example_skipped_file) + self.assertEqual(port._tests_from_skipped_file(skipped_file), self.example_skipped_tests) + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py new file mode 100644 index 0000000..41b2ba0 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py @@ -0,0 +1,99 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""QtWebKit implementation of the Port interface.""" + +import logging +import os +import signal + +import webkit + +from webkitpy.layout_tests.port.webkit import WebKitPort + +_log = logging.getLogger("webkitpy.layout_tests.port.qt") + + +class QtPort(WebKitPort): + """QtWebKit implementation of the Port class.""" + + def __init__(self, port_name=None, options=None): + if port_name is None: + port_name = 'qt' + WebKitPort.__init__(self, port_name, options) + + def _tests_for_other_platforms(self): + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + # This really need to be automated. + return [ + "platform/chromium", + "platform/win", + "platform/gtk", + "platform/mac", + ] + + def _path_to_apache_config_file(self): + # FIXME: This needs to detect the distribution and change config files. + return os.path.join(self.layout_tests_dir(), 'http', 'conf', + 'apache2-debian-httpd.conf') + + def _shut_down_http_server(self, server_pid): + """Shut down the httpd web server. Blocks until it's fully + shut down. + + Args: + server_pid: The process ID of the running server. + """ + # server_pid is not set when "http_server.py stop" is run manually. + if server_pid is None: + # FIXME: This isn't ideal, since it could conflict with + # lighttpd processes not started by http_server.py, + # but good enough for now. + self._executive.kill_all('apache2') + else: + try: + os.kill(server_pid, signal.SIGTERM) + # TODO(mmoss) Maybe throw in a SIGKILL just to be sure? + except OSError: + # Sometimes we get a bad PID (e.g. from a stale httpd.pid + # file), so if kill fails on the given PID, just try to + # 'killall' web servers. + self._shut_down_http_server(None) + + def _build_driver(self): + # The Qt port builds DRT as part of the main build step + return True + + def _path_to_driver(self): + return self._build_path('bin/DumpRenderTree') + + def setup_environ_for_server(self): + env = webkit.WebKitPort.setup_environ_for_server(self) + env['QTWEBKIT_PLUGIN_PATH'] = self._build_path('lib/plugins') + return env diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py new file mode 100644 index 0000000..62ca693 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Package that implements the ServerProcess wrapper class""" + +import fcntl +import logging +import os +import select +import signal +import subprocess +import sys +import time + +from webkitpy.common.system.executive import Executive + +_log = logging.getLogger("webkitpy.layout_tests.port.server_process") + + +class ServerProcess: + """This class provides a wrapper around a subprocess that + implements a simple request/response usage model. The primary benefit + is that reading responses takes a timeout, so that we don't ever block + indefinitely. The class also handles transparently restarting processes + as necessary to keep issuing commands.""" + + def __init__(self, port_obj, name, cmd, env=None, executive=Executive()): + self._port = port_obj + self._name = name + self._cmd = cmd + self._env = env + self._reset() + self._executive = executive + + def _reset(self): + self._proc = None + self._output = '' + self.crashed = False + self.timed_out = False + self.error = '' + + def _start(self): + if self._proc: + raise ValueError("%s already running" % self._name) + self._reset() + # close_fds is a workaround for http://bugs.python.org/issue2320 + close_fds = sys.platform not in ('win32', 'cygwin') + self._proc = subprocess.Popen(self._cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=close_fds, + env=self._env) + fd = self._proc.stdout.fileno() + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) + fd = self._proc.stderr.fileno() + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) + + def handle_interrupt(self): + """This routine checks to see if the process crashed or exited + because of a keyboard interrupt and raises KeyboardInterrupt + accordingly.""" + if self.crashed: + # This is hex code 0xc000001d, which is used for abrupt + # termination. This happens if we hit ctrl+c from the prompt + # and we happen to be waiting on the DumpRenderTree. + # sdoyon: Not sure for which OS and in what circumstances the + # above code is valid. What works for me under Linux to detect + # ctrl+c is for the subprocess returncode to be negative + # SIGINT. And that agrees with the subprocess documentation. + if (-1073741510 == self._proc.returncode or + - signal.SIGINT == self._proc.returncode): + raise KeyboardInterrupt + return + + def poll(self): + """Check to see if the underlying process is running; returns None + if it still is (wrapper around subprocess.poll).""" + if self._proc: + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + return self._proc.poll() + return None + + def returncode(self): + """Returns the exit code from the subprcoess; returns None if the + process hasn't exited (this is a wrapper around subprocess.returncode). + """ + if self._proc: + return self._proc.returncode + return None + + def write(self, input): + """Write a request to the subprocess. The subprocess is (re-)start()'ed + if is not already running.""" + if not self._proc: + self._start() + self._proc.stdin.write(input) + + def read_line(self, timeout): + """Read a single line from the subprocess, waiting until the deadline. + If the deadline passes, the call times out. Note that even if the + subprocess has crashed or the deadline has passed, if there is output + pending, it will be returned. + + Args: + timeout: floating-point number of seconds the call is allowed + to block for. A zero or negative number will attempt to read + any existing data, but will not block. There is no way to + block indefinitely. + Returns: + output: data returned, if any. If no data is available and the + call times out or crashes, an empty string is returned. Note + that the returned string includes the newline ('\n').""" + return self._read(timeout, size=0) + + def read(self, timeout, size): + """Attempts to read size characters from the subprocess, waiting until + the deadline passes. If the deadline passes, any available data will be + returned. Note that even if the deadline has passed or if the + subprocess has crashed, any available data will still be returned. + + Args: + timeout: floating-point number of seconds the call is allowed + to block for. A zero or negative number will attempt to read + any existing data, but will not block. There is no way to + block indefinitely. + size: amount of data to read. Must be a postive integer. + Returns: + output: data returned, if any. If no data is available, an empty + string is returned. + """ + if size <= 0: + raise ValueError('ServerProcess.read() called with a ' + 'non-positive size: %d ' % size) + return self._read(timeout, size) + + def _read(self, timeout, size): + """Internal routine that actually does the read.""" + index = -1 + out_fd = self._proc.stdout.fileno() + err_fd = self._proc.stderr.fileno() + select_fds = (out_fd, err_fd) + deadline = time.time() + timeout + while not self.timed_out and not self.crashed: + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + if self._proc.poll() != None: + self.crashed = True + self.handle_interrupt() + + now = time.time() + if now > deadline: + self.timed_out = True + + # Check to see if we have any output we can return. + if size and len(self._output) >= size: + index = size + elif size == 0: + index = self._output.find('\n') + 1 + + if index or self.crashed or self.timed_out: + output = self._output[0:index] + self._output = self._output[index:] + return output + + # Nope - wait for more data. + (read_fds, write_fds, err_fds) = select.select(select_fds, [], + select_fds, + deadline - now) + try: + if out_fd in read_fds: + self._output += self._proc.stdout.read() + if err_fd in read_fds: + self.error += self._proc.stderr.read() + except IOError, e: + pass + + def stop(self): + """Stop (shut down) the subprocess), if it is running.""" + pid = self._proc.pid + self._proc.stdin.close() + self._proc.stdout.close() + if self._proc.stderr: + self._proc.stderr.close() + if sys.platform not in ('win32', 'cygwin'): + # Closing stdin/stdout/stderr hangs sometimes on OS X, + # (see restart(), above), and anyway we don't want to hang + # the harness if DumpRenderTree is buggy, so we wait a couple + # seconds to give DumpRenderTree a chance to clean up, but then + # force-kill the process if necessary. + KILL_TIMEOUT = 3.0 + timeout = time.time() + KILL_TIMEOUT + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + while self._proc.poll() is None and time.time() < timeout: + time.sleep(0.1) + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + if self._proc.poll() is None: + _log.warning('stopping %s timed out, killing it' % + self._name) + self._executive.kill_process(self._proc.pid) + _log.warning('killed') + self._reset() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py index 0bc6e7c..6eef54e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py @@ -28,7 +28,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Dummy Port implementation used for testing.""" +from __future__ import with_statement +import codecs import os import time @@ -45,37 +47,43 @@ class TestPort(base.Port): return ('test',) def baseline_path(self): - curdir = os.path.abspath(__file__) - self.topdir = curdir[0:curdir.index("WebKitTools")] - return os.path.join(self.topdir, 'LayoutTests', 'platform', 'test') + return os.path.join(self.layout_tests_dir(), 'platform', + self.name()) def baseline_search_path(self): return [self.baseline_path()] - def check_sys_deps(self): + def check_build(self, needs_http): return True - def diff_image(self, actual_filename, expected_filename, diff_filename): + def compare_text(self, expected_text, actual_text): return False - def compare_text(self, actual_text, expected_text): + def diff_image(self, expected_filename, actual_filename, + diff_filename=None, tolerance=0): return False - def diff_text(self, actual_text, expected_text, - actual_filename, expected_filename): + def diff_text(self, expected_text, actual_text, + expected_filename, actual_filename): return '' + def layout_tests_dir(self): + return self.path_from_webkit_base('WebKitTools', 'Scripts', + 'webkitpy', 'layout_tests', 'data') + def name(self): return self._name - def num_cores(self): - return int(os.popen2("sysctl -n hw.ncpu")[1].read()) - def options(self): return self._options + def path_to_test_expectations_file(self): + return self.path_from_webkit_base('WebKitTools', 'Scripts', + 'webkitpy', 'layout_tests', 'data', 'platform', 'test', + 'test_expectations.txt') + def results_directory(self): - return '/tmp' + self._options.results_directory + return '/tmp/' + self._options.results_directory def setup_test_run(self): pass @@ -83,7 +91,7 @@ class TestPort(base.Port): def show_results_html_file(self, filename): pass - def start_driver(self, image_path, options): + def create_driver(self, image_path, options): return TestDriver(image_path, options, self) def start_http_server(self): @@ -92,20 +100,20 @@ class TestPort(base.Port): def start_websocket_server(self): pass - def start_helper(self): - pass - def stop_http_server(self): pass def stop_websocket_server(self): pass - def stop_helper(self): - pass - def test_expectations(self): - return '' + """Returns the test expectations for this port. + + Basically this string should contain the equivalent of a + test_expectations file. See test_expectations.py for more details.""" + expectations_path = self.path_to_test_expectations_file() + with codecs.open(expectations_path, "r", "utf-8") as file: + return file.read() def test_base_platform_names(self): return ('test',) @@ -119,7 +127,7 @@ class TestPort(base.Port): def version(): return '' - def wdiff_text(self, actual_filename, expected_filename): + def wdiff_text(self, expected_filename, actual_filename): return '' @@ -130,6 +138,7 @@ class TestDriver(base.Driver): self._driver_options = test_driver_options self._image_path = image_path self._port = port + self._image_written = False def poll(self): return True @@ -138,7 +147,19 @@ class TestDriver(base.Driver): return 0 def run_test(self, uri, timeoutms, image_hash): + if not self._image_written and self._port._options.pixel_tests: + with open(self._image_path, "w") as f: + f.write("bad png file from TestDriver") + self._image_written = True + + # We special-case this because we can't fake an image hash for a + # missing expectation. + if uri.find('misc/missing-expectation') != -1: + return (False, False, 'deadbeefdeadbeefdeadbeefdeadbeef', '', None) return (False, False, image_hash, '', None) + def start(self): + pass + def stop(self): pass diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py new file mode 100644 index 0000000..e1151a6 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -0,0 +1,476 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""WebKit implementations of the Port interface.""" + + +from __future__ import with_statement + +import codecs +import logging +import os +import re +import shutil +import signal +import sys +import time +import webbrowser + +from webkitpy.common.system.executive import Executive + +import webkitpy.common.system.ospath as ospath +import webkitpy.layout_tests.port.base as base +import webkitpy.layout_tests.port.server_process as server_process + +_log = logging.getLogger("webkitpy.layout_tests.port.webkit") + + +class WebKitPort(base.Port): + """WebKit implementation of the Port class.""" + + def __init__(self, port_name=None, options=None, **kwargs): + base.Port.__init__(self, port_name, options, **kwargs) + self._cached_build_root = None + self._cached_apache_path = None + + # FIXME: disable pixel tests until they are run by default on the + # build machines. + if options and (not hasattr(options, "pixel_tests") or + options.pixel_tests is None): + options.pixel_tests = False + + def baseline_path(self): + return self._webkit_baseline_path(self._name) + + def baseline_search_path(self): + return [self._webkit_baseline_path(self._name)] + + def path_to_test_expectations_file(self): + return os.path.join(self._webkit_baseline_path(self._name), + 'test_expectations.txt') + + # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard) + def version(self): + return '' + + def _build_driver(self): + return not self._executive.run_command([ + self.script_path("build-dumprendertree"), + self.flag_from_configuration(self._options.configuration), + ], return_exit_code=True) + + def _check_driver(self): + driver_path = self._path_to_driver() + if not os.path.exists(driver_path): + _log.error("DumpRenderTree was not found at %s" % driver_path) + return False + return True + + def check_build(self, needs_http): + if self._options.build and not self._build_driver(): + return False + if not self._check_driver(): + return False + if self._options.pixel_tests: + if not self.check_image_diff(): + return False + if not self._check_port_build(): + return False + return True + + def _check_port_build(self): + # Ports can override this method to do additional checks. + return True + + def check_image_diff(self, override_step=None, logging=True): + image_diff_path = self._path_to_image_diff() + if not os.path.exists(image_diff_path): + _log.error("ImageDiff was not found at %s" % image_diff_path) + return False + return True + + def diff_image(self, expected_filename, actual_filename, + diff_filename=None, tolerance=0.1): + """Return True if the two files are different. Also write a delta + image of the two images into |diff_filename| if it is not None.""" + + # FIXME: either expose the tolerance argument as a command-line + # parameter, or make it go away and always use exact matches. + + # Handle the case where the test didn't actually generate an image. + actual_length = os.stat(actual_filename).st_size + if actual_length == 0: + if diff_filename: + shutil.copyfile(actual_filename, expected_filename) + return True + + sp = self._diff_image_request(expected_filename, actual_filename, tolerance) + return self._diff_image_reply(sp, expected_filename, diff_filename) + + def _diff_image_request(self, expected_filename, actual_filename, tolerance): + command = [self._path_to_image_diff(), '--tolerance', str(tolerance)] + sp = server_process.ServerProcess(self, 'ImageDiff', command) + + actual_length = os.stat(actual_filename).st_size + with open(actual_filename) as file: + actual_file = file.read() + expected_length = os.stat(expected_filename).st_size + with open(expected_filename) as file: + expected_file = file.read() + sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' % + (actual_length, actual_file, expected_length, expected_file)) + + return sp + + def _diff_image_reply(self, sp, expected_filename, diff_filename): + timeout = 2.0 + deadline = time.time() + timeout + output = sp.read_line(timeout) + while not sp.timed_out and not sp.crashed and output: + if output.startswith('Content-Length'): + m = re.match('Content-Length: (\d+)', output) + content_length = int(m.group(1)) + timeout = deadline - time.time() + output = sp.read(timeout, content_length) + break + elif output.startswith('diff'): + break + else: + timeout = deadline - time.time() + output = sp.read_line(deadline) + + result = True + if output.startswith('diff'): + m = re.match('diff: (.+)% (passed|failed)', output) + if m.group(2) == 'passed': + result = False + elif output and diff_filename: + with open(diff_filename, 'w') as file: + file.write(output) + elif sp.timed_out: + _log.error("ImageDiff timed out on %s" % expected_filename) + elif sp.crashed: + _log.error("ImageDiff crashed") + sp.stop() + return result + + def results_directory(self): + # Results are store relative to the built products to make it easy + # to have multiple copies of webkit checked out and built. + return self._build_path(self._options.results_directory) + + def setup_test_run(self): + # This port doesn't require any specific configuration. + pass + + def show_results_html_file(self, results_filename): + uri = self.filename_to_uri(results_filename) + # FIXME: We should open results in the version of WebKit we built. + webbrowser.open(uri, new=1) + + def create_driver(self, image_path, options): + return WebKitDriver(self, image_path, options, executive=self._executive) + + def test_base_platform_names(self): + # At the moment we don't use test platform names, but we have + # to return something. + return ('mac', 'win') + + def _tests_for_other_platforms(self): + raise NotImplementedError('WebKitPort._tests_for_other_platforms') + # The original run-webkit-tests builds up a "whitelist" of tests to + # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes + # we run *all* tests and test_expectations.txt functions as a + # blacklist. + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + return [ + "platform/chromium", + "platform/gtk", + "platform/qt", + "platform/win", + ] + + def _tests_for_disabled_features(self): + # FIXME: This should use the feature detection from + # webkitperl/features.pm to match run-webkit-tests. + # For now we hard-code a list of features known to be disabled on + # the Mac platform. + disabled_feature_tests = [ + "fast/xhtmlmp", + "http/tests/wml", + "mathml", + "wml", + ] + # FIXME: webarchive tests expect to read-write from + # -expected.webarchive files instead of .txt files. + # This script doesn't know how to do that yet, so pretend they're + # just "disabled". + webarchive_tests = [ + "webarchive", + "svg/webarchive", + "http/tests/webarchive", + "svg/custom/image-with-prefix-in-webarchive.svg", + ] + return disabled_feature_tests + webarchive_tests + + def _tests_from_skipped_file(self, skipped_file): + tests_to_skip = [] + for line in skipped_file.readlines(): + line = line.strip() + if line.startswith('#') or not len(line): + continue + tests_to_skip.append(line) + return tests_to_skip + + def _skipped_file_paths(self): + return [os.path.join(self._webkit_baseline_path(self._name), + 'Skipped')] + + def _expectations_from_skipped_files(self): + tests_to_skip = [] + for filename in self._skipped_file_paths(): + if not os.path.exists(filename): + _log.warn("Failed to open Skipped file: %s" % filename) + continue + with codecs.open(filename, "r", "utf-8") as skipped_file: + tests_to_skip.extend(self._tests_from_skipped_file(skipped_file)) + return tests_to_skip + + def test_expectations(self): + # The WebKit mac port uses a combination of a test_expectations file + # and 'Skipped' files. + expectations_path = self.path_to_test_expectations_file() + with codecs.open(expectations_path, "r", "utf-8") as file: + return file.read() + self._skips() + + def _skips(self): + # Each Skipped file contains a list of files + # or directories to be skipped during the test run. The total list + # of tests to skipped is given by the contents of the generic + # Skipped file found in platform/X plus a version-specific file + # found in platform/X-version. Duplicate entries are allowed. + # This routine reads those files and turns contents into the + # format expected by test_expectations. + + # Use a set to allow duplicates + tests_to_skip = set(self._expectations_from_skipped_files()) + + tests_to_skip.update(self._tests_for_other_platforms()) + tests_to_skip.update(self._tests_for_disabled_features()) + skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" % + test_path, tests_to_skip) + return "\n".join(skip_lines) + + def test_platform_name(self): + return self._name + self.version() + + def test_platform_names(self): + return self.test_base_platform_names() + ( + 'mac-tiger', 'mac-leopard', 'mac-snowleopard') + + def _configuration_file_path(self): + build_root = self._webkit_build_directory(["--top-level"]) + return os.path.join(build_root, "Configuration") + + # Easy override for unit tests + def _open_configuration_file(self): + configuration_path = self._configuration_file_path() + return codecs.open(configuration_path, "r", "utf-8") + + def _read_configuration(self): + try: + with self._open_configuration_file() as file: + return file.readline().rstrip() + except IOError, e: + return None + + # FIXME: This list may be incomplete as Apple has some sekret configs. + _RECOGNIZED_CONFIGURATIONS = ("Debug", "Release") + + def default_configuration(self): + # FIXME: Unify this with webkitdir.pm configuration reading code. + configuration = self._read_configuration() + if not configuration: + configuration = "Release" + if configuration not in self._RECOGNIZED_CONFIGURATIONS: + _log.warn("Configuration \"%s\" found in %s is not a recognized value.\n" % (configuration, self._configuration_file_path())) + _log.warn("Scripts may fail. See 'set-webkit-configuration --help'.") + return configuration + + def _webkit_build_directory(self, args): + args = [self.script_path("webkit-build-directory")] + args + return self._executive.run_command(args).rstrip() + + def _build_path(self, *comps): + if not self._cached_build_root: + self._cached_build_root = self._webkit_build_directory([ + "--configuration", + self.flag_from_configuration(self._options.configuration), + ]) + return os.path.join(self._cached_build_root, *comps) + + def _path_to_driver(self): + return self._build_path('DumpRenderTree') + + def _path_to_helper(self): + return None + + def _path_to_image_diff(self): + return self._build_path('ImageDiff') + + def _path_to_wdiff(self): + # FIXME: This does not exist on a default Mac OS X Leopard install. + return 'wdiff' + + def _path_to_apache(self): + if not self._cached_apache_path: + # The Apache binary path can vary depending on OS and distribution + # See http://wiki.apache.org/httpd/DistrosDefaultLayout + for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]: + if os.path.exists(path): + self._cached_apache_path = path + break + + if not self._cached_apache_path: + _log.error("Could not find apache. Not installed or unknown path.") + + return self._cached_apache_path + + +class WebKitDriver(base.Driver): + """WebKit implementation of the DumpRenderTree interface.""" + + def __init__(self, port, image_path, driver_options, executive=Executive()): + self._port = port + # FIXME: driver_options is never used. + self._image_path = image_path + + def start(self): + command = [] + # FIXME: We should not be grabbing at self._port._options.wrapper directly. + command += self._command_wrapper(self._port._options.wrapper) + command += [self._port._path_to_driver(), '-'] + if self._image_path: + command.append('--pixel-tests') + environment = self._port.setup_environ_for_server() + environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path() + self._server_process = server_process.ServerProcess(self._port, + "DumpRenderTree", command, environment) + + def poll(self): + return self._server_process.poll() + + def restart(self): + self._server_process.stop() + self._server_process.start() + return + + def returncode(self): + return self._server_process.returncode() + + # FIXME: This function is huge. + def run_test(self, uri, timeoutms, image_hash): + if uri.startswith("file:///"): + command = uri[7:] + else: + command = uri + + if image_hash: + command += "'" + image_hash + command += "\n" + + self._server_process.write(command) + + have_seen_content_type = False + actual_image_hash = None + output = str() # Use a byte array for output, even though it should be UTF-8. + image = str() + + timeout = int(timeoutms) / 1000.0 + deadline = time.time() + timeout + line = self._server_process.read_line(timeout) + while (not self._server_process.timed_out + and not self._server_process.crashed + and line.rstrip() != "#EOF"): + if (line.startswith('Content-Type:') and not + have_seen_content_type): + have_seen_content_type = True + else: + # Note: Text output from DumpRenderTree is always UTF-8. + # However, some tests (e.g. webarchives) spit out binary + # data instead of text. So to make things simple, we + # always treat the output as binary. + output += line + line = self._server_process.read_line(timeout) + timeout = deadline - time.time() + + # Now read a second block of text for the optional image data + remaining_length = -1 + HASH_HEADER = 'ActualHash: ' + LENGTH_HEADER = 'Content-Length: ' + line = self._server_process.read_line(timeout) + while (not self._server_process.timed_out + and not self._server_process.crashed + and line.rstrip() != "#EOF"): + if line.startswith(HASH_HEADER): + actual_image_hash = line[len(HASH_HEADER):].strip() + elif line.startswith('Content-Type:'): + pass + elif line.startswith(LENGTH_HEADER): + timeout = deadline - time.time() + content_length = int(line[len(LENGTH_HEADER):]) + image = self._server_process.read(timeout, content_length) + timeout = deadline - time.time() + line = self._server_process.read_line(timeout) + + if self._image_path and len(self._image_path): + with open(self._image_path, "wb") as image_file: + image_file.write(image) + + error_lines = self._server_process.error.splitlines() + # FIXME: This is a hack. It is unclear why sometimes + # we do not get any error lines from the server_process + # probably we are not flushing stderr. + if error_lines and error_lines[-1] == "#EOF": + error_lines.pop() # Remove the expected "#EOF" + error = "\n".join(error_lines) + # FIXME: This seems like the wrong section of code to be doing + # this reset in. + self._server_process.error = "" + return (self._server_process.crashed, + self._server_process.timed_out, + actual_image_hash, + output, + error) + + def stop(self): + if self._server_process: + self._server_process.stop() + self._server_process = None diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py index ba8a5e9..81bf39e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/websocket_server.py @@ -30,6 +30,9 @@ """A class to help start/stop the PyWebSocket server used by layout tests.""" +from __future__ import with_statement + +import codecs import logging import optparse import os @@ -39,8 +42,15 @@ import tempfile import time import urllib +import factory import http_server +from webkitpy.common.system.executive import Executive +from webkitpy.thirdparty.autoinstalled.pywebsocket import mod_pywebsocket + + +_log = logging.getLogger("webkitpy.layout_tests.port.websocket_server") + _WS_LOG_PREFIX = 'pywebsocket.ws.log-' _WSS_LOG_PREFIX = 'pywebsocket.wss.log-' @@ -59,6 +69,7 @@ def url_is_alive(url): Return: True if the url is alive. """ + sleep_time = 0.5 wait_time = 5 while wait_time > 0: try: @@ -67,9 +78,9 @@ def url_is_alive(url): return True except IOError: pass - wait_time -= 1 - # Wait a second and try again. - time.sleep(1) + # Wait for sleep_time before trying again. + wait_time -= sleep_time + time.sleep(sleep_time) return False @@ -86,15 +97,13 @@ class PyWebSocket(http_server.Lighttpd): def __init__(self, port_obj, output_dir, port=_DEFAULT_WS_PORT, root=None, use_tls=False, - register_cygwin=None, pidfile=None): """Args: output_dir: the absolute path to the layout test result directory """ http_server.Lighttpd.__init__(self, port_obj, output_dir, port=_DEFAULT_WS_PORT, - root=root, - register_cygwin=register_cygwin) + root=root) self._output_dir = output_dir self._process = None self._port = port @@ -126,7 +135,7 @@ class PyWebSocket(http_server.Lighttpd): def start(self): if not self._web_socket_tests: - logging.info('No need to start %s server.' % self._server_name) + _log.info('No need to start %s server.' % self._server_name) return if self.is_running(): raise PyWebSocketNotStarted('%s is already running.' % @@ -145,74 +154,66 @@ class PyWebSocket(http_server.Lighttpd): error_log = os.path.join(self._output_dir, log_file_name + "-err.txt") output_log = os.path.join(self._output_dir, log_file_name + "-out.txt") - self._wsout = open(output_log, "w") + self._wsout = codecs.open(output_log, "w", "utf-8") python_interp = sys.executable pywebsocket_base = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))))), 'pywebsocket') + os.path.abspath(__file__)))), 'thirdparty', + 'autoinstalled', 'pywebsocket') pywebsocket_script = os.path.join(pywebsocket_base, 'mod_pywebsocket', 'standalone.py') start_cmd = [ python_interp, pywebsocket_script, - '-p', str(self._port), - '-d', self._layout_tests, - '-s', self._web_socket_tests, - '-l', error_log, + '--server-host', '127.0.0.1', + '--port', str(self._port), + '--document-root', self._layout_tests, + '--scan-dir', self._web_socket_tests, + '--cgi-paths', '/websocket/tests', + '--log-file', error_log, ] handler_map_file = os.path.join(self._web_socket_tests, 'handler_map.txt') if os.path.exists(handler_map_file): - logging.debug('Using handler_map_file: %s' % handler_map_file) - start_cmd.append('-m') + _log.debug('Using handler_map_file: %s' % handler_map_file) + start_cmd.append('--websock-handlers-map-file') start_cmd.append(handler_map_file) else: - logging.warning('No handler_map_file found') + _log.warning('No handler_map_file found') if self._use_tls: start_cmd.extend(['-t', '-k', self._private_key, '-c', self._certificate]) - # Put the cygwin directory first in the path to find cygwin1.dll - env = os.environ - if sys.platform in ('cygwin', 'win32'): - env['PATH'] = '%s;%s' % ( - self._port_obj.path_from_chromium_base('third_party', - 'cygwin', 'bin'), - env['PATH']) - - if sys.platform == 'win32' and self._register_cygwin: - setup_mount = self._port_obj.path_from_chromium_base( - 'third_party', 'cygwin', 'setup_mount.bat') - subprocess.Popen(setup_mount).wait() - + env = self._port_obj.setup_environ_for_server() env['PYTHONPATH'] = (pywebsocket_base + os.path.pathsep + env.get('PYTHONPATH', '')) - logging.debug('Starting %s server on %d.' % ( - self._server_name, self._port)) - logging.debug('cmdline: %s' % ' '.join(start_cmd)) - self._process = subprocess.Popen(start_cmd, stdout=self._wsout, + _log.debug('Starting %s server on %d.' % ( + self._server_name, self._port)) + _log.debug('cmdline: %s' % ' '.join(start_cmd)) + # FIXME: We should direct this call through Executive for testing. + # Note: Not thread safe: http://bugs.python.org/issue2320 + self._process = subprocess.Popen(start_cmd, + stdin=open(os.devnull, 'r'), + stdout=self._wsout, stderr=subprocess.STDOUT, env=env) - # Wait a bit before checking the liveness of the server. - time.sleep(0.5) - if self._use_tls: url = 'https' else: url = 'http' url = url + '://127.0.0.1:%d/' % self._port if not url_is_alive(url): - fp = open(output_log) - try: + if self._process.returncode == None: + # FIXME: We should use a non-static Executive for easier + # testing. + Executive().kill_process(self._process.pid) + with codecs.open(output_log, "r", "utf-8") as fp: for line in fp: - logging.error(line) - finally: - fp.close() + _log.error(line) raise PyWebSocketNotStarted( 'Failed to start %s server on port %s.' % (self._server_name, self._port)) @@ -222,82 +223,34 @@ class PyWebSocket(http_server.Lighttpd): raise PyWebSocketNotStarted( 'Failed to start %s server.' % self._server_name) if self._pidfile: - f = open(self._pidfile, 'w') - f.write("%d" % self._process.pid) - f.close() + with codecs.open(self._pidfile, "w", "ascii") as file: + file.write("%d" % self._process.pid) def stop(self, force=False): if not force and not self.is_running(): return + pid = None if self._process: pid = self._process.pid elif self._pidfile: - f = open(self._pidfile) - pid = int(f.read().strip()) - f.close() + with codecs.open(self._pidfile, "r", "ascii") as file: + pid = int(file.read().strip()) if not pid: raise PyWebSocketNotFound( 'Failed to find %s server pid.' % self._server_name) - logging.debug('Shutting down %s server %d.' % (self._server_name, pid)) - self._port_obj._kill_process(pid) + _log.debug('Shutting down %s server %d.' % (self._server_name, pid)) + # FIXME: We should use a non-static Executive for easier testing. + Executive().kill_process(pid) if self._process: + # wait() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 self._process.wait() self._process = None if self._wsout: self._wsout.close() self._wsout = None - - -if '__main__' == __name__: - # Provide some command line params for starting the PyWebSocket server - # manually. - option_parser = optparse.OptionParser() - option_parser.add_option('--server', type='choice', - choices=['start', 'stop'], default='start', - help='Server action (start|stop)') - option_parser.add_option('-p', '--port', dest='port', - default=None, help='Port to listen on') - option_parser.add_option('-r', '--root', - help='Absolute path to DocumentRoot ' - '(overrides layout test roots)') - option_parser.add_option('-t', '--tls', dest='use_tls', - action='store_true', - default=False, help='use TLS (wss://)') - option_parser.add_option('-k', '--private_key', dest='private_key', - default='', help='TLS private key file.') - option_parser.add_option('-c', '--certificate', dest='certificate', - default='', help='TLS certificate file.') - option_parser.add_option('--register_cygwin', action="store_true", - dest="register_cygwin", - help='Register Cygwin paths (on Win try bots)') - option_parser.add_option('--pidfile', help='path to pid file.') - options, args = option_parser.parse_args() - - if not options.port: - if options.use_tls: - options.port = _DEFAULT_WSS_PORT - else: - options.port = _DEFAULT_WS_PORT - - kwds = {'port': options.port, 'use_tls': options.use_tls} - if options.root: - kwds['root'] = options.root - if options.private_key: - kwds['private_key'] = options.private_key - if options.certificate: - kwds['certificate'] = options.certificate - kwds['register_cygwin'] = options.register_cygwin - if options.pidfile: - kwds['pidfile'] = options.pidfile - - pywebsocket = PyWebSocket(tempfile.gettempdir(), **kwds) - - if 'start' == options.server: - pywebsocket.start() - else: - pywebsocket.stop(force=True) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py new file mode 100644 index 0000000..e05a69d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py @@ -0,0 +1,76 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""WebKit Win implementation of the Port interface.""" + +import logging +import os + +from webkitpy.layout_tests.port.webkit import WebKitPort + +_log = logging.getLogger("webkitpy.layout_tests.port.win") + + +class WinPort(WebKitPort): + """WebKit Win implementation of the Port class.""" + + def __init__(self, port_name=None, options=None): + if port_name is None: + port_name = 'win' + WebKitPort.__init__(self, port_name, options) + + def baseline_search_path(self): + # Based on code from old-run-webkit-tests expectedDirectoryForTest() + port_names = ["win", "mac-snowleopard", "mac"] + return map(self._webkit_baseline_path, port_names) + + def _tests_for_other_platforms(self): + # FIXME: This list could be dynamic based on platform name and + # pushed into base.Port. + # This really need to be automated. + return [ + "platform/chromium", + "platform/gtk", + "platform/qt", + "platform/mac", + ] + + def _path_to_apache_config_file(self): + return os.path.join(self.layout_tests_dir(), 'http', 'conf', + 'cygwin-httpd.conf') + + def _shut_down_http_server(self, server_pid): + """Shut down the httpd web server. Blocks until it's fully + shut down. + + Args: + server_pid: The process ID of the running server. + """ + # Looks like we ignore server_pid. + # Copy/pasted from chromium-win. + self._executive.kill_all("httpd.exe") diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py index 83cf99de..fa4df9b 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py @@ -41,6 +41,10 @@ The script does the following for each platform specified: At the end, the script generates a html that compares old and new baselines. """ +from __future__ import with_statement + +import codecs +import copy import logging import optparse import os @@ -54,13 +58,17 @@ import urllib import webbrowser import zipfile -from layout_package import path_utils +from webkitpy.common.system.executive import run_command, ScriptError +from webkitpy.common.checkout.scm import detect_scm_system +import webkitpy.common.checkout.scm as scm + +import port from layout_package import test_expectations from test_types import image_diff from test_types import text_diff -# Repository type constants. -REPO_SVN, REPO_UNKNOWN = range(2) +_log = logging.getLogger("webkitpy.layout_tests." + "rebaseline_chromium_webkit_tests") BASELINE_SUFFIXES = ['.txt', '.png', '.checksum'] REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux'] @@ -89,7 +97,9 @@ def run_shell_with_return_code(command, print_output=False): """ # Use a shell for subcommands on Windows to get a PATH search. + # FIXME: shell=True is a trail of tears, and should be removed. use_shell = sys.platform.startswith('win') + # Note: Not thread safe: http://bugs.python.org/issue2320 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=use_shell) if print_output: @@ -137,11 +147,11 @@ def log_dashed_string(text, platform, logging_level=logging.INFO): msg = '%s %s %s' % (dashes, msg, dashes) if logging_level == logging.ERROR: - logging.error(msg) + _log.error(msg) elif logging_level == logging.WARNING: - logging.warn(msg) + _log.warn(msg) else: - logging.info(msg) + _log.info(msg) def setup_html_directory(html_directory): @@ -163,11 +173,11 @@ def setup_html_directory(html_directory): os.mkdir(html_directory) html_directory = os.path.join(html_directory, 'rebaseline_html') - logging.info('Html directory: "%s"', html_directory) + _log.info('Html directory: "%s"', html_directory) if os.path.exists(html_directory): shutil.rmtree(html_directory, True) - logging.info('Deleted file at html directory: "%s"', html_directory) + _log.info('Deleted file at html directory: "%s"', html_directory) if not os.path.exists(html_directory): os.mkdir(html_directory) @@ -191,7 +201,7 @@ def get_result_file_fullpath(html_directory, baseline_filename, platform, base, ext = os.path.splitext(baseline_filename) result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext) fullpath = os.path.join(html_directory, result_filename) - logging.debug(' Result file full path: "%s".', fullpath) + _log.debug(' Result file full path: "%s".', fullpath) return fullpath @@ -200,11 +210,21 @@ class Rebaseliner(object): REVISION_REGEX = r'<a href=\"(\d+)/\">' - def __init__(self, platform, options): - self._file_dir = path_utils.path_from_base('webkit', 'tools', - 'layout_tests') + def __init__(self, running_port, target_port, platform, options): + """ + Args: + running_port: the Port the script is running on. + target_port: the Port the script uses to find port-specific + configuration information like the test_expectations.txt + file location and the list of test platforms. + platform: the test platform to rebaseline + options: the command-line options object.""" self._platform = platform self._options = options + self._port = running_port + self._target_port = target_port + self._rebaseline_port = port.get( + self._target_port.test_platform_name_to_name(platform), options) self._rebaselining_tests = [] self._rebaselined_tests = [] @@ -212,14 +232,15 @@ class Rebaseliner(object): # -. compile list of tests that need rebaselining. # -. update the tests in test_expectations file after rebaseline # is done. + expectations_str = self._rebaseline_port.test_expectations() self._test_expectations = \ - test_expectations.TestExpectations(None, - self._file_dir, - platform, + test_expectations.TestExpectations(self._rebaseline_port, + None, + expectations_str, + self._platform, False, False) - - self._repo_type = self._get_repo_type() + self._scm = detect_scm_system(os.getcwd()) def run(self, backup): """Run rebaseline process.""" @@ -230,9 +251,9 @@ class Rebaseliner(object): log_dashed_string('Downloading archive', self._platform) archive_file = self._download_buildbot_archive() - logging.info('') + _log.info('') if not archive_file: - logging.error('No archive found.') + _log.error('No archive found.') return False log_dashed_string('Extracting and adding new baselines', @@ -243,34 +264,25 @@ class Rebaseliner(object): log_dashed_string('Updating rebaselined tests in file', self._platform) self._update_rebaselined_tests_in_file(backup) - logging.info('') + _log.info('') if len(self._rebaselining_tests) != len(self._rebaselined_tests): - logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN ' - 'REBASELINED.') - logging.warning(' Total tests needing rebaselining: %d', - len(self._rebaselining_tests)) - logging.warning(' Total tests rebaselined: %d', - len(self._rebaselined_tests)) + _log.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN ' + 'REBASELINED.') + _log.warning(' Total tests needing rebaselining: %d', + len(self._rebaselining_tests)) + _log.warning(' Total tests rebaselined: %d', + len(self._rebaselined_tests)) return False - logging.warning('All tests needing rebaselining were successfully ' - 'rebaselined.') + _log.warning('All tests needing rebaselining were successfully ' + 'rebaselined.') return True def get_rebaselining_tests(self): return self._rebaselining_tests - def _get_repo_type(self): - """Get the repository type that client is using.""" - output, return_code = run_shell_with_return_code(['svn', 'info'], - False) - if return_code == 0: - return REPO_SVN - - return REPO_UNKNOWN - def _compile_rebaselining_tests(self): """Compile list of tests that need rebaselining for the platform. @@ -282,16 +294,16 @@ class Rebaseliner(object): self._rebaselining_tests = \ self._test_expectations.get_rebaselining_failures() if not self._rebaselining_tests: - logging.warn('No tests found that need rebaselining.') + _log.warn('No tests found that need rebaselining.') return None - logging.info('Total number of tests needing rebaselining ' - 'for "%s": "%d"', self._platform, - len(self._rebaselining_tests)) + _log.info('Total number of tests needing rebaselining ' + 'for "%s": "%d"', self._platform, + len(self._rebaselining_tests)) test_no = 1 for test in self._rebaselining_tests: - logging.info(' %d: %s', test_no, test) + _log.info(' %d: %s', test_no, test) test_no += 1 return self._rebaselining_tests @@ -307,7 +319,7 @@ class Rebaseliner(object): None on failure. """ - logging.debug('Url to retrieve revision: "%s"', url) + _log.debug('Url to retrieve revision: "%s"', url) f = urllib.urlopen(url) content = f.read() @@ -315,11 +327,11 @@ class Rebaseliner(object): revisions = re.findall(self.REVISION_REGEX, content) if not revisions: - logging.error('Failed to find revision, content: "%s"', content) + _log.error('Failed to find revision, content: "%s"', content) return None revisions.sort(key=int) - logging.info('Latest revision: "%s"', revisions[len(revisions) - 1]) + _log.info('Latest revision: "%s"', revisions[len(revisions) - 1]) return revisions[len(revisions) - 1] def _get_archive_dir_name(self, platform, webkit_canary): @@ -336,8 +348,8 @@ class Rebaseliner(object): if platform in ARCHIVE_DIR_NAME_DICT: return ARCHIVE_DIR_NAME_DICT[platform] else: - logging.error('Cannot find platform key %s in archive ' - 'directory name dictionary', platform) + _log.error('Cannot find platform key %s in archive ' + 'directory name dictionary', platform) return None def _get_archive_url(self): @@ -348,21 +360,23 @@ class Rebaseliner(object): None on failure """ + if self._options.force_archive_url: + return self._options.force_archive_url + dir_name = self._get_archive_dir_name(self._platform, self._options.webkit_canary) if not dir_name: return None - logging.debug('Buildbot platform dir name: "%s"', dir_name) + _log.debug('Buildbot platform dir name: "%s"', dir_name) url_base = '%s/%s/' % (self._options.archive_url, dir_name) latest_revision = self._get_latest_revision(url_base) if latest_revision is None or latest_revision <= 0: return None - archive_url = ('%s%s/layout-test-results.zip' % (url_base, latest_revision)) - logging.info('Archive url: "%s"', archive_url) + _log.info('Archive url: "%s"', archive_url) return archive_url def _download_buildbot_archive(self): @@ -378,7 +392,7 @@ class Rebaseliner(object): return None fn = urllib.urlretrieve(url)[0] - logging.info('Archive downloaded and saved to file: "%s"', fn) + _log.info('Archive downloaded and saved to file: "%s"', fn) return fn def _extract_and_add_new_baselines(self, archive_file): @@ -395,32 +409,33 @@ class Rebaseliner(object): zip_file = zipfile.ZipFile(archive_file, 'r') zip_namelist = zip_file.namelist() - logging.debug('zip file namelist:') + _log.debug('zip file namelist:') for name in zip_namelist: - logging.debug(' ' + name) + _log.debug(' ' + name) - platform = path_utils.platform_name(self._platform) - logging.debug('Platform dir: "%s"', platform) + platform = self._rebaseline_port.test_platform_name_to_name( + self._platform) + _log.debug('Platform dir: "%s"', platform) test_no = 1 self._rebaselined_tests = [] for test in self._rebaselining_tests: - logging.info('Test %d: %s', test_no, test) + _log.info('Test %d: %s', test_no, test) found = False - svn_error = False + scm_error = False test_basename = os.path.splitext(test)[0] for suffix in BASELINE_SUFFIXES: archive_test_name = ('layout-test-results/%s-actual%s' % - (test_basename, suffix)) - logging.debug(' Archive test file name: "%s"', - archive_test_name) + (test_basename, suffix)) + _log.debug(' Archive test file name: "%s"', + archive_test_name) if not archive_test_name in zip_namelist: - logging.info(' %s file not in archive.', suffix) + _log.info(' %s file not in archive.', suffix) continue found = True - logging.info(' %s file found in archive.', suffix) + _log.info(' %s file found in archive.', suffix) # Extract new baseline from archive and save it to a temp file. data = zip_file.read(archive_test_name) @@ -431,11 +446,10 @@ class Rebaseliner(object): expected_filename = '%s-expected%s' % (test_basename, suffix) expected_fullpath = os.path.join( - path_utils.chromium_baseline_path(platform), - expected_filename) + self._rebaseline_port.baseline_path(), expected_filename) expected_fullpath = os.path.normpath(expected_fullpath) - logging.debug(' Expected file full path: "%s"', - expected_fullpath) + _log.debug(' Expected file full path: "%s"', + expected_fullpath) # TODO(victorw): for now, the rebaselining tool checks whether # or not THIS baseline is duplicate and should be skipped. @@ -443,33 +457,34 @@ class Rebaseliner(object): # and lower # levels and remove all duplicated baselines. if self._is_dup_baseline(temp_name, - expected_fullpath, - test, - suffix, - self._platform): + expected_fullpath, + test, + suffix, + self._platform): os.remove(temp_name) self._delete_baseline(expected_fullpath) continue # Create the new baseline directory if it doesn't already # exist. - path_utils.maybe_make_directory( + self._port.maybe_make_directory( os.path.dirname(expected_fullpath)) shutil.move(temp_name, expected_fullpath) - if not self._svn_add(expected_fullpath): - svn_error = True + if 0 != self._scm.add(expected_fullpath, return_exit_code=True): + # FIXME: print detailed diagnose messages + scm_error = True elif suffix != '.checksum': self._create_html_baseline_files(expected_fullpath) if not found: - logging.warn(' No new baselines found in archive.') + _log.warn(' No new baselines found in archive.') else: - if svn_error: - logging.warn(' Failed to add baselines to SVN.') + if scm_error: + _log.warn(' Failed to add baselines to your repository.') else: - logging.info(' Rebaseline succeeded.') + _log.info(' Rebaseline succeeded.') self._rebaselined_tests.append(test) test_no += 1 @@ -497,9 +512,10 @@ class Rebaseliner(object): True if the baseline is unnecessary. False otherwise. """ - test_filepath = os.path.join(path_utils.layout_tests_dir(), test) - all_baselines = path_utils.expected_baselines(test_filepath, - suffix, platform, True) + test_filepath = os.path.join(self._target_port.layout_tests_dir(), + test) + all_baselines = self._rebaseline_port.expected_baselines( + test_filepath, suffix, True) for (fallback_dir, fallback_file) in all_baselines: if fallback_dir and fallback_file: fallback_fullpath = os.path.normpath( @@ -507,8 +523,8 @@ class Rebaseliner(object): if fallback_fullpath.lower() != baseline_path.lower(): if not self._diff_baselines(new_baseline, fallback_fullpath): - logging.info(' Found same baseline at %s', - fallback_fullpath) + _log.info(' Found same baseline at %s', + fallback_fullpath) return True else: return False @@ -529,16 +545,16 @@ class Rebaseliner(object): ext1 = os.path.splitext(file1)[1].upper() ext2 = os.path.splitext(file2)[1].upper() if ext1 != ext2: - logging.warn('Files to compare have different ext. ' - 'File1: %s; File2: %s', file1, file2) + _log.warn('Files to compare have different ext. ' + 'File1: %s; File2: %s', file1, file2) return True if ext1 == '.PNG': - return image_diff.ImageDiff(self._platform, '').diff_files(file1, - file2) + return image_diff.ImageDiff(self._port, + '').diff_files(self._port, file1, file2) else: - return text_diff.TestTextDiff(self._platform, '').diff_files(file1, - file2) + return text_diff.TestTextDiff(self._port, + '').diff_files(self._port, file1, file2) def _delete_baseline(self, filename): """Remove the file from repository and delete it from disk. @@ -549,15 +565,7 @@ class Rebaseliner(object): if not filename or not os.path.isfile(filename): return - - if self._repo_type == REPO_SVN: - parent_dir, basename = os.path.split(filename) - original_dir = os.getcwd() - os.chdir(parent_dir) - run_shell(['svn', 'delete', '--force', basename], False) - os.chdir(original_dir) - else: - os.remove(filename) + self._scm.delete(filename) def _update_rebaselined_tests_in_file(self, backup): """Update the rebaselined tests in test expectations file. @@ -570,91 +578,25 @@ class Rebaseliner(object): """ if self._rebaselined_tests: - self._test_expectations.remove_platform_from_file( - self._rebaselined_tests, self._platform, backup) - else: - logging.info('No test was rebaselined so nothing to remove.') - - def _svn_add(self, filename): - """Add the file to SVN repository. - - Args: - filename: full path of the file to add. - - Returns: - True if the file already exists in SVN or is sucessfully added - to SVN. - False otherwise. - """ - - if not filename: - return False - - parent_dir, basename = os.path.split(filename) - if self._repo_type != REPO_SVN or parent_dir == filename: - logging.info("No svn checkout found, skip svn add.") - return True - - original_dir = os.getcwd() - os.chdir(parent_dir) - status_output = run_shell(['svn', 'status', basename], False) - os.chdir(original_dir) - output = status_output.upper() - if output.startswith('A') or output.startswith('M'): - logging.info(' File already added to SVN: "%s"', filename) - return True - - if output.find('IS NOT A WORKING COPY') >= 0: - logging.info(' File is not a working copy, add its parent: "%s"', - parent_dir) - return self._svn_add(parent_dir) - - os.chdir(parent_dir) - add_output = run_shell(['svn', 'add', basename], True) - os.chdir(original_dir) - output = add_output.upper().rstrip() - if output.startswith('A') and output.find(basename.upper()) >= 0: - logging.info(' Added new file: "%s"', filename) - self._svn_prop_set(filename) - return True - - if (not status_output) and (add_output.upper().find( - 'ALREADY UNDER VERSION CONTROL') >= 0): - logging.info(' File already under SVN and has no change: "%s"', - filename) - return True - - logging.warn(' Failed to add file to SVN: "%s"', filename) - logging.warn(' Svn status output: "%s"', status_output) - logging.warn(' Svn add output: "%s"', add_output) - return False - - def _svn_prop_set(self, filename): - """Set the baseline property - - Args: - filename: full path of the file to add. - - Returns: - True if the file already exists in SVN or is sucessfully added - to SVN. - False otherwise. - """ - ext = os.path.splitext(filename)[1].upper() - if ext != '.TXT' and ext != '.PNG' and ext != '.CHECKSUM': - return - - parent_dir, basename = os.path.split(filename) - original_dir = os.getcwd() - os.chdir(parent_dir) - if ext == '.PNG': - cmd = ['svn', 'pset', 'svn:mime-type', 'image/png', basename] + new_expectations = ( + self._test_expectations.remove_platform_from_expectations( + self._rebaselined_tests, self._platform)) + path = self._target_port.path_to_test_expectations_file() + if backup: + date_suffix = time.strftime('%Y%m%d%H%M%S', + time.localtime(time.time())) + backup_file = ('%s.orig.%s' % (path, date_suffix)) + if os.path.exists(backup_file): + os.remove(backup_file) + _log.info('Saving original file to "%s"', backup_file) + os.rename(path, backup_file) + # FIXME: What encoding are these files? + # Or is new_expectations always a byte array? + with open(path, "w") as file: + file.write(new_expectations) + self._scm.add(path) else: - cmd = ['svn', 'pset', 'svn:eol-style', 'LF', basename] - - logging.debug(' Set svn prop: %s', ' '.join(cmd)) - run_shell(cmd, False) - os.chdir(original_dir) + _log.info('No test was rebaselined so nothing to remove.') def _create_html_baseline_files(self, baseline_fullpath): """Create baseline files (old, new and diff) in html directory. @@ -674,62 +616,42 @@ class Rebaseliner(object): baseline_filename, self._platform, 'new') shutil.copyfile(baseline_fullpath, new_file) - logging.info(' Html: copied new baseline file from "%s" to "%s".', - baseline_fullpath, new_file) + _log.info(' Html: copied new baseline file from "%s" to "%s".', + baseline_fullpath, new_file) + + # Get the old baseline from the repository and save to the html directory. + try: + output = self._scm.show_head(baseline_fullpath) + except ScriptError, e: + _log.info(e) + output = "" - # Get the old baseline from SVN and save to the html directory. - output = run_shell(['svn', 'cat', '-r', 'BASE', baseline_fullpath]) if (not output) or (output.upper().rstrip().endswith( 'NO SUCH FILE OR DIRECTORY')): - logging.info(' No base file: "%s"', baseline_fullpath) + _log.info(' No base file: "%s"', baseline_fullpath) return base_file = get_result_file_fullpath(self._options.html_directory, baseline_filename, self._platform, 'old') - f = open(base_file, 'wb') - f.write(output) - f.close() - logging.info(' Html: created old baseline file: "%s".', - base_file) + # FIXME: This assumes run_shell returns a byte array. + # We should be using an explicit encoding here. + with open(base_file, "wb") as file: + file.write(output) + _log.info(' Html: created old baseline file: "%s".', + base_file) # Get the diff between old and new baselines and save to the html dir. if baseline_filename.upper().endswith('.TXT'): - # If the user specified a custom diff command in their svn config - # file, then it'll be used when we do svn diff, which we don't want - # to happen since we want the unified diff. Using --diff-cmd=diff - # doesn't always work, since they can have another diff executable - # in their path that gives different line endings. So we use a - # bogus temp directory as the config directory, which gets - # around these problems. - if sys.platform.startswith("win"): - parent_dir = tempfile.gettempdir() - else: - parent_dir = sys.path[0] # tempdir is not secure. - bogus_dir = os.path.join(parent_dir, "temp_svn_config") - logging.debug(' Html: temp config dir: "%s".', bogus_dir) - if not os.path.exists(bogus_dir): - os.mkdir(bogus_dir) - delete_bogus_dir = True - else: - delete_bogus_dir = False - - output = run_shell(["svn", "diff", "--config-dir", bogus_dir, - baseline_fullpath]) + output = self._scm.diff_for_file(baseline_fullpath, log=_log) if output: diff_file = get_result_file_fullpath( self._options.html_directory, baseline_filename, self._platform, 'diff') - f = open(diff_file, 'wb') - f.write(output) - f.close() - logging.info(' Html: created baseline diff file: "%s".', - diff_file) - - if delete_bogus_dir: - shutil.rmtree(bogus_dir, True) - logging.debug(' Html: removed temp config dir: "%s".', - bogus_dir) - + # FIXME: This assumes run_shell returns a byte array, not unicode() + with open(diff_file, 'wb') as file: + file.write(output) + _log.info(' Html: created baseline diff file: "%s".', + diff_file) class HtmlGenerator(object): """Class to generate rebaselining result comparison html.""" @@ -777,8 +699,9 @@ class HtmlGenerator(object): '<img style="width: 200" src="%(uri)s" /></a></td>') HTML_TR = '<tr>%s</tr>' - def __init__(self, options, platforms, rebaselining_tests): + def __init__(self, target_port, options, platforms, rebaselining_tests): self._html_directory = options.html_directory + self._target_port = target_port self._platforms = platforms self._rebaselining_tests = rebaselining_tests self._html_file = os.path.join(options.html_directory, @@ -787,7 +710,7 @@ class HtmlGenerator(object): def generate_html(self): """Generate html file for rebaselining result comparison.""" - logging.info('Generating html file') + _log.info('Generating html file') html_body = '' if not self._rebaselining_tests: @@ -798,29 +721,28 @@ class HtmlGenerator(object): test_no = 1 for test in tests: - logging.info('Test %d: %s', test_no, test) + _log.info('Test %d: %s', test_no, test) html_body += self._generate_html_for_one_test(test) html = self.HTML_REBASELINE % ({'time': time.asctime(), 'body': html_body}) - logging.debug(html) + _log.debug(html) - f = open(self._html_file, 'w') - f.write(html) - f.close() + with codecs.open(self._html_file, "w", "utf-8") as file: + file.write(html) - logging.info('Baseline comparison html generated at "%s"', - self._html_file) + _log.info('Baseline comparison html generated at "%s"', + self._html_file) def show_html(self): """Launch the rebaselining html in brwoser.""" - logging.info('Launching html: "%s"', self._html_file) + _log.info('Launching html: "%s"', self._html_file) - html_uri = path_utils.filename_to_uri(self._html_file) + html_uri = self._target_port.filename_to_uri(self._html_file) webbrowser.open(html_uri, 1) - logging.info('Html launched.') + _log.info('Html launched.') def _generate_baseline_links(self, test_basename, suffix, platform): """Generate links for baseline results (old, new and diff). @@ -835,18 +757,18 @@ class HtmlGenerator(object): """ baseline_filename = '%s-expected%s' % (test_basename, suffix) - logging.debug(' baseline filename: "%s"', baseline_filename) + _log.debug(' baseline filename: "%s"', baseline_filename) new_file = get_result_file_fullpath(self._html_directory, baseline_filename, platform, 'new') - logging.info(' New baseline file: "%s"', new_file) + _log.info(' New baseline file: "%s"', new_file) if not os.path.exists(new_file): - logging.info(' No new baseline file: "%s"', new_file) + _log.info(' No new baseline file: "%s"', new_file) return '' old_file = get_result_file_fullpath(self._html_directory, baseline_filename, platform, 'old') - logging.info(' Old baseline file: "%s"', old_file) + _log.info(' Old baseline file: "%s"', old_file) if suffix == '.png': html_td_link = self.HTML_TD_LINK_IMG else: @@ -855,24 +777,25 @@ class HtmlGenerator(object): links = '' if os.path.exists(old_file): links += html_td_link % { - 'uri': path_utils.filename_to_uri(old_file), + 'uri': self._target_port.filename_to_uri(old_file), 'name': baseline_filename} else: - logging.info(' No old baseline file: "%s"', old_file) + _log.info(' No old baseline file: "%s"', old_file) links += self.HTML_TD_NOLINK % '' - links += html_td_link % {'uri': path_utils.filename_to_uri(new_file), + links += html_td_link % {'uri': self._target_port.filename_to_uri( + new_file), 'name': baseline_filename} diff_file = get_result_file_fullpath(self._html_directory, baseline_filename, platform, 'diff') - logging.info(' Baseline diff file: "%s"', diff_file) + _log.info(' Baseline diff file: "%s"', diff_file) if os.path.exists(diff_file): - links += html_td_link % {'uri': path_utils.filename_to_uri( + links += html_td_link % {'uri': self._target_port.filename_to_uri( diff_file), 'name': 'Diff'} else: - logging.info(' No baseline diff file: "%s"', diff_file) + _log.info(' No baseline diff file: "%s"', diff_file) links += self.HTML_TD_NOLINK % '' return links @@ -888,13 +811,13 @@ class HtmlGenerator(object): """ test_basename = os.path.basename(os.path.splitext(test)[0]) - logging.info(' basename: "%s"', test_basename) + _log.info(' basename: "%s"', test_basename) rows = [] for suffix in BASELINE_SUFFIXES: if suffix == '.checksum': continue - logging.info(' Checking %s files', suffix) + _log.info(' Checking %s files', suffix) for platform in self._platforms: links = self._generate_baseline_links(test_basename, suffix, platform) @@ -903,17 +826,18 @@ class HtmlGenerator(object): suffix) row += self.HTML_TD_NOLINK % platform row += links - logging.debug(' html row: %s', row) + _log.debug(' html row: %s', row) rows.append(self.HTML_TR % row) if rows: - test_path = os.path.join(path_utils.layout_tests_dir(), test) - html = self.HTML_TR_TEST % (path_utils.filename_to_uri(test_path), - test) + test_path = os.path.join(self._target_port.layout_tests_dir(), + test) + html = self.HTML_TR_TEST % ( + self._target_port.filename_to_uri(test_path), test) html += self.HTML_TEST_DETAIL % ' '.join(rows) - logging.debug(' html for test: %s', html) + _log.debug(' html for test: %s', html) return self.HTML_TABLE_TEST % html return '' @@ -929,6 +853,28 @@ class HtmlGenerator(object): return 'Other' +def get_host_port_object(options): + """Return a port object for the platform we're running on.""" + # The only thing we really need on the host is a way to diff + # text files and image files, which means we need to check that some + # version of ImageDiff has been built. We will look for either Debug + # or Release versions of the default port on the platform. + options.configuration = "Release" + port_obj = port.get(None, options) + if not port_obj.check_image_diff(override_step=None, logging=False): + _log.debug('No release version of the image diff binary was found.') + options.configuration = "Debug" + port_obj = port.get(None, options) + if not port_obj.check_image_diff(override_step=None, logging=False): + _log.error('No version of image diff was found. Check your build.') + return None + else: + _log.debug('Found the debug version of the image diff binary.') + else: + _log.debug('Found the release version of the image diff binary.') + return port_obj + + def main(): """Main function to produce new baselines.""" @@ -938,6 +884,10 @@ def main(): default=False, help='include debug-level logging.') + option_parser.add_option('-q', '--quiet', + action='store_true', + help='Suppress result HTML viewing') + option_parser.add_option('-p', '--platforms', default='mac,win,win-xp,win-vista,linux', help=('Comma delimited list of platforms ' @@ -948,6 +898,9 @@ def main(): 'layout_test_results'), help=('Url to find the layout test result archive' ' file.')) + option_parser.add_option('-U', '--force_archive_url', + help=('Url of result zip file. This option is for debugging ' + 'purposes')) option_parser.add_option('-w', '--webkit_canary', action='store_true', @@ -963,11 +916,33 @@ def main(): option_parser.add_option('-d', '--html_directory', default='', - help=('The directory that stores the results for' - ' rebaselining comparison.')) + help=('The directory that stores the results for ' + 'rebaselining comparison.')) + option_parser.add_option('', '--use_drt', + action='store_true', + default=False, + help=('Use ImageDiff from DumpRenderTree instead ' + 'of image_diff for pixel tests.')) + + option_parser.add_option('', '--target-platform', + default='chromium', + help=('The target platform to rebaseline ' + '("mac", "chromium", "qt", etc.). Defaults ' + 'to "chromium".')) options = option_parser.parse_args()[0] + # We need to create three different Port objects over the life of this + # script. |target_port_obj| is used to determine configuration information: + # location of the expectations file, names of ports to rebaseline, etc. + # |port_obj| is used for runtime functionality like actually diffing + # Then we create a rebaselining port to actual find and manage the + # baselines. + target_options = copy.copy(options) + if options.target_platform == 'chromium': + target_options.chromium = True + target_port_obj = port.get(None, target_options) + # Set up our logging format. log_level = logging.INFO if options.verbose: @@ -977,15 +952,19 @@ def main(): '%(levelname)s %(message)s'), datefmt='%y%m%d %H:%M:%S') - # Verify 'platforms' option is valid + host_port_obj = get_host_port_object(options) + if not host_port_obj: + sys.exit(1) + + # Verify 'platforms' option is valid. if not options.platforms: - logging.error('Invalid "platforms" option. --platforms must be ' - 'specified in order to rebaseline.') + _log.error('Invalid "platforms" option. --platforms must be ' + 'specified in order to rebaseline.') sys.exit(1) platforms = [p.strip().lower() for p in options.platforms.split(',')] for platform in platforms: if not platform in REBASELINE_PLATFORM_ORDER: - logging.error('Invalid platform: "%s"' % (platform)) + _log.error('Invalid platform: "%s"' % (platform)) sys.exit(1) # Adjust the platform order so rebaseline tool is running at the order of @@ -1002,9 +981,10 @@ def main(): rebaselining_tests = set() backup = options.backup for platform in rebaseline_platforms: - rebaseliner = Rebaseliner(platform, options) + rebaseliner = Rebaseliner(host_port_obj, target_port_obj, + platform, options) - logging.info('') + _log.info('') log_dashed_string('Rebaseline started', platform) if rebaseliner.run(backup): # Only need to backup one original copy of test expectation file. @@ -1015,13 +995,15 @@ def main(): rebaselining_tests |= set(rebaseliner.get_rebaselining_tests()) - logging.info('') + _log.info('') log_dashed_string('Rebaselining result comparison started', None) - html_generator = HtmlGenerator(options, + html_generator = HtmlGenerator(target_port_obj, + options, rebaseline_platforms, rebaselining_tests) html_generator.generate_html() - html_generator.show_html() + if not options.quiet: + html_generator.show_html() log_dashed_string('Rebaselining result comparison done', None) sys.exit(0) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py new file mode 100644 index 0000000..fa03238 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py @@ -0,0 +1,88 @@ +#!/usr/bin/python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for rebaseline_chromium_webkit_tests.py.""" + +import unittest + +from webkitpy.layout_tests import port +from webkitpy.layout_tests import rebaseline_chromium_webkit_tests + + +class MockPort(object): + def __init__(self, image_diff_exists): + self.image_diff_exists = image_diff_exists + + def check_image_diff(self, override_step, logging): + return self.image_diff_exists + + +class MockOptions(object): + def __init__(self): + self.configuration = None + + +def get_mock_get(config_expectations): + def mock_get(port_name, options): + return MockPort(config_expectations[options.configuration]) + return mock_get + + +class TestGetHostPortObject(unittest.TestCase): + def assert_result(self, release_present, debug_present, valid_port_obj): + # Tests whether we get a valid port object returned when we claim + # that Image diff is (or isn't) present in the two configs. + port.get = get_mock_get({'Release': release_present, + 'Debug': debug_present}) + options = MockOptions() + port_obj = rebaseline_chromium_webkit_tests.get_host_port_object( + options) + if valid_port_obj: + self.assertNotEqual(port_obj, None) + else: + self.assertEqual(port_obj, None) + + def test_get_host_port_object(self): + # Save the normal port.get() function for future testing. + old_get = port.get + + # Test whether we get a valid port object back for the four + # possible cases of having ImageDiffs built. It should work when + # there is at least one binary present. + self.assert_result(False, False, False) + self.assert_result(True, False, True) + self.assert_result(False, True, True) + self.assert_result(True, True, True) + + # Restore the normal port.get() function. + port.get = old_get + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_chromium_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_chromium_webkit_tests.py deleted file mode 100755 index f0b68ee..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_chromium_webkit_tests.py +++ /dev/null @@ -1,1624 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Run layout tests using the test_shell. - -This is a port of the existing webkit test script run-webkit-tests. - -The TestRunner class runs a series of tests (TestType interface) against a set -of test files. If a test file fails a TestType, it returns a list TestFailure -objects to the TestRunner. The TestRunner then aggregates the TestFailures to -create a final report. - -This script reads several files, if they exist in the test_lists subdirectory -next to this script itself. Each should contain a list of paths to individual -tests or entire subdirectories of tests, relative to the outermost test -directory. Entire lines starting with '//' (comments) will be ignored. - -For details of the files' contents and purposes, see test_lists/README. -""" - -import errno -import glob -import logging -import math -import optparse -import os -import Queue -import random -import re -import shutil -import sys -import time -import traceback - -import simplejson - -from layout_package import test_expectations -from layout_package import json_layout_results_generator -from layout_package import metered_stream -from layout_package import test_failures -from layout_package import test_shell_thread -from layout_package import test_files -from test_types import fuzzy_image_diff -from test_types import image_diff -from test_types import test_type_base -from test_types import text_diff - -import port - -# Indicates that we want detailed progress updates in the output (prints -# directory-by-directory feedback). -LOG_DETAILED_PROGRESS = 'detailed-progress' - -# Log any unexpected results while running (instead of just at the end). -LOG_UNEXPECTED = 'unexpected' - -# Builder base URL where we have the archived test results. -BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/" - -TestExpectationsFile = test_expectations.TestExpectationsFile - - -class TestInfo: - """Groups information about a test for easy passing of data.""" - - def __init__(self, port, filename, timeout): - """Generates the URI and stores the filename and timeout for this test. - Args: - filename: Full path to the test. - timeout: Timeout for running the test in TestShell. - """ - self.filename = filename - self.uri = port.filename_to_uri(filename) - self.timeout = timeout - expected_hash_file = port.expected_filename(filename, '.checksum') - try: - self.image_hash = open(expected_hash_file, "r").read() - except IOError, e: - if errno.ENOENT != e.errno: - raise - self.image_hash = None - - -class ResultSummary(object): - """A class for partitioning the test results we get into buckets. - - This class is basically a glorified struct and it's private to this file - so we don't bother with any information hiding.""" - - def __init__(self, expectations, test_files): - self.total = len(test_files) - self.remaining = self.total - self.expectations = expectations - self.expected = 0 - self.unexpected = 0 - self.tests_by_expectation = {} - self.tests_by_timeline = {} - self.results = {} - self.unexpected_results = {} - self.failures = {} - self.tests_by_expectation[test_expectations.SKIP] = set() - for expectation in TestExpectationsFile.EXPECTATIONS.values(): - self.tests_by_expectation[expectation] = set() - for timeline in TestExpectationsFile.TIMELINES.values(): - self.tests_by_timeline[timeline] = ( - expectations.get_tests_with_timeline(timeline)) - - def add(self, test, failures, result, expected): - """Add a result into the appropriate bin. - - Args: - test: test file name - failures: list of failure objects from test execution - result: result of test (PASS, IMAGE, etc.). - expected: whether the result was what we expected it to be. - """ - - self.tests_by_expectation[result].add(test) - self.results[test] = result - self.remaining -= 1 - if len(failures): - self.failures[test] = failures - if expected: - self.expected += 1 - else: - self.unexpected_results[test] = result - self.unexpected += 1 - - -class TestRunner: - """A class for managing running a series of tests on a series of layout - test files.""" - - HTTP_SUBDIR = os.sep.join(['', 'http', '']) - WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', '']) - - # The per-test timeout in milliseconds, if no --time-out-ms option was - # given to run_webkit_tests. This should correspond to the default timeout - # in test_shell.exe. - DEFAULT_TEST_TIMEOUT_MS = 6 * 1000 - - NUM_RETRY_ON_UNEXPECTED_FAILURE = 1 - - def __init__(self, port, options, meter): - """Initialize test runner data structures. - - Args: - port: an object implementing port-specific - options: a dictionary of command line options - meter: a MeteredStream object to record updates to. - """ - self._port = port - self._options = options - self._meter = meter - - # disable wss server. need to install pyOpenSSL on buildbots. - # self._websocket_secure_server = websocket_server.PyWebSocket( - # options.results_directory, use_tls=True, port=9323) - - # a list of TestType objects - self._test_types = [] - - # a set of test files, and the same tests as a list - self._test_files = set() - self._test_files_list = None - self._result_queue = Queue.Queue() - - # These are used for --log detailed-progress to track status by - # directory. - self._current_dir = None - self._current_progress_str = "" - self._current_test_number = 0 - - def __del__(self): - logging.debug("flushing stdout") - sys.stdout.flush() - logging.debug("flushing stderr") - sys.stderr.flush() - logging.debug("stopping http server") - self._port.stop_http_server() - logging.debug("stopping websocket server") - self._port.stop_websocket_server() - - def gather_file_paths(self, paths): - """Find all the files to test. - - Args: - paths: a list of globs to use instead of the defaults.""" - self._test_files = test_files.gather_test_files(self._port, paths) - - def parse_expectations(self, test_platform_name, is_debug_mode): - """Parse the expectations from the test_list files and return a data - structure holding them. Throws an error if the test_list files have - invalid syntax.""" - if self._options.lint_test_files: - test_files = None - else: - test_files = self._test_files - - try: - expectations_str = self._port.test_expectations() - self._expectations = test_expectations.TestExpectations( - self._port, test_files, expectations_str, test_platform_name, - is_debug_mode, self._options.lint_test_files) - return self._expectations - except Exception, err: - if self._options.lint_test_files: - print str(err) - else: - raise err - - def prepare_lists_and_print_output(self, write): - """Create appropriate subsets of test lists and returns a - ResultSummary object. Also prints expected test counts. - - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - """ - - # Remove skipped - both fixable and ignored - files from the - # top-level list of files to test. - num_all_test_files = len(self._test_files) - write("Found: %d tests" % (len(self._test_files))) - skipped = set() - if num_all_test_files > 1 and not self._options.force: - skipped = self._expectations.get_tests_with_result_type( - test_expectations.SKIP) - self._test_files -= skipped - - # Create a sorted list of test files so the subset chunk, - # if used, contains alphabetically consecutive tests. - self._test_files_list = list(self._test_files) - if self._options.randomize_order: - random.shuffle(self._test_files_list) - else: - self._test_files_list.sort() - - # If the user specifies they just want to run a subset of the tests, - # just grab a subset of the non-skipped tests. - if self._options.run_chunk or self._options.run_part: - chunk_value = self._options.run_chunk or self._options.run_part - test_files = self._test_files_list - try: - (chunk_num, chunk_len) = chunk_value.split(":") - chunk_num = int(chunk_num) - assert(chunk_num >= 0) - test_size = int(chunk_len) - assert(test_size > 0) - except: - logging.critical("invalid chunk '%s'" % chunk_value) - sys.exit(1) - - # Get the number of tests - num_tests = len(test_files) - - # Get the start offset of the slice. - if self._options.run_chunk: - chunk_len = test_size - # In this case chunk_num can be really large. We need - # to make the slave fit in the current number of tests. - slice_start = (chunk_num * chunk_len) % num_tests - else: - # Validate the data. - assert(test_size <= num_tests) - assert(chunk_num <= test_size) - - # To count the chunk_len, and make sure we don't skip - # some tests, we round to the next value that fits exactly - # all the parts. - rounded_tests = num_tests - if rounded_tests % test_size != 0: - rounded_tests = (num_tests + test_size - - (num_tests % test_size)) - - chunk_len = rounded_tests / test_size - slice_start = chunk_len * (chunk_num - 1) - # It does not mind if we go over test_size. - - # Get the end offset of the slice. - slice_end = min(num_tests, slice_start + chunk_len) - - files = test_files[slice_start:slice_end] - - tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % ( - (slice_end - slice_start), slice_start, slice_end, num_tests) - write(tests_run_msg) - - # If we reached the end and we don't have enough tests, we run some - # from the beginning. - if (self._options.run_chunk and - (slice_end - slice_start < chunk_len)): - extra = 1 + chunk_len - (slice_end - slice_start) - extra_msg = (' last chunk is partial, appending [0:%d]' % - extra) - write(extra_msg) - tests_run_msg += "\n" + extra_msg - files.extend(test_files[0:extra]) - tests_run_filename = os.path.join(self._options.results_directory, - "tests_run.txt") - tests_run_file = open(tests_run_filename, "w") - tests_run_file.write(tests_run_msg + "\n") - tests_run_file.close() - - len_skip_chunk = int(len(files) * len(skipped) / - float(len(self._test_files))) - skip_chunk_list = list(skipped)[0:len_skip_chunk] - skip_chunk = set(skip_chunk_list) - - # Update expectations so that the stats are calculated correctly. - # We need to pass a list that includes the right # of skipped files - # to ParseExpectations so that ResultSummary() will get the correct - # stats. So, we add in the subset of skipped files, and then - # subtract them back out. - self._test_files_list = files + skip_chunk_list - self._test_files = set(self._test_files_list) - - self._expectations = self.parse_expectations( - self._port.test_platform_name(), - self._options.target == 'Debug') - - self._test_files = set(files) - self._test_files_list = files - else: - skip_chunk = skipped - - result_summary = ResultSummary(self._expectations, - self._test_files | skip_chunk) - self._print_expected_results_of_type(write, result_summary, - test_expectations.PASS, "passes") - self._print_expected_results_of_type(write, result_summary, - test_expectations.FAIL, "failures") - self._print_expected_results_of_type(write, result_summary, - test_expectations.FLAKY, "flaky") - self._print_expected_results_of_type(write, result_summary, - test_expectations.SKIP, "skipped") - - - if self._options.force: - write('Running all tests, including skips (--force)') - else: - # Note that we don't actually run the skipped tests (they were - # subtracted out of self._test_files, above), but we stub out the - # results here so the statistics can remain accurate. - for test in skip_chunk: - result_summary.add(test, [], test_expectations.SKIP, - expected=True) - write("") - - return result_summary - - def add_test_type(self, test_type): - """Add a TestType to the TestRunner.""" - self._test_types.append(test_type) - - def _get_dir_for_test_file(self, test_file): - """Returns the highest-level directory by which to shard the given - test file.""" - index = test_file.rfind(os.sep + 'LayoutTests' + os.sep) - - test_file = test_file[index + len('LayoutTests/'):] - test_file_parts = test_file.split(os.sep, 1) - directory = test_file_parts[0] - test_file = test_file_parts[1] - - # The http tests are very stable on mac/linux. - # TODO(ojan): Make the http server on Windows be apache so we can - # turn shard the http tests there as well. Switching to apache is - # what made them stable on linux/mac. - return_value = directory - while ((directory != 'http' or sys.platform in ('darwin', 'linux2')) - and test_file.find(os.sep) >= 0): - test_file_parts = test_file.split(os.sep, 1) - directory = test_file_parts[0] - return_value = os.path.join(return_value, directory) - test_file = test_file_parts[1] - - return return_value - - def _get_test_info_for_file(self, test_file): - """Returns the appropriate TestInfo object for the file. Mostly this - is used for looking up the timeout value (in ms) to use for the given - test.""" - if self._expectations.has_modifier(test_file, test_expectations.SLOW): - return TestInfo(self._port, test_file, - self._options.slow_time_out_ms) - return TestInfo(self._port, test_file, self._options.time_out_ms) - - def _get_test_file_queue(self, test_files): - """Create the thread safe queue of lists of (test filenames, test URIs) - tuples. Each TestShellThread pulls a list from this queue and runs - those tests in order before grabbing the next available list. - - Shard the lists by directory. This helps ensure that tests that depend - on each other (aka bad tests!) continue to run together as most - cross-tests dependencies tend to occur within the same directory. - - Return: - The Queue of lists of TestInfo objects. - """ - - if (self._options.experimental_fully_parallel or - self._is_single_threaded()): - filename_queue = Queue.Queue() - for test_file in test_files: - filename_queue.put( - ('.', [self._get_test_info_for_file(test_file)])) - return filename_queue - - tests_by_dir = {} - for test_file in test_files: - directory = self._get_dir_for_test_file(test_file) - tests_by_dir.setdefault(directory, []) - tests_by_dir[directory].append( - self._get_test_info_for_file(test_file)) - - # Sort by the number of tests in the dir so that the ones with the - # most tests get run first in order to maximize parallelization. - # Number of tests is a good enough, but not perfect, approximation - # of how long that set of tests will take to run. We can't just use - # a PriorityQueue until we move # to Python 2.6. - test_lists = [] - http_tests = None - for directory in tests_by_dir: - test_list = tests_by_dir[directory] - # Keep the tests in alphabetical order. - # TODO: Remove once tests are fixed so they can be run in any - # order. - test_list.reverse() - test_list_tuple = (directory, test_list) - if directory == 'LayoutTests' + os.sep + 'http': - http_tests = test_list_tuple - else: - test_lists.append(test_list_tuple) - test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1]))) - - # Put the http tests first. There are only a couple hundred of them, - # but each http test takes a very long time to run, so sorting by the - # number of tests doesn't accurately capture how long they take to run. - if http_tests: - test_lists.insert(0, http_tests) - - filename_queue = Queue.Queue() - for item in test_lists: - filename_queue.put(item) - return filename_queue - - def _get_test_shell_args(self, index): - """Returns the tuple of arguments for tests and for test_shell.""" - shell_args = [] - test_args = test_type_base.TestArguments() - png_path = None - if not self._options.no_pixel_tests: - png_path = os.path.join(self._options.results_directory, - "png_result%s.png" % index) - shell_args.append("--pixel-tests=" + png_path) - test_args.png_path = png_path - - test_args.new_baseline = self._options.new_baseline - - test_args.show_sources = self._options.sources - - if self._options.startup_dialog: - shell_args.append('--testshell-startup-dialog') - - if self._options.gp_fault_error_box: - shell_args.append('--gp-fault-error-box') - - return test_args, png_path, shell_args - - def _contains_tests(self, subdir): - for test_file in self._test_files_list: - if test_file.find(subdir) >= 0: - return True - return False - - def _instantiate_test_shell_threads(self, test_files, result_summary): - """Instantitates and starts the TestShellThread(s). - - Return: - The list of threads. - """ - filename_queue = self._get_test_file_queue(test_files) - - # Instantiate TestShellThreads and start them. - threads = [] - for i in xrange(int(self._options.num_test_shells)): - # Create separate TestTypes instances for each thread. - test_types = [] - for t in self._test_types: - test_types.append(t(self._port, self._options.platform, - self._options.results_directory)) - - test_args, png_path, shell_args = self._get_test_shell_args(i) - thread = test_shell_thread.TestShellThread(self._port, - filename_queue, - self._result_queue, - test_types, - test_args, - png_path, - shell_args, - self._options) - if self._is_single_threaded(): - thread.run_in_main_thread(self, result_summary) - else: - thread.start() - threads.append(thread) - - return threads - - def _is_single_threaded(self): - """Returns whether we should run all the tests in the main thread.""" - return int(self._options.num_test_shells) == 1 - - def _run_tests(self, file_list, result_summary): - """Runs the tests in the file_list. - - Return: A tuple (failures, thread_timings, test_timings, - individual_test_timings) - failures is a map from test to list of failure types - thread_timings is a list of dicts with the total runtime - of each thread with 'name', 'num_tests', 'total_time' properties - test_timings is a list of timings for each sharded subdirectory - of the form [time, directory_name, num_tests] - individual_test_timings is a list of run times for each test - in the form {filename:filename, test_run_time:test_run_time} - result_summary: summary object to populate with the results - """ - threads = self._instantiate_test_shell_threads(file_list, - result_summary) - - # Wait for the threads to finish and collect test failures. - failures = {} - test_timings = {} - individual_test_timings = [] - thread_timings = [] - try: - for thread in threads: - while thread.isAlive(): - # Let it timeout occasionally so it can notice a - # KeyboardInterrupt. Actually, the timeout doesn't - # really matter: apparently it suffices to not use - # an indefinite blocking join for it to - # be interruptible by KeyboardInterrupt. - thread.join(0.1) - self.update_summary(result_summary) - thread_timings.append({'name': thread.getName(), - 'num_tests': thread.get_num_tests(), - 'total_time': thread.get_total_time()}) - test_timings.update(thread.get_directory_timing_stats()) - individual_test_timings.extend( - thread.get_individual_test_stats()) - except KeyboardInterrupt: - for thread in threads: - thread.cancel() - self._port.stop_helper() - raise - for thread in threads: - # Check whether a TestShellThread died before normal completion. - exception_info = thread.get_exception_info() - if exception_info is not None: - # Re-raise the thread's exception here to make it clear that - # testing was aborted. Otherwise, the tests that did not run - # would be assumed to have passed. - raise exception_info[0], exception_info[1], exception_info[2] - - # Make sure we pick up any remaining tests. - self.update_summary(result_summary) - return (thread_timings, test_timings, individual_test_timings) - - def run(self, result_summary): - """Run all our tests on all our test files. - - For each test file, we run each test type. If there are any failures, - we collect them for reporting. - - Args: - result_summary: a summary object tracking the test results. - - Return: - We return nonzero if there are regressions compared to the last run. - """ - if not self._test_files: - return 0 - start_time = time.time() - - # Start up any helper needed - if not self._options.no_pixel_tests: - self._port.start_helper() - - if self._contains_tests(self.HTTP_SUBDIR): - self._port.start_http_server() - - if self._contains_tests(self.WEBSOCKET_SUBDIR): - self._port.start_websocket_server() - # self._websocket_secure_server.Start() - - thread_timings, test_timings, individual_test_timings = ( - self._run_tests(self._test_files_list, result_summary)) - - # We exclude the crashes from the list of results to retry, because - # we want to treat even a potentially flaky crash as an error. - failures = self._get_failures(result_summary, include_crashes=False) - retries = 0 - retry_summary = result_summary - while (retries < self.NUM_RETRY_ON_UNEXPECTED_FAILURE and - len(failures)): - logging.debug("Retrying %d unexpected failure(s)" % len(failures)) - retries += 1 - retry_summary = ResultSummary(self._expectations, failures.keys()) - self._run_tests(failures.keys(), retry_summary) - failures = self._get_failures(retry_summary, include_crashes=True) - - self._port.stop_helper() - end_time = time.time() - - write = create_logging_writer(self._options, 'timing') - self._print_timing_statistics(write, end_time - start_time, - thread_timings, test_timings, - individual_test_timings, - result_summary) - - self._meter.update("") - - if self._options.verbose: - # We write this block to stdout for compatibility with the - # buildbot log parser, which only looks at stdout, not stderr :( - write = lambda s: sys.stdout.write("%s\n" % s) - else: - write = create_logging_writer(self._options, 'actual') - - self._print_result_summary(write, result_summary) - - sys.stdout.flush() - sys.stderr.flush() - - if (LOG_DETAILED_PROGRESS in self._options.log or - (LOG_UNEXPECTED in self._options.log and - result_summary.total != result_summary.expected)): - print - - # This summary data gets written to stdout regardless of log level - self._print_one_line_summary(result_summary.total, - result_summary.expected) - - unexpected_results = self._summarize_unexpected_results(result_summary, - retry_summary) - self._print_unexpected_results(unexpected_results) - - # Write the same data to log files. - self._write_json_files(unexpected_results, result_summary, - individual_test_timings) - - # Write the summary to disk (results.html) and maybe open the - # test_shell to this file. - wrote_results = self._write_results_html_file(result_summary) - if not self._options.noshow_results and wrote_results: - self._show_results_html_file() - - # Ignore flaky failures and unexpected passes so we don't turn the - # bot red for those. - return unexpected_results['num_regressions'] - - def update_summary(self, result_summary): - """Update the summary while running tests.""" - while True: - try: - (test, fail_list) = self._result_queue.get_nowait() - result = test_failures.determine_result_type(fail_list) - expected = self._expectations.matches_an_expected_result(test, - result) - result_summary.add(test, fail_list, result, expected) - if (LOG_DETAILED_PROGRESS in self._options.log and - (self._options.experimental_fully_parallel or - self._is_single_threaded())): - self._display_detailed_progress(result_summary) - else: - if not expected and LOG_UNEXPECTED in self._options.log: - self._print_unexpected_test_result(test, result) - self._display_one_line_progress(result_summary) - except Queue.Empty: - return - - def _display_one_line_progress(self, result_summary): - """Displays the progress through the test run.""" - self._meter.update("Testing: %d ran as expected, %d didn't, %d left" % - (result_summary.expected, result_summary.unexpected, - result_summary.remaining)) - - def _display_detailed_progress(self, result_summary): - """Display detailed progress output where we print the directory name - and one dot for each completed test. This is triggered by - "--log detailed-progress".""" - if self._current_test_number == len(self._test_files_list): - return - - next_test = self._test_files_list[self._current_test_number] - next_dir = os.path.dirname( - self._port.relative_test_filename(next_test)) - if self._current_progress_str == "": - self._current_progress_str = "%s: " % (next_dir) - self._current_dir = next_dir - - while next_test in result_summary.results: - if next_dir != self._current_dir: - self._meter.write("%s\n" % (self._current_progress_str)) - self._current_progress_str = "%s: ." % (next_dir) - self._current_dir = next_dir - else: - self._current_progress_str += "." - - if (next_test in result_summary.unexpected_results and - LOG_UNEXPECTED in self._options.log): - result = result_summary.unexpected_results[next_test] - self._meter.write("%s\n" % self._current_progress_str) - self._print_unexpected_test_result(next_test, result) - self._current_progress_str = "%s: " % self._current_dir - - self._current_test_number += 1 - if self._current_test_number == len(self._test_files_list): - break - - next_test = self._test_files_list[self._current_test_number] - next_dir = os.path.dirname( - self._port.relative_test_filename(next_test)) - - if result_summary.remaining: - remain_str = " (%d)" % (result_summary.remaining) - self._meter.update("%s%s" % - (self._current_progress_str, remain_str)) - else: - self._meter.write("%s\n" % (self._current_progress_str)) - - def _get_failures(self, result_summary, include_crashes): - """Filters a dict of results and returns only the failures. - - Args: - result_summary: the results of the test run - include_crashes: whether crashes are included in the output. - We use False when finding the list of failures to retry - to see if the results were flaky. Although the crashes may also be - flaky, we treat them as if they aren't so that they're not ignored. - Returns: - a dict of files -> results - """ - failed_results = {} - for test, result in result_summary.unexpected_results.iteritems(): - if (result == test_expectations.PASS or - result == test_expectations.CRASH and not include_crashes): - continue - failed_results[test] = result - - return failed_results - - def _summarize_unexpected_results(self, result_summary, retry_summary): - """Summarize any unexpected results as a dict. - - TODO(dpranke): split this data structure into a separate class? - - Args: - result_summary: summary object from initial test runs - retry_summary: summary object from final test run of retried tests - Returns: - A dictionary containing a summary of the unexpected results from the - run, with the following fields: - 'version': a version indicator (1 in this version) - 'fixable': # of fixable tests (NOW - PASS) - 'skipped': # of skipped tests (NOW & SKIPPED) - 'num_regressions': # of non-flaky failures - 'num_flaky': # of flaky failures - 'num_passes': # of unexpected passes - 'tests': a dict of tests -> {'expected': '...', 'actual': '...'} - """ - results = {} - results['version'] = 1 - - tbe = result_summary.tests_by_expectation - tbt = result_summary.tests_by_timeline - results['fixable'] = len(tbt[test_expectations.NOW] - - tbe[test_expectations.PASS]) - results['skipped'] = len(tbt[test_expectations.NOW] & - tbe[test_expectations.SKIP]) - - num_passes = 0 - num_flaky = 0 - num_regressions = 0 - keywords = {} - for k, v in TestExpectationsFile.EXPECTATIONS.iteritems(): - keywords[v] = k.upper() - - tests = {} - for filename, result in result_summary.unexpected_results.iteritems(): - # Note that if a test crashed in the original run, we ignore - # whether or not it crashed when we retried it (if we retried it), - # and always consider the result not flaky. - test = self._port.relative_test_filename(filename) - expected = self._expectations.get_expectations_string(filename) - actual = [keywords[result]] - - if result == test_expectations.PASS: - num_passes += 1 - elif result == test_expectations.CRASH: - num_regressions += 1 - else: - if filename not in retry_summary.unexpected_results: - actual.extend( - self._expectations.get_expectations_string( - filename).split(" ")) - num_flaky += 1 - else: - retry_result = retry_summary.unexpected_results[filename] - if result != retry_result: - actual.append(keywords[retry_result]) - num_flaky += 1 - else: - num_regressions += 1 - - tests[test] = {} - tests[test]['expected'] = expected - tests[test]['actual'] = " ".join(actual) - - results['tests'] = tests - results['num_passes'] = num_passes - results['num_flaky'] = num_flaky - results['num_regressions'] = num_regressions - - return results - - def _write_json_files(self, unexpected_results, result_summary, - individual_test_timings): - """Writes the results of the test run as JSON files into the results - dir. - - There are three different files written into the results dir: - unexpected_results.json: A short list of any unexpected results. - This is used by the buildbots to display results. - expectations.json: This is used by the flakiness dashboard. - results.json: A full list of the results - used by the flakiness - dashboard and the aggregate results dashboard. - - Args: - unexpected_results: dict of unexpected results - result_summary: full summary object - individual_test_timings: list of test times (used by the flakiness - dashboard). - """ - logging.debug("Writing JSON files in %s." % - self._options.results_directory) - unexpected_file = open(os.path.join(self._options.results_directory, - "unexpected_results.json"), "w") - unexpected_file.write(simplejson.dumps(unexpected_results, - sort_keys=True, indent=2)) - unexpected_file.close() - - # Write a json file of the test_expectations.txt file for the layout - # tests dashboard. - expectations_file = open(os.path.join(self._options.results_directory, - "expectations.json"), "w") - expectations_json = \ - self._expectations.get_expectations_json_for_all_platforms() - expectations_file.write("ADD_EXPECTATIONS(" + expectations_json + ");") - expectations_file.close() - - json_layout_results_generator.JSONLayoutResultsGenerator( - self._port, self._options.builder_name, self._options.build_name, - self._options.build_number, self._options.results_directory, - BUILDER_BASE_URL, individual_test_timings, - self._expectations, result_summary, self._test_files_list) - - logging.debug("Finished writing JSON files.") - - def _print_expected_results_of_type(self, write, result_summary, - result_type, result_type_str): - """Print the number of the tests in a given result class. - - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - result_summary - the object containing all the results to report on - result_type - the particular result type to report in the summary. - result_type_str - a string description of the result_type. - """ - tests = self._expectations.get_tests_with_result_type(result_type) - now = result_summary.tests_by_timeline[test_expectations.NOW] - wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX] - defer = result_summary.tests_by_timeline[test_expectations.DEFER] - - # We use a fancy format string in order to print the data out in a - # nicely-aligned table. - fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd defer, %%%dd wontfix)" - % (self._num_digits(now), self._num_digits(defer), - self._num_digits(wontfix))) - write(fmtstr % (len(tests), result_type_str, len(tests & now), - len(tests & defer), len(tests & wontfix))) - - def _num_digits(self, num): - """Returns the number of digits needed to represent the length of a - sequence.""" - ndigits = 1 - if len(num): - ndigits = int(math.log10(len(num))) + 1 - return ndigits - - def _print_timing_statistics(self, write, total_time, thread_timings, - directory_test_timings, individual_test_timings, - result_summary): - """Record timing-specific information for the test run. - - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - total_time: total elapsed time (in seconds) for the test run - thread_timings: wall clock time each thread ran for - directory_test_timings: timing by directory - individual_test_timings: timing by file - result_summary: summary object for the test run - """ - write("Test timing:") - write(" %6.2f total testing time" % total_time) - write("") - write("Thread timing:") - cuml_time = 0 - for t in thread_timings: - write(" %10s: %5d tests, %6.2f secs" % - (t['name'], t['num_tests'], t['total_time'])) - cuml_time += t['total_time'] - write(" %6.2f cumulative, %6.2f optimal" % - (cuml_time, cuml_time / int(self._options.num_test_shells))) - write("") - - self._print_aggregate_test_statistics(write, individual_test_timings) - self._print_individual_test_times(write, individual_test_timings, - result_summary) - self._print_directory_timings(write, directory_test_timings) - - def _print_aggregate_test_statistics(self, write, individual_test_timings): - """Prints aggregate statistics (e.g. median, mean, etc.) for all tests. - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - individual_test_timings: List of test_shell_thread.TestStats for all - tests. - """ - test_types = individual_test_timings[0].time_for_diffs.keys() - times_for_test_shell = [] - times_for_diff_processing = [] - times_per_test_type = {} - for test_type in test_types: - times_per_test_type[test_type] = [] - - for test_stats in individual_test_timings: - times_for_test_shell.append(test_stats.test_run_time) - times_for_diff_processing.append( - test_stats.total_time_for_all_diffs) - time_for_diffs = test_stats.time_for_diffs - for test_type in test_types: - times_per_test_type[test_type].append( - time_for_diffs[test_type]) - - self._print_statistics_for_test_timings(write, - "PER TEST TIME IN TESTSHELL (seconds):", times_for_test_shell) - self._print_statistics_for_test_timings(write, - "PER TEST DIFF PROCESSING TIMES (seconds):", - times_for_diff_processing) - for test_type in test_types: - self._print_statistics_for_test_timings(write, - "PER TEST TIMES BY TEST TYPE: %s" % test_type, - times_per_test_type[test_type]) - - def _print_individual_test_times(self, write, individual_test_timings, - result_summary): - """Prints the run times for slow, timeout and crash tests. - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - individual_test_timings: List of test_shell_thread.TestStats for all - tests. - result_summary: summary object for test run - """ - # Reverse-sort by the time spent in test_shell. - individual_test_timings.sort(lambda a, b: - cmp(b.test_run_time, a.test_run_time)) - - num_printed = 0 - slow_tests = [] - timeout_or_crash_tests = [] - unexpected_slow_tests = [] - for test_tuple in individual_test_timings: - filename = test_tuple.filename - is_timeout_crash_or_slow = False - if self._expectations.has_modifier(filename, - test_expectations.SLOW): - is_timeout_crash_or_slow = True - slow_tests.append(test_tuple) - - if filename in result_summary.failures: - result = result_summary.results[filename] - if (result == test_expectations.TIMEOUT or - result == test_expectations.CRASH): - is_timeout_crash_or_slow = True - timeout_or_crash_tests.append(test_tuple) - - if (not is_timeout_crash_or_slow and - num_printed < self._options.num_slow_tests_to_log): - num_printed = num_printed + 1 - unexpected_slow_tests.append(test_tuple) - - write("") - self._print_test_list_timing(write, "%s slowest tests that are not " - "marked as SLOW and did not timeout/crash:" % - self._options.num_slow_tests_to_log, unexpected_slow_tests) - write("") - self._print_test_list_timing(write, "Tests marked as SLOW:", - slow_tests) - write("") - self._print_test_list_timing(write, "Tests that timed out or crashed:", - timeout_or_crash_tests) - write("") - - def _print_test_list_timing(self, write, title, test_list): - """Print timing info for each test. - - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - title: section heading - test_list: tests that fall in this section - """ - write(title) - for test_tuple in test_list: - filename = test_tuple.filename[len( - self._port.layout_tests_dir()) + 1:] - filename = filename.replace('\\', '/') - test_run_time = round(test_tuple.test_run_time, 1) - write(" %s took %s seconds" % (filename, test_run_time)) - - def _print_directory_timings(self, write, directory_test_timings): - """Print timing info by directory for any directories that - take > 10 seconds to run. - - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - directory_test_timing: time info for each directory - """ - timings = [] - for directory in directory_test_timings: - num_tests, time_for_directory = directory_test_timings[directory] - timings.append((round(time_for_directory, 1), directory, - num_tests)) - timings.sort() - - write("Time to process slowest subdirectories:") - min_seconds_to_print = 10 - for timing in timings: - if timing[0] > min_seconds_to_print: - write(" %s took %s seconds to run %s tests." % (timing[1], - timing[0], timing[2])) - write("") - - def _print_statistics_for_test_timings(self, write, title, timings): - """Prints the median, mean and standard deviation of the values in - timings. - - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - title: Title for these timings. - timings: A list of floats representing times. - """ - write(title) - timings.sort() - - num_tests = len(timings) - percentile90 = timings[int(.9 * num_tests)] - percentile99 = timings[int(.99 * num_tests)] - - if num_tests % 2 == 1: - median = timings[((num_tests - 1) / 2) - 1] - else: - lower = timings[num_tests / 2 - 1] - upper = timings[num_tests / 2] - median = (float(lower + upper)) / 2 - - mean = sum(timings) / num_tests - - for time in timings: - sum_of_deviations = math.pow(time - mean, 2) - - std_deviation = math.sqrt(sum_of_deviations / num_tests) - write(" Median: %6.3f" % median) - write(" Mean: %6.3f" % mean) - write(" 90th percentile: %6.3f" % percentile90) - write(" 99th percentile: %6.3f" % percentile99) - write(" Standard dev: %6.3f" % std_deviation) - write("") - - def _print_result_summary(self, write, result_summary): - """Print a short summary about how many tests passed. - - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - result_summary: information to log - """ - failed = len(result_summary.failures) - skipped = len( - result_summary.tests_by_expectation[test_expectations.SKIP]) - total = result_summary.total - passed = total - failed - skipped - pct_passed = 0.0 - if total > 0: - pct_passed = float(passed) * 100 / total - - write("") - write("=> Results: %d/%d tests passed (%.1f%%)" % - (passed, total, pct_passed)) - write("") - self._print_result_summary_entry(write, result_summary, - test_expectations.NOW, "Tests to be fixed for the current release") - - write("") - self._print_result_summary_entry(write, result_summary, - test_expectations.DEFER, - "Tests we'll fix in the future if they fail (DEFER)") - - write("") - self._print_result_summary_entry(write, result_summary, - test_expectations.WONTFIX, - "Tests that will only be fixed if they crash (WONTFIX)") - - def _print_result_summary_entry(self, write, result_summary, timeline, - heading): - """Print a summary block of results for a particular timeline of test. - - Args: - write: A callback to write info to (e.g., a LoggingWriter) or - sys.stdout.write. - result_summary: summary to print results for - timeline: the timeline to print results for (NOT, WONTFIX, etc.) - heading: a textual description of the timeline - """ - total = len(result_summary.tests_by_timeline[timeline]) - not_passing = (total - - len(result_summary.tests_by_expectation[test_expectations.PASS] & - result_summary.tests_by_timeline[timeline])) - write("=> %s (%d):" % (heading, not_passing)) - - for result in TestExpectationsFile.EXPECTATION_ORDER: - if result == test_expectations.PASS: - continue - results = (result_summary.tests_by_expectation[result] & - result_summary.tests_by_timeline[timeline]) - desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result] - if not_passing and len(results): - pct = len(results) * 100.0 / not_passing - write(" %5d %-24s (%4.1f%%)" % (len(results), - desc[len(results) != 1], pct)) - - def _print_one_line_summary(self, total, expected): - """Print a one-line summary of the test run to stdout. - - Args: - total: total number of tests run - expected: number of expected results - """ - unexpected = total - expected - if unexpected == 0: - print "All %d tests ran as expected." % expected - elif expected == 1: - print "1 test ran as expected, %d didn't:" % unexpected - else: - print "%d tests ran as expected, %d didn't:" % (expected, - unexpected) - - def _print_unexpected_results(self, unexpected_results): - """Prints any unexpected results in a human-readable form to stdout.""" - passes = {} - flaky = {} - regressions = {} - - if len(unexpected_results['tests']): - print "" - - for test, results in unexpected_results['tests'].iteritems(): - actual = results['actual'].split(" ") - expected = results['expected'].split(" ") - if actual == ['PASS']: - if 'CRASH' in expected: - _add_to_dict_of_lists(passes, - 'Expected to crash, but passed', - test) - elif 'TIMEOUT' in expected: - _add_to_dict_of_lists(passes, - 'Expected to timeout, but passed', - test) - else: - _add_to_dict_of_lists(passes, - 'Expected to fail, but passed', - test) - elif len(actual) > 1: - # We group flaky tests by the first actual result we got. - _add_to_dict_of_lists(flaky, actual[0], test) - else: - _add_to_dict_of_lists(regressions, results['actual'], test) - - if len(passes): - for key, tests in passes.iteritems(): - print "%s: (%d)" % (key, len(tests)) - tests.sort() - for test in tests: - print " %s" % test - print - - if len(flaky): - descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS - for key, tests in flaky.iteritems(): - result = TestExpectationsFile.EXPECTATIONS[key.lower()] - print "Unexpected flakiness: %s (%d)" % ( - descriptions[result][1], len(tests)) - tests.sort() - - for test in tests: - result = unexpected_results['tests'][test] - actual = result['actual'].split(" ") - expected = result['expected'].split(" ") - result = TestExpectationsFile.EXPECTATIONS[key.lower()] - new_expectations_list = list(set(actual) | set(expected)) - print " %s = %s" % (test, " ".join(new_expectations_list)) - print - - if len(regressions): - descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS - for key, tests in regressions.iteritems(): - result = TestExpectationsFile.EXPECTATIONS[key.lower()] - print "Regressions: Unexpected %s : (%d)" % ( - descriptions[result][1], len(tests)) - tests.sort() - for test in tests: - print " %s = %s" % (test, key) - print - - if len(unexpected_results['tests']) and self._options.verbose: - print "-" * 78 - - def _print_unexpected_test_result(self, test, result): - """Prints one unexpected test result line.""" - desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result][0] - self._meter.write(" %s -> unexpected %s\n" % - (self._port.relative_test_filename(test), desc)) - - def _write_results_html_file(self, result_summary): - """Write results.html which is a summary of tests that failed. - - Args: - result_summary: a summary of the results :) - - Returns: - True if any results were written (since expected failures may be - omitted) - """ - # test failures - if self._options.full_results_html: - test_files = result_summary.failures.keys() - else: - unexpected_failures = self._get_failures(result_summary, - include_crashes=True) - test_files = unexpected_failures.keys() - if not len(test_files): - return False - - out_filename = os.path.join(self._options.results_directory, - "results.html") - out_file = open(out_filename, 'w') - # header - if self._options.full_results_html: - h2 = "Test Failures" - else: - h2 = "Unexpected Test Failures" - out_file.write("<html><head><title>Layout Test Results (%(time)s)" - "</title></head><body><h2>%(h2)s (%(time)s)</h2>\n" - % {'h2': h2, 'time': time.asctime()}) - - test_files.sort() - for test_file in test_files: - test_failures = result_summary.failures.get(test_file, []) - out_file.write("<p><a href='%s'>%s</a><br />\n" - % (self._port.filename_to_uri(test_file), - self._port.relative_test_filename(test_file))) - for failure in test_failures: - out_file.write(" %s<br/>" - % failure.result_html_output( - self._port.relative_test_filename(test_file))) - out_file.write("</p>\n") - - # footer - out_file.write("</body></html>\n") - return True - - def _show_results_html_file(self): - """Launches the test shell open to the results.html page.""" - results_filename = os.path.join(self._options.results_directory, - "results.html") - self._port.show_results_html_file(results_filename) - - -def _add_to_dict_of_lists(dict, key, value): - dict.setdefault(key, []).append(value) - - -def read_test_files(files): - tests = [] - for file in files: - for line in open(file): - line = test_expectations.strip_comments(line) - if line: - tests.append(line) - return tests - - -def create_logging_writer(options, log_option): - """Returns a write() function that will write the string to logging.info() - if comp was specified in --log or if --verbose is true. Otherwise the - message is dropped. - - Args: - options: list of command line options from optparse - log_option: option to match in options.log in order for the messages - to be logged (e.g., 'actual' or 'expected') - """ - if options.verbose or log_option in options.log.split(","): - return logging.info - return lambda str: 1 - - -def main(options, args): - """Run the tests. Will call sys.exit when complete. - - Args: - options: a dictionary of command line options - args: a list of sub directories or files to test - """ - - if options.sources: - options.verbose = True - - # Set up our logging format. - meter = metered_stream.MeteredStream(options.verbose, sys.stderr) - log_fmt = '%(message)s' - log_datefmt = '%y%m%d %H:%M:%S' - log_level = logging.INFO - if options.verbose: - log_fmt = ('%(asctime)s %(filename)s:%(lineno)-4d %(levelname)s ' - '%(message)s') - log_level = logging.DEBUG - logging.basicConfig(level=log_level, format=log_fmt, datefmt=log_datefmt, - stream=meter) - - if not options.target: - if options.debug: - options.target = "Debug" - else: - options.target = "Release" - - port_obj = port.get(options.platform, options) - - if not options.use_apache: - options.use_apache = sys.platform in ('darwin', 'linux2') - - if options.results_directory.startswith("/"): - # Assume it's an absolute path and normalize. - options.results_directory = port_obj.get_absolute_path( - options.results_directory) - else: - # If it's a relative path, make the output directory relative to - # Debug or Release. - options.results_directory = port_obj.results_directory() - - if options.clobber_old_results: - # Just clobber the actual test results directories since the other - # files in the results directory are explicitly used for cross-run - # tracking. - path = os.path.join(options.results_directory, 'LayoutTests') - if os.path.exists(path): - shutil.rmtree(path) - - if not options.num_test_shells: - # TODO(ojan): Investigate perf/flakiness impact of using numcores + 1. - options.num_test_shells = port_obj.num_cores() - - write = create_logging_writer(options, 'config') - write("Running %s test_shells in parallel" % options.num_test_shells) - - if not options.time_out_ms: - if options.target == "Debug": - options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS) - else: - options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS) - - options.slow_time_out_ms = str(5 * int(options.time_out_ms)) - write("Regular timeout: %s, slow test timeout: %s" % - (options.time_out_ms, options.slow_time_out_ms)) - - # Include all tests if none are specified. - new_args = [] - for arg in args: - if arg and arg != '': - new_args.append(arg) - - paths = new_args - if not paths: - paths = [] - if options.test_list: - paths += read_test_files(options.test_list) - - # Create the output directory if it doesn't already exist. - port_obj.maybe_make_directory(options.results_directory) - meter.update("Gathering files ...") - - test_runner = TestRunner(port_obj, options, meter) - test_runner.gather_file_paths(paths) - - if options.lint_test_files: - # Creating the expecations for each platform/target pair does all the - # test list parsing and ensures it's correct syntax (e.g. no dupes). - for platform in port_obj.test_platform_names(): - test_runner.parse_expectations(platform, is_debug_mode=True) - test_runner.parse_expectations(platform, is_debug_mode=False) - print ("If there are no fail messages, errors or exceptions, then the " - "lint succeeded.") - sys.exit(0) - - # Check that the system dependencies (themes, fonts, ...) are correct. - if not options.nocheck_sys_deps: - if not port_obj.check_sys_deps(): - sys.exit(1) - - write = create_logging_writer(options, "config") - write("Using port '%s'" % port_obj.name()) - write("Placing test results in %s" % options.results_directory) - if options.new_baseline: - write("Placing new baselines in %s" % port_obj.baseline_path()) - write("Using %s build" % options.target) - if options.no_pixel_tests: - write("Not running pixel tests") - write("") - - meter.update("Parsing expectations ...") - test_runner.parse_expectations(port_obj.test_platform_name(), - options.target == 'Debug') - - meter.update("Preparing tests ...") - write = create_logging_writer(options, "expected") - result_summary = test_runner.prepare_lists_and_print_output(write) - - port_obj.setup_test_run() - - test_runner.add_test_type(text_diff.TestTextDiff) - if not options.no_pixel_tests: - test_runner.add_test_type(image_diff.ImageDiff) - if options.fuzzy_pixel_tests: - test_runner.add_test_type(fuzzy_image_diff.FuzzyImageDiff) - - meter.update("Starting ...") - has_new_failures = test_runner.run(result_summary) - - logging.debug("Exit status: %d" % has_new_failures) - sys.exit(has_new_failures) - - -def parse_args(args=None): - """Provides a default set of command line args. - - Returns a tuple of options, args from optparse""" - option_parser = optparse.OptionParser() - option_parser.add_option("", "--no-pixel-tests", action="store_true", - default=False, - help="disable pixel-to-pixel PNG comparisons") - option_parser.add_option("", "--fuzzy-pixel-tests", action="store_true", - default=False, - help="Also use fuzzy matching to compare pixel " - "test outputs.") - option_parser.add_option("", "--results-directory", - default="layout-test-results", - help="Output results directory source dir," - " relative to Debug or Release") - option_parser.add_option("", "--new-baseline", action="store_true", - default=False, - help="save all generated results as new baselines" - " into the platform directory, overwriting " - "whatever's already there.") - option_parser.add_option("", "--noshow-results", action="store_true", - default=False, help="don't launch the test_shell" - " with results after the tests are done") - option_parser.add_option("", "--full-results-html", action="store_true", - default=False, help="show all failures in " - "results.html, rather than only regressions") - option_parser.add_option("", "--clobber-old-results", action="store_true", - default=False, help="Clobbers test results from " - "previous runs.") - option_parser.add_option("", "--lint-test-files", action="store_true", - default=False, help="Makes sure the test files " - "parse for all configurations. Does not run any " - "tests.") - option_parser.add_option("", "--force", action="store_true", - default=False, - help="Run all tests, even those marked SKIP " - "in the test list") - option_parser.add_option("", "--num-test-shells", - help="Number of testshells to run in parallel.") - option_parser.add_option("", "--use-apache", action="store_true", - default=False, - help="Whether to use apache instead of lighttpd.") - option_parser.add_option("", "--time-out-ms", default=None, - help="Set the timeout for each test") - option_parser.add_option("", "--run-singly", action="store_true", - default=False, - help="run a separate test_shell for each test") - option_parser.add_option("", "--debug", action="store_true", default=False, - help="use the debug binary instead of the release" - " binary") - option_parser.add_option("", "--num-slow-tests-to-log", default=50, - help="Number of slow tests whose timings " - "to print.") - option_parser.add_option("", "--platform", - help="Override the platform for expected results") - option_parser.add_option("", "--target", default="", - help="Set the build target configuration " - "(overrides --debug)") - option_parser.add_option("", "--log", action="store", - default="detailed-progress,unexpected", - help="log various types of data. The param should" - " be a comma-separated list of values from: " - "actual,config," + LOG_DETAILED_PROGRESS + - ",expected,timing," + LOG_UNEXPECTED + " " - "(defaults to " + - "--log detailed-progress,unexpected)") - option_parser.add_option("-v", "--verbose", action="store_true", - default=False, help="include debug-level logging") - option_parser.add_option("", "--sources", action="store_true", - help="show expected result file path for each " - "test (implies --verbose)") - option_parser.add_option("", "--startup-dialog", action="store_true", - default=False, - help="create a dialog on test_shell.exe startup") - option_parser.add_option("", "--gp-fault-error-box", action="store_true", - default=False, - help="enable Windows GP fault error box") - option_parser.add_option("", "--wrapper", - help="wrapper command to insert before " - "invocations of test_shell; option is split " - "on whitespace before running. (Example: " - "--wrapper='valgrind --smc-check=all')") - option_parser.add_option("", "--test-list", action="append", - help="read list of tests to run from file", - metavar="FILE") - option_parser.add_option("", "--nocheck-sys-deps", action="store_true", - default=False, - help="Don't check the system dependencies " - "(themes)") - option_parser.add_option("", "--randomize-order", action="store_true", - default=False, - help=("Run tests in random order (useful for " - "tracking down corruption)")) - option_parser.add_option("", "--run-chunk", - default=None, - help=("Run a specified chunk (n:l), the " - "nth of len l, of the layout tests")) - option_parser.add_option("", "--run-part", - default=None, - help=("Run a specified part (n:m), the nth of m" - " parts, of the layout tests")) - option_parser.add_option("", "--batch-size", - default=None, - help=("Run a the tests in batches (n), after " - "every n tests, the test shell is " - "relaunched.")) - option_parser.add_option("", "--builder-name", - default="DUMMY_BUILDER_NAME", - help=("The name of the builder shown on the " - "waterfall running this script e.g. " - "WebKit.")) - option_parser.add_option("", "--build-name", - default="DUMMY_BUILD_NAME", - help=("The name of the builder used in its path, " - "e.g. webkit-rel.")) - option_parser.add_option("", "--build-number", - default="DUMMY_BUILD_NUMBER", - help=("The build number of the builder running" - "this script.")) - option_parser.add_option("", "--experimental-fully-parallel", - action="store_true", default=False, - help="run all tests in parallel") - return option_parser.parse_args(args) - -if '__main__' == __name__: - options, args = parse_args() - main(options, args) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py new file mode 100755 index 0000000..41aab62 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -0,0 +1,1684 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Run layout tests. + +This is a port of the existing webkit test script run-webkit-tests. + +The TestRunner class runs a series of tests (TestType interface) against a set +of test files. If a test file fails a TestType, it returns a list TestFailure +objects to the TestRunner. The TestRunner then aggregates the TestFailures to +create a final report. + +This script reads several files, if they exist in the test_lists subdirectory +next to this script itself. Each should contain a list of paths to individual +tests or entire subdirectories of tests, relative to the outermost test +directory. Entire lines starting with '//' (comments) will be ignored. + +For details of the files' contents and purposes, see test_lists/README. +""" + +from __future__ import with_statement + +import codecs +import errno +import glob +import logging +import math +import optparse +import os +import platform +import Queue +import random +import re +import shutil +import signal +import sys +import time +import traceback + +from layout_package import dump_render_tree_thread +from layout_package import json_layout_results_generator +from layout_package import printing +from layout_package import test_expectations +from layout_package import test_failures +from layout_package import test_files +from layout_package import test_results_uploader +from test_types import fuzzy_image_diff +from test_types import image_diff +from test_types import text_diff +from test_types import test_type_base + +from webkitpy.common.system.executive import Executive +from webkitpy.thirdparty import simplejson + +import port + +_log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests") + +# Builder base URL where we have the archived test results. +BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/" + +TestExpectationsFile = test_expectations.TestExpectationsFile + + +class TestInfo: + """Groups information about a test for easy passing of data.""" + + def __init__(self, port, filename, timeout): + """Generates the URI and stores the filename and timeout for this test. + Args: + filename: Full path to the test. + timeout: Timeout for running the test in TestShell. + """ + self.filename = filename + self.uri = port.filename_to_uri(filename) + self.timeout = timeout + # FIXME: Confusing that the file is .checksum and we call it "hash" + self._expected_hash_path = port.expected_filename(filename, '.checksum') + self._have_read_expected_hash = False + self._image_hash = None + + def _read_image_hash(self): + try: + with codecs.open(self._expected_hash_path, "r", "ascii") as hash_file: + return hash_file.read() + except IOError, e: + if errno.ENOENT != e.errno: + raise + + def image_hash(self): + # Read the image_hash lazily to reduce startup time. + # This class is accessed across threads, but only one thread should + # ever be dealing with any given TestInfo so no locking is needed. + if not self._have_read_expected_hash: + self._have_read_expected_hash = True + self._image_hash = self._read_image_hash() + return self._image_hash + + +class ResultSummary(object): + """A class for partitioning the test results we get into buckets. + + This class is basically a glorified struct and it's private to this file + so we don't bother with any information hiding.""" + + def __init__(self, expectations, test_files): + self.total = len(test_files) + self.remaining = self.total + self.expectations = expectations + self.expected = 0 + self.unexpected = 0 + self.tests_by_expectation = {} + self.tests_by_timeline = {} + self.results = {} + self.unexpected_results = {} + self.failures = {} + self.tests_by_expectation[test_expectations.SKIP] = set() + for expectation in TestExpectationsFile.EXPECTATIONS.values(): + self.tests_by_expectation[expectation] = set() + for timeline in TestExpectationsFile.TIMELINES.values(): + self.tests_by_timeline[timeline] = ( + expectations.get_tests_with_timeline(timeline)) + + def add(self, result, expected): + """Add a TestResult into the appropriate bin. + + Args: + result: TestResult from dump_render_tree_thread. + expected: whether the result was what we expected it to be. + """ + + self.tests_by_expectation[result.type].add(result.filename) + self.results[result.filename] = result + self.remaining -= 1 + if len(result.failures): + self.failures[result.filename] = result.failures + if expected: + self.expected += 1 + else: + self.unexpected_results[result.filename] = result.type + self.unexpected += 1 + + +def summarize_unexpected_results(port_obj, expectations, result_summary, + retry_summary): + """Summarize any unexpected results as a dict. + + FIXME: split this data structure into a separate class? + + Args: + port_obj: interface to port-specific hooks + expectations: test_expectations.TestExpectations object + result_summary: summary object from initial test runs + retry_summary: summary object from final test run of retried tests + Returns: + A dictionary containing a summary of the unexpected results from the + run, with the following fields: + 'version': a version indicator (1 in this version) + 'fixable': # of fixable tests (NOW - PASS) + 'skipped': # of skipped tests (NOW & SKIPPED) + 'num_regressions': # of non-flaky failures + 'num_flaky': # of flaky failures + 'num_passes': # of unexpected passes + 'tests': a dict of tests -> {'expected': '...', 'actual': '...'} + """ + results = {} + results['version'] = 1 + + tbe = result_summary.tests_by_expectation + tbt = result_summary.tests_by_timeline + results['fixable'] = len(tbt[test_expectations.NOW] - + tbe[test_expectations.PASS]) + results['skipped'] = len(tbt[test_expectations.NOW] & + tbe[test_expectations.SKIP]) + + num_passes = 0 + num_flaky = 0 + num_regressions = 0 + keywords = {} + for k, v in TestExpectationsFile.EXPECTATIONS.iteritems(): + keywords[v] = k.upper() + + tests = {} + for filename, result in result_summary.unexpected_results.iteritems(): + # Note that if a test crashed in the original run, we ignore + # whether or not it crashed when we retried it (if we retried it), + # and always consider the result not flaky. + test = port_obj.relative_test_filename(filename) + expected = expectations.get_expectations_string(filename) + actual = [keywords[result]] + + if result == test_expectations.PASS: + num_passes += 1 + elif result == test_expectations.CRASH: + num_regressions += 1 + else: + if filename not in retry_summary.unexpected_results: + actual.extend(expectations.get_expectations_string( + filename).split(" ")) + num_flaky += 1 + else: + retry_result = retry_summary.unexpected_results[filename] + if result != retry_result: + actual.append(keywords[retry_result]) + num_flaky += 1 + else: + num_regressions += 1 + + tests[test] = {} + tests[test]['expected'] = expected + tests[test]['actual'] = " ".join(actual) + + results['tests'] = tests + results['num_passes'] = num_passes + results['num_flaky'] = num_flaky + results['num_regressions'] = num_regressions + + return results + + +class TestRunner: + """A class for managing running a series of tests on a series of layout + test files.""" + + HTTP_SUBDIR = os.sep.join(['', 'http', '']) + WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', '']) + + # The per-test timeout in milliseconds, if no --time-out-ms option was + # given to run_webkit_tests. This should correspond to the default timeout + # in DumpRenderTree. + DEFAULT_TEST_TIMEOUT_MS = 6 * 1000 + + def __init__(self, port, options, printer): + """Initialize test runner data structures. + + Args: + port: an object implementing port-specific + options: a dictionary of command line options + printer: a Printer object to record updates to. + """ + self._port = port + self._options = options + self._printer = printer + + # disable wss server. need to install pyOpenSSL on buildbots. + # self._websocket_secure_server = websocket_server.PyWebSocket( + # options.results_directory, use_tls=True, port=9323) + + # a list of TestType objects + self._test_types = [] + + # a set of test files, and the same tests as a list + self._test_files = set() + self._test_files_list = None + self._result_queue = Queue.Queue() + + self._retrying = False + + # Hack for dumping threads on the bots + self._last_thread_dump = None + + def __del__(self): + _log.debug("flushing stdout") + sys.stdout.flush() + _log.debug("flushing stderr") + sys.stderr.flush() + _log.debug("stopping http server") + self._port.stop_http_server() + _log.debug("stopping websocket server") + self._port.stop_websocket_server() + + def gather_file_paths(self, paths): + """Find all the files to test. + + Args: + paths: a list of globs to use instead of the defaults.""" + self._test_files = test_files.gather_test_files(self._port, paths) + + def parse_expectations(self, test_platform_name, is_debug_mode): + """Parse the expectations from the test_list files and return a data + structure holding them. Throws an error if the test_list files have + invalid syntax.""" + if self._options.lint_test_files: + test_files = None + else: + test_files = self._test_files + + try: + expectations_str = self._port.test_expectations() + overrides_str = self._port.test_expectations_overrides() + self._expectations = test_expectations.TestExpectations( + self._port, test_files, expectations_str, test_platform_name, + is_debug_mode, self._options.lint_test_files, + tests_are_present=True, overrides=overrides_str) + return self._expectations + except SyntaxError, err: + if self._options.lint_test_files: + print str(err) + else: + raise err + + def prepare_lists_and_print_output(self): + """Create appropriate subsets of test lists and returns a + ResultSummary object. Also prints expected test counts. + """ + + # Remove skipped - both fixable and ignored - files from the + # top-level list of files to test. + num_all_test_files = len(self._test_files) + self._printer.print_expected("Found: %d tests" % + (len(self._test_files))) + if not num_all_test_files: + _log.critical("No tests to run.") + sys.exit(1) + + skipped = set() + if num_all_test_files > 1 and not self._options.force: + skipped = self._expectations.get_tests_with_result_type( + test_expectations.SKIP) + self._test_files -= skipped + + # Create a sorted list of test files so the subset chunk, + # if used, contains alphabetically consecutive tests. + self._test_files_list = list(self._test_files) + if self._options.randomize_order: + random.shuffle(self._test_files_list) + else: + self._test_files_list.sort() + + # If the user specifies they just want to run a subset of the tests, + # just grab a subset of the non-skipped tests. + if self._options.run_chunk or self._options.run_part: + chunk_value = self._options.run_chunk or self._options.run_part + test_files = self._test_files_list + try: + (chunk_num, chunk_len) = chunk_value.split(":") + chunk_num = int(chunk_num) + assert(chunk_num >= 0) + test_size = int(chunk_len) + assert(test_size > 0) + except: + _log.critical("invalid chunk '%s'" % chunk_value) + sys.exit(1) + + # Get the number of tests + num_tests = len(test_files) + + # Get the start offset of the slice. + if self._options.run_chunk: + chunk_len = test_size + # In this case chunk_num can be really large. We need + # to make the slave fit in the current number of tests. + slice_start = (chunk_num * chunk_len) % num_tests + else: + # Validate the data. + assert(test_size <= num_tests) + assert(chunk_num <= test_size) + + # To count the chunk_len, and make sure we don't skip + # some tests, we round to the next value that fits exactly + # all the parts. + rounded_tests = num_tests + if rounded_tests % test_size != 0: + rounded_tests = (num_tests + test_size - + (num_tests % test_size)) + + chunk_len = rounded_tests / test_size + slice_start = chunk_len * (chunk_num - 1) + # It does not mind if we go over test_size. + + # Get the end offset of the slice. + slice_end = min(num_tests, slice_start + chunk_len) + + files = test_files[slice_start:slice_end] + + tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % ( + (slice_end - slice_start), slice_start, slice_end, num_tests) + self._printer.print_expected(tests_run_msg) + + # If we reached the end and we don't have enough tests, we run some + # from the beginning. + if (self._options.run_chunk and + (slice_end - slice_start < chunk_len)): + extra = 1 + chunk_len - (slice_end - slice_start) + extra_msg = (' last chunk is partial, appending [0:%d]' % + extra) + self._printer.print_expected(extra_msg) + tests_run_msg += "\n" + extra_msg + files.extend(test_files[0:extra]) + tests_run_filename = os.path.join(self._options.results_directory, + "tests_run.txt") + with codecs.open(tests_run_filename, "w", "utf-8") as file: + file.write(tests_run_msg + "\n") + + len_skip_chunk = int(len(files) * len(skipped) / + float(len(self._test_files))) + skip_chunk_list = list(skipped)[0:len_skip_chunk] + skip_chunk = set(skip_chunk_list) + + # Update expectations so that the stats are calculated correctly. + # We need to pass a list that includes the right # of skipped files + # to ParseExpectations so that ResultSummary() will get the correct + # stats. So, we add in the subset of skipped files, and then + # subtract them back out. + self._test_files_list = files + skip_chunk_list + self._test_files = set(self._test_files_list) + + self._expectations = self.parse_expectations( + self._port.test_platform_name(), + self._options.configuration == 'Debug') + + self._test_files = set(files) + self._test_files_list = files + else: + skip_chunk = skipped + + result_summary = ResultSummary(self._expectations, + self._test_files | skip_chunk) + self._print_expected_results_of_type(result_summary, + test_expectations.PASS, "passes") + self._print_expected_results_of_type(result_summary, + test_expectations.FAIL, "failures") + self._print_expected_results_of_type(result_summary, + test_expectations.FLAKY, "flaky") + self._print_expected_results_of_type(result_summary, + test_expectations.SKIP, "skipped") + + if self._options.force: + self._printer.print_expected('Running all tests, including ' + 'skips (--force)') + else: + # Note that we don't actually run the skipped tests (they were + # subtracted out of self._test_files, above), but we stub out the + # results here so the statistics can remain accurate. + for test in skip_chunk: + result = dump_render_tree_thread.TestResult(test, + failures=[], test_run_time=0, total_time_for_all_diffs=0, + time_for_diffs=0) + result.type = test_expectations.SKIP + result_summary.add(result, expected=True) + self._printer.print_expected('') + + return result_summary + + def add_test_type(self, test_type): + """Add a TestType to the TestRunner.""" + self._test_types.append(test_type) + + def _get_dir_for_test_file(self, test_file): + """Returns the highest-level directory by which to shard the given + test file.""" + index = test_file.rfind(os.sep + 'LayoutTests' + os.sep) + + test_file = test_file[index + len('LayoutTests/'):] + test_file_parts = test_file.split(os.sep, 1) + directory = test_file_parts[0] + test_file = test_file_parts[1] + + # The http tests are very stable on mac/linux. + # TODO(ojan): Make the http server on Windows be apache so we can + # turn shard the http tests there as well. Switching to apache is + # what made them stable on linux/mac. + return_value = directory + while ((directory != 'http' or sys.platform in ('darwin', 'linux2')) + and test_file.find(os.sep) >= 0): + test_file_parts = test_file.split(os.sep, 1) + directory = test_file_parts[0] + return_value = os.path.join(return_value, directory) + test_file = test_file_parts[1] + + return return_value + + def _get_test_info_for_file(self, test_file): + """Returns the appropriate TestInfo object for the file. Mostly this + is used for looking up the timeout value (in ms) to use for the given + test.""" + if self._expectations.has_modifier(test_file, test_expectations.SLOW): + return TestInfo(self._port, test_file, + self._options.slow_time_out_ms) + return TestInfo(self._port, test_file, self._options.time_out_ms) + + def _get_test_file_queue(self, test_files): + """Create the thread safe queue of lists of (test filenames, test URIs) + tuples. Each TestShellThread pulls a list from this queue and runs + those tests in order before grabbing the next available list. + + Shard the lists by directory. This helps ensure that tests that depend + on each other (aka bad tests!) continue to run together as most + cross-tests dependencies tend to occur within the same directory. + + Return: + The Queue of lists of TestInfo objects. + """ + + if (self._options.experimental_fully_parallel or + self._is_single_threaded()): + filename_queue = Queue.Queue() + for test_file in test_files: + filename_queue.put( + ('.', [self._get_test_info_for_file(test_file)])) + return filename_queue + + tests_by_dir = {} + for test_file in test_files: + directory = self._get_dir_for_test_file(test_file) + tests_by_dir.setdefault(directory, []) + tests_by_dir[directory].append( + self._get_test_info_for_file(test_file)) + + # Sort by the number of tests in the dir so that the ones with the + # most tests get run first in order to maximize parallelization. + # Number of tests is a good enough, but not perfect, approximation + # of how long that set of tests will take to run. We can't just use + # a PriorityQueue until we move # to Python 2.6. + test_lists = [] + http_tests = None + for directory in tests_by_dir: + test_list = tests_by_dir[directory] + # Keep the tests in alphabetical order. + # TODO: Remove once tests are fixed so they can be run in any + # order. + test_list.reverse() + test_list_tuple = (directory, test_list) + if directory == 'LayoutTests' + os.sep + 'http': + http_tests = test_list_tuple + else: + test_lists.append(test_list_tuple) + test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1]))) + + # Put the http tests first. There are only a couple hundred of them, + # but each http test takes a very long time to run, so sorting by the + # number of tests doesn't accurately capture how long they take to run. + if http_tests: + test_lists.insert(0, http_tests) + + filename_queue = Queue.Queue() + for item in test_lists: + filename_queue.put(item) + return filename_queue + + def _get_dump_render_tree_args(self, index): + """Returns the tuple of arguments for tests and for DumpRenderTree.""" + shell_args = [] + test_args = test_type_base.TestArguments() + png_path = None + if self._options.pixel_tests: + png_path = os.path.join(self._options.results_directory, + "png_result%s.png" % index) + shell_args.append("--pixel-tests=" + png_path) + test_args.png_path = png_path + + test_args.new_baseline = self._options.new_baseline + test_args.reset_results = self._options.reset_results + + test_args.show_sources = self._options.sources + + if self._options.startup_dialog: + shell_args.append('--testshell-startup-dialog') + + if self._options.gp_fault_error_box: + shell_args.append('--gp-fault-error-box') + + return test_args, png_path, shell_args + + def _contains_tests(self, subdir): + for test_file in self._test_files: + if test_file.find(subdir) >= 0: + return True + return False + + def _instantiate_dump_render_tree_threads(self, test_files, + result_summary): + """Instantitates and starts the TestShellThread(s). + + Return: + The list of threads. + """ + filename_queue = self._get_test_file_queue(test_files) + + # Instantiate TestShellThreads and start them. + threads = [] + for i in xrange(int(self._options.child_processes)): + # Create separate TestTypes instances for each thread. + test_types = [] + for test_type in self._test_types: + test_types.append(test_type(self._port, + self._options.results_directory)) + + test_args, png_path, shell_args = \ + self._get_dump_render_tree_args(i) + thread = dump_render_tree_thread.TestShellThread(self._port, + filename_queue, self._result_queue, test_types, test_args, + png_path, shell_args, self._options) + if self._is_single_threaded(): + thread.run_in_main_thread(self, result_summary) + else: + thread.start() + threads.append(thread) + + return threads + + def _is_single_threaded(self): + """Returns whether we should run all the tests in the main thread.""" + return int(self._options.child_processes) == 1 + + def _dump_thread_states(self): + for thread_id, stack in sys._current_frames().items(): + # FIXME: Python 2.6 has thread.ident which we could + # use to map from thread_id back to thread.name + print "\n# Thread: %d" % thread_id + for filename, lineno, name, line in traceback.extract_stack(stack): + print 'File: "%s", line %d, in %s' % (filename, lineno, name) + if line: + print " %s" % (line.strip()) + + def _dump_thread_states_if_necessary(self): + # HACK: Dump thread states every minute to figure out what's + # hanging on the bots. + if not self._options.verbose: + return + dump_threads_every = 60 # Dump every minute + if not self._last_thread_dump: + self._last_thread_dump = time.time() + time_since_last_dump = time.time() - self._last_thread_dump + if time_since_last_dump > dump_threads_every: + self._dump_thread_states() + self._last_thread_dump = time.time() + + def _run_tests(self, file_list, result_summary): + """Runs the tests in the file_list. + + Return: A tuple (failures, thread_timings, test_timings, + individual_test_timings) + failures is a map from test to list of failure types + thread_timings is a list of dicts with the total runtime + of each thread with 'name', 'num_tests', 'total_time' properties + test_timings is a list of timings for each sharded subdirectory + of the form [time, directory_name, num_tests] + individual_test_timings is a list of run times for each test + in the form {filename:filename, test_run_time:test_run_time} + result_summary: summary object to populate with the results + """ + # FIXME: We should use webkitpy.tool.grammar.pluralize here. + plural = "" + if self._options.child_processes > 1: + plural = "s" + self._printer.print_update('Starting %s%s ...' % + (self._port.driver_name(), plural)) + threads = self._instantiate_dump_render_tree_threads(file_list, + result_summary) + self._printer.print_update("Starting testing ...") + + # Wait for the threads to finish and collect test failures. + failures = {} + test_timings = {} + individual_test_timings = [] + thread_timings = [] + keyboard_interrupted = False + try: + # Loop through all the threads waiting for them to finish. + for thread in threads: + # FIXME: We'll end up waiting on the first thread the whole + # time. That means we won't notice exceptions on other + # threads until the first one exits. + # We should instead while True: in the outer loop + # and then loop through threads joining and checking + # isAlive and get_exception_info. Exiting on any exception. + while thread.isAlive(): + # Wake the main thread every 0.1 seconds so we + # can call update_summary in a timely fashion. + thread.join(0.1) + # HACK: Used for debugging threads on the bots. + self._dump_thread_states_if_necessary() + self.update_summary(result_summary) + + except KeyboardInterrupt: + keyboard_interrupted = True + for thread in threads: + thread.cancel() + + if not keyboard_interrupted: + for thread in threads: + # Check whether a thread died before normal completion. + exception_info = thread.get_exception_info() + if exception_info is not None: + # Re-raise the thread's exception here to make it clear + # something went wrong. Otherwise, the tests that did not + # run would be assumed to have passed. + raise (exception_info[0], exception_info[1], + exception_info[2]) + + for thread in threads: + thread_timings.append({'name': thread.getName(), + 'num_tests': thread.get_num_tests(), + 'total_time': thread.get_total_time()}) + test_timings.update(thread.get_directory_timing_stats()) + individual_test_timings.extend(thread.get_test_results()) + return (keyboard_interrupted, thread_timings, test_timings, + individual_test_timings) + + def needs_http(self): + """Returns whether the test runner needs an HTTP server.""" + return self._contains_tests(self.HTTP_SUBDIR) + + def run(self, result_summary): + """Run all our tests on all our test files. + + For each test file, we run each test type. If there are any failures, + we collect them for reporting. + + Args: + result_summary: a summary object tracking the test results. + + Return: + The number of unexpected results (0 == success) + """ + if not self._test_files: + return 0 + start_time = time.time() + + if self.needs_http(): + self._printer.print_update('Starting HTTP server ...') + + self._port.start_http_server() + + if self._contains_tests(self.WEBSOCKET_SUBDIR): + self._printer.print_update('Starting WebSocket server ...') + self._port.start_websocket_server() + # self._websocket_secure_server.Start() + + keyboard_interrupted, thread_timings, test_timings, \ + individual_test_timings = ( + self._run_tests(self._test_files_list, result_summary)) + + # We exclude the crashes from the list of results to retry, because + # we want to treat even a potentially flaky crash as an error. + failures = self._get_failures(result_summary, include_crashes=False) + retry_summary = result_summary + while (len(failures) and self._options.retry_failures and + not self._retrying and not keyboard_interrupted): + _log.info('') + _log.info("Retrying %d unexpected failure(s) ..." % len(failures)) + _log.info('') + self._retrying = True + retry_summary = ResultSummary(self._expectations, failures.keys()) + # Note that we intentionally ignore the return value here. + self._run_tests(failures.keys(), retry_summary) + failures = self._get_failures(retry_summary, include_crashes=True) + + end_time = time.time() + + self._print_timing_statistics(end_time - start_time, + thread_timings, test_timings, + individual_test_timings, + result_summary) + + self._print_result_summary(result_summary) + + sys.stdout.flush() + sys.stderr.flush() + + self._printer.print_one_line_summary(result_summary.total, + result_summary.expected, + result_summary.unexpected) + + unexpected_results = summarize_unexpected_results(self._port, + self._expectations, result_summary, retry_summary) + self._printer.print_unexpected_results(unexpected_results) + + # Write the same data to log files. + self._write_json_files(unexpected_results, result_summary, + individual_test_timings) + + # Upload generated JSON files to appengine server. + self._upload_json_files() + + # Write the summary to disk (results.html) and display it if requested. + wrote_results = self._write_results_html_file(result_summary) + if self._options.show_results and wrote_results: + self._show_results_html_file() + + # Now that we've completed all the processing we can, we re-raise + # a KeyboardInterrupt if necessary so the caller can handle it. + if keyboard_interrupted: + raise KeyboardInterrupt + + # Ignore flaky failures and unexpected passes so we don't turn the + # bot red for those. + return unexpected_results['num_regressions'] + + def update_summary(self, result_summary): + """Update the summary and print results with any completed tests.""" + while True: + try: + result = self._result_queue.get_nowait() + except Queue.Empty: + return + + expected = self._expectations.matches_an_expected_result( + result.filename, result.type, self._options.pixel_tests) + result_summary.add(result, expected) + exp_str = self._expectations.get_expectations_string( + result.filename) + got_str = self._expectations.expectation_to_string(result.type) + self._printer.print_test_result(result, expected, exp_str, got_str) + self._printer.print_progress(result_summary, self._retrying, + self._test_files_list) + + def _get_failures(self, result_summary, include_crashes): + """Filters a dict of results and returns only the failures. + + Args: + result_summary: the results of the test run + include_crashes: whether crashes are included in the output. + We use False when finding the list of failures to retry + to see if the results were flaky. Although the crashes may also be + flaky, we treat them as if they aren't so that they're not ignored. + Returns: + a dict of files -> results + """ + failed_results = {} + for test, result in result_summary.unexpected_results.iteritems(): + if (result == test_expectations.PASS or + result == test_expectations.CRASH and not include_crashes): + continue + failed_results[test] = result + + return failed_results + + def _write_json_files(self, unexpected_results, result_summary, + individual_test_timings): + """Writes the results of the test run as JSON files into the results + dir. + + There are three different files written into the results dir: + unexpected_results.json: A short list of any unexpected results. + This is used by the buildbots to display results. + expectations.json: This is used by the flakiness dashboard. + results.json: A full list of the results - used by the flakiness + dashboard and the aggregate results dashboard. + + Args: + unexpected_results: dict of unexpected results + result_summary: full summary object + individual_test_timings: list of test times (used by the flakiness + dashboard). + """ + results_directory = self._options.results_directory + _log.debug("Writing JSON files in %s." % results_directory) + unexpected_json_path = os.path.join(results_directory, "unexpected_results.json") + with codecs.open(unexpected_json_path, "w", "utf-8") as file: + simplejson.dump(unexpected_results, file, sort_keys=True, indent=2) + + # Write a json file of the test_expectations.txt file for the layout + # tests dashboard. + expectations_path = os.path.join(results_directory, "expectations.json") + expectations_json = \ + self._expectations.get_expectations_json_for_all_platforms() + with codecs.open(expectations_path, "w", "utf-8") as file: + file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json) + + json_layout_results_generator.JSONLayoutResultsGenerator( + self._port, self._options.builder_name, self._options.build_name, + self._options.build_number, self._options.results_directory, + BUILDER_BASE_URL, individual_test_timings, + self._expectations, result_summary, self._test_files_list) + + _log.debug("Finished writing JSON files.") + + def _upload_json_files(self): + if not self._options.test_results_server: + return + + _log.info("Uploading JSON files for builder: %s", + self._options.builder_name) + + attrs = [('builder', self._options.builder_name)] + json_files = ["expectations.json", "results.json"] + + files = [(file, os.path.join(self._options.results_directory, file)) + for file in json_files] + + uploader = test_results_uploader.TestResultsUploader( + self._options.test_results_server) + try: + # Set uploading timeout in case appengine server is having problem. + # 120 seconds are more than enough to upload test results. + uploader.upload(attrs, files, 120) + except Exception, err: + _log.error("Upload failed: %s" % err) + return + + _log.info("JSON files uploaded.") + + def _print_expected_results_of_type(self, result_summary, + result_type, result_type_str): + """Print the number of the tests in a given result class. + + Args: + result_summary - the object containing all the results to report on + result_type - the particular result type to report in the summary. + result_type_str - a string description of the result_type. + """ + tests = self._expectations.get_tests_with_result_type(result_type) + now = result_summary.tests_by_timeline[test_expectations.NOW] + wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX] + defer = result_summary.tests_by_timeline[test_expectations.DEFER] + + # We use a fancy format string in order to print the data out in a + # nicely-aligned table. + fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd defer, %%%dd wontfix)" + % (self._num_digits(now), self._num_digits(defer), + self._num_digits(wontfix))) + self._printer.print_expected(fmtstr % + (len(tests), result_type_str, len(tests & now), + len(tests & defer), len(tests & wontfix))) + + def _num_digits(self, num): + """Returns the number of digits needed to represent the length of a + sequence.""" + ndigits = 1 + if len(num): + ndigits = int(math.log10(len(num))) + 1 + return ndigits + + def _print_timing_statistics(self, total_time, thread_timings, + directory_test_timings, individual_test_timings, + result_summary): + """Record timing-specific information for the test run. + + Args: + total_time: total elapsed time (in seconds) for the test run + thread_timings: wall clock time each thread ran for + directory_test_timings: timing by directory + individual_test_timings: timing by file + result_summary: summary object for the test run + """ + self._printer.print_timing("Test timing:") + self._printer.print_timing(" %6.2f total testing time" % total_time) + self._printer.print_timing("") + self._printer.print_timing("Thread timing:") + cuml_time = 0 + for t in thread_timings: + self._printer.print_timing(" %10s: %5d tests, %6.2f secs" % + (t['name'], t['num_tests'], t['total_time'])) + cuml_time += t['total_time'] + self._printer.print_timing(" %6.2f cumulative, %6.2f optimal" % + (cuml_time, cuml_time / int(self._options.child_processes))) + self._printer.print_timing("") + + self._print_aggregate_test_statistics(individual_test_timings) + self._print_individual_test_times(individual_test_timings, + result_summary) + self._print_directory_timings(directory_test_timings) + + def _print_aggregate_test_statistics(self, individual_test_timings): + """Prints aggregate statistics (e.g. median, mean, etc.) for all tests. + Args: + individual_test_timings: List of dump_render_tree_thread.TestStats + for all tests. + """ + test_types = [] # Unit tests don't actually produce any timings. + if individual_test_timings: + test_types = individual_test_timings[0].time_for_diffs.keys() + times_for_dump_render_tree = [] + times_for_diff_processing = [] + times_per_test_type = {} + for test_type in test_types: + times_per_test_type[test_type] = [] + + for test_stats in individual_test_timings: + times_for_dump_render_tree.append(test_stats.test_run_time) + times_for_diff_processing.append( + test_stats.total_time_for_all_diffs) + time_for_diffs = test_stats.time_for_diffs + for test_type in test_types: + times_per_test_type[test_type].append( + time_for_diffs[test_type]) + + self._print_statistics_for_test_timings( + "PER TEST TIME IN TESTSHELL (seconds):", + times_for_dump_render_tree) + self._print_statistics_for_test_timings( + "PER TEST DIFF PROCESSING TIMES (seconds):", + times_for_diff_processing) + for test_type in test_types: + self._print_statistics_for_test_timings( + "PER TEST TIMES BY TEST TYPE: %s" % test_type, + times_per_test_type[test_type]) + + def _print_individual_test_times(self, individual_test_timings, + result_summary): + """Prints the run times for slow, timeout and crash tests. + Args: + individual_test_timings: List of dump_render_tree_thread.TestStats + for all tests. + result_summary: summary object for test run + """ + # Reverse-sort by the time spent in DumpRenderTree. + individual_test_timings.sort(lambda a, b: + cmp(b.test_run_time, a.test_run_time)) + + num_printed = 0 + slow_tests = [] + timeout_or_crash_tests = [] + unexpected_slow_tests = [] + for test_tuple in individual_test_timings: + filename = test_tuple.filename + is_timeout_crash_or_slow = False + if self._expectations.has_modifier(filename, + test_expectations.SLOW): + is_timeout_crash_or_slow = True + slow_tests.append(test_tuple) + + if filename in result_summary.failures: + result = result_summary.results[filename].type + if (result == test_expectations.TIMEOUT or + result == test_expectations.CRASH): + is_timeout_crash_or_slow = True + timeout_or_crash_tests.append(test_tuple) + + if (not is_timeout_crash_or_slow and + num_printed < printing.NUM_SLOW_TESTS_TO_LOG): + num_printed = num_printed + 1 + unexpected_slow_tests.append(test_tuple) + + self._printer.print_timing("") + self._print_test_list_timing("%s slowest tests that are not " + "marked as SLOW and did not timeout/crash:" % + printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests) + self._printer.print_timing("") + self._print_test_list_timing("Tests marked as SLOW:", slow_tests) + self._printer.print_timing("") + self._print_test_list_timing("Tests that timed out or crashed:", + timeout_or_crash_tests) + self._printer.print_timing("") + + def _print_test_list_timing(self, title, test_list): + """Print timing info for each test. + + Args: + title: section heading + test_list: tests that fall in this section + """ + if self._printer.disabled('slowest'): + return + + self._printer.print_timing(title) + for test_tuple in test_list: + filename = test_tuple.filename[len( + self._port.layout_tests_dir()) + 1:] + filename = filename.replace('\\', '/') + test_run_time = round(test_tuple.test_run_time, 1) + self._printer.print_timing(" %s took %s seconds" % + (filename, test_run_time)) + + def _print_directory_timings(self, directory_test_timings): + """Print timing info by directory for any directories that + take > 10 seconds to run. + + Args: + directory_test_timing: time info for each directory + """ + timings = [] + for directory in directory_test_timings: + num_tests, time_for_directory = directory_test_timings[directory] + timings.append((round(time_for_directory, 1), directory, + num_tests)) + timings.sort() + + self._printer.print_timing("Time to process slowest subdirectories:") + min_seconds_to_print = 10 + for timing in timings: + if timing[0] > min_seconds_to_print: + self._printer.print_timing( + " %s took %s seconds to run %s tests." % (timing[1], + timing[0], timing[2])) + self._printer.print_timing("") + + def _print_statistics_for_test_timings(self, title, timings): + """Prints the median, mean and standard deviation of the values in + timings. + + Args: + title: Title for these timings. + timings: A list of floats representing times. + """ + self._printer.print_timing(title) + timings.sort() + + num_tests = len(timings) + if not num_tests: + return + percentile90 = timings[int(.9 * num_tests)] + percentile99 = timings[int(.99 * num_tests)] + + if num_tests % 2 == 1: + median = timings[((num_tests - 1) / 2) - 1] + else: + lower = timings[num_tests / 2 - 1] + upper = timings[num_tests / 2] + median = (float(lower + upper)) / 2 + + mean = sum(timings) / num_tests + + for time in timings: + sum_of_deviations = math.pow(time - mean, 2) + + std_deviation = math.sqrt(sum_of_deviations / num_tests) + self._printer.print_timing(" Median: %6.3f" % median) + self._printer.print_timing(" Mean: %6.3f" % mean) + self._printer.print_timing(" 90th percentile: %6.3f" % percentile90) + self._printer.print_timing(" 99th percentile: %6.3f" % percentile99) + self._printer.print_timing(" Standard dev: %6.3f" % std_deviation) + self._printer.print_timing("") + + def _print_result_summary(self, result_summary): + """Print a short summary about how many tests passed. + + Args: + result_summary: information to log + """ + failed = len(result_summary.failures) + skipped = len( + result_summary.tests_by_expectation[test_expectations.SKIP]) + total = result_summary.total + passed = total - failed - skipped + pct_passed = 0.0 + if total > 0: + pct_passed = float(passed) * 100 / total + + self._printer.print_actual("") + self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" % + (passed, total, pct_passed)) + self._printer.print_actual("") + self._print_result_summary_entry(result_summary, + test_expectations.NOW, "Tests to be fixed for the current release") + + self._printer.print_actual("") + self._print_result_summary_entry(result_summary, + test_expectations.DEFER, + "Tests we'll fix in the future if they fail (DEFER)") + + self._printer.print_actual("") + self._print_result_summary_entry(result_summary, + test_expectations.WONTFIX, + "Tests that will only be fixed if they crash (WONTFIX)") + self._printer.print_actual("") + + def _print_result_summary_entry(self, result_summary, timeline, + heading): + """Print a summary block of results for a particular timeline of test. + + Args: + result_summary: summary to print results for + timeline: the timeline to print results for (NOT, WONTFIX, etc.) + heading: a textual description of the timeline + """ + total = len(result_summary.tests_by_timeline[timeline]) + not_passing = (total - + len(result_summary.tests_by_expectation[test_expectations.PASS] & + result_summary.tests_by_timeline[timeline])) + self._printer.print_actual("=> %s (%d):" % (heading, not_passing)) + + for result in TestExpectationsFile.EXPECTATION_ORDER: + if result == test_expectations.PASS: + continue + results = (result_summary.tests_by_expectation[result] & + result_summary.tests_by_timeline[timeline]) + desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result] + if not_passing and len(results): + pct = len(results) * 100.0 / not_passing + self._printer.print_actual(" %5d %-24s (%4.1f%%)" % + (len(results), desc[len(results) != 1], pct)) + + def _results_html(self, test_files, failures, title="Test Failures", override_time=None): + """ + test_files = a list of file paths + failures = dictionary mapping test paths to failure objects + title = title printed at top of test + override_time = current time (used by unit tests) + """ + page = """<html> + <head> + <title>Layout Test Results (%(time)s)</title> + </head> + <body> + <h2>%(title)s (%(time)s)</h2> + """ % {'title': title, 'time': override_time or time.asctime()} + + for test_file in sorted(test_files): + test_name = self._port.relative_test_filename(test_file) + test_url = self._port.filename_to_uri(test_file) + page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name) + test_failures = failures.get(test_file, []) + for failure in test_failures: + page += u" %s<br/>" % failure.result_html_output(test_name) + page += "</p>\n" + page += "</body></html>\n" + return page + + def _write_results_html_file(self, result_summary): + """Write results.html which is a summary of tests that failed. + + Args: + result_summary: a summary of the results :) + + Returns: + True if any results were written (since expected failures may be + omitted) + """ + # test failures + if self._options.full_results_html: + results_title = "Test Failures" + test_files = result_summary.failures.keys() + else: + results_title = "Unexpected Test Failures" + unexpected_failures = self._get_failures(result_summary, + include_crashes=True) + test_files = unexpected_failures.keys() + if not len(test_files): + return False + + out_filename = os.path.join(self._options.results_directory, + "results.html") + with codecs.open(out_filename, "w", "utf-8") as results_file: + html = self._results_html(test_files, result_summary.failures, results_title) + results_file.write(html) + + return True + + def _show_results_html_file(self): + """Shows the results.html page.""" + results_filename = os.path.join(self._options.results_directory, + "results.html") + self._port.show_results_html_file(results_filename) + + +def read_test_files(files): + tests = [] + for file in files: + # FIXME: This could be cleaner using a list comprehension. + for line in codecs.open(file, "r", "utf-8"): + line = test_expectations.strip_comments(line) + if line: + tests.append(line) + return tests + + +def run(port_obj, options, args, regular_output=sys.stderr, + buildbot_output=sys.stdout): + """Run the tests. + + Args: + port_obj: Port object for port-specific behavior + options: a dictionary of command line options + args: a list of sub directories or files to test + regular_output: a stream-like object that we can send logging/debug + output to + buildbot_output: a stream-like object that we can write all output that + is intended to be parsed by the buildbot to + Returns: + the number of unexpected results that occurred, or -1 if there is an + error. + """ + + # Configure the printing subsystem for printing output, logging debug + # info, and tracing tests. + + if not options.child_processes: + # FIXME: Investigate perf/flakiness impact of using cpu_count + 1. + options.child_processes = port_obj.default_child_processes() + + printer = printing.Printer(port_obj, options, regular_output=regular_output, + buildbot_output=buildbot_output, + child_processes=int(options.child_processes), + is_fully_parallel=options.experimental_fully_parallel) + if options.help_printing: + printer.help_printing() + return 0 + + executive = Executive() + + if not options.configuration: + options.configuration = port_obj.default_configuration() + + if options.pixel_tests is None: + options.pixel_tests = True + + if not options.use_apache: + options.use_apache = sys.platform in ('darwin', 'linux2') + + if options.results_directory.startswith("/"): + # Assume it's an absolute path and normalize. + options.results_directory = port_obj.get_absolute_path( + options.results_directory) + else: + # If it's a relative path, make the output directory relative to + # Debug or Release. + options.results_directory = port_obj.results_directory() + + last_unexpected_results = [] + if options.print_last_failures or options.retest_last_failures: + unexpected_results_filename = os.path.join( + options.results_directory, "unexpected_results.json") + with codecs.open(unexpected_results_filename, "r", "utf-8") as file: + results = simplejson.load(file) + last_unexpected_results = results['tests'].keys() + if options.print_last_failures: + printer.write("\n".join(last_unexpected_results) + "\n") + return 0 + + if options.clobber_old_results: + # Just clobber the actual test results directories since the other + # files in the results directory are explicitly used for cross-run + # tracking. + printer.print_update("Clobbering old results in %s" % + options.results_directory) + layout_tests_dir = port_obj.layout_tests_dir() + possible_dirs = os.listdir(layout_tests_dir) + for dirname in possible_dirs: + if os.path.isdir(os.path.join(layout_tests_dir, dirname)): + shutil.rmtree(os.path.join(options.results_directory, dirname), + ignore_errors=True) + + if not options.time_out_ms: + if options.configuration == "Debug": + options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS) + else: + options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS) + + options.slow_time_out_ms = str(5 * int(options.time_out_ms)) + printer.print_config("Regular timeout: %s, slow test timeout: %s" % + (options.time_out_ms, options.slow_time_out_ms)) + + if int(options.child_processes) == 1: + printer.print_config("Running one %s" % port_obj.driver_name()) + else: + printer.print_config("Running %s %ss in parallel" % ( + options.child_processes, port_obj.driver_name())) + + # Include all tests if none are specified. + new_args = [] + for arg in args: + if arg and arg != '': + new_args.append(arg) + + paths = new_args + if not paths: + paths = [] + paths += last_unexpected_results + if options.test_list: + paths += read_test_files(options.test_list) + + # Create the output directory if it doesn't already exist. + port_obj.maybe_make_directory(options.results_directory) + printer.print_update("Collecting tests ...") + + test_runner = TestRunner(port_obj, options, printer) + test_runner.gather_file_paths(paths) + + if options.lint_test_files: + # Creating the expecations for each platform/configuration pair does + # all the test list parsing and ensures it's correct syntax (e.g. no + # dupes). + for platform_name in port_obj.test_platform_names(): + test_runner.parse_expectations(platform_name, is_debug_mode=True) + test_runner.parse_expectations(platform_name, is_debug_mode=False) + printer.write("") + _log.info("If there are no fail messages, errors or exceptions, " + "then the lint succeeded.") + return 0 + + printer.print_config("Using port '%s'" % port_obj.name()) + printer.print_config("Placing test results in %s" % + options.results_directory) + if options.new_baseline: + printer.print_config("Placing new baselines in %s" % + port_obj.baseline_path()) + printer.print_config("Using %s build" % options.configuration) + if options.pixel_tests: + printer.print_config("Pixel tests enabled") + else: + printer.print_config("Pixel tests disabled") + printer.print_config("") + + printer.print_update("Parsing expectations ...") + test_runner.parse_expectations(port_obj.test_platform_name(), + options.configuration == 'Debug') + + printer.print_update("Checking build ...") + if not port_obj.check_build(test_runner.needs_http()): + return -1 + + printer.print_update("Starting helper ...") + port_obj.start_helper() + + # Check that the system dependencies (themes, fonts, ...) are correct. + if not options.nocheck_sys_deps: + printer.print_update("Checking system dependencies ...") + if not port_obj.check_sys_deps(test_runner.needs_http()): + return -1 + + printer.print_update("Preparing tests ...") + result_summary = test_runner.prepare_lists_and_print_output() + + port_obj.setup_test_run() + + test_runner.add_test_type(text_diff.TestTextDiff) + if options.pixel_tests: + test_runner.add_test_type(image_diff.ImageDiff) + if options.fuzzy_pixel_tests: + test_runner.add_test_type(fuzzy_image_diff.FuzzyImageDiff) + + num_unexpected_results = test_runner.run(result_summary) + + port_obj.stop_helper() + + _log.debug("Exit status: %d" % num_unexpected_results) + return num_unexpected_results + + +def _compat_shim_callback(option, opt_str, value, parser): + print "Ignoring unsupported option: %s" % opt_str + + +def _compat_shim_option(option_name, **kwargs): + return optparse.make_option(option_name, action="callback", + callback=_compat_shim_callback, + help="Ignored, for old-run-webkit-tests compat only.", **kwargs) + + +def parse_args(args=None): + """Provides a default set of command line args. + + Returns a tuple of options, args from optparse""" + + # FIXME: All of these options should be stored closer to the code which + # FIXME: actually uses them. configuration_options should move + # FIXME: to WebKitPort and be shared across all scripts. + configuration_options = [ + optparse.make_option("-t", "--target", dest="configuration", + help="(DEPRECATED)"), + # FIXME: --help should display which configuration is default. + optparse.make_option('--debug', action='store_const', const='Debug', + dest="configuration", + help='Set the configuration to Debug'), + optparse.make_option('--release', action='store_const', + const='Release', dest="configuration", + help='Set the configuration to Release'), + # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION. + ] + + print_options = printing.print_options() + + # FIXME: These options should move onto the ChromiumPort. + chromium_options = [ + optparse.make_option("--chromium", action="store_true", default=False, + help="use the Chromium port"), + optparse.make_option("--startup-dialog", action="store_true", + default=False, help="create a dialog on DumpRenderTree startup"), + optparse.make_option("--gp-fault-error-box", action="store_true", + default=False, help="enable Windows GP fault error box"), + optparse.make_option("--nocheck-sys-deps", action="store_true", + default=False, + help="Don't check the system dependencies (themes)"), + optparse.make_option("--use-drt", action="store_true", + default=False, + help="Use DumpRenderTree instead of test_shell"), + ] + + # Missing Mac-specific old-run-webkit-tests options: + # FIXME: Need: -g, --guard for guard malloc support on Mac. + # FIXME: Need: -l --leaks Enable leaks checking. + # FIXME: Need: --sample-on-timeout Run sample on timeout + + old_run_webkit_tests_compat = [ + # NRWT doesn't generate results by default anyway. + _compat_shim_option("--no-new-test-results"), + # NRWT doesn't sample on timeout yet anyway. + _compat_shim_option("--no-sample-on-timeout"), + # FIXME: NRWT needs to support remote links eventually. + _compat_shim_option("--use-remote-links-to-tests"), + # FIXME: NRWT doesn't need this option as much since failures are + # designed to be cheap. We eventually plan to add this support. + _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"), + ] + + results_options = [ + # NEED for bots: --use-remote-links-to-tests Link to test files + # within the SVN repository in the results. + optparse.make_option("-p", "--pixel-tests", action="store_true", + dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), + optparse.make_option("--no-pixel-tests", action="store_false", + dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), + optparse.make_option("--fuzzy-pixel-tests", action="store_true", + default=False, + help="Also use fuzzy matching to compare pixel test outputs."), + # old-run-webkit-tests allows a specific tolerance: --tolerance t + # Ignore image differences less than this percentage (default: 0.1) + optparse.make_option("--results-directory", + default="layout-test-results", + help="Output results directory source dir, relative to Debug or " + "Release"), + optparse.make_option("--new-baseline", action="store_true", + default=False, help="Save all generated results as new baselines " + "into the platform directory, overwriting whatever's " + "already there."), + optparse.make_option("--reset-results", action="store_true", + default=False, help="Reset any existing baselines to the " + "generated results"), + optparse.make_option("--no-show-results", action="store_false", + default=True, dest="show_results", + help="Don't launch a browser with results after the tests " + "are done"), + # FIXME: We should have a helper function to do this sort of + # deprectated mapping and automatically log, etc. + optparse.make_option("--noshow-results", action="store_false", + dest="show_results", + help="Deprecated, same as --no-show-results."), + optparse.make_option("--no-launch-safari", action="store_false", + dest="show_results", + help="old-run-webkit-tests compat, same as --noshow-results."), + # old-run-webkit-tests: + # --[no-]launch-safari Launch (or do not launch) Safari to display + # test results (default: launch) + optparse.make_option("--full-results-html", action="store_true", + default=False, + help="Show all failures in results.html, rather than only " + "regressions"), + optparse.make_option("--clobber-old-results", action="store_true", + default=False, help="Clobbers test results from previous runs."), + optparse.make_option("--platform", + help="Override the platform for expected results"), + # old-run-webkit-tests also has HTTP toggle options: + # --[no-]http Run (or do not run) http tests + # (default: run) + # --[no-]wait-for-httpd Wait for httpd if some other test + # session is using it already (same + # as WEBKIT_WAIT_FOR_HTTPD=1). + # (default: 0) + ] + + test_options = [ + optparse.make_option("--build", dest="build", + action="store_true", default=True, + help="Check to ensure the DumpRenderTree build is up-to-date " + "(default)."), + optparse.make_option("--no-build", dest="build", + action="store_false", help="Don't check to see if the " + "DumpRenderTree build is up-to-date."), + # old-run-webkit-tests has --valgrind instead of wrapper. + optparse.make_option("--wrapper", + help="wrapper command to insert before invocations of " + "DumpRenderTree; option is split on whitespace before " + "running. (Example: --wrapper='valgrind --smc-check=all')"), + # old-run-webkit-tests: + # -i|--ignore-tests Comma-separated list of directories + # or tests to ignore + optparse.make_option("--test-list", action="append", + help="read list of tests to run from file", metavar="FILE"), + # old-run-webkit-tests uses --skipped==[default|ignore|only] + # instead of --force: + optparse.make_option("--force", action="store_true", default=False, + help="Run all tests, even those marked SKIP in the test list"), + optparse.make_option("--use-apache", action="store_true", + default=False, help="Whether to use apache instead of lighttpd."), + optparse.make_option("--time-out-ms", + help="Set the timeout for each test"), + # old-run-webkit-tests calls --randomize-order --random: + optparse.make_option("--randomize-order", action="store_true", + default=False, help=("Run tests in random order (useful " + "for tracking down corruption)")), + optparse.make_option("--run-chunk", + help=("Run a specified chunk (n:l), the nth of len l, " + "of the layout tests")), + optparse.make_option("--run-part", help=("Run a specified part (n:m), " + "the nth of m parts, of the layout tests")), + # old-run-webkit-tests calls --batch-size: --nthly n + # Restart DumpRenderTree every n tests (default: 1000) + optparse.make_option("--batch-size", + help=("Run a the tests in batches (n), after every n tests, " + "DumpRenderTree is relaunched.")), + # old-run-webkit-tests calls --run-singly: -1|--singly + # Isolate each test case run (implies --nthly 1 --verbose) + optparse.make_option("--run-singly", action="store_true", + default=False, help="run a separate DumpRenderTree for each test"), + optparse.make_option("--child-processes", + help="Number of DumpRenderTrees to run in parallel."), + # FIXME: Display default number of child processes that will run. + optparse.make_option("--experimental-fully-parallel", + action="store_true", default=False, + help="run all tests in parallel"), + # FIXME: Need --exit-after-n-failures N + # Exit after the first N failures instead of running all tests + # FIXME: Need --exit-after-n-crashes N + # Exit after the first N crashes instead of running all tests + # FIXME: consider: --iterations n + # Number of times to run the set of tests (e.g. ABCABCABC) + optparse.make_option("--print-last-failures", action="store_true", + default=False, help="Print the tests in the last run that " + "had unexpected failures (or passes)."), + optparse.make_option("--retest-last-failures", action="store_true", + default=False, help="re-test the tests in the last run that " + "had unexpected failures (or passes)."), + optparse.make_option("--retry-failures", action="store_true", + default=True, + help="Re-try any tests that produce unexpected results (default)"), + optparse.make_option("--no-retry-failures", action="store_false", + dest="retry_failures", + help="Don't re-try any tests that produce unexpected results."), + ] + + misc_options = [ + optparse.make_option("--lint-test-files", action="store_true", + default=False, help=("Makes sure the test files parse for all " + "configurations. Does not run any tests.")), + ] + + # FIXME: Move these into json_results_generator.py + results_json_options = [ + optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME", + help=("The name of the builder shown on the waterfall running " + "this script e.g. WebKit.")), + optparse.make_option("--build-name", default="DUMMY_BUILD_NAME", + help=("The name of the builder used in its path, e.g. " + "webkit-rel.")), + optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER", + help=("The build number of the builder running this script.")), + optparse.make_option("--test-results-server", default="", + help=("If specified, upload results json files to this appengine " + "server.")), + ] + + option_list = (configuration_options + print_options + + chromium_options + results_options + test_options + + misc_options + results_json_options + + old_run_webkit_tests_compat) + option_parser = optparse.OptionParser(option_list=option_list) + + options, args = option_parser.parse_args(args) + if options.sources: + options.verbose = True + + return options, args + + +def main(): + options, args = parse_args() + port_obj = port.get(options.platform, options) + return run(port_obj, options, args) + +if '__main__' == __name__: + try: + sys.exit(main()) + except KeyboardInterrupt: + # this mirrors what the shell normally does + sys.exit(signal.SIGINT + 128) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py new file mode 100644 index 0000000..1c751d6 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py @@ -0,0 +1,195 @@ +#!/usr/bin/python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for run_webkit_tests.""" + +import codecs +import os +import sys +import unittest + +from webkitpy.common import array_stream +from webkitpy.layout_tests import port +from webkitpy.layout_tests import run_webkit_tests + +from webkitpy.thirdparty.mock import Mock + + +def passing_run(args, port_obj=None, logging_included=False): + if not logging_included: + args.extend(['--print', 'nothing']) + options, args = run_webkit_tests.parse_args(args) + if port_obj is None: + port_obj = port.get(options.platform, options) + res = run_webkit_tests.run(port_obj, options, args) + return res == 0 + +def logging_run(args): + options, args = run_webkit_tests.parse_args(args) + port_obj = port.get(options.platform, options) + buildbot_output = array_stream.ArrayStream() + regular_output = array_stream.ArrayStream() + res = run_webkit_tests.run(port_obj, options, args, + buildbot_output=buildbot_output, + regular_output=regular_output) + return (res, buildbot_output, regular_output) + + +class MainTest(unittest.TestCase): + def test_fast(self): + self.assertTrue(passing_run(['--platform', 'test'])) + self.assertTrue(passing_run(['--platform', 'test', '--run-singly'])) + self.assertTrue(passing_run(['--platform', 'test', + 'text/article-element.html'])) + self.assertTrue(passing_run(['--platform', 'test', + '--child-processes', '1', + '--print', 'unexpected'])) + + def test_child_processes(self): + (res, buildbot_output, regular_output) = logging_run( + ['--platform', 'test', '--print', 'config', '--child-processes', + '1']) + self.assertTrue('Running one DumpRenderTree\n' + in regular_output.get()) + + (res, buildbot_output, regular_output) = logging_run( + ['--platform', 'test', '--print', 'config', '--child-processes', + '2']) + self.assertTrue('Running 2 DumpRenderTrees in parallel\n' + in regular_output.get()) + + def test_last_results(self): + passing_run(['--platform', 'test']) + (res, buildbot_output, regular_output) = logging_run( + ['--platform', 'test', '--print-last-failures']) + self.assertEqual(regular_output.get(), ['\n\n']) + self.assertEqual(buildbot_output.get(), []) + + +def _mocked_open(original_open, file_list): + def _wrapper(name, mode, encoding): + if name.find("-expected.") != -1 and mode == "w": + # we don't want to actually write new baselines, so stub these out + name.replace('\\', '/') + file_list.append(name) + return original_open(os.devnull, mode, encoding) + return original_open(name, mode, encoding) + return _wrapper + + +class RebaselineTest(unittest.TestCase): + def assertBaselines(self, file_list, file): + "assert that the file_list contains the baselines.""" + for ext in [".txt", ".png", ".checksum"]: + baseline = file + "-expected" + ext + self.assertTrue(any(f.find(baseline) != -1 for f in file_list)) + + def test_reset_results(self): + file_list = [] + original_open = codecs.open + try: + # Test that we update expectations in place. If the expectation + # is mssing, update the expected generic location. + file_list = [] + codecs.open = _mocked_open(original_open, file_list) + passing_run(['--platform', 'test', '--pixel-tests', + '--reset-results', + 'image/canvas-bg.html', + 'image/canvas-zoom.html', + 'misc/missing-expectation.html']) + self.assertEqual(len(file_list), 9) + self.assertBaselines(file_list, + "data/image/canvas-zoom") + self.assertBaselines(file_list, + "data/platform/test/image/canvas-bg") + self.assertBaselines(file_list, + "data/misc/missing-expectation") + finally: + codecs.open = original_open + + def test_new_baseline(self): + file_list = [] + original_open = codecs.open + try: + # Test that we update the platform expectations. If the expectation + # is mssing, then create a new expectation in the platform dir. + file_list = [] + codecs.open = _mocked_open(original_open, file_list) + passing_run(['--platform', 'test', '--pixel-tests', + '--new-baseline', + 'image/canvas-zoom.html', + 'image/canvas-bg.html', + 'misc/missing-expectation.html']) + self.assertEqual(len(file_list), 9) + self.assertBaselines(file_list, + "data/platform/test/image/canvas-zoom") + self.assertBaselines(file_list, + "data/platform/test/image/canvas-bg") + self.assertBaselines(file_list, + "data/platform/test/misc/missing-expectation") + finally: + codecs.open = original_open + +class TestRunnerTest(unittest.TestCase): + def test_results_html(self): + mock_port = Mock() + mock_port.relative_test_filename = lambda name: name + mock_port.filename_to_uri = lambda name: name + + runner = run_webkit_tests.TestRunner(port=mock_port, options=Mock(), printer=Mock()) + expected_html = u"""<html> + <head> + <title>Layout Test Results (time)</title> + </head> + <body> + <h2>Title (time)</h2> + <p><a href='test_path'>test_path</a><br /> +</p> +</body></html> +""" + html = runner._results_html(["test_path"], {}, "Title", override_time="time") + self.assertEqual(html, expected_html) + + +class DryrunTest(unittest.TestCase): + def test_basics(self): + # FIXME: it's hard to know which platforms are safe to test; the + # chromium platforms require a chromium checkout, and the mac platform + # requires fcntl, so it can't be tested on win32, etc. There is + # probably a better way of handling this. + if sys.platform != "mac": + return + self.assertTrue(passing_run(['--platform', 'dryrun', + 'fast/html'])) + self.assertTrue(passing_run(['--platform', 'dryrun-mac', + 'fast/html'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/fuzzy_image_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/fuzzy_image_diff.py index 89dd192..64dfb20 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/fuzzy_image_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/fuzzy_image_diff.py @@ -36,13 +36,15 @@ import logging import os import shutil -from layout_package import test_failures -from test_types import test_type_base +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.test_types import test_type_base + +_log = logging.getLogger("webkitpy.layout_tests.test_types.fuzzy_image_diff") class FuzzyImageDiff(test_type_base.TestTypeBase): - def compare_output(self, filename, output, test_args, target): + def compare_output(self, filename, output, test_args, configuration): """Implementation of CompareOutput that checks the output image and checksum against the expected files from the LayoutTest directory. """ @@ -55,14 +57,14 @@ class FuzzyImageDiff(test_type_base.TestTypeBase): expected_png_file = self._port.expected_filename(filename, '.png') if test_args.show_sources: - logging.debug('Using %s' % expected_png_file) + _log.debug('Using %s' % expected_png_file) # Also report a missing expected PNG file. if not os.path.isfile(expected_png_file): failures.append(test_failures.FailureMissingImage(self)) # Run the fuzzymatcher - r = port.fuzzy_diff(test_args.png_path, expected_png_file) + r = self._port.fuzzy_diff(test_args.png_path, expected_png_file) if r != 0: failures.append(test_failures.FailureFuzzyFailure(self)) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py index 1df7ca3..65f8f3a 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py @@ -34,18 +34,23 @@ match, returns FailureImageHashMismatch and outputs both hashes into the layout test results directory. """ +from __future__ import with_statement + +import codecs import errno import logging import os import shutil -from layout_package import test_failures -from test_types import test_type_base +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.test_types import test_type_base # Cache whether we have the image_diff executable available. _compare_available = True _compare_msg_printed = False +_log = logging.getLogger("webkitpy.layout_tests.test_types.image_diff") + class ImageDiff(test_type_base.TestTypeBase): @@ -68,26 +73,32 @@ class ImageDiff(test_type_base.TestTypeBase): if errno.ENOENT != e.errno: raise - def _save_baseline_files(self, filename, png_path, checksum): + def _save_baseline_files(self, filename, png_path, checksum, + generate_new_baseline): """Saves new baselines for the PNG and checksum. Args: filename: test filename png_path: path to the actual PNG result file checksum: value of the actual checksum result + generate_new_baseline: whether to generate a new, platform-specific + baseline, or update the existing one """ - png_file = open(png_path, "rb") - png_data = png_file.read() - png_file.close() - self._save_baseline_data(filename, png_data, ".png") - self._save_baseline_data(filename, checksum, ".checksum") - - def _create_image_diff(self, port, filename, target): + with open(png_path, "rb") as png_file: + png_data = png_file.read() + self._save_baseline_data(filename, png_data, ".png", encoding=None, + generate_new_baseline=generate_new_baseline) + self._save_baseline_data(filename, checksum, ".checksum", + encoding="ascii", + generate_new_baseline=generate_new_baseline) + + def _create_image_diff(self, port, filename, configuration): """Creates the visual diff of the expected/actual PNGs. Args: filename: the name of the test - target: Debug or Release + configuration: Debug or Release + Returns True if the files are different, False if they match """ diff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_COMPARE) @@ -96,9 +107,10 @@ class ImageDiff(test_type_base.TestTypeBase): expected_filename = self.output_filename(filename, self.FILENAME_SUFFIX_EXPECTED + '.png') + result = True try: _compare_available = True - result = port.diff_image(actual_filename, expected_filename, + result = port.diff_image(expected_filename, actual_filename, diff_filename) except ValueError: _compare_available = False @@ -106,12 +118,12 @@ class ImageDiff(test_type_base.TestTypeBase): global _compare_msg_printed if not _compare_available and not _compare_msg_printed: _compare_msg_printed = True - print('image_diff not found. Make sure you have a ' + target + - ' build of the image_diff executable.') + print('image_diff not found. Make sure you have a ' + + configuration + ' build of the image_diff executable.') return result - def compare_output(self, port, filename, output, test_args, target): + def compare_output(self, port, filename, output, test_args, configuration): """Implementation of CompareOutput that checks the output image and checksum against the expected files from the LayoutTest directory. """ @@ -122,9 +134,9 @@ class ImageDiff(test_type_base.TestTypeBase): return failures # If we're generating a new baseline, we pass. - if test_args.new_baseline: + if test_args.new_baseline or test_args.reset_results: self._save_baseline_files(filename, test_args.png_path, - test_args.hash) + test_args.hash, test_args.new_baseline) return failures # Compare hashes. @@ -133,11 +145,13 @@ class ImageDiff(test_type_base.TestTypeBase): expected_png_file = self._port.expected_filename(filename, '.png') if test_args.show_sources: - logging.debug('Using %s' % expected_hash_file) - logging.debug('Using %s' % expected_png_file) + _log.debug('Using %s' % expected_hash_file) + _log.debug('Using %s' % expected_png_file) + # FIXME: We repeat this pattern often, we should share code. try: - expected_hash = open(expected_hash_file, "r").read() + with codecs.open(expected_hash_file, "r", "ascii") as file: + expected_hash = file.read() except IOError, e: if errno.ENOENT != e.errno: raise @@ -146,9 +160,10 @@ class ImageDiff(test_type_base.TestTypeBase): if not os.path.isfile(expected_png_file): # Report a missing expected PNG file. - self.write_output_files(port, filename, '', '.checksum', + self.write_output_files(port, filename, '.checksum', test_args.hash, expected_hash, - diff=False, wdiff=False) + encoding="ascii", + print_text_diffs=False) self._copy_output_png(filename, test_args.png_path, '-actual.png') failures.append(test_failures.FailureMissingImage(self)) return failures @@ -156,30 +171,28 @@ class ImageDiff(test_type_base.TestTypeBase): # Hash matched (no diff needed, okay to return). return failures - - self.write_output_files(port, filename, '', '.checksum', + self.write_output_files(port, filename, '.checksum', test_args.hash, expected_hash, - diff=False, wdiff=False) + encoding="ascii", + print_text_diffs=False) self._copy_output_png(filename, test_args.png_path, '-actual.png') self._copy_output_png(filename, expected_png_file, '-expected.png') - # Even though we only use result in one codepath below but we + # Even though we only use the result in one codepath below but we # still need to call CreateImageDiff for other codepaths. - result = self._create_image_diff(port, filename, target) + images_are_different = self._create_image_diff(port, filename, configuration) if expected_hash == '': failures.append(test_failures.FailureMissingImageHash(self)) elif test_args.hash != expected_hash: - # Hashes don't match, so see if the images match. If they do, then - # the hash is wrong. - if result == 0: - failures.append(test_failures.FailureImageHashIncorrect(self)) - else: + if images_are_different: failures.append(test_failures.FailureImageHashMismatch(self)) + else: + failures.append(test_failures.FailureImageHashIncorrect(self)) return failures def diff_files(self, port, file1, file2): - """Diff two image files. + """Diff two image files exactly. Args: file1, file2: full paths of the files to compare. @@ -188,10 +201,7 @@ class ImageDiff(test_type_base.TestTypeBase): True if two files are different. False otherwise. """ - try: - result = port.diff_image(file1, file2) + return port.diff_image(file1, file2, None, 0) except ValueError, e: return True - - return result == 1 diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py index efa2e8c..8db2e3d 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py @@ -32,11 +32,16 @@ Also defines the TestArguments "struct" to pass them additional arguments. """ +from __future__ import with_statement + +import codecs import cgi import errno import logging import os.path +_log = logging.getLogger("webkitpy.layout_tests.test_types.test_type_base") + class TestArguments(object): """Struct-like wrapper for additional arguments needed by @@ -68,19 +73,18 @@ class TestTypeBase(object): FILENAME_SUFFIX_EXPECTED = "-expected" FILENAME_SUFFIX_DIFF = "-diff" FILENAME_SUFFIX_WDIFF = "-wdiff.html" + FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html" FILENAME_SUFFIX_COMPARE = "-diff.png" - def __init__(self, port, platform, root_output_dir): + def __init__(self, port, root_output_dir): """Initialize a TestTypeBase object. Args: - platform: the platform (e.g., 'chromium-mac-leopard') - identifying the platform-specific results to be used. + port: object implementing port-specific information and methods root_output_dir: The unix style path to the output dir. """ self._root_output_dir = root_output_dir self._port = port - self._platform = platform def _make_output_directory(self, filename): """Creates the output directory (if needed) for a given test @@ -89,8 +93,9 @@ class TestTypeBase(object): self._port.relative_test_filename(filename)) self._port.maybe_make_directory(os.path.split(output_filename)[0]) - def _save_baseline_data(self, filename, data, modifier): - """Saves a new baseline file into the platform directory. + def _save_baseline_data(self, filename, data, modifier, encoding, + generate_new_baseline=True): + """Saves a new baseline file into the port's baseline directory. The file will be named simply "<test>-expected<modifier>", suitable for use as the expected results in a later run. @@ -99,18 +104,26 @@ class TestTypeBase(object): filename: path to the test file data: result to be saved as the new baseline modifier: type of the result file, e.g. ".txt" or ".png" + encoding: file encoding (none, "utf-8", etc.) + generate_new_baseline: whether to enerate a new, platform-specific + baseline, or update the existing one """ - relative_dir = os.path.dirname( - self._port.relative_test_filename(filename)) - output_dir = os.path.join( - self._port.chromium_baseline_path(self._platform), relative_dir) - output_file = os.path.basename(os.path.splitext(filename)[0] + - self.FILENAME_SUFFIX_EXPECTED + modifier) - self._port.maybe_make_directory(output_dir) - output_path = os.path.join(output_dir, output_file) - logging.debug('writing new baseline to "%s"' % (output_path)) - open(output_path, "wb").write(data) + if generate_new_baseline: + relative_dir = os.path.dirname( + self._port.relative_test_filename(filename)) + baseline_path = self._port.baseline_path() + output_dir = os.path.join(baseline_path, relative_dir) + output_file = os.path.basename(os.path.splitext(filename)[0] + + self.FILENAME_SUFFIX_EXPECTED + modifier) + self._port.maybe_make_directory(output_dir) + output_path = os.path.join(output_dir, output_file) + _log.debug('writing new baseline result "%s"' % (output_path)) + else: + output_path = self._port.expected_filename(filename, modifier) + _log.debug('resetting baseline result "%s"' % output_path) + + self._write_into_file_at_path(output_path, data, encoding) def output_filename(self, filename, modifier): """Returns a filename inside the output dir that contains modifier. @@ -130,7 +143,7 @@ class TestTypeBase(object): self._port.relative_test_filename(filename)) return os.path.splitext(output_filename)[0] + modifier - def compare_output(self, port, filename, output, test_args, target): + def compare_output(self, port, filename, output, test_args, configuration): """Method that compares the output from the test with the expected value. @@ -141,56 +154,66 @@ class TestTypeBase(object): output: a string containing the output of the test test_args: a TestArguments object holding optional additional arguments - target: Debug or Release + configuration: Debug or Release Return: a list of TestFailure objects, empty if the test passes """ raise NotImplemented - def write_output_files(self, port, filename, test_type, file_type, - output, expected, diff=True, wdiff=False): + def _write_into_file_at_path(self, file_path, contents, encoding): + """This method assumes that byte_array is already encoded + into the right format.""" + with codecs.open(file_path, "w", encoding=encoding) as file: + file.write(contents) + + def write_output_files(self, port, filename, file_type, + output, expected, encoding, + print_text_diffs=False): """Writes the test output, the expected output and optionally the diff between the two to files in the results directory. The full output filename of the actual, for example, will be - <filename><test_type>-actual<file_type> + <filename>-actual<file_type> For instance, - my_test-simp-actual.txt + my_test-actual.txt Args: filename: The test filename - test_type: A string describing the test type, e.g. "simp" file_type: A string describing the test output file type, e.g. ".txt" output: A string containing the test output expected: A string containing the expected test output - diff: if True, write a file containing the diffs too. This should be - False for results that are not text - wdiff: if True, write an HTML file containing word-by-word diffs + print_text_diffs: True for text diffs. (FIXME: We should be able to get this from the file type?) """ self._make_output_directory(filename) - actual_filename = self.output_filename(filename, - test_type + self.FILENAME_SUFFIX_ACTUAL + file_type) - expected_filename = self.output_filename(filename, - test_type + self.FILENAME_SUFFIX_EXPECTED + file_type) + actual_filename = self.output_filename(filename, self.FILENAME_SUFFIX_ACTUAL + file_type) + expected_filename = self.output_filename(filename, self.FILENAME_SUFFIX_EXPECTED + file_type) + # FIXME: This function is poorly designed. We should be passing in some sort of + # encoding information from the callers. if output: - open(actual_filename, "wb").write(output) + self._write_into_file_at_path(actual_filename, output, encoding) if expected: - open(expected_filename, "wb").write(expected) + self._write_into_file_at_path(expected_filename, expected, encoding) if not output or not expected: return - if diff: - diff = port.diff_text(expected, output, expected_filename, - actual_filename) - diff_filename = self.output_filename(filename, - test_type + self.FILENAME_SUFFIX_DIFF + file_type) - open(diff_filename, "wb").write(diff) - - if wdiff: - # Shell out to wdiff to get colored inline diffs. - wdiff = port.wdiff_text(expected_filename, actual_filename) - filename = self.output_filename(filename, test_type + - self.FILENAME_SUFFIX_WDIFF) - out = open(filename, 'wb').write(wdiff) + if not print_text_diffs: + return + + # Note: We pass encoding=None for all diff writes, as we treat diff + # output as binary. Diff output may contain multiple files in + # conflicting encodings. + diff = port.diff_text(expected, output, expected_filename, actual_filename) + diff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_DIFF + file_type) + self._write_into_file_at_path(diff_filename, diff, encoding=None) + + # Shell out to wdiff to get colored inline diffs. + wdiff = port.wdiff_text(expected_filename, actual_filename) + wdiff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_WDIFF) + self._write_into_file_at_path(wdiff_filename, wdiff, encoding=None) + + # Use WebKit's PrettyPatch.rb to get an HTML diff. + pretty_patch = port.pretty_patch_text(diff_filename) + pretty_patch_filename = self.output_filename(filename, self.FILENAME_SUFFIX_PRETTY_PATCH) + self._write_into_file_at_path(pretty_patch_filename, pretty_patch, encoding=None) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py index 54b332b..18f74b8 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py @@ -33,18 +33,17 @@ If the output doesn't match, returns FailureTextMismatch and outputs the diff files into the layout test results directory. """ +from __future__ import with_statement + +import codecs import errno import logging import os.path -from layout_package import test_failures -from test_types import test_type_base - +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.test_types import test_type_base -def is_render_tree_dump(data): - """Returns true if data appears to be a render tree dump as opposed to a - plain text dump.""" - return data.find("RenderView at (0,0)") != -1 +_log = logging.getLogger("webkitpy.layout_tests.test_types.text_diff") class TestTextDiff(test_type_base.TestTypeBase): @@ -63,13 +62,19 @@ class TestTextDiff(test_type_base.TestTypeBase): # Read the port-specific expected text. expected_filename = self._port.expected_filename(filename, '.txt') if show_sources: - logging.debug('Using %s' % expected_filename) + _log.debug('Using %s' % expected_filename) return self.get_normalized_text(expected_filename) def get_normalized_text(self, filename): + # FIXME: We repeat this pattern often, we should share code. try: - text = open(filename).read() + # NOTE: -expected.txt files are ALWAYS utf-8. However, + # we do not decode the output from DRT, so we should not + # decode the -expected.txt values either to allow comparisons. + with codecs.open(filename, "r", encoding=None) as file: + text = file.read() + # We could assert that the text is valid utf-8. except IOError, e: if errno.ENOENT != e.errno: raise @@ -78,14 +83,18 @@ class TestTextDiff(test_type_base.TestTypeBase): # Normalize line endings return text.strip("\r\n").replace("\r\n", "\n") + "\n" - def compare_output(self, port, filename, output, test_args, target): + def compare_output(self, port, filename, output, test_args, configuration): """Implementation of CompareOutput that checks the output text against the expected text from the LayoutTest directory.""" failures = [] # If we're generating a new baseline, we pass. - if test_args.new_baseline: - self._save_baseline_data(filename, output, ".txt") + if test_args.new_baseline or test_args.reset_results: + # Although all test_shell/DumpRenderTree output should be utf-8, + # we do not ever decode it inside run-webkit-tests. For some tests + # DumpRenderTree may not output utf-8 text (e.g. webarchives). + self._save_baseline_data(filename, output, ".txt", encoding=None, + generate_new_baseline=test_args.new_baseline) return failures # Normalize text to diff @@ -96,8 +105,9 @@ class TestTextDiff(test_type_base.TestTypeBase): # Write output files for new tests, too. if port.compare_text(output, expected): # Text doesn't match, write output files. - self.write_output_files(port, filename, "", ".txt", output, - expected, diff=True, wdiff=True) + self.write_output_files(port, filename, ".txt", output, + expected, encoding=None, + print_text_diffs=True) if expected == '': failures.append(test_failures.FailureMissingResult(self)) diff --git a/WebKitTools/Scripts/webkitpy/mock.pyc b/WebKitTools/Scripts/webkitpy/mock.pyc Binary files differdeleted file mode 100644 index c39d3f4..0000000 --- a/WebKitTools/Scripts/webkitpy/mock.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/multicommandtool.pyc b/WebKitTools/Scripts/webkitpy/multicommandtool.pyc Binary files differdeleted file mode 100644 index 4584643..0000000 --- a/WebKitTools/Scripts/webkitpy/multicommandtool.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/networktransaction.pyc b/WebKitTools/Scripts/webkitpy/networktransaction.pyc Binary files differdeleted file mode 100644 index fb45bcb..0000000 --- a/WebKitTools/Scripts/webkitpy/networktransaction.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/patchcollection.pyc b/WebKitTools/Scripts/webkitpy/patchcollection.pyc Binary files differdeleted file mode 100644 index 18058d3..0000000 --- a/WebKitTools/Scripts/webkitpy/patchcollection.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/python24/__init__.py b/WebKitTools/Scripts/webkitpy/python24/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/python24/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/python24/versioning.py b/WebKitTools/Scripts/webkitpy/python24/versioning.py new file mode 100644 index 0000000..8b1f21b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/python24/versioning.py @@ -0,0 +1,133 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports Python version checking.""" + +import logging +import sys + +_log = logging.getLogger("webkitpy.python24.versioning") + +# The minimum Python version the webkitpy package supports. +_MINIMUM_SUPPORTED_PYTHON_VERSION = "2.5" + + +def compare_version(sysmodule=None, target_version=None): + """Compare the current Python version with a target version. + + Args: + sysmodule: An object with version and version_info data attributes + used to detect the current Python version. The attributes + should have the same semantics as sys.version and + sys.version_info. This parameter should only be used + for unit testing. Defaults to sys. + target_version: A string representing the Python version to compare + the current version against. The string should have + one of the following three forms: 2, 2.5, or 2.5.3. + Defaults to the minimum version that the webkitpy + package supports. + + Returns: + A triple of (comparison, current_version, target_version). + + comparison: An integer representing the result of comparing the + current version with the target version. A positive + number means the current version is greater than the + target, 0 means they are the same, and a negative number + means the current version is less than the target. + This method compares version information only up + to the precision of the given target version. For + example, if the target version is 2.6 and the current + version is 2.5.3, this method uses 2.5 for the purposes + of comparing with the target. + current_version: A string representing the current Python version, for + example 2.5.3. + target_version: A string representing the version that the current + version was compared against, for example 2.5. + + """ + if sysmodule is None: + sysmodule = sys + if target_version is None: + target_version = _MINIMUM_SUPPORTED_PYTHON_VERSION + + # The number of version parts to compare. + precision = len(target_version.split(".")) + + # We use sys.version_info rather than sys.version since its first + # three elements are guaranteed to be integers. + current_version_info_to_compare = sysmodule.version_info[:precision] + # Convert integers to strings. + current_version_info_to_compare = map(str, current_version_info_to_compare) + current_version_to_compare = ".".join(current_version_info_to_compare) + + # Compare version strings lexicographically. + if current_version_to_compare > target_version: + comparison = 1 + elif current_version_to_compare == target_version: + comparison = 0 + else: + comparison = -1 + + # The version number portion of the current version string, for + # example "2.6.4". + current_version = sysmodule.version.split()[0] + + return (comparison, current_version, target_version) + + +# FIXME: Add a logging level parameter to allow the version message +# to be logged at levels other than WARNING, for example CRITICAL. +def check_version(log=None, sysmodule=None, target_version=None): + """Check the current Python version against a target version. + + Logs a warning message if the current version is less than the + target version. + + Args: + log: A logging.logger instance to use when logging the version warning. + Defaults to the logger of this module. + sysmodule: See the compare_version() docstring. + target_version: See the compare_version() docstring. + + Returns: + A boolean value of whether the current version is greater than + or equal to the target version. + + """ + if log is None: + log = _log + + (comparison, current_version, target_version) = \ + compare_version(sysmodule, target_version) + + if comparison >= 0: + # Then the current version is at least the minimum version. + return True + + message = ("WebKit Python scripts do not support your current Python " + "version (%s). The minimum supported version is %s.\n" + " See the following page to upgrade your Python version:\n\n" + " http://trac.webkit.org/wiki/PythonGuidelines\n" + % (current_version, target_version)) + log.warn(message) + return False diff --git a/WebKitTools/Scripts/webkitpy/python24/versioning_unittest.py b/WebKitTools/Scripts/webkitpy/python24/versioning_unittest.py new file mode 100644 index 0000000..6939e2d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/python24/versioning_unittest.py @@ -0,0 +1,134 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains unit tests for versioning.py.""" + +import logging +import unittest + +from webkitpy.common.system.logtesting import LogTesting +from webkitpy.python24.versioning import check_version +from webkitpy.python24.versioning import compare_version + +class MockSys(object): + + """A mock sys module for passing to version-checking methods.""" + + def __init__(self, current_version): + """Create an instance. + + current_version: A version string with major, minor, and micro + version parts. + + """ + version_info = current_version.split(".") + version_info = map(int, version_info) + + self.version = current_version + " Version details." + self.version_info = version_info + + +class CompareVersionTest(unittest.TestCase): + + """Tests compare_version().""" + + def _mock_sys(self, current_version): + return MockSys(current_version) + + def test_default_minimum_version(self): + """Test the configured minimum version that webkitpy supports.""" + (comparison, current_version, min_version) = compare_version() + self.assertEquals(min_version, "2.5") + + def compare_version(self, target_version, current_version=None): + """Call compare_version().""" + if current_version is None: + current_version = "2.5.3" + mock_sys = self._mock_sys(current_version) + return compare_version(mock_sys, target_version) + + def compare(self, target_version, current_version=None): + """Call compare_version(), and return the comparison.""" + return self.compare_version(target_version, current_version)[0] + + def test_returned_current_version(self): + """Test the current_version return value.""" + current_version = self.compare_version("2.5")[1] + self.assertEquals(current_version, "2.5.3") + + def test_returned_target_version(self): + """Test the current_version return value.""" + target_version = self.compare_version("2.5")[2] + self.assertEquals(target_version, "2.5") + + def test_target_version_major(self): + """Test major version for target.""" + self.assertEquals(-1, self.compare("3")) + self.assertEquals(0, self.compare("2")) + self.assertEquals(1, self.compare("2", "3.0.0")) + + def test_target_version_minor(self): + """Test minor version for target.""" + self.assertEquals(-1, self.compare("2.6")) + self.assertEquals(0, self.compare("2.5")) + self.assertEquals(1, self.compare("2.4")) + + def test_target_version_micro(self): + """Test minor version for target.""" + self.assertEquals(-1, self.compare("2.5.4")) + self.assertEquals(0, self.compare("2.5.3")) + self.assertEquals(1, self.compare("2.5.2")) + + +class CheckVersionTest(unittest.TestCase): + + """Tests check_version().""" + + def setUp(self): + self._log = LogTesting.setUp(self) + + def tearDown(self): + self._log.tearDown() + + def _check_version(self, minimum_version): + """Call check_version().""" + mock_sys = MockSys("2.5.3") + return check_version(sysmodule=mock_sys, target_version=minimum_version) + + def test_true_return_value(self): + """Test the configured minimum version that webkitpy supports.""" + is_current = self._check_version("2.4") + self.assertEquals(True, is_current) + self._log.assertMessages([]) # No warning was logged. + + def test_false_return_value(self): + """Test the configured minimum version that webkitpy supports.""" + is_current = self._check_version("2.6") + self.assertEquals(False, is_current) + expected_message = ('WARNING: WebKit Python scripts do not support ' + 'your current Python version (2.5.3). ' + 'The minimum supported version is 2.6.\n ' + 'See the following page to upgrade your Python ' + 'version:\n\n ' + 'http://trac.webkit.org/wiki/PythonGuidelines\n\n') + self._log.assertMessages([expected_message]) + diff --git a/WebKitTools/Scripts/webkitpy/queueengine.pyc b/WebKitTools/Scripts/webkitpy/queueengine.pyc Binary files differdeleted file mode 100644 index 635bb57..0000000 --- a/WebKitTools/Scripts/webkitpy/queueengine.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/scm.py b/WebKitTools/Scripts/webkitpy/scm.py deleted file mode 100644 index 743f3fe..0000000 --- a/WebKitTools/Scripts/webkitpy/scm.py +++ /dev/null @@ -1,513 +0,0 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. -# Copyright (c) 2009 Apple Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Python module for interacting with an SCM system (like SVN or Git) - -import os -import re -import subprocess - -# Import WebKit-specific modules. -from webkitpy.changelogs import ChangeLog -from webkitpy.executive import Executive, run_command, ScriptError -from webkitpy.webkit_logging import error, log - -def detect_scm_system(path): - if SVN.in_working_directory(path): - return SVN(cwd=path) - - if Git.in_working_directory(path): - return Git(cwd=path) - - return None - -def first_non_empty_line_after_index(lines, index=0): - first_non_empty_line = index - for line in lines[index:]: - if re.match("^\s*$", line): - first_non_empty_line += 1 - else: - break - return first_non_empty_line - - -class CommitMessage: - def __init__(self, message): - self.message_lines = message[first_non_empty_line_after_index(message, 0):] - - def body(self, lstrip=False): - lines = self.message_lines[first_non_empty_line_after_index(self.message_lines, 1):] - if lstrip: - lines = [line.lstrip() for line in lines] - return "\n".join(lines) + "\n" - - def description(self, lstrip=False, strip_url=False): - line = self.message_lines[0] - if lstrip: - line = line.lstrip() - if strip_url: - line = re.sub("^(\s*)<.+> ", "\1", line) - return line - - def message(self): - return "\n".join(self.message_lines) + "\n" - - -class CheckoutNeedsUpdate(ScriptError): - def __init__(self, script_args, exit_code, output, cwd): - ScriptError.__init__(self, script_args=script_args, exit_code=exit_code, output=output, cwd=cwd) - - -def commit_error_handler(error): - if re.search("resource out of date", error.output): - raise CheckoutNeedsUpdate(script_args=error.script_args, exit_code=error.exit_code, output=error.output, cwd=error.cwd) - Executive.default_error_handler(error) - - -class SCM: - def __init__(self, cwd, dryrun=False): - self.cwd = cwd - self.checkout_root = self.find_checkout_root(self.cwd) - self.dryrun = dryrun - - def scripts_directory(self): - return os.path.join(self.checkout_root, "WebKitTools", "Scripts") - - def script_path(self, script_name): - return os.path.join(self.scripts_directory(), script_name) - - def ensure_clean_working_directory(self, force_clean): - if not force_clean and not self.working_directory_is_clean(): - print run_command(self.status_command(), error_handler=Executive.ignore_error) - raise ScriptError(message="Working directory has modifications, pass --force-clean or --no-clean to continue.") - - log("Cleaning working directory") - self.clean_working_directory() - - def ensure_no_local_commits(self, force): - if not self.supports_local_commits(): - return - commits = self.local_commits() - if not len(commits): - return - if not force: - error("Working directory has local commits, pass --force-clean to continue.") - self.discard_local_commits() - - def apply_patch(self, patch, force=False): - # It's possible that the patch was not made from the root directory. - # We should detect and handle that case. - # FIXME: scm.py should not deal with fetching Attachment data. Attachment should just have a .data() accessor. - curl_process = subprocess.Popen(['curl', '--location', '--silent', '--show-error', patch.url()], stdout=subprocess.PIPE) - args = [self.script_path('svn-apply')] - if patch.reviewer(): - args += ['--reviewer', patch.reviewer().full_name] - if force: - args.append('--force') - - run_command(args, input=curl_process.stdout) - - def run_status_and_extract_filenames(self, status_command, status_regexp): - filenames = [] - for line in run_command(status_command).splitlines(): - match = re.search(status_regexp, line) - if not match: - continue - # status = match.group('status') - filename = match.group('filename') - filenames.append(filename) - return filenames - - def strip_r_from_svn_revision(self, svn_revision): - match = re.match("^r(?P<svn_revision>\d+)", svn_revision) - if (match): - return match.group('svn_revision') - return svn_revision - - def svn_revision_from_commit_text(self, commit_text): - match = re.search(self.commit_success_regexp(), commit_text, re.MULTILINE) - return match.group('svn_revision') - - # ChangeLog-specific code doesn't really belong in scm.py, but this function is very useful. - def modified_changelogs(self): - changelog_paths = [] - paths = self.changed_files() - for path in paths: - if os.path.basename(path) == "ChangeLog": - changelog_paths.append(path) - return changelog_paths - - # FIXME: Requires unit test - # FIXME: commit_message_for_this_commit and modified_changelogs don't - # really belong here. We should have a separate module for - # handling ChangeLogs. - def commit_message_for_this_commit(self): - changelog_paths = self.modified_changelogs() - if not len(changelog_paths): - raise ScriptError(message="Found no modified ChangeLogs, cannot create a commit message.\n" - "All changes require a ChangeLog. See:\n" - "http://webkit.org/coding/contributing.html") - - changelog_messages = [] - for changelog_path in changelog_paths: - log("Parsing ChangeLog: %s" % changelog_path) - changelog_entry = ChangeLog(changelog_path).latest_entry() - if not changelog_entry: - raise ScriptError(message="Failed to parse ChangeLog: " + os.path.abspath(changelog_path)) - changelog_messages.append(changelog_entry) - - # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does. - return CommitMessage("".join(changelog_messages).splitlines()) - - @staticmethod - def in_working_directory(path): - raise NotImplementedError, "subclasses must implement" - - @staticmethod - def find_checkout_root(path): - raise NotImplementedError, "subclasses must implement" - - @staticmethod - def commit_success_regexp(): - raise NotImplementedError, "subclasses must implement" - - def working_directory_is_clean(self): - raise NotImplementedError, "subclasses must implement" - - def clean_working_directory(self): - raise NotImplementedError, "subclasses must implement" - - def status_command(self): - raise NotImplementedError, "subclasses must implement" - - def changed_files(self): - raise NotImplementedError, "subclasses must implement" - - def display_name(self): - raise NotImplementedError, "subclasses must implement" - - def create_patch(self): - raise NotImplementedError, "subclasses must implement" - - def diff_for_revision(self, revision): - raise NotImplementedError, "subclasses must implement" - - def apply_reverse_diff(self, revision): - raise NotImplementedError, "subclasses must implement" - - def revert_files(self, file_paths): - raise NotImplementedError, "subclasses must implement" - - def commit_with_message(self, message): - raise NotImplementedError, "subclasses must implement" - - def svn_commit_log(self, svn_revision): - raise NotImplementedError, "subclasses must implement" - - def last_svn_commit_log(self): - raise NotImplementedError, "subclasses must implement" - - # Subclasses must indicate if they support local commits, - # but the SCM baseclass will only call local_commits methods when this is true. - @staticmethod - def supports_local_commits(): - raise NotImplementedError, "subclasses must implement" - - def create_patch_from_local_commit(self, commit_id): - error("Your source control manager does not support creating a patch from a local commit.") - - def create_patch_since_local_commit(self, commit_id): - error("Your source control manager does not support creating a patch from a local commit.") - - def commit_locally_with_message(self, message): - error("Your source control manager does not support local commits.") - - def discard_local_commits(self): - pass - - def local_commits(self): - return [] - - -class SVN(SCM): - def __init__(self, cwd, dryrun=False): - SCM.__init__(self, cwd, dryrun) - self.cached_version = None - - @staticmethod - def in_working_directory(path): - return os.path.isdir(os.path.join(path, '.svn')) - - @classmethod - def find_uuid(cls, path): - if not cls.in_working_directory(path): - return None - return cls.value_from_svn_info(path, 'Repository UUID') - - @classmethod - def value_from_svn_info(cls, path, field_name): - svn_info_args = ['svn', 'info', path] - info_output = run_command(svn_info_args).rstrip() - match = re.search("^%s: (?P<value>.+)$" % field_name, info_output, re.MULTILINE) - if not match: - raise ScriptError(script_args=svn_info_args, message='svn info did not contain a %s.' % field_name) - return match.group('value') - - @staticmethod - def find_checkout_root(path): - uuid = SVN.find_uuid(path) - # If |path| is not in a working directory, we're supposed to return |path|. - if not uuid: - return path - # Search up the directory hierarchy until we find a different UUID. - last_path = None - while True: - if uuid != SVN.find_uuid(path): - return last_path - last_path = path - (path, last_component) = os.path.split(path) - if last_path == path: - return None - - @staticmethod - def commit_success_regexp(): - return "^Committed revision (?P<svn_revision>\d+)\.$" - - def svn_version(self): - if not self.cached_version: - self.cached_version = run_command(['svn', '--version', '--quiet']) - - return self.cached_version - - def working_directory_is_clean(self): - return run_command(['svn', 'diff']) == "" - - def clean_working_directory(self): - run_command(['svn', 'revert', '-R', '.']) - - def status_command(self): - return ['svn', 'status'] - - def changed_files(self): - if self.svn_version() > "1.6": - status_regexp = "^(?P<status>[ACDMR]).{6} (?P<filename>.+)$" - else: - status_regexp = "^(?P<status>[ACDMR]).{5} (?P<filename>.+)$" - return self.run_status_and_extract_filenames(self.status_command(), status_regexp) - - @staticmethod - def supports_local_commits(): - return False - - def display_name(self): - return "svn" - - def create_patch(self): - return run_command(self.script_path("svn-create-patch"), cwd=self.checkout_root, return_stderr=False) - - def diff_for_revision(self, revision): - return run_command(['svn', 'diff', '-c', str(revision)]) - - def _repository_url(self): - return self.value_from_svn_info(self.checkout_root, 'URL') - - def apply_reverse_diff(self, revision): - # '-c -revision' applies the inverse diff of 'revision' - svn_merge_args = ['svn', 'merge', '--non-interactive', '-c', '-%s' % revision, self._repository_url()] - log("WARNING: svn merge has been known to take more than 10 minutes to complete. It is recommended you use git for rollouts.") - log("Running '%s'" % " ".join(svn_merge_args)) - run_command(svn_merge_args) - - def revert_files(self, file_paths): - run_command(['svn', 'revert'] + file_paths) - - def commit_with_message(self, message): - if self.dryrun: - # Return a string which looks like a commit so that things which parse this output will succeed. - return "Dry run, no commit.\nCommitted revision 0." - return run_command(['svn', 'commit', '-m', message], error_handler=commit_error_handler) - - def svn_commit_log(self, svn_revision): - svn_revision = self.strip_r_from_svn_revision(str(svn_revision)) - return run_command(['svn', 'log', '--non-interactive', '--revision', svn_revision]); - - def last_svn_commit_log(self): - # BASE is the checkout revision, HEAD is the remote repository revision - # http://svnbook.red-bean.com/en/1.0/ch03s03.html - return self.svn_commit_log('BASE') - -# All git-specific logic should go here. -class Git(SCM): - def __init__(self, cwd, dryrun=False): - SCM.__init__(self, cwd, dryrun) - - @classmethod - def in_working_directory(cls, path): - return run_command(['git', 'rev-parse', '--is-inside-work-tree'], cwd=path, error_handler=Executive.ignore_error).rstrip() == "true" - - @classmethod - def find_checkout_root(cls, path): - # "git rev-parse --show-cdup" would be another way to get to the root - (checkout_root, dot_git) = os.path.split(run_command(['git', 'rev-parse', '--git-dir'], cwd=path)) - # If we were using 2.6 # checkout_root = os.path.relpath(checkout_root, path) - if not os.path.isabs(checkout_root): # Sometimes git returns relative paths - checkout_root = os.path.join(path, checkout_root) - return checkout_root - - @staticmethod - def commit_success_regexp(): - return "^Committed r(?P<svn_revision>\d+)$" - - - def discard_local_commits(self): - run_command(['git', 'reset', '--hard', 'trunk']) - - def local_commits(self): - return run_command(['git', 'log', '--pretty=oneline', 'HEAD...trunk']).splitlines() - - def rebase_in_progress(self): - return os.path.exists(os.path.join(self.checkout_root, '.git/rebase-apply')) - - def working_directory_is_clean(self): - return run_command(['git', 'diff-index', 'HEAD']) == "" - - def clean_working_directory(self): - # Could run git clean here too, but that wouldn't match working_directory_is_clean - run_command(['git', 'reset', '--hard', 'HEAD']) - # Aborting rebase even though this does not match working_directory_is_clean - if self.rebase_in_progress(): - run_command(['git', 'rebase', '--abort']) - - def status_command(self): - return ['git', 'status'] - - def changed_files(self): - status_command = ['git', 'diff', '-r', '--name-status', '-C', '-M', 'HEAD'] - status_regexp = '^(?P<status>[ADM])\t(?P<filename>.+)$' - return self.run_status_and_extract_filenames(status_command, status_regexp) - - @staticmethod - def supports_local_commits(): - return True - - def display_name(self): - return "git" - - def create_patch(self): - return run_command(['git', 'diff', '--binary', 'HEAD']) - - @classmethod - def git_commit_from_svn_revision(cls, revision): - # git svn find-rev always exits 0, even when the revision is not found. - return run_command(['git', 'svn', 'find-rev', 'r%s' % revision]).rstrip() - - def diff_for_revision(self, revision): - git_commit = self.git_commit_from_svn_revision(revision) - return self.create_patch_from_local_commit(git_commit) - - def apply_reverse_diff(self, revision): - # Assume the revision is an svn revision. - git_commit = self.git_commit_from_svn_revision(revision) - if not git_commit: - raise ScriptError(message='Failed to find git commit for revision %s, git svn log output: "%s"' % (revision, git_commit)) - - # I think this will always fail due to ChangeLogs. - # FIXME: We need to detec specific failure conditions and handle them. - run_command(['git', 'revert', '--no-commit', git_commit], error_handler=Executive.ignore_error) - - # Fix any ChangeLogs if necessary. - changelog_paths = self.modified_changelogs() - if len(changelog_paths): - run_command([self.script_path('resolve-ChangeLogs')] + changelog_paths) - - def revert_files(self, file_paths): - run_command(['git', 'checkout', 'HEAD'] + file_paths) - - def commit_with_message(self, message): - self.commit_locally_with_message(message) - return self.push_local_commits_to_server() - - def svn_commit_log(self, svn_revision): - svn_revision = self.strip_r_from_svn_revision(svn_revision) - return run_command(['git', 'svn', 'log', '-r', svn_revision]) - - def last_svn_commit_log(self): - return run_command(['git', 'svn', 'log', '--limit=1']) - - # Git-specific methods: - - def create_patch_from_local_commit(self, commit_id): - return run_command(['git', 'diff', '--binary', commit_id + "^.." + commit_id]) - - def create_patch_since_local_commit(self, commit_id): - return run_command(['git', 'diff', '--binary', commit_id]) - - def commit_locally_with_message(self, message): - run_command(['git', 'commit', '--all', '-F', '-'], input=message) - - def push_local_commits_to_server(self): - if self.dryrun: - # Return a string which looks like a commit so that things which parse this output will succeed. - return "Dry run, no remote commit.\nCommitted r0" - return run_command(['git', 'svn', 'dcommit'], error_handler=commit_error_handler) - - # This function supports the following argument formats: - # no args : rev-list trunk..HEAD - # A..B : rev-list A..B - # A...B : error! - # A B : [A, B] (different from git diff, which would use "rev-list A..B") - def commit_ids_from_commitish_arguments(self, args): - if not len(args): - # FIXME: trunk is not always the remote branch name, need a way to detect the name. - args.append('trunk..HEAD') - - commit_ids = [] - for commitish in args: - if '...' in commitish: - raise ScriptError(message="'...' is not supported (found in '%s'). Did you mean '..'?" % commitish) - elif '..' in commitish: - commit_ids += reversed(run_command(['git', 'rev-list', commitish]).splitlines()) - else: - # Turn single commits or branch or tag names into commit ids. - commit_ids += run_command(['git', 'rev-parse', '--revs-only', commitish]).splitlines() - return commit_ids - - def commit_message_for_local_commit(self, commit_id): - commit_lines = run_command(['git', 'cat-file', 'commit', commit_id]).splitlines() - - # Skip the git headers. - first_line_after_headers = 0 - for line in commit_lines: - first_line_after_headers += 1 - if line == "": - break - return CommitMessage(commit_lines[first_line_after_headers:]) - - def files_changed_summary_for_commit(self, commit_id): - return run_command(['git', 'diff-tree', '--shortstat', '--no-commit-id', commit_id]) diff --git a/WebKitTools/Scripts/webkitpy/scm.pyc b/WebKitTools/Scripts/webkitpy/scm.pyc Binary files differdeleted file mode 100644 index 520f611..0000000 --- a/WebKitTools/Scripts/webkitpy/scm.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/scm_unittest.py b/WebKitTools/Scripts/webkitpy/scm_unittest.py deleted file mode 100644 index 73faf40..0000000 --- a/WebKitTools/Scripts/webkitpy/scm_unittest.py +++ /dev/null @@ -1,595 +0,0 @@ -# Copyright (C) 2009 Google Inc. All rights reserved. -# Copyright (C) 2009 Apple Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import base64 -import os -import os.path -import re -import stat -import subprocess -import tempfile -import unittest -import urllib - -from datetime import date -from webkitpy.executive import Executive, run_command, ScriptError -from webkitpy.scm import detect_scm_system, SCM, CheckoutNeedsUpdate, commit_error_handler -from webkitpy.bugzilla import Attachment # FIXME: This should not be needed - -# Eventually we will want to write tests which work for both scms. (like update_webkit, changed_files, etc.) -# Perhaps through some SCMTest base-class which both SVNTest and GitTest inherit from. - -# FIXME: This should be unified into one of the executive.py commands! -def run_silent(args, cwd=None): - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) - process.communicate() # ignore output - exit_code = process.wait() - if exit_code: - raise ScriptError('Failed to run "%s" exit_code: %d cwd: %s' % (args, exit_code, cwd)) - -def write_into_file_at_path(file_path, contents): - file = open(file_path, 'w') - file.write(contents) - file.close() - -def read_from_path(file_path): - file = open(file_path, 'r') - contents = file.read() - file.close() - return contents - -# Exists to share svn repository creation code between the git and svn tests -class SVNTestRepository: - @staticmethod - def _setup_test_commits(test_object): - # Add some test commits - os.chdir(test_object.svn_checkout_path) - test_file = open('test_file', 'w') - test_file.write("test1") - test_file.flush() - - run_command(['svn', 'add', 'test_file']) - run_command(['svn', 'commit', '--quiet', '--message', 'initial commit']) - - test_file.write("test2") - test_file.flush() - - run_command(['svn', 'commit', '--quiet', '--message', 'second commit']) - - test_file.write("test3\n") - test_file.flush() - - run_command(['svn', 'commit', '--quiet', '--message', 'third commit']) - - test_file.write("test4\n") - test_file.close() - - run_command(['svn', 'commit', '--quiet', '--message', 'fourth commit']) - - # svn does not seem to update after commit as I would expect. - run_command(['svn', 'update']) - - @classmethod - def setup(cls, test_object): - # Create an test SVN repository - test_object.svn_repo_path = tempfile.mkdtemp(suffix="svn_test_repo") - test_object.svn_repo_url = "file://%s" % test_object.svn_repo_path # Not sure this will work on windows - # git svn complains if we don't pass --pre-1.5-compatible, not sure why: - # Expected FS format '2'; found format '3' at /usr/local/libexec/git-core//git-svn line 1477 - run_command(['svnadmin', 'create', '--pre-1.5-compatible', test_object.svn_repo_path]) - - # Create a test svn checkout - test_object.svn_checkout_path = tempfile.mkdtemp(suffix="svn_test_checkout") - run_command(['svn', 'checkout', '--quiet', test_object.svn_repo_url, test_object.svn_checkout_path]) - - cls._setup_test_commits(test_object) - - @classmethod - def tear_down(cls, test_object): - run_command(['rm', '-rf', test_object.svn_repo_path]) - run_command(['rm', '-rf', test_object.svn_checkout_path]) - -# For testing the SCM baseclass directly. -class SCMClassTests(unittest.TestCase): - def setUp(self): - self.dev_null = open(os.devnull, "w") # Used to make our Popen calls quiet. - - def tearDown(self): - self.dev_null.close() - - def test_run_command_with_pipe(self): - input_process = subprocess.Popen(['echo', 'foo\nbar'], stdout=subprocess.PIPE, stderr=self.dev_null) - self.assertEqual(run_command(['grep', 'bar'], input=input_process.stdout), "bar\n") - - # Test the non-pipe case too: - self.assertEqual(run_command(['grep', 'bar'], input="foo\nbar"), "bar\n") - - command_returns_non_zero = ['/bin/sh', '--invalid-option'] - # Test when the input pipe process fails. - input_process = subprocess.Popen(command_returns_non_zero, stdout=subprocess.PIPE, stderr=self.dev_null) - self.assertTrue(input_process.poll() != 0) - self.assertRaises(ScriptError, run_command, ['grep', 'bar'], input=input_process.stdout) - - # Test when the run_command process fails. - input_process = subprocess.Popen(['echo', 'foo\nbar'], stdout=subprocess.PIPE, stderr=self.dev_null) # grep shows usage and calls exit(2) when called w/o arguments. - self.assertRaises(ScriptError, run_command, command_returns_non_zero, input=input_process.stdout) - - def test_error_handlers(self): - git_failure_message="Merge conflict during commit: Your file or directory 'WebCore/ChangeLog' is probably out-of-date: resource out of date; try updating at /usr/local/libexec/git-core//git-svn line 469" - svn_failure_message="""svn: Commit failed (details follow): -svn: File or directory 'ChangeLog' is out of date; try updating -svn: resource out of date; try updating -""" - command_does_not_exist = ['does_not_exist', 'invalid_option'] - self.assertRaises(OSError, run_command, command_does_not_exist) - self.assertRaises(OSError, run_command, command_does_not_exist, error_handler=Executive.ignore_error) - - command_returns_non_zero = ['/bin/sh', '--invalid-option'] - self.assertRaises(ScriptError, run_command, command_returns_non_zero) - # Check if returns error text: - self.assertTrue(run_command(command_returns_non_zero, error_handler=Executive.ignore_error)) - - self.assertRaises(CheckoutNeedsUpdate, commit_error_handler, ScriptError(output=git_failure_message)) - self.assertRaises(CheckoutNeedsUpdate, commit_error_handler, ScriptError(output=svn_failure_message)) - self.assertRaises(ScriptError, commit_error_handler, ScriptError(output='blah blah blah')) - - -# GitTest and SVNTest inherit from this so any test_ methods here will be run once for this class and then once for each subclass. -class SCMTest(unittest.TestCase): - def _create_patch(self, patch_contents): - patch_path = os.path.join(self.svn_checkout_path, 'patch.diff') - write_into_file_at_path(patch_path, patch_contents) - patch = {} - patch['reviewer'] = 'Joe Cool' - patch['bug_id'] = '12345' - patch['url'] = 'file://%s' % urllib.pathname2url(patch_path) - return Attachment(patch, None) # FIXME: This is a hack, scm.py shouldn't be fetching attachment data. - - def _setup_webkittools_scripts_symlink(self, local_scm): - webkit_scm = detect_scm_system(os.path.dirname(os.path.abspath(__file__))) - webkit_scripts_directory = webkit_scm.scripts_directory() - local_scripts_directory = local_scm.scripts_directory() - os.mkdir(os.path.dirname(local_scripts_directory)) - os.symlink(webkit_scripts_directory, local_scripts_directory) - - # Tests which both GitTest and SVNTest should run. - # FIXME: There must be a simpler way to add these w/o adding a wrapper method to both subclasses - def _shared_test_commit_with_message(self): - write_into_file_at_path('test_file', 'more test content') - commit_text = self.scm.commit_with_message('another test commit') - self.assertEqual(self.scm.svn_revision_from_commit_text(commit_text), '5') - - self.scm.dryrun = True - write_into_file_at_path('test_file', 'still more test content') - commit_text = self.scm.commit_with_message('yet another test commit') - self.assertEqual(self.scm.svn_revision_from_commit_text(commit_text), '0') - - def _shared_test_reverse_diff(self): - self._setup_webkittools_scripts_symlink(self.scm) # Git's apply_reverse_diff uses resolve-ChangeLogs - # Only test the simple case, as any other will end up with conflict markers. - self.scm.apply_reverse_diff('4') - self.assertEqual(read_from_path('test_file'), "test1test2test3\n") - - def _shared_test_diff_for_revision(self): - # Patch formats are slightly different between svn and git, so just regexp for things we know should be there. - r3_patch = self.scm.diff_for_revision(3) - self.assertTrue(re.search('test3', r3_patch)) - self.assertFalse(re.search('test4', r3_patch)) - self.assertTrue(re.search('test2', r3_patch)) - self.assertTrue(re.search('test2', self.scm.diff_for_revision(2))) - - def _shared_test_svn_apply_git_patch(self): - self._setup_webkittools_scripts_symlink(self.scm) - git_binary_addition = """diff --git a/fizzbuzz7.gif b/fizzbuzz7.gif -new file mode 100644 -index 0000000000000000000000000000000000000000..64a9532e7794fcd791f6f12157406d90 -60151690 -GIT binary patch -literal 512 -zcmZ?wbhEHbRAx|MU|?iW{Kxc~?KofD;ckY;H+&5HnHl!!GQMD7h+sU{_)e9f^V3c? -zhJP##HdZC#4K}7F68@!1jfWQg2daCm-gs#3|JREDT>c+pG4L<_2;w##WMO#ysPPap -zLqpAf1OE938xAsSp4!5f-o><?VKe(#0jEcwfHGF4%M1^kRs14oVBp2ZEL{E1N<-zJ -zsfLmOtKta;2_;2c#^S1-8cf<nb!QnGl>c!Xe6RXvrEtAWBvSDTgTO1j3vA31Puw!A -zs(87q)j_mVDTqBo-P+03-P5mHCEnJ+x}YdCuS7#bCCyePUe(ynK+|4b-3qK)T?Z&) -zYG+`tl4h?GZv_$t82}X4*DTE|$;{DEiPyF@)U-1+FaX++T9H{&%cag`W1|zVP@`%b -zqiSkp6{BTpWTkCr!=<C6Q=?#~R8^JfrliAF6Q^gV9Iup8RqCXqqhqC`qsyhk<-nlB -z00f{QZvfK&|Nm#oZ0TQl`Yr$BIa6A@16O26ud7H<QM=xl`toLKnz-3h@9c9q&wm|X -z{89I|WPyD!*M?gv?q`;L=2YFeXrJQNti4?}s!zFo=5CzeBxC69xA<zrjP<wUcCRh4 -ptUl-ZG<%a~#LwkIWv&q!KSCH7tQ8cJDiw+|GV?MN)RjY50RTb-xvT&H - -literal 0 -HcmV?d00001 - -""" - self.scm.apply_patch(self._create_patch(git_binary_addition)) - added = read_from_path('fizzbuzz7.gif') - self.assertEqual(512, len(added)) - self.assertTrue(added.startswith('GIF89a')) - self.assertTrue('fizzbuzz7.gif' in self.scm.changed_files()) - - # The file already exists. - self.assertRaises(ScriptError, self.scm.apply_patch, self._create_patch(git_binary_addition)) - - git_binary_modification = """diff --git a/fizzbuzz7.gif b/fizzbuzz7.gif -index 64a9532e7794fcd791f6f12157406d9060151690..323fae03f4606ea9991df8befbb2fca7 -GIT binary patch -literal 7 -OcmYex&reD$;sO8*F9L)B - -literal 512 -zcmZ?wbhEHbRAx|MU|?iW{Kxc~?KofD;ckY;H+&5HnHl!!GQMD7h+sU{_)e9f^V3c? -zhJP##HdZC#4K}7F68@!1jfWQg2daCm-gs#3|JREDT>c+pG4L<_2;w##WMO#ysPPap -zLqpAf1OE938xAsSp4!5f-o><?VKe(#0jEcwfHGF4%M1^kRs14oVBp2ZEL{E1N<-zJ -zsfLmOtKta;2_;2c#^S1-8cf<nb!QnGl>c!Xe6RXvrEtAWBvSDTgTO1j3vA31Puw!A -zs(87q)j_mVDTqBo-P+03-P5mHCEnJ+x}YdCuS7#bCCyePUe(ynK+|4b-3qK)T?Z&) -zYG+`tl4h?GZv_$t82}X4*DTE|$;{DEiPyF@)U-1+FaX++T9H{&%cag`W1|zVP@`%b -zqiSkp6{BTpWTkCr!=<C6Q=?#~R8^JfrliAF6Q^gV9Iup8RqCXqqhqC`qsyhk<-nlB -z00f{QZvfK&|Nm#oZ0TQl`Yr$BIa6A@16O26ud7H<QM=xl`toLKnz-3h@9c9q&wm|X -z{89I|WPyD!*M?gv?q`;L=2YFeXrJQNti4?}s!zFo=5CzeBxC69xA<zrjP<wUcCRh4 -ptUl-ZG<%a~#LwkIWv&q!KSCH7tQ8cJDiw+|GV?MN)RjY50RTb-xvT&H - -""" - self.scm.apply_patch(self._create_patch(git_binary_modification)) - modified = read_from_path('fizzbuzz7.gif') - self.assertEqual('foobar\n', modified) - self.assertTrue('fizzbuzz7.gif' in self.scm.changed_files()) - - # Applying the same modification should fail. - self.assertRaises(ScriptError, self.scm.apply_patch, self._create_patch(git_binary_modification)) - - git_binary_deletion = """diff --git a/fizzbuzz7.gif b/fizzbuzz7.gif -deleted file mode 100644 -index 323fae0..0000000 -GIT binary patch -literal 0 -HcmV?d00001 - -literal 7 -OcmYex&reD$;sO8*F9L)B - -""" - self.scm.apply_patch(self._create_patch(git_binary_deletion)) - self.assertFalse(os.path.exists('fizzbuzz7.gif')) - self.assertFalse('fizzbuzz7.gif' in self.scm.changed_files()) - - # Cannot delete again. - self.assertRaises(ScriptError, self.scm.apply_patch, self._create_patch(git_binary_deletion)) - - -class SVNTest(SCMTest): - - @staticmethod - def _set_date_and_reviewer(changelog_entry): - # Joe Cool matches the reviewer set in SCMTest._create_patch - changelog_entry = changelog_entry.replace('REVIEWER_HERE', 'Joe Cool') - # svn-apply will update ChangeLog entries with today's date. - return changelog_entry.replace('DATE_HERE', date.today().isoformat()) - - def test_svn_apply(self): - first_entry = """2009-10-26 Eric Seidel <eric@webkit.org> - - Reviewed by Foo Bar. - - Most awesome change ever. - - * scm_unittest.py: -""" - intermediate_entry = """2009-10-27 Eric Seidel <eric@webkit.org> - - Reviewed by Baz Bar. - - A more awesomer change yet! - - * scm_unittest.py: -""" - one_line_overlap_patch = """Index: ChangeLog -=================================================================== ---- ChangeLog (revision 5) -+++ ChangeLog (working copy) -@@ -1,5 +1,13 @@ - 2009-10-26 Eric Seidel <eric@webkit.org> - -+ Reviewed by NOBODY (OOPS!). -+ -+ Second most awsome change ever. -+ -+ * scm_unittest.py: -+ -+2009-10-26 Eric Seidel <eric@webkit.org> -+ - Reviewed by Foo Bar. - - Most awesome change ever. -""" - one_line_overlap_entry = """DATE_HERE Eric Seidel <eric@webkit.org> - - Reviewed by REVIEWER_HERE. - - Second most awsome change ever. - - * scm_unittest.py: -""" - two_line_overlap_patch = """Index: ChangeLog -=================================================================== ---- ChangeLog (revision 5) -+++ ChangeLog (working copy) -@@ -2,6 +2,14 @@ - - Reviewed by Foo Bar. - -+ Second most awsome change ever. -+ -+ * scm_unittest.py: -+ -+2009-10-26 Eric Seidel <eric@webkit.org> -+ -+ Reviewed by Foo Bar. -+ - Most awesome change ever. - - * scm_unittest.py: -""" - two_line_overlap_entry = """DATE_HERE Eric Seidel <eric@webkit.org> - - Reviewed by Foo Bar. - - Second most awsome change ever. - - * scm_unittest.py: -""" - write_into_file_at_path('ChangeLog', first_entry) - run_command(['svn', 'add', 'ChangeLog']) - run_command(['svn', 'commit', '--quiet', '--message', 'ChangeLog commit']) - - # Patch files were created against just 'first_entry'. - # Add a second commit to make svn-apply have to apply the patches with fuzz. - changelog_contents = "%s\n%s" % (intermediate_entry, first_entry) - write_into_file_at_path('ChangeLog', changelog_contents) - run_command(['svn', 'commit', '--quiet', '--message', 'Intermediate commit']) - - self._setup_webkittools_scripts_symlink(self.scm) - self.scm.apply_patch(self._create_patch(one_line_overlap_patch)) - expected_changelog_contents = "%s\n%s" % (self._set_date_and_reviewer(one_line_overlap_entry), changelog_contents) - self.assertEquals(read_from_path('ChangeLog'), expected_changelog_contents) - - self.scm.revert_files(['ChangeLog']) - self.scm.apply_patch(self._create_patch(two_line_overlap_patch)) - expected_changelog_contents = "%s\n%s" % (self._set_date_and_reviewer(two_line_overlap_entry), changelog_contents) - self.assertEquals(read_from_path('ChangeLog'), expected_changelog_contents) - - def setUp(self): - SVNTestRepository.setup(self) - os.chdir(self.svn_checkout_path) - self.scm = detect_scm_system(self.svn_checkout_path) - - def tearDown(self): - SVNTestRepository.tear_down(self) - - def test_create_patch_is_full_patch(self): - test_dir_path = os.path.join(self.svn_checkout_path, 'test_dir') - os.mkdir(test_dir_path) - test_file_path = os.path.join(test_dir_path, 'test_file2') - write_into_file_at_path(test_file_path, 'test content') - run_command(['svn', 'add', 'test_dir']) - - # create_patch depends on 'svn-create-patch', so make a dummy version. - scripts_path = os.path.join(self.svn_checkout_path, 'WebKitTools', 'Scripts') - os.makedirs(scripts_path) - create_patch_path = os.path.join(scripts_path, 'svn-create-patch') - write_into_file_at_path(create_patch_path, '#!/bin/sh\necho $PWD') # We could pass -n to prevent the \n, but not all echo accept -n. - os.chmod(create_patch_path, stat.S_IXUSR | stat.S_IRUSR) - - # Change into our test directory and run the create_patch command. - os.chdir(test_dir_path) - scm = detect_scm_system(test_dir_path) - self.assertEqual(scm.checkout_root, self.svn_checkout_path) # Sanity check that detection worked right. - patch_contents = scm.create_patch() - # Our fake 'svn-create-patch' returns $PWD instead of a patch, check that it was executed from the root of the repo. - self.assertEqual("%s\n" % os.path.realpath(scm.checkout_root), patch_contents) # Add a \n because echo adds a \n. - - def test_detection(self): - scm = detect_scm_system(self.svn_checkout_path) - self.assertEqual(scm.display_name(), "svn") - self.assertEqual(scm.supports_local_commits(), False) - - def test_apply_small_binary_patch(self): - patch_contents = """Index: test_file.swf -=================================================================== -Cannot display: file marked as a binary type. -svn:mime-type = application/octet-stream - -Property changes on: test_file.swf -___________________________________________________________________ -Name: svn:mime-type - + application/octet-stream - - -Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA== -""" - expected_contents = base64.b64decode("Q1dTBx0AAAB42itg4GlgYJjGwMDDyODMxMDw34GBgQEAJPQDJA==") - self._setup_webkittools_scripts_symlink(self.scm) - patch_file = self._create_patch(patch_contents) - self.scm.apply_patch(patch_file) - actual_contents = read_from_path("test_file.swf") - self.assertEqual(actual_contents, expected_contents) - - def test_apply_svn_patch(self): - scm = detect_scm_system(self.svn_checkout_path) - patch = self._create_patch(run_command(['svn', 'diff', '-r4:3'])) - self._setup_webkittools_scripts_symlink(scm) - scm.apply_patch(patch) - - def test_apply_svn_patch_force(self): - scm = detect_scm_system(self.svn_checkout_path) - patch = self._create_patch(run_command(['svn', 'diff', '-r2:4'])) - self._setup_webkittools_scripts_symlink(scm) - self.assertRaises(ScriptError, scm.apply_patch, patch, force=True) - - def test_commit_logs(self): - # Commits have dates and usernames in them, so we can't just direct compare. - self.assertTrue(re.search('fourth commit', self.scm.last_svn_commit_log())) - self.assertTrue(re.search('second commit', self.scm.svn_commit_log(2))) - - def test_commit_text_parsing(self): - self._shared_test_commit_with_message() - - def test_reverse_diff(self): - self._shared_test_reverse_diff() - - def test_diff_for_revision(self): - self._shared_test_diff_for_revision() - - def test_svn_apply_git_patch(self): - self._shared_test_svn_apply_git_patch() - -class GitTest(SCMTest): - - def _setup_git_clone_of_svn_repository(self): - self.git_checkout_path = tempfile.mkdtemp(suffix="git_test_checkout") - # --quiet doesn't make git svn silent, so we use run_silent to redirect output - run_silent(['git', 'svn', '--quiet', 'clone', self.svn_repo_url, self.git_checkout_path]) - - def _tear_down_git_clone_of_svn_repository(self): - run_command(['rm', '-rf', self.git_checkout_path]) - - def setUp(self): - SVNTestRepository.setup(self) - self._setup_git_clone_of_svn_repository() - os.chdir(self.git_checkout_path) - self.scm = detect_scm_system(self.git_checkout_path) - - def tearDown(self): - SVNTestRepository.tear_down(self) - self._tear_down_git_clone_of_svn_repository() - - def test_detection(self): - scm = detect_scm_system(self.git_checkout_path) - self.assertEqual(scm.display_name(), "git") - self.assertEqual(scm.supports_local_commits(), True) - - def test_rebase_in_progress(self): - svn_test_file = os.path.join(self.svn_checkout_path, 'test_file') - write_into_file_at_path(svn_test_file, "svn_checkout") - run_command(['svn', 'commit', '--message', 'commit to conflict with git commit'], cwd=self.svn_checkout_path) - - git_test_file = os.path.join(self.git_checkout_path, 'test_file') - write_into_file_at_path(git_test_file, "git_checkout") - run_command(['git', 'commit', '-a', '-m', 'commit to be thrown away by rebase abort']) - - # --quiet doesn't make git svn silent, so use run_silent to redirect output - self.assertRaises(ScriptError, run_silent, ['git', 'svn', '--quiet', 'rebase']) # Will fail due to a conflict leaving us mid-rebase. - - scm = detect_scm_system(self.git_checkout_path) - self.assertTrue(scm.rebase_in_progress()) - - # Make sure our cleanup works. - scm.clean_working_directory() - self.assertFalse(scm.rebase_in_progress()) - - # Make sure cleanup doesn't throw when no rebase is in progress. - scm.clean_working_directory() - - def test_commitish_parsing(self): - scm = detect_scm_system(self.git_checkout_path) - - # Multiple revisions are cherry-picked. - self.assertEqual(len(scm.commit_ids_from_commitish_arguments(['HEAD~2'])), 1) - self.assertEqual(len(scm.commit_ids_from_commitish_arguments(['HEAD', 'HEAD~2'])), 2) - - # ... is an invalid range specifier - self.assertRaises(ScriptError, scm.commit_ids_from_commitish_arguments, ['trunk...HEAD']) - - def test_commitish_order(self): - scm = detect_scm_system(self.git_checkout_path) - - commit_range = 'HEAD~3..HEAD' - - actual_commits = scm.commit_ids_from_commitish_arguments([commit_range]) - expected_commits = [] - expected_commits += reversed(run_command(['git', 'rev-list', commit_range]).splitlines()) - - self.assertEqual(actual_commits, expected_commits) - - def test_apply_git_patch(self): - scm = detect_scm_system(self.git_checkout_path) - patch = self._create_patch(run_command(['git', 'diff', 'HEAD..HEAD^'])) - self._setup_webkittools_scripts_symlink(scm) - scm.apply_patch(patch) - - def test_apply_git_patch_force(self): - scm = detect_scm_system(self.git_checkout_path) - patch = self._create_patch(run_command(['git', 'diff', 'HEAD~2..HEAD'])) - self._setup_webkittools_scripts_symlink(scm) - self.assertRaises(ScriptError, scm.apply_patch, patch, force=True) - - def test_commit_text_parsing(self): - self._shared_test_commit_with_message() - - def test_reverse_diff(self): - self._shared_test_reverse_diff() - - def test_diff_for_revision(self): - self._shared_test_diff_for_revision() - - def test_svn_apply_git_patch(self): - self._shared_test_svn_apply_git_patch() - - def test_create_binary_patch(self): - # Create a git binary patch and check the contents. - scm = detect_scm_system(self.git_checkout_path) - test_file_name = 'binary_file' - test_file_path = os.path.join(self.git_checkout_path, test_file_name) - file_contents = ''.join(map(chr, range(256))) - write_into_file_at_path(test_file_path, file_contents) - run_command(['git', 'add', test_file_name]) - patch = scm.create_patch() - self.assertTrue(re.search(r'\nliteral 0\n', patch)) - self.assertTrue(re.search(r'\nliteral 256\n', patch)) - - # Check if we can apply the created patch. - run_command(['git', 'rm', '-f', test_file_name]) - self._setup_webkittools_scripts_symlink(scm) - self.scm.apply_patch(self._create_patch(patch)) - self.assertEqual(file_contents, read_from_path(test_file_path)) - - # Check if we can create a patch from a local commit. - write_into_file_at_path(test_file_path, file_contents) - run_command(['git', 'add', test_file_name]) - run_command(['git', 'commit', '-m', 'binary diff']) - patch_from_local_commit = scm.create_patch_from_local_commit('HEAD') - self.assertTrue(re.search(r'\nliteral 0\n', patch_from_local_commit)) - self.assertTrue(re.search(r'\nliteral 256\n', patch_from_local_commit)) - patch_since_local_commit = scm.create_patch_since_local_commit('HEAD^1') - self.assertTrue(re.search(r'\nliteral 0\n', patch_since_local_commit)) - self.assertTrue(re.search(r'\nliteral 256\n', patch_since_local_commit)) - self.assertEqual(patch_from_local_commit, patch_since_local_commit) - - -if __name__ == '__main__': - unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/statusserver.pyc b/WebKitTools/Scripts/webkitpy/statusserver.pyc Binary files differdeleted file mode 100644 index 2ba11b0..0000000 --- a/WebKitTools/Scripts/webkitpy/statusserver.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/__init__.py b/WebKitTools/Scripts/webkitpy/steps/__init__.py deleted file mode 100644 index 5ae4bea..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/__init__.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# FIXME: Is this the right way to do this? -from webkitpy.steps.applypatch import ApplyPatch -from webkitpy.steps.applypatchwithlocalcommit import ApplyPatchWithLocalCommit -from webkitpy.steps.build import Build -from webkitpy.steps.checkstyle import CheckStyle -from webkitpy.steps.cleanworkingdirectory import CleanWorkingDirectory -from webkitpy.steps.cleanworkingdirectorywithlocalcommits import CleanWorkingDirectoryWithLocalCommits -from webkitpy.steps.closebug import CloseBug -from webkitpy.steps.closebugforlanddiff import CloseBugForLandDiff -from webkitpy.steps.closepatch import ClosePatch -from webkitpy.steps.commit import Commit -from webkitpy.steps.completerollout import CompleteRollout -from webkitpy.steps.confirmdiff import ConfirmDiff -from webkitpy.steps.createbug import CreateBug -from webkitpy.steps.editchangelog import EditChangeLog -from webkitpy.steps.ensurebuildersaregreen import EnsureBuildersAreGreen -from webkitpy.steps.ensurelocalcommitifneeded import EnsureLocalCommitIfNeeded -from webkitpy.steps.obsoletepatches import ObsoletePatches -from webkitpy.steps.options import Options -from webkitpy.steps.postdiff import PostDiff -from webkitpy.steps.postdiffforcommit import PostDiffForCommit -from webkitpy.steps.preparechangelogforrevert import PrepareChangeLogForRevert -from webkitpy.steps.preparechangelog import PrepareChangeLog -from webkitpy.steps.promptforbugortitle import PromptForBugOrTitle -from webkitpy.steps.revertrevision import RevertRevision -from webkitpy.steps.runtests import RunTests -from webkitpy.steps.updatechangelogswithreviewer import UpdateChangeLogsWithReviewer -from webkitpy.steps.update import Update diff --git a/WebKitTools/Scripts/webkitpy/steps/__init__.pyc b/WebKitTools/Scripts/webkitpy/steps/__init__.pyc Binary files differdeleted file mode 100644 index ccf513b..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/__init__.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/abstractstep.pyc b/WebKitTools/Scripts/webkitpy/steps/abstractstep.pyc Binary files differdeleted file mode 100644 index d172c92..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/abstractstep.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/applypatch.pyc b/WebKitTools/Scripts/webkitpy/steps/applypatch.pyc Binary files differdeleted file mode 100644 index 0c5212d..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/applypatch.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/applypatchwithlocalcommit.pyc b/WebKitTools/Scripts/webkitpy/steps/applypatchwithlocalcommit.pyc Binary files differdeleted file mode 100644 index 67afd10..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/applypatchwithlocalcommit.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/build.pyc b/WebKitTools/Scripts/webkitpy/steps/build.pyc Binary files differdeleted file mode 100644 index 8e9e5ee..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/build.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/checkstyle.pyc b/WebKitTools/Scripts/webkitpy/steps/checkstyle.pyc Binary files differdeleted file mode 100644 index 561036b..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/checkstyle.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectory.pyc b/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectory.pyc Binary files differdeleted file mode 100644 index 11383f2..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectory.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectorywithlocalcommits.pyc b/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectorywithlocalcommits.pyc Binary files differdeleted file mode 100644 index 757d19a..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectorywithlocalcommits.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/closebug.pyc b/WebKitTools/Scripts/webkitpy/steps/closebug.pyc Binary files differdeleted file mode 100644 index 356f430..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/closebug.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff.pyc b/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff.pyc Binary files differdeleted file mode 100644 index 2dd3814..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/closepatch.pyc b/WebKitTools/Scripts/webkitpy/steps/closepatch.pyc Binary files differdeleted file mode 100644 index 3a2a75b..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/closepatch.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/commit.pyc b/WebKitTools/Scripts/webkitpy/steps/commit.pyc Binary files differdeleted file mode 100644 index fe9ef1a..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/commit.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/completerollout.py b/WebKitTools/Scripts/webkitpy/steps/completerollout.py deleted file mode 100644 index 8534956..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/completerollout.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from webkitpy.comments import bug_comment_from_commit_text -from webkitpy.steps.build import Build -from webkitpy.steps.commit import Commit -from webkitpy.steps.metastep import MetaStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log - - -class CompleteRollout(MetaStep): - substeps = [ - Build, - Commit, - ] - - @classmethod - def options(cls): - collected_options = cls._collect_options_from_steps(cls.substeps) - collected_options.append(Options.complete_rollout) - return collected_options - - def run(self, state): - bug_id = state["bug_id"] - # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout. - # Once we trust rollout we will remove this option. - if not self._options.complete_rollout: - log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"webkit-patch land %s\" to commit the rollout." % bug_id) - return - - MetaStep.run(self, state) - - commit_comment = bug_comment_from_commit_text(self._tool.scm(), state["commit_text"]) - comment_text = "Reverted r%s for reason:\n\n%s\n\n%s" % (state["revision"], state["reason"], commit_comment) - - if not bug_id: - log(comment_text) - log("No bugs were updated.") - return - self._tool.bugs.reopen_bug(bug_id, comment_text) diff --git a/WebKitTools/Scripts/webkitpy/steps/completerollout.pyc b/WebKitTools/Scripts/webkitpy/steps/completerollout.pyc Binary files differdeleted file mode 100644 index 47312b8..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/completerollout.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/confirmdiff.pyc b/WebKitTools/Scripts/webkitpy/steps/confirmdiff.pyc Binary files differdeleted file mode 100644 index d3fc1d4..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/confirmdiff.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/createbug.pyc b/WebKitTools/Scripts/webkitpy/steps/createbug.pyc Binary files differdeleted file mode 100644 index e27f5ec..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/createbug.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/editchangelog.pyc b/WebKitTools/Scripts/webkitpy/steps/editchangelog.pyc Binary files differdeleted file mode 100644 index 2ca5dbf..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/editchangelog.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/ensurebuildersaregreen.pyc b/WebKitTools/Scripts/webkitpy/steps/ensurebuildersaregreen.pyc Binary files differdeleted file mode 100644 index dd98935..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/ensurebuildersaregreen.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/ensurelocalcommitifneeded.pyc b/WebKitTools/Scripts/webkitpy/steps/ensurelocalcommitifneeded.pyc Binary files differdeleted file mode 100644 index 18ce98a..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/ensurelocalcommitifneeded.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/metastep.pyc b/WebKitTools/Scripts/webkitpy/steps/metastep.pyc Binary files differdeleted file mode 100644 index 21d2bf6..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/metastep.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/obsoletepatches.pyc b/WebKitTools/Scripts/webkitpy/steps/obsoletepatches.pyc Binary files differdeleted file mode 100644 index 4586950..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/obsoletepatches.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/options.pyc b/WebKitTools/Scripts/webkitpy/steps/options.pyc Binary files differdeleted file mode 100644 index 7634605..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/options.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/postdiff.pyc b/WebKitTools/Scripts/webkitpy/steps/postdiff.pyc Binary files differdeleted file mode 100644 index 82f1c09..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/postdiff.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/preparechangelog.pyc b/WebKitTools/Scripts/webkitpy/steps/preparechangelog.pyc Binary files differdeleted file mode 100644 index 2f6edbd..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/preparechangelog.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/preparechangelogforrevert.pyc b/WebKitTools/Scripts/webkitpy/steps/preparechangelogforrevert.pyc Binary files differdeleted file mode 100644 index c1f0ca4..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/preparechangelogforrevert.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/promptforbugortitle.pyc b/WebKitTools/Scripts/webkitpy/steps/promptforbugortitle.pyc Binary files differdeleted file mode 100644 index fdca409..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/promptforbugortitle.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/revertrevision.pyc b/WebKitTools/Scripts/webkitpy/steps/revertrevision.pyc Binary files differdeleted file mode 100644 index ec08b1f..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/revertrevision.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/runtests.pyc b/WebKitTools/Scripts/webkitpy/steps/runtests.pyc Binary files differdeleted file mode 100644 index 16908fb..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/runtests.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/update.pyc b/WebKitTools/Scripts/webkitpy/steps/update.pyc Binary files differdeleted file mode 100644 index 0b9e7e9..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/update.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreviewer.pyc b/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreviewer.pyc Binary files differdeleted file mode 100644 index 6c9b7fd..0000000 --- a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreviewer.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/stepsequence.pyc b/WebKitTools/Scripts/webkitpy/stepsequence.pyc Binary files differdeleted file mode 100644 index 4b3505e..0000000 --- a/WebKitTools/Scripts/webkitpy/stepsequence.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/style/checker.py b/WebKitTools/Scripts/webkitpy/style/checker.py index fbda8cb..e3c56c5 100644 --- a/WebKitTools/Scripts/webkitpy/style/checker.py +++ b/WebKitTools/Scripts/webkitpy/style/checker.py @@ -1,5 +1,6 @@ # Copyright (C) 2009 Google Inc. All rights reserved. # Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# Copyright (C) 2010 ProFUSION embedded systems # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -29,25 +30,26 @@ """Front end of some style-checker modules.""" -import codecs -import getopt +import logging import os.path import sys -from .. style_references import parse_patch +from checkers.common import categories as CommonCategories +from checkers.common import CarriageReturnChecker +from checkers.cpp import CppChecker +from checkers.python import PythonChecker +from checkers.text import TextChecker from error_handlers import DefaultStyleErrorHandler -from error_handlers import PatchStyleErrorHandler -from filter import validate_filter_rules from filter import FilterConfiguration -from processors.common import check_no_carriage_return -from processors.common import categories as CommonCategories -from processors.cpp import CppProcessor -from processors.text import TextProcessor +from optparser import ArgumentParser +from optparser import DefaultCommandOptionValues +from webkitpy.style_references import configure_logging as _configure_logging +_log = logging.getLogger("webkitpy.style.checker") -# These defaults are used by check-webkit-style. -WEBKIT_DEFAULT_VERBOSITY = 1 -WEBKIT_DEFAULT_OUTPUT_FORMAT = 'emacs' +# These are default option values for the command-line option parser. +_DEFAULT_MIN_CONFIDENCE = 1 +_DEFAULT_OUTPUT_FORMAT = 'emacs' # FIXME: For style categories we will never want to have, remove them. @@ -55,14 +57,16 @@ WEBKIT_DEFAULT_OUTPUT_FORMAT = 'emacs' # modify the implementation and enable them. # # Throughout this module, we use "filter rule" rather than "filter" -# for an individual boolean filter flag like "+foo". This allows us to +# for an individual boolean filter flag like "+foo". This allows us to # reserve "filter" for what one gets by collectively applying all of # the filter rules. # -# The _WEBKIT_FILTER_RULES are prepended to any user-specified filter -# rules. Since by default all errors are on, only include rules that -# begin with a - sign. -WEBKIT_DEFAULT_FILTER_RULES = [ +# The base filter rules are the filter rules that begin the list of +# filter rules used to check style. For example, these rules precede +# any user-specified filter rules. Since by default all categories are +# checked, this list should normally include only rules that begin +# with a "-" sign. +_BASE_FILTER_RULES = [ '-build/endif_comment', '-build/include_what_you_use', # <string> for std::string '-build/storage_class', # const static @@ -83,32 +87,41 @@ WEBKIT_DEFAULT_FILTER_RULES = [ '-whitespace/blank_line', '-whitespace/end_of_line', '-whitespace/labels', + # List Python pep8 categories last. + # + # Because much of WebKit's Python code base does not abide by the + # PEP8 79 character limit, we ignore the 79-character-limit category + # pep8/E501 for now. + # + # FIXME: Consider bringing WebKit's Python code base into conformance + # with the 79 character limit, or some higher limit that is + # agreeable to the WebKit project. + '-pep8/E501', ] -# FIXME: Change the second value of each tuple from a tuple to a list, -# and alter the filter code so it accepts lists instead. (The -# filter code will need to convert incoming values from a list -# to a tuple prior to caching). This will make this -# configuration setting a bit simpler since tuples have an -# unusual syntax case. -# # The path-specific filter rules. # # This list is order sensitive. Only the first path substring match # is used. See the FilterConfiguration documentation in filter.py # for more information on this list. +# +# Each string appearing in this nested list should have at least +# one associated unit test assertion. These assertions are located, +# for example, in the test_path_rules_specifier() unit test method of +# checker_unittest.py. _PATH_RULES_SPECIFIER = [ # Files in these directories are consumers of the WebKit # API and therefore do not follow the same header including # discipline as WebCore. (["WebKitTools/WebKitAPITest/", "WebKit/qt/QGVLauncher/"], - ("-build/include", - "-readability/streams")), - ([# The GTK+ APIs use GTK+ naming style, which includes - # lower-cased, underscore-separated values. - "WebKit/gtk/webkit/", + ["-build/include", + "-readability/streams"]), + ([# The EFL APIs use EFL naming style, which includes + # both lower-cased and camel-cased, underscore-sparated + # values. + "WebKit/efl/ewk/", # There is no clean way to avoid "yy_*" names used by flex. "WebCore/css/CSSParser.cpp", # There is no clean way to avoid "xxx_data" methods inside @@ -116,515 +129,282 @@ _PATH_RULES_SPECIFIER = [ # QtTest module. "WebKit/qt/tests/", "JavaScriptCore/qt/tests"], - ("-readability/naming",)), - # These are test file patterns. - (["_test.cpp", - "_unittest.cpp", - "_regtest.cpp"], - ("-readability/streams", # Many unit tests use cout. - "-runtime/rtti")), + ["-readability/naming"]), + ([# The GTK+ APIs use GTK+ naming style, which includes + # lower-cased, underscore-separated values. + # Also, GTK+ allows the use of NULL. + "WebKit/gtk/webkit/", + "WebKitTools/DumpRenderTree/gtk/"], + ["-readability/naming", + "-readability/null"]), + ([# Header files in ForwardingHeaders have no header guards or + # exceptional header guards (e.g., WebCore_FWD_Debugger_h). + "/ForwardingHeaders/"], + ["-build/header_guard"]), + + # For third-party Python code, keep only the following checks-- + # + # No tabs: to avoid having to set the SVN allow-tabs property. + # No trailing white space: since this is easy to correct. + # No carriage-return line endings: since this is easy to correct. + # + (["webkitpy/thirdparty/"], + ["-", + "+pep8/W191", # Tabs + "+pep8/W291", # Trailing white space + "+whitespace/carriage_return"]), ] +_CPP_FILE_EXTENSIONS = [ + 'c', + 'cpp', + 'h', + ] + +_PYTHON_FILE_EXTENSION = 'py' + +# FIXME: Include 'vcproj' files as text files after creating a mechanism +# for exempting them from the carriage-return checker (since they +# are Windows-only files). +_TEXT_FILE_EXTENSIONS = [ + 'ac', + 'cc', + 'cgi', + 'css', + 'exp', + 'flex', + 'gyp', + 'gypi', + 'html', + 'idl', + 'in', + 'js', + 'mm', + 'php', + 'pl', + 'pm', + 'pri', + 'pro', + 'rb', + 'sh', + 'txt', +# 'vcproj', # See FIXME above. + 'wm', + 'xhtml', + 'y', + ] + + +# Files to skip that are less obvious. +# # Some files should be skipped when checking style. For example, # WebKit maintains some files in Mozilla style on purpose to ease # future merges. -# -# Include a warning for skipped files that are less obvious. -SKIPPED_FILES_WITH_WARNING = [ +_SKIPPED_FILES_WITH_WARNING = [ # The Qt API and tests do not follow WebKit style. # They follow Qt style. :) "gtk2drawing.c", # WebCore/platform/gtk/gtk2drawing.c - "gtk2drawing.h", # WebCore/platform/gtk/gtk2drawing.h + "gtkdrawing.h", # WebCore/platform/gtk/gtkdrawing.h "JavaScriptCore/qt/api/", "WebKit/gtk/tests/", "WebKit/qt/Api/", "WebKit/qt/tests/", + "WebKit/qt/examples/", ] -# Don't include a warning for skipped files that are more common -# and more obvious. -SKIPPED_FILES_WITHOUT_WARNING = [ - "LayoutTests/" +# Files to skip that are more common or obvious. +# +# This list should be in addition to files with FileType.NONE. Files +# with FileType.NONE are automatically skipped without warning. +_SKIPPED_FILES_WITHOUT_WARNING = [ + "LayoutTests/", ] # The maximum number of errors to report per file, per category. # If a category is not a key, then it has no maximum. -MAX_REPORTS_PER_CATEGORY = { +_MAX_REPORTS_PER_CATEGORY = { "whitespace/carriage_return": 1 } -def style_categories(): +def _all_categories(): """Return the set of all categories used by check-webkit-style.""" - # Take the union across all processors. - return CommonCategories.union(CppProcessor.categories) - - -def webkit_argument_defaults(): - """Return the DefaultArguments instance for use by check-webkit-style.""" - return ArgumentDefaults(WEBKIT_DEFAULT_OUTPUT_FORMAT, - WEBKIT_DEFAULT_VERBOSITY, - WEBKIT_DEFAULT_FILTER_RULES) - - -def _create_usage(defaults): - """Return the usage string to display for command help. + # Take the union across all checkers. + categories = CommonCategories.union(CppChecker.categories) - Args: - defaults: An ArgumentDefaults instance. - - """ - usage = """ -Syntax: %(program_name)s [--verbose=#] [--git-commit=<SingleCommit>] [--output=vs7] - [--filter=-x,+y,...] [file] ... - - The style guidelines this tries to follow are here: - http://webkit.org/coding/coding-style.html - - Every style error is given a confidence score from 1-5, with 5 meaning - we are certain of the problem, and 1 meaning it could be a legitimate - construct. This can miss some errors and does not substitute for - code review. - - To prevent specific lines from being linted, add a '// NOLINT' comment to the - end of the line. - - Linted extensions are .cpp, .c and .h. Other file types are ignored. - - The file parameter is optional and accepts multiple files. Leaving - out the file parameter applies the check to all files considered changed - by your source control management system. - - Flags: - - verbose=# - A number 1-5 that restricts output to errors with a confidence - score at or above this value. In particular, the value 1 displays - all errors. The default is %(default_verbosity)s. + # FIXME: Consider adding all of the pep8 categories. Since they + # are not too meaningful for documentation purposes, for + # now we add only the categories needed for the unit tests + # (which validate the consistency of the configuration + # settings against the known categories, etc). + categories = categories.union(["pep8/W191", "pep8/W291", "pep8/E501"]) - git-commit=<SingleCommit> - Checks the style of everything from the given commit to the local tree. + return categories - output=vs7 - The output format, which may be one of - emacs : to ease emacs parsing - vs7 : compatible with Visual Studio - Defaults to "%(default_output_format)s". Other formats are unsupported. - filter=-x,+y,... - A comma-separated list of boolean filter rules used to filter - which categories of style guidelines to check. The script checks - a category if the category passes the filter rules, as follows. +def _check_webkit_style_defaults(): + """Return the default command-line options for check-webkit-style.""" + return DefaultCommandOptionValues(min_confidence=_DEFAULT_MIN_CONFIDENCE, + output_format=_DEFAULT_OUTPUT_FORMAT) - Any webkit category starts out passing. All filter rules are then - evaluated left to right, with later rules taking precedence. For - example, the rule "+foo" passes any category that starts with "foo", - and "-foo" fails any such category. The filter input "-whitespace, - +whitespace/braces" fails the category "whitespace/tab" and passes - "whitespace/braces". - - Examples: --filter=-whitespace,+whitespace/braces - --filter=-whitespace,-runtime/printf,+runtime/printf_format - --filter=-,+build/include_what_you_use - - Category names appear in error messages in brackets, for example - [whitespace/indent]. To see a list of all categories available to - %(program_name)s, along with which are enabled by default, pass - the empty filter as follows: - --filter= -""" % {'program_name': os.path.basename(sys.argv[0]), - 'default_verbosity': defaults.verbosity, - 'default_output_format': defaults.output_format} - - return usage - - -# FIXME: Eliminate support for "extra_flag_values". -# -# FIXME: Remove everything from ProcessorOptions except for the -# information that can be passed via the command line, and -# rename to something like CheckWebKitStyleOptions. This -# includes, but is not limited to, removing the -# max_reports_per_error attribute and the is_reportable() -# method. See also the FIXME below to create a new class -# called something like CheckerConfiguration. -# -# This class should not have knowledge of the flag key names. -class ProcessorOptions(object): - - """A container to store options passed via the command line. - - Attributes: - extra_flag_values: A string-string dictionary of all flag key-value - pairs that are not otherwise represented by this - class. The default is the empty dictionary. - filter_configuration: A FilterConfiguration instance. The default - is the "empty" filter configuration, which - means that all errors should be checked. +# This function assists in optparser not having to import from checker. +def check_webkit_style_parser(): + all_categories = _all_categories() + default_options = _check_webkit_style_defaults() + return ArgumentParser(all_categories=all_categories, + base_filter_rules=_BASE_FILTER_RULES, + default_options=default_options) - git_commit: A string representing the git commit to check. - The default is None. - max_reports_per_error: The maximum number of errors to report - per file, per category. +def check_webkit_style_configuration(options): + """Return a StyleProcessorConfiguration instance for check-webkit-style. - output_format: A string that is the output format. The supported - output formats are "emacs" which emacs can parse - and "vs7" which Microsoft Visual Studio 7 can parse. - - verbosity: An integer between 1-5 inclusive that restricts output - to errors with a confidence score at or above this value. - The default is 1, which reports all errors. + Args: + options: A CommandOptionValues instance. """ - def __init__(self, - extra_flag_values=None, - filter_configuration = None, - git_commit=None, - max_reports_per_category=None, - output_format="emacs", - verbosity=1): - if extra_flag_values is None: - extra_flag_values = {} - if filter_configuration is None: - filter_configuration = FilterConfiguration() - if max_reports_per_category is None: - max_reports_per_category = {} - - if output_format not in ("emacs", "vs7"): - raise ValueError('Invalid "output_format" parameter: ' - 'value must be "emacs" or "vs7". ' - 'Value given: "%s".' % output_format) - - if (verbosity < 1) or (verbosity > 5): - raise ValueError('Invalid "verbosity" parameter: ' - "value must be an integer between 1-5 inclusive. " - 'Value given: "%s".' % verbosity) - - self.extra_flag_values = extra_flag_values - self.filter_configuration = filter_configuration - self.git_commit = git_commit - self.max_reports_per_category = max_reports_per_category - self.output_format = output_format - self.verbosity = verbosity - - # Useful for unit testing. - def __eq__(self, other): - """Return whether this ProcessorOptions instance is equal to another.""" - if self.extra_flag_values != other.extra_flag_values: - return False - if self.filter_configuration != other.filter_configuration: - return False - if self.git_commit != other.git_commit: - return False - if self.max_reports_per_category != other.max_reports_per_category: - return False - if self.output_format != other.output_format: - return False - if self.verbosity != other.verbosity: - return False - - return True - - # Useful for unit testing. - def __ne__(self, other): - # Python does not automatically deduce this from __eq__(). - return not self.__eq__(other) - - def is_reportable(self, category, confidence_in_error, path): - """Return whether an error is reportable. + filter_configuration = FilterConfiguration( + base_rules=_BASE_FILTER_RULES, + path_specific=_PATH_RULES_SPECIFIER, + user_rules=options.filter_rules) - An error is reportable if the confidence in the error - is at least the current verbosity level, and if the current - filter says that the category should be checked for the - given path. + return StyleProcessorConfiguration(filter_configuration=filter_configuration, + max_reports_per_category=_MAX_REPORTS_PER_CATEGORY, + min_confidence=options.min_confidence, + output_format=options.output_format, + stderr_write=sys.stderr.write) - Args: - category: A string that is a style category. - confidence_in_error: An integer between 1 and 5, inclusive, that - represents the application's confidence in - the error. A higher number signifies greater - confidence. - path: The path of the file being checked - """ - if confidence_in_error < self.verbosity: - return False +def _create_log_handlers(stream): + """Create and return a default list of logging.Handler instances. - return self.filter_configuration.should_check(category, path) + Format WARNING messages and above to display the logging level, and + messages strictly below WARNING not to display it. - -# This class should not have knowledge of the flag key names. -class ArgumentDefaults(object): - - """A container to store default argument values. - - Attributes: - output_format: A string that is the default output format. - verbosity: An integer that is the default verbosity level. - base_filter_rules: A list of strings that are boolean filter rules - to prepend to any user-specified rules. + Args: + stream: See the configure_logging() docstring. """ + # Handles logging.WARNING and above. + error_handler = logging.StreamHandler(stream) + error_handler.setLevel(logging.WARNING) + formatter = logging.Formatter("%(levelname)s: %(message)s") + error_handler.setFormatter(formatter) - def __init__(self, default_output_format, default_verbosity, - default_base_filter_rules): - self.output_format = default_output_format - self.verbosity = default_verbosity - self.base_filter_rules = default_base_filter_rules + # Create a logging.Filter instance that only accepts messages + # below WARNING (i.e. filters out anything WARNING or above). + non_error_filter = logging.Filter() + # The filter method accepts a logging.LogRecord instance. + non_error_filter.filter = lambda record: record.levelno < logging.WARNING + non_error_handler = logging.StreamHandler(stream) + non_error_handler.addFilter(non_error_filter) + formatter = logging.Formatter("%(message)s") + non_error_handler.setFormatter(formatter) -class ArgumentPrinter(object): + return [error_handler, non_error_handler] - """Supports the printing of check-webkit-style command arguments.""" - def _flag_pair_to_string(self, flag_key, flag_value): - return '--%(key)s=%(val)s' % {'key': flag_key, 'val': flag_value } +def _create_debug_log_handlers(stream): + """Create and return a list of logging.Handler instances for debugging. - def to_flag_string(self, options): - """Return a flag string yielding the given ProcessorOptions instance. - - This method orders the flag values alphabetically by the flag key. - - Args: - options: A ProcessorOptions instance. - - """ - flags = options.extra_flag_values.copy() - - flags['output'] = options.output_format - flags['verbose'] = options.verbosity - # Only include the filter flag if user-provided rules are present. - user_rules = options.filter_configuration.user_rules - if user_rules: - flags['filter'] = ",".join(user_rules) - if options.git_commit: - flags['git-commit'] = options.git_commit - - flag_string = '' - # Alphabetizing lets us unit test this method. - for key in sorted(flags.keys()): - flag_string += self._flag_pair_to_string(key, flags[key]) + ' ' - - return flag_string.strip() - - -class ArgumentParser(object): - - """Supports the parsing of check-webkit-style command arguments. - - Attributes: - defaults: An ArgumentDefaults instance. - create_usage: A function that accepts an ArgumentDefaults instance - and returns a string of usage instructions. - This defaults to the function used to generate the - usage string for check-webkit-style. - doc_print: A function that accepts a string parameter and that is - called to display help messages. This defaults to - sys.stderr.write(). + Args: + stream: See the configure_logging() docstring. """ + handler = logging.StreamHandler(stream) + formatter = logging.Formatter("%(name)s: %(levelname)-8s %(message)s") + handler.setFormatter(formatter) - def __init__(self, argument_defaults, create_usage=None, doc_print=None): - if create_usage is None: - create_usage = _create_usage - if doc_print is None: - doc_print = sys.stderr.write + return [handler] - self.defaults = argument_defaults - self.create_usage = create_usage - self.doc_print = doc_print - def _exit_with_usage(self, error_message=''): - """Exit and print a usage string with an optional error message. +def configure_logging(stream, logger=None, is_verbose=False): + """Configure logging, and return the list of handlers added. - Args: - error_message: A string that is an error message to print. - - """ - usage = self.create_usage(self.defaults) - self.doc_print(usage) - if error_message: - sys.exit('\nFATAL ERROR: ' + error_message) - else: - sys.exit(1) - - def _exit_with_categories(self): - """Exit and print the style categories and default filter rules.""" - self.doc_print('\nAll categories:\n') - categories = style_categories() - for category in sorted(categories): - self.doc_print(' ' + category + '\n') - - self.doc_print('\nDefault filter rules**:\n') - for filter_rule in sorted(self.defaults.base_filter_rules): - self.doc_print(' ' + filter_rule + '\n') - self.doc_print('\n**The command always evaluates the above rules, ' - 'and before any --filter flag.\n\n') - - sys.exit(0) - - def _parse_filter_flag(self, flag_value): - """Parse the --filter flag, and return a list of filter rules. - - Args: - flag_value: A string of comma-separated filter rules, for - example "-whitespace,+whitespace/indent". + Returns: + A list of references to the logging handlers added to the root + logger. This allows the caller to later remove the handlers + using logger.removeHandler. This is useful primarily during unit + testing where the caller may want to configure logging temporarily + and then undo the configuring. - """ - filters = [] - for uncleaned_filter in flag_value.split(','): - filter = uncleaned_filter.strip() - if not filter: - continue - filters.append(filter) - return filters - - def parse(self, args, extra_flags=None): - """Parse the command line arguments to check-webkit-style. - - Args: - args: A list of command-line arguments as returned by sys.argv[1:]. - extra_flags: A list of flags whose values we want to extract, but - are not supported by the ProcessorOptions class. - An example flag "new_flag=". This defaults to the - empty list. - - Returns: - A tuple of (filenames, options) - - filenames: The list of filenames to check. - options: A ProcessorOptions instance. + Args: + stream: A file-like object to which to log. The stream must + define an "encoding" data attribute, or else logging + raises an error. + logger: A logging.logger instance to configure. This parameter + should be used only in unit tests. Defaults to the + root logger. + is_verbose: A boolean value of whether logging should be verbose. - """ - if extra_flags is None: - extra_flags = [] - - output_format = self.defaults.output_format - verbosity = self.defaults.verbosity - base_rules = self.defaults.base_filter_rules - - # The flags already supported by the ProcessorOptions class. - flags = ['help', 'output=', 'verbose=', 'filter=', 'git-commit='] - - for extra_flag in extra_flags: - if extra_flag in flags: - raise ValueError('Flag \'%(extra_flag)s is duplicated ' - 'or already supported.' % - {'extra_flag': extra_flag}) - flags.append(extra_flag) - - try: - (opts, filenames) = getopt.getopt(args, '', flags) - except getopt.GetoptError: - # FIXME: Settle on an error handling approach: come up - # with a consistent guideline as to when and whether - # a ValueError should be raised versus calling - # sys.exit when needing to interrupt execution. - self._exit_with_usage('Invalid arguments.') - - extra_flag_values = {} - git_commit = None - user_rules = [] - - for (opt, val) in opts: - if opt == '--help': - self._exit_with_usage() - elif opt == '--output': - output_format = val - elif opt == '--verbose': - verbosity = val - elif opt == '--git-commit': - git_commit = val - elif opt == '--filter': - if not val: - self._exit_with_categories() - # Prepend the defaults. - user_rules = self._parse_filter_flag(val) - else: - extra_flag_values[opt] = val - - # Check validity of resulting values. - if filenames and (git_commit != None): - self._exit_with_usage('It is not possible to check files and a ' - 'specific commit at the same time.') - - if output_format not in ('emacs', 'vs7'): - raise ValueError('Invalid --output value "%s": The only ' - 'allowed output formats are emacs and vs7.' % - output_format) - - all_categories = style_categories() - validate_filter_rules(user_rules, all_categories) - - verbosity = int(verbosity) - if (verbosity < 1) or (verbosity > 5): - raise ValueError('Invalid --verbose value %s: value must ' - 'be between 1-5.' % verbosity) - - filter_configuration = FilterConfiguration(base_rules=base_rules, - path_specific=_PATH_RULES_SPECIFIER, - user_rules=user_rules) - - options = ProcessorOptions(extra_flag_values=extra_flag_values, - filter_configuration=filter_configuration, - git_commit=git_commit, - max_reports_per_category=MAX_REPORTS_PER_CATEGORY, - output_format=output_format, - verbosity=verbosity) - - return (filenames, options) + """ + # If the stream does not define an "encoding" data attribute, the + # logging module can throw an error like the following: + # + # Traceback (most recent call last): + # File "/System/Library/Frameworks/Python.framework/Versions/2.6/... + # lib/python2.6/logging/__init__.py", line 761, in emit + # self.stream.write(fs % msg.encode(self.stream.encoding)) + # LookupError: unknown encoding: unknown + if logger is None: + logger = logging.getLogger() + + if is_verbose: + logging_level = logging.DEBUG + handlers = _create_debug_log_handlers(stream) + else: + logging_level = logging.INFO + handlers = _create_log_handlers(stream) + + handlers = _configure_logging(logging_level=logging_level, logger=logger, + handlers=handlers) + + return handlers # Enum-like idiom class FileType: - NONE = 1 + NONE = 0 # FileType.NONE evaluates to False. # Alphabetize remaining types - CPP = 2 + CPP = 1 + PYTHON = 2 TEXT = 3 -class ProcessorDispatcher(object): +class CheckerDispatcher(object): """Supports determining whether and how to check style, based on path.""" - cpp_file_extensions = ( - 'c', - 'cpp', - 'h', - ) - - text_file_extensions = ( - 'css', - 'html', - 'idl', - 'js', - 'mm', - 'php', - 'pm', - 'py', - 'txt', - ) - def _file_extension(self, file_path): """Return the file extension without the leading dot.""" return os.path.splitext(file_path)[1].lstrip(".") def should_skip_with_warning(self, file_path): """Return whether the given file should be skipped with a warning.""" - for skipped_file in SKIPPED_FILES_WITH_WARNING: + for skipped_file in _SKIPPED_FILES_WITH_WARNING: if file_path.find(skipped_file) >= 0: return True return False def should_skip_without_warning(self, file_path): """Return whether the given file should be skipped without a warning.""" - for skipped_file in SKIPPED_FILES_WITHOUT_WARNING: + if not self._file_type(file_path): # FileType.NONE. + return True + # Since "LayoutTests" is in _SKIPPED_FILES_WITHOUT_WARNING, make + # an exception to prevent files like "LayoutTests/ChangeLog" and + # "LayoutTests/ChangeLog-2009-06-16" from being skipped. + # + # FIXME: Figure out a good way to avoid having to add special logic + # for this special case. + if os.path.basename(file_path).startswith('ChangeLog'): + return False + for skipped_file in _SKIPPED_FILES_WITHOUT_WARNING: if file_path.find(skipped_file) >= 0: return True return False @@ -633,7 +413,7 @@ class ProcessorDispatcher(object): """Return the file type corresponding to the given file.""" file_extension = self._file_extension(file_path) - if (file_extension in self.cpp_file_extensions) or (file_path == '-'): + if (file_extension in _CPP_FILE_EXTENSIONS) or (file_path == '-'): # FIXME: Do something about the comment below and the issue it # raises since cpp_style already relies on the extension. # @@ -641,22 +421,28 @@ class ProcessorDispatcher(object): # reading from stdin, cpp_style tests should not rely on # the extension. return FileType.CPP - elif ("ChangeLog" in file_path - or "WebKitTools/Scripts/" in file_path - or file_extension in self.text_file_extensions): + elif file_extension == _PYTHON_FILE_EXTENSION: + return FileType.PYTHON + elif (os.path.basename(file_path).startswith('ChangeLog') or + (not file_extension and "WebKitTools/Scripts/" in file_path) or + file_extension in _TEXT_FILE_EXTENSIONS): return FileType.TEXT else: return FileType.NONE - def _create_processor(self, file_type, file_path, handle_style_error, verbosity): - """Instantiate and return a style processor based on file type.""" + def _create_checker(self, file_type, file_path, handle_style_error, + min_confidence): + """Instantiate and return a style checker based on file type.""" if file_type == FileType.NONE: - processor = None + checker = None elif file_type == FileType.CPP: file_extension = self._file_extension(file_path) - processor = CppProcessor(file_path, file_extension, handle_style_error, verbosity) + checker = CppChecker(file_path, file_extension, + handle_style_error, min_confidence) + elif file_type == FileType.PYTHON: + checker = PythonChecker(file_path, handle_style_error) elif file_type == FileType.TEXT: - processor = TextProcessor(file_path, handle_style_error) + checker = TextChecker(file_path, handle_style_error) else: raise ValueError('Invalid file type "%(file_type)s": the only valid file types ' "are %(NONE)s, %(CPP)s, and %(TEXT)s." @@ -665,164 +451,249 @@ class ProcessorDispatcher(object): "CPP": FileType.CPP, "TEXT": FileType.TEXT}) - return processor + return checker - def dispatch_processor(self, file_path, handle_style_error, verbosity): - """Instantiate and return a style processor based on file path.""" + def dispatch(self, file_path, handle_style_error, min_confidence): + """Instantiate and return a style checker based on file path.""" file_type = self._file_type(file_path) - processor = self._create_processor(file_type, - file_path, - handle_style_error, - verbosity) - return processor + checker = self._create_checker(file_type, + file_path, + handle_style_error, + min_confidence) + return checker -# FIXME: When creating the new CheckWebKitStyleOptions class as -# described in a FIXME above, add a new class here called -# something like CheckerConfiguration. The class should contain -# attributes for options needed to process a file. This includes -# a subset of the CheckWebKitStyleOptions attributes, a -# FilterConfiguration attribute, an stderr_write attribute, a -# max_reports_per_category attribute, etc. It can also include -# the is_reportable() method. The StyleChecker should accept -# an instance of this class rather than a ProcessorOptions -# instance. +# FIXME: Remove the stderr_write attribute from this class and replace +# its use with calls to a logging module logger. +class StyleProcessorConfiguration(object): + """Stores configuration values for the StyleProcessor class. -class StyleChecker(object): + Attributes: + min_confidence: An integer between 1 and 5 inclusive that is the + minimum confidence level of style errors to report. - """Supports checking style in files and patches. + max_reports_per_category: The maximum number of errors to report + per category, per file. - Attributes: - error_count: An integer that is the total number of reported - errors for the lifetime of this StyleChecker - instance. - options: A ProcessorOptions instance that controls the behavior - of style checking. + stderr_write: A function that takes a string as a parameter and + serves as stderr.write. """ - def __init__(self, options, stderr_write=None): - """Create a StyleChecker instance. + def __init__(self, + filter_configuration, + max_reports_per_category, + min_confidence, + output_format, + stderr_write): + """Create a StyleProcessorConfiguration instance. Args: - options: See options attribute. - stderr_write: A function that takes a string as a parameter - and that is called when a style error occurs. - Defaults to sys.stderr.write. This should be - used only for unit tests. + filter_configuration: A FilterConfiguration instance. The default + is the "empty" filter configuration, which + means that all errors should be checked. + + max_reports_per_category: The maximum number of errors to report + per category, per file. + + min_confidence: An integer between 1 and 5 inclusive that is the + minimum confidence level of style errors to report. + The default is 1, which reports all style errors. + + output_format: A string that is the output format. The supported + output formats are "emacs" which emacs can parse + and "vs7" which Microsoft Visual Studio 7 can parse. + + stderr_write: A function that takes a string as a parameter and + serves as stderr.write. """ - if stderr_write is None: - stderr_write = sys.stderr.write + self._filter_configuration = filter_configuration + self._output_format = output_format - self._stderr_write = stderr_write - self.error_count = 0 - self.options = options - - def _increment_error_count(self): - """Increment the total count of reported errors.""" - self.error_count += 1 - - def _process_file(self, processor, file_path, handle_style_error): - """Process the file using the given processor.""" - try: - # Support the UNIX convention of using "-" for stdin. Note that - # we are not opening the file with universal newline support - # (which codecs doesn't support anyway), so the resulting lines do - # contain trailing '\r' characters if we are reading a file that - # has CRLF endings. - # If after the split a trailing '\r' is present, it is removed - # below. If it is not expected to be present (i.e. os.linesep != - # '\r\n' as in Windows), a warning is issued below if this file - # is processed. - if file_path == '-': - file = codecs.StreamReaderWriter(sys.stdin, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace') - else: - file = codecs.open(file_path, 'r', 'utf8', 'replace') - - contents = file.read() - - except IOError: - self._stderr_write("Skipping input '%s': Can't open for reading\n" % file_path) - return - - lines = contents.split("\n") - - for line_number in range(len(lines)): - # FIXME: We should probably use the SVN "eol-style" property - # or a white list to decide whether or not to do - # the carriage-return check. Originally, we did the - # check only if (os.linesep != '\r\n'). - # - # FIXME: As a minor optimization, we can have - # check_no_carriage_return() return whether - # the line ends with "\r". - check_no_carriage_return(lines[line_number], line_number, - handle_style_error) - if lines[line_number].endswith("\r"): - lines[line_number] = lines[line_number].rstrip("\r") + self.max_reports_per_category = max_reports_per_category + self.min_confidence = min_confidence + self.stderr_write = stderr_write + + def is_reportable(self, category, confidence_in_error, file_path): + """Return whether an error is reportable. + + An error is reportable if both the confidence in the error is + at least the minimum confidence level and the current filter + says the category should be checked for the given path. + + Args: + category: A string that is a style category. + confidence_in_error: An integer between 1 and 5 inclusive that is + the application's confidence in the error. + A higher number means greater confidence. + file_path: The path of the file being checked + + """ + if confidence_in_error < self.min_confidence: + return False - processor.process(lines) + return self._filter_configuration.should_check(category, file_path) + + def write_style_error(self, + category, + confidence_in_error, + file_path, + line_number, + message): + """Write a style error to the configured stderr.""" + if self._output_format == 'vs7': + format_string = "%s(%s): %s [%s] [%d]\n" + else: + format_string = "%s:%s: %s [%s] [%d]\n" + + self.stderr_write(format_string % (file_path, + line_number, + message, + category, + confidence_in_error)) + + +class ProcessorBase(object): + + """The base class for processors of lists of lines.""" - def check_file(self, file_path, handle_style_error=None, process_file=None): - """Check style in the given file. + def should_process(self, file_path): + """Return whether the file at file_path should be processed. + + The TextFileReader class calls this method prior to reading in + the lines of a file. Use this method, for example, to prevent + the style checker from reading binary files into memory. + + """ + raise NotImplementedError('Subclasses should implement.') + + def process(self, lines, file_path, **kwargs): + """Process lines of text read from a file. Args: - file_path: A string that is the path of the file to process. - handle_style_error: The function to call when a style error - occurs. This parameter is meant for internal - use within this class. Defaults to a - DefaultStyleErrorHandler instance. - process_file: The function to call to process the file. This - parameter should be used only for unit tests. - Defaults to the file processing method of this class. + lines: A list of lines of text to process. + file_path: The path from which the lines were read. + **kwargs: This argument signifies that the process() method of + subclasses of ProcessorBase may support additional + keyword arguments. + For example, a style checker's check() method + may support a "reportable_lines" parameter that represents + the line numbers of the lines for which style errors + should be reported. """ - if handle_style_error is None: - handle_style_error = DefaultStyleErrorHandler(file_path, - self.options, - self._increment_error_count, - self._stderr_write) - if process_file is None: - process_file = self._process_file - - dispatcher = ProcessorDispatcher() - - if dispatcher.should_skip_without_warning(file_path): - return - if dispatcher.should_skip_with_warning(file_path): - self._stderr_write('Ignoring "%s": this file is exempt from the ' - "style guide.\n" % file_path) - return - - verbosity = self.options.verbosity - processor = dispatcher.dispatch_processor(file_path, - handle_style_error, - verbosity) - if processor is None: - return - - process_file(processor, file_path, handle_style_error) - - def check_patch(self, patch_string): - """Check style in the given patch. + raise NotImplementedError('Subclasses should implement.') + + +class StyleProcessor(ProcessorBase): + + """A ProcessorBase for checking style. + + Attributes: + error_count: An integer that is the total number of reported + errors for the lifetime of this instance. + + """ + + def __init__(self, configuration, mock_dispatcher=None, + mock_increment_error_count=None, + mock_carriage_checker_class=None): + """Create an instance. Args: - patch_string: A string that is a patch string. + configuration: A StyleProcessorConfiguration instance. + mock_dispatcher: A mock CheckerDispatcher instance. This + parameter is for unit testing. Defaults to a + CheckerDispatcher instance. + mock_increment_error_count: A mock error-count incrementer. + mock_carriage_checker_class: A mock class for checking and + transforming carriage returns. + This parameter is for unit testing. + Defaults to CarriageReturnChecker. """ - patch_files = parse_patch(patch_string) - for file_path, diff in patch_files.iteritems(): - style_error_handler = PatchStyleErrorHandler(diff, - file_path, - self.options, - self._increment_error_count, - self._stderr_write) + if mock_dispatcher is None: + dispatcher = CheckerDispatcher() + else: + dispatcher = mock_dispatcher + + if mock_increment_error_count is None: + # The following blank line is present to avoid flagging by pep8.py. + + def increment_error_count(): + """Increment the total count of reported errors.""" + self.error_count += 1 + else: + increment_error_count = mock_increment_error_count + + if mock_carriage_checker_class is None: + # This needs to be a class rather than an instance since the + # process() method instantiates one using parameters. + carriage_checker_class = CarriageReturnChecker + else: + carriage_checker_class = mock_carriage_checker_class + + self.error_count = 0 + + self._carriage_checker_class = carriage_checker_class + self._configuration = configuration + self._dispatcher = dispatcher + self._increment_error_count = increment_error_count + + def should_process(self, file_path): + """Return whether the file should be checked for style.""" + if self._dispatcher.should_skip_without_warning(file_path): + return False + if self._dispatcher.should_skip_with_warning(file_path): + _log.warn('File exempt from style guide. Skipping: "%s"' + % file_path) + return False + return True + + def process(self, lines, file_path, line_numbers=None): + """Check the given lines for style. + + Arguments: + lines: A list of all lines in the file to check. + file_path: The path of the file to process. If possible, the path + should be relative to the source root. Otherwise, + path-specific logic may not behave as expected. + line_numbers: A list of line numbers of the lines for which + style errors should be reported, or None if errors + for all lines should be reported. When not None, this + list normally contains the line numbers corresponding + to the modified lines of a patch. + + """ + _log.debug("Checking style: " + file_path) + + style_error_handler = DefaultStyleErrorHandler( + configuration=self._configuration, + file_path=file_path, + increment_error_count=self._increment_error_count, + line_numbers=line_numbers) + + carriage_checker = self._carriage_checker_class(style_error_handler) + + # FIXME: We should probably use the SVN "eol-style" property + # or a white list to decide whether or not to do + # the carriage-return check. Originally, we did the + # check only if (os.linesep != '\r\n'). + # + # Check for and remove trailing carriage returns ("\r"). + lines = carriage_checker.check(lines) + + min_confidence = self._configuration.min_confidence + checker = self._dispatcher.dispatch(file_path, + style_error_handler, + min_confidence) + + if checker is None: + raise AssertionError("File should not be checked: '%s'" % file_path) - self.check_file(file_path, style_error_handler) + _log.debug("Using class: " + checker.__class__.__name__) + checker.check(lines) diff --git a/WebKitTools/Scripts/webkitpy/style/checker_unittest.py b/WebKitTools/Scripts/webkitpy/style/checker_unittest.py index e1c9baf..5254275 100755 --- a/WebKitTools/Scripts/webkitpy/style/checker_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/checker_unittest.py @@ -34,103 +34,119 @@ """Unit tests for style.py.""" +import logging +import os import unittest import checker as style +from webkitpy.style_references import LogTesting +from webkitpy.style_references import TestLogStream +from checker import _BASE_FILTER_RULES +from checker import _MAX_REPORTS_PER_CATEGORY from checker import _PATH_RULES_SPECIFIER as PATH_RULES_SPECIFIER -from checker import style_categories -from checker import ProcessorDispatcher -from checker import ProcessorOptions -from checker import StyleChecker +from checker import _all_categories +from checker import check_webkit_style_configuration +from checker import check_webkit_style_parser +from checker import configure_logging +from checker import CheckerDispatcher +from checker import ProcessorBase +from checker import StyleProcessor +from checker import StyleProcessorConfiguration +from checkers.cpp import CppChecker +from checkers.python import PythonChecker +from checkers.text import TextChecker +from error_handlers import DefaultStyleErrorHandler from filter import validate_filter_rules from filter import FilterConfiguration -from processors.cpp import CppProcessor -from processors.text import TextProcessor +from optparser import ArgumentParser +from optparser import CommandOptionValues +from webkitpy.common.system.logtesting import LoggingTestCase +from webkitpy.style.filereader import TextFileReader -class ProcessorOptionsTest(unittest.TestCase): +class ConfigureLoggingTestBase(unittest.TestCase): - """Tests ProcessorOptions class.""" + """Base class for testing configure_logging(). - def test_init(self): - """Test __init__ constructor.""" - # Check default parameters. - options = ProcessorOptions() - self.assertEquals(options.extra_flag_values, {}) - self.assertEquals(options.filter_configuration, FilterConfiguration()) - self.assertEquals(options.git_commit, None) - self.assertEquals(options.max_reports_per_category, {}) - self.assertEquals(options.output_format, "emacs") - self.assertEquals(options.verbosity, 1) - - # Check argument validation. - self.assertRaises(ValueError, ProcessorOptions, output_format="bad") - ProcessorOptions(output_format="emacs") # No ValueError: works - ProcessorOptions(output_format="vs7") # works - self.assertRaises(ValueError, ProcessorOptions, verbosity=0) - self.assertRaises(ValueError, ProcessorOptions, verbosity=6) - ProcessorOptions(verbosity=1) # works - ProcessorOptions(verbosity=5) # works - - # Check attributes. - filter_configuration = FilterConfiguration(base_rules=["+"]) - options = ProcessorOptions(extra_flag_values={"extra_value" : 2}, - filter_configuration=filter_configuration, - git_commit="commit", - max_reports_per_category={"category": 3}, - output_format="vs7", - verbosity=3) - self.assertEquals(options.extra_flag_values, {"extra_value" : 2}) - self.assertEquals(options.filter_configuration, filter_configuration) - self.assertEquals(options.git_commit, "commit") - self.assertEquals(options.max_reports_per_category, {"category": 3}) - self.assertEquals(options.output_format, "vs7") - self.assertEquals(options.verbosity, 3) - - def test_eq(self): - """Test __eq__ equality function.""" - # == calls __eq__. - self.assertTrue(ProcessorOptions() == ProcessorOptions()) - - # Verify that a difference in any argument causes equality to fail. - filter_configuration = FilterConfiguration(base_rules=["+"]) - options = ProcessorOptions(extra_flag_values={"extra_value" : 1}, - filter_configuration=filter_configuration, - git_commit="commit", - max_reports_per_category={"category": 3}, - output_format="vs7", - verbosity=1) - self.assertFalse(options == ProcessorOptions(extra_flag_values={"extra_value" : 2})) - new_config = FilterConfiguration(base_rules=["-"]) - self.assertFalse(options == - ProcessorOptions(filter_configuration=new_config)) - self.assertFalse(options == ProcessorOptions(git_commit="commit2")) - self.assertFalse(options == ProcessorOptions(max_reports_per_category= - {"category": 2})) - self.assertFalse(options == ProcessorOptions(output_format="emacs")) - self.assertFalse(options == ProcessorOptions(verbosity=2)) - - def test_ne(self): - """Test __ne__ inequality function.""" - # != calls __ne__. - # By default, __ne__ always returns true on different objects. - # Thus, just check the distinguishing case to verify that the - # code defines __ne__. - self.assertFalse(ProcessorOptions() != ProcessorOptions()) + Sub-classes should implement: - def test_is_reportable(self): - """Test is_reportable().""" - filter_configuration = FilterConfiguration(base_rules=["-xyz"]) - options = ProcessorOptions(filter_configuration=filter_configuration, - verbosity=3) + is_verbose: The is_verbose value to pass to configure_logging(). + + """ + + def setUp(self): + is_verbose = self.is_verbose + + log_stream = TestLogStream(self) + + # Use a logger other than the root logger or one prefixed with + # webkit so as not to conflict with test-webkitpy logging. + logger = logging.getLogger("unittest") + + # Configure the test logger not to pass messages along to the + # root logger. This prevents test messages from being + # propagated to loggers used by test-webkitpy logging (e.g. + # the root logger). + logger.propagate = False + + self._handlers = configure_logging(stream=log_stream, logger=logger, + is_verbose=is_verbose) + self._log = logger + self._log_stream = log_stream + + def tearDown(self): + """Reset logging to its original state. + + This method ensures that the logging configuration set up + for a unit test does not affect logging in other unit tests. + + """ + logger = self._log + for handler in self._handlers: + logger.removeHandler(handler) + + def assert_log_messages(self, messages): + """Assert that the logged messages equal the given messages.""" + self._log_stream.assertMessages(messages) + + +class ConfigureLoggingTest(ConfigureLoggingTestBase): + + """Tests the configure_logging() function.""" + + is_verbose = False + + def test_warning_message(self): + self._log.warn("test message") + self.assert_log_messages(["WARNING: test message\n"]) + + def test_below_warning_message(self): + # We test the boundary case of a logging level equal to 29. + # In practice, we will probably only be calling log.info(), + # which corresponds to a logging level of 20. + level = logging.WARNING - 1 # Equals 29. + self._log.log(level, "test message") + self.assert_log_messages(["test message\n"]) + + def test_debug_message(self): + self._log.debug("test message") + self.assert_log_messages([]) + + def test_two_messages(self): + self._log.info("message1") + self._log.info("message2") + self.assert_log_messages(["message1\n", "message2\n"]) + + +class ConfigureLoggingVerboseTest(ConfigureLoggingTestBase): - # Test verbosity - self.assertTrue(options.is_reportable("abc", 3, "foo.h")) - self.assertFalse(options.is_reportable("abc", 2, "foo.h")) + """Tests the configure_logging() function with is_verbose True.""" - # Test filter - self.assertTrue(options.is_reportable("xy", 3, "foo.h")) - self.assertFalse(options.is_reportable("xyz", 3, "foo.h")) + is_verbose = True + + def test_debug_message(self): + self._log.debug("test message") + self.assert_log_messages(["unittest: DEBUG test message\n"]) class GlobalVariablesTest(unittest.TestCase): @@ -138,18 +154,18 @@ class GlobalVariablesTest(unittest.TestCase): """Tests validity of the global variables.""" def _all_categories(self): - return style.style_categories() + return _all_categories() def defaults(self): - return style.webkit_argument_defaults() + return style._check_webkit_style_defaults() - def test_filter_rules(self): + def test_webkit_base_filter_rules(self): + base_filter_rules = _BASE_FILTER_RULES defaults = self.defaults() already_seen = [] - validate_filter_rules(defaults.base_filter_rules, - self._all_categories()) + validate_filter_rules(base_filter_rules, self._all_categories()) # Also do some additional checks. - for rule in defaults.base_filter_rules: + for rule in base_filter_rules: # Check no leading or trailing white space. self.assertEquals(rule, rule.strip()) # All categories are on by default, so defaults should @@ -161,274 +177,186 @@ class GlobalVariablesTest(unittest.TestCase): def test_defaults(self): """Check that default arguments are valid.""" - defaults = self.defaults() + default_options = self.defaults() # FIXME: We should not need to call parse() to determine # whether the default arguments are valid. - parser = style.ArgumentParser(defaults) + parser = ArgumentParser(all_categories=self._all_categories(), + base_filter_rules=[], + default_options=default_options) # No need to test the return value here since we test parse() # on valid arguments elsewhere. - parser.parse([]) # arguments valid: no error or SystemExit + # + # The default options are valid: no error or SystemExit. + parser.parse(args=[]) def test_path_rules_specifier(self): - all_categories = style_categories() + all_categories = self._all_categories() for (sub_paths, path_rules) in PATH_RULES_SPECIFIER: - self.assertTrue(isinstance(path_rules, tuple), - "Checking: " + str(path_rules)) validate_filter_rules(path_rules, self._all_categories()) - # Try using the path specifier (as an "end-to-end" check). config = FilterConfiguration(path_specific=PATH_RULES_SPECIFIER) - self.assertTrue(config.should_check("xxx_any_category", - "xxx_non_matching_path")) - self.assertTrue(config.should_check("xxx_any_category", - "WebKitTools/WebKitAPITest/")) - self.assertFalse(config.should_check("build/include", - "WebKitTools/WebKitAPITest/")) - self.assertFalse(config.should_check("readability/naming", - "WebKit/qt/tests/qwebelement/tst_qwebelement.cpp")) + + def assertCheck(path, category): + """Assert that the given category should be checked.""" + message = ('Should check category "%s" for path "%s".' + % (category, path)) + self.assertTrue(config.should_check(category, path)) + + def assertNoCheck(path, category): + """Assert that the given category should not be checked.""" + message = ('Should not check category "%s" for path "%s".' + % (category, path)) + self.assertFalse(config.should_check(category, path), message) + + assertCheck("random_path.cpp", + "build/include") + assertNoCheck("WebKitTools/WebKitAPITest/main.cpp", + "build/include") + assertNoCheck("WebKit/qt/QGVLauncher/main.cpp", + "build/include") + assertNoCheck("WebKit/qt/QGVLauncher/main.cpp", + "readability/streams") + + assertCheck("random_path.cpp", + "readability/naming") + assertNoCheck("WebKit/gtk/webkit/webkit.h", + "readability/naming") + assertNoCheck("WebKitTools/DumpRenderTree/gtk/DumpRenderTree.cpp", + "readability/null") + assertNoCheck("WebKit/efl/ewk/ewk_view.h", + "readability/naming") + assertNoCheck("WebCore/css/CSSParser.cpp", + "readability/naming") + assertNoCheck("WebKit/qt/tests/qwebelement/tst_qwebelement.cpp", + "readability/naming") + assertNoCheck( + "JavaScriptCore/qt/tests/qscriptengine/tst_qscriptengine.cpp", + "readability/naming") + assertNoCheck("WebCore/ForwardingHeaders/debugger/Debugger.h", + "build/header_guard") + + # Third-party Python code: webkitpy/thirdparty + path = "WebKitTools/Scripts/webkitpy/thirdparty/mock.py" + assertNoCheck(path, "build/include") + assertNoCheck(path, "pep8/E401") # A random pep8 category. + assertCheck(path, "pep8/W191") + assertCheck(path, "pep8/W291") + assertCheck(path, "whitespace/carriage_return") def test_max_reports_per_category(self): - """Check that MAX_REPORTS_PER_CATEGORY is valid.""" + """Check that _MAX_REPORTS_PER_CATEGORY is valid.""" all_categories = self._all_categories() - for category in style.MAX_REPORTS_PER_CATEGORY.iterkeys(): + for category in _MAX_REPORTS_PER_CATEGORY.iterkeys(): self.assertTrue(category in all_categories, 'Key "%s" is not a category' % category) -class ArgumentPrinterTest(unittest.TestCase): - - """Tests the ArgumentPrinter class.""" - - _printer = style.ArgumentPrinter() - - def _create_options(self, - output_format='emacs', - verbosity=3, - user_rules=[], - git_commit=None, - extra_flag_values={}): - filter_configuration = FilterConfiguration(user_rules=user_rules) - return style.ProcessorOptions(extra_flag_values=extra_flag_values, - filter_configuration=filter_configuration, - git_commit=git_commit, - output_format=output_format, - verbosity=verbosity) +class CheckWebKitStyleFunctionTest(unittest.TestCase): + + """Tests the functions with names of the form check_webkit_style_*.""" + + def test_check_webkit_style_configuration(self): + # Exercise the code path to make sure the function does not error out. + option_values = CommandOptionValues() + configuration = check_webkit_style_configuration(option_values) - def test_to_flag_string(self): - options = self._create_options('vs7', 5, ['+foo', '-bar'], 'git', - {'a': 0, 'z': 1}) - self.assertEquals('--a=0 --filter=+foo,-bar --git-commit=git ' - '--output=vs7 --verbose=5 --z=1', - self._printer.to_flag_string(options)) - - # This is to check that --filter and --git-commit do not - # show up when not user-specified. - options = self._create_options() - self.assertEquals('--output=emacs --verbose=3', - self._printer.to_flag_string(options)) - - -class ArgumentParserTest(unittest.TestCase): - - """Test the ArgumentParser class.""" - - def _parse(self): - """Return a default parse() function for testing.""" - return self._create_parser().parse - - def _create_defaults(self, default_output_format='vs7', - default_verbosity=3, - default_filter_rules=['-', '+whitespace']): - """Return a default ArgumentDefaults instance for testing.""" - return style.ArgumentDefaults(default_output_format, - default_verbosity, - default_filter_rules) - - def _create_parser(self, defaults=None): - """Return an ArgumentParser instance for testing.""" - def create_usage(_defaults): - """Return a usage string for testing.""" - return "usage" - - def doc_print(message): - # We do not want the usage string or style categories - # to print during unit tests, so print nothing. - return - - if defaults is None: - defaults = self._create_defaults() - - return style.ArgumentParser(defaults, create_usage, doc_print) - - def test_parse_documentation(self): - parse = self._parse() - - # FIXME: Test both the printing of the usage string and the - # filter categories help. - - # Request the usage string. - self.assertRaises(SystemExit, parse, ['--help']) - # Request default filter rules and available style categories. - self.assertRaises(SystemExit, parse, ['--filter=']) - - def test_parse_bad_values(self): - parse = self._parse() - - # Pass an unsupported argument. - self.assertRaises(SystemExit, parse, ['--bad']) - - self.assertRaises(ValueError, parse, ['--verbose=bad']) - self.assertRaises(ValueError, parse, ['--verbose=0']) - self.assertRaises(ValueError, parse, ['--verbose=6']) - parse(['--verbose=1']) # works - parse(['--verbose=5']) # works - - self.assertRaises(ValueError, parse, ['--output=bad']) - parse(['--output=vs7']) # works - - # Pass a filter rule not beginning with + or -. - self.assertRaises(ValueError, parse, ['--filter=build']) - parse(['--filter=+build']) # works - # Pass files and git-commit at the same time. - self.assertRaises(SystemExit, parse, ['--git-commit=', 'file.txt']) - # Pass an extra flag already supported. - self.assertRaises(ValueError, parse, [], ['filter=']) - parse([], ['extra=']) # works - # Pass an extra flag with typo. - self.assertRaises(SystemExit, parse, ['--extratypo='], ['extra=']) - parse(['--extra='], ['extra=']) # works - self.assertRaises(ValueError, parse, [], ['extra=', 'extra=']) - - - def test_parse_default_arguments(self): - parse = self._parse() - - (files, options) = parse([]) - - self.assertEquals(files, []) - - self.assertEquals(options.output_format, 'vs7') - self.assertEquals(options.verbosity, 3) - self.assertEquals(options.filter_configuration, - FilterConfiguration(base_rules=["-", "+whitespace"], - path_specific=PATH_RULES_SPECIFIER)) - self.assertEquals(options.git_commit, None) - - def test_parse_explicit_arguments(self): - parse = self._parse() - - # Pass non-default explicit values. - (files, options) = parse(['--output=emacs']) - self.assertEquals(options.output_format, 'emacs') - (files, options) = parse(['--verbose=4']) - self.assertEquals(options.verbosity, 4) - (files, options) = parse(['--git-commit=commit']) - self.assertEquals(options.git_commit, 'commit') + def test_check_webkit_style_parser(self): + # Exercise the code path to make sure the function does not error out. + parser = check_webkit_style_parser() - # Pass user_rules. - (files, options) = parse(['--filter=+build,-whitespace']) - config = options.filter_configuration - self.assertEquals(options.filter_configuration, - FilterConfiguration(base_rules=["-", "+whitespace"], - path_specific=PATH_RULES_SPECIFIER, - user_rules=["+build", "-whitespace"])) - - # Pass spurious white space in user rules. - (files, options) = parse(['--filter=+build, -whitespace']) - self.assertEquals(options.filter_configuration, - FilterConfiguration(base_rules=["-", "+whitespace"], - path_specific=PATH_RULES_SPECIFIER, - user_rules=["+build", "-whitespace"])) - - # Pass extra flag values. - (files, options) = parse(['--extra'], ['extra']) - self.assertEquals(options.extra_flag_values, {'--extra': ''}) - (files, options) = parse(['--extra='], ['extra=']) - self.assertEquals(options.extra_flag_values, {'--extra': ''}) - (files, options) = parse(['--extra=x'], ['extra=']) - self.assertEquals(options.extra_flag_values, {'--extra': 'x'}) - - def test_parse_files(self): - parse = self._parse() - - (files, options) = parse(['foo.cpp']) - self.assertEquals(files, ['foo.cpp']) - # Pass multiple files. - (files, options) = parse(['--output=emacs', 'foo.cpp', 'bar.cpp']) - self.assertEquals(files, ['foo.cpp', 'bar.cpp']) +class CheckerDispatcherSkipTest(unittest.TestCase): + """Tests the "should skip" methods of the CheckerDispatcher class.""" + + def setUp(self): + self._dispatcher = CheckerDispatcher() -class ProcessorDispatcherSkipTest(unittest.TestCase): - - """Tests the "should skip" methods of the ProcessorDispatcher class.""" - def test_should_skip_with_warning(self): """Test should_skip_with_warning().""" - dispatcher = ProcessorDispatcher() - # Check a non-skipped file. - self.assertFalse(dispatcher.should_skip_with_warning("foo.txt")) + self.assertFalse(self._dispatcher.should_skip_with_warning("foo.txt")) # Check skipped files. paths_to_skip = [ "gtk2drawing.c", - "gtk2drawing.h", + "gtkdrawing.h", "JavaScriptCore/qt/api/qscriptengine_p.h", "WebCore/platform/gtk/gtk2drawing.c", - "WebCore/platform/gtk/gtk2drawing.h", + "WebCore/platform/gtk/gtkdrawing.h", "WebKit/gtk/tests/testatk.c", "WebKit/qt/Api/qwebpage.h", "WebKit/qt/tests/qwebsecurityorigin/tst_qwebsecurityorigin.cpp", ] for path in paths_to_skip: - self.assertTrue(dispatcher.should_skip_with_warning(path), + self.assertTrue(self._dispatcher.should_skip_with_warning(path), "Checking: " + path) - def test_should_skip_without_warning(self): - """Test should_skip_without_warning().""" - dispatcher = ProcessorDispatcher() - - # Check a non-skipped file. - self.assertFalse(dispatcher.should_skip_without_warning("foo.txt")) - - # Check skipped files. - paths_to_skip = [ - # LayoutTests folder - "LayoutTests/foo.txt", - ] + def _assert_should_skip_without_warning(self, path, is_checker_none, + expected): + # Check the file type before asserting the return value. + checker = self._dispatcher.dispatch(file_path=path, + handle_style_error=None, + min_confidence=3) + message = 'while checking: %s' % path + self.assertEquals(checker is None, is_checker_none, message) + self.assertEquals(self._dispatcher.should_skip_without_warning(path), + expected, message) + + def test_should_skip_without_warning__true(self): + """Test should_skip_without_warning() for True return values.""" + # Check a file with NONE file type. + path = 'foo.asdf' # Non-sensical file extension. + self._assert_should_skip_without_warning(path, + is_checker_none=True, + expected=True) + + # Check files with non-NONE file type. These examples must be + # drawn from the _SKIPPED_FILES_WITHOUT_WARNING configuration + # variable. + path = os.path.join('LayoutTests', 'foo.txt') + self._assert_should_skip_without_warning(path, + is_checker_none=False, + expected=True) + + def test_should_skip_without_warning__false(self): + """Test should_skip_without_warning() for False return values.""" + paths = ['foo.txt', + os.path.join('LayoutTests', 'ChangeLog'), + ] - for path in paths_to_skip: - self.assertTrue(dispatcher.should_skip_without_warning(path), - "Checking: " + path) + for path in paths: + self._assert_should_skip_without_warning(path, + is_checker_none=False, + expected=False) -class ProcessorDispatcherDispatchTest(unittest.TestCase): +class CheckerDispatcherDispatchTest(unittest.TestCase): - """Tests dispatch_processor() method of ProcessorDispatcher class.""" + """Tests dispatch() method of CheckerDispatcher class.""" def mock_handle_style_error(self): pass - def dispatch_processor(self, file_path): - """Call dispatch_processor() with the given file path.""" - dispatcher = ProcessorDispatcher() - processor = dispatcher.dispatch_processor(file_path, - self.mock_handle_style_error, - verbosity=3) - return processor - - def assert_processor_none(self, file_path): - """Assert that the dispatched processor is None.""" - processor = self.dispatch_processor(file_path) - self.assertTrue(processor is None, 'Checking: "%s"' % file_path) - - def assert_processor(self, file_path, expected_class): - """Assert the type of the dispatched processor.""" - processor = self.dispatch_processor(file_path) - got_class = processor.__class__ + def dispatch(self, file_path): + """Call dispatch() with the given file path.""" + dispatcher = CheckerDispatcher() + checker = dispatcher.dispatch(file_path, + self.mock_handle_style_error, + min_confidence=3) + return checker + + def assert_checker_none(self, file_path): + """Assert that the dispatched checker is None.""" + checker = self.dispatch(file_path) + self.assertTrue(checker is None, 'Checking: "%s"' % file_path) + + def assert_checker(self, file_path, expected_class): + """Assert the type of the dispatched checker.""" + checker = self.dispatch(file_path) + got_class = checker.__class__ self.assertEquals(got_class, expected_class, 'For path "%(file_path)s" got %(got_class)s when ' "expecting %(expected_class)s." @@ -436,13 +364,17 @@ class ProcessorDispatcherDispatchTest(unittest.TestCase): "got_class": got_class, "expected_class": expected_class}) - def assert_processor_cpp(self, file_path): - """Assert that the dispatched processor is a CppProcessor.""" - self.assert_processor(file_path, CppProcessor) + def assert_checker_cpp(self, file_path): + """Assert that the dispatched checker is a CppChecker.""" + self.assert_checker(file_path, CppChecker) + + def assert_checker_python(self, file_path): + """Assert that the dispatched checker is a PythonChecker.""" + self.assert_checker(file_path, PythonChecker) - def assert_processor_text(self, file_path): - """Assert that the dispatched processor is a TextProcessor.""" - self.assert_processor(file_path, TextProcessor) + def assert_checker_text(self, file_path): + """Assert that the dispatched checker is a TextChecker.""" + self.assert_checker(file_path, TextChecker) def test_cpp_paths(self): """Test paths that should be checked as C++.""" @@ -454,228 +386,384 @@ class ProcessorDispatcherDispatchTest(unittest.TestCase): ] for path in paths: - self.assert_processor_cpp(path) + self.assert_checker_cpp(path) - # Check processor attributes on a typical input. + # Check checker attributes on a typical input. file_base = "foo" file_extension = "c" file_path = file_base + "." + file_extension - self.assert_processor_cpp(file_path) - processor = self.dispatch_processor(file_path) - self.assertEquals(processor.file_extension, file_extension) - self.assertEquals(processor.file_path, file_path) - self.assertEquals(processor.handle_style_error, self.mock_handle_style_error) - self.assertEquals(processor.verbosity, 3) + self.assert_checker_cpp(file_path) + checker = self.dispatch(file_path) + self.assertEquals(checker.file_extension, file_extension) + self.assertEquals(checker.file_path, file_path) + self.assertEquals(checker.handle_style_error, self.mock_handle_style_error) + self.assertEquals(checker.min_confidence, 3) # Check "-" for good measure. file_base = "-" file_extension = "" file_path = file_base - self.assert_processor_cpp(file_path) - processor = self.dispatch_processor(file_path) - self.assertEquals(processor.file_extension, file_extension) - self.assertEquals(processor.file_path, file_path) + self.assert_checker_cpp(file_path) + checker = self.dispatch(file_path) + self.assertEquals(checker.file_extension, file_extension) + self.assertEquals(checker.file_path, file_path) + + def test_python_paths(self): + """Test paths that should be checked as Python.""" + paths = [ + "foo.py", + "WebKitTools/Scripts/modules/text_style.py", + ] + + for path in paths: + self.assert_checker_python(path) + + # Check checker attributes on a typical input. + file_base = "foo" + file_extension = "css" + file_path = file_base + "." + file_extension + self.assert_checker_text(file_path) + checker = self.dispatch(file_path) + self.assertEquals(checker.file_path, file_path) + self.assertEquals(checker.handle_style_error, + self.mock_handle_style_error) def test_text_paths(self): """Test paths that should be checked as text.""" paths = [ "ChangeLog", + "ChangeLog-2009-06-16", + "foo.ac", + "foo.cc", + "foo.cgi", "foo.css", + "foo.exp", + "foo.flex", + "foo.gyp", + "foo.gypi", "foo.html", "foo.idl", + "foo.in", "foo.js", "foo.mm", "foo.php", + "foo.pl", "foo.pm", - "foo.py", + "foo.pri", + "foo.pro", + "foo.rb", + "foo.sh", "foo.txt", - "FooChangeLog.bak", - "WebCore/ChangeLog", - "WebCore/inspector/front-end/inspector.js", - "WebKitTools/Scripts/check-webkit=style", - "WebKitTools/Scripts/modules/text_style.py", - ] + "foo.wm", + "foo.xhtml", + "foo.y", + os.path.join("WebCore", "ChangeLog"), + os.path.join("WebCore", "inspector", "front-end", "inspector.js"), + os.path.join("WebKitTools", "Scripts", "check-webkit-style"), + ] for path in paths: - self.assert_processor_text(path) + self.assert_checker_text(path) - # Check processor attributes on a typical input. + # Check checker attributes on a typical input. file_base = "foo" file_extension = "css" file_path = file_base + "." + file_extension - self.assert_processor_text(file_path) - processor = self.dispatch_processor(file_path) - self.assertEquals(processor.file_path, file_path) - self.assertEquals(processor.handle_style_error, self.mock_handle_style_error) + self.assert_checker_text(file_path) + checker = self.dispatch(file_path) + self.assertEquals(checker.file_path, file_path) + self.assertEquals(checker.handle_style_error, self.mock_handle_style_error) def test_none_paths(self): """Test paths that have no file type..""" paths = [ "Makefile", + "foo.asdf", # Non-sensical file extension. "foo.png", "foo.exe", + "foo.vcproj", ] for path in paths: - self.assert_processor_none(path) + self.assert_checker_none(path) -class StyleCheckerTest(unittest.TestCase): +class StyleProcessorConfigurationTest(unittest.TestCase): - """Test the StyleChecker class. + """Tests the StyleProcessorConfiguration class.""" - Attributes: - error_messages: A string containing all of the warning messages - written to the mock_stderr_write method of - this class. - - """ + def setUp(self): + self._error_messages = [] + """The messages written to _mock_stderr_write() of this class.""" def _mock_stderr_write(self, message): - pass + self._error_messages.append(message) + + def _style_checker_configuration(self, output_format="vs7"): + """Return a StyleProcessorConfiguration instance for testing.""" + base_rules = ["-whitespace", "+whitespace/tab"] + filter_configuration = FilterConfiguration(base_rules=base_rules) - def _style_checker(self, options): - return StyleChecker(options, self._mock_stderr_write) + return StyleProcessorConfiguration( + filter_configuration=filter_configuration, + max_reports_per_category={"whitespace/newline": 1}, + min_confidence=3, + output_format=output_format, + stderr_write=self._mock_stderr_write) + + def test_init(self): + """Test the __init__() method.""" + configuration = self._style_checker_configuration() + + # Check that __init__ sets the "public" data attributes correctly. + self.assertEquals(configuration.max_reports_per_category, + {"whitespace/newline": 1}) + self.assertEquals(configuration.stderr_write, self._mock_stderr_write) + self.assertEquals(configuration.min_confidence, 3) + + def test_is_reportable(self): + """Test the is_reportable() method.""" + config = self._style_checker_configuration() + + self.assertTrue(config.is_reportable("whitespace/tab", 3, "foo.txt")) + + # Test the confidence check code path by varying the confidence. + self.assertFalse(config.is_reportable("whitespace/tab", 2, "foo.txt")) + + # Test the category check code path by varying the category. + self.assertFalse(config.is_reportable("whitespace/line", 4, "foo.txt")) + + def _call_write_style_error(self, output_format): + config = self._style_checker_configuration(output_format=output_format) + config.write_style_error(category="whitespace/tab", + confidence_in_error=5, + file_path="foo.h", + line_number=100, + message="message") + + def test_write_style_error_emacs(self): + """Test the write_style_error() method.""" + self._call_write_style_error("emacs") + self.assertEquals(self._error_messages, + ["foo.h:100: message [whitespace/tab] [5]\n"]) + + def test_write_style_error_vs7(self): + """Test the write_style_error() method.""" + self._call_write_style_error("vs7") + self.assertEquals(self._error_messages, + ["foo.h(100): message [whitespace/tab] [5]\n"]) + + +class StyleProcessor_EndToEndTest(LoggingTestCase): + + """Test the StyleProcessor class with an emphasis on end-to-end tests.""" + + def setUp(self): + LoggingTestCase.setUp(self) + self._messages = [] + + def _mock_stderr_write(self, message): + """Save a message so it can later be asserted.""" + self._messages.append(message) def test_init(self): """Test __init__ constructor.""" - options = ProcessorOptions() - style_checker = self._style_checker(options) + configuration = StyleProcessorConfiguration( + filter_configuration=FilterConfiguration(), + max_reports_per_category={}, + min_confidence=3, + output_format="vs7", + stderr_write=self._mock_stderr_write) + processor = StyleProcessor(configuration) - self.assertEquals(style_checker.error_count, 0) - self.assertEquals(style_checker.options, options) + self.assertEquals(processor.error_count, 0) + self.assertEquals(self._messages, []) + def test_process(self): + configuration = StyleProcessorConfiguration( + filter_configuration=FilterConfiguration(), + max_reports_per_category={}, + min_confidence=3, + output_format="vs7", + stderr_write=self._mock_stderr_write) + processor = StyleProcessor(configuration) -class StyleCheckerCheckFileTest(unittest.TestCase): + processor.process(lines=['line1', 'Line with tab:\t'], + file_path='foo.txt') + self.assertEquals(processor.error_count, 1) + expected_messages = ['foo.txt(2): Line contains tab character. ' + '[whitespace/tab] [5]\n'] + self.assertEquals(self._messages, expected_messages) - """Test the check_file() method of the StyleChecker class. - The check_file() method calls its process_file parameter when - given a file that should not be skipped. +class StyleProcessor_CodeCoverageTest(LoggingTestCase): - The "got_*" attributes of this class are the parameters passed - to process_file by calls to check_file() made by this test - class. These attributes allow us to check the parameter values - passed internally to the process_file function. + """Test the StyleProcessor class with an emphasis on code coverage. - Attributes: - got_file_path: The file_path parameter passed by check_file() - to its process_file parameter. - got_handle_style_error: The handle_style_error parameter passed - by check_file() to its process_file - parameter. - got_processor: The processor parameter passed by check_file() to - its process_file parameter. - warning_messages: A string containing all of the warning messages - written to the mock_stderr_write method of - this class. + This class makes heavy use of mock objects. """ - def setUp(self): - self.got_file_path = None - self.got_handle_style_error = None - self.got_processor = None - self.warning_messages = "" - def mock_stderr_write(self, warning_message): - self.warning_messages += warning_message + class MockDispatchedChecker(object): - def mock_handle_style_error(self): + """A mock checker dispatched by the MockDispatcher.""" + + def __init__(self, file_path, min_confidence, style_error_handler): + self.file_path = file_path + self.min_confidence = min_confidence + self.style_error_handler = style_error_handler + + def check(self, lines): + self.lines = lines + + class MockDispatcher(object): + + """A mock CheckerDispatcher class.""" + + def __init__(self): + self.dispatched_checker = None + + def should_skip_with_warning(self, file_path): + return file_path.endswith('skip_with_warning.txt') + + def should_skip_without_warning(self, file_path): + return file_path.endswith('skip_without_warning.txt') + + def dispatch(self, file_path, style_error_handler, min_confidence): + if file_path.endswith('do_not_process.txt'): + return None + + checker = StyleProcessor_CodeCoverageTest.MockDispatchedChecker( + file_path, + min_confidence, + style_error_handler) + + # Save the dispatched checker so the current test case has a + # way to access and check it. + self.dispatched_checker = checker + + return checker + + def setUp(self): + LoggingTestCase.setUp(self) + # We can pass an error-message swallower here because error message + # output is tested instead in the end-to-end test case above. + configuration = StyleProcessorConfiguration( + filter_configuration=FilterConfiguration(), + max_reports_per_category={"whitespace/newline": 1}, + min_confidence=3, + output_format="vs7", + stderr_write=self._swallow_stderr_message) + + mock_carriage_checker_class = self._create_carriage_checker_class() + mock_dispatcher = self.MockDispatcher() + # We do not need to use a real incrementer here because error-count + # incrementing is tested instead in the end-to-end test case above. + mock_increment_error_count = self._do_nothing + + processor = StyleProcessor(configuration=configuration, + mock_carriage_checker_class=mock_carriage_checker_class, + mock_dispatcher=mock_dispatcher, + mock_increment_error_count=mock_increment_error_count) + + self._configuration = configuration + self._mock_dispatcher = mock_dispatcher + self._processor = processor + + def _do_nothing(self): + # We provide this function so the caller can pass it to the + # StyleProcessor constructor. This lets us assert the equality of + # the DefaultStyleErrorHandler instance generated by the process() + # method with an expected instance. pass - def mock_process_file(self, processor, file_path, handle_style_error): - """A mock _process_file(). + def _swallow_stderr_message(self, message): + """Swallow a message passed to stderr.write().""" + # This is a mock stderr.write() for passing to the constructor + # of the StyleProcessorConfiguration class. + pass - See the documentation for this class for more information - on this function. + def _create_carriage_checker_class(self): - """ - self.got_file_path = file_path - self.got_handle_style_error = handle_style_error - self.got_processor = processor - - def assert_attributes(self, - expected_file_path, - expected_handle_style_error, - expected_processor, - expected_warning_messages): - """Assert that the attributes of this class equal the given values.""" - self.assertEquals(self.got_file_path, expected_file_path) - self.assertEquals(self.got_handle_style_error, expected_handle_style_error) - self.assertEquals(self.got_processor, expected_processor) - self.assertEquals(self.warning_messages, expected_warning_messages) - - def call_check_file(self, file_path): - """Call the check_file() method of a test StyleChecker instance.""" - # Confirm that the attributes are reset. - self.assert_attributes(None, None, None, "") - - # Create a test StyleChecker instance. - # - # The verbosity attribute is the only ProcessorOptions - # attribute that needs to be checked in this test. - # This is because it is the only option is directly - # passed to the constructor of a style processor. - options = ProcessorOptions(verbosity=3) + # Create a reference to self with a new name so its name does not + # conflict with the self introduced below. + test_case = self - style_checker = StyleChecker(options, self.mock_stderr_write) + class MockCarriageChecker(object): - style_checker.check_file(file_path, - self.mock_handle_style_error, - self.mock_process_file) + """A mock carriage-return checker.""" - def test_check_file_on_skip_without_warning(self): - """Test check_file() for a skipped-without-warning file.""" + def __init__(self, style_error_handler): + self.style_error_handler = style_error_handler - file_path = "LayoutTests/foo.txt" + # This gives the current test case access to the + # instantiated carriage checker. + test_case.carriage_checker = self - dispatcher = ProcessorDispatcher() - # Confirm that the input file is truly a skipped-without-warning file. - self.assertTrue(dispatcher.should_skip_without_warning(file_path)) + def check(self, lines): + # Save the lines so the current test case has a way to access + # and check them. + self.lines = lines - # Check the outcome. - self.call_check_file(file_path) - self.assert_attributes(None, None, None, "") + return lines - def test_check_file_on_skip_with_warning(self): - """Test check_file() for a skipped-with-warning file.""" + return MockCarriageChecker - file_path = "gtk2drawing.c" + def test_should_process__skip_without_warning(self): + """Test should_process() for a skip-without-warning file.""" + file_path = "foo/skip_without_warning.txt" - dispatcher = ProcessorDispatcher() - # Check that the input file is truly a skipped-with-warning file. - self.assertTrue(dispatcher.should_skip_with_warning(file_path)) + self.assertFalse(self._processor.should_process(file_path)) - # Check the outcome. - self.call_check_file(file_path) - self.assert_attributes(None, None, None, - 'Ignoring "gtk2drawing.c": this file is exempt from the style guide.\n') + def test_should_process__skip_with_warning(self): + """Test should_process() for a skip-with-warning file.""" + file_path = "foo/skip_with_warning.txt" - def test_check_file_on_non_skipped(self): + self.assertFalse(self._processor.should_process(file_path)) - # We use a C++ file since by using a CppProcessor, we can check - # that all of the possible information is getting passed to - # process_file (in particular, the verbosity). - file_base = "foo" - file_extension = "cpp" - file_path = file_base + "." + file_extension + self.assertLog(['WARNING: File exempt from style guide. ' + 'Skipping: "foo/skip_with_warning.txt"\n']) + + def test_should_process__true_result(self): + """Test should_process() for a file that should be processed.""" + file_path = "foo/skip_process.txt" - dispatcher = ProcessorDispatcher() - # Check that the input file is truly a C++ file. - self.assertEquals(dispatcher._file_type(file_path), style.FileType.CPP) + self.assertTrue(self._processor.should_process(file_path)) - # Check the outcome. - self.call_check_file(file_path) + def test_process__checker_dispatched(self): + """Test the process() method for a path with a dispatched checker.""" + file_path = 'foo.txt' + lines = ['line1', 'line2'] + line_numbers = [100] - expected_processor = CppProcessor(file_path, file_extension, self.mock_handle_style_error, 3) + expected_error_handler = DefaultStyleErrorHandler( + configuration=self._configuration, + file_path=file_path, + increment_error_count=self._do_nothing, + line_numbers=line_numbers) - self.assert_attributes(file_path, - self.mock_handle_style_error, - expected_processor, - "") + self._processor.process(lines=lines, + file_path=file_path, + line_numbers=line_numbers) + # Check that the carriage-return checker was instantiated correctly + # and was passed lines correctly. + carriage_checker = self.carriage_checker + self.assertEquals(carriage_checker.style_error_handler, + expected_error_handler) + self.assertEquals(carriage_checker.lines, ['line1', 'line2']) -if __name__ == '__main__': - import sys + # Check that the style checker was dispatched correctly and was + # passed lines correctly. + checker = self._mock_dispatcher.dispatched_checker + self.assertEquals(checker.file_path, 'foo.txt') + self.assertEquals(checker.min_confidence, 3) + self.assertEquals(checker.style_error_handler, expected_error_handler) - unittest.main() + self.assertEquals(checker.lines, ['line1', 'line2']) + def test_process__no_checker_dispatched(self): + """Test the process() method for a path with no dispatched checker.""" + path = os.path.join('foo', 'do_not_process.txt') + self.assertRaises(AssertionError, self._processor.process, + lines=['line1', 'line2'], file_path=path, + line_numbers=[100]) diff --git a/WebKitTools/Scripts/webkitpy/style/checkers/__init__.py b/WebKitTools/Scripts/webkitpy/style/checkers/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/checkers/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/style/processors/common.py b/WebKitTools/Scripts/webkitpy/style/checkers/common.py index dbf4bea..a2d933f 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/common.py +++ b/WebKitTools/Scripts/webkitpy/style/checkers/common.py @@ -20,10 +20,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Supports style checking not specific to any one processor.""" +"""Supports style checking not specific to any one file type.""" -# FIXME: Test this list in the same way that the list of CppProcessor +# FIXME: Test this list in the same way that the list of CppChecker # categories is tested, for example by checking that all of its # elements appear in the unit tests. This should probably be done # after moving the relevant cpp_unittest.ErrorCollector code @@ -33,27 +33,25 @@ categories = set([ ]) -def check_no_carriage_return(line, line_number, error): - """Check that a line does not end with a carriage return. +class CarriageReturnChecker(object): - Returns true if the check is successful (i.e. if the line does not - end with a carriage return), and false otherwise. + """Supports checking for and handling carriage returns.""" - Args: - line: A string that is the line to check. - line_number: The line number. - error: The function to call with any errors found. + def __init__(self, handle_style_error): + self._handle_style_error = handle_style_error - """ + def check(self, lines): + """Check for and strip trailing carriage returns from lines.""" + for line_number in range(len(lines)): + if not lines[line_number].endswith("\r"): + continue - if line.endswith("\r"): - error(line_number, - "whitespace/carriage_return", - 1, - "One or more unexpected \\r (^M) found; " - "better to use only a \\n") - return False - - return True + self._handle_style_error(line_number + 1, # Correct for offset. + "whitespace/carriage_return", + 1, + "One or more unexpected \\r (^M) found; " + "better to use only a \\n") + lines[line_number] = lines[line_number].rstrip("\r") + return lines diff --git a/WebKitTools/Scripts/webkitpy/style/processors/common_unittest.py b/WebKitTools/Scripts/webkitpy/style/checkers/common_unittest.py index 9362b65..b67b7b0 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/common_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/checkers/common_unittest.py @@ -22,24 +22,25 @@ """Unit tests for common.py.""" - import unittest -from common import check_no_carriage_return +from common import CarriageReturnChecker -# FIXME: The unit tests for the cpp, text, and common processors should +# FIXME: The unit tests for the cpp, text, and common checkers should # share supporting test code. This can include, for example, the # mock style error handling code and the code to check that all -# of a processor's categories are covered by the unit tests. +# of a checker's categories are covered by the unit tests. # Such shared code can be located in a shared test file, perhaps -# ilke this one. -class CarriageReturnTest(unittest.TestCase): +# even this file. +class CarriageReturnCheckerTest(unittest.TestCase): """Tests check_no_carriage_return().""" _category = "whitespace/carriage_return" _confidence = 1 + _expected_message = ("One or more unexpected \\r (^M) found; " + "better to use only a \\n") def setUp(self): self._style_errors = [] # The list of accumulated style errors. @@ -50,33 +51,44 @@ class CarriageReturnTest(unittest.TestCase): error = (line_number, category, confidence, message) self._style_errors.append(error) - def assert_carriage_return(self, line, is_error): - """Call check_no_carriage_return() and assert the result.""" - line_number = 100 + def assert_carriage_return(self, input_lines, expected_lines, error_lines): + """Process the given line and assert that the result is correct.""" handle_style_error = self._mock_style_error_handler - check_no_carriage_return(line, line_number, handle_style_error) + checker = CarriageReturnChecker(handle_style_error) + output_lines = checker.check(input_lines) - expected_message = ("One or more unexpected \\r (^M) found; " - "better to use only a \\n") + # Check both the return value and error messages. + self.assertEquals(output_lines, expected_lines) - if is_error: - expected_errors = [(line_number, self._category, self._confidence, - expected_message)] - self.assertEquals(self._style_errors, expected_errors) - else: - self.assertEquals(self._style_errors, []) + expected_errors = [(line_number, self._category, self._confidence, + self._expected_message) + for line_number in error_lines] + self.assertEquals(self._style_errors, expected_errors) def test_ends_with_carriage(self): - self.assert_carriage_return("carriage return\r", is_error=True) + self.assert_carriage_return(["carriage return\r"], + ["carriage return"], + [1]) def test_ends_with_nothing(self): - self.assert_carriage_return("no carriage return", is_error=False) + self.assert_carriage_return(["no carriage return"], + ["no carriage return"], + []) def test_ends_with_newline(self): - self.assert_carriage_return("no carriage return\n", is_error=False) - - def test_ends_with_carriage_newline(self): - # Check_no_carriage_return only() checks the final character. - self.assert_carriage_return("carriage\r in a string", is_error=False) - + self.assert_carriage_return(["no carriage return\n"], + ["no carriage return\n"], + []) + + def test_carriage_in_middle(self): + # The CarriageReturnChecker checks only the final character + # of each line. + self.assert_carriage_return(["carriage\r in a string"], + ["carriage\r in a string"], + []) + + def test_multiple_errors(self): + self.assert_carriage_return(["line1", "line2\r", "line3\r"], + ["line1", "line2", "line3"], + [2, 3]) diff --git a/WebKitTools/Scripts/webkitpy/style/processors/cpp.py b/WebKitTools/Scripts/webkitpy/style/checkers/cpp.py index 182c967..770ab40 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/cpp.py +++ b/WebKitTools/Scripts/webkitpy/style/checkers/cpp.py @@ -161,7 +161,7 @@ def up_to_unmatched_closing_paren(s): Returns: A pair of strings (prefix before first unmatched ')', - reminder of s after first unmatched ')'), e.g., + remainder of s after first unmatched ')'), e.g., up_to_unmatched_closing_paren("a == (b + c)) { ") returns "a == (b + c)", " {". Returns None, None if there is no unmatched ')' @@ -272,18 +272,18 @@ class _FunctionState(object): """Tracks current function name and the number of lines in its body. Attributes: - verbosity: The verbosity level to use while checking style. + min_confidence: The minimum confidence level to use while checking style. """ _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. - def __init__(self, verbosity): - self.verbosity = verbosity + def __init__(self, min_confidence): + self.min_confidence = min_confidence + self.current_function = '' self.in_a_function = False self.lines_in_function = 0 - self.current_function = '' def begin(self, function_name): """Start analyzing function body. @@ -311,7 +311,7 @@ class _FunctionState(object): base_trigger = self._TEST_TRIGGER else: base_trigger = self._NORMAL_TRIGGER - trigger = base_trigger * 2 ** self.verbosity + trigger = base_trigger * 2 ** self.min_confidence if self.lines_in_function > trigger: error_level = int(math.log(self.lines_in_function / base_trigger, 2)) @@ -642,6 +642,10 @@ def get_header_guard_cpp_variable(filename): """ + # Restores original filename in case that style checker is invoked from Emacs's + # flymake. + filename = re.sub(r'_flymake\.h$', '.h', filename) + return sub(r'[-.\s]', '_', os.path.basename(filename)) @@ -1333,26 +1337,25 @@ def check_spacing(file_extension, clean_lines, line_number, error): # there should either be zero or one spaces inside the parens. # We don't want: "if ( foo)" or "if ( foo )". # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. - matched = search(r'\b(?P<statement>if|for|foreach|while|switch)\s*\((?P<reminder>.*)$', line) + matched = search(r'\b(?P<statement>if|for|foreach|while|switch)\s*\((?P<remainder>.*)$', line) if matched: statement = matched.group('statement') - condition, rest = up_to_unmatched_closing_paren(matched.group('reminder')) + condition, rest = up_to_unmatched_closing_paren(matched.group('remainder')) if condition is not None: condition_match = search(r'(?P<leading>[ ]*)(?P<separator>.).*[^ ]+(?P<trailing>[ ]*)', condition) if condition_match: n_leading = len(condition_match.group('leading')) n_trailing = len(condition_match.group('trailing')) - if n_leading != n_trailing: - for_exception = statement == 'for' and ( - (condition.startswith(' ;') and n_trailing == 0) or - (condition.endswith('; ') and n_leading == 0)) + if n_leading != 0: + for_exception = statement == 'for' and condition.startswith(' ;') + if not for_exception: + error(line_number, 'whitespace/parens', 5, + 'Extra space after ( in %s' % statement) + if n_trailing != 0: + for_exception = statement == 'for' and condition.endswith('; ') if not for_exception: error(line_number, 'whitespace/parens', 5, - 'Mismatching spaces inside () in %s' % statement) - if n_leading > 1: - error(line_number, 'whitespace/parens', 5, - 'Should have zero or one spaces inside ( and ) in %s' % - statement) + 'Extra space before ) in %s' % statement) # Do not check for more than one command in macros in_macro = match(r'\s*#define', line) @@ -1365,9 +1368,14 @@ def check_spacing(file_extension, clean_lines, line_number, error): error(line_number, 'whitespace/comma', 3, 'Missing space after ,') + matched = search(r'^\s*(?P<token1>[a-zA-Z0-9_\*&]+)\s\s+(?P<token2>[a-zA-Z0-9_\*&]+)', line) + if matched: + error(line_number, 'whitespace/declaration', 3, + 'Extra space between %s and %s' % (matched.group('token1'), matched.group('token2'))) + if file_extension == 'cpp': # C++ should have the & or * beside the type not the variable name. - matched = match(r'\s*\w+(?<!\breturn)\s+(?P<pointer_operator>\*|\&)\w+', line) + matched = match(r'\s*\w+(?<!\breturn|\bdelete)\s+(?P<pointer_operator>\*|\&)\w+', line) if matched: error(line_number, 'whitespace/declaration', 3, 'Declaration has space between type name and %s in %s' % (matched.group('pointer_operator'), matched.group(0).strip())) @@ -1652,7 +1660,7 @@ def check_braces(clean_lines, line_number, error): # We check if a closed brace has started a line to see if a # one line control statement was previous. previous_line = clean_lines.elided[line_number - 2] - if (previous_line.find('{') > 0 + if (previous_line.find('{') > 0 and previous_line.find('}') < 0 and search(r'\b(if|for|foreach|while|else)\b', previous_line)): error(line_number, 'whitespace/braces', 4, 'One line control clauses should not use braces.') @@ -1864,8 +1872,20 @@ def check_for_null(file_extension, clean_lines, line_number, error): line = clean_lines.elided[line_number] - # Don't warn about NULL usage in g_object_{get,set}(). See Bug 32858 - if search(r'\bg_object_[sg]et\b', line): + # Don't warn about NULL usage in g_*(). See Bug 32858 and 39372. + if search(r'\bg(_[a-z]+)+\b', line): + return + + # Don't warn about NULL usage in gst_*_many(). See Bug 39740 + if search(r'\bgst_\w+_many\b', line): + return + + # Don't warn about NULL usage in g_str{join,concat}(). See Bug 34834 + if search(r'\bg_str(join|concat)\b', line): + return + + # Don't warn about NULL usage in gdk_pixbuf_save_to_*{join,concat}(). See Bug 43090. + if search(r'\bgdk_pixbuf_save_to\w+\b', line): return if search(r'\bNULL\b', line): @@ -1900,7 +1920,7 @@ def get_line_width(line): return len(line) -def check_style(clean_lines, line_number, file_extension, file_state, error): +def check_style(clean_lines, line_number, file_extension, class_state, file_state, error): """Checks rules from the 'C++ style rules' section of cppguide.html. Most of these rules are hard to test (naming, comment style), but we @@ -1911,6 +1931,8 @@ def check_style(clean_lines, line_number, file_extension, file_state, error): clean_lines: A CleansedLines instance containing the file. line_number: The number of the line to check. file_extension: The extension (without the dot) of the filename. + class_state: A _ClassState instance which maintains information about + the current stack of nested class declarations being parsed. file_state: A _FileState instance which maintains information about the state of things in the file. error: The function to call with any errors found. @@ -1971,6 +1993,10 @@ def check_style(clean_lines, line_number, file_extension, file_state, error): and not ((cleansed_line.find('case ') != -1 or cleansed_line.find('default:') != -1) and cleansed_line.find('break;') != -1) + # Also it's ok to have many commands in trivial single-line accessors in class definitions. + and not (match(r'.*\(.*\).*{.*.}', line) + and class_state.classinfo_stack + and line.count('{') == line.count('}')) and not cleansed_line.startswith('#define ')): error(line_number, 'whitespace/newline', 4, 'More than one command on the same line') @@ -2421,7 +2447,9 @@ def check_identifier_name_in_declaration(filename, line_number, line, error): # Convert "long long", "long double", and "long long int" to # simple types, but don't remove simple "long". line = sub(r'long (long )?(?=long|double|int)', '', line) - line = sub(r'\b(unsigned|signed|inline|using|static|const|volatile|auto|register|extern|typedef|restrict|struct|class|virtual)(?=\W)', '', line) + # Convert unsigned/signed types to simple types, too. + line = sub(r'(unsigned|signed) (?=char|short|int|long)', '', line) + line = sub(r'\b(inline|using|static|const|volatile|auto|register|extern|typedef|restrict|struct|class|virtual)(?=\W)', '', line) # Remove all template parameters by removing matching < and >. # Loop until no templates are removed to remove nested templates. @@ -2449,8 +2477,9 @@ def check_identifier_name_in_declaration(filename, line_number, line, error): # Detect variable and functions. type_regexp = r'\w([\w]|\s*[*&]\s*|::)+' identifier_regexp = r'(?P<identifier>[\w:]+)' + maybe_bitfield_regexp = r'(:\s*\d+\s*)?' character_after_identifier_regexp = r'(?P<character_after_identifier>[[;()=,])(?!=)' - declaration_without_type_regexp = r'\s*' + identifier_regexp + r'\s*' + character_after_identifier_regexp + declaration_without_type_regexp = r'\s*' + identifier_regexp + r'\s*' + maybe_bitfield_regexp + character_after_identifier_regexp declaration_with_type_regexp = r'\s*' + type_regexp + r'\s' + declaration_without_type_regexp is_function_arguments = False number_of_identifiers = 0 @@ -2488,6 +2517,10 @@ def check_identifier_name_in_declaration(filename, line_number, line, error): and not modified_identifier == "const_iterator"): error(line_number, 'readability/naming', 4, identifier + " is incorrectly named. Don't use underscores in your identifier names.") + # Check for variables named 'l', these are too easy to confuse with '1' in some fonts + if modified_identifier == 'l': + error(line_number, 'readability/naming', 4, identifier + " is incorrectly named. Don't use the single letter 'l' as an identifier name.") + # There can be only one declaration in non-for-control statements. if control_statement: return @@ -2500,7 +2533,6 @@ def check_identifier_name_in_declaration(filename, line_number, line, error): number_of_identifiers += 1 line = line[matched.end():] - def check_c_style_cast(line_number, line, raw_line, cast_type, pattern, error): """Checks for a C-style cast by looking for the pattern. @@ -2762,9 +2794,7 @@ def check_for_include_what_you_use(filename, clean_lines, include_state, error, # found. # e.g. If the file name is 'foo_flymake.cpp', we should search for 'foo.h' # instead of 'foo_flymake.h' - emacs_flymake_suffix = '_flymake.cpp' - if abs_filename.endswith(emacs_flymake_suffix): - abs_filename = abs_filename[:-len(emacs_flymake_suffix)] + '.cpp' + abs_filename = re.sub(r'_flymake\.cpp$', '.cpp', abs_filename) # include_state is modified during iteration, so we iterate over a copy of # the keys. @@ -2821,7 +2851,7 @@ def process_line(filename, file_extension, if search(r'\bNOLINT\b', raw_lines[line]): # ignore nolint lines return check_for_multiline_comments_and_strings(clean_lines, line, error) - check_style(clean_lines, line, file_extension, file_state, error) + check_style(clean_lines, line, file_extension, class_state, file_state, error) check_language(filename, clean_lines, line, file_extension, include_state, error) check_for_non_standard_constructs(clean_lines, line, class_state, error) @@ -2829,7 +2859,7 @@ def process_line(filename, file_extension, check_invalid_increment(clean_lines, line, error) -def _process_lines(filename, file_extension, lines, error, verbosity): +def _process_lines(filename, file_extension, lines, error, min_confidence): """Performs lint checks and reports any errors to the given error function. Args: @@ -2843,7 +2873,7 @@ def _process_lines(filename, file_extension, lines, error, verbosity): ['// marker so line numbers end in a known way']) include_state = _IncludeState() - function_state = _FunctionState(verbosity) + function_state = _FunctionState(min_confidence) class_state = _ClassState() file_state = _FileState() @@ -2868,7 +2898,7 @@ def _process_lines(filename, file_extension, lines, error, verbosity): check_for_new_line_at_eof(lines, error) -class CppProcessor(object): +class CppChecker(object): """Processes C++ lines for checking style.""" @@ -2941,8 +2971,9 @@ class CppProcessor(object): 'whitespace/todo', ]) - def __init__(self, file_path, file_extension, handle_style_error, verbosity): - """Create a CppProcessor instance. + def __init__(self, file_path, file_extension, handle_style_error, + min_confidence): + """Create a CppChecker instance. Args: file_extension: A string that is the file extension, without @@ -2952,18 +2983,18 @@ class CppProcessor(object): self.file_extension = file_extension self.file_path = file_path self.handle_style_error = handle_style_error - self.verbosity = verbosity + self.min_confidence = min_confidence # Useful for unit testing. def __eq__(self, other): - """Return whether this CppProcessor instance is equal to another.""" + """Return whether this CppChecker instance is equal to another.""" if self.file_extension != other.file_extension: return False if self.file_path != other.file_path: return False if self.handle_style_error != other.handle_style_error: return False - if self.verbosity != other.verbosity: + if self.min_confidence != other.min_confidence: return False return True @@ -2973,13 +3004,12 @@ class CppProcessor(object): # Python does not automatically deduce __ne__() from __eq__(). return not self.__eq__(other) - def process(self, lines): + def check(self, lines): _process_lines(self.file_path, self.file_extension, lines, - self.handle_style_error, self.verbosity) + self.handle_style_error, self.min_confidence) # FIXME: Remove this function (requires refactoring unit tests). -def process_file_data(filename, file_extension, lines, error, verbosity): - processor = CppProcessor(filename, file_extension, error, verbosity) - processor.process(lines) - +def process_file_data(filename, file_extension, lines, error, min_confidence): + checker = CppChecker(filename, file_extension, error, min_confidence) + checker.check(lines) diff --git a/WebKitTools/Scripts/webkitpy/style/processors/cpp_unittest.py b/WebKitTools/Scripts/webkitpy/style/checkers/cpp_unittest.py index fb5a487..ee829aa 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/cpp_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/checkers/cpp_unittest.py @@ -42,13 +42,13 @@ import random import re import unittest import cpp as cpp_style -from cpp import CppProcessor +from cpp import CppChecker # This class works as an error collector and replaces cpp_style.Error # function for the unit tests. We also verify each category we see # is in STYLE_CATEGORIES, to help keep that list up to date. class ErrorCollector: - _all_style_categories = CppProcessor.categories + _all_style_categories = CppChecker.categories # This is a list including all categories seen in any unit test. _seen_style_categories = {} @@ -119,20 +119,21 @@ class CppStyleTestBase(unittest.TestCase): """Provides some useful helper functions for cpp_style tests. Attributes: - verbosity: An integer that is the current verbosity level for - the tests. + min_confidence: An integer that is the current minimum confidence + level for the tests. """ - # FIXME: Refactor the unit tests so the verbosity level is passed + # FIXME: Refactor the unit tests so the confidence level is passed # explicitly, just like it is in the real code. - verbosity = 1; + min_confidence = 1; - # Helper function to avoid needing to explicitly pass verbosity + # Helper function to avoid needing to explicitly pass confidence # in all the unit test calls to cpp_style.process_file_data(). def process_file_data(self, filename, file_extension, lines, error): - """Call cpp_style.process_file_data() with the current verbosity.""" - return cpp_style.process_file_data(filename, file_extension, lines, error, self.verbosity) + """Call cpp_style.process_file_data() with the min_confidence.""" + return cpp_style.process_file_data(filename, file_extension, lines, + error, self.min_confidence) # Perform lint on single line of input and return the error message. def perform_single_line_lint(self, code, file_name): @@ -141,7 +142,7 @@ class CppStyleTestBase(unittest.TestCase): cpp_style.remove_multi_line_comments(lines, error_collector) clean_lines = cpp_style.CleansedLines(lines) include_state = cpp_style._IncludeState() - function_state = cpp_style._FunctionState(self.verbosity) + function_state = cpp_style._FunctionState(self.min_confidence) ext = file_name[file_name.rfind('.') + 1:] class_state = cpp_style._ClassState() file_state = cpp_style._FileState() @@ -163,7 +164,7 @@ class CppStyleTestBase(unittest.TestCase): class_state = cpp_style._ClassState() file_state = cpp_style._FileState() for i in xrange(lines.num_lines()): - cpp_style.check_style(lines, i, file_extension, file_state, error_collector) + cpp_style.check_style(lines, i, file_extension, class_state, file_state, error_collector) cpp_style.check_for_non_standard_constructs(lines, i, class_state, error_collector) class_state.check_finished(error_collector) @@ -199,7 +200,7 @@ class CppStyleTestBase(unittest.TestCase): The accumulated errors. """ error_collector = ErrorCollector(self.assert_) - function_state = cpp_style._FunctionState(self.verbosity) + function_state = cpp_style._FunctionState(self.min_confidence) lines = code.split('\n') cpp_style.remove_multi_line_comments(lines, error_collector) lines = cpp_style.CleansedLines(lines) @@ -238,7 +239,7 @@ class CppStyleTestBase(unittest.TestCase): if re.search(expected_message_re, message): return - self.assertEquals(expected_message, messages) + self.assertEquals(expected_message_re, messages) def assert_multi_line_lint(self, code, expected_message, file_name='foo.h'): file_extension = file_name[file_name.rfind('.') + 1:] @@ -1163,28 +1164,30 @@ class CppStyleTest(CppStyleTestBase): '') def test_mismatching_spaces_in_parens(self): - self.assert_lint('if (foo ) {', 'Mismatching spaces inside () in if' + self.assert_lint('if (foo ) {', 'Extra space before ) in if' ' [whitespace/parens] [5]') - self.assert_lint('switch ( foo) {', 'Mismatching spaces inside () in switch' + self.assert_lint('switch ( foo) {', 'Extra space after ( in switch' ' [whitespace/parens] [5]') - self.assert_lint('for (foo; ba; bar ) {', 'Mismatching spaces inside () in for' + self.assert_lint('for (foo; ba; bar ) {', 'Extra space before ) in for' ' [whitespace/parens] [5]') - self.assert_lint('for ((foo); (ba); (bar) ) {', 'Mismatching spaces inside () in for' + self.assert_lint('for ((foo); (ba); (bar) ) {', 'Extra space before ) in for' ' [whitespace/parens] [5]') self.assert_lint('for (; foo; bar) {', '') self.assert_lint('for (; (foo); (bar)) {', '') self.assert_lint('for ( ; foo; bar) {', '') self.assert_lint('for ( ; (foo); (bar)) {', '') - self.assert_lint('for ( ; foo; bar ) {', '') - self.assert_lint('for ( ; (foo); (bar) ) {', '') + self.assert_lint('for ( ; foo; bar ) {', 'Extra space before ) in for' + ' [whitespace/parens] [5]') + self.assert_lint('for ( ; (foo); (bar) ) {', 'Extra space before ) in for' + ' [whitespace/parens] [5]') self.assert_lint('for (foo; bar; ) {', '') self.assert_lint('for ((foo); (bar); ) {', '') - self.assert_lint('foreach (foo, foos ) {', 'Mismatching spaces inside () in foreach' + self.assert_lint('foreach (foo, foos ) {', 'Extra space before ) in foreach' ' [whitespace/parens] [5]') - self.assert_lint('foreach ( foo, foos) {', 'Mismatching spaces inside () in foreach' + self.assert_lint('foreach ( foo, foos) {', 'Extra space after ( in foreach' + ' [whitespace/parens] [5]') + self.assert_lint('while ( foo) {', 'Extra space after ( in while' ' [whitespace/parens] [5]') - self.assert_lint('while ( foo ) {', 'Should have zero or one spaces inside' - ' ( and ) in while [whitespace/parens] [5]') def test_spacing_for_fncall(self): self.assert_lint('if (foo) {', '') @@ -1541,12 +1544,20 @@ class CppStyleTest(CppStyleTestBase): self.assert_lint('f(a, /* name */ b);', '') self.assert_lint('f(a, /* name */b);', '') + def test_declaration(self): + self.assert_lint('int a;', '') + self.assert_lint('int a;', 'Extra space between int and a [whitespace/declaration] [3]') + self.assert_lint('int* a;', 'Extra space between int* and a [whitespace/declaration] [3]') + self.assert_lint('else if { }', '') + self.assert_lint('else if { }', 'Extra space between else and if [whitespace/declaration] [3]') + def test_pointer_reference_marker_location(self): self.assert_lint('int* b;', '', 'foo.cpp') self.assert_lint('int *b;', 'Declaration has space between type name and * in int *b [whitespace/declaration] [3]', 'foo.cpp') self.assert_lint('return *b;', '', 'foo.cpp') + self.assert_lint('delete *b;', '', 'foo.cpp') self.assert_lint('int *b;', '', 'foo.c') self.assert_lint('int* b;', 'Declaration has space between * and variable name in int* b [whitespace/declaration] [3]', @@ -1734,6 +1745,26 @@ class CppStyleTest(CppStyleTestBase): ' [build/header_guard] [5]' % expected_guard), error_collector.result_list()) + # Special case for flymake + error_collector = ErrorCollector(self.assert_) + self.process_file_data('mydir/Foo_flymake.h', 'h', + ['#ifndef %s' % expected_guard, + '#define %s' % expected_guard, + '#endif // %s' % expected_guard], + error_collector) + for line in error_collector.result_list(): + if line.find('build/header_guard') != -1: + self.fail('Unexpected error: %s' % line) + + error_collector = ErrorCollector(self.assert_) + self.process_file_data('mydir/Foo_flymake.h', 'h', [], error_collector) + self.assertEquals( + 1, + error_collector.result_list().count( + 'No #ifndef header guard found, suggested CPP variable is: %s' + ' [build/header_guard] [5]' % expected_guard), + error_collector.result_list()) + def test_build_printf_format(self): self.assert_lint( r'printf("\%%d", value);', @@ -2227,11 +2258,11 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): cpp_style._FunctionState._TEST_TRIGGER = self.old_test_trigger # FIXME: Eliminate the need for this function. - def set_verbosity(self, verbosity): - """Set new test verbosity and return old test verbosity.""" - old_verbosity = self.verbosity - self.verbosity = verbosity - return old_verbosity + def set_min_confidence(self, min_confidence): + """Set new test confidence and return old test confidence.""" + old_min_confidence = self.min_confidence + self.min_confidence = min_confidence + return old_min_confidence def assert_function_lengths_check(self, code, expected_message): """Check warnings for long function bodies are as expected. @@ -2272,7 +2303,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): lines: Number of lines to generate. error_level: --v setting for cpp_style. """ - trigger_level = self.trigger_lines(self.verbosity) + trigger_level = self.trigger_lines(self.min_confidence) self.assert_function_lengths_check( 'void test(int x)' + self.function_body(lines), ('Small and focused functions are preferred: ' @@ -2355,29 +2386,29 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): '') def test_function_length_check_definition_below_severity0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_definition_ok(self.trigger_lines(0) - 1) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_at_severity0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_definition_ok(self.trigger_lines(0)) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_above_severity0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_above_error_level(0) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_below_severity1v0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_below_error_level(1) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_at_severity1v0(self): - old_verbosity = self.set_verbosity(0) + old_min_confidence = self.set_min_confidence(0) self.assert_function_length_check_at_error_level(1) - self.set_verbosity(old_verbosity) + self.set_min_confidence(old_min_confidence) def test_function_length_check_definition_below_severity1(self): self.assert_function_length_check_definition_ok(self.trigger_lines(1) - 1) @@ -2391,7 +2422,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_plus_blanks(self): error_level = 1 error_lines = self.trigger_lines(error_level) + 1 - trigger_level = self.trigger_lines(self.verbosity) + trigger_level = self.trigger_lines(self.min_confidence) self.assert_function_lengths_check( 'void test_blanks(int x)' + self.function_body(error_lines), ('Small and focused functions are preferred: ' @@ -2403,7 +2434,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_complex_definition_severity1(self): error_level = 1 error_lines = self.trigger_lines(error_level) + 1 - trigger_level = self.trigger_lines(self.verbosity) + trigger_level = self.trigger_lines(self.min_confidence) self.assert_function_lengths_check( ('my_namespace::my_other_namespace::MyVeryLongTypeName*\n' 'my_namespace::my_other_namespace::MyFunction(int arg1, char* arg2)' @@ -2418,7 +2449,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_for_test(self): error_level = 1 error_lines = self.trigger_test_lines(error_level) + 1 - trigger_level = self.trigger_test_lines(self.verbosity) + trigger_level = self.trigger_test_lines(self.min_confidence) self.assert_function_lengths_check( 'TEST_F(Test, Mutator)' + self.function_body(error_lines), ('Small and focused functions are preferred: ' @@ -2430,7 +2461,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_for_split_line_test(self): error_level = 1 error_lines = self.trigger_test_lines(error_level) + 1 - trigger_level = self.trigger_test_lines(self.verbosity) + trigger_level = self.trigger_test_lines(self.min_confidence) self.assert_function_lengths_check( ('TEST_F(GoogleUpdateRecoveryRegistryProtectedTest,\n' ' FixGoogleUpdate_AllValues_MachineApp)' # note: 4 spaces @@ -2445,7 +2476,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_for_bad_test_doesnt_break(self): error_level = 1 error_lines = self.trigger_test_lines(error_level) + 1 - trigger_level = self.trigger_test_lines(self.verbosity) + trigger_level = self.trigger_test_lines(self.min_confidence) self.assert_function_lengths_check( ('TEST_F(' + self.function_body(error_lines)), @@ -2458,7 +2489,7 @@ class CheckForFunctionLengthsTest(CppStyleTestBase): def test_function_length_check_definition_severity1_with_embedded_no_lints(self): error_level = 1 error_lines = self.trigger_lines(error_level) + 1 - trigger_level = self.trigger_lines(self.verbosity) + trigger_level = self.trigger_lines(self.min_confidence) self.assert_function_lengths_check( 'void test(int x)' + self.function_body_with_no_lints(error_lines), ('Small and focused functions are preferred: ' @@ -2563,6 +2594,19 @@ class NoNonVirtualDestructorsTest(CppStyleTestBase): self.assert_multi_line_lint( 'class Foo { void foo(); };', 'More than one command on the same line [whitespace/newline] [4]') + self.assert_multi_line_lint( + 'class MyClass {\n' + ' int getIntValue() { ASSERT(m_ptr); return *m_ptr; }\n' + '};\n', + '') + self.assert_multi_line_lint( + 'class MyClass {\n' + ' int getIntValue()\n' + ' {\n' + ' ASSERT(m_ptr); return *m_ptr;\n' + ' }\n' + '};\n', + 'More than one command on the same line [whitespace/newline] [4]') self.assert_multi_line_lint( '''class Qualified::Goo : public Foo { @@ -3037,7 +3081,7 @@ class WebKitStyleTest(CppStyleTestBase): '') self.assert_multi_line_lint( '#define TEST_ASSERT(expression) do { if ( !(expression)) { TestsController::shared().testFailed(__FILE__, __LINE__, #expression); return; } } while (0)\n', - 'Mismatching spaces inside () in if [whitespace/parens] [5]') + 'Extra space after ( in if [whitespace/parens] [5]') # FIXME: currently we only check first conditional, so we cannot detect errors in next ones. # self.assert_multi_line_lint( # '#define TEST_ASSERT(expression) do { if (!(expression)) { TestsController::shared().testFailed(__FILE__, __LINE__, #expression); return; } } while (0 )\n', @@ -3063,6 +3107,20 @@ class WebKitStyleTest(CppStyleTestBase): '}\n', ['More than one command on the same line in if [whitespace/parens] [4]', 'One line control clauses should not use braces. [whitespace/braces] [4]']) + self.assert_multi_line_lint( + 'void func()\n' + '{\n' + ' while (condition) { }\n' + ' return 0;\n' + '}\n', + '') + self.assert_multi_line_lint( + 'void func()\n' + '{\n' + ' for (i = 0; i < 42; i++) { foobar(); }\n' + ' return 0;\n' + '}\n', + 'More than one command on the same line in for [whitespace/parens] [4]') # 3. An else if statement should be written as an if statement # when the prior if concludes with a return statement. @@ -3392,13 +3450,50 @@ class WebKitStyleTest(CppStyleTestBase): '', 'foo.m') - # Make sure that the NULL check does not apply to g_object_{set,get} + # Make sure that the NULL check does not apply to g_object_{set,get} and + # g_str{join,concat} self.assert_lint( 'g_object_get(foo, "prop", &bar, NULL);', '') self.assert_lint( 'g_object_set(foo, "prop", bar, NULL);', '') + self.assert_lint( + 'g_build_filename(foo, bar, NULL);', + '') + self.assert_lint( + 'gst_bin_add_many(foo, bar, boo, NULL);', + '') + self.assert_lint( + 'gst_bin_remove_many(foo, bar, boo, NULL);', + '') + self.assert_lint( + 'gst_element_link_many(foo, bar, boo, NULL);', + '') + self.assert_lint( + 'gst_element_unlink_many(foo, bar, boo, NULL);', + '') + self.assert_lint( + 'gchar* result = g_strconcat("part1", "part2", "part3", NULL);', + '') + self.assert_lint( + 'gchar* result = g_strconcat("part1", NULL);', + '') + self.assert_lint( + 'gchar* result = g_strjoin(",", "part1", "part2", "part3", NULL);', + '') + self.assert_lint( + 'gchar* result = g_strjoin(",", "part1", NULL);', + '') + self.assert_lint( + 'gchar* result = gdk_pixbuf_save_to_callback(pixbuf, function, data, type, error, NULL);', + '') + self.assert_lint( + 'gchar* result = gdk_pixbuf_save_to_buffer(pixbuf, function, data, type, error, NULL);', + '') + self.assert_lint( + 'gchar* result = gdk_pixbuf_save_to_stream(pixbuf, function, data, type, error, NULL);', + '') # 2. C++ and C bool values should be written as true and # false. Objective-C BOOL values should be written as YES and NO. @@ -3489,7 +3584,8 @@ class WebKitStyleTest(CppStyleTestBase): 'foo.h') def test_names(self): - name_error_message = " is incorrectly named. Don't use underscores in your identifier names. [readability/naming] [4]" + name_underscore_error_message = " is incorrectly named. Don't use underscores in your identifier names. [readability/naming] [4]" + name_tooshort_error_message = " is incorrectly named. Don't use the single letter 'l' as an identifier name. [readability/naming] [4]" # Basic cases from WebKit style guide. self.assert_lint('struct Data;', '') @@ -3497,54 +3593,65 @@ class WebKitStyleTest(CppStyleTestBase): self.assert_lint('class HTMLDocument;', '') self.assert_lint('String mimeType();', '') self.assert_lint('size_t buffer_size;', - 'buffer_size' + name_error_message) + 'buffer_size' + name_underscore_error_message) self.assert_lint('short m_length;', '') self.assert_lint('short _length;', - '_length' + name_error_message) + '_length' + name_underscore_error_message) self.assert_lint('short length_;', - 'length_' + name_error_message) + 'length_' + name_underscore_error_message) + self.assert_lint('unsigned _length;', + '_length' + name_underscore_error_message) + self.assert_lint('unsigned int _length;', + '_length' + name_underscore_error_message) + self.assert_lint('unsigned long long _length;', + '_length' + name_underscore_error_message) + + # Variable name 'l' is easy to confuse with '1' + self.assert_lint('int l;', 'l' + name_tooshort_error_message) + self.assert_lint('size_t l;', 'l' + name_tooshort_error_message) + self.assert_lint('long long l;', 'l' + name_tooshort_error_message) # Pointers, references, functions, templates, and adjectives. self.assert_lint('char* under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('const int UNDER_SCORE;', - 'UNDER_SCORE' + name_error_message) + 'UNDER_SCORE' + name_underscore_error_message) self.assert_lint('static inline const char const& const under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('WebCore::RenderObject* under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('int func_name();', - 'func_name' + name_error_message) + 'func_name' + name_underscore_error_message) self.assert_lint('RefPtr<RenderObject*> under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('WTF::Vector<WTF::RefPtr<const RenderObject* const> > under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('int under_score[];', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('struct dirent* under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('long under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('long long under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('long double under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('long long int under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) # Declarations in control statement. self.assert_lint('if (int under_score = 42) {', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('else if (int under_score = 42) {', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('for (int under_score = 42; cond; i++) {', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('while (foo & under_score = bar) {', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('for (foo * under_score = p; cond; i++) {', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('for (foo * under_score; cond; i++) {', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('while (foo & value_in_thirdparty_library) {', '') self.assert_lint('while (foo * value_in_thirdparty_library) {', '') self.assert_lint('if (mli && S_OK == mli->foo()) {', '') @@ -3552,38 +3659,38 @@ class WebKitStyleTest(CppStyleTestBase): # More member variables and functions. self.assert_lint('int SomeClass::s_validName', '') self.assert_lint('int m_under_score;', - 'm_under_score' + name_error_message) + 'm_under_score' + name_underscore_error_message) self.assert_lint('int SomeClass::s_under_score = 0;', - 'SomeClass::s_under_score' + name_error_message) + 'SomeClass::s_under_score' + name_underscore_error_message) self.assert_lint('int SomeClass::under_score = 0;', - 'SomeClass::under_score' + name_error_message) + 'SomeClass::under_score' + name_underscore_error_message) # Other statements. self.assert_lint('return INT_MAX;', '') self.assert_lint('return_t under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('goto under_score;', - 'under_score' + name_error_message) + 'under_score' + name_underscore_error_message) self.assert_lint('delete static_cast<Foo*>(p);', '') # Multiple variables in one line. self.assert_lint('void myFunction(int variable1, int another_variable);', - 'another_variable' + name_error_message) + 'another_variable' + name_underscore_error_message) self.assert_lint('int variable1, another_variable;', - 'another_variable' + name_error_message) + 'another_variable' + name_underscore_error_message) self.assert_lint('int first_variable, secondVariable;', - 'first_variable' + name_error_message) + 'first_variable' + name_underscore_error_message) self.assert_lint('void my_function(int variable_1, int variable_2);', - ['my_function' + name_error_message, - 'variable_1' + name_error_message, - 'variable_2' + name_error_message]) + ['my_function' + name_underscore_error_message, + 'variable_1' + name_underscore_error_message, + 'variable_2' + name_underscore_error_message]) self.assert_lint('for (int variable_1, variable_2;;) {', - ['variable_1' + name_error_message, - 'variable_2' + name_error_message]) + ['variable_1' + name_underscore_error_message, + 'variable_2' + name_underscore_error_message]) # There is an exception for op code functions but only in the JavaScriptCore directory. self.assert_lint('void this_op_code(int var1, int var2)', '', 'JavaScriptCore/foo.cpp') - self.assert_lint('void this_op_code(int var1, int var2)', 'this_op_code' + name_error_message) + self.assert_lint('void this_op_code(int var1, int var2)', 'this_op_code' + name_underscore_error_message) # GObject requires certain magical names in class declarations. self.assert_lint('void webkit_dom_object_init();', '') @@ -3599,6 +3706,10 @@ class WebKitStyleTest(CppStyleTestBase): # const_iterator is allowed as well. self.assert_lint('typedef VectorType::const_iterator const_iterator;', '') + # Bitfields. + self.assert_lint('unsigned _fillRule : 1;', + '_fillRule' + name_underscore_error_message) + def test_comments(self): # A comment at the beginning of a line is ok. @@ -3614,52 +3725,52 @@ class WebKitStyleTest(CppStyleTestBase): pass -class CppProcessorTest(unittest.TestCase): +class CppCheckerTest(unittest.TestCase): - """Tests CppProcessor class.""" + """Tests CppChecker class.""" def mock_handle_style_error(self): pass - def _processor(self): - return CppProcessor("foo", "h", self.mock_handle_style_error, 3) + def _checker(self): + return CppChecker("foo", "h", self.mock_handle_style_error, 3) def test_init(self): """Test __init__ constructor.""" - processor = self._processor() - self.assertEquals(processor.file_extension, "h") - self.assertEquals(processor.file_path, "foo") - self.assertEquals(processor.handle_style_error, self.mock_handle_style_error) - self.assertEquals(processor.verbosity, 3) + checker = self._checker() + self.assertEquals(checker.file_extension, "h") + self.assertEquals(checker.file_path, "foo") + self.assertEquals(checker.handle_style_error, self.mock_handle_style_error) + self.assertEquals(checker.min_confidence, 3) def test_eq(self): """Test __eq__ equality function.""" - processor1 = self._processor() - processor2 = self._processor() + checker1 = self._checker() + checker2 = self._checker() # == calls __eq__. - self.assertTrue(processor1 == processor2) + self.assertTrue(checker1 == checker2) def mock_handle_style_error2(self): pass # Verify that a difference in any argument cause equality to fail. - processor = CppProcessor("foo", "h", self.mock_handle_style_error, 3) - self.assertFalse(processor == CppProcessor("bar", "h", self.mock_handle_style_error, 3)) - self.assertFalse(processor == CppProcessor("foo", "c", self.mock_handle_style_error, 3)) - self.assertFalse(processor == CppProcessor("foo", "h", mock_handle_style_error2, 3)) - self.assertFalse(processor == CppProcessor("foo", "h", self.mock_handle_style_error, 4)) + checker = CppChecker("foo", "h", self.mock_handle_style_error, 3) + self.assertFalse(checker == CppChecker("bar", "h", self.mock_handle_style_error, 3)) + self.assertFalse(checker == CppChecker("foo", "c", self.mock_handle_style_error, 3)) + self.assertFalse(checker == CppChecker("foo", "h", mock_handle_style_error2, 3)) + self.assertFalse(checker == CppChecker("foo", "h", self.mock_handle_style_error, 4)) def test_ne(self): """Test __ne__ inequality function.""" - processor1 = self._processor() - processor2 = self._processor() + checker1 = self._checker() + checker2 = self._checker() # != calls __ne__. # By default, __ne__ always returns true on different objects. # Thus, just check the distinguishing case to verify that the # code defines __ne__. - self.assertFalse(processor1 != processor2) + self.assertFalse(checker1 != checker2) def tearDown(): diff --git a/WebKitTools/Scripts/webkitpy/style/checkers/python.py b/WebKitTools/Scripts/webkitpy/style/checkers/python.py new file mode 100644 index 0000000..70d4450 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/checkers/python.py @@ -0,0 +1,56 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports checking WebKit style in Python files.""" + +from ...style_references import pep8 + + +class PythonChecker(object): + + """Processes text lines for checking style.""" + + def __init__(self, file_path, handle_style_error): + self._file_path = file_path + self._handle_style_error = handle_style_error + + def check(self, lines): + # Initialize pep8.options, which is necessary for + # Checker.check_all() to execute. + pep8.process_options(arglist=[self._file_path]) + + checker = pep8.Checker(self._file_path) + + def _pep8_handle_error(line_number, offset, text, check): + # FIXME: Incorporate the character offset into the error output. + # This will require updating the error handler __call__ + # signature to include an optional "offset" parameter. + pep8_code = text[:4] + pep8_message = text[5:] + + category = "pep8/" + pep8_code + + self._handle_style_error(line_number, category, 5, pep8_message) + + checker.report_error = _pep8_handle_error + + errors = checker.check_all() diff --git a/WebKitTools/Scripts/webkitpy/style/checkers/python_unittest.py b/WebKitTools/Scripts/webkitpy/style/checkers/python_unittest.py new file mode 100644 index 0000000..e003eb8 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/checkers/python_unittest.py @@ -0,0 +1,62 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for python.py.""" + +import os +import unittest + +from python import PythonChecker + + +class PythonCheckerTest(unittest.TestCase): + + """Tests the PythonChecker class.""" + + def test_init(self): + """Test __init__() method.""" + def _mock_handle_style_error(self): + pass + + checker = PythonChecker("foo.txt", _mock_handle_style_error) + self.assertEquals(checker._file_path, "foo.txt") + self.assertEquals(checker._handle_style_error, + _mock_handle_style_error) + + def test_check(self): + """Test check() method.""" + errors = [] + + def _mock_handle_style_error(line_number, category, confidence, + message): + error = (line_number, category, confidence, message) + errors.append(error) + + current_dir = os.path.dirname(__file__) + file_path = os.path.join(current_dir, "python_unittest_input.py") + + checker = PythonChecker(file_path, _mock_handle_style_error) + checker.check(lines=[]) + + self.assertEquals(len(errors), 1) + self.assertEquals(errors[0], + (2, "pep8/W291", 5, "trailing whitespace")) diff --git a/WebKitTools/Scripts/webkitpy/style/checkers/python_unittest_input.py b/WebKitTools/Scripts/webkitpy/style/checkers/python_unittest_input.py new file mode 100644 index 0000000..9f1d118 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/checkers/python_unittest_input.py @@ -0,0 +1,2 @@ +# This file is sample input for python_unittest.py and includes a single +# error which is an extra space at the end of this line. diff --git a/WebKitTools/Scripts/webkitpy/style/processors/text.py b/WebKitTools/Scripts/webkitpy/style/checkers/text.py index 307e5b8..0d03938 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/text.py +++ b/WebKitTools/Scripts/webkitpy/style/checkers/text.py @@ -30,7 +30,7 @@ """Checks WebKit style for text files.""" -class TextProcessor(object): +class TextChecker(object): """Processes text lines for checking style.""" @@ -38,7 +38,7 @@ class TextProcessor(object): self.file_path = file_path self.handle_style_error = handle_style_error - def process(self, lines): + def check(self, lines): lines = (["// adjust line numbers to make the first line 1."] + lines) # FIXME: share with cpp_style. @@ -51,6 +51,6 @@ class TextProcessor(object): # FIXME: Remove this function (requires refactoring unit tests). def process_file_data(filename, lines, error): - processor = TextProcessor(filename, error) - processor.process(lines) + checker = TextChecker(filename, error) + checker.check(lines) diff --git a/WebKitTools/Scripts/webkitpy/style/processors/text_unittest.py b/WebKitTools/Scripts/webkitpy/style/checkers/text_unittest.py index 62f825b..ced49a9 100644 --- a/WebKitTools/Scripts/webkitpy/style/processors/text_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/checkers/text_unittest.py @@ -32,7 +32,7 @@ import unittest import text as text_style -from text import TextProcessor +from text import TextChecker class TextStyleTestCase(unittest.TestCase): """TestCase for text_style.py""" @@ -76,18 +76,18 @@ class TextStyleTestCase(unittest.TestCase): '\tReviewed by NOBODY.'], 3) -class TextProcessorTest(unittest.TestCase): +class TextCheckerTest(unittest.TestCase): - """Tests TextProcessor class.""" + """Tests TextChecker class.""" def mock_handle_style_error(self): pass def test_init(self): """Test __init__ constructor.""" - processor = TextProcessor("foo.txt", self.mock_handle_style_error) - self.assertEquals(processor.file_path, "foo.txt") - self.assertEquals(processor.handle_style_error, self.mock_handle_style_error) + checker = TextChecker("foo.txt", self.mock_handle_style_error) + self.assertEquals(checker.file_path, "foo.txt") + self.assertEquals(checker.handle_style_error, self.mock_handle_style_error) if __name__ == '__main__': diff --git a/WebKitTools/Scripts/webkitpy/style/error_handlers.py b/WebKitTools/Scripts/webkitpy/style/error_handlers.py index 1940e03..0bede24 100644 --- a/WebKitTools/Scripts/webkitpy/style/error_handlers.py +++ b/WebKitTools/Scripts/webkitpy/style/error_handlers.py @@ -40,10 +40,10 @@ Methods: line_number: The integer line number of the line containing the error. category: The name of the category of the error, for example "whitespace/newline". - confidence: An integer between 1-5 that represents the level of - confidence in the error. The value 5 means that we are - certain of the problem, and the value 1 means that it - could be a legitimate construct. + confidence: An integer between 1 and 5 inclusive that represents the + application's level of confidence in the error. The value + 5 means that we are certain of the problem, and the + value 1 means that it could be a legitimate construct. message: The error message to report. """ @@ -56,35 +56,55 @@ class DefaultStyleErrorHandler(object): """The default style error handler.""" - def __init__(self, file_path, options, increment_error_count, - stderr_write=None): + def __init__(self, file_path, configuration, increment_error_count, + line_numbers=None): """Create a default style error handler. Args: file_path: The path to the file containing the error. This is used for reporting to the user. - options: A ProcessorOptions instance. + configuration: A StyleProcessorConfiguration instance. increment_error_count: A function that takes no arguments and increments the total count of reportable errors. - stderr_write: A function that takes a string as a parameter - and that is called when a style error occurs. - Defaults to sys.stderr.write. This should be - used only for unit tests. + line_numbers: An array of line numbers of the lines for which + style errors should be reported, or None if errors + for all lines should be reported. When it is not + None, this array normally contains the line numbers + corresponding to the modified lines of a patch. """ - if stderr_write is None: - stderr_write = sys.stderr.write + if line_numbers is not None: + line_numbers = set(line_numbers) self._file_path = file_path + self._configuration = configuration self._increment_error_count = increment_error_count - self._options = options - self._stderr_write = stderr_write + self._line_numbers = line_numbers # A string to integer dictionary cache of the number of reportable # errors per category passed to this instance. self._category_totals = {} + # Useful for unit testing. + def __eq__(self, other): + """Return whether this instance is equal to another.""" + if self._configuration != other._configuration: + return False + if self._file_path != other._file_path: + return False + if self._increment_error_count != other._increment_error_count: + return False + if self._line_numbers != other._line_numbers: + return False + + return True + + # Useful for unit testing. + def __ne__(self, other): + # Python does not automatically deduce __ne__ from __eq__. + return not self.__eq__(other) + def _add_reportable_error(self, category): """Increment the error count and return the new category total.""" self._increment_error_count() # Increment the total. @@ -99,9 +119,9 @@ class DefaultStyleErrorHandler(object): def _max_reports(self, category): """Return the maximum number of errors to report.""" - if not category in self._options.max_reports_per_category: + if not category in self._configuration.max_reports_per_category: return None - return self._options.max_reports_per_category[category] + return self._configuration.max_reports_per_category[category] def __call__(self, line_number, category, confidence, message): """Handle the occurrence of a style error. @@ -109,9 +129,15 @@ class DefaultStyleErrorHandler(object): See the docstring of this module for more information. """ - if not self._options.is_reportable(category, - confidence, - self._file_path): + if (self._line_numbers is not None and + line_number not in self._line_numbers): + # Then the error occurred in a line that was not modified, so + # the error is not reportable. + return + + if not self._configuration.is_reportable(category=category, + confidence_in_error=confidence, + file_path=self._file_path): return category_total = self._add_reportable_error(category) @@ -122,69 +148,12 @@ class DefaultStyleErrorHandler(object): # Then suppress displaying the error. return - if self._options.output_format == 'vs7': - format_string = "%s(%s): %s [%s] [%d]\n" - else: - format_string = "%s:%s: %s [%s] [%d]\n" + self._configuration.write_style_error(category=category, + confidence_in_error=confidence, + file_path=self._file_path, + line_number=line_number, + message=message) if category_total == max_reports: - format_string += ("Suppressing further [%s] reports for this " - "file.\n" % category) - - self._stderr_write(format_string % (self._file_path, - line_number, - message, - category, - confidence)) - - -class PatchStyleErrorHandler(object): - - """The style error function for patch files.""" - - def __init__(self, diff, file_path, options, increment_error_count, - stderr_write): - """Create a patch style error handler for the given path. - - Args: - diff: A DiffFile instance. - Other arguments: see the DefaultStyleErrorHandler.__init__() - documentation for the other arguments. - - """ - self._diff = diff - self._default_error_handler = DefaultStyleErrorHandler(file_path, - options, - increment_error_count, - stderr_write) - - # The line numbers of the modified lines. This is set lazily. - self._line_numbers = set() - - def _get_line_numbers(self): - """Return the line numbers of the modified lines.""" - if not self._line_numbers: - for line in self._diff.lines: - # When deleted line is not set, it means that - # the line is newly added (or modified). - if not line[0]: - self._line_numbers.add(line[1]) - - return self._line_numbers - - def __call__(self, line_number, category, confidence, message): - """Handle the occurrence of a style error. - - This function does not report errors occurring in lines not - marked as modified or added in the patch. - - See the docstring of this module for more information. - - """ - if line_number not in self._get_line_numbers(): - # Then the error is not reportable. - return - - self._default_error_handler(line_number, category, confidence, - message) - + self._configuration.stderr_write("Suppressing further [%s] reports " + "for this file.\n" % category) diff --git a/WebKitTools/Scripts/webkitpy/style/error_handlers_unittest.py b/WebKitTools/Scripts/webkitpy/style/error_handlers_unittest.py index 1d7e998..23619cc 100644 --- a/WebKitTools/Scripts/webkitpy/style/error_handlers_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/error_handlers_unittest.py @@ -25,172 +25,163 @@ import unittest -from .. style_references import parse_patch -from checker import ProcessorOptions +from checker import StyleProcessorConfiguration from error_handlers import DefaultStyleErrorHandler -from error_handlers import PatchStyleErrorHandler +from filter import FilterConfiguration -class StyleErrorHandlerTestBase(unittest.TestCase): +class DefaultStyleErrorHandlerTest(unittest.TestCase): + + """Tests the DefaultStyleErrorHandler class.""" def setUp(self): - self._error_messages = "" + self._error_messages = [] self._error_count = 0 - def _mock_increment_error_count(self): - self._error_count += 1 - - def _mock_stderr_write(self, message): - self._error_messages += message - - -class DefaultStyleErrorHandlerTest(StyleErrorHandlerTestBase): - - """Tests DefaultStyleErrorHandler class.""" + _category = "whitespace/tab" + """The category name for the tests in this class.""" _file_path = "foo.h" + """The file path for the tests in this class.""" - _category = "whitespace/tab" + def _mock_increment_error_count(self): + self._error_count += 1 - def _error_handler(self, options): - return DefaultStyleErrorHandler(self._file_path, - options, - self._mock_increment_error_count, - self._mock_stderr_write) + def _mock_stderr_write(self, message): + self._error_messages.append(message) + + def _style_checker_configuration(self): + """Return a StyleProcessorConfiguration instance for testing.""" + base_rules = ["-whitespace", "+whitespace/tab"] + filter_configuration = FilterConfiguration(base_rules=base_rules) + + return StyleProcessorConfiguration( + filter_configuration=filter_configuration, + max_reports_per_category={"whitespace/tab": 2}, + min_confidence=3, + output_format="vs7", + stderr_write=self._mock_stderr_write) + + def _error_handler(self, configuration, line_numbers=None): + return DefaultStyleErrorHandler(configuration=configuration, + file_path=self._file_path, + increment_error_count=self._mock_increment_error_count, + line_numbers=line_numbers) def _check_initialized(self): """Check that count and error messages are initialized.""" self.assertEquals(0, self._error_count) - self.assertEquals("", self._error_messages) - - def _call(self, handle_error, options, confidence): - """Handle an error with the given error handler.""" - line_number = 100 - message = "message" - - handle_error(line_number, self._category, confidence, message) - - def _call_error_handler(self, options, confidence): - """Handle an error using a new error handler.""" - handle_error = self._error_handler(options) - self._call(handle_error, options, confidence) - - def test_call_non_reportable(self): - """Test __call__() method with a non-reportable error.""" - confidence = 1 - options = ProcessorOptions(verbosity=3) + self.assertEquals(0, len(self._error_messages)) + + def _call_error_handler(self, handle_error, confidence, line_number=100): + """Call the given error handler with a test error.""" + handle_error(line_number=line_number, + category=self._category, + confidence=confidence, + message="message") + + def test_eq__true_return_value(self): + """Test the __eq__() method for the return value of True.""" + handler1 = self._error_handler(configuration=None) + handler2 = self._error_handler(configuration=None) + + self.assertTrue(handler1.__eq__(handler2)) + + def test_eq__false_return_value(self): + """Test the __eq__() method for the return value of False.""" + def make_handler(configuration=self._style_checker_configuration(), + file_path='foo.txt', increment_error_count=lambda: True, + line_numbers=[100]): + return DefaultStyleErrorHandler(configuration=configuration, + file_path=file_path, + increment_error_count=increment_error_count, + line_numbers=line_numbers) + + handler = make_handler() + + # Establish a baseline for our comparisons below. + self.assertTrue(handler.__eq__(make_handler())) + + # Verify that a difference in any argument causes equality to fail. + self.assertFalse(handler.__eq__(make_handler(configuration=None))) + self.assertFalse(handler.__eq__(make_handler(file_path='bar.txt'))) + self.assertFalse(handler.__eq__(make_handler(increment_error_count=None))) + self.assertFalse(handler.__eq__(make_handler(line_numbers=[50]))) + + def test_ne(self): + """Test the __ne__() method.""" + # By default, __ne__ always returns true on different objects. + # Thus, check just the distinguishing case to verify that the + # code defines __ne__. + handler1 = self._error_handler(configuration=None) + handler2 = self._error_handler(configuration=None) + + self.assertFalse(handler1.__ne__(handler2)) + + def test_non_reportable_error(self): + """Test __call__() with a non-reportable error.""" self._check_initialized() + configuration = self._style_checker_configuration() + confidence = 1 # Confirm the error is not reportable. - self.assertFalse(options.is_reportable(self._category, - confidence, - self._file_path)) - - self._call_error_handler(options, confidence) + self.assertFalse(configuration.is_reportable(self._category, + confidence, + self._file_path)) + error_handler = self._error_handler(configuration) + self._call_error_handler(error_handler, confidence) self.assertEquals(0, self._error_count) - self.assertEquals("", self._error_messages) - - def test_call_reportable_emacs(self): - """Test __call__() method with a reportable error and emacs format.""" - confidence = 5 - options = ProcessorOptions(verbosity=3, output_format="emacs") - self._check_initialized() - - self._call_error_handler(options, confidence) - - self.assertEquals(1, self._error_count) - self.assertEquals(self._error_messages, - "foo.h:100: message [whitespace/tab] [5]\n") + self.assertEquals([], self._error_messages) - def test_call_reportable_vs7(self): - """Test __call__() method with a reportable error and vs7 format.""" - confidence = 5 - options = ProcessorOptions(verbosity=3, output_format="vs7") + # Also serves as a reportable error test. + def test_max_reports_per_category(self): + """Test error report suppression in __call__() method.""" self._check_initialized() + configuration = self._style_checker_configuration() + error_handler = self._error_handler(configuration) - self._call_error_handler(options, confidence) - - self.assertEquals(1, self._error_count) - self.assertEquals(self._error_messages, - "foo.h(100): message [whitespace/tab] [5]\n") - - def test_call_max_reports_per_category(self): - """Test error report suppression in __call__() method.""" confidence = 5 - options = ProcessorOptions(verbosity=3, - max_reports_per_category={self._category: 2}) - error_handler = self._error_handler(options) - - self._check_initialized() # First call: usual reporting. - self._call(error_handler, options, confidence) + self._call_error_handler(error_handler, confidence) self.assertEquals(1, self._error_count) + self.assertEquals(1, len(self._error_messages)) self.assertEquals(self._error_messages, - "foo.h:100: message [whitespace/tab] [5]\n") + ["foo.h(100): message [whitespace/tab] [5]\n"]) # Second call: suppression message reported. - self._error_messages = "" - self._call(error_handler, options, confidence) + self._call_error_handler(error_handler, confidence) + # The "Suppressing further..." message counts as an additional + # message (but not as an addition to the error count). self.assertEquals(2, self._error_count) - self.assertEquals(self._error_messages, - "foo.h:100: message [whitespace/tab] [5]\n" - "Suppressing further [%s] reports for this file.\n" - % self._category) + self.assertEquals(3, len(self._error_messages)) + self.assertEquals(self._error_messages[-2], + "foo.h(100): message [whitespace/tab] [5]\n") + self.assertEquals(self._error_messages[-1], + "Suppressing further [whitespace/tab] reports " + "for this file.\n") # Third call: no report. - self._error_messages = "" - self._call(error_handler, options, confidence) + self._call_error_handler(error_handler, confidence) self.assertEquals(3, self._error_count) - self.assertEquals(self._error_messages, "") - - -class PatchStyleErrorHandlerTest(StyleErrorHandlerTestBase): + self.assertEquals(3, len(self._error_messages)) - """Tests PatchStyleErrorHandler class.""" - - _file_path = "__init__.py" - - _patch_string = """diff --git a/__init__.py b/__init__.py -index ef65bee..e3db70e 100644 ---- a/__init__.py -+++ b/__init__.py -@@ -1 +1,2 @@ - # Required for Python to search this directory for module files -+# New line - -""" - - def test_call(self): - patch_files = parse_patch(self._patch_string) - diff = patch_files[self._file_path] - - options = ProcessorOptions(verbosity=3) - - handle_error = PatchStyleErrorHandler(diff, - self._file_path, - options, - self._mock_increment_error_count, - self._mock_stderr_write) - - category = "whitespace/tab" + def test_line_numbers(self): + """Test the line_numbers parameter.""" + self._check_initialized() + configuration = self._style_checker_configuration() + error_handler = self._error_handler(configuration, + line_numbers=[50]) confidence = 5 - message = "message" - - # Confirm error is reportable. - self.assertTrue(options.is_reportable(category, - confidence, - self._file_path)) - - # Confirm error count initialized to zero. - self.assertEquals(0, self._error_count) - # Test error in unmodified line (error count does not increment). - handle_error(1, category, confidence, message) + # Error on non-modified line: no error. + self._call_error_handler(error_handler, confidence, line_number=60) self.assertEquals(0, self._error_count) + self.assertEquals([], self._error_messages) - # Test error in modified line (error count increments). - handle_error(2, category, confidence, message) + # Error on modified line: error. + self._call_error_handler(error_handler, confidence, line_number=50) self.assertEquals(1, self._error_count) - + self.assertEquals(self._error_messages, + ["foo.h(50): message [whitespace/tab] [5]\n"]) diff --git a/WebKitTools/Scripts/webkitpy/style/filereader.py b/WebKitTools/Scripts/webkitpy/style/filereader.py new file mode 100644 index 0000000..48455b3 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/filereader.py @@ -0,0 +1,148 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# Copyright (C) 2010 ProFUSION embedded systems +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports reading and processing text files.""" + +import codecs +import logging +import os +import sys + + +_log = logging.getLogger(__name__) + + +class TextFileReader(object): + + """Supports reading and processing text files. + + Attributes: + file_count: The total number of files passed to this instance + for processing, including non-text files and files + that should be skipped. + + """ + + def __init__(self, processor): + """Create an instance. + + Arguments: + processor: A ProcessorBase instance. + + """ + self._processor = processor + self.file_count = 0 + + def _read_lines(self, file_path): + """Read the file at a path, and return its lines. + + Raises: + IOError: If the file does not exist or cannot be read. + + """ + # Support the UNIX convention of using "-" for stdin. + if file_path == '-': + file = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace') + else: + # We do not open the file with universal newline support + # (codecs does not support it anyway), so the resulting + # lines contain trailing "\r" characters if we are reading + # a file with CRLF endings. + file = codecs.open(file_path, 'r', 'utf8', 'replace') + + try: + contents = file.read() + finally: + file.close() + + lines = contents.split('\n') + return lines + + def process_file(self, file_path, **kwargs): + """Process the given file by calling the processor's process() method. + + Args: + file_path: The path of the file to process. + **kwargs: Any additional keyword parameters that should be passed + to the processor's process() method. The process() + method should support these keyword arguments. + + Raises: + SystemExit: If no file at file_path exists. + + """ + self.file_count += 1 + + if not os.path.exists(file_path) and file_path != "-": + _log.error("File does not exist: '%s'" % file_path) + sys.exit(1) + + if not self._processor.should_process(file_path): + _log.debug("Skipping file: '%s'" % file_path) + return + _log.debug("Processing file: '%s'" % file_path) + + try: + lines = self._read_lines(file_path) + except IOError, err: + message = ("Could not read file. Skipping: '%s'\n %s" + % (file_path, err)) + _log.warn(message) + return + + self._processor.process(lines, file_path, **kwargs) + + def _process_directory(self, directory): + """Process all files in the given directory, recursively. + + Args: + directory: A directory path. + + """ + for dir_path, dir_names, file_names in os.walk(directory): + for file_name in file_names: + file_path = os.path.join(dir_path, file_name) + self.process_file(file_path) + + def process_paths(self, paths): + """Process the given file and directory paths. + + Args: + paths: A list of file and directory paths. + + """ + for path in paths: + if os.path.isdir(path): + self._process_directory(directory=path) + else: + self.process_file(path) diff --git a/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py b/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py new file mode 100644 index 0000000..558ec5a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/filereader_unittest.py @@ -0,0 +1,161 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains unit tests for filereader.py.""" + +from __future__ import with_statement + +import codecs +import os +import shutil +import tempfile +import unittest + +from webkitpy.common.system.logtesting import LoggingTestCase +from webkitpy.style.checker import ProcessorBase +from webkitpy.style.filereader import TextFileReader + + +class TextFileReaderTest(LoggingTestCase): + + class MockProcessor(ProcessorBase): + + """A processor for test purposes. + + This processor simply records the parameters passed to its process() + method for later checking by the unittest test methods. + + """ + + def __init__(self): + self.processed = [] + """The parameters passed for all calls to the process() method.""" + + def should_process(self, file_path): + return not file_path.endswith('should_not_process.txt') + + def process(self, lines, file_path, test_kwarg=None): + self.processed.append((lines, file_path, test_kwarg)) + + def setUp(self): + LoggingTestCase.setUp(self) + processor = TextFileReaderTest.MockProcessor() + + temp_dir = tempfile.mkdtemp() + + self._file_reader = TextFileReader(processor) + self._processor = processor + self._temp_dir = temp_dir + + def tearDown(self): + LoggingTestCase.tearDown(self) + shutil.rmtree(self._temp_dir) + + def _create_file(self, rel_path, text, encoding="utf-8"): + """Create a file with given text and return the path to the file.""" + # FIXME: There are better/more secure APIs for creatin tmp file paths. + file_path = os.path.join(self._temp_dir, rel_path) + with codecs.open(file_path, "w", encoding) as file: + file.write(text) + return file_path + + def _passed_to_processor(self): + """Return the parameters passed to MockProcessor.process().""" + return self._processor.processed + + def _assert_file_reader(self, passed_to_processor, file_count): + """Assert the state of the file reader.""" + self.assertEquals(passed_to_processor, self._passed_to_processor()) + self.assertEquals(file_count, self._file_reader.file_count) + + def test_process_file__does_not_exist(self): + try: + self._file_reader.process_file('does_not_exist.txt') + except SystemExit, err: + self.assertEquals(str(err), '1') + else: + self.fail('No Exception raised.') + self._assert_file_reader([], 1) + self.assertLog(["ERROR: File does not exist: 'does_not_exist.txt'\n"]) + + def test_process_file__is_dir(self): + temp_dir = os.path.join(self._temp_dir, 'test_dir') + os.mkdir(temp_dir) + + self._file_reader.process_file(temp_dir) + + # Because the log message below contains exception text, it is + # possible that the text varies across platforms. For this reason, + # we check only the portion of the log message that we control, + # namely the text at the beginning. + log_messages = self.logMessages() + # We remove the message we are looking at to prevent the tearDown() + # from raising an exception when it asserts that no log messages + # remain. + message = log_messages.pop() + + self.assertTrue(message.startswith('WARNING: Could not read file. ' + "Skipping: '%s'\n " % temp_dir)) + + self._assert_file_reader([], 1) + + def test_process_file__should_not_process(self): + file_path = self._create_file('should_not_process.txt', 'contents') + + self._file_reader.process_file(file_path) + self._assert_file_reader([], 1) + + def test_process_file__multiple_lines(self): + file_path = self._create_file('foo.txt', 'line one\r\nline two\n') + + self._file_reader.process_file(file_path) + processed = [(['line one\r', 'line two', ''], file_path, None)] + self._assert_file_reader(processed, 1) + + def test_process_file__file_stdin(self): + file_path = self._create_file('-', 'file contents') + + self._file_reader.process_file(file_path=file_path, test_kwarg='foo') + processed = [(['file contents'], file_path, 'foo')] + self._assert_file_reader(processed, 1) + + def test_process_file__with_kwarg(self): + file_path = self._create_file('foo.txt', 'file contents') + + self._file_reader.process_file(file_path=file_path, test_kwarg='foo') + processed = [(['file contents'], file_path, 'foo')] + self._assert_file_reader(processed, 1) + + def test_process_paths(self): + # We test a list of paths that contains both a file and a directory. + dir = os.path.join(self._temp_dir, 'foo_dir') + os.mkdir(dir) + + file_path1 = self._create_file('file1.txt', 'foo') + + rel_path = os.path.join('foo_dir', 'file2.txt') + file_path2 = self._create_file(rel_path, 'bar') + + self._file_reader.process_paths([dir, file_path1]) + processed = [(['bar'], file_path2, None), + (['foo'], file_path1, None)] + self._assert_file_reader(processed, 2) diff --git a/WebKitTools/Scripts/webkitpy/style/filter.py b/WebKitTools/Scripts/webkitpy/style/filter.py index 19c2f4d..608a9e6 100644 --- a/WebKitTools/Scripts/webkitpy/style/filter.py +++ b/WebKitTools/Scripts/webkitpy/style/filter.py @@ -139,12 +139,8 @@ class FilterConfiguration(object): are appended. The first substring match takes precedence, i.e. only the first match triggers an append. - The "path_rules" value is the tuple of filter + The "path_rules" value is a list of filter rules that can be appended to the base rules. - The value is a tuple rather than a list so it - can be used as a dictionary key. The dictionary - is for caching purposes in the implementation of - this class. user_rules: A list of filter rules that is always appended to the base rules and any path rules. In other @@ -165,11 +161,7 @@ class FilterConfiguration(object): self._path_specific_lower = None """The backing store for self._get_path_specific_lower().""" - # FIXME: Make user rules internal after the FilterConfiguration - # attribute is removed from ProcessorOptions (since at - # that point ArgumentPrinter will no longer need to - # access FilterConfiguration.user_rules). - self.user_rules = user_rules + self._user_rules = user_rules self._path_rules_to_filter = {} """Cached dictionary of path rules to CategoryFilter instance.""" @@ -188,7 +180,7 @@ class FilterConfiguration(object): return False if self._path_specific != other._path_specific: return False - if self.user_rules != other.user_rules: + if self._user_rules != other._user_rules: return False return True @@ -210,22 +202,34 @@ class FilterConfiguration(object): return self._path_specific_lower def _path_rules_from_path(self, path): - """Determine the path-specific rules to use, and return as a tuple.""" + """Determine the path-specific rules to use, and return as a tuple. + + This method returns a tuple rather than a list so the return + value can be passed to _filter_from_path_rules() without change. + + """ path = path.lower() for (sub_paths, path_rules) in self._get_path_specific_lower(): for sub_path in sub_paths: if path.find(sub_path) > -1: - return path_rules + return tuple(path_rules) return () # Default to the empty tuple. def _filter_from_path_rules(self, path_rules): - """Return the CategoryFilter associated to a path rules tuple.""" + """Return the CategoryFilter associated to the given path rules. + + Args: + path_rules: A tuple of path rules. We require a tuple rather + than a list so the value can be used as a dictionary + key in self._path_rules_to_filter. + + """ # We reuse the same CategoryFilter where possible to take # advantage of the caching they do. if path_rules not in self._path_rules_to_filter: rules = list(self._base_rules) # Make a copy rules.extend(path_rules) - rules.extend(self.user_rules) + rules.extend(self._user_rules) self._path_rules_to_filter[path_rules] = _CategoryFilter(rules) return self._path_rules_to_filter[path_rules] diff --git a/WebKitTools/Scripts/webkitpy/style/filter_unittest.py b/WebKitTools/Scripts/webkitpy/style/filter_unittest.py index 84760a5..7b8a5402 100644 --- a/WebKitTools/Scripts/webkitpy/style/filter_unittest.py +++ b/WebKitTools/Scripts/webkitpy/style/filter_unittest.py @@ -157,14 +157,14 @@ class FilterConfigurationTest(unittest.TestCase): # Test that the attributes are getting set correctly. # We use parameter values that are different from the defaults. base_rules = ["-"] - path_specific = [(["path"], ("+a",))] + path_specific = [(["path"], ["+a"])] user_rules = ["+"] config = self._config(base_rules, path_specific, user_rules) self.assertEquals(base_rules, config._base_rules) self.assertEquals(path_specific, config._path_specific) - self.assertEquals(user_rules, config.user_rules) + self.assertEquals(user_rules, config._user_rules) def test_default_arguments(self): # Test that the attributes are getting set correctly to the defaults. @@ -172,7 +172,7 @@ class FilterConfigurationTest(unittest.TestCase): self.assertEquals([], config._base_rules) self.assertEquals([], config._path_specific) - self.assertEquals([], config.user_rules) + self.assertEquals([], config._user_rules) def test_eq(self): """Test __eq__ method.""" @@ -185,7 +185,7 @@ class FilterConfigurationTest(unittest.TestCase): # These parameter values are different from the defaults. base_rules = ["-"] - path_specific = [(["path"], ("+a",))] + path_specific = [(["path"], ["+a"])] user_rules = ["+"] self.assertFalse(config.__eq__(FilterConfiguration( @@ -219,8 +219,8 @@ class FilterConfigurationTest(unittest.TestCase): def test_path_specific(self): """Test effect of path_rules_specifier on should_check().""" base_rules = ["-"] - path_specific = [(["path1"], ("+b",)), - (["path2"], ("+c",))] + path_specific = [(["path1"], ["+b"]), + (["path2"], ["+c"])] user_rules = [] config = self._config(base_rules, path_specific, user_rules) @@ -233,7 +233,7 @@ class FilterConfigurationTest(unittest.TestCase): def test_path_with_different_case(self): """Test a path that differs only in case.""" base_rules = ["-"] - path_specific = [(["Foo/"], ("+whitespace",))] + path_specific = [(["Foo/"], ["+whitespace"])] user_rules = [] config = self._config(base_rules, path_specific, user_rules) diff --git a/WebKitTools/Scripts/webkitpy/style/main.py b/WebKitTools/Scripts/webkitpy/style/main.py new file mode 100644 index 0000000..c933c6d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/main.py @@ -0,0 +1,130 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +import os +import sys + +from webkitpy.common.system.ospath import relpath as _relpath + + +_log = logging.getLogger(__name__) + + +def change_directory(checkout_root, paths, mock_os=None): + """Change the working directory to the WebKit checkout root, if possible. + + If every path in the paths parameter is below the checkout root (or if + the paths parameter is empty or None), this method changes the current + working directory to the checkout root and converts the paths parameter + as described below. + This allows the paths being checked to be displayed relative to the + checkout root, and for path-specific style checks to work as expected. + Path-specific checks include whether files should be skipped, whether + custom style rules should apply to certain files, etc. + If the checkout root is None or the empty string, this method returns + the paths parameter unchanged. + + Returns: + paths: A copy of the paths parameter -- possibly converted, as follows. + If this method changed the current working directory to the + checkout root, then the list is the paths parameter converted to + normalized paths relative to the checkout root. Otherwise, the + paths are not converted. + + Args: + paths: A list of paths to the files that should be checked for style. + This argument can be None or the empty list if a git commit + or all changes under the checkout root should be checked. + checkout_root: The path to the root of the WebKit checkout, or None or + the empty string if no checkout could be detected. + mock_os: A replacement module for unit testing. Defaults to os. + + """ + os_module = os if mock_os is None else mock_os + + if paths is not None: + paths = list(paths) + + if not checkout_root: + if not paths: + raise Exception("The paths parameter must be non-empty if " + "there is no checkout root.") + + # FIXME: Consider trying to detect the checkout root for each file + # being checked rather than only trying to detect the checkout + # root for the current working directory. This would allow + # files to be checked correctly even if the script is being + # run from outside any WebKit checkout. + # + # Moreover, try to find the "source root" for each file + # using path-based heuristics rather than using only the + # presence of a WebKit checkout. For example, we could + # examine parent directories until a directory is found + # containing JavaScriptCore, WebCore, WebKit, WebKitSite, + # and WebKitTools. + # Then log an INFO message saying that a source root not + # in a WebKit checkout was found. This will allow us to check + # the style of non-scm copies of the source tree (e.g. + # nightlies). + _log.warn("WebKit checkout root not found:\n" + " Path-dependent style checks may not work correctly.\n" + " See the help documentation for more info.") + + return paths + + if paths: + # Then try converting all of the paths to paths relative to + # the checkout root. + rel_paths = [] + for path in paths: + rel_path = _relpath(path, checkout_root) + if rel_path is None: + # Then the path is not below the checkout root. Since all + # paths should be interpreted relative to the same root, + # do not interpret any of the paths as relative to the + # checkout root. Interpret all of them relative to the + # current working directory, and do not change the current + # working directory. + _log.warn( +"""Path-dependent style checks may not work correctly: + + One of the given paths is outside the WebKit checkout of the current + working directory: + + Path: %s + Checkout root: %s + + Pass only files below the checkout root to ensure correct results. + See the help documentation for more info. +""" + % (path, checkout_root)) + + return paths + rel_paths.append(rel_path) + # If we got here, the conversion was successful. + paths = rel_paths + + _log.debug("Changing to checkout root: " + checkout_root) + os_module.chdir(checkout_root) + + return paths diff --git a/WebKitTools/Scripts/webkitpy/style/main_unittest.py b/WebKitTools/Scripts/webkitpy/style/main_unittest.py new file mode 100644 index 0000000..fe448f5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/main_unittest.py @@ -0,0 +1,124 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for main.py.""" + +import os +import unittest + +from main import change_directory +from webkitpy.style_references import LogTesting + + +class ChangeDirectoryTest(unittest.TestCase): + + """Tests change_directory().""" + + _original_directory = "/original" + _checkout_root = "/WebKit" + + class _MockOs(object): + + """A mock os module for unit testing.""" + + def __init__(self, test_case): + self._test_case = test_case + self._current_directory = \ + ChangeDirectoryTest._original_directory + + def chdir(self, current_directory): + self._current_directory = current_directory + + def assertCurrentDirectory(self, expected_directory): + self._test_case.assertEquals(expected_directory, + self._current_directory) + + def setUp(self): + self._log = LogTesting.setUp(self) + self._mock_os = self._MockOs(self) + + def tearDown(self): + self._log.tearDown() + + # This method is a convenient wrapper for change_working_directory() that + # passes the mock_os for this unit testing class. + def _change_directory(self, paths, checkout_root): + return change_directory(paths=paths, + checkout_root=checkout_root, + mock_os=self._mock_os) + + def _assert_result(self, actual_return_value, expected_return_value, + expected_log_messages, expected_current_directory): + self.assertEquals(actual_return_value, expected_return_value) + self._log.assertMessages(expected_log_messages) + self._mock_os.assertCurrentDirectory(expected_current_directory) + + def test_checkout_root_none_paths_none(self): + self.assertRaises(Exception, self._change_directory, + checkout_root=None, paths=None) + self._log.assertMessages([]) + self._mock_os.assertCurrentDirectory(self._original_directory) + + def test_checkout_root_none(self): + paths = self._change_directory(checkout_root=None, + paths=["path1"]) + log_messages = [ +"""WARNING: WebKit checkout root not found: + Path-dependent style checks may not work correctly. + See the help documentation for more info. +"""] + self._assert_result(paths, ["path1"], log_messages, + self._original_directory) + + def test_paths_none(self): + paths = self._change_directory(checkout_root=self._checkout_root, + paths=None) + self._assert_result(paths, None, [], self._checkout_root) + + def test_paths_convertible(self): + paths=["/WebKit/foo1.txt", + "/WebKit/foo2.txt"] + paths = self._change_directory(checkout_root=self._checkout_root, + paths=paths) + self._assert_result(paths, ["foo1.txt", "foo2.txt"], [], + self._checkout_root) + + def test_with_scm_paths_unconvertible(self): + paths=["/WebKit/foo1.txt", + "/outside/foo2.txt"] + paths = self._change_directory(checkout_root=self._checkout_root, + paths=paths) + log_messages = [ +"""WARNING: Path-dependent style checks may not work correctly: + + One of the given paths is outside the WebKit checkout of the current + working directory: + + Path: /outside/foo2.txt + Checkout root: /WebKit + + Pass only files below the checkout root to ensure correct results. + See the help documentation for more info. + +"""] + self._assert_result(paths, paths, log_messages, + self._original_directory) diff --git a/WebKitTools/Scripts/webkitpy/style/optparser.py b/WebKitTools/Scripts/webkitpy/style/optparser.py new file mode 100644 index 0000000..3ba0fae --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/optparser.py @@ -0,0 +1,450 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Supports the parsing of command-line options for check-webkit-style.""" + +import logging +from optparse import OptionParser +import os.path +import sys + +from filter import validate_filter_rules +# This module should not import anything from checker.py. + +_log = logging.getLogger(__name__) + +_USAGE = """usage: %prog [--help] [options] [path1] [path2] ... + +Overview: + Check coding style according to WebKit style guidelines: + + http://webkit.org/coding/coding-style.html + + Path arguments can be files and directories. If neither a git commit nor + paths are passed, then all changes in your source control working directory + are checked. + +Style errors: + This script assigns to every style error a confidence score from 1-5 and + a category name. A confidence score of 5 means the error is certainly + a problem, and 1 means it could be fine. + + Category names appear in error messages in brackets, for example + [whitespace/indent]. See the options section below for an option that + displays all available categories and which are reported by default. + +Filters: + Use filters to configure what errors to report. Filters are specified using + a comma-separated list of boolean filter rules. The script reports errors + in a category if the category passes the filter, as described below. + + All categories start out passing. Boolean filter rules are then evaluated + from left to right, with later rules taking precedence. For example, the + rule "+foo" passes any category that starts with "foo", and "-foo" fails + any such category. The filter input "-whitespace,+whitespace/braces" fails + the category "whitespace/tab" and passes "whitespace/braces". + + Examples: --filter=-whitespace,+whitespace/braces + --filter=-whitespace,-runtime/printf,+runtime/printf_format + --filter=-,+build/include_what_you_use + +Paths: + Certain style-checking behavior depends on the paths relative to + the WebKit source root of the files being checked. For example, + certain types of errors may be handled differently for files in + WebKit/gtk/webkit/ (e.g. by suppressing "readability/naming" errors + for files in this directory). + + Consequently, if the path relative to the source root cannot be + determined for a file being checked, then style checking may not + work correctly for that file. This can occur, for example, if no + WebKit checkout can be found, or if the source root can be detected, + but one of the files being checked lies outside the source tree. + + If a WebKit checkout can be detected and all files being checked + are in the source tree, then all paths will automatically be + converted to paths relative to the source root prior to checking. + This is also useful for display purposes. + + Currently, this command can detect the source root only if the + command is run from within a WebKit checkout (i.e. if the current + working directory is below the root of a checkout). In particular, + it is not recommended to run this script from a directory outside + a checkout. + + Running this script from a top-level WebKit source directory and + checking only files in the source tree will ensure that all style + checking behaves correctly -- whether or not a checkout can be + detected. This is because all file paths will already be relative + to the source root and so will not need to be converted.""" + +_EPILOG = ("This script can miss errors and does not substitute for " + "code review.") + + +# This class should not have knowledge of the flag key names. +class DefaultCommandOptionValues(object): + + """Stores the default check-webkit-style command-line options. + + Attributes: + output_format: A string that is the default output format. + min_confidence: An integer that is the default minimum confidence level. + + """ + + def __init__(self, min_confidence, output_format): + self.min_confidence = min_confidence + self.output_format = output_format + + +# This class should not have knowledge of the flag key names. +class CommandOptionValues(object): + + """Stores the option values passed by the user via the command line. + + Attributes: + is_verbose: A boolean value of whether verbose logging is enabled. + + filter_rules: The list of filter rules provided by the user. + These rules are appended to the base rules and + path-specific rules and so take precedence over + the base filter rules, etc. + + git_commit: A string representing the git commit to check. + The default is None. + + min_confidence: An integer between 1 and 5 inclusive that is the + minimum confidence level of style errors to report. + The default is 1, which reports all errors. + + output_format: A string that is the output format. The supported + output formats are "emacs" which emacs can parse + and "vs7" which Microsoft Visual Studio 7 can parse. + + """ + def __init__(self, + filter_rules=None, + git_commit=None, + is_verbose=False, + min_confidence=1, + output_format="emacs"): + if filter_rules is None: + filter_rules = [] + + if (min_confidence < 1) or (min_confidence > 5): + raise ValueError('Invalid "min_confidence" parameter: value ' + "must be an integer between 1 and 5 inclusive. " + 'Value given: "%s".' % min_confidence) + + if output_format not in ("emacs", "vs7"): + raise ValueError('Invalid "output_format" parameter: ' + 'value must be "emacs" or "vs7". ' + 'Value given: "%s".' % output_format) + + self.filter_rules = filter_rules + self.git_commit = git_commit + self.is_verbose = is_verbose + self.min_confidence = min_confidence + self.output_format = output_format + + # Useful for unit testing. + def __eq__(self, other): + """Return whether this instance is equal to another.""" + if self.filter_rules != other.filter_rules: + return False + if self.git_commit != other.git_commit: + return False + if self.is_verbose != other.is_verbose: + return False + if self.min_confidence != other.min_confidence: + return False + if self.output_format != other.output_format: + return False + + return True + + # Useful for unit testing. + def __ne__(self, other): + # Python does not automatically deduce this from __eq__(). + return not self.__eq__(other) + + +class ArgumentPrinter(object): + + """Supports the printing of check-webkit-style command arguments.""" + + def _flag_pair_to_string(self, flag_key, flag_value): + return '--%(key)s=%(val)s' % {'key': flag_key, 'val': flag_value } + + def to_flag_string(self, options): + """Return a flag string of the given CommandOptionValues instance. + + This method orders the flag values alphabetically by the flag key. + + Args: + options: A CommandOptionValues instance. + + """ + flags = {} + flags['min-confidence'] = options.min_confidence + flags['output'] = options.output_format + # Only include the filter flag if user-provided rules are present. + filter_rules = options.filter_rules + if filter_rules: + flags['filter'] = ",".join(filter_rules) + if options.git_commit: + flags['git-commit'] = options.git_commit + + flag_string = '' + # Alphabetizing lets us unit test this method. + for key in sorted(flags.keys()): + flag_string += self._flag_pair_to_string(key, flags[key]) + ' ' + + return flag_string.strip() + + +class ArgumentParser(object): + + # FIXME: Move the documentation of the attributes to the __init__ + # docstring after making the attributes internal. + """Supports the parsing of check-webkit-style command arguments. + + Attributes: + create_usage: A function that accepts a DefaultCommandOptionValues + instance and returns a string of usage instructions. + Defaults to the function that generates the usage + string for check-webkit-style. + default_options: A DefaultCommandOptionValues instance that provides + the default values for options not explicitly + provided by the user. + stderr_write: A function that takes a string as a parameter and + serves as stderr.write. Defaults to sys.stderr.write. + This parameter should be specified only for unit tests. + + """ + + def __init__(self, + all_categories, + default_options, + base_filter_rules=None, + mock_stderr=None, + usage=None): + """Create an ArgumentParser instance. + + Args: + all_categories: The set of all available style categories. + default_options: See the corresponding attribute in the class + docstring. + Keyword Args: + base_filter_rules: The list of filter rules at the beginning of + the list of rules used to check style. This + list has the least precedence when checking + style and precedes any user-provided rules. + The class uses this parameter only for display + purposes to the user. Defaults to the empty list. + create_usage: See the documentation of the corresponding + attribute in the class docstring. + stderr_write: See the documentation of the corresponding + attribute in the class docstring. + + """ + if base_filter_rules is None: + base_filter_rules = [] + stderr = sys.stderr if mock_stderr is None else mock_stderr + if usage is None: + usage = _USAGE + + self._all_categories = all_categories + self._base_filter_rules = base_filter_rules + + # FIXME: Rename these to reflect that they are internal. + self.default_options = default_options + self.stderr_write = stderr.write + + self._parser = self._create_option_parser(stderr=stderr, + usage=usage, + default_min_confidence=self.default_options.min_confidence, + default_output_format=self.default_options.output_format) + + def _create_option_parser(self, stderr, usage, + default_min_confidence, default_output_format): + # Since the epilog string is short, it is not necessary to replace + # the epilog string with a mock epilog string when testing. + # For this reason, we use _EPILOG directly rather than passing it + # as an argument like we do for the usage string. + parser = OptionParser(usage=usage, epilog=_EPILOG) + + filter_help = ('set a filter to control what categories of style ' + 'errors to report. Specify a filter using a comma-' + 'delimited list of boolean filter rules, for example ' + '"--filter -whitespace,+whitespace/braces". To display ' + 'all categories and which are enabled by default, pass ' + """no value (e.g. '-f ""' or '--filter=').""") + parser.add_option("-f", "--filter-rules", metavar="RULES", + dest="filter_value", help=filter_help) + + git_commit_help = ("check all changes in the given commit. " + "Use 'commit_id..' to check all changes after commmit_id") + parser.add_option("-g", "--git-diff", "--git-commit", + metavar="COMMIT", dest="git_commit", help=git_commit_help,) + + min_confidence_help = ("set the minimum confidence of style errors " + "to report. Can be an integer 1-5, with 1 " + "displaying all errors. Defaults to %default.") + parser.add_option("-m", "--min-confidence", metavar="INT", + type="int", dest="min_confidence", + default=default_min_confidence, + help=min_confidence_help) + + output_format_help = ('set the output format, which can be "emacs" ' + 'or "vs7" (for Visual Studio). ' + 'Defaults to "%default".') + parser.add_option("-o", "--output-format", metavar="FORMAT", + choices=["emacs", "vs7"], + dest="output_format", default=default_output_format, + help=output_format_help) + + verbose_help = "enable verbose logging." + parser.add_option("-v", "--verbose", dest="is_verbose", default=False, + action="store_true", help=verbose_help) + + # Override OptionParser's error() method so that option help will + # also display when an error occurs. Normally, just the usage + # string displays and not option help. + parser.error = self._parse_error + + # Override OptionParser's print_help() method so that help output + # does not render to the screen while running unit tests. + print_help = parser.print_help + parser.print_help = lambda: print_help(file=stderr) + + return parser + + def _parse_error(self, error_message): + """Print the help string and an error message, and exit.""" + # The method format_help() includes both the usage string and + # the flag options. + help = self._parser.format_help() + # Separate help from the error message with a single blank line. + self.stderr_write(help + "\n") + if error_message: + _log.error(error_message) + + # Since we are using this method to replace/override the Python + # module optparse's OptionParser.error() method, we match its + # behavior and exit with status code 2. + # + # As additional background, Python documentation says-- + # + # "Unix programs generally use 2 for command line syntax errors + # and 1 for all other kind of errors." + # + # (from http://docs.python.org/library/sys.html#sys.exit ) + sys.exit(2) + + def _exit_with_categories(self): + """Exit and print the style categories and default filter rules.""" + self.stderr_write('\nAll categories:\n') + for category in sorted(self._all_categories): + self.stderr_write(' ' + category + '\n') + + self.stderr_write('\nDefault filter rules**:\n') + for filter_rule in sorted(self._base_filter_rules): + self.stderr_write(' ' + filter_rule + '\n') + self.stderr_write('\n**The command always evaluates the above rules, ' + 'and before any --filter flag.\n\n') + + sys.exit(0) + + def _parse_filter_flag(self, flag_value): + """Parse the --filter flag, and return a list of filter rules. + + Args: + flag_value: A string of comma-separated filter rules, for + example "-whitespace,+whitespace/indent". + + """ + filters = [] + for uncleaned_filter in flag_value.split(','): + filter = uncleaned_filter.strip() + if not filter: + continue + filters.append(filter) + return filters + + def parse(self, args): + """Parse the command line arguments to check-webkit-style. + + Args: + args: A list of command-line arguments as returned by sys.argv[1:]. + + Returns: + A tuple of (paths, options) + + paths: The list of paths to check. + options: A CommandOptionValues instance. + + """ + (options, paths) = self._parser.parse_args(args=args) + + filter_value = options.filter_value + git_commit = options.git_commit + is_verbose = options.is_verbose + min_confidence = options.min_confidence + output_format = options.output_format + + if filter_value is not None and not filter_value: + # Then the user explicitly passed no filter, for + # example "-f ''" or "--filter=". + self._exit_with_categories() + + # Validate user-provided values. + + if paths and git_commit: + self._parse_error('You cannot provide both paths and a git ' + 'commit at the same time.') + + min_confidence = int(min_confidence) + if (min_confidence < 1) or (min_confidence > 5): + self._parse_error('option --min-confidence: invalid integer: ' + '%s: value must be between 1 and 5' + % min_confidence) + + if filter_value: + filter_rules = self._parse_filter_flag(filter_value) + else: + filter_rules = [] + + try: + validate_filter_rules(filter_rules, self._all_categories) + except ValueError, err: + self._parse_error(err) + + options = CommandOptionValues(filter_rules=filter_rules, + git_commit=git_commit, + is_verbose=is_verbose, + min_confidence=min_confidence, + output_format=output_format) + + return (paths, options) + diff --git a/WebKitTools/Scripts/webkitpy/style/optparser_unittest.py b/WebKitTools/Scripts/webkitpy/style/optparser_unittest.py new file mode 100644 index 0000000..b7e3eda --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/optparser_unittest.py @@ -0,0 +1,260 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for parser.py.""" + +import unittest + +from webkitpy.common.system.logtesting import LoggingTestCase +from webkitpy.style.optparser import ArgumentParser +from webkitpy.style.optparser import ArgumentPrinter +from webkitpy.style.optparser import CommandOptionValues as ProcessorOptions +from webkitpy.style.optparser import DefaultCommandOptionValues + + +class ArgumentPrinterTest(unittest.TestCase): + + """Tests the ArgumentPrinter class.""" + + _printer = ArgumentPrinter() + + def _create_options(self, + output_format='emacs', + min_confidence=3, + filter_rules=[], + git_commit=None): + return ProcessorOptions(filter_rules=filter_rules, + git_commit=git_commit, + min_confidence=min_confidence, + output_format=output_format) + + def test_to_flag_string(self): + options = self._create_options('vs7', 5, ['+foo', '-bar'], 'git') + self.assertEquals('--filter=+foo,-bar --git-commit=git ' + '--min-confidence=5 --output=vs7', + self._printer.to_flag_string(options)) + + # This is to check that --filter and --git-commit do not + # show up when not user-specified. + options = self._create_options() + self.assertEquals('--min-confidence=3 --output=emacs', + self._printer.to_flag_string(options)) + + +class ArgumentParserTest(LoggingTestCase): + + """Test the ArgumentParser class.""" + + class _MockStdErr(object): + + def write(self, message): + # We do not want the usage string or style categories + # to print during unit tests, so print nothing. + return + + def _parse(self, args): + """Call a test parser.parse().""" + parser = self._create_parser() + return parser.parse(args) + + def _create_defaults(self): + """Return a DefaultCommandOptionValues instance for testing.""" + base_filter_rules = ["-", "+whitespace"] + return DefaultCommandOptionValues(min_confidence=3, + output_format="vs7") + + def _create_parser(self): + """Return an ArgumentParser instance for testing.""" + default_options = self._create_defaults() + + all_categories = ["build" ,"whitespace"] + + mock_stderr = self._MockStdErr() + + return ArgumentParser(all_categories=all_categories, + base_filter_rules=[], + default_options=default_options, + mock_stderr=mock_stderr, + usage="test usage") + + def test_parse_documentation(self): + parse = self._parse + + # FIXME: Test both the printing of the usage string and the + # filter categories help. + + # Request the usage string. + self.assertRaises(SystemExit, parse, ['--help']) + # Request default filter rules and available style categories. + self.assertRaises(SystemExit, parse, ['--filter=']) + + def test_parse_bad_values(self): + parse = self._parse + + # Pass an unsupported argument. + self.assertRaises(SystemExit, parse, ['--bad']) + self.assertLog(['ERROR: no such option: --bad\n']) + + self.assertRaises(SystemExit, parse, ['--min-confidence=bad']) + self.assertLog(['ERROR: option --min-confidence: ' + "invalid integer value: 'bad'\n"]) + self.assertRaises(SystemExit, parse, ['--min-confidence=0']) + self.assertLog(['ERROR: option --min-confidence: invalid integer: 0: ' + 'value must be between 1 and 5\n']) + self.assertRaises(SystemExit, parse, ['--min-confidence=6']) + self.assertLog(['ERROR: option --min-confidence: invalid integer: 6: ' + 'value must be between 1 and 5\n']) + parse(['--min-confidence=1']) # works + parse(['--min-confidence=5']) # works + + self.assertRaises(SystemExit, parse, ['--output=bad']) + self.assertLog(['ERROR: option --output-format: invalid choice: ' + "'bad' (choose from 'emacs', 'vs7')\n"]) + parse(['--output=vs7']) # works + + # Pass a filter rule not beginning with + or -. + self.assertRaises(SystemExit, parse, ['--filter=build']) + self.assertLog(['ERROR: Invalid filter rule "build": ' + 'every rule must start with + or -.\n']) + parse(['--filter=+build']) # works + # Pass files and git-commit at the same time. + self.assertRaises(SystemExit, parse, ['--git-commit=committish', + 'file.txt']) + self.assertLog(['ERROR: You cannot provide both paths and ' + 'a git commit at the same time.\n']) + + def test_parse_default_arguments(self): + parse = self._parse + + (files, options) = parse([]) + + self.assertEquals(files, []) + + self.assertEquals(options.filter_rules, []) + self.assertEquals(options.git_commit, None) + self.assertEquals(options.is_verbose, False) + self.assertEquals(options.min_confidence, 3) + self.assertEquals(options.output_format, 'vs7') + + def test_parse_explicit_arguments(self): + parse = self._parse + + # Pass non-default explicit values. + (files, options) = parse(['--min-confidence=4']) + self.assertEquals(options.min_confidence, 4) + (files, options) = parse(['--output=emacs']) + self.assertEquals(options.output_format, 'emacs') + (files, options) = parse(['-g', 'commit']) + self.assertEquals(options.git_commit, 'commit') + (files, options) = parse(['--git-commit=commit']) + self.assertEquals(options.git_commit, 'commit') + (files, options) = parse(['--git-diff=commit']) + self.assertEquals(options.git_commit, 'commit') + (files, options) = parse(['--verbose']) + self.assertEquals(options.is_verbose, True) + + # Pass user_rules. + (files, options) = parse(['--filter=+build,-whitespace']) + self.assertEquals(options.filter_rules, + ["+build", "-whitespace"]) + + # Pass spurious white space in user rules. + (files, options) = parse(['--filter=+build, -whitespace']) + self.assertEquals(options.filter_rules, + ["+build", "-whitespace"]) + + def test_parse_files(self): + parse = self._parse + + (files, options) = parse(['foo.cpp']) + self.assertEquals(files, ['foo.cpp']) + + # Pass multiple files. + (files, options) = parse(['--output=emacs', 'foo.cpp', 'bar.cpp']) + self.assertEquals(files, ['foo.cpp', 'bar.cpp']) + + +class CommandOptionValuesTest(unittest.TestCase): + + """Tests CommandOptionValues class.""" + + def test_init(self): + """Test __init__ constructor.""" + # Check default parameters. + options = ProcessorOptions() + self.assertEquals(options.filter_rules, []) + self.assertEquals(options.git_commit, None) + self.assertEquals(options.is_verbose, False) + self.assertEquals(options.min_confidence, 1) + self.assertEquals(options.output_format, "emacs") + + # Check argument validation. + self.assertRaises(ValueError, ProcessorOptions, output_format="bad") + ProcessorOptions(output_format="emacs") # No ValueError: works + ProcessorOptions(output_format="vs7") # works + self.assertRaises(ValueError, ProcessorOptions, min_confidence=0) + self.assertRaises(ValueError, ProcessorOptions, min_confidence=6) + ProcessorOptions(min_confidence=1) # works + ProcessorOptions(min_confidence=5) # works + + # Check attributes. + options = ProcessorOptions(filter_rules=["+"], + git_commit="commit", + is_verbose=True, + min_confidence=3, + output_format="vs7") + self.assertEquals(options.filter_rules, ["+"]) + self.assertEquals(options.git_commit, "commit") + self.assertEquals(options.is_verbose, True) + self.assertEquals(options.min_confidence, 3) + self.assertEquals(options.output_format, "vs7") + + def test_eq(self): + """Test __eq__ equality function.""" + self.assertTrue(ProcessorOptions().__eq__(ProcessorOptions())) + + # Also verify that a difference in any argument causes equality to fail. + + # Explicitly create a ProcessorOptions instance with all default + # values. We do this to be sure we are assuming the right default + # values in our self.assertFalse() calls below. + options = ProcessorOptions(filter_rules=[], + git_commit=None, + is_verbose=False, + min_confidence=1, + output_format="emacs") + # Verify that we created options correctly. + self.assertTrue(options.__eq__(ProcessorOptions())) + + self.assertFalse(options.__eq__(ProcessorOptions(filter_rules=["+"]))) + self.assertFalse(options.__eq__(ProcessorOptions(git_commit="commit"))) + self.assertFalse(options.__eq__(ProcessorOptions(is_verbose=True))) + self.assertFalse(options.__eq__(ProcessorOptions(min_confidence=2))) + self.assertFalse(options.__eq__(ProcessorOptions(output_format="vs7"))) + + def test_ne(self): + """Test __ne__ inequality function.""" + # By default, __ne__ always returns true on different objects. + # Thus, just check the distinguishing case to verify that the + # code defines __ne__. + self.assertFalse(ProcessorOptions().__ne__(ProcessorOptions())) + diff --git a/WebKitTools/Scripts/webkitpy/style/patchreader.py b/WebKitTools/Scripts/webkitpy/style/patchreader.py new file mode 100644 index 0000000..7ba2b66 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/patchreader.py @@ -0,0 +1,75 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# Copyright (C) 2010 ProFUSION embedded systems +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging + +from webkitpy.common.checkout.diff_parser import DiffParser + + +_log = logging.getLogger("webkitpy.style.patchreader") + + +class PatchReader(object): + + """Supports checking style in patches.""" + + def __init__(self, text_file_reader): + """Create a PatchReader instance. + + Args: + text_file_reader: A TextFileReader instance. + + """ + self._text_file_reader = text_file_reader + + def check(self, patch_string): + """Check style in the given patch.""" + patch_files = DiffParser(patch_string.splitlines()).files + + # The diff variable is a DiffFile instance. + for path, diff in patch_files.iteritems(): + line_numbers = set() + for line in diff.lines: + # When deleted line is not set, it means that + # the line is newly added (or modified). + if not line[0]: + line_numbers.add(line[1]) + + _log.debug('Found %s new or modified lines in: %s' + % (len(line_numbers), path)) + + # If line_numbers is empty, the file has no new or + # modified lines. In this case, we don't check the file + # because we'll never output errors for the file. + # This optimization also prevents the program from exiting + # due to a deleted file. + if line_numbers: + self._text_file_reader.process_file(file_path=path, + line_numbers=line_numbers) diff --git a/WebKitTools/Scripts/webkitpy/style/patchreader_unittest.py b/WebKitTools/Scripts/webkitpy/style/patchreader_unittest.py new file mode 100644 index 0000000..10791e4 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/style/patchreader_unittest.py @@ -0,0 +1,85 @@ +#!/usr/bin/python +# +# Copyright (C) 2010 Google Inc. All rights reserved. +# Copyright (C) 2009 Torch Mobile Inc. +# Copyright (C) 2009 Apple Inc. All rights reserved. +# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.style.patchreader import PatchReader + + +class PatchReaderTest(unittest.TestCase): + + """Test the PatchReader class.""" + + class MockTextFileReader(object): + + def __init__(self): + self.passed_to_process_file = [] + """A list of (file_path, line_numbers) pairs.""" + + def process_file(self, file_path, line_numbers): + self.passed_to_process_file.append((file_path, line_numbers)) + + def setUp(self): + file_reader = self.MockTextFileReader() + self._file_reader = file_reader + self._patch_checker = PatchReader(file_reader) + + def _call_check_patch(self, patch_string): + self._patch_checker.check(patch_string) + + def _assert_checked(self, passed_to_process_file): + self.assertEquals(self._file_reader.passed_to_process_file, + passed_to_process_file) + + def test_check_patch(self): + # The modified line_numbers array for this patch is: [2]. + self._call_check_patch("""diff --git a/__init__.py b/__init__.py +index ef65bee..e3db70e 100644 +--- a/__init__.py ++++ b/__init__.py +@@ -1,1 +1,2 @@ + # Required for Python to search this directory for module files ++# New line +""") + self._assert_checked([("__init__.py", set([2]))]) + + def test_check_patch_with_deletion(self): + self._call_check_patch("""Index: __init__.py +=================================================================== +--- __init__.py (revision 3593) ++++ __init__.py (working copy) +@@ -1 +0,0 @@ +-foobar +""") + # _mock_check_file should not be called for the deletion patch. + self._assert_checked([]) diff --git a/WebKitTools/Scripts/webkitpy/style/unittests.py b/WebKitTools/Scripts/webkitpy/style/unittests.py deleted file mode 100644 index f8e3f71..0000000 --- a/WebKitTools/Scripts/webkitpy/style/unittests.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2010 Chris Jerdonek (chris.jerdonek@gmail.com) -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Apple Computer, Inc. ("Apple") nor the names of -# its contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Runs style package unit tests.""" - -# This module is imported by test-webkitpy. - -import sys -import unittest - -from checker_unittest import * -from error_handlers_unittest import * -from filter_unittest import * -from processors.common_unittest import * -from processors.cpp_unittest import * -from processors.text_unittest import * diff --git a/WebKitTools/Scripts/webkitpy/style_references.py b/WebKitTools/Scripts/webkitpy/style_references.py index 2528c4d..a42b69d 100644 --- a/WebKitTools/Scripts/webkitpy/style_references.py +++ b/WebKitTools/Scripts/webkitpy/style_references.py @@ -40,33 +40,33 @@ import os -from diff_parser import DiffParser -from scm import detect_scm_system +from webkitpy.common.checkout.diff_parser import DiffParser +from webkitpy.common.system.logtesting import LogTesting +from webkitpy.common.system.logtesting import TestLogStream +from webkitpy.common.system.logutils import configure_logging +from webkitpy.common.checkout.scm import detect_scm_system +from webkitpy.thirdparty.autoinstalled import pep8 -def parse_patch(patch_string): +def detect_checkout(): + """Return a WebKitCheckout instance, or None if it cannot be found.""" + cwd = os.path.abspath(os.curdir) + scm = detect_scm_system(cwd) - """Parse a patch string and return the affected files.""" + return None if scm is None else WebKitCheckout(scm) - patch = DiffParser(patch_string.splitlines()) - return patch.files +class WebKitCheckout(object): -class SimpleScm(object): + """Simple facade to the SCM class for use by style package.""" - """Simple facade to SCM for use by style package.""" + def __init__(self, scm): + self._scm = scm - def __init__(self): - cwd = os.path.abspath('.') - self._scm = detect_scm_system(cwd) - - def checkout_root(self): - """Return the source control root as an absolute path.""" + def root_path(self): + """Return the checkout root as an absolute path.""" return self._scm.checkout_root - def create_patch(self): - return self._scm.create_patch() - - def create_patch_since_local_commit(self, commit): - return self._scm.create_patch_since_local_commit(commit) + def create_patch(self, git_commit): + return self._scm.create_patch(git_commit) diff --git a/WebKitTools/Scripts/webkitpy/test/__init__.py b/WebKitTools/Scripts/webkitpy/test/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/test/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/test/main.py b/WebKitTools/Scripts/webkitpy/test/main.py new file mode 100644 index 0000000..daf255f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/test/main.py @@ -0,0 +1,129 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains the entry method for test-webkitpy.""" + +import logging +import os +import sys +import unittest + +import webkitpy + + +_log = logging.getLogger(__name__) + + +class Tester(object): + + """Discovers and runs webkitpy unit tests.""" + + def _find_unittest_files(self, webkitpy_dir): + """Return a list of paths to all unit-test files.""" + unittest_paths = [] # Return value. + + for dir_path, dir_names, file_names in os.walk(webkitpy_dir): + for file_name in file_names: + if not file_name.endswith("_unittest.py"): + continue + unittest_path = os.path.join(dir_path, file_name) + unittest_paths.append(unittest_path) + + return unittest_paths + + def _modules_from_paths(self, webkitpy_dir, paths): + """Return a list of fully-qualified module names given paths.""" + webkitpy_dir = os.path.abspath(webkitpy_dir) + webkitpy_name = os.path.split(webkitpy_dir)[1] # Equals "webkitpy". + + prefix_length = len(webkitpy_dir) + + modules = [] + for path in paths: + path = os.path.abspath(path) + # This gives us, for example: /common/config/ports_unittest.py + rel_path = path[prefix_length:] + # This gives us, for example: /common/config/ports_unittest + rel_path = os.path.splitext(rel_path)[0] + + parts = [] + while True: + (rel_path, tail) = os.path.split(rel_path) + if not tail: + break + parts.insert(0, tail) + # We now have, for example: common.config.ports_unittest + parts.insert(0, webkitpy_name) # Put "webkitpy" at the beginning. + module = ".".join(parts) + modules.append(module) + + return modules + + def run_tests(self, sys_argv): + """Run the unit tests in all *_unittest.py modules in webkitpy. + + This method excludes "webkitpy.common.checkout.scm_unittest" unless + the --all option is the second element of sys_argv. + + Args: + sys_argv: A reference to sys.argv. + + """ + if len(sys_argv) > 1 and not sys_argv[-1].startswith("-"): + # Then explicit modules or test names were provided, which + # the unittest module is equipped to handle. + unittest.main(argv=sys_argv, module=None) + # No need to return since unitttest.main() exits. + + # Otherwise, auto-detect all unit tests. + + webkitpy_dir = os.path.dirname(webkitpy.__file__) + unittest_paths = self._find_unittest_files(webkitpy_dir) + + modules = self._modules_from_paths(webkitpy_dir, unittest_paths) + modules.sort() + + # This is a sanity check to ensure that the unit-test discovery + # methods are working. + if len(modules) < 1: + raise Exception("No unit-test modules found.") + + for module in modules: + _log.debug("Found: %s" % module) + + # FIXME: This is a hack, but I'm tired of commenting out the test. + # See https://bugs.webkit.org/show_bug.cgi?id=31818 + if len(sys_argv) > 1 and sys.argv[1] == "--all": + sys.argv.remove("--all") + else: + excluded_module = "webkitpy.common.checkout.scm_unittest" + _log.info("Excluding: %s (use --all to include)" % excluded_module) + modules.remove(excluded_module) + + sys_argv.extend(modules) + + # We pass None for the module because we do not want the unittest + # module to resolve module names relative to a given module. + # (This would require importing all of the unittest modules from + # this module.) See the loadTestsFromName() method of the + # unittest.TestLoader class for more details on this parameter. + unittest.main(argv=sys_argv, module=None) diff --git a/WebKitTools/Scripts/webkitpy/BeautifulSoup.py b/WebKitTools/Scripts/webkitpy/thirdparty/BeautifulSoup.py index 34204e7..34204e7 100644 --- a/WebKitTools/Scripts/webkitpy/BeautifulSoup.py +++ b/WebKitTools/Scripts/webkitpy/thirdparty/BeautifulSoup.py diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py b/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py new file mode 100644 index 0000000..1cb554a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/__init__.py @@ -0,0 +1,106 @@ +# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This module is required for Python to treat this directory as a package. + +"""Autoinstalls third-party code required by WebKit.""" + +from __future__ import with_statement + +import codecs +import os + +from webkitpy.common.system.autoinstall import AutoInstaller + +# Putting the autoinstall code into webkitpy/thirdparty/__init__.py +# ensures that no autoinstalling occurs until a caller imports from +# webkitpy.thirdparty. This is useful if the caller wants to configure +# logging prior to executing autoinstall code. + +# FIXME: Ideally, a package should be autoinstalled only if the caller +# attempts to import from that individual package. This would +# make autoinstalling lazier than it is currently. This can +# perhaps be done using Python's import hooks as the original +# autoinstall implementation did. + +# FIXME: If any of these servers is offline, webkit-patch breaks (and maybe +# other scripts do, too). See <http://webkit.org/b/42080>. + +# We put auto-installed third-party modules in this directory-- +# +# webkitpy/thirdparty/autoinstalled +thirdparty_dir = os.path.dirname(__file__) +autoinstalled_dir = os.path.join(thirdparty_dir, "autoinstalled") + +# We need to download ClientForm since the mechanize package that we download +# below requires it. The mechanize package uses ClientForm, for example, +# in _html.py. Since mechanize imports ClientForm in the following way, +# +# > import sgmllib, ClientForm +# +# the search path needs to include ClientForm. We put ClientForm in +# its own directory so that we can include it in the search path without +# including other modules as a side effect. +clientform_dir = os.path.join(autoinstalled_dir, "clientform") +installer = AutoInstaller(append_to_search_path=True, + target_dir=clientform_dir) +installer.install(url="http://pypi.python.org/packages/source/C/ClientForm/ClientForm-0.2.10.zip", + url_subpath="ClientForm.py") + +# The remaining packages do not need to be in the search path, so we create +# a new AutoInstaller instance that does not append to the search path. +installer = AutoInstaller(target_dir=autoinstalled_dir) + +installer.install(url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.1.11.zip", + url_subpath="mechanize") +installer.install(url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", + url_subpath="pep8-0.5.0/pep8.py") +installer.install(url="http://www.adambarth.com/webkit/eliza", + target_name="eliza.py") + +rietveld_dir = os.path.join(autoinstalled_dir, "rietveld") +installer = AutoInstaller(target_dir=rietveld_dir) +installer.install(url="http://webkit-rietveld.googlecode.com/svn/trunk/static/upload.py", + target_name="upload.py") + + +# Since irclib and ircbot are two top-level packages, we need to import +# them separately. We group them into an irc package for better +# organization purposes. +irc_dir = os.path.join(autoinstalled_dir, "irc") +installer = AutoInstaller(target_dir=irc_dir) +installer.install(url="http://surfnet.dl.sourceforge.net/project/python-irclib/python-irclib/0.4.8/python-irclib-0.4.8.zip", + url_subpath="irclib.py") +installer.install(url="http://surfnet.dl.sourceforge.net/project/python-irclib/python-irclib/0.4.8/python-irclib-0.4.8.zip", + url_subpath="ircbot.py") + +pywebsocket_dir = os.path.join(autoinstalled_dir, "pywebsocket") +installer = AutoInstaller(target_dir=pywebsocket_dir) +installer.install(url="http://pywebsocket.googlecode.com/files/mod_pywebsocket-0.5.1.tar.gz", + url_subpath="pywebsocket-0.5.1/src/mod_pywebsocket") + +readme_path = os.path.join(autoinstalled_dir, "README") +if not os.path.exists(readme_path): + with codecs.open(readme_path, "w", "ascii") as file: + file.write("This directory is auto-generated by WebKit and is " + "safe to delete.\nIt contains needed third-party Python " + "packages automatically downloaded from the web.") diff --git a/WebKitTools/Scripts/webkitpy/mock.py b/WebKitTools/Scripts/webkitpy/thirdparty/mock.py index f6f328e..015c19e 100644 --- a/WebKitTools/Scripts/webkitpy/mock.py +++ b/WebKitTools/Scripts/webkitpy/thirdparty/mock.py @@ -1,309 +1,309 @@ -# mock.py
-# Test tools for mocking and patching.
-# Copyright (C) 2007-2009 Michael Foord
-# E-mail: fuzzyman AT voidspace DOT org DOT uk
-
-# mock 0.6.0
-# http://www.voidspace.org.uk/python/mock/
-
-# Released subject to the BSD License
-# Please see http://www.voidspace.org.uk/python/license.shtml
-
-# 2009-11-25: Licence downloaded from above URL.
-# BEGIN DOWNLOADED LICENSE
-#
-# Copyright (c) 2003-2009, Michael Foord
-# All rights reserved.
-# E-mail : fuzzyman AT voidspace DOT org DOT uk
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following
-# disclaimer in the documentation and/or other materials provided
-# with the distribution.
-#
-# * Neither the name of Michael Foord nor the name of Voidspace
-# may be used to endorse or promote products derived from this
-# software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# END DOWNLOADED LICENSE
-
-# Scripts maintained at http://www.voidspace.org.uk/python/index.shtml
-# Comments, suggestions and bug reports welcome.
-
-
-__all__ = (
- 'Mock',
- 'patch',
- 'patch_object',
- 'sentinel',
- 'DEFAULT'
-)
-
-__version__ = '0.6.0'
-
-class SentinelObject(object):
- def __init__(self, name):
- self.name = name
-
- def __repr__(self):
- return '<SentinelObject "%s">' % self.name
-
-
-class Sentinel(object):
- def __init__(self):
- self._sentinels = {}
-
- def __getattr__(self, name):
- return self._sentinels.setdefault(name, SentinelObject(name))
-
-
-sentinel = Sentinel()
-
-DEFAULT = sentinel.DEFAULT
-
-class OldStyleClass:
- pass
-ClassType = type(OldStyleClass)
-
-def _is_magic(name):
- return '__%s__' % name[2:-2] == name
-
-def _copy(value):
- if type(value) in (dict, list, tuple, set):
- return type(value)(value)
- return value
-
-
-class Mock(object):
-
- def __init__(self, spec=None, side_effect=None, return_value=DEFAULT,
- name=None, parent=None, wraps=None):
- self._parent = parent
- self._name = name
- if spec is not None and not isinstance(spec, list):
- spec = [member for member in dir(spec) if not _is_magic(member)]
-
- self._methods = spec
- self._children = {}
- self._return_value = return_value
- self.side_effect = side_effect
- self._wraps = wraps
-
- self.reset_mock()
-
-
- def reset_mock(self):
- self.called = False
- self.call_args = None
- self.call_count = 0
- self.call_args_list = []
- self.method_calls = []
- for child in self._children.itervalues():
- child.reset_mock()
- if isinstance(self._return_value, Mock):
- self._return_value.reset_mock()
-
-
- def __get_return_value(self):
- if self._return_value is DEFAULT:
- self._return_value = Mock()
- return self._return_value
-
- def __set_return_value(self, value):
- self._return_value = value
-
- return_value = property(__get_return_value, __set_return_value)
-
-
- def __call__(self, *args, **kwargs):
- self.called = True
- self.call_count += 1
- self.call_args = (args, kwargs)
- self.call_args_list.append((args, kwargs))
-
- parent = self._parent
- name = self._name
- while parent is not None:
- parent.method_calls.append((name, args, kwargs))
- if parent._parent is None:
- break
- name = parent._name + '.' + name
- parent = parent._parent
-
- ret_val = DEFAULT
- if self.side_effect is not None:
- if (isinstance(self.side_effect, Exception) or
- isinstance(self.side_effect, (type, ClassType)) and
- issubclass(self.side_effect, Exception)):
- raise self.side_effect
-
- ret_val = self.side_effect(*args, **kwargs)
- if ret_val is DEFAULT:
- ret_val = self.return_value
-
- if self._wraps is not None and self._return_value is DEFAULT:
- return self._wraps(*args, **kwargs)
- if ret_val is DEFAULT:
- ret_val = self.return_value
- return ret_val
-
-
- def __getattr__(self, name):
- if self._methods is not None:
- if name not in self._methods:
- raise AttributeError("Mock object has no attribute '%s'" % name)
- elif _is_magic(name):
- raise AttributeError(name)
-
- if name not in self._children:
- wraps = None
- if self._wraps is not None:
- wraps = getattr(self._wraps, name)
- self._children[name] = Mock(parent=self, name=name, wraps=wraps)
-
- return self._children[name]
-
-
- def assert_called_with(self, *args, **kwargs):
- assert self.call_args == (args, kwargs), 'Expected: %s\nCalled with: %s' % ((args, kwargs), self.call_args)
-
-
-def _dot_lookup(thing, comp, import_path):
- try:
- return getattr(thing, comp)
- except AttributeError:
- __import__(import_path)
- return getattr(thing, comp)
-
-
-def _importer(target):
- components = target.split('.')
- import_path = components.pop(0)
- thing = __import__(import_path)
-
- for comp in components:
- import_path += ".%s" % comp
- thing = _dot_lookup(thing, comp, import_path)
- return thing
-
-
-class _patch(object):
- def __init__(self, target, attribute, new, spec, create):
- self.target = target
- self.attribute = attribute
- self.new = new
- self.spec = spec
- self.create = create
- self.has_local = False
-
-
- def __call__(self, func):
- if hasattr(func, 'patchings'):
- func.patchings.append(self)
- return func
-
- def patched(*args, **keywargs):
- # don't use a with here (backwards compatability with 2.5)
- extra_args = []
- for patching in patched.patchings:
- arg = patching.__enter__()
- if patching.new is DEFAULT:
- extra_args.append(arg)
- args += tuple(extra_args)
- try:
- return func(*args, **keywargs)
- finally:
- for patching in getattr(patched, 'patchings', []):
- patching.__exit__()
-
- patched.patchings = [self]
- patched.__name__ = func.__name__
- patched.compat_co_firstlineno = getattr(func, "compat_co_firstlineno",
- func.func_code.co_firstlineno)
- return patched
-
-
- def get_original(self):
- target = self.target
- name = self.attribute
- create = self.create
-
- original = DEFAULT
- if _has_local_attr(target, name):
- try:
- original = target.__dict__[name]
- except AttributeError:
- # for instances of classes with slots, they have no __dict__
- original = getattr(target, name)
- elif not create and not hasattr(target, name):
- raise AttributeError("%s does not have the attribute %r" % (target, name))
- return original
-
-
- def __enter__(self):
- new, spec, = self.new, self.spec
- original = self.get_original()
- if new is DEFAULT:
- # XXXX what if original is DEFAULT - shouldn't use it as a spec
- inherit = False
- if spec == True:
- # set spec to the object we are replacing
- spec = original
- if isinstance(spec, (type, ClassType)):
- inherit = True
- new = Mock(spec=spec)
- if inherit:
- new.return_value = Mock(spec=spec)
- self.temp_original = original
- setattr(self.target, self.attribute, new)
- return new
-
-
- def __exit__(self, *_):
- if self.temp_original is not DEFAULT:
- setattr(self.target, self.attribute, self.temp_original)
- else:
- delattr(self.target, self.attribute)
- del self.temp_original
-
-
-def patch_object(target, attribute, new=DEFAULT, spec=None, create=False):
- return _patch(target, attribute, new, spec, create)
-
-
-def patch(target, new=DEFAULT, spec=None, create=False):
- try:
- target, attribute = target.rsplit('.', 1)
- except (TypeError, ValueError):
- raise TypeError("Need a valid target to patch. You supplied: %r" % (target,))
- target = _importer(target)
- return _patch(target, attribute, new, spec, create)
-
-
-
-def _has_local_attr(obj, name):
- try:
- return name in vars(obj)
- except TypeError:
- # objects without a __dict__
- return hasattr(obj, name)
+# mock.py +# Test tools for mocking and patching. +# Copyright (C) 2007-2009 Michael Foord +# E-mail: fuzzyman AT voidspace DOT org DOT uk + +# mock 0.6.0 +# http://www.voidspace.org.uk/python/mock/ + +# Released subject to the BSD License +# Please see http://www.voidspace.org.uk/python/license.shtml + +# 2009-11-25: Licence downloaded from above URL. +# BEGIN DOWNLOADED LICENSE +# +# Copyright (c) 2003-2009, Michael Foord +# All rights reserved. +# E-mail : fuzzyman AT voidspace DOT org DOT uk +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of Michael Foord nor the name of Voidspace +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# END DOWNLOADED LICENSE + +# Scripts maintained at http://www.voidspace.org.uk/python/index.shtml +# Comments, suggestions and bug reports welcome. + + +__all__ = ( + 'Mock', + 'patch', + 'patch_object', + 'sentinel', + 'DEFAULT' +) + +__version__ = '0.6.0' + +class SentinelObject(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return '<SentinelObject "%s">' % self.name + + +class Sentinel(object): + def __init__(self): + self._sentinels = {} + + def __getattr__(self, name): + return self._sentinels.setdefault(name, SentinelObject(name)) + + +sentinel = Sentinel() + +DEFAULT = sentinel.DEFAULT + +class OldStyleClass: + pass +ClassType = type(OldStyleClass) + +def _is_magic(name): + return '__%s__' % name[2:-2] == name + +def _copy(value): + if type(value) in (dict, list, tuple, set): + return type(value)(value) + return value + + +class Mock(object): + + def __init__(self, spec=None, side_effect=None, return_value=DEFAULT, + name=None, parent=None, wraps=None): + self._parent = parent + self._name = name + if spec is not None and not isinstance(spec, list): + spec = [member for member in dir(spec) if not _is_magic(member)] + + self._methods = spec + self._children = {} + self._return_value = return_value + self.side_effect = side_effect + self._wraps = wraps + + self.reset_mock() + + + def reset_mock(self): + self.called = False + self.call_args = None + self.call_count = 0 + self.call_args_list = [] + self.method_calls = [] + for child in self._children.itervalues(): + child.reset_mock() + if isinstance(self._return_value, Mock): + self._return_value.reset_mock() + + + def __get_return_value(self): + if self._return_value is DEFAULT: + self._return_value = Mock() + return self._return_value + + def __set_return_value(self, value): + self._return_value = value + + return_value = property(__get_return_value, __set_return_value) + + + def __call__(self, *args, **kwargs): + self.called = True + self.call_count += 1 + self.call_args = (args, kwargs) + self.call_args_list.append((args, kwargs)) + + parent = self._parent + name = self._name + while parent is not None: + parent.method_calls.append((name, args, kwargs)) + if parent._parent is None: + break + name = parent._name + '.' + name + parent = parent._parent + + ret_val = DEFAULT + if self.side_effect is not None: + if (isinstance(self.side_effect, Exception) or + isinstance(self.side_effect, (type, ClassType)) and + issubclass(self.side_effect, Exception)): + raise self.side_effect + + ret_val = self.side_effect(*args, **kwargs) + if ret_val is DEFAULT: + ret_val = self.return_value + + if self._wraps is not None and self._return_value is DEFAULT: + return self._wraps(*args, **kwargs) + if ret_val is DEFAULT: + ret_val = self.return_value + return ret_val + + + def __getattr__(self, name): + if self._methods is not None: + if name not in self._methods: + raise AttributeError("Mock object has no attribute '%s'" % name) + elif _is_magic(name): + raise AttributeError(name) + + if name not in self._children: + wraps = None + if self._wraps is not None: + wraps = getattr(self._wraps, name) + self._children[name] = Mock(parent=self, name=name, wraps=wraps) + + return self._children[name] + + + def assert_called_with(self, *args, **kwargs): + assert self.call_args == (args, kwargs), 'Expected: %s\nCalled with: %s' % ((args, kwargs), self.call_args) + + +def _dot_lookup(thing, comp, import_path): + try: + return getattr(thing, comp) + except AttributeError: + __import__(import_path) + return getattr(thing, comp) + + +def _importer(target): + components = target.split('.') + import_path = components.pop(0) + thing = __import__(import_path) + + for comp in components: + import_path += ".%s" % comp + thing = _dot_lookup(thing, comp, import_path) + return thing + + +class _patch(object): + def __init__(self, target, attribute, new, spec, create): + self.target = target + self.attribute = attribute + self.new = new + self.spec = spec + self.create = create + self.has_local = False + + + def __call__(self, func): + if hasattr(func, 'patchings'): + func.patchings.append(self) + return func + + def patched(*args, **keywargs): + # don't use a with here (backwards compatability with 2.5) + extra_args = [] + for patching in patched.patchings: + arg = patching.__enter__() + if patching.new is DEFAULT: + extra_args.append(arg) + args += tuple(extra_args) + try: + return func(*args, **keywargs) + finally: + for patching in getattr(patched, 'patchings', []): + patching.__exit__() + + patched.patchings = [self] + patched.__name__ = func.__name__ + patched.compat_co_firstlineno = getattr(func, "compat_co_firstlineno", + func.func_code.co_firstlineno) + return patched + + + def get_original(self): + target = self.target + name = self.attribute + create = self.create + + original = DEFAULT + if _has_local_attr(target, name): + try: + original = target.__dict__[name] + except AttributeError: + # for instances of classes with slots, they have no __dict__ + original = getattr(target, name) + elif not create and not hasattr(target, name): + raise AttributeError("%s does not have the attribute %r" % (target, name)) + return original + + + def __enter__(self): + new, spec, = self.new, self.spec + original = self.get_original() + if new is DEFAULT: + # XXXX what if original is DEFAULT - shouldn't use it as a spec + inherit = False + if spec == True: + # set spec to the object we are replacing + spec = original + if isinstance(spec, (type, ClassType)): + inherit = True + new = Mock(spec=spec) + if inherit: + new.return_value = Mock(spec=spec) + self.temp_original = original + setattr(self.target, self.attribute, new) + return new + + + def __exit__(self, *_): + if self.temp_original is not DEFAULT: + setattr(self.target, self.attribute, self.temp_original) + else: + delattr(self.target, self.attribute) + del self.temp_original + + +def patch_object(target, attribute, new=DEFAULT, spec=None, create=False): + return _patch(target, attribute, new, spec, create) + + +def patch(target, new=DEFAULT, spec=None, create=False): + try: + target, attribute = target.rsplit('.', 1) + except (TypeError, ValueError): + raise TypeError("Need a valid target to patch. You supplied: %r" % (target,)) + target = _importer(target) + return _patch(target, attribute, new, spec, create) + + + +def _has_local_attr(obj, name): + try: + return name in vars(obj) + except TypeError: + # objects without a __dict__ + return hasattr(obj, name) diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/LICENSE.txt b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/LICENSE.txt new file mode 100644 index 0000000..ad95f29 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2006 Bob Ippolito + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/README.txt b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/README.txt new file mode 100644 index 0000000..7f726ce --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/README.txt @@ -0,0 +1,11 @@ +URL: http://undefined.org/python/#simplejson +Version: 1.7.3 +License: MIT +License File: LICENSE.txt + +Description: +simplejson is a JSON encoder and decoder for Python. + + +Local Modifications: +Removed unit tests from current distribution. diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/__init__.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/__init__.py new file mode 100644 index 0000000..38d6229 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/__init__.py @@ -0,0 +1,287 @@ +r""" +A simple, fast, extensible JSON encoder and decoder + +JSON (JavaScript Object Notation) <http://json.org> is a subset of +JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data +interchange format. + +simplejson exposes an API familiar to uses of the standard library +marshal and pickle modules. + +Encoding basic Python object hierarchies:: + + >>> import simplejson + >>> simplejson.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) + '["foo", {"bar": ["baz", null, 1.0, 2]}]' + >>> print simplejson.dumps("\"foo\bar") + "\"foo\bar" + >>> print simplejson.dumps(u'\u1234') + "\u1234" + >>> print simplejson.dumps('\\') + "\\" + >>> print simplejson.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) + {"a": 0, "b": 0, "c": 0} + >>> from StringIO import StringIO + >>> io = StringIO() + >>> simplejson.dump(['streaming API'], io) + >>> io.getvalue() + '["streaming API"]' + +Compact encoding:: + + >>> import simplejson + >>> simplejson.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + '[1,2,3,{"4":5,"6":7}]' + +Pretty printing:: + + >>> import simplejson + >>> print simplejson.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4) + { + "4": 5, + "6": 7 + } + +Decoding JSON:: + + >>> import simplejson + >>> simplejson.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') + [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] + >>> simplejson.loads('"\\"foo\\bar"') + u'"foo\x08ar' + >>> from StringIO import StringIO + >>> io = StringIO('["streaming API"]') + >>> simplejson.load(io) + [u'streaming API'] + +Specializing JSON object decoding:: + + >>> import simplejson + >>> def as_complex(dct): + ... if '__complex__' in dct: + ... return complex(dct['real'], dct['imag']) + ... return dct + ... + >>> simplejson.loads('{"__complex__": true, "real": 1, "imag": 2}', + ... object_hook=as_complex) + (1+2j) + +Extending JSONEncoder:: + + >>> import simplejson + >>> class ComplexEncoder(simplejson.JSONEncoder): + ... def default(self, obj): + ... if isinstance(obj, complex): + ... return [obj.real, obj.imag] + ... return simplejson.JSONEncoder.default(self, obj) + ... + >>> dumps(2 + 1j, cls=ComplexEncoder) + '[2.0, 1.0]' + >>> ComplexEncoder().encode(2 + 1j) + '[2.0, 1.0]' + >>> list(ComplexEncoder().iterencode(2 + 1j)) + ['[', '2.0', ', ', '1.0', ']'] + + +Note that the JSON produced by this module's default settings +is a subset of YAML, so it may be used as a serializer for that as well. +""" +__version__ = '1.7.3' +__all__ = [ + 'dump', 'dumps', 'load', 'loads', + 'JSONDecoder', 'JSONEncoder', +] + +from decoder import JSONDecoder +from encoder import JSONEncoder + +_default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8' +) + +def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', **kw): + """ + Serialize ``obj`` as a JSON formatted stream to ``fp`` (a + ``.write()``-supporting file-like object). + + If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is ``False``, then the some chunks written to ``fp`` + may be ``unicode`` instances, subject to normal Python ``str`` to + ``unicode`` coercion rules. Unless ``fp.write()`` explicitly + understands ``unicode`` (as in ``codecs.getwriter()``) this is likely + to cause an error. + + If ``check_circular`` is ``False``, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is ``False``, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) + in strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a non-negative integer, then JSON array elements and object + members will be pretty-printed with that indent level. An indent level + of 0 will only insert newlines. ``None`` is the most compact representation. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + """ + # cached encoder + if (skipkeys is False and ensure_ascii is True and + check_circular is True and allow_nan is True and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and not kw): + iterable = _default_encoder.iterencode(obj) + else: + if cls is None: + cls = JSONEncoder + iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, **kw).iterencode(obj) + # could accelerate with writelines in some versions of Python, at + # a debuggability cost + for chunk in iterable: + fp.write(chunk) + + +def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', **kw): + """ + Serialize ``obj`` to a JSON formatted ``str``. + + If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is ``False``, then the return value will be a + ``unicode`` instance subject to normal Python ``str`` to ``unicode`` + coercion rules instead of being escaped to an ASCII ``str``. + + If ``check_circular`` is ``False``, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is ``False``, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in + strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a non-negative integer, then JSON array elements and + object members will be pretty-printed with that indent level. An indent + level of 0 will only insert newlines. ``None`` is the most compact + representation. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + """ + # cached encoder + if (skipkeys is False and ensure_ascii is True and + check_circular is True and allow_nan is True and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and not kw): + return _default_encoder.encode(obj) + if cls is None: + cls = JSONEncoder + return cls( + skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, + **kw).encode(obj) + +_default_decoder = JSONDecoder(encoding=None, object_hook=None) + +def load(fp, encoding=None, cls=None, object_hook=None, **kw): + """ + Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + a JSON document) to a Python object. + + If the contents of ``fp`` is encoded with an ASCII based encoding other + than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must + be specified. Encodings that are not ASCII based (such as UCS-2) are + not allowed, and should be wrapped with + ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode`` + object and passed to ``loads()`` + + ``object_hook`` is an optional function that will be called with the + result of any object literal decode (a ``dict``). The return value of + ``object_hook`` will be used instead of the ``dict``. This feature + can be used to implement custom decoders (e.g. JSON-RPC class hinting). + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + """ + return loads(fp.read(), + encoding=encoding, cls=cls, object_hook=object_hook, **kw) + +def loads(s, encoding=None, cls=None, object_hook=None, **kw): + """ + Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + document) to a Python object. + + If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding + other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name + must be specified. Encodings that are not ASCII based (such as UCS-2) + are not allowed and should be decoded to ``unicode`` first. + + ``object_hook`` is an optional function that will be called with the + result of any object literal decode (a ``dict``). The return value of + ``object_hook`` will be used instead of the ``dict``. This feature + can be used to implement custom decoders (e.g. JSON-RPC class hinting). + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + """ + if cls is None and encoding is None and object_hook is None and not kw: + return _default_decoder.decode(s) + if cls is None: + cls = JSONDecoder + if object_hook is not None: + kw['object_hook'] = object_hook + return cls(encoding=encoding, **kw).decode(s) + +def read(s): + """ + json-py API compatibility hook. Use loads(s) instead. + """ + import warnings + warnings.warn("simplejson.loads(s) should be used instead of read(s)", + DeprecationWarning) + return loads(s) + +def write(obj): + """ + json-py API compatibility hook. Use dumps(s) instead. + """ + import warnings + warnings.warn("simplejson.dumps(s) should be used instead of write(s)", + DeprecationWarning) + return dumps(obj) + + diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/_speedups.c b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/_speedups.c new file mode 100644 index 0000000..8f290bb --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/_speedups.c @@ -0,0 +1,215 @@ +#include "Python.h" +#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) +typedef int Py_ssize_t; +#define PY_SSIZE_T_MAX INT_MAX +#define PY_SSIZE_T_MIN INT_MIN +#endif + +static Py_ssize_t +ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars); +static PyObject * +ascii_escape_unicode(PyObject *pystr); +static PyObject * +ascii_escape_str(PyObject *pystr); +static PyObject * +py_encode_basestring_ascii(PyObject* self __attribute__((__unused__)), PyObject *pystr); +void init_speedups(void); + +#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '/' && c != '"') + +#define MIN_EXPANSION 6 +#ifdef Py_UNICODE_WIDE +#define MAX_EXPANSION (2 * MIN_EXPANSION) +#else +#define MAX_EXPANSION MIN_EXPANSION +#endif + +static Py_ssize_t +ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars) { + Py_UNICODE x; + output[chars++] = '\\'; + switch (c) { + case '/': output[chars++] = (char)c; break; + case '\\': output[chars++] = (char)c; break; + case '"': output[chars++] = (char)c; break; + case '\b': output[chars++] = 'b'; break; + case '\f': output[chars++] = 'f'; break; + case '\n': output[chars++] = 'n'; break; + case '\r': output[chars++] = 'r'; break; + case '\t': output[chars++] = 't'; break; + default: +#ifdef Py_UNICODE_WIDE + if (c >= 0x10000) { + /* UTF-16 surrogate pair */ + Py_UNICODE v = c - 0x10000; + c = 0xd800 | ((v >> 10) & 0x3ff); + output[chars++] = 'u'; + x = (c & 0xf000) >> 12; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x0f00) >> 8; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x00f0) >> 4; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x000f); + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + c = 0xdc00 | (v & 0x3ff); + output[chars++] = '\\'; + } +#endif + output[chars++] = 'u'; + x = (c & 0xf000) >> 12; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x0f00) >> 8; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x00f0) >> 4; + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + x = (c & 0x000f); + output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10); + } + return chars; +} + +static PyObject * +ascii_escape_unicode(PyObject *pystr) { + Py_ssize_t i; + Py_ssize_t input_chars; + Py_ssize_t output_size; + Py_ssize_t chars; + PyObject *rval; + char *output; + Py_UNICODE *input_unicode; + + input_chars = PyUnicode_GET_SIZE(pystr); + input_unicode = PyUnicode_AS_UNICODE(pystr); + /* One char input can be up to 6 chars output, estimate 4 of these */ + output_size = 2 + (MIN_EXPANSION * 4) + input_chars; + rval = PyString_FromStringAndSize(NULL, output_size); + if (rval == NULL) { + return NULL; + } + output = PyString_AS_STRING(rval); + chars = 0; + output[chars++] = '"'; + for (i = 0; i < input_chars; i++) { + Py_UNICODE c = input_unicode[i]; + if (S_CHAR(c)) { + output[chars++] = (char)c; + } else { + chars = ascii_escape_char(c, output, chars); + } + if (output_size - chars < (1 + MAX_EXPANSION)) { + /* There's more than four, so let's resize by a lot */ + output_size *= 2; + /* This is an upper bound */ + if (output_size > 2 + (input_chars * MAX_EXPANSION)) { + output_size = 2 + (input_chars * MAX_EXPANSION); + } + if (_PyString_Resize(&rval, output_size) == -1) { + return NULL; + } + output = PyString_AS_STRING(rval); + } + } + output[chars++] = '"'; + if (_PyString_Resize(&rval, chars) == -1) { + return NULL; + } + return rval; +} + +static PyObject * +ascii_escape_str(PyObject *pystr) { + Py_ssize_t i; + Py_ssize_t input_chars; + Py_ssize_t output_size; + Py_ssize_t chars; + PyObject *rval; + char *output; + char *input_str; + + input_chars = PyString_GET_SIZE(pystr); + input_str = PyString_AS_STRING(pystr); + /* One char input can be up to 6 chars output, estimate 4 of these */ + output_size = 2 + (MIN_EXPANSION * 4) + input_chars; + rval = PyString_FromStringAndSize(NULL, output_size); + if (rval == NULL) { + return NULL; + } + output = PyString_AS_STRING(rval); + chars = 0; + output[chars++] = '"'; + for (i = 0; i < input_chars; i++) { + Py_UNICODE c = (Py_UNICODE)input_str[i]; + if (S_CHAR(c)) { + output[chars++] = (char)c; + } else if (c > 0x7F) { + /* We hit a non-ASCII character, bail to unicode mode */ + PyObject *uni; + Py_DECREF(rval); + uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); + if (uni == NULL) { + return NULL; + } + rval = ascii_escape_unicode(uni); + Py_DECREF(uni); + return rval; + } else { + chars = ascii_escape_char(c, output, chars); + } + /* An ASCII char can't possibly expand to a surrogate! */ + if (output_size - chars < (1 + MIN_EXPANSION)) { + /* There's more than four, so let's resize by a lot */ + output_size *= 2; + if (output_size > 2 + (input_chars * MIN_EXPANSION)) { + output_size = 2 + (input_chars * MIN_EXPANSION); + } + if (_PyString_Resize(&rval, output_size) == -1) { + return NULL; + } + output = PyString_AS_STRING(rval); + } + } + output[chars++] = '"'; + if (_PyString_Resize(&rval, chars) == -1) { + return NULL; + } + return rval; +} + +PyDoc_STRVAR(pydoc_encode_basestring_ascii, + "encode_basestring_ascii(basestring) -> str\n" + "\n" + "..." +); + +static PyObject * +py_encode_basestring_ascii(PyObject* self __attribute__((__unused__)), PyObject *pystr) { + /* METH_O */ + if (PyString_Check(pystr)) { + return ascii_escape_str(pystr); + } else if (PyUnicode_Check(pystr)) { + return ascii_escape_unicode(pystr); + } + PyErr_SetString(PyExc_TypeError, "first argument must be a string"); + return NULL; +} + +#define DEFN(n, k) \ + { \ + #n, \ + (PyCFunction)py_ ##n, \ + k, \ + pydoc_ ##n \ + } +static PyMethodDef speedups_methods[] = { + DEFN(encode_basestring_ascii, METH_O), + {} +}; +#undef DEFN + +void +init_speedups(void) +{ + PyObject *m; + m = Py_InitModule4("_speedups", speedups_methods, NULL, NULL, PYTHON_API_VERSION); +} diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/decoder.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/decoder.py new file mode 100644 index 0000000..b887b58 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/decoder.py @@ -0,0 +1,273 @@ +""" +Implementation of JSONDecoder +""" +import re + +from .scanner import Scanner, pattern + +FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL + +def _floatconstants(): + import struct + import sys + _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') + if sys.byteorder != 'big': + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] + nan, inf = struct.unpack('dd', _BYTES) + return nan, inf, -inf + +NaN, PosInf, NegInf = _floatconstants() + +def linecol(doc, pos): + lineno = doc.count('\n', 0, pos) + 1 + if lineno == 1: + colno = pos + else: + colno = pos - doc.rindex('\n', 0, pos) + return lineno, colno + +def errmsg(msg, doc, pos, end=None): + lineno, colno = linecol(doc, pos) + if end is None: + return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos) + endlineno, endcolno = linecol(doc, end) + return '%s: line %d column %d - line %d column %d (char %d - %d)' % ( + msg, lineno, colno, endlineno, endcolno, pos, end) + +_CONSTANTS = { + '-Infinity': NegInf, + 'Infinity': PosInf, + 'NaN': NaN, + 'true': True, + 'false': False, + 'null': None, +} + +def JSONConstant(match, context, c=_CONSTANTS): + return c[match.group(0)], None +pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant) + +def JSONNumber(match, context): + match = JSONNumber.regex.match(match.string, *match.span()) + integer, frac, exp = match.groups() + if frac or exp: + res = float(integer + (frac or '') + (exp or '')) + else: + res = int(integer) + return res, None +pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber) + +STRINGCHUNK = re.compile(r'(.*?)(["\\])', FLAGS) +BACKSLASH = { + '"': u'"', '\\': u'\\', '/': u'/', + 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', +} + +DEFAULT_ENCODING = "utf-8" + +def scanstring(s, end, encoding=None, _b=BACKSLASH, _m=STRINGCHUNK.match): + if encoding is None: + encoding = DEFAULT_ENCODING + chunks = [] + _append = chunks.append + begin = end - 1 + while 1: + chunk = _m(s, end) + if chunk is None: + raise ValueError( + errmsg("Unterminated string starting at", s, begin)) + end = chunk.end() + content, terminator = chunk.groups() + if content: + if not isinstance(content, unicode): + content = unicode(content, encoding) + _append(content) + if terminator == '"': + break + try: + esc = s[end] + except IndexError: + raise ValueError( + errmsg("Unterminated string starting at", s, begin)) + if esc != 'u': + try: + m = _b[esc] + except KeyError: + raise ValueError( + errmsg("Invalid \\escape: %r" % (esc,), s, end)) + end += 1 + else: + esc = s[end + 1:end + 5] + try: + m = unichr(int(esc, 16)) + if len(esc) != 4 or not esc.isalnum(): + raise ValueError + except ValueError: + raise ValueError(errmsg("Invalid \\uXXXX escape", s, end)) + end += 5 + _append(m) + return u''.join(chunks), end + +def JSONString(match, context): + encoding = getattr(context, 'encoding', None) + return scanstring(match.string, match.end(), encoding) +pattern(r'"')(JSONString) + +WHITESPACE = re.compile(r'\s*', FLAGS) + +def JSONObject(match, context, _w=WHITESPACE.match): + pairs = {} + s = match.string + end = _w(s, match.end()).end() + nextchar = s[end:end + 1] + # trivial empty object + if nextchar == '}': + return pairs, end + 1 + if nextchar != '"': + raise ValueError(errmsg("Expecting property name", s, end)) + end += 1 + encoding = getattr(context, 'encoding', None) + iterscan = JSONScanner.iterscan + while True: + key, end = scanstring(s, end, encoding) + end = _w(s, end).end() + if s[end:end + 1] != ':': + raise ValueError(errmsg("Expecting : delimiter", s, end)) + end = _w(s, end + 1).end() + try: + value, end = iterscan(s, idx=end, context=context).next() + except StopIteration: + raise ValueError(errmsg("Expecting object", s, end)) + pairs[key] = value + end = _w(s, end).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == '}': + break + if nextchar != ',': + raise ValueError(errmsg("Expecting , delimiter", s, end - 1)) + end = _w(s, end).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar != '"': + raise ValueError(errmsg("Expecting property name", s, end - 1)) + object_hook = getattr(context, 'object_hook', None) + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end +pattern(r'{')(JSONObject) + +def JSONArray(match, context, _w=WHITESPACE.match): + values = [] + s = match.string + end = _w(s, match.end()).end() + # look-ahead for trivial empty array + nextchar = s[end:end + 1] + if nextchar == ']': + return values, end + 1 + iterscan = JSONScanner.iterscan + while True: + try: + value, end = iterscan(s, idx=end, context=context).next() + except StopIteration: + raise ValueError(errmsg("Expecting object", s, end)) + values.append(value) + end = _w(s, end).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == ']': + break + if nextchar != ',': + raise ValueError(errmsg("Expecting , delimiter", s, end)) + end = _w(s, end).end() + return values, end +pattern(r'\[')(JSONArray) + +ANYTHING = [ + JSONObject, + JSONArray, + JSONString, + JSONConstant, + JSONNumber, +] + +JSONScanner = Scanner(ANYTHING) + +class JSONDecoder(object): + """ + Simple JSON <http://json.org> decoder + + Performs the following translations in decoding: + + +---------------+-------------------+ + | JSON | Python | + +===============+===================+ + | object | dict | + +---------------+-------------------+ + | array | list | + +---------------+-------------------+ + | string | unicode | + +---------------+-------------------+ + | number (int) | int, long | + +---------------+-------------------+ + | number (real) | float | + +---------------+-------------------+ + | true | True | + +---------------+-------------------+ + | false | False | + +---------------+-------------------+ + | null | None | + +---------------+-------------------+ + + It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as + their corresponding ``float`` values, which is outside the JSON spec. + """ + + _scanner = Scanner(ANYTHING) + __all__ = ['__init__', 'decode', 'raw_decode'] + + def __init__(self, encoding=None, object_hook=None): + """ + ``encoding`` determines the encoding used to interpret any ``str`` + objects decoded by this instance (utf-8 by default). It has no + effect when decoding ``unicode`` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as ``unicode``. + + ``object_hook``, if specified, will be called with the result + of every JSON object decoded and its return value will be used in + place of the given ``dict``. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + """ + self.encoding = encoding + self.object_hook = object_hook + + def decode(self, s, _w=WHITESPACE.match): + """ + Return the Python representation of ``s`` (a ``str`` or ``unicode`` + instance containing a JSON document) + """ + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + end = _w(s, end).end() + if end != len(s): + raise ValueError(errmsg("Extra data", s, end, len(s))) + return obj + + def raw_decode(self, s, **kw): + """ + Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning + with a JSON document) and return a 2-tuple of the Python + representation and the index in ``s`` where the document ended. + + This can be used to decode a JSON document from a string that may + have extraneous data at the end. + """ + kw.setdefault('context', self) + try: + obj, end = self._scanner.iterscan(s, **kw).next() + except StopIteration: + raise ValueError("No JSON object could be decoded") + return obj, end + +__all__ = ['JSONDecoder'] diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/encoder.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/encoder.py new file mode 100644 index 0000000..d29919a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/encoder.py @@ -0,0 +1,371 @@ +""" +Implementation of JSONEncoder +""" +import re +try: + from simplejson import _speedups +except ImportError: + _speedups = None + +ESCAPE = re.compile(r'[\x00-\x19\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"/]|[^\ -~])') +ESCAPE_DCT = { + # escape all forward slashes to prevent </script> attack + '/': '\\/', + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +# assume this produces an infinity on all machines (probably not guaranteed) +INFINITY = float('1e66666') + +def floatstr(o, allow_nan=True): + # Check for specials. Note that this type of test is processor- and/or + # platform-specific, so do tests which don't depend on the internals. + + if o != o: + text = 'NaN' + elif o == INFINITY: + text = 'Infinity' + elif o == -INFINITY: + text = '-Infinity' + else: + return repr(o) + + if not allow_nan: + raise ValueError("Out of range float values are not JSON compliant: %r" + % (o,)) + + return text + + +def encode_basestring(s): + """ + Return a JSON representation of a Python string + """ + def replace(match): + return ESCAPE_DCT[match.group(0)] + return '"' + ESCAPE.sub(replace, s) + '"' + +def encode_basestring_ascii(s): + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + +try: + encode_basestring_ascii = _speedups.encode_basestring_ascii + _need_utf8 = True +except AttributeError: + _need_utf8 = False + +class JSONEncoder(object): + """ + Extensible JSON <http://json.org> encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str, unicode | string | + +-------------------+---------------+ + | int, long, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + """ + __all__ = ['__init__', 'default', 'encode', 'iterencode'] + item_separator = ', ' + key_separator = ': ' + def __init__(self, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, sort_keys=False, + indent=None, separators=None, encoding='utf-8'): + """ + Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is False, then it is a TypeError to attempt + encoding of keys that are not str, int, long, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is True, the output is guaranteed to be str + objects with all incoming unicode characters escaped. If + ensure_ascii is false, the output will be unicode object. + + If check_circular is True, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is True, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is True, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a non-negative integer, then JSON array + elements and object members will be pretty-printed with that + indent level. An indent level of 0 will only insert newlines. + None is the most compact representation. + + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. + + If encoding is not None, then all input strings will be + transformed into unicode using that encoding prior to JSON-encoding. + The default is UTF-8. + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.indent = indent + self.current_indent_level = 0 + if separators is not None: + self.item_separator, self.key_separator = separators + self.encoding = encoding + + def _newline_indent(self): + return '\n' + (' ' * (self.indent * self.current_indent_level)) + + def _iterencode_list(self, lst, markers=None): + if not lst: + yield '[]' + return + if markers is not None: + markerid = id(lst) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = lst + yield '[' + if self.indent is not None: + self.current_indent_level += 1 + newline_indent = self._newline_indent() + separator = self.item_separator + newline_indent + yield newline_indent + else: + newline_indent = None + separator = self.item_separator + first = True + for value in lst: + if first: + first = False + else: + yield separator + for chunk in self._iterencode(value, markers): + yield chunk + if newline_indent is not None: + self.current_indent_level -= 1 + yield self._newline_indent() + yield ']' + if markers is not None: + del markers[markerid] + + def _iterencode_dict(self, dct, markers=None): + if not dct: + yield '{}' + return + if markers is not None: + markerid = id(dct) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = dct + yield '{' + key_separator = self.key_separator + if self.indent is not None: + self.current_indent_level += 1 + newline_indent = self._newline_indent() + item_separator = self.item_separator + newline_indent + yield newline_indent + else: + newline_indent = None + item_separator = self.item_separator + first = True + if self.ensure_ascii: + encoder = encode_basestring_ascii + else: + encoder = encode_basestring + allow_nan = self.allow_nan + if self.sort_keys: + keys = dct.keys() + keys.sort() + items = [(k, dct[k]) for k in keys] + else: + items = dct.iteritems() + _encoding = self.encoding + _do_decode = (_encoding is not None + and not (_need_utf8 and _encoding == 'utf-8')) + for key, value in items: + if isinstance(key, str): + if _do_decode: + key = key.decode(_encoding) + elif isinstance(key, basestring): + pass + # JavaScript is weakly typed for these, so it makes sense to + # also allow them. Many encoders seem to do something like this. + elif isinstance(key, float): + key = floatstr(key, allow_nan) + elif isinstance(key, (int, long)): + key = str(key) + elif key is True: + key = 'true' + elif key is False: + key = 'false' + elif key is None: + key = 'null' + elif self.skipkeys: + continue + else: + raise TypeError("key %r is not a string" % (key,)) + if first: + first = False + else: + yield item_separator + yield encoder(key) + yield key_separator + for chunk in self._iterencode(value, markers): + yield chunk + if newline_indent is not None: + self.current_indent_level -= 1 + yield self._newline_indent() + yield '}' + if markers is not None: + del markers[markerid] + + def _iterencode(self, o, markers=None): + if isinstance(o, basestring): + if self.ensure_ascii: + encoder = encode_basestring_ascii + else: + encoder = encode_basestring + _encoding = self.encoding + if (_encoding is not None and isinstance(o, str) + and not (_need_utf8 and _encoding == 'utf-8')): + o = o.decode(_encoding) + yield encoder(o) + elif o is None: + yield 'null' + elif o is True: + yield 'true' + elif o is False: + yield 'false' + elif isinstance(o, (int, long)): + yield str(o) + elif isinstance(o, float): + yield floatstr(o, self.allow_nan) + elif isinstance(o, (list, tuple)): + for chunk in self._iterencode_list(o, markers): + yield chunk + elif isinstance(o, dict): + for chunk in self._iterencode_dict(o, markers): + yield chunk + else: + if markers is not None: + markerid = id(o) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = o + for chunk in self._iterencode_default(o, markers): + yield chunk + if markers is not None: + del markers[markerid] + + def _iterencode_default(self, o, markers=None): + newobj = self.default(o) + return self._iterencode(newobj, markers) + + def default(self, o): + """ + Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + """ + raise TypeError("%r is not JSON serializable" % (o,)) + + def encode(self, o): + """ + Return a JSON string representation of a Python data structure. + + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo":["bar", "baz"]}' + """ + # This is for extremely simple cases and benchmarks... + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8' and _need_utf8)): + o = o.decode(_encoding) + return encode_basestring_ascii(o) + # This doesn't pass the iterator directly to ''.join() because it + # sucks at reporting exceptions. It's going to do this internally + # anyway because it uses PySequence_Fast or similar. + chunks = list(self.iterencode(o)) + return ''.join(chunks) + + def iterencode(self, o): + """ + Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + """ + if self.check_circular: + markers = {} + else: + markers = None + return self._iterencode(o, markers) + +__all__ = ['JSONEncoder'] diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/jsonfilter.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/jsonfilter.py new file mode 100644 index 0000000..01ca21d --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/jsonfilter.py @@ -0,0 +1,40 @@ +import simplejson +import cgi + +class JSONFilter(object): + def __init__(self, app, mime_type='text/x-json'): + self.app = app + self.mime_type = mime_type + + def __call__(self, environ, start_response): + # Read JSON POST input to jsonfilter.json if matching mime type + response = {'status': '200 OK', 'headers': []} + def json_start_response(status, headers): + response['status'] = status + response['headers'].extend(headers) + environ['jsonfilter.mime_type'] = self.mime_type + if environ.get('REQUEST_METHOD', '') == 'POST': + if environ.get('CONTENT_TYPE', '') == self.mime_type: + args = [_ for _ in [environ.get('CONTENT_LENGTH')] if _] + data = environ['wsgi.input'].read(*map(int, args)) + environ['jsonfilter.json'] = simplejson.loads(data) + res = simplejson.dumps(self.app(environ, json_start_response)) + jsonp = cgi.parse_qs(environ.get('QUERY_STRING', '')).get('jsonp') + if jsonp: + content_type = 'text/javascript' + res = ''.join(jsonp + ['(', res, ')']) + elif 'Opera' in environ.get('HTTP_USER_AGENT', ''): + # Opera has bunk XMLHttpRequest support for most mime types + content_type = 'text/plain' + else: + content_type = self.mime_type + headers = [ + ('Content-type', content_type), + ('Content-length', len(res)), + ] + headers.extend(response['headers']) + start_response(response['status'], headers) + return [res] + +def factory(app, global_conf, **kw): + return JSONFilter(app, **kw) diff --git a/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/scanner.py b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/scanner.py new file mode 100644 index 0000000..64f4999 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/thirdparty/simplejson/scanner.py @@ -0,0 +1,63 @@ +""" +Iterator based sre token scanner +""" +import sre_parse, sre_compile, sre_constants +from sre_constants import BRANCH, SUBPATTERN +from re import VERBOSE, MULTILINE, DOTALL +import re + +__all__ = ['Scanner', 'pattern'] + +FLAGS = (VERBOSE | MULTILINE | DOTALL) +class Scanner(object): + def __init__(self, lexicon, flags=FLAGS): + self.actions = [None] + # combine phrases into a compound pattern + s = sre_parse.Pattern() + s.flags = flags + p = [] + for idx, token in enumerate(lexicon): + phrase = token.pattern + try: + subpattern = sre_parse.SubPattern(s, + [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))]) + except sre_constants.error: + raise + p.append(subpattern) + self.actions.append(token) + + p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) + self.scanner = sre_compile.compile(p) + + + def iterscan(self, string, idx=0, context=None): + """ + Yield match, end_idx for each match + """ + match = self.scanner.scanner(string, idx).match + actions = self.actions + lastend = idx + end = len(string) + while True: + m = match() + if m is None: + break + matchbegin, matchend = m.span() + if lastend == matchend: + break + action = actions[m.lastindex] + if action is not None: + rval, next_pos = action(m, context) + if next_pos is not None and next_pos != matchend: + # "fast forward" the scanner + matchend = next_pos + match = self.scanner.scanner(string, matchend).match + yield rval, matchend + lastend = matchend + +def pattern(pattern, flags=FLAGS): + def decorator(fn): + fn.pattern = pattern + fn.regex = re.compile(pattern, flags) + return fn + return decorator diff --git a/WebKitTools/Scripts/webkitpy/tool/__init__.py b/WebKitTools/Scripts/webkitpy/tool/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/__init__.py b/WebKitTools/Scripts/webkitpy/tool/bot/__init__.py new file mode 100644 index 0000000..ef65bee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/irc_command.py b/WebKitTools/Scripts/webkitpy/tool/bot/irc_command.py new file mode 100644 index 0000000..a848472 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/irc_command.py @@ -0,0 +1,109 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import random +import webkitpy.common.config.irc as config_irc + +from webkitpy.common.checkout.changelog import view_source_url +from webkitpy.tool.bot.queueengine import TerminateQueue +from webkitpy.common.net.bugzilla import parse_bug_id +from webkitpy.common.system.executive import ScriptError + +# FIXME: Merge with Command? +class IRCCommand(object): + def execute(self, nick, args, tool, sheriff): + raise NotImplementedError, "subclasses must implement" + + +class LastGreenRevision(IRCCommand): + def execute(self, nick, args, tool, sheriff): + return "%s: %s" % (nick, + view_source_url(tool.buildbot.last_green_revision())) + + +class Restart(IRCCommand): + def execute(self, nick, args, tool, sheriff): + tool.irc().post("Restarting...") + raise TerminateQueue() + + +class Rollout(IRCCommand): + def execute(self, nick, args, tool, sheriff): + if len(args) < 2: + tool.irc().post("%s: Usage: SVN_REVISION REASON" % nick) + return + svn_revision = args[0].lstrip("r") + rollout_reason = " ".join(args[1:]) + tool.irc().post("Preparing rollout for r%s..." % svn_revision) + try: + complete_reason = "%s (Requested by %s on %s)." % ( + rollout_reason, nick, config_irc.channel) + bug_id = sheriff.post_rollout_patch(svn_revision, complete_reason) + bug_url = tool.bugs.bug_url_for_bug_id(bug_id) + tool.irc().post("%s: Created rollout: %s" % (nick, bug_url)) + except ScriptError, e: + tool.irc().post("%s: Failed to create rollout patch:" % nick) + tool.irc().post("%s" % e) + bug_id = parse_bug_id(e.output) + if bug_id: + tool.irc().post("Ugg... Might have created %s" % + tool.bugs.bug_url_for_bug_id(bug_id)) + + +class Help(IRCCommand): + def execute(self, nick, args, tool, sheriff): + return "%s: Available commands: %s" % (nick, ", ".join(commands.keys())) + + +class Hi(IRCCommand): + def execute(self, nick, args, tool, sheriff): + quips = tool.bugs.quips() + quips.append('"Only you can prevent forest fires." -- Smokey the Bear') + return random.choice(quips) + + +class Eliza(IRCCommand): + therapist = None + + def __init__(self): + if not self.therapist: + import webkitpy.thirdparty.autoinstalled.eliza as eliza + Eliza.therapist = eliza.eliza() + + def execute(self, nick, args, tool, sheriff): + return "%s: %s" % (nick, self.therapist.respond(" ".join(args))) + + +# FIXME: Lame. We should have an auto-registering CommandCenter. +commands = { + "last-green-revision": LastGreenRevision, + "restart": Restart, + "rollout": Rollout, + "help": Help, + "hi": Hi, +} diff --git a/WebKitTools/Scripts/webkitpy/steps/commit.py b/WebKitTools/Scripts/webkitpy/tool/bot/irc_command_unittest.py index dd1fed7..7aeb6a0 100644 --- a/WebKitTools/Scripts/webkitpy/steps/commit.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/irc_command_unittest.py @@ -1,9 +1,9 @@ -# Copyright (C) 2010 Google Inc. All rights reserved. -# +# Copyright (c) 2010 Google Inc. All rights reserved. +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above @@ -13,7 +13,7 @@ # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -26,10 +26,13 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep +import unittest +from webkitpy.tool.bot.irc_command import * -class Commit(AbstractStep): - def run(self, state): - commit_message = self._tool.scm().commit_message_for_this_commit() - state["commit_text"] = self._tool.scm().commit_with_message(commit_message.message()) + +class IRCCommandTest(unittest.TestCase): + def test_eliza(self): + eliza = Eliza() + eliza.execute("tom", "hi", None, None) + eliza.execute("tom", "bye", None, None) diff --git a/WebKitTools/Scripts/webkitpy/patchcollection.py b/WebKitTools/Scripts/webkitpy/tool/bot/patchcollection.py index 7e8603c..6100cf8 100644 --- a/WebKitTools/Scripts/webkitpy/patchcollection.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/patchcollection.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -27,6 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + class PersistentPatchCollectionDelegate: def collection_name(self): raise NotImplementedError, "subclasses must implement" @@ -57,9 +57,22 @@ class PersistentPatchCollection: self._status_cache[patch_id] = status return status - def next(self): + def _is_active_patch_id(self, patch_id): + """Active patches are patches waiting to be processed from this collection.""" + status = self._cached_status(patch_id) + return not status or not self._delegate.is_terminal_status(status) + + def _fetch_active_patch_ids(self): patch_ids = self._delegate.fetch_potential_patch_ids() - for patch_id in patch_ids: - status = self._cached_status(patch_id) - if not status or not self._delegate.is_terminal_status(status): - return patch_id + return filter(lambda patch_id: self._is_active_patch_id(patch_id), patch_ids) + + def next(self): + # Note: We only fetch all the ids so we can post them back to the server. + # This will go away once we have a feeder queue and all other queues are + # just pulling their next work item from the server. + patch_ids = self._fetch_active_patch_ids() + # FIXME: We're assuming self._name is a valid queue-name. + self._status.update_work_items(self._name, patch_ids) + if not patch_ids: + return None + return patch_ids[0] diff --git a/WebKitTools/Scripts/webkitpy/patchcollection_unittest.py b/WebKitTools/Scripts/webkitpy/tool/bot/patchcollection_unittest.py index 811fed9..4ec6e25 100644 --- a/WebKitTools/Scripts/webkitpy/patchcollection_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/patchcollection_unittest.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -29,8 +28,8 @@ import unittest -from webkitpy.mock import Mock -from webkitpy.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate +from webkitpy.tool.bot.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate +from webkitpy.thirdparty.mock import Mock class TestPersistentPatchCollectionDelegate(PersistentPatchCollectionDelegate): diff --git a/WebKitTools/Scripts/webkitpy/queueengine.py b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine.py index d14177d..a1a66a1 100644 --- a/WebKitTools/Scripts/webkitpy/queueengine.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -34,9 +33,14 @@ import traceback from datetime import datetime, timedelta -from webkitpy.executive import ScriptError -from webkitpy.webkit_logging import log, OutputTee -from webkitpy.statusserver import StatusServer +from webkitpy.common.net.statusserver import StatusServer +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.system.deprecated_logging import log, OutputTee + + +class TerminateQueue(Exception): + pass + class QueueEngineDelegate: def queue_log_path(self): @@ -66,14 +70,15 @@ class QueueEngineDelegate: class QueueEngine: - def __init__(self, name, delegate): + def __init__(self, name, delegate, wakeup_event): self._name = name self._delegate = delegate + self._wakeup_event = wakeup_event self._output_tee = OutputTee() log_date_format = "%Y-%m-%d %H:%M:%S" - sleep_duration_text = "5 mins" - seconds_to_sleep = 300 + sleep_duration_text = "2 mins" # This could be generated from seconds_to_sleep + seconds_to_sleep = 120 handled_error_code = 2 # Child processes exit with a special code to the parent queue process can detect the error was handled. @@ -101,14 +106,18 @@ class QueueEngine: # This looks fixed, no? self._open_work_log(work_item) try: - self._delegate.process_work_item(work_item) + if not self._delegate.process_work_item(work_item): + self._sleep("Unable to process work item.") except ScriptError, e: # Use a special exit code to indicate that the error was already # handled in the child process and we should just keep looping. if e.exit_code == self.handled_error_code: continue - message = "Unexpected failure when landing patch! Please file a bug against webkit-patch.\n%s" % e.message_with_output() + message = "Unexpected failure when processing patch! Please file a bug against webkit-patch.\n%s" % e.message_with_output() self._delegate.handle_unexpected_error(work_item, message) + except TerminateQueue, e: + log("\nTerminateQueue exception received.") + return 0 except KeyboardInterrupt, e: log("\nUser terminated queue.") return 1 @@ -133,12 +142,15 @@ class QueueEngine: self._output_tee.remove_log(self._work_log) self._work_log = None - @classmethod - def _sleep_message(cls, message): - wake_time = datetime.now() + timedelta(seconds=cls.seconds_to_sleep) - return "%s Sleeping until %s (%s)." % (message, wake_time.strftime(cls.log_date_format), cls.sleep_duration_text) + def _now(self): + """Overriden by the unit tests to allow testing _sleep_message""" + return datetime.now() - @classmethod - def _sleep(cls, message): - log(cls._sleep_message(message)) - time.sleep(cls.seconds_to_sleep) + def _sleep_message(self, message): + wake_time = self._now() + timedelta(seconds=self.seconds_to_sleep) + return "%s Sleeping until %s (%s)." % (message, wake_time.strftime(self.log_date_format), self.sleep_duration_text) + + def _sleep(self, message): + log(self._sleep_message(message)) + self._wakeup_event.wait(self.seconds_to_sleep) + self._wakeup_event.clear() diff --git a/WebKitTools/Scripts/webkitpy/queueengine_unittest.py b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine_unittest.py index a4036ea..ec91bdb 100644 --- a/WebKitTools/Scripts/webkitpy/queueengine_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/bot/queueengine_unittest.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -27,13 +26,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import datetime import os import shutil import tempfile +import threading import unittest -from webkitpy.executive import ScriptError -from webkitpy.queueengine import QueueEngine, QueueEngineDelegate +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate class LoggingDelegate(QueueEngineDelegate): def __init__(self, test): @@ -86,6 +87,7 @@ class LoggingDelegate(QueueEngineDelegate): def process_work_item(self, work_item): self.record("process_work_item") self._test.assertEquals(work_item, "work_item") + return True def handle_unexpected_error(self, work_item, message): self.record("handle_unexpected_error") @@ -111,7 +113,7 @@ class NotSafeToProceedDelegate(LoggingDelegate): class FastQueueEngine(QueueEngine): def __init__(self, delegate): - QueueEngine.__init__(self, "fast-queue", delegate) + QueueEngine.__init__(self, "fast-queue", delegate, threading.Event()) # No sleep for the wicked. seconds_to_sleep = 0 @@ -123,7 +125,7 @@ class FastQueueEngine(QueueEngine): class QueueEngineTest(unittest.TestCase): def test_trivial(self): delegate = LoggingDelegate(self) - work_queue = QueueEngine("trivial-queue", delegate) + work_queue = QueueEngine("trivial-queue", delegate, threading.Event()) work_queue.run() self.assertEquals(delegate._callbacks, LoggingDelegate.expected_callbacks) self.assertTrue(os.path.exists(os.path.join(self.temp_dir, "queue_log_path"))) @@ -131,7 +133,7 @@ class QueueEngineTest(unittest.TestCase): def test_unexpected_error(self): delegate = ThrowErrorDelegate(self, 3) - work_queue = QueueEngine("error-queue", delegate) + work_queue = QueueEngine("error-queue", delegate, threading.Event()) work_queue.run() expected_callbacks = LoggingDelegate.expected_callbacks[:] work_item_index = expected_callbacks.index('process_work_item') @@ -142,7 +144,7 @@ class QueueEngineTest(unittest.TestCase): def test_handled_error(self): delegate = ThrowErrorDelegate(self, QueueEngine.handled_error_code) - work_queue = QueueEngine("handled-error-queue", delegate) + work_queue = QueueEngine("handled-error-queue", delegate, threading.Event()) work_queue.run() self.assertEquals(delegate._callbacks, LoggingDelegate.expected_callbacks) @@ -159,6 +161,17 @@ class QueueEngineTest(unittest.TestCase): expected_callbacks.append('should_continue_work_queue') self.assertEquals(delegate._callbacks, expected_callbacks) + def test_now(self): + """Make sure there are no typos in the QueueEngine.now() method.""" + engine = QueueEngine("test", None, None) + self.assertTrue(isinstance(engine._now(), datetime.datetime)) + + def test_sleep_message(self): + engine = QueueEngine("test", None, None) + engine._now = lambda: datetime.datetime(2010, 1, 1) + expected_sleep_message = "MESSAGE Sleeping until 2010-01-01 00:02:00 (2 mins)." + self.assertEqual(engine._sleep_message("MESSAGE"), expected_sleep_message) + def setUp(self): self.temp_dir = tempfile.mkdtemp(suffix="work_queue_test_logs") diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/sheriff.py b/WebKitTools/Scripts/webkitpy/tool/bot/sheriff.py new file mode 100644 index 0000000..a38c3cf --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/sheriff.py @@ -0,0 +1,131 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.common.checkout.changelog import view_source_url +from webkitpy.common.net.bugzilla import parse_bug_id +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.grammar import join_with_separators + + +class Sheriff(object): + def __init__(self, tool, sheriffbot): + self._tool = tool + self._sheriffbot = sheriffbot + + def post_irc_warning(self, commit_info, builders): + irc_nicknames = sorted([party.irc_nickname for + party in commit_info.responsible_parties() + if party.irc_nickname]) + irc_prefix = ": " if irc_nicknames else "" + irc_message = "%s%s%s might have broken %s" % ( + ", ".join(irc_nicknames), + irc_prefix, + view_source_url(commit_info.revision()), + join_with_separators([builder.name() for builder in builders])) + + self._tool.irc().post(irc_message) + + def post_rollout_patch(self, svn_revision, rollout_reason): + # Ensure that svn_revision is a number (and not an option to + # create-rollout). + try: + svn_revision = int(svn_revision) + except: + raise ScriptError(message="Invalid svn revision number \"%s\"." + % svn_revision) + + if rollout_reason.startswith("-"): + raise ScriptError(message="The rollout reason may not begin " + "with - (\"%s\")." % rollout_reason) + + output = self._sheriffbot.run_webkit_patch([ + "create-rollout", + "--force-clean", + # In principle, we should pass --non-interactive here, but it + # turns out that create-rollout doesn't need it yet. We can't + # pass it prophylactically because we reject unrecognized command + # line switches. + "--parent-command=sheriff-bot", + svn_revision, + rollout_reason, + ]) + return parse_bug_id(output) + + def _rollout_reason(self, builders): + # FIXME: This should explain which layout tests failed + # however, that would require Build objects here, either passed + # in through failure_info, or through Builder.latest_build. + names = [builder.name() for builder in builders] + return "Caused builders %s to fail." % join_with_separators(names) + + def post_automatic_rollout_patch(self, commit_info, builders): + # For now we're only posting rollout patches for commit-queue patches. + commit_bot_email = "eseidel@chromium.org" + if commit_bot_email == commit_info.committer_email(): + try: + self.post_rollout_patch(commit_info.revision(), + self._rollout_reason(builders)) + except ScriptError, e: + log("Failed to create-rollout.") + + def post_blame_comment_on_bug(self, commit_info, builders, blame_list): + if not commit_info.bug_id(): + return + comment = "%s might have broken %s" % ( + view_source_url(commit_info.revision()), + join_with_separators([builder.name() for builder in builders])) + if len(blame_list) > 1: + comment += "\nThe following changes are on the blame list:\n" + comment += "\n".join(map(view_source_url, blame_list)) + self._tool.bugs.post_comment_to_bug(commit_info.bug_id(), + comment, + cc=self._sheriffbot.watchers) + + # FIXME: Should some of this logic be on BuildBot? + def provoke_flaky_builders(self, revisions_causing_failures): + # We force_build builders that are red but have not "failed" (i.e., + # been red twice). We do this to avoid a deadlock situation where a + # flaky test blocks the commit-queue and there aren't any other + # patches being landed to re-spin the builder. + failed_builders = sum([revisions_causing_failures[key] for + key in revisions_causing_failures.keys()], []) + failed_builder_names = \ + set([builder.name() for builder in failed_builders]) + idle_red_builder_names = \ + set([builder["name"] + for builder in self._tool.buildbot.idle_red_core_builders()]) + + # We only want to provoke these builders if they are idle and have not + # yet "failed" (i.e., been red twice) to avoid overloading the bots. + flaky_builder_names = idle_red_builder_names - failed_builder_names + + for name in flaky_builder_names: + flaky_builder = self._tool.buildbot.builder_with_name(name) + flaky_builder.force_build(username=self._sheriffbot.name, + comments="Probe for flakiness.") diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/sheriff_unittest.py b/WebKitTools/Scripts/webkitpy/tool/bot/sheriff_unittest.py new file mode 100644 index 0000000..c375ff9 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/sheriff_unittest.py @@ -0,0 +1,105 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import unittest + +from webkitpy.common.net.buildbot import Builder +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.bot.sheriff import Sheriff +from webkitpy.tool.mocktool import MockTool + + +class MockSheriffBot(object): + name = "mock-sheriff-bot" + watchers = [ + "watcher@example.com", + ] + + def run_webkit_patch(self, args): + return "Created bug https://bugs.webkit.org/show_bug.cgi?id=36936\n" + + +class SheriffTest(unittest.TestCase): + def test_rollout_reason(self): + sheriff = Sheriff(MockTool(), MockSheriffBot()) + builders = [ + Builder("Foo", None), + Builder("Bar", None), + ] + reason = "Caused builders Foo and Bar to fail." + self.assertEquals(sheriff._rollout_reason(builders), reason) + + def test_post_blame_comment_on_bug(self): + def run(): + sheriff = Sheriff(MockTool(), MockSheriffBot()) + builders = [ + Builder("Foo", None), + Builder("Bar", None), + ] + commit_info = Mock() + commit_info.bug_id = lambda: None + commit_info.revision = lambda: 4321 + # Should do nothing with no bug_id + sheriff.post_blame_comment_on_bug(commit_info, builders, []) + sheriff.post_blame_comment_on_bug(commit_info, builders, [2468, 5646]) + # Should try to post a comment to the bug, but MockTool.bugs does nothing. + commit_info.bug_id = lambda: 1234 + sheriff.post_blame_comment_on_bug(commit_info, builders, []) + sheriff.post_blame_comment_on_bug(commit_info, builders, [3432]) + sheriff.post_blame_comment_on_bug(commit_info, builders, [841, 5646]) + + expected_stderr = u"MOCK bug comment: bug_id=1234, cc=['watcher@example.com']\n--- Begin comment ---\\http://trac.webkit.org/changeset/4321 might have broken Foo and Bar\n--- End comment ---\n\nMOCK bug comment: bug_id=1234, cc=['watcher@example.com']\n--- Begin comment ---\\http://trac.webkit.org/changeset/4321 might have broken Foo and Bar\n--- End comment ---\n\nMOCK bug comment: bug_id=1234, cc=['watcher@example.com']\n--- Begin comment ---\\http://trac.webkit.org/changeset/4321 might have broken Foo and Bar\nThe following changes are on the blame list:\nhttp://trac.webkit.org/changeset/841\nhttp://trac.webkit.org/changeset/5646\n--- End comment ---\n\n" + OutputCapture().assert_outputs(self, run, expected_stderr=expected_stderr) + + def test_provoke_flaky_builders(self): + def run(): + tool = MockTool() + tool.buildbot.light_tree_on_fire() + sheriff = Sheriff(tool, MockSheriffBot()) + revisions_causing_failures = {} + sheriff.provoke_flaky_builders(revisions_causing_failures) + expected_stderr = "MOCK: force_build: name=Builder2, username=mock-sheriff-bot, comments=Probe for flakiness.\n" + OutputCapture().assert_outputs(self, run, expected_stderr=expected_stderr) + + def test_post_blame_comment_on_bug(self): + sheriff = Sheriff(MockTool(), MockSheriffBot()) + builders = [ + Builder("Foo", None), + Builder("Bar", None), + ] + commit_info = Mock() + commit_info.bug_id = lambda: None + commit_info.revision = lambda: 4321 + commit_info.committer = lambda: None + commit_info.committer_email = lambda: "foo@example.com" + commit_info.reviewer = lambda: None + commit_info.author = lambda: None + sheriff.post_automatic_rollout_patch(commit_info, builders) + diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot.py b/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot.py new file mode 100644 index 0000000..de77222 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot.py @@ -0,0 +1,83 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import webkitpy.tool.bot.irc_command as irc_command + +from webkitpy.common.net.irc.ircbot import IRCBotDelegate +from webkitpy.common.thread.threadedmessagequeue import ThreadedMessageQueue + + +class _IRCThreadTearoff(IRCBotDelegate): + def __init__(self, password, message_queue, wakeup_event): + self._password = password + self._message_queue = message_queue + self._wakeup_event = wakeup_event + + # IRCBotDelegate methods + + def irc_message_received(self, nick, message): + self._message_queue.post([nick, message]) + self._wakeup_event.set() + + def irc_nickname(self): + return "sheriffbot" + + def irc_password(self): + return self._password + + +class SheriffIRCBot(object): + def __init__(self, tool, sheriff): + self._tool = tool + self._sheriff = sheriff + self._message_queue = ThreadedMessageQueue() + + def irc_delegate(self): + return _IRCThreadTearoff(self._tool.irc_password, + self._message_queue, + self._tool.wakeup_event) + + def process_message(self, message): + (nick, request) = message + tokenized_request = request.strip().split(" ") + if not tokenized_request: + return + command = irc_command.commands.get(tokenized_request[0]) + args = tokenized_request[1:] + if not command: + # Give the peoples someone to talk with. + command = irc_command.Eliza + args = tokenized_request + response = command().execute(nick, args, self._tool, self._sheriff) + if response: + self._tool.irc().post(response) + + def process_pending_messages(self): + (messages, is_running) = self._message_queue.take_all() + for message in messages: + self.process_message(message) diff --git a/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot_unittest.py b/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot_unittest.py new file mode 100644 index 0000000..08023bd --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/bot/sheriffircbot_unittest.py @@ -0,0 +1,95 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import random + +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.bot.sheriff import Sheriff +from webkitpy.tool.bot.sheriffircbot import SheriffIRCBot +from webkitpy.tool.bot.sheriff_unittest import MockSheriffBot +from webkitpy.tool.mocktool import MockTool + + +def run(message): + tool = MockTool() + tool.ensure_irc_connected(None) + bot = SheriffIRCBot(tool, Sheriff(tool, MockSheriffBot())) + bot._message_queue.post(["mock_nick", message]) + bot.process_pending_messages() + + +class SheriffIRCBotTest(unittest.TestCase): + def test_hi(self): + random.seed(23324) + expected_stderr = 'MOCK: irc.post: "Only you can prevent forest fires." -- Smokey the Bear\n' + OutputCapture().assert_outputs(self, run, args=["hi"], expected_stderr=expected_stderr) + + def test_help(self): + expected_stderr = "MOCK: irc.post: mock_nick: Available commands: rollout, hi, help, restart, last-green-revision\n" + OutputCapture().assert_outputs(self, run, args=["help"], expected_stderr=expected_stderr) + + def test_lgr(self): + expected_stderr = "MOCK: irc.post: mock_nick: http://trac.webkit.org/changeset/9479\n" + OutputCapture().assert_outputs(self, run, args=["last-green-revision"], expected_stderr=expected_stderr) + + def test_rollout(self): + expected_stderr = "MOCK: irc.post: Preparing rollout for r21654...\nMOCK: irc.post: mock_nick: Created rollout: http://example.com/36936\n" + OutputCapture().assert_outputs(self, run, args=["rollout 21654 This patch broke the world"], expected_stderr=expected_stderr) + + def test_rollout_with_r_in_svn_revision(self): + expected_stderr = "MOCK: irc.post: Preparing rollout for r21654...\nMOCK: irc.post: mock_nick: Created rollout: http://example.com/36936\n" + OutputCapture().assert_outputs(self, run, args=["rollout r21654 This patch broke the world"], expected_stderr=expected_stderr) + + def test_rollout_bananas(self): + expected_stderr = "MOCK: irc.post: mock_nick: Usage: SVN_REVISION REASON\n" + OutputCapture().assert_outputs(self, run, args=["rollout bananas"], expected_stderr=expected_stderr) + + def test_rollout_invalidate_revision(self): + expected_stderr = ("MOCK: irc.post: Preparing rollout for r--component=Tools...\n" + "MOCK: irc.post: mock_nick: Failed to create rollout patch:\n" + "MOCK: irc.post: Invalid svn revision number \"--component=Tools\".\n") + OutputCapture().assert_outputs(self, run, + args=["rollout " + "--component=Tools 21654"], + expected_stderr=expected_stderr) + + def test_rollout_invalidate_reason(self): + expected_stderr = ("MOCK: irc.post: Preparing rollout for " + "r21654...\nMOCK: irc.post: mock_nick: Failed to " + "create rollout patch:\nMOCK: irc.post: The rollout" + " reason may not begin with - (\"-bad (Requested " + "by mock_nick on #webkit).\").\n") + OutputCapture().assert_outputs(self, run, + args=["rollout " + "21654 -bad"], + expected_stderr=expected_stderr) + + def test_rollout_no_reason(self): + expected_stderr = "MOCK: irc.post: mock_nick: Usage: SVN_REVISION REASON\n" + OutputCapture().assert_outputs(self, run, args=["rollout 21654"], expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/__init__.py b/WebKitTools/Scripts/webkitpy/tool/commands/__init__.py new file mode 100644 index 0000000..9bdec8f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/__init__.py @@ -0,0 +1,5 @@ +# Required for Python to search this directory for module files + +from webkitpy.tool.commands.prettydiff import PrettyDiff +from webkitpy.tool.commands.rebaseline import Rebaseline +# FIXME: Add the rest of the commands here. diff --git a/WebKitTools/Scripts/webkitpy/commands/abstractsequencedcommand.py b/WebKitTools/Scripts/webkitpy/tool/commands/abstractsequencedcommand.py index 53af5b1..fc5a794 100644 --- a/WebKitTools/Scripts/webkitpy/commands/abstractsequencedcommand.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/abstractsequencedcommand.py @@ -26,8 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.multicommandtool import AbstractDeclarativeCommand -from webkitpy.stepsequence import StepSequence +from webkitpy.tool.commands.stepsequence import StepSequence +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand class AbstractSequencedCommand(AbstractDeclarativeCommand): diff --git a/WebKitTools/Scripts/webkitpy/commands/commandtest.py b/WebKitTools/Scripts/webkitpy/tool/commands/commandtest.py index a56cb05..de92cd3 100644 --- a/WebKitTools/Scripts/webkitpy/commands/commandtest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/commandtest.py @@ -28,11 +28,10 @@ import unittest -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.mocktool import MockOptions, MockTool class CommandsTest(unittest.TestCase): - def assert_execute_outputs(self, command, args, expected_stdout="", expected_stderr="", options=Mock(), tool=MockBugzillaTool()): + def assert_execute_outputs(self, command, args, expected_stdout="", expected_stderr="", options=MockOptions(), tool=MockTool()): command.bind_to_tool(tool) OutputCapture().assert_outputs(self, command.execute, [options, args, tool], expected_stdout=expected_stdout, expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/commands/download.py b/WebKitTools/Scripts/webkitpy/tool/commands/download.py index 49a6862..d27ab0e 100644 --- a/WebKitTools/Scripts/webkitpy/commands/download.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/download.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -31,18 +31,25 @@ import os from optparse import make_option -import webkitpy.steps as steps +import webkitpy.tool.steps as steps -from webkitpy.bugzilla import parse_bug_id -# We could instead use from modules import buildsteps and then prefix every buildstep with "buildsteps." -from webkitpy.changelogs import ChangeLog -from webkitpy.commands.abstractsequencedcommand import AbstractSequencedCommand -from webkitpy.comments import bug_comment_from_commit_text -from webkitpy.executive import ScriptError -from webkitpy.grammar import pluralize -from webkitpy.webkit_logging import error, log -from webkitpy.multicommandtool import AbstractDeclarativeCommand -from webkitpy.stepsequence import StepSequence +from webkitpy.common.checkout.changelog import ChangeLog, view_source_url +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand +from webkitpy.tool.commands.stepsequence import StepSequence +from webkitpy.tool.comments import bug_comment_from_commit_text +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand +from webkitpy.common.system.deprecated_logging import error, log + + +class Update(AbstractSequencedCommand): + name = "update" + help_text = "Update working copy (used internally)" + steps = [ + steps.CleanWorkingDirectory, + steps.Update, + ] class Build(AbstractSequencedCommand): @@ -74,22 +81,35 @@ class Land(AbstractSequencedCommand): steps = [ steps.EnsureBuildersAreGreen, steps.UpdateChangeLogsWithReviewer, - steps.EnsureBuildersAreGreen, + steps.ValidateReviewer, steps.Build, steps.RunTests, steps.Commit, steps.CloseBugForLandDiff, ] long_help = """land commits the current working copy diff (just as svn or git commit would). -land will build and run the tests before committing. +land will NOT build and run the tests before committing, but you can use the --build option for that. If a bug id is provided, or one can be found in the ChangeLog land will update the bug after committing.""" def _prepare_state(self, options, args, tool): return { - "bug_id" : (args and args[0]) or parse_bug_id(tool.scm().create_patch()), + "bug_id": (args and args[0]) or tool.checkout().bug_id_for_this_commit(options.git_commit), } +class LandCowboy(AbstractSequencedCommand): + name = "land-cowboy" + help_text = "Prepares a ChangeLog and lands the current working directory diff." + steps = [ + steps.PrepareChangeLog, + steps.EditChangeLog, + steps.ConfirmDiff, + steps.Build, + steps.RunTests, + steps.Commit, + ] + + class AbstractPatchProcessingCommand(AbstractDeclarativeCommand): # Subclasses must implement the methods below. We don't declare them here # because we want to be able to implement them with mix-ins. @@ -174,6 +194,18 @@ class BuildAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin): ] +class PostAttachmentToRietveld(AbstractPatchSequencingCommand, ProcessAttachmentsMixin): + name = "post-attachment-to-rietveld" + help_text = "Uploads a bugzilla attachment to rietveld" + arguments_names = "ATTACHMENTID" + main_steps = [ + steps.CleanWorkingDirectory, + steps.Update, + steps.ApplyPatch, + steps.PostCodeReview, + ] + + class AbstractPatchApplyingCommand(AbstractPatchSequencingCommand): prepare_steps = [ steps.EnsureLocalCommitIfNeeded, @@ -209,7 +241,7 @@ class AbstractPatchLandingCommand(AbstractPatchSequencingCommand): steps.CleanWorkingDirectory, steps.Update, steps.ApplyPatch, - steps.EnsureBuildersAreGreen, + steps.ValidateReviewer, steps.Build, steps.RunTests, steps.Commit, @@ -240,11 +272,94 @@ class LandFromBug(AbstractPatchLandingCommand, ProcessBugsMixin): show_in_main_help = True -class Rollout(AbstractSequencedCommand): +class AbstractRolloutPrepCommand(AbstractSequencedCommand): + argument_names = "REVISION REASON" + + def _commit_info(self, revision): + commit_info = self.tool.checkout().commit_info_for_revision(revision) + if commit_info and commit_info.bug_id(): + # Note: Don't print a bug URL here because it will confuse the + # SheriffBot because the SheriffBot just greps the output + # of create-rollout for bug URLs. It should do better + # parsing instead. + log("Preparing rollout for bug %s." % commit_info.bug_id()) + else: + log("Unable to parse bug number from diff.") + return commit_info + + def _prepare_state(self, options, args, tool): + revision = args[0] + commit_info = self._commit_info(revision) + cc_list = sorted([party.bugzilla_email() + for party in commit_info.responsible_parties() + if party.bugzilla_email()]) + return { + "revision": revision, + "bug_id": commit_info.bug_id(), + # FIXME: We should used the list as the canonical representation. + "bug_cc": ",".join(cc_list), + "reason": args[1], + } + + +class PrepareRollout(AbstractRolloutPrepCommand): + name = "prepare-rollout" + help_text = "Revert the given revision in the working copy and prepare ChangeLogs with revert reason" + long_help = """Updates the working copy. +Applies the inverse diff for the provided revision. +Creates an appropriate rollout ChangeLog, including a trac link and bug link. +""" + steps = [ + steps.CleanWorkingDirectory, + steps.Update, + steps.RevertRevision, + steps.PrepareChangeLogForRevert, + ] + + +class CreateRollout(AbstractRolloutPrepCommand): + name = "create-rollout" + help_text = "Creates a bug to track a broken SVN revision and uploads a rollout patch." + steps = [ + steps.CleanWorkingDirectory, + steps.Update, + steps.RevertRevision, + steps.CreateBug, + steps.PrepareChangeLogForRevert, + steps.PostDiffForRevert, + ] + + def _prepare_state(self, options, args, tool): + state = AbstractRolloutPrepCommand._prepare_state(self, options, args, tool) + # Currently, state["bug_id"] points to the bug that caused the + # regression. We want to create a new bug that blocks the old bug + # so we move state["bug_id"] to state["bug_blocked"] and delete the + # old state["bug_id"] so that steps.CreateBug will actually create + # the new bug that we want (and subsequently store its bug id into + # state["bug_id"]) + state["bug_blocked"] = state["bug_id"] + del state["bug_id"] + state["bug_title"] = "REGRESSION(r%s): %s" % (state["revision"], state["reason"]) + state["bug_description"] = "%s broke the build:\n%s" % (view_source_url(state["revision"]), state["reason"]) + # FIXME: If we had more context here, we could link to other open bugs + # that mention the test that regressed. + if options.parent_command == "sheriff-bot": + state["bug_description"] += """ + +This is an automatic bug report generated by the sheriff-bot. If this bug +report was created because of a flaky test, please file a bug for the flaky +test (if we don't already have one on file) and dup this bug against that bug +so that we can track how often these flaky tests case pain. + +"Only you can prevent forest fires." -- Smokey the Bear +""" + return state + + +class Rollout(AbstractRolloutPrepCommand): name = "rollout" show_in_main_help = True help_text = "Revert the given revision in the working copy and optionally commit the revert and re-open the original bug" - argument_names = "REVISION REASON" long_help = """Updates the working copy. Applies the inverse diff for the provided revision. Creates an appropriate rollout ChangeLog, including a trac link and bug link. @@ -258,27 +373,7 @@ Commits the revert and updates the bug (including re-opening the bug if necessar steps.PrepareChangeLogForRevert, steps.EditChangeLog, steps.ConfirmDiff, - steps.CompleteRollout, + steps.Build, + steps.Commit, + steps.ReopenBugAfterRollout, ] - - @staticmethod - def _parse_bug_id_from_revision_diff(tool, revision): - original_diff = tool.scm().diff_for_revision(revision) - return parse_bug_id(original_diff) - - def execute(self, options, args, tool): - revision = args[0] - reason = args[1] - bug_id = self._parse_bug_id_from_revision_diff(tool, revision) - if options.complete_rollout: - if bug_id: - log("Will re-open bug %s after rollout." % bug_id) - else: - log("Failed to parse bug number from diff. No bugs will be updated/reopened after the rollout.") - - state = { - "revision" : revision, - "bug_id" : bug_id, - "reason" : reason, - } - self._sequence.run_and_handle_errors(tool, options, state) diff --git a/WebKitTools/Scripts/webkitpy/commands/download_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/download_unittest.py index f60c5b8..75cd0f3 100644 --- a/WebKitTools/Scripts/webkitpy/commands/download_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/download_unittest.py @@ -26,13 +26,37 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.download import * -from webkitpy.mock import Mock +import unittest + +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.download import * +from webkitpy.tool.mocktool import MockOptions, MockTool + + +class AbstractRolloutPrepCommandTest(unittest.TestCase): + def test_commit_info(self): + command = AbstractRolloutPrepCommand() + tool = MockTool() + command.bind_to_tool(tool) + output = OutputCapture() + + expected_stderr = "Preparing rollout for bug 42.\n" + commit_info = output.assert_outputs(self, command._commit_info, [1234], expected_stderr=expected_stderr) + self.assertTrue(commit_info) + + mock_commit_info = Mock() + mock_commit_info.bug_id = lambda: None + tool._checkout.commit_info_for_revision = lambda revision: mock_commit_info + expected_stderr = "Unable to parse bug number from diff.\n" + commit_info = output.assert_outputs(self, command._commit_info, [1234], expected_stderr=expected_stderr) + self.assertEqual(commit_info, mock_commit_info) + class DownloadCommandsTest(CommandsTest): def _default_options(self): - options = Mock() + options = MockOptions() options.force_clean = False options.clean = True options.check_builders = True @@ -42,7 +66,6 @@ class DownloadCommandsTest(CommandsTest): options.build = True options.test = True options.close_bug = True - options.complete_rollout = False return options def test_build(self): @@ -69,7 +92,13 @@ class DownloadCommandsTest(CommandsTest): def test_land_diff(self): expected_stderr = "Building WebKit\nRunning Python unit tests\nRunning Perl unit tests\nRunning JavaScriptCore tests\nRunning run-webkit-tests\nUpdating bug 42\n" - self.assert_execute_outputs(Land(), [42], options=self._default_options(), expected_stderr=expected_stderr) + mock_tool = MockTool() + mock_tool.scm().create_patch = Mock() + mock_tool.checkout().modified_changelogs = Mock(return_value=[]) + self.assert_execute_outputs(Land(), [42], options=self._default_options(), expected_stderr=expected_stderr, tool=mock_tool) + # Make sure we're not calling expensive calls too often. + self.assertEqual(mock_tool.scm().create_patch.call_count, 0) + self.assertEqual(mock_tool.checkout().modified_changelogs.call_count, 1) def test_check_style(self): expected_stderr = "Processing 1 patch from 1 bug.\nUpdating working directory\nProcessing patch 197 from bug 42.\nRunning check-webkit-style\n" @@ -79,6 +108,10 @@ class DownloadCommandsTest(CommandsTest): expected_stderr = "Processing 1 patch from 1 bug.\nUpdating working directory\nProcessing patch 197 from bug 42.\nBuilding WebKit\n" self.assert_execute_outputs(BuildAttachment(), [197], options=self._default_options(), expected_stderr=expected_stderr) + def test_post_attachment_to_rietveld(self): + expected_stderr = "Processing 1 patch from 1 bug.\nUpdating working directory\nProcessing patch 197 from bug 42.\nMOCK: Uploading patch to rietveld\nMOCK setting flag 'in-rietveld' to '+' on attachment '197' with comment 'None' and additional comment 'None'\n" + self.assert_execute_outputs(PostAttachmentToRietveld(), [197], options=self._default_options(), expected_stderr=expected_stderr) + def test_land_attachment(self): # FIXME: This expected result is imperfect, notice how it's seeing the same patch as still there after it thought it would have cleared the flags. expected_stderr = """Processing 1 patch from 1 bug. @@ -116,12 +149,32 @@ Not closing bug 42 as attachment 197 has review=+. Assuming there are more patc """ self.assert_execute_outputs(LandFromBug(), [42], options=self._default_options(), expected_stderr=expected_stderr) + def test_prepare_rollout(self): + expected_stderr = "Preparing rollout for bug 42.\nUpdating working directory\nRunning prepare-ChangeLog\n" + self.assert_execute_outputs(PrepareRollout(), [852, "Reason"], options=self._default_options(), expected_stderr=expected_stderr) + + def test_create_rollout(self): + expected_stderr = """Preparing rollout for bug 42. +Updating working directory +MOCK create_bug +bug_title: REGRESSION(r852): Reason +bug_description: http://trac.webkit.org/changeset/852 broke the build: +Reason +Running prepare-ChangeLog +MOCK add_patch_to_bug: bug_id=None, description=ROLLOUT of r852, mark_for_review=False, mark_for_commit_queue=True, mark_for_landing=False +-- Begin comment -- +Any committer can land this patch automatically by marking it commit-queue+. The commit-queue will build and test the patch before landing to ensure that the rollout will be successful. This process takes approximately 15 minutes. + +If you would like to land the rollout faster, you can use the following command: + + webkit-patch land-attachment ATTACHMENT_ID --ignore-builders + +where ATTACHMENT_ID is the ID of this attachment. +-- End comment -- +""" + self.assert_execute_outputs(CreateRollout(), [852, "Reason"], options=self._default_options(), expected_stderr=expected_stderr) + def test_rollout(self): - expected_stderr = "Updating working directory\nRunning prepare-ChangeLog\n\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"webkit-patch land 12345\" to commit the rollout.\n" + expected_stderr = "Preparing rollout for bug 42.\nUpdating working directory\nRunning prepare-ChangeLog\nMOCK: user.open_url: file://...\nBuilding WebKit\n" self.assert_execute_outputs(Rollout(), [852, "Reason"], options=self._default_options(), expected_stderr=expected_stderr) - def test_complete_rollout(self): - options = self._default_options() - options.complete_rollout = True - expected_stderr = "Will re-open bug 12345 after rollout.\nUpdating working directory\nRunning prepare-ChangeLog\nBuilding WebKit\n" - self.assert_execute_outputs(Rollout(), [852, "Reason"], options=options, expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/commands/early_warning_system.py b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem.py index e3e14dd..9fbfda6 100644 --- a/WebKitTools/Scripts/webkitpy/commands/early_warning_system.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -27,13 +26,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from StringIO import StringIO - -from webkitpy.commands.queues import AbstractReviewQueue -from webkitpy.committers import CommitterList -from webkitpy.executive import ScriptError -from webkitpy.webkitport import WebKitPort -from webkitpy.queueengine import QueueEngine +from webkitpy.tool.commands.queues import AbstractReviewQueue +from webkitpy.common.config.committers import CommitterList +from webkitpy.common.config.ports import WebKitPort +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.bot.queueengine import QueueEngine class AbstractEarlyWarningSystem(AbstractReviewQueue): @@ -44,30 +41,51 @@ class AbstractEarlyWarningSystem(AbstractReviewQueue): self.port = WebKitPort.port(self.port_name) def should_proceed_with_work_item(self, patch): + return True + + def _can_build(self): try: self.run_webkit_patch([ "build", self.port.flag(), + "--build", "--build-style=%s" % self._build_style, "--force-clean", + "--no-update", "--quiet"]) - self._update_status("Building", patch) + return True except ScriptError, e: self._update_status("Unable to perform a build") return False - return True - def _review_patch(self, patch): - self.run_webkit_patch([ - "build-attachment", - self.port.flag(), - "--build-style=%s" % self._build_style, - "--force-clean", - "--quiet", - "--non-interactive", - "--parent-command=%s" % self.name, - "--no-update", - patch.id()]) + def _build(self, patch, first_run=False): + try: + args = [ + "build-attachment", + self.port.flag(), + "--build", + "--build-style=%s" % self._build_style, + "--force-clean", + "--quiet", + "--non-interactive", + patch.id()] + if not first_run: + # See commit-queue for an explanation of what we're doing here. + args.append("--no-update") + args.append("--parent-command=%s" % self.name) + self.run_webkit_patch(args) + return True + except ScriptError, e: + if first_run: + return False + raise + + def review_patch(self, patch): + if not self._build(patch, first_run=True): + if not self._can_build(): + return False + self._build(patch) + return True @classmethod def handle_script_error(cls, tool, state, script_error): @@ -95,14 +113,35 @@ class QtEWS(AbstractEarlyWarningSystem): port_name = "qt" -class ChromiumEWS(AbstractEarlyWarningSystem): - name = "chromium-ews" +class WinEWS(AbstractEarlyWarningSystem): + name = "win-ews" + port_name = "win" + # Use debug, the Apple Win port fails to link Release on 32-bit Windows. + # https://bugs.webkit.org/show_bug.cgi?id=39197 + _build_style = "debug" + + +class AbstractChromiumEWS(AbstractEarlyWarningSystem): port_name = "chromium" watchers = AbstractEarlyWarningSystem.watchers + [ "dglazkov@chromium.org", ] +class ChromiumLinuxEWS(AbstractChromiumEWS): + # FIXME: We should rename this command to cr-linux-ews, but that requires + # a database migration. :( + name = "chromium-ews" + + +class ChromiumWindowsEWS(AbstractChromiumEWS): + name = "cr-win-ews" + + +class ChromiumMacEWS(AbstractChromiumEWS): + name = "cr-mac-ews" + + # For platforms that we can't run inside a VM (like Mac OS X), we require # patches to be uploaded by committers, who are generally trustworthy folk. :) class AbstractCommitterOnlyEWS(AbstractEarlyWarningSystem): diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py new file mode 100644 index 0000000..67393d8 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py @@ -0,0 +1,85 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.earlywarningsystem import * +from webkitpy.tool.commands.queuestest import QueuesTest + +class EarlyWarningSytemTest(QueuesTest): + def test_failed_builds(self): + ews = ChromiumLinuxEWS() + ews._build = lambda patch, first_run=False: False + ews._can_build = lambda: True + ews.review_patch(Mock()) + + def _default_expected_stderr(self, ews): + string_replacemnts = { + "name": ews.name, + "checkout_dir": os.getcwd(), # FIXME: Use of os.getcwd() is wrong, should be scm.checkout_root + "port": ews.port_name, + "watchers": ews.watchers, + } + expected_stderr = { + "begin_work_queue": "CAUTION: %(name)s will discard all local changes in \"%(checkout_dir)s\"\nRunning WebKit %(name)s.\n" % string_replacemnts, + "handle_unexpected_error": "Mock error message\n", + "next_work_item": "MOCK: update_work_items: %(name)s [103]\n" % string_replacemnts, + "process_work_item": "MOCK: update_status: %(name)s Pass\n" % string_replacemnts, + "handle_script_error": "MOCK: update_status: %(name)s ScriptError error message\nMOCK bug comment: bug_id=345, cc=%(watchers)s\n--- Begin comment ---\\Attachment 1234 did not build on %(port)s:\nBuild output: http://dummy_url\n--- End comment ---\n\n" % string_replacemnts, + } + return expected_stderr + + def _test_ews(self, ews): + expected_exceptions = { + "handle_script_error": SystemExit, + } + self.assert_queue_outputs(ews, expected_stderr=self._default_expected_stderr(ews), expected_exceptions=expected_exceptions) + + # FIXME: If all EWSes are going to output the same text, we + # could test them all in one method with a for loop over an array. + def test_chromium_linux_ews(self): + self._test_ews(ChromiumLinuxEWS()) + + def test_chromium_windows_ews(self): + self._test_ews(ChromiumWindowsEWS()) + + def test_qt_ews(self): + self._test_ews(QtEWS()) + + def test_gtk_ews(self): + self._test_ews(GtkEWS()) + + def test_mac_ews(self): + ews = MacEWS() + expected_stderr = self._default_expected_stderr(ews) + expected_stderr["process_work_item"] = "MOCK: update_status: mac-ews Error: mac-ews cannot process patches from non-committers :(\n" + expected_exceptions = { + "handle_script_error": SystemExit, + } + self.assert_queue_outputs(ews, expected_stderr=expected_stderr, expected_exceptions=expected_exceptions) diff --git a/WebKitTools/Scripts/webkitpy/commands/openbugs.py b/WebKitTools/Scripts/webkitpy/tool/commands/openbugs.py index 25bdefc..5da5bbb 100644 --- a/WebKitTools/Scripts/webkitpy/commands/openbugs.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/openbugs.py @@ -29,8 +29,8 @@ import re import sys -from webkitpy.multicommandtool import AbstractDeclarativeCommand -from webkitpy.webkit_logging import log +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand +from webkitpy.common.system.deprecated_logging import log class OpenBugs(AbstractDeclarativeCommand): diff --git a/WebKitTools/Scripts/webkitpy/commands/openbugs_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/openbugs_unittest.py index 71fefd2..40a6e1b 100644 --- a/WebKitTools/Scripts/webkitpy/commands/openbugs_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/openbugs_unittest.py @@ -26,8 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.openbugs import OpenBugs +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.openbugs import OpenBugs class OpenBugsTest(CommandsTest): diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/prettydiff.py b/WebKitTools/Scripts/webkitpy/tool/commands/prettydiff.py new file mode 100644 index 0000000..e3fc00c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/prettydiff.py @@ -0,0 +1,38 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand +import webkitpy.tool.steps as steps + + +class PrettyDiff(AbstractSequencedCommand): + name = "pretty-diff" + help_text = "Shows the pretty diff in the default browser" + steps = [ + steps.ConfirmDiff, + ] diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/queries.py b/WebKitTools/Scripts/webkitpy/tool/commands/queries.py new file mode 100644 index 0000000..91ce5e9 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queries.py @@ -0,0 +1,286 @@ +# Copyright (c) 2009 Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from optparse import make_option + +from webkitpy.common.checkout.commitinfo import CommitInfo +from webkitpy.common.config.committers import CommitterList +from webkitpy.common.net.buildbot import BuildBot +from webkitpy.common.system.user import User +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand +from webkitpy.common.system.deprecated_logging import log + + +class BugsToCommit(AbstractDeclarativeCommand): + name = "bugs-to-commit" + help_text = "List bugs in the commit-queue" + + def execute(self, options, args, tool): + # FIXME: This command is poorly named. It's fetching the commit-queue list here. The name implies it's fetching pending-commit (all r+'d patches). + bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue() + for bug_id in bug_ids: + print "%s" % bug_id + + +class PatchesInCommitQueue(AbstractDeclarativeCommand): + name = "patches-in-commit-queue" + help_text = "List patches in the commit-queue" + + def execute(self, options, args, tool): + patches = tool.bugs.queries.fetch_patches_from_commit_queue() + log("Patches in commit queue:") + for patch in patches: + print patch.url() + + +class PatchesToCommitQueue(AbstractDeclarativeCommand): + name = "patches-to-commit-queue" + help_text = "Patches which should be added to the commit queue" + def __init__(self): + options = [ + make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"), + ] + AbstractDeclarativeCommand.__init__(self, options=options) + + @staticmethod + def _needs_commit_queue(patch): + if patch.commit_queue() == "+": # If it's already cq+, ignore the patch. + log("%s already has cq=%s" % (patch.id(), patch.commit_queue())) + return False + + # We only need to worry about patches from contributers who are not yet committers. + committer_record = CommitterList().committer_by_email(patch.attacher_email()) + if committer_record: + log("%s committer = %s" % (patch.id(), committer_record)) + return not committer_record + + def execute(self, options, args, tool): + patches = tool.bugs.queries.fetch_patches_from_pending_commit_list() + patches_needing_cq = filter(self._needs_commit_queue, patches) + if options.bugs: + bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq) + bugs_needing_cq = sorted(set(bugs_needing_cq)) + for bug_id in bugs_needing_cq: + print "%s" % tool.bugs.bug_url_for_bug_id(bug_id) + else: + for patch in patches_needing_cq: + print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit") + + +class PatchesToReview(AbstractDeclarativeCommand): + name = "patches-to-review" + help_text = "List patches that are pending review" + + def execute(self, options, args, tool): + patch_ids = tool.bugs.queries.fetch_attachment_ids_from_review_queue() + log("Patches pending review:") + for patch_id in patch_ids: + print patch_id + + +class LastGreenRevision(AbstractDeclarativeCommand): + name = "last-green-revision" + help_text = "Prints the last known good revision" + + def execute(self, options, args, tool): + print self.tool.buildbot.last_green_revision() + + +class WhatBroke(AbstractDeclarativeCommand): + name = "what-broke" + help_text = "Print failing buildbots (%s) and what revisions broke them" % BuildBot.default_host + + def _print_builder_line(self, builder_name, max_name_width, status_message): + print "%s : %s" % (builder_name.ljust(max_name_width), status_message) + + # FIXME: This is slightly different from Builder.suspect_revisions_for_green_to_red_transition + # due to needing to detect the "hit the limit" case an print a special message. + def _print_blame_information_for_builder(self, builder_status, name_width, avoid_flakey_tests=True): + builder = self.tool.buildbot.builder_with_name(builder_status["name"]) + red_build = builder.build(builder_status["build_number"]) + (last_green_build, first_red_build) = builder.find_failure_transition(red_build) + if not first_red_build: + self._print_builder_line(builder.name(), name_width, "FAIL (error loading build information)") + return + if not last_green_build: + self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: sometime before %s?)" % first_red_build.revision()) + return + + suspect_revisions = range(first_red_build.revision(), last_green_build.revision(), -1) + suspect_revisions.reverse() + first_failure_message = "" + if (first_red_build == builder.build(builder_status["build_number"])): + first_failure_message = " FIRST FAILURE, possibly a flaky test" + self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s%s)" % (suspect_revisions, first_failure_message)) + for revision in suspect_revisions: + commit_info = self.tool.checkout().commit_info_for_revision(revision) + if commit_info: + print commit_info.blame_string(self.tool.bugs) + else: + print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision + + def execute(self, options, args, tool): + builder_statuses = tool.buildbot.builder_statuses() + longest_builder_name = max(map(len, map(lambda builder: builder["name"], builder_statuses))) + failing_builders = 0 + for builder_status in builder_statuses: + # If the builder is green, print OK, exit. + if builder_status["is_green"]: + continue + self._print_blame_information_for_builder(builder_status, name_width=longest_builder_name) + failing_builders += 1 + if failing_builders: + print "%s of %s are failing" % (failing_builders, pluralize("builder", len(builder_statuses))) + else: + print "All builders are passing!" + + +class WhoBrokeIt(AbstractDeclarativeCommand): + name = "who-broke-it" + help_text = "Print a list of revisions causing failures on %s" % BuildBot.default_host + + def execute(self, options, args, tool): + for revision, builders in self.tool.buildbot.revisions_causing_failures(False).items(): + print "r%s appears to have broken %s" % (revision, [builder.name() for builder in builders]) + + +class ResultsFor(AbstractDeclarativeCommand): + name = "results-for" + help_text = "Print a list of failures for the passed revision from bots on %s" % BuildBot.default_host + argument_names = "REVISION" + + def _print_layout_test_results(self, results): + if not results: + print " No results." + return + for title, files in results.parsed_results().items(): + print " %s" % title + for filename in files: + print " %s" % filename + + def execute(self, options, args, tool): + builders = self.tool.buildbot.builders() + for builder in builders: + print "%s:" % builder.name() + build = builder.build_for_revision(args[0], allow_failed_lookups=True) + self._print_layout_test_results(build.layout_test_results()) + + +class FailureReason(AbstractDeclarativeCommand): + name = "failure-reason" + help_text = "Lists revisions where individual test failures started at %s" % BuildBot.default_host + + def _print_blame_information_for_transition(self, green_build, red_build, failing_tests): + suspect_revisions = green_build.builder().suspect_revisions_for_transition(green_build, red_build) + print "SUCCESS: Build %s (r%s) was the first to show failures: %s" % (red_build._number, red_build.revision(), failing_tests) + print "Suspect revisions:" + for revision in suspect_revisions: + commit_info = self.tool.checkout().commit_info_for_revision(revision) + if commit_info: + print commit_info.blame_string(self.tool.bugs) + else: + print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision + + def _explain_failures_for_builder(self, builder, start_revision): + print "Examining failures for \"%s\", starting at r%s" % (builder.name(), start_revision) + revision_to_test = start_revision + build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True) + layout_test_results = build.layout_test_results() + if not layout_test_results: + # FIXME: This could be made more user friendly. + print "Failed to load layout test results; can't continue. (start revision = r%s)" % start_revision + return 1 + + results_to_explain = set(layout_test_results.failing_tests()) + last_build_with_results = build + print "Starting at %s" % revision_to_test + while results_to_explain: + revision_to_test -= 1 + new_build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True) + if not new_build: + print "No build for %s" % revision_to_test + continue + build = new_build + latest_results = build.layout_test_results() + if not latest_results: + print "No results build %s (r%s)" % (build._number, build.revision()) + continue + failures = set(latest_results.failing_tests()) + if len(failures) >= 20: + # FIXME: We may need to move this logic into the LayoutTestResults class. + # The buildbot stops runs after 20 failures so we don't have full results to work with here. + print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision()) + continue + fixed_results = results_to_explain - failures + if not fixed_results: + print "No change in build %s (r%s), %s unexplained failures (%s in this build)" % (build._number, build.revision(), len(results_to_explain), len(failures)) + last_build_with_results = build + continue + self._print_blame_information_for_transition(build, last_build_with_results, fixed_results) + last_build_with_results = build + results_to_explain -= fixed_results + if results_to_explain: + print "Failed to explain failures: %s" % results_to_explain + return 1 + print "Explained all results for %s" % builder.name() + return 0 + + def _builder_to_explain(self): + builder_statuses = self.tool.buildbot.builder_statuses() + red_statuses = [status for status in builder_statuses if not status["is_green"]] + print "%s failing" % (pluralize("builder", len(red_statuses))) + builder_choices = [status["name"] for status in red_statuses] + # We could offer an "All" choice here. + chosen_name = User.prompt_with_list("Which builder to diagnose:", builder_choices) + # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object. + for status in red_statuses: + if status["name"] == chosen_name: + return (self.tool.buildbot.builder_with_name(chosen_name), status["built_revision"]) + + def execute(self, options, args, tool): + (builder, latest_revision) = self._builder_to_explain() + start_revision = self.tool.user.prompt("Revision to walk backwards from? [%s] " % latest_revision) or latest_revision + if not start_revision: + print "Revision required." + return 1 + return self._explain_failures_for_builder(builder, start_revision=int(start_revision)) + + +class TreeStatus(AbstractDeclarativeCommand): + name = "tree-status" + help_text = "Print the status of the %s buildbots" % BuildBot.default_host + long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder +and displayes the status of each builder.""" + + def execute(self, options, args, tool): + for builder in tool.buildbot.builder_statuses(): + status_string = "ok" if builder["is_green"] else "FAIL" + print "%s : %s" % (status_string.ljust(4), builder["name"]) diff --git a/WebKitTools/Scripts/webkitpy/commands/queries_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/queries_unittest.py index b858777..98ed545 100644 --- a/WebKitTools/Scripts/webkitpy/commands/queries_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queries_unittest.py @@ -26,11 +26,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.bugzilla import Bugzilla -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.queries import * -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool +from webkitpy.common.net.bugzilla import Bugzilla +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.queries import * +from webkitpy.tool.mocktool import MockTool class QueryCommandsTest(CommandsTest): def test_bugs_to_commit(self): diff --git a/WebKitTools/Scripts/webkitpy/commands/queues.py b/WebKitTools/Scripts/webkitpy/tool/commands/queues.py index 6ea1c48..97c3ddb 100644 --- a/WebKitTools/Scripts/webkitpy/commands/queues.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queues.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -35,19 +34,18 @@ from datetime import datetime from optparse import make_option from StringIO import StringIO -from webkitpy.bugzilla import CommitterValidator -from webkitpy.executive import ScriptError -from webkitpy.grammar import pluralize -from webkitpy.webkit_logging import error, log -from webkitpy.multicommandtool import Command -from webkitpy.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate -from webkitpy.statusserver import StatusServer -from webkitpy.stepsequence import StepSequenceErrorHandler -from webkitpy.queueengine import QueueEngine, QueueEngineDelegate +from webkitpy.common.net.bugzilla import CommitterValidator +from webkitpy.common.net.statusserver import StatusServer +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.system.deprecated_logging import error, log +from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler +from webkitpy.tool.bot.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate +from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.multicommandtool import Command, TryAgain class AbstractQueue(Command, QueueEngineDelegate): watchers = [ - "webkit-bot-watchers@googlegroups.com", ] _pass_status = "Pass" @@ -57,8 +55,10 @@ class AbstractQueue(Command, QueueEngineDelegate): def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations options_list = (options or []) + [ make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue. Dangerous!"), + make_option("--exit-after-iteration", action="store", type="int", dest="iterations", default=None, help="Stop running the queue after iterating this number of times."), ] Command.__init__(self, "Run the %s" % self.name, options=options_list) + self._iteration_count = 0 def _cc_watchers(self, bug_id): try: @@ -67,24 +67,26 @@ class AbstractQueue(Command, QueueEngineDelegate): traceback.print_exc() log("Failed to CC watchers.") - def _update_status(self, message, patch=None, results_file=None): - self.tool.status_server.update_status(self.name, message, patch, results_file) + def run_webkit_patch(self, args): + webkit_patch_args = [self.tool.path()] + # FIXME: This is a hack, we should have a more general way to pass global options. + # FIXME: We must always pass global options and their value in one argument + # because our global option code looks for the first argument which does + # not begin with "-" and assumes that is the command name. + webkit_patch_args += ["--status-host=%s" % self.tool.status_server.host] + webkit_patch_args.extend(args) + return self.tool.executive.run_and_throw_if_fail(webkit_patch_args) - def _did_pass(self, patch): - self._update_status(self._pass_status, patch) + def _log_directory(self): + return "%s-logs" % self.name - def _did_fail(self, patch): - self._update_status(self._fail_status, patch) - - def _did_error(self, patch, reason): - message = "%s: %s" % (self._error_status, reason) - self._update_status(message, patch) + # QueueEngineDelegate methods def queue_log_path(self): - return "%s.log" % self.name + return os.path.join(self._log_directory(), "%s.log" % self.name) - def work_item_log_path(self, patch): - return os.path.join("%s-logs" % self.name, "%s.log" % patch.bug_id()) + def work_item_log_path(self, work_item): + raise NotImplementedError, "subclasses must implement" def begin_work_queue(self): log("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self.tool.scm().checkout_root)) @@ -95,7 +97,8 @@ class AbstractQueue(Command, QueueEngineDelegate): log("Running WebKit %s." % self.name) def should_continue_work_queue(self): - return True + self._iteration_count += 1 + return not self.options.iterations or self._iteration_count <= self.options.iterations def next_work_item(self): raise NotImplementedError, "subclasses must implement" @@ -109,39 +112,58 @@ class AbstractQueue(Command, QueueEngineDelegate): def handle_unexpected_error(self, work_item, message): raise NotImplementedError, "subclasses must implement" - def run_webkit_patch(self, args): - webkit_patch_args = [self.tool.path()] - # FIXME: This is a hack, we should have a more general way to pass global options. - webkit_patch_args += ["--status-host=%s" % self.tool.status_server.host] - webkit_patch_args += map(str, args) - self.tool.executive.run_and_throw_if_fail(webkit_patch_args) - - def log_progress(self, patch_ids): - log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self.name, ", ".join(map(str, patch_ids)))) + # Command methods def execute(self, options, args, tool, engine=QueueEngine): self.options = options self.tool = tool - return engine(self.name, self).run() + return engine(self.name, self, self.tool.wakeup_event).run() @classmethod def _update_status_for_script_error(cls, tool, state, script_error, is_error=False): - message = script_error.message + message = str(script_error) if is_error: message = "Error: %s" % message - output = script_error.message_with_output(output_limit=5*1024*1024) # 5MB - return tool.status_server.update_status(cls.name, message, state["patch"], StringIO(output)) + output = script_error.message_with_output(output_limit=1024*1024) # 1MB + # We pre-encode the string to a byte array before passing it + # to status_server, because ClientForm (part of mechanize) + # wants a file-like object with pre-encoded data. + return tool.status_server.update_status(cls.name, message, state["patch"], StringIO(output.encode("utf-8"))) + + +class AbstractPatchQueue(AbstractQueue): + def _update_status(self, message, patch=None, results_file=None): + self.tool.status_server.update_status(self.name, message, patch, results_file) + + def _update_work_items(self, patch_ids): + self.tool.status_server.update_work_items(self.name, patch_ids) + def _did_pass(self, patch): + self._update_status(self._pass_status, patch) + + def _did_fail(self, patch): + self._update_status(self._fail_status, patch) -class CommitQueue(AbstractQueue, StepSequenceErrorHandler): + def _did_error(self, patch, reason): + message = "%s: %s" % (self._error_status, reason) + self._update_status(message, patch) + + def work_item_log_path(self, patch): + return os.path.join(self._log_directory(), "%s.log" % patch.bug_id()) + + def log_progress(self, patch_ids): + log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self.name, ", ".join(map(str, patch_ids)))) + + +class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler): name = "commit-queue" def __init__(self): - AbstractQueue.__init__(self) + AbstractPatchQueue.__init__(self) - # AbstractQueue methods + # AbstractPatchQueue methods def begin_work_queue(self): - AbstractQueue.begin_work_queue(self) + AbstractPatchQueue.begin_work_queue(self) self.committer_validator = CommitterValidator(self.tool.bugs) def _validate_patches_in_commit_queue(self): @@ -150,9 +172,18 @@ class CommitQueue(AbstractQueue, StepSequenceErrorHandler): all_patches = sum([self.tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True) for bug_id in bug_ids], []) return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches) + def _patch_cmp(self, a, b): + # Sort first by is_rollout, then by attach_date. + # Reversing the order so that is_rollout is first. + rollout_cmp = cmp(b.is_rollout(), a.is_rollout()) + if (rollout_cmp != 0): + return rollout_cmp + return cmp(a.attach_date(), b.attach_date()) + def next_work_item(self): patches = self._validate_patches_in_commit_queue() - # FIXME: We could sort the patches in a specific order here, was suggested by https://bugs.webkit.org/show_bug.cgi?id=33395 + patches = sorted(patches, self._patch_cmp) + self._update_work_items([patch.id() for patch in patches]) if not patches: self._update_status("Empty queue") return None @@ -162,47 +193,79 @@ class CommitQueue(AbstractQueue, StepSequenceErrorHandler): def _can_build_and_test(self): try: - self.run_webkit_patch(["build-and-test", "--force-clean", "--non-interactive", "--build-style=both", "--quiet"]) + self.run_webkit_patch([ + "build-and-test", + "--force-clean", + "--build", + "--test", + "--non-interactive", + "--no-update", + "--build-style=both", + "--quiet"]) except ScriptError, e: - self._update_status("Unabled to successfully build and test", None) - return False - return True - - def _builders_are_green(self): - red_builders_names = self.tool.buildbot.red_core_builders_names() - if red_builders_names: - red_builders_names = map(lambda name: "\"%s\"" % name, red_builders_names) # Add quotes around the names. - self._update_status("Builders [%s] are red. See http://build.webkit.org" % ", ".join(red_builders_names), None) + self._update_status("Unable to successfully build and test", None) return False return True def should_proceed_with_work_item(self, patch): - if not self._builders_are_green(): - return False - if not self._can_build_and_test(): - return False - if not self._builders_are_green(): - return False - self._update_status("Landing patch", patch) + patch_text = "rollout patch" if patch.is_rollout() else "patch" + self._update_status("Landing %s" % patch_text, patch) return True - def process_work_item(self, patch): + def _land(self, patch, first_run=False): try: - self._cc_watchers(patch.bug_id()) - # We pass --no-update here because we've already validated - # that the current revision actually builds and passes the tests. - # If we update, we risk moving to a revision that doesn't! - self.run_webkit_patch(["land-attachment", "--force-clean", "--non-interactive", "--no-update", "--parent-command=commit-queue", "--build-style=both", "--quiet", patch.id()]) + args = [ + "land-attachment", + "--force-clean", + "--build", + "--non-interactive", + "--ignore-builders", + "--build-style=both", + "--quiet", + patch.id() + ] + # We don't bother to run tests for rollouts as that makes them too slow. + if not patch.is_rollout(): + args.append("--test") + if not first_run: + # The first time through, we don't reject the patch from the + # commit queue because we want to make sure we can build and + # test ourselves. However, the second time through, we + # register ourselves as the parent-command so we can reject + # the patch on failure. + args.append("--parent-command=commit-queue") + # The second time through, we also don't want to update so we + # know we're testing the same revision that we successfully + # built and tested. + args.append("--no-update") + self.run_webkit_patch(args) self._did_pass(patch) + return True except ScriptError, e: + if first_run: + return False self._did_fail(patch) - raise e + raise + + def process_work_item(self, patch): + self._cc_watchers(patch.bug_id()) + if not self._land(patch, first_run=True): + # The patch failed to land, but the bots were green. It's possible + # that the bots were behind. To check that case, we try to build and + # test ourselves. + if not self._can_build_and_test(): + return False + # Hum, looks like the patch is actually bad. Of course, we could + # have been bitten by a flaky test the first time around. We try + # to land again. If it fails a second time, we're pretty sure its + # a bad test and re can reject it outright. + self._land(patch) + return True def handle_unexpected_error(self, patch, message): self.committer_validator.reject_patch_from_commit_queue(patch.id(), message) # StepSequenceErrorHandler methods - @staticmethod def _error_message_for_bug(tool, status_id, script_error): if not script_error.output: @@ -216,12 +279,68 @@ class CommitQueue(AbstractQueue, StepSequenceErrorHandler): validator = CommitterValidator(tool.bugs) validator.reject_patch_from_commit_queue(state["patch"].id(), cls._error_message_for_bug(tool, status_id, script_error)) + @classmethod + def handle_checkout_needs_update(cls, tool, state, options, error): + # The only time when we find out that out checkout needs update is + # when we were ready to actually pull the trigger and land the patch. + # Rather than spinning in the master process, we retry without + # building or testing, which is much faster. + options.build = False + options.test = False + options.update = True + raise TryAgain() -class AbstractReviewQueue(AbstractQueue, PersistentPatchCollectionDelegate, StepSequenceErrorHandler): + +class RietveldUploadQueue(AbstractPatchQueue, StepSequenceErrorHandler): + name = "rietveld-upload-queue" + + def __init__(self): + AbstractPatchQueue.__init__(self) + + # AbstractPatchQueue methods + + def next_work_item(self): + patch_id = self.tool.bugs.queries.fetch_first_patch_from_rietveld_queue() + if patch_id: + return patch_id + self._update_status("Empty queue") + + def should_proceed_with_work_item(self, patch): + self._update_status("Uploading patch", patch) + return True + + def process_work_item(self, patch): + try: + self.run_webkit_patch(["post-attachment-to-rietveld", "--force-clean", "--non-interactive", "--parent-command=rietveld-upload-queue", patch.id()]) + self._did_pass(patch) + return True + except ScriptError, e: + if e.exit_code != QueueEngine.handled_error_code: + self._did_fail(patch) + raise e + + @classmethod + def _reject_patch(cls, tool, patch_id): + tool.bugs.set_flag_on_attachment(patch_id, "in-rietveld", "-") + + def handle_unexpected_error(self, patch, message): + log(message) + self._reject_patch(self.tool, patch.id()) + + # StepSequenceErrorHandler methods + + @classmethod + def handle_script_error(cls, tool, state, script_error): + log(script_error.message_with_output()) + cls._update_status_for_script_error(tool, state, script_error) + cls._reject_patch(tool, state["patch"].id()) + + +class AbstractReviewQueue(AbstractPatchQueue, PersistentPatchCollectionDelegate, StepSequenceErrorHandler): def __init__(self, options=None): - AbstractQueue.__init__(self, options) + AbstractPatchQueue.__init__(self, options) - def _review_patch(self, patch): + def review_patch(self, patch): raise NotImplementedError, "subclasses must implement" # PersistentPatchCollectionDelegate methods @@ -238,10 +357,10 @@ class AbstractReviewQueue(AbstractQueue, PersistentPatchCollectionDelegate, Step def is_terminal_status(self, status): return status == "Pass" or status == "Fail" or status.startswith("Error:") - # AbstractQueue methods + # AbstractPatchQueue methods def begin_work_queue(self): - AbstractQueue.begin_work_queue(self) + AbstractPatchQueue.begin_work_queue(self) self._patches = PersistentPatchCollection(self) def next_work_item(self): @@ -255,8 +374,10 @@ class AbstractReviewQueue(AbstractQueue, PersistentPatchCollectionDelegate, Step def process_work_item(self, patch): try: - self._review_patch(patch) + if not self.review_patch(patch): + return False self._did_pass(patch) + return True except ScriptError, e: if e.exit_code != QueueEngine.handled_error_code: self._did_fail(patch) @@ -281,8 +402,9 @@ class StyleQueue(AbstractReviewQueue): self._update_status("Checking style", patch) return True - def _review_patch(self, patch): + def review_patch(self, patch): self.run_webkit_patch(["check-style", "--force-clean", "--non-interactive", "--parent-command=style-queue", patch.id()]) + return True @classmethod def handle_script_error(cls, tool, state, script_error): diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/queues_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/queues_unittest.py new file mode 100644 index 0000000..f82eb19 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queues_unittest.py @@ -0,0 +1,236 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + +from webkitpy.common.net.bugzilla import Attachment +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.queues import * +from webkitpy.tool.commands.queuestest import QueuesTest +from webkitpy.tool.mocktool import MockTool, MockSCM + + +class TestQueue(AbstractPatchQueue): + name = "test-queue" + + +class TestReviewQueue(AbstractReviewQueue): + name = "test-review-queue" + + +class MockPatch(object): + def is_rollout(self): + return True + + def bug_id(self): + return 12345 + + def id(self): + return 76543 + + +class AbstractQueueTest(CommandsTest): + def _assert_log_progress_output(self, patch_ids, progress_output): + OutputCapture().assert_outputs(self, TestQueue().log_progress, [patch_ids], expected_stderr=progress_output) + + def test_log_progress(self): + self._assert_log_progress_output([1,2,3], "3 patches in test-queue [1, 2, 3]\n") + self._assert_log_progress_output(["1","2","3"], "3 patches in test-queue [1, 2, 3]\n") + self._assert_log_progress_output([1], "1 patch in test-queue [1]\n") + + def test_log_directory(self): + self.assertEquals(TestQueue()._log_directory(), "test-queue-logs") + + def _assert_run_webkit_patch(self, run_args): + queue = TestQueue() + tool = MockTool() + tool.executive = Mock() + queue.bind_to_tool(tool) + + queue.run_webkit_patch(run_args) + expected_run_args = ["echo", "--status-host=example.com"] + run_args + tool.executive.run_and_throw_if_fail.assert_called_with(expected_run_args) + + def test_run_webkit_patch(self): + self._assert_run_webkit_patch([1]) + self._assert_run_webkit_patch(["one", 2]) + + def test_iteration_count(self): + queue = TestQueue() + queue.options = Mock() + queue.options.iterations = 3 + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + self.assertFalse(queue.should_continue_work_queue()) + + def test_no_iteration_count(self): + queue = TestQueue() + queue.options = Mock() + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + self.assertTrue(queue.should_continue_work_queue()) + + +class AbstractReviewQueueTest(CommandsTest): + def test_patch_collection_delegate_methods(self): + queue = TestReviewQueue() + tool = MockTool() + queue.bind_to_tool(tool) + self.assertEquals(queue.collection_name(), "test-review-queue") + self.assertEquals(queue.fetch_potential_patch_ids(), [103]) + queue.status_server() + self.assertTrue(queue.is_terminal_status("Pass")) + self.assertTrue(queue.is_terminal_status("Fail")) + self.assertTrue(queue.is_terminal_status("Error: Your patch exploded")) + self.assertFalse(queue.is_terminal_status("Foo")) + + +class CommitQueueTest(QueuesTest): + def test_commit_queue(self): + expected_stderr = { + "begin_work_queue" : "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % MockSCM.fake_checkout_root, + "should_proceed_with_work_item": "MOCK: update_status: commit-queue Landing patch\n", + # FIXME: The commit-queue warns about bad committers twice. This is due to the fact that we access Attachment.reviewer() twice and it logs each time. + "next_work_item" : """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Rejecting patch 128 from commit-queue.' and additional comment 'non-committer@example.com does not have committer permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/common/config/committers.py.\n\n- If you do not have committer rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags.\n\n- If you have committer rights please correct the error in WebKitTools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed). Due to bug 30084 the commit-queue will require a restart after your change. Please contact eseidel@chromium.org to request a commit-queue restart. After restart the commit-queue will correctly respect your committer rights.' +MOCK: update_work_items: commit-queue [106, 197] +2 patches in commit-queue [106, 197] +""", + "process_work_item" : "MOCK: update_status: commit-queue Pass\n", + "handle_unexpected_error" : "MOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'Mock error message'\n", + "handle_script_error": "MOCK: update_status: commit-queue ScriptError error message\nMOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'ScriptError error message'\n", + } + self.assert_queue_outputs(CommitQueue(), expected_stderr=expected_stderr) + + def test_rollout(self): + tool = MockTool(log_executive=True) + tool.buildbot.light_tree_on_fire() + expected_stderr = { + "begin_work_queue" : "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % MockSCM.fake_checkout_root, + "should_proceed_with_work_item": "MOCK: update_status: commit-queue Landing patch\n", + # FIXME: The commit-queue warns about bad committers twice. This is due to the fact that we access Attachment.reviewer() twice and it logs each time. + "next_work_item" : """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Rejecting patch 128 from commit-queue.' and additional comment 'non-committer@example.com does not have committer permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/common/config/committers.py. + +- If you do not have committer rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags. + +- If you have committer rights please correct the error in WebKitTools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed). Due to bug 30084 the commit-queue will require a restart after your change. Please contact eseidel@chromium.org to request a commit-queue restart. After restart the commit-queue will correctly respect your committer rights.' +MOCK: update_work_items: commit-queue [106, 197] +2 patches in commit-queue [106, 197] +""", + "process_work_item" : "MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--build', '--non-interactive', '--ignore-builders', '--build-style=both', '--quiet', 1234, '--test']\nMOCK: update_status: commit-queue Pass\n", + "handle_unexpected_error" : "MOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'Mock error message'\n", + "handle_script_error": "MOCK: update_status: commit-queue ScriptError error message\nMOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'ScriptError error message'\n", + } + self.assert_queue_outputs(CommitQueue(), tool=tool, expected_stderr=expected_stderr) + + def test_rollout_lands(self): + tool = MockTool(log_executive=True) + tool.buildbot.light_tree_on_fire() + rollout_patch = MockPatch() + expected_stderr = { + "begin_work_queue": "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % MockSCM.fake_checkout_root, + "should_proceed_with_work_item": "MOCK: update_status: commit-queue Landing rollout patch\n", + # FIXME: The commit-queue warns about bad committers twice. This is due to the fact that we access Attachment.reviewer() twice and it logs each time. + "next_work_item": """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com) +MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Rejecting patch 128 from commit-queue.' and additional comment 'non-committer@example.com does not have committer permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/common/config/committers.py. + +- If you do not have committer rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags. + +- If you have committer rights please correct the error in WebKitTools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed). Due to bug 30084 the commit-queue will require a restart after your change. Please contact eseidel@chromium.org to request a commit-queue restart. After restart the commit-queue will correctly respect your committer rights.' +MOCK: update_work_items: commit-queue [106, 197] +2 patches in commit-queue [106, 197] +""", + "process_work_item": "MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--build', '--non-interactive', '--ignore-builders', '--build-style=both', '--quiet', 76543]\nMOCK: update_status: commit-queue Pass\n", + "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '76543' with comment 'Rejecting patch 76543 from commit-queue.' and additional comment 'Mock error message'\n", + "handle_script_error": "MOCK: update_status: commit-queue ScriptError error message\nMOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'ScriptError error message'\n", + } + self.assert_queue_outputs(CommitQueue(), tool=tool, work_item=rollout_patch, expected_stderr=expected_stderr) + + def test_can_build_and_test(self): + queue = CommitQueue() + tool = MockTool() + tool.executive = Mock() + queue.bind_to_tool(tool) + self.assertTrue(queue._can_build_and_test()) + expected_run_args = ["echo", "--status-host=example.com", "build-and-test", "--force-clean", "--build", "--test", "--non-interactive", "--no-update", "--build-style=both", "--quiet"] + tool.executive.run_and_throw_if_fail.assert_called_with(expected_run_args) + + def _mock_attachment(self, is_rollout, attach_date): + attachment = Mock() + attachment.is_rollout = lambda: is_rollout + attachment.attach_date = lambda: attach_date + return attachment + + def test_patch_cmp(self): + long_ago_date = datetime(1900, 1, 21) + recent_date = datetime(2010, 1, 21) + attachment1 = self._mock_attachment(is_rollout=False, attach_date=recent_date) + attachment2 = self._mock_attachment(is_rollout=False, attach_date=long_ago_date) + attachment3 = self._mock_attachment(is_rollout=True, attach_date=recent_date) + attachment4 = self._mock_attachment(is_rollout=True, attach_date=long_ago_date) + attachments = [attachment1, attachment2, attachment3, attachment4] + expected_sort = [attachment4, attachment3, attachment2, attachment1] + queue = CommitQueue() + attachments.sort(queue._patch_cmp) + self.assertEqual(attachments, expected_sort) + + +class RietveldUploadQueueTest(QueuesTest): + def test_rietveld_upload_queue(self): + expected_stderr = { + "begin_work_queue": "CAUTION: rietveld-upload-queue will discard all local changes in \"%s\"\nRunning WebKit rietveld-upload-queue.\n" % MockSCM.fake_checkout_root, + "should_proceed_with_work_item": "MOCK: update_status: rietveld-upload-queue Uploading patch\n", + "process_work_item": "MOCK: update_status: rietveld-upload-queue Pass\n", + "handle_unexpected_error": "Mock error message\nMOCK setting flag 'in-rietveld' to '-' on attachment '1234' with comment 'None' and additional comment 'None'\n", + "handle_script_error": "ScriptError error message\nMOCK: update_status: rietveld-upload-queue ScriptError error message\nMOCK setting flag 'in-rietveld' to '-' on attachment '1234' with comment 'None' and additional comment 'None'\n", + } + self.assert_queue_outputs(RietveldUploadQueue(), expected_stderr=expected_stderr) + + +class StyleQueueTest(QueuesTest): + def test_style_queue(self): + expected_stderr = { + "begin_work_queue" : "CAUTION: style-queue will discard all local changes in \"%s\"\nRunning WebKit style-queue.\n" % MockSCM.fake_checkout_root, + "next_work_item": "MOCK: update_work_items: style-queue [103]\n", + "should_proceed_with_work_item": "MOCK: update_status: style-queue Checking style\n", + "process_work_item" : "MOCK: update_status: style-queue Pass\n", + "handle_unexpected_error" : "Mock error message\n", + "handle_script_error": "MOCK: update_status: style-queue ScriptError error message\nMOCK bug comment: bug_id=345, cc=[]\n--- Begin comment ---\\Attachment 1234 did not pass style-queue:\n\nScriptError error message\n\nIf any of these errors are false positives, please file a bug against check-webkit-style.\n--- End comment ---\n\n", + } + expected_exceptions = { + "handle_script_error": SystemExit, + } + self.assert_queue_outputs(StyleQueue(), expected_stderr=expected_stderr, expected_exceptions=expected_exceptions) diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/queuestest.py b/WebKitTools/Scripts/webkitpy/tool/commands/queuestest.py new file mode 100644 index 0000000..9e17c5c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/queuestest.py @@ -0,0 +1,95 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.net.bugzilla import Attachment +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.common.system.executive import ScriptError +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.mocktool import MockTool + + +class MockQueueEngine(object): + def __init__(self, name, queue, wakeup_event): + pass + + def run(self): + pass + + +class MockPatch(): + def id(self): + return 1234 + + def bug_id(self): + return 345 + + +class QueuesTest(unittest.TestCase): + mock_work_item = Attachment({ + "id": 1234, + "bug_id": 345, + "name": "Patch", + "attacher_email": "adam@example.com", + }, None) + + def assert_outputs(self, func, func_name, args, expected_stdout, expected_stderr, expected_exceptions): + exception = None + if expected_exceptions and func_name in expected_exceptions: + exception = expected_exceptions[func_name] + + OutputCapture().assert_outputs(self, + func, + args=args, + expected_stdout=expected_stdout.get(func_name, ""), + expected_stderr=expected_stderr.get(func_name, ""), + expected_exception=exception) + + def assert_queue_outputs(self, queue, args=None, work_item=None, expected_stdout=None, expected_stderr=None, expected_exceptions=None, options=Mock(), tool=MockTool()): + if not expected_stdout: + expected_stdout = {} + if not expected_stderr: + expected_stderr = {} + if not args: + args = [] + if not work_item: + work_item = self.mock_work_item + tool.user.prompt = lambda message: "yes" + + queue.execute(options, args, tool, engine=MockQueueEngine) + + self.assert_outputs(queue.queue_log_path, "queue_log_path", [], expected_stdout, expected_stderr, expected_exceptions) + self.assert_outputs(queue.work_item_log_path, "work_item_log_path", [work_item], expected_stdout, expected_stderr, expected_exceptions) + self.assert_outputs(queue.begin_work_queue, "begin_work_queue", [], expected_stdout, expected_stderr, expected_exceptions) + self.assert_outputs(queue.should_continue_work_queue, "should_continue_work_queue", [], expected_stdout, expected_stderr, expected_exceptions) + self.assert_outputs(queue.next_work_item, "next_work_item", [], expected_stdout, expected_stderr, expected_exceptions) + self.assert_outputs(queue.should_proceed_with_work_item, "should_proceed_with_work_item", [work_item], expected_stdout, expected_stderr, expected_exceptions) + self.assert_outputs(queue.process_work_item, "process_work_item", [work_item], expected_stdout, expected_stderr, expected_exceptions) + self.assert_outputs(queue.handle_unexpected_error, "handle_unexpected_error", [work_item, "Mock error message"], expected_stdout, expected_stderr, expected_exceptions) + self.assert_outputs(queue.handle_script_error, "handle_script_error", [tool, {"patch": MockPatch()}, ScriptError(message="ScriptError error message", script_args="MockErrorCommand")], expected_stdout, expected_stderr, expected_exceptions) diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/rebaseline.py b/WebKitTools/Scripts/webkitpy/tool/commands/rebaseline.py new file mode 100644 index 0000000..78e06c6 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/rebaseline.py @@ -0,0 +1,113 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os.path +import re +import shutil +import urllib + +from webkitpy.common.net.buildbot import BuildBot, LayoutTestResults +from webkitpy.common.system.user import User +from webkitpy.layout_tests.port import factory +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand + + +# FIXME: I'm not sure where this logic should go in the end. +# For now it's here, until we have a second need for it. +class BuilderToPort(object): + _builder_name_to_port_name = { + r"SnowLeopard": "mac-snowleopard", + r"Leopard": "mac-leopard", + r"Tiger": "mac-tiger", + r"Windows": "win", + r"GTK": "gtk", + r"Qt": "qt", + r"Chromium Mac": "chromium-mac", + r"Chromium Linux": "chromium-linux", + r"Chromium Win": "chromium-win", + } + + def _port_name_for_builder_name(self, builder_name): + for regexp, port_name in self._builder_name_to_port_name.items(): + if re.match(regexp, builder_name): + return port_name + + def port_for_builder(self, builder_name): + port_name = self._port_name_for_builder_name(builder_name) + assert(port_name) # Need to update _builder_name_to_port_name + port = factory.get(port_name) + assert(port) # Need to update _builder_name_to_port_name + return port + + +class Rebaseline(AbstractDeclarativeCommand): + name = "rebaseline" + help_text = "Replaces local expected.txt files with new results from build bots" + + # FIXME: This should share more code with FailureReason._builder_to_explain + def _builder_to_pull_from(self): + builder_statuses = self.tool.buildbot.builder_statuses() + red_statuses = [status for status in builder_statuses if not status["is_green"]] + print "%s failing" % (pluralize("builder", len(red_statuses))) + builder_choices = [status["name"] for status in red_statuses] + chosen_name = self.tool.user.prompt_with_list("Which builder to pull results from:", builder_choices) + # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object. + for status in red_statuses: + if status["name"] == chosen_name: + return (self.tool.buildbot.builder_with_name(chosen_name), status["build_number"]) + + def _replace_expectation_with_remote_result(self, local_file, remote_file): + (downloaded_file, headers) = urllib.urlretrieve(remote_file) + shutil.move(downloaded_file, local_file) + + def _tests_to_update(self, build): + parsed_results = build.layout_test_results().parsed_results() + # FIXME: This probably belongs as API on LayoutTestResults + # but .failing_tests() already means something else. + return parsed_results[LayoutTestResults.fail_key] + + def _results_url_for_test(self, build, test): + test_base = os.path.splitext(test)[0] + actual_path = test_base + "-actual.txt" + return build.results_url() + "/" + actual_path + + def execute(self, options, args, tool): + builder, build_number = self._builder_to_pull_from() + build = builder.build(build_number) + port = BuilderToPort().port_for_builder(builder.name()) + + for test in self._tests_to_update(build): + results_url = self._results_url_for_test(build, test) + # Port operates with absolute paths. + absolute_path = os.path.join(port.layout_tests_dir(), test) + expected_file = port.expected_filename(absolute_path, ".txt") + print test + self._replace_expectation_with_remote_result(expected_file, results_url) + + # FIXME: We should handle new results too. diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py new file mode 100644 index 0000000..d6582a7 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py @@ -0,0 +1,38 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.tool.commands.rebaseline import BuilderToPort + + +class BuilderToPortTest(unittest.TestCase): + def test_port_for_builder(self): + converter = BuilderToPort() + port = converter.port_for_builder("Leopard Intel Debug (Tests)") + self.assertEqual(port.name(), "mac-leopard") diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot.py b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot.py new file mode 100644 index 0000000..24c8517 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot.py @@ -0,0 +1,136 @@ +# Copyright (c) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.config.ports import WebKitPort +from webkitpy.tool.bot.sheriff import Sheriff +from webkitpy.tool.bot.sheriffircbot import SheriffIRCBot +from webkitpy.tool.commands.queues import AbstractQueue +from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler + + +class SheriffBot(AbstractQueue, StepSequenceErrorHandler): + name = "sheriff-bot" + watchers = AbstractQueue.watchers + [ + "abarth@webkit.org", + "eric@webkit.org", + ] + + def _update(self): + self.run_webkit_patch(["update", "--force-clean", "--quiet"]) + + # AbstractQueue methods + + def begin_work_queue(self): + AbstractQueue.begin_work_queue(self) + self._sheriff = Sheriff(self.tool, self) + self._irc_bot = SheriffIRCBot(self.tool, self._sheriff) + self.tool.ensure_irc_connected(self._irc_bot.irc_delegate()) + + def work_item_log_path(self, new_failures): + return os.path.join("%s-logs" % self.name, "%s.log" % new_failures.keys()[0]) + + def _new_failures(self, revisions_causing_failures, old_failing_svn_revisions): + # We ignore failures that might have been caused by svn_revisions that + # we've already complained about. This is conservative in the sense + # that we might be ignoring some new failures, but our experience has + # been that skipping this check causes a lot of spam for builders that + # take a long time to cycle. + old_failing_builder_names = [] + for svn_revision in old_failing_svn_revisions: + old_failing_builder_names.extend( + [builder.name() for builder in revisions_causing_failures[svn_revision]]) + + new_failures = {} + for svn_revision, builders in revisions_causing_failures.items(): + if svn_revision in old_failing_svn_revisions: + # FIXME: We should re-process the work item after some time delay. + # https://bugs.webkit.org/show_bug.cgi?id=36581 + continue + new_builders = [builder for builder in builders + if builder.name() not in old_failing_builder_names] + if new_builders: + new_failures[svn_revision] = new_builders + + return new_failures + + def next_work_item(self): + self._irc_bot.process_pending_messages() + self._update() + + # We do one read from buildbot to ensure a consistent view. + revisions_causing_failures = self.tool.buildbot.revisions_causing_failures() + + # Similarly, we read once from our the status_server. + old_failing_svn_revisions = [] + for svn_revision in revisions_causing_failures.keys(): + if self.tool.status_server.svn_revision(svn_revision): + old_failing_svn_revisions.append(svn_revision) + + new_failures = self._new_failures(revisions_causing_failures, + old_failing_svn_revisions) + + self._sheriff.provoke_flaky_builders(revisions_causing_failures) + return new_failures + + def should_proceed_with_work_item(self, new_failures): + # Currently, we don't have any reasons not to proceed with work items. + return True + + def process_work_item(self, new_failures): + blame_list = new_failures.keys() + for svn_revision, builders in new_failures.items(): + try: + commit_info = self.tool.checkout().commit_info_for_revision(svn_revision) + if not commit_info: + print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision + continue + self._sheriff.post_irc_warning(commit_info, builders) + self._sheriff.post_blame_comment_on_bug(commit_info, + builders, + blame_list) + self._sheriff.post_automatic_rollout_patch(commit_info, + builders) + finally: + for builder in builders: + self.tool.status_server.update_svn_revision(svn_revision, + builder.name()) + return True + + def handle_unexpected_error(self, new_failures, message): + log(message) + + # StepSequenceErrorHandler methods + + @classmethod + def handle_script_error(cls, tool, state, script_error): + # Ideally we would post some information to IRC about what went wrong + # here, but we don't have the IRC password in the child process. + pass diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py new file mode 100644 index 0000000..4b4b8b6 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py @@ -0,0 +1,73 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + +from webkitpy.tool.commands.queuestest import QueuesTest +from webkitpy.tool.commands.sheriffbot import SheriffBot +from webkitpy.tool.mocktool import MockBuilder + + +class SheriffBotTest(QueuesTest): + builder1 = MockBuilder("Builder1") + builder2 = MockBuilder("Builder2") + + def test_sheriff_bot(self): + mock_work_item = { + 29837: [self.builder1], + } + expected_stderr = { + "begin_work_queue": "CAUTION: sheriff-bot will discard all local changes in \"%s\"\nRunning WebKit sheriff-bot.\n" % os.getcwd(), + "next_work_item": "", + "process_work_item": "MOCK: irc.post: abarth, darin, eseidel: http://trac.webkit.org/changeset/29837 might have broken Builder1\nMOCK bug comment: bug_id=42, cc=['abarth@webkit.org', 'eric@webkit.org']\n--- Begin comment ---\\http://trac.webkit.org/changeset/29837 might have broken Builder1\n--- End comment ---\n\n", + "handle_unexpected_error": "Mock error message\n" + } + self.assert_queue_outputs(SheriffBot(), work_item=mock_work_item, expected_stderr=expected_stderr) + + revisions_causing_failures = { + 1234: [builder1], + 1235: [builder1, builder2], + } + + def test_new_failures(self): + old_failing_svn_revisions = [] + self.assertEquals(SheriffBot()._new_failures(self.revisions_causing_failures, + old_failing_svn_revisions), + self.revisions_causing_failures) + + def test_new_failures_with_old_revisions(self): + old_failing_svn_revisions = [1234] + self.assertEquals(SheriffBot()._new_failures(self.revisions_causing_failures, + old_failing_svn_revisions), + {1235: [builder2]}) + + def test_new_failures_with_old_revisions(self): + old_failing_svn_revisions = [1235] + self.assertEquals(SheriffBot()._new_failures(self.revisions_causing_failures, + old_failing_svn_revisions), + {}) diff --git a/WebKitTools/Scripts/webkitpy/stepsequence.py b/WebKitTools/Scripts/webkitpy/tool/commands/stepsequence.py index 008b366..be2ed4c 100644 --- a/WebKitTools/Scripts/webkitpy/stepsequence.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/stepsequence.py @@ -26,12 +26,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import webkitpy.steps as steps +import webkitpy.tool.steps as steps -from webkitpy.executive import ScriptError -from webkitpy.webkit_logging import log -from webkitpy.scm import CheckoutNeedsUpdate -from webkitpy.queueengine import QueueEngine +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.checkout.scm import CheckoutNeedsUpdate +from webkitpy.tool.bot.queueengine import QueueEngine +from webkitpy.common.system.deprecated_logging import log class StepSequenceErrorHandler(): @@ -39,6 +39,10 @@ class StepSequenceErrorHandler(): def handle_script_error(cls, tool, patch, script_error): raise NotImplementedError, "subclasses must implement" + @classmethod + def handle_checkout_needs_update(cls, tool, state, options, error): + raise NotImplementedError, "subclasses must implement" + class StepSequence(object): def __init__(self, steps): @@ -66,7 +70,9 @@ class StepSequence(object): self._run(tool, options, state) except CheckoutNeedsUpdate, e: log("Commit failed because the checkout is out of date. Please update and try again.") - log("You can pass --no-build to skip building/testing after update if you believe the new commits did not affect the results.") + if options.parent_command: + command = tool.command_by_name(options.parent_command) + command.handle_checkout_needs_update(tool, state, options, e) QueueEngine.exit_after_handled_error(e) except ScriptError, e: if not options.quiet: diff --git a/WebKitTools/Scripts/webkitpy/commands/upload.py b/WebKitTools/Scripts/webkitpy/tool/commands/upload.py index 15bdfbb..4a15ed6 100644 --- a/WebKitTools/Scripts/webkitpy/commands/upload.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/upload.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009, 2010 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -30,30 +30,38 @@ import os import re -import StringIO import sys from optparse import make_option -import webkitpy.steps as steps +import webkitpy.tool.steps as steps + +from webkitpy.common.config.committers import CommitterList +from webkitpy.common.net.bugzilla import parse_bug_id +from webkitpy.common.system.user import User +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand +from webkitpy.tool.grammar import pluralize, join_with_separators +from webkitpy.tool.comments import bug_comment_from_svn_revision +from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand +from webkitpy.common.system.deprecated_logging import error, log -from webkitpy.bugzilla import parse_bug_id -from webkitpy.commands.abstractsequencedcommand import AbstractSequencedCommand -from webkitpy.comments import bug_comment_from_svn_revision -from webkitpy.committers import CommitterList -from webkitpy.grammar import pluralize, join_with_separators -from webkitpy.webkit_logging import error, log -from webkitpy.mock import Mock -from webkitpy.multicommandtool import AbstractDeclarativeCommand -from webkitpy.user import User class CommitMessageForCurrentDiff(AbstractDeclarativeCommand): name = "commit-message" help_text = "Print a commit message suitable for the uncommitted changes" + def __init__(self): + options = [ + steps.Options.git_commit, + ] + AbstractDeclarativeCommand.__init__(self, options=options) + def execute(self, options, args, tool): - os.chdir(tool.scm().checkout_root) - print "%s" % tool.scm().commit_message_for_this_commit().message() + # This command is a useful test to make sure commit_message_for_this_commit + # always returns the right value regardless of the current working directory. + print "%s" % tool.checkout().commit_message_for_this_commit(options.git_commit).message() + class CleanPendingCommit(AbstractDeclarativeCommand): name = "clean-pending-commit" @@ -97,8 +105,8 @@ class AssignToCommitter(AbstractDeclarativeCommand): def _assign_bug_to_last_patch_attacher(self, bug_id): committers = CommitterList() bug = self.tool.bugs.fetch_bug(bug_id) - assigned_to_email = bug.assigned_to_email() - if assigned_to_email != self.tool.bugs.unassigned_email: + if not bug.is_unassigned(): + assigned_to_email = bug.assigned_to_email() log("Bug %s is already assigned to %s (%s)." % (bug_id, assigned_to_email, committers.committer_by_email(assigned_to_email))) return @@ -140,19 +148,18 @@ class ObsoleteAttachments(AbstractSequencedCommand): class AbstractPatchUploadingCommand(AbstractSequencedCommand): - def _bug_id(self, args, tool, state): + def _bug_id(self, options, args, tool, state): # Perfer a bug id passed as an argument over a bug url in the diff (i.e. ChangeLogs). bug_id = args and args[0] if not bug_id: - state["diff"] = tool.scm().create_patch() - bug_id = parse_bug_id(state["diff"]) + bug_id = tool.checkout().bug_id_for_this_commit(options.git_commit) return bug_id def _prepare_state(self, options, args, tool): state = {} - state["bug_id"] = self._bug_id(args, tool, state) + state["bug_id"] = self._bug_id(options, args, tool, state) if not state["bug_id"]: - error("No bug id passed and no bug url found in diff.") + error("No bug id passed and no bug url found in ChangeLogs.") return state @@ -160,7 +167,6 @@ class Post(AbstractPatchUploadingCommand): name = "post" help_text = "Attach the current working directory diff to a bug as a patch file" argument_names = "[BUGID]" - show_in_main_help = True steps = [ steps.CheckStyle, steps.ConfirmDiff, @@ -171,8 +177,13 @@ class Post(AbstractPatchUploadingCommand): class LandSafely(AbstractPatchUploadingCommand): name = "land-safely" - help_text = "Land the current diff via the commit-queue (Experimental)" + help_text = "Land the current diff via the commit-queue" argument_names = "[BUGID]" + long_help = """land-safely updates the ChangeLog with the reviewer listed + in bugs.webkit.org for BUGID (or the bug ID detected from the ChangeLog). + The command then uploads the current diff to the bug and marks it for + commit by the commit-queue.""" + show_in_main_help = True steps = [ steps.UpdateChangeLogsWithReviewer, steps.ObsoletePatches, @@ -184,7 +195,6 @@ class Prepare(AbstractSequencedCommand): name = "prepare" help_text = "Creates a bug (or prompts for an existing bug) and prepares the ChangeLogs" argument_names = "[BUGID]" - show_in_main_help = True steps = [ steps.PromptForBugOrTitle, steps.CreateBug, @@ -221,7 +231,7 @@ class Upload(AbstractPatchUploadingCommand): def _prepare_state(self, options, args, tool): state = {} - state["bug_id"] = self._bug_id(args, tool, state) + state["bug_id"] = self._bug_id(options, args, tool, state) return state @@ -258,10 +268,6 @@ class PostCommits(AbstractDeclarativeCommand): comment_text += tool.scm().files_changed_summary_for_commit(commit_id) return comment_text - def _diff_file_for_commit(self, tool, commit_id): - diff = tool.scm().create_patch_from_local_commit(commit_id) - return StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object - def execute(self, options, args, tool): commit_ids = tool.scm().commit_ids_from_commitish_arguments(args) if len(commit_ids) > 10: # We could lower this limit, 10 is too many for one bug as-is. @@ -272,7 +278,7 @@ class PostCommits(AbstractDeclarativeCommand): commit_message = tool.scm().commit_message_for_local_commit(commit_id) # Prefer --bug-id=, then a bug url in the commit message, then a bug url in the entire commit diff (i.e. ChangeLogs). - bug_id = options.bug_id or parse_bug_id(commit_message.message()) or parse_bug_id(tool.scm().create_patch_from_local_commit(commit_id)) + bug_id = options.bug_id or parse_bug_id(commit_message.message()) or parse_bug_id(tool.scm().create_patch(git_commit=commit_id)) if not bug_id: log("Skipping %s: No bug id found in commit or specified with --bug-id." % commit_id) continue @@ -282,12 +288,13 @@ class PostCommits(AbstractDeclarativeCommand): steps.ObsoletePatches(tool, options).run(state) have_obsoleted_patches.add(bug_id) - diff_file = self._diff_file_for_commit(tool, commit_id) + diff = tool.scm().create_patch(git_commit=commit_id) description = options.description or commit_message.description(lstrip=True, strip_url=True) comment_text = self._comment_text_for_commit(options, commit_message, tool, commit_id) - tool.bugs.add_patch_to_bug(bug_id, diff_file, description, comment_text, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) + tool.bugs.add_patch_to_bug(bug_id, diff, description, comment_text, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) +# FIXME: This command needs to be brought into the modern age with steps and CommitInfo. class MarkBugFixed(AbstractDeclarativeCommand): name = "mark-bug-fixed" help_text = "Mark the specified bug as fixed" @@ -301,6 +308,7 @@ class MarkBugFixed(AbstractDeclarativeCommand): ] AbstractDeclarativeCommand.__init__(self, options=options) + # FIXME: We should be using checkout().changelog_entries_for_revision(...) instead here. def _fetch_commit_log(self, tool, svn_revision): if not svn_revision: return tool.scm().last_svn_commit_log() @@ -399,9 +407,8 @@ class CreateBug(AbstractDeclarativeCommand): comment_text += "---\n" comment_text += tool.scm().files_changed_summary_for_commit(commit_id) - diff = tool.scm().create_patch_from_local_commit(commit_id) - diff_file = StringIO.StringIO(diff) # create_bug expects a file-like object - bug_id = tool.bugs.create_bug(bug_title, comment_text, options.component, diff_file, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) + diff = tool.scm().create_patch(git_commit=commit_id) + bug_id = tool.bugs.create_bug(bug_title, comment_text, options.component, diff, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) if bug_id and len(commit_ids) > 1: options.bug_id = bug_id @@ -415,13 +422,12 @@ class CreateBug(AbstractDeclarativeCommand): if options.prompt: (bug_title, comment_text) = self.prompt_for_bug_title_and_comment() else: - commit_message = tool.scm().commit_message_for_this_commit() + commit_message = tool.checkout().commit_message_for_this_commit(options.git_commit) bug_title = commit_message.description(lstrip=True, strip_url=True) comment_text = commit_message.body(lstrip=True) - diff = tool.scm().create_patch() - diff_file = StringIO.StringIO(diff) # create_bug expects a file-like object - bug_id = tool.bugs.create_bug(bug_title, comment_text, options.component, diff_file, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) + diff = tool.scm().create_patch(options.git_commit) + bug_id = tool.bugs.create_bug(bug_title, comment_text, options.component, diff, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) def prompt_for_bug_title_and_comment(self): bug_title = User.prompt("Bug title: ") diff --git a/WebKitTools/Scripts/webkitpy/commands/upload_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/upload_unittest.py index 7fa8797..5f3f400 100644 --- a/WebKitTools/Scripts/webkitpy/commands/upload_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/commands/upload_unittest.py @@ -26,25 +26,22 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.commands.commandtest import CommandsTest -from webkitpy.commands.upload import * -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool +from webkitpy.thirdparty.mock import Mock +from webkitpy.tool.commands.commandtest import CommandsTest +from webkitpy.tool.commands.upload import * +from webkitpy.tool.mocktool import MockOptions, MockTool class UploadCommandsTest(CommandsTest): def test_commit_message_for_current_diff(self): - tool = MockBugzillaTool() - mock_commit_message_for_this_commit = Mock() - mock_commit_message_for_this_commit.message = lambda: "Mock message" - tool._scm.commit_message_for_this_commit = lambda: mock_commit_message_for_this_commit - expected_stdout = "Mock message\n" + tool = MockTool() + expected_stdout = "This is a fake commit message that is at least 50 characters.\n" self.assert_execute_outputs(CommitMessageForCurrentDiff(), [], expected_stdout=expected_stdout, tool=tool) def test_clean_pending_commit(self): self.assert_execute_outputs(CleanPendingCommit(), []) def test_assign_to_committer(self): - tool = MockBugzillaTool() + tool = MockTool() expected_stderr = "Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)\nBug 77 is already assigned to foo@foo.com (None).\nBug 76 has no non-obsolete patches, ignoring.\n" self.assert_execute_outputs(AssignToCommitter(), [], expected_stderr=expected_stderr, tool=tool) tool.bugs.reassign_bug.assert_called_with(42, "eric@webkit.org", "Attachment 128 was posted by a committer and has review+, assigning to Eric Seidel for commit.") @@ -54,33 +51,59 @@ class UploadCommandsTest(CommandsTest): self.assert_execute_outputs(ObsoleteAttachments(), [42], expected_stderr=expected_stderr) def test_post(self): - expected_stderr = "Running check-webkit-style\nObsoleting 2 old patches on bug 42\n" - self.assert_execute_outputs(Post(), [42], expected_stderr=expected_stderr) + options = MockOptions() + options.description = "MOCK description" + options.request_commit = False + options.review = True + options.comment = None + options.cc = None + expected_stderr = """Running check-webkit-style +MOCK: user.open_url: file://... +Obsoleting 2 old patches on bug 42 +MOCK add_patch_to_bug: bug_id=42, description=MOCK description, mark_for_review=True, mark_for_commit_queue=False, mark_for_landing=False +-- Begin comment -- +None +-- End comment -- +MOCK: user.open_url: http://example.com/42 +""" + self.assert_execute_outputs(Post(), [42], options=options, expected_stderr=expected_stderr) - def test_post(self): - expected_stderr = "Obsoleting 2 old patches on bug 42\n" + def test_land_safely(self): + expected_stderr = "Obsoleting 2 old patches on bug 42\nMOCK add_patch_to_bug: bug_id=42, description=Patch for landing, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=True\n-- Begin comment --\nNone\n-- End comment --\n" self.assert_execute_outputs(LandSafely(), [42], expected_stderr=expected_stderr) def test_prepare_diff_with_arg(self): self.assert_execute_outputs(Prepare(), [42]) def test_prepare(self): - self.assert_execute_outputs(Prepare(), []) + expected_stderr = "MOCK create_bug\nbug_title: Mock user response\nbug_description: Mock user response\n" + self.assert_execute_outputs(Prepare(), [], expected_stderr=expected_stderr) def test_upload(self): - expected_stderr = "Running check-webkit-style\nObsoleting 2 old patches on bug 42\nMOCK: user.open_url: http://example.com/42\n" - self.assert_execute_outputs(Upload(), [42], expected_stderr=expected_stderr) + options = MockOptions() + options.description = "MOCK description" + options.request_commit = False + options.review = True + options.comment = None + options.cc = None + expected_stderr = """Running check-webkit-style +MOCK: user.open_url: file://... +Obsoleting 2 old patches on bug 42 +MOCK add_patch_to_bug: bug_id=42, description=MOCK description, mark_for_review=True, mark_for_commit_queue=False, mark_for_landing=False +-- Begin comment -- +None +-- End comment -- +MOCK: user.open_url: http://example.com/42 +""" + self.assert_execute_outputs(Upload(), [42], options=options, expected_stderr=expected_stderr) def test_mark_bug_fixed(self): - tool = MockBugzillaTool() + tool = MockTool() tool._scm.last_svn_commit_log = lambda: "r9876 |" options = Mock() options.bug_id = 42 - expected_stderr = """Bug: <http://example.com/42> Bug with two r+'d and cq+'d patches, one of which has an invalid commit-queue setter. -Revision: 9876 -MOCK: user.open_url: http://example.com/42 -Adding comment to Bug 42. -""" + options.comment = "MOCK comment" + expected_stderr = "Bug: <http://example.com/42> Bug with two r+'d and cq+'d patches, one of which has an invalid commit-queue setter.\nRevision: 9876\nMOCK: user.open_url: http://example.com/42\nAdding comment to Bug 42.\nMOCK bug comment: bug_id=42, cc=None\n--- Begin comment ---\\MOCK comment\n\nCommitted r9876: <http://trac.webkit.org/changeset/9876>\n--- End comment ---\n\n" self.assert_execute_outputs(MarkBugFixed(), [], expected_stderr=expected_stderr, tool=tool, options=options) def test_edit_changelog(self): diff --git a/WebKitTools/Scripts/webkitpy/comments.py b/WebKitTools/Scripts/webkitpy/tool/comments.py index 77ad239..83f2be8 100755 --- a/WebKitTools/Scripts/webkitpy/comments.py +++ b/WebKitTools/Scripts/webkitpy/tool/comments.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -30,7 +30,7 @@ # A tool for automating dealing with bugzilla, posting patches, committing # patches, etc. -from webkitpy.changelogs import view_source_url +from webkitpy.common.checkout.changelog import view_source_url def bug_comment_from_svn_revision(svn_revision): diff --git a/WebKitTools/Scripts/webkitpy/grammar.py b/WebKitTools/Scripts/webkitpy/tool/grammar.py index 651bbc9..8db9826 100644 --- a/WebKitTools/Scripts/webkitpy/grammar.py +++ b/WebKitTools/Scripts/webkitpy/tool/grammar.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -45,9 +44,11 @@ def pluralize(noun, count): return "%d %s" % (count, noun) -def join_with_separators(list_of_strings, separator=', ', last_separator=', and '): +def join_with_separators(list_of_strings, separator=', ', only_two_separator=" and ", last_separator=', and '): if not list_of_strings: return "" if len(list_of_strings) == 1: return list_of_strings[0] + if len(list_of_strings) == 2: + return only_two_separator.join(list_of_strings) return "%s%s%s" % (separator.join(list_of_strings[:-1]), last_separator, list_of_strings[-1]) diff --git a/WebKitTools/Scripts/webkitpy/grammar_unittest.py b/WebKitTools/Scripts/webkitpy/tool/grammar_unittest.py index 3d8b179..cab71db 100644 --- a/WebKitTools/Scripts/webkitpy/grammar_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/grammar_unittest.py @@ -27,11 +27,14 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from webkitpy.grammar import join_with_separators + +from webkitpy.tool.grammar import join_with_separators class GrammarTest(unittest.TestCase): def test_join_with_separators(self): + self.assertEqual(join_with_separators(["one"]), "one") + self.assertEqual(join_with_separators(["one", "two"]), "one and two") self.assertEqual(join_with_separators(["one", "two", "three"]), "one, two, and three") if __name__ == '__main__': diff --git a/WebKitTools/Scripts/webkitpy/tool/main.py b/WebKitTools/Scripts/webkitpy/tool/main.py new file mode 100755 index 0000000..1f43145 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/main.py @@ -0,0 +1,141 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# A tool for automating dealing with bugzilla, posting patches, committing patches, etc. + +import os +import threading + +from webkitpy.common.checkout.api import Checkout +from webkitpy.common.checkout.scm import detect_scm_system +from webkitpy.common.net.bugzilla import Bugzilla +from webkitpy.common.net.buildbot import BuildBot +from webkitpy.common.net.rietveld import Rietveld +from webkitpy.common.net.irc.ircproxy import IRCProxy +from webkitpy.common.system.executive import Executive +from webkitpy.common.system.user import User +import webkitpy.tool.commands as commands +# FIXME: Remove these imports once all the commands are in the root of the +# command package. +from webkitpy.tool.commands.download import * +from webkitpy.tool.commands.earlywarningsystem import * +from webkitpy.tool.commands.openbugs import OpenBugs +from webkitpy.tool.commands.queries import * +from webkitpy.tool.commands.queues import * +from webkitpy.tool.commands.sheriffbot import * +from webkitpy.tool.commands.upload import * +from webkitpy.tool.multicommandtool import MultiCommandTool +from webkitpy.common.system.deprecated_logging import log + + +class WebKitPatch(MultiCommandTool): + global_options = [ + make_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="enable all logging"), + make_option("--dry-run", action="store_true", dest="dry_run", default=False, help="do not touch remote servers"), + make_option("--status-host", action="store", dest="status_host", type="string", nargs=1, help="Hostname (e.g. localhost or commit.webkit.org) where status updates should be posted."), + make_option("--irc-password", action="store", dest="irc_password", type="string", nargs=1, help="Password to use when communicating via IRC."), + ] + + def __init__(self, path): + MultiCommandTool.__init__(self) + + self._path = path + self.wakeup_event = threading.Event() + self.bugs = Bugzilla() + self.buildbot = BuildBot() + self.executive = Executive() + self._irc = None + self.user = User() + self._scm = None + self._checkout = None + self.status_server = StatusServer() + self.codereview = Rietveld(self.executive) + + def scm(self): + # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands). + original_cwd = os.path.abspath(".") + if not self._scm: + self._scm = detect_scm_system(original_cwd) + + if not self._scm: + script_directory = os.path.abspath(sys.path[0]) + self._scm = detect_scm_system(script_directory) + if self._scm: + log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, self._scm.checkout_root)) + else: + error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, script_directory)) + + return self._scm + + def checkout(self): + if not self._checkout: + self._checkout = Checkout(self.scm()) + return self._checkout + + def ensure_irc_connected(self, irc_delegate): + if not self._irc: + self._irc = IRCProxy(irc_delegate) + + def irc(self): + # We don't automatically construct IRCProxy here because constructing + # IRCProxy actually connects to IRC. We want clients to explicitly + # connect to IRC. + return self._irc + + def path(self): + return self._path + + def command_completed(self): + if self._irc: + self._irc.disconnect() + + def should_show_in_main_help(self, command): + if not command.show_in_main_help: + return False + if command.requires_local_commits: + return self.scm().supports_local_commits() + return True + + # FIXME: This may be unnecessary since we pass global options to all commands during execute() as well. + def handle_global_options(self, options): + self._options = options + if options.dry_run: + self.scm().dryrun = True + self.bugs.dryrun = True + self.codereview.dryrun = True + if options.status_host: + self.status_server.set_host(options.status_host) + if options.irc_password: + self.irc_password = options.irc_password + + def should_execute_command(self, command): + if command.requires_local_commits and not self.scm().supports_local_commits(): + failure_reason = "%s requires local commits using %s in %s." % (command.name, self.scm().display_name(), self.scm().checkout_root) + return (False, failure_reason) + return (True, None) diff --git a/WebKitTools/Scripts/webkitpy/mock_bugzillatool.py b/WebKitTools/Scripts/webkitpy/tool/mocktool.py index f522e40..2114c30 100644 --- a/WebKitTools/Scripts/webkitpy/mock_bugzillatool.py +++ b/WebKitTools/Scripts/webkitpy/tool/mocktool.py @@ -27,12 +27,15 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os +import threading -from webkitpy.bugzilla import Bug, Attachment -from webkitpy.committers import CommitterList, Reviewer -from webkitpy.mock import Mock -from webkitpy.scm import CommitMessage -from webkitpy.webkit_logging import log +from webkitpy.common.config.committers import CommitterList, Reviewer +from webkitpy.common.checkout.commitinfo import CommitInfo +from webkitpy.common.checkout.scm import CommitMessage +from webkitpy.common.net.bugzilla import Bug, Attachment +from webkitpy.common.net.rietveld import Rietveld +from webkitpy.thirdparty.mock import Mock +from webkitpy.common.system.deprecated_logging import log def _id_to_object_dictionary(*objects): @@ -41,6 +44,7 @@ def _id_to_object_dictionary(*objects): dictionary[thing["id"]] = thing return dictionary +# Testing # FIXME: The ids should be 1, 2, 3 instead of crazy numbers. @@ -49,6 +53,7 @@ _patch1 = { "id": 197, "bug_id": 42, "url": "http://example.com/197", + "name": "Patch1", "is_obsolete": False, "is_patch": True, "review": "+", @@ -63,6 +68,7 @@ _patch2 = { "id": 128, "bug_id": 42, "url": "http://example.com/128", + "name": "Patch2", "is_obsolete": False, "is_patch": True, "review": "+", @@ -77,8 +83,10 @@ _patch3 = { "id": 103, "bug_id": 75, "url": "http://example.com/103", + "name": "Patch3", "is_obsolete": False, "is_patch": True, + "in-rietveld": "?", "review": "?", "attacher_email": "eric@webkit.org", } @@ -88,6 +96,7 @@ _patch4 = { "id": 104, "bug_id": 77, "url": "http://example.com/103", + "name": "Patch3", "is_obsolete": False, "is_patch": True, "review": "+", @@ -101,8 +110,10 @@ _patch5 = { "id": 105, "bug_id": 77, "url": "http://example.com/103", + "name": "Patch5", "is_obsolete": False, "is_patch": True, + "in-rietveld": "?", "review": "+", "reviewer_email": "foo@bar.com", "attacher_email": "eric@webkit.org", @@ -113,8 +124,10 @@ _patch6 = { # Valid committer, but no reviewer. "id": 106, "bug_id": 77, "url": "http://example.com/103", + "name": "ROLLOUT of r3489", "is_obsolete": False, "is_patch": True, + "in-rietveld": "-", "commit-queue": "+", "committer_email": "foo@bar.com", "attacher_email": "eric@webkit.org", @@ -125,17 +138,18 @@ _patch7 = { # Valid review, patch is marked obsolete. "id": 107, "bug_id": 76, "url": "http://example.com/103", + "name": "Patch7", "is_obsolete": True, "is_patch": True, + "in-rietveld": "+", "review": "+", "reviewer_email": "foo@bar.com", "attacher_email": "eric@webkit.org", } -# This must be defined before we define the bugs, thus we don't use -# MockBugzilla.unassigned_email directly. -_unassigned_email = "unassigned@example.com" +# This matches one of Bug.unassigned_emails +_unassigned_email = "webkit-unassigned@lists.webkit.org" # FIXME: The ids should be 1, 2, 3 instead of crazy numbers. @@ -211,6 +225,12 @@ class MockBugzillaQueries(Mock): def fetch_patches_from_pending_commit_list(self): return sum([bug.reviewed_patches() for bug in self._all_bugs()], []) + def fetch_first_patch_from_rietveld_queue(self): + for bug in self._all_bugs(): + patches = bug.in_rietveld_queue_patches() + if len(patches): + return patches[0] + raise Exception('No patches in the rietveld queue') # FIXME: Bugzilla is the wrong Mock-point. Once we have a BugzillaNetwork # class we should mock that instead. @@ -221,8 +241,6 @@ class MockBugzilla(Mock): bug_server_url = "http://example.com" - unassigned_email = _unassigned_email - bug_cache = _id_to_object_dictionary(_bug1, _bug2, _bug3, _bug4) attachment_cache = _id_to_object_dictionary(_patch1, @@ -239,6 +257,23 @@ class MockBugzilla(Mock): self.committers = CommitterList(reviewers=[Reviewer("Foo Bar", "foo@bar.com")]) + def create_bug(self, + bug_title, + bug_description, + component=None, + diff=None, + patch_description=None, + cc=None, + blocked=None, + mark_for_review=False, + mark_for_commit_queue=False): + log("MOCK create_bug") + log("bug_title: %s" % bug_title) + log("bug_description: %s" % bug_description) + + def quips(self): + return ["Good artists copy. Great artists steal. - Pablo Picasso"] + def fetch_bug(self, bug_id): return Bug(self.bug_cache.get(bug_id), self) @@ -262,29 +297,110 @@ class MockBugzilla(Mock): action_param = "&action=%s" % action return "%s/%s%s" % (self.bug_server_url, attachment_id, action_param) - -class MockBuildBot(Mock): - - def builder_statuses(self): - return [{ + def set_flag_on_attachment(self, + attachment_id, + flag_name, + flag_value, + comment_text=None, + additional_comment_text=None): + log("MOCK setting flag '%s' to '%s' on attachment '%s' with comment '%s' and additional comment '%s'" % ( + flag_name, flag_value, attachment_id, comment_text, additional_comment_text)) + + def post_comment_to_bug(self, bug_id, comment_text, cc=None): + log("MOCK bug comment: bug_id=%s, cc=%s\n--- Begin comment ---\%s\n--- End comment ---\n" % ( + bug_id, cc, comment_text)) + + def add_patch_to_bug(self, + bug_id, + diff, + description, + comment_text=None, + mark_for_review=False, + mark_for_commit_queue=False, + mark_for_landing=False): + log("MOCK add_patch_to_bug: bug_id=%s, description=%s, mark_for_review=%s, mark_for_commit_queue=%s, mark_for_landing=%s" % + (bug_id, description, mark_for_review, mark_for_commit_queue, mark_for_landing)) + log("-- Begin comment --") + log(comment_text) + log("-- End comment --") + + +class MockBuilder(object): + def __init__(self, name): + self._name = name + + def name(self): + return self._name + + def results_url(self): + return "http://example.com/builders/%s/results/" % self.name() + + def force_build(self, username, comments): + log("MOCK: force_build: name=%s, username=%s, comments=%s" % ( + self._name, username, comments)) + + +class MockBuildBot(object): + def __init__(self): + self._mock_builder1_status = { "name": "Builder1", "is_green": True, - }, { + "activity": "building", + } + self._mock_builder2_status = { "name": "Builder2", "is_green": True, - }] + "activity": "idle", + } + + def builder_with_name(self, name): + return MockBuilder(name) + + def builder_statuses(self): + return [ + self._mock_builder1_status, + self._mock_builder2_status, + ] def red_core_builders_names(self): + if not self._mock_builder2_status["is_green"]: + return [self._mock_builder2_status["name"]] + return [] + + def red_core_builders(self): + if not self._mock_builder2_status["is_green"]: + return [self._mock_builder2_status] return [] + def idle_red_core_builders(self): + if not self._mock_builder2_status["is_green"]: + return [self._mock_builder2_status] + return [] + + def last_green_revision(self): + return 9479 + + def light_tree_on_fire(self): + self._mock_builder2_status["is_green"] = False + + def revisions_causing_failures(self): + return { + "29837": [self.builder_with_name("Builder1")], + } + class MockSCM(Mock): + fake_checkout_root = os.path.realpath("/tmp") # realpath is needed to allow for Mac OS X's /private/tmp + def __init__(self): Mock.__init__(self) - self.checkout_root = os.getcwd() + # FIXME: We should probably use real checkout-root detection logic here. + # os.getcwd() can't work here because other parts of the code assume that "checkout_root" + # will actually be the root. Since getcwd() is wrong, use a globally fake root for now. + self.checkout_root = self.fake_checkout_root - def create_patch(self): + def create_patch(self, git_commit): return "Patch1" def commit_ids_from_commitish_arguments(self, args): @@ -299,13 +415,6 @@ class MockSCM(Mock): "https://bugs.example.org/show_bug.cgi?id=75\n") raise Exception("Bogus commit_id in commit_message_for_local_commit.") - def create_patch_from_local_commit(self, commit_id): - if commit_id == "Commitish1": - return "Patch1" - if commit_id == "Commitish2": - return "Patch2" - raise Exception("Bogus commit_id in commit_message_for_local_commit.") - def diff_for_revision(self, revision): return "DiffForRevision%s\n" \ "http://bugs.webkit.org/show_bug.cgi?id=12345" % revision @@ -313,11 +422,40 @@ class MockSCM(Mock): def svn_revision_from_commit_text(self, commit_text): return "49824" - def modified_changelogs(self): + +class MockCheckout(object): + + _committer_list = CommitterList() + + def commit_info_for_revision(self, svn_revision): + return CommitInfo(svn_revision, "eric@webkit.org", { + "bug_id": 42, + "author_name": "Adam Barth", + "author_email": "abarth@webkit.org", + "author": self._committer_list.committer_by_email("abarth@webkit.org"), + "reviewer_text": "Darin Adler", + "reviewer": self._committer_list.committer_by_name("Darin Adler"), + }) + + def bug_id_for_revision(self, svn_revision): + return 12345 + + def modified_changelogs(self, git_commit): # Ideally we'd return something more interesting here. The problem is - # that LandDiff will try to actually read the path from disk! + # that LandDiff will try to actually read the patch from disk! return [] + def commit_message_for_this_commit(self, git_commit): + commit_message = Mock() + commit_message.message = lambda:"This is a fake commit message that is at least 50 characters." + return commit_message + + def apply_patch(self, patch, force=False): + pass + + def apply_reverse_diff(self, revision): + pass + class MockUser(object): @@ -328,15 +466,32 @@ class MockUser(object): def edit(self, files): pass + def edit_changelog(self, files): + pass + def page(self, message): pass def confirm(self, message=None): return True + def can_open_url(self): + return True + def open_url(self, url): + if url.startswith("file://"): + log("MOCK: user.open_url: file://...") + return log("MOCK: user.open_url: %s" % url) - pass + + +class MockIRC(object): + + def post(self, message): + log("MOCK: irc.post: %s" % message) + + def disconnect(self): + log("MOCK: irc.disconnect") class MockStatusServer(object): @@ -347,22 +502,86 @@ class MockStatusServer(object): def patch_status(self, queue_name, patch_id): return None + def svn_revision(self, svn_revision): + return None + + def update_work_items(self, queue_name, work_items): + log("MOCK: update_work_items: %s %s" % (queue_name, work_items)) + def update_status(self, queue_name, status, patch=None, results_file=None): + log("MOCK: update_status: %s %s" % (queue_name, status)) return 187 + def update_svn_revision(self, svn_revision, broken_bot): + return 191 -class MockBugzillaTool(): + def results_url_for_status(self, status_id): + return "http://dummy_url" - def __init__(self): + +class MockExecute(Mock): + def __init__(self, should_log): + self._should_log = should_log + + def run_and_throw_if_fail(self, args, quiet=False): + if self._should_log: + log("MOCK run_and_throw_if_fail: %s" % args) + return "MOCK output of child process" + + def run_command(self, + args, + cwd=None, + input=None, + error_handler=None, + return_exit_code=False, + return_stderr=True, + decode_output=False): + if self._should_log: + log("MOCK run_command: %s" % args) + return "MOCK output of child process" + + +class MockOptions(Mock): + no_squash = False + squash = False + + +class MockRietveld(): + + def __init__(self, executive, dryrun=False): + pass + + def post(self, diff, message=None, codereview_issue=None, cc=None): + log("MOCK: Uploading patch to rietveld") + + +class MockTool(): + + def __init__(self, log_executive=False): + self.wakeup_event = threading.Event() self.bugs = MockBugzilla() self.buildbot = MockBuildBot() - self.executive = Mock() + self.executive = MockExecute(should_log=log_executive) + self._irc = None self.user = MockUser() self._scm = MockSCM() + self._checkout = MockCheckout() self.status_server = MockStatusServer() + self.irc_password = "MOCK irc password" + self.codereview = MockRietveld(self.executive) def scm(self): return self._scm + def checkout(self): + return self._checkout + + def ensure_irc_connected(self, delegate): + if not self._irc: + self._irc = MockIRC() + + def irc(self): + return self._irc + def path(self): return "echo" diff --git a/WebKitTools/Scripts/webkitpy/multicommandtool.py b/WebKitTools/Scripts/webkitpy/tool/multicommandtool.py index 10cf426..12ede2e 100644 --- a/WebKitTools/Scripts/webkitpy/multicommandtool.py +++ b/WebKitTools/Scripts/webkitpy/tool/multicommandtool.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -35,8 +35,12 @@ import sys from optparse import OptionParser, IndentedHelpFormatter, SUPPRESS_USAGE, make_option -from webkitpy.grammar import pluralize -from webkitpy.webkit_logging import log +from webkitpy.tool.grammar import pluralize +from webkitpy.common.system.deprecated_logging import log + + +class TryAgain(Exception): + pass class Command(object): @@ -263,6 +267,9 @@ class MultiCommandTool(object): def path(self): raise NotImplementedError, "subclasses must implement" + def command_completed(self): + pass + def should_show_in_main_help(self, command): return command.show_in_main_help @@ -296,4 +303,12 @@ class MultiCommandTool(object): log(failure_reason) return 0 # FIXME: Should this really be 0? - return command.check_arguments_and_execute(options, args, self) + while True: + try: + result = command.check_arguments_and_execute(options, args, self) + break + except TryAgain, e: + pass + + self.command_completed() + return result diff --git a/WebKitTools/Scripts/webkitpy/multicommandtool_unittest.py b/WebKitTools/Scripts/webkitpy/tool/multicommandtool_unittest.py index ae77e73..c19095c 100644 --- a/WebKitTools/Scripts/webkitpy/multicommandtool_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/multicommandtool_unittest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -28,11 +28,13 @@ import sys import unittest -from multicommandtool import MultiCommandTool, Command -from webkitpy.outputcapture import OutputCapture from optparse import make_option +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.multicommandtool import MultiCommandTool, Command, TryAgain + + class TrivialCommand(Command): name = "trivial" show_in_main_help = True @@ -42,10 +44,26 @@ class TrivialCommand(Command): def execute(self, options, args, tool): pass + class UncommonCommand(TrivialCommand): name = "uncommon" show_in_main_help = False + +class LikesToRetry(Command): + name = "likes-to-retry" + show_in_main_help = True + + def __init__(self, **kwargs): + Command.__init__(self, "help text", **kwargs) + self.execute_count = 0 + + def execute(self, options, args, tool): + self.execute_count += 1 + if self.execute_count < 2: + raise TryAgain() + + class CommandTest(unittest.TestCase): def test_name_with_arguments(self): command_with_args = TrivialCommand(argument_names="ARG1 ARG2") @@ -72,7 +90,7 @@ class TrivialTool(MultiCommandTool): def __init__(self, commands=None): MultiCommandTool.__init__(self, name="trivial-tool", commands=commands) - def path(): + def path(self): return __file__ def should_execute_command(self, command): @@ -107,6 +125,12 @@ class MultiCommandToolTest(unittest.TestCase): exit_code = OutputCapture().assert_outputs(self, tool.main, [main_args], expected_stdout=expected_stdout, expected_stderr=expected_stderr) self.assertEqual(exit_code, expected_exit_code) + def test_retry(self): + likes_to_retry = LikesToRetry() + tool = TrivialTool(commands=[likes_to_retry]) + tool.main(["tool", "likes-to-retry"]) + self.assertEqual(likes_to_retry.execute_count, 2) + def test_global_help(self): tool = TrivialTool(commands=[TrivialCommand(), UncommonCommand()]) expected_common_commands_help = """Usage: trivial-tool [options] COMMAND [ARGS] diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/__init__.py b/WebKitTools/Scripts/webkitpy/tool/steps/__init__.py new file mode 100644 index 0000000..d59cdc5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/__init__.py @@ -0,0 +1,59 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# FIXME: Is this the right way to do this? +from webkitpy.tool.steps.applypatch import ApplyPatch +from webkitpy.tool.steps.applypatchwithlocalcommit import ApplyPatchWithLocalCommit +from webkitpy.tool.steps.build import Build +from webkitpy.tool.steps.checkstyle import CheckStyle +from webkitpy.tool.steps.cleanworkingdirectory import CleanWorkingDirectory +from webkitpy.tool.steps.cleanworkingdirectorywithlocalcommits import CleanWorkingDirectoryWithLocalCommits +from webkitpy.tool.steps.closebug import CloseBug +from webkitpy.tool.steps.closebugforlanddiff import CloseBugForLandDiff +from webkitpy.tool.steps.closepatch import ClosePatch +from webkitpy.tool.steps.commit import Commit +from webkitpy.tool.steps.confirmdiff import ConfirmDiff +from webkitpy.tool.steps.createbug import CreateBug +from webkitpy.tool.steps.editchangelog import EditChangeLog +from webkitpy.tool.steps.ensurebuildersaregreen import EnsureBuildersAreGreen +from webkitpy.tool.steps.ensurelocalcommitifneeded import EnsureLocalCommitIfNeeded +from webkitpy.tool.steps.obsoletepatches import ObsoletePatches +from webkitpy.tool.steps.options import Options +from webkitpy.tool.steps.postcodereview import PostCodeReview +from webkitpy.tool.steps.postdiff import PostDiff +from webkitpy.tool.steps.postdiffforcommit import PostDiffForCommit +from webkitpy.tool.steps.postdiffforrevert import PostDiffForRevert +from webkitpy.tool.steps.preparechangelogforrevert import PrepareChangeLogForRevert +from webkitpy.tool.steps.preparechangelog import PrepareChangeLog +from webkitpy.tool.steps.promptforbugortitle import PromptForBugOrTitle +from webkitpy.tool.steps.reopenbugafterrollout import ReopenBugAfterRollout +from webkitpy.tool.steps.revertrevision import RevertRevision +from webkitpy.tool.steps.runtests import RunTests +from webkitpy.tool.steps.updatechangelogswithreviewer import UpdateChangeLogsWithReviewer +from webkitpy.tool.steps.update import Update +from webkitpy.tool.steps.validatereviewer import ValidateReviewer diff --git a/WebKitTools/Scripts/webkitpy/steps/abstractstep.py b/WebKitTools/Scripts/webkitpy/tool/steps/abstractstep.py index 639cf55..9ceb2cb 100644 --- a/WebKitTools/Scripts/webkitpy/steps/abstractstep.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/abstractstep.py @@ -26,8 +26,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.webkit_logging import log -from webkitpy.webkitport import WebKitPort +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.config.ports import WebKitPort +from webkitpy.tool.steps.options import Options class AbstractStep(object): @@ -36,10 +38,13 @@ class AbstractStep(object): self._options = options self._port = None - def _run_script(self, script_name, quiet=False, port=WebKitPort): + def _run_script(self, script_name, args=None, quiet=False, port=WebKitPort): log("Running %s" % script_name) + command = [port.script_path(script_name)] + if args: + command.extend(args) # FIXME: This should use self.port() - self._tool.executive.run_and_throw_if_fail(port.script_path(script_name), quiet) + self._tool.executive.run_and_throw_if_fail(command, quiet) # FIXME: The port should live on the tool. def port(self): @@ -49,8 +54,9 @@ class AbstractStep(object): return self._port _well_known_keys = { - "diff" : lambda self: self._tool.scm().create_patch(), - "changelogs" : lambda self: self._tool.scm().modified_changelogs(), + "diff": lambda self, state: self._tool.scm().create_patch(self._options.git_commit), + "changelogs": lambda self, state: self._tool.checkout().modified_changelogs(self._options.git_commit), + "bug_title": lambda self, state: self._tool.bugs.fetch_bug(state["bug_id"]).title(), } def cached_lookup(self, state, key, promise=None): @@ -58,12 +64,15 @@ class AbstractStep(object): return state[key] if not promise: promise = self._well_known_keys.get(key) - state[key] = promise(self) + state[key] = promise(self, state) return state[key] @classmethod def options(cls): - return [] + return [ + # We need this option here because cached_lookup uses it. :( + Options.git_commit, + ] def run(self, state): raise NotImplementedError, "subclasses must implement" diff --git a/WebKitTools/Scripts/webkitpy/steps/applypatch.py b/WebKitTools/Scripts/webkitpy/tool/steps/applypatch.py index aba81ae..6cded27 100644 --- a/WebKitTools/Scripts/webkitpy/steps/applypatch.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/applypatch.py @@ -26,17 +26,17 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class ApplyPatch(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.non_interactive, ] def run(self, state): log("Processing patch %s from bug %s." % (state["patch"].id(), state["patch"].bug_id())) - self._tool.scm().apply_patch(state["patch"], force=self._options.non_interactive) + self._tool.checkout().apply_patch(state["patch"], force=self._options.non_interactive) diff --git a/WebKitTools/Scripts/webkitpy/steps/applypatchwithlocalcommit.py b/WebKitTools/Scripts/webkitpy/tool/steps/applypatchwithlocalcommit.py index bfaf52a..3dcd8d9 100644 --- a/WebKitTools/Scripts/webkitpy/steps/applypatchwithlocalcommit.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/applypatchwithlocalcommit.py @@ -26,18 +26,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.applypatch import ApplyPatch -from webkitpy.steps.options import Options +from webkitpy.tool.steps.applypatch import ApplyPatch +from webkitpy.tool.steps.options import Options class ApplyPatchWithLocalCommit(ApplyPatch): @classmethod def options(cls): - return [ + return ApplyPatch.options() + [ Options.local_commit, - ] + ApplyPatch.options() + ] def run(self, state): ApplyPatch.run(self, state) if self._options.local_commit: - commit_message = self._tool.scm().commit_message_for_this_commit() + commit_message = self._tool.checkout().commit_message_for_this_commit(git_commit=None) self._tool.scm().commit_locally_with_message(commit_message.message() or state["patch"].name()) diff --git a/WebKitTools/Scripts/webkitpy/steps/build.py b/WebKitTools/Scripts/webkitpy/tool/steps/build.py index 1823cff..456db25 100644 --- a/WebKitTools/Scripts/webkitpy/steps/build.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/build.py @@ -26,15 +26,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class Build(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.build, Options.quiet, Options.build_style, diff --git a/WebKitTools/Scripts/webkitpy/steps/checkstyle.py b/WebKitTools/Scripts/webkitpy/tool/steps/checkstyle.py index c8e20f8..af38214 100644 --- a/WebKitTools/Scripts/webkitpy/steps/checkstyle.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/checkstyle.py @@ -28,25 +28,32 @@ import os -from webkitpy.executive import ScriptError -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import error class CheckStyle(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.non_interactive, Options.check_style, + Options.git_commit, ] def run(self, state): if not self._options.check_style: return os.chdir(self._tool.scm().checkout_root) + + args = [] + if self._options.git_commit: + args.append("--git-commit") + args.append(self._options.git_commit) + try: - self._run_script("check-webkit-style") + self._run_script("check-webkit-style", args) except ScriptError, e: if self._options.non_interactive: # We need to re-raise the exception here to have the diff --git a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectory.py b/WebKitTools/Scripts/webkitpy/tool/steps/cleanworkingdirectory.py index 88e38f5..e13fbc2 100644 --- a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectory.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/cleanworkingdirectory.py @@ -28,8 +28,8 @@ import os -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options class CleanWorkingDirectory(AbstractStep): @@ -39,7 +39,7 @@ class CleanWorkingDirectory(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.force_clean, Options.clean, ] diff --git a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectorywithlocalcommits.py b/WebKitTools/Scripts/webkitpy/tool/steps/cleanworkingdirectorywithlocalcommits.py index cabeba2..f06f94e 100644 --- a/WebKitTools/Scripts/webkitpy/steps/cleanworkingdirectorywithlocalcommits.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/cleanworkingdirectorywithlocalcommits.py @@ -26,7 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.cleanworkingdirectory import CleanWorkingDirectory +from webkitpy.tool.steps.cleanworkingdirectory import CleanWorkingDirectory class CleanWorkingDirectoryWithLocalCommits(CleanWorkingDirectory): def __init__(self, tool, options): diff --git a/WebKitTools/Scripts/webkitpy/steps/closebug.py b/WebKitTools/Scripts/webkitpy/tool/steps/closebug.py index 2640ee3..e77bc24 100644 --- a/WebKitTools/Scripts/webkitpy/steps/closebug.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/closebug.py @@ -26,15 +26,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class CloseBug(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.close_bug, ] diff --git a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff.py b/WebKitTools/Scripts/webkitpy/tool/steps/closebugforlanddiff.py index 43a0c66..e5a68db 100644 --- a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/closebugforlanddiff.py @@ -26,16 +26,16 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.comments import bug_comment_from_commit_text -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.comments import bug_comment_from_commit_text +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class CloseBugForLandDiff(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.close_bug, ] diff --git a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff_unittest.py b/WebKitTools/Scripts/webkitpy/tool/steps/closebugforlanddiff_unittest.py index 73561ab..0a56564 100644 --- a/WebKitTools/Scripts/webkitpy/steps/closebugforlanddiff_unittest.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/closebugforlanddiff_unittest.py @@ -28,14 +28,13 @@ import unittest -from webkitpy.steps.closebugforlanddiff import CloseBugForLandDiff -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.mocktool import MockOptions, MockTool +from webkitpy.tool.steps.closebugforlanddiff import CloseBugForLandDiff class CloseBugForLandDiffTest(unittest.TestCase): def test_empty_state(self): capture = OutputCapture() - step = CloseBugForLandDiff(MockBugzillaTool(), Mock()) + step = CloseBugForLandDiff(MockTool(), MockOptions()) expected_stderr = "Committed r49824: <http://trac.webkit.org/changeset/49824>\nNo bug id provided.\n" capture.assert_outputs(self, step.run, [{"commit_text" : "Mock commit text"}], expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/steps/closepatch.py b/WebKitTools/Scripts/webkitpy/tool/steps/closepatch.py index f20fe7e..ff94df8 100644 --- a/WebKitTools/Scripts/webkitpy/steps/closepatch.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/closepatch.py @@ -26,8 +26,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.comments import bug_comment_from_commit_text -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.comments import bug_comment_from_commit_text +from webkitpy.tool.steps.abstractstep import AbstractStep class ClosePatch(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/commit.py b/WebKitTools/Scripts/webkitpy/tool/steps/commit.py new file mode 100644 index 0000000..9f93120 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/commit.py @@ -0,0 +1,75 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.common.checkout.scm import AuthenticationError, AmbiguousCommitError +from webkitpy.common.system.executive import ScriptError +from webkitpy.common.system.user import User +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options + + +class Commit(AbstractStep): + @classmethod + def options(cls): + return AbstractStep.options() + [ + Options.git_commit, + ] + + def _commit_warning(self, error): + working_directory_message = "" if error.working_directory_is_clean else " and working copy changes" + return ('There are %s local commits%s. Everything will be committed as a single commit. ' + 'To avoid this prompt, set "git config webkit-patch.squash true".' % ( + error.num_local_commits, working_directory_message)) + + def run(self, state): + self._commit_message = self._tool.checkout().commit_message_for_this_commit(self._options.git_commit).message() + if len(self._commit_message) < 50: + raise Exception("Attempted to commit with a commit message shorter than 50 characters. Either your patch is missing a ChangeLog or webkit-patch may have a bug.") + + self._state = state + + username = None + force_squash = False + + num_tries = 0 + while num_tries < 3: + num_tries += 1 + + try: + self._state["commit_text"] = self._tool.scm().commit_with_message(self._commit_message, git_commit=self._options.git_commit, username=username, force_squash=force_squash) + break; + except AmbiguousCommitError, e: + if self._tool.user.confirm(self._commit_warning(e)): + force_squash = True + else: + # This will correctly interrupt the rest of the commit process. + raise ScriptError(message="Did not commit") + except AuthenticationError, e: + username = self._tool.user.prompt("%s login: " % e.server_host, repeat=5) + if not username: + raise ScriptError("You need to specify the username on %s to perform the commit as." % self.svn_server_host) diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/confirmdiff.py b/WebKitTools/Scripts/webkitpy/tool/steps/confirmdiff.py new file mode 100644 index 0000000..7e8e348 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/confirmdiff.py @@ -0,0 +1,77 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import urllib + +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.prettypatch import PrettyPatch +from webkitpy.common.system import logutils +from webkitpy.common.system.executive import ScriptError + + +_log = logutils.get_logger(__file__) + + +class ConfirmDiff(AbstractStep): + @classmethod + def options(cls): + return AbstractStep.options() + [ + Options.confirm, + ] + + def _show_pretty_diff(self, diff): + if not self._tool.user.can_open_url(): + return None + + try: + pretty_patch = PrettyPatch(self._tool.executive, + self._tool.scm().checkout_root) + pretty_diff_file = pretty_patch.pretty_diff_file(diff) + url = "file://%s" % urllib.quote(pretty_diff_file.name) + self._tool.user.open_url(url) + # We return the pretty_diff_file here because we need to keep the + # file alive until the user has had a chance to confirm the diff. + return pretty_diff_file + except ScriptError, e: + _log.warning("PrettyPatch failed. :(") + except OSError, e: + _log.warning("PrettyPatch unavailable.") + + def run(self, state): + if not self._options.confirm: + return + diff = self.cached_lookup(state, "diff") + pretty_diff_file = self._show_pretty_diff(diff) + if not pretty_diff_file: + self._tool.user.page(diff) + diff_correct = self._tool.user.confirm("Was that diff correct?") + if pretty_diff_file: + pretty_diff_file.close() + if not diff_correct: + exit(1) diff --git a/WebKitTools/Scripts/webkitpy/steps/createbug.py b/WebKitTools/Scripts/webkitpy/tool/steps/createbug.py index 75bf17f..0ab6f68 100644 --- a/WebKitTools/Scripts/webkitpy/steps/createbug.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/createbug.py @@ -26,20 +26,27 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options class CreateBug(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.cc, Options.component, + Options.blocks, ] def run(self, state): # No need to create a bug if we already have one. if state.get("bug_id"): return - state["bug_id"] = self._tool.bugs.create_bug(state["bug_title"], state["bug_description"], component=self._options.component, cc=self._options.cc) + cc = self._options.cc + if not cc: + cc = state.get("bug_cc") + blocks = self._options.blocks + if not blocks: + blocks = state.get("bug_blocked") + state["bug_id"] = self._tool.bugs.create_bug(state["bug_title"], state["bug_description"], blocked=blocks, component=self._options.component, cc=cc) diff --git a/WebKitTools/Scripts/webkitpy/steps/editchangelog.py b/WebKitTools/Scripts/webkitpy/tool/steps/editchangelog.py index d545c72..de9b4e4 100644 --- a/WebKitTools/Scripts/webkitpy/steps/editchangelog.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/editchangelog.py @@ -28,10 +28,10 @@ import os -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep class EditChangeLog(AbstractStep): def run(self, state): os.chdir(self._tool.scm().checkout_root) - self._tool.user.edit(self.cached_lookup(state, "changelogs")) + self._tool.user.edit_changelog(self.cached_lookup(state, "changelogs")) diff --git a/WebKitTools/Scripts/webkitpy/steps/ensurebuildersaregreen.py b/WebKitTools/Scripts/webkitpy/tool/steps/ensurebuildersaregreen.py index 96f265a..7b717ef 100644 --- a/WebKitTools/Scripts/webkitpy/steps/ensurebuildersaregreen.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/ensurebuildersaregreen.py @@ -26,15 +26,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log, error class EnsureBuildersAreGreen(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.check_builders, ] @@ -45,4 +45,6 @@ class EnsureBuildersAreGreen(AbstractStep): if not red_builders_names: return red_builders_names = map(lambda name: "\"%s\"" % name, red_builders_names) # Add quotes around the names. - error("Builders [%s] are red, please do not commit.\nSee http://%s.\nPass --ignore-builders to bypass this check." % (", ".join(red_builders_names), self._tool.buildbot.buildbot_host)) + log("\nBuilders [%s] are red, please do not commit.\nSee http://%s/console?category=core\n" % (", ".join(red_builders_names), self._tool.buildbot.buildbot_host)) + if not self._tool.user.confirm("Are you sure you want to continue?"): + error("User aborted.") diff --git a/WebKitTools/Scripts/webkitpy/steps/ensurelocalcommitifneeded.py b/WebKitTools/Scripts/webkitpy/tool/steps/ensurelocalcommitifneeded.py index cecf891..d0cda46 100644 --- a/WebKitTools/Scripts/webkitpy/steps/ensurelocalcommitifneeded.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/ensurelocalcommitifneeded.py @@ -26,15 +26,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import error class EnsureLocalCommitIfNeeded(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.local_commit, ] diff --git a/WebKitTools/Scripts/webkitpy/steps/metastep.py b/WebKitTools/Scripts/webkitpy/tool/steps/metastep.py index 9f368de..7cbd1c5 100644 --- a/WebKitTools/Scripts/webkitpy/steps/metastep.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/metastep.py @@ -26,7 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep # FIXME: Unify with StepSequence? I'm not sure yet which is the better design. diff --git a/WebKitTools/Scripts/webkitpy/steps/obsoletepatches.py b/WebKitTools/Scripts/webkitpy/tool/steps/obsoletepatches.py index dbdbabd..de508c6 100644 --- a/WebKitTools/Scripts/webkitpy/steps/obsoletepatches.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/obsoletepatches.py @@ -26,16 +26,16 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.grammar import pluralize -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class ObsoletePatches(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.obsolete_patches, ] diff --git a/WebKitTools/Scripts/webkitpy/steps/options.py b/WebKitTools/Scripts/webkitpy/tool/steps/options.py index 8b28f27..e7e3855 100644 --- a/WebKitTools/Scripts/webkitpy/steps/options.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/options.py @@ -29,19 +29,21 @@ from optparse import make_option class Options(object): - build = make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test.") + blocks = make_option("--blocks", action="store", dest="blocks", default=None, help="Bug number which the created bug blocks.") + build = make_option("--build", action="store_true", dest="build", default=False, help="Build and run run-webkit-tests before committing.") build_style = make_option("--build-style", action="store", dest="build_style", default=None, help="Whether to build debug, release, or both.") cc = make_option("--cc", action="store", type="string", dest="cc", help="Comma-separated list of email addresses to carbon-copy.") check_builders = make_option("--ignore-builders", action="store_false", dest="check_builders", default=True, help="Don't check to see if the build.webkit.org builders are green before landing.") check_style = make_option("--ignore-style", action="store_false", dest="check_style", default=True, help="Don't check to see if the patch has proper style before uploading.") clean = make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches") close_bug = make_option("--no-close", action="store_false", dest="close_bug", default=True, help="Leave bug open after landing.") - complete_rollout = make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Commit the revert and re-open the original bug.") + comment = make_option("--comment", action="store", type="string", dest="comment", help="Comment to post to bug.") component = make_option("--component", action="store", type="string", dest="component", help="Component for the new bug.") confirm = make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Skip confirmation steps.") description = make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: \"patch\")") email = make_option("--email", action="store", type="string", dest="email", help="Email address to use in ChangeLogs.") force_clean = make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)") + git_commit = make_option("-g", "--git-commit", action="store", dest="git_commit", help="Operate on a local commit. If a range, the commits are squashed into one. HEAD.. operates on working copy changes only.") local_commit = make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch") non_interactive = make_option("--non-interactive", action="store_true", dest="non_interactive", default=False, help="Never prompt the user, fail as fast as possible.") obsolete_patches = make_option("--no-obsolete", action="store_false", dest="obsolete_patches", default=True, help="Do not obsolete old patches before posting this one.") @@ -52,5 +54,5 @@ class Options(object): request_commit = make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review.") review = make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review.") reviewer = make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER.") - test = make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without running run-webkit-tests.") + test = make_option("--test", action="store_true", dest="test", default=False, help="Run run-webkit-tests before committing.") update = make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory.") diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/postcodereview.py b/WebKitTools/Scripts/webkitpy/tool/steps/postcodereview.py new file mode 100644 index 0000000..f9bc685 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/postcodereview.py @@ -0,0 +1,69 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options + + +class PostCodeReview(AbstractStep): + @classmethod + def options(cls): + return AbstractStep.options() + [ + Options.cc, + Options.description, + ] + + def run(self, state): + patch = state.get("patch") + bug_id = patch.bug_id() + title = patch.name() + + # If the issue already exists, then the message becomes the label + # of the new patch. Otherwise, it becomes the title of the whole + # issue. + if title: + # This is the common case for the the first "upload" command. + message = title + elif bug_id: + # This is the common case for the "post" command and + # subsequent runs of the "upload" command. + message = "Code review for %s" % self._tool.bugs.bug_url_for_bug_id(bug_id) + else: + # Unreachable with our current commands, but we might hit + # this case if we support bug-less code reviews. + message = "Code review" + + # Use the bug ID as the rietveld issue number. This means rietveld code reviews + # when there are multiple different patches on a bug will be a bit wonky, but + # webkit-patch assumes one-patch-per-bug. + created_issue = self._tool.codereview.post(diff=self.cached_lookup(state, "diff"), + message=message, + codereview_issue=bug_id, + cc=self._options.cc) + + self._tool.bugs.set_flag_on_attachment(patch.id(), 'in-rietveld', '+') diff --git a/WebKitTools/Scripts/webkitpy/steps/postdiff.py b/WebKitTools/Scripts/webkitpy/tool/steps/postdiff.py index a5ba2a4..c40b6ff 100644 --- a/WebKitTools/Scripts/webkitpy/steps/postdiff.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/postdiff.py @@ -26,17 +26,16 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import StringIO - -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options class PostDiff(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.description, + Options.comment, Options.review, Options.request_commit, Options.open_bug, @@ -44,8 +43,8 @@ class PostDiff(AbstractStep): def run(self, state): diff = self.cached_lookup(state, "diff") - diff_file = StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object description = self._options.description or "Patch" - self._tool.bugs.add_patch_to_bug(state["bug_id"], diff_file, description, mark_for_review=self._options.review, mark_for_commit_queue=self._options.request_commit) + comment_text = self._options.comment + self._tool.bugs.add_patch_to_bug(state["bug_id"], diff, description, comment_text=comment_text, mark_for_review=self._options.review, mark_for_commit_queue=self._options.request_commit) if self._options.open_bug: self._tool.user.open_url(self._tool.bugs.bug_url_for_bug_id(state["bug_id"])) diff --git a/WebKitTools/Scripts/webkitpy/steps/postdiffforcommit.py b/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforcommit.py index 449381c..13bc00c 100644 --- a/WebKitTools/Scripts/webkitpy/steps/postdiffforcommit.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforcommit.py @@ -26,16 +26,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import StringIO - -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep class PostDiffForCommit(AbstractStep): def run(self, state): self._tool.bugs.add_patch_to_bug( state["bug_id"], - StringIO.StringIO(self.cached_lookup(state, "diff")), + self.cached_lookup(state, "diff"), "Patch for landing", mark_for_review=False, mark_for_landing=True) diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforrevert.py b/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforrevert.py new file mode 100644 index 0000000..bfa631f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/postdiffforrevert.py @@ -0,0 +1,49 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.common.net.bugzilla import Attachment +from webkitpy.tool.steps.abstractstep import AbstractStep + + +class PostDiffForRevert(AbstractStep): + def run(self, state): + comment_text = "Any committer can land this patch automatically by \ +marking it commit-queue+. The commit-queue will build and test \ +the patch before landing to ensure that the rollout will be \ +successful. This process takes approximately 15 minutes.\n\n\ +If you would like to land the rollout faster, you can use the \ +following command:\n\n\ + webkit-patch land-attachment ATTACHMENT_ID --ignore-builders\n\n\ +where ATTACHMENT_ID is the ID of this attachment." + self._tool.bugs.add_patch_to_bug( + state["bug_id"], + self.cached_lookup(state, "diff"), + "%s%s" % (Attachment.rollout_preamble, state["revision"]), + comment_text=comment_text, + mark_for_review=False, + mark_for_commit_queue=True) diff --git a/WebKitTools/Scripts/webkitpy/steps/preparechangelog.py b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelog.py index bd41f0b..ce04024 100644 --- a/WebKitTools/Scripts/webkitpy/steps/preparechangelog.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelog.py @@ -28,30 +28,49 @@ import os -from webkitpy.executive import ScriptError -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.common.checkout.changelog import ChangeLog +from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import error class PrepareChangeLog(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.port, Options.quiet, Options.email, + Options.git_commit, ] + def _ensure_bug_url(self, state): + if not state.get("bug_id"): + return + bug_id = state.get("bug_id") + changelogs = self.cached_lookup(state, "changelogs") + for changelog_path in changelogs: + changelog = ChangeLog(changelog_path) + if not changelog.latest_entry().bug_id(): + changelog.set_short_description_and_bug_url( + self.cached_lookup(state, "bug_title"), + self._tool.bugs.bug_url_for_bug_id(bug_id)) + def run(self, state): if self.cached_lookup(state, "changelogs"): + self._ensure_bug_url(state) return os.chdir(self._tool.scm().checkout_root) args = [self.port().script_path("prepare-ChangeLog")] - if state["bug_id"]: + if state.get("bug_id"): args.append("--bug=%s" % state["bug_id"]) if self._options.email: args.append("--email=%s" % self._options.email) + + if self._tool.scm().supports_local_commits(): + args.append("--merge-base=%s" % self._tool.scm().merge_base(self._options.git_commit)) + try: self._tool.executive.run_and_throw_if_fail(args, self._options.quiet) except ScriptError, e: diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelog_unittest.py b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelog_unittest.py new file mode 100644 index 0000000..eceffdf --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelog_unittest.py @@ -0,0 +1,54 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import unittest + +from webkitpy.common.checkout.changelog_unittest import ChangeLogTest +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.mocktool import MockOptions, MockTool +from webkitpy.tool.steps.preparechangelog import PrepareChangeLog + + +class PrepareChangeLogTest(ChangeLogTest): + def test_ensure_bug_url(self): + capture = OutputCapture() + step = PrepareChangeLog(MockTool(), MockOptions()) + changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate, self._example_changelog) + changelog_path = self._write_tmp_file_with_contents(changelog_contents.encode("utf-8")) + state = { + "bug_title": "Example title", + "bug_id": 1234, + "changelogs": [changelog_path], + } + capture.assert_outputs(self, step.run, [state]) + actual_contents = self._read_file_contents(changelog_path, "utf-8") + expected_message = "Example title\n http://example.com/1234" + expected_contents = changelog_contents.replace("Need a short description and bug URL (OOPS!)", expected_message) + os.remove(changelog_path) + self.assertEquals(actual_contents, expected_contents) diff --git a/WebKitTools/Scripts/webkitpy/steps/preparechangelogforrevert.py b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelogforrevert.py index 88e5134..0e78bc2 100644 --- a/WebKitTools/Scripts/webkitpy/steps/preparechangelogforrevert.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/preparechangelogforrevert.py @@ -28,20 +28,15 @@ import os -from webkitpy.changelogs import ChangeLog -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.common.checkout.changelog import ChangeLog +from webkitpy.tool.steps.abstractstep import AbstractStep class PrepareChangeLogForRevert(AbstractStep): def run(self, state): - # First, discard the ChangeLog changes from the rollout. - os.chdir(self._tool.scm().checkout_root) - changelog_paths = self._tool.scm().modified_changelogs() - self._tool.scm().revert_files(changelog_paths) - - # Second, make new ChangeLog entries for this rollout. # This could move to prepare-ChangeLog by adding a --revert= option. self._run_script("prepare-ChangeLog") + changelog_paths = self._tool.checkout().modified_changelogs(git_commit=None) bug_url = self._tool.bugs.bug_url_for_bug_id(state["bug_id"]) if state["bug_id"] else None for changelog_path in changelog_paths: # FIXME: Seems we should prepare the message outside of changelogs.py and then just pass in diff --git a/WebKitTools/Scripts/webkitpy/steps/promptforbugortitle.py b/WebKitTools/Scripts/webkitpy/tool/steps/promptforbugortitle.py index fb2f877..31c913c 100644 --- a/WebKitTools/Scripts/webkitpy/steps/promptforbugortitle.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/promptforbugortitle.py @@ -26,7 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep class PromptForBugOrTitle(AbstractStep): diff --git a/WebKitTools/Scripts/webkitpy/steps/confirmdiff.py b/WebKitTools/Scripts/webkitpy/tool/steps/reopenbugafterrollout.py index fc28f8f..f369ca9 100644 --- a/WebKitTools/Scripts/webkitpy/steps/confirmdiff.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/reopenbugafterrollout.py @@ -26,22 +26,19 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import error +from webkitpy.tool.comments import bug_comment_from_commit_text +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.common.system.deprecated_logging import log -class ConfirmDiff(AbstractStep): - @classmethod - def options(cls): - return [ - Options.confirm, - ] - +class ReopenBugAfterRollout(AbstractStep): def run(self, state): - if not self._options.confirm: + commit_comment = bug_comment_from_commit_text(self._tool.scm(), state["commit_text"]) + comment_text = "Reverted r%s for reason:\n\n%s\n\n%s" % (state["revision"], state["reason"], commit_comment) + + bug_id = state["bug_id"] + if not bug_id: + log(comment_text) + log("No bugs were updated.") return - diff = self.cached_lookup(state, "diff") - self._tool.user.page(diff) - if not self._tool.user.confirm("Was that diff correct?"): - exit(1) + self._tool.bugs.reopen_bug(bug_id, comment_text) diff --git a/WebKitTools/Scripts/webkitpy/steps/revertrevision.py b/WebKitTools/Scripts/webkitpy/tool/steps/revertrevision.py index ce6c263..81b6bcb 100644 --- a/WebKitTools/Scripts/webkitpy/steps/revertrevision.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/revertrevision.py @@ -26,9 +26,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.abstractstep import AbstractStep class RevertRevision(AbstractStep): def run(self, state): - self._tool.scm().apply_reverse_diff(state["revision"]) + self._tool.checkout().apply_reverse_diff(state["revision"]) diff --git a/WebKitTools/Scripts/webkitpy/steps/runtests.py b/WebKitTools/Scripts/webkitpy/tool/steps/runtests.py index ebe809f..22b9452 100644 --- a/WebKitTools/Scripts/webkitpy/steps/runtests.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/runtests.py @@ -26,15 +26,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class RunTests(AbstractStep): @classmethod def options(cls): - return [ - Options.build, + return AbstractStep.options() + [ Options.test, Options.non_interactive, Options.quiet, @@ -42,8 +41,6 @@ class RunTests(AbstractStep): ] def run(self, state): - if not self._options.build: - return if not self._options.test: return @@ -60,6 +57,12 @@ class RunTests(AbstractStep): if self._options.non_interactive: args.append("--no-launch-safari") args.append("--exit-after-n-failures=1") + # FIXME: Hack to work around https://bugs.webkit.org/show_bug.cgi?id=38912 + # when running the commit-queue on a mac leopard machine since compositing + # does not work reliably on Leopard due to various graphics driver/system bugs. + if self.port().name() == "Mac" and self.port().is_leopard(): + args.extend(["--ignore-tests", "compositing"]) + if self._options.quiet: args.append("--quiet") self._tool.executive.run_and_throw_if_fail(args) diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/steps_unittest.py b/WebKitTools/Scripts/webkitpy/tool/steps/steps_unittest.py new file mode 100644 index 0000000..766801b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/steps_unittest.py @@ -0,0 +1,82 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.common.config.ports import WebKitPort +from webkitpy.tool.mocktool import MockOptions, MockTool +from webkitpy.tool.steps.update import Update +from webkitpy.tool.steps.runtests import RunTests +from webkitpy.tool.steps.promptforbugortitle import PromptForBugOrTitle + + +class StepsTest(unittest.TestCase): + def _run_step(self, step, tool=None, options=None, state=None): + if not tool: + tool = MockTool() + if not options: + options = MockOptions() + if not state: + state = {} + step(tool, options).run(state) + + def test_update_step(self): + options = MockOptions() + options.update = True + expected_stderr = "Updating working directory\n" + OutputCapture().assert_outputs(self, self._run_step, [Update, options], expected_stderr=expected_stderr) + + def test_prompt_for_bug_or_title_step(self): + tool = MockTool() + tool.user.prompt = lambda message: 42 + self._run_step(PromptForBugOrTitle, tool=tool) + + def test_runtests_leopard_commit_queue_hack(self): + expected_stderr = "Running Python unit tests\nRunning Perl unit tests\nRunning JavaScriptCore tests\nRunning run-webkit-tests\n" + OutputCapture().assert_outputs(self, self._run_step, [RunTests], expected_stderr=expected_stderr) + + def test_runtests_leopard_commit_queue_hack(self): + mock_options = MockOptions() + mock_options.non_interactive = True + step = RunTests(MockTool(log_executive=True), mock_options) + # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment. + mock_port = WebKitPort() + mock_port.name = lambda: "Mac" + mock_port.is_leopard = lambda: True + step.port = lambda: mock_port + expected_stderr = """Running Python unit tests +MOCK run_and_throw_if_fail: ['WebKitTools/Scripts/test-webkitpy'] +Running Perl unit tests +MOCK run_and_throw_if_fail: ['WebKitTools/Scripts/test-webkitperl'] +Running JavaScriptCore tests +MOCK run_and_throw_if_fail: ['WebKitTools/Scripts/run-javascriptcore-tests'] +Running run-webkit-tests +MOCK run_and_throw_if_fail: ['WebKitTools/Scripts/run-webkit-tests', '--no-launch-safari', '--exit-after-n-failures=1', '--ignore-tests', 'compositing', '--quiet'] +""" + OutputCapture().assert_outputs(self, step.run, [{}], expected_stderr=expected_stderr) diff --git a/WebKitTools/Scripts/webkitpy/steps/update.py b/WebKitTools/Scripts/webkitpy/tool/steps/update.py index 0f45671..0f450f3 100644 --- a/WebKitTools/Scripts/webkitpy/steps/update.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/update.py @@ -26,15 +26,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log class Update(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ Options.update, Options.port, ] diff --git a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreview_unittests.py b/WebKitTools/Scripts/webkitpy/tool/steps/updatechangelogswithreview_unittest.py index 102a454..a037422 100644 --- a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreview_unittests.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/updatechangelogswithreview_unittest.py @@ -28,19 +28,18 @@ import unittest -from webkitpy.steps.updatechangelogswithreviewer import UpdateChangeLogsWithReviewer -from webkitpy.mock import Mock -from webkitpy.mock_bugzillatool import MockBugzillaTool -from webkitpy.outputcapture import OutputCapture +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.mocktool import MockOptions, MockTool +from webkitpy.tool.steps.updatechangelogswithreviewer import UpdateChangeLogsWithReviewer class UpdateChangeLogsWithReviewerTest(unittest.TestCase): def test_guess_reviewer_from_bug(self): capture = OutputCapture() - step = UpdateChangeLogsWithReviewer(MockBugzillaTool(), Mock()) + step = UpdateChangeLogsWithReviewer(MockTool(), MockOptions()) expected_stderr = "0 reviewed patches on bug 75, cannot infer reviewer.\n" capture.assert_outputs(self, step._guess_reviewer_from_bug, [75], expected_stderr=expected_stderr) def test_empty_state(self): capture = OutputCapture() - step = UpdateChangeLogsWithReviewer(MockBugzillaTool(), Mock()) + step = UpdateChangeLogsWithReviewer(MockTool(), MockOptions()) capture.assert_outputs(self, step.run, [{}]) diff --git a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreviewer.py b/WebKitTools/Scripts/webkitpy/tool/steps/updatechangelogswithreviewer.py index 90fdc35..e46b790 100644 --- a/WebKitTools/Scripts/webkitpy/steps/updatechangelogswithreviewer.py +++ b/WebKitTools/Scripts/webkitpy/tool/steps/updatechangelogswithreviewer.py @@ -28,16 +28,17 @@ import os -from webkitpy.changelogs import ChangeLog -from webkitpy.grammar import pluralize -from webkitpy.steps.abstractstep import AbstractStep -from webkitpy.steps.options import Options -from webkitpy.webkit_logging import log, error +from webkitpy.common.checkout.changelog import ChangeLog +from webkitpy.tool.grammar import pluralize +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import log, error class UpdateChangeLogsWithReviewer(AbstractStep): @classmethod def options(cls): - return [ + return AbstractStep.options() + [ + Options.git_commit, Options.reviewer, ] @@ -67,5 +68,5 @@ class UpdateChangeLogsWithReviewer(AbstractStep): return os.chdir(self._tool.scm().checkout_root) - for changelog_path in self._tool.scm().modified_changelogs(): + for changelog_path in self.cached_lookup(state, "changelogs"): ChangeLog(changelog_path).set_reviewer(reviewer) diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer.py b/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer.py new file mode 100644 index 0000000..bdf729e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer.py @@ -0,0 +1,71 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import re + +from webkitpy.common.checkout.changelog import ChangeLog +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.tool.steps.options import Options +from webkitpy.common.system.deprecated_logging import error, log + + +# FIXME: Some of this logic should probably be unified with CommitterValidator? +class ValidateReviewer(AbstractStep): + @classmethod + def options(cls): + return AbstractStep.options() + [ + Options.git_commit, + ] + + # FIXME: This should probably move onto ChangeLogEntry + def _has_valid_reviewer(self, changelog_entry): + if changelog_entry.reviewer(): + return True + if re.search("unreviewed", changelog_entry.contents(), re.IGNORECASE): + return True + if re.search("rubber[ -]stamp", changelog_entry.contents(), re.IGNORECASE): + return True + return False + + def run(self, state): + # FIXME: For now we disable this check when a user is driving the script + # this check is too draconian (and too poorly tested) to foist upon users. + if not self._options.non_interactive: + return + # FIXME: We should figure out how to handle the current working + # directory issue more globally. + os.chdir(self._tool.scm().checkout_root) + for changelog_path in self.cached_lookup(state, "changelogs"): + changelog_entry = ChangeLog(changelog_path).latest_entry() + if self._has_valid_reviewer(changelog_entry): + continue + reviewer_text = changelog_entry.reviewer_text() + if reviewer_text: + log("%s found in %s does not appear to be a valid reviewer according to committers.py." % (reviewer_text, changelog_path)) + error('%s neither lists a valid reviewer nor contains the string "Unreviewed" or "Rubber stamp" (case insensitive).' % changelog_path) diff --git a/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer_unittest.py b/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer_unittest.py new file mode 100644 index 0000000..d9b856a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/tool/steps/validatereviewer_unittest.py @@ -0,0 +1,57 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.checkout.changelog import ChangeLogEntry +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.mocktool import MockOptions, MockTool +from webkitpy.tool.steps.validatereviewer import ValidateReviewer + +class ValidateReviewerTest(unittest.TestCase): + _boilerplate_entry = '''2009-08-19 Eric Seidel <eric@webkit.org> + + REVIEW_LINE + + * Scripts/bugzilla-tool: +''' + + def _test_review_text(self, step, text, expected): + contents = self._boilerplate_entry.replace("REVIEW_LINE", text) + entry = ChangeLogEntry(contents) + self.assertEqual(step._has_valid_reviewer(entry), expected) + + def test_has_valid_reviewer(self): + step = ValidateReviewer(MockTool(), MockOptions()) + self._test_review_text(step, "Reviewed by Eric Seidel.", True) + self._test_review_text(step, "Reviewed by Eric Seidel", True) # Not picky about the '.' + self._test_review_text(step, "Reviewed by Eric.", False) + self._test_review_text(step, "Reviewed by Eric C Seidel.", False) + self._test_review_text(step, "Rubber-stamped by Eric.", True) + self._test_review_text(step, "Rubber stamped by Eric.", True) + self._test_review_text(step, "Unreviewed build fix.", True) diff --git a/WebKitTools/Scripts/webkitpy/user.pyc b/WebKitTools/Scripts/webkitpy/user.pyc Binary files differdeleted file mode 100644 index 7d6b687..0000000 --- a/WebKitTools/Scripts/webkitpy/user.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/webkit_logging.pyc b/WebKitTools/Scripts/webkitpy/webkit_logging.pyc Binary files differdeleted file mode 100644 index 137f042..0000000 --- a/WebKitTools/Scripts/webkitpy/webkit_logging.pyc +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/webkitport.pyc b/WebKitTools/Scripts/webkitpy/webkitport.pyc Binary files differdeleted file mode 100644 index e344aca..0000000 --- a/WebKitTools/Scripts/webkitpy/webkitport.pyc +++ /dev/null |
