diff options
| author | Steve Block <steveblock@google.com> | 2010-08-27 11:02:25 +0100 |
|---|---|---|
| committer | Steve Block <steveblock@google.com> | 2010-09-02 17:17:20 +0100 |
| commit | e8b154fd68f9b33be40a3590e58347f353835f5c (patch) | |
| tree | 0733ce26384183245aaa5656af26c653636fe6c1 /WebKitTools/Scripts/webkitpy/layout_tests | |
| parent | da56157816334089526a7a115a85fd85a6e9a1dc (diff) | |
| download | external_webkit-e8b154fd68f9b33be40a3590e58347f353835f5c.zip external_webkit-e8b154fd68f9b33be40a3590e58347f353835f5c.tar.gz external_webkit-e8b154fd68f9b33be40a3590e58347f353835f5c.tar.bz2 | |
Merge WebKit at r66079 : Initial merge by git
Change-Id: Ie2e1440fb9d487d24e52c247342c076fecaecac7
Diffstat (limited to 'WebKitTools/Scripts/webkitpy/layout_tests')
73 files changed, 708 insertions, 442 deletions
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum-expected.checksum new file mode 100644 index 0000000..5890112 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum-expected.checksum @@ -0,0 +1 @@ +checksum-checksum diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum-expected.png new file mode 100644 index 0000000..83a5de3 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum-expected.png @@ -0,0 +1 @@ +checksum-png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum-expected.txt new file mode 100644 index 0000000..5628d69 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum-expected.txt @@ -0,0 +1 @@ +checksum-txt diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum.html new file mode 100644 index 0000000..2b78d31 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/checksum.html @@ -0,0 +1 @@ +image_checksum diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/crash.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/crash.html new file mode 100644 index 0000000..0bc3798 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/crash.html @@ -0,0 +1 @@ +crash diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image-expected.checksum new file mode 100644 index 0000000..24b887a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image-expected.checksum @@ -0,0 +1 @@ +image-checksum diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image-expected.png new file mode 100644 index 0000000..4c23996 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image-expected.png @@ -0,0 +1 @@ +image-png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image-expected.txt new file mode 100644 index 0000000..c6ee718 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image-expected.txt @@ -0,0 +1 @@ +image-txt diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image.html new file mode 100644 index 0000000..53e4b27 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image.html @@ -0,0 +1 @@ +image_failure diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum-expected.checksum new file mode 100644 index 0000000..8fa0851 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum-expected.checksum @@ -0,0 +1 @@ +image_checksum-checksum diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum-expected.png new file mode 100644 index 0000000..d677d2e --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum-expected.png @@ -0,0 +1 @@ +image_checksum-png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum-expected.txt new file mode 100644 index 0000000..453f213 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum-expected.txt @@ -0,0 +1 @@ +image_checksum-txt diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum.html new file mode 100644 index 0000000..2b78d31 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/image_checksum.html @@ -0,0 +1 @@ +image_checksum diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_check-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_check-expected.png new file mode 100644 index 0000000..e45c7af --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_check-expected.png @@ -0,0 +1 @@ +missing_check-png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_check-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_check-expected.txt new file mode 100644 index 0000000..0ea9227 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_check-expected.txt @@ -0,0 +1 @@ +missing_check-txt diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_check.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_check.html new file mode 100644 index 0000000..0af8000 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_check.html @@ -0,0 +1 @@ +missing_image diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_image.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_image.html new file mode 100644 index 0000000..0af8000 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_image.html @@ -0,0 +1 @@ +missing_image diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_text.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_text.html new file mode 100644 index 0000000..47b8ad6 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/missing_text.html @@ -0,0 +1 @@ +missing_text diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/text-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/text-expected.txt new file mode 100644 index 0000000..e21ea45 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/text-expected.txt @@ -0,0 +1 @@ +text_failures-txt diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/text.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/text.html new file mode 100644 index 0000000..91f5fc7 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/text.html @@ -0,0 +1 @@ +text_failure diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/timeout.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/timeout.html new file mode 100644 index 0000000..790851a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/timeout.html @@ -0,0 +1 @@ +timeout diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum-expected.checksum new file mode 100644 index 0000000..0c4f6da --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum-expected.checksum @@ -0,0 +1 @@ +fail_checksum diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum-expected.png new file mode 100644 index 0000000..db483ee --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum-expected.png @@ -0,0 +1 @@ +fail_png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum-expected.txt new file mode 100644 index 0000000..a1f3c24 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum-expected.txt @@ -0,0 +1 @@ +fail_output diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum.html new file mode 100644 index 0000000..b325924 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/unexpected/text-image-checksum.html @@ -0,0 +1 @@ +Google diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-bg.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-bg.html deleted file mode 100644 index 2022676..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-bg.html +++ /dev/null @@ -1,22 +0,0 @@ -<html> - <head> - <style> - div { background: -webkit-canvas(squares); width:600px; height:600px; border:2px solid black } - </style> - - <script type="application/x-javascript"> -function draw(w, h) { - var ctx = document.getCSSCanvasContext("2d", "squares", w, h); - - ctx.fillStyle = "rgb(200,0,0)"; - ctx.fillRect (10, 10, 55, 50); - - ctx.fillStyle = "rgba(0, 0, 200, 0.5)"; - ctx.fillRect (30, 30, 55, 50); -} - </script> - </head> - <body onload="draw(300, 300)"> - <div></div> - </body> -</html>
\ No newline at end of file diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.checksum deleted file mode 100644 index 7373fe2..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.checksum +++ /dev/null @@ -1 +0,0 @@ -afa0f2d246120c180005d67d47636b92
\ No newline at end of file diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.png Binary files differdeleted file mode 100644 index 44952b4..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.png +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.txt deleted file mode 100644 index 288458d..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom-expected.txt +++ /dev/null @@ -1,22 +0,0 @@ -layer at (0,0) size 800x600 - RenderView at (0,0) size 800x600 -layer at (0,0) size 800x600 - RenderBlock {HTML} at (0,0) size 800x600 - RenderBody {BODY} at (8,8) size 784x584 - RenderBlock {P} at (0,0) size 784x18 - RenderText {#text} at (0,0) size 624x18 - text run at (0,0) width 624: "These should be four green hollow boxes with dimensions 600x300, 100x300, 600x100, 100x100." - RenderBlock (anonymous) at (0,34) size 784x420 - RenderHTMLCanvas {CANVAS} at (0,0) size 606x306 [border: (3px solid #008000)] - RenderText {#text} at (606,292) size 4x18 - text run at (606,292) width 4: " " - RenderText {#text} at (0,0) size 0x0 - RenderHTMLCanvas {CANVAS} at (610,0) size 106x306 [border: (3px solid #008000)] - RenderText {#text} at (0,0) size 0x0 - RenderText {#text} at (0,0) size 0x0 - RenderHTMLCanvas {CANVAS} at (0,310) size 606x106 [border: (3px solid #008000)] - RenderText {#text} at (606,402) size 4x18 - text run at (606,402) width 4: " " - RenderText {#text} at (0,0) size 0x0 - RenderHTMLCanvas {CANVAS} at (610,310) size 106x106 [border: (3px solid #008000)] - RenderText {#text} at (0,0) size 0x0 diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom.html deleted file mode 100644 index 4dabce1..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/image/canvas-zoom.html +++ /dev/null @@ -1,15 +0,0 @@ -<style> - canvas { border: solid green; - zoom: 2; } -</style> -<p> - These should be four green hollow boxes with dimensions 600x300, 100x300, 600x100, 100x100. -</p> -<!-- 300x150 --> -<canvas id="canvas"></canvas> -<!-- 50x150 --> -<canvas id="canvas" width="50"></canvas> -<!-- 300x50 --> -<canvas id="canvas" height="50"></canvas> -<!-- 50x50 --> -<canvas id="canvas" width="50" height="50"></canvas> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash-expected.txt deleted file mode 100644 index 521c3f5..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash-expected.txt +++ /dev/null @@ -1 +0,0 @@ -This test is expected to crash. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash.html deleted file mode 100644 index b9820d6..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/crash.html +++ /dev/null @@ -1,5 +0,0 @@ -<html> -<body> -This test is expected to crash. -</body> -</html> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/missing-expectation.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/missing-expectation.html deleted file mode 100644 index 1f00b50..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/missing-expectation.html +++ /dev/null @@ -1,5 +0,0 @@ -<html> -<body> -This test intentionally doesn't have an expected result checked in. -</body> -</html> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing-expected.txt deleted file mode 100644 index 26bd316..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing-expected.txt +++ /dev/null @@ -1 +0,0 @@ -This test is expected to pass. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing.html deleted file mode 100644 index db7e3de..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/misc/passing.html +++ /dev/null @@ -1,5 +0,0 @@ -<html> -<body> -This test is expected to pass. -</body> -</html> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image-expected.checksum new file mode 100644 index 0000000..24b887a --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image-expected.checksum @@ -0,0 +1 @@ +image-checksum diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image-expected.png new file mode 100644 index 0000000..4c23996 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image-expected.png @@ -0,0 +1 @@ +image-png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image-expected.txt new file mode 100644 index 0000000..c6ee718 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image-expected.txt @@ -0,0 +1 @@ +image-txt diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image.html new file mode 100644 index 0000000..773b222 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/image.html @@ -0,0 +1 @@ +image diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image-expected.checksum new file mode 100644 index 0000000..52038ae --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image-expected.checksum @@ -0,0 +1 @@ +platform_image-generic-checksum diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image-expected.png new file mode 100644 index 0000000..087872b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image-expected.png @@ -0,0 +1 @@ +platform_image-generic-png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image-expected.txt new file mode 100644 index 0000000..f71680c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image-expected.txt @@ -0,0 +1 @@ +platform_image-generic-txt diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image.html new file mode 100644 index 0000000..ca48a7b --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/platform_image.html @@ -0,0 +1 @@ +platform_image diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text-expected.txt new file mode 100644 index 0000000..2b38a06 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text-expected.txt @@ -0,0 +1 @@ +text-txt diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text.html new file mode 100644 index 0000000..8e27be7 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text.html @@ -0,0 +1 @@ +text diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.checksum deleted file mode 100644 index 4cd8dac..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.checksum +++ /dev/null @@ -1 +0,0 @@ -790b681a41697634fcf2a2587afb89c6
\ No newline at end of file diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.png Binary files differdeleted file mode 100644 index 3d00450..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.png +++ /dev/null diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.txt deleted file mode 100644 index 2411c0a..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/image/canvas-bg-expected.txt +++ /dev/null @@ -1,6 +0,0 @@ -layer at (0,0) size 785x620 - RenderView at (0,0) size 785x600 -layer at (0,0) size 785x620 - RenderBlock {HTML} at (0,0) size 785x620 - RenderBody {BODY} at (8,8) size 769x604 - RenderBlock {DIV} at (0,0) size 604x604 [border: (2px solid #000000)] diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/passes/platform_image-expected.checksum b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/passes/platform_image-expected.checksum new file mode 100644 index 0000000..ea557cf --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/passes/platform_image-expected.checksum @@ -0,0 +1 @@ +platform_image-checksum diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/passes/platform_image-expected.png b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/passes/platform_image-expected.png new file mode 100644 index 0000000..ec42fc1 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/passes/platform_image-expected.png @@ -0,0 +1 @@ +platform_image-png diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/passes/platform_image-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/passes/platform_image-expected.txt new file mode 100644 index 0000000..ff8bf43 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/passes/platform_image-expected.txt @@ -0,0 +1 @@ +platform_image-txt diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt index b78a01c..6e66caa 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt @@ -1 +1,10 @@ -WONTFIX : misc/missing-expectation.html = MISSING PASS +WONTFIX : failures/expected/checksum.html = IMAGE +WONTFIX : failures/expected/crash.html = CRASH +// This one actually passes because the checksums will match. +WONTFIX : failures/expected/image.html = PASS +WONTFIX : failures/expected/image_checksum.html = IMAGE +WONTFIX : failures/expected/missing_check.html = MISSING PASS +WONTFIX : failures/expected/missing_image.html = MISSING PASS +WONTFIX : failures/expected/missing_text.html = MISSING PASS +WONTFIX : failures/expected/text.html = TEXT +WONTFIX : failures/expected/timeout.html = TIMEOUT diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/resources/README.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/resources/README.txt new file mode 100644 index 0000000..b806b06 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/data/resources/README.txt @@ -0,0 +1,2 @@ +This directory exists solely to make sure that when we gather the lists of +tests, we skip over directories named 'resources'. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element-expected.txt deleted file mode 100644 index f60ac38..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element-expected.txt +++ /dev/null @@ -1,20 +0,0 @@ -Various tests for the article element. - -On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE". - - -<article> closes <p>: -PASS article1.parentNode.nodeName == "p" is false -<p> does not close <article>: -PASS p1.parentNode.nodeName is "ARTICLE" -<article> can be nested inside <article>: -PASS article3.parentNode.id is "article2" -Residual style: -PASS getWeight("article4") is "bold" -PASS getWeight("span1") is "bold" -FormatBlock: -PASS document.getElementById("span2").parentNode.nodeName is "ARTICLE" -PASS successfullyParsed is true - -TEST COMPLETE - diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element.html deleted file mode 100644 index c0f4547..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/data/text/article-element.html +++ /dev/null @@ -1,13 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN"> -<html> -<head> -<link rel="stylesheet" href="../../fast/js/resources/js-test-style.css"> -<script src="../../fast/js/resources/js-test-pre.js"></script> -</head> -<body> -<p id="description"></p> -<div id="console"></div> -<script src="script-tests/article-element.js"></script> -<script src="../../fast/js/resources/js-test-post.js"></script> -</body> -</html> diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py index 81cdc9b..a9e015f 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py @@ -298,7 +298,20 @@ class Printer(object): self._write("") def print_test_result(self, result, expected, exp_str, got_str): - """Print the result of the test as determined by --print.""" + """Print the result of the test as determined by --print. + + This routine is used to print the details of each test as it completes. + + Args: + result - The actual TestResult object + expected - Whether the result we got was an expected result + exp_str - What we expected to get (used for tracing) + got_str - What we actually got (used for tracing) + + Note that we need all of these arguments even though they seem + somewhat redundant, in order to keep this routine from having to + known anything about the set of expectations. + """ if (self.enabled('trace-everything') or self.enabled('trace-unexpected') and not expected): self._print_test_trace(result, exp_str, got_str) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py index c8648bc..40c691f 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py @@ -51,27 +51,6 @@ def get_options(args): return option_parser.parse_args(args) -def get_result(filename, result_type=test_expectations.PASS, run_time=0): - failures = [] - if result_type == test_expectations.TIMEOUT: - failures = [test_failures.FailureTimeout()] - elif result_type == test_expectations.CRASH: - failures = [test_failures.FailureCrash()] - return dump_render_tree_thread.TestResult(filename, failures, run_time, - total_time_for_all_diffs=0, - time_for_diffs=0) - - -def get_result_summary(port_obj, test_files, expectations_str): - expectations = test_expectations.TestExpectations( - port_obj, test_files, expectations_str, - port_obj.test_platform_name(), is_debug_mode=False, - is_lint_mode=False, tests_are_present=False) - - rs = run_webkit_tests.ResultSummary(expectations, test_files) - return rs, expectations - - class TestUtilityFunctions(unittest.TestCase): def test_configure_logging(self): # FIXME: We need to figure out how to reset the basic logger. @@ -155,6 +134,28 @@ class Testprinter(unittest.TestCase): is_fully_parallel) return printer, regular_output, buildbot_output + def get_result(self, test, result_type=test_expectations.PASS, run_time=0): + failures = [] + if result_type == test_expectations.TIMEOUT: + failures = [test_failures.FailureTimeout()] + elif result_type == test_expectations.CRASH: + failures = [test_failures.FailureCrash()] + path = os.path.join(self._port.layout_tests_dir(), test) + return dump_render_tree_thread.TestResult(path, failures, run_time, + total_time_for_all_diffs=0, + time_for_diffs=0) + + def get_result_summary(self, tests, expectations_str): + test_paths = [os.path.join(self._port.layout_tests_dir(), test) for + test in tests] + expectations = test_expectations.TestExpectations( + self._port, test_paths, expectations_str, + self._port.test_platform_name(), is_debug_mode=False, + is_lint_mode=False, tests_are_present=False) + + rs = run_webkit_tests.ResultSummary(expectations, test_paths) + return test_paths, rs, expectations + def test_help_printer(self): # Here and below we'll call the "regular" printer err and the # buildbot printer out; this corresponds to how things run on the @@ -244,10 +245,18 @@ class Testprinter(unittest.TestCase): def test_print_test_result(self): - result = get_result('foo.html') + # Note here that we don't use meaningful exp_str and got_str values; + # the actual contents of the string are treated opaquely by + # print_test_result() when tracing, and usually we don't want + # to test what exactly is printed, just that something + # was printed (or that nothing was printed). + # + # FIXME: this is actually some goofy layering; it would be nice + # we could refactor it so that the args weren't redundant. Maybe + # the TestResult should contain what was expected, and the + # strings could be derived from the TestResult? printer, err, out = self.get_printer(['--print', 'nothing']) - result = get_result(os.path.join(self._port.layout_tests_dir(), - 'foo.html')) + result = self.get_result('passes/image.html') printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertTrue(err.empty()) @@ -259,7 +268,7 @@ class Testprinter(unittest.TestCase): printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertEquals(err.get(), - [' foo.html -> unexpected pass\n']) + [' passes/image.html -> unexpected pass\n']) printer, err, out = self.get_printer(['--print', 'everything']) printer.print_test_result(result, expected=True, exp_str='', @@ -269,7 +278,7 @@ class Testprinter(unittest.TestCase): printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertEquals(err.get(), - [' foo.html -> unexpected pass\n']) + [' passes/image.html -> unexpected pass\n']) printer, err, out = self.get_printer(['--print', 'nothing']) printer.print_test_result(result, expected=False, exp_str='', @@ -277,11 +286,24 @@ class Testprinter(unittest.TestCase): self.assertTrue(err.empty()) printer, err, out = self.get_printer(['--print', - 'trace-unexpected']) + 'trace-unexpected']) printer.print_test_result(result, expected=True, exp_str='', got_str='') self.assertTrue(err.empty()) + printer, err, out = self.get_printer(['--print', + 'trace-unexpected']) + printer.print_test_result(result, expected=False, exp_str='', + got_str='') + self.assertFalse(err.empty()) + + printer, err, out = self.get_printer(['--print', + 'trace-unexpected']) + result = self.get_result("passes/text.html") + printer.print_test_result(result, expected=False, exp_str='', + got_str='') + self.assertFalse(err.empty()) + err.reset() printer.print_test_result(result, expected=False, exp_str='', got_str='') @@ -297,104 +319,106 @@ class Testprinter(unittest.TestCase): got_str='') def test_print_progress(self): - test_files = ['foo.html', 'bar.html'] expectations = '' # test that we print nothing printer, err, out = self.get_printer(['--print', 'nothing']) - rs, exp = get_result_summary(self._port, test_files, expectations) + tests = ['passes/text.html', 'failures/expected/timeout.html', + 'failures/expected/crash.html'] + paths, rs, exp = self.get_result_summary(tests, expectations) - printer.print_progress(rs, False, test_files) + printer.print_progress(rs, False, paths) self.assertTrue(out.empty()) self.assertTrue(err.empty()) - printer.print_progress(rs, True, test_files) + printer.print_progress(rs, True, paths) self.assertTrue(out.empty()) self.assertTrue(err.empty()) # test regular functionality printer, err, out = self.get_printer(['--print', 'one-line-progress']) - printer.print_progress(rs, False, test_files) + printer.print_progress(rs, False, paths) self.assertTrue(out.empty()) self.assertFalse(err.empty()) err.reset() out.reset() - printer.print_progress(rs, True, test_files) + printer.print_progress(rs, True, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) def test_print_progress__detailed(self): - test_files = ['pass/pass.html', 'pass/timeout.html', 'fail/crash.html'] - expectations = 'pass/timeout.html = TIMEOUT' + tests = ['passes/text.html', 'failures/expected/timeout.html', + 'failures/expected/crash.html'] + expectations = 'failures/expected/timeout.html = TIMEOUT' # first, test that it is disabled properly # should still print one-line-progress printer, err, out = self.get_printer( ['--print', 'detailed-progress'], single_threaded=False) - rs, exp = get_result_summary(self._port, test_files, expectations) - printer.print_progress(rs, False, test_files) + paths, rs, exp = self.get_result_summary(tests, expectations) + printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) # now test the enabled paths printer, err, out = self.get_printer( ['--print', 'detailed-progress'], single_threaded=True) - rs, exp = get_result_summary(self._port, test_files, expectations) - printer.print_progress(rs, False, test_files) + paths, rs, exp = self.get_result_summary(tests, expectations) + printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) err.reset() out.reset() - printer.print_progress(rs, True, test_files) + printer.print_progress(rs, True, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) - rs.add(get_result('pass/pass.html', test_expectations.TIMEOUT), False) - rs.add(get_result('pass/timeout.html'), True) - rs.add(get_result('fail/crash.html', test_expectations.CRASH), True) + rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) + rs.add(self.get_result('failures/expected/timeout.html'), True) + rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) err.reset() out.reset() - printer.print_progress(rs, False, test_files) + printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) # We only clear the meter when retrying w/ detailed-progress. err.reset() out.reset() - printer.print_progress(rs, True, test_files) - self.assertEqual(err.get(), ['']) + printer.print_progress(rs, True, paths) + self.assertFalse(err.empty()) self.assertTrue(out.empty()) printer, err, out = self.get_printer( ['--print', 'detailed-progress,unexpected'], single_threaded=True) - rs, exp = get_result_summary(self._port, test_files, expectations) - printer.print_progress(rs, False, test_files) + paths, rs, exp = self.get_result_summary(tests, expectations) + printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) err.reset() out.reset() - printer.print_progress(rs, True, test_files) + printer.print_progress(rs, True, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) - rs.add(get_result('pass/pass.html', test_expectations.TIMEOUT), False) - rs.add(get_result('pass/timeout.html'), True) - rs.add(get_result('fail/crash.html', test_expectations.CRASH), True) + rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) + rs.add(self.get_result('failures/expected/timeout.html'), True) + rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) err.reset() out.reset() - printer.print_progress(rs, False, test_files) + printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) # We only clear the meter when retrying w/ detailed-progress. err.reset() out.reset() - printer.print_progress(rs, True, test_files) - self.assertEqual(err.get(), ['']) + printer.print_progress(rs, True, paths) + self.assertFalse(err.empty()) self.assertTrue(out.empty()) def test_write(self): @@ -416,42 +440,72 @@ class Testprinter(unittest.TestCase): printer.write("foo", "config") self.assertFalse(err.empty()) + # FIXME: this should be logged somewhere, but it actually + # disappears into the ether in the logging subsystem. + printer, err, out = self.get_printer(['--verbose']) + printer.write("foo") + self.assertTrue(err.empty()) + self.assertTrue(out.empty()) + def test_print_unexpected_results(self): # This routine is the only one that prints stuff that the bots # care about. + # + # FIXME: there's some weird layering going on here. It seems + # like we shouldn't be both using an expectations string and + # having to specify whether or not the result was expected. + # This whole set of tests should probably be rewritten. + # + # FIXME: Plus, the fact that we're having to call into + # run_webkit_tests is clearly a layering inversion. def get_unexpected_results(expected, passing, flaky): - rs, exp = get_result_summary(self._port, test_files, expectations) + """Return an unexpected results summary matching the input description. + + There are a lot of different combinations of test results that + can be tested; this routine produces various combinations based + on the values of the input flags. + + Args + expected: whether the tests ran as expected + passing: whether the tests should all pass + flaky: whether the tests should be flaky (if False, they + produce the same results on both runs; if True, they + all pass on the second run). + + """ + paths, rs, exp = self.get_result_summary(tests, expectations) if expected: - rs.add(get_result('pass/pass.html', test_expectations.PASS), + rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected) - rs.add(get_result('pass/timeout.html', + rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected) - rs.add(get_result('fail/crash.html', test_expectations.CRASH), + rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected) elif passing: - rs.add(get_result('pass/pass.html'), expected) - rs.add(get_result('pass/timeout.html'), expected) - rs.add(get_result('fail/crash.html'), expected) + rs.add(self.get_result('passes/text.html'), expected) + rs.add(self.get_result('failures/expected/timeout.html'), expected) + rs.add(self.get_result('failures/expected/crash.html'), expected) else: - rs.add(get_result('pass/pass.html', test_expectations.TIMEOUT), + rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected) - rs.add(get_result('pass/timeout.html', + rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected) - rs.add(get_result('fail/crash.html', + rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected) retry = rs if flaky: - retry, exp = get_result_summary(self._port, test_files, + paths, retry, exp = self.get_result_summary(tests, expectations) - retry.add(get_result('pass/pass.html'), True) - retry.add(get_result('pass/timeout.html'), True) - retry.add(get_result('fail/crash.html'), True) + retry.add(self.get_result('passes/text.html'), True) + retry.add(self.get_result('failures/expected/timeout.html'), True) + retry.add(self.get_result('failures/expected/crash.html'), True) unexpected_results = run_webkit_tests.summarize_unexpected_results( self._port, exp, rs, retry) return unexpected_results - test_files = ['pass/pass.html', 'pass/timeout.html', 'fail/crash.html'] + tests = ['passes/text.html', 'failures/expected/timeout.html', + 'failures/expected/crash.html'] expectations = '' printer, err, out = self.get_printer(['--print', 'nothing']) @@ -501,6 +555,33 @@ class Testprinter(unittest.TestCase): self.assertTrue(err.empty()) self.assertFalse(out.empty()) + expectations = """ +failures/expected/crash.html = CRASH +failures/expected/timeout.html = TIMEOUT +""" + err.reset() + out.reset() + ur = get_unexpected_results(expected=False, passing=False, flaky=False) + printer.print_unexpected_results(ur) + self.assertTrue(err.empty()) + self.assertFalse(out.empty()) + + err.reset() + out.reset() + ur = get_unexpected_results(expected=False, passing=True, flaky=False) + printer.print_unexpected_results(ur) + self.assertTrue(err.empty()) + self.assertFalse(out.empty()) + + # Test handling of --verbose as well. + err.reset() + out.reset() + printer, err, out = self.get_printer(['--verbose']) + ur = get_unexpected_results(expected=False, passing=False, flaky=False) + printer.print_unexpected_results(ur) + self.assertTrue(err.empty()) + self.assertFalse(out.empty()) + def test_print_unexpected_results_buildbot(self): # FIXME: Test that print_unexpected_results() produces the printer the # buildbot is expecting. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py index e154932..086321d 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py @@ -152,10 +152,7 @@ class TestExpectations: for item in TestExpectationsFile.EXPECTATIONS.items(): if item[1] == expectation: return item[0].upper() - return "" - - def get_timeline_for_test(self, test): - return self._expected_failures.get_timeline_for_test(test) + raise ValueError(expectation) def get_tests_with_result_type(self, result_type): return self._expected_failures.get_tests_with_result_type(result_type) @@ -208,15 +205,15 @@ class ModifiersAndExpectations: class ExpectationsJsonEncoder(simplejson.JSONEncoder): - """JSON encoder that can handle ModifiersAndExpectations objects. - """ - + """JSON encoder that can handle ModifiersAndExpectations objects.""" def default(self, obj): - if isinstance(obj, ModifiersAndExpectations): - return {"modifiers": obj.modifiers, - "expectations": obj.expectations} - else: - return JSONEncoder.default(self, obj) + # A ModifiersAndExpectations object has two fields, each of which + # is a dict. Since JSONEncoders handle all the builtin types directly, + # the only time this routine should be called is on the top level + # object (i.e., the encoder shouldn't recurse). + assert isinstance(obj, ModifiersAndExpectations) + return {"modifiers": obj.modifiers, + "expectations": obj.expectations} class TestExpectationsFile: @@ -463,9 +460,6 @@ class TestExpectationsFile: def get_non_fatal_errors(self): return self._non_fatal_errors - def contains(self, test): - return test in self._test_to_expectations - def remove_platform_from_expectations(self, tests, platform): """Returns a copy of the expectations with the tests matching the platform remove. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py index cf3c560..22214b0 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py @@ -33,16 +33,8 @@ import os import sys import unittest -try: - d = os.path.dirname(__file__) -except NameError: - d = os.path.dirname(sys.argv[0]) - -sys.path.append(os.path.abspath(os.path.join(d, '..'))) -sys.path.append(os.path.abspath(os.path.join(d, '../../thirdparty'))) - -import port -from test_expectations import * +from webkitpy.layout_tests import port +from webkitpy.layout_tests.layout_package.test_expectations import * class FunctionsTest(unittest.TestCase): def test_result_was_expected(self): @@ -87,8 +79,7 @@ class FunctionsTest(unittest.TestCase): set([PASS, CRASH])) -class TestExpectationsTest(unittest.TestCase): - +class Base(unittest.TestCase): def __init__(self, testFunc, setUp=None, tearDown=None, description=None): self._port = port.get('test', None) self._exp = None @@ -98,53 +89,151 @@ class TestExpectationsTest(unittest.TestCase): return os.path.join(self._port.layout_tests_dir(), test_name) def get_basic_tests(self): - return [self.get_test('text/article-element.html'), - self.get_test('image/canvas-bg.html'), - self.get_test('image/canvas-zoom.html'), - self.get_test('misc/crash.html'), - self.get_test('misc/passing.html')] + return [self.get_test('failures/expected/text.html'), + self.get_test('failures/expected/image_checksum.html'), + self.get_test('failures/expected/crash.html'), + self.get_test('failures/expected/missing_text.html'), + self.get_test('passes/text.html')] def get_basic_expectations(self): return """ -BUG_TEST : text/article-element.html = TEXT -BUG_TEST SKIP : misc/crash.html = CRASH -BUG_TEST REBASELINE : misc/missing-expectation.html = MISSING -BUG_TEST : image = IMAGE +BUG_TEST : failures/expected/text.html = TEXT +BUG_TEST SKIP : failures/expected/crash.html = CRASH +BUG_TEST REBASELINE : failure/expected/missing_image.html = MISSING +BUG_TEST : failures/expected/image_checksum.html = IMAGE """ - def parse_exp(self, expectations, overrides=None): + def parse_exp(self, expectations, overrides=None, is_lint_mode=False, + is_debug_mode=False, tests_are_present=True): self._exp = TestExpectations(self._port, tests=self.get_basic_tests(), expectations=expectations, test_platform_name=self._port.test_platform_name(), - is_debug_mode=False, - is_lint_mode=False, - tests_are_present=True, + is_debug_mode=is_debug_mode, + is_lint_mode=is_lint_mode, + tests_are_present=tests_are_present, overrides=overrides) def assert_exp(self, test, result): self.assertEquals(self._exp.get_expectations(self.get_test(test)), set([result])) + +class TestExpectationsTest(Base): def test_basic(self): self.parse_exp(self.get_basic_expectations()) - self.assert_exp('text/article-element.html', TEXT) - self.assert_exp('image/canvas-zoom.html', IMAGE) - self.assert_exp('misc/passing.html', PASS) + self.assert_exp('failures/expected/text.html', TEXT) + self.assert_exp('failures/expected/image_checksum.html', IMAGE) + self.assert_exp('passes/text.html', PASS) + + def test_defer(self): + self.parse_exp('BUGX DEFER : failures/expected/text.html = TEXT') + self.assertEqual(self._exp.get_options( + self.get_test('failures/expected/text.html')), ['bugx', 'defer']) + + def test_precedence(self): + # This tests handling precedence of specific lines over directories + # and tests expectations covering entire directories. + exp_str = """ +BUGX : failures/expected/text.html = TEXT +BUGX DEFER : failures/expected = IMAGE +""" + self.parse_exp(exp_str) + self.assert_exp('failures/expected/text.html', TEXT) + self.assert_exp('failures/expected/crash.html', IMAGE) + + self.parse_exp(exp_str, tests_are_present=False) + self.assert_exp('failures/expected/text.html', TEXT) + self.assert_exp('failures/expected/crash.html', IMAGE) + + def test_release_mode(self): + self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT', + is_debug_mode=True) + self.assert_exp('failures/expected/text.html', TEXT) + self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT', + is_debug_mode=True) + self.assert_exp('failures/expected/text.html', PASS) + self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT', + is_debug_mode=False) + self.assert_exp('failures/expected/text.html', PASS) + self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT', + is_debug_mode=False) + self.assert_exp('failures/expected/text.html', TEXT) + + def test_get_options(self): + self.parse_exp(self.get_basic_expectations()) + self.assertEqual(self._exp.get_options( + self.get_test('passes/text.html')), []) - def test_duplicates(self): + def test_expectations_json_for_all_platforms(self): + self.parse_exp(self.get_basic_expectations()) + json_str = self._exp.get_expectations_json_for_all_platforms() + # FIXME: test actual content? + self.assertTrue(json_str) + + def test_get_expectations_string(self): + self.parse_exp(self.get_basic_expectations()) + self.assertEquals(self._exp.get_expectations_string( + self.get_test('failures/expected/text.html')), + 'TEXT') + + def test_expectation_to_string(self): + # Normal cases are handled by other tests. + self.parse_exp(self.get_basic_expectations()) + self.assertRaises(ValueError, self._exp.expectation_to_string, + -1) + + def test_syntax_missing_expectation(self): + # This is missing the expectation. + self.assertRaises(SyntaxError, self.parse_exp, + 'BUG_TEST: failures/expected/text.html', + is_debug_mode=True) + + def test_syntax_invalid_option(self): + self.assertRaises(SyntaxError, self.parse_exp, + 'BUG_TEST FOO: failures/expected/text.html = PASS') + + def test_syntax_invalid_expectation(self): + # This is missing the expectation. + self.assertRaises(SyntaxError, self.parse_exp, + 'BUG_TEST: failures/expected/text.html = FOO') + + def test_syntax_missing_bugid(self): + # This should log a non-fatal error. + self.parse_exp('SLOW : failures/expected/text.html = TEXT') + self.assertEqual( + len(self._exp._expected_failures.get_non_fatal_errors()), 1) + + def test_semantic_slow_and_timeout(self): + # A test cannot be SLOW and expected to TIMEOUT. + self.assertRaises(SyntaxError, self.parse_exp, + 'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT') + + def test_semantic_wontfix_defer(self): + # A test cannot be WONTFIX and DEFER. + self.assertRaises(SyntaxError, self.parse_exp, + 'BUG_TEST WONTFIX DEFER : failures/expected/text.html = TEXT') + + def test_semantic_rebaseline(self): + # Can't lint a file w/ 'REBASELINE' in it. + self.assertRaises(SyntaxError, self.parse_exp, + 'BUG_TEST REBASELINE : failures/expected/text.html = TEXT', + is_lint_mode=True) + + def test_semantic_duplicates(self): self.assertRaises(SyntaxError, self.parse_exp, """ -BUG_TEST : text/article-element.html = TEXT -BUG_TEST : text/article-element.html = IMAGE""") +BUG_TEST : failures/expected/text.html = TEXT +BUG_TEST : failures/expected/text.html = IMAGE""") + self.assertRaises(SyntaxError, self.parse_exp, self.get_basic_expectations(), """ -BUG_TEST : text/article-element.html = TEXT -BUG_TEST : text/article-element.html = IMAGE""") +BUG_TEST : failures/expected/text.html = TEXT +BUG_TEST : failures/expected/text.html = IMAGE""") def test_overrides(self): self.parse_exp(self.get_basic_expectations(), """ -BUG_OVERRIDE : text/article-element.html = IMAGE""") - self.assert_exp('text/article-element.html', IMAGE) +BUG_OVERRIDE : failures/expected/text.html = IMAGE""") + self.assert_exp('failures/expected/text.html', IMAGE) def test_matches_an_expected_result(self): @@ -153,16 +242,34 @@ BUG_OVERRIDE : text/article-element.html = IMAGE""") self.get_test(test), result, pixel_tests_enabled) self.parse_exp(self.get_basic_expectations()) - self.assertTrue(match('text/article-element.html', TEXT, True)) - self.assertTrue(match('text/article-element.html', TEXT, False)) - self.assertFalse(match('text/article-element.html', CRASH, True)) - self.assertFalse(match('text/article-element.html', CRASH, False)) + self.assertTrue(match('failures/expected/text.html', TEXT, True)) + self.assertTrue(match('failures/expected/text.html', TEXT, False)) + self.assertFalse(match('failures/expected/text.html', CRASH, True)) + self.assertFalse(match('failures/expected/text.html', CRASH, False)) + self.assertTrue(match('failures/expected/image_checksum.html', IMAGE, + True)) + self.assertTrue(match('failures/expected/image_checksum.html', PASS, + False)) + self.assertTrue(match('failures/expected/crash.html', SKIP, False)) + self.assertTrue(match('passes/text.html', PASS, False)) - self.assertTrue(match('image/canvas-bg.html', IMAGE, True)) - self.assertTrue(match('image/canvas-bg.html', PASS, False)) - self.assertTrue(match('misc/crash.html', SKIP, False)) - self.assertTrue(match('misc/passing.html', PASS, False)) +class RebaseliningTest(Base): + """Test rebaselining-specific functionality.""" + def test_no_get_rebaselining_failures(self): + self.parse_exp(self.get_basic_expectations()) + self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) + + def test_basic(self): + self.parse_exp(""" +BUG_TEST REBASELINE : failures/expected/text.html = TEXT +""") + self.assertEqual(len(self._exp.get_rebaselining_failures()), 1) + + new_exp_str = self._exp.remove_platform_from_expectations( + self.get_test('failures/expected/text.html'), 'TEST') + # FIXME: actually test rebaselining + # self.assertEqual(new_exp_str, '\n') if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py index 3be9240..340d075 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py @@ -73,11 +73,11 @@ class TestFailure(object): @staticmethod def message(): """Returns a string describing the failure in more detail.""" - raise NotImplemented + raise NotImplementedError def result_html_output(self, filename): """Returns an HTML string to be included on the results.html page.""" - raise NotImplemented + raise NotImplementedError def should_kill_dump_render_tree(self): """Returns True if we should kill DumpRenderTree before the next @@ -108,10 +108,8 @@ class FailureWithType(TestFailure): use the standard OutputLinks. """ - def __init__(self, test_type): + def __init__(self): TestFailure.__init__(self) - # FIXME: This class no longer needs to know the test_type. - self._test_type = test_type # Filename suffixes used by ResultHtmlOutput. OUT_FILENAMES = [] @@ -202,8 +200,8 @@ class FailureTextMismatch(FailureWithType): OUT_FILENAMES_WDIFF = ["-actual.txt", "-expected.txt", "-diff.txt", "-wdiff.html", "-pretty-diff.html"] - def __init__(self, test_type, has_wdiff): - FailureWithType.__init__(self, test_type) + def __init__(self, has_wdiff): + FailureWithType.__init__(self) if has_wdiff: self.OUT_FILENAMES = self.OUT_FILENAMES_WDIFF diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py new file mode 100644 index 0000000..92fe276 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py @@ -0,0 +1,68 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""""Tests code paths not covered by the regular unit tests.""" + +from webkitpy.layout_tests.layout_package.test_failures import * +import unittest + +class Test(unittest.TestCase): + def assertResultHtml(self, failure_obj): + self.assertNotEqual(failure_obj.result_html_output('foo'), None) + + def test_crash(self): + self.assertResultHtml(FailureCrash()) + + def test_hash_incorrect(self): + self.assertResultHtml(FailureImageHashIncorrect()) + + def test_missing(self): + self.assertResultHtml(FailureMissingResult()) + + def test_missing_image(self): + self.assertResultHtml(FailureMissingImage()) + + def test_missing_image_hash(self): + self.assertResultHtml(FailureMissingImageHash()) + + def test_timeout(self): + self.assertResultHtml(FailureTimeout()) + + def test_unknown_failure_type(self): + class UnknownFailure(TestFailure): + pass + + failure_obj = UnknownFailure() + self.assertRaises(ValueError, determine_result_type, [failure_obj]) + self.assertRaises(NotImplementedError, failure_obj.message) + self.assertRaises(NotImplementedError, failure_obj.result_html_output, + "foo.txt") + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py index d226e64..af1af93 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py @@ -30,7 +30,10 @@ """Abstract base class of Port-specific entrypoints for the layout tests test infrastructure (the Port and Driver classes).""" +from __future__ import with_statement + import cgi +import codecs import difflib import errno import os @@ -633,8 +636,38 @@ class Port(object): _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output())) return self._pretty_patch_error_html + def _webkit_build_directory(self, args): + args = [self.script_path("webkit-build-directory")] + args + return self._executive.run_command(args).rstrip() + + def _configuration_file_path(self): + build_root = self._webkit_build_directory(["--top-level"]) + return os.path.join(build_root, "Configuration") + + # Easy override for unit tests + def _open_configuration_file(self): + configuration_path = self._configuration_file_path() + return codecs.open(configuration_path, "r", "utf-8") + + def _read_configuration(self): + try: + with self._open_configuration_file() as file: + return file.readline().rstrip() + except IOError, e: + return None + + # FIXME: This list may be incomplete as Apple has some sekret configs. + _RECOGNIZED_CONFIGURATIONS = ("Debug", "Release") + def default_configuration(self): - return "Release" + # FIXME: Unify this with webkitdir.pm configuration reading code. + configuration = self._read_configuration() + if not configuration: + configuration = "Release" + if configuration not in self._RECOGNIZED_CONFIGURATIONS: + _log.warn("Configuration \"%s\" found in %s is not a recognized value.\n" % (configuration, self._configuration_file_path())) + _log.warn("Scripts may fail. See 'set-webkit-configuration --help'.") + return configuration # # PROTECTED ROUTINES diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome.py index 1ea053b..46ab3ed 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome.py @@ -36,9 +36,10 @@ def GetGoogleChromePort(port_name, options): def baseline_search_path(self): paths = chromium_linux.ChromiumLinuxPort.baseline_search_path( self) - paths.insert(0, self._webkit_baseline_path(self._name)) + paths.insert(0, self._webkit_baseline_path( + 'google-chrome-linux32')) return paths - return GoogleChromeLinux32Port(port_name, options) + return GoogleChromeLinux32Port(None, options) elif port_name == 'google-chrome-linux64': import chromium_linux @@ -46,9 +47,10 @@ def GetGoogleChromePort(port_name, options): def baseline_search_path(self): paths = chromium_linux.ChromiumLinuxPort.baseline_search_path( self) - paths.insert(0, self._webkit_baseline_path(self._name)) + paths.insert(0, self._webkit_baseline_path( + 'google-chrome-linux64')) return paths - return GoogleChromeLinux64Port(port_name, options) + return GoogleChromeLinux64Port(None, options) elif port_name.startswith('google-chrome-mac'): import chromium_mac @@ -59,7 +61,7 @@ def GetGoogleChromePort(port_name, options): paths.insert(0, self._webkit_baseline_path( 'google-chrome-mac')) return paths - return GoogleChromeMacPort(port_name, options) + return GoogleChromeMacPort(None, options) elif port_name.startswith('google-chrome-win'): import chromium_win @@ -70,5 +72,5 @@ def GetGoogleChromePort(port_name, options): paths.insert(0, self._webkit_baseline_path( 'google-chrome-win')) return paths - return GoogleChromeWinPort(port_name, options) + return GoogleChromeWinPort(None, options) raise NotImplementedError('unsupported port: %s' % port_name) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py index 9c9ab0a..4fe3ec1 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py @@ -43,12 +43,9 @@ class TestPort(base.Port): def __init__(self, port_name=None, options=None): base.Port.__init__(self, port_name, options) - def base_platforms(self): - return ('test',) - def baseline_path(self): return os.path.join(self.layout_tests_dir(), 'platform', - self.name()) + self.name() + self.version()) def baseline_search_path(self): return [self.baseline_path()] @@ -56,16 +53,18 @@ class TestPort(base.Port): def check_build(self, needs_http): return True - def compare_text(self, expected_text, actual_text): - return False - def diff_image(self, expected_filename, actual_filename, diff_filename=None, tolerance=0): - return False - - def diff_text(self, expected_text, actual_text, - expected_filename, actual_filename): - return '' + with codecs.open(actual_filename, "r", "utf-8") as actual_fh: + actual_contents = actual_fh.read() + with codecs.open(expected_filename, "r", "utf-8") as expected_fh: + expected_contents = expected_fh.read() + diffed = actual_contents != expected_contents + if diffed and diff_filename: + with codecs.open(diff_filename, "w", "utf-8") as diff_fh: + diff_fh.write("< %s\n---\n> %s\n" % + (expected_contents, actual_contents)) + return diffed def layout_tests_dir(self): return self.path_from_webkit_base('WebKitTools', 'Scripts', @@ -82,6 +81,9 @@ class TestPort(base.Port): 'webkitpy', 'layout_tests', 'data', 'platform', 'test', 'test_expectations.txt') + def _path_to_wdiff(self): + return None + def results_directory(self): return '/tmp/' + self._options.results_directory @@ -127,10 +129,7 @@ class TestPort(base.Port): def test_platform_name_to_name(self, test_platform_name): return test_platform_name - def version(): - return '' - - def wdiff_text(self, expected_filename, actual_filename): + def version(self): return '' @@ -150,16 +149,58 @@ class TestDriver(base.Driver): return 0 def run_test(self, uri, timeoutms, image_hash): - if not self._image_written and self._port._options.pixel_tests: - with open(self._image_path, "w") as f: - f.write("bad png file from TestDriver") - self._image_written = True - - # We special-case this because we can't fake an image hash for a - # missing expectation. - if uri.find('misc/missing-expectation') != -1: - return (False, False, 'deadbeefdeadbeefdeadbeefdeadbeef', '', None) - return (False, False, image_hash, '', None) + basename = uri[(uri.rfind("/") + 1):uri.rfind(".html")] + + error = '' + checksum = None + # There are four currently supported types of tests: text, image, + # image hash (checksum), and stderr output. The fake output + # is the basename of the file + "-" plus the type of test output + # (or a blank string for stderr). + # + # If 'image' or 'check' appears in the basename, we assume this is + # simulating a pixel test. + # + # If 'failures' appears in the URI, then we assume this test should + # fail. Which type of failures are determined by which strings appear + # in the basename of the test. For failures that produce outputs, + # we change the fake output to basename + "_failed-". + # + # The fact that each test produces (more or less) unique output data + # will allow us to see if any results get crossed by the rest of the + # program. + if 'failures' in uri: + crash = 'crash' in basename + timeout = 'timeout' in basename + if 'error' in basename: + error = basename + "_error\n" + if 'text' in basename: + output = basename + '_failed-txt\n' + else: + output = basename + '-txt\n' + if self._port.options().pixel_tests: + if ('image' in basename or 'check' in basename): + checksum = basename + "-checksum\n" + + if 'image' in basename: + with open(self._image_path, "w") as f: + f.write(basename + "_failed-png\n") + elif 'check' in basename: + with open(self._image_path, "w") as f: + f.write(basename + "-png\n") + if 'checksum' in basename: + checksum = basename + "_failed-checksum\n" + else: + crash = False + timeout = False + output = basename + '-txt\n' + if self._port.options().pixel_tests and ( + 'image' in basename or 'check' in basename): + checksum = basename + '-checksum\n' + with open(self._image_path, "w") as f: + f.write(basename + "-png") + + return (crash, timeout, checksum, output, error) def start(self): pass diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py index e1151a6..cedc028 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -295,39 +295,6 @@ class WebKitPort(base.Port): return self.test_base_platform_names() + ( 'mac-tiger', 'mac-leopard', 'mac-snowleopard') - def _configuration_file_path(self): - build_root = self._webkit_build_directory(["--top-level"]) - return os.path.join(build_root, "Configuration") - - # Easy override for unit tests - def _open_configuration_file(self): - configuration_path = self._configuration_file_path() - return codecs.open(configuration_path, "r", "utf-8") - - def _read_configuration(self): - try: - with self._open_configuration_file() as file: - return file.readline().rstrip() - except IOError, e: - return None - - # FIXME: This list may be incomplete as Apple has some sekret configs. - _RECOGNIZED_CONFIGURATIONS = ("Debug", "Release") - - def default_configuration(self): - # FIXME: Unify this with webkitdir.pm configuration reading code. - configuration = self._read_configuration() - if not configuration: - configuration = "Release" - if configuration not in self._RECOGNIZED_CONFIGURATIONS: - _log.warn("Configuration \"%s\" found in %s is not a recognized value.\n" % (configuration, self._configuration_file_path())) - _log.warn("Scripts may fail. See 'set-webkit-configuration --help'.") - return configuration - - def _webkit_build_directory(self, args): - args = [self.script_path("webkit-build-directory")] + args - return self._executive.run_command(args).rstrip() - def _build_path(self, *comps): if not self._cached_build_root: self._cached_build_root = self._webkit_build_directory([ diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py index 92f1032..3a9f923 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py @@ -63,8 +63,6 @@ import webkitpy.common.checkout.scm as scm import port from layout_package import test_expectations -from test_types import image_diff -from test_types import text_diff _log = logging.getLogger("webkitpy.layout_tests." "rebaseline_chromium_webkit_tests") @@ -549,11 +547,13 @@ class Rebaseliner(object): return True if ext1 == '.PNG': - return image_diff.ImageDiff(self._port, - '').diff_files(self._port, file1, file2) + return self._port.diff_image(file1, file2) else: - return text_diff.TestTextDiff(self._port, - '').diff_files(self._port, file1, file2) + with codecs.open(file1, "r", "utf8") as file_handle1: + output1 = file_handle1.read() + with codecs.open(file2, "r", "utf8") as file_handle2: + output2 = file_handle2.read() + return self._port.compare_text(output1, output2) def _delete_baseline(self, filename): """Remove the file from repository and delete it from disk. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py index 121b64e..dbb2b91 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py @@ -29,6 +29,7 @@ """Unit tests for rebaseline_chromium_webkit_tests.py.""" +import os import unittest from webkitpy.layout_tests import port @@ -85,18 +86,32 @@ class TestGetHostPortObject(unittest.TestCase): class TestRebaseliner(unittest.TestCase): - - def test_noop(self): - # this method tests that was can at least instantiate an object, even - # if there is nothing to do. + def make_rebaseliner(self): options = MockOptions() host_port_obj = port.get('test', options) target_options = options target_port_obj = port.get('test', target_options) platform = 'test' - rebaseliner = rebaseline_chromium_webkit_tests.Rebaseliner( + return rebaseline_chromium_webkit_tests.Rebaseliner( host_port_obj, target_port_obj, platform, options) + + def test_noop(self): + # this method tests that was can at least instantiate an object, even + # if there is nothing to do. + rebaseliner = self.make_rebaseliner() self.assertNotEqual(rebaseliner, None) + def test_diff_baselines_txt(self): + rebaseliner = self.make_rebaseliner() + path = os.path.join(rebaseliner._port.layout_tests_dir(), + "passes", "text-expected.txt") + self.assertFalse(rebaseliner._diff_baselines(path, path)) + + def test_diff_baselines_png(self): + rebaseliner = self.make_rebaseliner() + path = os.path.join(rebaseliner._port.layout_tests_dir(), + "passes", "image-expected.png") + self.assertFalse(rebaseliner._diff_baselines(path, path)) + if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index b26bc6c..7163e1b 100755 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -687,7 +687,7 @@ class TestRunner: self.update_summary(result_summary) if some_thread_is_alive: - time.sleep(0.1) + time.sleep(0.01) except KeyboardInterrupt: keyboard_interrupted = True @@ -779,12 +779,13 @@ class TestRunner: self._expectations, result_summary, retry_summary) self._printer.print_unexpected_results(unexpected_results) - # Write the same data to log files. - self._write_json_files(unexpected_results, result_summary, - individual_test_timings) + if self._options.record_results: + # Write the same data to log files. + self._write_json_files(unexpected_results, result_summary, + individual_test_timings) - # Upload generated JSON files to appengine server. - self._upload_json_files() + # Upload generated JSON files to appengine server. + self._upload_json_files() # Write the summary to disk (results.html) and display it if requested. wrote_results = self._write_results_html_file(result_summary) @@ -1545,6 +1546,9 @@ def parse_args(args=None): default=False, help="Clobbers test results from previous runs."), optparse.make_option("--platform", help="Override the platform for expected results"), + optparse.make_option("--no-record-results", action="store_false", + default=True, dest="record_results", + help="Don't record the results."), # old-run-webkit-tests also has HTTP toggle options: # --[no-]http Run (or do not run) http tests # (default: run) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py index e1b3746..3a3b14e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py @@ -48,16 +48,25 @@ from webkitpy.layout_tests.layout_package import dump_render_tree_thread from webkitpy.thirdparty.mock import Mock -def passing_run(args, port_obj=None, logging_included=False): - if not logging_included: - args.extend(['--print', 'nothing']) +def passing_run(args, port_obj=None, record_results=False, + tests_included=False): + args.extend(['--print', 'nothing']) + if not tests_included: + # We use the glob to test that globbing works. + args.extend(['passes', 'failures/expected/*']) + if not record_results: + args.append('--no-record-results') options, args = run_webkit_tests.parse_args(args) if port_obj is None: port_obj = port.get(options.platform, options) res = run_webkit_tests.run(port_obj, options, args) return res == 0 -def logging_run(args): + +def logging_run(args, tests_included=False): + args.extend(['--no-record-results']) + if not tests_included: + args.extend(['passes', 'failures/expected/*']) options, args = run_webkit_tests.parse_args(args) port_obj = port.get(options.platform, options) buildbot_output = array_stream.ArrayStream() @@ -73,18 +82,21 @@ class MainTest(unittest.TestCase): self.assertTrue(passing_run(['--platform', 'test'])) self.assertTrue(passing_run(['--platform', 'test', '--run-singly'])) self.assertTrue(passing_run(['--platform', 'test', - 'text/article-element.html'])) - self.assertTrue(passing_run(['--platform', 'test', - '--child-processes', '1', - '--print', 'unexpected'])) + 'passes/text.html'], tests_included=True)) - def test_child_processes(self): + def test_unexpected_failures(self): + # Run tests including the unexpected failures. + self.assertFalse(passing_run(['--platform', 'test'], + tests_included=True)) + + def test_one_child_process(self): (res, buildbot_output, regular_output) = logging_run( ['--platform', 'test', '--print', 'config', '--child-processes', '1']) self.assertTrue('Running one DumpRenderTree\n' in regular_output.get()) + def test_two_child_processes(self): (res, buildbot_output, regular_output) = logging_run( ['--platform', 'test', '--print', 'config', '--child-processes', '2']) @@ -92,13 +104,19 @@ class MainTest(unittest.TestCase): in regular_output.get()) def test_last_results(self): - passing_run(['--platform', 'test']) + passing_run(['--platform', 'test'], record_results=True) (res, buildbot_output, regular_output) = logging_run( ['--platform', 'test', '--print-last-failures']) self.assertEqual(regular_output.get(), ['\n\n']) self.assertEqual(buildbot_output.get(), []) - + def test_no_tests_found(self): + self.assertRaises(SystemExit, logging_run, + ['--platform', 'test', 'resources'], + tests_included=True) + self.assertRaises(SystemExit, logging_run, + ['--platform', 'test', 'foo'], + tests_included=True) def _mocked_open(original_open, file_list): def _wrapper(name, mode, encoding): @@ -123,21 +141,19 @@ class RebaselineTest(unittest.TestCase): original_open = codecs.open try: # Test that we update expectations in place. If the expectation - # is mssing, update the expected generic location. + # is missing, update the expected generic location. file_list = [] codecs.open = _mocked_open(original_open, file_list) passing_run(['--platform', 'test', '--pixel-tests', '--reset-results', - 'image/canvas-bg.html', - 'image/canvas-zoom.html', - 'misc/missing-expectation.html']) - self.assertEqual(len(file_list), 9) + 'passes/image.html', + 'failures/expected/missing_image.html'], + tests_included=True) + self.assertEqual(len(file_list), 6) self.assertBaselines(file_list, - "data/image/canvas-zoom") + "data/passes/image") self.assertBaselines(file_list, - "data/platform/test/image/canvas-bg") - self.assertBaselines(file_list, - "data/misc/missing-expectation") + "data/failures/expected/missing_image") finally: codecs.open = original_open @@ -151,16 +167,14 @@ class RebaselineTest(unittest.TestCase): codecs.open = _mocked_open(original_open, file_list) passing_run(['--platform', 'test', '--pixel-tests', '--new-baseline', - 'image/canvas-zoom.html', - 'image/canvas-bg.html', - 'misc/missing-expectation.html']) - self.assertEqual(len(file_list), 9) - self.assertBaselines(file_list, - "data/platform/test/image/canvas-zoom") + 'passes/image.html', + 'failures/expected/missing_image.html'], + tests_included=True) + self.assertEqual(len(file_list), 6) self.assertBaselines(file_list, - "data/platform/test/image/canvas-bg") + "data/platform/test/passes/image") self.assertBaselines(file_list, - "data/platform/test/misc/missing-expectation") + "data/platform/test/failures/expected/missing_image") finally: codecs.open = original_open @@ -186,13 +200,14 @@ class TestRunnerTest(unittest.TestCase): class DryrunTest(unittest.TestCase): - def test_basics(self): - # FIXME: it's hard to know which platforms are safe to test; the - # chromium platforms require a chromium checkout, and the mac platform - # requires fcntl, so it can't be tested on win32, etc. There is - # probably a better way of handling this. - if sys.platform != "mac": + # FIXME: it's hard to know which platforms are safe to test; the + # chromium platforms require a chromium checkout, and the mac platform + # requires fcntl, so it can't be tested on win32, etc. There is + # probably a better way of handling this. + def test_darwin(self): + if sys.platform != "darwin": return + self.assertTrue(passing_run(['--platform', 'dryrun', 'fast/html'])) self.assertTrue(passing_run(['--platform', 'dryrun-mac', diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py index c9f4107..879646c 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py @@ -66,12 +66,8 @@ class ImageDiff(test_type_base.TestTypeBase): self._make_output_directory(test_filename) dest_image = self.output_filename(test_filename, extension) - try: + if os.path.exists(source_image): shutil.copyfile(source_image, dest_image) - except IOError, e: - # A missing expected PNG has already been recorded as an error. - if errno.ENOENT != e.errno: - raise def _save_baseline_files(self, filename, png_path, checksum, generate_new_baseline): @@ -107,20 +103,8 @@ class ImageDiff(test_type_base.TestTypeBase): expected_filename = self.output_filename(filename, self.FILENAME_SUFFIX_EXPECTED + '.png') - result = True - try: - _compare_available = True - result = port.diff_image(expected_filename, actual_filename, - diff_filename) - except ValueError: - _compare_available = False - - global _compare_msg_printed - if not _compare_available and not _compare_msg_printed: - _compare_msg_printed = True - print('image_diff not found. Make sure you have a ' + - configuration + ' build of the image_diff executable.') - + result = port.diff_image(expected_filename, actual_filename, + diff_filename) return result def compare_output(self, port, filename, output, test_args, configuration): @@ -145,14 +129,10 @@ class ImageDiff(test_type_base.TestTypeBase): expected_png_file = self._port.expected_filename(filename, '.png') # FIXME: We repeat this pattern often, we should share code. - try: + expected_hash = '' + if os.path.exists(expected_hash_file): with codecs.open(expected_hash_file, "r", "ascii") as file: expected_hash = file.read() - except IOError, e: - if errno.ENOENT != e.errno: - raise - expected_hash = '' - if not os.path.isfile(expected_png_file): # Report a missing expected PNG file. @@ -161,7 +141,7 @@ class ImageDiff(test_type_base.TestTypeBase): encoding="ascii", print_text_diffs=False) self._copy_output_png(filename, test_args.png_path, '-actual.png') - failures.append(test_failures.FailureMissingImage(self)) + failures.append(test_failures.FailureMissingImage()) return failures elif test_args.hash == expected_hash: # Hash matched (no diff needed, okay to return). @@ -178,26 +158,11 @@ class ImageDiff(test_type_base.TestTypeBase): # still need to call CreateImageDiff for other codepaths. images_are_different = self._create_image_diff(port, filename, configuration) if expected_hash == '': - failures.append(test_failures.FailureMissingImageHash(self)) + failures.append(test_failures.FailureMissingImageHash()) elif test_args.hash != expected_hash: if images_are_different: - failures.append(test_failures.FailureImageHashMismatch(self)) + failures.append(test_failures.FailureImageHashMismatch()) else: - failures.append(test_failures.FailureImageHashIncorrect(self)) + failures.append(test_failures.FailureImageHashIncorrect()) return failures - - def diff_files(self, port, file1, file2): - """Diff two image files exactly. - - Args: - file1, file2: full paths of the files to compare. - - Returns: - True if two files are different. - False otherwise. - """ - try: - return port.diff_image(file1, file2, None, 0) - except ValueError, e: - return True diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py index dd44642..753dbee 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py @@ -156,7 +156,7 @@ class TestTypeBase(object): Return: a list of TestFailure objects, empty if the test passes """ - raise NotImplemented + raise NotImplementedError def _write_into_file_at_path(self, file_path, contents, encoding): """This method assumes that byte_array is already encoded diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py new file mode 100644 index 0000000..5dbfcb6 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py @@ -0,0 +1,47 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""""Tests stray tests not covered by regular code paths.""" + +import test_type_base +import unittest + +from webkitpy.thirdparty.mock import Mock + + +class Test(unittest.TestCase): + + def test_compare_output_notimplemented(self): + test_type = test_type_base.TestTypeBase(None, None) + self.assertRaises(NotImplementedError, test_type.compare_output, + None, "foo.txt", '', + test_type_base.TestArguments(), 'Debug') + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py index d06ec8d..50a9995 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py @@ -65,18 +65,16 @@ class TestTextDiff(test_type_base.TestTypeBase): def _get_normalized_text(self, filename): # FIXME: We repeat this pattern often, we should share code. - try: - # NOTE: -expected.txt files are ALWAYS utf-8. However, - # we do not decode the output from DRT, so we should not - # decode the -expected.txt values either to allow comparisons. - with codecs.open(filename, "r", encoding=None) as file: - text = file.read() - # We could assert that the text is valid utf-8. - except IOError, e: - if errno.ENOENT != e.errno: - raise + if not os.path.exists(filename): return '' + # NOTE: -expected.txt files are ALWAYS utf-8. However, + # we do not decode the output from DRT, so we should not + # decode the -expected.txt values either to allow comparisons. + with codecs.open(filename, "r", encoding=None) as file: + text = file.read() + # We could assert that the text is valid utf-8. + # Normalize line endings return text.strip("\r\n").replace("\r\n", "\n") + "\n" @@ -106,22 +104,8 @@ class TestTextDiff(test_type_base.TestTypeBase): print_text_diffs=True) if expected == '': - failures.append(test_failures.FailureMissingResult(self)) + failures.append(test_failures.FailureMissingResult()) else: - failures.append(test_failures.FailureTextMismatch(self, True)) + failures.append(test_failures.FailureTextMismatch(True)) return failures - - def diff_files(self, port, file1, file2): - """Diff two text files. - - Args: - file1, file2: full paths of the files to compare. - - Returns: - True if two files are different. - False otherwise. - """ - - return port.compare_text(self._get_normalized_text(file1), - self._get_normalized_text(file2)) |
