summaryrefslogtreecommitdiffstats
path: root/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py
blob: 2b25e296f473d0927774034943fee178d177fa5b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
#!/usr/bin/env python
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the Google name nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Abstract base class of Port-specific entrypoints for the layout tests
test infrastructure (the Port and Driver classes)."""

import cgi
import difflib
import errno
import os
import subprocess
import sys

import apache_http_server
import http_server
import websocket_server

# Python bug workaround.  See Port.wdiff_text() for an explanation.
_wdiff_available = True


# FIXME: This class should merge with webkitpy.webkit_port at some point.
class Port(object):
    """Abstract class for Port-specific hooks for the layout_test package.
    """

    def __init__(self, port_name=None, options=None):
        self._name = port_name
        self._options = options
        self._helper = None
        self._http_server = None
        self._webkit_base_dir = None
        self._websocket_server = None

    def baseline_path(self):
        """Return the absolute path to the directory to store new baselines
        in for this port."""
        raise NotImplementedError('Port.baseline_path')

    def baseline_search_path(self):
        """Return a list of absolute paths to directories to search under for
        baselines. The directories are searched in order."""
        raise NotImplementedError('Port.baseline_search_path')

    def check_sys_deps(self):
        """If the port needs to do some runtime checks to ensure that the
        tests can be run successfully, they should be done here.

        Returns whether the system is properly configured."""
        raise NotImplementedError('Port.check_sys_deps')

    def compare_text(self, actual_text, expected_text):
        """Return whether or not the two strings are *not* equal. This
        routine is used to diff text output.

        While this is a generic routine, we include it in the Port
        interface so that it can be overriden for testing purposes."""
        return actual_text != expected_text

    def diff_image(self, actual_filename, expected_filename,
                   diff_filename=None):
        """Compare two image files and produce a delta image file.

        Return 1 if the two files are different, 0 if they are the same.
        Also produce a delta image of the two images and write that into
        |diff_filename| if it is not None.

        While this is a generic routine, we include it in the Port
        interface so that it can be overriden for testing purposes."""
        executable = self._path_to_image_diff()
        cmd = [executable, '--diff', actual_filename, expected_filename]
        if diff_filename:
            cmd.append(diff_filename)
        result = 1
        try:
            result = subprocess.call(cmd)
        except OSError, e:
            if e.errno == errno.ENOENT or e.errno == errno.EACCES:
                _compare_available = False
            else:
                raise e
        except ValueError:
            # work around a race condition in Python 2.4's implementation
            # of subprocess.Popen. See http://bugs.python.org/issue1199282 .
            pass
        return result

    def diff_text(self, actual_text, expected_text,
                  actual_filename, expected_filename):
        """Returns a string containing the diff of the two text strings
        in 'unified diff' format.

        While this is a generic routine, we include it in the Port
        interface so that it can be overriden for testing purposes."""
        diff = difflib.unified_diff(expected_text.splitlines(True),
                                    actual_text.splitlines(True),
                                    expected_filename,
                                    actual_filename)
        return ''.join(diff)

    def expected_baselines(self, filename, suffix, all_baselines=False):
        """Given a test name, finds where the baseline results are located.

        Args:
        filename: absolute filename to test file
        suffix: file suffix of the expected results, including dot; e.g.
            '.txt' or '.png'.  This should not be None, but may be an empty
            string.
        all_baselines: If True, return an ordered list of all baseline paths
            for the given platform. If False, return only the first one.
        Returns
        a list of ( platform_dir, results_filename ), where
            platform_dir - abs path to the top of the results tree (or test
                tree)
            results_filename - relative path from top of tree to the results
                file
            (os.path.join of the two gives you the full path to the file,
                unless None was returned.)
        Return values will be in the format appropriate for the current
        platform (e.g., "\\" for path separators on Windows). If the results
        file is not found, then None will be returned for the directory,
        but the expected relative pathname will still be returned.

        This routine is generic but lives here since it is used in
        conjunction with the other baseline and filename routines that are
        platform specific.
        """
        testname = os.path.splitext(self.relative_test_filename(filename))[0]

        baseline_filename = testname + '-expected' + suffix

        baseline_search_path = self.baseline_search_path()

        baselines = []
        for platform_dir in baseline_search_path:
            if os.path.exists(os.path.join(platform_dir, baseline_filename)):
                baselines.append((platform_dir, baseline_filename))

            if not all_baselines and baselines:
                return baselines

        # If it wasn't found in a platform directory, return the expected
        # result in the test directory, even if no such file actually exists.
        platform_dir = self.layout_tests_dir()
        if os.path.exists(os.path.join(platform_dir, baseline_filename)):
            baselines.append((platform_dir, baseline_filename))

        if baselines:
            return baselines

        return [(None, baseline_filename)]

    def expected_filename(self, filename, suffix):
        """Given a test name, returns an absolute path to its expected results.

        If no expected results are found in any of the searched directories,
        the directory in which the test itself is located will be returned.
        The return value is in the format appropriate for the platform
        (e.g., "\\" for path separators on windows).

        Args:
        filename: absolute filename to test file
        suffix: file suffix of the expected results, including dot; e.g. '.txt'
            or '.png'.  This should not be None, but may be an empty string.
        platform: the most-specific directory name to use to build the
            search list of directories, e.g., 'chromium-win', or
            'chromium-mac-leopard' (we follow the WebKit format)

        This routine is generic but is implemented here to live alongside
        the other baseline and filename manipulation routines.
        """
        platform_dir, baseline_filename = self.expected_baselines(
            filename, suffix)[0]
        if platform_dir:
            return os.path.join(platform_dir, baseline_filename)
        return os.path.join(self.layout_tests_dir(), baseline_filename)

    def filename_to_uri(self, filename):
        """Convert a test file to a URI."""
        LAYOUTTEST_HTTP_DIR = "http/tests/"
        LAYOUTTEST_WEBSOCKET_DIR = "websocket/tests/"

        relative_path = self.relative_test_filename(filename)
        port = None
        use_ssl = False

        if relative_path.startswith(LAYOUTTEST_HTTP_DIR):
            # http/tests/ run off port 8000 and ssl/ off 8443
            relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
            port = 8000
        elif relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR):
            # websocket/tests/ run off port 8880 and 9323
            # Note: the root is /, not websocket/tests/
            port = 8880

        # Make http/tests/local run as local files. This is to mimic the
        # logic in run-webkit-tests.
        #
        # TODO(dpranke): remove the media reference and the SSL reference?
        if (port and not relative_path.startswith("local/") and
            not relative_path.startswith("media/")):
            if relative_path.startswith("ssl/"):
                port += 443
                protocol = "https"
            else:
                protocol = "http"
            return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)

        if sys.platform in ('cygwin', 'win32'):
            return "file:///" + self.get_absolute_path(filename)
        return "file://" + self.get_absolute_path(filename)

    def get_absolute_path(self, filename):
        """Return the absolute path in unix format for the given filename.

        This routine exists so that platforms that don't use unix filenames
        can convert accordingly."""
        return os.path.abspath(filename)

    def layout_tests_dir(self):
        """Return the absolute path to the top of the LayoutTests directory."""
        return self.path_from_webkit_base('LayoutTests')

    def maybe_make_directory(self, *path):
        """Creates the specified directory if it doesn't already exist."""
        try:
            os.makedirs(os.path.join(*path))
        except OSError, e:
            if e.errno != errno.EEXIST:
                raise

    def name(self):
        """Return the name of the port (e.g., 'mac', 'chromium-win-xp').

        Note that this is different from the test_platform_name(), which
        may be different (e.g., 'win-xp' instead of 'chromium-win-xp'."""
        return self._name

    def num_cores(self):
        """Return the number of cores/cpus available on this machine.

        This routine is used to determine the default amount of parallelism
        used by run-chromium-webkit-tests."""
        raise NotImplementedError('Port.num_cores')

    # FIXME: This could be replaced by functions in webkitpy.scm.
    def path_from_webkit_base(self, *comps):
        """Returns the full path to path made by joining the top of the
        WebKit source tree and the list of path components in |*comps|."""
        if not self._webkit_base_dir:
            abspath = os.path.abspath(__file__)
            self._webkit_base_dir = abspath[0:abspath.find('WebKitTools')]
        return os.path.join(self._webkit_base_dir, *comps)

    # FIXME: Callers should eventually move to scm.script_path.
    def script_path(self, script_name):
        return self.path_from_webkit_base("WebKitTools", "Scripts", script_name)

    def path_to_test_expectations_file(self):
        """Update the test expectations to the passed-in string.

        This is used by the rebaselining tool. Raises NotImplementedError
        if the port does not use expectations files."""
        raise NotImplementedError('Port.path_to_test_expectations_file')
 
    def remove_directory(self, *path):
        """Recursively removes a directory, even if it's marked read-only.

        Remove the directory located at *path, if it exists.

        shutil.rmtree() doesn't work on Windows if any of the files
        or directories are read-only, which svn repositories and
        some .svn files are.  We need to be able to force the files
        to be writable (i.e., deletable) as we traverse the tree.

        Even with all this, Windows still sometimes fails to delete a file,
        citing a permission error (maybe something to do with antivirus
        scans or disk indexing).  The best suggestion any of the user
        forums had was to wait a bit and try again, so we do that too.
        It's hand-waving, but sometimes it works. :/
        """
        file_path = os.path.join(*path)
        if not os.path.exists(file_path):
            return

        win32 = False
        if sys.platform == 'win32':
            win32 = True
            # Some people don't have the APIs installed. In that case we'll do
            # without.
            try:
                win32api = __import__('win32api')
                win32con = __import__('win32con')
            except ImportError:
                win32 = False

            def remove_with_retry(rmfunc, path):
                os.chmod(path, stat.S_IWRITE)
                if win32:
                    win32api.SetFileAttributes(path,
                                              win32con.FILE_ATTRIBUTE_NORMAL)
                try:
                    return rmfunc(path)
                except EnvironmentError, e:
                    if e.errno != errno.EACCES:
                        raise
                    print 'Failed to delete %s: trying again' % repr(path)
                    time.sleep(0.1)
                    return rmfunc(path)
        else:

            def remove_with_retry(rmfunc, path):
                if os.path.islink(path):
                    return os.remove(path)
                else:
                    return rmfunc(path)

        for root, dirs, files in os.walk(file_path, topdown=False):
            # For POSIX:  making the directory writable guarantees
            # removability. Windows will ignore the non-read-only
            # bits in the chmod value.
            os.chmod(root, 0770)
            for name in files:
                remove_with_retry(os.remove, os.path.join(root, name))
            for name in dirs:
                remove_with_retry(os.rmdir, os.path.join(root, name))

        remove_with_retry(os.rmdir, file_path)

    def test_platform_name(self):
        return self._name

    def relative_test_filename(self, filename):
        """Relative unix-style path for a filename under the LayoutTests
        directory. Filenames outside the LayoutTests directory should raise
        an error."""
        return filename[len(self.layout_tests_dir()) + 1:]

    def results_directory(self):
        """Absolute path to the place to store the test results."""
        raise NotImplemented('Port.results_directory')

    def setup_test_run(self):
        """This routine can be overridden to perform any port-specific
        work that shouuld be done at the beginning of a test run."""
        pass

    def show_html_results_file(self, results_filename):
        """This routine should display the HTML file pointed at by
        results_filename in a users' browser."""
        raise NotImplementedError('Port.show_html_results_file')

    def start_driver(self, png_path, options):
        """Starts a new test Driver and returns a handle to the object."""
        raise NotImplementedError('Port.start_driver')

    def start_helper(self):
        """Start a layout test helper if needed on this port. The test helper
        is used to reconfigure graphics settings and do other things that
        may be necessary to ensure a known test configuration."""
        raise NotImplementedError('Port.start_helper')

    def start_http_server(self):
        """Start a web server if it is available. Do nothing if
        it isn't. This routine is allowed to (and may) fail if a server
        is already running."""
        if self._options.use_apache:
            self._http_server = apache_http_server.LayoutTestApacheHttpd(self,
                self._options.results_directory)
        else:
            self._http_server = http_server.Lighttpd(self,
                self._options.results_directory)
        self._http_server.start()

    def start_websocket_server(self):
        """Start a websocket server if it is available. Do nothing if
        it isn't. This routine is allowed to (and may) fail if a server
        is already running."""
        self._websocket_server = websocket_server.PyWebSocket(self,
            self._options.results_directory)
        self._websocket_server.start()

    def stop_helper(self):
        """Shut down the test helper if it is running. Do nothing if
        it isn't, or it isn't available."""
        raise NotImplementedError('Port.stop_helper')

    def stop_http_server(self):
        """Shut down the http server if it is running. Do nothing if
        it isn't, or it isn't available."""
        if self._http_server:
            self._http_server.stop()

    def stop_websocket_server(self):
        """Shut down the websocket server if it is running. Do nothing if
        it isn't, or it isn't available."""
        if self._websocket_server:
            self._websocket_server.stop()

    def test_expectations(self):
        """Returns the test expectations for this port.

        Basically this string should contain the equivalent of a
        test_expectations file. See test_expectations.py for more details."""
        raise NotImplementedError('Port.test_expectations')

    def test_base_platform_names(self):
        """Return a list of the 'base' platforms on your port. The base
        platforms represent different architectures, operating systems,
        or implementations (as opposed to different versions of a single
        platform). For example, 'mac' and 'win' might be different base
        platforms, wherease 'mac-tiger' and 'mac-leopard' might be
        different platforms. This routine is used by the rebaselining tool
        and the dashboards, and the strings correspond to the identifiers
        in your test expectations (*not* necessarily the platform names
        themselves)."""
        raise NotImplementedError('Port.base_test_platforms')

    def test_platform_name(self):
        """Returns the string that corresponds to the given platform name
        in the test expectations. This may be the same as name(), or it
        may be different. For example, chromium returns 'mac' for
        'chromium-mac'."""
        raise NotImplementedError('Port.test_platform_name')

    def test_platforms(self):
        """Returns the list of test platform identifiers as used in the
        test_expectations and on dashboards, the rebaselining tool, etc.

        Note that this is not necessarily the same as the list of ports,
        which must be globally unique (e.g., both 'chromium-mac' and 'mac'
        might return 'mac' as a test_platform name'."""
        raise NotImplementedError('Port.platforms')

    def version(self):
        """Returns a string indicating the version of a given platform, e.g.
        '-leopard' or '-xp'.

        This is used to help identify the exact port when parsing test
        expectations, determining search paths, and logging information."""
        raise NotImplementedError('Port.version')

    def wdiff_text(self, actual_filename, expected_filename):
        """Returns a string of HTML indicating the word-level diff of the
        contents of the two filenames. Returns an empty string if word-level
        diffing isn't available."""
        executable = self._path_to_wdiff()
        cmd = [executable,
               '--start-delete=##WDIFF_DEL##',
               '--end-delete=##WDIFF_END##',
               '--start-insert=##WDIFF_ADD##',
               '--end-insert=##WDIFF_END##',
               expected_filename,
               actual_filename]
        global _wdiff_available
        result = ''
        try:
            # Python's Popen has a bug that causes any pipes opened to a
            # process that can't be executed to be leaked.  Since this
            # code is specifically designed to tolerate exec failures
            # to gracefully handle cases where wdiff is not installed,
            # the bug results in a massive file descriptor leak. As a
            # workaround, if an exec failure is ever experienced for
            # wdiff, assume it's not available.  This will leak one
            # file descriptor but that's better than leaking each time
            # wdiff would be run.
            #
            # http://mail.python.org/pipermail/python-list/
            #    2008-August/505753.html
            # http://bugs.python.org/issue3210
            #
            # It also has a threading bug, so we don't output wdiff if
            # the Popen raises a ValueError.
            # http://bugs.python.org/issue1236
            if _wdiff_available:
                try:
                    wdiff = subprocess.Popen(cmd,
                        stdout=subprocess.PIPE).communicate()[0]
                except ValueError, e:
                    # Working around a race in Python 2.4's implementation
                    # of Popen().
                    wdiff = ''
                wdiff = cgi.escape(wdiff)
                wdiff = wdiff.replace('##WDIFF_DEL##', '<span class=del>')
                wdiff = wdiff.replace('##WDIFF_ADD##', '<span class=add>')
                wdiff = wdiff.replace('##WDIFF_END##', '</span>')
                result = '<head><style>.del { background: #faa; } '
                result += '.add { background: #afa; }</style></head>'
                result += '<pre>' + wdiff + '</pre>'
        except OSError, e:
            if (e.errno == errno.ENOENT or e.errno == errno.EACCES or
                e.errno == errno.ECHILD):
                _wdiff_available = False
            else:
                raise e
        return result

    #
    # PROTECTED ROUTINES
    #
    # The routines below should only be called by routines in this class
    # or any of its subclasses.
    #

    def _kill_process(self, pid):
        """Forcefully kill a process.

        This routine should not be used or needed generically, but can be
        used in helper files like http_server.py."""
        raise NotImplementedError('Port.kill_process')

    def _path_to_apache(self):
        """Returns the full path to the apache binary.

        This is needed only by ports that use the apache_http_server module."""
        raise NotImplementedError('Port.path_to_apache')

    def _path_to_apache_config_file(self):
        """Returns the full path to the apache binary.

        This is needed only by ports that use the apache_http_server module."""
        raise NotImplementedError('Port.path_to_apache_config_file')

    def _path_to_driver(self):
        """Returns the full path to the test driver (DumpRenderTree)."""
        raise NotImplementedError('Port.path_to_driver')

    def _path_to_helper(self):
        """Returns the full path to the layout_test_helper binary, which
        is used to help configure the system for the test run, or None
        if no helper is needed.

        This is likely only used by start/stop_helper()."""
        raise NotImplementedError('Port._path_to_helper')

    def _path_to_image_diff(self):
        """Returns the full path to the image_diff binary, or None if it
        is not available.

        This is likely used only by diff_image()"""
        raise NotImplementedError('Port.path_to_image_diff')

    def _path_to_lighttpd(self):
        """Returns the path to the LigHTTPd binary.

        This is needed only by ports that use the http_server.py module."""
        raise NotImplementedError('Port._path_to_lighttpd')

    def _path_to_lighttpd_modules(self):
        """Returns the path to the LigHTTPd modules directory.

        This is needed only by ports that use the http_server.py module."""
        raise NotImplementedError('Port._path_to_lighttpd_modules')

    def _path_to_lighttpd_php(self):
        """Returns the path to the LigHTTPd PHP executable.

        This is needed only by ports that use the http_server.py module."""
        raise NotImplementedError('Port._path_to_lighttpd_php')

    def _path_to_wdiff(self):
        """Returns the full path to the wdiff binary, or None if it is
        not available.

        This is likely used only by wdiff_text()"""
        raise NotImplementedError('Port._path_to_wdiff')

    def _shut_down_http_server(self, pid):
        """Forcefully and synchronously kills the web server.

        This routine should only be called from http_server.py or its
        subclasses."""
        raise NotImplementedError('Port._shut_down_http_server')

    def _webkit_baseline_path(self, platform):
        """Return the  full path to the top of the baseline tree for a
        given platform."""
        return os.path.join(self.layout_tests_dir(), 'platform',
                            platform)


class Driver:
    """Abstract interface for the DumpRenderTree interface."""

    def __init__(self, port, png_path, options):
        """Initialize a Driver to subsequently run tests.

        Typically this routine will spawn DumpRenderTree in a config
        ready for subsequent input.

        port - reference back to the port object.
        png_path - an absolute path for the driver to write any image
            data for a test (as a PNG). If no path is provided, that
            indicates that pixel test results will not be checked.
        options - any port-specific driver options."""
        raise NotImplementedError('Driver.__init__')

    def run_test(self, uri, timeout, checksum):
        """Run a single test and return the results.

        Note that it is okay if a test times out or crashes and leaves
        the driver in an indeterminate state. The upper layers of the program
        are responsible for cleaning up and ensuring things are okay.

        uri - a full URI for the given test
        timeout - number of milliseconds to wait before aborting this test.
        checksum - if present, the expected checksum for the image for this
            test

        Returns a tuple of the following:
            crash - a boolean indicating whether the driver crashed on the test
            timeout - a boolean indicating whehter the test timed out
            checksum - a string containing the checksum of the image, if
                present
            output - any text output
            error - any unexpected or additional (or error) text output

        Note that the image itself should be written to the path that was
        specified in the __init__() call."""
        raise NotImplementedError('Driver.run_test')

    def poll(self):
        """Returns None if the Driver is still running. Returns the returncode
        if it has exited."""
        raise NotImplementedError('Driver.poll')

    def returncode(self):
        """Returns the system-specific returncode if the Driver has stopped or
        exited."""
        raise NotImplementedError('Driver.returncode')

    def stop(self):
        raise NotImplementedError('Driver.stop')