1 files changed, 240 insertions, 0 deletions
diff --git a/src/util/all_tests.go b/src/util/all_tests.go
new file mode 100644
index 0000000..91822d1
--- /dev/null
+++ b/src/util/all_tests.go
@@ -0,0 +1,240 @@
+/* Copyright (c) 2015, Google Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"time"
+)
+
+// TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
+
+var (
+	useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
+	buildDir    = flag.String("build-dir", "build", "The build directory to run the tests from.")
+	jsonOutput  = flag.String("json-output", "", "The file to output JSON results to.")
+)
+
+type test []string
+
+var tests = []test{
+	{"crypto/base64/base64_test"},
+	{"crypto/bio/bio_test"},
+	{"crypto/bn/bn_test"},
+	{"crypto/bytestring/bytestring_test"},
+	{"crypto/cipher/aead_test", "aes-128-gcm", "crypto/cipher/test/aes_128_gcm_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-128-key-wrap", "crypto/cipher/test/aes_128_key_wrap_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-256-gcm", "crypto/cipher/test/aes_256_gcm_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-256-key-wrap", "crypto/cipher/test/aes_256_key_wrap_tests.txt"},
+	{"crypto/cipher/aead_test", "chacha20-poly1305", "crypto/cipher/test/chacha20_poly1305_tests.txt"},
+	{"crypto/cipher/aead_test", "rc4-md5-tls", "crypto/cipher/test/rc4_md5_tls_tests.txt"},
+	{"crypto/cipher/aead_test", "rc4-sha1-tls", "crypto/cipher/test/rc4_sha1_tls_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls", "crypto/cipher/test/aes_128_cbc_sha1_tls_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_128_cbc_sha1_tls_implicit_iv_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-128-cbc-sha256-tls", "crypto/cipher/test/aes_128_cbc_sha256_tls_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls", "crypto/cipher/test/aes_256_cbc_sha1_tls_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_256_cbc_sha1_tls_implicit_iv_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-256-cbc-sha256-tls", "crypto/cipher/test/aes_256_cbc_sha256_tls_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-256-cbc-sha384-tls", "crypto/cipher/test/aes_256_cbc_sha384_tls_tests.txt"},
+	{"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls", "crypto/cipher/test/des_ede3_cbc_sha1_tls_tests.txt"},
+	{"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/des_ede3_cbc_sha1_tls_implicit_iv_tests.txt"},
+	{"crypto/cipher/aead_test", "rc4-md5-ssl3", "crypto/cipher/test/rc4_md5_ssl3_tests.txt"},
+	{"crypto/cipher/aead_test", "rc4-sha1-ssl3", "crypto/cipher/test/rc4_sha1_ssl3_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-128-cbc-sha1-ssl3", "crypto/cipher/test/aes_128_cbc_sha1_ssl3_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-256-cbc-sha1-ssl3", "crypto/cipher/test/aes_256_cbc_sha1_ssl3_tests.txt"},
+	{"crypto/cipher/aead_test", "des-ede3-cbc-sha1-ssl3", "crypto/cipher/test/des_ede3_cbc_sha1_ssl3_tests.txt"},
+	{"crypto/cipher/aead_test", "aes-128-ctr-hmac-sha256", "crypto/cipher/test/aes_128_ctr_hmac_sha256.txt"},
+	{"crypto/cipher/aead_test", "aes-256-ctr-hmac-sha256", "crypto/cipher/test/aes_256_ctr_hmac_sha256.txt"},
+	{"crypto/cipher/cipher_test", "crypto/cipher/test/cipher_test.txt"},
+	{"crypto/cmac/cmac_test"},
+	{"crypto/constant_time_test"},
+	{"crypto/dh/dh_test"},
+	{"crypto/digest/digest_test"},
+	{"crypto/dsa/dsa_test"},
+	{"crypto/ec/ec_test"},
+	{"crypto/ec/example_mul"},
+	{"crypto/ecdsa/ecdsa_test"},
+	{"crypto/err/err_test"},
+	{"crypto/evp/evp_extra_test"},
+	{"crypto/evp/evp_test", "crypto/evp/evp_tests.txt"},
+	{"crypto/evp/evp_test", "crypto/hmac/hmac_tests.txt"},
+	{"crypto/evp/pbkdf_test"},
+	{"crypto/hkdf/hkdf_test"},
+	{"crypto/hmac/hmac_test", "crypto/hmac/hmac_tests.txt"},
+	{"crypto/lhash/lhash_test"},
+	{"crypto/modes/gcm_test"},
+	{"crypto/pkcs8/pkcs12_test"},
+	{"crypto/rsa/rsa_test"},
+	{"crypto/thread_test"},
+	{"crypto/x509/pkcs7_test"},
+	{"crypto/x509v3/tab_test"},
+	{"crypto/x509v3/v3name_test"},
+	{"ssl/pqueue/pqueue_test"},
+	{"ssl/ssl_test"},
+}
+
+// testOutput is a representation of Chromium's JSON test result format. See
+// https://www.chromium.org/developers/the-json-test-results-format
+type testOutput struct {
+	Version           int                   `json:"version"`
+	Interrupted       bool                  `json:"interrupted"`
+	PathDelimiter     string                `json:"path_delimiter"`
+	SecondsSinceEpoch float64               `json:"seconds_since_epoch"`
+	NumFailuresByType map[string]int        `json:"num_failures_by_type"`
+	Tests             map[string]testResult `json:"tests"`
+}
+
+type testResult struct {
+	Actual       string `json:"actual"`
+	Expected     string `json:"expected"`
+	IsUnexpected bool   `json:"is_unexpected"`
+}
+
+func newTestOutput() *testOutput {
+	return &testOutput{
+		Version:           3,
+		PathDelimiter:     ".",
+		SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
+		NumFailuresByType: make(map[string]int),
+		Tests:             make(map[string]testResult),
+	}
+}
+
+func (t *testOutput) addResult(name, result string) {
+	if _, found := t.Tests[name]; found {
+		panic(name)
+	}
+	t.Tests[name] = testResult{
+		Actual:       result,
+		Expected:     "PASS",
+		IsUnexpected: result != "PASS",
+	}
+	t.NumFailuresByType[result]++
+}
+
+func (t *testOutput) writeTo(name string) error {
+	file, err := os.Create(name)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	out, err := json.MarshalIndent(t, "", "  ")
+	if err != nil {
+		return err
+	}
+	_, err = file.Write(out)
+	return err
+}
+
+func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
+	valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"}
+	if dbAttach {
+		valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p")
+	}
+	valgrindArgs = append(valgrindArgs, path)
+	valgrindArgs = append(valgrindArgs, args...)
+
+	return exec.Command("valgrind", valgrindArgs...)
+}
+
+func runTest(test test) (passed bool, err error) {
+	prog := path.Join(*buildDir, test[0])
+	args := test[1:]
+	var cmd *exec.Cmd
+	if *useValgrind {
+		cmd = valgrindOf(false, prog, args...)
+	} else {
+		cmd = exec.Command(prog, args...)
+	}
+	var stdoutBuf bytes.Buffer
+	cmd.Stdout = &stdoutBuf
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		return false, err
+	}
+	if err := cmd.Wait(); err != nil {
+		return false, err
+	}
+
+	// Account for Windows line-endings.
+	stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
+
+	if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
+		(len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
+		return true, nil
+	}
+	return false, nil
+}
+
+// shortTestName returns the short name of a test. Except for evp_test, it
+// assumes that any argument which ends in .txt is a path to a data file and not
+// relevant to the test's uniqueness.
+func shortTestName(test test) string {
+	var args []string
+	for _, arg := range test {
+		if test[0] == "crypto/evp/evp_test" || !strings.HasSuffix(arg, ".txt") {
+			args = append(args, arg)
+		}
+	}
+	return strings.Join(args, " ")
+}
+
+func main() {
+	flag.Parse()
+
+	testOutput := newTestOutput()
+	var failed []test
+	for _, test := range tests {
+		fmt.Printf("%s\n", strings.Join([]string(test), " "))
+
+		name := shortTestName(test)
+		passed, err := runTest(test)
+		if err != nil {
+			fmt.Printf("%s failed to complete: %s\n", test[0], err)
+			failed = append(failed, test)
+			testOutput.addResult(name, "CRASHED")
+		} else if !passed {
+			fmt.Printf("%s failed to print PASS on the last line.\n", test[0])
+			failed = append(failed, test)
+			testOutput.addResult(name, "FAIL")
+		} else {
+			testOutput.addResult(name, "PASS")
+		}
+	}
+
+	if *jsonOutput != "" {
+		if err := testOutput.writeTo(*jsonOutput); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %s\n", err)
+		}
+	}
+
+	if len(failed) > 0 {
+		fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(tests))
+		for _, test := range failed {
+			fmt.Printf("\t%s\n", strings.Join([]string(test), " "))
+		}
+		os.Exit(1)
+	}
+
+	fmt.Printf("\nAll tests passed!\n")
+}