From 65ec521c3cd08bfbfb17334cc2c6668524cea68d Mon Sep 17 00:00:00 2001 From: Bill Wendling <isanbard@gmail.com> Date: Sun, 23 Oct 2011 20:14:06 +0000 Subject: Rename the script to indicate that this is for the TEST=simple tests. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@142764 91177308-0d34-0410-b5e6-96231b3b80d8 --- utils/release/findRegressions-simple.py | 158 ++++++++++++++++++++++++++++++++ utils/release/findRegressions.py | 158 -------------------------------- 2 files changed, 158 insertions(+), 158 deletions(-) create mode 100755 utils/release/findRegressions-simple.py delete mode 100755 utils/release/findRegressions.py (limited to 'utils/release') diff --git a/utils/release/findRegressions-simple.py b/utils/release/findRegressions-simple.py new file mode 100755 index 0000000..7586231 --- /dev/null +++ b/utils/release/findRegressions-simple.py @@ -0,0 +1,158 @@ +#!/usr/bin/python +import re, string, sys, os, time, math + +DEBUG = 0 + +(tp, exp) = ('compile', 'exec') + +def parse(file): + f = open(file, 'r') + d = f.read() + + # Cleanup weird stuff + d = re.sub(r',\d+:\d', '', d) + + r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d) + + test = {} + fname = '' + for t in r: + if DEBUG: + print t + + if t[0] == 'PASS' or t[0] == 'FAIL' : + tmp = t[2].split('llvm-test/') + + if DEBUG: + print tmp + + if len(tmp) == 2: + fname = tmp[1].strip('\r\n') + else: + fname = tmp[0].strip('\r\n') + + if not test.has_key(fname): + test[fname] = {} + + test[fname][t[1] + ' state'] = t[0] + test[fname][t[1] + ' time'] = float('nan') + else : + try: + n = t[0].split('RESULT-')[1] + + if DEBUG: + print "n == ", n; + + if n == 'compile-success': + test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n')) + + elif n == 'exec-success': + test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n')) + if DEBUG: + print test[fname][string.replace(n, '-success', '')] + + else : + # print "ERROR!" + sys.exit(1) + + except: + continue + + return test + +# Diff results and look for regressions. +def diffResults(d_old, d_new): + regressions = {} + passes = {} + removed = '' + + for x in ['compile state', 'compile time', 'exec state', 'exec time']: + regressions[x] = '' + passes[x] = '' + + for t in sorted(d_old.keys()) : + if d_new.has_key(t): + + # Check if the test passed or failed. + for x in ['compile state', 'compile time', 'exec state', 'exec time']: + + if not d_old[t].has_key(x) and not d_new[t].has_key(x): + continue + + if d_old[t].has_key(x): + if d_new[t].has_key(x): + + if d_old[t][x] == 'PASS': + if d_new[t][x] != 'PASS': + regressions[x] += t + "\n" + else: + if d_new[t][x] == 'PASS': + passes[x] += t + "\n" + + else : + regressions[x] += t + "\n" + + if x == 'compile state' or x == 'exec state': + continue + + # For execution time, if there is no result it's a fail. + if not d_old[t].has_key(x) and not d_new[t].has_key(x): + continue + elif not d_new[t].has_key(x): + regressions[x] += t + "\n" + elif not d_old[t].has_key(x): + passes[x] += t + "\n" + + if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): + continue + + elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]): + passes[x] += t + "\n" + + elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): + regressions[x] += t + ": NaN%\n" + + if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \ + (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05: + regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n" + + else : + removed += t + "\n" + + if len(regressions['compile state']) != 0: + print 'REGRESSION: Compilation Failed' + print regressions['compile state'] + + if len(regressions['exec state']) != 0: + print 'REGRESSION: Execution Failed' + print regressions['exec state'] + + if len(regressions['compile time']) != 0: + print 'REGRESSION: Compilation Time' + print regressions['compile time'] + + if len(regressions['exec time']) != 0: + print 'REGRESSION: Execution Time' + print regressions['exec time'] + + if len(passes['compile state']) != 0: + print 'NEW PASSES: Compilation' + print passes['compile state'] + + if len(passes['exec state']) != 0: + print 'NEW PASSES: Execution' + print passes['exec state'] + + if len(removed) != 0: + print 'REMOVED TESTS' + print removed + +# Main +if len(sys.argv) < 3 : + print 'Usage:', sys.argv[0], '<old log> <new log>' + sys.exit(-1) + +d_old = parse(sys.argv[1]) +d_new = parse(sys.argv[2]) + +diffResults(d_old, d_new) diff --git a/utils/release/findRegressions.py b/utils/release/findRegressions.py deleted file mode 100755 index 7586231..0000000 --- a/utils/release/findRegressions.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/python -import re, string, sys, os, time, math - -DEBUG = 0 - -(tp, exp) = ('compile', 'exec') - -def parse(file): - f = open(file, 'r') - d = f.read() - - # Cleanup weird stuff - d = re.sub(r',\d+:\d', '', d) - - r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d) - - test = {} - fname = '' - for t in r: - if DEBUG: - print t - - if t[0] == 'PASS' or t[0] == 'FAIL' : - tmp = t[2].split('llvm-test/') - - if DEBUG: - print tmp - - if len(tmp) == 2: - fname = tmp[1].strip('\r\n') - else: - fname = tmp[0].strip('\r\n') - - if not test.has_key(fname): - test[fname] = {} - - test[fname][t[1] + ' state'] = t[0] - test[fname][t[1] + ' time'] = float('nan') - else : - try: - n = t[0].split('RESULT-')[1] - - if DEBUG: - print "n == ", n; - - if n == 'compile-success': - test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n')) - - elif n == 'exec-success': - test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n')) - if DEBUG: - print test[fname][string.replace(n, '-success', '')] - - else : - # print "ERROR!" - sys.exit(1) - - except: - continue - - return test - -# Diff results and look for regressions. -def diffResults(d_old, d_new): - regressions = {} - passes = {} - removed = '' - - for x in ['compile state', 'compile time', 'exec state', 'exec time']: - regressions[x] = '' - passes[x] = '' - - for t in sorted(d_old.keys()) : - if d_new.has_key(t): - - # Check if the test passed or failed. - for x in ['compile state', 'compile time', 'exec state', 'exec time']: - - if not d_old[t].has_key(x) and not d_new[t].has_key(x): - continue - - if d_old[t].has_key(x): - if d_new[t].has_key(x): - - if d_old[t][x] == 'PASS': - if d_new[t][x] != 'PASS': - regressions[x] += t + "\n" - else: - if d_new[t][x] == 'PASS': - passes[x] += t + "\n" - - else : - regressions[x] += t + "\n" - - if x == 'compile state' or x == 'exec state': - continue - - # For execution time, if there is no result it's a fail. - if not d_old[t].has_key(x) and not d_new[t].has_key(x): - continue - elif not d_new[t].has_key(x): - regressions[x] += t + "\n" - elif not d_old[t].has_key(x): - passes[x] += t + "\n" - - if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): - continue - - elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]): - passes[x] += t + "\n" - - elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): - regressions[x] += t + ": NaN%\n" - - if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \ - (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05: - regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n" - - else : - removed += t + "\n" - - if len(regressions['compile state']) != 0: - print 'REGRESSION: Compilation Failed' - print regressions['compile state'] - - if len(regressions['exec state']) != 0: - print 'REGRESSION: Execution Failed' - print regressions['exec state'] - - if len(regressions['compile time']) != 0: - print 'REGRESSION: Compilation Time' - print regressions['compile time'] - - if len(regressions['exec time']) != 0: - print 'REGRESSION: Execution Time' - print regressions['exec time'] - - if len(passes['compile state']) != 0: - print 'NEW PASSES: Compilation' - print passes['compile state'] - - if len(passes['exec state']) != 0: - print 'NEW PASSES: Execution' - print passes['exec state'] - - if len(removed) != 0: - print 'REMOVED TESTS' - print removed - -# Main -if len(sys.argv) < 3 : - print 'Usage:', sys.argv[0], '<old log> <new log>' - sys.exit(-1) - -d_old = parse(sys.argv[1]) -d_new = parse(sys.argv[2]) - -diffResults(d_old, d_new) -- cgit v1.1