# findRegressions-simple.py   [plain text]

```#!/usr/bin/python
import re, string, sys, os, time, math

DEBUG = 0

(tp, exp) = ('compile', 'exec')

def parse(file):
f = open(file, 'r')

# Cleanup weird stuff
d = re.sub(r',\d+:\d', '', d)

r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)

test = {}
fname = ''
for t in r:
if DEBUG:
print t

if t[0] == 'PASS' or t[0] == 'FAIL' :
tmp = t[2].split('llvm-test/')

if DEBUG:
print tmp

if len(tmp) == 2:
fname = tmp[1].strip('\r\n')
else:
fname = tmp[0].strip('\r\n')

if not test.has_key(fname):
test[fname] = {}

test[fname][t[1] + ' state'] = t[0]
test[fname][t[1] + ' time'] = float('nan')
else :
try:
n = t[0].split('RESULT-')[1]

if DEBUG:
print "n == ", n;

if n == 'compile-success':
test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n'))

elif n == 'exec-success':
test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n'))
if DEBUG:
print test[fname][string.replace(n, '-success', '')]

else :
# print "ERROR!"
sys.exit(1)

except:
continue

return test

# Diff results and look for regressions.
def diffResults(d_old, d_new):
regressions = {}
passes = {}
removed = ''

for x in ['compile state', 'compile time', 'exec state', 'exec time']:
regressions[x] = ''
passes[x] = ''

for t in sorted(d_old.keys()) :
if d_new.has_key(t):

# Check if the test passed or failed.
for x in ['compile state', 'compile time', 'exec state', 'exec time']:

if not d_old[t].has_key(x) and not d_new[t].has_key(x):
continue

if d_old[t].has_key(x):
if d_new[t].has_key(x):

if d_old[t][x] == 'PASS':
if d_new[t][x] != 'PASS':
regressions[x] += t + "\n"
else:
if d_new[t][x] == 'PASS':
passes[x] += t + "\n"

else :
regressions[x] += t + "\n"

if x == 'compile state' or x == 'exec state':
continue

# For execution time, if there is no result it's a fail.
if not d_old[t].has_key(x) and not d_new[t].has_key(x):
continue
elif not d_new[t].has_key(x):
regressions[x] += t + "\n"
elif not d_old[t].has_key(x):
passes[x] += t + "\n"

if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
continue

elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]):
passes[x] += t + "\n"

elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
regressions[x] += t + ": NaN%\n"

if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \
(d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05:
regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n"

else :
removed += t + "\n"

if len(regressions['compile state']) != 0:
print 'REGRESSION: Compilation Failed'
print regressions['compile state']

if len(regressions['exec state']) != 0:
print 'REGRESSION: Execution Failed'
print regressions['exec state']

if len(regressions['compile time']) != 0:
print 'REGRESSION: Compilation Time'
print regressions['compile time']

if len(regressions['exec time']) != 0:
print 'REGRESSION: Execution Time'
print regressions['exec time']

if len(passes['compile state']) != 0:
print 'NEW PASSES: Compilation'
print passes['compile state']

if len(passes['exec state']) != 0:
print 'NEW PASSES: Execution'
print passes['exec state']

if len(removed) != 0:
print 'REMOVED TESTS'
print removed

# Main
if len(sys.argv) < 3 :
print 'Usage:', sys.argv[0], '<old log> <new log>'
sys.exit(-1)

d_old = parse(sys.argv[1])
d_new = parse(sys.argv[2])

diffResults(d_old, d_new)
```