dg-cmp-results.sh revision 1.1.1.4 
1#!/bin/bash
2# Copyright (C) 2006, 2008 Free Software Foundation
3#
4# Analyze changes in GCC DejaGNU test logs for binutils, gcc, gdb, etc.
5# Original version written in 2005 by James Lemke <jwlemke@wasabisystems.com>.
6#
7# See usage() below.
8
9usage () {
10    cat <<EOF >&2
11Usage:
12    dg-cmp-results.sh [-v] [-v] [-v] <variant-name> <old-file> <new-file>
13    <variant-name> names the desired variant, "/" must be written as "\/".
14    Use the empty string ("") for the first variant in each file.
15    Output is to stdout.
16    Non-verbose output is degradation info like PASS->FAIL.
17    -v adds improvement info like FAIL->PASS.
18    -v -v adds info like tests that are no longer run.
19    -v -v -v adds info for tests that have not changed status.
20    -v -v -v -v is used for debugging.
21EOF
22}
23
24verbose=0
25while test "$1" = "-v"; do
26    verbose=`expr $verbose + 1`
27    shift
28done
29
30if test $# -ne 3 ; then
31    usage
32    exit 1
33fi
34
35if test ! -f "$2"; then
36    echo "unable to open $2" >&2
37    exit 1
38fi
39
40if test ! -f "$3"; then
41    echo "unable to open $3" >&2
42    exit 1
43fi
44
45# Command differences for various platforms.
46case `uname -s` in
47Darwin|NetBSD)
48    E=-E	# sed
49    ;;
50*)
51    E=-r	# sed
52    ;;
53esac
54
55# sections are identified by separator lines beginning with '\t\t==='.
56# section 0 identifies run date, target, and host.
57# section 1 and subsequent contain test data for a target variant.
58# -skip to /^Running target/ and use that line to identify the variant.
59# -subsequent lines contain the result data.  They begin with:
60# '(PASS|FAIL|XFAIL|XPASS|UNTESTED|UNSUPPORTED|UNRESOLVED):'
61VARIANT="$1"
62OFILE="$2"
63OBASE=`basename "$2"`
64NFILE="$3"
65NBASE=`basename "$3"`
66TMPDIR=${TMPDIR:-/tmp}
67
68echo "dg-cmp-results.sh: Verbosity is ${verbose}, Variant is \"${VARIANT}\""
69echo
70
71header="^Running target $VARIANT"
72
73temp=`grep "$header" $OFILE`
74if test -z "$temp"; then
75    echo "Error: variant \"$VARIANT\" not found in $OFILE."
76    exit 1
77fi
78temp=`grep "$header" $NFILE`
79if test -z "$temp"; then
80    echo "Error: variant \"$VARIANT\" not found in $NFILE."
81    exit 1
82fi
83unset temp
84
85# Copy out the old file's section 0.
86echo "Older log file: $OFILE"
87sed $E -e '/^[[:space:]]+===/,$d' $OFILE
88
89# Copy out the new file's section 0.
90echo "Newer log file: $NFILE"
91sed $E -e '/^[[:space:]]+===/,$d' $NFILE
92
93# Create a temporary file from the old file's interesting section.
94sed $E -e "/$header/,/^[[:space:]]+===.*Summary ===/!d" \
95  -e '/^[A-Z]+:/!d' \
96  -e '/^(WARNING|ERROR):/d' \
97  -e 's/\r$//' \
98  -e 's/^/O:/' \
99  $OFILE |
100  sort -s -t : -k 3b - \
101  >$TMPDIR/o$$-$OBASE
102
103# Create a temporary file from the new file's interesting section.
104sed $E -e "/$header/,/^[[:space:]]+===.*Summary ===/!d" \
105  -e '/^[A-Z]+:/!d' \
106  -e '/^(WARNING|ERROR):/d' \
107  -e 's/\r$//' \
108  -e 's/^/N:/' \
109  $NFILE |
110  sort -s -t : -k 3b - \
111  >$TMPDIR/n$$-$NBASE
112
113# Merge the two files, then compare adjacent lines.
114# Comparison is complicated by tests that may be run multiple times.
115# If that case, we assume that the order is the same in both files.
116cat <<EOF >compare-$$.awk
117BEGIN {
118    FS = ":"
119    queue1 = 1; queueN = 0; status[queue1] = ""; name[queue1] = ""
120    verbose = verbose + 0	# Make sure it's defined.
121}
122
123# FIFO circular queue
124function push(st, nm) {
125    queueN += 1; status[queueN] = st; name[queueN] = nm
126}
127function peek() {
128    result = 0
129    if (queueN >= queue1) result = queue1
130    return result
131}
132function drop() {
133    queue1 += 1
134    if (queue1 > queueN) { queue1 = 1; queueN = 0; }
135}
136
137function compare(st, nm) {
138    old = peek()
139    if (old == 0) {
140        # This new test wasn't run last time.
141        if (verbose >= 2) printf("NA->%s:%s\n", st, nm)
142    }
143    else {
144	# Compare this new test to the first queued old one.
145	if (verbose >= 4) {
146	    printf("Comparing two lines:\n O:%s:%s\n N:%s:%s\n",
147	     status[old], name[old], st, nm)
148	}
149        if (name[old] != nm) {
150	    # The old test wasn't run this time and
151	    # the new test wasn't run last time.
152	    if (verbose >= 2) {
153		printf("%s->NA:%s\n", status[old], name[old])
154		if (nm != "") printf("NA->%s:%s\n", st, nm)
155	    }
156	    drop()
157        }
158	else {
159	    notable = 0
160	    if (status[old] == st) {
161	        # Status of this test has not changed.
162		if (verbose >= 3) printf("%s:%s\n", st, nm)
163	    }
164	    else if(status[old] == "PASS" && st == "XFAIL") {
165	        if (verbose >= 1) notable = 1
166	    }
167	    else if(status[old] == "PASS" || st == "FAIL") {
168	        # Test did pass but doesn't now
169		# or didn't fail but does now.
170		notable = 1
171	    }
172	    else if(st == "PASS") {
173	        # Test didn't pass but does now.
174		if (verbose >= 1) notable = 1
175	    }
176	    else if(verbose >= 2) {
177	        # Miscellaneous status change.
178		notable = 1
179	    }
180	    if (notable > 0) printf("%s->%s:%s\n", status[old], st, nm)
181	    drop()
182	}
183    }
184}
185
186/^O:/ {
187    while (old = peek()) {
188	if (name[old] == \$3) break;
189	# The queued test is no longer run.
190	compare("", "");
191    }
192    # Save this test for later comparison.
193    push(\$2, \$3)
194}
195
196/^N:/ {
197    compare(\$2, \$3)
198}
199
200END {
201    while (old = peek()) compare("", "")
202}
203EOF
204sort -m -s -t : -k 3b $TMPDIR/o$$-$OBASE $TMPDIR/n$$-$NBASE |
205 awk -v verbose=$verbose -f compare-$$.awk /dev/stdin
206
207# Delete the temporary files.
208rm -f compare-$$.awk $TMPDIR/o$$-$OBASE $TMPDIR/n$$-$NBASE
209
210exit 0
211