1#!/usr/bin/python -u
2import glob, os, string, sys, thread, time
3# import difflib
4import libxml2
5
6###
7#
8# This is a "Work in Progress" attempt at a python script to run the
9# various regression tests.  The rationale for this is that it should be
10# possible to run this on most major platforms, including those (such as
11# Windows) which don't support gnu Make.
12#
13# The script is driven by a parameter file which defines the various tests
14# to be run, together with the unique settings for each of these tests.  A
15# script for Linux is included (regressions.xml), with comments indicating
16# the significance of the various parameters.  To run the tests under Windows,
17# edit regressions.xml and remove the comment around the default parameter
18# "<execpath>" (i.e. make it point to the location of the binary executables).
19#
20# Note that this current version requires the Python bindings for libxml2 to
21# have been previously installed and accessible
22#
23# See Copyright for the status of this software.
24# William Brack (wbrack@mmm.com.hk)
25#
26###
27defaultParams = {}	# will be used as a dictionary to hold the parsed params
28
29# This routine is used for comparing the expected stdout / stdin with the results.
30# The expected data has already been read in; the result is a file descriptor.
31# Within the two sets of data, lines may begin with a path string.  If so, the
32# code "relativises" it by removing the path component.  The first argument is a
33# list already read in by a separate thread; the second is a file descriptor.
34# The two 'base' arguments are to let me "relativise" the results files, allowing
35# the script to be run from any directory.
36def compFiles(res, expected, base1, base2):
37    l1 = len(base1)
38    exp = expected.readlines()
39    expected.close()
40    # the "relativisation" is done here
41    for i in range(len(res)):
42        j = string.find(res[i],base1)
43        if (j == 0) or ((j == 2) and (res[i][0:2] == './')):
44            col = string.find(res[i],':')
45            if col > 0:
46                start = string.rfind(res[i][:col], '/')
47                if start > 0:
48                    res[i] = res[i][start+1:]
49
50    for i in range(len(exp)):
51        j = string.find(exp[i],base2)
52        if (j == 0) or ((j == 2) and (exp[i][0:2] == './')):
53            col = string.find(exp[i],':')
54            if col > 0:
55                start = string.rfind(exp[i][:col], '/')
56                if start > 0:
57                    exp[i] = exp[i][start+1:]
58
59    ret = 0
60    # ideally we would like to use difflib functions here to do a
61    # nice comparison of the two sets.  Unfortunately, during testing
62    # (using python 2.3.3 and 2.3.4) the following code went into
63    # a dead loop under windows.  I'll pursue this later.
64#    diff = difflib.ndiff(res, exp)
65#    diff = list(diff)
66#    for line in diff:
67#        if line[:2] != '  ':
68#            print string.strip(line)
69#            ret = -1
70
71    # the following simple compare is fine for when the two data sets
72    # (actual result vs. expected result) are equal, which should be true for
73    # us.  Unfortunately, if the test fails it's not nice at all.
74    rl = len(res)
75    el = len(exp)
76    if el != rl:
77        print 'Length of expected is %d, result is %d' % (el, rl)
78	ret = -1
79    for i in range(min(el, rl)):
80        if string.strip(res[i]) != string.strip(exp[i]):
81            print '+:%s-:%s' % (res[i], exp[i])
82            ret = -1
83    if el > rl:
84        for i in range(rl, el):
85            print '-:%s' % exp[i]
86            ret = -1
87    elif rl > el:
88        for i in range (el, rl):
89            print '+:%s' % res[i]
90            ret = -1
91    return ret
92
93# Separate threads to handle stdout and stderr are created to run this function
94def readPfile(file, list, flag):
95    data = file.readlines()	# no call by reference, so I cheat
96    for l in data:
97        list.append(l)
98    file.close()
99    flag.append('ok')
100
101# This routine runs the test program (e.g. xmllint)
102def runOneTest(testDescription, filename, inbase, errbase):
103    if 'execpath' in testDescription:
104        dir = testDescription['execpath'] + '/'
105    else:
106        dir = ''
107    cmd = os.path.abspath(dir + testDescription['testprog'])
108    if 'flag' in testDescription:
109        for f in string.split(testDescription['flag']):
110            cmd += ' ' + f
111    if 'stdin' not in testDescription:
112        cmd += ' ' + inbase + filename
113    if 'extarg' in testDescription:
114        cmd += ' ' + testDescription['extarg']
115
116    noResult = 0
117    expout = None
118    if 'resext' in testDescription:
119        if testDescription['resext'] == 'None':
120            noResult = 1
121        else:
122            ext = '.' + testDescription['resext']
123    else:
124        ext = ''
125    if not noResult:
126        try:
127            fname = errbase + filename + ext
128            expout = open(fname, 'rt')
129        except:
130            print "Can't open result file %s - bypassing test" % fname
131            return
132
133    noErrors = 0
134    if 'reserrext' in testDescription:
135        if testDescription['reserrext'] == 'None':
136            noErrors = 1
137        else:
138            if len(testDescription['reserrext'])>0:
139                ext = '.' + testDescription['reserrext']
140            else:
141                ext = ''
142    else:
143        ext = ''
144    if not noErrors:
145        try:
146            fname = errbase + filename + ext
147            experr = open(fname, 'rt')
148        except:
149            experr = None
150    else:
151        experr = None
152
153    pin, pout, perr = os.popen3(cmd)
154    if 'stdin' in testDescription:
155        infile = open(inbase + filename, 'rt')
156        pin.writelines(infile.readlines())
157        infile.close()
158        pin.close()
159
160    # popen is great fun, but can lead to the old "deadly embrace", because
161    # synchronizing the writing (by the task being run) of stdout and stderr
162    # with respect to the reading (by this task) is basically impossible.  I
163    # tried several ways to cheat, but the only way I have found which works
164    # is to do a *very* elementary multi-threading approach.  We can only hope
165    # that Python threads are implemented on the target system (it's okay for
166    # Linux and Windows)
167
168    th1Flag = []	# flags to show when threads finish
169    th2Flag = []
170    outfile = []	# lists to contain the pipe data
171    errfile = []
172    th1 = thread.start_new_thread(readPfile, (pout, outfile, th1Flag))
173    th2 = thread.start_new_thread(readPfile, (perr, errfile, th2Flag))
174    while (len(th1Flag)==0) or (len(th2Flag)==0):
175        time.sleep(0.001)
176    if not noResult:
177        ret = compFiles(outfile, expout, inbase, 'test/')
178        if ret != 0:
179            print 'trouble with %s' % cmd
180    else:
181        if len(outfile) != 0:
182            for l in outfile:
183                print l
184            print 'trouble with %s' % cmd
185    if experr != None:
186        ret = compFiles(errfile, experr, inbase, 'test/')
187        if ret != 0:
188            print 'trouble with %s' % cmd
189    else:
190        if not noErrors:
191            if len(errfile) != 0:
192                for l in errfile:
193                    print l
194                print 'trouble with %s' % cmd
195
196    if 'stdin' not in testDescription:
197        pin.close()
198
199# This routine is called by the parameter decoding routine whenever the end of a
200# 'test' section is encountered.  Depending upon file globbing, a large number of
201# individual tests may be run.
202def runTest(description):
203    testDescription = defaultParams.copy()		# set defaults
204    testDescription.update(description)			# override with current ent
205    if 'testname' in testDescription:
206        print "## %s" % testDescription['testname']
207    if not 'file' in testDescription:
208        print "No file specified - can't run this test!"
209        return
210    # Set up the source and results directory paths from the decoded params
211    dir = ''
212    if 'srcdir' in testDescription:
213        dir += testDescription['srcdir'] + '/'
214    if 'srcsub' in testDescription:
215        dir += testDescription['srcsub'] + '/'
216
217    rdir = ''
218    if 'resdir' in testDescription:
219        rdir += testDescription['resdir'] + '/'
220    if 'ressub' in testDescription:
221        rdir += testDescription['ressub'] + '/'
222
223    testFiles = glob.glob(os.path.abspath(dir + testDescription['file']))
224    if testFiles == []:
225        print "No files result from '%s'" % testDescription['file']
226        return
227
228    # Some test programs just don't work (yet).  For now we exclude them.
229    count = 0
230    excl = []
231    if 'exclfile' in testDescription:
232        for f in string.split(testDescription['exclfile']):
233            glb = glob.glob(dir + f)
234            for g in glb:
235                excl.append(os.path.abspath(g))
236
237    # Run the specified test program
238    for f in testFiles:
239        if not os.path.isdir(f):
240            if f not in excl:
241                count = count + 1
242                runOneTest(testDescription, os.path.basename(f), dir, rdir)
243
244#
245# The following classes are used with the xmlreader interface to interpret the
246# parameter file.  Once a test section has been identified, runTest is called
247# with a dictionary containing the parsed results of the interpretation.
248#
249
250class testDefaults:
251    curText = ''	# accumulates text content of parameter
252
253    def addToDict(self, key):
254        txt = string.strip(self.curText)
255#        if txt == '':
256#            return
257        if key not in defaultParams:
258            defaultParams[key] = txt
259        else:
260            defaultParams[key] += ' ' + txt
261
262    def processNode(self, reader, curClass):
263        if reader.Depth() == 2:
264            if reader.NodeType() == 1:
265                self.curText = ''	# clear the working variable
266            elif reader.NodeType() == 15:
267                if (reader.Name() != '#text') and (reader.Name() != '#comment'):
268                    self.addToDict(reader.Name())
269        elif reader.Depth() == 3:
270            if reader.Name() == '#text':
271                self.curText += reader.Value()
272
273        elif reader.NodeType() == 15:	# end of element
274            print "Defaults have been set to:"
275            for k in defaultParams.keys():
276                print "   %s : '%s'" % (k, defaultParams[k])
277            curClass = rootClass()
278        return curClass
279
280
281class testClass:
282    def __init__(self):
283        self.testParams = {}	# start with an empty set of params
284        self.curText = ''	# and empty text
285
286    def addToDict(self, key):
287        data = string.strip(self.curText)
288        if key not in self.testParams:
289            self.testParams[key] = data
290        else:
291            if self.testParams[key] != '':
292                data = ' ' + data
293            self.testParams[key] += data
294
295    def processNode(self, reader, curClass):
296        if reader.Depth() == 2:
297            if reader.NodeType() == 1:
298                self.curText = ''	# clear the working variable
299                if reader.Name() not in self.testParams:
300                    self.testParams[reader.Name()] = ''
301            elif reader.NodeType() == 15:
302                if (reader.Name() != '#text') and (reader.Name() != '#comment'):
303                    self.addToDict(reader.Name())
304        elif reader.Depth() == 3:
305            if reader.Name() == '#text':
306                self.curText += reader.Value()
307
308        elif reader.NodeType() == 15:	# end of element
309            runTest(self.testParams)
310            curClass = rootClass()
311        return curClass
312
313
314class rootClass:
315    def processNode(self, reader, curClass):
316        if reader.Depth() == 0:
317            return curClass
318        if reader.Depth() != 1:
319            print "Unexpected junk: Level %d, type %d, name %s" % (
320                  reader.Depth(), reader.NodeType(), reader.Name())
321            return curClass
322        if reader.Name() == 'test':
323            curClass = testClass()
324            curClass.testParams = {}
325        elif reader.Name() == 'defaults':
326            curClass = testDefaults()
327        return curClass
328
329def streamFile(filename):
330    try:
331        reader = libxml2.newTextReaderFilename(filename)
332    except:
333        print "unable to open %s" % (filename)
334        return
335
336    curClass = rootClass()
337    ret = reader.Read()
338    while ret == 1:
339        curClass = curClass.processNode(reader, curClass)
340        ret = reader.Read()
341
342    if ret != 0:
343        print "%s : failed to parse" % (filename)
344
345# OK, we're finished with all the routines.  Now for the main program:-
346if len(sys.argv) != 2:
347    print "Usage: maketest {filename}"
348    sys.exit(-1)
349
350streamFile(sys.argv[1])
351