1/*
2 * runsuite.c: C program to run libxml2 againts published testsuites
3 *
4 * See Copyright for the status of this software.
5 *
6 * daniel@veillard.com
7 */
8
9#ifdef HAVE_CONFIG_H
10#include "libxml.h"
11#else
12#include <stdio.h>
13#endif
14
15#ifdef LIBXML_XPATH_ENABLED
16
17#if !defined(_WIN32) || defined(__CYGWIN__)
18#include <unistd.h>
19#endif
20#include <string.h>
21#include <sys/types.h>
22#include <sys/stat.h>
23#include <fcntl.h>
24
25#include <libxml/parser.h>
26#include <libxml/parserInternals.h>
27#include <libxml/tree.h>
28#include <libxml/uri.h>
29#include <libxml/xmlreader.h>
30
31#include <libxml/xpath.h>
32#include <libxml/xpathInternals.h>
33
34#define LOGFILE "runxmlconf.log"
35static FILE *logfile = NULL;
36static int verbose = 0;
37
38#define NB_EXPECTED_ERRORS 15
39
40#if defined(_WIN32) && !defined(__CYGWIN__)
41
42#define vsnprintf _vsnprintf
43
44#define snprintf _snprintf
45
46#endif
47
48const char *skipped_tests[] = {
49/* http://lists.w3.org/Archives/Public/public-xml-testsuite/2008Jul/0000.html */
50    "rmt-ns10-035",
51    NULL
52};
53
54/************************************************************************
55 *									*
56 *		File name and path utilities				*
57 *									*
58 ************************************************************************/
59
60static int checkTestFile(const char *filename) {
61    struct stat buf;
62
63    if (stat(filename, &buf) == -1)
64        return(0);
65
66#if defined(_WIN32) && !defined(__CYGWIN__)
67    if (!(buf.st_mode & _S_IFREG))
68        return(0);
69#else
70    if (!S_ISREG(buf.st_mode))
71        return(0);
72#endif
73
74    return(1);
75}
76
77static xmlChar *composeDir(const xmlChar *dir, const xmlChar *path) {
78    char buf[500];
79
80    if (dir == NULL) return(xmlStrdup(path));
81    if (path == NULL) return(NULL);
82
83    snprintf(buf, 500, "%s/%s", (const char *) dir, (const char *) path);
84    return(xmlStrdup((const xmlChar *) buf));
85}
86
87/************************************************************************
88 *									*
89 *		Libxml2 specific routines				*
90 *									*
91 ************************************************************************/
92
93static int nb_skipped = 0;
94static int nb_tests = 0;
95static int nb_errors = 0;
96static int nb_leaks = 0;
97
98/*
99 * We need to trap calls to the resolver to not account memory for the catalog
100 * and not rely on any external resources.
101 */
102static xmlParserInputPtr
103testExternalEntityLoader(const char *URL, const char *ID ATTRIBUTE_UNUSED,
104			 xmlParserCtxtPtr ctxt) {
105    xmlParserInputPtr ret;
106
107    ret = xmlNewInputFromFile(ctxt, (const char *) URL);
108
109    return(ret);
110}
111
112/*
113 * Trapping the error messages at the generic level to grab the equivalent of
114 * stderr messages on CLI tools.
115 */
116static char testErrors[32769];
117static int testErrorsSize = 0;
118static int nbError = 0;
119static int nbFatal = 0;
120
121static void test_log(const char *msg, ...) {
122    va_list args;
123    if (logfile != NULL) {
124        fprintf(logfile, "\n------------\n");
125	va_start(args, msg);
126	vfprintf(logfile, msg, args);
127	va_end(args);
128	fprintf(logfile, "%s", testErrors);
129	testErrorsSize = 0; testErrors[0] = 0;
130    }
131    if (verbose) {
132	va_start(args, msg);
133	vfprintf(stderr, msg, args);
134	va_end(args);
135    }
136}
137
138static void
139testErrorHandler(void *userData ATTRIBUTE_UNUSED, xmlErrorPtr error) {
140    int res;
141
142    if (testErrorsSize >= 32768)
143        return;
144    res = snprintf(&testErrors[testErrorsSize],
145                    32768 - testErrorsSize,
146		   "%s:%d: %s\n", (error->file ? error->file : "entity"),
147		   error->line, error->message);
148    if (error->level == XML_ERR_FATAL)
149        nbFatal++;
150    else if (error->level == XML_ERR_ERROR)
151        nbError++;
152    if (testErrorsSize + res >= 32768) {
153        /* buffer is full */
154	testErrorsSize = 32768;
155	testErrors[testErrorsSize] = 0;
156    } else {
157        testErrorsSize += res;
158    }
159    testErrors[testErrorsSize] = 0;
160}
161
162static xmlXPathContextPtr ctxtXPath;
163
164static void
165initializeLibxml2(void) {
166    xmlGetWarningsDefaultValue = 0;
167    xmlPedanticParserDefault(0);
168
169    xmlMemSetup(xmlMemFree, xmlMemMalloc, xmlMemRealloc, xmlMemoryStrdup);
170    xmlInitParser();
171    xmlSetExternalEntityLoader(testExternalEntityLoader);
172    ctxtXPath = xmlXPathNewContext(NULL);
173    /*
174    * Deactivate the cache if created; otherwise we have to create/free it
175    * for every test, since it will confuse the memory leak detection.
176    * Note that normally this need not be done, since the cache is not
177    * created until set explicitely with xmlXPathContextSetCache();
178    * but for test purposes it is sometimes usefull to activate the
179    * cache by default for the whole library.
180    */
181    if (ctxtXPath->cache != NULL)
182	xmlXPathContextSetCache(ctxtXPath, 0, -1, 0);
183    xmlSetStructuredErrorFunc(NULL, testErrorHandler);
184}
185
186/************************************************************************
187 *									*
188 *		Run the xmlconf test if found				*
189 *									*
190 ************************************************************************/
191
192static int
193xmlconfTestInvalid(const char *id, const char *filename, int options) {
194    xmlDocPtr doc;
195    xmlParserCtxtPtr ctxt;
196    int ret = 1;
197
198    ctxt = xmlNewParserCtxt();
199    if (ctxt == NULL) {
200        test_log("test %s : %s out of memory\n",
201	         id, filename);
202        return(0);
203    }
204    doc = xmlCtxtReadFile(ctxt, filename, NULL, options);
205    if (doc == NULL) {
206        test_log("test %s : %s invalid document turned not well-formed too\n",
207	         id, filename);
208    } else {
209    /* invalidity should be reported both in the context and in the document */
210        if ((ctxt->valid != 0) || (doc->properties & XML_DOC_DTDVALID)) {
211	    test_log("test %s : %s failed to detect invalid document\n",
212		     id, filename);
213	    nb_errors++;
214	    ret = 0;
215	}
216	xmlFreeDoc(doc);
217    }
218    xmlFreeParserCtxt(ctxt);
219    return(ret);
220}
221
222static int
223xmlconfTestValid(const char *id, const char *filename, int options) {
224    xmlDocPtr doc;
225    xmlParserCtxtPtr ctxt;
226    int ret = 1;
227
228    ctxt = xmlNewParserCtxt();
229    if (ctxt == NULL) {
230        test_log("test %s : %s out of memory\n",
231	         id, filename);
232        return(0);
233    }
234    doc = xmlCtxtReadFile(ctxt, filename, NULL, options);
235    if (doc == NULL) {
236        test_log("test %s : %s failed to parse a valid document\n",
237	         id, filename);
238        nb_errors++;
239	ret = 0;
240    } else {
241    /* validity should be reported both in the context and in the document */
242        if ((ctxt->valid == 0) || ((doc->properties & XML_DOC_DTDVALID) == 0)) {
243	    test_log("test %s : %s failed to validate a valid document\n",
244		     id, filename);
245	    nb_errors++;
246	    ret = 0;
247	}
248	xmlFreeDoc(doc);
249    }
250    xmlFreeParserCtxt(ctxt);
251    return(ret);
252}
253
254static int
255xmlconfTestNotNSWF(const char *id, const char *filename, int options) {
256    xmlDocPtr doc;
257    int ret = 1;
258
259    /*
260     * In case of Namespace errors, libxml2 will still parse the document
261     * but log a Namesapce error.
262     */
263    doc = xmlReadFile(filename, NULL, options);
264    if (doc == NULL) {
265        test_log("test %s : %s failed to parse the XML\n",
266	         id, filename);
267        nb_errors++;
268	ret = 0;
269    } else {
270	if ((xmlLastError.code == XML_ERR_OK) ||
271	    (xmlLastError.domain != XML_FROM_NAMESPACE)) {
272	    test_log("test %s : %s failed to detect namespace error\n",
273		     id, filename);
274	    nb_errors++;
275	    ret = 0;
276	}
277	xmlFreeDoc(doc);
278    }
279    return(ret);
280}
281
282static int
283xmlconfTestNotWF(const char *id, const char *filename, int options) {
284    xmlDocPtr doc;
285    int ret = 1;
286
287    doc = xmlReadFile(filename, NULL, options);
288    if (doc != NULL) {
289        test_log("test %s : %s failed to detect not well formedness\n",
290	         id, filename);
291        nb_errors++;
292	xmlFreeDoc(doc);
293	ret = 0;
294    }
295    return(ret);
296}
297
298static int
299xmlconfTestItem(xmlDocPtr doc, xmlNodePtr cur) {
300    int ret = -1;
301    xmlChar *type = NULL;
302    xmlChar *filename = NULL;
303    xmlChar *uri = NULL;
304    xmlChar *base = NULL;
305    xmlChar *id = NULL;
306    xmlChar *rec = NULL;
307    xmlChar *version = NULL;
308    xmlChar *entities = NULL;
309    xmlChar *edition = NULL;
310    int options = 0;
311    int nstest = 0;
312    int mem, final;
313    int i;
314
315    testErrorsSize = 0; testErrors[0] = 0;
316    nbError = 0;
317    nbFatal = 0;
318    id = xmlGetProp(cur, BAD_CAST "ID");
319    if (id == NULL) {
320        test_log("test missing ID, line %ld\n", xmlGetLineNo(cur));
321	goto error;
322    }
323    for (i = 0;skipped_tests[i] != NULL;i++) {
324        if (!strcmp(skipped_tests[i], (char *) id)) {
325	    test_log("Skipping test %s from skipped list\n", (char *) id);
326	    ret = 0;
327	    nb_skipped++;
328	    goto error;
329	}
330    }
331    type = xmlGetProp(cur, BAD_CAST "TYPE");
332    if (type == NULL) {
333        test_log("test %s missing TYPE\n", (char *) id);
334	goto error;
335    }
336    uri = xmlGetProp(cur, BAD_CAST "URI");
337    if (uri == NULL) {
338        test_log("test %s missing URI\n", (char *) id);
339	goto error;
340    }
341    base = xmlNodeGetBase(doc, cur);
342    filename = composeDir(base, uri);
343    if (!checkTestFile((char *) filename)) {
344        test_log("test %s missing file %s \n", id,
345	         (filename ? (char *)filename : "NULL"));
346	goto error;
347    }
348
349    version = xmlGetProp(cur, BAD_CAST "VERSION");
350
351    entities = xmlGetProp(cur, BAD_CAST "ENTITIES");
352    if (!xmlStrEqual(entities, BAD_CAST "none")) {
353        options |= XML_PARSE_DTDLOAD;
354        options |= XML_PARSE_NOENT;
355    }
356    rec = xmlGetProp(cur, BAD_CAST "RECOMMENDATION");
357    if ((rec == NULL) ||
358        (xmlStrEqual(rec, BAD_CAST "XML1.0")) ||
359	(xmlStrEqual(rec, BAD_CAST "XML1.0-errata2e")) ||
360	(xmlStrEqual(rec, BAD_CAST "XML1.0-errata3e")) ||
361	(xmlStrEqual(rec, BAD_CAST "XML1.0-errata4e"))) {
362	if ((version != NULL) && (!xmlStrEqual(version, BAD_CAST "1.0"))) {
363	    test_log("Skipping test %s for %s\n", (char *) id,
364	             (char *) version);
365	    ret = 0;
366	    nb_skipped++;
367	    goto error;
368	}
369	ret = 1;
370    } else if ((xmlStrEqual(rec, BAD_CAST "NS1.0")) ||
371	       (xmlStrEqual(rec, BAD_CAST "NS1.0-errata1e"))) {
372	ret = 1;
373	nstest = 1;
374    } else {
375        test_log("Skipping test %s for REC %s\n", (char *) id, (char *) rec);
376	ret = 0;
377	nb_skipped++;
378	goto error;
379    }
380    edition = xmlGetProp(cur, BAD_CAST "EDITION");
381    if ((edition != NULL) && (xmlStrchr(edition, '5') == NULL)) {
382        /* test limited to all versions before 5th */
383	options |= XML_PARSE_OLD10;
384    }
385
386    /*
387     * Reset errors and check memory usage before the test
388     */
389    xmlResetLastError();
390    testErrorsSize = 0; testErrors[0] = 0;
391    mem = xmlMemUsed();
392
393    if (xmlStrEqual(type, BAD_CAST "not-wf")) {
394        if (nstest == 0)
395	    xmlconfTestNotWF((char *) id, (char *) filename, options);
396        else
397	    xmlconfTestNotNSWF((char *) id, (char *) filename, options);
398    } else if (xmlStrEqual(type, BAD_CAST "valid")) {
399        options |= XML_PARSE_DTDVALID;
400	xmlconfTestValid((char *) id, (char *) filename, options);
401    } else if (xmlStrEqual(type, BAD_CAST "invalid")) {
402        options |= XML_PARSE_DTDVALID;
403	xmlconfTestInvalid((char *) id, (char *) filename, options);
404    } else if (xmlStrEqual(type, BAD_CAST "error")) {
405        test_log("Skipping error test %s \n", (char *) id);
406	ret = 0;
407	nb_skipped++;
408	goto error;
409    } else {
410        test_log("test %s unknown TYPE value %s\n", (char *) id, (char *)type);
411	ret = -1;
412	goto error;
413    }
414
415    /*
416     * Reset errors and check memory usage after the test
417     */
418    xmlResetLastError();
419    final = xmlMemUsed();
420    if (final > mem) {
421        test_log("test %s : %s leaked %d bytes\n",
422	         id, filename, final - mem);
423        nb_leaks++;
424	xmlMemDisplayLast(logfile, final - mem);
425    }
426    nb_tests++;
427
428error:
429    if (type != NULL)
430        xmlFree(type);
431    if (entities != NULL)
432        xmlFree(entities);
433    if (edition != NULL)
434        xmlFree(edition);
435    if (version != NULL)
436        xmlFree(version);
437    if (filename != NULL)
438        xmlFree(filename);
439    if (uri != NULL)
440        xmlFree(uri);
441    if (base != NULL)
442        xmlFree(base);
443    if (id != NULL)
444        xmlFree(id);
445    if (rec != NULL)
446        xmlFree(rec);
447    return(ret);
448}
449
450static int
451xmlconfTestCases(xmlDocPtr doc, xmlNodePtr cur, int level) {
452    xmlChar *profile;
453    int ret = 0;
454    int tests = 0;
455    int output = 0;
456
457    if (level == 1) {
458	profile = xmlGetProp(cur, BAD_CAST "PROFILE");
459	if (profile != NULL) {
460	    output = 1;
461	    level++;
462	    printf("Test cases: %s\n", (char *) profile);
463	    xmlFree(profile);
464	}
465    }
466    cur = cur->children;
467    while (cur != NULL) {
468        /* look only at elements we ignore everything else */
469        if (cur->type == XML_ELEMENT_NODE) {
470	    if (xmlStrEqual(cur->name, BAD_CAST "TESTCASES")) {
471	        ret += xmlconfTestCases(doc, cur, level);
472	    } else if (xmlStrEqual(cur->name, BAD_CAST "TEST")) {
473	        if (xmlconfTestItem(doc, cur) >= 0)
474		    ret++;
475		tests++;
476	    } else {
477	        fprintf(stderr, "Unhandled element %s\n", (char *)cur->name);
478	    }
479	}
480        cur = cur->next;
481    }
482    if (output == 1) {
483	if (tests > 0)
484	    printf("Test cases: %d tests\n", tests);
485    }
486    return(ret);
487}
488
489static int
490xmlconfTestSuite(xmlDocPtr doc, xmlNodePtr cur) {
491    xmlChar *profile;
492    int ret = 0;
493
494    profile = xmlGetProp(cur, BAD_CAST "PROFILE");
495    if (profile != NULL) {
496        printf("Test suite: %s\n", (char *) profile);
497	xmlFree(profile);
498    } else
499        printf("Test suite\n");
500    cur = cur->children;
501    while (cur != NULL) {
502        /* look only at elements we ignore everything else */
503        if (cur->type == XML_ELEMENT_NODE) {
504	    if (xmlStrEqual(cur->name, BAD_CAST "TESTCASES")) {
505	        ret += xmlconfTestCases(doc, cur, 1);
506	    } else {
507	        fprintf(stderr, "Unhandled element %s\n", (char *)cur->name);
508	    }
509	}
510        cur = cur->next;
511    }
512    return(ret);
513}
514
515static void
516xmlconfInfo(void) {
517    fprintf(stderr, "  you need to fetch and extract the\n");
518    fprintf(stderr, "  latest XML Conformance Test Suites\n");
519    fprintf(stderr, "  http://www.w3.org/XML/Test/xmlts20080205.tar.gz\n");
520    fprintf(stderr, "  see http://www.w3.org/XML/Test/ for informations\n");
521}
522
523static int
524xmlconfTest(void) {
525    const char *confxml = "xmlconf/xmlconf.xml";
526    xmlDocPtr doc;
527    xmlNodePtr cur;
528    int ret = 0;
529
530    if (!checkTestFile(confxml)) {
531        fprintf(stderr, "%s is missing \n", confxml);
532	xmlconfInfo();
533	return(-1);
534    }
535    doc = xmlReadFile(confxml, NULL, XML_PARSE_NOENT);
536    if (doc == NULL) {
537        fprintf(stderr, "%s is corrupted \n", confxml);
538	xmlconfInfo();
539	return(-1);
540    }
541
542    cur = xmlDocGetRootElement(doc);
543    if ((cur == NULL) || (!xmlStrEqual(cur->name, BAD_CAST "TESTSUITE"))) {
544        fprintf(stderr, "Unexpected format %s\n", confxml);
545	xmlconfInfo();
546	ret = -1;
547    } else {
548        ret = xmlconfTestSuite(doc, cur);
549    }
550    xmlFreeDoc(doc);
551    return(ret);
552}
553
554/************************************************************************
555 *									*
556 *		The driver for the tests				*
557 *									*
558 ************************************************************************/
559
560int
561main(int argc ATTRIBUTE_UNUSED, char **argv ATTRIBUTE_UNUSED) {
562    int ret = 0;
563    int old_errors, old_tests, old_leaks;
564
565    logfile = fopen(LOGFILE, "w");
566    if (logfile == NULL) {
567        fprintf(stderr,
568	        "Could not open the log file, running in verbose mode\n");
569	verbose = 1;
570    }
571    initializeLibxml2();
572
573    if ((argc >= 2) && (!strcmp(argv[1], "-v")))
574        verbose = 1;
575
576
577    old_errors = nb_errors;
578    old_tests = nb_tests;
579    old_leaks = nb_leaks;
580    xmlconfTest();
581    if ((nb_errors == old_errors) && (nb_leaks == old_leaks))
582	printf("Ran %d tests, no errors\n", nb_tests - old_tests);
583    else
584	printf("Ran %d tests, %d errors, %d leaks\n",
585	       nb_tests - old_tests,
586	       nb_errors - old_errors,
587	       nb_leaks - old_leaks);
588    if ((nb_errors == 0) && (nb_leaks == 0)) {
589        ret = 0;
590	printf("Total %d tests, no errors\n",
591	       nb_tests);
592    } else {
593	ret = 1;
594	printf("Total %d tests, %d errors, %d leaks\n",
595	       nb_tests, nb_errors, nb_leaks);
596	printf("See %s for detailed output\n", LOGFILE);
597	if ((nb_leaks == 0) && (nb_errors == NB_EXPECTED_ERRORS)) {
598	    printf("%d errors were expected\n", nb_errors);
599	    ret = 0;
600	}
601    }
602    xmlXPathFreeContext(ctxtXPath);
603    xmlCleanupParser();
604    xmlMemoryDump();
605
606    if (logfile != NULL)
607        fclose(logfile);
608    return(ret);
609}
610
611#else /* ! LIBXML_XPATH_ENABLED */
612#include <stdio.h>
613int
614main(int argc, char **argv) {
615    fprintf(stderr, "%s need XPath support\n", argv[0]);
616}
617#endif
618