1#!/usr/bin/python
2# -*- coding: utf-8 -*-
3#
4# Python Imports
5import os
6import sys
7import re
8
9"""
10xnu_raft_tests
11Automate testing of unit tests for xnu.
12
132012/02/23
14"""
15
16# this needs to be first thing for raft to load its environment correctly
17if __name__ == '__main__':
18	# The following code allows this test to be invoked outside the harness and should be left unchanged
19	args = [os.path.realpath(os.path.expanduser("/usr/local/bin/raft")), "-f"] + sys.argv
20	os.execv(args[0], args)
21
22
23# Library Imports
24from raftlibs.coreos import crashReporterStop, crashReporterStart, doPrivileged, runFunctionWithTestReRun
25from raftlibs.coreos import runUniversalLogProcess, spotlightStopSubtest, spotlightStartSubtest, svnCheckoutTestTool, svnCheckoutToPath, runSimpleProcess
26
27from raft.core.logging import log_note
28
29# Raft Imports
30from __test__ import __path__
31
32# This is a Raft test. For more information see http://raft.apple.com
33testDescription  = "Runs all tests defined as targets in Makefile"                 # Add a brief description of test functionality
34testVersion      = "0.1"              # Used to differentiate between results for different versions of the test
35testState        = DevelopmentState   # Possible values: DevelopmentState, ProductionState
36
37
38# class definitions
39class xnuTest:
40	""" A container to hold test and its result """
41	def __init__(self,testName):
42		self.name = str(testName)
43		self.buildStatus = False
44		self.executeStatus = False
45		self.exitValue = None
46		self.comments = ''
47
48	def getName(self):
49		return self.name
50
51	@staticmethod
52	def getSummaryHeader():
53		return "| {0: ^40s} |{1: >6s} |{2: >5s} |{3: >10s} |{4}".format("Test Name", "Build", "Run", "ExitVal", "Comments")
54
55	def getSummary(self):
56		formatString ="| {0: <40s} |{1: >6s} |{2: >5s} |{3: >10s} |{4}"
57		nameVal = str(self.name)
58		buildVal = str(self.buildStatus)
59		execVal = str(self.executeStatus)
60		exitVal = str(self.exitValue)
61		commentsVal = str(self.comments)
62		return formatString.format(nameVal, buildVal, execVal, exitVal, commentsVal)
63
64# global functions
65def getTestsFromMakeFile(makeFilePath):
66	makeTargets=[]
67	targetRegex = re.compile("^\s*([a-zA-Z0-9_.]+)\s*:\s*([a-zA-Z0-9_.]*).*",re.IGNORECASE|re.DOTALL)
68	fh = open(makeFilePath,"r");
69	for line in fh:
70		tmp_res = targetRegex.findall(line)
71		if len(tmp_res) == 1:
72			makeTargets.append(xnuTest(tmp_res[0][0]))
73	fh.close()
74	return makeTargets
75
76
77def buildTest(test, path):
78	os.chdir(path)
79	result = doCommand("/usr/bin/make",test)
80	if result['status'] != 0:
81		print "Failed to Build %s" % test
82		print "**STDOUT**\n%s" % result['stdout']
83		print "**STDERR**\n%s" % result['stderr']
84		raise StandardError
85	log_note("Built %s successfully" % test)
86
87def executeTest(testObject,path):
88	os.chdir(path)
89	test = testObject.getName()
90	executable_path = os.path.join(path, test)
91	print "[TEST] %s" % test
92	print "[BEGIN] %s" % test
93	try:
94		result = runSimpleProcess(executable_path,testName()+"_"+test, wait_time=120)
95		testObject.exitValue = result['status']
96		if result['status'] == 0:
97			print "[PASS] %s returned %d" % (test,result['status'])
98	except:
99		print "[FAIL] %s returned %d" % (test, result['status'])
100		testObject.comments = "Failed due to timeout or file not found error"
101	log_note("Completed running test %s" % test)
102
103def removeTestExecutable(test,path):
104	os.chdir(path)
105	doCommand("/bin/rm",test)
106
107def runTest(params):
108	# Change to /tmp, because make doesn't support directory paths with spaces
109	os.chdir("/private/tmp")
110	output= {'status': 1 }
111	try:
112		output = svnCheckoutTestTool("unit_tests")
113	except:
114		pass
115	if output['status'] != 0 :
116		# since we are not fully published yet. lets get data from a branch
117		print "Fetching unit_test roots from Branch instead of trunk"
118		baseURL = "http://src.apple.com/svn/xnu/branches/PR-10938974/tools/tests/unit_tests/"
119		output = svnCheckoutToPath(baseURL)
120		if output['status'] != 0 :
121			logFail("[FAIL] error in checkout from branch")
122			sys.exit(1)
123
124	local_path = os.path.join(os.getcwd(), "unit_tests")
125	makefile_path = os.path.join(local_path, "Makefile")
126	build_path = os.path.join(local_path, "BUILD")
127
128
129	tests_to_run = getTestsFromMakeFile(makefile_path)
130	log_note("Starting raft tests for XNU")
131	stats = {"total":len(tests_to_run) , "pass":0, "fail":0}
132	for testObject in tests_to_run:
133		test = testObject.getName()
134		if test == "clean":
135			stats["pass"]+=1
136			testObject.buildStatus = True
137			testObject.executeStatus = True
138			testObject.exitValue = 0
139			continue
140
141		log_note("Running test :%s" % test)
142		try:
143			buildTest(test,local_path)
144			testObject.buildStatus = True
145			res = executeTest(testObject,build_path)
146			testObject.executeStatus = True
147			if testObject.exitValue == 0 :
148				stats["pass"]+=1
149			else:
150				stats["fail"]+=1
151			removeTestExecutable(test,build_path)
152			logPass(test)
153		except:
154			logFail("[FAIL] %s failed." % test)
155	print "Finished running tests. Cleaning up"
156	doCommand("/usr/bin/make","clean")
157	#Now to print the Summary and statistics
158	print "\n\n Test Summary \n"
159	print xnuTest.getSummaryHeader()
160	for testObject in tests_to_run:
161		print testObject.getSummary()
162	print "\n===============================\n"
163	print "[SUMMARY]"
164	print "Total tests: %d" % stats["total"]
165	print "Passed     : %d" % stats["pass"]
166	print "Failed     : %d" % stats["fail"]
167	print "================================\n\n"
168
169	logPass() # This line is implicit and can be removed
170