1#!/usr/bin/python 2# -*- coding: utf-8 -*- 3# 4# Python Imports 5import os 6import sys 7import re 8 9""" 10xnu_raft_tests 11Automate testing of unit tests for xnu. 12 132012/02/23 14""" 15 16# this needs to be first thing for raft to load its environment correctly 17if __name__ == '__main__': 18 # The following code allows this test to be invoked outside the harness and should be left unchanged 19 args = [os.path.realpath(os.path.expanduser("/usr/local/bin/raft")), "-f"] + sys.argv 20 os.execv(args[0], args) 21 22 23# Library Imports 24from raftlibs.coreos import crashReporterStop, crashReporterStart, doPrivileged, runFunctionWithTestReRun 25from raftlibs.coreos import runUniversalLogProcess, spotlightStopSubtest, spotlightStartSubtest, svnCheckoutTestTool, svnCheckoutToPath, runSimpleProcess 26 27from raft.core.logging import log_note 28 29# Raft Imports 30from __test__ import __path__ 31 32# This is a Raft test. For more information see http://raft.apple.com 33testDescription = "Runs all tests defined as targets in Makefile" # Add a brief description of test functionality 34testVersion = "0.1" # Used to differentiate between results for different versions of the test 35testState = DevelopmentState # Possible values: DevelopmentState, ProductionState 36 37 38# class definitions 39class xnuTest: 40 """ A container to hold test and its result """ 41 def __init__(self,testName): 42 self.name = str(testName) 43 self.buildStatus = False 44 self.executeStatus = False 45 self.exitValue = None 46 self.comments = '' 47 48 def getName(self): 49 return self.name 50 51 @staticmethod 52 def getSummaryHeader(): 53 return "| {0: ^40s} |{1: >6s} |{2: >5s} |{3: >10s} |{4}".format("Test Name", "Build", "Run", "ExitVal", "Comments") 54 55 def getSummary(self): 56 formatString ="| {0: <40s} |{1: >6s} |{2: >5s} |{3: >10s} |{4}" 57 nameVal = str(self.name) 58 buildVal = str(self.buildStatus) 59 execVal = str(self.executeStatus) 60 exitVal = str(self.exitValue) 61 commentsVal = str(self.comments) 62 return formatString.format(nameVal, buildVal, execVal, exitVal, commentsVal) 63 64# global functions 65def getTestsFromMakeFile(makeFilePath): 66 makeTargets=[] 67 targetRegex = re.compile("^\s*([a-zA-Z0-9_.]+)\s*:\s*([a-zA-Z0-9_.]*).*",re.IGNORECASE|re.DOTALL) 68 fh = open(makeFilePath,"r"); 69 for line in fh: 70 tmp_res = targetRegex.findall(line) 71 if len(tmp_res) == 1: 72 makeTargets.append(xnuTest(tmp_res[0][0])) 73 fh.close() 74 return makeTargets 75 76 77def buildTest(test, path): 78 os.chdir(path) 79 result = doCommand("/usr/bin/make",test) 80 if result['status'] != 0: 81 print "Failed to Build %s" % test 82 print "**STDOUT**\n%s" % result['stdout'] 83 print "**STDERR**\n%s" % result['stderr'] 84 raise StandardError 85 log_note("Built %s successfully" % test) 86 87def executeTest(testObject,path): 88 os.chdir(path) 89 test = testObject.getName() 90 executable_path = os.path.join(path, test) 91 print "[TEST] %s" % test 92 print "[BEGIN] %s" % test 93 try: 94 result = runSimpleProcess(executable_path,testName()+"_"+test, wait_time=120) 95 testObject.exitValue = result['status'] 96 if result['status'] == 0: 97 print "[PASS] %s returned %d" % (test,result['status']) 98 except: 99 print "[FAIL] %s returned %d" % (test, result['status']) 100 testObject.comments = "Failed due to timeout or file not found error" 101 log_note("Completed running test %s" % test) 102 103def removeTestExecutable(test,path): 104 os.chdir(path) 105 doCommand("/bin/rm",test) 106 107def runTest(params): 108 # Change to /tmp, because make doesn't support directory paths with spaces 109 os.chdir("/private/tmp") 110 output= {'status': 1 } 111 try: 112 output = svnCheckoutTestTool("unit_tests") 113 except: 114 pass 115 if output['status'] != 0 : 116 # since we are not fully published yet. lets get data from a branch 117 print "Fetching unit_test roots from Branch instead of trunk" 118 baseURL = "http://src.apple.com/svn/xnu/branches/PR-10938974/tools/tests/unit_tests/" 119 output = svnCheckoutToPath(baseURL) 120 if output['status'] != 0 : 121 logFail("[FAIL] error in checkout from branch") 122 sys.exit(1) 123 124 local_path = os.path.join(os.getcwd(), "unit_tests") 125 makefile_path = os.path.join(local_path, "Makefile") 126 build_path = os.path.join(local_path, "BUILD") 127 128 129 tests_to_run = getTestsFromMakeFile(makefile_path) 130 log_note("Starting raft tests for XNU") 131 stats = {"total":len(tests_to_run) , "pass":0, "fail":0} 132 for testObject in tests_to_run: 133 test = testObject.getName() 134 if test == "clean": 135 stats["pass"]+=1 136 testObject.buildStatus = True 137 testObject.executeStatus = True 138 testObject.exitValue = 0 139 continue 140 141 log_note("Running test :%s" % test) 142 try: 143 buildTest(test,local_path) 144 testObject.buildStatus = True 145 res = executeTest(testObject,build_path) 146 testObject.executeStatus = True 147 if testObject.exitValue == 0 : 148 stats["pass"]+=1 149 else: 150 stats["fail"]+=1 151 removeTestExecutable(test,build_path) 152 logPass(test) 153 except: 154 logFail("[FAIL] %s failed." % test) 155 print "Finished running tests. Cleaning up" 156 doCommand("/usr/bin/make","clean") 157 #Now to print the Summary and statistics 158 print "\n\n Test Summary \n" 159 print xnuTest.getSummaryHeader() 160 for testObject in tests_to_run: 161 print testObject.getSummary() 162 print "\n===============================\n" 163 print "[SUMMARY]" 164 print "Total tests: %d" % stats["total"] 165 print "Passed : %d" % stats["pass"] 166 print "Failed : %d" % stats["fail"] 167 print "================================\n\n" 168 169 logPass() # This line is implicit and can be removed 170