RunTests.gmk revision 2790:32dc808e9918 
1#
2# Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
3# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4#
5# This code is free software; you can redistribute it and/or modify it
6# under the terms of the GNU General Public License version 2 only, as
7# published by the Free Software Foundation.  Oracle designates this
8# particular file as subject to the "Classpath" exception as provided
9# by Oracle in the LICENSE file that accompanied this code.
10#
11# This code is distributed in the hope that it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14# version 2 for more details (a copy is included in the LICENSE file that
15# accompanied this code).
16#
17# You should have received a copy of the GNU General Public License version
18# 2 along with this work; if not, write to the Free Software Foundation,
19# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20#
21# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22# or visit www.oracle.com if you need additional information or have any
23# questions.
24#
25
26default: all
27
28include $(SPEC)
29include MakeBase.gmk
30include FindTests.gmk
31
32# We will always run multiple tests serially
33.NOTPARALLEL:
34
35# Hook to include the corresponding custom file, if present.
36$(eval $(call IncludeCustomExtension, , RunTests.gmk))
37
38TEST_RESULTS_DIR := $(BUILD_OUTPUT)/test-results
39TEST_SUPPORT_DIR := $(BUILD_OUTPUT)/test-support
40
41
42################################################################################
43# Parse control variables
44################################################################################
45
46$(eval $(call ParseKeywordVariable, JTREG, \
47    KEYWORDS := JOBS TIMEOUT TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM, \
48    STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \
49))
50
51ifneq ($(JTREG), )
52  # Inform the user
53  $(info Running tests using JTREG control variable '$(JTREG)')
54endif
55
56$(eval $(call ParseKeywordVariable, GTEST, \
57    KEYWORDS := REPEAT, \
58    STRING_KEYWORDS := OPTIONS, \
59))
60
61ifneq ($(GTEST), )
62  # Inform the user
63  $(info Running tests using GTEST control variable '$(GTEST)')
64endif
65
66
67################################################################################
68# Component-specific Jtreg settings
69################################################################################
70
71ifeq ($(TEST_JOBS), 0)
72  # If TEST_JOBS is not specified, hotspot fallback default is
73  # min(num_cores / 2, 12).
74  hotspot_JTREG_JOBS := $(shell $(EXPR) $(NUM_CORES) / 2)
75  ifeq ($(hotspot_JTREG_JOBS), 0)
76    hotspot_JTREG_JOBS := 1
77  else ifeq ($(shell $(EXPR) $(hotspot_JTREG_JOBS) \> 12), 1)
78    hotspot_JTREG_JOBS := 12
79  endif
80endif
81
82hotspot_JTREG_MAX_MEM := 0
83hotspot_JTREG_ASSERT := false
84hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native
85jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native
86
87
88################################################################################
89# Parse test selection
90#
91# The user has given a test selection in the TEST variable. We must parse it
92# and determine what that means in terms of actual calls to the test framework.
93#
94# The parse functions take as argument a test specification as given by the
95# user, and returns a fully qualified test descriptor if it was a match, or
96# nothing if not. A single test specification can result in multiple test
97# descriptors being returned. A valid test descriptor must always be accepted
98# and returned identically.
99################################################################################
100
101# Helper function to determine if a test specification is a Gtest test
102#
103# It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional
104# test filter string.
105define ParseGtestTestSelection
106  $(if $(filter gtest%, $1), \
107    $(if $(filter gtest, $1), \
108      gtest:all \
109    , \
110      $(if $(filter gtest:, $1), \
111        gtest:all \
112      , \
113        $1 \
114      ) \
115    ) \
116  )
117endef
118
119# Helper function to determine if a test specification is a Jtreg test
120#
121# It is a Jtreg test if it optionally begins with jtreg:, and then is either
122# an unspecified group name (possibly prefixed by :), or a group in a
123# specified <component>/test directory, or a path to a test or test directory,
124# either absolute or relative to TOPDIR.
125define ParseJtregTestSelection
126  $(eval TEST_NAME := $(strip $(patsubst jtreg:%, %, $1))) \
127  $(if $(or $(findstring :, $(TEST_NAME)), $(findstring /, $(TEST_NAME))), , \
128    $(eval TEST_NAME := :$(TEST_NAME)) \
129  ) \
130  $(if $(findstring :, $(TEST_NAME)), \
131    $(if $(filter :%, $(TEST_NAME)), \
132      $(foreach component, $(JTREG_COMPONENTS), \
133        $(if $(filter $(patsubst :%, %, $(TEST_NAME)), \
134            $($(component)_JTREG_TEST_GROUPS)), \
135          jtreg:$(component)/test:$(patsubst :%,%,$(TEST_NAME)) \
136        ) \
137      ) \
138    , \
139      $(eval COMPONENT := $(word 1, $(subst /, $(SPACE), $(TEST_NAME)))) \
140      $(eval GROUP := $(word 2, $(subst :, $(SPACE), $(TEST_NAME)))) \
141      $(if $(filter $(COMPONENT), $(JTREG_COMPONENTS)), \
142        $(if $(filter $(GROUP), $($(COMPONENT)_JTREG_TEST_GROUPS)), \
143          jtreg:$(TEST_NAME) \
144        ) \
145      ) \
146    ) \
147  , \
148    $(if $(filter /%, $(TEST_NAME)), \
149      $(if $(wildcard $(TEST_NAME)), \
150        jtreg:$(TEST_NAME) \
151      ) \
152    , \
153      $(if $(wildcard $(TOPDIR)/$(TEST_NAME)), \
154        jtreg:$(TEST_NAME) \
155      ) \
156    ) \
157  )
158endef
159
160ifeq ($(TEST), )
161  $(info No test selection given in TEST!)
162  $(info Please use e.g. 'run-test TEST=tier1' or 'run-test-tier1')
163  $(info See common/doc/testing.[md|html] for help)
164  $(error Cannot continue)
165endif
166
167# Now intelligently convert the test selection given by the user in TEST
168# into a list of fully qualified test descriptors of the tests to run.
169TESTS_TO_RUN :=
170$(foreach test, $(TEST), \
171  $(eval PARSED_TESTS := $(call ParseCustomTestSelection, $(test))) \
172  $(if $(strip $(PARSED_TESTS)), , \
173    $(eval PARSED_TESTS += $(call ParseGtestTestSelection, $(test))) \
174  ) \
175  $(if $(strip $(PARSED_TESTS)), , \
176    $(eval PARSED_TESTS += $(call ParseJtregTestSelection, $(test))) \
177  ) \
178  $(if $(strip $(PARSED_TESTS)), , \
179    $(eval UNKNOWN_TEST := $(test)) \
180  ) \
181  $(eval TESTS_TO_RUN += $(PARSED_TESTS)) \
182)
183
184ifneq ($(UNKNOWN_TEST), )
185  $(info Unknown test selection: '$(UNKNOWN_TEST)')
186  $(info See common/doc/testing.[md|html] for help)
187  $(error Cannot continue)
188endif
189
190TESTS_TO_RUN := $(strip $(TESTS_TO_RUN))
191
192
193# Present the result of our parsing to the user
194$(info Test selection '$(TEST)', will run:)
195$(foreach test, $(TESTS_TO_RUN), $(info * $(test)))
196
197
198################################################################################
199# Functions for setting up rules for running the selected tests
200#
201# The SetupRun*Test functions all have the same interface:
202#
203# Parameter 1 is the name of the rule. This is the test id, based on the test
204# descriptor, and this is also used as variable prefix, and the targets
205# generated are listed in a variable by that name.
206#
207# Remaining parameters are named arguments. Currently this is only:
208#   TEST -- The properly formatted fully qualified test descriptor
209#
210# After the rule named by the test id has been executed, the following
211# variables will be available:
212# testid_TOTAL - the total number of tests run
213# testid_PASSED - the number of successful tests
214# testid_FAILED - the number of failed tests
215# testid_ERROR - the number of tests was neither successful or failed
216#
217################################################################################
218
219### Rules for Gtest
220
221SetupRunGtestTest = $(NamedParamsMacroTemplate)
222define SetupRunGtestTestBody
223  $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
224  $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
225
226  $1_TEST_NAME := $$(strip $$(patsubst gtest:%, %, $$($1_TEST)))
227  ifneq ($$($1_TEST_NAME), all)
228    $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)*
229  endif
230
231  ifneq ($$(GTEST_REPEAT), )
232    $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT)
233  endif
234
235  run-test-$1:
236	$$(call LogWarn)
237	$$(call LogWarn, Running test '$$($1_TEST)')
238	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
239	$$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, \
240	    $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/server/gtestLauncher \
241	    -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \
242	    --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
243	    $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) \
244	    > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) || true )
245
246  $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
247
248  parse-test-$1: run-test-$1
249	$$(call LogWarn, Finished running test '$$($1_TEST)')
250	$$(call LogWarn, Test report is stored in $$(strip \
251	    $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
252	$$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \
253	    test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE)))
254	$$(eval $1_PASSED := $$(shell $$(AWK) '/\[  PASSED  \] .* tests?./ \
255	    { print $$$$4 }' $$($1_RESULT_FILE)))
256	$$(eval $1_FAILED := $$(shell $$(AWK) '/\[  FAILED  \] .* tests?, \
257	    listed below/ { print $$$$4 }' $$($1_RESULT_FILE)))
258	$$(if $$($1_FAILED), , $$(eval $1_FAILED := 0))
259	$$(eval $1_ERROR := $$(shell \
260	    $$(EXPR) $$($1_TOTAL) - $$($1_PASSED) - $$($1_FAILED)))
261
262  $1: run-test-$1 parse-test-$1
263
264  TARGETS += $1
265endef
266
267################################################################################
268
269### Rules for Jtreg
270
271# Helper function for SetupRunJtregTest. Set a JTREG_* variable from, in order:
272# 1) Specified by user on command line
273# 2) Component-specific default
274# 3) Generic default
275#
276# Note: No spaces are allowed around the arguments.
277# Arg $1 The test ID (i.e. $1 in SetupRunJtregTest)
278# Arg $2 Base variable, e.g. JTREG_JOBS
279# Arg $3 The default value (optional)
280define SetJtregValue
281  ifneq ($$($2), )
282    $1_$2 := $$($2)
283  else
284    ifneq ($$($$($1_COMPONENT)_$2), )
285      $1_$2 := $$($$($1_COMPONENT)_$2)
286    else
287      ifneq ($3, )
288        $1_$2 := $3
289      endif
290    endif
291  endif
292endef
293
294SetupRunJtregTest = $(NamedParamsMacroTemplate)
295define SetupRunJtregTestBody
296  $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
297  $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
298
299  $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST)))
300  $1_COMPONENT := $$(firstword $$(subst /, $$(SPACE), $$($1_TEST_NAME)))
301
302  ifeq ($$(JT_HOME), )
303    $$(info Error: jtreg framework is not found.)
304    $$(info Please run configure using --with-jtreg.)
305    $$(error Cannot continue)
306  endif
307
308  # Unfortunately, we need different defaults for some JTREG values,
309  # depending on what component we're running.
310
311  # Convert JTREG_foo into $1_JTREG_foo with a suitable value.
312  $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm))
313  $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true))
314  $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,512m))
315  $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH))
316  $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS))
317
318  ifneq ($(TEST_JOBS), 0)
319    # User has specified TEST_JOBS, use that as fallback default
320    $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
321  else
322    # Use JOBS as default (except for hotspot)
323    $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS)))
324  endif
325
326  ifeq ($$(shell $$(EXPR) $$($1_JTREG_JOBS) \> 50), 1)
327    # Until CODETOOLS-7901892 is fixed, JTreg cannot handle more than 50 jobs
328    $1_JTREG_JOBS := 50
329  endif
330
331  # Make sure MaxRAMPercentage is high enough to not cause OOM or swapping since
332  # we may end up with a lot of JVM's
333  $1_JTREG_MAX_RAM_PERCENTAGE := $$(shell $$(EXPR) 25 / $$($1_JTREG_JOBS))
334
335  JTREG_TIMEOUT ?= 4
336  JTREG_VERBOSE ?= fail,error,summary
337  JTREG_RETAIN ?= fail,error
338
339  ifneq ($$($1_JTREG_MAX_MEM), 0)
340    $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM)
341    $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM)
342  endif
343
344  $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \
345      -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \
346      -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT) \
347      -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE)
348
349  $1_JTREG_BASIC_OPTIONS += -automatic -keywords:\!ignore -ignore:quiet
350
351  # Some tests needs to find a boot JDK using the JDK8_HOME variable.
352  $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK)
353
354  $1_JTREG_BASIC_OPTIONS += \
355      $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \
356      $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \
357      #
358
359  ifeq ($$($1_JTREG_ASSERT), true)
360    $1_JTREG_BASIC_OPTIONS += -ea -esa
361  endif
362
363  ifneq ($$($1_JTREG_NATIVEPATH), )
364    $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH)
365  endif
366
367  ifneq ($$(JIB_JAR), )
368    $1_JTREG_BASIC_OPTIONS += -cpa:$$(JIB_JAR)
369  endif
370
371  run-test-$1:
372	$$(call LogWarn)
373	$$(call LogWarn, Running test '$$($1_TEST)')
374	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
375	$$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, \
376	    $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
377	        -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
378	        $$($1_JTREG_BASIC_OPTIONS) \
379	        -testjdk:$$(JDK_IMAGE_DIR) \
380	        -dir:$$(TOPDIR) \
381	        -reportDir:$$($1_TEST_RESULTS_DIR) \
382	        -workDir:$$($1_TEST_SUPPORT_DIR) \
383	        $$(JTREG_OPTIONS) \
384	        $$($1_TEST_NAME) || true )
385
386  $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt
387
388  parse-test-$1: run-test-$1
389	$$(call LogWarn, Finished running test '$$($1_TEST)')
390	$$(call LogWarn, Test report is stored in $$(strip \
391	    $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
392	$$(eval $1_PASSED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \
393	    for (i=1; i<=NF; i++) { if ($$$$i == "passed:") \
394	    print $$$$(i+1) } }' $$($1_RESULT_FILE)))
395	$$(if $$($1_PASSED), , $$(eval $1_PASSED := 0))
396	$$(eval $1_FAILED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
397	    for (i=1; i<=NF; i++) { if ($$$$i == "failed:") \
398	    print $$$$(i+1) } }' $$($1_RESULT_FILE)))
399	$$(if $$($1_FAILED), , $$(eval $1_FAILED := 0))
400	$$(eval $1_ERROR := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
401	    for (i=1; i<=NF; i++) { if ($$$$i == "error:") \
402	    print $$$$(i+1) } }' $$($1_RESULT_FILE)))
403	$$(if $$($1_ERROR), , $$(eval $1_ERROR := 0))
404	$$(eval $1_TOTAL := $$(shell \
405	    $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR)))
406
407  $1: run-test-$1 parse-test-$1
408
409  TARGETS += $1
410endef
411
412
413################################################################################
414# Setup and execute make rules for all selected tests
415################################################################################
416
417# Helper function to determine which handler to use for the given test
418UseGtestTestHandler = \
419  $(if $(filter gtest:%, $1), true)
420
421UseJtregTestHandler = \
422  $(if $(filter jtreg:%, $1), true)
423
424# Now process each test to run and setup a proper make rule
425$(foreach test, $(TESTS_TO_RUN), \
426  $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
427      $(TR) -cs '[a-z][A-Z][0-9]\n' '_')) \
428  $(eval ALL_TEST_IDS += $(TEST_ID)) \
429  $(if $(call UseCustomTestHandler, $(test)), \
430    $(eval $(call SetupRunCustomTest, $(TEST_ID), \
431        TEST := $(test), \
432    )) \
433  ) \
434  $(if $(call UseGtestTestHandler, $(test)), \
435    $(eval $(call SetupRunGtestTest, $(TEST_ID), \
436        TEST := $(test), \
437    )) \
438  ) \
439  $(if $(call UseJtregTestHandler, $(test)), \
440    $(eval $(call SetupRunJtregTest, $(TEST_ID), \
441        TEST := $(test), \
442    )) \
443  ) \
444)
445
446# Sort also removes duplicates, so if there is any we'll get fewer words.
447ifneq ($(words $(ALL_TEST_IDS)), $(words $(sort $(ALL_TEST_IDS))))
448  $(error Duplicate test specification)
449endif
450
451
452################################################################################
453# The main target for RunTests.gmk
454################################################################################
455
456# The SetupRun*Test functions have populated TARGETS.
457
458TEST_FAILURE := false
459
460run-test: $(TARGETS)
461        # Print a table of the result of all tests run and their result
462	$(ECHO)
463	$(ECHO) ==============================
464	$(ECHO) Test summary
465	$(ECHO) ==============================
466	$(PRINTF) "%2s %-49s %5s %5s %5s %5s %2s\n" "  " TEST \
467	    TOTAL PASS FAIL ERROR " "
468	$(foreach test, $(TESTS_TO_RUN), \
469	  $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
470	      $(TR) -cs '[a-z][A-Z][0-9]\n' '_')) \
471	  $(if $(filter $($(TEST_ID)_PASSED), $($(TEST_ID)_TOTAL)), \
472	    $(PRINTF) "%2s %-49s %5d %5d %5d %5d %2s\n" "  " "$(test)" \
473	        $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) $($(TEST_ID)_FAILED) \
474	        $($(TEST_ID)_ERROR) "  " $(NEWLINE) \
475	  , \
476	    $(PRINTF) "%2s %-49s %5d %5d %5d %5d %2s\n" ">>" "$(test)" \
477	        $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) $($(TEST_ID)_FAILED) \
478	        $($(TEST_ID)_ERROR) "<<" $(NEWLINE) \
479	    $(eval TEST_FAILURE := true) \
480	  ) \
481	)
482	$(ECHO) ==============================
483	$(if $(filter true, $(TEST_FAILURE)), \
484	  $(ECHO) TEST FAILURE $(NEWLINE) \
485	  $(TOUCH) $(MAKESUPPORT_OUTPUTDIR)/exit-with-error \
486	, \
487	  $(ECHO) TEST SUCCESS \
488	)
489	$(ECHO)
490
491################################################################################
492
493all: run-test
494
495.PHONY: default all run-test $(TARGETS)
496