gbuild: Framework for performance unit tests using callgrind

Run them with "make perfcheck" (toplevel or in a module)

To add a new performance test:
  - Add it as a perfcheck target
  - Use gb_CppunitTest_set_performance_test
  - Use CALLGRIND_ macros from valgrind/callgrind.h
    http://valgrind.org/docs/manual/cl-manual.html#cl-manual.clientrequests

Change-Id: I67c776dbe4db0a686607efeee7a7e4f3aeae0e5c

Makefile.in

@@ -7,13 +7,15 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 
-.PHONY : all bootstrap gbuild build build-non-l10n-only build-l10n-only check clean clean-build clean-host test-install distclean distro-pack-install docs download fetch findunusedcode get-submodules id install install-strip subsequentcheck tags debugrun help slowcheck translations unitcheck packageinfo internal.clean
+gb_Top_MODULE_CHECK_TARGETS := slowcheck unitcheck subsequentcheck perfcheck
+
+.PHONY : all bootstrap gbuild build build-non-l10n-only build-l10n-only check clean clean-build clean-host test-install distclean distro-pack-install docs download fetch findunusedcode get-submodules id install install-strip tags debugrun help translations packageinfo internal.clean $(gb_Top_MODULE_CHECK_TARGETS)
 
 MAKECMDGOALS?=all
 build_goal:=$(if $(filter build check,$(MAKECMDGOALS)),all)\
  $(if $(filter build-nocheck,$(MAKECMDGOALS)),build)\
  $(if $(filter check,$(MAKECMDGOALS)),subsequentcheck)\
- $(filter all build-l10n-only build-non-l10n-only debugrun help slowcheck translations unitcheck subsequentcheck check packageinfo,$(MAKECMDGOALS))
+ $(filter all build-l10n-only build-non-l10n-only debugrun help translations $(gb_Top_MODULE_CHECK_TARGETS) check packageinfo,$(MAKECMDGOALS))
 
 SHELL := @SHELL_BASH@
 SRCDIR := @SRC_ROOT@
@@ -61,12 +63,12 @@ PARALLELISM_OPTION := $(if $(filter-out 0,$(PARALLELISM)),-j $(PARALLELISM),)
 # Partial Build
 #
 define gb_Top_GbuildModuleRules
-.PHONY: $(1) $(1).all $(1).build $(1).check $(1).clean $(1).showdeliverables $(1).subsequentcheck $(1).slowcheck
+.PHONY: $(1) $(1).all $(1).build $(1).check $(1).clean $(1).showdeliverables $(foreach target,$(gb_Top_MODULE_CHECK_TARGETS),$(1).$(target))
 
 $(1): bootstrap fetch
 	cd $(SRCDIR)/$(2) && $$(MAKE) $(PARALLELISM_OPTION) $(GMAKE_OPTIONS)
 
-$(1).build $(1).check $(1).clean $(1).showdeliverables $(1).subsequentcheck $(1).slowcheck:
+$(1).build $(1).check $(1).clean $(1).showdeliverables $(foreach target,$(gb_Top_MODULE_CHECK_TARGETS),$(1).$(target)):
 	cd $(SRCDIR)/$(2) && $$(MAKE) $(PARALLELISM_OPTION) $(GMAKE_OPTIONS) $$(patsubst $(1).%,%,$$@)
 
 $(1).all: bootstrap fetch
@@ -233,7 +235,7 @@ ifeq ($(OS),IOS)
 	$(MAKE) $(PARALLELISM_OPTION) $(GMAKE_OPTIONS) ios
 endif
 
-build-non-l10n-only build-l10n-only build-nocheck check debugrun help slowcheck translations unitcheck subsequentcheck packageinfo: build
+build-non-l10n-only build-l10n-only build-nocheck check debugrun help translations packageinfo $(gb_Top_MODULE_CHECK_TARGETS): build
 
 cross-toolset: bootstrap fetch
 	$(MAKE) gb_Side=build $(PARALLELISM_OPTION) $(GMAKE_OPTIONS) -f $(SRCDIR)/Makefile.gbuild build-tools

solenv/gbuild/CppunitTest.mk

@@ -94,7 +94,10 @@ $(call gb_CppunitTest_get_target,%) :| $(gb_CppunitTest_CPPTESTDEPS)
 			|| ($(if $(value gb_CppunitTest_postprocess), \
 					RET=$$?; \
 					$(call gb_CppunitTest_postprocess,$(gb_CppunitTest_CPPTESTCOMMAND),$@.core,$$RET) >> $@.log 2>&1;) \
-				cat $@.log; $(SRCDIR)/solenv/bin/unittest-failed.sh Cppunit $*))))
+				cat $@.log; $(SRCDIR)/solenv/bin/unittest-failed.sh Cppunit $*))) \
+		$(if $(PERFTEST), && VAL=$$(grep '^==.*== Collected : ' $@.log | sed "s/==.*== Collected : //") && \
+			$(if $(filter 0,$(PERFTEST)), expr "$$VAL" "*" "101" "/" "100", test $$VAL -le $(PERFTEST) || (echo "Unit test is slow! $$VAL instructions detected." && false))) \
+	)
 
 define gb_CppunitTest_CppunitTest
 $(call gb_CppunitTest__CppunitTest_impl,$(1),$(call gb_CppunitTest_get_linktarget,$(1)))
@@ -144,6 +147,21 @@ $(call gb_CppunitTest_get_target,$(1)) : ARGS += $(2)
 
 endef
 
+# Run this unit test with callgrind tool to measure performance. If you are
+# creating a new test, first set the number to 0, run the test, and it will
+# print an number for you to use as a second parameter here. The test will
+# fail if reported number of instructions will be bigger than this parameter.
+#
+# call gb_CppunitTest_set_performance_test,name,instructions_number
+define gb_CppunitTest_set_performance_test
+$(if $(ENABLE_VALGRIND),,$(call gb_Output_error,gb_CppunitTest_set_performance_test used with empty $$(ENABLE_VALGRIND)))
+
+$(call gb_CppunitTest_get_target,$(1)) : PERFTEST := $(2)
+$(call gb_CppunitTest_get_target,$(1)) : gb_CppunitTest_VALGRINDTOOL := valgrind --tool=callgrind --dump-instr=yes --instr-atstart=no
+$(call gb_CppunitTest_use_external,$(1),valgrind)
+
+endef
+
 define gb_CppunitTest_use_ure
 $(call gb_CppunitTest_use_rdb,$(1),ure/services)
 $(call gb_CppunitTest_get_target,$(1)) : URE := $(true)

solenv/gbuild/Module.mk

@@ -32,6 +32,7 @@
 # unitcheck (global)          run unit tests               top-level Module/unitcheck
 # slowcheck (global)          run slow unit tests          top-level Module/slowcheck
 # subsequentcheck (global)    run system tests             top-level Module/subsequentcheck
+# perfcheck (global)          run performance unit tests   top-level Module/perfcheck
 # all (global)                default goal                 build unitcheck
 
 
@@ -44,6 +45,7 @@ gb_Module_L10NTARGETSTACK :=
 gb_Module_CHECKTARGETSTACK :=
 gb_Module_SLOWCHECKTARGETSTACK :=
 gb_Module_SUBSEQUENTCHECKTARGETSTACK :=
+gb_Module_PERFCHECKTARGETSTACK :=
 gb_Module_CLEANTARGETSTACK :=
 
 # The currently read gbuild makefile.
@@ -65,7 +67,7 @@ $(call gb_Module_get_clean_target,%) :
 	$(call gb_Output_announce,$*,$(false),MOD,5)
 	$(call gb_Output_announce_title,module $* cleared.)
 	-$(call gb_Helper_abbreviate_dirs,\
-		rm -f $(call gb_Module_get_target,$*) $(call gb_Module_get_nonl10n_target,$*) $(call gb_Module_get_l10n_target,$*) $(call gb_Module_get_check_target,$*) $(call gb_Module_get_slowcheck_target,$*) $(call gb_Module_get_subsequentcheck_target,$*))
+		rm -f $(call gb_Module_get_target,$*) $(call gb_Module_get_nonl10n_target,$*) $(call gb_Module_get_l10n_target,$*) $(call gb_Module_get_check_target,$*) $(call gb_Module_get_slowcheck_target,$*) $(call gb_Module_get_subsequentcheck_target,$*) $(call gb_Module_get_perfcheck_target,$*))
 
 $(call gb_Module_get_l10n_target,%) :
 	$(call gb_Output_announce,$*,$(true),LOC,5)
@@ -95,6 +97,13 @@ $(call gb_Module_get_subsequentcheck_target,%) :
 		mkdir -p $(dir $@) && \
 		touch $@)
 
+$(call gb_Module_get_perfcheck_target,%) :
+	$(call gb_Output_announce,$*,$(true),PFC,5)
+	$(call gb_Output_announce_title,module $* perfchecks done.)
+	-$(call gb_Helper_abbreviate_dirs,\
+		mkdir -p $(dir $@) && \
+		touch $@)
+
 $(call gb_Module_get_target,%) :
 	$(call gb_Output_announce,$*,$(true),MOD,5)
 	$(call gb_Output_announce_title,module $* done.)
@@ -102,7 +111,7 @@ $(call gb_Module_get_target,%) :
 		mkdir -p $(dir $@) && \
 		touch $@)
 
-.PHONY : all build build-l10n-only build-non-l10n-only unitcheck slowcheck subsequentcheck clean check debugrun help showmodules translations
+.PHONY : all build build-l10n-only build-non-l10n-only unitcheck slowcheck subsequentcheck perfcheck clean check debugrun help showmodules translations
 .DEFAULT_GOAL := all
 
 all : build $(if $(CROSS_COMPILING),,unitcheck $(if $(gb_PARTIAL_BUILD),,slowcheck))
@@ -147,6 +156,11 @@ subsequentcheck :
 	$(call gb_Output_announce_title,all subsequent tests checked.)
 	$(call gb_Output_announce_bell)
 
+perfcheck :
+	$(call gb_Output_announce,loaded modules: $(sort $(gb_Module_ALLMODULES)),$(true),PFC,6)
+	$(call gb_Output_announce_title,all perftests checked.)
+	$(call gb_Output_announce_bell)
+
 clean :
 	$(call gb_Output_announce,top level modules: $(foreach module,$^,$(notdir $(module))),$(false),ALL,6)
 	$(call gb_Output_announce,loaded modules: $(sort $(gb_Module_ALLMODULES)),$(false),ALL,6)
@@ -191,6 +205,7 @@ gb_Module_L10NTARGETSTACK := $(call gb_Module_get_l10n_target,$(1)) $(gb_Module_
 gb_Module_CHECKTARGETSTACK := $(call gb_Module_get_check_target,$(1)) $(gb_Module_CHECKTARGETSTACK)
 gb_Module_SLOWCHECKTARGETSTACK := $(call gb_Module_get_slowcheck_target,$(1)) $(gb_Module_SLOWCHECKTARGETSTACK)
 gb_Module_SUBSEQUENTCHECKTARGETSTACK := $(call gb_Module_get_subsequentcheck_target,$(1)) $(gb_Module_SUBSEQUENTCHECKTARGETSTACK)
+gb_Module_PERFCHECKTARGETSTACK := $(call gb_Module_get_perfcheck_target,$(1)) $(gb_Module_PERFCHECKTARGETSTACK)
 gb_Module_CLEANTARGETSTACK := $(call gb_Module_get_clean_target,$(1)) $(gb_Module_CLEANTARGETSTACK)
 gb_Module_CURRENTMODULE_DEBUG_ENABLED := $(call gb_Module__debug_enabled,$(1))
 gb_Module_CURRENTMODULE_NAME := $(1)
@@ -272,6 +287,14 @@ $(call gb_Module_get_clean_target,$(1)) : $$(gb_Module_CURRENTCLEANTARGET)
 
 endef
 
+define gb_Module_add_perfcheck_target
+$(call gb_Module__read_targetfile,$(1),$(2),perfcheck target)
+
+$(call gb_Module_get_perfcheck_target,$(1)) : $$(gb_Module_CURRENTTARGET)
+$(call gb_Module_get_clean_target,$(1)) : $$(gb_Module_CURRENTCLEANTARGET)
+
+endef
+
 define gb_Module_add_moduledir
 include $(patsubst $(1):%,%,$(filter $(1):%,$(gb_Module_MODULELOCATIONS)))/$(2)/Module_$(2).mk
 $(call gb_Module_get_target,$(1)) : $$(firstword $$(gb_Module_TARGETSTACK))
@@ -279,12 +302,14 @@ $(call gb_Module_get_l10n_target,$(1)) : $$(firstword $$(gb_Module_L10NTARGETSTA
 $(call gb_Module_get_check_target,$(1)) : $$(firstword $$(gb_Module_CHECKTARGETSTACK))
 $(call gb_Module_get_slowcheck_target,$(1)) : $$(firstword $$(gb_Module_SLOWCHECKTARGETSTACK))
 $(call gb_Module_get_subsequentcheck_target,$(1)) : $$(firstword $$(gb_Module_SUBSEQUENTCHECKTARGETSTACK))
+$(call gb_Module_get_perfcheck_target,$(1)) : $$(firstword $$(gb_Module_PERFCHECKTARGETSTACK))
 $(call gb_Module_get_clean_target,$(1)) : $$(firstword $$(gb_Module_CLEANTARGETSTACK))
 gb_Module_TARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_TARGETSTACK)),$$(gb_Module_TARGETSTACK))
 gb_Module_L10NTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_L10NTARGETSTACK)),$$(gb_Module_L10NTARGETSTACK))
 gb_Module_CHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_CHECKTARGETSTACK)),$$(gb_Module_CHECKTARGETSTACK))
 gb_Module_SLOWCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_SLOWCHECKTARGETSTACK)),$$(gb_Module_SLOWCHECKTARGETSTACK))
 gb_Module_SUBSEQUENTCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_SUBSEQUENTCHECKTARGETSTACK)),$$(gb_Module_SUBSEQUENTCHECKTARGETSTACK))
+gb_Module_PERFCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_PERFCHECKTARGETSTACK)),$$(gb_Module_PERFCHECKTARGETSTACK))
 gb_Module_CLEANTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_CLEANTARGETSTACK)),$$(gb_Module_CLEANTARGETSTACK))
 
 endef
@@ -318,6 +343,11 @@ $(foreach target,$(2),$(call gb_Module_add_subsequentcheck_target,$(1),$(target)
 
 endef
 
+define gb_Module_add_perfcheck_targets
+$(foreach target,$(2),$(call gb_Module_add_perfcheck_target,$(1),$(target)))
+
+endef
+
 define gb_Module_add_moduledirs
 $(foreach target,$(sort $(2)),$(call gb_Module_add_moduledir,$(1),$(target)))
 
@@ -336,6 +366,7 @@ build-l10n-only : $$(firstword $$(gb_Module_L10NTARGETSTACK))
 unitcheck : $$(firstword $$(gb_Module_CHECKTARGETSTACK))
 slowcheck : $$(firstword $$(gb_Module_SLOWCHECKTARGETSTACK))
 subsequentcheck : $$(firstword $$(gb_Module_SUBSEQUENTCHECKTARGETSTACK))
+perfcheck : $$(firstword $$(gb_Module_PERFCHECKTARGETSTACK))
 clean : $$(firstword $$(gb_Module_CLEANTARGETSTACK))
 
 ifneq ($$(words $$(gb_Module_TARGETSTACK)),1)
@@ -347,9 +378,10 @@ gb_Module_L10NTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_L10NTARGETSTACK
 gb_Module_CHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_CHECKTARGETSTACK)),$$(gb_Module_CHECKTARGETSTACK))
 gb_Module_SLOWCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_SLOWCHECKTARGETSTACK)),$$(gb_Module_SLOWCHECKTARGETSTACK))
 gb_Module_SUBSEQUENTCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_SUBSEQUENTCHECKTARGETSTACK)),$$(gb_Module_SUBSEQUENTCHECKTARGETSTACK))
+gb_Module_PERFCHECKTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_PERFCHECKTARGETSTACK)),$$(gb_Module_PERFCHECKTARGETSTACK))
 gb_Module_CLEANTARGETSTACK := $$(wordlist 2,$$(words $$(gb_Module_CLEANTARGETSTACK)),$$(gb_Module_CLEANTARGETSTACK))
 
-ifneq ($$(and $$(gb_Module_TARGETSTACK),$$(gb_Module_CHECKTARGETSTACK),$$(gb_Module_SLOWCHECKTARGETSTACK),$$(gb_Module_SUBSEQUENTCHECKTARGETSTACK),$$(gb_Module_L10NTARGETSTACK)),)
+ifneq ($$(and $$(gb_Module_TARGETSTACK),$$(gb_Module_CHECKTARGETSTACK),$$(gb_Module_SLOWCHECKTARGETSTACK),$$(gb_Module_SUBSEQUENTCHECKTARGETSTACK),$$(gb_Module_PERFCHECKTARGETSTACK),$$(gb_Module_L10NTARGETSTACK)),)
 $$(eval $$(call gb_Output_error,Corrupted module target stack!3))
 endif
 

solenv/gbuild/TargetLocations.mk

@@ -119,6 +119,7 @@ gb_Module_get_l10n_target = $(WORKDIR)/Module/l10n/$(1)
 gb_Module_get_check_target = $(WORKDIR)/Module/check/$(1)
 gb_Module_get_slowcheck_target = $(WORKDIR)/Module/slowcheck/$(1)
 gb_Module_get_subsequentcheck_target = $(WORKDIR)/Module/subsequentcheck/$(1)
+gb_Module_get_perfcheck_target = $(WORKDIR)/Module/perfcheck/$(1)
 gb_Module_get_target = $(WORKDIR)/Module/$(1)
 gb_ObjCxxObject_get_target = $(WORKDIR)/ObjCxxObject/$(1).o
 gb_ObjCObject_get_target = $(WORKDIR)/ObjCObject/$(1).o

solenv/gbuild/extensions/post_SpeedUpTargets.mk

@@ -20,6 +20,10 @@ ifeq ($(MAKECMDGOALS),build)
 gb_Module_SKIPTARGETS := check slowcheck subsequentcheck
 endif
 
+ifeq (,$(filter perfcheck,$(MAKECMDGOALS)))
+gb_Module_SKIPTARGETS += perfcheck
+endif
+
 ifneq ($(strip $(MAKECMDGOALS)),)
 # speed up depending on the target
 gb_SpeedUpTargets_LEVEL_4 := debugrun help translations install-package-% packageinfo
@@ -63,6 +67,10 @@ ifneq (,$(filter subsequentcheck,$(gb_Module_SKIPTARGETS)))
 gb_Module_add_subsequentcheck_target =
 endif
 
+ifneq (,$(filter perfcheck,$(gb_Module_SKIPTARGETS)))
+gb_Module_add_perfcheck_target =
+endif
+
 ifneq (,$(filter module,$(gb_Module_SKIPTARGETS)))
 gb_Module_add_moduledir =
 endif

solenv/gbuild/gbuild.help.txt

@@ -32,6 +32,7 @@ AVAILABLE TARGETS
        unitcheck        run unit tests
        slowcheck        run slow unit tests
        subsequentcheck  run system tests (requires full installation)
+       perfcheck        run performance/callgrind unit tests
        check            run unit tests and if in toplevel subsequentcheck
        clean            remove all generated files
        debugrun         starts the INSTDIR instance and allows tests to be run