diff mbox

[3/3] benchtests:Enable BENCHSET to run subset of tests

Message ID 20171116225233.2697-3-victor.rodriguez.bahena@intel.com
State New, archived
Headers show

Commit Message

Victor Rodriguez Nov. 16, 2017, 10:52 p.m. UTC
This patch adds BENCHSET variable to benchtests/Makefile in order to
provide the capability to run a list of subsets of benchmark tests, ie;

    make bench BENCHSET="bench-pthread bench-math malloc-thread"

This Helps users to benchmark specific glibc area

Changelog:
2017-11-15  Victor Rodriguez  <victor.rodriguez.bahena@intel.com>

	(VERSION): Set to 2.27
        * benchtests/Makefile:Add BENCHSET to allow subsets of
        benchmarks to be run.
        * benchtests/README: Add documentation for: Running subsets of
        benchmarks.

Signed-off-by: Victor Rodriguez <victor.rodriguez.bahena@intel.com>
Signed-off-by: Icarus Sparry <icarus.w.sparry@intel.com>
---
 ChangeLog           |  9 +++++++
 benchtests/Makefile | 69 +++++++++++++++++++++++++++++++++++++----------------
 benchtests/README   | 20 ++++++++++++++++
 3 files changed, 78 insertions(+), 20 deletions(-)

Comments

Siddhesh Poyarekar Nov. 22, 2017, 10:34 a.m. UTC | #1
Hi,

Almost there, comments inline.  Please post the final version with those
changes.

Thanks,
Siddhesh

On Friday 17 November 2017 04:22 AM, Victor Rodriguez wrote:
> This patch adds BENCHSET variable to benchtests/Makefile in order to
> provide the capability to run a list of subsets of benchmark tests, ie;
> 
>     make bench BENCHSET="bench-pthread bench-math malloc-thread"
> 
> This Helps users to benchmark specific glibc area
> 
> Changelog:
> 2017-11-15  Victor Rodriguez  <victor.rodriguez.bahena@intel.com>
> 
> 	(VERSION): Set to 2.27
>         * benchtests/Makefile:Add BENCHSET to allow subsets of
>         benchmarks to be run.
>         * benchtests/README: Add documentation for: Running subsets of
>         benchmarks.
> 
> Signed-off-by: Victor Rodriguez <victor.rodriguez.bahena@intel.com>
> Signed-off-by: Icarus Sparry <icarus.w.sparry@intel.com>
> ---
>  ChangeLog           |  9 +++++++
>  benchtests/Makefile | 69 +++++++++++++++++++++++++++++++++++++----------------
>  benchtests/README   | 20 ++++++++++++++++
>  3 files changed, 78 insertions(+), 20 deletions(-)
> 
> diff --git a/ChangeLog b/ChangeLog
> index 6a6bb62..bf61c72 100644
> --- a/ChangeLog
> +++ b/ChangeLog
> @@ -1,3 +1,12 @@
> +Changelog:
> +2017-11-15  Victor Rodriguez  <victor.rodriguez.bahena@intel.com>
> +
> +	(VERSION): Set to 2.27

What is this line for?

> +		* benchtests/Makefile:Add BENCHSET to allow subsets of
> +		benchmarks to be run
> +		* benchtests/README: Add documentation for: Running subsets of
> +		benchmarks.
> +
>  2017-11-15  Victor Rodriguez  <victor.rodriguez.bahena@intel.com>
>  
>  	(VERSION): Set to 2.27
> diff --git a/benchtests/Makefile b/benchtests/Makefile
> index 37788e8..6712710 100644
> --- a/benchtests/Makefile
> +++ b/benchtests/Makefile
> @@ -29,7 +29,11 @@ bench-pthread := pthread_once thread_create
>  
>  bench-string := ffs ffsll
>  
> +ifeq (${BENCHSET},)
>  bench := $(bench-math) $(bench-pthread) $(bench-string)
> +else
> +bench := $(foreach B,$(filter bench-%,${BENCHSET}), ${${B}})
> +endif

OK.

>  
>  # String function benchmarks.
>  string-benchset := bcopy bzero memccpy memchr memcmp memcpy memmem memmove \
> @@ -66,8 +70,12 @@ stdio-common-benchset := sprintf
>  
>  math-benchset := math-inlines
>  
> +ifeq (${BENCHSET},)
>  benchset := $(string-benchset-all) $(stdlib-benchset) $(stdio-common-benchset) \
>  	    $(math-benchset)
> +else
> +benchset := $(foreach B,$(filter %-benchset,${BENCHSET}), ${${B}})
> +endif

OK.

>  
>  CFLAGS-bench-ffs.c += -fno-builtin
>  CFLAGS-bench-ffsll.c += -fno-builtin
> @@ -77,7 +85,11 @@ CFLAGS-bench-fminf.c += -fno-builtin
>  CFLAGS-bench-fmax.c += -fno-builtin
>  CFLAGS-bench-fmaxf.c += -fno-builtin
>  
> +ifeq (${BENCHSET},)
>  bench-malloc := malloc-thread
> +else
> +bench-malloc := $(filter malloc-%,${BENCHSET})
> +endif

OK.

>  
>  $(addprefix $(objpfx)bench-,$(bench-math)): $(libm)
>  $(addprefix $(objpfx)bench-,$(math-benchset)): $(libm)
> @@ -144,6 +156,19 @@ bench-clean:
>  	rm -f $(timing-type) $(addsuffix .o,$(timing-type))
>  	rm -f $(addprefix $(objpfx),$(bench-extra-objs))
>  
> +# Validate the passed in BENCHSET
> +ifneq ($(strip ${BENCHSET}),)
> +VALIDBENCHSETNAMES := bench-pthread bench-math bench-string string-benchset \
> +	wcsmbs-benchset stdlib-benchset stdio-common-benchset math-benchset \
> +	malloc-thread
> +INVALIDBENCHSETNAMES := $(filter-out ${VALIDBENCHSETNAMES},${BENCHSET})
> +ifneq (${INVALIDBENCHSETNAMES},)
> +$(info The following values in BENCHSET are invalid: ${INVALIDBENCHSETNAMES})
> +$(info The valid ones are: ${VALIDBENCHSETNAMES})
> +$(error Invalid BENCHSET value)
> +endif
> +endif
> +
>  # Define the bench target only if the target has a usable python installation.
>  ifdef PYTHON
>  bench: bench-build bench-set bench-func bench-malloc
> @@ -170,10 +195,11 @@ bench-set: $(binaries-benchset)
>  	done
>  
>  bench-malloc: $(binaries-bench-malloc)
> -	run=$(objpfx)bench-malloc-thread; \
> -	for thr in 1 8 16 32; do \
> -	  echo "Running $${run} $${thr}"; \
> +	for run in $^; do \
> +		for thr in 1 8 16 32; do \
> +		  echo "Running $${run} $${thr}"; \
>  	  $(run-bench) $${thr} > $${run}-$${thr}.out; \
> +		done;\
>  	done

OK.

>  
>  # Build and execute the benchmark functions.  This target generates JSON
> @@ -181,25 +207,28 @@ bench-malloc: $(binaries-bench-malloc)
>  # so one could even execute them individually and process it using any JSON
>  # capable language or tool.
>  bench-func: $(binaries-bench)
> +	if [ -n '$^' ] ; then \
>  	{ timing_type=$$($(timing-type)); \
> -	echo "{\"timing_type\": \"$${timing_type}\","; \
> -	echo " \"functions\": {"; \
> -	for run in $^; do \
> -	  if ! [ "x$${run}" = "x$<" ]; then \
> -	    echo ","; \
> +	  echo "{\"timing_type\": \"$${timing_type}\","; \
> +	  echo " \"functions\": {"; \
> +	  for run in $^; do \
> +	    if ! [ "x$${run}" = "x$<" ]; then \
> +	      echo ","; \
> +	    fi; \
> +	    echo "Running $${run}" >&2; \
> +	    $(run-bench) $(DETAILED_OPT); \
> +	  done; \
> +	  echo; \
> +	  echo " }"; \
> +	  echo "}"; \
> +	  } > $(objpfx)bench.out-tmp; \
> +	  if [ -f $(objpfx)bench.out ]; then \
> +	    mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
>  	  fi; \
> -	  echo "Running $${run}" >&2; \
> -	  $(run-bench) $(DETAILED_OPT); \
> -	done; \
> -	echo; \
> -	echo " }"; \
> -	echo "}"; } > $(objpfx)bench.out-tmp; \
> -	if [ -f $(objpfx)bench.out ]; then \
> -	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
> -	fi; \
> -	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
> -	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
> -		scripts/benchout.schema.json
> +	  mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out; \
> +	  $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
> +	  scripts/benchout.schema.json; \
> +	fi
>  
>  $(timing-type) $(binaries-bench) $(binaries-benchset) \
>  	$(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \
> diff --git a/benchtests/README b/benchtests/README
> index b015acf..20942c7 100644
> --- a/benchtests/README
> +++ b/benchtests/README
> @@ -53,6 +53,26 @@ otherwise the above command may try to build the benchmark again.  Benchmarks
>  that require generated code to be executed during the build are skipped when
>  cross-building.
>  
> +Running subsets of benchmarks:
> +==============================
> +
> +If a subset of benchmarks needs to be run one may run the benchmark by invoking
> +make as follows:

The following phrasing may be more succint:

"To run only a subset of benchmarks, one may invoke make as follows"

> +
> +  $ make bench BENCHSET="bench-pthread bench-math malloc-thread"
> +
> +Allowing to measure the performance of specific glibc area. Posible areas are:

Drop this line and add:

"where BENCHSET may be a space-separated list of the following values:"

> +
> +    bench-math
> +    bench-pthread
> +    bench-string
> +    string-benchset
> +    wcsmbs-benchset
> +    stdlib-benchset
> +    stdio-common-benchset
> +    math-benchset
> +    malloc-thread
> +
>  Adding a function to benchtests:
>  ===============================
>  
>
diff mbox

Patch

diff --git a/ChangeLog b/ChangeLog
index 6a6bb62..bf61c72 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@ 
+Changelog:
+2017-11-15  Victor Rodriguez  <victor.rodriguez.bahena@intel.com>
+
+	(VERSION): Set to 2.27
+		* benchtests/Makefile:Add BENCHSET to allow subsets of
+		benchmarks to be run
+		* benchtests/README: Add documentation for: Running subsets of
+		benchmarks.
+
 2017-11-15  Victor Rodriguez  <victor.rodriguez.bahena@intel.com>
 
 	(VERSION): Set to 2.27
diff --git a/benchtests/Makefile b/benchtests/Makefile
index 37788e8..6712710 100644
--- a/benchtests/Makefile
+++ b/benchtests/Makefile
@@ -29,7 +29,11 @@  bench-pthread := pthread_once thread_create
 
 bench-string := ffs ffsll
 
+ifeq (${BENCHSET},)
 bench := $(bench-math) $(bench-pthread) $(bench-string)
+else
+bench := $(foreach B,$(filter bench-%,${BENCHSET}), ${${B}})
+endif
 
 # String function benchmarks.
 string-benchset := bcopy bzero memccpy memchr memcmp memcpy memmem memmove \
@@ -66,8 +70,12 @@  stdio-common-benchset := sprintf
 
 math-benchset := math-inlines
 
+ifeq (${BENCHSET},)
 benchset := $(string-benchset-all) $(stdlib-benchset) $(stdio-common-benchset) \
 	    $(math-benchset)
+else
+benchset := $(foreach B,$(filter %-benchset,${BENCHSET}), ${${B}})
+endif
 
 CFLAGS-bench-ffs.c += -fno-builtin
 CFLAGS-bench-ffsll.c += -fno-builtin
@@ -77,7 +85,11 @@  CFLAGS-bench-fminf.c += -fno-builtin
 CFLAGS-bench-fmax.c += -fno-builtin
 CFLAGS-bench-fmaxf.c += -fno-builtin
 
+ifeq (${BENCHSET},)
 bench-malloc := malloc-thread
+else
+bench-malloc := $(filter malloc-%,${BENCHSET})
+endif
 
 $(addprefix $(objpfx)bench-,$(bench-math)): $(libm)
 $(addprefix $(objpfx)bench-,$(math-benchset)): $(libm)
@@ -144,6 +156,19 @@  bench-clean:
 	rm -f $(timing-type) $(addsuffix .o,$(timing-type))
 	rm -f $(addprefix $(objpfx),$(bench-extra-objs))
 
+# Validate the passed in BENCHSET
+ifneq ($(strip ${BENCHSET}),)
+VALIDBENCHSETNAMES := bench-pthread bench-math bench-string string-benchset \
+	wcsmbs-benchset stdlib-benchset stdio-common-benchset math-benchset \
+	malloc-thread
+INVALIDBENCHSETNAMES := $(filter-out ${VALIDBENCHSETNAMES},${BENCHSET})
+ifneq (${INVALIDBENCHSETNAMES},)
+$(info The following values in BENCHSET are invalid: ${INVALIDBENCHSETNAMES})
+$(info The valid ones are: ${VALIDBENCHSETNAMES})
+$(error Invalid BENCHSET value)
+endif
+endif
+
 # Define the bench target only if the target has a usable python installation.
 ifdef PYTHON
 bench: bench-build bench-set bench-func bench-malloc
@@ -170,10 +195,11 @@  bench-set: $(binaries-benchset)
 	done
 
 bench-malloc: $(binaries-bench-malloc)
-	run=$(objpfx)bench-malloc-thread; \
-	for thr in 1 8 16 32; do \
-	  echo "Running $${run} $${thr}"; \
+	for run in $^; do \
+		for thr in 1 8 16 32; do \
+		  echo "Running $${run} $${thr}"; \
 	  $(run-bench) $${thr} > $${run}-$${thr}.out; \
+		done;\
 	done
 
 # Build and execute the benchmark functions.  This target generates JSON
@@ -181,25 +207,28 @@  bench-malloc: $(binaries-bench-malloc)
 # so one could even execute them individually and process it using any JSON
 # capable language or tool.
 bench-func: $(binaries-bench)
+	if [ -n '$^' ] ; then \
 	{ timing_type=$$($(timing-type)); \
-	echo "{\"timing_type\": \"$${timing_type}\","; \
-	echo " \"functions\": {"; \
-	for run in $^; do \
-	  if ! [ "x$${run}" = "x$<" ]; then \
-	    echo ","; \
+	  echo "{\"timing_type\": \"$${timing_type}\","; \
+	  echo " \"functions\": {"; \
+	  for run in $^; do \
+	    if ! [ "x$${run}" = "x$<" ]; then \
+	      echo ","; \
+	    fi; \
+	    echo "Running $${run}" >&2; \
+	    $(run-bench) $(DETAILED_OPT); \
+	  done; \
+	  echo; \
+	  echo " }"; \
+	  echo "}"; \
+	  } > $(objpfx)bench.out-tmp; \
+	  if [ -f $(objpfx)bench.out ]; then \
+	    mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
 	  fi; \
-	  echo "Running $${run}" >&2; \
-	  $(run-bench) $(DETAILED_OPT); \
-	done; \
-	echo; \
-	echo " }"; \
-	echo "}"; } > $(objpfx)bench.out-tmp; \
-	if [ -f $(objpfx)bench.out ]; then \
-	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
-	fi; \
-	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
-	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
-		scripts/benchout.schema.json
+	  mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out; \
+	  $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
+	  scripts/benchout.schema.json; \
+	fi
 
 $(timing-type) $(binaries-bench) $(binaries-benchset) \
 	$(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \
diff --git a/benchtests/README b/benchtests/README
index b015acf..20942c7 100644
--- a/benchtests/README
+++ b/benchtests/README
@@ -53,6 +53,26 @@  otherwise the above command may try to build the benchmark again.  Benchmarks
 that require generated code to be executed during the build are skipped when
 cross-building.
 
+Running subsets of benchmarks:
+==============================
+
+If a subset of benchmarks needs to be run one may run the benchmark by invoking
+make as follows:
+
+  $ make bench BENCHSET="bench-pthread bench-math malloc-thread"
+
+Allowing to measure the performance of specific glibc area. Posible areas are:
+
+    bench-math
+    bench-pthread
+    bench-string
+    string-benchset
+    wcsmbs-benchset
+    stdlib-benchset
+    stdio-common-benchset
+    math-benchset
+    malloc-thread
+
 Adding a function to benchtests:
 ===============================