diff mbox

[RFC] Group benchtests in math/pthread/string

Message ID 20171017184524.8181-1-victor.rodriguez.bahena@intel.com
State New, archived
Headers

Commit Message

Victor Rodriguez Oct. 17, 2017, 6:45 p.m. UTC 
  This patch try to cluster the benchmarks tests in
bench-math/bench-pthread/bench-string . These new rules help
users to just run the benchmarks acording to the glibc area they are
measuring.

Signed-off-by: Victor Rodriguez <victor.rodriguez.bahena@intel.com>
---
 Makefile.in         |  2 +-
 benchtests/Makefile | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+), 1 deletion(-)

Comments

Victor Rodriguez Oct. 21, 2017, 4:55 p.m. UTC | #1 
>This patch try to cluster the benchmarks tests in
>bench-math/bench-pthread/bench-string . These new rules help
>users to just run the benchmarks acording to the glibc area they are
>measuring.
>
>Signed-off-by: Victor Rodriguez <victor.rodriguez.bahena@intel.com>
>---
> Makefile.in         |  2 +-
> benchtests/Makefile | 69
>+++++++++++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 70 insertions(+), 1 deletion(-)
>
>diff --git a/Makefile.in b/Makefile.in
>index 3fe9e73..f9d55ac 100644
>--- a/Makefile.in
>+++ b/Makefile.in
>@@ -12,7 +12,7 @@ install:
> 	LC_ALL=C; export LC_ALL; \
> 	$(MAKE) -r PARALLELMFLAGS="$(PARALLELMFLAGS)" -C $(srcdir) objdir=`pwd`
>$@
> 
>-bench bench-clean bench-build:
>+bench bench-clean bench-build bench-math bench-pthread bench-string:
> 	$(MAKE) -C $(srcdir)/benchtests $(PARALLELMFLAGS) objdir=`pwd` $@
> 
> # Convenience target to rebuild ULPs for all math tests.
>diff --git a/benchtests/Makefile b/benchtests/Makefile
>index 37788e8..a7ec04e 100644
>--- a/benchtests/Makefile
>+++ b/benchtests/Makefile
>@@ -96,6 +96,9 @@ $(objpfx)bench-malloc-thread: $(shared-thread-library)
> include ../Rules
> 
> binaries-bench := $(addprefix $(objpfx)bench-,$(bench))
>+binaries-bench-math := $(addprefix $(objpfx)bench-,$(bench-math))
>+binaries-bench-pthread := $(addprefix $(objpfx)bench-,$(bench-pthread))
>+binaries-bench-string := $(addprefix $(objpfx)bench-,$(bench-string))
> binaries-benchset := $(addprefix $(objpfx)bench-,$(benchset))
> binaries-bench-malloc := $(addprefix $(objpfx)bench-,$(bench-malloc))
> 
>@@ -147,6 +150,9 @@ bench-clean:
> # Define the bench target only if the target has a usable python
>installation.
> ifdef PYTHON
> bench: bench-build bench-set bench-func bench-malloc
>+bench-pthread: bench-func-pthread
>+bench-math: bench-func-math
>+bench-string: bench-func-string
> else
> bench:
> 	@echo "The bench target needs python to run."
>@@ -201,6 +207,69 @@ bench-func: $(binaries-bench)
> 	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
> 		scripts/benchout.schema.json
> 
>+bench-func-math: $(binaries-bench-math)
>+	{ timing_type=$$($(timing-type)); \
>+	echo "{\"timing_type\": \"$${timing_type}\","; \
>+	echo " \"functions\": {"; \
>+	for run in $^; do \
>+	  if ! [ "x$${run}" = "x$<" ]; then \
>+	    echo ","; \
>+	  fi; \
>+	  echo "Running $${run}" >&2; \
>+	  $(run-bench) $(DETAILED_OPT); \
>+	done; \
>+	echo; \
>+	echo " }"; \
>+	echo "}"; } > $(objpfx)bench.out-tmp; \
>+	if [ -f $(objpfx)bench.out ]; then \
>+	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
>+	fi; \
>+	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
>+	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
>+		scripts/benchout.schema.json
>+
>+bench-func-pthread: $(binaries-bench-pthread)
>+	{ timing_type=$$($(timing-type)); \
>+	echo "{\"timing_type\": \"$${timing_type}\","; \
>+	echo " \"functions\": {"; \
>+	for run in $^; do \
>+	  if ! [ "x$${run}" = "x$<" ]; then \
>+	    echo ","; \
>+	  fi; \
>+	  echo "Running $${run}" >&2; \
>+	  $(run-bench) $(DETAILED_OPT); \
>+	done; \
>+	echo; \
>+	echo " }"; \
>+	echo "}"; } > $(objpfx)bench.out-tmp; \
>+	if [ -f $(objpfx)bench.out ]; then \
>+	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
>+	fi; \
>+	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
>+	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
>+		scripts/benchout.schema.json
>+
>+bench-func-string: $(binaries-bench-string)
>+	{ timing_type=$$($(timing-type)); \
>+	echo "{\"timing_type\": \"$${timing_type}\","; \
>+	echo " \"functions\": {"; \
>+	for run in $^; do \
>+	  if ! [ "x$${run}" = "x$<" ]; then \
>+	    echo ","; \
>+	  fi; \
>+	  echo "Running $${run}" >&2; \
>+	  $(run-bench) $(DETAILED_OPT); \
>+	done; \
>+	echo; \
>+	echo " }"; \
>+	echo "}"; } > $(objpfx)bench.out-tmp; \
>+	if [ -f $(objpfx)bench.out ]; then \
>+	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
>+	fi; \
>+	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
>+	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
>+		scripts/benchout.schema.json
>+
> $(timing-type) $(binaries-bench) $(binaries-benchset) \
> 	$(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \
> 	$(link-extra-libs-tests) \
>-- 
>2.14.2

Any feedback ? 

Regards


>
Siddhesh Poyarekar Oct. 23, 2017, 10 a.m. UTC | #2 
On Wednesday 18 October 2017 12:15 AM, Victor Rodriguez wrote:
> This patch try to cluster the benchmarks tests in
> bench-math/bench-pthread/bench-string . These new rules help
> users to just run the benchmarks acording to the glibc area they are
> measuring.

Thanks, this is a useful feature but there's a better way to implement
it.  Something like the following would be far more extensible:

  make BENCHSET=math bench

or similar, where BENCHSET could be a space separated list of targets or
a shortcut 'all'.

Also, please review the patch submission guidelines[1] with your
submission; for one you need a ChangeLog entry for your patch.

Thanks,
Siddhesh

[1] https://sourceware.org/glibc/wiki/Contribution%20checklist

> Signed-off-by: Victor Rodriguez <victor.rodriguez.bahena@intel.com>
> ---
>  Makefile.in         |  2 +-
>  benchtests/Makefile | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 70 insertions(+), 1 deletion(-)
> 
> diff --git a/Makefile.in b/Makefile.in
> index 3fe9e73..f9d55ac 100644
> --- a/Makefile.in
> +++ b/Makefile.in
> @@ -12,7 +12,7 @@ install:
>  	LC_ALL=C; export LC_ALL; \
>  	$(MAKE) -r PARALLELMFLAGS="$(PARALLELMFLAGS)" -C $(srcdir) objdir=`pwd` $@
>  
> -bench bench-clean bench-build:
> +bench bench-clean bench-build bench-math bench-pthread bench-string:
>  	$(MAKE) -C $(srcdir)/benchtests $(PARALLELMFLAGS) objdir=`pwd` $@
>  
>  # Convenience target to rebuild ULPs for all math tests.
> diff --git a/benchtests/Makefile b/benchtests/Makefile
> index 37788e8..a7ec04e 100644
> --- a/benchtests/Makefile
> +++ b/benchtests/Makefile
> @@ -96,6 +96,9 @@ $(objpfx)bench-malloc-thread: $(shared-thread-library)
>  include ../Rules
>  
>  binaries-bench := $(addprefix $(objpfx)bench-,$(bench))
> +binaries-bench-math := $(addprefix $(objpfx)bench-,$(bench-math))
> +binaries-bench-pthread := $(addprefix $(objpfx)bench-,$(bench-pthread))
> +binaries-bench-string := $(addprefix $(objpfx)bench-,$(bench-string))
>  binaries-benchset := $(addprefix $(objpfx)bench-,$(benchset))
>  binaries-bench-malloc := $(addprefix $(objpfx)bench-,$(bench-malloc))
>  
> @@ -147,6 +150,9 @@ bench-clean:
>  # Define the bench target only if the target has a usable python installation.
>  ifdef PYTHON
>  bench: bench-build bench-set bench-func bench-malloc
> +bench-pthread: bench-func-pthread
> +bench-math: bench-func-math
> +bench-string: bench-func-string
>  else
>  bench:
>  	@echo "The bench target needs python to run."
> @@ -201,6 +207,69 @@ bench-func: $(binaries-bench)
>  	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
>  		scripts/benchout.schema.json
>  
> +bench-func-math: $(binaries-bench-math)
> +	{ timing_type=$$($(timing-type)); \
> +	echo "{\"timing_type\": \"$${timing_type}\","; \
> +	echo " \"functions\": {"; \
> +	for run in $^; do \
> +	  if ! [ "x$${run}" = "x$<" ]; then \
> +	    echo ","; \
> +	  fi; \
> +	  echo "Running $${run}" >&2; \
> +	  $(run-bench) $(DETAILED_OPT); \
> +	done; \
> +	echo; \
> +	echo " }"; \
> +	echo "}"; } > $(objpfx)bench.out-tmp; \
> +	if [ -f $(objpfx)bench.out ]; then \
> +	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
> +	fi; \
> +	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
> +	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
> +		scripts/benchout.schema.json
> +
> +bench-func-pthread: $(binaries-bench-pthread)
> +	{ timing_type=$$($(timing-type)); \
> +	echo "{\"timing_type\": \"$${timing_type}\","; \
> +	echo " \"functions\": {"; \
> +	for run in $^; do \
> +	  if ! [ "x$${run}" = "x$<" ]; then \
> +	    echo ","; \
> +	  fi; \
> +	  echo "Running $${run}" >&2; \
> +	  $(run-bench) $(DETAILED_OPT); \
> +	done; \
> +	echo; \
> +	echo " }"; \
> +	echo "}"; } > $(objpfx)bench.out-tmp; \
> +	if [ -f $(objpfx)bench.out ]; then \
> +	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
> +	fi; \
> +	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
> +	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
> +		scripts/benchout.schema.json
> +
> +bench-func-string: $(binaries-bench-string)
> +	{ timing_type=$$($(timing-type)); \
> +	echo "{\"timing_type\": \"$${timing_type}\","; \
> +	echo " \"functions\": {"; \
> +	for run in $^; do \
> +	  if ! [ "x$${run}" = "x$<" ]; then \
> +	    echo ","; \
> +	  fi; \
> +	  echo "Running $${run}" >&2; \
> +	  $(run-bench) $(DETAILED_OPT); \
> +	done; \
> +	echo; \
> +	echo " }"; \
> +	echo "}"; } > $(objpfx)bench.out-tmp; \
> +	if [ -f $(objpfx)bench.out ]; then \
> +	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
> +	fi; \
> +	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
> +	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
> +		scripts/benchout.schema.json
> +
>  $(timing-type) $(binaries-bench) $(binaries-benchset) \
>  	$(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \
>  	$(link-extra-libs-tests) \
>
Victor Rodriguez Nov. 9, 2017, 11:36 a.m. UTC | #3 
-----Original Message-----
From: Siddhesh Poyarekar <siddhesh@gotplt.org>
Date: Monday, October 23, 2017 at 4:00 AM
To: Victor Rodriguez Bahena <victor.rodriguez.bahena@intel.com>,
"libc-alpha@sourceware.org" <libc-alpha@sourceware.org>
Subject: Re: [RFC PATCH] Group benchtests in math/pthread/string

>On Wednesday 18 October 2017 12:15 AM, Victor Rodriguez wrote:
>> This patch try to cluster the benchmarks tests in
>> bench-math/bench-pthread/bench-string . These new rules help
>> users to just run the benchmarks acording to the glibc area they are
>> measuring.
>
>Thanks, this is a useful feature but there's a better way to implement
>it.  Something like the following would be far more extensible:
>
>  make BENCHSET=math bench
>
>or similar, where BENCHSET could be a space separated list of targets or
>a shortcut 'all'.

Thanks a lot for your feedback Siddhesh, really appreciated . I sent a new
path named:

benchtests:Add BENCHSET list of targets

This is more extensible, user can run:

make bench BENCHSET="bench-pthread bench-math malloc-thread²


The groups that are supported are taken from the Makefile sub groups:

bench-math
bench-pthread
bench-string
string-benchset
stdlib-benchset
stdio-common-benchset
math-benchset
bench-malloc

If user wants to run all the benchmark, the command line remains as usual:

make bench


It has been tested and is working fine , please let me know if is good for
commit or need something else

Regards

Victor 

>Also, please review the patch submission guidelines[1] with your
>submission; for one you need a ChangeLog entry for your patch.
>
>Thanks,
>Siddhesh
>
>[1] https://sourceware.org/glibc/wiki/Contribution%20checklist
>
>> Signed-off-by: Victor Rodriguez <victor.rodriguez.bahena@intel.com>
>> ---
>>  Makefile.in         |  2 +-
>>  benchtests/Makefile | 69
>>+++++++++++++++++++++++++++++++++++++++++++++++++++++
>>  2 files changed, 70 insertions(+), 1 deletion(-)
>> 
>> diff --git a/Makefile.in b/Makefile.in
>> index 3fe9e73..f9d55ac 100644
>> --- a/Makefile.in
>> +++ b/Makefile.in
>> @@ -12,7 +12,7 @@ install:
>>  	LC_ALL=C; export LC_ALL; \
>>  	$(MAKE) -r PARALLELMFLAGS="$(PARALLELMFLAGS)" -C $(srcdir)
>>objdir=`pwd` $@
>>  
>> -bench bench-clean bench-build:
>> +bench bench-clean bench-build bench-math bench-pthread bench-string:
>>  	$(MAKE) -C $(srcdir)/benchtests $(PARALLELMFLAGS) objdir=`pwd` $@
>>  
>>  # Convenience target to rebuild ULPs for all math tests.
>> diff --git a/benchtests/Makefile b/benchtests/Makefile
>> index 37788e8..a7ec04e 100644
>> --- a/benchtests/Makefile
>> +++ b/benchtests/Makefile
>> @@ -96,6 +96,9 @@ $(objpfx)bench-malloc-thread: $(shared-thread-library)
>>  include ../Rules
>>  
>>  binaries-bench := $(addprefix $(objpfx)bench-,$(bench))
>> +binaries-bench-math := $(addprefix $(objpfx)bench-,$(bench-math))
>> +binaries-bench-pthread := $(addprefix $(objpfx)bench-,$(bench-pthread))
>> +binaries-bench-string := $(addprefix $(objpfx)bench-,$(bench-string))
>>  binaries-benchset := $(addprefix $(objpfx)bench-,$(benchset))
>>  binaries-bench-malloc := $(addprefix $(objpfx)bench-,$(bench-malloc))
>>  
>> @@ -147,6 +150,9 @@ bench-clean:
>>  # Define the bench target only if the target has a usable python
>>installation.
>>  ifdef PYTHON
>>  bench: bench-build bench-set bench-func bench-malloc
>> +bench-pthread: bench-func-pthread
>> +bench-math: bench-func-math
>> +bench-string: bench-func-string
>>  else
>>  bench:
>>  	@echo "The bench target needs python to run."
>> @@ -201,6 +207,69 @@ bench-func: $(binaries-bench)
>>  	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
>>  		scripts/benchout.schema.json
>>  
>> +bench-func-math: $(binaries-bench-math)
>> +	{ timing_type=$$($(timing-type)); \
>> +	echo "{\"timing_type\": \"$${timing_type}\","; \
>> +	echo " \"functions\": {"; \
>> +	for run in $^; do \
>> +	  if ! [ "x$${run}" = "x$<" ]; then \
>> +	    echo ","; \
>> +	  fi; \
>> +	  echo "Running $${run}" >&2; \
>> +	  $(run-bench) $(DETAILED_OPT); \
>> +	done; \
>> +	echo; \
>> +	echo " }"; \
>> +	echo "}"; } > $(objpfx)bench.out-tmp; \
>> +	if [ -f $(objpfx)bench.out ]; then \
>> +	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
>> +	fi; \
>> +	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
>> +	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
>> +		scripts/benchout.schema.json
>> +
>> +bench-func-pthread: $(binaries-bench-pthread)
>> +	{ timing_type=$$($(timing-type)); \
>> +	echo "{\"timing_type\": \"$${timing_type}\","; \
>> +	echo " \"functions\": {"; \
>> +	for run in $^; do \
>> +	  if ! [ "x$${run}" = "x$<" ]; then \
>> +	    echo ","; \
>> +	  fi; \
>> +	  echo "Running $${run}" >&2; \
>> +	  $(run-bench) $(DETAILED_OPT); \
>> +	done; \
>> +	echo; \
>> +	echo " }"; \
>> +	echo "}"; } > $(objpfx)bench.out-tmp; \
>> +	if [ -f $(objpfx)bench.out ]; then \
>> +	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
>> +	fi; \
>> +	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
>> +	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
>> +		scripts/benchout.schema.json
>> +
>> +bench-func-string: $(binaries-bench-string)
>> +	{ timing_type=$$($(timing-type)); \
>> +	echo "{\"timing_type\": \"$${timing_type}\","; \
>> +	echo " \"functions\": {"; \
>> +	for run in $^; do \
>> +	  if ! [ "x$${run}" = "x$<" ]; then \
>> +	    echo ","; \
>> +	  fi; \
>> +	  echo "Running $${run}" >&2; \
>> +	  $(run-bench) $(DETAILED_OPT); \
>> +	done; \
>> +	echo; \
>> +	echo " }"; \
>> +	echo "}"; } > $(objpfx)bench.out-tmp; \
>> +	if [ -f $(objpfx)bench.out ]; then \
>> +	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
>> +	fi; \
>> +	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
>> +	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
>> +		scripts/benchout.schema.json
>> +
>>  $(timing-type) $(binaries-bench) $(binaries-benchset) \
>>  	$(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \
>>  	$(link-extra-libs-tests) \
>>
diff mbox

Patch

diff --git a/Makefile.in b/Makefile.in
index 3fe9e73..f9d55ac 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -12,7 +12,7 @@  install:
 	LC_ALL=C; export LC_ALL; \
 	$(MAKE) -r PARALLELMFLAGS="$(PARALLELMFLAGS)" -C $(srcdir) objdir=`pwd` $@
 
-bench bench-clean bench-build:
+bench bench-clean bench-build bench-math bench-pthread bench-string:
 	$(MAKE) -C $(srcdir)/benchtests $(PARALLELMFLAGS) objdir=`pwd` $@
 
 # Convenience target to rebuild ULPs for all math tests.
diff --git a/benchtests/Makefile b/benchtests/Makefile
index 37788e8..a7ec04e 100644
--- a/benchtests/Makefile
+++ b/benchtests/Makefile
@@ -96,6 +96,9 @@  $(objpfx)bench-malloc-thread: $(shared-thread-library)
 include ../Rules
 
 binaries-bench := $(addprefix $(objpfx)bench-,$(bench))
+binaries-bench-math := $(addprefix $(objpfx)bench-,$(bench-math))
+binaries-bench-pthread := $(addprefix $(objpfx)bench-,$(bench-pthread))
+binaries-bench-string := $(addprefix $(objpfx)bench-,$(bench-string))
 binaries-benchset := $(addprefix $(objpfx)bench-,$(benchset))
 binaries-bench-malloc := $(addprefix $(objpfx)bench-,$(bench-malloc))
 
@@ -147,6 +150,9 @@  bench-clean:
 # Define the bench target only if the target has a usable python installation.
 ifdef PYTHON
 bench: bench-build bench-set bench-func bench-malloc
+bench-pthread: bench-func-pthread
+bench-math: bench-func-math
+bench-string: bench-func-string
 else
 bench:
 	@echo "The bench target needs python to run."
@@ -201,6 +207,69 @@  bench-func: $(binaries-bench)
 	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
 		scripts/benchout.schema.json
 
+bench-func-math: $(binaries-bench-math)
+	{ timing_type=$$($(timing-type)); \
+	echo "{\"timing_type\": \"$${timing_type}\","; \
+	echo " \"functions\": {"; \
+	for run in $^; do \
+	  if ! [ "x$${run}" = "x$<" ]; then \
+	    echo ","; \
+	  fi; \
+	  echo "Running $${run}" >&2; \
+	  $(run-bench) $(DETAILED_OPT); \
+	done; \
+	echo; \
+	echo " }"; \
+	echo "}"; } > $(objpfx)bench.out-tmp; \
+	if [ -f $(objpfx)bench.out ]; then \
+	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
+	fi; \
+	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
+	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
+		scripts/benchout.schema.json
+
+bench-func-pthread: $(binaries-bench-pthread)
+	{ timing_type=$$($(timing-type)); \
+	echo "{\"timing_type\": \"$${timing_type}\","; \
+	echo " \"functions\": {"; \
+	for run in $^; do \
+	  if ! [ "x$${run}" = "x$<" ]; then \
+	    echo ","; \
+	  fi; \
+	  echo "Running $${run}" >&2; \
+	  $(run-bench) $(DETAILED_OPT); \
+	done; \
+	echo; \
+	echo " }"; \
+	echo "}"; } > $(objpfx)bench.out-tmp; \
+	if [ -f $(objpfx)bench.out ]; then \
+	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
+	fi; \
+	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
+	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
+		scripts/benchout.schema.json
+
+bench-func-string: $(binaries-bench-string)
+	{ timing_type=$$($(timing-type)); \
+	echo "{\"timing_type\": \"$${timing_type}\","; \
+	echo " \"functions\": {"; \
+	for run in $^; do \
+	  if ! [ "x$${run}" = "x$<" ]; then \
+	    echo ","; \
+	  fi; \
+	  echo "Running $${run}" >&2; \
+	  $(run-bench) $(DETAILED_OPT); \
+	done; \
+	echo; \
+	echo " }"; \
+	echo "}"; } > $(objpfx)bench.out-tmp; \
+	if [ -f $(objpfx)bench.out ]; then \
+	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
+	fi; \
+	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
+	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
+		scripts/benchout.schema.json
+
 $(timing-type) $(binaries-bench) $(binaries-benchset) \
 	$(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \
 	$(link-extra-libs-tests) \