diff mbox

[2/4] Detailed benchmark outputs for functions

Message ID 20140325102941.GB1850@spoyarek.pnq.redhat.com
State Committed
Headers

Commit Message

Siddhesh Poyarekar March 25, 2014, 10:29 a.m. UTC 
  Hi,

This patch adds an option to get detailed benchmark output for
functions.  Invoking the benchmark with 'make DETAILED=1 bench' causes
each benchmark program to store a mean execution time for each input
it works on.  This is useful to give a more comprehensive picture of
performance of functions compared to just the single mean figure.

Siddhesh

	* benchtests/Makefile (DETAILED_OPT): New make option.
	(bench-func): Run benchmark program with -d if DETAILED_OPT is
	set.
	* benchtests/bench-skeleton.c: Include stdbool.h.
	(main): Store and print timings per input.
	* benchtests/scripts/bench.py (STRUCT_TEMPLATE): Add timing
	member to each argument value.
	(EPILOGUE): Define new macros RESULT and RESULT_ACCUM.
	(_print_arg_data): Initialize per-input timing to 0.
	

---
 benchtests/Makefile         |  8 +++++++-
 benchtests/bench-skeleton.c | 22 ++++++++++++++++++++++
 benchtests/scripts/bench.py |  6 +++++-
 3 files changed, 34 insertions(+), 2 deletions(-)

Comments

Will Newton March 25, 2014, 12:29 p.m. UTC | #1 
On 25 March 2014 10:29, Siddhesh Poyarekar <siddhesh@redhat.com> wrote:
> Hi,
>
> This patch adds an option to get detailed benchmark output for
> functions.  Invoking the benchmark with 'make DETAILED=1 bench' causes
> each benchmark program to store a mean execution time for each input
> it works on.  This is useful to give a more comprehensive picture of
> performance of functions compared to just the single mean figure.
>
> Siddhesh
>
>         * benchtests/Makefile (DETAILED_OPT): New make option.
>         (bench-func): Run benchmark program with -d if DETAILED_OPT is
>         set.
>         * benchtests/bench-skeleton.c: Include stdbool.h.
>         (main): Store and print timings per input.
>         * benchtests/scripts/bench.py (STRUCT_TEMPLATE): Add timing
>         member to each argument value.
>         (EPILOGUE): Define new macros RESULT and RESULT_ACCUM.
>         (_print_arg_data): Initialize per-input timing to 0.

This looks ok to me.
diff mbox

Patch

diff --git a/benchtests/Makefile b/benchtests/Makefile
index be11708..f5488c1 100644
--- a/benchtests/Makefile
+++ b/benchtests/Makefile
@@ -86,6 +86,12 @@  ifdef USE_CLOCK_GETTIME
 CPPFLAGS-nonlib += -DUSE_CLOCK_GETTIME
 endif
 
+DETAILED_OPT :=
+
+ifdef DETAILED
+DETAILED_OPT := -d
+endif
+
 # This makes sure CPPFLAGS-nonlib and CFLAGS-nonlib are passed
 # for all these modules.
 cpp-srcs-left := $(binaries-benchset:=.c) $(binaries-bench:=.c)
@@ -126,7 +132,7 @@  bench-func: $(binaries-bench)
 	    echo ","; \
 	  fi; \
 	  echo "Running $${run}" >&2; \
-	  $(run-bench); \
+	  $(run-bench) $(DETAILED_OPT); \
 	done; \
 	echo "  }"; \
 	echo "}"; } > $(objpfx)bench.out-tmp; \
diff --git a/benchtests/bench-skeleton.c b/benchtests/bench-skeleton.c
index faef7eb..0c7d744 100644
--- a/benchtests/bench-skeleton.c
+++ b/benchtests/bench-skeleton.c
@@ -18,6 +18,7 @@ 
 
 #include <string.h>
 #include <stdint.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <time.h>
 #include <inttypes.h>
@@ -48,6 +49,10 @@  main (int argc, char **argv)
   unsigned long i, k;
   struct timespec runtime;
   timing_t start, end;
+  bool detailed = false;
+
+  if (argc == 2 && !strcmp (argv[1], "-d"))
+    detailed = true;
 
   startup();
 
@@ -72,6 +77,7 @@  main (int argc, char **argv)
 
       double d_total_i = 0;
       timing_t total = 0, max = 0, min = 0x7fffffffffffffff;
+      int64_t c = 0;
       while (1)
 	{
 	  for (i = 0; i < NUM_SAMPLES (v); i++)
@@ -91,8 +97,13 @@  main (int argc, char **argv)
 		min = cur;
 
 	      TIMING_ACCUM (total, cur);
+	      /* Accumulate timings for the value.  In the end we will divide
+	         by the total iterations.  */
+	      RESULT_ACCUM (cur, v, i, c * iters, (c + 1) * iters);
+
 	      d_total_i += iters;
 	    }
+	  c++;
 	  struct timespec curtime;
 
 	  memset (&curtime, 0, sizeof (curtime));
@@ -114,6 +125,17 @@  main (int argc, char **argv)
 	      d_total_s, d_total_i, max / d_iters, min / d_iters,
 	      d_total_s / d_total_i);
 
+      if (detailed)
+	{
+	  printf (",\n\"timings\": [");
+	  for (int i = 0; i < NUM_SAMPLES (v); i++)
+	    {
+	      if (i > 0)
+		putc (',', stdout);
+	      printf ("%g", RESULT (v, i));
+	    }
+	  puts ("]");
+	}
       puts ("}");
     }
 
diff --git a/benchtests/scripts/bench.py b/benchtests/scripts/bench.py
index 90317b5..492c764 100755
--- a/benchtests/scripts/bench.py
+++ b/benchtests/scripts/bench.py
@@ -50,6 +50,7 @@  STRUCT_TEMPLATE = '''
 struct args
 {
 %(args)s
+  double timing;
 };
 
 struct _variants
@@ -80,6 +81,9 @@  struct _variants variants[%(num_variants)d] = {
 
 # Epilogue for the generated source file.
 EPILOGUE = '''
+#define RESULT(__v, __i) (variants[(__v)].in[(__i)].timing)
+#define RESULT_ACCUM(r, v, i, old, new) \\
+        ((RESULT ((v), (i))) = (RESULT ((v), (i)) * (old) + (r)) / ((new) + 1))
 #define BENCH_FUNC(i, j) ({%(getret)s CALL_BENCH_FUNC (i, j);})
 #define FUNCNAME "%(func)s"
 #include "bench-skeleton.c"'''
@@ -168,7 +172,7 @@  def _print_arg_data(func, directives, all_vals):
     # Now print the values.
     variants = []
     for (k, vals), i in zip(all_vals.items(), itertools.count()):
-        out = ['  {%s},' % v for v in vals]
+        out = ['  {%s, 0},' % v for v in vals]
 
         # Members for the variants structure list that we will
         # print later.