Patchwork [2/3] benchtests: keep comparing even if value does not exist

login
register
mail settings
Submitter leonardo.sandoval.gonzalez@linux.intel.com
Date Dec. 10, 2018, 11:05 p.m.
Message ID <20181210230504.13571-3-leonardo.sandoval.gonzalez@linux.intel.com>
Download mbox | patch
Permalink /patch/30616/
State New
Headers show

Comments

leonardo.sandoval.gonzalez@linux.intel.com - Dec. 10, 2018, 11:05 p.m.
From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>

---
 benchtests/scripts/compare_bench.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)
Siddhesh Poyarekar - Dec. 11, 2018, 6:40 a.m.
On 11/12/18 4:35 AM, leonardo.sandoval.gonzalez@linux.intel.com wrote:
> From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>

Please add a ChangeLog entry and an explanation describing the rationale 
of the patch.

> ---
>   benchtests/scripts/compare_bench.py | 27 ++++++++++++++++++---------
>   1 file changed, 18 insertions(+), 9 deletions(-)
> 
> diff --git a/benchtests/scripts/compare_bench.py b/benchtests/scripts/compare_bench.py
> index 9cbbda6be6..add4023944 100755
> --- a/benchtests/scripts/compare_bench.py
> +++ b/benchtests/scripts/compare_bench.py
> @@ -42,17 +42,25 @@ def do_compare(func, var, tl1, tl2, par, threshold):
>           threshold: The threshold for differences, beyond which the script should
>           print a warning.
>       """
> -    d = abs(tl2[par] - tl1[par]) * 100 / tl1[str(par)]
> +    try:
> +        v1 = tl1[str(par)]
> +        v2 = tl2[str(par)]
> +        d = abs(v2 - v1) * 100 / v1
> +    except KeyError:
> +        return
> +    except ZeroDivisionError:
> +        return
> +
>       if d > threshold:
> -        if tl1[par] > tl2[par]:
> +        if v1 > v2:
>               ind = '+++'
>           else:
>               ind = '---'
>           print('%s %s(%s)[%s]: (%.2lf%%) from %g to %g' %
> -                (ind, func, var, par, d, tl1[par], tl2[par]))
> +                (ind, func, var, par, d, v1, v2))
>   
>   
> -def compare_runs(pts1, pts2, threshold):
> +def compare_runs(pts1, pts2, threshold, stats):
>       """Compare two benchmark runs
>   
>       Args:
> @@ -70,8 +78,8 @@ def compare_runs(pts1, pts2, threshold):
>   
>               # Compare the consolidated numbers
>               # do_compare(func, var, tl1, tl2, 'max', threshold)
> -            do_compare(func, var, tl1, tl2, 'min', threshold)
> -            do_compare(func, var, tl1, tl2, 'mean', threshold)
> +            for stat in stats.split():
> +                do_compare(func, var, tl1, tl2, stat, threshold)
>   
>               # Skip over to the next variant or function if there is no detailed
>               # timing info for the function variant.
> @@ -152,7 +160,7 @@ def plot_graphs(bench1, bench2):
>               print('Writing out %s' % filename)
>               pylab.savefig(filename)
>   
> -def main(bench1, bench2, schema, threshold):
> +def main(bench1, bench2, schema, threshold, stats):
>       bench1 = bench.parse_bench(bench1, schema)
>       bench2 = bench.parse_bench(bench2, schema)
>   
> @@ -161,7 +169,7 @@ def main(bench1, bench2, schema, threshold):
>       bench.compress_timings(bench1)
>       bench.compress_timings(bench2)
>   
> -    compare_runs(bench1, bench2, threshold)
> +    compare_runs(bench1, bench2, threshold, stats)
>   
>   
>   if __name__ == '__main__':
> @@ -176,7 +184,8 @@ if __name__ == '__main__':
>                           default=os.path.join(os.path.dirname(os.path.realpath(__file__)),'benchout.schema.json'),
>                           help='JSON file to validate source/dest files (default: %(default)s)')
>       parser.add_argument('--threshold', default=10.0, type=float, help='Only print those with equal or higher threshold (default: %(default)s)')
> +    parser.add_argument('--stats', default="min mean", type=str, help='Only consider the values from the specified statistics')

The help text needs to be fleshed out a bit and the argument needs to 
have a limited set of values.  Suggested text:

     Only consider the values from the statistics specified as a space
     separated list.  The currently identified statistics are 'min',
     'mean'.

Siddhesh

>   
>       args = parser.parse_args()
>   
> -    main(args.bench1, args.bench2, args.schema, args.threshold)
> +    main(args.bench1, args.bench2, args.schema, args.threshold, args.stats)
>

Patch

diff --git a/benchtests/scripts/compare_bench.py b/benchtests/scripts/compare_bench.py
index 9cbbda6be6..add4023944 100755
--- a/benchtests/scripts/compare_bench.py
+++ b/benchtests/scripts/compare_bench.py
@@ -42,17 +42,25 @@  def do_compare(func, var, tl1, tl2, par, threshold):
         threshold: The threshold for differences, beyond which the script should
         print a warning.
     """
-    d = abs(tl2[par] - tl1[par]) * 100 / tl1[str(par)]
+    try:
+        v1 = tl1[str(par)]
+        v2 = tl2[str(par)]
+        d = abs(v2 - v1) * 100 / v1
+    except KeyError:
+        return
+    except ZeroDivisionError:
+        return
+
     if d > threshold:
-        if tl1[par] > tl2[par]:
+        if v1 > v2:
             ind = '+++'
         else:
             ind = '---'
         print('%s %s(%s)[%s]: (%.2lf%%) from %g to %g' %
-                (ind, func, var, par, d, tl1[par], tl2[par]))
+                (ind, func, var, par, d, v1, v2))
 
 
-def compare_runs(pts1, pts2, threshold):
+def compare_runs(pts1, pts2, threshold, stats):
     """Compare two benchmark runs
 
     Args:
@@ -70,8 +78,8 @@  def compare_runs(pts1, pts2, threshold):
 
             # Compare the consolidated numbers
             # do_compare(func, var, tl1, tl2, 'max', threshold)
-            do_compare(func, var, tl1, tl2, 'min', threshold)
-            do_compare(func, var, tl1, tl2, 'mean', threshold)
+            for stat in stats.split():
+                do_compare(func, var, tl1, tl2, stat, threshold)
 
             # Skip over to the next variant or function if there is no detailed
             # timing info for the function variant.
@@ -152,7 +160,7 @@  def plot_graphs(bench1, bench2):
             print('Writing out %s' % filename)
             pylab.savefig(filename)
 
-def main(bench1, bench2, schema, threshold):
+def main(bench1, bench2, schema, threshold, stats):
     bench1 = bench.parse_bench(bench1, schema)
     bench2 = bench.parse_bench(bench2, schema)
 
@@ -161,7 +169,7 @@  def main(bench1, bench2, schema, threshold):
     bench.compress_timings(bench1)
     bench.compress_timings(bench2)
 
-    compare_runs(bench1, bench2, threshold)
+    compare_runs(bench1, bench2, threshold, stats)
 
 
 if __name__ == '__main__':
@@ -176,7 +184,8 @@  if __name__ == '__main__':
                         default=os.path.join(os.path.dirname(os.path.realpath(__file__)),'benchout.schema.json'),
                         help='JSON file to validate source/dest files (default: %(default)s)')
     parser.add_argument('--threshold', default=10.0, type=float, help='Only print those with equal or higher threshold (default: %(default)s)')
+    parser.add_argument('--stats', default="min mean", type=str, help='Only consider the values from the specified statistics')
 
     args = parser.parse_args()
 
-    main(args.bench1, args.bench2, args.schema, args.threshold)
+    main(args.bench1, args.bench2, args.schema, args.threshold, args.stats)