diff mbox series

[v1] benchtests: Improve benchtests for strstr, memmem, and memchr

Message ID 20220527210458.1376451-1-goldstein.w.n@gmail.com
State Superseded
Headers
Series [v1] benchtests: Improve benchtests for strstr, memmem, and memchr |

Checks

Context Check Description
dj/TryBot-apply_patch success Patch applied to master at the time it was sent
dj/TryBot-32bit success Build for i686

Commit Message

Noah Goldstein May 27, 2022, 9:04 p.m. UTC 
  1. Use json_ctx for output to help standardize format across all
   benchtests.

2. Add some additional tests to strstr and memchr expanding alignments
   and adding more small values.
---
 benchtests/bench-memchr.c | 108 ++++++++++++++-------
 benchtests/bench-memmem.c |  82 ++++++++++++----
 benchtests/bench-strstr.c | 191 +++++++++++++++++++++++++++++++-------
 3 files changed, 289 insertions(+), 92 deletions(-)

Comments

H.J. Lu May 27, 2022, 9:35 p.m. UTC | #1 
On Fri, May 27, 2022 at 2:05 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> 1. Use json_ctx for output to help standardize format across all
>    benchtests.
>
> 2. Add some additional tests to strstr and memchr expanding alignments
>    and adding more small values.
> ---
>  benchtests/bench-memchr.c | 108 ++++++++++++++-------
>  benchtests/bench-memmem.c |  82 ++++++++++++----
>  benchtests/bench-strstr.c | 191 +++++++++++++++++++++++++++++++-------
>  3 files changed, 289 insertions(+), 92 deletions(-)
>
> diff --git a/benchtests/bench-memchr.c b/benchtests/bench-memchr.c
> index fb0284f523..4d7212332f 100644
> --- a/benchtests/bench-memchr.c
> +++ b/benchtests/bench-memchr.c
> @@ -53,8 +53,11 @@ SIMPLE_MEMCHR (const CHAR *s, int c, size_t n)
>  }
>  #endif /* !USE_AS_MEMRCHR */
>
> +#include "json-lib.h"
> +
>  static void
> -do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
> +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const CHAR *s, int c,
> +            size_t n)
>  {
>    size_t i, iters = INNER_LOOP_ITERS_LARGE;
>    timing_t start, stop, cur;
> @@ -68,15 +71,16 @@ do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
>
>    TIMING_DIFF (cur, start, stop);
>
> -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> +  json_element_double (json_ctx, (double) cur / (double) iters);
>  }
>
>  static void
> -do_test (size_t align, size_t pos, size_t len, int seek_char)
> +do_test (json_ctx_t *json_ctx, size_t align, size_t pos, size_t len,
> +        int seek_char)
>  {
>    size_t i;
>
> -  align &= 7;
> +  align &= getpagesize () - 1;
>    if ((align + len) * sizeof (CHAR) >= page_size)
>      return;
>
> @@ -100,67 +104,99 @@ do_test (size_t align, size_t pos, size_t len, int seek_char)
>        buf[align + len] = seek_char;
>      }
>
> -  printf ("Length %4zd, position %4zd, alignment %2zd:",
> -         len, pos, align);
> +  json_element_object_begin (json_ctx);
> +  json_attr_uint (json_ctx, "align", align);
> +  json_attr_uint (json_ctx, "pos", pos);
> +  json_attr_uint (json_ctx, "len", len);
> +  json_attr_uint (json_ctx, "seek_char", seek_char);
> +
> +  json_array_begin (json_ctx, "timings");
>
>    FOR_EACH_IMPL (impl, 0)
> -    do_one_test (impl, (CHAR *) (buf + align), seek_char, len);
> +    do_one_test (json_ctx, impl, (CHAR *) (buf + align), seek_char, len);
>
> -  putchar ('\n');
> +  json_array_end (json_ctx);
> +  json_element_object_end (json_ctx);
>  }
>
>  int
>  test_main (void)
>  {
>    size_t i;
> -
> +  json_ctx_t json_ctx;
>    test_init ();
>
> -  printf ("%20s", "");
> +  json_init (&json_ctx, 0, stdout);
> +
> +  json_document_begin (&json_ctx);
> +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> +
> +  json_attr_object_begin (&json_ctx, "functions");
> +  json_attr_object_begin (&json_ctx, TEST_NAME);
> +  json_attr_string (&json_ctx, "bench-variant", "");
> +
> +  json_array_begin (&json_ctx, "ifuncs");
>    FOR_EACH_IMPL (impl, 0)
> -    printf ("\t%s", impl->name);
> -  putchar ('\n');
> +    json_element_string (&json_ctx, impl->name);
> +  json_array_end (&json_ctx);
> +
> +  json_array_begin (&json_ctx, "results");
>
>    for (i = 1; i < 8; ++i)
>      {
> -      do_test (0, 16 << i, 2048, 23);
> -      do_test (i, 64, 256, 23);
> -      do_test (0, 16 << i, 2048, 0);
> -      do_test (i, 64, 256, 0);
> +      do_test (&json_ctx, 0, 16 << i, 2048, 23);
> +      do_test (&json_ctx, i, 64, 256, 23);
> +      do_test (&json_ctx, 0, 16 << i, 2048, 0);
> +      do_test (&json_ctx, i, 64, 256, 0);
> +
> +      do_test (&json_ctx, getpagesize () - 15, 64, 256, 0);
>  #ifdef USE_AS_MEMRCHR
>        /* Also test the position close to the beginning for memrchr.  */
> -      do_test (0, i, 256, 23);
> -      do_test (0, i, 256, 0);
> -      do_test (i, i, 256, 23);
> -      do_test (i, i, 256, 0);
> +      do_test (&json_ctx, 0, i, 256, 23);
> +      do_test (&json_ctx, 0, i, 256, 0);
> +      do_test (&json_ctx, i, i, 256, 23);
> +      do_test (&json_ctx, i, i, 256, 0);
>  #endif
>      }
>    for (i = 1; i < 8; ++i)
>      {
> -      do_test (i, i << 5, 192, 23);
> -      do_test (i, i << 5, 192, 0);
> -      do_test (i, i << 5, 256, 23);
> -      do_test (i, i << 5, 256, 0);
> -      do_test (i, i << 5, 512, 23);
> -      do_test (i, i << 5, 512, 0);
> +      do_test (&json_ctx, i, i << 5, 192, 23);
> +      do_test (&json_ctx, i, i << 5, 192, 0);
> +      do_test (&json_ctx, i, i << 5, 256, 23);
> +      do_test (&json_ctx, i, i << 5, 256, 0);
> +      do_test (&json_ctx, i, i << 5, 512, 23);
> +      do_test (&json_ctx, i, i << 5, 512, 0);
> +
> +      do_test (&json_ctx, getpagesize () - 15, i << 5, 256, 23);
>      }
>    for (i = 1; i < 32; ++i)
>      {
> -      do_test (0, i, i + 1, 23);
> -      do_test (0, i, i + 1, 0);
> -      do_test (i, i, i + 1, 23);
> -      do_test (i, i, i + 1, 0);
> -      do_test (0, i, i - 1, 23);
> -      do_test (0, i, i - 1, 0);
> -      do_test (i, i, i - 1, 23);
> -      do_test (i, i, i - 1, 0);
> +      do_test (&json_ctx, 0, i, i + 1, 23);
> +      do_test (&json_ctx, 0, i, i + 1, 0);
> +      do_test (&json_ctx, i, i, i + 1, 23);
> +      do_test (&json_ctx, i, i, i + 1, 0);
> +      do_test (&json_ctx, 0, i, i - 1, 23);
> +      do_test (&json_ctx, 0, i, i - 1, 0);
> +      do_test (&json_ctx, i, i, i - 1, 23);
> +      do_test (&json_ctx, i, i, i - 1, 0);
> +
> +      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 23);
> +      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 0);
> +
> +      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 23);
> +      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 0);
>  #ifdef USE_AS_MEMRCHR
>        /* Also test the position close to the beginning for memrchr.  */
> -      do_test (0, 1, i + 1, 23);
> -      do_test (0, 2, i + 1, 0);
> +      do_test (&json_ctx, 0, 1, i + 1, 23);
> +      do_test (&json_ctx, 0, 2, i + 1, 0);
>  #endif
>      }
>
> +  json_array_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_document_end (&json_ctx);
> +
>    return ret;
>  }
>
> diff --git a/benchtests/bench-memmem.c b/benchtests/bench-memmem.c
> index 0fb87309c0..c2b92405dc 100644
> --- a/benchtests/bench-memmem.c
> +++ b/benchtests/bench-memmem.c
> @@ -21,6 +21,7 @@
>  #define BUF1PAGES 20
>  #define ITERATIONS 100
>  #include "bench-string.h"
> +#include "json-lib.h"
>
>  typedef char *(*proto_t) (const void *, size_t, const void *, size_t);
>
> @@ -102,44 +103,62 @@ IMPL (twoway_memmem, 0)
>  IMPL (basic_memmem, 0)
>
>  static void
> -do_one_test (impl_t *impl, const void *haystack, size_t haystack_len,
> -            const void *needle, size_t needle_len, const void *expected)
> +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const void *haystack,
> +            size_t haystack_len, const void *needle, size_t needle_len,
> +            const void *expected)
>  {
>    size_t i, iters = INNER_LOOP_ITERS_SMALL;
>    timing_t start, stop, cur;
> -
> +  void *res;
>    TIMING_NOW (start);
>    for (i = 0; i < iters; ++i)
>      {
> -      CALL (impl, haystack, haystack_len, needle, needle_len);
> +      res = CALL (impl, haystack, haystack_len, needle, needle_len);
>      }
>    TIMING_NOW (stop);
>
>    TIMING_DIFF (cur, start, stop);
>
> -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> +  json_element_double (json_ctx, (double) cur / (double) iters);
> +
> +  if (res != expected)
> +    {
> +      error (0, 0, "Wrong result in function (%p != %p) %s(%p, %zu, %p, %zu)",
> +            res, expected, impl->name, haystack, haystack_len, needle,
> +            needle_len);
> +      ret = 1;
> +    }
>  }
>
>  static void
> -do_test (const char *str, size_t len, size_t idx)
> +do_test (json_ctx_t *json_ctx, const char *str, size_t len, size_t idx)
>  {
>    char tmpbuf[len];
>
>    memcpy (tmpbuf, buf1 + idx, len);
>    memcpy (buf1 + idx, str, len);
>
> -  printf ("String %s, offset %zd:", str, idx);
> +  json_element_object_begin (json_ctx);
> +  json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
> +  json_attr_uint (json_ctx, "len_needle", len);
> +  json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
> +  json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) str);
> +  json_attr_uint (json_ctx, "fail", 0);
> +
> +  json_array_begin (json_ctx, "timings");
>
>    FOR_EACH_IMPL (impl, 0)
> -    do_one_test (impl, buf1, BUF1PAGES * page_size, str, len, buf1 + idx);
> +    do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, str, len,
> +                buf1 + idx);
>
>    memcpy (buf1 + idx, tmpbuf, len);
>
> -  putchar ('\n');
> +  json_array_end (json_ctx);
> +  json_element_object_end (json_ctx);
>  }
>
>  static void
> -do_random_tests (void)
> +do_random_tests (json_ctx_t *json_ctx)
>  {
>    for (size_t n = 0; n < ITERATIONS; ++n)
>      {
> @@ -159,13 +178,21 @@ do_random_tests (void)
>           buf1[idx + off] = ch;
>         }
>
> -      printf ("String %.*s, offset %zd:", (int) len, buf1 + idx, idx);
> +      json_element_object_begin (json_ctx);
> +      json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
> +      json_attr_uint (json_ctx, "len_needle", len);
> +      json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
> +      json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) (buf1 + idx));
> +      json_attr_uint (json_ctx, "fail", 0);
> +
> +      json_array_begin (json_ctx, "timings");
>
>        FOR_EACH_IMPL (impl, 0)
> -       do_one_test (impl, buf1, BUF1PAGES * page_size, buf1 + idx, len,
> -                    buf1 + idx);
> +       do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, buf1 + idx,
> +                    len, buf1 + idx);
>
> -      putchar ('\n');
> +      json_array_end (json_ctx);
> +      json_element_object_end (json_ctx);
>
>        memcpy (buf1 + idx, tmpbuf, len);
>      }
> @@ -178,18 +205,28 @@ static const char *const strs[] =
>      "abc0", "aaaa0", "abcabc0"
>    };
>
> -
>  int
>  test_main (void)
>  {
> +  json_ctx_t json_ctx;
>    size_t i;
>
>    test_init ();
> +  json_init (&json_ctx, 0, stdout);
> +
> +  json_document_begin (&json_ctx);
> +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> +
> +  json_attr_object_begin (&json_ctx, "functions");
> +  json_attr_object_begin (&json_ctx, TEST_NAME);
> +  json_attr_string (&json_ctx, "bench-variant", "");
>
> -  printf ("%23s", "");
> +  json_array_begin (&json_ctx, "ifuncs");
>    FOR_EACH_IMPL (impl, 0)
> -    printf ("\t%s", impl->name);
> -  putchar ('\n');
> +    json_element_string (&json_ctx, impl->name);
> +  json_array_end (&json_ctx);
> +
> +  json_array_begin (&json_ctx, "results");
>
>    for (i = 0; i < BUF1PAGES * page_size; ++i)
>      buf1[i] = 60 + random () % 32;
> @@ -199,10 +236,15 @@ test_main (void)
>        {
>         size_t len = strlen (strs[i]);
>
> -       do_test (strs[i], len, j);
> +       do_test (&json_ctx, strs[i], len, j);
>        }
>
> -  do_random_tests ();
> +  do_random_tests (&json_ctx);
> +
> +  json_array_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_document_end (&json_ctx);
>    return ret;
>  }
>
> diff --git a/benchtests/bench-strstr.c b/benchtests/bench-strstr.c
> index 467418b12c..3838a2b6ac 100644
> --- a/benchtests/bench-strstr.c
> +++ b/benchtests/bench-strstr.c
> @@ -21,6 +21,8 @@
>  #define TEST_NAME "strstr"
>  #include "bench-string.h"
>
> +#include "json-lib.h"
> +
>  static const char input[] =
>  "This manual is written with the assumption that you are at least "
>  "somewhat familiar with the C programming language and basic programming "
> @@ -129,7 +131,8 @@ IMPL (twoway_strstr, 0)
>  IMPL (basic_strstr, 0)
>
>  static void
> -do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1,
> +            const char *s2, char *exp_result)
>  {
>    size_t i, iters = INNER_LOOP_ITERS_SMALL / 8;
>    timing_t start, stop, cur;
> @@ -142,7 +145,7 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
>
>    TIMING_DIFF (cur, start, stop);
>
> -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> +  json_element_double (json_ctx, (double) cur / (double) iters);
>
>    if (res != exp_result)
>      {
> @@ -153,14 +156,16 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
>      }
>  }
>
> -
>  static void
> -do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> -        int fail)
> +do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1,
> +        size_t len2, int fail)
>  {
>    char *s1 = (char *) (buf1 + align1);
>    char *s2 = (char *) (buf2 + align2);
>
> +  if (align1 + len1 >= MIN_PAGE_SIZE || align1 + len1 >= MIN_PAGE_SIZE)

Is this really needed?  Did you mean align2 + len2?

> +    return;
> +
>    size_t size = sizeof (input) - 1;
>    size_t pos = (len1 + len2) % size;
>
> @@ -195,13 +200,21 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
>      if (fail || ss1 != s1 + len1 - len2)
>        ++ss1[len2 / 2];
>
> -  printf ("Length %4zd/%3zd, alignment %2zd/%2zd, %s:",
> -         len1, len2, align1, align2, fail ? "fail " : "found");
> +  json_element_object_begin (json_ctx);
> +  json_attr_uint (json_ctx, "len_haystack", len1);
> +  json_attr_uint (json_ctx, "len_needle", len2);
> +  json_attr_uint (json_ctx, "align_haystack", align1);
> +  json_attr_uint (json_ctx, "align_needle", align2);
> +  json_attr_uint (json_ctx, "fail", fail);
> +
> +  json_array_begin (json_ctx, "timings");
>
>    FOR_EACH_IMPL (impl, 0)
> -    do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> +    do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> +
> +  json_array_end (json_ctx);
> +  json_element_object_end (json_ctx);
>
> -  putchar ('\n');
>  }
>
>  /* Test needles which exhibit worst-case performance.  This shows that
> @@ -211,11 +224,14 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
>     within a factor of 2 on several different microarchitectures.  */
>
>  static void
> -test_hard_needle (size_t ne_len, size_t hs_len)
> +test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len)
>  {
>    char *ne = (char *) buf1;
>    char *hs = (char *) buf2;
>
> +  if (ne_len >= MIN_PAGE_SIZE || hs_len >= MIN_PAGE_SIZE)
> +    return;
> +
>    /* Hard needle for strstr algorithm using skip table.  This results in many
>       memcmp calls comparing most of the needle.  */
>    {
> @@ -226,15 +242,25 @@ test_hard_needle (size_t ne_len, size_t hs_len)
>      memset (hs, 'a', hs_len);
>      for (size_t i = ne_len; i <= hs_len; i += ne_len)
>        {
> -       hs[i-5] = 'b';
> -       hs[i-62] = 'b';
> +       hs[i - 5] = 'b';
> +       hs[i - 62] = 'b';
>        }
>
> -    printf ("Length %4zd/%3zd, complex needle 1:", hs_len, ne_len);
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult skiptable(0)");
> +
> +    json_array_begin (json_ctx, "timings");
>
>      FOR_EACH_IMPL (impl, 0)
> -      do_one_test (impl, hs, ne, NULL);
> -    putchar ('\n');
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
>    }
>
>    /* 2nd hard needle for strstr algorithm using skip table.  This results in
> @@ -247,15 +273,25 @@ test_hard_needle (size_t ne_len, size_t hs_len)
>      memset (hs, 'a', hs_len);
>      for (size_t i = ne_len; i <= hs_len; i += ne_len)
>        {
> -       hs[i-5] = 'b';
> -       hs[i-6] = 'b';
> +       hs[i - 5] = 'b';
> +       hs[i - 6] = 'b';
>        }
>
> -    printf ("Length %4zd/%3zd, complex needle 2:", hs_len, ne_len);
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult skiptable(1)");
> +
> +    json_array_begin (json_ctx, "timings");
>
>      FOR_EACH_IMPL (impl, 0)
> -      do_one_test (impl, hs, ne, NULL);
> -    putchar ('\n');
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
>    }
>
>    /* Hard needle for Two-way algorithm - the random input causes a large number
> @@ -267,45 +303,128 @@ test_hard_needle (size_t ne_len, size_t hs_len)
>      hs[hs_len] = 0;
>
>      memset (ne, 'a', ne_len);
> -    ne[ne_len-2] = 'b';
> +    ne[ne_len - 2] = 'b';
>      ne[0] = 'b';
>      ne[ne_len] = 0;
>
> -    printf ("Length %4zd/%3zd, complex needle 3:", hs_len, ne_len);
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult 2-way");
> +
> +    json_array_begin (json_ctx, "timings");
> +
> +    FOR_EACH_IMPL (impl, 0)
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
> +  }
> +
> +  /* Hard needle for standard algorithm testing first few characters of
> +   * needle.  */
> +  {
> +    for (int i = 0; i < hs_len; i++)
> +      hs[i] = (rand () & 255) >= 128 ? 'a' : 'b';
> +    hs[hs_len] = 0;
> +
> +    for (int i = 0; i < ne_len; i++)
> +      {
> +       if (i % 3 == 0)
> +         ne[i] = 'a';
> +       else if (i % 3 == 1)
> +         ne[i] = 'b';
> +       else
> +         ne[i] = 'c';
> +      }
> +    ne[ne_len] = 0;
> +
> +    json_element_object_begin (json_ctx);
> +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> +    json_attr_uint (json_ctx, "len_needle", ne_len);
> +    json_attr_uint (json_ctx, "align_haystack", 0);
> +    json_attr_uint (json_ctx, "align_needle", 0);
> +    json_attr_uint (json_ctx, "fail", 1);
> +    json_attr_string (json_ctx, "desc", "Difficult testing first 2");
> +
> +    json_array_begin (json_ctx, "timings");
>
>      FOR_EACH_IMPL (impl, 0)
> -      do_one_test (impl, hs, ne, NULL);
> -    putchar ('\n');
> +      do_one_test (json_ctx, impl, hs, ne, NULL);
> +
> +    json_array_end (json_ctx);
> +    json_element_object_end (json_ctx);
>    }
>  }
>
>  static int
>  test_main (void)
>  {
> +  json_ctx_t json_ctx;
>    test_init ();
>
> -  printf ("%23s", "");
> +  json_init (&json_ctx, 0, stdout);
> +
> +  json_document_begin (&json_ctx);
> +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> +
> +  json_attr_object_begin (&json_ctx, "functions");
> +  json_attr_object_begin (&json_ctx, TEST_NAME);
> +  json_attr_string (&json_ctx, "bench-variant", "");
> +
> +  json_array_begin (&json_ctx, "ifuncs");
>    FOR_EACH_IMPL (impl, 0)
> -    printf ("\t%s", impl->name);
> -  putchar ('\n');
> +    json_element_string (&json_ctx, impl->name);
> +  json_array_end (&json_ctx);
> +
> +  json_array_begin (&json_ctx, "results");
>
> -  for (size_t hlen = 64; hlen <= 256; hlen += 32)
> +  for (size_t hlen = 8; hlen <= 256;)
>      for (size_t klen = 1; klen <= 16; klen++)
>        {
> -       do_test (1, 3, hlen, klen, 0);
> -       do_test (0, 9, hlen, klen, 1);
> +       do_test (&json_ctx, 1, 3, hlen, klen, 0);
> +       do_test (&json_ctx, 0, 9, hlen, klen, 1);
> +
> +       do_test (&json_ctx, 1, 3, hlen + 1, klen, 0);
> +       do_test (&json_ctx, 0, 9, hlen + 1, klen, 1);
> +
> +       do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1);
> +       if (hlen < 64)
> +         {
> +           hlen += 8;
> +         }
> +       else
> +         {
> +           hlen += 32;
> +         }
>        }
>
>    for (size_t hlen = 256; hlen <= 65536; hlen *= 2)
> -    for (size_t klen = 16; klen <= 256; klen *= 2)
> +    for (size_t klen = 4; klen <= 256; klen *= 2)
>        {
> -       do_test (1, 11, hlen, klen, 0);
> -       do_test (14, 5, hlen, klen, 1);
> +       do_test (&json_ctx, 1, 11, hlen, klen, 0);
> +       do_test (&json_ctx, 14, 5, hlen, klen, 1);
> +
> +    do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0);
> +    do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1);
> +
> +       do_test (&json_ctx, 1, 11, hlen + 1, klen, 0);
> +       do_test (&json_ctx, 14, 5, hlen + 1, klen, 1);
> +
> +       do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1);
>        }
>
> -  test_hard_needle (64, 65536);
> -  test_hard_needle (256, 65536);
> -  test_hard_needle (1024, 65536);
> +  test_hard_needle (&json_ctx, 64, 65536);
> +  test_hard_needle (&json_ctx, 256, 65536);
> +  test_hard_needle (&json_ctx, 1024, 65536);
> +
> +  json_array_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_attr_object_end (&json_ctx);
> +  json_document_end (&json_ctx);
>
>    return ret;
>  }
> --
> 2.34.1
>
Noah Goldstein May 27, 2022, 9:46 p.m. UTC | #2 
On Fri, May 27, 2022 at 4:35 PM H.J. Lu <hjl.tools@gmail.com> wrote:
>
> On Fri, May 27, 2022 at 2:05 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
> >
> > 1. Use json_ctx for output to help standardize format across all
> >    benchtests.
> >
> > 2. Add some additional tests to strstr and memchr expanding alignments
> >    and adding more small values.
> > ---
> >  benchtests/bench-memchr.c | 108 ++++++++++++++-------
> >  benchtests/bench-memmem.c |  82 ++++++++++++----
> >  benchtests/bench-strstr.c | 191 +++++++++++++++++++++++++++++++-------
> >  3 files changed, 289 insertions(+), 92 deletions(-)
> >
> > diff --git a/benchtests/bench-memchr.c b/benchtests/bench-memchr.c
> > index fb0284f523..4d7212332f 100644
> > --- a/benchtests/bench-memchr.c
> > +++ b/benchtests/bench-memchr.c
> > @@ -53,8 +53,11 @@ SIMPLE_MEMCHR (const CHAR *s, int c, size_t n)
> >  }
> >  #endif /* !USE_AS_MEMRCHR */
> >
> > +#include "json-lib.h"
> > +
> >  static void
> > -do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
> > +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const CHAR *s, int c,
> > +            size_t n)
> >  {
> >    size_t i, iters = INNER_LOOP_ITERS_LARGE;
> >    timing_t start, stop, cur;
> > @@ -68,15 +71,16 @@ do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
> >
> >    TIMING_DIFF (cur, start, stop);
> >
> > -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> > +  json_element_double (json_ctx, (double) cur / (double) iters);
> >  }
> >
> >  static void
> > -do_test (size_t align, size_t pos, size_t len, int seek_char)
> > +do_test (json_ctx_t *json_ctx, size_t align, size_t pos, size_t len,
> > +        int seek_char)
> >  {
> >    size_t i;
> >
> > -  align &= 7;
> > +  align &= getpagesize () - 1;
> >    if ((align + len) * sizeof (CHAR) >= page_size)
> >      return;
> >
> > @@ -100,67 +104,99 @@ do_test (size_t align, size_t pos, size_t len, int seek_char)
> >        buf[align + len] = seek_char;
> >      }
> >
> > -  printf ("Length %4zd, position %4zd, alignment %2zd:",
> > -         len, pos, align);
> > +  json_element_object_begin (json_ctx);
> > +  json_attr_uint (json_ctx, "align", align);
> > +  json_attr_uint (json_ctx, "pos", pos);
> > +  json_attr_uint (json_ctx, "len", len);
> > +  json_attr_uint (json_ctx, "seek_char", seek_char);
> > +
> > +  json_array_begin (json_ctx, "timings");
> >
> >    FOR_EACH_IMPL (impl, 0)
> > -    do_one_test (impl, (CHAR *) (buf + align), seek_char, len);
> > +    do_one_test (json_ctx, impl, (CHAR *) (buf + align), seek_char, len);
> >
> > -  putchar ('\n');
> > +  json_array_end (json_ctx);
> > +  json_element_object_end (json_ctx);
> >  }
> >
> >  int
> >  test_main (void)
> >  {
> >    size_t i;
> > -
> > +  json_ctx_t json_ctx;
> >    test_init ();
> >
> > -  printf ("%20s", "");
> > +  json_init (&json_ctx, 0, stdout);
> > +
> > +  json_document_begin (&json_ctx);
> > +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> > +
> > +  json_attr_object_begin (&json_ctx, "functions");
> > +  json_attr_object_begin (&json_ctx, TEST_NAME);
> > +  json_attr_string (&json_ctx, "bench-variant", "");
> > +
> > +  json_array_begin (&json_ctx, "ifuncs");
> >    FOR_EACH_IMPL (impl, 0)
> > -    printf ("\t%s", impl->name);
> > -  putchar ('\n');
> > +    json_element_string (&json_ctx, impl->name);
> > +  json_array_end (&json_ctx);
> > +
> > +  json_array_begin (&json_ctx, "results");
> >
> >    for (i = 1; i < 8; ++i)
> >      {
> > -      do_test (0, 16 << i, 2048, 23);
> > -      do_test (i, 64, 256, 23);
> > -      do_test (0, 16 << i, 2048, 0);
> > -      do_test (i, 64, 256, 0);
> > +      do_test (&json_ctx, 0, 16 << i, 2048, 23);
> > +      do_test (&json_ctx, i, 64, 256, 23);
> > +      do_test (&json_ctx, 0, 16 << i, 2048, 0);
> > +      do_test (&json_ctx, i, 64, 256, 0);
> > +
> > +      do_test (&json_ctx, getpagesize () - 15, 64, 256, 0);
> >  #ifdef USE_AS_MEMRCHR
> >        /* Also test the position close to the beginning for memrchr.  */
> > -      do_test (0, i, 256, 23);
> > -      do_test (0, i, 256, 0);
> > -      do_test (i, i, 256, 23);
> > -      do_test (i, i, 256, 0);
> > +      do_test (&json_ctx, 0, i, 256, 23);
> > +      do_test (&json_ctx, 0, i, 256, 0);
> > +      do_test (&json_ctx, i, i, 256, 23);
> > +      do_test (&json_ctx, i, i, 256, 0);
> >  #endif
> >      }
> >    for (i = 1; i < 8; ++i)
> >      {
> > -      do_test (i, i << 5, 192, 23);
> > -      do_test (i, i << 5, 192, 0);
> > -      do_test (i, i << 5, 256, 23);
> > -      do_test (i, i << 5, 256, 0);
> > -      do_test (i, i << 5, 512, 23);
> > -      do_test (i, i << 5, 512, 0);
> > +      do_test (&json_ctx, i, i << 5, 192, 23);
> > +      do_test (&json_ctx, i, i << 5, 192, 0);
> > +      do_test (&json_ctx, i, i << 5, 256, 23);
> > +      do_test (&json_ctx, i, i << 5, 256, 0);
> > +      do_test (&json_ctx, i, i << 5, 512, 23);
> > +      do_test (&json_ctx, i, i << 5, 512, 0);
> > +
> > +      do_test (&json_ctx, getpagesize () - 15, i << 5, 256, 23);
> >      }
> >    for (i = 1; i < 32; ++i)
> >      {
> > -      do_test (0, i, i + 1, 23);
> > -      do_test (0, i, i + 1, 0);
> > -      do_test (i, i, i + 1, 23);
> > -      do_test (i, i, i + 1, 0);
> > -      do_test (0, i, i - 1, 23);
> > -      do_test (0, i, i - 1, 0);
> > -      do_test (i, i, i - 1, 23);
> > -      do_test (i, i, i - 1, 0);
> > +      do_test (&json_ctx, 0, i, i + 1, 23);
> > +      do_test (&json_ctx, 0, i, i + 1, 0);
> > +      do_test (&json_ctx, i, i, i + 1, 23);
> > +      do_test (&json_ctx, i, i, i + 1, 0);
> > +      do_test (&json_ctx, 0, i, i - 1, 23);
> > +      do_test (&json_ctx, 0, i, i - 1, 0);
> > +      do_test (&json_ctx, i, i, i - 1, 23);
> > +      do_test (&json_ctx, i, i, i - 1, 0);
> > +
> > +      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 23);
> > +      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 0);
> > +
> > +      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 23);
> > +      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 0);
> >  #ifdef USE_AS_MEMRCHR
> >        /* Also test the position close to the beginning for memrchr.  */
> > -      do_test (0, 1, i + 1, 23);
> > -      do_test (0, 2, i + 1, 0);
> > +      do_test (&json_ctx, 0, 1, i + 1, 23);
> > +      do_test (&json_ctx, 0, 2, i + 1, 0);
> >  #endif
> >      }
> >
> > +  json_array_end (&json_ctx);
> > +  json_attr_object_end (&json_ctx);
> > +  json_attr_object_end (&json_ctx);
> > +  json_document_end (&json_ctx);
> > +
> >    return ret;
> >  }
> >
> > diff --git a/benchtests/bench-memmem.c b/benchtests/bench-memmem.c
> > index 0fb87309c0..c2b92405dc 100644
> > --- a/benchtests/bench-memmem.c
> > +++ b/benchtests/bench-memmem.c
> > @@ -21,6 +21,7 @@
> >  #define BUF1PAGES 20
> >  #define ITERATIONS 100
> >  #include "bench-string.h"
> > +#include "json-lib.h"
> >
> >  typedef char *(*proto_t) (const void *, size_t, const void *, size_t);
> >
> > @@ -102,44 +103,62 @@ IMPL (twoway_memmem, 0)
> >  IMPL (basic_memmem, 0)
> >
> >  static void
> > -do_one_test (impl_t *impl, const void *haystack, size_t haystack_len,
> > -            const void *needle, size_t needle_len, const void *expected)
> > +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const void *haystack,
> > +            size_t haystack_len, const void *needle, size_t needle_len,
> > +            const void *expected)
> >  {
> >    size_t i, iters = INNER_LOOP_ITERS_SMALL;
> >    timing_t start, stop, cur;
> > -
> > +  void *res;
> >    TIMING_NOW (start);
> >    for (i = 0; i < iters; ++i)
> >      {
> > -      CALL (impl, haystack, haystack_len, needle, needle_len);
> > +      res = CALL (impl, haystack, haystack_len, needle, needle_len);
> >      }
> >    TIMING_NOW (stop);
> >
> >    TIMING_DIFF (cur, start, stop);
> >
> > -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> > +  json_element_double (json_ctx, (double) cur / (double) iters);
> > +
> > +  if (res != expected)
> > +    {
> > +      error (0, 0, "Wrong result in function (%p != %p) %s(%p, %zu, %p, %zu)",
> > +            res, expected, impl->name, haystack, haystack_len, needle,
> > +            needle_len);
> > +      ret = 1;
> > +    }
> >  }
> >
> >  static void
> > -do_test (const char *str, size_t len, size_t idx)
> > +do_test (json_ctx_t *json_ctx, const char *str, size_t len, size_t idx)
> >  {
> >    char tmpbuf[len];
> >
> >    memcpy (tmpbuf, buf1 + idx, len);
> >    memcpy (buf1 + idx, str, len);
> >
> > -  printf ("String %s, offset %zd:", str, idx);
> > +  json_element_object_begin (json_ctx);
> > +  json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
> > +  json_attr_uint (json_ctx, "len_needle", len);
> > +  json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
> > +  json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) str);
> > +  json_attr_uint (json_ctx, "fail", 0);
> > +
> > +  json_array_begin (json_ctx, "timings");
> >
> >    FOR_EACH_IMPL (impl, 0)
> > -    do_one_test (impl, buf1, BUF1PAGES * page_size, str, len, buf1 + idx);
> > +    do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, str, len,
> > +                buf1 + idx);
> >
> >    memcpy (buf1 + idx, tmpbuf, len);
> >
> > -  putchar ('\n');
> > +  json_array_end (json_ctx);
> > +  json_element_object_end (json_ctx);
> >  }
> >
> >  static void
> > -do_random_tests (void)
> > +do_random_tests (json_ctx_t *json_ctx)
> >  {
> >    for (size_t n = 0; n < ITERATIONS; ++n)
> >      {
> > @@ -159,13 +178,21 @@ do_random_tests (void)
> >           buf1[idx + off] = ch;
> >         }
> >
> > -      printf ("String %.*s, offset %zd:", (int) len, buf1 + idx, idx);
> > +      json_element_object_begin (json_ctx);
> > +      json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
> > +      json_attr_uint (json_ctx, "len_needle", len);
> > +      json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
> > +      json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) (buf1 + idx));
> > +      json_attr_uint (json_ctx, "fail", 0);
> > +
> > +      json_array_begin (json_ctx, "timings");
> >
> >        FOR_EACH_IMPL (impl, 0)
> > -       do_one_test (impl, buf1, BUF1PAGES * page_size, buf1 + idx, len,
> > -                    buf1 + idx);
> > +       do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, buf1 + idx,
> > +                    len, buf1 + idx);
> >
> > -      putchar ('\n');
> > +      json_array_end (json_ctx);
> > +      json_element_object_end (json_ctx);
> >
> >        memcpy (buf1 + idx, tmpbuf, len);
> >      }
> > @@ -178,18 +205,28 @@ static const char *const strs[] =
> >      "abc0", "aaaa0", "abcabc0"
> >    };
> >
> > -
> >  int
> >  test_main (void)
> >  {
> > +  json_ctx_t json_ctx;
> >    size_t i;
> >
> >    test_init ();
> > +  json_init (&json_ctx, 0, stdout);
> > +
> > +  json_document_begin (&json_ctx);
> > +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> > +
> > +  json_attr_object_begin (&json_ctx, "functions");
> > +  json_attr_object_begin (&json_ctx, TEST_NAME);
> > +  json_attr_string (&json_ctx, "bench-variant", "");
> >
> > -  printf ("%23s", "");
> > +  json_array_begin (&json_ctx, "ifuncs");
> >    FOR_EACH_IMPL (impl, 0)
> > -    printf ("\t%s", impl->name);
> > -  putchar ('\n');
> > +    json_element_string (&json_ctx, impl->name);
> > +  json_array_end (&json_ctx);
> > +
> > +  json_array_begin (&json_ctx, "results");
> >
> >    for (i = 0; i < BUF1PAGES * page_size; ++i)
> >      buf1[i] = 60 + random () % 32;
> > @@ -199,10 +236,15 @@ test_main (void)
> >        {
> >         size_t len = strlen (strs[i]);
> >
> > -       do_test (strs[i], len, j);
> > +       do_test (&json_ctx, strs[i], len, j);
> >        }
> >
> > -  do_random_tests ();
> > +  do_random_tests (&json_ctx);
> > +
> > +  json_array_end (&json_ctx);
> > +  json_attr_object_end (&json_ctx);
> > +  json_attr_object_end (&json_ctx);
> > +  json_document_end (&json_ctx);
> >    return ret;
> >  }
> >
> > diff --git a/benchtests/bench-strstr.c b/benchtests/bench-strstr.c
> > index 467418b12c..3838a2b6ac 100644
> > --- a/benchtests/bench-strstr.c
> > +++ b/benchtests/bench-strstr.c
> > @@ -21,6 +21,8 @@
> >  #define TEST_NAME "strstr"
> >  #include "bench-string.h"
> >
> > +#include "json-lib.h"
> > +
> >  static const char input[] =
> >  "This manual is written with the assumption that you are at least "
> >  "somewhat familiar with the C programming language and basic programming "
> > @@ -129,7 +131,8 @@ IMPL (twoway_strstr, 0)
> >  IMPL (basic_strstr, 0)
> >
> >  static void
> > -do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> > +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1,
> > +            const char *s2, char *exp_result)
> >  {
> >    size_t i, iters = INNER_LOOP_ITERS_SMALL / 8;
> >    timing_t start, stop, cur;
> > @@ -142,7 +145,7 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> >
> >    TIMING_DIFF (cur, start, stop);
> >
> > -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> > +  json_element_double (json_ctx, (double) cur / (double) iters);
> >
> >    if (res != exp_result)
> >      {
> > @@ -153,14 +156,16 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> >      }
> >  }
> >
> > -
> >  static void
> > -do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> > -        int fail)
> > +do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1,
> > +        size_t len2, int fail)
> >  {
> >    char *s1 = (char *) (buf1 + align1);
> >    char *s2 = (char *) (buf2 + align2);
> >
> > +  if (align1 + len1 >= MIN_PAGE_SIZE || align1 + len1 >= MIN_PAGE_SIZE)
>
> Is this really needed?  Did you mean align2 + len2?
Will fix align2/len2 in V2.

Not needed AFAICT for any of these but generally think it makes it easier
to add new tests w.o worrying about them failing.

Can remove if you in V2 if you want.
>
> > +    return;
> > +
> >    size_t size = sizeof (input) - 1;
> >    size_t pos = (len1 + len2) % size;
> >
> > @@ -195,13 +200,21 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> >      if (fail || ss1 != s1 + len1 - len2)
> >        ++ss1[len2 / 2];
> >
> > -  printf ("Length %4zd/%3zd, alignment %2zd/%2zd, %s:",
> > -         len1, len2, align1, align2, fail ? "fail " : "found");
> > +  json_element_object_begin (json_ctx);
> > +  json_attr_uint (json_ctx, "len_haystack", len1);
> > +  json_attr_uint (json_ctx, "len_needle", len2);
> > +  json_attr_uint (json_ctx, "align_haystack", align1);
> > +  json_attr_uint (json_ctx, "align_needle", align2);
> > +  json_attr_uint (json_ctx, "fail", fail);
> > +
> > +  json_array_begin (json_ctx, "timings");
> >
> >    FOR_EACH_IMPL (impl, 0)
> > -    do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> > +    do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> > +
> > +  json_array_end (json_ctx);
> > +  json_element_object_end (json_ctx);
> >
> > -  putchar ('\n');
> >  }
> >
> >  /* Test needles which exhibit worst-case performance.  This shows that
> > @@ -211,11 +224,14 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> >     within a factor of 2 on several different microarchitectures.  */
> >
> >  static void
> > -test_hard_needle (size_t ne_len, size_t hs_len)
> > +test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len)
> >  {
> >    char *ne = (char *) buf1;
> >    char *hs = (char *) buf2;
> >
> > +  if (ne_len >= MIN_PAGE_SIZE || hs_len >= MIN_PAGE_SIZE)
> > +    return;
> > +
> >    /* Hard needle for strstr algorithm using skip table.  This results in many
> >       memcmp calls comparing most of the needle.  */
> >    {
> > @@ -226,15 +242,25 @@ test_hard_needle (size_t ne_len, size_t hs_len)
> >      memset (hs, 'a', hs_len);
> >      for (size_t i = ne_len; i <= hs_len; i += ne_len)
> >        {
> > -       hs[i-5] = 'b';
> > -       hs[i-62] = 'b';
> > +       hs[i - 5] = 'b';
> > +       hs[i - 62] = 'b';
> >        }
> >
> > -    printf ("Length %4zd/%3zd, complex needle 1:", hs_len, ne_len);
> > +    json_element_object_begin (json_ctx);
> > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > +    json_attr_uint (json_ctx, "align_needle", 0);
> > +    json_attr_uint (json_ctx, "fail", 1);
> > +    json_attr_string (json_ctx, "desc", "Difficult skiptable(0)");
> > +
> > +    json_array_begin (json_ctx, "timings");
> >
> >      FOR_EACH_IMPL (impl, 0)
> > -      do_one_test (impl, hs, ne, NULL);
> > -    putchar ('\n');
> > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > +
> > +    json_array_end (json_ctx);
> > +    json_element_object_end (json_ctx);
> >    }
> >
> >    /* 2nd hard needle for strstr algorithm using skip table.  This results in
> > @@ -247,15 +273,25 @@ test_hard_needle (size_t ne_len, size_t hs_len)
> >      memset (hs, 'a', hs_len);
> >      for (size_t i = ne_len; i <= hs_len; i += ne_len)
> >        {
> > -       hs[i-5] = 'b';
> > -       hs[i-6] = 'b';
> > +       hs[i - 5] = 'b';
> > +       hs[i - 6] = 'b';
> >        }
> >
> > -    printf ("Length %4zd/%3zd, complex needle 2:", hs_len, ne_len);
> > +    json_element_object_begin (json_ctx);
> > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > +    json_attr_uint (json_ctx, "align_needle", 0);
> > +    json_attr_uint (json_ctx, "fail", 1);
> > +    json_attr_string (json_ctx, "desc", "Difficult skiptable(1)");
> > +
> > +    json_array_begin (json_ctx, "timings");
> >
> >      FOR_EACH_IMPL (impl, 0)
> > -      do_one_test (impl, hs, ne, NULL);
> > -    putchar ('\n');
> > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > +
> > +    json_array_end (json_ctx);
> > +    json_element_object_end (json_ctx);
> >    }
> >
> >    /* Hard needle for Two-way algorithm - the random input causes a large number
> > @@ -267,45 +303,128 @@ test_hard_needle (size_t ne_len, size_t hs_len)
> >      hs[hs_len] = 0;
> >
> >      memset (ne, 'a', ne_len);
> > -    ne[ne_len-2] = 'b';
> > +    ne[ne_len - 2] = 'b';
> >      ne[0] = 'b';
> >      ne[ne_len] = 0;
> >
> > -    printf ("Length %4zd/%3zd, complex needle 3:", hs_len, ne_len);
> > +    json_element_object_begin (json_ctx);
> > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > +    json_attr_uint (json_ctx, "align_needle", 0);
> > +    json_attr_uint (json_ctx, "fail", 1);
> > +    json_attr_string (json_ctx, "desc", "Difficult 2-way");
> > +
> > +    json_array_begin (json_ctx, "timings");
> > +
> > +    FOR_EACH_IMPL (impl, 0)
> > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > +
> > +    json_array_end (json_ctx);
> > +    json_element_object_end (json_ctx);
> > +  }
> > +
> > +  /* Hard needle for standard algorithm testing first few characters of
> > +   * needle.  */
> > +  {
> > +    for (int i = 0; i < hs_len; i++)
> > +      hs[i] = (rand () & 255) >= 128 ? 'a' : 'b';
> > +    hs[hs_len] = 0;
> > +
> > +    for (int i = 0; i < ne_len; i++)
> > +      {
> > +       if (i % 3 == 0)
> > +         ne[i] = 'a';
> > +       else if (i % 3 == 1)
> > +         ne[i] = 'b';
> > +       else
> > +         ne[i] = 'c';
> > +      }
> > +    ne[ne_len] = 0;
> > +
> > +    json_element_object_begin (json_ctx);
> > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > +    json_attr_uint (json_ctx, "align_needle", 0);
> > +    json_attr_uint (json_ctx, "fail", 1);
> > +    json_attr_string (json_ctx, "desc", "Difficult testing first 2");
> > +
> > +    json_array_begin (json_ctx, "timings");
> >
> >      FOR_EACH_IMPL (impl, 0)
> > -      do_one_test (impl, hs, ne, NULL);
> > -    putchar ('\n');
> > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > +
> > +    json_array_end (json_ctx);
> > +    json_element_object_end (json_ctx);
> >    }
> >  }
> >
> >  static int
> >  test_main (void)
> >  {
> > +  json_ctx_t json_ctx;
> >    test_init ();
> >
> > -  printf ("%23s", "");
> > +  json_init (&json_ctx, 0, stdout);
> > +
> > +  json_document_begin (&json_ctx);
> > +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> > +
> > +  json_attr_object_begin (&json_ctx, "functions");
> > +  json_attr_object_begin (&json_ctx, TEST_NAME);
> > +  json_attr_string (&json_ctx, "bench-variant", "");
> > +
> > +  json_array_begin (&json_ctx, "ifuncs");
> >    FOR_EACH_IMPL (impl, 0)
> > -    printf ("\t%s", impl->name);
> > -  putchar ('\n');
> > +    json_element_string (&json_ctx, impl->name);
> > +  json_array_end (&json_ctx);
> > +
> > +  json_array_begin (&json_ctx, "results");
> >
> > -  for (size_t hlen = 64; hlen <= 256; hlen += 32)
> > +  for (size_t hlen = 8; hlen <= 256;)
> >      for (size_t klen = 1; klen <= 16; klen++)
> >        {
> > -       do_test (1, 3, hlen, klen, 0);
> > -       do_test (0, 9, hlen, klen, 1);
> > +       do_test (&json_ctx, 1, 3, hlen, klen, 0);
> > +       do_test (&json_ctx, 0, 9, hlen, klen, 1);
> > +
> > +       do_test (&json_ctx, 1, 3, hlen + 1, klen, 0);
> > +       do_test (&json_ctx, 0, 9, hlen + 1, klen, 1);
> > +
> > +       do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1);
> > +       if (hlen < 64)
> > +         {
> > +           hlen += 8;
> > +         }
> > +       else
> > +         {
> > +           hlen += 32;
> > +         }
> >        }
> >
> >    for (size_t hlen = 256; hlen <= 65536; hlen *= 2)
> > -    for (size_t klen = 16; klen <= 256; klen *= 2)
> > +    for (size_t klen = 4; klen <= 256; klen *= 2)
> >        {
> > -       do_test (1, 11, hlen, klen, 0);
> > -       do_test (14, 5, hlen, klen, 1);
> > +       do_test (&json_ctx, 1, 11, hlen, klen, 0);
> > +       do_test (&json_ctx, 14, 5, hlen, klen, 1);
> > +
> > +    do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0);
> > +    do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1);
> > +
> > +       do_test (&json_ctx, 1, 11, hlen + 1, klen, 0);
> > +       do_test (&json_ctx, 14, 5, hlen + 1, klen, 1);
> > +
> > +       do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1);
> >        }
> >
> > -  test_hard_needle (64, 65536);
> > -  test_hard_needle (256, 65536);
> > -  test_hard_needle (1024, 65536);
> > +  test_hard_needle (&json_ctx, 64, 65536);
> > +  test_hard_needle (&json_ctx, 256, 65536);
> > +  test_hard_needle (&json_ctx, 1024, 65536);
> > +
> > +  json_array_end (&json_ctx);
> > +  json_attr_object_end (&json_ctx);
> > +  json_attr_object_end (&json_ctx);
> > +  json_document_end (&json_ctx);
> >
> >    return ret;
> >  }
> > --
> > 2.34.1
> >
>
>
> --
> H.J.
H.J. Lu May 27, 2022, 10:09 p.m. UTC | #3 
On Fri, May 27, 2022 at 2:46 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> On Fri, May 27, 2022 at 4:35 PM H.J. Lu <hjl.tools@gmail.com> wrote:
> >
> > On Fri, May 27, 2022 at 2:05 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
> > >
> > > 1. Use json_ctx for output to help standardize format across all
> > >    benchtests.
> > >
> > > 2. Add some additional tests to strstr and memchr expanding alignments
> > >    and adding more small values.
> > > ---
> > >  benchtests/bench-memchr.c | 108 ++++++++++++++-------
> > >  benchtests/bench-memmem.c |  82 ++++++++++++----
> > >  benchtests/bench-strstr.c | 191 +++++++++++++++++++++++++++++++-------
> > >  3 files changed, 289 insertions(+), 92 deletions(-)
> > >
> > > diff --git a/benchtests/bench-memchr.c b/benchtests/bench-memchr.c
> > > index fb0284f523..4d7212332f 100644
> > > --- a/benchtests/bench-memchr.c
> > > +++ b/benchtests/bench-memchr.c
> > > @@ -53,8 +53,11 @@ SIMPLE_MEMCHR (const CHAR *s, int c, size_t n)
> > >  }
> > >  #endif /* !USE_AS_MEMRCHR */
> > >
> > > +#include "json-lib.h"
> > > +
> > >  static void
> > > -do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
> > > +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const CHAR *s, int c,
> > > +            size_t n)
> > >  {
> > >    size_t i, iters = INNER_LOOP_ITERS_LARGE;
> > >    timing_t start, stop, cur;
> > > @@ -68,15 +71,16 @@ do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
> > >
> > >    TIMING_DIFF (cur, start, stop);
> > >
> > > -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> > > +  json_element_double (json_ctx, (double) cur / (double) iters);
> > >  }
> > >
> > >  static void
> > > -do_test (size_t align, size_t pos, size_t len, int seek_char)
> > > +do_test (json_ctx_t *json_ctx, size_t align, size_t pos, size_t len,
> > > +        int seek_char)
> > >  {
> > >    size_t i;
> > >
> > > -  align &= 7;
> > > +  align &= getpagesize () - 1;
> > >    if ((align + len) * sizeof (CHAR) >= page_size)
> > >      return;
> > >
> > > @@ -100,67 +104,99 @@ do_test (size_t align, size_t pos, size_t len, int seek_char)
> > >        buf[align + len] = seek_char;
> > >      }
> > >
> > > -  printf ("Length %4zd, position %4zd, alignment %2zd:",
> > > -         len, pos, align);
> > > +  json_element_object_begin (json_ctx);
> > > +  json_attr_uint (json_ctx, "align", align);
> > > +  json_attr_uint (json_ctx, "pos", pos);
> > > +  json_attr_uint (json_ctx, "len", len);
> > > +  json_attr_uint (json_ctx, "seek_char", seek_char);
> > > +
> > > +  json_array_begin (json_ctx, "timings");
> > >
> > >    FOR_EACH_IMPL (impl, 0)
> > > -    do_one_test (impl, (CHAR *) (buf + align), seek_char, len);
> > > +    do_one_test (json_ctx, impl, (CHAR *) (buf + align), seek_char, len);
> > >
> > > -  putchar ('\n');
> > > +  json_array_end (json_ctx);
> > > +  json_element_object_end (json_ctx);
> > >  }
> > >
> > >  int
> > >  test_main (void)
> > >  {
> > >    size_t i;
> > > -
> > > +  json_ctx_t json_ctx;
> > >    test_init ();
> > >
> > > -  printf ("%20s", "");
> > > +  json_init (&json_ctx, 0, stdout);
> > > +
> > > +  json_document_begin (&json_ctx);
> > > +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> > > +
> > > +  json_attr_object_begin (&json_ctx, "functions");
> > > +  json_attr_object_begin (&json_ctx, TEST_NAME);
> > > +  json_attr_string (&json_ctx, "bench-variant", "");
> > > +
> > > +  json_array_begin (&json_ctx, "ifuncs");
> > >    FOR_EACH_IMPL (impl, 0)
> > > -    printf ("\t%s", impl->name);
> > > -  putchar ('\n');
> > > +    json_element_string (&json_ctx, impl->name);
> > > +  json_array_end (&json_ctx);
> > > +
> > > +  json_array_begin (&json_ctx, "results");
> > >
> > >    for (i = 1; i < 8; ++i)
> > >      {
> > > -      do_test (0, 16 << i, 2048, 23);
> > > -      do_test (i, 64, 256, 23);
> > > -      do_test (0, 16 << i, 2048, 0);
> > > -      do_test (i, 64, 256, 0);
> > > +      do_test (&json_ctx, 0, 16 << i, 2048, 23);
> > > +      do_test (&json_ctx, i, 64, 256, 23);
> > > +      do_test (&json_ctx, 0, 16 << i, 2048, 0);
> > > +      do_test (&json_ctx, i, 64, 256, 0);
> > > +
> > > +      do_test (&json_ctx, getpagesize () - 15, 64, 256, 0);
> > >  #ifdef USE_AS_MEMRCHR
> > >        /* Also test the position close to the beginning for memrchr.  */
> > > -      do_test (0, i, 256, 23);
> > > -      do_test (0, i, 256, 0);
> > > -      do_test (i, i, 256, 23);
> > > -      do_test (i, i, 256, 0);
> > > +      do_test (&json_ctx, 0, i, 256, 23);
> > > +      do_test (&json_ctx, 0, i, 256, 0);
> > > +      do_test (&json_ctx, i, i, 256, 23);
> > > +      do_test (&json_ctx, i, i, 256, 0);
> > >  #endif
> > >      }
> > >    for (i = 1; i < 8; ++i)
> > >      {
> > > -      do_test (i, i << 5, 192, 23);
> > > -      do_test (i, i << 5, 192, 0);
> > > -      do_test (i, i << 5, 256, 23);
> > > -      do_test (i, i << 5, 256, 0);
> > > -      do_test (i, i << 5, 512, 23);
> > > -      do_test (i, i << 5, 512, 0);
> > > +      do_test (&json_ctx, i, i << 5, 192, 23);
> > > +      do_test (&json_ctx, i, i << 5, 192, 0);
> > > +      do_test (&json_ctx, i, i << 5, 256, 23);
> > > +      do_test (&json_ctx, i, i << 5, 256, 0);
> > > +      do_test (&json_ctx, i, i << 5, 512, 23);
> > > +      do_test (&json_ctx, i, i << 5, 512, 0);
> > > +
> > > +      do_test (&json_ctx, getpagesize () - 15, i << 5, 256, 23);
> > >      }
> > >    for (i = 1; i < 32; ++i)
> > >      {
> > > -      do_test (0, i, i + 1, 23);
> > > -      do_test (0, i, i + 1, 0);
> > > -      do_test (i, i, i + 1, 23);
> > > -      do_test (i, i, i + 1, 0);
> > > -      do_test (0, i, i - 1, 23);
> > > -      do_test (0, i, i - 1, 0);
> > > -      do_test (i, i, i - 1, 23);
> > > -      do_test (i, i, i - 1, 0);
> > > +      do_test (&json_ctx, 0, i, i + 1, 23);
> > > +      do_test (&json_ctx, 0, i, i + 1, 0);
> > > +      do_test (&json_ctx, i, i, i + 1, 23);
> > > +      do_test (&json_ctx, i, i, i + 1, 0);
> > > +      do_test (&json_ctx, 0, i, i - 1, 23);
> > > +      do_test (&json_ctx, 0, i, i - 1, 0);
> > > +      do_test (&json_ctx, i, i, i - 1, 23);
> > > +      do_test (&json_ctx, i, i, i - 1, 0);
> > > +
> > > +      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 23);
> > > +      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 0);
> > > +
> > > +      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 23);
> > > +      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 0);
> > >  #ifdef USE_AS_MEMRCHR
> > >        /* Also test the position close to the beginning for memrchr.  */
> > > -      do_test (0, 1, i + 1, 23);
> > > -      do_test (0, 2, i + 1, 0);
> > > +      do_test (&json_ctx, 0, 1, i + 1, 23);
> > > +      do_test (&json_ctx, 0, 2, i + 1, 0);
> > >  #endif
> > >      }
> > >
> > > +  json_array_end (&json_ctx);
> > > +  json_attr_object_end (&json_ctx);
> > > +  json_attr_object_end (&json_ctx);
> > > +  json_document_end (&json_ctx);
> > > +
> > >    return ret;
> > >  }
> > >
> > > diff --git a/benchtests/bench-memmem.c b/benchtests/bench-memmem.c
> > > index 0fb87309c0..c2b92405dc 100644
> > > --- a/benchtests/bench-memmem.c
> > > +++ b/benchtests/bench-memmem.c
> > > @@ -21,6 +21,7 @@
> > >  #define BUF1PAGES 20
> > >  #define ITERATIONS 100
> > >  #include "bench-string.h"
> > > +#include "json-lib.h"
> > >
> > >  typedef char *(*proto_t) (const void *, size_t, const void *, size_t);
> > >
> > > @@ -102,44 +103,62 @@ IMPL (twoway_memmem, 0)
> > >  IMPL (basic_memmem, 0)
> > >
> > >  static void
> > > -do_one_test (impl_t *impl, const void *haystack, size_t haystack_len,
> > > -            const void *needle, size_t needle_len, const void *expected)
> > > +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const void *haystack,
> > > +            size_t haystack_len, const void *needle, size_t needle_len,
> > > +            const void *expected)
> > >  {
> > >    size_t i, iters = INNER_LOOP_ITERS_SMALL;
> > >    timing_t start, stop, cur;
> > > -
> > > +  void *res;
> > >    TIMING_NOW (start);
> > >    for (i = 0; i < iters; ++i)
> > >      {
> > > -      CALL (impl, haystack, haystack_len, needle, needle_len);
> > > +      res = CALL (impl, haystack, haystack_len, needle, needle_len);
> > >      }
> > >    TIMING_NOW (stop);
> > >
> > >    TIMING_DIFF (cur, start, stop);
> > >
> > > -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> > > +  json_element_double (json_ctx, (double) cur / (double) iters);
> > > +
> > > +  if (res != expected)
> > > +    {
> > > +      error (0, 0, "Wrong result in function (%p != %p) %s(%p, %zu, %p, %zu)",
> > > +            res, expected, impl->name, haystack, haystack_len, needle,
> > > +            needle_len);
> > > +      ret = 1;
> > > +    }
> > >  }
> > >
> > >  static void
> > > -do_test (const char *str, size_t len, size_t idx)
> > > +do_test (json_ctx_t *json_ctx, const char *str, size_t len, size_t idx)
> > >  {
> > >    char tmpbuf[len];
> > >
> > >    memcpy (tmpbuf, buf1 + idx, len);
> > >    memcpy (buf1 + idx, str, len);
> > >
> > > -  printf ("String %s, offset %zd:", str, idx);
> > > +  json_element_object_begin (json_ctx);
> > > +  json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
> > > +  json_attr_uint (json_ctx, "len_needle", len);
> > > +  json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
> > > +  json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) str);
> > > +  json_attr_uint (json_ctx, "fail", 0);
> > > +
> > > +  json_array_begin (json_ctx, "timings");
> > >
> > >    FOR_EACH_IMPL (impl, 0)
> > > -    do_one_test (impl, buf1, BUF1PAGES * page_size, str, len, buf1 + idx);
> > > +    do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, str, len,
> > > +                buf1 + idx);
> > >
> > >    memcpy (buf1 + idx, tmpbuf, len);
> > >
> > > -  putchar ('\n');
> > > +  json_array_end (json_ctx);
> > > +  json_element_object_end (json_ctx);
> > >  }
> > >
> > >  static void
> > > -do_random_tests (void)
> > > +do_random_tests (json_ctx_t *json_ctx)
> > >  {
> > >    for (size_t n = 0; n < ITERATIONS; ++n)
> > >      {
> > > @@ -159,13 +178,21 @@ do_random_tests (void)
> > >           buf1[idx + off] = ch;
> > >         }
> > >
> > > -      printf ("String %.*s, offset %zd:", (int) len, buf1 + idx, idx);
> > > +      json_element_object_begin (json_ctx);
> > > +      json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
> > > +      json_attr_uint (json_ctx, "len_needle", len);
> > > +      json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
> > > +      json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) (buf1 + idx));
> > > +      json_attr_uint (json_ctx, "fail", 0);
> > > +
> > > +      json_array_begin (json_ctx, "timings");
> > >
> > >        FOR_EACH_IMPL (impl, 0)
> > > -       do_one_test (impl, buf1, BUF1PAGES * page_size, buf1 + idx, len,
> > > -                    buf1 + idx);
> > > +       do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, buf1 + idx,
> > > +                    len, buf1 + idx);
> > >
> > > -      putchar ('\n');
> > > +      json_array_end (json_ctx);
> > > +      json_element_object_end (json_ctx);
> > >
> > >        memcpy (buf1 + idx, tmpbuf, len);
> > >      }
> > > @@ -178,18 +205,28 @@ static const char *const strs[] =
> > >      "abc0", "aaaa0", "abcabc0"
> > >    };
> > >
> > > -
> > >  int
> > >  test_main (void)
> > >  {
> > > +  json_ctx_t json_ctx;
> > >    size_t i;
> > >
> > >    test_init ();
> > > +  json_init (&json_ctx, 0, stdout);
> > > +
> > > +  json_document_begin (&json_ctx);
> > > +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> > > +
> > > +  json_attr_object_begin (&json_ctx, "functions");
> > > +  json_attr_object_begin (&json_ctx, TEST_NAME);
> > > +  json_attr_string (&json_ctx, "bench-variant", "");
> > >
> > > -  printf ("%23s", "");
> > > +  json_array_begin (&json_ctx, "ifuncs");
> > >    FOR_EACH_IMPL (impl, 0)
> > > -    printf ("\t%s", impl->name);
> > > -  putchar ('\n');
> > > +    json_element_string (&json_ctx, impl->name);
> > > +  json_array_end (&json_ctx);
> > > +
> > > +  json_array_begin (&json_ctx, "results");
> > >
> > >    for (i = 0; i < BUF1PAGES * page_size; ++i)
> > >      buf1[i] = 60 + random () % 32;
> > > @@ -199,10 +236,15 @@ test_main (void)
> > >        {
> > >         size_t len = strlen (strs[i]);
> > >
> > > -       do_test (strs[i], len, j);
> > > +       do_test (&json_ctx, strs[i], len, j);
> > >        }
> > >
> > > -  do_random_tests ();
> > > +  do_random_tests (&json_ctx);
> > > +
> > > +  json_array_end (&json_ctx);
> > > +  json_attr_object_end (&json_ctx);
> > > +  json_attr_object_end (&json_ctx);
> > > +  json_document_end (&json_ctx);
> > >    return ret;
> > >  }
> > >
> > > diff --git a/benchtests/bench-strstr.c b/benchtests/bench-strstr.c
> > > index 467418b12c..3838a2b6ac 100644
> > > --- a/benchtests/bench-strstr.c
> > > +++ b/benchtests/bench-strstr.c
> > > @@ -21,6 +21,8 @@
> > >  #define TEST_NAME "strstr"
> > >  #include "bench-string.h"
> > >
> > > +#include "json-lib.h"
> > > +
> > >  static const char input[] =
> > >  "This manual is written with the assumption that you are at least "
> > >  "somewhat familiar with the C programming language and basic programming "
> > > @@ -129,7 +131,8 @@ IMPL (twoway_strstr, 0)
> > >  IMPL (basic_strstr, 0)
> > >
> > >  static void
> > > -do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> > > +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1,
> > > +            const char *s2, char *exp_result)
> > >  {
> > >    size_t i, iters = INNER_LOOP_ITERS_SMALL / 8;
> > >    timing_t start, stop, cur;
> > > @@ -142,7 +145,7 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> > >
> > >    TIMING_DIFF (cur, start, stop);
> > >
> > > -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> > > +  json_element_double (json_ctx, (double) cur / (double) iters);
> > >
> > >    if (res != exp_result)
> > >      {
> > > @@ -153,14 +156,16 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> > >      }
> > >  }
> > >
> > > -
> > >  static void
> > > -do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> > > -        int fail)
> > > +do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1,
> > > +        size_t len2, int fail)
> > >  {
> > >    char *s1 = (char *) (buf1 + align1);
> > >    char *s2 = (char *) (buf2 + align2);
> > >
> > > +  if (align1 + len1 >= MIN_PAGE_SIZE || align1 + len1 >= MIN_PAGE_SIZE)
> >
> > Is this really needed?  Did you mean align2 + len2?
> Will fix align2/len2 in V2.
>
> Not needed AFAICT for any of these but generally think it makes it easier
> to add new tests w.o worrying about them failing.

It means that the new tests are ignored.

> Can remove if you in V2 if you want.

Please remove it.

Thanks.

> >
> > > +    return;
> > > +
> > >    size_t size = sizeof (input) - 1;
> > >    size_t pos = (len1 + len2) % size;
> > >
> > > @@ -195,13 +200,21 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> > >      if (fail || ss1 != s1 + len1 - len2)
> > >        ++ss1[len2 / 2];
> > >
> > > -  printf ("Length %4zd/%3zd, alignment %2zd/%2zd, %s:",
> > > -         len1, len2, align1, align2, fail ? "fail " : "found");
> > > +  json_element_object_begin (json_ctx);
> > > +  json_attr_uint (json_ctx, "len_haystack", len1);
> > > +  json_attr_uint (json_ctx, "len_needle", len2);
> > > +  json_attr_uint (json_ctx, "align_haystack", align1);
> > > +  json_attr_uint (json_ctx, "align_needle", align2);
> > > +  json_attr_uint (json_ctx, "fail", fail);
> > > +
> > > +  json_array_begin (json_ctx, "timings");
> > >
> > >    FOR_EACH_IMPL (impl, 0)
> > > -    do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> > > +    do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> > > +
> > > +  json_array_end (json_ctx);
> > > +  json_element_object_end (json_ctx);
> > >
> > > -  putchar ('\n');
> > >  }
> > >
> > >  /* Test needles which exhibit worst-case performance.  This shows that
> > > @@ -211,11 +224,14 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> > >     within a factor of 2 on several different microarchitectures.  */
> > >
> > >  static void
> > > -test_hard_needle (size_t ne_len, size_t hs_len)
> > > +test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len)
> > >  {
> > >    char *ne = (char *) buf1;
> > >    char *hs = (char *) buf2;
> > >
> > > +  if (ne_len >= MIN_PAGE_SIZE || hs_len >= MIN_PAGE_SIZE)
> > > +    return;
> > > +
> > >    /* Hard needle for strstr algorithm using skip table.  This results in many
> > >       memcmp calls comparing most of the needle.  */
> > >    {
> > > @@ -226,15 +242,25 @@ test_hard_needle (size_t ne_len, size_t hs_len)
> > >      memset (hs, 'a', hs_len);
> > >      for (size_t i = ne_len; i <= hs_len; i += ne_len)
> > >        {
> > > -       hs[i-5] = 'b';
> > > -       hs[i-62] = 'b';
> > > +       hs[i - 5] = 'b';
> > > +       hs[i - 62] = 'b';
> > >        }
> > >
> > > -    printf ("Length %4zd/%3zd, complex needle 1:", hs_len, ne_len);
> > > +    json_element_object_begin (json_ctx);
> > > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > > +    json_attr_uint (json_ctx, "align_needle", 0);
> > > +    json_attr_uint (json_ctx, "fail", 1);
> > > +    json_attr_string (json_ctx, "desc", "Difficult skiptable(0)");
> > > +
> > > +    json_array_begin (json_ctx, "timings");
> > >
> > >      FOR_EACH_IMPL (impl, 0)
> > > -      do_one_test (impl, hs, ne, NULL);
> > > -    putchar ('\n');
> > > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > > +
> > > +    json_array_end (json_ctx);
> > > +    json_element_object_end (json_ctx);
> > >    }
> > >
> > >    /* 2nd hard needle for strstr algorithm using skip table.  This results in
> > > @@ -247,15 +273,25 @@ test_hard_needle (size_t ne_len, size_t hs_len)
> > >      memset (hs, 'a', hs_len);
> > >      for (size_t i = ne_len; i <= hs_len; i += ne_len)
> > >        {
> > > -       hs[i-5] = 'b';
> > > -       hs[i-6] = 'b';
> > > +       hs[i - 5] = 'b';
> > > +       hs[i - 6] = 'b';
> > >        }
> > >
> > > -    printf ("Length %4zd/%3zd, complex needle 2:", hs_len, ne_len);
> > > +    json_element_object_begin (json_ctx);
> > > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > > +    json_attr_uint (json_ctx, "align_needle", 0);
> > > +    json_attr_uint (json_ctx, "fail", 1);
> > > +    json_attr_string (json_ctx, "desc", "Difficult skiptable(1)");
> > > +
> > > +    json_array_begin (json_ctx, "timings");
> > >
> > >      FOR_EACH_IMPL (impl, 0)
> > > -      do_one_test (impl, hs, ne, NULL);
> > > -    putchar ('\n');
> > > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > > +
> > > +    json_array_end (json_ctx);
> > > +    json_element_object_end (json_ctx);
> > >    }
> > >
> > >    /* Hard needle for Two-way algorithm - the random input causes a large number
> > > @@ -267,45 +303,128 @@ test_hard_needle (size_t ne_len, size_t hs_len)
> > >      hs[hs_len] = 0;
> > >
> > >      memset (ne, 'a', ne_len);
> > > -    ne[ne_len-2] = 'b';
> > > +    ne[ne_len - 2] = 'b';
> > >      ne[0] = 'b';
> > >      ne[ne_len] = 0;
> > >
> > > -    printf ("Length %4zd/%3zd, complex needle 3:", hs_len, ne_len);
> > > +    json_element_object_begin (json_ctx);
> > > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > > +    json_attr_uint (json_ctx, "align_needle", 0);
> > > +    json_attr_uint (json_ctx, "fail", 1);
> > > +    json_attr_string (json_ctx, "desc", "Difficult 2-way");
> > > +
> > > +    json_array_begin (json_ctx, "timings");
> > > +
> > > +    FOR_EACH_IMPL (impl, 0)
> > > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > > +
> > > +    json_array_end (json_ctx);
> > > +    json_element_object_end (json_ctx);
> > > +  }
> > > +
> > > +  /* Hard needle for standard algorithm testing first few characters of
> > > +   * needle.  */
> > > +  {
> > > +    for (int i = 0; i < hs_len; i++)
> > > +      hs[i] = (rand () & 255) >= 128 ? 'a' : 'b';
> > > +    hs[hs_len] = 0;
> > > +
> > > +    for (int i = 0; i < ne_len; i++)
> > > +      {
> > > +       if (i % 3 == 0)
> > > +         ne[i] = 'a';
> > > +       else if (i % 3 == 1)
> > > +         ne[i] = 'b';
> > > +       else
> > > +         ne[i] = 'c';
> > > +      }
> > > +    ne[ne_len] = 0;
> > > +
> > > +    json_element_object_begin (json_ctx);
> > > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > > +    json_attr_uint (json_ctx, "align_needle", 0);
> > > +    json_attr_uint (json_ctx, "fail", 1);
> > > +    json_attr_string (json_ctx, "desc", "Difficult testing first 2");
> > > +
> > > +    json_array_begin (json_ctx, "timings");
> > >
> > >      FOR_EACH_IMPL (impl, 0)
> > > -      do_one_test (impl, hs, ne, NULL);
> > > -    putchar ('\n');
> > > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > > +
> > > +    json_array_end (json_ctx);
> > > +    json_element_object_end (json_ctx);
> > >    }
> > >  }
> > >
> > >  static int
> > >  test_main (void)
> > >  {
> > > +  json_ctx_t json_ctx;
> > >    test_init ();
> > >
> > > -  printf ("%23s", "");
> > > +  json_init (&json_ctx, 0, stdout);
> > > +
> > > +  json_document_begin (&json_ctx);
> > > +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> > > +
> > > +  json_attr_object_begin (&json_ctx, "functions");
> > > +  json_attr_object_begin (&json_ctx, TEST_NAME);
> > > +  json_attr_string (&json_ctx, "bench-variant", "");
> > > +
> > > +  json_array_begin (&json_ctx, "ifuncs");
> > >    FOR_EACH_IMPL (impl, 0)
> > > -    printf ("\t%s", impl->name);
> > > -  putchar ('\n');
> > > +    json_element_string (&json_ctx, impl->name);
> > > +  json_array_end (&json_ctx);
> > > +
> > > +  json_array_begin (&json_ctx, "results");
> > >
> > > -  for (size_t hlen = 64; hlen <= 256; hlen += 32)
> > > +  for (size_t hlen = 8; hlen <= 256;)
> > >      for (size_t klen = 1; klen <= 16; klen++)
> > >        {
> > > -       do_test (1, 3, hlen, klen, 0);
> > > -       do_test (0, 9, hlen, klen, 1);
> > > +       do_test (&json_ctx, 1, 3, hlen, klen, 0);
> > > +       do_test (&json_ctx, 0, 9, hlen, klen, 1);
> > > +
> > > +       do_test (&json_ctx, 1, 3, hlen + 1, klen, 0);
> > > +       do_test (&json_ctx, 0, 9, hlen + 1, klen, 1);
> > > +
> > > +       do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1);
> > > +       if (hlen < 64)
> > > +         {
> > > +           hlen += 8;
> > > +         }
> > > +       else
> > > +         {
> > > +           hlen += 32;
> > > +         }
> > >        }
> > >
> > >    for (size_t hlen = 256; hlen <= 65536; hlen *= 2)
> > > -    for (size_t klen = 16; klen <= 256; klen *= 2)
> > > +    for (size_t klen = 4; klen <= 256; klen *= 2)
> > >        {
> > > -       do_test (1, 11, hlen, klen, 0);
> > > -       do_test (14, 5, hlen, klen, 1);
> > > +       do_test (&json_ctx, 1, 11, hlen, klen, 0);
> > > +       do_test (&json_ctx, 14, 5, hlen, klen, 1);
> > > +
> > > +    do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0);
> > > +    do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1);
> > > +
> > > +       do_test (&json_ctx, 1, 11, hlen + 1, klen, 0);
> > > +       do_test (&json_ctx, 14, 5, hlen + 1, klen, 1);
> > > +
> > > +       do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1);
> > >        }
> > >
> > > -  test_hard_needle (64, 65536);
> > > -  test_hard_needle (256, 65536);
> > > -  test_hard_needle (1024, 65536);
> > > +  test_hard_needle (&json_ctx, 64, 65536);
> > > +  test_hard_needle (&json_ctx, 256, 65536);
> > > +  test_hard_needle (&json_ctx, 1024, 65536);
> > > +
> > > +  json_array_end (&json_ctx);
> > > +  json_attr_object_end (&json_ctx);
> > > +  json_attr_object_end (&json_ctx);
> > > +  json_document_end (&json_ctx);
> > >
> > >    return ret;
> > >  }
> > > --
> > > 2.34.1
> > >
> >
> >
> > --
> > H.J.
Noah Goldstein May 27, 2022, 11:26 p.m. UTC | #4 
On Fri, May 27, 2022 at 5:09 PM H.J. Lu <hjl.tools@gmail.com> wrote:
>
> On Fri, May 27, 2022 at 2:46 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
> >
> > On Fri, May 27, 2022 at 4:35 PM H.J. Lu <hjl.tools@gmail.com> wrote:
> > >
> > > On Fri, May 27, 2022 at 2:05 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
> > > >
> > > > 1. Use json_ctx for output to help standardize format across all
> > > >    benchtests.
> > > >
> > > > 2. Add some additional tests to strstr and memchr expanding alignments
> > > >    and adding more small values.
> > > > ---
> > > >  benchtests/bench-memchr.c | 108 ++++++++++++++-------
> > > >  benchtests/bench-memmem.c |  82 ++++++++++++----
> > > >  benchtests/bench-strstr.c | 191 +++++++++++++++++++++++++++++++-------
> > > >  3 files changed, 289 insertions(+), 92 deletions(-)
> > > >
> > > > diff --git a/benchtests/bench-memchr.c b/benchtests/bench-memchr.c
> > > > index fb0284f523..4d7212332f 100644
> > > > --- a/benchtests/bench-memchr.c
> > > > +++ b/benchtests/bench-memchr.c
> > > > @@ -53,8 +53,11 @@ SIMPLE_MEMCHR (const CHAR *s, int c, size_t n)
> > > >  }
> > > >  #endif /* !USE_AS_MEMRCHR */
> > > >
> > > > +#include "json-lib.h"
> > > > +
> > > >  static void
> > > > -do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
> > > > +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const CHAR *s, int c,
> > > > +            size_t n)
> > > >  {
> > > >    size_t i, iters = INNER_LOOP_ITERS_LARGE;
> > > >    timing_t start, stop, cur;
> > > > @@ -68,15 +71,16 @@ do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
> > > >
> > > >    TIMING_DIFF (cur, start, stop);
> > > >
> > > > -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> > > > +  json_element_double (json_ctx, (double) cur / (double) iters);
> > > >  }
> > > >
> > > >  static void
> > > > -do_test (size_t align, size_t pos, size_t len, int seek_char)
> > > > +do_test (json_ctx_t *json_ctx, size_t align, size_t pos, size_t len,
> > > > +        int seek_char)
> > > >  {
> > > >    size_t i;
> > > >
> > > > -  align &= 7;
> > > > +  align &= getpagesize () - 1;
> > > >    if ((align + len) * sizeof (CHAR) >= page_size)
> > > >      return;
> > > >
> > > > @@ -100,67 +104,99 @@ do_test (size_t align, size_t pos, size_t len, int seek_char)
> > > >        buf[align + len] = seek_char;
> > > >      }
> > > >
> > > > -  printf ("Length %4zd, position %4zd, alignment %2zd:",
> > > > -         len, pos, align);
> > > > +  json_element_object_begin (json_ctx);
> > > > +  json_attr_uint (json_ctx, "align", align);
> > > > +  json_attr_uint (json_ctx, "pos", pos);
> > > > +  json_attr_uint (json_ctx, "len", len);
> > > > +  json_attr_uint (json_ctx, "seek_char", seek_char);
> > > > +
> > > > +  json_array_begin (json_ctx, "timings");
> > > >
> > > >    FOR_EACH_IMPL (impl, 0)
> > > > -    do_one_test (impl, (CHAR *) (buf + align), seek_char, len);
> > > > +    do_one_test (json_ctx, impl, (CHAR *) (buf + align), seek_char, len);
> > > >
> > > > -  putchar ('\n');
> > > > +  json_array_end (json_ctx);
> > > > +  json_element_object_end (json_ctx);
> > > >  }
> > > >
> > > >  int
> > > >  test_main (void)
> > > >  {
> > > >    size_t i;
> > > > -
> > > > +  json_ctx_t json_ctx;
> > > >    test_init ();
> > > >
> > > > -  printf ("%20s", "");
> > > > +  json_init (&json_ctx, 0, stdout);
> > > > +
> > > > +  json_document_begin (&json_ctx);
> > > > +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> > > > +
> > > > +  json_attr_object_begin (&json_ctx, "functions");
> > > > +  json_attr_object_begin (&json_ctx, TEST_NAME);
> > > > +  json_attr_string (&json_ctx, "bench-variant", "");
> > > > +
> > > > +  json_array_begin (&json_ctx, "ifuncs");
> > > >    FOR_EACH_IMPL (impl, 0)
> > > > -    printf ("\t%s", impl->name);
> > > > -  putchar ('\n');
> > > > +    json_element_string (&json_ctx, impl->name);
> > > > +  json_array_end (&json_ctx);
> > > > +
> > > > +  json_array_begin (&json_ctx, "results");
> > > >
> > > >    for (i = 1; i < 8; ++i)
> > > >      {
> > > > -      do_test (0, 16 << i, 2048, 23);
> > > > -      do_test (i, 64, 256, 23);
> > > > -      do_test (0, 16 << i, 2048, 0);
> > > > -      do_test (i, 64, 256, 0);
> > > > +      do_test (&json_ctx, 0, 16 << i, 2048, 23);
> > > > +      do_test (&json_ctx, i, 64, 256, 23);
> > > > +      do_test (&json_ctx, 0, 16 << i, 2048, 0);
> > > > +      do_test (&json_ctx, i, 64, 256, 0);
> > > > +
> > > > +      do_test (&json_ctx, getpagesize () - 15, 64, 256, 0);
> > > >  #ifdef USE_AS_MEMRCHR
> > > >        /* Also test the position close to the beginning for memrchr.  */
> > > > -      do_test (0, i, 256, 23);
> > > > -      do_test (0, i, 256, 0);
> > > > -      do_test (i, i, 256, 23);
> > > > -      do_test (i, i, 256, 0);
> > > > +      do_test (&json_ctx, 0, i, 256, 23);
> > > > +      do_test (&json_ctx, 0, i, 256, 0);
> > > > +      do_test (&json_ctx, i, i, 256, 23);
> > > > +      do_test (&json_ctx, i, i, 256, 0);
> > > >  #endif
> > > >      }
> > > >    for (i = 1; i < 8; ++i)
> > > >      {
> > > > -      do_test (i, i << 5, 192, 23);
> > > > -      do_test (i, i << 5, 192, 0);
> > > > -      do_test (i, i << 5, 256, 23);
> > > > -      do_test (i, i << 5, 256, 0);
> > > > -      do_test (i, i << 5, 512, 23);
> > > > -      do_test (i, i << 5, 512, 0);
> > > > +      do_test (&json_ctx, i, i << 5, 192, 23);
> > > > +      do_test (&json_ctx, i, i << 5, 192, 0);
> > > > +      do_test (&json_ctx, i, i << 5, 256, 23);
> > > > +      do_test (&json_ctx, i, i << 5, 256, 0);
> > > > +      do_test (&json_ctx, i, i << 5, 512, 23);
> > > > +      do_test (&json_ctx, i, i << 5, 512, 0);
> > > > +
> > > > +      do_test (&json_ctx, getpagesize () - 15, i << 5, 256, 23);
> > > >      }
> > > >    for (i = 1; i < 32; ++i)
> > > >      {
> > > > -      do_test (0, i, i + 1, 23);
> > > > -      do_test (0, i, i + 1, 0);
> > > > -      do_test (i, i, i + 1, 23);
> > > > -      do_test (i, i, i + 1, 0);
> > > > -      do_test (0, i, i - 1, 23);
> > > > -      do_test (0, i, i - 1, 0);
> > > > -      do_test (i, i, i - 1, 23);
> > > > -      do_test (i, i, i - 1, 0);
> > > > +      do_test (&json_ctx, 0, i, i + 1, 23);
> > > > +      do_test (&json_ctx, 0, i, i + 1, 0);
> > > > +      do_test (&json_ctx, i, i, i + 1, 23);
> > > > +      do_test (&json_ctx, i, i, i + 1, 0);
> > > > +      do_test (&json_ctx, 0, i, i - 1, 23);
> > > > +      do_test (&json_ctx, 0, i, i - 1, 0);
> > > > +      do_test (&json_ctx, i, i, i - 1, 23);
> > > > +      do_test (&json_ctx, i, i, i - 1, 0);
> > > > +
> > > > +      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 23);
> > > > +      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 0);
> > > > +
> > > > +      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 23);
> > > > +      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 0);
> > > >  #ifdef USE_AS_MEMRCHR
> > > >        /* Also test the position close to the beginning for memrchr.  */
> > > > -      do_test (0, 1, i + 1, 23);
> > > > -      do_test (0, 2, i + 1, 0);
> > > > +      do_test (&json_ctx, 0, 1, i + 1, 23);
> > > > +      do_test (&json_ctx, 0, 2, i + 1, 0);
> > > >  #endif
> > > >      }
> > > >
> > > > +  json_array_end (&json_ctx);
> > > > +  json_attr_object_end (&json_ctx);
> > > > +  json_attr_object_end (&json_ctx);
> > > > +  json_document_end (&json_ctx);
> > > > +
> > > >    return ret;
> > > >  }
> > > >
> > > > diff --git a/benchtests/bench-memmem.c b/benchtests/bench-memmem.c
> > > > index 0fb87309c0..c2b92405dc 100644
> > > > --- a/benchtests/bench-memmem.c
> > > > +++ b/benchtests/bench-memmem.c
> > > > @@ -21,6 +21,7 @@
> > > >  #define BUF1PAGES 20
> > > >  #define ITERATIONS 100
> > > >  #include "bench-string.h"
> > > > +#include "json-lib.h"
> > > >
> > > >  typedef char *(*proto_t) (const void *, size_t, const void *, size_t);
> > > >
> > > > @@ -102,44 +103,62 @@ IMPL (twoway_memmem, 0)
> > > >  IMPL (basic_memmem, 0)
> > > >
> > > >  static void
> > > > -do_one_test (impl_t *impl, const void *haystack, size_t haystack_len,
> > > > -            const void *needle, size_t needle_len, const void *expected)
> > > > +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const void *haystack,
> > > > +            size_t haystack_len, const void *needle, size_t needle_len,
> > > > +            const void *expected)
> > > >  {
> > > >    size_t i, iters = INNER_LOOP_ITERS_SMALL;
> > > >    timing_t start, stop, cur;
> > > > -
> > > > +  void *res;
> > > >    TIMING_NOW (start);
> > > >    for (i = 0; i < iters; ++i)
> > > >      {
> > > > -      CALL (impl, haystack, haystack_len, needle, needle_len);
> > > > +      res = CALL (impl, haystack, haystack_len, needle, needle_len);
> > > >      }
> > > >    TIMING_NOW (stop);
> > > >
> > > >    TIMING_DIFF (cur, start, stop);
> > > >
> > > > -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> > > > +  json_element_double (json_ctx, (double) cur / (double) iters);
> > > > +
> > > > +  if (res != expected)
> > > > +    {
> > > > +      error (0, 0, "Wrong result in function (%p != %p) %s(%p, %zu, %p, %zu)",
> > > > +            res, expected, impl->name, haystack, haystack_len, needle,
> > > > +            needle_len);
> > > > +      ret = 1;
> > > > +    }
> > > >  }
> > > >
> > > >  static void
> > > > -do_test (const char *str, size_t len, size_t idx)
> > > > +do_test (json_ctx_t *json_ctx, const char *str, size_t len, size_t idx)
> > > >  {
> > > >    char tmpbuf[len];
> > > >
> > > >    memcpy (tmpbuf, buf1 + idx, len);
> > > >    memcpy (buf1 + idx, str, len);
> > > >
> > > > -  printf ("String %s, offset %zd:", str, idx);
> > > > +  json_element_object_begin (json_ctx);
> > > > +  json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
> > > > +  json_attr_uint (json_ctx, "len_needle", len);
> > > > +  json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
> > > > +  json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) str);
> > > > +  json_attr_uint (json_ctx, "fail", 0);
> > > > +
> > > > +  json_array_begin (json_ctx, "timings");
> > > >
> > > >    FOR_EACH_IMPL (impl, 0)
> > > > -    do_one_test (impl, buf1, BUF1PAGES * page_size, str, len, buf1 + idx);
> > > > +    do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, str, len,
> > > > +                buf1 + idx);
> > > >
> > > >    memcpy (buf1 + idx, tmpbuf, len);
> > > >
> > > > -  putchar ('\n');
> > > > +  json_array_end (json_ctx);
> > > > +  json_element_object_end (json_ctx);
> > > >  }
> > > >
> > > >  static void
> > > > -do_random_tests (void)
> > > > +do_random_tests (json_ctx_t *json_ctx)
> > > >  {
> > > >    for (size_t n = 0; n < ITERATIONS; ++n)
> > > >      {
> > > > @@ -159,13 +178,21 @@ do_random_tests (void)
> > > >           buf1[idx + off] = ch;
> > > >         }
> > > >
> > > > -      printf ("String %.*s, offset %zd:", (int) len, buf1 + idx, idx);
> > > > +      json_element_object_begin (json_ctx);
> > > > +      json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
> > > > +      json_attr_uint (json_ctx, "len_needle", len);
> > > > +      json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
> > > > +      json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) (buf1 + idx));
> > > > +      json_attr_uint (json_ctx, "fail", 0);
> > > > +
> > > > +      json_array_begin (json_ctx, "timings");
> > > >
> > > >        FOR_EACH_IMPL (impl, 0)
> > > > -       do_one_test (impl, buf1, BUF1PAGES * page_size, buf1 + idx, len,
> > > > -                    buf1 + idx);
> > > > +       do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, buf1 + idx,
> > > > +                    len, buf1 + idx);
> > > >
> > > > -      putchar ('\n');
> > > > +      json_array_end (json_ctx);
> > > > +      json_element_object_end (json_ctx);
> > > >
> > > >        memcpy (buf1 + idx, tmpbuf, len);
> > > >      }
> > > > @@ -178,18 +205,28 @@ static const char *const strs[] =
> > > >      "abc0", "aaaa0", "abcabc0"
> > > >    };
> > > >
> > > > -
> > > >  int
> > > >  test_main (void)
> > > >  {
> > > > +  json_ctx_t json_ctx;
> > > >    size_t i;
> > > >
> > > >    test_init ();
> > > > +  json_init (&json_ctx, 0, stdout);
> > > > +
> > > > +  json_document_begin (&json_ctx);
> > > > +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> > > > +
> > > > +  json_attr_object_begin (&json_ctx, "functions");
> > > > +  json_attr_object_begin (&json_ctx, TEST_NAME);
> > > > +  json_attr_string (&json_ctx, "bench-variant", "");
> > > >
> > > > -  printf ("%23s", "");
> > > > +  json_array_begin (&json_ctx, "ifuncs");
> > > >    FOR_EACH_IMPL (impl, 0)
> > > > -    printf ("\t%s", impl->name);
> > > > -  putchar ('\n');
> > > > +    json_element_string (&json_ctx, impl->name);
> > > > +  json_array_end (&json_ctx);
> > > > +
> > > > +  json_array_begin (&json_ctx, "results");
> > > >
> > > >    for (i = 0; i < BUF1PAGES * page_size; ++i)
> > > >      buf1[i] = 60 + random () % 32;
> > > > @@ -199,10 +236,15 @@ test_main (void)
> > > >        {
> > > >         size_t len = strlen (strs[i]);
> > > >
> > > > -       do_test (strs[i], len, j);
> > > > +       do_test (&json_ctx, strs[i], len, j);
> > > >        }
> > > >
> > > > -  do_random_tests ();
> > > > +  do_random_tests (&json_ctx);
> > > > +
> > > > +  json_array_end (&json_ctx);
> > > > +  json_attr_object_end (&json_ctx);
> > > > +  json_attr_object_end (&json_ctx);
> > > > +  json_document_end (&json_ctx);
> > > >    return ret;
> > > >  }
> > > >
> > > > diff --git a/benchtests/bench-strstr.c b/benchtests/bench-strstr.c
> > > > index 467418b12c..3838a2b6ac 100644
> > > > --- a/benchtests/bench-strstr.c
> > > > +++ b/benchtests/bench-strstr.c
> > > > @@ -21,6 +21,8 @@
> > > >  #define TEST_NAME "strstr"
> > > >  #include "bench-string.h"
> > > >
> > > > +#include "json-lib.h"
> > > > +
> > > >  static const char input[] =
> > > >  "This manual is written with the assumption that you are at least "
> > > >  "somewhat familiar with the C programming language and basic programming "
> > > > @@ -129,7 +131,8 @@ IMPL (twoway_strstr, 0)
> > > >  IMPL (basic_strstr, 0)
> > > >
> > > >  static void
> > > > -do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> > > > +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1,
> > > > +            const char *s2, char *exp_result)
> > > >  {
> > > >    size_t i, iters = INNER_LOOP_ITERS_SMALL / 8;
> > > >    timing_t start, stop, cur;
> > > > @@ -142,7 +145,7 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> > > >
> > > >    TIMING_DIFF (cur, start, stop);
> > > >
> > > > -  TIMING_PRINT_MEAN ((double) cur, (double) iters);
> > > > +  json_element_double (json_ctx, (double) cur / (double) iters);
> > > >
> > > >    if (res != exp_result)
> > > >      {
> > > > @@ -153,14 +156,16 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
> > > >      }
> > > >  }
> > > >
> > > > -
> > > >  static void
> > > > -do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> > > > -        int fail)
> > > > +do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1,
> > > > +        size_t len2, int fail)
> > > >  {
> > > >    char *s1 = (char *) (buf1 + align1);
> > > >    char *s2 = (char *) (buf2 + align2);
> > > >
> > > > +  if (align1 + len1 >= MIN_PAGE_SIZE || align1 + len1 >= MIN_PAGE_SIZE)
> > >
> > > Is this really needed?  Did you mean align2 + len2?
> > Will fix align2/len2 in V2.
> >
> > Not needed AFAICT for any of these but generally think it makes it easier
> > to add new tests w.o worrying about them failing.
>
> It means that the new tests are ignored.
>
> > Can remove if you in V2 if you want.
>
> Please remove it.
>
> Thanks.
>

Fixed in V2.
> > >
> > > > +    return;
> > > > +
> > > >    size_t size = sizeof (input) - 1;
> > > >    size_t pos = (len1 + len2) % size;
> > > >
> > > > @@ -195,13 +200,21 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> > > >      if (fail || ss1 != s1 + len1 - len2)
> > > >        ++ss1[len2 / 2];
> > > >
> > > > -  printf ("Length %4zd/%3zd, alignment %2zd/%2zd, %s:",
> > > > -         len1, len2, align1, align2, fail ? "fail " : "found");
> > > > +  json_element_object_begin (json_ctx);
> > > > +  json_attr_uint (json_ctx, "len_haystack", len1);
> > > > +  json_attr_uint (json_ctx, "len_needle", len2);
> > > > +  json_attr_uint (json_ctx, "align_haystack", align1);
> > > > +  json_attr_uint (json_ctx, "align_needle", align2);
> > > > +  json_attr_uint (json_ctx, "fail", fail);
> > > > +
> > > > +  json_array_begin (json_ctx, "timings");
> > > >
> > > >    FOR_EACH_IMPL (impl, 0)
> > > > -    do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> > > > +    do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2);
> > > > +
> > > > +  json_array_end (json_ctx);
> > > > +  json_element_object_end (json_ctx);
> > > >
> > > > -  putchar ('\n');
> > > >  }
> > > >
> > > >  /* Test needles which exhibit worst-case performance.  This shows that
> > > > @@ -211,11 +224,14 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2,
> > > >     within a factor of 2 on several different microarchitectures.  */
> > > >
> > > >  static void
> > > > -test_hard_needle (size_t ne_len, size_t hs_len)
> > > > +test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len)
> > > >  {
> > > >    char *ne = (char *) buf1;
> > > >    char *hs = (char *) buf2;
> > > >
> > > > +  if (ne_len >= MIN_PAGE_SIZE || hs_len >= MIN_PAGE_SIZE)
> > > > +    return;
> > > > +
> > > >    /* Hard needle for strstr algorithm using skip table.  This results in many
> > > >       memcmp calls comparing most of the needle.  */
> > > >    {
> > > > @@ -226,15 +242,25 @@ test_hard_needle (size_t ne_len, size_t hs_len)
> > > >      memset (hs, 'a', hs_len);
> > > >      for (size_t i = ne_len; i <= hs_len; i += ne_len)
> > > >        {
> > > > -       hs[i-5] = 'b';
> > > > -       hs[i-62] = 'b';
> > > > +       hs[i - 5] = 'b';
> > > > +       hs[i - 62] = 'b';
> > > >        }
> > > >
> > > > -    printf ("Length %4zd/%3zd, complex needle 1:", hs_len, ne_len);
> > > > +    json_element_object_begin (json_ctx);
> > > > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > > > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > > > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > > > +    json_attr_uint (json_ctx, "align_needle", 0);
> > > > +    json_attr_uint (json_ctx, "fail", 1);
> > > > +    json_attr_string (json_ctx, "desc", "Difficult skiptable(0)");
> > > > +
> > > > +    json_array_begin (json_ctx, "timings");
> > > >
> > > >      FOR_EACH_IMPL (impl, 0)
> > > > -      do_one_test (impl, hs, ne, NULL);
> > > > -    putchar ('\n');
> > > > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > > > +
> > > > +    json_array_end (json_ctx);
> > > > +    json_element_object_end (json_ctx);
> > > >    }
> > > >
> > > >    /* 2nd hard needle for strstr algorithm using skip table.  This results in
> > > > @@ -247,15 +273,25 @@ test_hard_needle (size_t ne_len, size_t hs_len)
> > > >      memset (hs, 'a', hs_len);
> > > >      for (size_t i = ne_len; i <= hs_len; i += ne_len)
> > > >        {
> > > > -       hs[i-5] = 'b';
> > > > -       hs[i-6] = 'b';
> > > > +       hs[i - 5] = 'b';
> > > > +       hs[i - 6] = 'b';
> > > >        }
> > > >
> > > > -    printf ("Length %4zd/%3zd, complex needle 2:", hs_len, ne_len);
> > > > +    json_element_object_begin (json_ctx);
> > > > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > > > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > > > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > > > +    json_attr_uint (json_ctx, "align_needle", 0);
> > > > +    json_attr_uint (json_ctx, "fail", 1);
> > > > +    json_attr_string (json_ctx, "desc", "Difficult skiptable(1)");
> > > > +
> > > > +    json_array_begin (json_ctx, "timings");
> > > >
> > > >      FOR_EACH_IMPL (impl, 0)
> > > > -      do_one_test (impl, hs, ne, NULL);
> > > > -    putchar ('\n');
> > > > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > > > +
> > > > +    json_array_end (json_ctx);
> > > > +    json_element_object_end (json_ctx);
> > > >    }
> > > >
> > > >    /* Hard needle for Two-way algorithm - the random input causes a large number
> > > > @@ -267,45 +303,128 @@ test_hard_needle (size_t ne_len, size_t hs_len)
> > > >      hs[hs_len] = 0;
> > > >
> > > >      memset (ne, 'a', ne_len);
> > > > -    ne[ne_len-2] = 'b';
> > > > +    ne[ne_len - 2] = 'b';
> > > >      ne[0] = 'b';
> > > >      ne[ne_len] = 0;
> > > >
> > > > -    printf ("Length %4zd/%3zd, complex needle 3:", hs_len, ne_len);
> > > > +    json_element_object_begin (json_ctx);
> > > > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > > > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > > > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > > > +    json_attr_uint (json_ctx, "align_needle", 0);
> > > > +    json_attr_uint (json_ctx, "fail", 1);
> > > > +    json_attr_string (json_ctx, "desc", "Difficult 2-way");
> > > > +
> > > > +    json_array_begin (json_ctx, "timings");
> > > > +
> > > > +    FOR_EACH_IMPL (impl, 0)
> > > > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > > > +
> > > > +    json_array_end (json_ctx);
> > > > +    json_element_object_end (json_ctx);
> > > > +  }
> > > > +
> > > > +  /* Hard needle for standard algorithm testing first few characters of
> > > > +   * needle.  */
> > > > +  {
> > > > +    for (int i = 0; i < hs_len; i++)
> > > > +      hs[i] = (rand () & 255) >= 128 ? 'a' : 'b';
> > > > +    hs[hs_len] = 0;
> > > > +
> > > > +    for (int i = 0; i < ne_len; i++)
> > > > +      {
> > > > +       if (i % 3 == 0)
> > > > +         ne[i] = 'a';
> > > > +       else if (i % 3 == 1)
> > > > +         ne[i] = 'b';
> > > > +       else
> > > > +         ne[i] = 'c';
> > > > +      }
> > > > +    ne[ne_len] = 0;
> > > > +
> > > > +    json_element_object_begin (json_ctx);
> > > > +    json_attr_uint (json_ctx, "len_haystack", hs_len);
> > > > +    json_attr_uint (json_ctx, "len_needle", ne_len);
> > > > +    json_attr_uint (json_ctx, "align_haystack", 0);
> > > > +    json_attr_uint (json_ctx, "align_needle", 0);
> > > > +    json_attr_uint (json_ctx, "fail", 1);
> > > > +    json_attr_string (json_ctx, "desc", "Difficult testing first 2");
> > > > +
> > > > +    json_array_begin (json_ctx, "timings");
> > > >
> > > >      FOR_EACH_IMPL (impl, 0)
> > > > -      do_one_test (impl, hs, ne, NULL);
> > > > -    putchar ('\n');
> > > > +      do_one_test (json_ctx, impl, hs, ne, NULL);
> > > > +
> > > > +    json_array_end (json_ctx);
> > > > +    json_element_object_end (json_ctx);
> > > >    }
> > > >  }
> > > >
> > > >  static int
> > > >  test_main (void)
> > > >  {
> > > > +  json_ctx_t json_ctx;
> > > >    test_init ();
> > > >
> > > > -  printf ("%23s", "");
> > > > +  json_init (&json_ctx, 0, stdout);
> > > > +
> > > > +  json_document_begin (&json_ctx);
> > > > +  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
> > > > +
> > > > +  json_attr_object_begin (&json_ctx, "functions");
> > > > +  json_attr_object_begin (&json_ctx, TEST_NAME);
> > > > +  json_attr_string (&json_ctx, "bench-variant", "");
> > > > +
> > > > +  json_array_begin (&json_ctx, "ifuncs");
> > > >    FOR_EACH_IMPL (impl, 0)
> > > > -    printf ("\t%s", impl->name);
> > > > -  putchar ('\n');
> > > > +    json_element_string (&json_ctx, impl->name);
> > > > +  json_array_end (&json_ctx);
> > > > +
> > > > +  json_array_begin (&json_ctx, "results");
> > > >
> > > > -  for (size_t hlen = 64; hlen <= 256; hlen += 32)
> > > > +  for (size_t hlen = 8; hlen <= 256;)
> > > >      for (size_t klen = 1; klen <= 16; klen++)
> > > >        {
> > > > -       do_test (1, 3, hlen, klen, 0);
> > > > -       do_test (0, 9, hlen, klen, 1);
> > > > +       do_test (&json_ctx, 1, 3, hlen, klen, 0);
> > > > +       do_test (&json_ctx, 0, 9, hlen, klen, 1);
> > > > +
> > > > +       do_test (&json_ctx, 1, 3, hlen + 1, klen, 0);
> > > > +       do_test (&json_ctx, 0, 9, hlen + 1, klen, 1);
> > > > +
> > > > +       do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1);
> > > > +       if (hlen < 64)
> > > > +         {
> > > > +           hlen += 8;
> > > > +         }
> > > > +       else
> > > > +         {
> > > > +           hlen += 32;
> > > > +         }
> > > >        }
> > > >
> > > >    for (size_t hlen = 256; hlen <= 65536; hlen *= 2)
> > > > -    for (size_t klen = 16; klen <= 256; klen *= 2)
> > > > +    for (size_t klen = 4; klen <= 256; klen *= 2)
> > > >        {
> > > > -       do_test (1, 11, hlen, klen, 0);
> > > > -       do_test (14, 5, hlen, klen, 1);
> > > > +       do_test (&json_ctx, 1, 11, hlen, klen, 0);
> > > > +       do_test (&json_ctx, 14, 5, hlen, klen, 1);
> > > > +
> > > > +    do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0);
> > > > +    do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1);
> > > > +
> > > > +       do_test (&json_ctx, 1, 11, hlen + 1, klen, 0);
> > > > +       do_test (&json_ctx, 14, 5, hlen + 1, klen, 1);
> > > > +
> > > > +       do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1);
> > > >        }
> > > >
> > > > -  test_hard_needle (64, 65536);
> > > > -  test_hard_needle (256, 65536);
> > > > -  test_hard_needle (1024, 65536);
> > > > +  test_hard_needle (&json_ctx, 64, 65536);
> > > > +  test_hard_needle (&json_ctx, 256, 65536);
> > > > +  test_hard_needle (&json_ctx, 1024, 65536);
> > > > +
> > > > +  json_array_end (&json_ctx);
> > > > +  json_attr_object_end (&json_ctx);
> > > > +  json_attr_object_end (&json_ctx);
> > > > +  json_document_end (&json_ctx);
> > > >
> > > >    return ret;
> > > >  }
> > > > --
> > > > 2.34.1
> > > >
> > >
> > >
> > > --
> > > H.J.
>
>
>
> --
> H.J.
diff mbox series

Patch

diff --git a/benchtests/bench-memchr.c b/benchtests/bench-memchr.c
index fb0284f523..4d7212332f 100644
--- a/benchtests/bench-memchr.c
+++ b/benchtests/bench-memchr.c
@@ -53,8 +53,11 @@  SIMPLE_MEMCHR (const CHAR *s, int c, size_t n)
 }
 #endif /* !USE_AS_MEMRCHR */
 
+#include "json-lib.h"
+
 static void
-do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
+do_one_test (json_ctx_t *json_ctx, impl_t *impl, const CHAR *s, int c,
+	     size_t n)
 {
   size_t i, iters = INNER_LOOP_ITERS_LARGE;
   timing_t start, stop, cur;
@@ -68,15 +71,16 @@  do_one_test (impl_t *impl, const CHAR *s, int c, size_t n)
 
   TIMING_DIFF (cur, start, stop);
 
-  TIMING_PRINT_MEAN ((double) cur, (double) iters);
+  json_element_double (json_ctx, (double) cur / (double) iters);
 }
 
 static void
-do_test (size_t align, size_t pos, size_t len, int seek_char)
+do_test (json_ctx_t *json_ctx, size_t align, size_t pos, size_t len,
+	 int seek_char)
 {
   size_t i;
 
-  align &= 7;
+  align &= getpagesize () - 1;
   if ((align + len) * sizeof (CHAR) >= page_size)
     return;
 
@@ -100,67 +104,99 @@  do_test (size_t align, size_t pos, size_t len, int seek_char)
       buf[align + len] = seek_char;
     }
 
-  printf ("Length %4zd, position %4zd, alignment %2zd:",
-	  len, pos, align);
+  json_element_object_begin (json_ctx);
+  json_attr_uint (json_ctx, "align", align);
+  json_attr_uint (json_ctx, "pos", pos);
+  json_attr_uint (json_ctx, "len", len);
+  json_attr_uint (json_ctx, "seek_char", seek_char);
+
+  json_array_begin (json_ctx, "timings");
 
   FOR_EACH_IMPL (impl, 0)
-    do_one_test (impl, (CHAR *) (buf + align), seek_char, len);
+    do_one_test (json_ctx, impl, (CHAR *) (buf + align), seek_char, len);
 
-  putchar ('\n');
+  json_array_end (json_ctx);
+  json_element_object_end (json_ctx);
 }
 
 int
 test_main (void)
 {
   size_t i;
-
+  json_ctx_t json_ctx;
   test_init ();
 
-  printf ("%20s", "");
+  json_init (&json_ctx, 0, stdout);
+
+  json_document_begin (&json_ctx);
+  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
+
+  json_attr_object_begin (&json_ctx, "functions");
+  json_attr_object_begin (&json_ctx, TEST_NAME);
+  json_attr_string (&json_ctx, "bench-variant", "");
+
+  json_array_begin (&json_ctx, "ifuncs");
   FOR_EACH_IMPL (impl, 0)
-    printf ("\t%s", impl->name);
-  putchar ('\n');
+    json_element_string (&json_ctx, impl->name);
+  json_array_end (&json_ctx);
+
+  json_array_begin (&json_ctx, "results");
 
   for (i = 1; i < 8; ++i)
     {
-      do_test (0, 16 << i, 2048, 23);
-      do_test (i, 64, 256, 23);
-      do_test (0, 16 << i, 2048, 0);
-      do_test (i, 64, 256, 0);
+      do_test (&json_ctx, 0, 16 << i, 2048, 23);
+      do_test (&json_ctx, i, 64, 256, 23);
+      do_test (&json_ctx, 0, 16 << i, 2048, 0);
+      do_test (&json_ctx, i, 64, 256, 0);
+
+      do_test (&json_ctx, getpagesize () - 15, 64, 256, 0);
 #ifdef USE_AS_MEMRCHR
       /* Also test the position close to the beginning for memrchr.  */
-      do_test (0, i, 256, 23);
-      do_test (0, i, 256, 0);
-      do_test (i, i, 256, 23);
-      do_test (i, i, 256, 0);
+      do_test (&json_ctx, 0, i, 256, 23);
+      do_test (&json_ctx, 0, i, 256, 0);
+      do_test (&json_ctx, i, i, 256, 23);
+      do_test (&json_ctx, i, i, 256, 0);
 #endif
     }
   for (i = 1; i < 8; ++i)
     {
-      do_test (i, i << 5, 192, 23);
-      do_test (i, i << 5, 192, 0);
-      do_test (i, i << 5, 256, 23);
-      do_test (i, i << 5, 256, 0);
-      do_test (i, i << 5, 512, 23);
-      do_test (i, i << 5, 512, 0);
+      do_test (&json_ctx, i, i << 5, 192, 23);
+      do_test (&json_ctx, i, i << 5, 192, 0);
+      do_test (&json_ctx, i, i << 5, 256, 23);
+      do_test (&json_ctx, i, i << 5, 256, 0);
+      do_test (&json_ctx, i, i << 5, 512, 23);
+      do_test (&json_ctx, i, i << 5, 512, 0);
+
+      do_test (&json_ctx, getpagesize () - 15, i << 5, 256, 23);
     }
   for (i = 1; i < 32; ++i)
     {
-      do_test (0, i, i + 1, 23);
-      do_test (0, i, i + 1, 0);
-      do_test (i, i, i + 1, 23);
-      do_test (i, i, i + 1, 0);
-      do_test (0, i, i - 1, 23);
-      do_test (0, i, i - 1, 0);
-      do_test (i, i, i - 1, 23);
-      do_test (i, i, i - 1, 0);
+      do_test (&json_ctx, 0, i, i + 1, 23);
+      do_test (&json_ctx, 0, i, i + 1, 0);
+      do_test (&json_ctx, i, i, i + 1, 23);
+      do_test (&json_ctx, i, i, i + 1, 0);
+      do_test (&json_ctx, 0, i, i - 1, 23);
+      do_test (&json_ctx, 0, i, i - 1, 0);
+      do_test (&json_ctx, i, i, i - 1, 23);
+      do_test (&json_ctx, i, i, i - 1, 0);
+
+      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 23);
+      do_test (&json_ctx, getpagesize () - 15, i, i - 1, 0);
+
+      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 23);
+      do_test (&json_ctx, getpagesize () - 15, i, i + 1, 0);
 #ifdef USE_AS_MEMRCHR
       /* Also test the position close to the beginning for memrchr.  */
-      do_test (0, 1, i + 1, 23);
-      do_test (0, 2, i + 1, 0);
+      do_test (&json_ctx, 0, 1, i + 1, 23);
+      do_test (&json_ctx, 0, 2, i + 1, 0);
 #endif
     }
 
+  json_array_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_document_end (&json_ctx);
+
   return ret;
 }
 
diff --git a/benchtests/bench-memmem.c b/benchtests/bench-memmem.c
index 0fb87309c0..c2b92405dc 100644
--- a/benchtests/bench-memmem.c
+++ b/benchtests/bench-memmem.c
@@ -21,6 +21,7 @@ 
 #define BUF1PAGES 20
 #define ITERATIONS 100
 #include "bench-string.h"
+#include "json-lib.h"
 
 typedef char *(*proto_t) (const void *, size_t, const void *, size_t);
 
@@ -102,44 +103,62 @@  IMPL (twoway_memmem, 0)
 IMPL (basic_memmem, 0)
 
 static void
-do_one_test (impl_t *impl, const void *haystack, size_t haystack_len,
-	     const void *needle, size_t needle_len, const void *expected)
+do_one_test (json_ctx_t *json_ctx, impl_t *impl, const void *haystack,
+	     size_t haystack_len, const void *needle, size_t needle_len,
+	     const void *expected)
 {
   size_t i, iters = INNER_LOOP_ITERS_SMALL;
   timing_t start, stop, cur;
-
+  void *res;
   TIMING_NOW (start);
   for (i = 0; i < iters; ++i)
     {
-      CALL (impl, haystack, haystack_len, needle, needle_len);
+      res = CALL (impl, haystack, haystack_len, needle, needle_len);
     }
   TIMING_NOW (stop);
 
   TIMING_DIFF (cur, start, stop);
 
-  TIMING_PRINT_MEAN ((double) cur, (double) iters);
+  json_element_double (json_ctx, (double) cur / (double) iters);
+
+  if (res != expected)
+    {
+      error (0, 0, "Wrong result in function (%p != %p) %s(%p, %zu, %p, %zu)",
+	     res, expected, impl->name, haystack, haystack_len, needle,
+	     needle_len);
+      ret = 1;
+    }
 }
 
 static void
-do_test (const char *str, size_t len, size_t idx)
+do_test (json_ctx_t *json_ctx, const char *str, size_t len, size_t idx)
 {
   char tmpbuf[len];
 
   memcpy (tmpbuf, buf1 + idx, len);
   memcpy (buf1 + idx, str, len);
 
-  printf ("String %s, offset %zd:", str, idx);
+  json_element_object_begin (json_ctx);
+  json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
+  json_attr_uint (json_ctx, "len_needle", len);
+  json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
+  json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) str);
+  json_attr_uint (json_ctx, "fail", 0);
+
+  json_array_begin (json_ctx, "timings");
 
   FOR_EACH_IMPL (impl, 0)
-    do_one_test (impl, buf1, BUF1PAGES * page_size, str, len, buf1 + idx);
+    do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, str, len,
+		 buf1 + idx);
 
   memcpy (buf1 + idx, tmpbuf, len);
 
-  putchar ('\n');
+  json_array_end (json_ctx);
+  json_element_object_end (json_ctx);
 }
 
 static void
-do_random_tests (void)
+do_random_tests (json_ctx_t *json_ctx)
 {
   for (size_t n = 0; n < ITERATIONS; ++n)
     {
@@ -159,13 +178,21 @@  do_random_tests (void)
 	  buf1[idx + off] = ch;
 	}
 
-      printf ("String %.*s, offset %zd:", (int) len, buf1 + idx, idx);
+      json_element_object_begin (json_ctx);
+      json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size);
+      json_attr_uint (json_ctx, "len_needle", len);
+      json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1);
+      json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) (buf1 + idx));
+      json_attr_uint (json_ctx, "fail", 0);
+
+      json_array_begin (json_ctx, "timings");
 
       FOR_EACH_IMPL (impl, 0)
-	do_one_test (impl, buf1, BUF1PAGES * page_size, buf1 + idx, len,
-		     buf1 + idx);
+	do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, buf1 + idx,
+		     len, buf1 + idx);
 
-      putchar ('\n');
+      json_array_end (json_ctx);
+      json_element_object_end (json_ctx);
 
       memcpy (buf1 + idx, tmpbuf, len);
     }
@@ -178,18 +205,28 @@  static const char *const strs[] =
     "abc0", "aaaa0", "abcabc0"
   };
 
-
 int
 test_main (void)
 {
+  json_ctx_t json_ctx;
   size_t i;
 
   test_init ();
+  json_init (&json_ctx, 0, stdout);
+
+  json_document_begin (&json_ctx);
+  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
+
+  json_attr_object_begin (&json_ctx, "functions");
+  json_attr_object_begin (&json_ctx, TEST_NAME);
+  json_attr_string (&json_ctx, "bench-variant", "");
 
-  printf ("%23s", "");
+  json_array_begin (&json_ctx, "ifuncs");
   FOR_EACH_IMPL (impl, 0)
-    printf ("\t%s", impl->name);
-  putchar ('\n');
+    json_element_string (&json_ctx, impl->name);
+  json_array_end (&json_ctx);
+
+  json_array_begin (&json_ctx, "results");
 
   for (i = 0; i < BUF1PAGES * page_size; ++i)
     buf1[i] = 60 + random () % 32;
@@ -199,10 +236,15 @@  test_main (void)
       {
 	size_t len = strlen (strs[i]);
 
-	do_test (strs[i], len, j);
+	do_test (&json_ctx, strs[i], len, j);
       }
 
-  do_random_tests ();
+  do_random_tests (&json_ctx);
+
+  json_array_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_document_end (&json_ctx);
   return ret;
 }
 
diff --git a/benchtests/bench-strstr.c b/benchtests/bench-strstr.c
index 467418b12c..3838a2b6ac 100644
--- a/benchtests/bench-strstr.c
+++ b/benchtests/bench-strstr.c
@@ -21,6 +21,8 @@ 
 #define TEST_NAME "strstr"
 #include "bench-string.h"
 
+#include "json-lib.h"
+
 static const char input[] =
 "This manual is written with the assumption that you are at least "
 "somewhat familiar with the C programming language and basic programming "
@@ -129,7 +131,8 @@  IMPL (twoway_strstr, 0)
 IMPL (basic_strstr, 0)
 
 static void
-do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
+do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1,
+	     const char *s2, char *exp_result)
 {
   size_t i, iters = INNER_LOOP_ITERS_SMALL / 8;
   timing_t start, stop, cur;
@@ -142,7 +145,7 @@  do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
 
   TIMING_DIFF (cur, start, stop);
 
-  TIMING_PRINT_MEAN ((double) cur, (double) iters);
+  json_element_double (json_ctx, (double) cur / (double) iters);
 
   if (res != exp_result)
     {
@@ -153,14 +156,16 @@  do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result)
     }
 }
 
-
 static void
-do_test (size_t align1, size_t align2, size_t len1, size_t len2,
-	 int fail)
+do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1,
+	 size_t len2, int fail)
 {
   char *s1 = (char *) (buf1 + align1);
   char *s2 = (char *) (buf2 + align2);
 
+  if (align1 + len1 >= MIN_PAGE_SIZE || align1 + len1 >= MIN_PAGE_SIZE)
+    return;
+
   size_t size = sizeof (input) - 1;
   size_t pos = (len1 + len2) % size;
 
@@ -195,13 +200,21 @@  do_test (size_t align1, size_t align2, size_t len1, size_t len2,
     if (fail || ss1 != s1 + len1 - len2)
       ++ss1[len2 / 2];
 
-  printf ("Length %4zd/%3zd, alignment %2zd/%2zd, %s:",
-	  len1, len2, align1, align2, fail ? "fail " : "found");
+  json_element_object_begin (json_ctx);
+  json_attr_uint (json_ctx, "len_haystack", len1);
+  json_attr_uint (json_ctx, "len_needle", len2);
+  json_attr_uint (json_ctx, "align_haystack", align1);
+  json_attr_uint (json_ctx, "align_needle", align2);
+  json_attr_uint (json_ctx, "fail", fail);
+
+  json_array_begin (json_ctx, "timings");
 
   FOR_EACH_IMPL (impl, 0)
-    do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2);
+    do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2);
+
+  json_array_end (json_ctx);
+  json_element_object_end (json_ctx);
 
-  putchar ('\n');
 }
 
 /* Test needles which exhibit worst-case performance.  This shows that
@@ -211,11 +224,14 @@  do_test (size_t align1, size_t align2, size_t len1, size_t len2,
    within a factor of 2 on several different microarchitectures.  */
 
 static void
-test_hard_needle (size_t ne_len, size_t hs_len)
+test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len)
 {
   char *ne = (char *) buf1;
   char *hs = (char *) buf2;
 
+  if (ne_len >= MIN_PAGE_SIZE || hs_len >= MIN_PAGE_SIZE)
+    return;
+
   /* Hard needle for strstr algorithm using skip table.  This results in many
      memcmp calls comparing most of the needle.  */
   {
@@ -226,15 +242,25 @@  test_hard_needle (size_t ne_len, size_t hs_len)
     memset (hs, 'a', hs_len);
     for (size_t i = ne_len; i <= hs_len; i += ne_len)
       {
-	hs[i-5] = 'b';
-	hs[i-62] = 'b';
+	hs[i - 5] = 'b';
+	hs[i - 62] = 'b';
       }
 
-    printf ("Length %4zd/%3zd, complex needle 1:", hs_len, ne_len);
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult skiptable(0)");
+
+    json_array_begin (json_ctx, "timings");
 
     FOR_EACH_IMPL (impl, 0)
-      do_one_test (impl, hs, ne, NULL);
-    putchar ('\n');
+      do_one_test (json_ctx, impl, hs, ne, NULL);
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
   }
 
   /* 2nd hard needle for strstr algorithm using skip table.  This results in
@@ -247,15 +273,25 @@  test_hard_needle (size_t ne_len, size_t hs_len)
     memset (hs, 'a', hs_len);
     for (size_t i = ne_len; i <= hs_len; i += ne_len)
       {
-	hs[i-5] = 'b';
-	hs[i-6] = 'b';
+	hs[i - 5] = 'b';
+	hs[i - 6] = 'b';
       }
 
-    printf ("Length %4zd/%3zd, complex needle 2:", hs_len, ne_len);
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult skiptable(1)");
+
+    json_array_begin (json_ctx, "timings");
 
     FOR_EACH_IMPL (impl, 0)
-      do_one_test (impl, hs, ne, NULL);
-    putchar ('\n');
+      do_one_test (json_ctx, impl, hs, ne, NULL);
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
   }
 
   /* Hard needle for Two-way algorithm - the random input causes a large number
@@ -267,45 +303,128 @@  test_hard_needle (size_t ne_len, size_t hs_len)
     hs[hs_len] = 0;
 
     memset (ne, 'a', ne_len);
-    ne[ne_len-2] = 'b';
+    ne[ne_len - 2] = 'b';
     ne[0] = 'b';
     ne[ne_len] = 0;
 
-    printf ("Length %4zd/%3zd, complex needle 3:", hs_len, ne_len);
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult 2-way");
+
+    json_array_begin (json_ctx, "timings");
+
+    FOR_EACH_IMPL (impl, 0)
+      do_one_test (json_ctx, impl, hs, ne, NULL);
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
+  }
+
+  /* Hard needle for standard algorithm testing first few characters of
+   * needle.  */
+  {
+    for (int i = 0; i < hs_len; i++)
+      hs[i] = (rand () & 255) >= 128 ? 'a' : 'b';
+    hs[hs_len] = 0;
+
+    for (int i = 0; i < ne_len; i++)
+      {
+	if (i % 3 == 0)
+	  ne[i] = 'a';
+	else if (i % 3 == 1)
+	  ne[i] = 'b';
+	else
+	  ne[i] = 'c';
+      }
+    ne[ne_len] = 0;
+
+    json_element_object_begin (json_ctx);
+    json_attr_uint (json_ctx, "len_haystack", hs_len);
+    json_attr_uint (json_ctx, "len_needle", ne_len);
+    json_attr_uint (json_ctx, "align_haystack", 0);
+    json_attr_uint (json_ctx, "align_needle", 0);
+    json_attr_uint (json_ctx, "fail", 1);
+    json_attr_string (json_ctx, "desc", "Difficult testing first 2");
+
+    json_array_begin (json_ctx, "timings");
 
     FOR_EACH_IMPL (impl, 0)
-      do_one_test (impl, hs, ne, NULL);
-    putchar ('\n');
+      do_one_test (json_ctx, impl, hs, ne, NULL);
+
+    json_array_end (json_ctx);
+    json_element_object_end (json_ctx);
   }
 }
 
 static int
 test_main (void)
 {
+  json_ctx_t json_ctx;
   test_init ();
 
-  printf ("%23s", "");
+  json_init (&json_ctx, 0, stdout);
+
+  json_document_begin (&json_ctx);
+  json_attr_string (&json_ctx, "timing_type", TIMING_TYPE);
+
+  json_attr_object_begin (&json_ctx, "functions");
+  json_attr_object_begin (&json_ctx, TEST_NAME);
+  json_attr_string (&json_ctx, "bench-variant", "");
+
+  json_array_begin (&json_ctx, "ifuncs");
   FOR_EACH_IMPL (impl, 0)
-    printf ("\t%s", impl->name);
-  putchar ('\n');
+    json_element_string (&json_ctx, impl->name);
+  json_array_end (&json_ctx);
+
+  json_array_begin (&json_ctx, "results");
 
-  for (size_t hlen = 64; hlen <= 256; hlen += 32)
+  for (size_t hlen = 8; hlen <= 256;)
     for (size_t klen = 1; klen <= 16; klen++)
       {
-	do_test (1, 3, hlen, klen, 0);
-	do_test (0, 9, hlen, klen, 1);
+	do_test (&json_ctx, 1, 3, hlen, klen, 0);
+	do_test (&json_ctx, 0, 9, hlen, klen, 1);
+
+	do_test (&json_ctx, 1, 3, hlen + 1, klen, 0);
+	do_test (&json_ctx, 0, 9, hlen + 1, klen, 1);
+
+	do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1);
+	if (hlen < 64)
+	  {
+	    hlen += 8;
+	  }
+	else
+	  {
+	    hlen += 32;
+	  }
       }
 
   for (size_t hlen = 256; hlen <= 65536; hlen *= 2)
-    for (size_t klen = 16; klen <= 256; klen *= 2)
+    for (size_t klen = 4; klen <= 256; klen *= 2)
       {
-	do_test (1, 11, hlen, klen, 0);
-	do_test (14, 5, hlen, klen, 1);
+	do_test (&json_ctx, 1, 11, hlen, klen, 0);
+	do_test (&json_ctx, 14, 5, hlen, klen, 1);
+
+    do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0);
+    do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1);
+
+	do_test (&json_ctx, 1, 11, hlen + 1, klen, 0);
+	do_test (&json_ctx, 14, 5, hlen + 1, klen, 1);
+
+	do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1);
       }
 
-  test_hard_needle (64, 65536);
-  test_hard_needle (256, 65536);
-  test_hard_needle (1024, 65536);
+  test_hard_needle (&json_ctx, 64, 65536);
+  test_hard_needle (&json_ctx, 256, 65536);
+  test_hard_needle (&json_ctx, 1024, 65536);
+
+  json_array_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_attr_object_end (&json_ctx);
+  json_document_end (&json_ctx);
 
   return ret;
 }