From patchwork Mon Oct 29 15:55:36 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Wilco Dijkstra X-Patchwork-Id: 29951 Received: (qmail 120713 invoked by alias); 29 Oct 2018 15:55:44 -0000 Mailing-List: contact libc-alpha-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Unsubscribe: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: libc-alpha-owner@sourceware.org Delivered-To: mailing list libc-alpha@sourceware.org Received: (qmail 120353 invoked by uid 89); 29 Oct 2018 15:55:43 -0000 Authentication-Results: sourceware.org; auth=none X-Spam-SWARE-Status: No, score=-26.9 required=5.0 tests=BAYES_00, GIT_PATCH_0, GIT_PATCH_1, GIT_PATCH_2, GIT_PATCH_3, RCVD_IN_DNSWL_NONE, SPF_HELO_PASS, SPF_PASS autolearn=ham version=3.3.2 spammy=average, 1-4, significantly, 2211 X-HELO: EUR02-AM5-obe.outbound.protection.outlook.com DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=armh.onmicrosoft.com; s=selector1-arm-com; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=bXqMGV6g7S1YJiPO2TfPlCHYwZ4Nc4xRuKaWvqnx9yk=; b=Ow0CutRuzPHfQS0fbQaODwJ9nR0D/6xfQ5D0BBYQLU4+Gl89Mw/n6Qm8IgtYuPtgdUfm7kuJ5Lnsf7Q+xBQ8pbNgWbSIrCV68dTKdLgAL/6qaaPKPH8Rmn+u9RP3BzGeX3bwnRfgozn6+1b0JoPSVn5/gJuHg+bJtb5ix/Ixfw8= From: Wilco Dijkstra To: 'GNU C Library' CC: nd Subject: [PATCH] Use Quicksearch in strstr Date: Mon, 29 Oct 2018 15:55:36 +0000 Message-ID: authentication-results: spf=none (sender IP is ) smtp.mailfrom=Wilco.Dijkstra@arm.com; received-spf: None (protection.outlook.com: arm.com does not designate permitted sender hosts) MIME-Version: 1.0 This patch significantly improves performance of strstr by using Sunday's Quick-Search algorithm. Due to its simplicity it has the best average performance of string matching algorithms on almost all inputs. It uses a bad-character shift table to skip past mismatches. The needle length is limited to 254 - combined with a hashed shift table this reduces the shift table memory 16 to 32 times, lowering preprocessing overhead and minimizing cache effects. The needle limit also implies worst-case performance remains linear. Small 1-4 byte needles use special case code which is typically faster. Very long needles continue to use the linear-time Two-Way algorithm. The performance gain using the improved bench-strstr on Cortex-A72 is 2.1x. Due to the low overhead the gain is larger on small haystacks: haystacks of 64 chars are ~3 times faster for needles of 5-16 chars. Tested against GLIBC testsuite, randomized tests and the GNULIB strstr test (https://git.savannah.gnu.org/cgit/gnulib.git/tree/tests/test-strstr.c). ChangeLog: 2018-10-29 Wilco Dijkstra * string/str-two-way.h (two_way_short_needle): Add inline to avoid warning. * string/strstr.c (strstr2): Add new function. (strstr3): Likewise. (strstr3): Likewise. (STRSTR): Completely rewrite strstr to improve performance diff --git a/string/str-two-way.h b/string/str-two-way.h index 523d946c59412e1f1f65b8ec3778553b83191952..31c3f18fb057cdd999c3ac9e9d894a8b62a98a70 100644 --- a/string/str-two-way.h +++ b/string/str-two-way.h @@ -221,7 +221,7 @@ critical_factorization (const unsigned char *needle, size_t needle_len, most 2 * HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching. If AVAILABLE modifies HAYSTACK_LEN (as in strstr), then at most 3 * HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching. */ -static RETURN_TYPE +static inline RETURN_TYPE two_way_short_needle (const unsigned char *haystack, size_t haystack_len, const unsigned char *needle, size_t needle_len) { diff --git a/string/strstr.c b/string/strstr.c index f74d7189ed1319f6225525cc2d32380745de1523..206f20ddfcf10f61ba68725b516ef00356e9028c 100644 --- a/string/strstr.c +++ b/string/strstr.c @@ -22,11 +22,8 @@ # include #endif -/* Specification of strstr. */ #include -#include - #ifndef _LIBC # define __builtin_expect(expr, val) (expr) #endif @@ -47,46 +44,115 @@ #define STRSTR strstr #endif -/* Return the first occurrence of NEEDLE in HAYSTACK. Return HAYSTACK - if NEEDLE is empty, otherwise NULL if NEEDLE is not found in - HAYSTACK. */ -char * -STRSTR (const char *haystack, const char *needle) +static inline char * +strstr2 (const unsigned char *hs, const unsigned char *ne) { - size_t needle_len; /* Length of NEEDLE. */ - size_t haystack_len; /* Known minimum length of HAYSTACK. */ + uint32_t h1 = (ne[0] << 16) | ne[1]; + uint32_t h2 = 0; + for (int c = hs[0]; h1 != h2 && c != 0; c = *++hs) + h2 = (h2 << 16) | c; + return h1 == h2 ? (char *)hs - 2 : NULL; +} - /* Handle empty NEEDLE special case. */ - if (needle[0] == '\0') - return (char *) haystack; +static inline char * +strstr3 (const unsigned char *hs, const unsigned char *ne) +{ + uint32_t h1 = (ne[0] << 24) | (ne[1] << 16) | (ne[2] << 8); + uint32_t h2 = 0; + for (int c = hs[0]; h1 != h2 && c != 0; c = *++hs) + h2 = (h2 | c) << 8; + return h1 == h2 ? (char *)hs - 3 : NULL; +} - /* Skip until we find the first matching char from NEEDLE. */ - haystack = strchr (haystack, needle[0]); - if (haystack == NULL || needle[1] == '\0') - return (char *) haystack; +static inline char * +strstr4 (const unsigned char *hs, const unsigned char *ne) +{ + uint32_t h1 = (ne[0] << 24) | (ne[1] << 16) | (ne[2] << 8) | ne[3]; + uint32_t h2 = 0; + for (int c = hs[0]; c != 0 && h1 != h2; c = *++hs) + h2 = (h2 << 8) | c; + return h1 == h2 ? (char *)hs - 4 : NULL; +} - /* Ensure HAYSTACK length is at least as long as NEEDLE length. - Since a match may occur early on in a huge HAYSTACK, use strnlen +/* Number of bits used to index shift table. */ +#define SHIFT_TABLE_BITS 6 + +/* Extremely fast strstr algorithm with guaranteed linear-time performance. + Small needles up to size 4 use a dedicated linear search. Longer needles + up to size 254 use Sunday's Quick-Search algorithm. Due to its simplicity + it has the best average performance of string matching algorithms on almost + all inputs. It uses a bad-character shift table to skip past mismatches. + By limiting the needle length to 254, the shift table can be reduced to 8 + bits per entry, lowering preprocessing overhead and minimizing cache effects. + The limit also implies the worst-case performance is linear. + Even larger needles are processed by the linear-time Two-Way algorithm. +*/ +char * +STRSTR (const char *haystack, const char *needle) +{ + const unsigned char *hs = (const unsigned char *) haystack; + const unsigned char *ne = (const unsigned char *) needle; + + /* Handle short needle special cases first. */ + if (ne[0] == '\0') + return (char *)hs; + hs = (const unsigned char *)strchr ((const char*)hs, ne[0]); + if (hs == NULL || ne[1] == '\0') + return (char*)hs; + if (ne[2] == '\0') + return strstr2 (hs, ne); + if (ne[3] == '\0') + return strstr3 (hs, ne); + if (ne[4] == '\0') + return strstr4 (hs, ne); + + /* Ensure haystack length is at least as long as needle length. + Since a match may occur early on in a huge haystack, use strnlen and read ahead a few cachelines for improved performance. */ - needle_len = strlen (needle); - haystack_len = __strnlen (haystack, needle_len + 256); - if (haystack_len < needle_len) + size_t ne_len = strlen ((const char*)ne); + size_t hs_len = strnlen ((const char*)hs, ne_len | 512); + if (hs_len < ne_len) return NULL; - /* Check whether we have a match. This improves performance since we avoid - the initialization overhead of the two-way algorithm. */ - if (memcmp (haystack, needle, needle_len) == 0) - return (char *) haystack; - - /* Perform the search. Abstract memory is considered to be an array - of 'unsigned char' values, not an array of 'char' values. See - ISO C 99 section 6.2.6.1. */ - if (needle_len < LONG_NEEDLE_THRESHOLD) - return two_way_short_needle ((const unsigned char *) haystack, - haystack_len, - (const unsigned char *) needle, needle_len); - return two_way_long_needle ((const unsigned char *) haystack, haystack_len, - (const unsigned char *) needle, needle_len); + /* Check whether we have a match. This improves performance since we + avoid initialization overheads. */ + if (memcmp (hs, ne, ne_len) == 0) + return (char *) hs; + + /* Use the Quick-Search algorithm for needle lengths less than 255. */ + if (__glibc_likely (ne_len < 255)) + { + uint8_t shift[1 << SHIFT_TABLE_BITS]; + const unsigned char *end = hs + hs_len - ne_len; + + /* Initialize bad character shift hash table. */ + memset (shift, ne_len + 1, sizeof (shift)); + for (int i = 0; i < ne_len; i++) + shift[ne[i] % sizeof (shift)] = ne_len - i; + + do + { + hs--; + + /* Search by skipping past bad characters. */ + size_t tmp = shift[hs[ne_len] % sizeof (shift)]; + for (hs += tmp; hs <= end; hs += tmp) + { + tmp = shift[hs[ne_len] % sizeof (shift)]; + if (memcmp (hs, ne, ne_len) == 0) + return (char*)hs; + } + if (end[ne_len] == '\0') + return NULL; + end += strnlen ((const char*)end + ne_len, 2048); + } + while (hs <= end); + + return NULL; + } + + /* Use Two-Way algorithm for very long needles. */ + return two_way_long_needle (hs, hs_len, ne, ne_len); } libc_hidden_builtin_def (strstr)