From patchwork Mon Jun 15 14:40:27 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Earnshaw X-Patchwork-Id: 39613 Return-Path: X-Original-To: patchwork@sourceware.org Delivered-To: patchwork@sourceware.org Received: from server2.sourceware.org (localhost [IPv6:::1]) by sourceware.org (Postfix) with ESMTP id 22163383E814; Mon, 15 Jun 2020 14:41:32 +0000 (GMT) X-Original-To: libc-alpha@sourceware.org Delivered-To: libc-alpha@sourceware.org Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by sourceware.org (Postfix) with ESMTP id D7FB6383F869 for ; Mon, 15 Jun 2020 14:41:28 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.3.2 sourceware.org D7FB6383F869 Authentication-Results: sourceware.org; dmarc=none (p=none dis=none) header.from=arm.com Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=rearnsha@arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 71259D6E; Mon, 15 Jun 2020 07:41:28 -0700 (PDT) Received: from eagle.buzzard.freeserve.co.uk (unknown [172.31.20.19]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id F1FB13F6CF; Mon, 15 Jun 2020 07:41:27 -0700 (PDT) From: Richard Earnshaw To: libc-alpha@sourceware.org Subject: [PATCH 5/7] aarch64: Mitigations for string functions when MTE is enabled. Date: Mon, 15 Jun 2020 15:40:27 +0100 Message-Id: <20200615144029.19771-6-rearnsha@arm.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200615144029.19771-1-rearnsha@arm.com> References: <20200615144029.19771-1-rearnsha@arm.com> MIME-Version: 1.0 X-Spam-Status: No, score=-15.6 required=5.0 tests=BAYES_00, GIT_PATCH_0, KAM_DMARC_STATUS, SPF_HELO_NONE, SPF_PASS, TXREP autolearn=ham autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on server2.sourceware.org X-BeenThere: libc-alpha@sourceware.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Libc-alpha mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Richard Earnshaw Errors-To: libc-alpha-bounces@sourceware.org Sender: "Libc-alpha" This is a place-holder patch for the changes needed to the string functions to make them safe when using memory tagging. It is expected that this patch will be replaced before the final series is committed. When memory tagging is enabled, functions must not fetch data beyond a granule boundary. Unfortunately, this affects a number of the optimized string operations for aarch64 which assume that provided a page boundary is not being crossed any amount of data within the page may be accessed. This patch replaces the existing string functions with variants that do not violate the granule size limitations that now exist. This patch has not been tuned for performance. --- sysdeps/aarch64/memchr.S | 21 ++++++++++++++++++++- sysdeps/aarch64/multiarch/strlen_asimd.S | 2 +- sysdeps/aarch64/strchr.S | 15 +++++++++++++++ sysdeps/aarch64/strchrnul.S | 14 +++++++++++++- sysdeps/aarch64/strcmp.S | 12 +++++++++--- sysdeps/aarch64/strcpy.S | 2 +- sysdeps/aarch64/strlen.S | 2 +- sysdeps/aarch64/strncmp.S | 10 ++++++++-- sysdeps/aarch64/strrchr.S | 15 ++++++++++++++- 9 files changed, 82 insertions(+), 11 deletions(-) diff --git a/sysdeps/aarch64/memchr.S b/sysdeps/aarch64/memchr.S index 85c65cbfca..6e01a0a0a9 100644 --- a/sysdeps/aarch64/memchr.S +++ b/sysdeps/aarch64/memchr.S @@ -64,6 +64,25 @@ */ ENTRY (MEMCHR) +#ifdef _LIBC_MTAG + /* Quick-and-dirty implementation for MTE. Needs a rewrite as + granules are only 16 bytes in size. */ + /* Do not dereference srcin if no bytes to compare. */ + cbz cntin, L(zero_length) + and chrin, chrin, #255 +L(next_byte): + ldrb wtmp2, [srcin], #1 + cmp wtmp2, chrin + b.eq L(found) + subs cntin, cntin, #1 + b.ne L(next_byte) +L(zero_length): + mov result, #0 + ret +L(found): + sub result, srcin, #1 + ret +#else /* Do not dereference srcin if no bytes to compare. */ cbz cntin, L(zero_length) /* @@ -152,10 +171,10 @@ L(tail): /* Select result or NULL */ csel result, xzr, result, eq ret - L(zero_length): mov result, #0 ret +#endif /* _LIBC_MTAG */ END (MEMCHR) weak_alias (MEMCHR, memchr) libc_hidden_builtin_def (memchr) diff --git a/sysdeps/aarch64/multiarch/strlen_asimd.S b/sysdeps/aarch64/multiarch/strlen_asimd.S index 236a2c96a6..c2c718e493 100644 --- a/sysdeps/aarch64/multiarch/strlen_asimd.S +++ b/sysdeps/aarch64/multiarch/strlen_asimd.S @@ -51,7 +51,7 @@ #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f -#ifdef TEST_PAGE_CROSS +#if defined _LIBC_MTAG || defined TEST_PAGE_CROSS # define MIN_PAGE_SIZE 16 #else # define MIN_PAGE_SIZE 4096 diff --git a/sysdeps/aarch64/strchr.S b/sysdeps/aarch64/strchr.S index 4a75e73945..32c500609e 100644 --- a/sysdeps/aarch64/strchr.S +++ b/sysdeps/aarch64/strchr.S @@ -63,6 +63,20 @@ ENTRY (strchr) DELOUSE (0) +#ifdef _LIBC_MTAG + /* Quick and dirty implementation for MTE */ + and chrin, chrin, #255 +L(next_byte): + ldrb wtmp2, [srcin], #1 + cbz wtmp2, L(end) + cmp wtmp2, chrin + b.ne L(next_byte) + sub result, srcin, #1 + ret +L(end): + mov result, #0 + ret +#else mov wtmp2, #0x0401 movk wtmp2, #0x4010, lsl #16 dup vrepchr.16b, chrin @@ -134,6 +148,7 @@ L(tail): add result, src, tmp1, lsr #1 csel result, result, xzr, eq ret +#endif END (strchr) libc_hidden_builtin_def (strchr) weak_alias (strchr, index) diff --git a/sysdeps/aarch64/strchrnul.S b/sysdeps/aarch64/strchrnul.S index a65be6cba8..78a9252eb8 100644 --- a/sysdeps/aarch64/strchrnul.S +++ b/sysdeps/aarch64/strchrnul.S @@ -61,6 +61,18 @@ ENTRY (__strchrnul) DELOUSE (0) +#ifdef _LIBC_MTAG + /* Quick and dirty implementation for MTE */ + and chrin, chrin, #255 +L(next_byte): + ldrb wtmp2, [srcin], #1 + cmp wtmp2, #0 + ccmp wtmp2, chrin, #4, ne /* NZCV = 0x0100 */ + b.ne L(next_byte) + + sub result, srcin, #1 + ret +#else /* Magic constant 0x40100401 to allow us to identify which lane matches the termination condition. */ mov wtmp2, #0x0401 @@ -126,6 +138,6 @@ L(tail): /* tmp1 is twice the offset into the fragment. */ add result, src, tmp1, lsr #1 ret - +#endif /* _LIBC_MTAG */ END(__strchrnul) weak_alias (__strchrnul, strchrnul) diff --git a/sysdeps/aarch64/strcmp.S b/sysdeps/aarch64/strcmp.S index d044c29e9b..d01b199ab3 100644 --- a/sysdeps/aarch64/strcmp.S +++ b/sysdeps/aarch64/strcmp.S @@ -46,6 +46,12 @@ #define zeroones x10 #define pos x11 +#if defined _LIBC_MTAG || defined TEST_PAGE_CROSS +# define MIN_PAGE_SIZE 16 +#else +# define MIN_PAGE_SIZE 4096 +#endif + /* Start of performance-critical section -- one 64B cache line. */ ENTRY_ALIGN(strcmp, 6) @@ -161,10 +167,10 @@ L(do_misaligned): b.ne L(do_misaligned) L(loop_misaligned): - /* Test if we are within the last dword of the end of a 4K page. If + /* Test if we are within the last dword of the end of a page. If yes then jump back to the misaligned loop to copy a byte at a time. */ - and tmp1, src2, #0xff8 - eor tmp1, tmp1, #0xff8 + and tmp1, src2, #(MIN_PAGE_SIZE - 8) + eor tmp1, tmp1, #(MIN_PAGE_SIZE - 8) cbz tmp1, L(do_misaligned) ldr data1, [src1], #8 ldr data2, [src2], #8 diff --git a/sysdeps/aarch64/strcpy.S b/sysdeps/aarch64/strcpy.S index 548130e413..82548f3d53 100644 --- a/sysdeps/aarch64/strcpy.S +++ b/sysdeps/aarch64/strcpy.S @@ -87,7 +87,7 @@ misaligned, crosses a page boundary - after that we move to aligned fetches for the remainder of the string. */ -#ifdef STRCPY_TEST_PAGE_CROSS +#if defined _LIBC_MTAG || defined STRCPY_TEST_PAGE_CROSS /* Make everything that isn't Qword aligned look like a page cross. */ #define MIN_PAGE_P2 4 #else diff --git a/sysdeps/aarch64/strlen.S b/sysdeps/aarch64/strlen.S index e01fab7c2a..7455a668bb 100644 --- a/sysdeps/aarch64/strlen.S +++ b/sysdeps/aarch64/strlen.S @@ -57,7 +57,7 @@ #define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_80 0x8080808080808080 -#ifdef TEST_PAGE_CROSS +#if defined _LIBC_MTAG || defined TEST_PAGE_CROSS # define MIN_PAGE_SIZE 16 #else # define MIN_PAGE_SIZE 4096 diff --git a/sysdeps/aarch64/strncmp.S b/sysdeps/aarch64/strncmp.S index c5141fab8a..40c805f609 100644 --- a/sysdeps/aarch64/strncmp.S +++ b/sysdeps/aarch64/strncmp.S @@ -51,6 +51,12 @@ #define endloop x15 #define count mask +#if defined _LIBC_MTAG || defined TEST_PAGE_CROSS +# define MIN_PAGE_SIZE 16 +#else +# define MIN_PAGE_SIZE 4096 +#endif + ENTRY_ALIGN_AND_PAD (strncmp, 6, 7) DELOUSE (0) DELOUSE (1) @@ -233,8 +239,8 @@ L(do_misaligned): subs limit_wd, limit_wd, #1 b.lo L(done_loop) L(loop_misaligned): - and tmp2, src2, #0xff8 - eor tmp2, tmp2, #0xff8 + and tmp2, src2, #(MIN_PAGE_SIZE - 8) + eor tmp2, tmp2, #(MIN_PAGE_SIZE - 8) cbz tmp2, L(page_end_loop) ldr data1, [src1], #8 diff --git a/sysdeps/aarch64/strrchr.S b/sysdeps/aarch64/strrchr.S index 94da08d351..ef00e969d9 100644 --- a/sysdeps/aarch64/strrchr.S +++ b/sysdeps/aarch64/strrchr.S @@ -70,6 +70,19 @@ ENTRY(strrchr) DELOUSE (0) cbz x1, L(null_search) +#ifdef _LIBC_MTAG + /* Quick and dirty version for MTE. */ + and chrin, chrin, #255 + mov src_match, #0 +L(next_byte): + ldrb wtmp2, [srcin] + cmp wtmp2, chrin + csel src_match, src_match, srcin, ne + add srcin, srcin, #1 + cbnz wtmp2, L(next_byte) + mov result, src_match + ret +#else /* Magic constant 0x40100401 to allow us to identify which lane matches the requested byte. Magic constant 0x80200802 used similarly for NUL termination. */ @@ -158,9 +171,9 @@ L(tail): csel result, result, xzr, ne ret +#endif L(null_search): b __strchrnul - END(strrchr) weak_alias (strrchr, rindex) libc_hidden_builtin_def (strrchr)