From patchwork Fri Jun 5 15:22:26 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrea Corallo X-Patchwork-Id: 39484 Return-Path: X-Original-To: patchwork@sourceware.org Delivered-To: patchwork@sourceware.org Received: from server2.sourceware.org (localhost [IPv6:::1]) by sourceware.org (Postfix) with ESMTP id 64868383E817; Fri, 5 Jun 2020 15:22:43 +0000 (GMT) X-Original-To: libc-alpha@sourceware.org Delivered-To: libc-alpha@sourceware.org Received: from EUR05-AM6-obe.outbound.protection.outlook.com (mail-am6eur05on2071.outbound.protection.outlook.com [40.107.22.71]) by sourceware.org (Postfix) with ESMTPS id 819A8384B0C0 for ; Fri, 5 Jun 2020 15:22:40 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.3.2 sourceware.org 819A8384B0C0 Authentication-Results: sourceware.org; dmarc=none (p=none dis=none) header.from=arm.com Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=Andrea.Corallo@arm.com DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=armh.onmicrosoft.com; s=selector2-armh-onmicrosoft-com; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=ZY5mb/uT27BcDRTqusSyrNmONkkcEkjJ1qpkT+DBGqQ=; b=Xy71s1pVCxecpIntN/Oeef03LVNEsr86pt/HgA20BmEf6yQYQdhNodld3rXMQgdmTR9T3ISPOjMHrPrMMSpXjD/6N57w4DnUusywHE10T/0BL227jqE0r1UpUTLnmg0JJt6RXNhuArtt7WQJ4oV2Jibphcx+QJWErI/J4f3rV90= Received: from AM0PR10CA0001.EURPRD10.PROD.OUTLOOK.COM (2603:10a6:208:17c::11) by AM0PR08MB3042.eurprd08.prod.outlook.com (2603:10a6:208:5e::27) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3066.18; Fri, 5 Jun 2020 15:22:38 +0000 Received: from AM5EUR03FT021.eop-EUR03.prod.protection.outlook.com (2603:10a6:208:17c:cafe::24) by AM0PR10CA0001.outlook.office365.com (2603:10a6:208:17c::11) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3066.18 via Frontend Transport; Fri, 5 Jun 2020 15:22:38 +0000 X-MS-Exchange-Authentication-Results: spf=pass (sender IP is 63.35.35.123) smtp.mailfrom=arm.com; sourceware.org; dkim=pass (signature was verified) header.d=armh.onmicrosoft.com; sourceware.org; dmarc=bestguesspass action=none header.from=arm.com; Received-SPF: Pass (protection.outlook.com: domain of arm.com designates 63.35.35.123 as permitted sender) receiver=protection.outlook.com; client-ip=63.35.35.123; helo=64aa7808-outbound-1.mta.getcheckrecipient.com; Received: from 64aa7808-outbound-1.mta.getcheckrecipient.com (63.35.35.123) by AM5EUR03FT021.mail.protection.outlook.com (10.152.16.105) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3066.18 via Frontend Transport; Fri, 5 Jun 2020 15:22:38 +0000 Received: ("Tessian outbound d3ae83885012:v59"); Fri, 05 Jun 2020 15:22:38 +0000 X-CheckRecipientChecked: true X-CR-MTA-CID: e990470694b6fc65 X-CR-MTA-TID: 64aa7808 Received: from b5c5cba37d86.2 by 64aa7808-outbound-1.mta.getcheckrecipient.com id E9C728B8-EC9F-43E4-94D9-AA32723984BF.1; Fri, 05 Jun 2020 15:22:32 +0000 Received: from EUR03-AM5-obe.outbound.protection.outlook.com by 64aa7808-outbound-1.mta.getcheckrecipient.com with ESMTPS id b5c5cba37d86.2 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384); Fri, 05 Jun 2020 15:22:32 +0000 ARC-Seal: i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none; b=SXQqIO8lp1P8WlCFeFourcxlgf/4eYtXDeon7OPdSimghmnIdWsDQEttSKy8OMUFpLA5x41PcRGpULr8ZZqrjF6dUzVbKNfoJxvqzRPbufsZy0fJrhOGg4r9deeSumnNKe2nrSRJsMDImMOC7jiMO0dWLcsoc/AH4np1h7WWVeEc/LC3k+z3xHXsTY6mAL/lf36RK2/xKeyzp4gKZv048655GWIGbhggjBZ+0AAMF3XEJMDl1W1iuWV5SYS6DMubBSg7xi89sReY6R/HLT0kwWws84rwljueBKE8+raJ24RcvCGCxifVqakW3uI/HYXIp3vrFESzyGPRQ7T3v2sRJQ== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; s=arcselector9901; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=ZY5mb/uT27BcDRTqusSyrNmONkkcEkjJ1qpkT+DBGqQ=; b=jbEVBZAuhqegp6XZiYOcx+mzon/haEYzb3HnhuCApQ9Ocn+QXLJXMpwoUrfjZkmm3s4P3Zp4/WphpYvRSdLY80JrKWwbBt5lg38oJFevCpMoCO0Cmyl8G2AugLpElb/WuAUBtoK9J6hXD+fKKSCNzErxiMpoyWg90WyD/NX4vUqBDGaxOVnaeZD5cIlGKAjoW2O41OQYf9Qnbxms0//n+udscnZ5PNhSOz+Zr0WboPML1sW5nW5WjTfwduAtBE130zg8tr8GqitOsiwvDpqhGIKdmDcKgHOabQ3J4NsjU7YEZqn8nE3xgguo8jWZn0F16xd0pxw2ifKEcFxMftMPWw== ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=pass smtp.mailfrom=arm.com; dmarc=pass action=none header.from=arm.com; dkim=pass header.d=arm.com; arc=none DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=armh.onmicrosoft.com; s=selector2-armh-onmicrosoft-com; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=ZY5mb/uT27BcDRTqusSyrNmONkkcEkjJ1qpkT+DBGqQ=; b=Xy71s1pVCxecpIntN/Oeef03LVNEsr86pt/HgA20BmEf6yQYQdhNodld3rXMQgdmTR9T3ISPOjMHrPrMMSpXjD/6N57w4DnUusywHE10T/0BL227jqE0r1UpUTLnmg0JJt6RXNhuArtt7WQJ4oV2Jibphcx+QJWErI/J4f3rV90= Authentication-Results-Original: arm.com; dkim=none (message not signed) header.d=none;arm.com; dmarc=none action=none header.from=arm.com; Received: from DB7PR08MB3594.eurprd08.prod.outlook.com (2603:10a6:10:4e::11) by DB7PR08MB3452.eurprd08.prod.outlook.com (2603:10a6:10:48::13) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3066.18; Fri, 5 Jun 2020 15:22:31 +0000 Received: from DB7PR08MB3594.eurprd08.prod.outlook.com ([fe80::5447:f1c0:97c:aa35]) by DB7PR08MB3594.eurprd08.prod.outlook.com ([fe80::5447:f1c0:97c:aa35%7]) with mapi id 15.20.3045.024; Fri, 5 Jun 2020 15:22:31 +0000 From: Andrea Corallo To: libc-alpha@sourceware.org Subject: [PATCH v2] aarch64: MTE compatible strlen Date: Fri, 05 Jun 2020 17:22:26 +0200 Message-ID: X-ClientProxiedBy: LO2P265CA0169.GBRP265.PROD.OUTLOOK.COM (2603:10a6:600:a::13) To DB7PR08MB3594.eurprd08.prod.outlook.com (2603:10a6:10:4e::11) MIME-Version: 1.0 X-MS-Exchange-MessageSentRepresentingType: 1 Received: from e112547 (217.140.96.140) by LO2P265CA0169.GBRP265.PROD.OUTLOOK.COM (2603:10a6:600:a::13) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3066.18 via Frontend Transport; Fri, 5 Jun 2020 15:22:30 +0000 X-Originating-IP: [217.140.96.140] X-MS-PublicTrafficType: Email X-MS-Office365-Filtering-HT: Tenant X-MS-Office365-Filtering-Correlation-Id: 0ae816ef-0b34-439b-b8b1-08d80964482c X-MS-TrafficTypeDiagnostic: DB7PR08MB3452:|AM0PR08MB3042: X-MS-Exchange-Transport-Forked: True X-Microsoft-Antispam-PRVS: x-checkrecipientrouted: true NoDisclaimer: true X-MS-Oob-TLC-OOBClassifiers: OLM:10000;OLM:10000; X-Forefront-PRVS: 0425A67DEF X-MS-Exchange-SenderADCheck: 1 X-Microsoft-Antispam-Untrusted: BCL:0; X-Microsoft-Antispam-Message-Info-Original: GfGGLY/COnRRrNbq6gRtpXjWZgPjY+ANT8X5H0LW/tlmGrhiU3nCeAB3BZtzcMCTsveFdY6SXJVsqNYdAZl8iOLnTzEnNZxLB7/TMK6C6dnakOCgb65IXuwLdRJjtc6SAIbf00scrQN/CFFTCP77tcHxAC0kdVBG3klLbNnpCmZHMfUY4G0SyNXuCAcI4c2LtZLABEWpL7Iar5eJOQLPmXxmUn7kyo/gKLoeZW22/XlRKS2bT1lzqrxn/y02BDax4MKFcyxIrdD6A+msGyHzRot+l2u7eaMZtlNZC00DkctF/ehRlvwphpDTqFPNBuaSKA8zvJsgetNg91D2mHmZtw== X-Forefront-Antispam-Report-Untrusted: CIP:255.255.255.255; CTRY:; LANG:en; SCL:1; SRV:; IPV:NLI; SFV:NSPM; H:DB7PR08MB3594.eurprd08.prod.outlook.com; PTR:; CAT:NONE; SFTY:; SFS:(4636009)(376002)(346002)(136003)(396003)(39860400002)(366004)(30864003)(83380400001)(86362001)(6486002)(36756003)(52116002)(6496006)(6916009)(5660300002)(316002)(2906002)(16526019)(2616005)(186003)(4326008)(6666004)(66556008)(8936002)(66616009)(44832011)(66946007)(26005)(478600001)(956004)(66476007)(8676002); DIR:OUT; SFP:1101; X-MS-Exchange-AntiSpam-MessageData: MlgQE0cP0oZtL5UJQCoZ2hxjDk03BUcNZYCYf4G6MSvDBPYTbnpankhc1Xdu+00jo4A5baaNLhTPQz//9/gQoCi+T2v/XeyC390WV3ymL8ro+ojDgJ5rhJBkVx/rlrWvbxEuxSX4DWywaVlXS7/3rlJ66GknskBunnkWOi5D/lmk5Tgu5J2a1J874RtuzG7WUZMjb5tzYXs3WmiEN9ceAR0LMkMz9FH8o6hDGWznnpeE1YLoTuGC4ZA/QcGQUPN1F2S9lU+X2/WXiCjOQ+ibpXYBQzaiqSYD48GprcIz8QlbamXGUX62REIHe4NolXCL/Dur0GL6Qekd2jVEz73tfsSuufL2EPuh7gGvlwaIMddQQylDUrMbg+Yj0w0ylI7RNVt9YSYSwzE9Tr5DmFavNa5aqzuUg4j8Eruh1p72Fx8QhBnOFTQdDUrlK1cRyU3+vrRPgFgT/jNNXf2h78uQhEExn9CCGQkVFjO0CZmeK6s= X-MS-Exchange-Transport-CrossTenantHeadersStamped: DB7PR08MB3452 Original-Authentication-Results: arm.com; dkim=none (message not signed) header.d=none;arm.com; dmarc=none action=none header.from=arm.com; X-EOPAttributedMessage: 0 X-MS-Exchange-Transport-CrossTenantHeadersStripped: AM5EUR03FT021.eop-EUR03.prod.protection.outlook.com X-Forefront-Antispam-Report: CIP:63.35.35.123; CTRY:IE; LANG:en; SCL:1; SRV:; IPV:CAL; SFV:NSPM; H:64aa7808-outbound-1.mta.getcheckrecipient.com; PTR:ec2-63-35-35-123.eu-west-1.compute.amazonaws.com; CAT:NONE; SFTY:; SFS:(4636009)(376002)(396003)(346002)(39860400002)(136003)(46966005)(16526019)(478600001)(70206006)(6916009)(70586007)(6496006)(66616009)(26005)(186003)(36756003)(30864003)(6486002)(2906002)(4326008)(5660300002)(336012)(36906005)(8936002)(356005)(8676002)(956004)(86362001)(82740400003)(2616005)(47076004)(81166007)(6666004)(82310400002)(83380400001)(44832011)(316002); DIR:OUT; SFP:1101; X-MS-Office365-Filtering-Correlation-Id-Prvs: 12048426-0a41-4643-9679-08d8096443f5 X-Forefront-PRVS: 0425A67DEF X-Microsoft-Antispam: BCL:0; X-Microsoft-Antispam-Message-Info: 46sl0q20bCJpFgXUz9tGYLAfPtLjzlQ/TYTRSkUn7xlERzz3utBPgFiPDwP6glKo/IR7EIKlD3DVLotmXHXMTdjR1AyDzL///AakT9TXjY4Ar+EjzUD0zs6tCHCoeiv07blWDpleKmwl0vLeYMUUflp5mDuERm7kzkY4+jHhjl5okegwDrox19zA7hNMHqWYsMWZVap3NquyD0+x130LD13yflGf3SKC6zaM6M8qetGwg5bded9uNoHxdYfCOxcKcgQgrkyWzLW1tnj1/bHytTVQntUGI6UO/IlEo/c4GMMsZGWrtgKK3ahFy6GQTdBUOivk2CWJDox7OcYwKmEqeCxYDlZgQfMYsm/lojw5QTeKK/Gc7tQSiXVaKfybl3PU5b2aza9OgopuFS3XgT8uCQ== X-OriginatorOrg: arm.com X-MS-Exchange-CrossTenant-OriginalArrivalTime: 05 Jun 2020 15:22:38.4465 (UTC) X-MS-Exchange-CrossTenant-Network-Message-Id: 0ae816ef-0b34-439b-b8b1-08d80964482c X-MS-Exchange-CrossTenant-Id: f34e5979-57d9-4aaa-ad4d-b122a662184d X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp: TenantId=f34e5979-57d9-4aaa-ad4d-b122a662184d; Ip=[63.35.35.123]; Helo=[64aa7808-outbound-1.mta.getcheckrecipient.com] X-MS-Exchange-CrossTenant-FromEntityHeader: HybridOnPrem X-MS-Exchange-Transport-CrossTenantHeadersStamped: AM0PR08MB3042 X-Spam-Status: No, score=-18.1 required=5.0 tests=BAYES_00, DKIM_SIGNED, DKIM_VALID, GIT_PATCH_0, MSGID_FROM_MTA_HEADER, RCVD_IN_DNSWL_LOW, RCVD_IN_MSPIKE_H2, SPF_HELO_PASS, SPF_PASS, TXREP, UNPARSEABLE_RELAY autolearn=ham autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on server2.sourceware.org X-BeenThere: libc-alpha@sourceware.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Libc-alpha mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: nd@arm.com, Wilco.Dijkstra@arm.com Errors-To: libc-alpha-bounces@sourceware.org Sender: "Libc-alpha" Hi all, I'd like to submit this patch introducing an Arm MTE compatible strlen implementation. Follows a performance comparison (obtained using glibc benchtests) of the strlen benchmark run on Cortex-A72, Cortex-A53, Neoverse N1. | length | alignment | perf-uplift A72 | perf-uplift A53 |perf-uplift | |--------+-----------+-----------------+-----------------|------------| | 1 | 1 | 1.00x | 0.96x | 1.13x | | 1 | 0 | 2.15x | 0.96x | 1.00x | | 2 | 2 | 1.16x | 0.95x | 1.09x | | 2 | 0 | 1.17x | 0.93x | 1.00x | | 3 | 3 | 1.30x | 0.95x | 1.09x | | 3 | 0 | 1.32x | 0.96x | 1.00x | | 4 | 4 | 1.14x | 0.87x | 0.99x | | 4 | 0 | 1.14x | 0.96x | 1.00x | | 5 | 5 | 1.15x | 0.89x | 1.09x | | 5 | 0 | 1.19x | 0.96x | 1.00x | | 6 | 6 | 1.14x | 0.96x | 1.39x | | 6 | 0 | 1.14x | 0.95x | 1.00x | | 7 | 7 | 1.03x | 0.90x | 1.09x | | 7 | 0 | 1.14x | 0.95x | 1.27x | | 4 | 0 | 1.15x | 0.87x | 1.00x | | 4 | 7 | 1.15x | 0.96x | 1.10x | | 4 | 2 | 1.27x | 0.95x | 1.39x | | 2 | 2 | 1.14x | 0.96x | 1.09x | | 8 | 0 | 1.15x | 0.96x | 1.00x | | 8 | 7 | 1.14x | 0.96x | 1.09x | | 8 | 3 | 1.17x | 0.96x | 1.39x | | 5 | 3 | 1.14x | 0.96x | 1.39x | | 16 | 0 | 1.15x | 0.83x | 1.48x | | 16 | 7 | 1.14x | 0.80x | 1.43x | | 16 | 4 | 1.15x | 0.83x | 1.48x | | 10 | 4 | 1.15x | 0.96x | 1.27x | | 32 | 0 | 1.04x | 0.88x | 1.16x | | 32 | 7 | 1.02x | 0.84x | 1.19x | | 32 | 5 | 1.04x | 0.84x | 1.23x | | 21 | 5 | 1.14x | 0.83x | 1.60x | | 64 | 0 | 1.17x | 0.80x | 1.75x | | 64 | 7 | 1.17x | 0.77x | 1.83x | | 64 | 6 | 1.17x | 0.77x | 1.57x | | 42 | 6 | 1.00x | 0.80x | 1.42x | | 128 | 0 | 0.96x | 0.68x | 1.80x | | 128 | 7 | 0.96x | 0.66x | 1.85x | | 128 | 7 | 0.96x | 0.67x | 1.86x | | 85 | 7 | 1.05x | 0.75x | 1.87x | | 256 | 0 | 0.98x | 0.69x | 1.88x | | 256 | 7 | 0.98x | 0.68x | 1.92x | | 256 | 8 | 0.99x | 0.69x | 1.88x | | 170 | 8 | 0.96x | 0.72x | 1.86x | | 512 | 0 | 0.99x | 0.65x | 1.90x | | 512 | 7 | 0.98x | 0.65x | 1.92x | | 512 | 9 | 0.99x | 0.65x | 1.92x | | 341 | 9 | 0.98x | 0.68x | 1.99x | | 1024 | 0 | 0.99x | 0.63x | 1.90x | | 1024 | 7 | 0.99x | 0.62x | 1.92x | | 1024 | 10 | 0.99x | 0.62x | 1.92x | | 682 | 10 | 0.99x | 0.64x | 1.96x | | 2048 | 0 | 0.99x | 0.61x | 1.92x | | 2048 | 7 | 1.01x | 0.61x | 1.93x | | 2048 | 11 | 1.00x | 0.61x | 1.95x | | 1365 | 11 | 1.00x | 0.62x | 1.94x | | 4096 | 0 | 1.00x | 0.61x | 1.93x | | 4096 | 7 | 1.00x | 0.61x | 1.94x | | 4096 | 12 | 1.00x | 0.61x | 1.95x | | 2730 | 12 | 1.00x | 0.61x | 1.94x | This patch is passing GLIBC tests. Regards Andrea 8< --- 8< --- 8< Introduce an Arm MTE compatible strlen implementation. The existing implementation assumes that any access to the pages in which the string resides is safe. This assumption is not true when MTE is enabled. This patch updates the algorithm to ensure that accesses remain within the bounds of an MTE tag (16-byte chunks) and improves overall performance. Benchmarked on Cortex-A72, Cortex-A53, Neoverse N1. Co-authored-by: Wilco Dijkstra diff --git a/sysdeps/aarch64/strlen.S b/sysdeps/aarch64/strlen.S index e01fab7c2a..e314fffed6 100644 --- a/sysdeps/aarch64/strlen.S +++ b/sysdeps/aarch64/strlen.S @@ -20,205 +20,78 @@ /* Assumptions: * - * ARMv8-a, AArch64, unaligned accesses, min page size 4k. + * ARMv8-a, AArch64, Advanced SIMD. + * MTE compatible. */ #ifndef STRLEN # define STRLEN __strlen #endif -/* To test the page crossing code path more thoroughly, compile with - -DTEST_PAGE_CROSS - this will force all calls through the slower - entry path. This option is not intended for production use. */ - -/* Arguments and results. */ #define srcin x0 -#define len x0 +#define result x0 -/* Locals and temporaries. */ #define src x1 -#define data1 x2 -#define data2 x3 -#define has_nul1 x4 -#define has_nul2 x5 -#define tmp1 x4 -#define tmp2 x5 -#define tmp3 x6 -#define tmp4 x7 -#define zeroones x8 - - /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 - (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - can be done in parallel across the entire word. A faster check - (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives - false hits for characters 129..255. */ - -#define REP8_01 0x0101010101010101 -#define REP8_7f 0x7f7f7f7f7f7f7f7f -#define REP8_80 0x8080808080808080 - -#ifdef TEST_PAGE_CROSS -# define MIN_PAGE_SIZE 16 -#else -# define MIN_PAGE_SIZE 4096 -#endif - - /* Since strings are short on average, we check the first 16 bytes - of the string for a NUL character. In order to do an unaligned ldp - safely we have to do a page cross check first. If there is a NUL - byte we calculate the length from the 2 8-byte words using - conditional select to reduce branch mispredictions (it is unlikely - strlen will be repeatedly called on strings with the same length). - - If the string is longer than 16 bytes, we align src so don't need - further page cross checks, and process 32 bytes per iteration - using the fast NUL check. If we encounter non-ASCII characters, - fallback to a second loop using the full NUL check. - - If the page cross check fails, we read 16 bytes from an aligned - address, remove any characters before the string, and continue - in the main loop using aligned loads. Since strings crossing a - page in the first 16 bytes are rare (probability of - 16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized. - - AArch64 systems have a minimum page size of 4k. We don't bother - checking for larger page sizes - the cost of setting up the correct - page size is just not worth the extra gain from a small reduction in - the cases taking the slow path. Note that we only care about - whether the first fetch, which may be misaligned, crosses a page - boundary. */ - -ENTRY_ALIGN (STRLEN, 6) +#define synd x2 +#define tmp x3 +#define wtmp w3 +#define shift x4 + +#define data q0 +#define vdata v0 +#define vhas_nul v1 +#define vrepmask v2 +#define vend v3 +#define dend d3 + +/* Core algorithm: + + For each 16-byte chunk we calculate a 64-bit syndrome value with four bits + per byte. For even bytes, bits 0-3 are set if the relevant byte matched the + requested character or the byte is NUL. Bits 4-7 must be zero. Bits 4-7 are + set likewise for odd bytes so that adjacent bytes can be merged. Since the + bits in the syndrome reflect the order in which things occur in the original + string, counting trailing zeros identifies exactly which byte matched. */ + +ENTRY (STRLEN) DELOUSE (0) DELOUSE (1) - and tmp1, srcin, MIN_PAGE_SIZE - 1 - mov zeroones, REP8_01 - cmp tmp1, MIN_PAGE_SIZE - 16 - b.gt L(page_cross) - ldp data1, data2, [srcin] -#ifdef __AARCH64EB__ - /* For big-endian, carry propagation (if the final byte in the - string is 0x01) means we cannot use has_nul1/2 directly. - Since we expect strings to be small and early-exit, - byte-swap the data now so has_null1/2 will be correct. */ - rev data1, data1 - rev data2, data2 -#endif - sub tmp1, data1, zeroones - orr tmp2, data1, REP8_7f - sub tmp3, data2, zeroones - orr tmp4, data2, REP8_7f - bics has_nul1, tmp1, tmp2 - bic has_nul2, tmp3, tmp4 - ccmp has_nul2, 0, 0, eq - beq L(main_loop_entry) - - /* Enter with C = has_nul1 == 0. */ - csel has_nul1, has_nul1, has_nul2, cc - mov len, 8 - rev has_nul1, has_nul1 - clz tmp1, has_nul1 - csel len, xzr, len, cc - add len, len, tmp1, lsr 3 + bic src, srcin, 15 + mov wtmp, 0xf00f + ld1 {vdata.16b}, [src] + dup vrepmask.8h, wtmp + cmeq vhas_nul.16b, vdata.16b, 0 + lsl shift, srcin, 2 + and vhas_nul.16b, vhas_nul.16b, vrepmask.16b + addp vend.16b, vhas_nul.16b, vhas_nul.16b /* 128->64 */ + fmov synd, dend + lsr synd, synd, shift + cbz synd, L(loop) + + rbit synd, synd + clz result, synd + lsr result, result, 2 ret - /* The inner loop processes 32 bytes per iteration and uses the fast - NUL check. If we encounter non-ASCII characters, use a second - loop with the accurate NUL check. */ - .p2align 4 -L(main_loop_entry): - bic src, srcin, 15 - sub src, src, 16 -L(main_loop): - ldp data1, data2, [src, 32]! -L(page_cross_entry): - sub tmp1, data1, zeroones - sub tmp3, data2, zeroones - orr tmp2, tmp1, tmp3 - tst tmp2, zeroones, lsl 7 - bne 1f - ldp data1, data2, [src, 16] - sub tmp1, data1, zeroones - sub tmp3, data2, zeroones - orr tmp2, tmp1, tmp3 - tst tmp2, zeroones, lsl 7 - beq L(main_loop) - add src, src, 16 -1: - /* The fast check failed, so do the slower, accurate NUL check. */ - orr tmp2, data1, REP8_7f - orr tmp4, data2, REP8_7f - bics has_nul1, tmp1, tmp2 - bic has_nul2, tmp3, tmp4 - ccmp has_nul2, 0, 0, eq - beq L(nonascii_loop) - - /* Enter with C = has_nul1 == 0. */ -L(tail): -#ifdef __AARCH64EB__ - /* For big-endian, carry propagation (if the final byte in the - string is 0x01) means we cannot use has_nul1/2 directly. The - easiest way to get the correct byte is to byte-swap the data - and calculate the syndrome a second time. */ - csel data1, data1, data2, cc - rev data1, data1 - sub tmp1, data1, zeroones - orr tmp2, data1, REP8_7f - bic has_nul1, tmp1, tmp2 -#else - csel has_nul1, has_nul1, has_nul2, cc + .p2align 5 +L(loop): + ldr data, [src, 16]! + cmeq vhas_nul.16b, vdata.16b, 0 + umaxp vend.16b, vhas_nul.16b, vhas_nul.16b + fmov synd, dend + cbz synd, L(loop) + + and vhas_nul.16b, vhas_nul.16b, vrepmask.16b + addp vend.16b, vhas_nul.16b, vhas_nul.16b /* 128->64 */ + sub result, src, srcin + fmov synd, dend +#ifndef __AARCH64EB__ + rbit synd, synd #endif - sub len, src, srcin - rev has_nul1, has_nul1 - add tmp2, len, 8 - clz tmp1, has_nul1 - csel len, len, tmp2, cc - add len, len, tmp1, lsr 3 + clz tmp, synd + add result, result, tmp, lsr 2 ret -L(nonascii_loop): - ldp data1, data2, [src, 16]! - sub tmp1, data1, zeroones - orr tmp2, data1, REP8_7f - sub tmp3, data2, zeroones - orr tmp4, data2, REP8_7f - bics has_nul1, tmp1, tmp2 - bic has_nul2, tmp3, tmp4 - ccmp has_nul2, 0, 0, eq - bne L(tail) - ldp data1, data2, [src, 16]! - sub tmp1, data1, zeroones - orr tmp2, data1, REP8_7f - sub tmp3, data2, zeroones - orr tmp4, data2, REP8_7f - bics has_nul1, tmp1, tmp2 - bic has_nul2, tmp3, tmp4 - ccmp has_nul2, 0, 0, eq - beq L(nonascii_loop) - b L(tail) - - /* Load 16 bytes from [srcin & ~15] and force the bytes that precede - srcin to 0x7f, so we ignore any NUL bytes before the string. - Then continue in the aligned loop. */ -L(page_cross): - bic src, srcin, 15 - ldp data1, data2, [src] - lsl tmp1, srcin, 3 - mov tmp4, -1 -#ifdef __AARCH64EB__ - /* Big-endian. Early bytes are at MSB. */ - lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */ -#else - /* Little-endian. Early bytes are at LSB. */ - lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */ -#endif - orr tmp1, tmp1, REP8_80 - orn data1, data1, tmp1 - orn tmp2, data2, tmp1 - tst srcin, 8 - csel data1, data1, tmp4, eq - csel data2, data2, tmp2, eq - b L(page_cross_entry) END (STRLEN) weak_alias (STRLEN, strlen) libc_hidden_builtin_def (strlen)