From patchwork Thu Oct 12 09:35:49 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Wilco Dijkstra X-Patchwork-Id: 23496 Received: (qmail 19063 invoked by alias); 12 Oct 2017 09:35:54 -0000 Mailing-List: contact libc-alpha-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Unsubscribe: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: libc-alpha-owner@sourceware.org Delivered-To: mailing list libc-alpha@sourceware.org Received: (qmail 18841 invoked by uid 89); 12 Oct 2017 09:35:53 -0000 Authentication-Results: sourceware.org; auth=none X-Virus-Found: No X-Spam-SWARE-Status: No, score=-25.1 required=5.0 tests=AWL, BAYES_00, GIT_PATCH_0, GIT_PATCH_1, GIT_PATCH_2, GIT_PATCH_3, RCVD_IN_DNSWL_NONE, SPF_HELO_PASS, SPF_PASS autolearn=ham version=3.3.2 spammy= X-HELO: EUR01-DB5-obe.outbound.protection.outlook.com From: Wilco Dijkstra To: "libc-alpha@sourceware.org" CC: nd Subject: [PATCH 5/5] Add single-threaded path to _int_malloc Date: Thu, 12 Oct 2017 09:35:49 +0000 Message-ID: authentication-results: spf=none (sender IP is ) smtp.mailfrom=Wilco.Dijkstra@arm.com; x-ms-publictraffictype: Email x-microsoft-exchange-diagnostics: 1; DB6SPR00MB2500; 6:MYqkVqbgg0z/1NJy7gwBtssBu7pq927qT5bGD/ysx8UMxYvXSTwDjreuXlk9CY3mVidXp14cvjiwNrDy5GzsVxG2dhgyoLSJUHrrUFYCJOEKmzwOzTWnKhmxyqGsS3aY3AHiR0pGXXV1p6rNMwWmVStGPINRJ8dfTK9l/IvADnCK2hseTPOKHQKCiwcBjYrJWmRsWDr+ARJtKndYHDk5ojBQUagbTpd0kiwzBvrHErKP4obZXPQu11flsY6qsWuG+S+W6GGKor7bvcPPeNL8cx7jaeOwRZUaK/pXk2i+B9QldEvt2ijSVWGNRuu245cxehC2hrC36jGLT0zTfVJiuw==; 5:pWkz+sCvkRFfFpIku6rytvk/6lRjADJBHNQ5R2HumQTKrF/TLwxV6yGxEs1hacoCXUJ3fgOfZNrpjkQyg3R39tci9HSLwJCgDFHT37w9TriLEM3sGKmJijrv/MnVj4vB7jAiC6E7ECJss5Qe85xikA==; 24:wbqh/tIFsqJHPrgPwjPsTk7ctKvhT6xpi+qy865KM+xaPJ4+W5l7gENGRHQ72lIwtZCJcpa8df85xDezJ8VV35ODy95BysaU5fR5i4rpD/E=; 7:HkMy9bNx0Am8vX8LsgpjVJkQPUbVGaJ87GXS4O7e2xGsclGDrZcVjvTdX0NIOnsGFjZbf46x5t2YfOjOIXw7SqMPFg9rMD25ncalJdGD/JFDanEa3zkGu2kH9jk936jh1hvxy76N6jujuIty6YjAuvMKgi96TYXGX7KT6s7Z8AsMQp20gZYnQjfPvdxozAKqzT/KJeUaII6PSBrMFIb3GooARBLp1ULGOIWsmD9d9Bo= x-ms-exchange-antispam-srfa-diagnostics: SSOS; x-ms-office365-filtering-correlation-id: 87037985-b34c-464b-1da6-08d511549f55 x-ms-office365-filtering-ht: Tenant x-microsoft-antispam: UriScan:; BCL:0; PCL:0; RULEID:(22001)(2017030254152)(48565401081)(2017052603199)(201703131423075)(201703031133081)(201702281549075); SRVR:DB6SPR00MB2500; x-ms-traffictypediagnostic: DB6SPR00MB2500: nodisclaimer: True x-exchange-antispam-report-test: UriScan:(180628864354917); x-microsoft-antispam-prvs: x-exchange-antispam-report-cfa-test: BCL:0; PCL:0; RULEID:(100000700101)(100105000095)(100000701101)(100105300095)(100000702101)(100105100095)(6040450)(2401047)(8121501046)(5005006)(93006095)(93001095)(10201501046)(100000703101)(100105400095)(3002001)(6055026)(6041248)(201703131423075)(201702281528075)(201703061421075)(201703061406153)(20161123558100)(20161123564025)(20161123562025)(20161123555025)(20161123560025)(6072148)(201708071742011)(100000704101)(100105200095)(100000705101)(100105500095); SRVR:DB6SPR00MB2500; BCL:0; PCL:0; RULEID:(100000800101)(100110000095)(100000801101)(100110300095)(100000802101)(100110100095)(100000803101)(100110400095)(100000804101)(100110200095)(100000805101)(100110500095); SRVR:DB6SPR00MB2500; x-forefront-prvs: 04583CED1A x-forefront-antispam-report: SFV:NSPM; SFS:(10009020)(6009001)(346002)(376002)(39860400002)(189002)(377424004)(199003)(54534003)(68736007)(189998001)(99286003)(2900100001)(14454004)(2351001)(5660300001)(6116002)(3846002)(102836003)(8676002)(105586002)(106356001)(5640700003)(53936002)(6436002)(2501003)(81156014)(6506006)(8936002)(575784001)(86362001)(3280700002)(3660700001)(81166006)(97736004)(5250100002)(478600001)(101416001)(74316002)(305945005)(66066001)(7736002)(50986999)(6916009)(54356999)(72206003)(33656002)(4326008)(4001150100001)(9686003)(55016002)(25786009)(7696004)(2906002)(316002); DIR:OUT; SFP:1101; SCL:1; SRVR:DB6SPR00MB2500; H:DB6PR0801MB2053.eurprd08.prod.outlook.com; FPR:; SPF:None; PTR:InfoNoRecords; MX:1; A:1; LANG:en; received-spf: None (protection.outlook.com: arm.com does not designate permitted sender hosts) spamdiagnosticoutput: 1:99 spamdiagnosticmetadata: NSPM MIME-Version: 1.0 X-OriginatorOrg: arm.com X-MS-Exchange-CrossTenant-originalarrivaltime: 12 Oct 2017 09:35:49.0092 (UTC) X-MS-Exchange-CrossTenant-fromentityheader: Hosted X-MS-Exchange-CrossTenant-id: f34e5979-57d9-4aaa-ad4d-b122a662184d X-MS-Exchange-Transport-CrossTenantHeadersStamped: DB6SPR00MB2500 This patch adds a single-threaded fast path to _int_malloc. Since the goal is to use a single test to skip multiple atomic operations, some of the fastbin allocation code has to be duplicated. Passes GLIBC tests, OK for commit? ChangeLog: 2017-10-11 Wilco Dijkstra * malloc/malloc.c (_int_malloc): Add SINGLE_THREAD_P path. diff --git a/malloc/malloc.c b/malloc/malloc.c index f4f44400d120188c4d0bece996380e04b35c8fac..11fb93fbc983d5895ea05a9c9dc262d2d9d8923c 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -3565,37 +3565,77 @@ _int_malloc (mstate av, size_t bytes) { idx = fastbin_index (nb); mfastbinptr *fb = &fastbin (av, idx); - mchunkptr pp = *fb; - REMOVE_FB (fb, victim, pp); - if (victim != 0) - { - if (__builtin_expect (fastbin_index (chunksize (victim)) != idx, 0)) - malloc_printerr ("malloc(): memory corruption (fast)"); - check_remalloced_chunk (av, victim, nb); -#if USE_TCACHE - /* While we're here, if we see other chunks of the same size, - stash them in the tcache. */ - size_t tc_idx = csize2tidx (nb); - if (tcache && tc_idx < mp_.tcache_bins) - { - mchunkptr tc_victim; + mchunkptr next; - /* While bin not empty and tcache not full, copy chunks over. */ - while (tcache->counts[tc_idx] < mp_.tcache_count - && (pp = *fb) != NULL) + if (SINGLE_THREAD_P) + { + victim = *fb; + if (victim != NULL) + { + size_t victim_idx = fastbin_index (chunksize (victim)); + next = victim->fd; + if (__builtin_expect (victim_idx != idx, 0)) + malloc_printerr ("malloc(): memory corruption (fast)"); + check_remalloced_chunk (av, victim, nb); +#if USE_TCACHE + /* While we're here, if we see other chunks of the same size, + stash them in the tcache. */ + size_t tc_idx = csize2tidx (nb); + if (next != NULL && tcache && tc_idx < mp_.tcache_bins) { - REMOVE_FB (fb, tc_victim, pp); - if (tc_victim != 0) + mchunkptr tc_victim; + + /* While bin not empty and tcache not full, copy chunks. */ + while (tcache->counts[tc_idx] < mp_.tcache_count) { + tc_victim = next; + next = tc_victim->fd; tcache_put (tc_victim, tc_idx); - } + if (next == NULL) + break; + } } - } #endif - void *p = chunk2mem (victim); - alloc_perturb (p, bytes); - return p; + *fb = next; + void *p = chunk2mem (victim); + alloc_perturb (p, bytes); + return p; + } } + else + { + next = *fb; + REMOVE_FB (fb, victim, next); + if (victim != 0) + { + size_t victim_idx = fastbin_index (chunksize (victim)); + if (__builtin_expect (victim_idx != idx, 0)) + malloc_printerr ("malloc(): memory corruption (fast)"); + check_remalloced_chunk (av, victim, nb); +#if USE_TCACHE + /* While we're here, if we see other chunks of the same size, + stash them in the tcache. */ + size_t tc_idx = csize2tidx (nb); + if (tcache && tc_idx < mp_.tcache_bins) + { + mchunkptr tc_victim; + + /* While bin not empty and tcache not full, copy chunks. */ + while (tcache->counts[tc_idx] < mp_.tcache_count + && (next = *fb) != NULL) + { + REMOVE_FB (fb, tc_victim, next); + if (tc_victim == NULL) + break; + tcache_put (tc_victim, tc_idx); + } + } +#endif + void *p = chunk2mem (victim); + alloc_perturb (p, bytes); + return p; + } + } } /*