@@ -22,6 +22,7 @@
#include <pthread.h>
#include <pthreadP.h>
#include <stap-probe.h>
+#include <elide.h>
/* Acquire read lock for RWLOCK. Slow path. */
@@ -102,6 +103,12 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
LIBC_PROBE (rdlock_entry, 1, rwlock);
+ if (ELIDE_LOCK (rwlock->__data.__rwelision,
+ rwlock->__data.__lock == 0
+ && rwlock->__data.__writer == 0
+ && rwlock->__data.__nr_readers == 0))
+ return 0;
+
/* Make sure we are alone. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
@@ -19,6 +19,7 @@
#include <errno.h>
#include "pthreadP.h"
#include <lowlevellock.h>
+#include <elide.h>
int
@@ -26,6 +27,12 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
{
int result = EBUSY;
+ if (ELIDE_TRYLOCK (rwlock->__data.__rwelision,
+ rwlock->__data.__lock == 0
+ && rwlock->__data.__nr_readers == 0
+ && rwlock->__data.__writer, 0))
+ return 0;
+
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
if (rwlock->__data.__writer == 0
@@ -19,6 +19,7 @@
#include <errno.h>
#include "pthreadP.h"
#include <lowlevellock.h>
+#include <elide.h>
int
@@ -26,6 +27,12 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
{
int result = EBUSY;
+ if (ELIDE_TRYLOCK (rwlock->__data.__rwelision,
+ rwlock->__data.__lock == 0
+ && rwlock->__data.__nr_readers == 0
+ && rwlock->__data.__writer, 1))
+ return 0;
+
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
if (rwlock->__data.__writer == 0 && rwlock->__data.__nr_readers == 0)
@@ -22,6 +22,8 @@
#include <pthread.h>
#include <pthreadP.h>
#include <stap-probe.h>
+#include <elide.h>
+
/* Unlock RWLOCK. */
int
@@ -29,6 +31,10 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
{
LIBC_PROBE (rwlock_unlock, 1, rwlock);
+ if (ELIDE_UNLOCK (rwlock->__data.__writer == 0
+ && rwlock->__data.__nr_readers == 0))
+ return 0;
+
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
if (rwlock->__data.__writer)
rwlock->__data.__writer = 0;
@@ -22,6 +22,7 @@
#include <pthread.h>
#include <pthreadP.h>
#include <stap-probe.h>
+#include <elide.h>
/* Acquire write lock for RWLOCK. */
@@ -91,6 +92,12 @@ __pthread_rwlock_wrlock (pthread_rwlock_t *rwlock)
{
LIBC_PROBE (wrlock_entry, 1, rwlock);
+ if (ELIDE_LOCK (rwlock->__data.__rwelision,
+ rwlock->__data.__lock == 0
+ && rwlock->__data.__writer == 0
+ && rwlock->__data.__nr_readers == 0))
+ return 0;
+
/* Make sure we are alone. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
@@ -132,17 +132,17 @@ enum
/* Read-write lock initializers. */
# define PTHREAD_RWLOCK_INITIALIZER \
- { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }
+ { { 0, 0, 0, 0, 0, 0, 0, 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, 0 } }
# ifdef __USE_GNU
# ifdef __PTHREAD_RWLOCK_INT_FLAGS_SHARED
# define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \
- { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
+ { { 0, 0, 0, 0, 0, 0, 0, 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, \
PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP } }
# else
# if __BYTE_ORDER == __LITTLE_ENDIAN
# define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \
{ { 0, 0, 0, 0, 0, 0, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP, \
- 0, 0, 0, 0 } }
+ 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, 0 } }
# else
# define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \
{ { 0, 0, 0, 0, 0, 0, 0, 0, 0, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP,\
@@ -214,6 +214,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -194,6 +194,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -155,6 +155,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -155,6 +155,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -194,6 +194,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -141,6 +141,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -142,6 +142,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -196,6 +196,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -143,6 +143,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -146,6 +146,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -150,6 +150,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -203,6 +203,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -194,6 +194,8 @@ typedef union
long int __align;
} pthread_rwlock_t;
+#define __PTHREAD_RWLOCK_ELISION_EXTRA 0
+
typedef union
{
char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T];
@@ -66,6 +66,8 @@ elision_init (int argc __attribute__ ((unused)),
#ifdef ENABLE_LOCK_ELISION
__pthread_force_elision = __libc_enable_secure ? 0 : __elision_available;
#endif
+ if (!HAS_RTM)
+ __elision_aconf.retry_try_xbegin = 0; /* Disable elision on rwlocks */
}
#ifdef SHARED
@@ -184,11 +184,13 @@ typedef union
unsigned int __nr_writers_queued;
int __writer;
int __shared;
- unsigned long int __pad1;
+ signed char __rwelision;
+ unsigned char __pad1[7];
unsigned long int __pad2;
/* FLAGS must stay at this position in the structure to maintain
binary compatibility. */
unsigned int __flags;
+# define __PTHREAD_RWLOCK_ELISION_EXTRA 0, {0, 0, 0, 0, 0, 0, 0 }
# define __PTHREAD_RWLOCK_INT_FLAGS_SHARED 1
} __data;
# else
@@ -204,7 +206,8 @@ typedef union
binary compatibility. */
unsigned char __flags;
unsigned char __shared;
- unsigned char __pad1;
+ signed char __rwelision;
+# define __PTHREAD_RWLOCK_ELISION_EXTRA 0
unsigned char __pad2;
int __writer;
} __data;
new file mode 100644
@@ -0,0 +1,109 @@
+/* elide.h: Generic lock elision support.
+ Copyright (C) 2014 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#ifndef ELIDE_H
+#define ELIDE_H 1
+
+#include <hle.h>
+#include <elision-conf.h>
+
+#define ACCESS_ONCE(x) (* (volatile typeof(x) *) &(x))
+
+/* Adapt elision with ADAPT_COUNT and STATUS and decide retries. */
+
+static inline bool
+elision_adapt(uint8_t *adapt_count, unsigned int status)
+{
+ if (status & _XABORT_RETRY)
+ return false;
+ if ((status & _XABORT_EXPLICIT)
+ && _XABORT_CODE (status) == _ABORT_LOCK_BUSY)
+ {
+ /* Right now we skip here. Better would be to wait a bit
+ and retry. This likely needs some spinning. Be careful
+ to avoid writing the lock. */
+ if (*adapt_count != __elision_aconf.skip_lock_busy)
+ ACCESS_ONCE (*adapt_count) = __elision_aconf.skip_lock_busy;
+ }
+ /* Internal abort. There is no chance for retry.
+ Use the normal locking and next time use lock.
+ Be careful to avoid writing to the lock. */
+ else if (*adapt_count != __elision_aconf.skip_lock_internal_abort)
+ ACCESS_ONCE (*adapt_count) = __elision_aconf.skip_lock_internal_abort;
+ return true;
+}
+
+/* is_lock_free must be executed inside the transaction */
+
+/* Returns true if lock defined by IS_LOCK_FREE was elided.
+ ADAPT_COUNT is a pointer to per-lock state variable. */
+
+#define ELIDE_LOCK(adapt_count, is_lock_free) \
+ ({ \
+ int ret = 0; \
+ \
+ if ((adapt_count) <= 0) \
+ { \
+ for (int i = __elision_aconf.retry_try_xbegin; i > 0; i--) \
+ { \
+ unsigned int status; \
+ if ((status = _xbegin ()) == _XBEGIN_STARTED) \
+ { \
+ if (is_lock_free) \
+ { \
+ ret = 1; \
+ break; \
+ } \
+ _xabort (_ABORT_LOCK_BUSY); \
+ } \
+ if (!elision_adapt (&(adapt_count), status)) \
+ break; \
+ } \
+ } \
+ else \
+ (adapt_count)--; /* missing updates ok */ \
+ ret; \
+ })
+
+/* Returns true if lock defined by IS_LOCK_FREE was try-elided.
+ ADAPT_COUNT is a pointer to per-lock state variable. */
+
+#define ELIDE_TRYLOCK(adapt_count, is_lock_free, write) ({ \
+ int ret = 0; \
+ if (__elision_aconf.retry_try_xbegin > 0) \
+ { \
+ if (write) \
+ _xabort (_ABORT_NESTED_TRYLOCK); \
+ ret = ELIDE_LOCK (adapt_count, is_lock_free); \
+ } \
+ ret; \
+ })
+
+/* Returns true if lock defined by IS_LOCK_FREE was elided. */
+
+#define ELIDE_UNLOCK(is_lock_free) \
+ ({ \
+ int ret = 0; \
+ if (is_lock_free) \
+ { \
+ _xend (); \
+ ret = 1; \
+ } \
+ ret; \
+ })
+
+#endif