diff --git a/nptl/pthread_rwlock_rdlock.c b/nptl/pthread_rwlock_rdlock.c
index e07581b..3073dfc 100644
--- a/nptl/pthread_rwlock_rdlock.c
+++ b/nptl/pthread_rwlock_rdlock.c
@@ -16,6 +16,8 @@
License along with the GNU C Library; if not, see
. */
+#include
+
#include "pthread_rwlock_common.c"
/* See pthread_rwlock_common.c. */
@@ -24,6 +26,12 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
{
LIBC_PROBE (rdlock_entry, 1, rwlock);
+ if (ELIDE_LOCK (rwlock->__data.__rwelision,
+ ((rwlock->__data.__readers & PTHREAD_RWLOCK_WRPHASE) &&
(rwlock->__data.__readers & PTHREAD_RWLOCK_WRLOCKED)) == 0
+ && (rwlock->__data.__readers >> PTHREAD_RWLOCK_READER_SHIFT) == 0
+ && rwlock->__data.__writers_futex == 0))
+ return 0;
+
int result = __pthread_rwlock_rdlock_full (rwlock, NULL);
LIBC_PROBE (rdlock_acquire_read, 1, rwlock);
return result;
diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c
index 6c3014c..7cff149 100644
--- a/nptl/pthread_rwlock_tryrdlock.c
+++ b/nptl/pthread_rwlock_tryrdlock.c
@@ -20,6 +20,8 @@
#include "pthreadP.h"
#include
#include
+#include
+
#include "pthread_rwlock_common.c"
@@ -27,6 +29,12 @@
int
__pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
{
+ if (ELIDE_TRYLOCK (rwlock->__data.__rwelision,
+ ((rwlock->__data.__readers & PTHREAD_RWLOCK_WRPHASE) &&
(rwlock->__data.__readers & PTHREAD_RWLOCK_WRLOCKED)) == 0
+ && (rwlock->__data.__readers >> PTHREAD_RWLOCK_READER_SHIFT) == 0
+ && rwlock->__data.__writers_futex, 0))
+ return 0;
+
/* For tryrdlock, we could speculate that we will succeed and go ahead and
register as a reader. However, if we misspeculate, we have to do the
same steps as a timed-out rdlock, which will increase contention.
diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c
index 0d9ccaf..dce32da 100644
--- a/nptl/pthread_rwlock_trywrlock.c
+++ b/nptl/pthread_rwlock_trywrlock.c
@@ -19,11 +19,18 @@
#include
#include "pthreadP.h"
#include
+#include
/* See pthread_rwlock_common.c for an overview. */
int
__pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
{
+ if (ELIDE_TRYLOCK (rwlock->__data.__rwelision,
+ ((rwlock->__data.__readers & PTHREAD_RWLOCK_WRPHASE) &&
(rwlock->__data.__readers & PTHREAD_RWLOCK_WRLOCKED)) == 0
+ && (rwlock->__data.__readers >> PTHREAD_RWLOCK_READER_SHIFT) == 0
+ && rwlock->__data.__writers_futex, 1))
+ return 0;
+
/* When in a trywrlock, we can acquire the write lock if it is in states
#1 (idle and read phase) and #5 (idle and write phase), and also in #6
(readers waiting, write phase) if we prefer writers.
diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c
index ef46e88..fa4555d 100644
--- a/nptl/pthread_rwlock_unlock.c
+++ b/nptl/pthread_rwlock_unlock.c
@@ -22,6 +22,7 @@
#include
#include
#include
+#include
#include "pthread_rwlock_common.c"
@@ -31,6 +32,10 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
{
LIBC_PROBE (rwlock_unlock, 1, rwlock);
+ if (ELIDE_UNLOCK ((rwlock->__data.__readers >>
PTHREAD_RWLOCK_READER_SHIFT) == 0
+ && rwlock->__data.__writers_futex == 0))
+ return 0;
+
/* We distinguish between having acquired a read vs. a write lock by looking
at the writer TID. If it's equal to our TID, we must be the writer
because nobody else can have stored this value. Also, if we are a
diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c
index 335fcd1..d041aad 100644
--- a/nptl/pthread_rwlock_wrlock.c
+++ b/nptl/pthread_rwlock_wrlock.c
@@ -16,6 +16,8 @@
License along with the GNU C Library; if not, see
. */
+#include
+
#include "pthread_rwlock_common.c"
/* See pthread_rwlock_common.c. */
@@ -24,6 +26,12 @@ __pthread_rwlock_wrlock (pthread_rwlock_t *rwlock)
{
LIBC_PROBE (wrlock_entry, 1, rwlock);
+ if (ELIDE_LOCK (rwlock->__data.__rwelision,
+ ((rwlock->__data.__readers & PTHREAD_RWLOCK_WRPHASE) &&
(rwlock->__data.__readers & PTHREAD_RWLOCK_WRLOCKED)) == 0
+ && (rwlock->__data.__readers >> PTHREAD_RWLOCK_READER_SHIFT) == 0
+ && rwlock->__data.__writers_futex == 0))
+ return 0;
+
int result = __pthread_rwlock_wrlock_full (rwlock, NULL);
LIBC_PROBE (wrlock_acquire_write, 1, rwlock);
return result;
diff --git a/sysdeps/x86/elide.h b/sysdeps/x86/elide.h
index 53de418..3eaf15c 100644
--- a/sysdeps/x86/elide.h
+++ b/sysdeps/x86/elide.h
@@ -77,7 +77,7 @@ elision_adapt(signed char *adapt_count, unsigned int status)
} \
_xabort (_ABORT_LOCK_BUSY); \
} \
- if (!elision_adapt (&(adapt_count), status)) \
+ if (elision_adapt (&(adapt_count), status)) \
break; \
} \
} \