1
0
mirror of https://sourceware.org/git/glibc.git synced 2025-07-30 22:43:12 +03:00

Add __builtin_expect where appropriate.

This commit is contained in:
Ulrich Drepper
2003-03-18 00:31:30 +00:00
parent 0a8d800411
commit bd0fa4cefd
6 changed files with 146 additions and 154 deletions

View File

@ -76,20 +76,22 @@ typedef uintmax_t uatomic_max_t;
__val = *__memp; \ __val = *__memp; \
if (sizeof (*mem) == 4) \ if (sizeof (*mem) == 4) \
do \ do \
__oldval = __val; \ { \
while ((__val \ __oldval = __val; \
= __arch_compare_and_exchange_32_val_acq (__memp, \ __val = __arch_compare_and_exchange_32_val_acq (__memp, \
__oldval + __value, \ __oldval + __value, \
__oldval)) \ __oldval); \
!= __oldval); \ } \
while (__builtin_expect (__val != __oldval, 0)); \
else if (sizeof (*mem) == 8) \ else if (sizeof (*mem) == 8) \
do \ do \
__oldval = __val; \ { \
while ((__val \ __oldval = __val; \
= __arch_compare_and_exchange_64_val_acq (__memp, \ __val = __arch_compare_and_exchange_64_val_acq (__memp, \
__oldval + __value, \ __oldval + __value, \
__oldval)) \ __oldval); \
!= __oldval); \ } \
while (__builtin_expect (__val != __oldval, 0)); \
else \ else \
abort (); \ abort (); \
__oldval + __value; }) __oldval + __value; })

View File

@ -17,15 +17,13 @@
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
02111-1307 USA. */ 02111-1307 USA. */
/* Ugly hack to avoid the declaration of pthread_spin_init. */
#define pthread_spin_init pthread_spin_init_XXX
#include "pthreadP.h" #include "pthreadP.h"
#undef pthread_spin_init
int int
pthread_spin_unlock (pthread_spinlock_t *lock) pthread_spin_unlock (pthread_spinlock_t *lock)
{ {
*lock = 0; __sync_lock_release_si ((int *) lock);
return 0; return 0;
} }
strong_alias (pthread_spin_unlock, pthread_spin_init) strong_alias (pthread_spin_unlock, pthread_spin_init)

View File

@ -27,9 +27,9 @@
void void
__lll_lock_wait (int *futex, int val) __lll_lock_wait (int *futex, int val)
{ {
do { do
lll_futex_wait (futex, val+1); lll_futex_wait (futex, val + 1);
} while ((val = __lll_add (futex, 1)) != 0); while ((val = __lll_add (futex, 1)) != 0);
*futex = 2; *futex = 2;
} }
hidden_proto (__lll_lock_wait) hidden_proto (__lll_lock_wait)
@ -38,8 +38,6 @@ hidden_proto (__lll_lock_wait)
int int
__lll_timedlock_wait (int *futex, int val, const struct timespec *abstime) __lll_timedlock_wait (int *futex, int val, const struct timespec *abstime)
{ {
int err;
/* Reject invalid timeouts. */ /* Reject invalid timeouts. */
if (abstime->tv_nsec >= 1000000000) if (abstime->tv_nsec >= 1000000000)
return EINVAL; return EINVAL;
@ -48,33 +46,31 @@ __lll_timedlock_wait (int *futex, int val, const struct timespec *abstime)
{ {
struct timeval tv; struct timeval tv;
struct timespec rt; struct timespec rt;
int sec, nsec;
/* Get the current time. */ /* Get the current time. */
__gettimeofday (&tv, NULL); (void) __gettimeofday (&tv, NULL);
/* Compute relative timeout. */ /* Compute relative timeout. */
sec = abstime->tv_sec - tv.tv_sec; rt.tv_sec = abstime->tv_sec - tv.tv_sec;
nsec = abstime->tv_nsec - tv.tv_usec * 1000; rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
if (nsec < 0) if (rt.tv_nsec < 0)
{ {
nsec += 1000000000; rt.tv_nsec += 1000000000;
--sec; --rt.tv_sec;
} }
/* Already timed out? */ /* Already timed out? */
err = -ETIMEDOUT; if (rt.tv_sec < 0)
if (sec < 0) return ETIMEDOUT;
break;
/* Wait. */ /* Wait. */
rt.tv_sec = sec; if (lll_futex_timed_wait (futex, val + 1, &rt) == -ETIMEDOUT)
rt.tv_nsec = nsec; return ETIMEDOUT;
err = lll_futex_timed_wait (futex, val+1, &rt); }
} while (err == 0 && (val = __lll_add (futex, 1)) != 0); while ((val = __lll_add (futex, 1)) != 0);
*futex = 2; *futex = 2;
return -err; return 0;
} }
hidden_proto (__lll_timedlock_wait) hidden_proto (__lll_timedlock_wait)
@ -84,7 +80,9 @@ hidden_proto (__lll_timedlock_wait)
int int
lll_unlock_wake_cb (int *futex) lll_unlock_wake_cb (int *futex)
{ {
__lll_add (futex, 1); if (__lll_add (futex, 1) + 1 != 0)
lll_futex_wake (futex, 1);
return 0; return 0;
} }
hidden_proto (lll_unlock_wake_cb) hidden_proto (lll_unlock_wake_cb)
@ -94,7 +92,6 @@ int
__lll_timedwait_tid (int *tidp, const struct timespec *abstime) __lll_timedwait_tid (int *tidp, const struct timespec *abstime)
{ {
int tid; int tid;
int err = 0;
if (abstime == NULL || abstime->tv_nsec >= 1000000000) if (abstime == NULL || abstime->tv_nsec >= 1000000000)
return EINVAL; return EINVAL;
@ -104,36 +101,29 @@ __lll_timedwait_tid (int *tidp, const struct timespec *abstime)
{ {
struct timeval tv; struct timeval tv;
struct timespec rt; struct timespec rt;
int sec, nsec;
/* Get the current time. */ /* Get the current time. */
__gettimeofday (&tv, NULL); (void) __gettimeofday (&tv, NULL);
/* Compute relative timeout. */ /* Compute relative timeout. */
sec = abstime->tv_sec - tv.tv_sec; rt.tv_sec = abstime->tv_sec - tv.tv_sec;
nsec = abstime->tv_nsec - tv.tv_usec * 1000; rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
if (nsec < 0) if (rt.tv_nsec < 0)
{ {
nsec += 1000000000; rt.tv_nsec += 1000000000;
--sec; --rt.tv_sec;
} }
/* Already timed out? */ /* Already timed out? */
err = -ETIMEDOUT; if (rt.tv_sec < 0)
if (sec < 0) return ETIMEDOUT;
break;
/* Wait. */
rt.tv_sec = sec;
rt.tv_nsec = nsec;
/* Wait until thread terminates. */ /* Wait until thread terminates. */
err = lll_futex_timed_wait (tidp, tid, &rt); if (lll_futex_timed_wait (tidp, tid, &rt) == -ETIMEDOUT)
if (err != 0) return ETIMEDOUT;
break;
} }
return -err; return 0;
} }
hidden_proto (__lll_timedwait_tid) hidden_proto (__lll_timedwait_tid)

View File

@ -31,107 +31,107 @@
/* Initializer for compatibility lock. */ /* Initializer for compatibility lock. */
#define LLL_MUTEX_LOCK_INITIALIZER (0) #define LLL_MUTEX_LOCK_INITIALIZER (0)
#define lll_futex_wait(futexp, val) \ #define lll_futex_wait(futexp, val) \
({ \ ({ \
INTERNAL_SYSCALL_DECL (__err); \ INTERNAL_SYSCALL_DECL (__err); \
long int __ret; \ long int __ret; \
\ \
__ret = INTERNAL_SYSCALL (futex, __err, 4, \ __ret = INTERNAL_SYSCALL (futex, __err, 4, \
(futexp), FUTEX_WAIT, (val), 0); \ (futexp), FUTEX_WAIT, (val), 0); \
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \ INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \
}) })
#define lll_futex_timed_wait(futexp, val, timespec) \ #define lll_futex_timed_wait(futexp, val, timespec) \
({ \ ({ \
INTERNAL_SYSCALL_DECL (__err); \ INTERNAL_SYSCALL_DECL (__err); \
long int __ret; \ long int __ret; \
\ \
__ret = INTERNAL_SYSCALL (futex, __err, 4, \ __ret = INTERNAL_SYSCALL (futex, __err, 4, \
(futexp), FUTEX_WAIT, (val), (timespec)); \ (futexp), FUTEX_WAIT, (val), (timespec)); \
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \ INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \
}) })
#define lll_futex_wake(futexp, nr) \ #define lll_futex_wake(futexp, nr) \
({ \ ({ \
INTERNAL_SYSCALL_DECL (__err); \ INTERNAL_SYSCALL_DECL (__err); \
long int __ret; \ long int __ret; \
\ \
__ret = INTERNAL_SYSCALL (futex, __err, 4, \ __ret = INTERNAL_SYSCALL (futex, __err, 4, \
(futexp), FUTEX_WAKE, (nr), 0); \ (futexp), FUTEX_WAKE, (nr), 0); \
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \ INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \
}) })
#ifdef UP #ifdef UP
#define __lll_acq_instr "" # define __lll_acq_instr ""
#define __lll_rel_instr "" # define __lll_rel_instr ""
#else #else
#define __lll_acq_instr "isync" # define __lll_acq_instr "isync"
#define __lll_rel_instr "sync" # define __lll_rel_instr "sync"
#endif #endif
/* Set *futex to 1 if it is 0, atomically. Returns the old value */ /* Set *futex to 1 if it is 0, atomically. Returns the old value */
#define __lll_trylock(futex) \ #define __lll_trylock(futex) \
({ int __val; \ ({ int __val; \
__asm __volatile ("1: lwarx %0,0,%2\n" \ __asm __volatile ("1: lwarx %0,0,%2\n" \
" cmpwi 0,%0,0\n" \ " cmpwi 0,%0,0\n" \
" bne 2f\n" \ " bne 2f\n" \
" stwcx. %3,0,%2\n" \ " stwcx. %3,0,%2\n" \
" bne- 1b\n" \ " bne- 1b\n" \
"2: " __lll_acq_instr \ "2: " __lll_acq_instr \
: "=&r" (__val), "=m" (*futex) \ : "=&r" (__val), "=m" (*futex) \
: "r" (futex), "r" (1), "1" (*futex) \ : "r" (futex), "r" (1), "1" (*futex) \
: "cr0", "memory"); \ : "cr0", "memory"); \
__val; \ __val; \
}) })
#define lll_mutex_trylock(lock) __lll_trylock(&(lock)) #define lll_mutex_trylock(lock) __lll_trylock(&(lock))
/* Add inc to *futex atomically and return the old value. */ /* Add inc to *futex atomically and return the old value. */
#define __lll_add(futex, inc) \ #define __lll_add(futex, inc) \
({ int __val, __tmp; \ ({ int __val, __tmp; \
__asm __volatile ("1: lwarx %0,0,%3\n" \ __asm __volatile ("1: lwarx %0,0,%3\n" \
" addi %1,%0,%4\n" \ " addi %1,%0,%4\n" \
" stwcx. %1,0,%3\n" \ " stwcx. %1,0,%3\n" \
" bne- 1b" \ " bne- 1b" \
: "=&b" (__val), "=&r" (__tmp), "=m" (*futex) \ : "=&b" (__val), "=&r" (__tmp), "=m" (*futex) \
: "r" (futex), "I" (inc), "2" (*futex) \ : "r" (futex), "I" (inc), "2" (*futex) \
: "cr0"); \ : "cr0"); \
__val; \ __val; \
}) })
extern void __lll_lock_wait (int *futex, int val) attribute_hidden; extern void __lll_lock_wait (int *futex, int val) attribute_hidden;
#define lll_mutex_lock(lock) \ #define lll_mutex_lock(lock) \
(void) ({ \ (void) ({ \
int *__futex = &(lock); \ int *__futex = &(lock); \
int __val = __lll_add (__futex, 1); \ int __val = __lll_add (__futex, 1); \
__asm __volatile (__lll_acq_instr ::: "memory"); \ __asm __volatile (__lll_acq_instr ::: "memory"); \
if (__builtin_expect (__val != 0, 0)) \ if (__builtin_expect (__val != 0, 0)) \
__lll_lock_wait (__futex, __val); \ __lll_lock_wait (__futex, __val); \
}) })
extern int __lll_timedlock_wait extern int __lll_timedlock_wait
(int *futex, int val, const struct timespec *) attribute_hidden; (int *futex, int val, const struct timespec *) attribute_hidden;
#define lll_mutex_timedlock(lock, abstime) \ #define lll_mutex_timedlock(lock, abstime) \
({ int *__futex = &(lock); \ ({ int *__futex = &(lock); \
int __val = __lll_add (__futex, 1); \ int __val = __lll_add (__futex, 1); \
__asm __volatile (__lll_acq_instr ::: "memory"); \ __asm __volatile (__lll_acq_instr ::: "memory"); \
if (__builtin_expect (__val != 0, 0)) \ if (__builtin_expect (__val != 0, 0)) \
__val = __lll_timedlock_wait (__futex, __val, (abstime)); \ __val = __lll_timedlock_wait (__futex, __val, (abstime)); \
__val; \ __val; \
}) })
#define lll_mutex_unlock(lock) \ #define lll_mutex_unlock(lock) \
(void) ({ \ (void) ({ \
int *__futex = &(lock); \ int *__futex = &(lock); \
__asm __volatile (__lll_rel_instr ::: "memory"); \ __asm __volatile (__lll_rel_instr ::: "memory"); \
int __val = __lll_add (__futex, -1); \ int __val = __lll_add (__futex, -1); \
if (__builtin_expect (__val != 1, 0)) \ if (__builtin_expect (__val != 1, 0)) \
{ \ { \
*__futex = 0; \ *__futex = 0; \
lll_futex_wake (__futex, 1); \ lll_futex_wake (__futex, 1); \
} \ } \
}) })
#define lll_mutex_islocked(futex) \ #define lll_mutex_islocked(futex) \
@ -164,39 +164,39 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
wakeup when the clone terminates. The memory location contains the wakeup when the clone terminates. The memory location contains the
thread ID while the clone is running and is reset to zero thread ID while the clone is running and is reset to zero
afterwards. */ afterwards. */
#define lll_wait_tid(tid) \ #define lll_wait_tid(tid) \
do { \ do { \
__typeof (tid) __tid; \ __typeof (tid) __tid; \
while ((__tid = (tid)) != 0) \ while ((__tid = (tid)) != 0) \
lll_futex_wait (&(tid), __tid); \ lll_futex_wait (&(tid), __tid); \
} while (0) } while (0)
extern int __lll_timedwait_tid (int *, const struct timespec *) extern int __lll_timedwait_tid (int *, const struct timespec *)
attribute_hidden; attribute_hidden;
#define lll_timedwait_tid(tid, abstime) \ #define lll_timedwait_tid(tid, abstime) \
({ \ ({ \
int __res = 0; \ int __res = 0; \
if ((tid) != 0) \ if ((tid) != 0) \
__res = __lll_timedwait_tid (&(tid), (abstime)); \ __res = __lll_timedwait_tid (&(tid), (abstime)); \
__res; \ __res; \
}) })
/* Decrement *futex if it is > 0, and return the old value */ /* Decrement *futex if it is > 0, and return the old value */
#define __lll_dec_if_positive(futex) \ #define __lll_dec_if_positive(futex) \
({ int __val, __tmp; \ ({ int __val, __tmp; \
__asm __volatile ("1: lwarx %0,0,%3\n" \ __asm __volatile ("1: lwarx %0,0,%3\n" \
" cmpwi 0,%0,0\n" \ " cmpwi 0,%0,0\n" \
" addi %1,%0,-1\n" \ " addi %1,%0,-1\n" \
" ble 2f\n" \ " ble 2f\n" \
" stwcx. %1,0,%3\n" \ " stwcx. %1,0,%3\n" \
" bne- 1b\n" \ " bne- 1b\n" \
"2: " __lll_acq_instr \ "2: " __lll_acq_instr \
: "=&b" (__val), "=&r" (__tmp), "=m" (*futex) \ : "=&b" (__val), "=&r" (__tmp), "=m" (*futex) \
: "r" (futex), "2" (*futex) \ : "r" (futex), "2" (*futex) \
: "cr0"); \ : "cr0"); \
__val; \ __val; \
}) })

View File

@ -0,0 +1 @@
#include <nptl/sysdeps/x86_64/pthread_spin_init.c>

View File

@ -0,0 +1 @@
#include <nptl/sysdeps/x86_64/pthread_spin_unlock.S>