mirror of
https://sourceware.org/git/glibc.git
synced 2025-07-30 22:43:12 +03:00
Add __builtin_expect where appropriate.
This commit is contained in:
@ -76,20 +76,22 @@ typedef uintmax_t uatomic_max_t;
|
|||||||
__val = *__memp; \
|
__val = *__memp; \
|
||||||
if (sizeof (*mem) == 4) \
|
if (sizeof (*mem) == 4) \
|
||||||
do \
|
do \
|
||||||
__oldval = __val; \
|
{ \
|
||||||
while ((__val \
|
__oldval = __val; \
|
||||||
= __arch_compare_and_exchange_32_val_acq (__memp, \
|
__val = __arch_compare_and_exchange_32_val_acq (__memp, \
|
||||||
__oldval + __value, \
|
__oldval + __value, \
|
||||||
__oldval)) \
|
__oldval); \
|
||||||
!= __oldval); \
|
} \
|
||||||
|
while (__builtin_expect (__val != __oldval, 0)); \
|
||||||
else if (sizeof (*mem) == 8) \
|
else if (sizeof (*mem) == 8) \
|
||||||
do \
|
do \
|
||||||
__oldval = __val; \
|
{ \
|
||||||
while ((__val \
|
__oldval = __val; \
|
||||||
= __arch_compare_and_exchange_64_val_acq (__memp, \
|
__val = __arch_compare_and_exchange_64_val_acq (__memp, \
|
||||||
__oldval + __value, \
|
__oldval + __value, \
|
||||||
__oldval)) \
|
__oldval); \
|
||||||
!= __oldval); \
|
} \
|
||||||
|
while (__builtin_expect (__val != __oldval, 0)); \
|
||||||
else \
|
else \
|
||||||
abort (); \
|
abort (); \
|
||||||
__oldval + __value; })
|
__oldval + __value; })
|
||||||
|
@ -17,15 +17,13 @@
|
|||||||
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
||||||
02111-1307 USA. */
|
02111-1307 USA. */
|
||||||
|
|
||||||
/* Ugly hack to avoid the declaration of pthread_spin_init. */
|
|
||||||
#define pthread_spin_init pthread_spin_init_XXX
|
|
||||||
#include "pthreadP.h"
|
#include "pthreadP.h"
|
||||||
#undef pthread_spin_init
|
|
||||||
|
|
||||||
int
|
int
|
||||||
pthread_spin_unlock (pthread_spinlock_t *lock)
|
pthread_spin_unlock (pthread_spinlock_t *lock)
|
||||||
{
|
{
|
||||||
*lock = 0;
|
__sync_lock_release_si ((int *) lock);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
strong_alias (pthread_spin_unlock, pthread_spin_init)
|
strong_alias (pthread_spin_unlock, pthread_spin_init)
|
||||||
|
@ -27,9 +27,9 @@
|
|||||||
void
|
void
|
||||||
__lll_lock_wait (int *futex, int val)
|
__lll_lock_wait (int *futex, int val)
|
||||||
{
|
{
|
||||||
do {
|
do
|
||||||
lll_futex_wait (futex, val+1);
|
lll_futex_wait (futex, val + 1);
|
||||||
} while ((val = __lll_add (futex, 1)) != 0);
|
while ((val = __lll_add (futex, 1)) != 0);
|
||||||
*futex = 2;
|
*futex = 2;
|
||||||
}
|
}
|
||||||
hidden_proto (__lll_lock_wait)
|
hidden_proto (__lll_lock_wait)
|
||||||
@ -38,8 +38,6 @@ hidden_proto (__lll_lock_wait)
|
|||||||
int
|
int
|
||||||
__lll_timedlock_wait (int *futex, int val, const struct timespec *abstime)
|
__lll_timedlock_wait (int *futex, int val, const struct timespec *abstime)
|
||||||
{
|
{
|
||||||
int err;
|
|
||||||
|
|
||||||
/* Reject invalid timeouts. */
|
/* Reject invalid timeouts. */
|
||||||
if (abstime->tv_nsec >= 1000000000)
|
if (abstime->tv_nsec >= 1000000000)
|
||||||
return EINVAL;
|
return EINVAL;
|
||||||
@ -48,33 +46,31 @@ __lll_timedlock_wait (int *futex, int val, const struct timespec *abstime)
|
|||||||
{
|
{
|
||||||
struct timeval tv;
|
struct timeval tv;
|
||||||
struct timespec rt;
|
struct timespec rt;
|
||||||
int sec, nsec;
|
|
||||||
|
|
||||||
/* Get the current time. */
|
/* Get the current time. */
|
||||||
__gettimeofday (&tv, NULL);
|
(void) __gettimeofday (&tv, NULL);
|
||||||
|
|
||||||
/* Compute relative timeout. */
|
/* Compute relative timeout. */
|
||||||
sec = abstime->tv_sec - tv.tv_sec;
|
rt.tv_sec = abstime->tv_sec - tv.tv_sec;
|
||||||
nsec = abstime->tv_nsec - tv.tv_usec * 1000;
|
rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
|
||||||
if (nsec < 0)
|
if (rt.tv_nsec < 0)
|
||||||
{
|
{
|
||||||
nsec += 1000000000;
|
rt.tv_nsec += 1000000000;
|
||||||
--sec;
|
--rt.tv_sec;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Already timed out? */
|
/* Already timed out? */
|
||||||
err = -ETIMEDOUT;
|
if (rt.tv_sec < 0)
|
||||||
if (sec < 0)
|
return ETIMEDOUT;
|
||||||
break;
|
|
||||||
|
|
||||||
/* Wait. */
|
/* Wait. */
|
||||||
rt.tv_sec = sec;
|
if (lll_futex_timed_wait (futex, val + 1, &rt) == -ETIMEDOUT)
|
||||||
rt.tv_nsec = nsec;
|
return ETIMEDOUT;
|
||||||
err = lll_futex_timed_wait (futex, val+1, &rt);
|
}
|
||||||
} while (err == 0 && (val = __lll_add (futex, 1)) != 0);
|
while ((val = __lll_add (futex, 1)) != 0);
|
||||||
|
|
||||||
*futex = 2;
|
*futex = 2;
|
||||||
return -err;
|
return 0;
|
||||||
}
|
}
|
||||||
hidden_proto (__lll_timedlock_wait)
|
hidden_proto (__lll_timedlock_wait)
|
||||||
|
|
||||||
@ -84,7 +80,9 @@ hidden_proto (__lll_timedlock_wait)
|
|||||||
int
|
int
|
||||||
lll_unlock_wake_cb (int *futex)
|
lll_unlock_wake_cb (int *futex)
|
||||||
{
|
{
|
||||||
__lll_add (futex, 1);
|
if (__lll_add (futex, 1) + 1 != 0)
|
||||||
|
lll_futex_wake (futex, 1);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
hidden_proto (lll_unlock_wake_cb)
|
hidden_proto (lll_unlock_wake_cb)
|
||||||
@ -94,7 +92,6 @@ int
|
|||||||
__lll_timedwait_tid (int *tidp, const struct timespec *abstime)
|
__lll_timedwait_tid (int *tidp, const struct timespec *abstime)
|
||||||
{
|
{
|
||||||
int tid;
|
int tid;
|
||||||
int err = 0;
|
|
||||||
|
|
||||||
if (abstime == NULL || abstime->tv_nsec >= 1000000000)
|
if (abstime == NULL || abstime->tv_nsec >= 1000000000)
|
||||||
return EINVAL;
|
return EINVAL;
|
||||||
@ -104,36 +101,29 @@ __lll_timedwait_tid (int *tidp, const struct timespec *abstime)
|
|||||||
{
|
{
|
||||||
struct timeval tv;
|
struct timeval tv;
|
||||||
struct timespec rt;
|
struct timespec rt;
|
||||||
int sec, nsec;
|
|
||||||
|
|
||||||
/* Get the current time. */
|
/* Get the current time. */
|
||||||
__gettimeofday (&tv, NULL);
|
(void) __gettimeofday (&tv, NULL);
|
||||||
|
|
||||||
/* Compute relative timeout. */
|
/* Compute relative timeout. */
|
||||||
sec = abstime->tv_sec - tv.tv_sec;
|
rt.tv_sec = abstime->tv_sec - tv.tv_sec;
|
||||||
nsec = abstime->tv_nsec - tv.tv_usec * 1000;
|
rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
|
||||||
if (nsec < 0)
|
if (rt.tv_nsec < 0)
|
||||||
{
|
{
|
||||||
nsec += 1000000000;
|
rt.tv_nsec += 1000000000;
|
||||||
--sec;
|
--rt.tv_sec;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Already timed out? */
|
/* Already timed out? */
|
||||||
err = -ETIMEDOUT;
|
if (rt.tv_sec < 0)
|
||||||
if (sec < 0)
|
return ETIMEDOUT;
|
||||||
break;
|
|
||||||
|
|
||||||
/* Wait. */
|
|
||||||
rt.tv_sec = sec;
|
|
||||||
rt.tv_nsec = nsec;
|
|
||||||
|
|
||||||
/* Wait until thread terminates. */
|
/* Wait until thread terminates. */
|
||||||
err = lll_futex_timed_wait (tidp, tid, &rt);
|
if (lll_futex_timed_wait (tidp, tid, &rt) == -ETIMEDOUT)
|
||||||
if (err != 0)
|
return ETIMEDOUT;
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return -err;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
hidden_proto (__lll_timedwait_tid)
|
hidden_proto (__lll_timedwait_tid)
|
||||||
|
@ -31,107 +31,107 @@
|
|||||||
/* Initializer for compatibility lock. */
|
/* Initializer for compatibility lock. */
|
||||||
#define LLL_MUTEX_LOCK_INITIALIZER (0)
|
#define LLL_MUTEX_LOCK_INITIALIZER (0)
|
||||||
|
|
||||||
#define lll_futex_wait(futexp, val) \
|
#define lll_futex_wait(futexp, val) \
|
||||||
({ \
|
({ \
|
||||||
INTERNAL_SYSCALL_DECL (__err); \
|
INTERNAL_SYSCALL_DECL (__err); \
|
||||||
long int __ret; \
|
long int __ret; \
|
||||||
\
|
\
|
||||||
__ret = INTERNAL_SYSCALL (futex, __err, 4, \
|
__ret = INTERNAL_SYSCALL (futex, __err, 4, \
|
||||||
(futexp), FUTEX_WAIT, (val), 0); \
|
(futexp), FUTEX_WAIT, (val), 0); \
|
||||||
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \
|
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \
|
||||||
})
|
})
|
||||||
|
|
||||||
#define lll_futex_timed_wait(futexp, val, timespec) \
|
#define lll_futex_timed_wait(futexp, val, timespec) \
|
||||||
({ \
|
({ \
|
||||||
INTERNAL_SYSCALL_DECL (__err); \
|
INTERNAL_SYSCALL_DECL (__err); \
|
||||||
long int __ret; \
|
long int __ret; \
|
||||||
\
|
\
|
||||||
__ret = INTERNAL_SYSCALL (futex, __err, 4, \
|
__ret = INTERNAL_SYSCALL (futex, __err, 4, \
|
||||||
(futexp), FUTEX_WAIT, (val), (timespec)); \
|
(futexp), FUTEX_WAIT, (val), (timespec)); \
|
||||||
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \
|
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \
|
||||||
})
|
})
|
||||||
|
|
||||||
#define lll_futex_wake(futexp, nr) \
|
#define lll_futex_wake(futexp, nr) \
|
||||||
({ \
|
({ \
|
||||||
INTERNAL_SYSCALL_DECL (__err); \
|
INTERNAL_SYSCALL_DECL (__err); \
|
||||||
long int __ret; \
|
long int __ret; \
|
||||||
\
|
\
|
||||||
__ret = INTERNAL_SYSCALL (futex, __err, 4, \
|
__ret = INTERNAL_SYSCALL (futex, __err, 4, \
|
||||||
(futexp), FUTEX_WAKE, (nr), 0); \
|
(futexp), FUTEX_WAKE, (nr), 0); \
|
||||||
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \
|
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0; \
|
||||||
})
|
})
|
||||||
|
|
||||||
#ifdef UP
|
#ifdef UP
|
||||||
#define __lll_acq_instr ""
|
# define __lll_acq_instr ""
|
||||||
#define __lll_rel_instr ""
|
# define __lll_rel_instr ""
|
||||||
#else
|
#else
|
||||||
#define __lll_acq_instr "isync"
|
# define __lll_acq_instr "isync"
|
||||||
#define __lll_rel_instr "sync"
|
# define __lll_rel_instr "sync"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/* Set *futex to 1 if it is 0, atomically. Returns the old value */
|
/* Set *futex to 1 if it is 0, atomically. Returns the old value */
|
||||||
#define __lll_trylock(futex) \
|
#define __lll_trylock(futex) \
|
||||||
({ int __val; \
|
({ int __val; \
|
||||||
__asm __volatile ("1: lwarx %0,0,%2\n" \
|
__asm __volatile ("1: lwarx %0,0,%2\n" \
|
||||||
" cmpwi 0,%0,0\n" \
|
" cmpwi 0,%0,0\n" \
|
||||||
" bne 2f\n" \
|
" bne 2f\n" \
|
||||||
" stwcx. %3,0,%2\n" \
|
" stwcx. %3,0,%2\n" \
|
||||||
" bne- 1b\n" \
|
" bne- 1b\n" \
|
||||||
"2: " __lll_acq_instr \
|
"2: " __lll_acq_instr \
|
||||||
: "=&r" (__val), "=m" (*futex) \
|
: "=&r" (__val), "=m" (*futex) \
|
||||||
: "r" (futex), "r" (1), "1" (*futex) \
|
: "r" (futex), "r" (1), "1" (*futex) \
|
||||||
: "cr0", "memory"); \
|
: "cr0", "memory"); \
|
||||||
__val; \
|
__val; \
|
||||||
})
|
})
|
||||||
|
|
||||||
#define lll_mutex_trylock(lock) __lll_trylock(&(lock))
|
#define lll_mutex_trylock(lock) __lll_trylock(&(lock))
|
||||||
|
|
||||||
/* Add inc to *futex atomically and return the old value. */
|
/* Add inc to *futex atomically and return the old value. */
|
||||||
#define __lll_add(futex, inc) \
|
#define __lll_add(futex, inc) \
|
||||||
({ int __val, __tmp; \
|
({ int __val, __tmp; \
|
||||||
__asm __volatile ("1: lwarx %0,0,%3\n" \
|
__asm __volatile ("1: lwarx %0,0,%3\n" \
|
||||||
" addi %1,%0,%4\n" \
|
" addi %1,%0,%4\n" \
|
||||||
" stwcx. %1,0,%3\n" \
|
" stwcx. %1,0,%3\n" \
|
||||||
" bne- 1b" \
|
" bne- 1b" \
|
||||||
: "=&b" (__val), "=&r" (__tmp), "=m" (*futex) \
|
: "=&b" (__val), "=&r" (__tmp), "=m" (*futex) \
|
||||||
: "r" (futex), "I" (inc), "2" (*futex) \
|
: "r" (futex), "I" (inc), "2" (*futex) \
|
||||||
: "cr0"); \
|
: "cr0"); \
|
||||||
__val; \
|
__val; \
|
||||||
})
|
})
|
||||||
|
|
||||||
extern void __lll_lock_wait (int *futex, int val) attribute_hidden;
|
extern void __lll_lock_wait (int *futex, int val) attribute_hidden;
|
||||||
|
|
||||||
#define lll_mutex_lock(lock) \
|
#define lll_mutex_lock(lock) \
|
||||||
(void) ({ \
|
(void) ({ \
|
||||||
int *__futex = &(lock); \
|
int *__futex = &(lock); \
|
||||||
int __val = __lll_add (__futex, 1); \
|
int __val = __lll_add (__futex, 1); \
|
||||||
__asm __volatile (__lll_acq_instr ::: "memory"); \
|
__asm __volatile (__lll_acq_instr ::: "memory"); \
|
||||||
if (__builtin_expect (__val != 0, 0)) \
|
if (__builtin_expect (__val != 0, 0)) \
|
||||||
__lll_lock_wait (__futex, __val); \
|
__lll_lock_wait (__futex, __val); \
|
||||||
})
|
})
|
||||||
|
|
||||||
extern int __lll_timedlock_wait
|
extern int __lll_timedlock_wait
|
||||||
(int *futex, int val, const struct timespec *) attribute_hidden;
|
(int *futex, int val, const struct timespec *) attribute_hidden;
|
||||||
|
|
||||||
#define lll_mutex_timedlock(lock, abstime) \
|
#define lll_mutex_timedlock(lock, abstime) \
|
||||||
({ int *__futex = &(lock); \
|
({ int *__futex = &(lock); \
|
||||||
int __val = __lll_add (__futex, 1); \
|
int __val = __lll_add (__futex, 1); \
|
||||||
__asm __volatile (__lll_acq_instr ::: "memory"); \
|
__asm __volatile (__lll_acq_instr ::: "memory"); \
|
||||||
if (__builtin_expect (__val != 0, 0)) \
|
if (__builtin_expect (__val != 0, 0)) \
|
||||||
__val = __lll_timedlock_wait (__futex, __val, (abstime)); \
|
__val = __lll_timedlock_wait (__futex, __val, (abstime)); \
|
||||||
__val; \
|
__val; \
|
||||||
})
|
})
|
||||||
|
|
||||||
#define lll_mutex_unlock(lock) \
|
#define lll_mutex_unlock(lock) \
|
||||||
(void) ({ \
|
(void) ({ \
|
||||||
int *__futex = &(lock); \
|
int *__futex = &(lock); \
|
||||||
__asm __volatile (__lll_rel_instr ::: "memory"); \
|
__asm __volatile (__lll_rel_instr ::: "memory"); \
|
||||||
int __val = __lll_add (__futex, -1); \
|
int __val = __lll_add (__futex, -1); \
|
||||||
if (__builtin_expect (__val != 1, 0)) \
|
if (__builtin_expect (__val != 1, 0)) \
|
||||||
{ \
|
{ \
|
||||||
*__futex = 0; \
|
*__futex = 0; \
|
||||||
lll_futex_wake (__futex, 1); \
|
lll_futex_wake (__futex, 1); \
|
||||||
} \
|
} \
|
||||||
})
|
})
|
||||||
|
|
||||||
#define lll_mutex_islocked(futex) \
|
#define lll_mutex_islocked(futex) \
|
||||||
@ -164,39 +164,39 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
|
|||||||
wakeup when the clone terminates. The memory location contains the
|
wakeup when the clone terminates. The memory location contains the
|
||||||
thread ID while the clone is running and is reset to zero
|
thread ID while the clone is running and is reset to zero
|
||||||
afterwards. */
|
afterwards. */
|
||||||
#define lll_wait_tid(tid) \
|
#define lll_wait_tid(tid) \
|
||||||
do { \
|
do { \
|
||||||
__typeof (tid) __tid; \
|
__typeof (tid) __tid; \
|
||||||
while ((__tid = (tid)) != 0) \
|
while ((__tid = (tid)) != 0) \
|
||||||
lll_futex_wait (&(tid), __tid); \
|
lll_futex_wait (&(tid), __tid); \
|
||||||
} while (0)
|
} while (0)
|
||||||
|
|
||||||
extern int __lll_timedwait_tid (int *, const struct timespec *)
|
extern int __lll_timedwait_tid (int *, const struct timespec *)
|
||||||
attribute_hidden;
|
attribute_hidden;
|
||||||
|
|
||||||
#define lll_timedwait_tid(tid, abstime) \
|
#define lll_timedwait_tid(tid, abstime) \
|
||||||
({ \
|
({ \
|
||||||
int __res = 0; \
|
int __res = 0; \
|
||||||
if ((tid) != 0) \
|
if ((tid) != 0) \
|
||||||
__res = __lll_timedwait_tid (&(tid), (abstime)); \
|
__res = __lll_timedwait_tid (&(tid), (abstime)); \
|
||||||
__res; \
|
__res; \
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
/* Decrement *futex if it is > 0, and return the old value */
|
/* Decrement *futex if it is > 0, and return the old value */
|
||||||
#define __lll_dec_if_positive(futex) \
|
#define __lll_dec_if_positive(futex) \
|
||||||
({ int __val, __tmp; \
|
({ int __val, __tmp; \
|
||||||
__asm __volatile ("1: lwarx %0,0,%3\n" \
|
__asm __volatile ("1: lwarx %0,0,%3\n" \
|
||||||
" cmpwi 0,%0,0\n" \
|
" cmpwi 0,%0,0\n" \
|
||||||
" addi %1,%0,-1\n" \
|
" addi %1,%0,-1\n" \
|
||||||
" ble 2f\n" \
|
" ble 2f\n" \
|
||||||
" stwcx. %1,0,%3\n" \
|
" stwcx. %1,0,%3\n" \
|
||||||
" bne- 1b\n" \
|
" bne- 1b\n" \
|
||||||
"2: " __lll_acq_instr \
|
"2: " __lll_acq_instr \
|
||||||
: "=&b" (__val), "=&r" (__tmp), "=m" (*futex) \
|
: "=&b" (__val), "=&r" (__tmp), "=m" (*futex) \
|
||||||
: "r" (futex), "2" (*futex) \
|
: "r" (futex), "2" (*futex) \
|
||||||
: "cr0"); \
|
: "cr0"); \
|
||||||
__val; \
|
__val; \
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
1
nptl/sysdeps/unix/sysv/linux/x86_64/pthread_spin_init.c
Normal file
1
nptl/sysdeps/unix/sysv/linux/x86_64/pthread_spin_init.c
Normal file
@ -0,0 +1 @@
|
|||||||
|
#include <nptl/sysdeps/x86_64/pthread_spin_init.c>
|
@ -0,0 +1 @@
|
|||||||
|
#include <nptl/sysdeps/x86_64/pthread_spin_unlock.S>
|
Reference in New Issue
Block a user