From 1e46d6a032d238bf3ca749289ca8abc8432c4e23 Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Tue, 17 Nov 2009 17:11:32 -0700 Subject: [PATCH] WL#2595 kernel-independent atomic operations Backport from 6.0.14 to 5.6.0 Original code from Sergei Golubchik --- configure.in | 90 ++++++----- include/Makefile.am | 6 +- include/atomic/gcc_builtins.h | 5 +- include/atomic/generic-msvc.h | 116 ++++++++++++++ include/atomic/nolock.h | 45 +++--- include/atomic/rwlock.h | 70 +++++++-- include/atomic/solaris.h | 8 +- include/atomic/x86-gcc.h | 28 ++-- include/my_atomic.h | 285 +++++++++++++++++++++++++--------- mysys/my_atomic.c | 33 +++- mysys/my_getncpus.c | 34 ++-- unittest/mysys/Makefile.am | 2 + unittest/mysys/my_atomic-t.c | 155 ++++++------------ unittest/mysys/thr_template.c | 92 +++++++++++ 14 files changed, 681 insertions(+), 288 deletions(-) create mode 100644 include/atomic/generic-msvc.h create mode 100644 unittest/mysys/thr_template.c diff --git a/configure.in b/configure.in index 3910422a0f7..dd2c53a2b52 100644 --- a/configure.in +++ b/configure.in @@ -1750,64 +1750,74 @@ then fi AC_ARG_WITH([atomic-ops], - AC_HELP_STRING([--with-atomic-ops=rwlocks|smp|up], - [Implement atomic operations using pthread rwlocks or atomic CPU - instructions for multi-processor (default) or uniprocessor - configuration]), , [with_atomic_ops=smp]) + AS_HELP_STRING([--with-atomic-ops=rwlocks|smp|up], + [Implement atomic operations using pthread rwlocks or atomic CPU + instructions for multi-processor or uniprocessor + configuration. By default gcc built-in sync functions are used, + if available and 'smp' configuration otherwise.])) case "$with_atomic_ops" in "up") AC_DEFINE([MY_ATOMIC_MODE_DUMMY], [1], [Assume single-CPU mode, no concurrency]) ;; "rwlocks") AC_DEFINE([MY_ATOMIC_MODE_RWLOCKS], [1], [Use pthread rwlocks for atomic ops]) ;; "smp") ;; + "") + ;; *) AC_MSG_ERROR(["$with_atomic_ops" is not a valid value for --with-atomic-ops]) ;; esac AC_CACHE_CHECK([whether the compiler provides atomic builtins], - [mysql_cv_gcc_atomic_builtins], [AC_TRY_RUN([ - int main() - { - int foo= -10; int bar= 10; - if (!__sync_fetch_and_add(&foo, bar) || foo) - return -1; - bar= __sync_lock_test_and_set(&foo, bar); - if (bar || foo != 10) - return -1; - bar= __sync_val_compare_and_swap(&bar, foo, 15); - if (bar) - return -1; - return 0; - } -], [mysql_cv_gcc_atomic_builtins=yes], + [mysql_cv_gcc_atomic_builtins], + [AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [ + ], + [[ + int foo= -10; int bar= 10; + if (!__sync_fetch_and_add(&foo, bar) || foo) + return -1; + bar= __sync_lock_test_and_set(&foo, bar); + if (bar || foo != 10) + return -1; + bar= __sync_val_compare_and_swap(&bar, foo, 15); + if (bar) + return -1; + return 0; + ]] + )], + [mysql_cv_gcc_atomic_builtins=yes], [mysql_cv_gcc_atomic_builtins=no], - [mysql_cv_gcc_atomic_builtins=no])]) - + [mysql_cv_gcc_atomic_builtins=no] +)]) if test "x$mysql_cv_gcc_atomic_builtins" = xyes; then AC_DEFINE(HAVE_GCC_ATOMIC_BUILTINS, 1, [Define to 1 if compiler provides atomic builtins.]) fi AC_CACHE_CHECK([whether the OS provides atomic_* functions like Solaris], - [mysql_cv_solaris_atomic], [AC_TRY_RUN([ -#include -int -main() -{ - int foo = -10; int bar = 10; - if (atomic_add_int_nv((uint_t *)&foo, bar) || foo) - return -1; - bar = atomic_swap_uint((uint_t *)&foo, (uint_t)bar); - if (bar || foo != 10) - return -1; - bar = atomic_cas_uint((uint_t *)&bar, (uint_t)foo, 15); - if (bar) - return -1; - return 0; -} -], [mysql_cv_solaris_atomic=yes], + [mysql_cv_solaris_atomic], + [AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [ + #include + ] + [[ + int foo = -10; int bar = 10; + if (atomic_add_int_nv((uint_t *)&foo, bar) || foo) + return -1; + bar = atomic_swap_uint((uint_t *)&foo, (uint_t)bar); + if (bar || foo != 10) + return -1; + bar = atomic_cas_uint((uint_t *)&bar, (uint_t)foo, 15); + if (bar) + return -1; + return 0; + ]] + )], + [mysql_cv_solaris_atomic=yes], [mysql_cv_solaris_atomic=no], - [mysql_cv_solaris_atomic=no])]) - + [mysql_cv_solaris_atomic=no] +)]) if test "x$mysql_cv_solaris_atomic" = xyes; then AC_DEFINE(HAVE_SOLARIS_ATOMIC, 1, [Define to 1 if OS provides atomic_* functions like Solaris.]) diff --git a/include/Makefile.am b/include/Makefile.am index 0a7a35bd6fa..e20b08afe35 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -39,9 +39,9 @@ noinst_HEADERS = config-win.h config-netware.h my_bit.h \ thr_lock.h t_ctype.h violite.h my_md5.h base64.h \ my_handler.h my_time.h service_versions.h \ my_vle.h my_user.h my_atomic.h atomic/nolock.h \ - atomic/rwlock.h atomic/x86-gcc.h atomic/x86-msvc.h \ - atomic/solaris.h \ - atomic/gcc_builtins.h my_libwrap.h my_stacktrace.h + atomic/rwlock.h atomic/x86-gcc.h atomic/generic-msvc.h \ + atomic/gcc_builtins.h my_libwrap.h my_stacktrace.h \ + atomic/solaris.h EXTRA_DIST = mysql.h.pp mysql/plugin.h.pp probes_mysql.d.base diff --git a/include/atomic/gcc_builtins.h b/include/atomic/gcc_builtins.h index 01ebc38707e..100ff80cacd 100644 --- a/include/atomic/gcc_builtins.h +++ b/include/atomic/gcc_builtins.h @@ -18,7 +18,7 @@ #define make_atomic_add_body(S) \ v= __sync_fetch_and_add(a, v); -#define make_atomic_swap_body(S) \ +#define make_atomic_fas_body(S) \ v= __sync_lock_test_and_set(a, v); #define make_atomic_cas_body(S) \ int ## S sav; \ @@ -28,7 +28,10 @@ #ifdef MY_ATOMIC_MODE_DUMMY #define make_atomic_load_body(S) ret= *a #define make_atomic_store_body(S) *a= v +#define MY_ATOMIC_MODE "gcc-builtins-up" + #else +#define MY_ATOMIC_MODE "gcc-builtins-smp" #define make_atomic_load_body(S) \ ret= __sync_fetch_and_or(a, 0); #define make_atomic_store_body(S) \ diff --git a/include/atomic/generic-msvc.h b/include/atomic/generic-msvc.h new file mode 100644 index 00000000000..f1e1b0e88c9 --- /dev/null +++ b/include/atomic/generic-msvc.h @@ -0,0 +1,116 @@ +/* Copyright (C) 2006-2008 MySQL AB, 2008-2009 Sun Microsystems, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#ifndef _atomic_h_cleanup_ +#define _atomic_h_cleanup_ "atomic/generic-msvc.h" + +/* + We don't implement anything specific for MY_ATOMIC_MODE_DUMMY, always use + intrinsics. + 8 and 16-bit atomics are not implemented, but it can be done if necessary. +*/ +#undef MY_ATOMIC_HAS_8_16 + +/* + x86 compilers (both VS2003 or VS2005) never use instrinsics, but generate + function calls to kernel32 instead, even in the optimized build. + We force intrinsics as described in MSDN documentation for + _InterlockedCompareExchange. +*/ +#ifdef _M_IX86 + +#if (_MSC_VER >= 1500) +#include +#else +C_MODE_START +/*Visual Studio 2003 and earlier do not have prototypes for atomic intrinsics*/ +LONG _InterlockedExchange (LONG volatile *Target,LONG Value); +LONG _InterlockedCompareExchange (LONG volatile *Target, LONG Value, LONG Comp); +LONG _InterlockedExchangeAdd (LONG volatile *Addend, LONG Value); +C_MODE_END + +#pragma intrinsic(_InterlockedExchangeAdd) +#pragma intrinsic(_InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchange) +#endif + +#define InterlockedExchange _InterlockedExchange +#define InterlockedExchangeAdd _InterlockedExchangeAdd +#define InterlockedCompareExchange _InterlockedCompareExchange +/* + No need to do something special for InterlockedCompareExchangePointer + as it is a #define to InterlockedCompareExchange. The same applies to + InterlockedExchangePointer. +*/ +#endif /*_M_IX86*/ + +#define MY_ATOMIC_MODE "msvc-intrinsics" +#define IL_EXCHG_ADD32(X,Y) InterlockedExchangeAdd((volatile LONG *)(X),(Y)) +#define IL_COMP_EXCHG32(X,Y,Z) InterlockedCompareExchange((volatile LONG *)(X),(Y),(Z)) +#define IL_COMP_EXCHGptr InterlockedCompareExchangePointer +#define IL_EXCHG32(X,Y) InterlockedExchange((volatile LONG *)(X),(Y)) +#define IL_EXCHGptr InterlockedExchangePointer +#define make_atomic_add_body(S) \ + v= IL_EXCHG_ADD ## S (a, v) +#define make_atomic_cas_body(S) \ + int ## S initial_cmp= *cmp; \ + int ## S initial_a= IL_COMP_EXCHG ## S (a, set, initial_cmp); \ + if (!(ret= (initial_a == initial_cmp))) *cmp= initial_a; +#define make_atomic_swap_body(S) \ + v= IL_EXCHG ## S (a, v) +#define make_atomic_load_body(S) \ + ret= 0; /* avoid compiler warning */ \ + ret= IL_COMP_EXCHG ## S (a, ret, ret); + +/* + my_yield_processor (equivalent of x86 PAUSE instruction) should be used + to improve performance on hyperthreaded CPUs. Intel recommends to use it in + spin loops also on non-HT machines to reduce power consumption (see e.g + http://softwarecommunity.intel.com/articles/eng/2004.htm) + + Running benchmarks for spinlocks implemented with InterlockedCompareExchange + and YieldProcessor shows that much better performance is achieved by calling + YieldProcessor in a loop - that is, yielding longer. On Intel boxes setting + loop count in the range 200-300 brought best results. + */ +#ifndef YIELD_LOOPS +#define YIELD_LOOPS 200 +#endif + +static __inline int my_yield_processor() +{ + int i; + for(i=0; irw) -#define my_atomic_rwlock_init(name) pthread_rwlock_init(& (name)->rw, 0) -#define my_atomic_rwlock_rdlock(name) pthread_rwlock_rdlock(& (name)->rw) -#define my_atomic_rwlock_wrlock(name) pthread_rwlock_wrlock(& (name)->rw) -#define my_atomic_rwlock_rdunlock(name) pthread_rwlock_unlock(& (name)->rw) -#define my_atomic_rwlock_wrunlock(name) pthread_rwlock_unlock(& (name)->rw) -#define MY_ATOMIC_MODE "rwlocks" +#else /* not MY_ATOMIC_MODE_DUMMY */ + +typedef struct {pthread_mutex_t rw;} my_atomic_rwlock_t; + +#ifndef SAFE_MUTEX + +/* + we're using read-write lock macros but map them to mutex locks, and they're + faster. Still, having semantically rich API we can change the + underlying implementation, if necessary. +*/ +#define my_atomic_rwlock_destroy(name) pthread_mutex_destroy(& (name)->rw) +#define my_atomic_rwlock_init(name) pthread_mutex_init(& (name)->rw, 0) +#define my_atomic_rwlock_rdlock(name) pthread_mutex_lock(& (name)->rw) +#define my_atomic_rwlock_wrlock(name) pthread_mutex_lock(& (name)->rw) +#define my_atomic_rwlock_rdunlock(name) pthread_mutex_unlock(& (name)->rw) +#define my_atomic_rwlock_wrunlock(name) pthread_mutex_unlock(& (name)->rw) + +#else /* SAFE_MUTEX */ + +/* + SAFE_MUTEX pollutes the compiling name space with macros + that alter pthread_mutex_t, pthread_mutex_init, etc. + Atomic operations should never use the safe mutex wrappers. + Unfortunately, there is no way to have both: + - safe mutex macros expanding pthread_mutex_lock to safe_mutex_lock + - my_atomic macros expanding to unmodified pthread_mutex_lock + inlined in the same compilation unit. + So, in case of SAFE_MUTEX, a function call is required. + Given that SAFE_MUTEX is a debugging facility, + this extra function call is not a performance concern for + production builds. +*/ +C_MODE_START +extern void plain_pthread_mutex_init(safe_mutex_t *); +extern void plain_pthread_mutex_destroy(safe_mutex_t *); +extern void plain_pthread_mutex_lock(safe_mutex_t *); +extern void plain_pthread_mutex_unlock(safe_mutex_t *); +C_MODE_END + +#define my_atomic_rwlock_destroy(name) plain_pthread_mutex_destroy(&(name)->rw) +#define my_atomic_rwlock_init(name) plain_pthread_mutex_init(&(name)->rw) +#define my_atomic_rwlock_rdlock(name) plain_pthread_mutex_lock(&(name)->rw) +#define my_atomic_rwlock_wrlock(name) plain_pthread_mutex_lock(&(name)->rw) +#define my_atomic_rwlock_rdunlock(name) plain_pthread_mutex_unlock(&(name)->rw) +#define my_atomic_rwlock_wrunlock(name) plain_pthread_mutex_unlock(&(name)->rw) + +#endif /* SAFE_MUTEX */ + +#define MY_ATOMIC_MODE "mutex" +#ifndef MY_ATOMIC_MODE_RWLOCKS +#define MY_ATOMIC_MODE_RWLOCKS 1 +#endif #endif #define make_atomic_add_body(S) int ## S sav; sav= *a; *a+= v; v=sav; -#define make_atomic_swap_body(S) int ## S sav; sav= *a; *a= v; v=sav; +#define make_atomic_fas_body(S) int ## S sav; sav= *a; *a= v; v=sav; #define make_atomic_cas_body(S) if ((ret= (*a == *cmp))) *a= set; else *cmp=*a; #define make_atomic_load_body(S) ret= *a; #define make_atomic_store_body(S) *a= v; diff --git a/include/atomic/solaris.h b/include/atomic/solaris.h index 4c51253d2d5..45efd9faaba 100644 --- a/include/atomic/solaris.h +++ b/include/atomic/solaris.h @@ -186,25 +186,25 @@ my_atomic_storeptr(void * volatile *a, void *v) /* ------------------------------------------------------------------------ */ STATIC_INLINE int8 -my_atomic_swap8(int8 volatile *a, int8 v) +my_atomic_fas8(int8 volatile *a, int8 v) { return ((int8) atomic_swap_8((volatile uint8_t *)a, (uint8_t)v)); } STATIC_INLINE int16 -my_atomic_swap16(int16 volatile *a, int16 v) +my_atomic_fas16(int16 volatile *a, int16 v) { return ((int16) atomic_swap_16((volatile uint16_t *)a, (uint16_t)v)); } STATIC_INLINE int32 -my_atomic_swap32(int32 volatile *a, int32 v) +my_atomic_fas32(int32 volatile *a, int32 v) { return ((int32) atomic_swap_32((volatile uint32_t *)a, (uint32_t)v)); } STATIC_INLINE void * -my_atomic_swapptr(void * volatile *a, void *v) +my_atomic_fasptr(void * volatile *a, void *v) { return (atomic_swap_ptr(a, v)); } diff --git a/include/atomic/x86-gcc.h b/include/atomic/x86-gcc.h index c3029f9c1b4..59090bc26b7 100644 --- a/include/atomic/x86-gcc.h +++ b/include/atomic/x86-gcc.h @@ -22,10 +22,18 @@ architectures support double-word (128-bit) cas. */ -#ifdef MY_ATOMIC_NO_XADD -#define MY_ATOMIC_MODE "gcc-x86" LOCK "-no-xadd" +#ifdef __x86_64__ +# ifdef MY_ATOMIC_NO_XADD +# define MY_ATOMIC_MODE "gcc-amd64" LOCK_prefix "-no-xadd" +# else +# define MY_ATOMIC_MODE "gcc-amd64" LOCK_prefix +# endif #else -#define MY_ATOMIC_MODE "gcc-x86" LOCK +# ifdef MY_ATOMIC_NO_XADD +# define MY_ATOMIC_MODE "gcc-x86" LOCK_prefix "-no-xadd" +# else +# define MY_ATOMIC_MODE "gcc-x86" LOCK_prefix +# endif #endif /* fix -ansi errors while maintaining readability */ @@ -35,12 +43,12 @@ #ifndef MY_ATOMIC_NO_XADD #define make_atomic_add_body(S) \ - asm volatile (LOCK "; xadd %0, %1;" : "+r" (v) , "+m" (*a)) + asm volatile (LOCK_prefix "; xadd %0, %1;" : "+r" (v) , "+m" (*a)) #endif -#define make_atomic_swap_body(S) \ - asm volatile ("; xchg %0, %1;" : "+q" (v) , "+m" (*a)) +#define make_atomic_fas_body(S) \ + asm volatile ("xchg %0, %1;" : "+q" (v) , "+m" (*a)) #define make_atomic_cas_body(S) \ - asm volatile (LOCK "; cmpxchg %3, %0; setz %2;" \ + asm volatile (LOCK_prefix "; cmpxchg %3, %0; setz %2;" \ : "+m" (*a), "+a" (*cmp), "=q" (ret): "r" (set)) #ifdef MY_ATOMIC_MODE_DUMMY @@ -49,14 +57,14 @@ #else /* Actually 32-bit reads/writes are always atomic on x86 - But we add LOCK here anyway to force memory barriers + But we add LOCK_prefix here anyway to force memory barriers */ #define make_atomic_load_body(S) \ ret=0; \ - asm volatile (LOCK "; cmpxchg %2, %0" \ + asm volatile (LOCK_prefix "; cmpxchg %2, %0" \ : "+m" (*a), "+a" (ret): "r" (ret)) #define make_atomic_store_body(S) \ - asm volatile ("; xchg %0, %1;" : "+m" (*a) : "r" (v)) + asm volatile ("; xchg %0, %1;" : "+m" (*a), "+r" (v)) #endif #endif /* ATOMIC_X86_GCC_INCLUDED */ diff --git a/include/my_atomic.h b/include/my_atomic.h index f5da6e6a0d9..85cf87165fb 100644 --- a/include/my_atomic.h +++ b/include/my_atomic.h @@ -16,9 +16,51 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +/* + This header defines five atomic operations: + + my_atomic_add#(&var, what) + add 'what' to *var, and return the old value of *var + + my_atomic_fas#(&var, what) + 'Fetch And Store' + store 'what' in *var, and return the old value of *var + + my_atomic_cas#(&var, &old, new) + 'Compare And Swap' + if *var is equal to *old, then store 'new' in *var, and return TRUE + otherwise store *var in *old, and return FALSE + + my_atomic_load#(&var) + return *var + + my_atomic_store#(&var, what) + store 'what' in *var + + '#' is substituted by a size suffix - 8, 16, 32, or ptr + (e.g. my_atomic_add8, my_atomic_fas32, my_atomic_casptr). + + NOTE This operations are not always atomic, so they always must be + enclosed in my_atomic_rwlock_rdlock(lock)/my_atomic_rwlock_rdunlock(lock) + or my_atomic_rwlock_wrlock(lock)/my_atomic_rwlock_wrunlock(lock). + Hint: if a code block makes intensive use of atomic ops, it make sense + to take/release rwlock once for the whole block, not for every statement. + + On architectures where these operations are really atomic, rwlocks will + be optimized away. + 8- and 16-bit atomics aren't implemented for windows (see generic-msvc.h), + but can be added, if necessary. +*/ + #ifndef my_atomic_rwlock_init #define intptr void * +/** + On most platforms we implement 8-bit, 16-bit, 32-bit and "pointer" + operations. Thus the symbol below is defined by default; platforms + where we leave out 8-bit or 16-bit operations should undefine it. +*/ +#define MY_ATOMIC_HAS_8_16 1 #ifndef MY_ATOMIC_MODE_RWLOCKS /* @@ -27,126 +69,225 @@ #include "atomic/nolock.h" #endif -#ifndef MY_ATOMIC_NOLOCK -/* - * Have to use rw-locks for atomic ops - */ +#ifndef make_atomic_cas_body +/* nolock.h was not able to generate even a CAS function, fall back */ #include "atomic/rwlock.h" -#endif - -#ifndef MY_ATOMICS_MADE - +#else +/* define missing functions by using the already generated ones */ #ifndef make_atomic_add_body -#define make_atomic_add_body(S) \ +#define make_atomic_add_body(S) \ int ## S tmp=*a; \ while (!my_atomic_cas ## S(a, &tmp, tmp+v)); \ v=tmp; #endif +#ifndef make_atomic_fas_body +#define make_atomic_fas_body(S) \ + int ## S tmp=*a; \ + while (!my_atomic_cas ## S(a, &tmp, v)); \ + v=tmp; +#endif +#ifndef make_atomic_load_body +#define make_atomic_load_body(S) \ + ret= 0; /* avoid compiler warning */ \ + (void)(my_atomic_cas ## S(a, &ret, ret)); +#endif +#ifndef make_atomic_store_body +#define make_atomic_store_body(S) \ + (void)(my_atomic_fas ## S (a, v)); +#endif +#endif + +/* + transparent_union doesn't work in g++ + Bug ? + + Darwin's gcc doesn't want to put pointers in a transparent_union + when built with -arch ppc64. Complains: + warning: 'transparent_union' attribute ignored +*/ +#if defined(__GNUC__) && !defined(__cplusplus) && \ + ! (defined(__APPLE__) && defined(_ARCH_PPC64)) +/* + we want to be able to use my_atomic_xxx functions with + both signed and unsigned integers. But gcc will issue a warning + "passing arg N of `my_atomic_XXX' as [un]signed due to prototype" + if the signedness of the argument doesn't match the prototype, or + "pointer targets in passing argument N of my_atomic_XXX differ in signedness" + if int* is used where uint* is expected (or vice versa). + Let's shut these warnings up +*/ +#define make_transparent_unions(S) \ + typedef union { \ + int ## S i; \ + uint ## S u; \ + } U_ ## S __attribute__ ((transparent_union)); \ + typedef union { \ + int ## S volatile *i; \ + uint ## S volatile *u; \ + } Uv_ ## S __attribute__ ((transparent_union)); +#define uintptr intptr +make_transparent_unions(8) +make_transparent_unions(16) +make_transparent_unions(32) +make_transparent_unions(ptr) +#undef uintptr +#undef make_transparent_unions +#define a U_a.i +#define cmp U_cmp.i +#define v U_v.i +#define set U_set.i +#else +#define U_8 int8 +#define U_16 int16 +#define U_32 int32 +#define U_ptr intptr +#define Uv_8 int8 +#define Uv_16 int16 +#define Uv_32 int32 +#define Uv_ptr intptr +#define U_a volatile *a +#define U_cmp *cmp +#define U_v v +#define U_set set +#endif /* __GCC__ transparent_union magic */ #ifdef HAVE_INLINE -#define make_atomic_add(S) \ -STATIC_INLINE int ## S my_atomic_add ## S( \ - int ## S volatile *a, int ## S v) \ -{ \ - make_atomic_add_body(S); \ - return v; \ +#define make_atomic_cas(S) \ +STATIC_INLINE int my_atomic_cas ## S(Uv_ ## S U_a, \ + Uv_ ## S U_cmp, U_ ## S U_set) \ +{ \ + int8 ret; \ + make_atomic_cas_body(S); \ + return ret; \ } -#define make_atomic_swap(S) \ -STATIC_INLINE int ## S my_atomic_swap ## S( \ - int ## S volatile *a, int ## S v) \ -{ \ - make_atomic_swap_body(S); \ - return v; \ +#define make_atomic_add(S) \ +STATIC_INLINE int ## S my_atomic_add ## S( \ + Uv_ ## S U_a, U_ ## S U_v) \ +{ \ + make_atomic_add_body(S); \ + return v; \ } -#define make_atomic_cas(S) \ -STATIC_INLINE int my_atomic_cas ## S(int ## S volatile *a, \ - int ## S *cmp, int ## S set) \ -{ \ - int8 ret; \ - make_atomic_cas_body(S); \ - return ret; \ +#define make_atomic_fas(S) \ +STATIC_INLINE int ## S my_atomic_fas ## S( \ + Uv_ ## S U_a, U_ ## S U_v) \ +{ \ + make_atomic_fas_body(S); \ + return v; \ } -#define make_atomic_load(S) \ -STATIC_INLINE int ## S my_atomic_load ## S(int ## S volatile *a) \ -{ \ - int ## S ret; \ - make_atomic_load_body(S); \ - return ret; \ +#define make_atomic_load(S) \ +STATIC_INLINE int ## S my_atomic_load ## S(Uv_ ## S U_a) \ +{ \ + int ## S ret; \ + make_atomic_load_body(S); \ + return ret; \ } -#define make_atomic_store(S) \ -STATIC_INLINE void my_atomic_store ## S( \ - int ## S volatile *a, int ## S v) \ -{ \ - make_atomic_store_body(S); \ +#define make_atomic_store(S) \ +STATIC_INLINE void my_atomic_store ## S( \ + Uv_ ## S U_a, U_ ## S U_v) \ +{ \ + make_atomic_store_body(S); \ } #else /* no inline functions */ -#define make_atomic_add(S) \ -extern int ## S my_atomic_add ## S(int ## S volatile *a, int ## S v); +#define make_atomic_add(S) \ +extern int ## S my_atomic_add ## S(Uv_ ## S U_a, U_ ## S U_v); -#define make_atomic_swap(S) \ -extern int ## S my_atomic_swap ## S(int ## S volatile *a, int ## S v); +#define make_atomic_fas(S) \ +extern int ## S my_atomic_fas ## S(Uv_ ## S U_a, U_ ## S U_v); -#define make_atomic_cas(S) \ -extern int my_atomic_cas ## S(int ## S volatile *a, int ## S *cmp, int ## S set); +#define make_atomic_cas(S) \ +extern int my_atomic_cas ## S(Uv_ ## S U_a, Uv_ ## S U_cmp, U_ ## S U_set); -#define make_atomic_load(S) \ -extern int ## S my_atomic_load ## S(int ## S volatile *a); +#define make_atomic_load(S) \ +extern int ## S my_atomic_load ## S(Uv_ ## S U_a); -#define make_atomic_store(S) \ -extern void my_atomic_store ## S(int ## S volatile *a, int ## S v); +#define make_atomic_store(S) \ +extern void my_atomic_store ## S(Uv_ ## S U_a, U_ ## S U_v); #endif /* HAVE_INLINE */ -make_atomic_cas( 8) +#ifdef MY_ATOMIC_HAS_8_16 +make_atomic_cas(8) make_atomic_cas(16) +#endif make_atomic_cas(32) make_atomic_cas(ptr) -make_atomic_add( 8) +#ifdef MY_ATOMIC_HAS_8_16 +make_atomic_add(8) make_atomic_add(16) +#endif make_atomic_add(32) -make_atomic_load( 8) +#ifdef MY_ATOMIC_HAS_8_16 +make_atomic_load(8) make_atomic_load(16) +#endif make_atomic_load(32) make_atomic_load(ptr) -make_atomic_store( 8) +#ifdef MY_ATOMIC_HAS_8_16 +make_atomic_fas(8) +make_atomic_fas(16) +#endif +make_atomic_fas(32) +make_atomic_fas(ptr) + +#ifdef MY_ATOMIC_HAS_8_16 +make_atomic_store(8) make_atomic_store(16) +#endif make_atomic_store(32) make_atomic_store(ptr) -make_atomic_swap( 8) -make_atomic_swap(16) -make_atomic_swap(32) -make_atomic_swap(ptr) - -#undef make_atomic_add -#undef make_atomic_cas -#undef make_atomic_load -#undef make_atomic_store -#undef make_atomic_swap -#undef make_atomic_add_body -#undef make_atomic_cas_body -#undef make_atomic_load_body -#undef make_atomic_store_body -#undef make_atomic_swap_body -#undef intptr - -#endif /* MY_ATOMICS_MADE */ - #ifdef _atomic_h_cleanup_ #include _atomic_h_cleanup_ #undef _atomic_h_cleanup_ #endif +#undef U_8 +#undef U_16 +#undef U_32 +#undef U_ptr +#undef Uv_8 +#undef Uv_16 +#undef Uv_32 +#undef Uv_ptr +#undef a +#undef cmp +#undef v +#undef set +#undef U_a +#undef U_cmp +#undef U_v +#undef U_set +#undef make_atomic_add +#undef make_atomic_cas +#undef make_atomic_load +#undef make_atomic_store +#undef make_atomic_fas +#undef make_atomic_add_body +#undef make_atomic_cas_body +#undef make_atomic_load_body +#undef make_atomic_store_body +#undef make_atomic_fas_body +#undef intptr + +/* + the macro below defines (as an expression) the code that + will be run in spin-loops. Intel manuals recummend to have PAUSE there. + It is expected to be defined in include/atomic/ *.h files +*/ +#ifndef LF_BACKOFF +#define LF_BACKOFF (1) +#endif + #define MY_ATOMIC_OK 0 #define MY_ATOMIC_NOT_1CPU 1 extern int my_atomic_initialize(); diff --git a/mysys/my_atomic.c b/mysys/my_atomic.c index aa04d55f624..6bc76f0de3c 100644 --- a/mysys/my_atomic.c +++ b/mysys/my_atomic.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2006 MySQL AB +/* Copyright (C) 2006 MySQL AB, 2008-2009 Sun Microsystems, Inc This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -14,7 +14,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include -#include +#include #ifndef HAVE_INLINE /* the following will cause all inline functions to be instantiated */ @@ -43,3 +43,32 @@ int my_atomic_initialize() #endif } +#ifdef SAFE_MUTEX +#undef pthread_mutex_init +#undef pthread_mutex_destroy +#undef pthread_mutex_lock +#undef pthread_mutex_unlock + +void plain_pthread_mutex_init(safe_mutex_t *m) +{ + pthread_mutex_init(& m->mutex, NULL); +} + +void plain_pthread_mutex_destroy(safe_mutex_t *m) +{ + pthread_mutex_destroy(& m->mutex); +} + +void plain_pthread_mutex_lock(safe_mutex_t *m) +{ + pthread_mutex_lock(& m->mutex); +} + +void plain_pthread_mutex_unlock(safe_mutex_t *m) +{ + pthread_mutex_unlock(& m->mutex); +} + +#endif + + diff --git a/mysys/my_getncpus.c b/mysys/my_getncpus.c index 82e87dee2e4..5be961e3bc9 100644 --- a/mysys/my_getncpus.c +++ b/mysys/my_getncpus.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2006 MySQL AB +/* Copyright (C) 2006 MySQL AB, 2008-2009 Sun Microsystems, Inc This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -16,24 +16,34 @@ /* get the number of (online) CPUs */ #include "mysys_priv.h" +#ifdef HAVE_UNISTD_H #include +#endif static int ncpus=0; -#ifdef _SC_NPROCESSORS_ONLN int my_getncpus() { if (!ncpus) + { +#ifdef _SC_NPROCESSORS_ONLN ncpus= sysconf(_SC_NPROCESSORS_ONLN); +#elif defined(__WIN__) + SYSTEM_INFO sysinfo; + + /* + * We are not calling GetNativeSystemInfo here because (1) we + * don't believe that they return different values for number + * of processors and (2) if WOW64 limits processors for Win32 + * then we don't want to try to override that. + */ + GetSystemInfo(&sysinfo); + + ncpus= sysinfo.dwNumberOfProcessors; +#else +/* unknown so play safe: assume SMP and forbid uniprocessor build */ + ncpus= 2; +#endif + } return ncpus; } - -#else -/* unknown */ -int my_getncpus() -{ - return 2; -} - -#endif - diff --git a/unittest/mysys/Makefile.am b/unittest/mysys/Makefile.am index 56c65d71396..6e8058c4d9b 100644 --- a/unittest/mysys/Makefile.am +++ b/unittest/mysys/Makefile.am @@ -16,6 +16,8 @@ AM_CPPFLAGS = @ZLIB_INCLUDES@ -I$(top_builddir)/include AM_CPPFLAGS += -I$(top_srcdir)/include -I$(top_srcdir)/unittest/mytap +noinst_HEADERS = thr_template.c + LDADD = $(top_builddir)/unittest/mytap/libmytap.a \ $(top_builddir)/mysys/libmysys.a \ $(top_builddir)/dbug/libdbug.a \ diff --git a/unittest/mysys/my_atomic-t.c b/unittest/mysys/my_atomic-t.c index f2bcd360508..1a558e8cb73 100644 --- a/unittest/mysys/my_atomic-t.c +++ b/unittest/mysys/my_atomic-t.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2006 MySQL AB +/* Copyright (C) 2006-2008 MySQL AB, 2008 Sun Microsystems, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -13,10 +13,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#include -#include -#include -#include +#include "thr_template.c" /* at least gcc 3.4.5 and 3.4.6 (but not 3.2.3) on RHEL */ #if __GNUC__ == 3 && __GNUC_MINOR__ == 4 @@ -25,181 +22,125 @@ #define GCC_BUG_WORKAROUND #endif -int32 a32,b32,c32; +volatile uint32 b32; +volatile int32 c32; my_atomic_rwlock_t rwl; -pthread_attr_t thr_attr; -pthread_mutex_t mutex; -pthread_cond_t cond; -int N; - /* add and sub a random number in a loop. Must get 0 at the end */ -pthread_handler_t test_atomic_add_handler(void *arg) +pthread_handler_t test_atomic_add(void *arg) { - int m=*(int *)arg; + int m= (*(int *)arg)/2; GCC_BUG_WORKAROUND int32 x; - for (x=((int)((long)(&m))); m ; m--) + for (x= ((int)(intptr)(&m)); m ; m--) { - x=x*m+0x87654321; + x= (x*m+0x87654321) & INT_MAX32; my_atomic_rwlock_wrlock(&rwl); - my_atomic_add32(&a32, x); + my_atomic_add32(&bad, x); my_atomic_rwlock_wrunlock(&rwl); my_atomic_rwlock_wrlock(&rwl); - my_atomic_add32(&a32, -x); + my_atomic_add32(&bad, -x); my_atomic_rwlock_wrunlock(&rwl); } pthread_mutex_lock(&mutex); - N--; - if (!N) pthread_cond_signal(&cond); + if (!--running_threads) pthread_cond_signal(&cond); pthread_mutex_unlock(&mutex); return 0; } /* 1. generate thread number 0..N-1 from b32 - 2. add it to a32 + 2. add it to bad 3. swap thread numbers in c32 4. (optionally) one more swap to avoid 0 as a result - 5. subtract result from a32 - must get 0 in a32 at the end + 5. subtract result from bad + must get 0 in bad at the end */ -pthread_handler_t test_atomic_swap_handler(void *arg) +pthread_handler_t test_atomic_fas(void *arg) { - int m=*(int *)arg; - int32 x; + int m= *(int *)arg; + int32 x; my_atomic_rwlock_wrlock(&rwl); - x=my_atomic_add32(&b32, 1); + x= my_atomic_add32(&b32, 1); my_atomic_rwlock_wrunlock(&rwl); my_atomic_rwlock_wrlock(&rwl); - my_atomic_add32(&a32, x); + my_atomic_add32(&bad, x); my_atomic_rwlock_wrunlock(&rwl); for (; m ; m--) { my_atomic_rwlock_wrlock(&rwl); - x=my_atomic_swap32(&c32, x); + x= my_atomic_fas32(&c32, x); my_atomic_rwlock_wrunlock(&rwl); } if (!x) { my_atomic_rwlock_wrlock(&rwl); - x=my_atomic_swap32(&c32, x); + x= my_atomic_fas32(&c32, x); my_atomic_rwlock_wrunlock(&rwl); } my_atomic_rwlock_wrlock(&rwl); - my_atomic_add32(&a32, -x); + my_atomic_add32(&bad, -x); my_atomic_rwlock_wrunlock(&rwl); pthread_mutex_lock(&mutex); - N--; - if (!N) pthread_cond_signal(&cond); + if (!--running_threads) pthread_cond_signal(&cond); pthread_mutex_unlock(&mutex); return 0; } /* - same as test_atomic_add_handler, but my_atomic_add32 is emulated with - (slower) my_atomic_cas32 + same as test_atomic_add, but my_atomic_add32 is emulated with + my_atomic_cas32 - notice that the slowdown is proportional to the + number of CPUs */ -pthread_handler_t test_atomic_cas_handler(void *arg) +pthread_handler_t test_atomic_cas(void *arg) { - int m=*(int *)arg, ok; - GCC_BUG_WORKAROUND int32 x,y; - for (x=((int)((long)(&m))); m ; m--) + int m= (*(int *)arg)/2, ok= 0; + GCC_BUG_WORKAROUND int32 x, y; + for (x= ((int)(intptr)(&m)); m ; m--) { my_atomic_rwlock_wrlock(&rwl); - y=my_atomic_load32(&a32); + y= my_atomic_load32(&bad); my_atomic_rwlock_wrunlock(&rwl); - - x=x*m+0x87654321; + x= (x*m+0x87654321) & INT_MAX32; do { my_atomic_rwlock_wrlock(&rwl); - ok=my_atomic_cas32(&a32, &y, y+x); + ok= my_atomic_cas32(&bad, &y, (uint32)y+x); my_atomic_rwlock_wrunlock(&rwl); - } while (!ok); + } while (!ok) ; do { my_atomic_rwlock_wrlock(&rwl); - ok=my_atomic_cas32(&a32, &y, y-x); + ok= my_atomic_cas32(&bad, &y, y-x); my_atomic_rwlock_wrunlock(&rwl); - } while (!ok); + } while (!ok) ; } pthread_mutex_lock(&mutex); - N--; - if (!N) pthread_cond_signal(&cond); + if (!--running_threads) pthread_cond_signal(&cond); pthread_mutex_unlock(&mutex); return 0; } -void test_atomic(const char *test, pthread_handler handler, int n, int m) + +void do_tests() { - pthread_t t; - ulonglong now=my_getsystime(); - - a32= 0; - b32= 0; - c32= 0; - - diag("Testing %s with %d threads, %d iterations... ", test, n, m); - for (N=n ; n ; n--) - { - if (pthread_create(&t, &thr_attr, handler, &m) != 0) - { - diag("Could not create thread"); - a32= 1; - goto err; - } - } - - pthread_mutex_lock(&mutex); - while (N) - pthread_cond_wait(&cond, &mutex); - pthread_mutex_unlock(&mutex); - now=my_getsystime()-now; -err: - ok(a32 == 0, "tested %s in %g secs", test, ((double)now)/1e7); -} - -int main() -{ - int err; - MY_INIT("my_atomic-t.c"); - - diag("N CPUs: %d", my_getncpus()); - err= my_atomic_initialize(); - plan(4); - ok(err == 0, "my_atomic_initialize() returned %d", err); - pthread_attr_init(&thr_attr); - pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED); - pthread_mutex_init(&mutex, 0); - pthread_cond_init(&cond, 0); + bad= my_atomic_initialize(); + ok(!bad, "my_atomic_initialize() returned %d", bad); + my_atomic_rwlock_init(&rwl); -#ifdef HPUX11 -#define CYCLES 1000 -#else -#define CYCLES 10000 -#endif -#define THREADS 100 - test_atomic("my_atomic_add32", test_atomic_add_handler, THREADS, CYCLES); - test_atomic("my_atomic_swap32", test_atomic_swap_handler, THREADS, CYCLES); - test_atomic("my_atomic_cas32", test_atomic_cas_handler, THREADS, CYCLES); - /* - workaround until we know why it crashes randomly on some machine - (BUG#22320). - */ - sleep(2); + b32= c32= 0; + test_concurrently("my_atomic_add32", test_atomic_add, THREADS, CYCLES); + b32= c32= 0; + test_concurrently("my_atomic_fas32", test_atomic_fas, THREADS, CYCLES); + b32= c32= 0; + test_concurrently("my_atomic_cas32", test_atomic_cas, THREADS, CYCLES); - pthread_mutex_destroy(&mutex); - pthread_cond_destroy(&cond); - pthread_attr_destroy(&thr_attr); my_atomic_rwlock_destroy(&rwl); - return exit_status(); } - diff --git a/unittest/mysys/thr_template.c b/unittest/mysys/thr_template.c new file mode 100644 index 00000000000..1ac03e474fd --- /dev/null +++ b/unittest/mysys/thr_template.c @@ -0,0 +1,92 @@ +/* Copyright (C) 2006-2008 MySQL AB, 2008 Sun Microsystems, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include +#include +#include +#include + +volatile uint32 bad; +pthread_attr_t thr_attr; +pthread_mutex_t mutex; +pthread_cond_t cond; +uint running_threads; + +void do_tests(); + +void test_concurrently(const char *test, pthread_handler handler, int n, int m) +{ + pthread_t t; + ulonglong now= my_getsystime(); + + bad= 0; + + diag("Testing %s with %d threads, %d iterations... ", test, n, m); + for (running_threads= n ; n ; n--) + { + if (pthread_create(&t, &thr_attr, handler, &m) != 0) + { + diag("Could not create thread"); + abort(); + } + } + pthread_mutex_lock(&mutex); + while (running_threads) + pthread_cond_wait(&cond, &mutex); + pthread_mutex_unlock(&mutex); + + now= my_getsystime()-now; + ok(!bad, "tested %s in %g secs (%d)", test, ((double)now)/1e7, bad); +} + +int main(int argc __attribute__((unused)), char **argv) +{ + MY_INIT("thd_template"); + + if (argv[1] && *argv[1]) + DBUG_SET_INITIAL(argv[1]); + + pthread_mutex_init(&mutex, 0); + pthread_cond_init(&cond, 0); + pthread_attr_init(&thr_attr); + pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED); + +#ifdef MY_ATOMIC_MODE_RWLOCKS +#if defined(HPUX11) || defined(__POWERPC__) /* showed to be very slow (scheduler-related) */ +#define CYCLES 300 +#else +#define CYCLES 3000 +#endif +#else +#define CYCLES 3000 +#endif +#define THREADS 30 + + diag("N CPUs: %d, atomic ops: %s", my_getncpus(), MY_ATOMIC_MODE); + + do_tests(); + + /* + workaround until we know why it crashes randomly on some machine + (BUG#22320). + */ + sleep(2); + pthread_mutex_destroy(&mutex); + pthread_cond_destroy(&cond); + pthread_attr_destroy(&thr_attr); + my_end(0); + return exit_status(); +} +