Update.

2003-04-04  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/pthread/createthread.c (create_thread): Add some more
	comments explaining when to set multiple_threads and when not.

	* pthreadP.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
	THREAD_ATOMIC_BIT_SET if not already defined.
	* sysdeps/i386/tls.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
	THREAD_ATOMIC_BIT_SET:
	* sysdeps/x86_64/tls.h: Likewise.
	* cleanup_defer.c (_pthread_cleanup_push_defer): Rewrite to use
	THREAD_ATOMIC_CMPXCHG_VAL.
	(_pthread_cleanup_pop_restore): Likewise.
	* cancellation.c (__pthread_enable_asynccancel): Likewise.
	(__pthread_enable_asynccancel_2): Likewise.
	(__pthread_disable_asynccancel): Likewise.
	* libc-cancellation.c (__libc_enable_asynccancel): Likewise.
	(__libc_disable_asynccancel): Likewise.
	* init.c (sigcancel_handler): Likewise.
	* pthread_setcancelstate.c (__pthread_setcancelstate): Likewise.
	* pthread_setcanceltype.c (__pthread_setcanceltype): Likewise.

nptl/ChangeLog

@@ -1,3 +1,25 @@
+2003-04-04  Ulrich Drepper  <drepper@redhat.com>
+
+	* sysdeps/pthread/createthread.c (create_thread): Add some more
+	comments explaining when to set multiple_threads and when not.
+
+	* pthreadP.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
+	THREAD_ATOMIC_BIT_SET if not already defined.
+	* sysdeps/i386/tls.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
+	THREAD_ATOMIC_BIT_SET:
+	* sysdeps/x86_64/tls.h: Likewise.
+	* cleanup_defer.c (_pthread_cleanup_push_defer): Rewrite to use
+	THREAD_ATOMIC_CMPXCHG_VAL.
+	(_pthread_cleanup_pop_restore): Likewise.
+	* cancellation.c (__pthread_enable_asynccancel): Likewise.
+	(__pthread_enable_asynccancel_2): Likewise.
+	(__pthread_disable_asynccancel): Likewise.
+	* libc-cancellation.c (__libc_enable_asynccancel): Likewise.
+	(__libc_disable_asynccancel): Likewise.
+	* init.c (sigcancel_handler): Likewise.
+	* pthread_setcancelstate.c (__pthread_setcancelstate): Likewise.
+	* pthread_setcanceltype.c (__pthread_setcanceltype): Likewise.
+
 2003-04-03  Ulrich Drepper  <drepper@redhat.com>
 
 	* init.c (sigcancel_handler): Don't set EXITING_BIT here.

nptl/cancellation.c

@@ -20,7 +20,6 @@
 #include <setjmp.h>
 #include <stdlib.h>
 #include "pthreadP.h"
-#include "atomic.h"
 
 
 /* The next two functions are similar to pthread_setcanceltype() but
@@ -31,18 +30,18 @@ attribute_hidden
 __pthread_enable_asynccancel (void)
 {
   struct pthread *self = THREAD_SELF;
-  int oldval;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
     {
-      oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval | CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
-						  newval, oldval))
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    {
@@ -52,6 +51,9 @@ __pthread_enable_asynccancel (void)
 
 	  break;
 	}
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
 
   return oldval;
@@ -63,17 +65,22 @@ internal_function attribute_hidden
 __pthread_enable_asynccancel_2 (int *oldvalp)
 {
   struct pthread *self = THREAD_SELF;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
     {
-      int oldval = *oldvalp = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval | CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
+      /* We have to store the value before enablying asynchronous
-						  newval, oldval))
+	 cancellation.  */
+      *oldvalp = oldval;
+
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    {
@@ -97,17 +104,21 @@ __pthread_disable_asynccancel (int oldtype)
     return;
 
   struct pthread *self = THREAD_SELF;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
     {
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval & ~CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
-						  newval, oldval))
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	break;
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
 }

nptl/cleanup_defer.c

@@ -18,7 +18,6 @@
    02111-1307 USA.  */
 
 #include "pthreadP.h"
-#include <atomic.h>
 
 
 void
@@ -37,12 +36,18 @@ _pthread_cleanup_push_defer (buffer, routine, arg)
 
   /* Disable asynchronous cancellation for now.  */
   if (__builtin_expect (cancelhandling & CANCELTYPE_BITMASK, 0))
+    while (1)
       {
-      while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
+	int newval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
 						cancelhandling
 						& ~CANCELTYPE_BITMASK,
-						   cancelhandling))
+						cancelhandling);
-	cancelhandling = self->cancelhandling;
+	if (__builtin_expect (newval == cancelhandling, 1))
+	  /* Successfully replaced the value.  */
+	  break;
+
+	/* Prepare for the next round.  */
+	cancelhandling = newval;
       }
 
   buffer->__canceltype = (cancelhandling & CANCELTYPE_BITMASK
@@ -53,6 +58,7 @@ _pthread_cleanup_push_defer (buffer, routine, arg)
 }
 strong_alias (_pthread_cleanup_push_defer, __pthread_cleanup_push_defer)
 
+
 void
 _pthread_cleanup_pop_restore (buffer, execute)
      struct _pthread_cleanup_buffer *buffer;
@@ -67,11 +73,19 @@ _pthread_cleanup_pop_restore (buffer, execute)
       && ((cancelhandling = THREAD_GETMEM (self, cancelhandling))
 	  & CANCELTYPE_BITMASK) == 0)
     {
-      while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
+      while (1)
+	{
+	  int newval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
 						  cancelhandling
 						  | CANCELTYPE_BITMASK,
-						   cancelhandling))
+						  cancelhandling);
-	cancelhandling = self->cancelhandling;
+	  if (__builtin_expect (newval == cancelhandling, 1))
+	    /* Successfully replaced the value.  */
+	    break;
+
+	  /* Prepare for the next round.  */
+	  cancelhandling = newval;
+	}
 
       CANCELLATION_P (self);
     }

nptl/init.c

@@ -135,20 +135,21 @@ sigcancel_handler (int sig __attribute ((unused)))
 {
   struct pthread *self = THREAD_SELF;
 
+  int oldval = THREAD_GETMEM (self, cancelhandling);
   while (1)
     {
       /* We are canceled now.  When canceled by another thread this flag
 	 is already set but if the signal is directly send (internally or
 	 from another process) is has to be done here.  */
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval | CANCELING_BITMASK | CANCELED_BITMASK;
 
       if (oldval == newval || (oldval & EXITING_BITMASK) != 0)
 	/* Already canceled or exiting.  */
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
-						  newval, oldval))
+					      oldval);
+      if (curval == oldval)
 	{
 	  /* Set the return value.  */
 	  THREAD_SETMEM (self, result, PTHREAD_CANCELED);
@@ -160,6 +161,8 @@ sigcancel_handler (int sig __attribute ((unused)))
 
 	  break;
 	}
+
+      oldval = curval;
     }
 }
 

nptl/libc-cancellation.c

@@ -33,13 +33,11 @@ attribute_hidden
 __libc_enable_asynccancel (void)
 {
   struct pthread *self = THREAD_SELF;
-  int oldval;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
-  int newval;
 
-  do
+  while (1)
     {
-      oldval = THREAD_GETMEM (self, cancelhandling);
+      int newval = oldval | CANCELTYPE_BITMASK;
-      newval = oldval | CANCELTYPE_BITMASK;
 
       if (__builtin_expect ((oldval & CANCELED_BITMASK) != 0, 0))
 	{
@@ -47,10 +45,14 @@ __libc_enable_asynccancel (void)
 	  if ((oldval & EXITING_BITMASK) != 0)
 	    break;
 
-	  if (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
+	  int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
-						    newval, oldval))
+						  newval, oldval);
+	  if (__builtin_expect (curval != oldval, 0))
+	    {
 	      /* Somebody else modified the word, try again.  */
+	      oldval = curval;
 	      continue;
+	    }
 
 	  THREAD_SETMEM (self, result, PTHREAD_CANCELED);
 
@@ -58,9 +60,15 @@ __libc_enable_asynccancel (void)
 
 	  /* NOTREACHED */
 	}
+
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
+	break;
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
-  while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-					       newval, oldval));
 
   return oldval;
 }
@@ -76,19 +84,23 @@ __libc_disable_asynccancel (int oldtype)
     return;
 
   struct pthread *self = THREAD_SELF;
-  int oldval;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
-  int newval;
 
-  do
+  while (1)
     {
-      oldval = THREAD_GETMEM (self, cancelhandling);
+      int newval = oldval & ~CANCELTYPE_BITMASK;
-      newval = oldval & ~CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
+
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
+	break;
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
-  while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling, newval,
-					       oldval));
 }
 
 #endif

nptl/pthread_setcancelstate.c

@@ -34,9 +34,9 @@ __pthread_setcancelstate (state, oldstate)
 
   self = THREAD_SELF;
 
+  int oldval = THREAD_GETMEM (self, cancelhandling);
   while (1)
     {
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = (state == PTHREAD_CANCEL_DISABLE
 		    ? oldval | CANCELSTATE_BITMASK
 		    : oldval & ~CANCELSTATE_BITMASK);
@@ -54,14 +54,18 @@ __pthread_setcancelstate (state, oldstate)
 
       /* Update the cancel handling word.  This has to be done
 	 atomically since other bits could be modified as well.  */
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
-						  newval, oldval))
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    __do_cancel ();
 
 	  break;
 	}
+
+      /* Prepare for the next round.  */
+      oldval = curval;
     }
 
   return 0;

nptl/pthread_setcanceltype.c

@@ -34,9 +34,9 @@ __pthread_setcanceltype (type, oldtype)
 
   self = THREAD_SELF;
 
+  int oldval = THREAD_GETMEM (self, cancelhandling);
   while (1)
     {
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = (type == PTHREAD_CANCEL_ASYNCHRONOUS
 		    ? oldval | CANCELTYPE_BITMASK
 		    : oldval & ~CANCELTYPE_BITMASK);
@@ -54,8 +54,9 @@ __pthread_setcanceltype (type, oldtype)
 
       /* Update the cancel handling word.  This has to be done
 	 atomically since other bits could be modified as well.  */
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
-						  newval, oldval))
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    {
@@ -65,6 +66,9 @@ __pthread_setcanceltype (type, oldtype)
 
 	  break;
 	}
+
+      /* Prepare for the next round.  */
+      oldval = curval;
     }
 
   return 0;

nptl/sysdeps/i386/tls.h

@@ -169,6 +169,14 @@ union user_desc_init
 # define INIT_SYSINFO
 #endif
 
+#ifndef LOCK
+# ifdef UP
+#  define LOCK  /* nothing */
+# else
+#  define LOCK "lock;"
+# endif
+#endif
+
 /* Code to initially initialize the thread pointer.  This might need
    special attention since 'errno' is not yet available and if the
    operation can cause a failure 'errno' must not be touched.  */
@@ -352,6 +360,32 @@ union user_desc_init
        }})
 
 
+/* Atomic compare and exchange on TLS, returning old value.  */
+#define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \
+  ({ __typeof (descr->member) __ret;					      \
+     __typeof (oldval) __old = (oldval);				      \
+     if (sizeof (descr->member) == 4)					      \
+       asm volatile (LOCK "cmpxchgl %2, %%gs:%P3"			      \
+		     : "=a" (__ret)					      \
+		     : "0" (__old), "r" (newval),			      \
+		       "i" (offsetof (struct pthread, member)));	      \
+     else								      \
+       /* Not necessary for other sizes in the moment.  */		      \
+       abort ();							      \
+     __ret; })
+
+
+/* Atomic set bit.  */
+#define THREAD_ATOMIC_BIT_SET(descr, member, bit) \
+  (void) ({ if (sizeof ((descr)->member) == 4)				      \
+	      asm volatile (LOCK "orl %1, %%gs:%P0"			      \
+			    :: "i" (offsetof (struct pthread, member)),	      \
+			       "ir" (1 << (bit)));			      \
+	    else							      \
+	      /* Not necessary for other sizes in the moment.  */	      \
+	      abort (); })
+
+
 /* Call the user-provided thread function.  */
 #define CALL_THREAD_FCT(descr) \
   ({ void *__res;							      \

nptl/sysdeps/pthread/createthread.c

@@ -83,11 +83,11 @@ create_thread (struct pthread *pd, STACK_VARIABLES_PARMS)
 	    /* Failed.  */
 	    return errno;
 
+	  /* We now have for sure more than one thread.  The main
+	     thread might not yet have the flag set.  No need to set
+	     the global variable again if this is what we use.  */
 #ifdef TLS_MULTIPLE_THREADS_IN_TCB
-	  /* We now have for sure more than one thread.  */
+	  THREAD_SETMEM (THREAD_SELF, header.multiple_threads, 1);
-	  pd->header.multiple_threads = 1;
-#else
-	  __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 #endif
 
 	  /* Now fill in the information about the new thread in
@@ -155,8 +155,10 @@ create_thread (struct pthread *pd, STACK_VARIABLES_PARMS)
     /* Failed.  */
     return errno;
 
+  /* We now have for sure more than one thread.  The main thread might
+     not yet have the flag set.  No need to set the global variable
+     again if this is what we use.  */
 #ifdef TLS_MULTIPLE_THREADS_IN_TCB
-  /* We now have for sure more than one thread.  */
   THREAD_SETMEM (THREAD_SELF, header.multiple_threads, 1);
 #endif
 

nptl/sysdeps/x86_64/tls.h

@@ -253,6 +253,32 @@ typedef struct
        }})
 
 
+/* Atomic compare and exchange on TLS, returning old value.  */
+#define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \
+  ({ __typeof (descr->member) __ret;					      \
+     __typeof (oldval) __old = (oldval);				      \
+     if (sizeof (descr->member) == 4)					      \
+       asm volatile (LOCK "cmpxchgl %2, %%fs:%P3"			      \
+		     : "=a" (__ret)					      \
+		     : "0" (__old), "r" (newval),			      \
+		       "i" (offsetof (struct pthread, member)));	      \
+     else								      \
+       /* Not necessary for other sizes in the moment.  */		      \
+       abort ();							      \
+     __ret; })
+
+
+/* Atomic set bit.  */
+#define THREAD_ATOMIC_BIT_SET(descr, member, bit) \
+  (void) ({ if (sizeof ((descr)->member) == 4)				      \
+	      asm volatile (LOCK "orl %1, %%fs:%P0"			      \
+			    :: "i" (offsetof (struct pthread, member)),	      \
+			       "ir" (1 << (bit)));			      \
+	    else							      \
+	      /* Not necessary for other sizes in the moment.  */	      \
+	      abort (); })
+
+
 #define CALL_THREAD_FCT(descr) \
   ({ void *__res;							      \
      asm volatile ("movq %%fs:%P2, %%rdi\n\t"				      \