wait.h (do_spin): New inline...

	* config/linux/wait.h (do_spin): New inline, largely copied
	from do_wait, just don't do futex_wait here, instead return true if
	it should be done.
	(do_wait): Implement using do_spin.
	* config/linux/mutex.h (gomp_mutex_lock_slow): Add an int argument
	to prototype.
	(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
	__sync_bool_compare_and_swap, pass the oldval to
	gomp_mutex_lock_slow.
	* config/linux/mutex.c (gomp_mutex_lock_slow): Add oldval argument.
	If all mutex contenders are just spinning and not sleeping, don't
	change state to 2 unnecessarily.  Optimize the loop when state has
	already become 2 to use just one atomic operation per loop instead
	of two.
	* config/linux/ia64/mutex.h (gomp_mutex_lock_slow): Add an int argument
	to prototype.
	(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
	__sync_bool_compare_and_swap, pass the oldval to
	gomp_mutex_lock_slow.

From-SVN: r176326

diff --git a/libgomp/ChangeLog b/libgomp/ChangeLog
index 1d93273b919042e3797e634353363064e7d3afa6..36c716c518eb1e34f1f0c04cc8a0ebe783a2f038 100644 (file)
--- a/libgomp/ChangeLog
+++ b/libgomp/ChangeLog
@@ -1,3 +1,25 @@
+2011-07-15  Jakub Jelinek  <jakub@redhat.com>
+
+       * config/linux/wait.h (do_spin): New inline, largely copied
+       from do_wait, just don't do futex_wait here, instead return true if
+       it should be done.
+       (do_wait): Implement using do_spin.
+       * config/linux/mutex.h (gomp_mutex_lock_slow): Add an int argument
+       to prototype.
+       (gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
+       __sync_bool_compare_and_swap, pass the oldval to
+       gomp_mutex_lock_slow.
+       * config/linux/mutex.c (gomp_mutex_lock_slow): Add oldval argument.
+       If all mutex contenders are just spinning and not sleeping, don't
+       change state to 2 unnecessarily.  Optimize the loop when state has
+       already become 2 to use just one atomic operation per loop instead
+       of two.
+       * config/linux/ia64/mutex.h (gomp_mutex_lock_slow): Add an int argument
+       to prototype.
+       (gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
+       __sync_bool_compare_and_swap, pass the oldval to
+       gomp_mutex_lock_slow.
+
 2011-06-22  Jakub Jelinek  <jakub@redhat.com>
 
        PR libgomp/49490

diff --git a/libgomp/config/linux/ia64/mutex.h b/libgomp/config/linux/ia64/mutex.h
index 6e294059b23a7d03c4bc5a9c9dace41bf0df7987..8a67673df40238dac0df8b13caa8d29a1845238b 100644 (file)
--- a/libgomp/config/linux/ia64/mutex.h
+++ b/libgomp/config/linux/ia64/mutex.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU OpenMP Library (libgomp).
@@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex)
   *mutex = 0;
 }
 
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex);
+extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
 static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
 {
-  if (!__sync_bool_compare_and_swap (mutex, 0, 1))
-    gomp_mutex_lock_slow (mutex);
+  int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+  if (__builtin_expect (oldval, 0))
+    gomp_mutex_lock_slow (mutex, oldval);
 }
 
 extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);

diff --git a/libgomp/config/linux/mutex.c b/libgomp/config/linux/mutex.c
index 3ca37c19f5f633607eb5d93907d38ef73f13e050..2574f7be42a0d35aa81f03a30b2be014e2a1326b 100644 (file)
--- a/libgomp/config/linux/mutex.c
+++ b/libgomp/config/linux/mutex.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU OpenMP Library (libgomp).
@@ -32,15 +32,27 @@ long int gomp_futex_wake = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
 long int gomp_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
 
 void
-gomp_mutex_lock_slow (gomp_mutex_t *mutex)
+gomp_mutex_lock_slow (gomp_mutex_t *mutex, int oldval)
 {
-  do
+  while (oldval == 1)
     {
-      int oldval = __sync_val_compare_and_swap (mutex, 1, 2);
-      if (oldval != 0)
-       do_wait (mutex, 2);
+      if (do_spin (mutex, 1))
+       {
+         oldval = __sync_lock_test_and_set (mutex, 2);
+         if (oldval == 0)
+           return;
+         futex_wait (mutex, 2);
+         break;
+       }
+      else
+       {
+         oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+         if (oldval == 0)
+           return;
+       }
     }
-  while (!__sync_bool_compare_and_swap (mutex, 0, 2));
+  while ((oldval = __sync_lock_test_and_set (mutex, 2)))
+    do_wait (mutex, 2);
 }
 
 void

diff --git a/libgomp/config/linux/mutex.h b/libgomp/config/linux/mutex.h
index 1905ce012fde79b8b57ccc5a338be8f2b895eebf..eafb0e7673401332329b7d5eaedd59eac9232228 100644 (file)
--- a/libgomp/config/linux/mutex.h
+++ b/libgomp/config/linux/mutex.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2009, 2011 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU OpenMP Library (libgomp).
@@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex)
   *mutex = 0;
 }
 
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex);
+extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
 static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
 {
-  if (!__sync_bool_compare_and_swap (mutex, 0, 1))
-    gomp_mutex_lock_slow (mutex);
+  int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+  if (__builtin_expect (oldval, 0))
+    gomp_mutex_lock_slow (mutex, oldval);
 }
 
 extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);

diff --git a/libgomp/config/linux/wait.h b/libgomp/config/linux/wait.h
index 0e8abf111b09362f41173893747f268629a1ba58..4f65985581128eae3707e82794790cd86f6ede09 100644 (file)
--- a/libgomp/config/linux/wait.h
+++ b/libgomp/config/linux/wait.h
@@ -44,7 +44,7 @@ extern long int gomp_futex_wait, gomp_futex_wake;
 
 #include <futex.h>
 
-static inline void do_wait (int *addr, int val)
+static inline int do_spin (int *addr, int val)
 {
   unsigned long long i, count = gomp_spin_count_var;
 
@@ -52,10 +52,16 @@ static inline void do_wait (int *addr, int val)
     count = gomp_throttled_spin_count_var;
   for (i = 0; i < count; i++)
     if (__builtin_expect (*addr != val, 0))
-      return;
+      return 0;
     else
       cpu_relax ();
-  futex_wait (addr, val);
+  return 1;
+}
+
+static inline void do_wait (int *addr, int val)
+{
+  if (do_spin (addr, val))
+    futex_wait (addr, val);
 }
 
 #ifdef HAVE_ATTRIBUTE_VISIBILITY