1 files changed, 87 insertions, 47 deletions

diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h
index 83e5d216105e..84faa13aa7ae 100644
--- a/arch/s390/include/asm/spinlock.h
+++ b/arch/s390/include/asm/spinlock.h
@@ -1,7 +1,8 @@
 /*
  *  S390 version
- *    Copyright IBM Corp. 1999
+ *    Copyright IBM Corp. 1999, 2014
  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
+ *		 Philipp Hachtmann (phacht@linux.vnet.ibm.com)
  *
  *  Derived from "include/asm-i386/spinlock.h"
  */
@@ -11,80 +12,121 @@
 
 #include <linux/smp.h>
 
+#define SPINLOCK_LOCKVAL (S390_lowcore.spinlock_lockval)
+
 extern int spin_retry;
 
 static inline int
-_raw_compare_and_swap(volatile unsigned int *lock,
-		      unsigned int old, unsigned int new)
+_raw_compare_and_swap(unsigned int *lock, unsigned int old, unsigned int new)
 {
+	unsigned int old_expected = old;
+
 	asm volatile(
 		"	cs	%0,%3,%1"
 		: "=d" (old), "=Q" (*lock)
 		: "0" (old), "d" (new), "Q" (*lock)
 		: "cc", "memory" );
-	return old;
+	return old == old_expected;
 }
 
 /*
  * Simple spin lock operations.  There are two variants, one clears IRQ's
  * on the local processor, one does not.
  *
- * We make no fairness assumptions. They have a cost.
- *
  * (the type definitions are in asm/spinlock_types.h)
  */
 
-#define arch_spin_is_locked(x) ((x)->owner_cpu != 0)
-#define arch_spin_unlock_wait(lock) \
-	do { while (arch_spin_is_locked(lock)) \
-		 arch_spin_relax(lock); } while (0)
+void arch_spin_lock_wait(arch_spinlock_t *);
+int arch_spin_trylock_retry(arch_spinlock_t *);
+void arch_spin_relax(arch_spinlock_t *);
+
+#ifdef CONFIG_S390_TICKET_SPINLOCK
+
+void arch_spin_unlock_slow(arch_spinlock_t *lp);
+
+static inline u32 arch_spin_lockval(u32 cpu)
+{
+	arch_spinlock_t new;
+
+	new.tickets.owner = ~cpu;
+	new.tickets.head = 0;
+	new.tickets.tail = 0;
+	return new.lock;
+}
+
+static inline void arch_spin_lock_wait_flags(arch_spinlock_t *lp,
+					     unsigned long flags)
+{
+	arch_spin_lock_wait(lp);
+}
+
+#else /* CONFIG_S390_TICKET_SPINLOCK */
+
+void arch_spin_lock_wait_flags(arch_spinlock_t *, unsigned long flags);
+
+static inline u32 arch_spin_lockval(int cpu)
+{
+	return ~cpu;
+}
+
+static inline void arch_spin_unlock_slow(arch_spinlock_t *lp)
+{
+}
 
-extern void arch_spin_lock_wait(arch_spinlock_t *);
-extern void arch_spin_lock_wait_flags(arch_spinlock_t *, unsigned long flags);
-extern int arch_spin_trylock_retry(arch_spinlock_t *);
-extern void arch_spin_relax(arch_spinlock_t *lock);
+#endif /* CONFIG_S390_TICKET_SPINLOCK */
 
 static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
 {
-	return lock.owner_cpu == 0;
+	return lock.lock == 0;
 }
 
-static inline void arch_spin_lock(arch_spinlock_t *lp)
+static inline int arch_spin_is_locked(arch_spinlock_t *lp)
 {
-	int old;
+	return ACCESS_ONCE(lp->lock) != 0;
+}
 
-	old = _raw_compare_and_swap(&lp->owner_cpu, 0, ~smp_processor_id());
-	if (likely(old == 0))
-		return;
-	arch_spin_lock_wait(lp);
+static inline int arch_spin_trylock_once(arch_spinlock_t *lp)
+{
+	return _raw_compare_and_swap(&lp->lock, 0, SPINLOCK_LOCKVAL);
 }
 
-static inline void arch_spin_lock_flags(arch_spinlock_t *lp,
-					 unsigned long flags)
+static inline int arch_spin_tryrelease_once(arch_spinlock_t *lp)
 {
-	int old;
+	return _raw_compare_and_swap(&lp->lock, SPINLOCK_LOCKVAL, 0);
+}
 
-	old = _raw_compare_and_swap(&lp->owner_cpu, 0, ~smp_processor_id());
-	if (likely(old == 0))
-		return;
-	arch_spin_lock_wait_flags(lp, flags);
+static inline void arch_spin_lock(arch_spinlock_t *lp)
+{
+	if (unlikely(!arch_spin_trylock_once(lp)))
+		arch_spin_lock_wait(lp);
 }
 
-static inline int arch_spin_trylock(arch_spinlock_t *lp)
+static inline void arch_spin_lock_flags(arch_spinlock_t *lp,
+					unsigned long flags)
 {
-	int old;
+	if (unlikely(!arch_spin_trylock_once(lp)))
+		arch_spin_lock_wait_flags(lp, flags);
+}
 
-	old = _raw_compare_and_swap(&lp->owner_cpu, 0, ~smp_processor_id());
-	if (likely(old == 0))
-		return 1;
-	return arch_spin_trylock_retry(lp);
+static inline int arch_spin_trylock(arch_spinlock_t *lp)
+{
+	if (unlikely(!arch_spin_trylock_once(lp)))
+		return arch_spin_trylock_retry(lp);
+	return 1;
 }
 
 static inline void arch_spin_unlock(arch_spinlock_t *lp)
 {
-	_raw_compare_and_swap(&lp->owner_cpu, lp->owner_cpu, 0);
+	if (unlikely(!arch_spin_tryrelease_once(lp)))
+		arch_spin_unlock_slow(lp);
 }
-		
+
+static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
+{
+	while (arch_spin_is_locked(lock))
+		arch_spin_relax(lock);
+}
+
 /*
  * Read-write spinlocks, allowing multiple readers
  * but only one writer.
@@ -119,7 +161,7 @@ static inline void arch_read_lock(arch_rwlock_t *rw)
 {
 	unsigned int old;
 	old = rw->lock & 0x7fffffffU;
-	if (_raw_compare_and_swap(&rw->lock, old, old + 1) != old)
+	if (!_raw_compare_and_swap(&rw->lock, old, old + 1))
 		_raw_read_lock_wait(rw);
 }
 
@@ -127,30 +169,28 @@ static inline void arch_read_lock_flags(arch_rwlock_t *rw, unsigned long flags)
 {
 	unsigned int old;
 	old = rw->lock & 0x7fffffffU;
-	if (_raw_compare_and_swap(&rw->lock, old, old + 1) != old)
+	if (!_raw_compare_and_swap(&rw->lock, old, old + 1))
 		_raw_read_lock_wait_flags(rw, flags);
 }
 
 static inline void arch_read_unlock(arch_rwlock_t *rw)
 {
-	unsigned int old, cmp;
+	unsigned int old;
 
-	old = rw->lock;
 	do {
-		cmp = old;
-		old = _raw_compare_and_swap(&rw->lock, old, old - 1);
-	} while (cmp != old);
+		old = ACCESS_ONCE(rw->lock);
+	} while (!_raw_compare_and_swap(&rw->lock, old, old - 1));
 }
 
 static inline void arch_write_lock(arch_rwlock_t *rw)
 {
-	if (unlikely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000) != 0))
+	if (unlikely(!_raw_compare_and_swap(&rw->lock, 0, 0x80000000)))
 		_raw_write_lock_wait(rw);
 }
 
 static inline void arch_write_lock_flags(arch_rwlock_t *rw, unsigned long flags)
 {
-	if (unlikely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000) != 0))
+	if (unlikely(!_raw_compare_and_swap(&rw->lock, 0, 0x80000000)))
 		_raw_write_lock_wait_flags(rw, flags);
 }
 
@@ -163,14 +203,14 @@ static inline int arch_read_trylock(arch_rwlock_t *rw)
 {
 	unsigned int old;
 	old = rw->lock & 0x7fffffffU;
-	if (likely(_raw_compare_and_swap(&rw->lock, old, old + 1) == old))
+	if (likely(_raw_compare_and_swap(&rw->lock, old, old + 1)))
 		return 1;
 	return _raw_read_trylock_retry(rw);
 }
 
 static inline int arch_write_trylock(arch_rwlock_t *rw)
 {
-	if (likely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0))
+	if (likely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000)))
 		return 1;
 	return _raw_write_trylock_retry(rw);
 }