1 files changed, 344 insertions, 299 deletions

diff --git a/linux/six.c b/linux/six.c
index 3d366a84..a47cd6d0 100644
--- a/linux/six.c
+++ b/linux/six.c
@@ -14,9 +14,9 @@
 #include <trace/events/lock.h>
 
 #ifdef DEBUG
-#define EBUG_ON(cond)		BUG_ON(cond)
+#define EBUG_ON(cond)			BUG_ON(cond)
 #else
-#define EBUG_ON(cond)		do {} while (0)
+#define EBUG_ON(cond)			do {} while (0)
 #endif
 
 #define six_acquire(l, t, r, ip)	lock_acquire(l, 0, t, r, 1, NULL, ip)
@@ -24,59 +24,69 @@
 
 static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type);
 
+#define SIX_LOCK_HELD_read_OFFSET	0
+#define SIX_LOCK_HELD_read		~(~0U << 26)
+#define SIX_LOCK_HELD_intent		(1U << 26)
+#define SIX_LOCK_HELD_write		(1U << 27)
+#define SIX_LOCK_WAITING_read		(1U << (28 + SIX_LOCK_read))
+#define SIX_LOCK_WAITING_intent		(1U << (28 + SIX_LOCK_intent))
+#define SIX_LOCK_WAITING_write		(1U << (28 + SIX_LOCK_write))
+#define SIX_LOCK_NOSPIN			(1U << 31)
+
 struct six_lock_vals {
 	/* Value we add to the lock in order to take the lock: */
-	u64			lock_val;
+	u32			lock_val;
 
 	/* If the lock has this value (used as a mask), taking the lock fails: */
-	u64			lock_fail;
-
-	/* Value we add to the lock in order to release the lock: */
-	u64			unlock_val;
+	u32			lock_fail;
 
 	/* Mask that indicates lock is held for this type: */
-	u64			held_mask;
+	u32			held_mask;
 
 	/* Waitlist we wakeup when releasing the lock: */
 	enum six_lock_type	unlock_wakeup;
 };
 
-#define __SIX_LOCK_HELD_read	__SIX_VAL(read_lock, ~0)
-#define __SIX_LOCK_HELD_intent	__SIX_VAL(intent_lock, ~0)
-#define __SIX_LOCK_HELD_write	__SIX_VAL(seq, 1)
-
 #define LOCK_VALS {							\
 	[SIX_LOCK_read] = {						\
-		.lock_val	= __SIX_VAL(read_lock, 1),		\
-		.lock_fail	= __SIX_LOCK_HELD_write + __SIX_VAL(write_locking, 1),\
-		.unlock_val	= -__SIX_VAL(read_lock, 1),		\
-		.held_mask	= __SIX_LOCK_HELD_read,			\
+		.lock_val	= 1U << SIX_LOCK_HELD_read_OFFSET,	\
+		.lock_fail	= SIX_LOCK_HELD_write,			\
+		.held_mask	= SIX_LOCK_HELD_read,			\
 		.unlock_wakeup	= SIX_LOCK_write,			\
 	},								\
 	[SIX_LOCK_intent] = {						\
-		.lock_val	= __SIX_VAL(intent_lock, 1),		\
-		.lock_fail	= __SIX_LOCK_HELD_intent,		\
-		.unlock_val	= -__SIX_VAL(intent_lock, 1),		\
-		.held_mask	= __SIX_LOCK_HELD_intent,		\
+		.lock_val	= SIX_LOCK_HELD_intent,			\
+		.lock_fail	= SIX_LOCK_HELD_intent,			\
+		.held_mask	= SIX_LOCK_HELD_intent,			\
 		.unlock_wakeup	= SIX_LOCK_intent,			\
 	},								\
 	[SIX_LOCK_write] = {						\
-		.lock_val	= __SIX_VAL(seq, 1),			\
-		.lock_fail	= __SIX_LOCK_HELD_read,			\
-		.unlock_val	= __SIX_VAL(seq, 1),			\
-		.held_mask	= __SIX_LOCK_HELD_write,		\
+		.lock_val	= SIX_LOCK_HELD_write,			\
+		.lock_fail	= SIX_LOCK_HELD_read,			\
+		.held_mask	= SIX_LOCK_HELD_write,			\
 		.unlock_wakeup	= SIX_LOCK_read,			\
 	},								\
 }
 
+static inline void six_set_bitmask(struct six_lock *lock, u32 mask)
+{
+	if ((atomic_read(&lock->state) & mask) != mask)
+		atomic_or(mask, &lock->state);
+}
+
+static inline void six_clear_bitmask(struct six_lock *lock, u32 mask)
+{
+	if (atomic_read(&lock->state) & mask)
+		atomic_and(~mask, &lock->state);
+}
+
 static inline void six_set_owner(struct six_lock *lock, enum six_lock_type type,
-				 union six_lock_state old,
-				 struct task_struct *owner)
+				 u32 old, struct task_struct *owner)
 {
 	if (type != SIX_LOCK_intent)
 		return;
 
-	if (!old.intent_lock) {
+	if (!(old & SIX_LOCK_HELD_intent)) {
 		EBUG_ON(lock->owner);
 		lock->owner = owner;
 	} else {
@@ -94,22 +104,25 @@ static inline unsigned pcpu_read_count(struct six_lock *lock)
 	return read_count;
 }
 
-/* This is probably up there with the more evil things I've done */
-#define waitlist_bitnr(id) ilog2((((union six_lock_state) { .waiters = 1 << (id) }).l))
-
-static int __do_six_trylock_type(struct six_lock *lock,
-				 enum six_lock_type type,
-				 struct task_struct *task,
-				 bool try)
+/*
+ * __do_six_trylock() - main trylock routine
+ *
+ * Returns 1 on success, 0 on failure
+ *
+ * In percpu reader mode, a failed trylock may cause a spurious trylock failure
+ * for anoter thread taking the competing lock type, and we may havve to do a
+ * wakeup: when a wakeup is required, we return -1 - wakeup_type.
+ */
+static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type,
+			    struct task_struct *task, bool try)
 {
 	const struct six_lock_vals l[] = LOCK_VALS;
-	union six_lock_state old, new;
 	int ret;
-	u64 v;
+	u32 old, new, v;
 
 	EBUG_ON(type == SIX_LOCK_write && lock->owner != task);
-	EBUG_ON(type == SIX_LOCK_write && (lock->state.seq & 1));
-	EBUG_ON(type == SIX_LOCK_write && (try != !(lock->state.write_locking)));
+	EBUG_ON(type == SIX_LOCK_write &&
+		(try != !(atomic_read(&lock->state) & SIX_LOCK_HELD_write)));
 
 	/*
 	 * Percpu reader mode:
@@ -124,101 +137,75 @@ static int __do_six_trylock_type(struct six_lock *lock,
 	 * the lock, then issues a full memory barrier, then reads from the
 	 * other thread's variable to check if the other thread thinks it has
 	 * the lock. If we raced, we backoff and retry/sleep.
+	 *
+	 * Failure to take the lock may cause a spurious trylock failure in
+	 * another thread, because we temporarily set the lock to indicate that
+	 * we held it. This would be a problem for a thread in six_lock(), when
+	 * they are calling trylock after adding themself to the waitlist and
+	 * prior to sleeping.
+	 *
+	 * Therefore, if we fail to get the lock, and there were waiters of the
+	 * type we conflict with, we will have to issue a wakeup.
+	 *
+	 * Since we may be called under wait_lock (and by the wakeup code
+	 * itself), we return that the wakeup has to be done instead of doing it
+	 * here.
 	 */
-
 	if (type == SIX_LOCK_read && lock->readers) {
 		preempt_disable();
 		this_cpu_inc(*lock->readers); /* signal that we own lock */
 
 		smp_mb();
 
-		old.v = READ_ONCE(lock->state.v);
-		ret = !(old.v & l[type].lock_fail);
+		old = atomic_read(&lock->state);
+		ret = !(old & l[type].lock_fail);
 
 		this_cpu_sub(*lock->readers, !ret);
 		preempt_enable();
 
-		/*
-		 * If we failed because a writer was trying to take the
-		 * lock, issue a wakeup because we might have caused a
-		 * spurious trylock failure:
-		 */
-#if 0
-		/*
-		 * This code should be sufficient, but we're seeing unexplained
-		 * lost wakeups:
-		 */
-		if (old.write_locking)
+		if (!ret && (old & SIX_LOCK_WAITING_write))
 			ret = -1 - SIX_LOCK_write;
-#else
-		if (!ret)
-			ret = -1 - SIX_LOCK_write;
-#endif
 	} else if (type == SIX_LOCK_write && lock->readers) {
 		if (try) {
-			atomic64_add(__SIX_VAL(write_locking, 1),
-				     &lock->state.counter);
-			smp_mb__after_atomic();
-		} else if (!(lock->state.waiters & (1 << SIX_LOCK_write))) {
-			atomic64_add(__SIX_VAL(waiters, 1 << SIX_LOCK_write),
-				     &lock->state.counter);
-			/*
-			 * pairs with barrier after unlock and before checking
-			 * for readers in unlock path
-			 */
+			atomic_add(SIX_LOCK_HELD_write, &lock->state);
 			smp_mb__after_atomic();
 		}
 
 		ret = !pcpu_read_count(lock);
 
-		/*
-		 * On success, we increment lock->seq; also we clear
-		 * write_locking unless we failed from the lock path:
-		 */
-		v = 0;
-		if (ret)
-			v += __SIX_VAL(seq, 1);
-		if (ret || try)
-			v -= __SIX_VAL(write_locking, 1);
-
 		if (try && !ret) {
-			old.v = atomic64_add_return(v, &lock->state.counter);
-			if (old.waiters & (1 << SIX_LOCK_read))
+			old = atomic_sub_return(SIX_LOCK_HELD_write, &lock->state);
+			if (old & SIX_LOCK_WAITING_read)
 				ret = -1 - SIX_LOCK_read;
-		} else {
-			atomic64_add(v, &lock->state.counter);
 		}
 	} else {
-		v = READ_ONCE(lock->state.v);
+		v = atomic_read(&lock->state);
 		do {
-			new.v = old.v = v;
+			new = old = v;
 
-			if (!(old.v & l[type].lock_fail)) {
-				new.v += l[type].lock_val;
+			ret = !(old & l[type].lock_fail);
 
-				if (type == SIX_LOCK_write)
-					new.write_locking = 0;
-			} else if (!try && !(new.waiters & (1 << type)))
-				new.waiters |= 1 << type;
-			else
-				break; /* waiting bit already set */
-		} while ((v = atomic64_cmpxchg_acquire(&lock->state.counter,
-					old.v, new.v)) != old.v);
+			if (!ret || (type == SIX_LOCK_write && !try)) {
+				smp_mb();
+				break;
+			}
 
-		ret = !(old.v & l[type].lock_fail);
+			new += l[type].lock_val;
+		} while ((v = atomic_cmpxchg_acquire(&lock->state, old, new)) != old);
 
-		EBUG_ON(ret && !(lock->state.v & l[type].held_mask));
+		EBUG_ON(ret && !(atomic_read(&lock->state) & l[type].held_mask));
 	}
 
 	if (ret > 0)
 		six_set_owner(lock, type, old, task);
 
-	EBUG_ON(type == SIX_LOCK_write && (try || ret > 0) && (lock->state.write_locking));
+	EBUG_ON(type == SIX_LOCK_write && try && ret <= 0 &&
+		(atomic_read(&lock->state) & SIX_LOCK_HELD_write));
 
 	return ret;
 }
 
-static inline void __six_lock_wakeup(struct six_lock *lock, enum six_lock_type lock_type)
+static void __six_lock_wakeup(struct six_lock *lock, enum six_lock_type lock_type)
 {
 	struct six_lock_waiter *w, *next;
 	struct task_struct *task;
@@ -237,7 +224,7 @@ again:
 			goto unlock;
 		saw_one = true;
 
-		ret = __do_six_trylock_type(lock, lock_type, w->task, false);
+		ret = __do_six_trylock(lock, lock_type, w->task, false);
 		if (ret <= 0)
 			goto unlock;
 
@@ -252,7 +239,7 @@ again:
 		wake_up_process(task);
 	}
 
-	clear_bit(waitlist_bitnr(lock_type), (unsigned long *) &lock->state.v);
+	six_clear_bitmask(lock, SIX_LOCK_WAITING_read << lock_type);
 unlock:
 	raw_spin_unlock(&lock->wait_lock);
 
@@ -262,96 +249,74 @@ unlock:
 	}
 }
 
-static inline void six_lock_wakeup(struct six_lock *lock,
-				   union six_lock_state state,
-				   enum six_lock_type lock_type)
+__always_inline
+static void six_lock_wakeup(struct six_lock *lock, u32 state,
+			    enum six_lock_type lock_type)
 {
-	if (lock_type == SIX_LOCK_write && state.read_lock)
+	if (lock_type == SIX_LOCK_write && (state & SIX_LOCK_HELD_read))
 		return;
 
-	if (!(state.waiters & (1 << lock_type)))
+	if (!(state & (SIX_LOCK_WAITING_read << lock_type)))
 		return;
 
 	__six_lock_wakeup(lock, lock_type);
 }
 
-static bool do_six_trylock_type(struct six_lock *lock,
-				enum six_lock_type type,
-				bool try)
+__always_inline
+static bool do_six_trylock(struct six_lock *lock, enum six_lock_type type, bool try)
 {
 	int ret;
 
-	ret = __do_six_trylock_type(lock, type, current, try);
+	ret = __do_six_trylock(lock, type, current, try);
 	if (ret < 0)
 		__six_lock_wakeup(lock, -ret - 1);
 
 	return ret > 0;
 }
 
-__always_inline __flatten
-static bool __six_trylock_type(struct six_lock *lock, enum six_lock_type type,
-			       unsigned long ip)
+/**
+ * six_trylock_ip - attempt to take a six lock without blocking
+ * @lock:	lock to take
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ * @ip:		ip parameter for lockdep/lockstat, i.e. _THIS_IP_
+ *
+ * Return: true on success, false on failure.
+ */
+bool six_trylock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip)
 {
-	if (!do_six_trylock_type(lock, type, true))
+	if (!do_six_trylock(lock, type, true))
 		return false;
 
 	if (type != SIX_LOCK_write)
 		six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip);
 	return true;
 }
-
-__always_inline __flatten
-static bool __six_relock_type(struct six_lock *lock, enum six_lock_type type,
-			      unsigned seq, unsigned long ip)
+EXPORT_SYMBOL_GPL(six_trylock_ip);
+
+/**
+ * six_relock_ip - attempt to re-take a lock that was held previously
+ * @lock:	lock to take
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ * @seq:	lock sequence number obtained from six_lock_seq() while lock was
+ *		held previously
+ * @ip:		ip parameter for lockdep/lockstat, i.e. _THIS_IP_
+ *
+ * Return: true on success, false on failure.
+ */
+bool six_relock_ip(struct six_lock *lock, enum six_lock_type type,
+		   unsigned seq, unsigned long ip)
 {
-	const struct six_lock_vals l[] = LOCK_VALS;
-	union six_lock_state old;
-	u64 v;
-
-	EBUG_ON(type == SIX_LOCK_write);
-
-	if (type == SIX_LOCK_read &&
-	    lock->readers) {
-		bool ret;
-
-		preempt_disable();
-		this_cpu_inc(*lock->readers);
-
-		smp_mb();
-
-		old.v = READ_ONCE(lock->state.v);
-		ret = !(old.v & l[type].lock_fail) && old.seq == seq;
-
-		this_cpu_sub(*lock->readers, !ret);
-		preempt_enable();
-
-		/*
-		 * Similar to the lock path, we may have caused a spurious write
-		 * lock fail and need to issue a wakeup:
-		 */
-		if (ret)
-			six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip);
-		else
-			six_lock_wakeup(lock, old, SIX_LOCK_write);
+	if (lock->seq != seq || !six_trylock_ip(lock, type, ip))
+		return false;
 
-		return ret;
+	if (lock->seq != seq) {
+		six_unlock_ip(lock, type, ip);
+		return false;
 	}
 
-	v = READ_ONCE(lock->state.v);
-	do {
-		old.v = v;
-
-		if (old.seq != seq || old.v & l[type].lock_fail)
-			return false;
-	} while ((v = atomic64_cmpxchg_acquire(&lock->state.counter,
-				old.v,
-				old.v + l[type].lock_val)) != old.v);
-
-	six_set_owner(lock, type, old, current);
-	if (type != SIX_LOCK_write)
-		six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip);
 	return true;
 }
+EXPORT_SYMBOL_GPL(six_relock_ip);
 
 #ifdef CONFIG_LOCK_SPIN_ON_OWNER
 
@@ -371,17 +336,6 @@ static inline bool six_can_spin_on_owner(struct six_lock *lock)
 	return ret;
 }
 
-static inline void six_set_nospin(struct six_lock *lock)
-{
-	union six_lock_state old, new;
-	u64 v = READ_ONCE(lock->state.v);
-
-	do {
-		new.v = old.v = v;
-		new.nospin = true;
-	} while ((v = atomic64_cmpxchg(&lock->state.counter, old.v, new.v)) != old.v);
-}
-
 static inline bool six_spin_on_owner(struct six_lock *lock,
 				     struct task_struct *owner,
 				     u64 end_time)
@@ -405,7 +359,7 @@ static inline bool six_spin_on_owner(struct six_lock *lock,
 		}
 
 		if (!(++loop & 0xf) && (time_after64(sched_clock(), end_time))) {
-			six_set_nospin(lock);
+			six_set_bitmask(lock, SIX_LOCK_NOSPIN);
 			ret = false;
 			break;
 		}
@@ -445,7 +399,7 @@ static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type
 		if (owner && !six_spin_on_owner(lock, owner, end_time))
 			break;
 
-		if (do_six_trylock_type(lock, type, false)) {
+		if (do_six_trylock(lock, type, false)) {
 			osq_unlock(&lock->osq);
 			preempt_enable();
 			return true;
@@ -494,17 +448,16 @@ static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type
 #endif
 
 noinline
-static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type type,
-				    struct six_lock_waiter *wait,
-				    six_lock_should_sleep_fn should_sleep_fn, void *p,
-				    unsigned long ip)
+static int six_lock_slowpath(struct six_lock *lock, enum six_lock_type type,
+			     struct six_lock_waiter *wait,
+			     six_lock_should_sleep_fn should_sleep_fn, void *p,
+			     unsigned long ip)
 {
-	union six_lock_state old;
 	int ret = 0;
 
 	if (type == SIX_LOCK_write) {
-		EBUG_ON(lock->state.write_locking);
-		atomic64_add(__SIX_VAL(write_locking, 1), &lock->state.counter);
+		EBUG_ON(atomic_read(&lock->state) & SIX_LOCK_HELD_write);
+		atomic_add(SIX_LOCK_HELD_write, &lock->state);
 		smp_mb__after_atomic();
 	}
 
@@ -519,13 +472,12 @@ static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type ty
 	wait->lock_acquired	= false;
 
 	raw_spin_lock(&lock->wait_lock);
-	if (!(lock->state.waiters & (1 << type)))
-		set_bit(waitlist_bitnr(type), (unsigned long *) &lock->state.v);
+	six_set_bitmask(lock, SIX_LOCK_WAITING_read << type);
 	/*
-	 * Retry taking the lock after taking waitlist lock, have raced with an
-	 * unlock:
+	 * Retry taking the lock after taking waitlist lock, in case we raced
+	 * with an unlock:
 	 */
-	ret = __do_six_trylock_type(lock, type, current, false);
+	ret = __do_six_trylock(lock, type, current, false);
 	if (ret <= 0) {
 		wait->start_time = local_clock();
 
@@ -565,7 +517,7 @@ static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type ty
 				list_del(&wait->list);
 			raw_spin_unlock(&lock->wait_lock);
 
-			if (wait->lock_acquired)
+			if (unlikely(wait->lock_acquired))
 				do_six_unlock_type(lock, type);
 			break;
 		}
@@ -575,21 +527,49 @@ static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type ty
 
 	__set_current_state(TASK_RUNNING);
 out:
-	if (ret && type == SIX_LOCK_write && lock->state.write_locking) {
-		old.v = atomic64_sub_return(__SIX_VAL(write_locking, 1),
-					    &lock->state.counter);
-		six_lock_wakeup(lock, old, SIX_LOCK_read);
+	if (ret && type == SIX_LOCK_write) {
+		six_clear_bitmask(lock, SIX_LOCK_HELD_write);
+		six_lock_wakeup(lock, atomic_read(&lock->state), SIX_LOCK_read);
 	}
 	trace_contention_end(lock, 0);
 
 	return ret;
 }
 
-__always_inline __flatten
-static int __six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type,
-			 struct six_lock_waiter *wait,
-			 six_lock_should_sleep_fn should_sleep_fn, void *p,
-			 unsigned long ip)
+/**
+ * six_lock_ip_waiter - take a lock, with full waitlist interface
+ * @lock:	lock to take
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ * @wait:	pointer to wait object, which will be added to lock's waitlist
+ * @should_sleep_fn: callback run after adding to waitlist, immediately prior
+ *		to scheduling
+ * @p:		passed through to @should_sleep_fn
+ * @ip:		ip parameter for lockdep/lockstat, i.e. _THIS_IP_
+ *
+ * This is the most general six_lock() variant, with parameters to support full
+ * cycle detection for deadlock avoidance.
+ *
+ * The code calling this function must implement tracking of held locks, and the
+ * @wait object should be embedded into the struct that tracks held locks -
+ * which must also be accessible in a thread-safe way.
+ *
+ * @should_sleep_fn should invoke the cycle detector; it should walk each
+ * lock's waiters, and for each waiter recursively walk their held locks.
+ *
+ * When this function must block, @wait will be added to @lock's waitlist before
+ * calling trylock, and before calling @should_sleep_fn, and @wait will not be
+ * removed from the lock waitlist until the lock has been successfully acquired,
+ * or we abort.
+ *
+ * @wait.start_time will be monotonically increasing for any given waitlist, and
+ * thus may be used as a loop cursor.
+ *
+ * Return: 0 on success, or the return code from @should_sleep_fn on failure.
+ */
+int six_lock_ip_waiter(struct six_lock *lock, enum six_lock_type type,
+		       struct six_lock_waiter *wait,
+		       six_lock_should_sleep_fn should_sleep_fn, void *p,
+		       unsigned long ip)
 {
 	int ret;
 
@@ -598,8 +578,8 @@ static int __six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type
 	if (type != SIX_LOCK_write)
 		six_acquire(&lock->dep_map, 0, type == SIX_LOCK_read, ip);
 
-	ret = do_six_trylock_type(lock, type, true) ? 0
-		: __six_lock_type_slowpath(lock, type, wait, should_sleep_fn, p, ip);
+	ret = do_six_trylock(lock, type, true) ? 0
+		: six_lock_slowpath(lock, type, wait, should_sleep_fn, p, ip);
 
 	if (ret && type != SIX_LOCK_write)
 		six_release(&lock->dep_map, ip);
@@ -608,22 +588,13 @@ static int __six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(six_lock_ip_waiter);
 
 __always_inline
-static int __six_lock_type(struct six_lock *lock, enum six_lock_type type,
-			   six_lock_should_sleep_fn should_sleep_fn, void *p,
-			   unsigned long ip)
-{
-	struct six_lock_waiter wait;
-
-	return __six_lock_type_waiter(lock, type, &wait, should_sleep_fn, p, ip);
-}
-
-__always_inline __flatten
 static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 {
 	const struct six_lock_vals l[] = LOCK_VALS;
-	union six_lock_state state;
+	u32 state;
 
 	if (type == SIX_LOCK_intent)
 		lock->owner = NULL;
@@ -633,26 +604,39 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 		smp_mb(); /* unlock barrier */
 		this_cpu_dec(*lock->readers);
 		smp_mb(); /* between unlocking and checking for waiters */
-		state.v = READ_ONCE(lock->state.v);
+		state = atomic_read(&lock->state);
 	} else {
-		u64 v = l[type].unlock_val;
+		u32 v = l[type].lock_val;
 
 		if (type != SIX_LOCK_read)
-			v -= lock->state.v & __SIX_VAL(nospin, 1);
+			v += atomic_read(&lock->state) & SIX_LOCK_NOSPIN;
 
-		EBUG_ON(!(lock->state.v & l[type].held_mask));
-		state.v = atomic64_add_return_release(v, &lock->state.counter);
+		EBUG_ON(!(atomic_read(&lock->state) & l[type].held_mask));
+		state = atomic_sub_return_release(v, &lock->state);
 	}
 
 	six_lock_wakeup(lock, state, l[type].unlock_wakeup);
 }
 
-__always_inline __flatten
-static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type,
-			      unsigned long ip)
+/**
+ * six_unlock_ip - drop a six lock
+ * @lock:	lock to unlock
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ * @ip:		ip parameter for lockdep/lockstat, i.e. _THIS_IP_
+ *
+ * When a lock is held multiple times (because six_lock_incement()) was used),
+ * this decrements the 'lock held' counter by one.
+ *
+ * For example:
+ * six_lock_read(&foo->lock);				read count 1
+ * six_lock_increment(&foo->lock, SIX_LOCK_read);	read count 2
+ * six_lock_unlock(&foo->lock, SIX_LOCK_read);		read count 1
+ * six_lock_unlock(&foo->lock, SIX_LOCK_read);		read count 0
+ */
+void six_unlock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip)
 {
 	EBUG_ON(type == SIX_LOCK_write &&
-		!(lock->state.v & __SIX_LOCK_HELD_intent));
+		!(atomic_read(&lock->state) & SIX_LOCK_HELD_intent));
 	EBUG_ON((type == SIX_LOCK_write ||
 		 type == SIX_LOCK_intent) &&
 		lock->owner != current);
@@ -666,52 +650,18 @@ static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type,
 		return;
 	}
 
+	lock->seq += type == SIX_LOCK_write;
+
 	do_six_unlock_type(lock, type);
 }
+EXPORT_SYMBOL_GPL(six_unlock_ip);
 
-#define __SIX_LOCK(type)						\
-bool six_trylock_ip_##type(struct six_lock *lock, unsigned long ip)	\
-{									\
-	return __six_trylock_type(lock, SIX_LOCK_##type, ip);		\
-}									\
-EXPORT_SYMBOL_GPL(six_trylock_ip_##type);				\
-									\
-bool six_relock_ip_##type(struct six_lock *lock, u32 seq, unsigned long ip)\
-{									\
-	return __six_relock_type(lock, SIX_LOCK_##type, seq, ip);	\
-}									\
-EXPORT_SYMBOL_GPL(six_relock_ip_##type);				\
-									\
-int six_lock_ip_##type(struct six_lock *lock,				\
-		    six_lock_should_sleep_fn should_sleep_fn, void *p,	\
-		    unsigned long ip)					\
-{									\
-	return __six_lock_type(lock, SIX_LOCK_##type, should_sleep_fn, p, ip);\
-}									\
-EXPORT_SYMBOL_GPL(six_lock_ip_##type);					\
-									\
-int six_lock_ip_waiter_##type(struct six_lock *lock,			\
-			   struct six_lock_waiter *wait,		\
-			   six_lock_should_sleep_fn should_sleep_fn, void *p,\
-			   unsigned long ip)				\
-{									\
-	return __six_lock_type_waiter(lock, SIX_LOCK_##type, wait, should_sleep_fn, p, ip);\
-}									\
-EXPORT_SYMBOL_GPL(six_lock_ip_waiter_##type);				\
-									\
-void six_unlock_ip_##type(struct six_lock *lock, unsigned long ip)	\
-{									\
-	__six_unlock_type(lock, SIX_LOCK_##type, ip);			\
-}									\
-EXPORT_SYMBOL_GPL(six_unlock_ip_##type);
-
-__SIX_LOCK(read)
-__SIX_LOCK(intent)
-__SIX_LOCK(write)
-
-#undef __SIX_LOCK
-
-/* Convert from intent to read: */
+/**
+ * six_lock_downgrade - convert an intent lock to a read lock
+ * @lock:	lock to dowgrade
+ *
+ * @lock will have read count incremented and intent count decremented
+ */
 void six_lock_downgrade(struct six_lock *lock)
 {
 	six_lock_increment(lock, SIX_LOCK_read);
@@ -719,25 +669,33 @@ void six_lock_downgrade(struct six_lock *lock)
 }
 EXPORT_SYMBOL_GPL(six_lock_downgrade);
 
+/**
+ * six_lock_tryupgrade - attempt to convert read lock to an intent lock
+ * @lock:	lock to upgrade
+ *
+ * On success, @lock will have intent count incremented and read count
+ * decremented
+ *
+ * Return: true on success, false on failure
+ */
 bool six_lock_tryupgrade(struct six_lock *lock)
 {
-	union six_lock_state old, new;
-	u64 v = READ_ONCE(lock->state.v);
+	const struct six_lock_vals l[] = LOCK_VALS;
+	u32 old, new, v = atomic_read(&lock->state);
 
 	do {
-		new.v = old.v = v;
+		new = old = v;
 
-		if (new.intent_lock)
+		if (new & SIX_LOCK_HELD_intent)
 			return false;
 
 		if (!lock->readers) {
-			EBUG_ON(!new.read_lock);
-			new.read_lock--;
+			EBUG_ON(!(new & SIX_LOCK_HELD_read));
+			new -= l[SIX_LOCK_read].lock_val;
 		}
 
-		new.intent_lock = 1;
-	} while ((v = atomic64_cmpxchg_acquire(&lock->state.counter,
-				old.v, new.v)) != old.v);
+		new |= SIX_LOCK_HELD_intent;
+	} while ((v = atomic_cmpxchg_acquire(&lock->state, old, new)) != old);
 
 	if (lock->readers)
 		this_cpu_dec(*lock->readers);
@@ -748,6 +706,17 @@ bool six_lock_tryupgrade(struct six_lock *lock)
 }
 EXPORT_SYMBOL_GPL(six_lock_tryupgrade);
 
+/**
+ * six_trylock_convert - attempt to convert a held lock from one type to another
+ * @lock:	lock to upgrade
+ * @from:	SIX_LOCK_read or SIX_LOCK_intent
+ * @to:		SIX_LOCK_read or SIX_LOCK_intent
+ *
+ * On success, @lock will have intent count incremented and read count
+ * decremented
+ *
+ * Return: true on success, false on failure
+ */
 bool six_trylock_convert(struct six_lock *lock,
 			 enum six_lock_type from,
 			 enum six_lock_type to)
@@ -766,9 +735,16 @@ bool six_trylock_convert(struct six_lock *lock,
 }
 EXPORT_SYMBOL_GPL(six_trylock_convert);
 
-/*
- * Increment read/intent lock count, assuming we already have it read or intent
- * locked:
+/**
+ * six_lock_increment - increase held lock count on a lock that is already held
+ * @lock:	lock to increment
+ * @type:	SIX_LOCK_read or SIX_LOCK_intent
+ *
+ * @lock must already be held, with a lock type that is greater than or equal to
+ * @type
+ *
+ * A corresponding six_unlock_type() call will be required for @lock to be fully
+ * unlocked.
  */
 void six_lock_increment(struct six_lock *lock, enum six_lock_type type)
 {
@@ -783,13 +759,14 @@ void six_lock_increment(struct six_lock *lock, enum six_lock_type type)
 		if (lock->readers) {
 			this_cpu_inc(*lock->readers);
 		} else {
-			EBUG_ON(!lock->state.read_lock &&
-				!lock->state.intent_lock);
-			atomic64_add(l[type].lock_val, &lock->state.counter);
+			EBUG_ON(!(atomic_read(&lock->state) &
+				  (SIX_LOCK_HELD_read|
+				   SIX_LOCK_HELD_intent)));
+			atomic_add(l[type].lock_val, &lock->state);
 		}
 		break;
 	case SIX_LOCK_intent:
-		EBUG_ON(!lock->state.intent_lock);
+		EBUG_ON(!(atomic_read(&lock->state) & SIX_LOCK_HELD_intent));
 		lock->intent_lock_recurse++;
 		break;
 	case SIX_LOCK_write:
@@ -799,9 +776,19 @@ void six_lock_increment(struct six_lock *lock, enum six_lock_type type)
 }
 EXPORT_SYMBOL_GPL(six_lock_increment);
 
+/**
+ * six_lock_wakeup_all - wake up all waiters on @lock
+ * @lock:	lock to wake up waiters for
+ *
+ * Wakeing up waiters will cause them to re-run should_sleep_fn, which may then
+ * abort the lock operation.
+ *
+ * This function is never needed in a bug-free program; it's only useful in
+ * debug code, e.g. to determine if a cycle detector is at fault.
+ */
 void six_lock_wakeup_all(struct six_lock *lock)
 {
-	union six_lock_state state = lock->state;
+	u32 state = atomic_read(&lock->state);
 	struct six_lock_waiter *w;
 
 	six_lock_wakeup(lock, state, SIX_LOCK_read);
@@ -815,38 +802,96 @@ void six_lock_wakeup_all(struct six_lock *lock)
 }
 EXPORT_SYMBOL_GPL(six_lock_wakeup_all);
 
-void six_lock_pcpu_free(struct six_lock *lock)
-{
-	BUG_ON(lock->readers && pcpu_read_count(lock));
-	BUG_ON(lock->state.read_lock);
-
-	free_percpu(lock->readers);
-	lock->readers = NULL;
-}
-EXPORT_SYMBOL_GPL(six_lock_pcpu_free);
-
-void six_lock_pcpu_alloc(struct six_lock *lock)
-{
-#ifdef __KERNEL__
-	if (!lock->readers)
-		lock->readers = alloc_percpu(unsigned);
-#endif
-}
-EXPORT_SYMBOL_GPL(six_lock_pcpu_alloc);
-
-/*
- * Returns lock held counts, for both read and intent
+/**
+ * six_lock_counts - return held lock counts, for each lock type
+ * @lock:	lock to return counters for
+ *
+ * Return: the number of times a lock is held for read, intent and write.
  */
 struct six_lock_count six_lock_counts(struct six_lock *lock)
 {
 	struct six_lock_count ret;
 
 	ret.n[SIX_LOCK_read]	= !lock->readers
-		? lock->state.read_lock
+		? atomic_read(&lock->state) & SIX_LOCK_HELD_read
 		: pcpu_read_count(lock);
-	ret.n[SIX_LOCK_intent]	= lock->state.intent_lock + lock->intent_lock_recurse;
-	ret.n[SIX_LOCK_write]	= lock->state.seq & 1;
+	ret.n[SIX_LOCK_intent]	= !!(atomic_read(&lock->state) & SIX_LOCK_HELD_intent) +
+		lock->intent_lock_recurse;
+	ret.n[SIX_LOCK_write]	= !!(atomic_read(&lock->state) & SIX_LOCK_HELD_write);
 
 	return ret;
 }
 EXPORT_SYMBOL_GPL(six_lock_counts);
+
+/**
+ * six_lock_readers_add - directly manipulate reader count of a lock
+ * @lock:	lock to add/subtract readers for
+ * @nr:		reader count to add/subtract
+ *
+ * When an upper layer is implementing lock reentrency, we may have both read
+ * and intent locks on the same lock.
+ *
+ * When we need to take a write lock, the read locks will cause self-deadlock,
+ * because six locks themselves do not track which read locks are held by the
+ * current thread and which are held by a different thread - it does no
+ * per-thread tracking of held locks.
+ *
+ * The upper layer that is tracking held locks may however, if trylock() has
+ * failed, count up its own read locks, subtract them, take the write lock, and
+ * then re-add them.
+ *
+ * As in any other situation when taking a write lock, @lock must be held for
+ * intent one (or more) times, so @lock will never be left unlocked.
+ */
+void six_lock_readers_add(struct six_lock *lock, int nr)
+{
+	if (lock->readers) {
+		this_cpu_add(*lock->readers, nr);
+	} else {
+		EBUG_ON((int) (atomic_read(&lock->state) & SIX_LOCK_HELD_read) + nr < 0);
+		/* reader count starts at bit 0 */
+		atomic_add(nr, &lock->state);
+	}
+}
+EXPORT_SYMBOL_GPL(six_lock_readers_add);
+
+/**
+ * six_lock_exit - release resources held by a lock prior to freeing
+ * @lock:	lock to exit
+ *
+ * When a lock was initialized in percpu mode (SIX_OLCK_INIT_PCPU), this is
+ * required to free the percpu read counts.
+ */
+void six_lock_exit(struct six_lock *lock)
+{
+	WARN_ON(lock->readers && pcpu_read_count(lock));
+	WARN_ON(atomic_read(&lock->state) & SIX_LOCK_HELD_read);
+
+	free_percpu(lock->readers);
+	lock->readers = NULL;
+}
+EXPORT_SYMBOL_GPL(six_lock_exit);
+
+void __six_lock_init(struct six_lock *lock, const char *name,
+		     struct lock_class_key *key, enum six_lock_init_flags flags)
+{
+	atomic_set(&lock->state, 0);
+	raw_spin_lock_init(&lock->wait_lock);
+	INIT_LIST_HEAD(&lock->wait_list);
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+	debug_check_no_locks_freed((void *) lock, sizeof(*lock));
+	lockdep_init_map(&lock->dep_map, name, key, 0);
+#endif
+
+	if (flags & SIX_LOCK_INIT_PCPU) {
+		/*
+		 * We don't return an error here on memory allocation failure
+		 * since percpu is an optimization, and locks will work with the
+		 * same semantics in non-percpu mode: callers can check for
+		 * failure if they wish by checking lock->readers, but generally
+		 * will not want to treat it as an error.
+		 */
+		lock->readers = alloc_percpu(unsigned);
+	}
+}
+EXPORT_SYMBOL_GPL(__six_lock_init);