Update bcachefs sources to 31c09369cd six locks: Fix an unitialized var

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

3 files changed, 398 insertions, 284 deletions

diff --git a/include/linux/atomic.h b/include/linux/atomic.h
index a9852fa1..79cf5aa9 100644
--- a/include/linux/atomic.h
+++ b/include/linux/atomic.h
@@ -32,6 +32,8 @@ typedef struct {
 #define __ATOMIC_SUB(v, p)		uatomic_sub(p, v)
 #define __ATOMIC_INC(p)			uatomic_inc(p)
 #define __ATOMIC_DEC(p)			uatomic_dec(p)
+#define __ATOMIC_AND(v, p)		uatomic_and(p, v)
+#define __ATOMIC_OR(v, p)		uatomic_or(p, v)
 
 #define xchg(p, v)			uatomic_xchg(p, v)
 #define xchg_acquire(p, v)		uatomic_xchg(p, v)
@@ -56,6 +58,8 @@ typedef struct {
 #define __ATOMIC_SUB_RETURN(v, p)	__atomic_sub_fetch(p, v, __ATOMIC_RELAXED)
 #define __ATOMIC_SUB_RETURN_RELEASE(v, p)				\
 					__atomic_sub_fetch(p, v, __ATOMIC_RELEASE)
+#define __ATOMIC_AND(p)			__atomic_and_fetch(p, v, __ATOMIC_RELAXED)
+#define __ATOMIC_OR(p)			__atomic_or_fetch(p, v, __ATOMIC_RELAXED)
 
 #define xchg(p, v)			__atomic_exchange_n(p, v, __ATOMIC_SEQ_CST)
 #define xchg_acquire(p, v)		__atomic_exchange_n(p, v, __ATOMIC_ACQUIRE)
@@ -244,6 +248,16 @@ static inline bool a_type##_inc_not_zero(a_type##_t *v)			\
 	return a_type##_add_unless(v, 1, 0);				\
 }									\
 									\
+static inline void a_type##_and(i_type a, a_type##_t *v)		\
+{									\
+	__ATOMIC_AND(a, v);						\
+}									\
+									\
+static inline void a_type##_or(i_type a, a_type##_t *v)			\
+{									\
+	__ATOMIC_OR(a, v);						\
+}									\
+									\
 static inline i_type a_type##_xchg(a_type##_t *v, i_type i)		\
 {									\
 	return xchg(&v->counter, i);					\
diff --git a/include/linux/mean_and_variance.h b/include/linux/mean_and_variance.h
index 756eb3d1..9ed79f42 100644
--- a/include/linux/mean_and_variance.h
+++ b/include/linux/mean_and_variance.h
@@ -2,122 +2,112 @@
 #ifndef MEAN_AND_VARIANCE_H_
 #define MEAN_AND_VARIANCE_H_
 
-#include <linux/kernel.h>
 #include <linux/types.h>
+#include <linux/kernel.h>
 #include <linux/limits.h>
 #include <linux/math64.h>
+#include <stdlib.h>
 
 #define SQRT_U64_MAX 4294967295ULL
 
-/**
- * abs - return absolute value of an argument
- * @x: the value.  If it is unsigned type, it is converted to signed type first.
- *     char is treated as if it was signed (regardless of whether it really is)
- *     but the macro's return type is preserved as char.
- *
- * Return: an absolute value of x.
+/*
+ * u128_u: u128 user mode, because not all architectures support a real int128
+ * type
  */
-#define abs(x)	__abs_choose_expr(x, long long,				\
-		__abs_choose_expr(x, long,				\
-		__abs_choose_expr(x, int,				\
-		__abs_choose_expr(x, short,				\
-		__abs_choose_expr(x, char,				\
-		__builtin_choose_expr(					\
-			__builtin_types_compatible_p(typeof(x), char),	\
-			(char)({ signed char __x = (x); __x<0?-__x:__x; }), \
-			((void)0)))))))
 
-#define __abs_choose_expr(x, type, other) __builtin_choose_expr(	\
-	__builtin_types_compatible_p(typeof(x),   signed type) ||	\
-	__builtin_types_compatible_p(typeof(x), unsigned type),		\
-	({ signed type __x = (x); __x < 0 ? -__x : __x; }), other)
+#ifdef __SIZEOF_INT128__
 
-#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
-
-typedef unsigned __int128 u128;
+typedef struct {
+	unsigned __int128 v;
+} __aligned(16) u128_u;
 
-static inline u128 u64_to_u128(u64 a)
+static inline u128_u u64_to_u128(u64 a)
 {
-	return (u128)a;
+	return (u128_u) { .v = a };
 }
 
-static inline u64 u128_to_u64(u128 a)
+static inline u64 u128_lo(u128_u a)
 {
-	return (u64)a;
+	return a.v;
 }
 
-static inline u64 u128_shr64_to_u64(u128 a)
+static inline u64 u128_hi(u128_u a)
 {
-	return (u64)(a >> 64);
+	return a.v >> 64;
 }
 
-static inline u128 u128_add(u128 a, u128 b)
+static inline u128_u u128_add(u128_u a, u128_u b)
 {
-	return a + b;
+	a.v += b.v;
+	return a;
 }
 
-static inline u128 u128_sub(u128 a, u128 b)
+static inline u128_u u128_sub(u128_u a, u128_u b)
 {
-	return a - b;
+	a.v -= b.v;
+	return a;
 }
 
-static inline u128 u128_shl(u128 i, s8 shift)
+static inline u128_u u128_shl(u128_u a, s8 shift)
 {
-	return i << shift;
+	a.v <<= shift;
+	return a;
 }
 
-static inline u128 u128_shl64_add(u64 a, u64 b)
+static inline u128_u u128_square(u64 a)
 {
-	return ((u128)a << 64) + b;
-}
+	u128_u b = u64_to_u128(a);
 
-static inline u128 u128_square(u64 i)
-{
-	return i*i;
+	b.v *= b.v;
+	return b;
 }
 
 #else
 
 typedef struct {
 	u64 hi, lo;
-} u128;
+} __aligned(16) u128_u;
+
+/* conversions */
 
-static inline u128 u64_to_u128(u64 a)
+static inline u128_u u64_to_u128(u64 a)
 {
-	return (u128){ .lo = a };
+	return (u128_u) { .lo = a };
 }
 
-static inline u64 u128_to_u64(u128 a)
+static inline u64 u128_lo(u128_u a)
 {
 	return a.lo;
 }
 
-static inline u64 u128_shr64_to_u64(u128 a)
+static inline u64 u128_hi(u128_u a)
 {
 	return a.hi;
 }
 
-static inline u128 u128_add(u128 a, u128 b)
+/* arithmetic */
+
+static inline u128_u u128_add(u128_u a, u128_u b)
 {
-	u128 c;
+	u128_u c;
 
 	c.lo = a.lo + b.lo;
 	c.hi = a.hi + b.hi + (c.lo < a.lo);
 	return c;
 }
 
-static inline u128 u128_sub(u128 a, u128 b)
+static inline u128_u u128_sub(u128_u a, u128_u b)
 {
-	u128 c;
+	u128_u c;
 
 	c.lo = a.lo - b.lo;
 	c.hi = a.hi - b.hi - (c.lo > a.lo);
 	return c;
 }
 
-static inline u128 u128_shl(u128 i, s8 shift)
+static inline u128_u u128_shl(u128_u i, s8 shift)
 {
-	u128 r;
+	u128_u r;
 
 	r.lo = i.lo << shift;
 	if (shift < 64)
@@ -129,15 +119,10 @@ static inline u128 u128_shl(u128 i, s8 shift)
 	return r;
 }
 
-static inline u128 u128_shl64_add(u64 a, u64 b)
+static inline u128_u u128_square(u64 i)
 {
-	return u128_add(u128_shl(u64_to_u128(a), 64), u64_to_u128(b));
-}
-
-static inline u128 u128_square(u64 i)
-{
-	u128 r;
-	u64  h = i >> 32, l = i & (u64)U32_MAX;
+	u128_u r;
+	u64  h = i >> 32, l = i & U32_MAX;
 
 	r =             u128_shl(u64_to_u128(h*h), 64);
 	r = u128_add(r, u128_shl(u64_to_u128(h*l), 32));
@@ -148,85 +133,69 @@ static inline u128 u128_square(u64 i)
 
 #endif
 
-static inline u128 u128_div(u128 n, u64 d)
+static inline u128_u u64s_to_u128(u64 hi, u64 lo)
 {
-	u128 r;
-	u64 rem;
-	u64 hi = u128_shr64_to_u64(n);
-	u64 lo = u128_to_u64(n);
-	u64  h =  hi & ((u64)U32_MAX  << 32);
-	u64  l = (hi &  (u64)U32_MAX) << 32;
+	u128_u c = u64_to_u128(hi);
 
-	r =             u128_shl(u64_to_u128(div64_u64_rem(h,                d, &rem)), 64);
-	r = u128_add(r, u128_shl(u64_to_u128(div64_u64_rem(l  + (rem << 32), d, &rem)), 32));
-	r = u128_add(r,          u64_to_u128(div64_u64_rem(lo + (rem << 32), d, &rem)));
-	return r;
+	c = u128_shl(c, 64);
+	c = u128_add(c, u64_to_u128(lo));
+	return c;
 }
 
+u128_u u128_div(u128_u n, u64 d);
+
 struct mean_and_variance {
-	s64 n;
-	s64 sum;
-	u128 sum_squares;
+	s64	n;
+	s64	sum;
+	u128_u	sum_squares;
 };
 
 /* expontentially weighted variant */
 struct mean_and_variance_weighted {
-	bool init;
-	u8 w;
-	s64 mean;
-	u64 variance;
+	bool	init;
+	u8	weight;	/* base 2 logarithim */
+	s64	mean;
+	u64	variance;
 };
 
-s64 fast_divpow2(s64 n, u8 d);
+/**
+ * fast_divpow2() - fast approximation for n / (1 << d)
+ * @n: numerator
+ * @d: the power of 2 denominator.
+ *
+ * note: this rounds towards 0.
+ */
+static inline s64 fast_divpow2(s64 n, u8 d)
+{
+	return (n + ((n < 0) ? ((1 << d) - 1) : 0)) >> d;
+}
 
+/**
+ * mean_and_variance_update() - update a mean_and_variance struct @s1 with a new sample @v1
+ * and return it.
+ * @s1: the mean_and_variance to update.
+ * @v1: the new sample.
+ *
+ * see linked pdf equation 12.
+ */
 static inline struct mean_and_variance
-mean_and_variance_update_inlined(struct mean_and_variance s1, s64 v1)
-{
-	struct mean_and_variance s2;
-	u64 v2 = abs(v1);
-
-	s2.n           = s1.n + 1;
-	s2.sum         = s1.sum + v1;
-	s2.sum_squares = u128_add(s1.sum_squares, u128_square(v2));
-	return s2;
-}
-
-static inline struct mean_and_variance_weighted
-mean_and_variance_weighted_update_inlined(struct mean_and_variance_weighted s1, s64 x)
-{
-	struct mean_and_variance_weighted s2;
-	// previous weighted variance.
-	u64 var_w0 = s1.variance;
-	u8 w = s2.w = s1.w;
-	// new value weighted.
-	s64 x_w = x << w;
-	s64 diff_w = x_w - s1.mean;
-	s64 diff = fast_divpow2(diff_w, w);
-	// new mean weighted.
-	s64 u_w1     = s1.mean + diff;
-
-	BUG_ON(w % 2 != 0);
-
-	if (!s1.init) {
-		s2.mean = x_w;
-		s2.variance = 0;
-	} else {
-		s2.mean = u_w1;
-		s2.variance = ((var_w0 << w) - var_w0 + ((diff_w * (x_w - u_w1)) >> w)) >> w;
-	}
-	s2.init = true;
-
-	return s2;
+mean_and_variance_update(struct mean_and_variance s, s64 v)
+{
+	return (struct mean_and_variance) {
+		.n           = s.n + 1,
+		.sum         = s.sum + v,
+		.sum_squares = u128_add(s.sum_squares, u128_square(abs(v))),
+	};
 }
 
-struct mean_and_variance mean_and_variance_update(struct mean_and_variance s1, s64 v1);
-       s64		 mean_and_variance_get_mean(struct mean_and_variance s);
-       u64		 mean_and_variance_get_variance(struct mean_and_variance s1);
-       u32		 mean_and_variance_get_stddev(struct mean_and_variance s);
+s64 mean_and_variance_get_mean(struct mean_and_variance s);
+u64 mean_and_variance_get_variance(struct mean_and_variance s1);
+u32 mean_and_variance_get_stddev(struct mean_and_variance s);
+
+void mean_and_variance_weighted_update(struct mean_and_variance_weighted *s, s64 v);
 
-struct mean_and_variance_weighted mean_and_variance_weighted_update(struct mean_and_variance_weighted s1, s64 v1);
-       s64			  mean_and_variance_weighted_get_mean(struct mean_and_variance_weighted s);
-       u64			  mean_and_variance_weighted_get_variance(struct mean_and_variance_weighted s);
-       u32			  mean_and_variance_weighted_get_stddev(struct mean_and_variance_weighted s);
+s64 mean_and_variance_weighted_get_mean(struct mean_and_variance_weighted s);
+u64 mean_and_variance_weighted_get_variance(struct mean_and_variance_weighted s);
+u32 mean_and_variance_weighted_get_stddev(struct mean_and_variance_weighted s);
 
 #endif // MEAN_AND_VAIRANCE_H_
diff --git a/include/linux/six.h b/include/linux/six.h
index 83023f64..394da423 100644
--- a/include/linux/six.h
+++ b/include/linux/six.h
@@ -3,59 +3,124 @@
 #ifndef _LINUX_SIX_H
 #define _LINUX_SIX_H
 
-/*
- * Shared/intent/exclusive locks: sleepable read/write locks, much like rw
- * semaphores, except with a third intermediate state, intent. Basic operations
- * are:
+/**
+ * DOC: SIX locks overview
  *
- * six_lock_read(&foo->lock);
- * six_unlock_read(&foo->lock);
+ * Shared/intent/exclusive locks: sleepable read/write locks, like rw semaphores
+ * but with an additional state: read/shared, intent, exclusive/write
  *
- * six_lock_intent(&foo->lock);
- * six_unlock_intent(&foo->lock);
+ * The purpose of the intent state is to allow for greater concurrency on tree
+ * structures without deadlocking. In general, a read can't be upgraded to a
+ * write lock without deadlocking, so an operation that updates multiple nodes
+ * will have to take write locks for the full duration of the operation.
  *
- * six_lock_write(&foo->lock);
- * six_unlock_write(&foo->lock);
+ * But by adding an intent state, which is exclusive with other intent locks but
+ * not with readers, we can take intent locks at thte start of the operation,
+ * and then take write locks only for the actual update to each individual
+ * nodes, without deadlocking.
  *
- * Intent locks block other intent locks, but do not block read locks, and you
- * must have an intent lock held before taking a write lock, like so:
+ * Example usage:
+ *   six_lock_read(&foo->lock);
+ *   six_unlock_read(&foo->lock);
  *
- * six_lock_intent(&foo->lock);
- * six_lock_write(&foo->lock);
- * six_unlock_write(&foo->lock);
- * six_unlock_intent(&foo->lock);
+ * An intent lock must be held before taking a write lock:
+ *   six_lock_intent(&foo->lock);
+ *   six_lock_write(&foo->lock);
+ *   six_unlock_write(&foo->lock);
+ *   six_unlock_intent(&foo->lock);
  *
  * Other operations:
- *
  *   six_trylock_read()
  *   six_trylock_intent()
  *   six_trylock_write()
  *
- *   six_lock_downgrade():	convert from intent to read
- *   six_lock_tryupgrade():	attempt to convert from read to intent
- *
- * Locks also embed a sequence number, which is incremented when the lock is
- * locked or unlocked for write. The current sequence number can be grabbed
- * while a lock is held from lock->state.seq; then, if you drop the lock you can
- * use six_relock_(read|intent_write)(lock, seq) to attempt to retake the lock
- * iff it hasn't been locked for write in the meantime.
- *
- * There are also operations that take the lock type as a parameter, where the
- * type is one of SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write:
- *
- *   six_lock_type(lock, type)
- *   six_unlock_type(lock, type)
- *   six_relock(lock, type, seq)
- *   six_trylock_type(lock, type)
- *   six_trylock_convert(lock, from, to)
- *
- * A lock may be held multiple times by the same thread (for read or intent,
- * not write). However, the six locks code does _not_ implement the actual
- * recursive checks itself though - rather, if your code (e.g. btree iterator
- * code) knows that the current thread already has a lock held, and for the
- * correct type, six_lock_increment() may be used to bump up the counter for
- * that type - the only effect is that one more call to unlock will be required
- * before the lock is unlocked.
+ *   six_lock_downgrade()	convert from intent to read
+ *   six_lock_tryupgrade()	attempt to convert from read to intent, may fail
+ *
+ * There are also interfaces that take the lock type as an enum:
+ *
+ *   six_lock_type(&foo->lock, SIX_LOCK_read);
+ *   six_trylock_convert(&foo->lock, SIX_LOCK_read, SIX_LOCK_intent)
+ *   six_lock_type(&foo->lock, SIX_LOCK_write);
+ *   six_unlock_type(&foo->lock, SIX_LOCK_write);
+ *   six_unlock_type(&foo->lock, SIX_LOCK_intent);
+ *
+ * Lock sequence numbers - unlock(), relock():
+ *
+ *   Locks embed sequences numbers, which are incremented on write lock/unlock.
+ *   This allows locks to be dropped and the retaken iff the state they protect
+ *   hasn't changed; this makes it much easier to avoid holding locks while e.g.
+ *   doing IO or allocating memory.
+ *
+ *   Example usage:
+ *     six_lock_read(&foo->lock);
+ *     u32 seq = six_lock_seq(&foo->lock);
+ *     six_unlock_read(&foo->lock);
+ *
+ *     some_operation_that_may_block();
+ *
+ *     if (six_relock_read(&foo->lock, seq)) { ... }
+ *
+ *   If the relock operation succeeds, it is as if the lock was never unlocked.
+ *
+ * Reentrancy:
+ *
+ *   Six locks are not by themselves reentrent, but have counters for both the
+ *   read and intent states that can be used to provide reentrency by an upper
+ *   layer that tracks held locks. If a lock is known to already be held in the
+ *   read or intent state, six_lock_increment() can be used to bump the "lock
+ *   held in this state" counter, increasing the number of unlock calls that
+ *   will be required to fully unlock it.
+ *
+ *   Example usage:
+ *     six_lock_read(&foo->lock);
+ *     six_lock_increment(&foo->lock, SIX_LOCK_read);
+ *     six_unlock_read(&foo->lock);
+ *     six_unlock_read(&foo->lock);
+ *   foo->lock is now fully unlocked.
+ *
+ *   Since the intent state supercedes read, it's legal to increment the read
+ *   counter when holding an intent lock, but not the reverse.
+ *
+ *   A lock may only be held once for write: six_lock_increment(.., SIX_LOCK_write)
+ *   is not legal.
+ *
+ * should_sleep_fn:
+ *
+ *   There is a six_lock() variant that takes a function pointer that is called
+ *   immediately prior to schedule() when blocking, and may return an error to
+ *   abort.
+ *
+ *   One possible use for this feature is when objects being locked are part of
+ *   a cache and may reused, and lock ordering is based on a property of the
+ *   object that will change when the object is reused - i.e. logical key order.
+ *
+ *   If looking up an object in the cache may race with object reuse, and lock
+ *   ordering is required to prevent deadlock, object reuse may change the
+ *   correct lock order for that object and cause a deadlock. should_sleep_fn
+ *   can be used to check if the object is still the object we want and avoid
+ *   this deadlock.
+ *
+ * Wait list entry interface:
+ *
+ *   There is a six_lock() variant, six_lock_waiter(), that takes a pointer to a
+ *   wait list entry. By embedding six_lock_waiter into another object, and by
+ *   traversing lock waitlists, it is then possible for an upper layer to
+ *   implement full cycle detection for deadlock avoidance.
+ *
+ *   should_sleep_fn should be used for invoking the cycle detector, walking the
+ *   graph of held locks to check for a deadlock. The upper layer must track
+ *   held locks for each thread, and each thread's held locks must be reachable
+ *   from its six_lock_waiter object.
+ *
+ *   six_lock_waiter() will add the wait object to the waitlist re-trying taking
+ *   the lock, and before calling should_sleep_fn, and the wait object will not
+ *   be removed from the waitlist until either the lock has been successfully
+ *   acquired, or we aborted because should_sleep_fn returned an error.
+ *
+ *   Also, six_lock_waiter contains a timestamp, and waiters on a waitlist will
+ *   have timestamps in strictly ascending order - this is so the timestamp can
+ *   be used as a cursor for lock graph traverse.
  */
 
 #include <linux/lockdep.h>
@@ -63,41 +128,6 @@
 #include <linux/sched.h>
 #include <linux/types.h>
 
-#define SIX_LOCK_SEPARATE_LOCKFNS
-
-union six_lock_state {
-	struct {
-		atomic64_t	counter;
-	};
-
-	struct {
-		u64		v;
-	};
-
-	struct {
-		/* for waitlist_bitnr() */
-		unsigned long	l;
-	};
-
-	struct {
-		unsigned	read_lock:26;
-		unsigned	write_locking:1;
-		unsigned	intent_lock:1;
-		unsigned	nospin:1;
-		unsigned	waiters:3;
-		/*
-		 * seq works much like in seqlocks: it's incremented every time
-		 * we lock and unlock for write.
-		 *
-		 * If it's odd write lock is held, even unlocked.
-		 *
-		 * Thus readers can unlock, and then lock again later iff it
-		 * hasn't been modified in the meantime.
-		 */
-		u32		seq;
-	};
-};
-
 enum six_lock_type {
 	SIX_LOCK_read,
 	SIX_LOCK_intent,
@@ -105,7 +135,8 @@ enum six_lock_type {
 };
 
 struct six_lock {
-	union six_lock_state	state;
+	atomic_t		state;
+	u32			seq;
 	unsigned		intent_lock_recurse;
 	struct task_struct	*owner;
 	unsigned __percpu	*readers;
@@ -127,59 +158,210 @@ struct six_lock_waiter {
 
 typedef int (*six_lock_should_sleep_fn)(struct six_lock *lock, void *);
 
-static __always_inline void __six_lock_init(struct six_lock *lock,
-					    const char *name,
-					    struct lock_class_key *key)
-{
-	atomic64_set(&lock->state.counter, 0);
-	raw_spin_lock_init(&lock->wait_lock);
-	INIT_LIST_HEAD(&lock->wait_list);
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-	debug_check_no_locks_freed((void *) lock, sizeof(*lock));
-	lockdep_init_map(&lock->dep_map, name, key, 0);
-#endif
-}
+void six_lock_exit(struct six_lock *lock);
+
+enum six_lock_init_flags {
+	SIX_LOCK_INIT_PCPU	= 1U << 0,
+};
 
-#define six_lock_init(lock)						\
+void __six_lock_init(struct six_lock *lock, const char *name,
+		     struct lock_class_key *key, enum six_lock_init_flags flags);
+
+/**
+ * six_lock_init - initialize a six lock
+ * @lock:	lock to initialize
+ * @flags:	optional flags, i.e. SIX_LOCK_INIT_PCPU
+ */
+#define six_lock_init(lock, flags)					\
 do {									\
 	static struct lock_class_key __key;				\
 									\
-	__six_lock_init((lock), #lock, &__key);				\
+	__six_lock_init((lock), #lock, &__key, flags);			\
 } while (0)
 
-#define __SIX_VAL(field, _v)	(((union six_lock_state) { .field = _v }).v)
+/**
+ * six_lock_seq - obtain current lock sequence number
+ * @lock:	six_lock to obtain sequence number for
+ *
+ * @lock should be held for read or intent, and not write
+ *
+ * By saving the lock sequence number, we can unlock @lock and then (typically
+ * after some blocking operation) attempt to relock it: the relock will succeed
+ * if the sequence number hasn't changed, meaning no write locks have been taken
+ * and state corresponding to what @lock protects is still valid.
+ */
+static inline u32 six_lock_seq(const struct six_lock *lock)
+{
+	return lock->seq;
+}
+
+bool six_trylock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip);
+
+/**
+ * six_trylock_type - attempt to take a six lock without blocking
+ * @lock:	lock to take
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ *
+ * Return: true on success, false on failure.
+ */
+static inline bool six_trylock_type(struct six_lock *lock, enum six_lock_type type)
+{
+	return six_trylock_ip(lock, type, _THIS_IP_);
+}
+
+int six_lock_ip_waiter(struct six_lock *lock, enum six_lock_type type,
+		       struct six_lock_waiter *wait,
+		       six_lock_should_sleep_fn should_sleep_fn, void *p,
+		       unsigned long ip);
+
+/**
+ * six_lock_waiter - take a lock, with full waitlist interface
+ * @lock:	lock to take
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ * @wait:	pointer to wait object, which will be added to lock's waitlist
+ * @should_sleep_fn: callback run after adding to waitlist, immediately prior
+ *		to scheduling
+ * @p:		passed through to @should_sleep_fn
+ *
+ * This is a convenience wrapper around six_lock_ip_waiter(), see that function
+ * for full documentation.
+ *
+ * Return: 0 on success, or the return code from @should_sleep_fn on failure.
+ */
+static inline int six_lock_waiter(struct six_lock *lock, enum six_lock_type type,
+				  struct six_lock_waiter *wait,
+				  six_lock_should_sleep_fn should_sleep_fn, void *p)
+{
+	return six_lock_ip_waiter(lock, type, wait, should_sleep_fn, p, _THIS_IP_);
+}
+
+/**
+ * six_lock_ip - take a six lock lock
+ * @lock:	lock to take
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ * @should_sleep_fn: callback run after adding to waitlist, immediately prior
+ *		to scheduling
+ * @p:		passed through to @should_sleep_fn
+ * @ip:		ip parameter for lockdep/lockstat, i.e. _THIS_IP_
+ *
+ * Return: 0 on success, or the return code from @should_sleep_fn on failure.
+ */
+static inline int six_lock_ip(struct six_lock *lock, enum six_lock_type type,
+			      six_lock_should_sleep_fn should_sleep_fn, void *p,
+			      unsigned long ip)
+{
+	struct six_lock_waiter wait;
+
+	return six_lock_ip_waiter(lock, type, &wait, should_sleep_fn, p, ip);
+}
+
+/**
+ * six_lock_type - take a six lock lock
+ * @lock:	lock to take
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ * @should_sleep_fn: callback run after adding to waitlist, immediately prior
+ *		to scheduling
+ * @p:		passed through to @should_sleep_fn
+ *
+ * Return: 0 on success, or the return code from @should_sleep_fn on failure.
+ */
+static inline int six_lock_type(struct six_lock *lock, enum six_lock_type type,
+				six_lock_should_sleep_fn should_sleep_fn, void *p)
+{
+	struct six_lock_waiter wait;
+
+	return six_lock_ip_waiter(lock, type, &wait, should_sleep_fn, p, _THIS_IP_);
+}
+
+bool six_relock_ip(struct six_lock *lock, enum six_lock_type type,
+		   unsigned seq, unsigned long ip);
+
+/**
+ * six_relock_type - attempt to re-take a lock that was held previously
+ * @lock:	lock to take
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ * @seq:	lock sequence number obtained from six_lock_seq() while lock was
+ *		held previously
+ *
+ * Return: true on success, false on failure.
+ */
+static inline bool six_relock_type(struct six_lock *lock, enum six_lock_type type,
+				   unsigned seq)
+{
+	return six_relock_ip(lock, type, seq, _THIS_IP_);
+}
+
+void six_unlock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip);
+
+/**
+ * six_unlock_type - drop a six lock
+ * @lock:	lock to unlock
+ * @type:	SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
+ *
+ * When a lock is held multiple times (because six_lock_incement()) was used),
+ * this decrements the 'lock held' counter by one.
+ *
+ * For example:
+ * six_lock_read(&foo->lock);				read count 1
+ * six_lock_increment(&foo->lock, SIX_LOCK_read);	read count 2
+ * six_lock_unlock(&foo->lock, SIX_LOCK_read);		read count 1
+ * six_lock_unlock(&foo->lock, SIX_LOCK_read);		read count 0
+ */
+static inline void six_unlock_type(struct six_lock *lock, enum six_lock_type type)
+{
+	six_unlock_ip(lock, type, _THIS_IP_);
+}
 
 #define __SIX_LOCK(type)						\
-bool six_trylock_ip_##type(struct six_lock *, unsigned long);		\
-bool six_relock_ip_##type(struct six_lock *, u32, unsigned long);	\
-int six_lock_ip_##type(struct six_lock *, six_lock_should_sleep_fn,	\
-		       void *, unsigned long);				\
-int six_lock_ip_waiter_##type(struct six_lock *, struct six_lock_waiter *,\
-			six_lock_should_sleep_fn, void *, unsigned long);\
-void six_unlock_ip_##type(struct six_lock *, unsigned long);		\
+static inline bool six_trylock_ip_##type(struct six_lock *lock, unsigned long ip)\
+{									\
+	return six_trylock_ip(lock, SIX_LOCK_##type, ip);		\
+}									\
 									\
 static inline bool six_trylock_##type(struct six_lock *lock)		\
 {									\
-	return six_trylock_ip_##type(lock, _THIS_IP_);			\
+	return six_trylock_ip(lock, SIX_LOCK_##type, _THIS_IP_);	\
+}									\
+									\
+static inline int six_lock_ip_waiter_##type(struct six_lock *lock,	\
+			   struct six_lock_waiter *wait,		\
+			   six_lock_should_sleep_fn should_sleep_fn, void *p,\
+			   unsigned long ip)				\
+{									\
+	return six_lock_ip_waiter(lock, SIX_LOCK_##type, wait, should_sleep_fn, p, ip);\
+}									\
+									\
+static inline int six_lock_ip_##type(struct six_lock *lock,		\
+		    six_lock_should_sleep_fn should_sleep_fn, void *p,	\
+		    unsigned long ip)					\
+{									\
+	return six_lock_ip(lock, SIX_LOCK_##type, should_sleep_fn, p, ip);\
+}									\
+									\
+static inline bool six_relock_ip_##type(struct six_lock *lock, u32 seq, unsigned long ip)\
+{									\
+	return six_relock_ip(lock, SIX_LOCK_##type, seq, ip);		\
 }									\
+									\
 static inline bool six_relock_##type(struct six_lock *lock, u32 seq)	\
 {									\
-	return six_relock_ip_##type(lock, seq, _THIS_IP_);		\
+	return six_relock_ip(lock, SIX_LOCK_##type, seq, _THIS_IP_);	\
 }									\
+									\
 static inline int six_lock_##type(struct six_lock *lock,		\
 				  six_lock_should_sleep_fn fn, void *p)\
 {									\
 	return six_lock_ip_##type(lock, fn, p, _THIS_IP_);		\
 }									\
-static inline int six_lock_waiter_##type(struct six_lock *lock,		\
-			struct six_lock_waiter *wait,			\
-			six_lock_should_sleep_fn fn, void *p)		\
+									\
+static inline void six_unlock_ip_##type(struct six_lock *lock, unsigned long ip)	\
 {									\
-	return six_lock_ip_waiter_##type(lock, wait, fn, p, _THIS_IP_);	\
+	six_unlock_ip(lock, SIX_LOCK_##type, ip);			\
 }									\
+									\
 static inline void six_unlock_##type(struct six_lock *lock)		\
 {									\
-	return six_unlock_ip_##type(lock, _THIS_IP_);			\
+	six_unlock_ip(lock, SIX_LOCK_##type, _THIS_IP_);		\
 }
 
 __SIX_LOCK(read)
@@ -187,55 +369,6 @@ __SIX_LOCK(intent)
 __SIX_LOCK(write)
 #undef __SIX_LOCK
 
-#define SIX_LOCK_DISPATCH(type, fn, ...)			\
-	switch (type) {						\
-	case SIX_LOCK_read:					\
-		return fn##_read(__VA_ARGS__);			\
-	case SIX_LOCK_intent:					\
-		return fn##_intent(__VA_ARGS__);		\
-	case SIX_LOCK_write:					\
-		return fn##_write(__VA_ARGS__);			\
-	default:						\
-		BUG();						\
-	}
-
-static inline bool six_trylock_type(struct six_lock *lock, enum six_lock_type type)
-{
-	SIX_LOCK_DISPATCH(type, six_trylock, lock);
-}
-
-static inline bool six_relock_type(struct six_lock *lock, enum six_lock_type type,
-				   unsigned seq)
-{
-	SIX_LOCK_DISPATCH(type, six_relock, lock, seq);
-}
-
-static inline int six_lock_type(struct six_lock *lock, enum six_lock_type type,
-				six_lock_should_sleep_fn should_sleep_fn, void *p)
-{
-	SIX_LOCK_DISPATCH(type, six_lock, lock, should_sleep_fn, p);
-}
-
-static inline int six_lock_type_ip_waiter(struct six_lock *lock, enum six_lock_type type,
-				struct six_lock_waiter *wait,
-				six_lock_should_sleep_fn should_sleep_fn, void *p,
-				unsigned long ip)
-{
-	SIX_LOCK_DISPATCH(type, six_lock_ip_waiter, lock, wait, should_sleep_fn, p, ip);
-}
-
-static inline int six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type,
-				struct six_lock_waiter *wait,
-				six_lock_should_sleep_fn should_sleep_fn, void *p)
-{
-	SIX_LOCK_DISPATCH(type, six_lock_waiter, lock, wait, should_sleep_fn, p);
-}
-
-static inline void six_unlock_type(struct six_lock *lock, enum six_lock_type type)
-{
-	SIX_LOCK_DISPATCH(type, six_unlock, lock);
-}
-
 void six_lock_downgrade(struct six_lock *);
 bool six_lock_tryupgrade(struct six_lock *);
 bool six_trylock_convert(struct six_lock *, enum six_lock_type,
@@ -245,13 +378,11 @@ void six_lock_increment(struct six_lock *, enum six_lock_type);
 
 void six_lock_wakeup_all(struct six_lock *);
 
-void six_lock_pcpu_free(struct six_lock *);
-void six_lock_pcpu_alloc(struct six_lock *);
-
 struct six_lock_count {
 	unsigned n[3];
 };
 
 struct six_lock_count six_lock_counts(struct six_lock *);
+void six_lock_readers_add(struct six_lock *, int);
 
 #endif /* _LINUX_SIX_H */