1 files changed, 64 insertions, 39 deletions

diff --git a/libbcachefs/btree_iter.h b/libbcachefs/btree_iter.h
index 0097a2a..99e51b2 100644
--- a/libbcachefs/btree_iter.h
+++ b/libbcachefs/btree_iter.h
@@ -28,40 +28,47 @@ static inline bool btree_iter_linked(const struct btree_iter *iter)
 	return iter->next != iter;
 }
 
-/**
- * for_each_linked_btree_iter - iterate over all iterators linked with @_iter
- */
-#define for_each_linked_btree_iter(_iter, _linked)			\
-	for ((_linked) = (_iter)->next;					\
-	     (_linked) != (_iter);					\
-	     (_linked) = (_linked)->next)
+static inline bool __iter_has_node(const struct btree_iter *iter,
+				   const struct btree *b)
+{
+	/*
+	 * We don't compare the low bits of the lock sequence numbers because
+	 * @iter might have taken a write lock on @b, and we don't want to skip
+	 * the linked iterator if the sequence numbers were equal before taking
+	 * that write lock. The lock sequence number is incremented by taking
+	 * and releasing write locks and is even when unlocked:
+	 */
+
+	return iter->l[b->level].b == b &&
+		iter->lock_seq[b->level] >> 1 == b->lock.state.seq >> 1;
+}
 
 static inline struct btree_iter *
-__next_linked_btree_node(struct btree_iter *iter, struct btree *b,
-			 struct btree_iter *linked)
-{
-	do {
-		linked = linked->next;
-
-		if (linked == iter)
-			return NULL;
-
-		/*
-		 * We don't compare the low bits of the lock sequence numbers
-		 * because @iter might have taken a write lock on @b, and we
-		 * don't want to skip the linked iterator if the sequence
-		 * numbers were equal before taking that write lock. The lock
-		 * sequence number is incremented by taking and releasing write
-		 * locks and is even when unlocked:
-		 */
-	} while (linked->l[b->level].b != b ||
-		 linked->lock_seq[b->level] >> 1 != b->lock.state.seq >> 1);
+__next_linked_iter(struct btree_iter *iter, struct btree_iter *linked)
+{
+	return linked->next != iter ? linked->next : NULL;
+}
+
+static inline struct btree_iter *
+__next_iter_with_node(struct btree_iter *iter, struct btree *b,
+		      struct btree_iter *linked)
+{
+	while (linked && !__iter_has_node(linked, b))
+		linked = __next_linked_iter(iter, linked);
 
 	return linked;
 }
 
 /**
- * for_each_linked_btree_node - iterate over all iterators linked with @_iter
+ * for_each_btree_iter - iterate over all iterators linked with @_iter,
+ * including @_iter
+ */
+#define for_each_btree_iter(_iter, _linked)				\
+	for ((_linked) = (_iter); (_linked);				\
+	     (_linked) = __next_linked_iter(_iter, _linked))
+
+/**
+ * for_each_btree_iter_with_node - iterate over all iterators linked with @_iter
  * that also point to @_b
  *
  * @_b is assumed to be locked by @_iter
@@ -69,15 +76,27 @@ __next_linked_btree_node(struct btree_iter *iter, struct btree *b,
  * Filters out iterators that don't have a valid btree_node iterator for @_b -
  * i.e. iterators for which bch2_btree_node_relock() would not succeed.
  */
-#define for_each_linked_btree_node(_iter, _b, _linked)			\
+#define for_each_btree_iter_with_node(_iter, _b, _linked)		\
 	for ((_linked) = (_iter);					\
-	     ((_linked) = __next_linked_btree_node(_iter, _b, _linked));)
+	     ((_linked) = __next_iter_with_node(_iter, _b, _linked));	\
+	     (_linked) = __next_linked_iter(_iter, _linked))
+
+/**
+ * for_each_linked_btree_iter - iterate over all iterators linked with @_iter,
+ * _not_ including @_iter
+ */
+#define for_each_linked_btree_iter(_iter, _linked)			\
+	for ((_linked) = (_iter)->next;					\
+	     (_linked) != (_iter);					\
+	     (_linked) = (_linked)->next)
 
 #ifdef CONFIG_BCACHEFS_DEBUG
 void bch2_btree_iter_verify(struct btree_iter *, struct btree *);
+void bch2_btree_iter_verify_locks(struct btree_iter *);
 #else
 static inline void bch2_btree_iter_verify(struct btree_iter *iter,
-					 struct btree *b) {}
+					  struct btree *b) {}
+static inline void bch2_btree_iter_verify_locks(struct btree_iter *iter) {}
 #endif
 
 void bch2_btree_node_iter_fix(struct btree_iter *, struct btree *,
@@ -85,22 +104,28 @@ void bch2_btree_node_iter_fix(struct btree_iter *, struct btree *,
 			     struct bkey_packed *, unsigned, unsigned);
 
 int bch2_btree_iter_unlock(struct btree_iter *);
-bool __bch2_btree_iter_set_locks_want(struct btree_iter *, unsigned);
 
-static inline bool bch2_btree_iter_set_locks_want(struct btree_iter *iter,
-						 unsigned new_locks_want)
+bool __bch2_btree_iter_upgrade(struct btree_iter *, unsigned);
+
+static inline bool bch2_btree_iter_upgrade(struct btree_iter *iter,
+					   unsigned new_locks_want)
 {
 	new_locks_want = min(new_locks_want, BTREE_MAX_DEPTH);
 
-	if (iter->locks_want == new_locks_want &&
-	    iter->nodes_intent_locked == (1 << new_locks_want) - 1)
-		return true;
+	return iter->locks_want < new_locks_want
+		?  __bch2_btree_iter_upgrade(iter, new_locks_want)
+		: iter->uptodate <= BTREE_ITER_NEED_PEEK;
+}
+
+void __bch2_btree_iter_downgrade(struct btree_iter *, unsigned);
 
-	return __bch2_btree_iter_set_locks_want(iter, new_locks_want);
+static inline void bch2_btree_iter_downgrade(struct btree_iter *iter)
+{
+	if (iter->locks_want > (iter->flags & BTREE_ITER_INTENT) ? 1 : 0)
+		__bch2_btree_iter_downgrade(iter, 0);
 }
 
-bool bch2_btree_iter_node_replace(struct btree_iter *, struct btree *);
-void bch2_btree_iter_node_drop_linked(struct btree_iter *, struct btree *);
+void bch2_btree_iter_node_replace(struct btree_iter *, struct btree *);
 void bch2_btree_iter_node_drop(struct btree_iter *, struct btree *);
 
 void bch2_btree_iter_reinit_node(struct btree_iter *, struct btree *);