Update bcachefs sources to f67089dc9b bcachefs: Convert bch2_sb_to_text to master option list

1 files changed, 85 insertions, 60 deletions

diff --git a/libbcachefs/btree_cache.c b/libbcachefs/btree_cache.c
index 1347b1fc..d91408c2 100644
--- a/libbcachefs/btree_cache.c
+++ b/libbcachefs/btree_cache.c
@@ -42,6 +42,14 @@ static inline unsigned btree_cache_can_free(struct btree_cache *bc)
 	return max_t(int, 0, bc->used - bc->reserve);
 }
 
+static void btree_node_to_freedlist(struct btree_cache *bc, struct btree *b)
+{
+	if (b->c.lock.readers)
+		list_move(&b->list, &bc->freed_pcpu);
+	else
+		list_move(&b->list, &bc->freed_nonpcpu);
+}
+
 static void btree_node_data_free(struct bch_fs *c, struct btree *b)
 {
 	struct btree_cache *bc = &c->btree_cache;
@@ -58,7 +66,8 @@ static void btree_node_data_free(struct bch_fs *c, struct btree *b)
 	b->aux_data = NULL;
 
 	bc->used--;
-	list_move(&b->list, &bc->freed);
+
+	btree_node_to_freedlist(bc, b);
 }
 
 static int bch2_btree_cache_cmp_fn(struct rhashtable_compare_arg *arg,
@@ -163,11 +172,6 @@ int bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b,
 	b->c.level	= level;
 	b->c.btree_id	= id;
 
-	if (level)
-		six_lock_pcpu_alloc(&b->c.lock);
-	else
-		six_lock_pcpu_free_rcu(&b->c.lock);
-
 	mutex_lock(&bc->lock);
 	ret = __bch2_btree_node_hash_insert(bc, b);
 	if (!ret)
@@ -328,17 +332,13 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink,
 	}
 restart:
 	list_for_each_entry_safe(b, t, &bc->live, list) {
-		touched++;
-
-		if (touched >= nr) {
-			/* Save position */
-			if (&t->list != &bc->live)
-				list_move_tail(&bc->live, &t->list);
-			break;
+		/* tweak this */
+		if (btree_node_accessed(b)) {
+			clear_btree_node_accessed(b);
+			goto touched;
 		}
 
-		if (!btree_node_accessed(b) &&
-		    !btree_node_reclaim(c, b)) {
+		if (!btree_node_reclaim(c, b)) {
 			/* can't call bch2_btree_node_hash_remove under lock  */
 			freed++;
 			if (&t->list != &bc->live)
@@ -359,8 +359,18 @@ restart:
 			else if (!mutex_trylock(&bc->lock))
 				goto out;
 			goto restart;
-		} else
-			clear_btree_node_accessed(b);
+		} else {
+			continue;
+		}
+touched:
+		touched++;
+
+		if (touched >= nr) {
+			/* Save position */
+			if (&t->list != &bc->live)
+				list_move_tail(&bc->live, &t->list);
+			break;
+		}
 	}
 
 	mutex_unlock(&bc->lock);
@@ -427,8 +437,10 @@ void bch2_fs_btree_cache_exit(struct bch_fs *c)
 
 	BUG_ON(atomic_read(&c->btree_cache.dirty));
 
-	while (!list_empty(&bc->freed)) {
-		b = list_first_entry(&bc->freed, struct btree, list);
+	list_splice(&bc->freed_pcpu, &bc->freed_nonpcpu);
+
+	while (!list_empty(&bc->freed_nonpcpu)) {
+		b = list_first_entry(&bc->freed_nonpcpu, struct btree, list);
 		list_del(&b->list);
 		six_lock_pcpu_free(&b->c.lock);
 		kfree(b);
@@ -482,7 +494,8 @@ void bch2_fs_btree_cache_init_early(struct btree_cache *bc)
 	mutex_init(&bc->lock);
 	INIT_LIST_HEAD(&bc->live);
 	INIT_LIST_HEAD(&bc->freeable);
-	INIT_LIST_HEAD(&bc->freed);
+	INIT_LIST_HEAD(&bc->freed_pcpu);
+	INIT_LIST_HEAD(&bc->freed_nonpcpu);
 }
 
 /*
@@ -557,10 +570,13 @@ static struct btree *btree_node_cannibalize(struct bch_fs *c)
 	}
 }
 
-struct btree *bch2_btree_node_mem_alloc(struct bch_fs *c)
+struct btree *bch2_btree_node_mem_alloc(struct bch_fs *c, bool pcpu_read_locks)
 {
 	struct btree_cache *bc = &c->btree_cache;
-	struct btree *b;
+	struct list_head *freed = pcpu_read_locks
+		? &bc->freed_pcpu
+		: &bc->freed_nonpcpu;
+	struct btree *b, *b2;
 	u64 start_time = local_clock();
 	unsigned flags;
 
@@ -568,44 +584,49 @@ struct btree *bch2_btree_node_mem_alloc(struct bch_fs *c)
 	mutex_lock(&bc->lock);
 
 	/*
-	 * btree_free() doesn't free memory; it sticks the node on the end of
-	 * the list. Check if there's any freed nodes there:
-	 */
-	list_for_each_entry(b, &bc->freeable, list)
-		if (!btree_node_reclaim(c, b))
-			goto got_node;
-
-	/*
 	 * We never free struct btree itself, just the memory that holds the on
 	 * disk node. Check the freed list before allocating a new one:
 	 */
-	list_for_each_entry(b, &bc->freed, list)
-		if (!btree_node_reclaim(c, b))
+	list_for_each_entry(b, freed, list)
+		if (!btree_node_reclaim(c, b)) {
+			list_del_init(&b->list);
 			goto got_node;
+		}
+
+	b = __btree_node_mem_alloc(c);
+	if (!b)
+		goto err_locked;
 
-	b = NULL;
+	if (pcpu_read_locks)
+		six_lock_pcpu_alloc(&b->c.lock);
+
+	BUG_ON(!six_trylock_intent(&b->c.lock));
+	BUG_ON(!six_trylock_write(&b->c.lock));
 got_node:
-	if (b)
-		list_del_init(&b->list);
-	mutex_unlock(&bc->lock);
 
-	if (!b) {
-		b = __btree_node_mem_alloc(c);
-		if (!b)
-			goto err;
+	/*
+	 * btree_free() doesn't free memory; it sticks the node on the end of
+	 * the list. Check if there's any freed nodes there:
+	 */
+	list_for_each_entry(b2, &bc->freeable, list)
+		if (!btree_node_reclaim(c, b2)) {
+			swap(b->data, b2->data);
+			swap(b->aux_data, b2->aux_data);
+			btree_node_to_freedlist(bc, b2);
+			six_unlock_write(&b2->c.lock);
+			six_unlock_intent(&b2->c.lock);
+			goto got_mem;
+		}
 
-		BUG_ON(!six_trylock_intent(&b->c.lock));
-		BUG_ON(!six_trylock_write(&b->c.lock));
-	}
+	mutex_unlock(&bc->lock);
 
-	if (!b->data) {
-		if (btree_node_data_alloc(c, b, __GFP_NOWARN|GFP_KERNEL))
-			goto err;
+	if (btree_node_data_alloc(c, b, __GFP_NOWARN|GFP_KERNEL))
+		goto err;
 
-		mutex_lock(&bc->lock);
-		bc->used++;
-		mutex_unlock(&bc->lock);
-	}
+	mutex_lock(&bc->lock);
+	bc->used++;
+got_mem:
+	mutex_unlock(&bc->lock);
 
 	BUG_ON(btree_node_hashed(b));
 	BUG_ON(btree_node_dirty(b));
@@ -627,17 +648,21 @@ out:
 	return b;
 err:
 	mutex_lock(&bc->lock);
-
-	if (b) {
-		list_add(&b->list, &bc->freed);
-		six_unlock_write(&b->c.lock);
-		six_unlock_intent(&b->c.lock);
-	}
-
+err_locked:
 	/* Try to cannibalize another cached btree node: */
 	if (bc->alloc_lock == current) {
-		b = btree_node_cannibalize(c);
-		list_del_init(&b->list);
+		b2 = btree_node_cannibalize(c);
+		if (b) {
+			swap(b->data, b2->data);
+			swap(b->aux_data, b2->aux_data);
+			btree_node_to_freedlist(bc, b2);
+			six_unlock_write(&b2->c.lock);
+			six_unlock_intent(&b2->c.lock);
+		} else {
+			b = b2;
+			list_del_init(&b->list);
+		}
+
 		mutex_unlock(&bc->lock);
 
 		bch2_btree_node_hash_remove(bc, b);
@@ -677,7 +702,7 @@ static noinline struct btree *bch2_btree_node_fill(struct bch_fs *c,
 		return ERR_PTR(-EINTR);
 	}
 
-	b = bch2_btree_node_mem_alloc(c);
+	b = bch2_btree_node_mem_alloc(c, level != 0);
 
 	if (trans && b == ERR_PTR(-ENOMEM)) {
 		trans->memory_allocation_failure = true;