summaryrefslogtreecommitdiff
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/md/bcache/btree_cache.c30
-rw-r--r--drivers/md/bcache/btree_cache.h2
-rw-r--r--drivers/md/bcache/btree_gc.c8
-rw-r--r--drivers/md/bcache/btree_io.c2
-rw-r--r--drivers/md/bcache/btree_iter.c418
-rw-r--r--drivers/md/bcache/btree_iter.h67
-rw-r--r--drivers/md/bcache/btree_locking.h58
-rw-r--r--drivers/md/bcache/btree_update.c39
-rw-r--r--drivers/md/bcache/extents.c67
-rw-r--r--drivers/md/bcache/snapshot.h6
10 files changed, 434 insertions, 263 deletions
diff --git a/drivers/md/bcache/btree_cache.c b/drivers/md/bcache/btree_cache.c
index c36366cd777c..eaa482231b43 100644
--- a/drivers/md/bcache/btree_cache.c
+++ b/drivers/md/bcache/btree_cache.c
@@ -560,9 +560,10 @@ err:
/* Slowpath, don't want it inlined into btree_iter_traverse() */
static noinline struct btree *bch_btree_node_fill(struct btree_iter *iter,
- const struct bkey_i *k,
- unsigned level,
- struct closure *cl)
+ struct btree_iter_state *_iter,
+ const struct bkey_i *k,
+ unsigned level,
+ struct closure *cl)
{
struct cache_set *c = iter->c;
struct btree *b;
@@ -598,13 +599,13 @@ static noinline struct btree *bch_btree_node_fill(struct btree_iter *iter,
* But the deadlock described below doesn't exist in this case,
* so it's safe to not drop the parent lock until here:
*/
- if (btree_node_read_locked(iter, level + 1))
- btree_node_unlock(iter, level + 1);
+ if (btree_node_read_locked(_iter, level + 1))
+ btree_node_unlock(_iter, level + 1);
bch_btree_node_read(c, b);
six_unlock_write(&b->lock);
- mark_btree_node_locked(iter, level, btree_lock_want(iter, level));
+ mark_btree_node_locked(_iter, level, btree_lock_want(iter, level));
if (btree_lock_want(iter, level) == SIX_LOCK_read)
BUG_ON(!six_trylock_convert(&b->lock,
@@ -624,11 +625,12 @@ static noinline struct btree *bch_btree_node_fill(struct btree_iter *iter,
* the @write parameter.
*/
struct btree *bch_btree_node_get(struct btree_iter *iter,
+ struct btree_iter_state *_iter,
const struct bkey_i *k, unsigned level,
struct closure *cl)
{
- int i = 0;
struct btree *b;
+ int i = 0;
BUG_ON(level >= BTREE_MAX_DEPTH);
retry:
@@ -642,7 +644,7 @@ retry:
* else we could read in a btree node from disk that's been
* freed:
*/
- b = bch_btree_node_fill(iter, k, level, cl);
+ b = bch_btree_node_fill(iter, _iter, k, level, cl);
/* We raced and found the btree node in the cache */
if (!b)
@@ -657,8 +659,8 @@ retry:
* But we still have to drop read locks before we return, for
* deadlock avoidance:
*/
- if (btree_node_read_locked(iter, level + 1))
- btree_node_unlock(iter, level + 1);
+ if (btree_node_read_locked(_iter, level + 1))
+ btree_node_unlock(_iter, level + 1);
} else {
/*
* There's a potential deadlock with splits and insertions into
@@ -688,12 +690,12 @@ retry:
* the parent was modified, when the pointer to the node we want
* was removed - and we'll bail out:
*/
- if (btree_node_read_locked(iter, level + 1))
- btree_node_unlock(iter, level + 1);
+ if (btree_node_read_locked(_iter, level + 1))
+ btree_node_unlock(_iter, level + 1);
if (!btree_node_lock(b, iter, level,
PTR_HASH(&b->key) != PTR_HASH(k))) {
- if (!btree_node_relock(iter, level + 1)) {
+ if (!bch_btree_node_relock(iter, _iter, level + 1)) {
trace_bcache_btree_intent_lock_fail(b, iter);
return ERR_PTR(-EINTR);
}
@@ -712,7 +714,7 @@ retry:
}
if (btree_node_read_error(b)) {
- __btree_node_unlock(iter, level, b);
+ __btree_node_unlock(_iter, level, b);
return ERR_PTR(-EIO);
}
diff --git a/drivers/md/bcache/btree_cache.h b/drivers/md/bcache/btree_cache.h
index e3950bf4cfb3..4dad87b53a3b 100644
--- a/drivers/md/bcache/btree_cache.h
+++ b/drivers/md/bcache/btree_cache.h
@@ -5,6 +5,7 @@
#include "btree_types.h"
struct btree_iter;
+struct btree_iter_state;
extern const char *bch_btree_id_names[BTREE_ID_NR];
@@ -20,6 +21,7 @@ int mca_cannibalize_lock(struct cache_set *, struct closure *);
struct btree *mca_alloc(struct cache_set *, struct closure *);
struct btree *bch_btree_node_get(struct btree_iter *,
+ struct btree_iter_state *,
const struct bkey_i *, unsigned,
struct closure *);
diff --git a/drivers/md/bcache/btree_gc.c b/drivers/md/bcache/btree_gc.c
index 886d3c3acd76..df814c07ee71 100644
--- a/drivers/md/bcache/btree_gc.c
+++ b/drivers/md/bcache/btree_gc.c
@@ -451,7 +451,7 @@ static void recalc_packed_keys(struct btree *b)
static void bch_coalesce_nodes(struct btree *old_nodes[GC_MERGE_NODES],
struct btree_iter *iter)
{
- struct btree *parent = iter->l[old_nodes[0]->level + 1].node;
+ struct btree *parent = iter_s(iter)->l[old_nodes[0]->level + 1].node;
struct cache_set *c = iter->c;
unsigned i, nr_old_nodes, nr_new_nodes, u64s = 0;
unsigned blocks = btree_blocks(c) * 2 / 3;
@@ -625,7 +625,7 @@ next:
BUG_ON(!bch_keylist_empty(&keylist));
- BUG_ON(iter->l[old_nodes[0]->level].node != old_nodes[0]);
+ BUG_ON(!btree_iter_has_node(iter_s(iter), old_nodes[0]));
BUG_ON(!bch_btree_iter_node_replace(iter, new_nodes[0]));
@@ -716,8 +716,8 @@ static int bch_coalesce_btree(struct cache_set *c, enum btree_id btree_id)
* and the nodes in our sliding window might not have the same
* parent anymore - blow away the sliding window:
*/
- if (iter.l[iter.level + 1].node &&
- !btree_node_intent_locked(&iter, iter.level + 1))
+ if (iter_s(&iter)->l[iter.level + 1].node &&
+ !btree_node_intent_locked(iter_s(&iter), iter.level + 1))
memset(merge + 1, 0,
(GC_MERGE_NODES - 1) * sizeof(merge[0]));
}
diff --git a/drivers/md/bcache/btree_io.c b/drivers/md/bcache/btree_io.c
index 052364a042cd..a458fc537ae5 100644
--- a/drivers/md/bcache/btree_io.c
+++ b/drivers/md/bcache/btree_io.c
@@ -144,7 +144,7 @@ void bch_btree_init_next(struct cache_set *c, struct btree *b,
{
bool did_sort;
- BUG_ON(iter && iter->l[b->level].node != b);
+ BUG_ON(iter && !btree_iter_has_node(iter_s(iter), b));
did_sort = btree_node_compact(c, b, iter);
diff --git a/drivers/md/bcache/btree_iter.c b/drivers/md/bcache/btree_iter.c
index 2c469ad4936a..343b44439f59 100644
--- a/drivers/md/bcache/btree_iter.c
+++ b/drivers/md/bcache/btree_iter.c
@@ -12,47 +12,54 @@
#define BTREE_ITER_NOT_END ((struct btree *) 1)
-static inline bool is_btree_node(struct btree_iter *iter, unsigned l)
+static inline bool is_btree_node(struct btree_iter_state *_iter, unsigned l)
{
- return iter->l[l].node && iter->l[l].node != BTREE_ITER_NOT_END;
+ struct btree *b = _iter->l[l].node;
+
+ return b && b != BTREE_ITER_NOT_END;
}
/* Btree node locking: */
/*
- * Updates the saved lock sequence number, so that btree_node_relock() will
+ * Updates the saved lock sequence number, so that bch_btree_node_relock() will
* succeed:
*/
void btree_node_unlock_write(struct btree *b, struct btree_iter *iter)
{
+ struct btree_iter_state *_iter = iter_s(iter);
struct btree_iter *linked;
- EBUG_ON(iter->l[b->level].node != b);
- EBUG_ON(iter->l[b->level].lock_seq + 1 != b->lock.state.seq);
+ EBUG_ON(!btree_iter_has_node(_iter, b));
+ EBUG_ON(_iter->l[b->level].lock_seq + 1 != b->lock.state.seq);
for_each_linked_btree_node(iter, b, linked)
- linked->l[b->level].lock_seq += 2;
+ iter_s(linked)->l[b->level].lock_seq += 2;
- iter->l[b->level].lock_seq += 2;
+ _iter->l[b->level].lock_seq += 2;
six_unlock_write(&b->lock);
}
void btree_node_lock_write(struct btree *b, struct btree_iter *iter)
{
+ struct btree_iter_state *_iter = iter_s(iter);
struct btree_iter *linked;
unsigned readers = 0;
- EBUG_ON(iter->l[b->level].node != b);
- EBUG_ON(iter->l[b->level].lock_seq != b->lock.state.seq);
+ EBUG_ON(!btree_iter_has_node(_iter, b));
+ EBUG_ON(_iter->l[b->level].lock_seq != b->lock.state.seq);
if (six_trylock_write(&b->lock))
return;
- for_each_linked_btree_iter(iter, linked)
- if (linked->l[b->level].node == b &&
- btree_node_read_locked(linked, b->level))
+ for_each_linked_btree_iter(iter, linked) {
+ struct btree_iter_state *_linked = iter_s(linked);
+
+ if (btree_iter_has_node(_linked, b) &&
+ btree_node_read_locked(_linked, b->level))
readers++;
+ }
if (likely(!readers)) {
six_lock_write(&b->lock);
@@ -88,27 +95,29 @@ void __btree_node_lock_write(struct btree *b, struct btree_iter *iter)
six_lock_write(&b->lock);
}
-static bool btree_lock_upgrade(struct btree_iter *iter, unsigned level)
+static bool btree_lock_upgrade(struct btree_iter *iter,
+ struct btree_iter_state *_iter,
+ unsigned level)
{
struct btree_iter *linked;
- struct btree *b = iter->l[level].node;
+ struct btree *b = _iter->l[level].node;
- if (btree_node_intent_locked(iter, level))
+ if (btree_node_intent_locked(_iter, level))
return true;
- if (!is_btree_node(iter, level))
+ if (!is_btree_node(_iter, level))
return false;
- if (btree_node_locked(iter, level)
+ if (btree_node_locked(_iter, level)
? six_trylock_convert(&b->lock, SIX_LOCK_read, SIX_LOCK_intent)
- : six_relock_intent(&b->lock, iter->l[level].lock_seq))
+ : six_relock_intent(&b->lock, _iter->l[level].lock_seq))
goto success;
for_each_linked_btree_iter(iter, linked)
- if (linked->l[level].node == b &&
- btree_node_intent_locked(linked, level) &&
- iter->l[level].lock_seq == b->lock.state.seq) {
- btree_node_unlock(iter, level);
+ if (iter_s(linked)->l[level].node == b &&
+ btree_node_intent_locked(iter_s(linked), level) &&
+ _iter->l[level].lock_seq == b->lock.state.seq) {
+ btree_node_unlock(_iter, level);
six_lock_increment(&b->lock, SIX_LOCK_intent);
goto success;
}
@@ -117,30 +126,31 @@ static bool btree_lock_upgrade(struct btree_iter *iter, unsigned level)
trace_bcache_btree_upgrade_lock_fail(b, iter);
return false;
success:
- mark_btree_node_intent_locked(iter, level);
+ mark_btree_node_intent_locked(_iter, level);
trace_bcache_btree_upgrade_lock(b, iter);
return true;
}
/* Btree iterator locking: */
-bool bch_btree_iter_upgrade(struct btree_iter *iter)
+static bool __btree_iter_upgrade(struct btree_iter *iter,
+ struct btree_iter_state *_iter)
{
int i;
for (i = iter->level;
i < min_t(int, iter->locks_want, BTREE_MAX_DEPTH);
i++)
- if (iter->l[i].node && !btree_lock_upgrade(iter, i)) {
+ if (_iter->l[i].node && !btree_lock_upgrade(iter, _iter, i)) {
do {
- btree_node_unlock(iter, i);
+ btree_node_unlock(_iter, i);
/*
- * Make sure btree_node_relock() in
+ * Make sure bch_btree_node_relock() in
* btree_iter_traverse() fails, so that we keep
* going up and get all the intent locks we need
*/
- iter->l[i].lock_seq--;
+ _iter->l[i].lock_seq--;
} while (--i >= 0);
return false;
@@ -149,40 +159,50 @@ bool bch_btree_iter_upgrade(struct btree_iter *iter)
return true;
}
-int bch_btree_iter_unlock(struct btree_iter *iter)
+bool bch_btree_iter_upgrade(struct btree_iter *iter)
+{
+ return __btree_iter_upgrade(iter, iter_s(iter));
+}
+
+void __btree_iter_unlock(struct btree_iter_state *_iter)
{
- unsigned l = 0;
+ while (_iter->nodes_locked)
+ btree_node_unlock(_iter, __ffs(_iter->nodes_locked));
+}
- while (iter->nodes_locked)
- btree_node_unlock(iter, l++);
+int bch_btree_iter_unlock(struct btree_iter *iter)
+{
+ __btree_iter_unlock(iter_s(iter));
return iter->error;
}
-bool btree_node_relock(struct btree_iter *iter, unsigned level)
+bool bch_btree_node_relock(struct btree_iter *iter,
+ struct btree_iter_state *_iter,
+ unsigned level)
{
struct btree_iter *linked;
- struct btree *b = iter->l[level].node;
+ struct btree *b = _iter->l[level].node;
enum six_lock_type type = btree_lock_want(iter, level);
- if (btree_node_locked(iter, level))
+ if (btree_node_locked(_iter, level))
return true;
if (race_fault())
return false;
- if (is_btree_node(iter, level) &&
- six_relock_type(&b->lock, iter->l[level].lock_seq, type)) {
- mark_btree_node_locked(iter, level, type);
+ if (is_btree_node(_iter, level) &&
+ six_relock_type(&b->lock, _iter->l[level].lock_seq, type)) {
+ mark_btree_node_locked(_iter, level, type);
return true;
}
for_each_linked_btree_iter(iter, linked)
- if (linked->l[level].node == b &&
- btree_node_locked_type(linked, level) == type &&
- iter->l[level].lock_seq == b->lock.state.seq) {
+ if (iter_s(linked)->l[level].node == b &&
+ btree_node_locked_type(iter_s(linked), level) == type &&
+ _iter->l[level].lock_seq == b->lock.state.seq) {
six_lock_increment(&b->lock, type);
- mark_btree_node_locked(iter, level, type);
+ mark_btree_node_locked(_iter, level, type);
return true;
}
@@ -327,9 +347,9 @@ void bch_btree_node_iter_fix(struct btree_iter *iter,
}
/* peek_all() doesn't skip deleted keys */
-static inline struct bkey_s_c __btree_iter_peek_all(struct btree_iter *iter)
+static inline struct bkey_s_c __btree_iter_peek_all(struct btree_iter *iter,
+ struct btree_iter_level *l)
{
- struct btree_iter_level *l = &iter->l[iter->level];
const struct bkey_format *f = &l->node->keys.format;
struct bkey_packed *k =
bch_btree_node_iter_peek_all(&l->node_iter, &l->node->keys);
@@ -346,9 +366,9 @@ static inline struct bkey_s_c __btree_iter_peek_all(struct btree_iter *iter)
return ret;
}
-static inline struct bkey_s_c __btree_iter_peek(struct btree_iter *iter)
+static inline struct bkey_s_c __btree_iter_peek(struct btree_iter *iter,
+ struct btree_iter_level *l)
{
- struct btree_iter_level *l = &iter->l[iter->level];
const struct bkey_format *f = &l->node->keys.format;
struct bkey_packed *k =
bch_btree_node_iter_peek(&l->node_iter, &l->node->keys);
@@ -365,10 +385,8 @@ static inline struct bkey_s_c __btree_iter_peek(struct btree_iter *iter)
return ret;
}
-static inline void __btree_iter_advance(struct btree_iter *iter)
+static inline void __btree_iter_advance(struct btree_iter_level *l)
{
- struct btree_iter_level *l = &iter->l[iter->level];
-
bch_btree_node_iter_advance(&l->node_iter, &l->node->keys);
}
@@ -381,10 +399,11 @@ static inline void btree_iter_node_iter_init(struct btree_iter *iter,
}
static inline void btree_iter_node_set(struct btree_iter *iter,
+ struct btree_iter_state *_iter,
struct btree *b,
struct bpos pos)
{
- struct btree_iter_level *l = &iter->l[b->level];
+ struct btree_iter_level *l = &_iter->l[b->level];
BUG_ON(b->lock.state.seq & 1);
@@ -416,7 +435,7 @@ bool bch_btree_iter_node_replace(struct btree_iter *iter, struct btree *b)
* the old node we're replacing has already been
* unlocked and the pointer invalidated
*/
- BUG_ON(btree_node_locked(linked, b->level));
+ EBUG_ON(btree_node_locked(iter_s(linked), b->level));
/*
* If @linked wants this node read locked, we don't want
@@ -426,15 +445,16 @@ bool bch_btree_iter_node_replace(struct btree_iter *iter, struct btree *b)
* progress...
*
* Instead, btree_iter_node_set() sets things up so
- * btree_node_relock() will succeed:
+ * bch_btree_node_relock() will succeed:
*/
if (btree_want_intent(linked, b->level)) {
six_lock_increment(&b->lock, SIX_LOCK_intent);
- mark_btree_node_intent_locked(linked, b->level);
+ mark_btree_node_intent_locked(iter_s(linked), b->level);
}
- btree_iter_node_set(linked, b, linked->pos);
+ btree_iter_node_set(linked, iter_s(linked),
+ b, linked->pos);
}
if (!btree_iter_pos_in_node(iter, b)) {
@@ -442,8 +462,8 @@ bool bch_btree_iter_node_replace(struct btree_iter *iter, struct btree *b)
return false;
}
- mark_btree_node_intent_locked(iter, b->level);
- btree_iter_node_set(iter, b, iter->pos);
+ mark_btree_node_intent_locked(iter_s(iter), b->level);
+ btree_iter_node_set(iter, iter_s(iter), b, iter->pos);
return true;
}
@@ -452,21 +472,25 @@ void bch_btree_iter_node_drop_linked(struct btree_iter *iter, struct btree *b)
struct btree_iter *linked;
unsigned level = b->level;
- for_each_linked_btree_iter(iter, linked)
- if (linked->l[level].node == b) {
- btree_node_unlock(linked, level);
- linked->l[level].node = BTREE_ITER_NOT_END;
+ for_each_linked_btree_iter(iter, linked) {
+ struct btree_iter_state *_linked = iter_s(linked);
+
+ if (_linked->l[level].node == b) {
+ btree_node_unlock(_linked, level);
+ _linked->l[level].node = BTREE_ITER_NOT_END;
}
+ }
}
void bch_btree_iter_node_drop(struct btree_iter *iter, struct btree *b)
{
+ struct btree_iter_state *_iter = iter_s(iter);
unsigned level = b->level;
- if (iter->l[level].node == b) {
+ if (_iter->l[level].node == b) {
BUG_ON(b->lock.state.intent_lock != 1);
- btree_node_unlock(iter, level);
- iter->l[level].node = BTREE_ITER_NOT_END;
+ btree_node_unlock(_iter, level);
+ _iter->l[level].node = BTREE_ITER_NOT_END;
}
}
@@ -480,15 +504,17 @@ void bch_btree_iter_reinit_node(struct btree_iter *iter, struct btree *b)
for_each_linked_btree_node(iter, b, linked)
btree_iter_node_iter_init(linked,
- &linked->l[b->level],
+ &iter_s(linked)->l[b->level],
linked->pos);
btree_iter_node_iter_init(iter,
- &iter->l[b->level],
+ &iter_s(iter)->l[b->level],
iter->pos);
}
-static void btree_iter_verify_locking(struct btree_iter *iter, unsigned level)
+static void btree_iter_verify_locking(struct btree_iter *iter,
+ struct btree_iter_state *_iter,
+ unsigned level)
{
#ifdef CONFIG_BCACHE_DEBUG
struct btree_iter *linked;
@@ -505,24 +531,27 @@ static void btree_iter_verify_locking(struct btree_iter *iter, unsigned level)
* retaking fails then return -EINTR... but, let's keep things simple
* for now:
*/
- BUG_ON(iter->nodes_locked != iter->nodes_intent_locked);
+ BUG_ON(_iter->nodes_locked != _iter->nodes_intent_locked);
for_each_linked_btree_iter(iter, linked)
- BUG_ON(linked->nodes_locked != linked->nodes_intent_locked);
+ BUG_ON(iter_s(linked)->nodes_locked !=
+ iter_s(linked)->nodes_intent_locked);
/* Lock ordering: */
for_each_linked_btree_iter(iter, linked) {
- BUG_ON(linked->nodes_locked &&
+ BUG_ON(iter_s(linked)->nodes_locked &&
btree_iter_cmp(linked, iter) > 0);
- BUG_ON(linked->nodes_locked &&
+ BUG_ON(iter_s(linked)->nodes_locked &&
linked->btree_id == iter->btree_id &&
- level > __fls(linked->nodes_locked));
+ level > __fls(iter_s(linked)->nodes_locked));
}
#endif
}
-static inline void btree_iter_lock_root(struct btree_iter *iter, struct bpos pos)
+static inline void btree_iter_lock_root(struct btree_iter *iter,
+ struct btree_iter_state *_iter,
+ struct bpos pos)
{
struct cache_set *c = iter->c;
@@ -530,42 +559,45 @@ static inline void btree_iter_lock_root(struct btree_iter *iter, struct bpos pos
struct btree *b = c->btree_roots[iter->btree_id].b;
unsigned level = READ_ONCE(b->level);
- btree_iter_verify_locking(iter, level);
+ btree_iter_verify_locking(iter, _iter, level);
if (btree_node_lock(b, iter, level,
(b != c->btree_roots[iter->btree_id].b))) {
iter->level = level;
if (level + 1 < BTREE_MAX_DEPTH)
- iter->l[level + 1].node = NULL;
- btree_iter_node_set(iter, b, pos);
+ _iter->l[level + 1].node = NULL;
+ btree_iter_node_set(iter, _iter, b, pos);
break;
}
}
}
-static inline int btree_iter_down(struct btree_iter *iter, struct bpos pos,
- struct closure *cl)
+static inline int btree_iter_down(struct btree_iter *iter,
+ struct btree_iter_state *_iter,
+ struct bpos pos, struct closure *cl)
{
struct btree *b;
- struct bkey_s_c k = __btree_iter_peek(iter);
+ struct bkey_s_c k = __btree_iter_peek(iter,
+ &_iter->l[iter->level]);
BKEY_PADDED(k) tmp;
bkey_reassemble(&tmp.k, k);
- b = bch_btree_node_get(iter, &tmp.k, iter->level - 1, cl);
+ b = bch_btree_node_get(iter, _iter, &tmp.k, iter->level - 1, cl);
if (unlikely(IS_ERR(b)))
return PTR_ERR(b);
- btree_iter_verify_locking(iter, iter->level - 1);
+ btree_iter_verify_locking(iter, _iter, iter->level - 1);
--iter->level;
- btree_iter_node_set(iter, b, pos);
+ btree_iter_node_set(iter, _iter, b, pos);
return 0;
}
-static void btree_iter_up(struct btree_iter *iter)
+static void btree_iter_up(struct btree_iter *iter,
+ struct btree_iter_state *_iter)
{
- btree_node_unlock(iter, iter->level++);
+ btree_node_unlock(_iter, iter->level++);
}
/*
@@ -578,9 +610,13 @@ static void btree_iter_up(struct btree_iter *iter)
* stashed in the iterator and returned from bch_btree_iter_unlock().
*/
static int __must_check __bch_btree_iter_traverse(struct btree_iter *iter,
- unsigned l, struct bpos pos)
+ struct btree_iter_state *_iter,
+ unsigned traverse_to,
+ struct bpos pos)
{
- if (!iter->l[iter->level].node)
+ struct btree_iter_level *l;
+
+ if (!_iter->l[iter->level].node)
return 0;
iter->at_end_of_leaf = false;
@@ -589,25 +625,25 @@ retry:
* If the current node isn't locked, go up until we have a locked node
* or run out of nodes:
*/
- while (iter->l[iter->level].node &&
- !(is_btree_node(iter, iter->level) &&
- btree_node_relock(iter, iter->level) &&
+ while (_iter->l[iter->level].node &&
+ !(is_btree_node(_iter, iter->level) &&
+ bch_btree_node_relock(iter, _iter, iter->level) &&
btree_iter_pos_cmp(pos,
- &iter->l[iter->level].node->key.k,
- iter->is_extents)))
- btree_iter_up(iter);
+ &_iter->l[iter->level].node->key.k),
+ iter->is_extents))
+ btree_iter_up(iter, _iter);
/*
* If we've got a btree node locked (i.e. we aren't about to relock the
* root) - advance its node iterator if necessary:
*/
if (iter->level < BTREE_MAX_DEPTH &&
- iter->l[iter->level].node) {
+ (l = &_iter->l[iter->level])->node) {
struct bkey_s_c k;
- while ((k = __btree_iter_peek_all(iter)).k &&
+ while ((k = __btree_iter_peek_all(iter, l)).k &&
!btree_iter_pos_cmp(pos, k.k, iter->is_extents))
- __btree_iter_advance(iter);
+ __btree_iter_advance(l);
}
/*
@@ -616,17 +652,17 @@ retry:
* here it indicates that relocking the root failed - it's critical that
* btree_iter_lock_root() comes next and that it can't fail
*/
- while (iter->level > l)
+ while (iter->level > traverse_to)
if (iter->level < BTREE_MAX_DEPTH &&
- iter->l[iter->level].node) {
+ _iter->l[iter->level].node) {
struct closure cl;
int ret;
closure_init_stack(&cl);
- ret = btree_iter_down(iter, pos, &cl);
+ ret = btree_iter_down(iter, _iter, pos, &cl);
if (unlikely(ret)) {
- bch_btree_iter_unlock(iter);
+ __btree_iter_unlock(_iter);
closure_sync(&cl);
/*
@@ -634,7 +670,7 @@ retry:
* intent locks, make sure to get them again:
*/
if (ret == -EAGAIN || ret == -EINTR) {
- bch_btree_iter_upgrade(iter);
+ __btree_iter_upgrade(iter, _iter);
goto retry;
}
@@ -643,7 +679,7 @@ retry:
return ret;
}
} else {
- btree_iter_lock_root(iter, pos);
+ btree_iter_lock_root(iter, _iter, pos);
}
return 0;
@@ -651,13 +687,15 @@ retry:
int __must_check bch_btree_iter_traverse(struct btree_iter *iter)
{
- return __bch_btree_iter_traverse(iter, iter->level, iter->pos);
+ return __bch_btree_iter_traverse(iter, iter_s(iter),
+ iter->level, iter->pos);
}
/* Iterate across nodes (leaf and interior nodes) */
struct btree *bch_btree_iter_peek_node(struct btree_iter *iter)
{
+ struct btree_iter_state *_iter = iter_s(iter);
struct btree *b;
int ret;
@@ -667,7 +705,7 @@ struct btree *bch_btree_iter_peek_node(struct btree_iter *iter)
if (ret)
return NULL;
- b = iter->l[iter->level].node;
+ b = _iter->l[iter->level].node;
EBUG_ON(bkey_cmp(b->key.k.p, iter->pos) < 0);
iter->pos = b->key.k.p;
@@ -677,15 +715,16 @@ struct btree *bch_btree_iter_peek_node(struct btree_iter *iter)
struct btree *bch_btree_iter_next_node(struct btree_iter *iter)
{
+ struct btree_iter_state *_iter = iter_s(iter);
struct btree *b;
int ret;
EBUG_ON(iter->is_extents);
- btree_iter_up(iter);
+ btree_iter_up(iter, _iter);
if (iter->level >= BTREE_MAX_DEPTH ||
- !iter->l[iter->level].node)
+ !_iter->l[iter->level].node)
return NULL;
/* parent node usually won't be locked: redo traversal if necessary */
@@ -693,16 +732,16 @@ struct btree *bch_btree_iter_next_node(struct btree_iter *iter)
if (ret)
return NULL;
- b = iter->l[iter->level].node;
+ b = _iter->l[iter->level].node;
if (bkey_cmp(iter->pos, b->key.k.p) < 0) {
struct bpos pos = bkey_successor(iter->pos);
- ret = __bch_btree_iter_traverse(iter, 0, pos);
+ ret = __bch_btree_iter_traverse(iter, iter_s(iter), 0, pos);
if (ret)
return NULL;
- b = iter->l[iter->level].node;
+ b = _iter->l[iter->level].node;
}
iter->pos = b->key.k.p;
@@ -750,7 +789,8 @@ void bch_btree_iter_advance_pos(struct btree_iter *iter)
/* XXX: expensive */
void bch_btree_iter_rewind(struct btree_iter *iter, struct bpos pos)
{
- struct btree_iter_level *l = &iter->l[iter->level];
+ struct btree_iter_state *_iter = iter_s(iter);
+ struct btree_iter_level *l = &_iter->l[iter->level];
/* incapable of rewinding across nodes: */
BUG_ON(bkey_cmp(pos, l->node->data->min_key) < 0);
@@ -760,18 +800,19 @@ void bch_btree_iter_rewind(struct btree_iter *iter, struct bpos pos)
btree_iter_node_iter_init(iter, l, pos);
}
-struct bkey_s_c bch_btree_iter_peek(struct btree_iter *iter)
+struct bkey_s_c __bch_btree_iter_peek(struct btree_iter *iter,
+ struct btree_iter_state *_iter)
{
- struct bkey_s_c k;
struct bpos pos = iter->pos;
+ struct bkey_s_c k;
int ret;
while (1) {
- ret = __bch_btree_iter_traverse(iter, 0, pos);
+ ret = __bch_btree_iter_traverse(iter, _iter, 0, pos);
if (unlikely(ret))
return bkey_s_c_null;
- k = __btree_iter_peek(iter);
+ k = __btree_iter_peek(iter, &_iter->l[0]);
if (likely(k.k)) {
/*
* iter->pos should always be equal to the key we just
@@ -783,26 +824,106 @@ struct bkey_s_c bch_btree_iter_peek(struct btree_iter *iter)
return k;
}
- pos = iter->l[0].node->key.k.p;
+ pos = btree_iter_leaf(iter)->key.k.p;
if (!bkey_cmp(pos, POS_MAX))
return (struct bkey_s_c) { NULL, NULL };
pos = btree_type_successor(iter->btree_id, pos);
}
+
}
+struct bkey_s_c bch_btree_iter_peek(struct btree_iter *iter)
+{
+ return __bch_btree_iter_peek(iter, iter_s(iter));
+}
+
+/*
+ * iter_s(iter) is the cursor for iter->pos - the key we return
+ *
+ * iter->pos.snapshot may be different than snapshot->id (but must be an
+ * ancestor)
+ *
+ * if iter->have_alternate != 0, then iter_a(iter) is the cursor for inserting a
+ * key at iter.pos, except with iter.pos.snapshot = snapshot->id
+ */
+
struct bkey_s_c bch_btree_iter_peek_snapshot(struct btree_iter *iter,
struct snapshot *snapshot)
{
+ struct btree_iter_state *_iter = iter_s(iter);
+ struct btree_iter_state *_alternate = iter_a(iter);
struct bkey_s_c k;
- do
+ if (iter->have_alternate) {
+ __btree_iter_unlock(_alternate);
+ iter->have_alternate = 0;
+ }
+
+ k = bch_btree_iter_peek(iter);
+ if (unlikely(!k.k))
+ return k;
+
+ if (likely(bch_is_snapshot_ancestor(iter->c,
+ snapshot, k.k->p.snapshot)))
+ return k;
+
+ /*
+ * keep current position locked, advance until we find a key that is an
+ * ancestor:
+ */
+ *_alternate = *_iter;
+ _alternate->nodes_locked = 0;
+ _alternate->nodes_intent_locked = 0;
+ iter->have_alternate = 1;
+
+ iter->s_idx ^= 1;
+ swap(_iter, _alternate);
+
+ while (1) {
+ bch_btree_iter_advance_pos(iter);
+
k = bch_btree_iter_peek(iter);
- while (k.k &&
- !bch_snapshot_is_descendant(iter->c, snapshot, k.k->p.snapshot));
+ if (unlikely(!k.k))
+ return k;
+
+ if (likely(bch_is_snapshot_ancestor(iter->c,
+ snapshot, k.k->p.snapshot)))
+ return k;
+ }
return k;
+#if 0
+ struct bpos pos = iter->pos;
+ struct bkey_s_c k;
+ int ret;
+
+ while (1) {
+ ret = __bch_btree_iter_traverse(iter, 0, pos);
+ if (unlikely(ret))
+ return bkey_s_c_null;
+
+ k = __btree_iter_peek(iter);
+ if (likely(k.k)) {
+ /*
+ * iter->pos should always be equal to the key we just
+ * returned - except extents can straddle iter->pos:
+ */
+ if (!iter->is_extents ||
+ bkey_cmp(bkey_start_pos(k.k), iter->pos) > 0)
+ bch_btree_iter_set_pos(iter, bkey_start_pos(k.k));
+ return k;
+ }
+
+ pos = btree_iter_leaf(iter)->key.k.p;
+
+ if (!bkey_cmp(pos, POS_MAX))
+ return (struct bkey_s_c) { NULL, NULL };
+
+ pos = btree_type_successor(iter->btree_id, pos);
+ }
+#endif
}
struct bkey_s_c bch_btree_iter_peek_with_holes(struct btree_iter *iter)
@@ -812,13 +933,14 @@ struct bkey_s_c bch_btree_iter_peek_with_holes(struct btree_iter *iter)
int ret;
while (1) {
- ret = __bch_btree_iter_traverse(iter, 0, iter->pos);
+ ret = __bch_btree_iter_traverse(iter, iter_s(iter),
+ 0, iter->pos);
if (unlikely(ret))
return bkey_s_c_null;
k = iter->is_extents
- ? __btree_iter_peek(iter)
- : __btree_iter_peek_all(iter);
+ ? __btree_iter_peek(iter, &iter_s(iter)->l[0])
+ : __btree_iter_peek_all(iter, &iter_s(iter)->l[0]);
recheck:
if (!k.k || bkey_cmp(bkey_start_pos(k.k), iter->pos) > 0) {
/* hole */
@@ -832,7 +954,7 @@ recheck:
}
if (!k.k)
- k.k = &iter->l[0].node->key.k;
+ k.k = &btree_iter_leaf(iter)->key.k;
bch_key_resize(&n,
min_t(u64, KEY_SIZE_MAX,
@@ -849,7 +971,7 @@ recheck:
} else if (!bkey_deleted(k.k)) {
return k;
} else {
- __btree_iter_advance(iter);
+ __btree_iter_advance(&iter_s(iter)->l[0]);
}
}
@@ -869,26 +991,30 @@ struct bkey_s_c bch_btree_iter_peek_snapshot_with_holes(struct btree_iter *iter,
int ret;
while (1) {
- ret = __bch_btree_iter_traverse(iter, 0, iter->pos);
- if (ret)
+ ret = __bch_btree_iter_traverse(iter, iter_s(iter),
+ 0, iter->pos);
+ if (unlikely(ret))
return bkey_s_c_null;
- k = __btree_iter_peek_all(iter);
+ k = __btree_iter_peek_all(iter, &iter_s(iter)->l[0]);
+
+ /* hrm */
+
recheck:
if (!k.k || bkey_cmp(bkey_start_pos(k.k), iter->pos) > 0) {
/* hole */
bkey_init(&n);
n.p = iter->pos;
- if (!k.k)
- k.k = &iter->l[0].node->key.k;
-
- if (iter->btree_id == BTREE_ID_EXTENTS) {
+ if (iter->is_extents) {
if (n.p.offset == KEY_OFFSET_MAX) {
iter->pos = bkey_successor(iter->pos);
goto recheck;
}
+ if (!k.k)
+ k.k = &btree_iter_leaf(iter)->key.k;
+
bch_key_resize(&n,
min_t(u64, KEY_SIZE_MAX,
(k.k->p.inode == n.p.inode
@@ -904,7 +1030,7 @@ recheck:
} else if (!bkey_deleted(k.k)) {
return k;
} else {
- __btree_iter_advance(iter);
+ __btree_iter_advance(&iter_s(iter)->l[0]);
}
}
@@ -920,19 +1046,25 @@ void __bch_btree_iter_init(struct btree_iter *iter, struct cache_set *c,
enum btree_id btree_id, struct bpos pos,
int locks_want)
{
- iter->level = 0;
- iter->is_extents = btree_id == BTREE_ID_EXTENTS;
- iter->nodes_locked = 0;
- iter->nodes_intent_locked = 0;
- iter->locks_want = locks_want;
- iter->btree_id = btree_id;
- iter->at_end_of_leaf = 0;
- iter->error = 0;
+ struct btree_iter_state *_iter;
+
iter->c = c;
- iter->pos = pos;
- iter->l[iter->level].node = BTREE_ITER_NOT_END;
- iter->l[iter->level + 1].node = NULL;
iter->next = iter;
+ iter->pos = pos;
+
+ iter->error = 0;
+ iter->btree_id = btree_id;
+ iter->at_end_of_leaf = 0;
+ iter->is_extents = btree_id == BTREE_ID_EXTENTS;
+ iter->locks_want = locks_want;
+ iter->level = 0;
+ iter->s_idx = 0;
+
+ _iter = iter_s(iter);
+ _iter->nodes_locked = 0;
+ _iter->nodes_intent_locked = 0;
+ _iter->l[iter->level].node = BTREE_ITER_NOT_END;
+ _iter->l[iter->level + 1].node = NULL;
}
int bch_btree_iter_unlink(struct btree_iter *iter)
@@ -975,8 +1107,10 @@ void bch_btree_iter_init_copy(struct btree_iter *dst, struct btree_iter *src)
{
*dst = *src;
dst->next = dst;
- dst->nodes_locked = 0;
- dst->nodes_intent_locked = 0;
+ dst->s[0].nodes_locked = 0;
+ dst->s[0].nodes_intent_locked = 0;
+ dst->s[1].nodes_locked = 0;
+ dst->s[1].nodes_intent_locked = 0;
bch_btree_iter_link(src, dst);
bch_btree_iter_upgrade(dst);
diff --git a/drivers/md/bcache/btree_iter.h b/drivers/md/bcache/btree_iter.h
index 96356558812f..7f8aa4710ffe 100644
--- a/drivers/md/bcache/btree_iter.h
+++ b/drivers/md/bcache/btree_iter.h
@@ -9,6 +9,27 @@ struct btree_iter_level {
u32 lock_seq;
};
+struct btree_iter_state {
+ /* Bitmasks for read/intent locks held per level */
+ u8 nodes_locked;
+ u8 nodes_intent_locked;
+
+ /*
+ * NOTE: Never set iter->nodes to NULL except in btree_iter_lock_root().
+ *
+ * This is because iter->nodes[iter->level] == NULL is how
+ * btree_iter_next_node() knows that it's finished with a depth first
+ * traversal. Just unlocking a node (with btree_node_unlock()) is fine,
+ * and if you really don't want that node used again (e.g. btree_split()
+ * freed it) decrementing lock_seq will cause bch_btree_node_relock() to
+ * always fail (but since freeing a btree node takes a write lock on the
+ * node, which increments the node's lock seq, that's not actually
+ * necessary in that example).
+ */
+
+ struct btree_iter_level l[BTREE_MAX_DEPTH];
+};
+
struct btree_iter {
struct cache_set *c;
@@ -52,25 +73,33 @@ struct btree_iter {
/* Current btree depth */
u8 level;
- /* Bitmasks for read/intent locks held per level */
- u8 nodes_locked;
- u8 nodes_intent_locked;
+ unsigned s_idx:1,
+ have_alternate;
- /*
- * NOTE: Never set iter->nodes to NULL except in btree_iter_lock_root().
- *
- * This is because iter->nodes[iter->level] == NULL is how
- * btree_iter_next_node() knows that it's finished with a depth first
- * traversal. Just unlocking a node (with btree_node_unlock()) is fine,
- * and if you really don't want that node used again (e.g. btree_split()
- * freed it) decrementing lock_seq will cause btree_node_relock() to
- * always fail (but since freeing a btree node takes a write lock on the
- * node, which increments the node's lock seq, that's not actually
- * necessary in that example).
- */
- struct btree_iter_level l[BTREE_MAX_DEPTH];
+ struct btree_iter_state s[2];
};
+static inline struct btree_iter_state *iter_s(struct btree_iter *iter)
+{
+ return &iter->s[iter->s_idx];
+}
+
+static inline struct btree_iter_state *iter_a(struct btree_iter *iter)
+{
+ return &iter->s[iter->s_idx ^ 1];
+}
+
+static inline struct btree *btree_iter_leaf(struct btree_iter *iter)
+{
+ return iter_s(iter)->l[0].node;
+}
+
+static inline bool btree_iter_has_node(struct btree_iter_state *_iter,
+ struct btree *b)
+{
+ return _iter->l[b->level].node == b;
+}
+
/**
* for_each_linked_btree_iter - iterate over all iterators linked with @_iter
*/
@@ -97,8 +126,8 @@ __next_linked_btree_node(struct btree_iter *iter, struct btree *b,
* sequence number is incremented by taking and releasing write
* locks and is even when unlocked:
*/
- } while (linked->l[b->level].node != b ||
- linked->l[b->level].lock_seq >> 1 != b->lock.state.seq >> 1);
+ } while (!btree_iter_has_node(iter_s(linked), b) ||
+ iter_s(linked)->l[b->level].lock_seq >> 1 != b->lock.state.seq >> 1);
return linked;
}
@@ -110,7 +139,7 @@ __next_linked_btree_node(struct btree_iter *iter, struct btree *b,
* @_b is assumed to be locked by @_iter
*
* Filters out iterators that don't have a valid btree_node iterator for @_b -
- * i.e. iterators for which btree_node_relock() would not succeed.
+ * i.e. iterators for which bch_btree_node_relock() would not succeed.
*/
#define for_each_linked_btree_node(_iter, _b, _linked) \
for ((_linked) = (_iter); \
diff --git a/drivers/md/bcache/btree_locking.h b/drivers/md/bcache/btree_locking.h
index f6789bb4a20b..34c12597377d 100644
--- a/drivers/md/bcache/btree_locking.h
+++ b/drivers/md/bcache/btree_locking.h
@@ -19,7 +19,7 @@ enum btree_node_locked_type {
BTREE_NODE_INTENT_LOCKED = SIX_LOCK_intent,
};
-static inline int btree_node_locked_type(struct btree_iter *iter,
+static inline int btree_node_locked_type(struct btree_iter_state *_iter,
unsigned level)
{
/*
@@ -28,35 +28,35 @@ static inline int btree_node_locked_type(struct btree_iter *iter,
* branches:
*/
return BTREE_NODE_UNLOCKED +
- ((iter->nodes_locked >> level) & 1) +
- ((iter->nodes_intent_locked >> level) & 1);
+ ((_iter->nodes_locked >> level) & 1) +
+ ((_iter->nodes_intent_locked >> level) & 1);
}
-static inline bool btree_node_intent_locked(struct btree_iter *iter,
+static inline bool btree_node_intent_locked(struct btree_iter_state *_iter,
unsigned level)
{
- return btree_node_locked_type(iter, level) == BTREE_NODE_INTENT_LOCKED;
+ return btree_node_locked_type(_iter, level) == BTREE_NODE_INTENT_LOCKED;
}
-static inline bool btree_node_read_locked(struct btree_iter *iter,
+static inline bool btree_node_read_locked(struct btree_iter_state *_iter,
unsigned level)
{
- return btree_node_locked_type(iter, level) == BTREE_NODE_READ_LOCKED;
+ return btree_node_locked_type(_iter, level) == BTREE_NODE_READ_LOCKED;
}
-static inline bool btree_node_locked(struct btree_iter *iter, unsigned level)
+static inline bool btree_node_locked(struct btree_iter_state *_iter, unsigned level)
{
- return iter->nodes_locked & (1 << level);
+ return _iter->nodes_locked & (1 << level);
}
-static inline void mark_btree_node_unlocked(struct btree_iter *iter,
+static inline void mark_btree_node_unlocked(struct btree_iter_state *_iter,
unsigned level)
{
- iter->nodes_locked &= ~(1 << level);
- iter->nodes_intent_locked &= ~(1 << level);
+ _iter->nodes_locked &= ~(1 << level);
+ _iter->nodes_intent_locked &= ~(1 << level);
}
-static inline void mark_btree_node_locked(struct btree_iter *iter,
+static inline void mark_btree_node_locked(struct btree_iter_state *_iter,
unsigned level,
enum six_lock_type type)
{
@@ -64,14 +64,14 @@ static inline void mark_btree_node_locked(struct btree_iter *iter,
BUILD_BUG_ON(SIX_LOCK_read != 0);
BUILD_BUG_ON(SIX_LOCK_intent != 1);
- iter->nodes_locked |= 1 << level;
- iter->nodes_intent_locked |= type << level;
+ _iter->nodes_locked |= 1 << level;
+ _iter->nodes_intent_locked |= type << level;
}
-static inline void mark_btree_node_intent_locked(struct btree_iter *iter,
+static inline void mark_btree_node_intent_locked(struct btree_iter_state *_iter,
unsigned level)
{
- mark_btree_node_locked(iter, level, SIX_LOCK_intent);
+ mark_btree_node_locked(_iter, level, SIX_LOCK_intent);
}
static inline enum six_lock_type
@@ -87,10 +87,10 @@ static inline bool btree_want_intent(struct btree_iter *iter, int level)
return btree_lock_want(iter, level) == SIX_LOCK_intent;
}
-static inline void __btree_node_unlock(struct btree_iter *iter, unsigned level,
+static inline void __btree_node_unlock(struct btree_iter_state *_iter, unsigned level,
struct btree *b)
{
- switch (btree_node_locked_type(iter, level)) {
+ switch (btree_node_locked_type(_iter, level)) {
case BTREE_NODE_READ_LOCKED:
six_unlock_read(&b->lock);
break;
@@ -99,12 +99,12 @@ static inline void __btree_node_unlock(struct btree_iter *iter, unsigned level,
break;
}
- mark_btree_node_unlocked(iter, level);
+ mark_btree_node_unlocked(_iter, level);
}
-static inline void btree_node_unlock(struct btree_iter *iter, unsigned level)
+static inline void btree_node_unlock(struct btree_iter_state *_iter, unsigned level)
{
- __btree_node_unlock(iter, level, iter->l[level].node);
+ __btree_node_unlock(_iter, level, _iter->l[level].node);
}
static inline void btree_node_lock_type(struct btree *b, struct btree_iter *iter,
@@ -115,12 +115,15 @@ static inline void btree_node_lock_type(struct btree *b, struct btree_iter *iter
if (six_trylock_type(&b->lock, type))
return;
- for_each_linked_btree_iter(iter, linked)
- if (linked->l[b->level].node == b &&
- btree_node_locked_type(linked, b->level) == type) {
+ for_each_linked_btree_iter(iter, linked) {
+ struct btree_iter_state *_linked = iter_s(linked);
+
+ if (_linked->l[b->level].node == b &&
+ btree_node_locked_type(_linked, b->level) == type) {
six_lock_increment(&b->lock, type);
return;
}
+ }
six_lock_type(&b->lock, type);
}
@@ -134,7 +137,7 @@ static inline void btree_node_lock_type(struct btree *b, struct btree_iter *iter
if ((_raced = ((check_if_raced) || ((b)->level != _level)))) \
six_unlock_type(&(b)->lock, _type); \
else \
- mark_btree_node_locked(_iter, _level, _type); \
+ mark_btree_node_locked(iter_s(_iter), _level, _type); \
\
!_raced; \
})
@@ -143,7 +146,8 @@ static inline void btree_node_lock_type(struct btree *b, struct btree_iter *iter
(!race_fault() && \
__btree_node_lock(b, iter, level, check_if_raced))
-bool btree_node_relock(struct btree_iter *, unsigned);
+bool bch_btree_node_relock(struct btree_iter *, struct btree_iter_state *,
+ unsigned);
void btree_node_unlock_write(struct btree *, struct btree_iter *);
void btree_node_lock_write(struct btree *, struct btree_iter *);
diff --git a/drivers/md/bcache/btree_update.c b/drivers/md/bcache/btree_update.c
index 2d15e4d4c994..b37ac35fe015 100644
--- a/drivers/md/bcache/btree_update.c
+++ b/drivers/md/bcache/btree_update.c
@@ -191,7 +191,7 @@ static void __btree_node_free(struct cache_set *c, struct btree *b,
/*
* By using six_unlock_write() directly instead of
* btree_node_unlock_write(), we don't update the iterator's sequence
- * numbers and cause future btree_node_relock() calls to fail:
+ * numbers and cause future bch_btree_node_relock() calls to fail:
*/
six_unlock_write(&b->lock);
}
@@ -723,7 +723,7 @@ void bch_btree_journal_key(struct btree_iter *iter,
struct journal_res *res)
{
struct cache_set *c = iter->c;
- struct btree_iter_level *l = &iter->l[0];
+ struct btree_iter_level *l = &iter_s(iter)->l[0];
struct btree *b = l->node;
EBUG_ON(iter->level || b->level);
@@ -821,7 +821,7 @@ struct async_split *__bch_async_split_alloc(struct btree *nodes[],
*
* So far this is the only place where we have this issue:
*/
- if (iter->l[nodes[i]->level].node == nodes[i])
+ if (btree_iter_has_node(iter_s(iter), nodes[i]))
btree_node_lock_write(nodes[i], iter);
else
six_lock_write(&nodes[i]->lock);
@@ -854,7 +854,7 @@ struct async_split *__bch_async_split_alloc(struct btree *nodes[],
journal_pin_add(&c->journal, pin_list, &as->journal, NULL);
for (i = 0; i < nr_nodes; i++) {
- if (iter->l[nodes[i]->level].node == nodes[i])
+ if (btree_iter_has_node(iter_s(iter), nodes[i]))
btree_node_unlock_write(nodes[i], iter);
else
six_unlock_write(&nodes[i]->lock);
@@ -1105,7 +1105,7 @@ bch_btree_insert_keys_interior(struct btree *b,
struct bkey_i *insert = bch_keylist_front(insert_keys);
struct bkey_packed *k;
- BUG_ON(!btree_node_intent_locked(iter, btree_node_root(b)->level));
+ BUG_ON(!btree_node_intent_locked(iter_s(iter), btree_node_root(b)->level));
BUG_ON(!b->level);
BUG_ON(!as || as->b);
verify_keys_sorted(insert_keys);
@@ -1119,7 +1119,7 @@ bch_btree_insert_keys_interior(struct btree *b,
}
/* Don't screw up @iter's position: */
- node_iter = iter->l[b->level].node_iter;
+ node_iter = iter_s(iter)->l[b->level].node_iter;
/*
* btree_split(), btree_gc_coalesce() will insert keys before
@@ -1268,14 +1268,14 @@ static void btree_split(struct btree *b, struct btree_iter *iter,
struct async_split *as)
{
struct cache_set *c = iter->c;
- struct btree *parent = iter->l[b->level + 1].node;
+ struct btree *parent = iter_s(iter)->l[b->level + 1].node;
struct btree *n1, *n2 = NULL, *n3 = NULL;
uint64_t start_time = local_clock();
unsigned u64s_to_insert = b->level
? bch_keylist_nkeys(insert_keys) : 0;
BUG_ON(!parent && (b != btree_node_root(b)));
- BUG_ON(!btree_node_intent_locked(iter, btree_node_root(b)->level));
+ BUG_ON(!btree_node_intent_locked(iter_s(iter), btree_node_root(b)->level));
bch_async_split_will_free_node(as, b);
@@ -1446,7 +1446,7 @@ static int bch_btree_split_leaf(struct btree_iter *iter, unsigned flags,
{
struct btree_iter *linked;
struct cache_set *c = iter->c;
- struct btree *b = iter->l[0].node;
+ struct btree *b = iter_s(iter)->l[0].node;
struct btree_reserve *reserve;
struct async_split *as;
int ret = 0;
@@ -1497,8 +1497,8 @@ out_get_locks:
unsigned i;
for (i = 0; i < BTREE_MAX_DEPTH; i++) {
- btree_node_unlock(linked, i);
- linked->l[i].lock_seq--;
+ btree_node_unlock(iter_s(linked), i);
+ iter_s(linked)->l[i].lock_seq--;
}
linked->locks_want = U8_MAX;
}
@@ -1516,7 +1516,8 @@ btree_insert_key(struct btree_insert_trans *trans,
struct journal_res *res,
unsigned flags)
{
- struct btree *b = insert->iter->l[0].node;
+ struct btree_iter_level *l = &iter_s(insert->iter)->l[0];
+ struct btree *b = l->node;
s64 oldsize = bch_count_data(&b->keys);
enum btree_insert_ret ret;
@@ -1539,7 +1540,7 @@ static bool same_leaf_as_prev(struct btree_insert_trans *trans,
* point to the same leaf node they'll always be adjacent now:
*/
return i != trans->entries &&
- i[0].iter->l[0].node == i[-1].iter->l[0].node;
+ btree_iter_leaf(i[0].iter) == btree_iter_leaf(i[-1].iter);
}
#define trans_for_each_entry(trans, i) \
@@ -1551,7 +1552,7 @@ static void multi_lock_write(struct btree_insert_trans *trans)
trans_for_each_entry(trans, i)
if (!same_leaf_as_prev(trans, i))
- btree_node_lock_for_insert(i->iter->l[0].node, i->iter);
+ btree_node_lock_for_insert(btree_iter_leaf(i->iter), i->iter);
}
static void multi_unlock_write(struct btree_insert_trans *trans)
@@ -1560,7 +1561,7 @@ static void multi_unlock_write(struct btree_insert_trans *trans)
trans_for_each_entry(trans, i)
if (!same_leaf_as_prev(trans, i))
- btree_node_unlock_write(i->iter->l[0].node, i->iter);
+ btree_node_unlock_write(btree_iter_leaf(i->iter), i->iter);
}
static int btree_trans_entry_cmp(const void *_l, const void *_r)
@@ -1633,7 +1634,7 @@ retry:
if (!i->done) {
u64s += i->k->k.u64s;
if (!bch_btree_node_insert_fits(c,
- i->iter->l[0].node, u64s))
+ btree_iter_leaf(i->iter), u64s))
goto unlock_split;
}
}
@@ -1679,7 +1680,7 @@ retry:
trans_for_each_entry(trans, i)
if (!same_leaf_as_prev(trans, i))
- bch_btree_node_write_lazy(i->iter->l[0].node, i->iter);
+ bch_btree_node_write_lazy(btree_iter_leaf(i->iter), i->iter);
out:
percpu_ref_put(&c->writes);
return ret;
@@ -1965,7 +1966,7 @@ int bch_btree_delete_range(struct cache_set *c,
delete.k.p = iter.pos;
delete.k.version = version;
- if (iter.is_extents) {
+ if (btree_iter_leaf(&iter)->keys.ops->is_extents) {
/*
* The extents btree is special - KEY_TYPE_DISCARD is
* used for deletions, not KEY_TYPE_DELETED. This is an
@@ -2002,7 +2003,7 @@ int bch_btree_node_rewrite(struct btree_iter *iter, struct btree *b,
struct closure *cl)
{
struct cache_set *c = iter->c;
- struct btree *n, *parent = iter->l[b->level + 1].node;
+ struct btree *n, *parent = iter_s(iter)->l[b->level + 1].node;
struct btree_reserve *reserve;
struct async_split *as;
diff --git a/drivers/md/bcache/extents.c b/drivers/md/bcache/extents.c
index 817346a2db13..541f985c2b1a 100644
--- a/drivers/md/bcache/extents.c
+++ b/drivers/md/bcache/extents.c
@@ -114,7 +114,7 @@ bch_insert_fixup_key(struct btree_insert_trans *trans,
struct journal_res *res)
{
struct btree_iter *iter = insert->iter;
- struct btree_iter_level *l = &iter->l[0];
+ struct btree_iter_level *l = &iter_s(insert->iter)->l[0];
BUG_ON(iter->level);
@@ -816,14 +816,10 @@ struct btree_nr_keys bch_extent_sort_fix_overlapping(struct btree_keys *b,
return nr;
}
-static void bch_add_sectors(struct btree_iter *iter, struct bkey_s_c k,
- u64 offset, s64 sectors,
+static void bch_add_sectors(struct cache_set *c, struct btree *b,
+ struct bkey_s_c k, u64 offset, s64 sectors,
struct bucket_stats_cache_set *stats)
{
- struct cache_set *c = iter->c;
- struct btree *b = iter->l[0].node;
-
- EBUG_ON(iter->level);
EBUG_ON(bkey_cmp(bkey_start_pos(k.k), b->data->min_key) < 0);
if (!sectors)
@@ -836,39 +832,40 @@ static void bch_add_sectors(struct btree_iter *iter, struct bkey_s_c k,
bcache_dev_sectors_dirty_add(c, k.k->p.inode, offset, sectors);
}
-static void bch_subtract_sectors(struct btree_iter *iter, struct bkey_s_c k,
- u64 offset, s64 sectors,
+static void bch_subtract_sectors(struct cache_set *c, struct btree *b,
+ struct bkey_s_c k, u64 offset, s64 sectors,
struct bucket_stats_cache_set *stats)
{
- bch_add_sectors(iter, k, offset, -sectors, stats);
+ bch_add_sectors(c, b, k, offset, -sectors, stats);
}
/* These wrappers subtract exactly the sectors that we're removing from @k */
-static void bch_cut_subtract_back(struct btree_iter *iter,
+static void bch_cut_subtract_back(struct cache_set *c, struct btree *b,
struct bpos where, struct bkey_s k,
struct bucket_stats_cache_set *stats)
{
- bch_subtract_sectors(iter, k.s_c, where.offset,
+ bch_subtract_sectors(c, b, k.s_c, where.offset,
k.k->p.offset - where.offset,
stats);
bch_cut_back(where, k.k);
}
-static void bch_cut_subtract_front(struct btree_iter *iter,
+static void bch_cut_subtract_front(struct cache_set *c, struct btree *b,
struct bpos where, struct bkey_s k,
struct bucket_stats_cache_set *stats)
{
- bch_subtract_sectors(iter, k.s_c, bkey_start_offset(k.k),
+ bch_subtract_sectors(c, b, k.s_c, bkey_start_offset(k.k),
where.offset - bkey_start_offset(k.k),
stats);
__bch_cut_front(where, k);
}
-static void bch_drop_subtract(struct btree_iter *iter, struct bkey_s k,
+static void bch_drop_subtract(struct cache_set *c, struct btree *b,
+ struct bkey_s k,
struct bucket_stats_cache_set *stats)
{
if (k.k->size)
- bch_subtract_sectors(iter, k.s_c,
+ bch_subtract_sectors(c, b, k.s_c,
bkey_start_offset(k.k), k.k->size,
stats);
k.k->size = 0;
@@ -994,13 +991,14 @@ static bool bch_extent_merge_inline(struct btree_iter *iter,
#define MAX_LOCK_HOLD_TIME (5 * NSEC_PER_MSEC)
-static enum btree_insert_ret extent_insert_should_stop(struct btree_insert_trans *trans,
- struct btree_trans_entry *insert,
- struct journal_res *res,
- u64 start_time,
- unsigned nr_done)
+static enum btree_insert_ret extent_insert_should_stop(struct cache_set *c,
+ struct btree *b,
+ struct btree_insert_trans *trans,
+ struct btree_trans_entry *insert,
+ struct journal_res *res,
+ u64 start_time,
+ unsigned nr_done)
{
- struct btree *b = insert->iter->l[0].node;
/*
* Check if we have sufficient space in both the btree node and the
* journal reservation:
@@ -1014,7 +1012,7 @@ static enum btree_insert_ret extent_insert_should_stop(struct btree_insert_trans
* doing a lot of work under the btree node write lock - bail out if
* we've been running for too long and readers are waiting on the lock:
*/
- if (!bch_btree_node_insert_fits(insert->iter->c, b, insert->k->k.u64s))
+ if (!bch_btree_node_insert_fits(c, b, insert->k->k.u64s))
return BTREE_INSERT_BTREE_NODE_FULL;
else if (!journal_res_insert_fits(trans, insert, res))
return BTREE_INSERT_JOURNAL_RES_FULL; /* XXX worth tracing */
@@ -1041,9 +1039,10 @@ static void extent_do_insert(struct btree_iter *iter,
bset_bkey_last(i);
if (!(flags & BTREE_INSERT_NO_MARK_KEY))
- bch_add_sectors(iter, bkey_i_to_s_c(insert),
+ bch_add_sectors(iter->c, btree_iter_leaf(iter),
+ bkey_i_to_s_c(insert),
bkey_start_offset(&insert->k),
- insert->k.size, stats);
+ split->k.size, stats);
bch_btree_journal_key(iter, insert, res);
@@ -1141,7 +1140,7 @@ extent_insert_advance_pos(struct btree_insert_trans *trans,
struct journal_res *res, unsigned flags,
struct bucket_stats_cache_set *stats)
{
- struct btree *b = insert->iter->l[0].node;
+ struct btree *b = btree_iter_leaf(insert->iter);
struct bpos next_pos =
bpos_min(insert->k->k.p, k.k ? k.k->p : b->key.k.p);
@@ -1234,7 +1233,7 @@ bch_insert_fixup_extent(struct btree_insert_trans *trans,
{
struct btree_iter *iter = insert->iter;
struct cache_set *c = iter->c;
- struct btree_iter_level *l = &iter->l[0];
+ struct btree_iter_level *l = &iter_s(iter)->l[0];
struct btree *b = l->node;
struct btree_node_iter *node_iter = &l->node_iter;
struct btree_node_iter node_insert_iter = *node_iter;
@@ -1260,7 +1259,7 @@ bch_insert_fixup_extent(struct btree_insert_trans *trans,
EBUG_ON(bkey_cmp(iter->pos, bkey_start_pos(&insert->k->k)));
while (bkey_cmp(committed_pos, insert->k->k.p) < 0 &&
- (ret = extent_insert_should_stop(trans, insert, res,
+ (ret = extent_insert_should_stop(c, b, trans, insert, res,
start_time, nr_done)) == BTREE_INSERT_OK &&
(_k = bch_btree_node_iter_peek_all(node_iter, &b->keys))) {
struct bkey_s k = __bkey_disassemble(f, _k, &unpacked);
@@ -1283,8 +1282,8 @@ bch_insert_fixup_extent(struct btree_insert_trans *trans,
* bkey_start_pos(k.k) not monotonically increasing except for
* ancestors of a given snapshot with nonzero size:
*/
- if (!bch_snapshot_is_descendant(c, trans->snapshot,
- k.k->p.snapshot))
+ if (!bch_is_snapshot_ancestor(c, trans->snapshot,
+ k.k->p.snapshot))
continue;
if (bkey_cmp(bkey_start_pos(k.k), insert->k->k.p) >= 0)
@@ -1341,14 +1340,14 @@ bch_insert_fixup_extent(struct btree_insert_trans *trans,
switch (overlap) {
case BCH_EXTENT_OVERLAP_FRONT:
/* insert overlaps with start of k: */
- bch_cut_subtract_front(iter, insert->k->k.p, k, &stats);
+ bch_cut_subtract_front(c, b, insert->k->k.p, k, &stats);
BUG_ON(bkey_deleted(k.k));
extent_save(&b->keys, node_iter, _k, k.k);
break;
case BCH_EXTENT_OVERLAP_BACK:
/* insert overlaps with end of k: */
- bch_cut_subtract_back(iter,
+ bch_cut_subtract_back(c, b,
bkey_start_pos(&insert->k->k),
k, &stats);
BUG_ON(bkey_deleted(k.k));
@@ -1368,7 +1367,7 @@ bch_insert_fixup_extent(struct btree_insert_trans *trans,
if (!bkey_deleted(_k))
btree_keys_account_key_drop(&b->keys.nr, _k);
- bch_drop_subtract(iter, k, &stats);
+ bch_drop_subtract(c, b, k, &stats);
extent_save(&b->keys, node_iter, _k, k.k);
break;
@@ -1393,7 +1392,7 @@ bch_insert_fixup_extent(struct btree_insert_trans *trans,
BUG_ON(bkey_deleted(&split.k.k));
__bch_cut_front(bkey_start_pos(&insert->k->k), k);
- bch_cut_subtract_front(iter, insert->k->k.p, k, &stats);
+ bch_cut_subtract_front(c, b, insert->k->k.p, k, &stats);
BUG_ON(bkey_deleted(k.k));
extent_save(&b->keys, node_iter, _k, k.k);
diff --git a/drivers/md/bcache/snapshot.h b/drivers/md/bcache/snapshot.h
index 168f52271f05..81db6886ac71 100644
--- a/drivers/md/bcache/snapshot.h
+++ b/drivers/md/bcache/snapshot.h
@@ -36,9 +36,9 @@ bch_snapshot_entry(struct cache_set *c, u32 id)
?: __bch_snapshot_entry(c, id);
}
-static inline bool bch_snapshot_is_descendant(struct cache_set *c,
- struct snapshot *snapshot,
- u32 ancestor_id)
+static inline bool bch_is_snapshot_ancestor(struct cache_set *c,
+ struct snapshot *snapshot,
+ u32 ancestor_id)
{
struct snapshot_cached_entry *child, *ancestor;
u32 child_id = snapshot->id;
generated by cgit v1.2.3 (git 2.25.1) at 2021-01-25 06:57:46 +0000