summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKent Overstreet <kent.overstreet@gmail.com>2016-09-19 20:17:11 -0800
committerKent Overstreet <kent.overstreet@gmail.com>2016-09-19 20:17:11 -0800
commitdcc5a0164ea0bee90e3a7e1c542d544d34620d40 (patch)
treefb663dd30411fe48a8586b357da3a1f6bcb0a640
parent8e907d018708980a66f524a9695ebbede2df56f9 (diff)
bcache: add bch_journal_pin_add_if_older()
also uninline the journal pin stuff, there's enough callers (or will be soon) that it shouldn't be inlined.
-rw-r--r--drivers/md/bcache/btree_io.c2
-rw-r--r--drivers/md/bcache/btree_update.c58
-rw-r--r--drivers/md/bcache/journal.c130
-rw-r--r--drivers/md/bcache/journal.h61
4 files changed, 135 insertions, 116 deletions
diff --git a/drivers/md/bcache/btree_io.c b/drivers/md/bcache/btree_io.c
index 2c5542a494fd..aa6c1c3fa75c 100644
--- a/drivers/md/bcache/btree_io.c
+++ b/drivers/md/bcache/btree_io.c
@@ -483,7 +483,7 @@ int bch_btree_root_read(struct cache_set *c, enum btree_id id,
void bch_btree_complete_write(struct cache_set *c, struct btree *b,
struct btree_write *w)
{
- journal_pin_drop(&c->journal, &w->journal);
+ bch_journal_pin_drop(&c->journal, &w->journal);
closure_wake_up(&w->wait);
}
diff --git a/drivers/md/bcache/btree_update.c b/drivers/md/bcache/btree_update.c
index 48e062b7acbc..eee969f74c61 100644
--- a/drivers/md/bcache/btree_update.c
+++ b/drivers/md/bcache/btree_update.c
@@ -726,7 +726,7 @@ void bch_btree_journal_key(struct btree_iter *iter,
BUG_ON(!res->ref && test_bit(JOURNAL_REPLAY_DONE, &j->flags));
if (!journal_pin_active(&w->journal))
- journal_pin_add(j, &w->journal, btree_node_flush);
+ bch_journal_pin_add(j, &w->journal, btree_node_flush);
if (test_bit(JOURNAL_REPLAY_DONE, &j->flags)) {
bch_journal_add_keys(j, res, b->btree_id, insert);
@@ -783,8 +783,7 @@ __bch_btree_interior_update_alloc(struct btree *nodes[], unsigned nr_nodes,
{
struct cache_set *c = iter->c;
struct btree_interior_update *as;
- struct journal_entry_pin_list *pin_list = NULL;
- unsigned i, pin_idx = UINT_MAX;
+ unsigned i;
as = mempool_alloc(&c->btree_interior_update_pool, GFP_NOIO);
memset(as, 0, sizeof(*as));
@@ -795,52 +794,13 @@ __bch_btree_interior_update_alloc(struct btree *nodes[], unsigned nr_nodes,
bch_keylist_init(&as->parent_keys, as->inline_keys,
ARRAY_SIZE(as->inline_keys));
- /* block btree node from being written and write_idx changing: */
for (i = 0; i < nr_nodes; i++) {
- /*
- * It's not legal to call btree_node_lock_write() when @iter
- * does not point to nodes[i] - which happens in
- * bch_coalesce_nodes(), unfortunately.
- *
- * So far this is the only place where we have this issue:
- */
- if (iter->nodes[nodes[i]->level] == nodes[i])
- btree_node_lock_write(nodes[i], iter);
- else
- six_lock_write(&nodes[i]->lock);
- }
-
- for (i = 0; i < nr_nodes; i++) {
- struct btree_write *w = btree_current_write(nodes[i]);
-
- if (journal_pin_active(&w->journal)) {
- unsigned idx = fifo_entry_idx(&c->journal.pin,
- w->journal.pin_list);
-
- if (idx < pin_idx) {
- pin_list = w->journal.pin_list;
- pin_idx = idx;
- }
- }
- }
-
- if (!pin_list) {
- /*
- * We don't have a journal reservation to block cur_pin_list
- * from changing, need to use a barrier to make sure it points
- * to an initialised pin_list:
- */
- pin_list = c->journal.cur_pin_list;
- smp_rmb();
- }
-
- __journal_pin_add(&c->journal, pin_list, &as->journal, NULL);
-
- for (i = 0; i < nr_nodes; i++) {
- if (iter->nodes[nodes[i]->level] == nodes[i])
- btree_node_unlock_write(nodes[i], iter);
- else
- six_unlock_write(&nodes[i]->lock);
+ bch_journal_pin_add_if_older(&c->journal,
+ &nodes[i]->writes[0].journal,
+ &as->journal, NULL);
+ bch_journal_pin_add_if_older(&c->journal,
+ &nodes[i]->writes[1].journal,
+ &as->journal, NULL);
}
mutex_lock(&c->btree_interior_update_lock);
@@ -870,7 +830,7 @@ static void btree_interior_update_pointers_written(struct closure *cl)
struct cache_set *c = as->c;
unsigned i;
- journal_pin_drop(&c->journal, &as->journal);
+ bch_journal_pin_drop(&c->journal, &as->journal);
mutex_lock(&c->btree_interior_update_lock);
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index 30e1eb13dc77..03052ea7cbe5 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -23,6 +23,10 @@
static void journal_write(struct closure *);
static void journal_reclaim_fast(struct journal *);
+static void journal_pin_add_entry(struct journal *,
+ struct journal_entry_pin_list *,
+ struct journal_entry_pin *,
+ journal_pin_flush_fn);
static inline struct journal_buf *journal_cur_buf(struct journal *j)
{
@@ -248,7 +252,7 @@ redo_peek:
mutex_lock(&j->blacklist_lock);
- journal_pin_drop(j, &bl->pin);
+ bch_journal_pin_drop(j, &bl->pin);
list_del(&bl->list);
kfree(bl->entries);
kfree(bl);
@@ -271,9 +275,8 @@ journal_seq_blacklist_find(struct journal *j, u64 seq)
}
static struct journal_seq_blacklist *
-bch_journal_seq_blacklisted_new(struct cache_set *c, u64 seq)
+bch_journal_seq_blacklisted_new(struct journal *j, u64 seq)
{
- struct journal *j = &c->journal;
struct journal_seq_blacklist *bl;
lockdep_assert_held(&j->blacklist_lock);
@@ -334,7 +337,7 @@ int bch_journal_seq_should_ignore(struct cache_set *c, u64 seq, struct btree *b)
/*
* When we start the journal, bch_journal_start() will skip over @seq:
*/
- bl = bch_journal_seq_blacklisted_new(c, seq);
+ bl = bch_journal_seq_blacklisted_new(&c->journal, seq);
if (!bl) {
ret = -ENOMEM;
goto out;
@@ -762,7 +765,7 @@ static void journal_entries_free(struct journal *j,
}
}
-static int journal_seq_blacklist_read(struct cache_set *c,
+static int journal_seq_blacklist_read(struct journal *j,
struct journal_replay *i,
struct journal_entry_pin_list *p)
{
@@ -774,13 +777,13 @@ static int journal_seq_blacklist_read(struct cache_set *c,
JOURNAL_ENTRY_JOURNAL_SEQ_BLACKLISTED) {
seq = entry->_data[0];
- bl = bch_journal_seq_blacklisted_new(c, seq);
+ bl = bch_journal_seq_blacklisted_new(j, seq);
if (!bl) {
- mutex_unlock(&c->journal.blacklist_lock);
+ mutex_unlock(&j->blacklist_lock);
return -ENOMEM;
}
- __journal_pin_add(&c->journal, p, &bl->pin,
+ journal_pin_add_entry(j, p, &bl->pin,
journal_seq_blacklist_flush);
bl->written = true;
}
@@ -874,7 +877,7 @@ const char *bch_journal_read(struct cache_set *c, struct list_head *list)
if (i && le64_to_cpu(i->j.seq) == seq) {
atomic_set(&p->count, 1);
- if (journal_seq_blacklist_read(c, i, p))
+ if (journal_seq_blacklist_read(&c->journal, i, p))
return "insufficient memory";
i = list_is_last(&i->list, list)
@@ -999,6 +1002,8 @@ static void __bch_journal_next_entry(struct journal *j)
jset = journal_cur_buf(j)->data;
jset->seq = cpu_to_le64(++j->seq);
jset->u64s = 0;
+
+ BUG_ON(journal_pin_seq(j, p) != j->seq);
}
static enum {
@@ -1281,8 +1286,10 @@ void bch_journal_start(struct cache_set *c)
JOURNAL_ENTRY_JOURNAL_SEQ_BLACKLISTED,
0, 0);
- __journal_pin_add(j, &fifo_peek_back(&j->pin), &bl->pin,
- journal_seq_blacklist_flush);
+ journal_pin_add_entry(j,
+ &fifo_peek_back(&j->pin),
+ &bl->pin,
+ journal_seq_blacklist_flush);
bl->written = true;
}
@@ -1451,6 +1458,105 @@ static void journal_reclaim_fast(struct journal *j)
}
}
+/*
+ * Journal entry pinning - machinery for holding a reference on a given journal
+ * entry, marking it as dirty:
+ */
+
+static inline void __journal_pin_add(struct journal *j,
+ struct journal_entry_pin_list *pin_list,
+ struct journal_entry_pin *pin,
+ journal_pin_flush_fn flush_fn)
+{
+ BUG_ON(journal_pin_active(pin));
+
+ atomic_inc(&pin_list->count);
+ pin->pin_list = pin_list;
+ pin->flush = flush_fn;
+
+ if (flush_fn)
+ list_add(&pin->list, &pin_list->list);
+ else
+ INIT_LIST_HEAD(&pin->list);
+}
+
+static void journal_pin_add_entry(struct journal *j,
+ struct journal_entry_pin_list *pin_list,
+ struct journal_entry_pin *pin,
+ journal_pin_flush_fn flush_fn)
+{
+ spin_lock_irq(&j->pin_lock);
+ __journal_pin_add(j, pin_list, pin, flush_fn);
+ spin_unlock_irq(&j->pin_lock);
+}
+
+void bch_journal_pin_add(struct journal *j,
+ struct journal_entry_pin *pin,
+ journal_pin_flush_fn flush_fn)
+{
+ spin_lock_irq(&j->pin_lock);
+ __journal_pin_add(j, j->cur_pin_list, pin, flush_fn);
+ spin_unlock_irq(&j->pin_lock);
+}
+
+static inline bool __journal_pin_drop(struct journal *j,
+ struct journal_entry_pin *pin)
+{
+ struct journal_entry_pin_list *pin_list = pin->pin_list;
+
+ pin->pin_list = NULL;
+
+ /* journal_reclaim_work() might have already taken us off the list */
+ if (!list_empty_careful(&pin->list))
+ list_del_init(&pin->list);
+
+ return atomic_dec_and_test(&pin_list->count);
+}
+
+void bch_journal_pin_drop(struct journal *j,
+ struct journal_entry_pin *pin)
+{
+ unsigned long flags;
+ bool wakeup;
+
+ if (!journal_pin_active(pin))
+ return;
+
+ spin_lock_irqsave(&j->pin_lock, flags);
+ wakeup = __journal_pin_drop(j, pin);
+ spin_unlock_irqrestore(&j->pin_lock, flags);
+
+ /*
+ * Unpinning a journal entry make make journal_next_bucket() succeed, if
+ * writing a new last_seq will now make another bucket available:
+ *
+ * Nested irqsave is expensive, don't do the wakeup with lock held:
+ */
+ if (wakeup)
+ wake_up(&j->wait);
+}
+
+void bch_journal_pin_add_if_older(struct journal *j,
+ struct journal_entry_pin *src_pin,
+ struct journal_entry_pin *pin,
+ journal_pin_flush_fn flush_fn)
+{
+ spin_lock_irq(&j->pin_lock);
+
+ if (journal_pin_active(src_pin) &&
+ (!journal_pin_active(pin) ||
+ fifo_entry_idx(&j->pin, src_pin->pin_list) <
+ fifo_entry_idx(&j->pin, pin->pin_list))) {
+ if (journal_pin_active(pin))
+ __journal_pin_drop(j, pin);
+ __journal_pin_add(j, src_pin->pin_list,
+ pin, NULL);
+ }
+
+ spin_unlock_irq(&j->pin_lock);
+}
+
+
static struct journal_entry_pin *
journal_get_next_pin(struct journal *j, u64 seq_to_flush)
{
@@ -1473,7 +1579,7 @@ journal_get_next_pin(struct journal *j, u64 seq_to_flush)
ret = list_first_entry_or_null(&pin_list->list,
struct journal_entry_pin, list);
if (ret) {
- /* must be list_del_init(), see journal_pin_drop() */
+ /* must be list_del_init(), see bch_journal_pin_drop() */
list_del_init(&ret->list);
break;
}
diff --git a/drivers/md/bcache/journal.h b/drivers/md/bcache/journal.h
index 0813e8d7d12b..e851abe8440d 100644
--- a/drivers/md/bcache/journal.h
+++ b/drivers/md/bcache/journal.h
@@ -131,60 +131,13 @@ static inline bool journal_pin_active(struct journal_entry_pin *pin)
return pin->pin_list != NULL;
}
-static inline void __journal_pin_add(struct journal *j,
- struct journal_entry_pin_list *pin_list,
- struct journal_entry_pin *pin,
- journal_pin_flush_fn flush_fn)
-{
- BUG_ON(journal_pin_active(pin));
-
- spin_lock_irq(&j->pin_lock);
-
- atomic_inc(&pin_list->count);
- pin->pin_list = pin_list;
- pin->flush = flush_fn;
-
- if (flush_fn)
- list_add(&pin->list, &pin_list->list);
- else
- INIT_LIST_HEAD(&pin->list);
-
- spin_unlock_irq(&j->pin_lock);
-}
-
-static inline void journal_pin_add(struct journal *j,
- struct journal_entry_pin *pin,
- journal_pin_flush_fn flush_fn)
-{
- __journal_pin_add(j, j->cur_pin_list, pin, flush_fn);
-}
-
-static inline void journal_pin_drop(struct journal *j,
- struct journal_entry_pin *pin)
-{
- unsigned long flags;
-
- if (!journal_pin_active(pin))
- return;
-
- /* journal_reclaim_work() might have already taken us off the list */
- if (!list_empty_careful(&pin->list)) {
- spin_lock_irqsave(&j->pin_lock, flags);
- list_del_init(&pin->list);
- spin_unlock_irqrestore(&j->pin_lock, flags);
- }
-
- if (atomic_dec_and_test(&pin->pin_list->count)) {
- /*
- * Unpinning a journal entry make make journal_next_bucket()
- * succeed, if writing a new last_seq will now make another
- * bucket available:
- */
- wake_up(&j->wait);
- }
-
- pin->pin_list = NULL;
-}
+void bch_journal_pin_add(struct journal *, struct journal_entry_pin *,
+ journal_pin_flush_fn);
+void bch_journal_pin_drop(struct journal *, struct journal_entry_pin *);
+void bch_journal_pin_add_if_older(struct journal *,
+ struct journal_entry_pin *,
+ struct journal_entry_pin *,
+ journal_pin_flush_fn);
struct closure;
struct cache_set;