diff options
| author | Kent Overstreet <kent.overstreet@linux.dev> | 2024-06-03 17:03:54 -0400 |
|---|---|---|
| committer | Kent Overstreet <kent.overstreet@linux.dev> | 2024-06-03 20:08:06 -0400 |
| commit | 7d79fba1af3d41b148009d3dc76d813b07f01f69 (patch) | |
| tree | bee9ba49d247a1c74d1c1e7c9c14b37a243bba89 | |
| parent | d456f9e97aebe649a59cc2858d5f28749b77d1d0 (diff) | |
Update bcachefs sources to f81dc88f0c80 bcachefs: bch2_btree_insert() - add btree iter flags
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
88 files changed, 3258 insertions, 2222 deletions
diff --git a/.bcachefs_revision b/.bcachefs_revision index e0161a6c..a9d4e51a 100644 --- a/.bcachefs_revision +++ b/.bcachefs_revision @@ -1 +1 @@ -8528bde1b66bab9a0abc2f521523abd00049c81b +f81dc88f0c80be08f1703852be83a9b75e626228 diff --git a/bch_bindgen/src/bkey.rs b/bch_bindgen/src/bkey.rs index b9cab4a1..eab070c6 100644 --- a/bch_bindgen/src/bkey.rs +++ b/bch_bindgen/src/bkey.rs @@ -49,6 +49,7 @@ pub enum BkeyValC<'a> { snapshot_tree(&'a c::bch_snapshot_tree), logged_op_truncate(&'a c::bch_logged_op_truncate), logged_op_finsert(&'a c::bch_logged_op_finsert), + accounting(&'a c::bch_accounting), } impl<'a, 'b> BkeySC<'a> { @@ -104,6 +105,7 @@ impl<'a, 'b> BkeySC<'a> { KEY_TYPE_snapshot_tree => snapshot_tree(transmute(self.v)), KEY_TYPE_logged_op_truncate => logged_op_truncate(transmute(self.v)), KEY_TYPE_logged_op_finsert => logged_op_finsert(transmute(self.v)), + KEY_TYPE_accounting => accounting(transmute(self.v)), KEY_TYPE_MAX => unreachable!(), } } diff --git a/c_src/cmd_dump.c b/c_src/cmd_dump.c index f54ea85d..f89d83ae 100644 --- a/c_src/cmd_dump.c +++ b/c_src/cmd_dump.c @@ -153,7 +153,7 @@ int cmd_dump(int argc, char *argv[]) if (IS_ERR(c)) die("error opening devices: %s", bch2_err_str(PTR_ERR(c))); - down_read(&c->gc_lock); + down_read(&c->state_lock); for_each_online_member(c, ca) nr_devices++; @@ -176,7 +176,7 @@ int cmd_dump(int argc, char *argv[]) close(fd); } - up_read(&c->gc_lock); + up_read(&c->state_lock); bch2_fs_stop(c); return 0; diff --git a/c_src/cmd_fsck.c b/c_src/cmd_fsck.c index 580df0d5..3d47af6c 100644 --- a/c_src/cmd_fsck.c +++ b/c_src/cmd_fsck.c @@ -236,6 +236,7 @@ int cmd_fsck(int argc, char *argv[]) kernel_probed = should_use_kernel_fsck(devs); struct bch_opts opts = bch2_opts_empty(); + struct printbuf parse_later = PRINTBUF; if (kernel_probed) { struct bch_ioctl_fsck_offline *fsck = calloc(sizeof(*fsck) + @@ -259,7 +260,7 @@ int cmd_fsck(int argc, char *argv[]) ret = splice_fd_to_stdinout(fsck_fd); } else { userland_fsck: - ret = bch2_parse_mount_opts(NULL, &opts, opts_str.buf); + ret = bch2_parse_mount_opts(NULL, &opts, &parse_later, opts_str.buf); if (ret) return ret; diff --git a/c_src/cmd_migrate.c b/c_src/cmd_migrate.c index 81c57c09..24937822 100644 --- a/c_src/cmd_migrate.c +++ b/c_src/cmd_migrate.c @@ -126,7 +126,7 @@ static void update_inode(struct bch_fs *c, bch2_inode_pack(&packed, inode); packed.inode.k.p.snapshot = U32_MAX; ret = bch2_btree_insert(c, BTREE_ID_inodes, &packed.inode.k_i, - NULL, 0); + NULL, 0, 0); if (ret) die("error updating inode: %s", bch2_err_str(ret)); } @@ -340,7 +340,7 @@ static void link_data(struct bch_fs *c, struct bch_inode_unpacked *dst, die("error reserving space in new filesystem: %s", bch2_err_str(ret)); - ret = bch2_btree_insert(c, BTREE_ID_extents, &e->k_i, &res, 0); + ret = bch2_btree_insert(c, BTREE_ID_extents, &e->k_i, &res, 0, 0); if (ret) die("btree insert error %s", bch2_err_str(ret)); diff --git a/include/linux/atomic.h b/include/linux/atomic.h index 5313f850..73024023 100644 --- a/include/linux/atomic.h +++ b/include/linux/atomic.h @@ -41,6 +41,17 @@ typedef struct { #define cmpxchg_acquire(p, old, new) uatomic_cmpxchg(p, old, new) #define cmpxchg_release(p, old, new) uatomic_cmpxchg(p, old, new) +#define try_cmpxchg(p, _old, _new) \ +({ \ + typeof(*(_old)) _v = cmpxchg(p, *(_old), _new); \ + bool _ret = _v == *(_old); \ + *(_old) = _v; \ + _ret; \ +}) + +#define try_cmpxchg_acquire(p, _old, _new) \ + try_cmpxchg(p, _old, _new) + #define smp_mb__before_atomic() cmm_smp_mb__before_uatomic_add() #define smp_mb__after_atomic() cmm_smp_mb__after_uatomic_add() #define smp_wmb() cmm_smp_wmb() @@ -65,6 +76,11 @@ typedef struct { #define xchg(p, v) __atomic_exchange_n(p, v, __ATOMIC_SEQ_CST) #define xchg_acquire(p, v) __atomic_exchange_n(p, v, __ATOMIC_ACQUIRE) +#define try_cmpxchg(p, old, new) \ + __atomic_compare_exchange_n((p), __old, new, false, \ + __ATOMIC_SEQ_CST, \ + __ATOMIC_SEQ_CST) + #define cmpxchg(p, old, new) \ ({ \ typeof(*(p)) __old = (old); \ @@ -281,6 +297,11 @@ static inline i_type a_type##_cmpxchg(a_type##_t *v, i_type old, i_type new)\ return cmpxchg(&v->counter, old, new); \ } \ \ +static inline bool a_type##_try_cmpxchg(a_type##_t *v, i_type *old, i_type new)\ +{ \ + return try_cmpxchg(&v->counter, old, new); \ +} \ + \ static inline i_type a_type##_cmpxchg_acquire(a_type##_t *v, i_type old, i_type new)\ { \ return cmpxchg_acquire(&v->counter, old, new); \ diff --git a/libbcachefs/alloc_background.c b/libbcachefs/alloc_background.c index 346cd91f..c220e9c9 100644 --- a/libbcachefs/alloc_background.c +++ b/libbcachefs/alloc_background.c @@ -14,6 +14,7 @@ #include "buckets_waiting_for_journal.h" #include "clock.h" #include "debug.h" +#include "disk_accounting.h" #include "ec.h" #include "error.h" #include "lru.h" @@ -259,27 +260,41 @@ int bch2_alloc_v4_invalid(struct bch_fs *c, struct bkey_s_c k, "invalid data type (got %u should be %u)", a.v->data_type, alloc_data_type(*a.v, a.v->data_type)); + unsigned stripe_sectors = BCH_ALLOC_V4_BACKPOINTERS_START(a.v) * sizeof(u64) > + offsetof(struct bch_alloc_v4, stripe_sectors) + ? a.v->stripe_sectors + : 0; + switch (a.v->data_type) { case BCH_DATA_free: case BCH_DATA_need_gc_gens: case BCH_DATA_need_discard: - bkey_fsck_err_on(bch2_bucket_sectors_total(*a.v) || a.v->stripe, + bkey_fsck_err_on(stripe_sectors || + a.v->dirty_sectors || + a.v->cached_sectors || + a.v->stripe, c, err, alloc_key_empty_but_have_data, - "empty data type free but have data"); + "empty data type free but have data %u.%u.%u %u", + stripe_sectors, + a.v->dirty_sectors, + a.v->cached_sectors, + a.v->stripe); break; case BCH_DATA_sb: case BCH_DATA_journal: case BCH_DATA_btree: case BCH_DATA_user: case BCH_DATA_parity: - bkey_fsck_err_on(!bch2_bucket_sectors_dirty(*a.v), + bkey_fsck_err_on(!a.v->dirty_sectors && + !stripe_sectors, c, err, alloc_key_dirty_sectors_0, "data_type %s but dirty_sectors==0", bch2_data_type_str(a.v->data_type)); break; case BCH_DATA_cached: bkey_fsck_err_on(!a.v->cached_sectors || - bch2_bucket_sectors_dirty(*a.v) || + a.v->dirty_sectors || + stripe_sectors || a.v->stripe, c, err, alloc_key_cached_inconsistency, "data type inconsistency"); @@ -310,6 +325,7 @@ void bch2_alloc_v4_swab(struct bkey_s k) a->stripe = swab32(a->stripe); a->nr_external_backpointers = swab32(a->nr_external_backpointers); a->fragmentation_lru = swab64(a->fragmentation_lru); + a->stripe_sectors = swab32(a->stripe_sectors); bps = alloc_v4_backpointers(a); for (bp = bps; bp < bps + BCH_ALLOC_V4_NR_BACKPOINTERS(a); bp++) { @@ -334,6 +350,7 @@ void bch2_alloc_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c prt_printf(out, "need_discard %llu\n", BCH_ALLOC_V4_NEED_DISCARD(a)); prt_printf(out, "need_inc_gen %llu\n", BCH_ALLOC_V4_NEED_INC_GEN(a)); prt_printf(out, "dirty_sectors %u\n", a->dirty_sectors); + prt_printf(out, "stripe_sectors %u\n", a->stripe_sectors); prt_printf(out, "cached_sectors %u\n", a->cached_sectors); prt_printf(out, "stripe %u\n", a->stripe); prt_printf(out, "stripe_redundancy %u\n", a->stripe_redundancy); @@ -570,8 +587,6 @@ int bch2_alloc_read(struct bch_fs *c) struct bch_dev *ca = NULL; int ret; - down_read(&c->gc_lock); - if (c->sb.version_upgrade_complete >= bcachefs_metadata_version_bucket_gens) { ret = for_each_btree_key(trans, iter, BTREE_ID_bucket_gens, POS_MIN, BTREE_ITER_prefetch, k, ({ @@ -620,7 +635,6 @@ int bch2_alloc_read(struct bch_fs *c) bch2_dev_put(ca); bch2_trans_put(trans); - up_read(&c->gc_lock); bch_err_fn(c, ret); return ret; @@ -735,6 +749,61 @@ static noinline int bch2_bucket_gen_update(struct btree_trans *trans, return ret; } +static inline int bch2_dev_data_type_accounting_mod(struct btree_trans *trans, struct bch_dev *ca, + enum bch_data_type data_type, + s64 delta_buckets, + s64 delta_sectors, + s64 delta_fragmented, unsigned flags) +{ + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_dev_data_type, + .dev_data_type.dev = ca->dev_idx, + .dev_data_type.data_type = data_type, + }; + s64 d[3] = { delta_buckets, delta_sectors, delta_fragmented }; + + return bch2_disk_accounting_mod(trans, &acc, d, 3, flags & BTREE_TRIGGER_gc); +} + +int bch2_alloc_key_to_dev_counters(struct btree_trans *trans, struct bch_dev *ca, + const struct bch_alloc_v4 *old, + const struct bch_alloc_v4 *new, + unsigned flags) +{ + s64 old_sectors = bch2_bucket_sectors(*old); + s64 new_sectors = bch2_bucket_sectors(*new); + if (old->data_type != new->data_type) { + int ret = bch2_dev_data_type_accounting_mod(trans, ca, new->data_type, + 1, new_sectors, bch2_bucket_sectors_fragmented(ca, *new), flags) ?: + bch2_dev_data_type_accounting_mod(trans, ca, old->data_type, + -1, -old_sectors, -bch2_bucket_sectors_fragmented(ca, *old), flags); + if (ret) + return ret; + } else if (old_sectors != new_sectors) { + int ret = bch2_dev_data_type_accounting_mod(trans, ca, new->data_type, + 0, + new_sectors - old_sectors, + bch2_bucket_sectors_fragmented(ca, *new) - + bch2_bucket_sectors_fragmented(ca, *old), flags); + if (ret) + return ret; + } + + s64 old_unstriped = bch2_bucket_sectors_unstriped(*old); + s64 new_unstriped = bch2_bucket_sectors_unstriped(*new); + if (old_unstriped != new_unstriped) { + int ret = bch2_dev_data_type_accounting_mod(trans, ca, BCH_DATA_unstriped, + !!new_unstriped - !!old_unstriped, + new_unstriped - old_unstriped, + 0, + flags); + if (ret) + return ret; + } + + return 0; +} + int bch2_trigger_alloc(struct btree_trans *trans, enum btree_id btree, unsigned level, struct bkey_s_c old, struct bkey_s new, @@ -749,10 +818,9 @@ int bch2_trigger_alloc(struct btree_trans *trans, struct bch_alloc_v4 old_a_convert; const struct bch_alloc_v4 *old_a = bch2_alloc_to_v4(old, &old_a_convert); + struct bch_alloc_v4 *new_a = bkey_s_to_alloc_v4(new).v; if (flags & BTREE_TRIGGER_transactional) { - struct bch_alloc_v4 *new_a = bkey_s_to_alloc_v4(new).v; - alloc_data_type_set(new_a, new_a->data_type); if (bch2_bucket_sectors_total(*new_a) > bch2_bucket_sectors_total(*old_a)) { @@ -808,22 +876,21 @@ int bch2_trigger_alloc(struct btree_trans *trans, goto err; } - /* - * need to know if we're getting called from the invalidate path or - * not: - */ - if ((flags & BTREE_TRIGGER_bucket_invalidate) && old_a->cached_sectors) { - ret = bch2_update_cached_sectors_list(trans, new.k->p.inode, - -((s64) old_a->cached_sectors)); + ret = bch2_mod_dev_cached_sectors(trans, ca->dev_idx, + -((s64) old_a->cached_sectors), + flags & BTREE_TRIGGER_gc); if (ret) goto err; } + + ret = bch2_alloc_key_to_dev_counters(trans, ca, old_a, new_a, flags); + if (ret) + goto err; } if ((flags & BTREE_TRIGGER_atomic) && (flags & BTREE_TRIGGER_insert)) { - struct bch_alloc_v4 *new_a = bkey_s_to_alloc_v4(new).v; u64 journal_seq = trans->journal_res.seq; u64 bucket_journal_seq = new_a->journal_seq; @@ -852,19 +919,16 @@ int bch2_trigger_alloc(struct btree_trans *trans, c->journal.flushed_seq_ondisk, new.k->p.inode, new.k->p.offset, bucket_journal_seq); - if (ret) { - bch2_fs_fatal_error(c, - "setting bucket_needs_journal_commit: %s", bch2_err_str(ret)); + if (bch2_fs_fatal_err_on(ret, c, + "setting bucket_needs_journal_commit: %s", bch2_err_str(ret))) goto err; - } } - percpu_down_read(&c->mark_lock); - if (new_a->gen != old_a->gen) + if (new_a->gen != old_a->gen) { + percpu_down_read(&c->mark_lock); *bucket_gen(ca, new.k->p.offset) = new_a->gen; - - bch2_dev_usage_update(c, ca, old_a, new_a, journal_seq, false); - percpu_up_read(&c->mark_lock); + percpu_up_read(&c->mark_lock); + } #define eval_state(_a, expr) ({ const struct bch_alloc_v4 *a = _a; expr; }) #define statechange(expr) !eval_state(old_a, expr) && eval_state(new_a, expr) @@ -888,26 +952,12 @@ int bch2_trigger_alloc(struct btree_trans *trans, bch2_gc_gens_async(c); } - if ((flags & BTREE_TRIGGER_gc) && - (flags & BTREE_TRIGGER_bucket_invalidate)) { - struct bch_alloc_v4 new_a_convert; - const struct bch_alloc_v4 *new_a = bch2_alloc_to_v4(new.s_c, &new_a_convert); - - percpu_down_read(&c->mark_lock); + if ((flags & BTREE_TRIGGER_gc) && (flags & BTREE_TRIGGER_insert)) { + rcu_read_lock(); struct bucket *g = gc_bucket(ca, new.k->p.offset); - - bucket_lock(g); - - g->gen_valid = 1; - g->gen = new_a->gen; - g->data_type = new_a->data_type; - g->stripe = new_a->stripe; - g->stripe_redundancy = new_a->stripe_redundancy; - g->dirty_sectors = new_a->dirty_sectors; - g->cached_sectors = new_a->cached_sectors; - - bucket_unlock(g); - percpu_up_read(&c->mark_lock); + g->gen_valid = 1; + g->gen = new_a->gen; + rcu_read_unlock(); } err: bch2_dev_put(ca); @@ -1035,7 +1085,7 @@ int bch2_check_alloc_key(struct btree_trans *trans, struct bch_dev *ca = bch2_dev_bucket_tryget_noerror(c, alloc_k.k->p); if (fsck_err_on(!ca, - c, alloc_key_to_missing_dev_bucket, + trans, alloc_key_to_missing_dev_bucket, "alloc key for invalid device:bucket %llu:%llu", alloc_k.k->p.inode, alloc_k.k->p.offset)) ret = bch2_btree_delete_at(trans, alloc_iter, 0); @@ -1055,7 +1105,7 @@ int bch2_check_alloc_key(struct btree_trans *trans, goto err; if (fsck_err_on(k.k->type != discard_key_type, - c, need_discard_key_wrong, + trans, need_discard_key_wrong, "incorrect key in need_discard btree (got %s should be %s)\n" " %s", bch2_bkey_types[k.k->type], @@ -1085,7 +1135,7 @@ int bch2_check_alloc_key(struct btree_trans *trans, goto err; if (fsck_err_on(k.k->type != freespace_key_type, - c, freespace_key_wrong, + trans, freespace_key_wrong, "incorrect key in freespace btree (got %s should be %s)\n" " %s", bch2_bkey_types[k.k->type], @@ -1116,7 +1166,7 @@ int bch2_check_alloc_key(struct btree_trans *trans, goto err; if (fsck_err_on(a->gen != alloc_gen(k, gens_offset), - c, bucket_gens_key_wrong, + trans, bucket_gens_key_wrong, "incorrect gen in bucket_gens btree (got %u should be %u)\n" " %s", alloc_gen(k, gens_offset), a->gen, @@ -1157,7 +1207,6 @@ int bch2_check_alloc_hole_freespace(struct btree_trans *trans, struct bpos *end, struct btree_iter *freespace_iter) { - struct bch_fs *c = trans->c; struct bkey_s_c k; struct printbuf buf = PRINTBUF; int ret; @@ -1175,7 +1224,7 @@ int bch2_check_alloc_hole_freespace(struct btree_trans *trans, *end = bkey_min(k.k->p, *end); if (fsck_err_on(k.k->type != KEY_TYPE_set, - c, freespace_hole_missing, + trans, freespace_hole_missing, "hole in alloc btree missing in freespace btree\n" " device %llu buckets %llu-%llu", freespace_iter->pos.inode, @@ -1211,7 +1260,6 @@ int bch2_check_alloc_hole_bucket_gens(struct btree_trans *trans, struct bpos *end, struct btree_iter *bucket_gens_iter) { - struct bch_fs *c = trans->c; struct bkey_s_c k; struct printbuf buf = PRINTBUF; unsigned i, gens_offset, gens_end_offset; @@ -1235,7 +1283,7 @@ int bch2_check_alloc_hole_bucket_gens(struct btree_trans *trans, bkey_reassemble(&g.k_i, k); for (i = gens_offset; i < gens_end_offset; i++) { - if (fsck_err_on(g.v.gens[i], c, + if (fsck_err_on(g.v.gens[i], trans, bucket_gens_hole_wrong, "hole in alloc btree at %llu:%llu with nonzero gen in bucket_gens btree (%u)", bucket_gens_pos_to_alloc(k.k->p, i).inode, @@ -1293,8 +1341,8 @@ static noinline_for_stack int bch2_check_discard_freespace_key(struct btree_tran if (ret) return ret; - if (fsck_err_on(!bch2_dev_bucket_exists(c, pos), c, - need_discard_freespace_key_to_invalid_dev_bucket, + if (fsck_err_on(!bch2_dev_bucket_exists(c, pos), + trans, need_discard_freespace_key_to_invalid_dev_bucket, "entry in %s btree for nonexistant dev:bucket %llu:%llu", bch2_btree_id_str(iter->btree_id), pos.inode, pos.offset)) goto delete; @@ -1303,8 +1351,8 @@ static noinline_for_stack int bch2_check_discard_freespace_key(struct btree_tran if (fsck_err_on(a->data_type != state || (state == BCH_DATA_free && - genbits != alloc_freespace_genbits(*a)), c, - need_discard_freespace_key_bad, + genbits != alloc_freespace_genbits(*a)), + trans, need_discard_freespace_key_bad, "%s\n incorrectly set at %s:%llu:%llu:0 (free %u, genbits %llu should be %llu)", (bch2_bkey_val_to_text(&buf, c, alloc_k), buf.buf), bch2_btree_id_str(iter->btree_id), @@ -1351,7 +1399,7 @@ int bch2_check_bucket_gens_key(struct btree_trans *trans, struct bch_dev *ca = bch2_dev_tryget_noerror(c, k.k->p.inode); if (!ca) { - if (fsck_err(c, bucket_gens_to_invalid_dev, + if (fsck_err(trans, bucket_gens_to_invalid_dev, "bucket_gens key for invalid device:\n %s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) ret = bch2_btree_delete_at(trans, iter, 0); @@ -1359,8 +1407,8 @@ int bch2_check_bucket_gens_key(struct btree_trans *trans, } if (fsck_err_on(end <= ca->mi.first_bucket || - start >= ca->mi.nbuckets, c, - bucket_gens_to_invalid_buckets, + start >= ca->mi.nbuckets, + trans, bucket_gens_to_invalid_buckets, "bucket_gens key for invalid buckets:\n %s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { ret = bch2_btree_delete_at(trans, iter, 0); @@ -1368,16 +1416,16 @@ int bch2_check_bucket_gens_key(struct btree_trans *trans, } for (b = start; b < ca->mi.first_bucket; b++) - if (fsck_err_on(g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK], c, - bucket_gens_nonzero_for_invalid_buckets, + if (fsck_err_on(g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK], + trans, bucket_gens_nonzero_for_invalid_buckets, "bucket_gens key has nonzero gen for invalid bucket")) { g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK] = 0; need_update = true; } for (b = ca->mi.nbuckets; b < end; b++) - if (fsck_err_on(g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK], c, - bucket_gens_nonzero_for_invalid_buckets, + if (fsck_err_on(g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK], + trans, bucket_gens_nonzero_for_invalid_buckets, "bucket_gens key has nonzero gen for invalid bucket")) { g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK] = 0; need_update = true; @@ -1549,8 +1597,8 @@ static int bch2_check_alloc_to_lru_ref(struct btree_trans *trans, if (a->data_type != BCH_DATA_cached) return 0; - if (fsck_err_on(!a->io_time[READ], c, - alloc_key_cached_but_read_time_zero, + if (fsck_err_on(!a->io_time[READ], + trans, alloc_key_cached_but_read_time_zero, "cached bucket with read_time 0\n" " %s", (printbuf_reset(&buf), @@ -1578,8 +1626,8 @@ static int bch2_check_alloc_to_lru_ref(struct btree_trans *trans, if (ret) return ret; - if (fsck_err_on(lru_k.k->type != KEY_TYPE_set, c, - alloc_key_to_missing_lru_entry, + if (fsck_err_on(lru_k.k->type != KEY_TYPE_set, + trans, alloc_key_to_missing_lru_entry, "missing lru entry\n" " %s", (printbuf_reset(&buf), @@ -1616,7 +1664,7 @@ static int discard_in_flight_add(struct bch_fs *c, struct bpos bucket) mutex_lock(&c->discard_buckets_in_flight_lock); darray_for_each(c->discard_buckets_in_flight, i) if (bkey_eq(*i, bucket)) { - ret = -EEXIST; + ret = -BCH_ERR_EEXIST_discard_in_flight_add; goto out; } @@ -1956,6 +2004,7 @@ static int invalidate_one_bucket(struct btree_trans *trans, a->v.gen++; a->v.data_type = 0; a->v.dirty_sectors = 0; + a->v.stripe_sectors = 0; a->v.cached_sectors = 0; a->v.io_time[READ] = atomic64_read(&c->io_clock[READ].now); a->v.io_time[WRITE] = atomic64_read(&c->io_clock[WRITE].now); diff --git a/libbcachefs/alloc_background.h b/libbcachefs/alloc_background.h index ae31a94b..59c08c79 100644 --- a/libbcachefs/alloc_background.h +++ b/libbcachefs/alloc_background.h @@ -41,6 +41,7 @@ static inline void alloc_to_bucket(struct bucket *dst, struct bch_alloc_v4 src) { dst->gen = src.gen; dst->data_type = src.data_type; + dst->stripe_sectors = src.stripe_sectors; dst->dirty_sectors = src.dirty_sectors; dst->cached_sectors = src.cached_sectors; dst->stripe = src.stripe; @@ -50,6 +51,7 @@ static inline void __bucket_m_to_alloc(struct bch_alloc_v4 *dst, struct bucket s { dst->gen = src.gen; dst->data_type = src.data_type; + dst->stripe_sectors = src.stripe_sectors; dst->dirty_sectors = src.dirty_sectors; dst->cached_sectors = src.cached_sectors; dst->stripe = src.stripe; @@ -80,30 +82,49 @@ static inline bool bucket_data_type_mismatch(enum bch_data_type bucket, bucket_data_type(bucket) != bucket_data_type(ptr); } -static inline unsigned bch2_bucket_sectors_total(struct bch_alloc_v4 a) +static inline s64 bch2_bucket_sectors_total(struct bch_alloc_v4 a) { - return a.dirty_sectors + a.cached_sectors; + return a.stripe_sectors + a.dirty_sectors + a.cached_sectors; } -static inline unsigned bch2_bucket_sectors_dirty(struct bch_alloc_v4 a) +static inline s64 bch2_bucket_sectors_dirty(struct bch_alloc_v4 a) { - return a.dirty_sectors; + return a.stripe_sectors + a.dirty_sectors; } -static inline unsigned bch2_bucket_sectors_fragmented(struct bch_dev *ca, +static inline s64 bch2_bucket_sectors(struct bch_alloc_v4 a) +{ + return a.data_type == BCH_DATA_cached + ? a.cached_sectors + : bch2_bucket_sectors_dirty(a); +} + +static inline s64 bch2_bucket_sectors_fragmented(struct bch_dev *ca, struct bch_alloc_v4 a) { - int d = bch2_bucket_sectors_dirty(a); + int d = bch2_bucket_sectors(a); return d ? max(0, ca->mi.bucket_size - d) : 0; } +static inline s64 bch2_gc_bucket_sectors_fragmented(struct bch_dev *ca, struct bucket a) +{ + int d = a.stripe_sectors + a.dirty_sectors; + + return d ? max(0, ca->mi.bucket_size - d) : 0; +} + +static inline s64 bch2_bucket_sectors_unstriped(struct bch_alloc_v4 a) +{ + return a.data_type == BCH_DATA_stripe ? a.dirty_sectors : 0; +} + static inline enum bch_data_type alloc_data_type(struct bch_alloc_v4 a, enum bch_data_type data_type) { if (a.stripe) return data_type == BCH_DATA_parity ? data_type : BCH_DATA_stripe; - if (a.dirty_sectors) + if (bch2_bucket_sectors_dirty(a)) return data_type; if (a.cached_sectors) return BCH_DATA_cached; @@ -264,6 +285,9 @@ static inline bool bkey_is_alloc(const struct bkey *k) int bch2_alloc_read(struct bch_fs *); +int bch2_alloc_key_to_dev_counters(struct btree_trans *, struct bch_dev *, + const struct bch_alloc_v4 *, + const struct bch_alloc_v4 *, unsigned); int bch2_trigger_alloc(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_s, enum btree_iter_update_trigger_flags); diff --git a/libbcachefs/alloc_background_format.h b/libbcachefs/alloc_background_format.h index b4ec20be..47d9d006 100644 --- a/libbcachefs/alloc_background_format.h +++ b/libbcachefs/alloc_background_format.h @@ -70,6 +70,8 @@ struct bch_alloc_v4 { __u32 stripe; __u32 nr_external_backpointers; __u64 fragmentation_lru; + __u32 stripe_sectors; + __u32 pad; } __packed __aligned(8); #define BCH_ALLOC_V4_U64s_V0 6 diff --git a/libbcachefs/alloc_foreground.c b/libbcachefs/alloc_foreground.c index 927a5f30..a352a671 100644 --- a/libbcachefs/alloc_foreground.c +++ b/libbcachefs/alloc_foreground.c @@ -1705,15 +1705,13 @@ void bch2_fs_alloc_debug_to_text(struct printbuf *out, struct bch_fs *c) printbuf_tabstop_push(out, 24); - percpu_down_read(&c->mark_lock); - prt_printf(out, "hidden\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.hidden)); - prt_printf(out, "btree\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.btree)); - prt_printf(out, "data\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.data)); - prt_printf(out, "cached\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.cached)); - prt_printf(out, "reserved\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.reserved)); - prt_printf(out, "online_reserved\t%llu\n", percpu_u64_get(c->online_reserved)); - prt_printf(out, "nr_inodes\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.nr_inodes)); - percpu_up_read(&c->mark_lock); + prt_printf(out, "hidden\t%llu\n", percpu_u64_get(&c->usage->hidden)); + prt_printf(out, "btree\t%llu\n", percpu_u64_get(&c->usage->btree)); + prt_printf(out, "data\t%llu\n", percpu_u64_get(&c->usage->data)); + prt_printf(out, "cached\t%llu\n", percpu_u64_get(&c->usage->cached)); + prt_printf(out, "reserved\t%llu\n", percpu_u64_get(&c->usage->reserved)); + prt_printf(out, "online_reserved\t%llu\n", percpu_u64_get(c->online_reserved)); + prt_printf(out, "nr_inodes\t%llu\n", percpu_u64_get(&c->usage->nr_inodes)); prt_newline(out); prt_printf(out, "freelist_wait\t%s\n", c->freelist_wait.list.first ? "waiting" : "empty"); diff --git a/libbcachefs/backpointers.c b/libbcachefs/backpointers.c index 4321f9fb..ca16fa5d 100644 --- a/libbcachefs/backpointers.c +++ b/libbcachefs/backpointers.c @@ -395,7 +395,7 @@ static int bch2_check_btree_backpointer(struct btree_trans *trans, struct btree_ struct bpos bucket; if (!bp_pos_to_bucket_nodev_noerror(c, k.k->p, &bucket)) { - if (fsck_err(c, backpointer_to_missing_device, + if (fsck_err(trans, backpointer_to_missing_device, "backpointer for missing device:\n%s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) ret = bch2_btree_delete_at(trans, bp_iter, 0); @@ -407,8 +407,8 @@ static int bch2_check_btree_backpointer(struct btree_trans *trans, struct btree_ if (ret) goto out; - if (fsck_err_on(alloc_k.k->type != KEY_TYPE_alloc_v4, c, - backpointer_to_missing_alloc, + if (fsck_err_on(alloc_k.k->type != KEY_TYPE_alloc_v4, + trans, backpointer_to_missing_alloc, "backpointer for nonexistent alloc key: %llu:%llu:0\n%s", alloc_iter.pos.inode, alloc_iter.pos.offset, (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { @@ -512,7 +512,7 @@ found: struct nonce nonce = extent_nonce(extent.k->version, p.crc); struct bch_csum csum = bch2_checksum(c, p.crc.csum_type, nonce, data_buf, bytes); if (fsck_err_on(bch2_crc_cmp(csum, p.crc.csum), - c, dup_backpointer_to_bad_csum_extent, + trans, dup_backpointer_to_bad_csum_extent, "%s", buf.buf)) ret = drop_dev_and_update(trans, btree, extent, dev) ?: 1; fsck_err: @@ -671,7 +671,7 @@ missing: prt_printf(&buf, "\n want: "); bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&n_bp_k.k_i)); - if (fsck_err(c, ptr_to_missing_backpointer, "%s", buf.buf)) + if (fsck_err(trans, ptr_to_missing_backpointer, "%s", buf.buf)) ret = bch2_bucket_backpointer_mod(trans, ca, bucket, bp, orig_k, true); goto out; @@ -932,8 +932,8 @@ static int check_one_backpointer(struct btree_trans *trans, goto out; } - if (fsck_err_on(!k.k, c, - backpointer_to_missing_ptr, + if (fsck_err_on(!k.k, + trans, backpointer_to_missing_ptr, "backpointer for missing %s\n %s", bp.v->level ? "btree node" : "extent", (bch2_bkey_val_to_text(&buf, c, bp.s_c), buf.buf))) { diff --git a/libbcachefs/bcachefs.h b/libbcachefs/bcachefs.h index 2a538eb2..1650446a 100644 --- a/libbcachefs/bcachefs.h +++ b/libbcachefs/bcachefs.h @@ -205,6 +205,7 @@ #include <linux/zstd.h> #include "bcachefs_format.h" +#include "disk_accounting_types.h" #include "errcode.h" #include "fifo.h" #include "nocow_locking_types.h" @@ -266,6 +267,8 @@ do { \ #define bch2_fmt(_c, fmt) bch2_log_msg(_c, fmt "\n") +void bch2_print_str(struct bch_fs *, const char *); + __printf(2, 3) void bch2_print_opts(struct bch_opts *, const char *, ...); @@ -530,8 +533,8 @@ struct bch_dev { /* * Buckets: * Per-bucket arrays are protected by c->mark_lock, bucket_lock and - * gc_lock, for device resize - holding any is sufficient for access: - * Or rcu_read_lock(), but only for dev_ptr_stale(): + * gc_gens_lock, for device resize - holding any is sufficient for + * access: Or rcu_read_lock(), but only for dev_ptr_stale(): */ struct bucket_array __rcu *buckets_gc; struct bucket_gens __rcu *bucket_gens; @@ -539,9 +542,7 @@ struct bch_dev { unsigned long *buckets_nouse; struct rw_semaphore bucket_lock; - struct bch_dev_usage *usage_base; - struct bch_dev_usage __percpu *usage[JOURNAL_BUF_NR]; - struct bch_dev_usage __percpu *usage_gc; + struct bch_dev_usage __percpu *usage; /* Allocator: */ u64 new_fs_bucket_idx; @@ -581,6 +582,8 @@ struct bch_dev { #define BCH_FS_FLAGS() \ x(new_fs) \ x(started) \ + x(btree_running) \ + x(accounting_replay_done) \ x(may_go_rw) \ x(rw) \ x(was_rw) \ @@ -659,8 +662,6 @@ struct btree_trans_buf { struct btree_trans *trans; }; -#define REPLICAS_DELTA_LIST_MAX (1U << 16) - #define BCACHEFS_ROOT_SUBVOL_INUM \ ((subvol_inum) { BCACHEFS_ROOT_SUBVOL, BCACHEFS_ROOT_INO }) @@ -730,15 +731,14 @@ struct bch_fs { struct bch_dev __rcu *devs[BCH_SB_MEMBERS_MAX]; + struct bch_accounting_mem accounting[2]; + struct bch_replicas_cpu replicas; struct bch_replicas_cpu replicas_gc; struct mutex replicas_gc_lock; - mempool_t replicas_delta_pool; struct journal_entry_res btree_root_journal_res; - struct journal_entry_res replicas_journal_res; struct journal_entry_res clock_journal_res; - struct journal_entry_res dev_usage_journal_res; struct bch_disk_groups_cpu __rcu *disk_groups; @@ -877,15 +877,9 @@ struct bch_fs { struct percpu_rw_semaphore mark_lock; seqcount_t usage_lock; - struct bch_fs_usage *usage_base; - struct bch_fs_usage __percpu *usage[JOURNAL_BUF_NR]; - struct bch_fs_usage __percpu *usage_gc; + struct bch_fs_usage_base __percpu *usage; u64 __percpu *online_reserved; - /* single element mempool: */ - struct mutex usage_scratch_lock; - struct bch_fs_usage_online *usage_scratch; - struct io_clock io_clock[2]; /* JOURNAL SEQ BLACKLIST */ @@ -943,7 +937,6 @@ struct bch_fs { * The allocation code needs gc_mark in struct bucket to be correct, but * it's not while a gc is in progress. */ - struct rw_semaphore gc_lock; struct mutex gc_gens_lock; /* IO PATH */ diff --git a/libbcachefs/bcachefs_format.h b/libbcachefs/bcachefs_format.h index 90c12fe2..0da82744 100644 --- a/libbcachefs/bcachefs_format.h +++ b/libbcachefs/bcachefs_format.h @@ -417,7 +417,8 @@ static inline void bkey_init(struct bkey *k) x(bucket_gens, 30) \ x(snapshot_tree, 31) \ x(logged_op_truncate, 32) \ - x(logged_op_finsert, 33) + x(logged_op_finsert, 33) \ + x(accounting, 34) enum bch_bkey_type { #define x(name, nr) KEY_TYPE_##name = nr, @@ -502,6 +503,9 @@ struct bch_sb_field { x(downgrade, 14) #include "alloc_background_format.h" +#include "dirent_format.h" +#include "disk_accounting_format.h" +#include "disk_groups_format.h" #include "extents_format.h" #include "ec_format.h" #include "dirent_format.h" @@ -599,48 +603,6 @@ LE64_BITMASK(BCH_KDF_SCRYPT_N, struct bch_sb_field_crypt, kdf_flags, 0, 16); LE64_BITMASK(BCH_KDF_SCRYPT_R, struct bch_sb_field_crypt, kdf_flags, 16, 32); LE64_BITMASK(BCH_KDF_SCRYPT_P, struct bch_sb_field_crypt, kdf_flags, 32, 48); -#define BCH_DATA_TYPES() \ - x(free, 0) \ - x(sb, 1) \ - x(journal, 2) \ - x(btree, 3) \ - x(user, 4) \ - x(cached, 5) \ - x(parity, 6) \ - x(stripe, 7) \ - x(need_gc_gens, 8) \ - x(need_discard, 9) - -enum bch_data_type { -#define x(t, n) BCH_DATA_##t, - BCH_DATA_TYPES() -#undef x - BCH_DATA_NR -}; - -static inline bool data_type_is_empty(enum bch_data_type type) -{ - switch (type) { - case BCH_DATA_free: - case BCH_DATA_need_gc_gens: - case BCH_DATA_need_discard: - return true; - default: - return false; - } -} - -static inline bool data_type_is_hidden(enum bch_data_type type) -{ - switch (type) { - case BCH_DATA_sb: - case BCH_DATA_journal: - return true; - default: - return false; - } -} - /* * On clean shutdown, store btree roots and current journal sequence number in * the superblock: @@ -719,7 +681,9 @@ struct bch_sb_field_ext { x(member_seq, BCH_VERSION(1, 4)) \ x(subvolume_fs_parent, BCH_VERSION(1, 5)) \ x(btree_subvolume_children, BCH_VERSION(1, 6)) \ - x(mi_btree_bitmap, BCH_VERSION(1, 7)) + x(mi_btree_bitmap, BCH_VERSION(1, 7)) \ + x(bucket_stripe_sectors, BCH_VERSION(1, 8)) \ + x(disk_accounting_v2, BCH_VERSION(1, 9)) enum bcachefs_metadata_version { bcachefs_metadata_version_min = 9, @@ -1170,7 +1134,6 @@ static inline bool jset_entry_is_key(struct jset_entry *e) switch (e->type) { case BCH_JSET_ENTRY_btree_keys: case BCH_JSET_ENTRY_btree_root: - case BCH_JSET_ENTRY_overwrite: case BCH_JSET_ENTRY_write_buffer_keys: return true; } @@ -1371,7 +1334,9 @@ enum btree_id_flags { x(rebalance_work, 18, BTREE_ID_SNAPSHOT_FIELD, \ BIT_ULL(KEY_TYPE_set)|BIT_ULL(KEY_TYPE_cookie)) \ x(subvolume_children, 19, 0, \ - BIT_ULL(KEY_TYPE_set)) + BIT_ULL(KEY_TYPE_set)) \ + x(accounting, 20, BTREE_ID_SNAPSHOT_FIELD, \ + BIT_ULL(KEY_TYPE_accounting)) \ enum btree_id { #define x(name, nr, ...) BTREE_ID_##name = nr, diff --git a/libbcachefs/bcachefs_ioctl.h b/libbcachefs/bcachefs_ioctl.h index 4b8fba75..3c23bdf7 100644 --- a/libbcachefs/bcachefs_ioctl.h +++ b/libbcachefs/bcachefs_ioctl.h @@ -5,6 +5,7 @@ #include <linux/uuid.h> #include <asm/ioctl.h> #include "bcachefs_format.h" +#include "bkey_types.h" /* * Flags common to multiple ioctls: @@ -85,6 +86,7 @@ struct bch_ioctl_incremental { #define BCH_IOCTL_FSCK_OFFLINE _IOW(0xbc, 19, struct bch_ioctl_fsck_offline) #define BCH_IOCTL_FSCK_ONLINE _IOW(0xbc, 20, struct bch_ioctl_fsck_online) +#define BCH_IOCTL_QUERY_ACCOUNTING _IOW(0xbc, 21, struct bch_ioctl_query_accounting) /* ioctl below act on a particular file, not the filesystem as a whole: */ @@ -251,12 +253,18 @@ struct bch_replicas_usage { struct bch_replicas_entry_v1 r; } __packed; +static inline unsigned replicas_usage_bytes(struct bch_replicas_usage *u) +{ + return offsetof(struct bch_replicas_usage, r) + replicas_entry_bytes(&u->r); +} + static inline struct bch_replicas_usage * replicas_usage_next(struct bch_replicas_usage *u) { - return (void *) u + replicas_entry_bytes(&u->r) + 8; + return (void *) u + replicas_usage_bytes(u); } +/* Obsolete */ /* * BCH_IOCTL_FS_USAGE: query filesystem disk space usage * @@ -282,6 +290,7 @@ struct bch_ioctl_fs_usage { struct bch_replicas_usage replicas[]; }; +/* Obsolete */ /* * BCH_IOCTL_DEV_USAGE: query device disk space usage * @@ -306,6 +315,7 @@ struct bch_ioctl_dev_usage { } d[10]; }; +/* Obsolete */ struct bch_ioctl_dev_usage_v2 { __u64 dev; __u32 flags; @@ -409,4 +419,28 @@ struct bch_ioctl_fsck_online { __u64 opts; /* string */ }; +/* + * BCH_IOCTL_QUERY_ACCOUNTING: query filesystem disk accounting + * + * Returns disk space usage broken out by data type, number of replicas, and + * by component device + * + * @replica_entries_bytes - size, in bytes, allocated for replica usage entries + * + * On success, @replica_entries_bytes will be changed to indicate the number of + * bytes actually used. + * + * Returns -ERANGE if @replica_entries_bytes was too small + */ +struct bch_ioctl_query_accounting { + __u64 capacity; + __u64 used; + __u64 online_reserved; + + __u32 accounting_u64s; /* input parameter */ + __u32 accounting_types_mask; /* input parameter */ + + struct bkey_i_accounting accounting[]; +}; + #endif /* _BCACHEFS_IOCTL_H */ diff --git a/libbcachefs/bkey_methods.c b/libbcachefs/bkey_methods.c index c2c3dae5..e838ac8f 100644 --- a/libbcachefs/bkey_methods.c +++ b/libbcachefs/bkey_methods.c @@ -7,6 +7,7 @@ #include "btree_types.h" #include "alloc_background.h" #include "dirent.h" +#include "disk_accounting.h" #include "ec.h" #include "error.h" #include "extents.h" diff --git a/libbcachefs/btree_cache.c b/libbcachefs/btree_cache.c index 9e4ed75d..6017e95d 100644 --- a/libbcachefs/btree_cache.c +++ b/libbcachefs/btree_cache.c @@ -601,8 +601,8 @@ int bch2_btree_cache_cannibalize_lock(struct btree_trans *trans, struct closure struct btree_cache *bc = &c->btree_cache; struct task_struct *old; - old = cmpxchg(&bc->alloc_lock, NULL, current); - if (old == NULL || old == current) + old = NULL; + if (try_cmpxchg(&bc->alloc_lock, &old, current) || old == current) goto success; if (!cl) { @@ -613,8 +613,8 @@ int bch2_btree_cache_cannibalize_lock(struct btree_trans *trans, struct closure closure_wait(&bc->alloc_wait, cl); /* Try again, after adding ourselves to waitlist */ - old = cmpxchg(&bc->alloc_lock, NULL, current); - if (old == NULL || old == current) { + old = NULL; + if (try_cmpxchg(&bc->alloc_lock, &old, current) || old == current) { /* We raced */ closure_wake_up(&bc->alloc_wait); goto success; diff --git a/libbcachefs/btree_gc.c b/libbcachefs/btree_gc.c index dc97991b..517c2ca2 100644 --- a/libbcachefs/btree_gc.c +++ b/libbcachefs/btree_gc.c @@ -20,6 +20,7 @@ #include "buckets.h" #include "clock.h" #include "debug.h" +#include "disk_accounting.h" #include "ec.h" #include "error.h" #include "extents.h" @@ -174,10 +175,11 @@ static int set_node_max(struct bch_fs *c, struct btree *b, struct bpos new_max) return 0; } -static int btree_check_node_boundaries(struct bch_fs *c, struct btree *b, +static int btree_check_node_boundaries(struct btree_trans *trans, struct btree *b, struct btree *prev, struct btree *cur, struct bpos *pulled_from_scan) { + struct bch_fs *c = trans->c; struct bpos expected_start = !prev ? b->data->min_key : bpos_successor(prev->key.k.p); @@ -215,29 +217,29 @@ static int btree_check_node_boundaries(struct bch_fs *c, struct btree *b, *pulled_from_scan = cur->data->min_key; ret = DID_FILL_FROM_SCAN; } else { - if (mustfix_fsck_err(c, btree_node_topology_bad_min_key, + if (mustfix_fsck_err(trans, btree_node_topology_bad_min_key, "btree node with incorrect min_key%s", buf.buf)) ret = set_node_min(c, cur, expected_start); } } else { /* overlap */ if (prev && BTREE_NODE_SEQ(cur->data) > BTREE_NODE_SEQ(prev->data)) { /* cur overwrites prev */ if (bpos_ge(prev->data->min_key, cur->data->min_key)) { /* fully? */ - if (mustfix_fsck_err(c, btree_node_topology_overwritten_by_next_node, + if (mustfix_fsck_err(trans, btree_node_topology_overwritten_by_next_node, "btree node overwritten by next node%s", buf.buf)) ret = DROP_PREV_NODE; } else { - if (mustfix_fsck_err(c, btree_node_topology_bad_max_key, + if (mustfix_fsck_err(trans, btree_node_topology_bad_max_key, "btree node with incorrect max_key%s", buf.buf)) ret = set_node_max(c, prev, bpos_predecessor(cur->data->min_key)); } } else { if (bpos_ge(expected_start, cur->data->max_key)) { /* fully? */ - if (mustfix_fsck_err(c, btree_node_topology_overwritten_by_prev_node, + if (mustfix_fsck_err(trans, btree_node_topology_overwritten_by_prev_node, "btree node overwritten by prev node%s", buf.buf)) ret = DROP_THIS_NODE; } else { - if (mustfix_fsck_err(c, btree_node_topology_bad_min_key, + if (mustfix_fsck_err(trans, btree_node_topology_bad_min_key, "btree node with incorrect min_key%s", buf.buf)) ret = set_node_min(c, cur, expected_start); } @@ -249,9 +251,10 @@ fsck_err: return ret; } -static int btree_repair_node_end(struct bch_fs *c, struct btree *b, +static int btree_repair_node_end(struct btree_trans *trans, struct btree *b, struct btree *child, struct bpos *pulled_from_scan) { + struct bch_fs *c = trans->c; struct printbuf buf = PRINTBUF; int ret = 0; @@ -265,7 +268,7 @@ static int btree_repair_node_end(struct bch_fs *c, struct btree *b, prt_str(&buf, "\n child: "); bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&child->key)); - if (mustfix_fsck_err(c, btree_node_topology_bad_max_key, + if (mustfix_fsck_err(trans, btree_node_topology_bad_max_key, "btree node with incorrect max_key%s", buf.buf)) { if (b->c.level == 1 && bpos_lt(*pulled_from_scan, b->key.k.p)) { @@ -324,8 +327,8 @@ again: printbuf_reset(&buf); bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(cur_k.k)); - if (mustfix_fsck_err_on(bch2_err_matches(ret, EIO), c, - btree_node_unreadable, + if (mustfix_fsck_err_on(bch2_err_matches(ret, EIO), + trans, btree_node_unreadable, "Topology repair: unreadable btree node at btree %s level %u:\n" " %s", bch2_btree_id_str(b->c.btree_id), @@ -362,7 +365,7 @@ again: continue; } - ret = btree_check_node_boundaries(c, b, prev, cur, pulled_from_scan); + ret = btree_check_node_boundaries(trans, b, prev, cur, pulled_from_scan); if (ret == DID_FILL_FROM_SCAN) { new_pass = true; ret = 0; @@ -403,7 +406,7 @@ again: if (!ret && !IS_ERR_OR_NULL(prev)) { BUG_ON(cur); - ret = btree_repair_node_end(c, b, prev, pulled_from_scan); + ret = btree_repair_node_end(trans, b, prev, pulled_from_scan); if (ret == DID_FILL_FROM_SCAN) { new_pass = true; ret = 0; @@ -461,8 +464,8 @@ again: printbuf_reset(&buf); bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); - if (mustfix_fsck_err_on(!have_child, c, - btree_node_topology_interior_node_empty, + if (mustfix_fsck_err_on(!have_child, + trans, btree_node_topology_interior_node_empty, "empty interior btree node at btree %s level %u\n" " %s", bch2_btree_id_str(b->c.btree_id), @@ -509,7 +512,7 @@ reconstruct_root: r->error = 0; if (!bch2_btree_has_scanned_nodes(c, i)) { - mustfix_fsck_err(c, btree_root_unreadable_and_scan_found_nothing, + mustfix_fsck_err(trans, btree_root_unreadable_and_scan_found_nothing, "no nodes found for btree %s, continue?", bch2_btree_id_str(i)); bch2_btree_root_alloc_fake_trans(trans, i, 0); } else { @@ -583,8 +586,9 @@ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, BUG_ON(bch2_journal_seq_verify && k.k->version.lo > atomic64_read(&c->journal.seq)); - if (fsck_err_on(k.k->version.lo > atomic64_read(&c->key_version), c, - bkey_version_in_future, + if (fsck_err_on(btree_id != BTREE_ID_accounting && + k.k->version.lo > atomic64_read(&c->key_version), + trans, bkey_version_in_future, "key version number higher than recorded %llu\n %s", atomic64_read(&c->key_version), (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) @@ -592,7 +596,7 @@ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, } if (mustfix_fsck_err_on(level && !bch2_dev_btree_bitmap_marked(c, k), - c, btree_bitmap_not_marked, + trans, btree_bitmap_not_marked, "btree ptr not marked in member info btree allocated bitmap\n %s", (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, k), @@ -622,7 +626,7 @@ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, } ret = bch2_key_trigger(trans, btree_id, level, old, unsafe_bkey_s_c_to_s(k), - BTREE_TRIGGER_gc|flags); + BTREE_TRIGGER_gc|BTREE_TRIGGER_insert|flags); out: fsck_err: printbuf_exit(&buf); @@ -633,29 +637,14 @@ fsck_err: static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree, bool initial) { struct bch_fs *c = trans->c; - int level = 0, target_depth = btree_node_type_needs_gc(__btree_node_type(0, btree)) ? 0 : 1; + unsigned target_depth = btree_node_type_has_triggers(__btree_node_type(0, btree)) ? 0 : 1; int ret = 0; /* We need to make sure every leaf node is readable before going RW */ if (initial) target_depth = 0; - /* root */ - mutex_lock(&c->btree_root_lock); - struct btree *b = bch2_btree_id_root(c, btree)->b; - if (!btree_node_fake(b)) { - gc_pos_set(c, gc_pos_btree(btree, b->c.level + 1, SPOS_MAX)); - ret = lockrestart_do(trans, - bch2_gc_mark_key(trans, b->c.btree_id, b->c.level + 1, - NULL, NULL, bkey_i_to_s_c(&b->key), initial)); - level = b->c.level; - } - mutex_unlock(&c->btree_root_lock); - - if (ret) - return ret; - - for (; level >= target_depth; --level) { + for (unsigned level = target_depth; level < BTREE_MAX_DEPTH; level++) { struct btree *prev = NULL; struct btree_iter iter; bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 0, level, @@ -666,9 +655,21 @@ static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree, bool in bch2_gc_mark_key(trans, btree, level, &prev, &iter, k, initial); })); if (ret) - break; + goto err; } + /* root */ + mutex_lock(&c->btree_root_lock); + struct btree *b = bch2_btree_id_root(c, btree)->b; + if (!btree_node_fake(b)) { + gc_pos_set(c, gc_pos_btree(btree, b->c.level + 1, SPOS_MAX)); + ret = lockrestart_do(trans, + bch2_gc_mark_key(trans, b->c.btree_id, b->c.level + 1, + NULL, NULL, bkey_i_to_s_c(&b->key), initial)); + } + mutex_unlock(&c->btree_root_lock); +err: + bch_err_fn(c, ret); return ret; } @@ -697,7 +698,7 @@ static int bch2_gc_btrees(struct bch_fs *c) ret = bch2_gc_btree(trans, btree, true); if (mustfix_fsck_err_on(bch2_err_matches(ret, EIO), - c, btree_node_read_error, + trans, btree_node_read_error, "btree node read error for %s", bch2_btree_id_str(btree))) ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology); @@ -720,131 +721,25 @@ static int bch2_mark_superblocks(struct bch_fs *c) static void bch2_gc_free(struct bch_fs *c) { + bch2_accounting_free(&c->accounting[1]); + genradix_free(&c->reflink_gc_table); genradix_free(&c->gc_stripes); for_each_member_device(c, ca) { kvfree(rcu_dereference_protected(ca->buckets_gc, 1)); ca->buckets_gc = NULL; - - free_percpu(ca->usage_gc); - ca->usage_gc = NULL; } - - free_percpu(c->usage_gc); - c->usage_gc = NULL; -} - -static int bch2_gc_done(struct bch_fs *c) -{ - struct bch_dev *ca = NULL; - struct printbuf buf = PRINTBUF; - unsigned i; - int ret = 0; - - percpu_down_write(&c->mark_lock); - -#define copy_field(_err, _f, _msg, ...) \ - if (fsck_err_on(dst->_f != src->_f, c, _err, \ - _msg ": got %llu, should be %llu" , ##__VA_ARGS__, \ - dst->_f, src->_f)) \ - dst->_f = src->_f -#define copy_dev_field(_err, _f, _msg, ...) \ - copy_field(_err, _f, "dev %u has wrong " _msg, ca->dev_idx, ##__VA_ARGS__) -#define copy_fs_field(_err, _f, _msg, ...) \ - copy_field(_err, _f, "fs has wrong " _msg, ##__VA_ARGS__) - - for (i = 0; i < ARRAY_SIZE(c->usage); i++) - bch2_fs_usage_acc_to_base(c, i); - - __for_each_member_device(c, ca) { - struct bch_dev_usage *dst = ca->usage_base; - struct bch_dev_usage *src = (void *) - bch2_acc_percpu_u64s((u64 __percpu *) ca->usage_gc, - dev_usage_u64s()); - - for (i = 0; i < BCH_DATA_NR; i++) { - copy_dev_field(dev_usage_buckets_wrong, - d[i].buckets, "%s buckets", bch2_data_type_str(i)); - copy_dev_field(dev_usage_sectors_wrong, - d[i].sectors, "%s sectors", bch2_data_type_str(i)); - copy_dev_field(dev_usage_fragmented_wrong, - d[i].fragmented, "%s fragmented", bch2_data_type_str(i)); - } - } - - { - unsigned nr = fs_usage_u64s(c); - struct bch_fs_usage *dst = c->usage_base; - struct bch_fs_usage *src = (void *) - bch2_acc_percpu_u64s((u64 __percpu *) c->usage_gc, nr); - - copy_fs_field(fs_usage_hidden_wrong, - b.hidden, "hidden"); - copy_fs_field(fs_usage_btree_wrong, - b.btree, "btree"); - - copy_fs_field(fs_usage_data_wrong, - b.data, "data"); - copy_fs_field(fs_usage_cached_wrong, - b.cached, "cached"); - copy_fs_field(fs_usage_reserved_wrong, - b.reserved, "reserved"); - copy_fs_field(fs_usage_nr_inodes_wrong, - b.nr_inodes,"nr_inodes"); - - for (i = 0; i < BCH_REPLICAS_MAX; i++) - copy_fs_field(fs_usage_persistent_reserved_wrong, - persistent_reserved[i], - "persistent_reserved[%i]", i); - - for (i = 0; i < c->replicas.nr; i++) { - struct bch_replicas_entry_v1 *e = - cpu_replicas_entry(&c->replicas, i); - - printbuf_reset(&buf); - bch2_replicas_entry_to_text(&buf, e); - - copy_fs_field(fs_usage_replicas_wrong, - replicas[i], "%s", buf.buf); - } - } - -#undef copy_fs_field -#undef copy_dev_field -#undef copy_stripe_field -#undef copy_field -fsck_err: - bch2_dev_put(ca); - bch_err_fn(c, ret); - percpu_up_write(&c->mark_lock); - printbuf_exit(&buf); - return ret; } static int bch2_gc_start(struct bch_fs *c) { - BUG_ON(c->usage_gc); - - c->usage_gc = __alloc_percpu_gfp(fs_usage_u64s(c) * sizeof(u64), - sizeof(u64), GFP_KERNEL); - if (!c->usage_gc) { - bch_err(c, "error allocating c->usage_gc"); - return -BCH_ERR_ENOMEM_gc_start; - } - for_each_member_device(c, ca) { - BUG_ON(ca->usage_gc); - - ca->usage_gc = alloc_percpu(struct bch_dev_usage); - if (!ca->usage_gc) { - bch_err(c, "error allocating ca->usage_gc"); + int ret = bch2_dev_usage_init(ca, true); + if (ret) { bch2_dev_put(ca); - return -BCH_ERR_ENOMEM_gc_start; + return ret; } - - this_cpu_write(ca->usage_gc->d[BCH_DATA_free].buckets, - ca->mi.nbuckets - ca->mi.first_bucket); } return 0; @@ -858,6 +753,7 @@ static inline bool bch2_alloc_v4_cmp(struct bch_alloc_v4 l, l.oldest_gen != r.oldest_gen || l.data_type != r.data_type || l.dirty_sectors != r.dirty_sectors || + l.stripe_sectors != r.stripe_sectors || l.cached_sectors != r.cached_sectors || l.stripe_redundancy != r.stripe_redundancy || l.stripe != r.stripe; @@ -888,6 +784,7 @@ static int bch2_alloc_write_key(struct btree_trans *trans, gc.data_type = old->data_type; gc.dirty_sectors = old->dirty_sectors; } + percpu_up_read(&c->mark_lock); /* * gc.data_type doesn't yet include need_discard & need_gc_gen states - @@ -896,12 +793,14 @@ static int bch2_alloc_write_key(struct btree_trans *trans, alloc_data_type_set(&gc, gc.data_type); if (gc.data_type != old_gc.data_type || - gc.dirty_sectors != old_gc.dirty_sectors) - bch2_dev_usage_update(c, ca, &old_gc, &gc, 0, true); - percpu_up_read(&c->mark_lock); + gc.dirty_sectors != old_gc.dirty_sectors) { + ret = bch2_alloc_key_to_dev_counters(trans, ca, &old_gc, &gc, BTREE_TRIGGER_gc); + if (ret) + return ret; + } - if (fsck_err_on(new.data_type != gc.data_type, c, - alloc_key_data_type_wrong, + if (fsck_err_on(new.data_type != gc.data_type, + trans, alloc_key_data_type_wrong, "bucket %llu:%llu gen %u has wrong data_type" ": got %s, should be %s", iter->pos.inode, iter->pos.offset, @@ -911,7 +810,8 @@ static int bch2_alloc_write_key(struct btree_trans *trans, new.data_type = gc.data_type; #define copy_bucket_field(_errtype, _f) \ - if (fsck_err_on(new._f != gc._f, c, _errtype, \ + if (fsck_err_on(new._f != gc._f, \ + trans, _errtype, \ "bucket %llu:%llu gen %u data type %s has wrong " #_f \ ": got %u, should be %u", \ iter->pos.inode, iter->pos.offset, \ @@ -924,6 +824,8 @@ static int bch2_alloc_write_key(struct btree_trans *trans, gen); copy_bucket_field(alloc_key_dirty_sectors_wrong, dirty_sectors); + copy_bucket_field(alloc_key_stripe_sectors_wrong, + stripe_sectors); copy_bucket_field(alloc_key_cached_sectors_wrong, cached_sectors); copy_bucket_field(alloc_key_stripe_wrong, @@ -978,14 +880,16 @@ static int bch2_gc_alloc_done(struct bch_fs *c) static int bch2_gc_alloc_start(struct bch_fs *c) { + int ret = 0; + for_each_member_device(c, ca) { struct bucket_array *buckets = kvmalloc(sizeof(struct bucket_array) + ca->mi.nbuckets * sizeof(struct bucket), GFP_KERNEL|__GFP_ZERO); if (!buckets) { bch2_dev_put(ca); - bch_err(c, "error allocating ca->buckets[gc]"); - return -BCH_ERR_ENOMEM_gc_alloc_start; + ret = -BCH_ERR_ENOMEM_gc_alloc_start; + break; } buckets->first_bucket = ca->mi.first_bucket; @@ -993,25 +897,6 @@ static int bch2_gc_alloc_start(struct bch_fs *c) rcu_assign_pointer(ca->buckets_gc, buckets); } - struct bch_dev *ca = NULL; - int ret = bch2_trans_run(c, - for_each_btree_key(trans, iter, BTREE_ID_alloc, POS_MIN, - BTREE_ITER_prefetch, k, ({ - ca = bch2_dev_iterate(c, ca, k.k->p.inode); - if (!ca) { - bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); - continue; - } - - struct bch_alloc_v4 a_convert; - const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &a_convert); - - struct bucket *g = gc_bucket(ca, k.k->p.offset); - g->gen_valid = 1; - g->gen = a->gen; - 0; - }))); - bch2_dev_put(ca); bch_err_fn(c, ret); return ret; } @@ -1041,8 +926,8 @@ static int bch2_gc_write_reflink_key(struct btree_trans *trans, return -EINVAL; } - if (fsck_err_on(r->refcount != le64_to_cpu(*refcount), c, - reflink_v_refcount_wrong, + if (fsck_err_on(r->refcount != le64_to_cpu(*refcount), + trans, reflink_v_refcount_wrong, "reflink key has wrong refcount:\n" " %s\n" " should be %u", @@ -1140,7 +1025,8 @@ static int bch2_gc_write_stripes_key(struct btree_trans *trans, if (bad) bch2_bkey_val_to_text(&buf, c, k); - if (fsck_err_on(bad, c, stripe_sector_count_wrong, + if (fsck_err_on(bad, + trans, stripe_sector_count_wrong, "%s", buf.buf)) { struct bkey_i_stripe *new; @@ -1199,8 +1085,6 @@ int bch2_check_allocations(struct bch_fs *c) lockdep_assert_held(&c->state_lock); - down_write(&c->gc_lock); - bch2_btree_interior_updates_flush(c); ret = bch2_gc_start(c) ?: @@ -1212,7 +1096,9 @@ int bch2_check_allocations(struct bch_fs *c) gc_pos_set(c, gc_phase(GC_PHASE_start)); ret = bch2_mark_superblocks(c); - BUG_ON(ret); + bch_err_msg(c, ret, "marking superblocks"); + if (ret) + goto out; ret = bch2_gc_btrees(c); if (ret) @@ -1220,15 +1106,11 @@ int bch2_check_allocations(struct bch_fs *c) c->gc_count++; - bch2_journal_block(&c->journal); -out: ret = bch2_gc_alloc_done(c) ?: - bch2_gc_done(c) ?: + bch2_accounting_gc_done(c) ?: bch2_gc_stripes_done(c) ?: bch2_gc_reflink_done(c); - - bch2_journal_unblock(&c->journal); - +out: percpu_down_write(&c->mark_lock); /* Indicates that gc is no longer in progress: */ __gc_pos_set(c, gc_phase(GC_PHASE_not_running)); @@ -1236,13 +1118,6 @@ out: bch2_gc_free(c); percpu_up_write(&c->mark_lock); - up_write(&c->gc_lock); - - /* - * At startup, allocations can happen directly instead of via the - * allocator thread - issue wakeup in case they blocked on gc_lock: - */ - closure_wake_up(&c->freelist_wait); bch_err_fn(c, ret); return ret; } @@ -1323,7 +1198,7 @@ int bch2_gc_gens(struct bch_fs *c) int ret; /* - * Ideally we would be using state_lock and not gc_lock here, but that + * Ideally we would be using state_lock and not gc_gens_lock here, but that * introduces a deadlock in the RO path - we currently take the state * lock at the start of going RO, thus the gc thread may get stuck: */ @@ -1331,7 +1206,8 @@ int bch2_gc_gens(struct bch_fs *c) return 0; trace_and_count(c, gc_gens_start, c); - down_read(&c->gc_lock); + + down_read(&c->state_lock); for_each_member_device(c, ca) { struct bucket_gens *gens = bucket_gens(ca); @@ -1400,7 +1276,7 @@ err: ca->oldest_gen = NULL; } - up_read(&c->gc_lock); + up_read(&c->state_lock); mutex_unlock(&c->gc_gens_lock); if (!bch2_err_matches(ret, EROFS)) bch_err_fn(c, ret); diff --git a/libbcachefs/btree_gc.h b/libbcachefs/btree_gc.h index 876d81e2..1bdf841d 100644 --- a/libbcachefs/btree_gc.h +++ b/libbcachefs/btree_gc.h @@ -58,6 +58,8 @@ static inline struct gc_pos gc_pos_btree_node(struct btree *b) static inline int gc_btree_order(enum btree_id btree) { + if (btree == BTREE_ID_alloc) + return -2; if (btree == BTREE_ID_stripes) return -1; return btree; @@ -65,11 +67,11 @@ static inline int gc_btree_order(enum btree_id btree) static inline int gc_pos_cmp(struct gc_pos l, struct gc_pos r) { - return cmp_int(l.phase, r.phase) ?: - cmp_int(gc_btree_order(l.btree), - gc_btree_order(r.btree)) ?: - -cmp_int(l.level, r.level) ?: - bpos_cmp(l.pos, r.pos); + return cmp_int(l.phase, r.phase) ?: + cmp_int(gc_btree_order(l.btree), + gc_btree_order(r.btree)) ?: + cmp_int(l.level, r.level) ?: + bpos_cmp(l.pos, r.pos); } static inline bool gc_visited(struct bch_fs *c, struct gc_pos pos) diff --git a/libbcachefs/btree_io.c b/libbcachefs/btree_io.c index 829c1b91..b79dfcd2 100644 --- a/libbcachefs/btree_io.c +++ b/libbcachefs/btree_io.c @@ -534,7 +534,7 @@ static void btree_err_msg(struct printbuf *out, struct bch_fs *c, printbuf_indent_add(out, 2); prt_printf(out, "\nnode offset %u/%u", - b->written, btree_ptr_sectors_written(&b->key)); + b->written, btree_ptr_sectors_written(bkey_i_to_s_c(&b->key))); if (i) prt_printf(out, " bset u64s %u", le16_to_cpu(i->u64s)); if (k) @@ -585,7 +585,7 @@ static int __btree_err(int ret, switch (ret) { case -BCH_ERR_btree_node_read_err_fixable: ret = !silent - ? bch2_fsck_err(c, FSCK_CAN_FIX, err_type, "%s", out.buf) + ? __bch2_fsck_err(c, NULL, FSCK_CAN_FIX, err_type, "%s", out.buf) : -BCH_ERR_fsck_fix; if (ret != -BCH_ERR_fsck_fix && ret != -BCH_ERR_fsck_ignore) @@ -689,6 +689,7 @@ static int validate_bset(struct bch_fs *c, struct bch_dev *ca, int write, bool have_retry, bool *saw_error) { unsigned version = le16_to_cpu(i->version); + unsigned ptr_written = btree_ptr_sectors_written(bkey_i_to_s_c(&b->key)); struct printbuf buf1 = PRINTBUF; struct printbuf buf2 = PRINTBUF; int ret = 0; @@ -732,11 +733,13 @@ static int validate_bset(struct bch_fs *c, struct bch_dev *ca, btree_node_unsupported_version, "BSET_SEPARATE_WHITEOUTS no longer supported"); - if (btree_err_on(offset + sectors > btree_sectors(c), + if (!write && + btree_err_on(offset + sectors > (ptr_written ?: btree_sectors(c)), -BCH_ERR_btree_node_read_err_fixable, c, ca, b, i, NULL, bset_past_end_of_btree_node, - "bset past end of btree node")) { + "bset past end of btree node (offset %u len %u but written %zu)", + offset, sectors, ptr_written ?: btree_sectors(c))) { i->u64s = 0; ret = 0; goto out; @@ -1002,7 +1005,7 @@ int bch2_btree_node_read_done(struct bch_fs *c, struct bch_dev *ca, bool updated_range = b->key.k.type == KEY_TYPE_btree_ptr_v2 && BTREE_PTR_RANGE_UPDATED(&bkey_i_to_btree_ptr_v2(&b->key)->v); unsigned u64s; - unsigned ptr_written = btree_ptr_sectors_written(&b->key); + unsigned ptr_written = btree_ptr_sectors_written(bkey_i_to_s_c(&b->key)); struct printbuf buf = PRINTBUF; int ret = 0, retry_read = 0, write = READ; u64 start_time = local_clock(); @@ -1796,15 +1799,16 @@ int bch2_btree_root_read(struct bch_fs *c, enum btree_id id, static void bch2_btree_complete_write(struct bch_fs *c, struct btree *b, struct btree_write *w) { - unsigned long old, new, v = READ_ONCE(b->will_make_reachable); + unsigned long old, new; + old = READ_ONCE(b->will_make_reachable); do { - old = new = v; + new = old; if (!(old & 1)) break; new &= ~1UL; - } while ((v = cmpxchg(&b->will_make_reachable, old, new)) != old); + } while (!try_cmpxchg(&b->will_make_reachable, &old, new)); if (old & 1) closure_put(&((struct btree_update *) new)->cl); @@ -1815,14 +1819,14 @@ static void bch2_btree_complete_write(struct bch_fs *c, struct btree *b, static void __btree_node_write_done(struct bch_fs *c, struct btree *b) { struct btree_write *w = btree_prev_write(b); - unsigned long old, new, v; + unsigned long old, new; unsigned type = 0; bch2_btree_complete_write(c, b, w); - v = READ_ONCE(b->flags); + old = READ_ONCE(b->flags); do { - old = new = v; + new = old; if ((old & (1U << BTREE_NODE_dirty)) && (old & (1U << BTREE_NODE_need_write)) && @@ -1842,7 +1846,7 @@ static void __btree_node_write_done(struct bch_fs *c, struct btree *b) new &= ~(1U << BTREE_NODE_write_in_flight); new &= ~(1U << BTREE_NODE_write_in_flight_inner); } - } while ((v = cmpxchg(&b->flags, old, new)) != old); + } while (!try_cmpxchg(&b->flags, &old, new)); if (new & (1U << BTREE_NODE_write_in_flight)) __bch2_btree_node_write(c, b, BTREE_WRITE_ALREADY_STARTED|type); @@ -2014,8 +2018,9 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) * dirty bit requires a write lock, we can't race with other threads * redirtying it: */ + old = READ_ONCE(b->flags); do { - old = new = READ_ONCE(b->flags); + new = old; if (!(old & (1 << BTREE_NODE_dirty))) return; @@ -2046,7 +2051,7 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) new |= (1 << BTREE_NODE_write_in_flight_inner); new |= (1 << BTREE_NODE_just_written); new ^= (1 << BTREE_NODE_write_idx); - } while (cmpxchg_acquire(&b->flags, old, new) != old); + } while (!try_cmpxchg_acquire(&b->flags, &old, new)); if (new & (1U << BTREE_NODE_need_write)) return; @@ -2133,7 +2138,7 @@ do_write: if (!b->written && b->key.k.type == KEY_TYPE_btree_ptr_v2) - BUG_ON(btree_ptr_sectors_written(&b->key) != sectors_to_write); + BUG_ON(btree_ptr_sectors_written(bkey_i_to_s_c(&b->key)) != sectors_to_write); memset(data + bytes_to_write, 0, (sectors_to_write << 9) - bytes_to_write); diff --git a/libbcachefs/btree_io.h b/libbcachefs/btree_io.h index 2b8b564f..63d76f5c 100644 --- a/libbcachefs/btree_io.h +++ b/libbcachefs/btree_io.h @@ -27,10 +27,10 @@ static inline void clear_btree_node_dirty_acct(struct bch_fs *c, struct btree *b atomic_dec(&c->btree_cache.dirty); } -static inline unsigned btree_ptr_sectors_written(struct bkey_i *k) +static inline unsigned btree_ptr_sectors_written(struct bkey_s_c k) { - return k->k.type == KEY_TYPE_btree_ptr_v2 - ? le16_to_cpu(bkey_i_to_btree_ptr_v2(k)->v.sectors_written) + return k.k->type == KEY_TYPE_btree_ptr_v2 + ? le16_to_cpu(bkey_s_c_to_btree_ptr_v2(k).v->sectors_written) : 0; } diff --git a/libbcachefs/btree_iter.c b/libbcachefs/btree_iter.c index d3bcb4e4..bd6bd073 100644 --- a/libbcachefs/btree_iter.c +++ b/libbcachefs/btree_iter.c @@ -1002,6 +1002,7 @@ retry_all: bch2_trans_unlock(trans); cond_resched(); trans->locked = true; + trans->last_unlock_ip = 0; if (unlikely(trans->memory_allocation_failure)) { struct closure cl; @@ -1470,7 +1471,7 @@ void bch2_dump_trans_updates(struct btree_trans *trans) struct printbuf buf = PRINTBUF; bch2_trans_updates_to_text(&buf, trans); - bch2_print_string_as_lines(KERN_ERR, buf.buf); + bch2_print_str(trans->c, buf.buf); printbuf_exit(&buf); } @@ -1562,7 +1563,7 @@ void __bch2_dump_trans_paths_updates(struct btree_trans *trans, bool nosort) __bch2_trans_paths_to_text(&buf, trans, nosort); bch2_trans_updates_to_text(&buf, trans); - bch2_print_string_as_lines(KERN_ERR, buf.buf); + bch2_print_str(trans->c, buf.buf); printbuf_exit(&buf); } @@ -3095,6 +3096,7 @@ u32 bch2_trans_begin(struct btree_trans *trans) trans->last_begin_ip = _RET_IP_; trans->locked = true; + trans->last_unlock_ip = 0; if (trans->restarted) { bch2_btree_path_traverse_all(trans); @@ -3249,15 +3251,6 @@ void bch2_trans_put(struct btree_trans *trans) srcu_read_unlock(&c->btree_trans_barrier, trans->srcu_idx); } - if (trans->fs_usage_deltas) { - if (trans->fs_usage_deltas->size + sizeof(trans->fs_usage_deltas) == - REPLICAS_DELTA_LIST_MAX) - mempool_free(trans->fs_usage_deltas, - &c->replicas_delta_pool); - else - kfree(trans->fs_usage_deltas); - } - if (unlikely(trans->journal_replay_not_finished)) bch2_journal_keys_put(c); @@ -3288,6 +3281,20 @@ void bch2_trans_put(struct btree_trans *trans) } } +bool bch2_current_has_btree_trans(struct bch_fs *c) +{ + seqmutex_lock(&c->btree_trans_lock); + struct btree_trans *trans; + bool ret = false; + list_for_each_entry(trans, &c->btree_trans_list, list) + if (trans->locking_wait.task == current) { + ret = true; + break; + } + seqmutex_unlock(&c->btree_trans_lock); + return ret; +} + static void __maybe_unused bch2_btree_bkey_cached_common_to_text(struct printbuf *out, struct btree_bkey_cached_common *b) diff --git a/libbcachefs/btree_iter.h b/libbcachefs/btree_iter.h index 798eb1c4..bdb3cd2e 100644 --- a/libbcachefs/btree_iter.h +++ b/libbcachefs/btree_iter.h @@ -866,6 +866,14 @@ __bch2_btree_iter_peek_and_restart(struct btree_trans *trans, _p; \ }) +#define bch2_trans_run(_c, _do) \ +({ \ + struct btree_trans *trans = bch2_trans_get(_c); \ + int _ret = (_do); \ + bch2_trans_put(trans); \ + _ret; \ +}) + void bch2_trans_updates_to_text(struct printbuf *, struct btree_trans *); void bch2_btree_path_to_text(struct printbuf *, struct btree_trans *, btree_path_idx_t); void bch2_trans_paths_to_text(struct printbuf *, struct btree_trans *); @@ -875,6 +883,8 @@ void bch2_dump_trans_paths_updates(struct btree_trans *); struct btree_trans *__bch2_trans_get(struct bch_fs *, unsigned); void bch2_trans_put(struct btree_trans *); +bool bch2_current_has_btree_trans(struct bch_fs *); + extern const char *bch2_btree_transaction_fns[BCH_TRANSACTIONS_NR]; unsigned bch2_trans_get_fn_idx(const char *); diff --git a/libbcachefs/btree_journal_iter.c b/libbcachefs/btree_journal_iter.c index 332dbf16..74933490 100644 --- a/libbcachefs/btree_journal_iter.c +++ b/libbcachefs/btree_journal_iter.c @@ -16,21 +16,6 @@ * operations for the regular btree iter code to use: */ -static int __journal_key_cmp(enum btree_id l_btree_id, - unsigned l_level, - struct bpos l_pos, - const struct journal_key *r) -{ - return (cmp_int(l_btree_id, r->btree_id) ?: - cmp_int(l_level, r->level) ?: - bpos_cmp(l_pos, r->k->k.p)); -} - -static int journal_key_cmp(const struct journal_key *l, const struct journal_key *r) -{ - return __journal_key_cmp(l->btree_id, l->level, l->k->k.p, r); -} - static inline size_t idx_to_pos(struct journal_keys *keys, size_t idx) { size_t gap_size = keys->size - keys->nr; @@ -548,7 +533,13 @@ static void __journal_keys_sort(struct journal_keys *keys) struct journal_key *dst = keys->data; darray_for_each(*keys, src) { - if (src + 1 < &darray_top(*keys) && + /* + * We don't accumulate accounting keys here because we have to + * compare each individual accounting key against the version in + * the btree during replay: + */ + if (src->k->k.type != KEY_TYPE_accounting && + src + 1 < &darray_top(*keys) && !journal_key_cmp(src, src + 1)) continue; diff --git a/libbcachefs/btree_journal_iter.h b/libbcachefs/btree_journal_iter.h index 1ba4a79b..5b66c8f8 100644 --- a/libbcachefs/btree_journal_iter.h +++ b/libbcachefs/btree_journal_iter.h @@ -26,6 +26,21 @@ struct btree_and_journal_iter { bool prefetch; }; +static inline int __journal_key_cmp(enum btree_id l_btree_id, + unsigned l_level, + struct bpos l_pos, + const struct journal_key *r) +{ + return (cmp_int(l_btree_id, r->btree_id) ?: + cmp_int(l_level, r->level) ?: + bpos_cmp(l_pos, r->k->k.p)); +} + +static inline int journal_key_cmp(const struct journal_key *l, const struct journal_key *r) +{ + return __journal_key_cmp(l->btree_id, l->level, l->k->k.p, r); +} + struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *, enum btree_id, unsigned, struct bpos, struct bpos, size_t *); struct bkey_i *bch2_journal_keys_peek_slot(struct bch_fs *, enum btree_id, diff --git a/libbcachefs/btree_locking.c b/libbcachefs/btree_locking.c index c3e9b0cc..79057cde 100644 --- a/libbcachefs/btree_locking.c +++ b/libbcachefs/btree_locking.c @@ -215,6 +215,7 @@ static noinline int break_cycle(struct lock_graph *g, struct printbuf *cycle) if (unlikely(!best)) { struct printbuf buf = PRINTBUF; + buf.atomic++; prt_printf(&buf, bch2_fmt(g->g->trans->c, "cycle of nofail locks")); @@ -792,6 +793,7 @@ static inline int __bch2_trans_relock(struct btree_trans *trans, bool trace) } trans->locked = true; + trans->last_unlock_ip = 0; out: bch2_trans_verify_locks(trans); return 0; diff --git a/libbcachefs/btree_trans_commit.c b/libbcachefs/btree_trans_commit.c index 74e1ff22..843558d9 100644 --- a/libbcachefs/btree_trans_commit.c +++ b/libbcachefs/btree_trans_commit.c @@ -10,6 +10,7 @@ #include "btree_update_interior.h" #include "btree_write_buffer.h" #include "buckets.h" +#include "disk_accounting.h" #include "errcode.h" #include "error.h" #include "journal.h" @@ -228,14 +229,14 @@ static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, struct btree_write *w = container_of(pin, struct btree_write, journal); struct btree *b = container_of(w, struct btree, writes[i]); struct btree_trans *trans = bch2_trans_get(c); - unsigned long old, new, v; + unsigned long old, new; unsigned idx = w - b->writes; btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read); - v = READ_ONCE(b->flags); + old = READ_ONCE(b->flags); do { - old = new = v; + new = old; if (!(old & (1 << BTREE_NODE_dirty)) || !!(old & (1 << BTREE_NODE_write_idx)) != idx || @@ -245,7 +246,7 @@ static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, new &= ~BTREE_WRITE_TYPE_MASK; new |= BTREE_WRITE_journal_reclaim; new |= 1 << BTREE_NODE_need_write; - } while ((v = cmpxchg(&b->flags, old, new)) != old); + } while (!try_cmpxchg(&b->flags, &old, new)); btree_node_write_if_need(c, b, SIX_LOCK_read); six_unlock_read(&b->c.lock); @@ -456,34 +457,36 @@ static int run_one_mem_trigger(struct btree_trans *trans, struct btree_insert_entry *i, unsigned flags) { - struct bkey_s_c old = { &i->old_k, i->old_v }; - struct bkey_i *new = i->k; - const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type); - const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); - int ret; - verify_update_old_key(trans, i); if (unlikely(flags & BTREE_TRIGGER_norun)) return 0; - if (old_ops->trigger == new_ops->trigger) { - ret = bch2_key_trigger(trans, i->btree_id, i->level, + struct bkey_s_c old = { &i->old_k, i->old_v }; + struct bkey_i *new = i->k; + const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type); + const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); + + if (old_ops->trigger == new_ops->trigger) + return bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(new), BTREE_TRIGGER_insert|BTREE_TRIGGER_overwrite|flags); - } else { - ret = bch2_key_trigger_new(trans, i->btree_id, i->level, + else + return bch2_key_trigger_new(trans, i->btree_id, i->level, bkey_i_to_s(new), flags) ?: - bch2_key_trigger_old(trans, i->btree_id, i->level, + bch2_key_trigger_old(trans, i->btree_id, i->level, old, flags); - } - - return ret; } static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i, bool overwrite) { + verify_update_old_key(trans, i); + + if ((i->flags & BTREE_TRIGGER_norun) || + !btree_node_type_has_trans_triggers(i->bkey_type)) + return 0; + /* * Transactional triggers create new btree_insert_entries, so we can't * pass them a pointer to a btree_insert_entry, that memory is going to @@ -495,12 +498,6 @@ static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_ const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); unsigned flags = i->flags|BTREE_TRIGGER_transactional; - verify_update_old_key(trans, i); - - if ((i->flags & BTREE_TRIGGER_norun) || - !(BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type))) - return 0; - if (!i->insert_trigger_run && !i->overwrite_trigger_run && old_ops->trigger == new_ops->trigger) { @@ -523,10 +520,8 @@ static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_ static int run_btree_triggers(struct btree_trans *trans, enum btree_id btree_id, unsigned btree_id_start) { - bool trans_trigger_run; - int ret, overwrite; - - for (overwrite = 1; overwrite >= 0; --overwrite) { + for (int overwrite = 1; overwrite >= 0; --overwrite) { + bool trans_trigger_run; /* * Running triggers will append more updates to the list of updates as @@ -541,7 +536,7 @@ static int run_btree_triggers(struct btree_trans *trans, enum btree_id btree_id, if (trans->updates[i].btree_id != btree_id) continue; - ret = run_one_trans_trigger(trans, trans->updates + i, overwrite); + int ret = run_one_trans_trigger(trans, trans->updates + i, overwrite); if (ret < 0) return ret; if (ret) @@ -594,7 +589,7 @@ static int bch2_trans_commit_run_triggers(struct btree_trans *trans) #ifdef CONFIG_BCACHEFS_DEBUG trans_for_each_update(trans, i) BUG_ON(!(i->flags & BTREE_TRIGGER_norun) && - (BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type)) && + btree_node_type_has_trans_triggers(i->bkey_type) && (!i->insert_trigger_run || !i->overwrite_trigger_run)); #endif return 0; @@ -602,24 +597,25 @@ static int bch2_trans_commit_run_triggers(struct btree_trans *trans) static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans) { - trans_for_each_update(trans, i) { - /* - * XXX: synchronization of cached update triggers with gc - * XXX: synchronization of interior node updates with gc - */ - BUG_ON(i->cached || i->level); - - if (btree_node_type_needs_gc(__btree_node_type(i->level, i->btree_id)) && + trans_for_each_update(trans, i) + if (btree_node_type_has_triggers(i->bkey_type) && gc_visited(trans->c, gc_pos_btree_node(insert_l(trans, i)->b))) { int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc); if (ret) return ret; } - } return 0; } +static struct bversion journal_pos_to_bversion(struct journal_res *res, unsigned offset) +{ + return (struct bversion) { + .hi = res->seq >> 32, + .lo = (res->seq << 32) | (res->offset + offset), + }; +} + static inline int bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, struct btree_insert_entry **stopped_at, @@ -628,7 +624,7 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, struct bch_fs *c = trans->c; struct btree_trans_commit_hook *h; unsigned u64s = 0; - int ret; + int ret = 0; bch2_trans_verify_not_unlocked(trans); bch2_trans_verify_not_in_restart(trans); @@ -693,23 +689,40 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, i->k->k.version = MAX_VERSION; } - if (trans->fs_usage_deltas && - bch2_trans_fs_usage_apply(trans, trans->fs_usage_deltas)) - return -BCH_ERR_btree_insert_need_mark_replicas; - - /* XXX: we only want to run this if deltas are nonzero */ - bch2_trans_account_disk_usage_change(trans); - h = trans->hooks; while (h) { ret = h->fn(trans, h); if (ret) - goto revert_fs_usage; + return ret; h = h->next; } + struct jset_entry *entry = trans->journal_entries; + + if (likely(!(flags & BCH_TRANS_COMMIT_skip_accounting_apply))) { + percpu_down_read(&c->mark_lock); + + for (entry = trans->journal_entries; + entry != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); + entry = vstruct_next(entry)) + if (jset_entry_is_key(entry) && entry->start->k.type == KEY_TYPE_accounting) { + struct bkey_i_accounting *a = bkey_i_to_accounting(entry->start); + + a->k.version = journal_pos_to_bversion(&trans->journal_res, + (u64 *) entry - (u64 *) trans->journal_entries); + BUG_ON(bversion_zero(a->k.version)); + ret = bch2_accounting_mem_mod_locked(trans, accounting_i_to_s_c(a), false); + if (ret) + goto revert_fs_usage; + } + percpu_up_read(&c->mark_lock); + + /* XXX: we only want to run this if deltas are nonzero */ + bch2_trans_account_disk_usage_change(trans); + } + trans_for_each_update(trans, i) - if (BTREE_NODE_TYPE_HAS_ATOMIC_TRIGGERS & (1U << i->bkey_type)) { + if (btree_node_type_has_atomic_triggers(i->bkey_type)) { ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags); if (ret) goto fatal_err; @@ -776,17 +789,34 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, return 0; fatal_err: - bch2_fatal_error(c); + bch2_fs_fatal_error(c, "fatal error in transaction commit: %s", bch2_err_str(ret)); + percpu_down_read(&c->mark_lock); revert_fs_usage: - if (trans->fs_usage_deltas) - bch2_trans_fs_usage_revert(trans, trans->fs_usage_deltas); + for (struct jset_entry *entry2 = trans->journal_entries; + entry2 != entry; + entry2 = vstruct_next(entry2)) + if (jset_entry_is_key(entry2) && entry2->start->k.type == KEY_TYPE_accounting) { + struct bkey_s_accounting a = bkey_i_to_s_accounting(entry2->start); + + bch2_accounting_neg(a); + bch2_accounting_mem_mod_locked(trans, a.c, false); + bch2_accounting_neg(a); + } + percpu_up_read(&c->mark_lock); return ret; } static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans) { + /* + * Accounting keys aren't deduped in the journal: we have to compare + * each individual update against what's in the btree to see if it has + * been applied yet, and accounting updates also don't overwrite, + * they're deltas that accumulate. + */ trans_for_each_update(trans, i) - bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p); + if (i->k->k.type != KEY_TYPE_accounting) + bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p); } static noinline int bch2_trans_commit_bkey_invalid(struct btree_trans *trans, @@ -922,7 +952,7 @@ int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags, break; case -BCH_ERR_btree_insert_need_mark_replicas: ret = drop_locks_do(trans, - bch2_replicas_delta_list_mark(c, trans->fs_usage_deltas)); + bch2_accounting_update_sb(trans)); break; case -BCH_ERR_journal_res_get_blocked: /* @@ -993,15 +1023,24 @@ static noinline int do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans) { struct bch_fs *c = trans->c; - int ret = 0; trans_for_each_update(trans, i) { - ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->k); + int ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->k); if (ret) - break; + return ret; } - return ret; + for (struct jset_entry *i = trans->journal_entries; + i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); + i = vstruct_next(i)) + if (i->type == BCH_JSET_ENTRY_btree_keys || + i->type == BCH_JSET_ENTRY_write_buffer_keys) { + int ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->start); + if (ret) + return ret; + } + + return 0; } int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) @@ -1017,8 +1056,6 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) !trans->journal_entries_u64s) goto out_reset; - memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta)); - ret = bch2_trans_commit_run_triggers(trans); if (ret) goto out_reset; @@ -1115,6 +1152,7 @@ retry: bch2_trans_verify_not_in_restart(trans); if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) memset(&trans->journal_res, 0, sizeof(trans->journal_res)); + memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta)); ret = do_bch2_trans_commit(trans, flags, &errored_at, _RET_IP_); diff --git a/libbcachefs/btree_types.h b/libbcachefs/btree_types.h index d63db4fe..e7a78ef6 100644 --- a/libbcachefs/btree_types.h +++ b/libbcachefs/btree_types.h @@ -522,7 +522,6 @@ struct btree_trans { unsigned journal_u64s; unsigned extra_disk_res; /* XXX kill */ - struct replicas_delta_list *fs_usage_deltas; /* Entries before this are zeroed out on every bch2_trans_get() call */ @@ -754,20 +753,30 @@ const char *bch2_btree_node_type_str(enum btree_node_type); (BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS| \ BTREE_NODE_TYPE_HAS_ATOMIC_TRIGGERS) -static inline bool btree_node_type_needs_gc(enum btree_node_type type) +static inline bool btree_node_type_has_trans_triggers(enum btree_node_type type) { - return BTREE_NODE_TYPE_HAS_TRIGGERS & BIT_ULL(type); + return BIT_ULL(type) & BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS; +} + +static inline bool btree_node_type_has_atomic_triggers(enum btree_node_type type) +{ + return BIT_ULL(type) & BTREE_NODE_TYPE_HAS_ATOMIC_TRIGGERS; +} + +static inline bool btree_node_type_has_triggers(enum btree_node_type type) +{ + return BIT_ULL(type) & BTREE_NODE_TYPE_HAS_TRIGGERS; } static inline bool btree_node_type_is_extents(enum btree_node_type type) { - const unsigned mask = 0 + const u64 mask = 0 #define x(name, nr, flags, ...) |((!!((flags) & BTREE_ID_EXTENTS)) << (nr + 1)) BCH_BTREE_IDS() #undef x ; - return (1U << type) & mask; + return BIT_ULL(type) & mask; } static inline bool btree_id_is_extents(enum btree_id btree) @@ -777,35 +786,35 @@ static inline bool btree_id_is_extents(enum btree_id btree) static inline bool btree_type_has_snapshots(enum btree_id id) { - const unsigned mask = 0 + const u64 mask = 0 #define x(name, nr, flags, ...) |((!!((flags) & BTREE_ID_SNAPSHOTS)) << nr) BCH_BTREE_IDS() #undef x ; - return (1U << id) & mask; + return BIT_ULL(id) & mask; } static inline bool btree_type_has_snapshot_field(enum btree_id id) { - const unsigned mask = 0 + const u64 mask = 0 #define x(name, nr, flags, ...) |((!!((flags) & (BTREE_ID_SNAPSHOT_FIELD|BTREE_ID_SNAPSHOTS))) << nr) BCH_BTREE_IDS() #undef x ; - return (1U << id) & mask; + return BIT_ULL(id) & mask; } static inline bool btree_type_has_ptrs(enum btree_id id) { - const unsigned mask = 0 + const u64 mask = 0 #define x(name, nr, flags, ...) |((!!((flags) & BTREE_ID_DATA)) << nr) BCH_BTREE_IDS() #undef x ; - return (1U << id) & mask; + return BIT_ULL(id) & mask; } struct btree_root { diff --git a/libbcachefs/btree_update.c b/libbcachefs/btree_update.c index f3c645a4..d6f6df10 100644 --- a/libbcachefs/btree_update.c +++ b/libbcachefs/btree_update.c @@ -656,14 +656,16 @@ int bch2_btree_insert_trans(struct btree_trans *trans, enum btree_id id, * @disk_res: must be non-NULL whenever inserting or potentially * splitting data extents * @flags: transaction commit flags + * @iter_flags: btree iter update trigger flags * * Returns: 0 on success, error code on failure */ int bch2_btree_insert(struct bch_fs *c, enum btree_id id, struct bkey_i *k, - struct disk_reservation *disk_res, int flags) + struct disk_reservation *disk_res, int flags, + enum btree_iter_update_trigger_flags iter_flags) { return bch2_trans_do(c, disk_res, NULL, flags, - bch2_btree_insert_trans(trans, id, k, 0)); + bch2_btree_insert_trans(trans, id, k, iter_flags)); } int bch2_btree_delete_extent_at(struct btree_trans *trans, struct btree_iter *iter, diff --git a/libbcachefs/btree_update.h b/libbcachefs/btree_update.h index b4894e4d..60393e98 100644 --- a/libbcachefs/btree_update.h +++ b/libbcachefs/btree_update.h @@ -29,6 +29,7 @@ void bch2_btree_insert_key_leaf(struct btree_trans *, struct btree_path *, "pin journal entry referred to by trans->journal_res.seq") \ x(journal_reclaim, "operation required for journal reclaim; may return error" \ "instead of deadlocking if BCH_WATERMARK_reclaim not specified")\ + x(skip_accounting_apply, "we're in journal replay - accounting updates have already been applied") enum __bch_trans_commit_flags { /* First bits for bch_watermark: */ @@ -56,8 +57,9 @@ int bch2_btree_insert_nonextent(struct btree_trans *, enum btree_id, int bch2_btree_insert_trans(struct btree_trans *, enum btree_id, struct bkey_i *, enum btree_iter_update_trigger_flags); -int bch2_btree_insert(struct bch_fs *, enum btree_id, struct bkey_i *, - struct disk_reservation *, int flags); +int bch2_btree_insert(struct bch_fs *, enum btree_id, struct bkey_i *, struct + disk_reservation *, int flags, enum + btree_iter_update_trigger_flags iter_flags); int bch2_btree_delete_range_trans(struct btree_trans *, enum btree_id, struct bpos, struct bpos, unsigned, u64 *); @@ -130,7 +132,19 @@ static inline int __must_check bch2_trans_update_buffered(struct btree_trans *tr enum btree_id btree, struct bkey_i *k) { - if (unlikely(trans->journal_replay_not_finished)) + /* + * Most updates skip the btree write buffer until journal replay is + * finished because synchronization with journal replay relies on having + * a btree node locked - if we're overwriting a key in the journal that + * journal replay hasn't yet replayed, we have to mark it as + * overwritten. + * + * But accounting updates don't overwrite, they're deltas, and they have + * to be flushed to the btree strictly in order for journal replay to be + * able to tell which updates need to be applied: + */ + if (k->k.type != KEY_TYPE_accounting && + unlikely(trans->journal_replay_not_finished)) return bch2_btree_insert_clone_trans(trans, btree, k); struct jset_entry *e = bch2_trans_jset_entry_alloc(trans, jset_u64s(k->k.u64s)); @@ -178,14 +192,6 @@ static inline int bch2_trans_commit(struct btree_trans *trans, nested_lockrestart_do(_trans, _do ?: bch2_trans_commit(_trans, (_disk_res),\ (_journal_seq), (_flags))) -#define bch2_trans_run(_c, _do) \ -({ \ - struct btree_trans *trans = bch2_trans_get(_c); \ - int _ret = (_do); \ - bch2_trans_put(trans); \ - _ret; \ -}) - #define bch2_trans_do(_c, _disk_res, _journal_seq, _flags, _do) \ bch2_trans_run(_c, commit_do(trans, _disk_res, _journal_seq, _flags, _do)) @@ -203,14 +209,6 @@ static inline void bch2_trans_reset_updates(struct btree_trans *trans) trans->journal_entries_u64s = 0; trans->hooks = NULL; trans->extra_disk_res = 0; - - if (trans->fs_usage_deltas) { - trans->fs_usage_deltas->used = 0; - memset((void *) trans->fs_usage_deltas + - offsetof(struct replicas_delta_list, memset_start), 0, - (void *) &trans->fs_usage_deltas->memset_end - - (void *) &trans->fs_usage_deltas->memset_start); - } } static inline struct bkey_i *__bch2_bkey_make_mut_noupdate(struct btree_trans *trans, struct bkey_s_c k, diff --git a/libbcachefs/btree_update_interior.c b/libbcachefs/btree_update_interior.c index 60b8544c..a4c92753 100644 --- a/libbcachefs/btree_update_interior.c +++ b/libbcachefs/btree_update_interior.c @@ -61,7 +61,7 @@ int bch2_btree_node_check_topology(struct btree_trans *trans, struct btree *b) if (!bpos_eq(b->data->min_key, POS_MIN)) { printbuf_reset(&buf); bch2_bpos_to_text(&buf, b->data->min_key); - need_fsck_err(c, btree_root_bad_min_key, + need_fsck_err(trans, btree_root_bad_min_key, "btree root with incorrect min_key: %s", buf.buf); goto topology_repair; } @@ -69,7 +69,7 @@ int bch2_btree_node_check_topology(struct btree_trans *trans, struct btree *b) if (!bpos_eq(b->data->max_key, SPOS_MAX)) { printbuf_reset(&buf); bch2_bpos_to_text(&buf, b->data->max_key); - need_fsck_err(c, btree_root_bad_max_key, + need_fsck_err(trans, btree_root_bad_max_key, "btree root with incorrect max_key: %s", buf.buf); goto topology_repair; } @@ -105,7 +105,7 @@ int bch2_btree_node_check_topology(struct btree_trans *trans, struct btree *b) prt_str(&buf, "\n next "); bch2_bkey_val_to_text(&buf, c, k); - need_fsck_err(c, btree_node_topology_bad_min_key, "%s", buf.buf); + need_fsck_err(trans, btree_node_topology_bad_min_key, "%s", buf.buf); goto topology_repair; } @@ -122,7 +122,7 @@ int bch2_btree_node_check_topology(struct btree_trans *trans, struct btree *b) bch2_btree_id_str(b->c.btree_id), b->c.level); bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); - need_fsck_err(c, btree_node_topology_empty_interior_node, "%s", buf.buf); + need_fsck_err(trans, btree_node_topology_empty_interior_node, "%s", buf.buf); goto topology_repair; } else if (!bpos_eq(prev.k->k.p, b->key.k.p)) { bch2_topology_error(c); @@ -135,7 +135,7 @@ int bch2_btree_node_check_topology(struct btree_trans *trans, struct btree *b) prt_str(&buf, "\n last key "); bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(prev.k)); - need_fsck_err(c, btree_node_topology_bad_max_key, "%s", buf.buf); + need_fsck_err(trans, btree_node_topology_bad_max_key, "%s", buf.buf); goto topology_repair; } out: @@ -565,10 +565,6 @@ static void bch2_btree_update_free(struct btree_update *as, struct btree_trans * { struct bch_fs *c = as->c; - if (as->took_gc_lock) - up_read(&c->gc_lock); - as->took_gc_lock = false; - bch2_journal_pin_drop(&c->journal, &as->journal); bch2_journal_pin_flush(&c->journal, &as->journal); bch2_disk_reservation_put(c, &as->disk_res); @@ -1117,10 +1113,6 @@ static void bch2_btree_update_done(struct btree_update *as, struct btree_trans * BUG_ON(as->mode == BTREE_UPDATE_none); - if (as->took_gc_lock) - up_read(&as->c->gc_lock); - as->took_gc_lock = false; - bch2_btree_reserve_put(as, trans); continue_at(&as->cl, btree_update_set_nodes_written, @@ -1192,14 +1184,6 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path, split = path->l[level_end].b->nr.live_u64s > BTREE_SPLIT_THRESHOLD(c); } - if (!down_read_trylock(&c->gc_lock)) { - ret = drop_locks_do(trans, (down_read(&c->gc_lock), 0)); - if (ret) { - up_read(&c->gc_lock); - return ERR_PTR(ret); - } - } - as = mempool_alloc(&c->btree_interior_update_pool, GFP_NOFS); memset(as, 0, sizeof(*as)); closure_init(&as->cl, NULL); @@ -1208,7 +1192,6 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path, as->ip_started = _RET_IP_; as->mode = BTREE_UPDATE_none; as->flags = flags; - as->took_gc_lock = true; as->btree_id = path->btree_id; as->update_level_start = level_start; as->update_level_end = level_end; @@ -1356,10 +1339,10 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, struct bch_fs *c = as->c; struct bkey_packed *k; struct printbuf buf = PRINTBUF; - unsigned long old, new, v; + unsigned long old, new; BUG_ON(insert->k.type == KEY_TYPE_btree_ptr_v2 && - !btree_ptr_sectors_written(insert)); + !btree_ptr_sectors_written(bkey_i_to_s_c(insert))); if (unlikely(!test_bit(JOURNAL_replay_done, &c->journal.flags))) bch2_journal_key_overwritten(c, b->c.btree_id, b->c.level, insert->k.p); @@ -1395,14 +1378,14 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, bch2_btree_bset_insert_key(trans, path, b, node_iter, insert); set_btree_node_dirty_acct(c, b); - v = READ_ONCE(b->flags); + old = READ_ONCE(b->flags); do { - old = new = v; + new = old; new &= ~BTREE_WRITE_TYPE_MASK; new |= BTREE_WRITE_interior; new |= 1 << BTREE_NODE_need_write; - } while ((v = cmpxchg(&b->flags, old, new)) != old); + } while (!try_cmpxchg(&b->flags, &old, new)); printbuf_exit(&buf); } @@ -1777,7 +1760,6 @@ static int bch2_btree_insert_node(struct btree_update *as, struct btree_trans *t int live_u64s_added, u64s_added; int ret; - lockdep_assert_held(&c->gc_lock); BUG_ON(!btree_node_intent_locked(path, b->c.level)); BUG_ON(!b->c.level); BUG_ON(!as || as->b); diff --git a/libbcachefs/btree_update_interior.h b/libbcachefs/btree_update_interior.h index b5b76ce0..9caf45b2 100644 --- a/libbcachefs/btree_update_interior.h +++ b/libbcachefs/btree_update_interior.h @@ -54,7 +54,6 @@ struct btree_update { enum btree_update_mode mode; enum bch_trans_commit_flags flags; unsigned nodes_written:1; - unsigned took_gc_lock:1; enum btree_id btree_id; unsigned update_level_start; diff --git a/libbcachefs/btree_write_buffer.c b/libbcachefs/btree_write_buffer.c index 75c8a196..5e1262a5 100644 --- a/libbcachefs/btree_write_buffer.c +++ b/libbcachefs/btree_write_buffer.c @@ -5,6 +5,7 @@ #include "btree_update.h" #include "btree_update_interior.h" #include "btree_write_buffer.h" +#include "disk_accounting.h" #include "error.h" #include "journal.h" #include "journal_io.h" @@ -132,7 +133,9 @@ static noinline int wb_flush_one_slowpath(struct btree_trans *trans, static inline int wb_flush_one(struct btree_trans *trans, struct btree_iter *iter, struct btree_write_buffered_key *wb, - bool *write_locked, size_t *fast) + bool *write_locked, + bool *accounting_accumulated, + size_t *fast) { struct btree_path *path; int ret; @@ -145,6 +148,16 @@ static inline int wb_flush_one(struct btree_trans *trans, struct btree_iter *ite if (ret) return ret; + if (!*accounting_accumulated && wb->k.k.type == KEY_TYPE_accounting) { + struct bkey u; + struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, iter), &u); + + if (k.k->type == KEY_TYPE_accounting) + bch2_accounting_accumulate(bkey_i_to_accounting(&wb->k), + bkey_s_c_to_accounting(k)); + } + *accounting_accumulated = true; + /* * We can't clone a path that has write locks: unshare it now, before * set_pos and traverse(): @@ -257,8 +270,9 @@ static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans) struct journal *j = &c->journal; struct btree_write_buffer *wb = &c->btree_write_buffer; struct btree_iter iter = { NULL }; - size_t skipped = 0, fast = 0, slowpath = 0; + size_t overwritten = 0, fast = 0, slowpath = 0, could_not_insert = 0; bool write_locked = false; + bool accounting_replay_done = test_bit(BCH_FS_accounting_replay_done, &c->flags); int ret = 0; bch2_trans_unlock(trans); @@ -299,11 +313,22 @@ static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans) BUG_ON(!k->journal_seq); + if (!accounting_replay_done && + k->k.k.type == KEY_TYPE_accounting) { + slowpath++; + continue; + } + if (i + 1 < &darray_top(wb->sorted) && wb_key_eq(i, i + 1)) { struct btree_write_buffered_key *n = &wb->flushing.keys.data[i[1].idx]; - skipped++; + if (k->k.k.type == KEY_TYPE_accounting && + n->k.k.type == KEY_TYPE_accounting) + bch2_accounting_accumulate(bkey_i_to_accounting(&n->k), + bkey_i_to_s_c_accounting(&k->k)); + + overwritten++; n->journal_seq = min_t(u64, n->journal_seq, k->journal_seq); k->journal_seq = 0; continue; @@ -338,13 +363,15 @@ static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans) bch2_btree_iter_set_pos(&iter, k->k.k.p); btree_iter_path(trans, &iter)->preserve = false; + bool accounting_accumulated = false; do { if (race_fault()) { ret = -BCH_ERR_journal_reclaim_would_deadlock; break; } - ret = wb_flush_one(trans, &iter, k, &write_locked, &fast); + ret = wb_flush_one(trans, &iter, k, &write_locked, + &accounting_accumulated, &fast); if (!write_locked) bch2_trans_begin(trans); } while (bch2_err_matches(ret, BCH_ERR_transaction_restart)); @@ -385,8 +412,15 @@ static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans) if (!i->journal_seq) continue; - bch2_journal_pin_update(j, i->journal_seq, &wb->flushing.pin, - bch2_btree_write_buffer_journal_flush); + if (!accounting_replay_done && + i->k.k.type == KEY_TYPE_accounting) { + could_not_insert++; + continue; + } + + if (!could_not_insert) + bch2_journal_pin_update(j, i->journal_seq, &wb->flushing.pin, + bch2_btree_write_buffer_journal_flush); bch2_trans_begin(trans); @@ -399,13 +433,45 @@ static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans) btree_write_buffered_insert(trans, i)); if (ret) goto err; + + i->journal_seq = 0; + } + + /* + * If journal replay hasn't finished with accounting keys we + * can't flush accounting keys at all - condense them and leave + * them for next time. + * + * Q: Can the write buffer overflow? + * A Shouldn't be any actual risk. It's just new accounting + * updates that the write buffer can't flush, and those are only + * going to be generated by interior btree node updates as + * journal replay has to split/rewrite nodes to make room for + * its updates. + * + * And for those new acounting updates, updates to the same + * counters get accumulated as they're flushed from the journal + * to the write buffer - see the patch for eytzingcer tree + * accumulated. So we could only overflow if the number of + * distinct counters touched somehow was very large. + */ + if (could_not_insert) { + struct btree_write_buffered_key *dst = wb->flushing.keys.data; + + darray_for_each(wb->flushing.keys, i) + if (i->journal_seq) + *dst++ = *i; + wb->flushing.keys.nr = dst - wb->flushing.keys.data; } } err: + if (ret || !could_not_insert) { + bch2_journal_pin_drop(j, &wb->flushing.pin); + wb->flushing.keys.nr = 0; + } + bch2_fs_fatal_err_on(ret, c, "%s", bch2_err_str(ret)); - trace_write_buffer_flush(trans, wb->flushing.keys.nr, skipped, fast, 0); - bch2_journal_pin_drop(j, &wb->flushing.pin); - wb->flushing.keys.nr = 0; + trace_write_buffer_flush(trans, wb->flushing.keys.nr, overwritten, fast, 0); return ret; } @@ -507,6 +573,29 @@ static void bch2_btree_write_buffer_flush_work(struct work_struct *work) bch2_write_ref_put(c, BCH_WRITE_REF_btree_write_buffer); } +static void wb_accounting_sort(struct btree_write_buffer *wb) +{ + eytzinger0_sort(wb->accounting.data, wb->accounting.nr, + sizeof(wb->accounting.data[0]), + wb_key_cmp, NULL); +} + +int bch2_accounting_key_to_wb_slowpath(struct bch_fs *c, enum btree_id btree, + struct bkey_i_accounting *k) +{ + struct btree_write_buffer *wb = &c->btree_write_buffer; + struct btree_write_buffered_key new = { .btree = btree }; + + bkey_copy(&new.k, &k->k_i); + + int ret = darray_push(&wb->accounting, new); + if (ret) + return ret; + + wb_accounting_sort(wb); + return 0; +} + int bch2_journal_key_to_wb_slowpath(struct bch_fs *c, struct journal_keys_to_wb *dst, enum btree_id btree, struct bkey_i *k) @@ -576,11 +665,35 @@ void bch2_journal_keys_to_write_buffer_start(struct bch_fs *c, struct journal_ke bch2_journal_pin_add(&c->journal, seq, &dst->wb->pin, bch2_btree_write_buffer_journal_flush); + + darray_for_each(wb->accounting, i) + memset(&i->k.v, 0, bkey_val_bytes(&i->k.k)); } -void bch2_journal_keys_to_write_buffer_end(struct bch_fs *c, struct journal_keys_to_wb *dst) +int bch2_journal_keys_to_write_buffer_end(struct bch_fs *c, struct journal_keys_to_wb *dst) { struct btree_write_buffer *wb = &c->btree_write_buffer; + unsigned live_accounting_keys = 0; + int ret = 0; + + darray_for_each(wb->accounting, i) + if (!bch2_accounting_key_is_zero(bkey_i_to_s_c_accounting(&i->k))) { + i->journal_seq = dst->seq; + live_accounting_keys++; + ret = __bch2_journal_key_to_wb(c, dst, i->btree, &i->k); + if (ret) + break; + } + + if (live_accounting_keys * 2 < wb->accounting.nr) { + struct btree_write_buffered_key *dst = wb->accounting.data; + + darray_for_each(wb->accounting, src) + if (!bch2_accounting_key_is_zero(bkey_i_to_s_c_accounting(&src->k))) + *dst++ = *src; + wb->accounting.nr = dst - wb->accounting.data; + wb_accounting_sort(wb); + } if (!dst->wb->keys.nr) bch2_journal_pin_drop(&c->journal, &dst->wb->pin); @@ -593,6 +706,8 @@ void bch2_journal_keys_to_write_buffer_end(struct bch_fs *c, struct journal_keys if (dst->wb == &wb->flushing) mutex_unlock(&wb->flushing.lock); mutex_unlock(&wb->inc.lock); + + return ret; } static int bch2_journal_keys_to_write_buffer(struct bch_fs *c, struct journal_buf *buf) @@ -616,7 +731,7 @@ static int bch2_journal_keys_to_write_buffer(struct bch_fs *c, struct journal_bu buf->need_flush_to_write_buffer = false; spin_unlock(&c->journal.lock); out: - bch2_journal_keys_to_write_buffer_end(c, &dst); + ret = bch2_journal_keys_to_write_buffer_end(c, &dst) ?: ret; return ret; } @@ -648,6 +763,7 @@ void bch2_fs_btree_write_buffer_exit(struct bch_fs *c) BUG_ON((wb->inc.keys.nr || wb->flushing.keys.nr) && !bch2_journal_error(&c->journal)); + darray_exit(&wb->accounting); darray_exit(&wb->sorted); darray_exit(&wb->flushing.keys); darray_exit(&wb->inc.keys); diff --git a/libbcachefs/btree_write_buffer.h b/libbcachefs/btree_write_buffer.h index eebcd2b1..ee3710ea 100644 --- a/libbcachefs/btree_write_buffer.h +++ b/libbcachefs/btree_write_buffer.h @@ -3,6 +3,7 @@ #define _BCACHEFS_BTREE_WRITE_BUFFER_H #include "bkey.h" +#include "disk_accounting.h" static inline bool bch2_btree_write_buffer_should_flush(struct bch_fs *c) { @@ -29,16 +30,45 @@ struct journal_keys_to_wb { u64 seq; }; +static inline int wb_key_cmp(const void *_l, const void *_r) +{ + const struct btree_write_buffered_key *l = _l; + const struct btree_write_buffered_key *r = _r; + + return cmp_int(l->btree, r->btree) ?: bpos_cmp(l->k.k.p, r->k.k.p); +} + +int bch2_accounting_key_to_wb_slowpath(struct bch_fs *, + enum btree_id, struct bkey_i_accounting *); + +static inline int bch2_accounting_key_to_wb(struct bch_fs *c, + enum btree_id btree, struct bkey_i_accounting *k) +{ + struct btree_write_buffer *wb = &c->btree_write_buffer; + struct btree_write_buffered_key search; + search.btree = btree; + search.k.k.p = k->k.p; + + unsigned idx = eytzinger0_find(wb->accounting.data, wb->accounting.nr, + sizeof(wb->accounting.data[0]), + wb_key_cmp, &search); + + if (idx >= wb->accounting.nr) + return bch2_accounting_key_to_wb_slowpath(c, btree, k); + + struct bkey_i_accounting *dst = bkey_i_to_accounting(&wb->accounting.data[idx].k); + bch2_accounting_accumulate(dst, accounting_i_to_s_c(k)); + return 0; +} + int bch2_journal_key_to_wb_slowpath(struct bch_fs *, struct journal_keys_to_wb *, enum btree_id, struct bkey_i *); -static inline int bch2_journal_key_to_wb(struct bch_fs *c, +static inline int __bch2_journal_key_to_wb(struct bch_fs *c, struct journal_keys_to_wb *dst, enum btree_id btree, struct bkey_i *k) { - EBUG_ON(!dst->seq); - if (unlikely(!dst->room)) return bch2_journal_key_to_wb_slowpath(c, dst, btree, k); @@ -51,8 +81,19 @@ static inline int bch2_journal_key_to_wb(struct bch_fs *c, return 0; } +static inline int bch2_journal_key_to_wb(struct bch_fs *c, + struct journal_keys_to_wb *dst, + enum btree_id btree, struct bkey_i *k) +{ + EBUG_ON(!dst->seq); + + return k->k.type == KEY_TYPE_accounting + ? bch2_accounting_key_to_wb(c, btree, bkey_i_to_accounting(k)) + : __bch2_journal_key_to_wb(c, dst, btree, k); +} + void bch2_journal_keys_to_write_buffer_start(struct bch_fs *, struct journal_keys_to_wb *, u64); -void bch2_journal_keys_to_write_buffer_end(struct bch_fs *, struct journal_keys_to_wb *); +int bch2_journal_keys_to_write_buffer_end(struct bch_fs *, struct journal_keys_to_wb *); int bch2_btree_write_buffer_resize(struct bch_fs *, size_t); void bch2_fs_btree_write_buffer_exit(struct bch_fs *); diff --git a/libbcachefs/btree_write_buffer_types.h b/libbcachefs/btree_write_buffer_types.h index 9b9433de..e9e76e20 100644 --- a/libbcachefs/btree_write_buffer_types.h +++ b/libbcachefs/btree_write_buffer_types.h @@ -52,6 +52,8 @@ struct btree_write_buffer { struct btree_write_buffer_keys inc; struct btree_write_buffer_keys flushing; struct work_struct flush_work; + + DARRAY(struct btree_write_buffered_key) accounting; }; #endif /* _BCACHEFS_BTREE_WRITE_BUFFER_TYPES_H */ diff --git a/libbcachefs/buckets.c b/libbcachefs/buckets.c index ed97712d..7c6cf94d 100644 --- a/libbcachefs/buckets.c +++ b/libbcachefs/buckets.c @@ -13,6 +13,7 @@ #include "btree_update.h" #include "buckets.h" #include "buckets_waiting_for_journal.h" +#include "disk_accounting.h" #include "ec.h" #include "error.h" #include "inode.h" @@ -25,197 +26,10 @@ #include <linux/preempt.h> -static inline void fs_usage_data_type_to_base(struct bch_fs_usage_base *fs_usage, - enum bch_data_type data_type, - s64 sectors) -{ - switch (data_type) { - case BCH_DATA_btree: - fs_usage->btree += sectors; - break; - case BCH_DATA_user: - case BCH_DATA_parity: - fs_usage->data += sectors; - break; - case BCH_DATA_cached: - fs_usage->cached += sectors; - break; - default: - break; - } -} - -void bch2_fs_usage_initialize(struct bch_fs *c) -{ - percpu_down_write(&c->mark_lock); - struct bch_fs_usage *usage = c->usage_base; - - for (unsigned i = 0; i < ARRAY_SIZE(c->usage); i++) - bch2_fs_usage_acc_to_base(c, i); - - for (unsigned i = 0; i < BCH_REPLICAS_MAX; i++) - usage->b.reserved += usage->persistent_reserved[i]; - - for (unsigned i = 0; i < c->replicas.nr; i++) { - struct bch_replicas_entry_v1 *e = - cpu_replicas_entry(&c->replicas, i); - - fs_usage_data_type_to_base(&usage->b, e->data_type, usage->replicas[i]); - } - - for_each_member_device(c, ca) { - struct bch_dev_usage dev = bch2_dev_usage_read(ca); - - usage->b.hidden += (dev.d[BCH_DATA_sb].buckets + - dev.d[BCH_DATA_journal].buckets) * - ca->mi.bucket_size; - } - - percpu_up_write(&c->mark_lock); -} - -static inline struct bch_dev_usage *dev_usage_ptr(struct bch_dev *ca, - unsigned journal_seq, - bool gc) -{ - BUG_ON(!gc && !journal_seq); - - return this_cpu_ptr(gc - ? ca->usage_gc - : ca->usage[journal_seq & JOURNAL_BUF_MASK]); -} - void bch2_dev_usage_read_fast(struct bch_dev *ca, struct bch_dev_usage *usage) { - struct bch_fs *c = ca->fs; - unsigned seq, i, u64s = dev_usage_u64s(); - - do { - seq = read_seqcount_begin(&c->usage_lock); - memcpy(usage, ca->usage_base, u64s * sizeof(u64)); - for (i = 0; i < ARRAY_SIZE(ca->usage); i++) - acc_u64s_percpu((u64 *) usage, (u64 __percpu *) ca->usage[i], u64s); - } while (read_seqcount_retry(&c->usage_lock, seq)); -} - -u64 bch2_fs_usage_read_one(struct bch_fs *c, u64 *v) -{ - ssize_t offset = v - (u64 *) c->usage_base; - unsigned i, seq; - u64 ret; - - BUG_ON(offset < 0 || offset >= fs_usage_u64s(c)); - percpu_rwsem_assert_held(&c->mark_lock); - - do { - seq = read_seqcount_begin(&c->usage_lock); - ret = *v; - - for (i = 0; i < ARRAY_SIZE(c->usage); i++) - ret += percpu_u64_get((u64 __percpu *) c->usage[i] + offset); - } while (read_seqcount_retry(&c->usage_lock, seq)); - - return ret; -} - -struct bch_fs_usage_online *bch2_fs_usage_read(struct bch_fs *c) -{ - struct bch_fs_usage_online *ret; - unsigned nr_replicas = READ_ONCE(c->replicas.nr); - unsigned seq, i; -retry: - ret = kmalloc(__fs_usage_online_u64s(nr_replicas) * sizeof(u64), GFP_KERNEL); - if (unlikely(!ret)) - return NULL; - - percpu_down_read(&c->mark_lock); - - if (nr_replicas != c->replicas.nr) { - nr_replicas = c->replicas.nr; - percpu_up_read(&c->mark_lock); - kfree(ret); - goto retry; - } - - ret->online_reserved = percpu_u64_get(c->online_reserved); - - do { - seq = read_seqcount_begin(&c->usage_lock); - unsafe_memcpy(&ret->u, c->usage_base, - __fs_usage_u64s(nr_replicas) * sizeof(u64), - "embedded variable length struct"); - for (i = 0; i < ARRAY_SIZE(c->usage); i++) - acc_u64s_percpu((u64 *) &ret->u, (u64 __percpu *) c->usage[i], - __fs_usage_u64s(nr_replicas)); - } while (read_seqcount_retry(&c->usage_lock, seq)); - - return ret; -} - -void bch2_fs_usage_acc_to_base(struct bch_fs *c, unsigned idx) -{ - unsigned u64s = fs_usage_u64s(c); - - BUG_ON(idx >= ARRAY_SIZE(c->usage)); - - preempt_disable(); - write_seqcount_begin(&c->usage_lock); - - acc_u64s_percpu((u64 *) c->usage_base, - (u64 __percpu *) c->usage[idx], u64s); - percpu_memset(c->usage[idx], 0, u64s * sizeof(u64)); - - rcu_read_lock(); - for_each_member_device_rcu(c, ca, NULL) { - u64s = dev_usage_u64s(); - - acc_u64s_percpu((u64 *) ca->usage_base, - (u64 __percpu *) ca->usage[idx], u64s); - percpu_memset(ca->usage[idx], 0, u64s * sizeof(u64)); - } - rcu_read_unlock(); - - write_seqcount_end(&c->usage_lock); - preempt_enable(); -} - -void bch2_fs_usage_to_text(struct printbuf *out, - struct bch_fs *c, - struct bch_fs_usage_online *fs_usage) -{ - unsigned i; - - prt_printf(out, "capacity:\t\t\t%llu\n", c->capacity); - - prt_printf(out, "hidden:\t\t\t\t%llu\n", - fs_usage->u.b.hidden); - prt_printf(out, "data:\t\t\t\t%llu\n", - fs_usage->u.b.data); - prt_printf(out, "cached:\t\t\t\t%llu\n", - fs_usage->u.b.cached); - prt_printf(out, "reserved:\t\t\t%llu\n", - fs_usage->u.b.reserved); - prt_printf(out, "nr_inodes:\t\t\t%llu\n", - fs_usage->u.b.nr_inodes); - prt_printf(out, "online reserved:\t\t%llu\n", - fs_usage->online_reserved); - - for (i = 0; - i < ARRAY_SIZE(fs_usage->u.persistent_reserved); - i++) { - prt_printf(out, "%u replicas:\n", i + 1); - prt_printf(out, "\treserved:\t\t%llu\n", - fs_usage->u.persistent_reserved[i]); - } - - for (i = 0; i < c->replicas.nr; i++) { - struct bch_replicas_entry_v1 *e = - cpu_replicas_entry(&c->replicas, i); - - prt_printf(out, "\t"); - bch2_replicas_entry_to_text(out, e); - prt_printf(out, ":\t%llu\n", fs_usage->u.replicas[i]); - } + memset(usage, 0, sizeof(*usage)); + acc_u64s_percpu((u64 *) usage, (u64 __percpu *) ca->usage, dev_usage_u64s()); } static u64 reserve_factor(u64 r) @@ -223,16 +37,6 @@ static u64 reserve_factor(u64 r) return r + (round_up(r, (1 << RESERVE_FACTOR)) >> RESERVE_FACTOR); } -u64 bch2_fs_sectors_used(struct bch_fs *c, struct bch_fs_usage_online *fs_usage) -{ - return min(fs_usage->u.b.hidden + - fs_usage->u.b.btree + - fs_usage->u.b.data + - reserve_factor(fs_usage->u.b.reserved + - fs_usage->online_reserved), - c->capacity); -} - static struct bch_fs_usage_short __bch2_fs_usage_read_short(struct bch_fs *c) { @@ -240,17 +44,17 @@ __bch2_fs_usage_read_short(struct bch_fs *c) u64 data, reserved; ret.capacity = c->capacity - - bch2_fs_usage_read_one(c, &c->usage_base->b.hidden); + percpu_u64_get(&c->usage->hidden); - data = bch2_fs_usage_read_one(c, &c->usage_base->b.data) + - bch2_fs_usage_read_one(c, &c->usage_base->b.btree); - reserved = bch2_fs_usage_read_one(c, &c->usage_base->b.reserved) + + data = percpu_u64_get(&c->usage->data) + + percpu_u64_get(&c->usage->btree); + reserved = percpu_u64_get(&c->usage->reserved) + percpu_u64_get(c->online_reserved); ret.used = min(ret.capacity, data + reserve_factor(reserved)); ret.free = ret.capacity - ret.used; - ret.nr_inodes = bch2_fs_usage_read_one(c, &c->usage_base->b.nr_inodes); + ret.nr_inodes = percpu_u64_get(&c->usage->nr_inodes); return ret; } @@ -267,11 +71,6 @@ bch2_fs_usage_read_short(struct bch_fs *c) return ret; } -void bch2_dev_usage_init(struct bch_dev *ca) -{ - ca->usage_base->d[BCH_DATA_free].buckets = ca->mi.nbuckets - ca->mi.first_bucket; -} - void bch2_dev_usage_to_text(struct printbuf *out, struct bch_dev_usage *usage) { prt_printf(out, "\tbuckets\rsectors\rfragmented\r\n"); @@ -285,186 +84,6 @@ void bch2_dev_usage_to_text(struct printbuf *out, struct bch_dev_usage *usage) } } -void bch2_dev_usage_update(struct bch_fs *c, struct bch_dev *ca, - const struct bch_alloc_v4 *old, - const struct bch_alloc_v4 *new, - u64 journal_seq, bool gc) -{ - struct bch_fs_usage *fs_usage; - struct bch_dev_usage *u; - - preempt_disable(); - fs_usage = fs_usage_ptr(c, journal_seq, gc); - - if (data_type_is_hidden(old->data_type)) - fs_usage->b.hidden -= ca->mi.bucket_size; - if (data_type_is_hidden(new->data_type)) - fs_usage->b.hidden += ca->mi.bucket_size; - - u = dev_usage_ptr(ca, journal_seq, gc); - - u->d[old->data_type].buckets--; - u->d[new->data_type].buckets++; - - u->d[old->data_type].sectors -= bch2_bucket_sectors_dirty(*old); - u->d[new->data_type].sectors += bch2_bucket_sectors_dirty(*new); - - u->d[BCH_DATA_cached].sectors += new->cached_sectors; - u->d[BCH_DATA_cached].sectors -= old->cached_sectors; - - u->d[old->data_type].fragmented -= bch2_bucket_sectors_fragmented(ca, *old); - u->d[new->data_type].fragmented += bch2_bucket_sectors_fragmented(ca, *new); - - preempt_enable(); -} - -static inline int __update_replicas(struct bch_fs *c, - struct bch_fs_usage *fs_usage, - struct bch_replicas_entry_v1 *r, - s64 sectors) -{ - int idx = bch2_replicas_entry_idx(c, r); - - if (idx < 0) - return -1; - - fs_usage_data_type_to_base(&fs_usage->b, r->data_type, sectors); - fs_usage->replicas[idx] += sectors; - return 0; -} - -int bch2_update_replicas(struct bch_fs *c, struct bkey_s_c k, - struct bch_replicas_entry_v1 *r, s64 sectors, - unsigned journal_seq, bool gc) -{ - struct bch_fs_usage *fs_usage; - int idx, ret = 0; - struct printbuf buf = PRINTBUF; - - percpu_down_read(&c->mark_lock); - - idx = bch2_replicas_entry_idx(c, r); - if (idx < 0 && - fsck_err(c, ptr_to_missing_replicas_entry, - "no replicas entry\n while marking %s", - (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { - percpu_up_read(&c->mark_lock); - ret = bch2_mark_replicas(c, r); - percpu_down_read(&c->mark_lock); - - if (ret) - goto err; - idx = bch2_replicas_entry_idx(c, r); - } - if (idx < 0) { - ret = -1; - goto err; - } - - preempt_disable(); - fs_usage = fs_usage_ptr(c, journal_seq, gc); - fs_usage_data_type_to_base(&fs_usage->b, r->data_type, sectors); - fs_usage->replicas[idx] += sectors; - preempt_enable(); -err: -fsck_err: - percpu_up_read(&c->mark_lock); - printbuf_exit(&buf); - return ret; -} - -static inline int update_cached_sectors(struct bch_fs *c, - struct bkey_s_c k, - unsigned dev, s64 sectors, - unsigned journal_seq, bool gc) -{ - struct bch_replicas_padded r; - - bch2_replicas_entry_cached(&r.e, dev); - - return bch2_update_replicas(c, k, &r.e, sectors, journal_seq, gc); -} - -static int __replicas_deltas_realloc(struct btree_trans *trans, unsigned more, - gfp_t gfp) -{ - struct replicas_delta_list *d = trans->fs_usage_deltas; - unsigned new_size = d ? (d->size + more) * 2 : 128; - unsigned alloc_size = sizeof(*d) + new_size; - - WARN_ON_ONCE(alloc_size > REPLICAS_DELTA_LIST_MAX); - - if (!d || d->used + more > d->size) { - d = krealloc(d, alloc_size, gfp|__GFP_ZERO); - - if (unlikely(!d)) { - if (alloc_size > REPLICAS_DELTA_LIST_MAX) - return -ENOMEM; - - d = mempool_alloc(&trans->c->replicas_delta_pool, gfp); - if (!d) - return -ENOMEM; - - memset(d, 0, REPLICAS_DELTA_LIST_MAX); - - if (trans->fs_usage_deltas) - memcpy(d, trans->fs_usage_deltas, - trans->fs_usage_deltas->size + sizeof(*d)); - - new_size = REPLICAS_DELTA_LIST_MAX - sizeof(*d); - kfree(trans->fs_usage_deltas); - } - - d->size = new_size; - trans->fs_usage_deltas = d; - } - - return 0; -} - -int bch2_replicas_deltas_realloc(struct btree_trans *trans, unsigned more) -{ - return allocate_dropping_locks_errcode(trans, - __replicas_deltas_realloc(trans, more, _gfp)); -} - -int bch2_update_replicas_list(struct btree_trans *trans, - struct bch_replicas_entry_v1 *r, - s64 sectors) -{ - struct replicas_delta_list *d; - struct replicas_delta *n; - unsigned b; - int ret; - - if (!sectors) - return 0; - - b = replicas_entry_bytes(r) + 8; - ret = bch2_replicas_deltas_realloc(trans, b); - if (ret) - return ret; - - d = trans->fs_usage_deltas; - n = (void *) d->d + d->used; - n->delta = sectors; - unsafe_memcpy((void *) n + offsetof(struct replicas_delta, r), - r, replicas_entry_bytes(r), - "flexible array member embedded in strcuct with padding"); - bch2_replicas_entry_sort(&n->r); - d->used += b; - return 0; -} - -int bch2_update_cached_sectors_list(struct btree_trans *trans, unsigned dev, s64 sectors) -{ - struct bch_replicas_padded r; - - bch2_replicas_entry_cached(&r.e, dev); - - return bch2_update_replicas_list(trans, &r.e, sectors); -} - int bch2_check_fix_ptrs(struct btree_trans *trans, enum btree_id btree, unsigned level, struct bkey_s_c k, enum btree_iter_update_trigger_flags flags) @@ -482,7 +101,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, bkey_for_each_ptr_decode(k.k, ptrs_c, p, entry_c) { struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev); if (!ca) { - if (fsck_err(c, ptr_to_invalid_device, + if (fsck_err(trans, ptr_to_invalid_device, "pointer to missing device %u\n" "while marking %s", p.ptr.dev, @@ -496,7 +115,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry_c); if (fsck_err_on(!g->gen_valid, - c, ptr_to_missing_alloc_key, + trans, ptr_to_missing_alloc_key, "bucket %u:%zu data type %s ptr gen %u missing in alloc btree\n" "while marking %s", p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), @@ -513,7 +132,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, } if (fsck_err_on(gen_cmp(p.ptr.gen, g->gen) > 0, - c, ptr_gen_newer_than_bucket_gen, + trans, ptr_gen_newer_than_bucket_gen, "bucket %u:%zu data type %s ptr gen in the future: %u > %u\n" "while marking %s", p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), @@ -527,6 +146,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, g->gen_valid = true; g->gen = p.ptr.gen; g->data_type = 0; + g->stripe_sectors = 0; g->dirty_sectors = 0; g->cached_sectors = 0; } else { @@ -535,7 +155,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, } if (fsck_err_on(gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX, - c, ptr_gen_newer_than_bucket_gen, + trans, ptr_gen_newer_than_bucket_gen, "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n" "while marking %s", p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen, @@ -546,7 +166,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, do_update = true; if (fsck_err_on(!p.ptr.cached && gen_cmp(p.ptr.gen, g->gen) < 0, - c, stale_dirty_ptr, + trans, stale_dirty_ptr, "bucket %u:%zu data type %s stale dirty ptr: %u < %u\n" "while marking %s", p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), @@ -560,7 +180,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, goto next; if (fsck_err_on(bucket_data_type_mismatch(g->data_type, data_type), - c, ptr_bucket_data_type_mismatch, + trans, ptr_bucket_data_type_mismatch, "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n" "while marking %s", p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen, @@ -572,6 +192,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, g->gen_valid = true; g->gen = p.ptr.gen; g->data_type = data_type; + g->stripe_sectors = 0; g->dirty_sectors = 0; g->cached_sectors = 0; } else { @@ -582,8 +203,8 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, if (p.has_ec) { struct gc_stripe *m = genradix_ptr(&c->gc_stripes, p.ec.idx); - if (fsck_err_on(!m || !m->alive, c, - ptr_to_missing_stripe, + if (fsck_err_on(!m || !m->alive, + trans, ptr_to_missing_stripe, "pointer to nonexistent stripe %llu\n" "while marking %s", (u64) p.ec.idx, @@ -591,8 +212,8 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, bch2_bkey_val_to_text(&buf, c, k), buf.buf))) do_update = true; - if (fsck_err_on(m && m->alive && !bch2_ptr_matches_stripe_m(m, p), c, - ptr_to_incorrect_stripe, + if (fsck_err_on(m && m->alive && !bch2_ptr_matches_stripe_m(m, p), + trans, ptr_to_incorrect_stripe, "pointer does not match stripe %llu\n" "while marking %s", (u64) p.ec.idx, @@ -738,8 +359,8 @@ int bch2_bucket_ref_update(struct btree_trans *trans, struct bch_dev *ca, BUG_ON(!sectors); if (gen_after(ptr->gen, b_gen)) { - bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, - BCH_FSCK_ERR_ptr_gen_newer_than_bucket_gen, + bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, + ptr_gen_newer_than_bucket_gen, "bucket %u:%zu gen %u data type %s: ptr gen %u newer than bucket gen\n" "while marking %s", ptr->dev, bucket_nr, b_gen, @@ -752,8 +373,8 @@ int bch2_bucket_ref_update(struct btree_trans *trans, struct bch_dev *ca, } if (gen_cmp(b_gen, ptr->gen) > BUCKET_GC_GEN_MAX) { - bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, - BCH_FSCK_ERR_ptr_too_stale, + bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, + ptr_too_stale, "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n" "while marking %s", ptr->dev, bucket_nr, b_gen, @@ -772,8 +393,8 @@ int bch2_bucket_ref_update(struct btree_trans *trans, struct bch_dev *ca, } if (b_gen != ptr->gen) { - bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, - BCH_FSCK_ERR_stale_dirty_ptr, + bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, + stale_dirty_ptr, "bucket %u:%zu gen %u (mem gen %u) data type %s: stale dirty ptr (gen %u)\n" "while marking %s", ptr->dev, bucket_nr, b_gen, @@ -788,8 +409,8 @@ int bch2_bucket_ref_update(struct btree_trans *trans, struct bch_dev *ca, } if (bucket_data_type_mismatch(bucket_data_type, ptr_data_type)) { - bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, - BCH_FSCK_ERR_ptr_bucket_data_type_mismatch, + bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, + ptr_bucket_data_type_mismatch, "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n" "while marking %s", ptr->dev, bucket_nr, b_gen, @@ -803,8 +424,8 @@ int bch2_bucket_ref_update(struct btree_trans *trans, struct bch_dev *ca, } if ((u64) *bucket_sectors + sectors > U32_MAX) { - bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, - BCH_FSCK_ERR_bucket_sector_count_overflow, + bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, + bucket_sector_count_overflow, "bucket %u:%zu gen %u data type %s sector count overflow: %u + %lli > U32_MAX\n" "while marking %s", ptr->dev, bucket_nr, b_gen, @@ -827,47 +448,6 @@ err: goto out; } -void bch2_trans_fs_usage_revert(struct btree_trans *trans, - struct replicas_delta_list *deltas) -{ - struct bch_fs *c = trans->c; - struct bch_fs_usage *dst; - struct replicas_delta *d, *top = (void *) deltas->d + deltas->used; - s64 added = 0; - unsigned i; - - percpu_down_read(&c->mark_lock); - preempt_disable(); - dst = fs_usage_ptr(c, trans->journal_res.seq, false); - - /* revert changes: */ - for (d = deltas->d; d != top; d = replicas_delta_next(d)) { - switch (d->r.data_type) { - case BCH_DATA_btree: - case BCH_DATA_user: - case BCH_DATA_parity: - added += d->delta; - } - BUG_ON(__update_replicas(c, dst, &d->r, -d->delta)); - } - - dst->b.nr_inodes -= deltas->nr_inodes; - - for (i = 0; i < BCH_REPLICAS_MAX; i++) { - added -= deltas->persistent_reserved[i]; - dst->b.reserved -= deltas->persistent_reserved[i]; - dst->persistent_reserved[i] -= deltas->persistent_reserved[i]; - } - - if (added > 0) { - trans->disk_res->sectors += added; - this_cpu_add(*c->online_reserved, added); - } - - preempt_enable(); - percpu_up_read(&c->mark_lock); -} - void bch2_trans_account_disk_usage_change(struct btree_trans *trans) { struct bch_fs *c = trans->c; @@ -876,8 +456,6 @@ void bch2_trans_account_disk_usage_change(struct btree_trans *trans) bool warn = false; percpu_down_read(&c->mark_lock); - preempt_disable(); - struct bch_fs_usage_base *dst = &fs_usage_ptr(c, trans->journal_res.seq, false)->b; struct bch_fs_usage_base *src = &trans->fs_usage_delta; s64 added = src->btree + src->data + src->reserved; @@ -888,13 +466,13 @@ void bch2_trans_account_disk_usage_change(struct btree_trans *trans) */ s64 should_not_have_added = added - (s64) disk_res_sectors; if (unlikely(should_not_have_added > 0)) { - u64 old, new, v = atomic64_read(&c->sectors_available); + u64 old, new; + old = atomic64_read(&c->sectors_available); do { - old = v; new = max_t(s64, 0, old - should_not_have_added); - } while ((v = atomic64_cmpxchg(&c->sectors_available, - old, new)) != old); + } while (!atomic64_try_cmpxchg(&c->sectors_available, + &old, new)); added -= should_not_have_added; warn = true; @@ -905,13 +483,9 @@ void bch2_trans_account_disk_usage_change(struct btree_trans *trans) this_cpu_sub(*c->online_reserved, added); } - dst->hidden += src->hidden; - dst->btree += src->btree; - dst->data += src->data; - dst->cached += src->cached; - dst->reserved += src->reserved; - dst->nr_inodes += src->nr_inodes; - + preempt_disable(); + struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage); + acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64)); preempt_enable(); percpu_up_read(&c->mark_lock); @@ -921,55 +495,18 @@ void bch2_trans_account_disk_usage_change(struct btree_trans *trans) should_not_have_added, disk_res_sectors); } -int bch2_trans_fs_usage_apply(struct btree_trans *trans, - struct replicas_delta_list *deltas) -{ - struct bch_fs *c = trans->c; - struct replicas_delta *d, *d2; - struct replicas_delta *top = (void *) deltas->d + deltas->used; - struct bch_fs_usage *dst; - unsigned i; - - percpu_down_read(&c->mark_lock); - preempt_disable(); - dst = fs_usage_ptr(c, trans->journal_res.seq, false); - - for (d = deltas->d; d != top; d = replicas_delta_next(d)) - if (__update_replicas(c, dst, &d->r, d->delta)) - goto need_mark; - - dst->b.nr_inodes += deltas->nr_inodes; - - for (i = 0; i < BCH_REPLICAS_MAX; i++) { - dst->b.reserved += deltas->persistent_reserved[i]; - dst->persistent_reserved[i] += deltas->persistent_reserved[i]; - } - - preempt_enable(); - percpu_up_read(&c->mark_lock); - return 0; -need_mark: - /* revert changes: */ - for (d2 = deltas->d; d2 != d; d2 = replicas_delta_next(d2)) - BUG_ON(__update_replicas(c, dst, &d2->r, -d2->delta)); - - preempt_enable(); - percpu_up_read(&c->mark_lock); - return -1; -} - /* KEY_TYPE_extent: */ static int __mark_pointer(struct btree_trans *trans, struct bch_dev *ca, struct bkey_s_c k, - const struct bch_extent_ptr *ptr, + const struct extent_ptr_decoded *p, s64 sectors, enum bch_data_type ptr_data_type, struct bch_alloc_v4 *a) { - u32 *dst_sectors = !ptr->cached - ? &a->dirty_sectors - : &a->cached_sectors; - int ret = bch2_bucket_ref_update(trans, ca, k, ptr, sectors, ptr_data_type, + u32 *dst_sectors = p->has_ec ? &a->stripe_sectors : + !p->ptr.cached ? &a->dirty_sectors : + &a->cached_sectors; + int ret = bch2_bucket_ref_update(trans, ca, k, &p->ptr, sectors, ptr_data_type, a->gen, a->data_type, dst_sectors); if (ret) @@ -1005,7 +542,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, if (flags & BTREE_TRIGGER_transactional) { struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, bucket); ret = PTR_ERR_OR_ZERO(a) ?: - __mark_pointer(trans, ca, k, &p.ptr, *sectors, bp.data_type, &a->v); + __mark_pointer(trans, ca, k, &p, *sectors, bp.data_type, &a->v); if (ret) goto err; @@ -1021,13 +558,13 @@ static int bch2_trigger_pointer(struct btree_trans *trans, struct bucket *g = gc_bucket(ca, bucket.offset); bucket_lock(g); struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old; - ret = __mark_pointer(trans, ca, k, &p.ptr, *sectors, bp.data_type, &new); - if (!ret) { - alloc_to_bucket(g, new); - bch2_dev_usage_update(c, ca, &old, &new, 0, true); - } + ret = __mark_pointer(trans, ca, k, &p, *sectors, bp.data_type, &new); + alloc_to_bucket(g, new); bucket_unlock(g); percpu_up_read(&c->mark_lock); + + if (!ret) + ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags); } err: bch2_dev_put(ca); @@ -1066,10 +603,12 @@ static int bch2_trigger_stripe_ptr(struct btree_trans *trans, stripe_blockcount_get(&s->v, p.ec.block) + sectors); - struct bch_replicas_padded r; - bch2_bkey_to_replicas(&r.e, bkey_i_to_s_c(&s->k_i)); - r.e.data_type = data_type; - ret = bch2_update_replicas_list(trans, &r.e, sectors); + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_replicas, + }; + bch2_bkey_to_replicas(&acc.replicas, bkey_i_to_s_c(&s->k_i)); + acc.replicas.data_type = data_type; + ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, false); err: bch2_trans_iter_exit(trans, &iter); return ret; @@ -1078,8 +617,6 @@ err: if (flags & BTREE_TRIGGER_gc) { struct bch_fs *c = trans->c; - BUG_ON(!(flags & BTREE_TRIGGER_gc)); - struct gc_stripe *m = genradix_ptr_alloc(&c->gc_stripes, p.ec.idx, GFP_KERNEL); if (!m) { bch_err(c, "error allocating memory for gc_stripes, idx %llu", @@ -1102,11 +639,16 @@ err: m->block_sectors[p.ec.block] += sectors; - struct bch_replicas_padded r = m->r; + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_replicas, + }; + memcpy(&acc.replicas, &m->r.e, replicas_entry_bytes(&m->r.e)); mutex_unlock(&c->ec_stripes_heap_lock); - r.e.data_type = data_type; - bch2_update_replicas(c, k, &r.e, sectors, trans->journal_res.seq, true); + acc.replicas.data_type = data_type; + int ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, true); + if (ret) + return ret; } return 0; @@ -1118,20 +660,26 @@ static int __trigger_extent(struct btree_trans *trans, enum btree_iter_update_trigger_flags flags) { bool gc = flags & BTREE_TRIGGER_gc; - struct bch_fs *c = trans->c; struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); const union bch_extent_entry *entry; struct extent_ptr_decoded p; - struct bch_replicas_padded r; enum bch_data_type data_type = bkey_is_btree_ptr(k.k) ? BCH_DATA_btree : BCH_DATA_user; s64 replicas_sectors = 0; int ret = 0; - r.e.data_type = data_type; - r.e.nr_devs = 0; - r.e.nr_required = 1; + struct disk_accounting_pos acc_replicas_key = { + .type = BCH_DISK_ACCOUNTING_replicas, + .replicas.data_type = data_type, + .replicas.nr_devs = 0, + .replicas.nr_required = 1, + }; + + struct disk_accounting_pos acct_compression_key = { + .type = BCH_DISK_ACCOUNTING_compression, + }; + u64 compression_acct[3] = { 1, 0, 0 }; bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { s64 disk_sectors = 0; @@ -1141,19 +689,16 @@ static int __trigger_extent(struct btree_trans *trans, bool stale = ret > 0; + if (p.ptr.cached && stale) + continue; + if (p.ptr.cached) { - if (!stale) { - ret = !gc - ? bch2_update_cached_sectors_list(trans, p.ptr.dev, disk_sectors) - : update_cached_sectors(c, k, p.ptr.dev, disk_sectors, 0, true); - bch2_fs_fatal_err_on(ret && gc, c, "%s: no replicas entry while updating cached sectors", - bch2_err_str(ret)); - if (ret) - return ret; - } + ret = bch2_mod_dev_cached_sectors(trans, p.ptr.dev, disk_sectors, gc); + if (ret) + return ret; } else if (!p.has_ec) { replicas_sectors += disk_sectors; - r.e.devs[r.e.nr_devs++] = p.ptr.dev; + acc_replicas_key.replicas.devs[acc_replicas_key.replicas.nr_devs++] = p.ptr.dev; } else { ret = bch2_trigger_stripe_ptr(trans, k, p, data_type, disk_sectors, flags); if (ret) @@ -1164,21 +709,72 @@ static int __trigger_extent(struct btree_trans *trans, * if so they're not required for mounting if we have an * erasure coded pointer in this extent: */ - r.e.nr_required = 0; + acc_replicas_key.replicas.nr_required = 0; } - } - if (r.e.nr_devs) { - ret = !gc - ? bch2_update_replicas_list(trans, &r.e, replicas_sectors) - : bch2_update_replicas(c, k, &r.e, replicas_sectors, 0, true); - if (unlikely(ret && gc)) { - struct printbuf buf = PRINTBUF; + if (acct_compression_key.compression.type && + acct_compression_key.compression.type != p.crc.compression_type) { + if (flags & BTREE_TRIGGER_overwrite) + bch2_u64s_neg(compression_acct, ARRAY_SIZE(compression_acct)); - bch2_bkey_val_to_text(&buf, c, k); - bch2_fs_fatal_error(c, ": no replicas entry for %s", buf.buf); - printbuf_exit(&buf); + ret = bch2_disk_accounting_mod(trans, &acct_compression_key, compression_acct, + ARRAY_SIZE(compression_acct), gc); + if (ret) + return ret; + + compression_acct[0] = 1; + compression_acct[1] = 0; + compression_acct[2] = 0; + } + + acct_compression_key.compression.type = p.crc.compression_type; + if (p.crc.compression_type) { + compression_acct[1] += p.crc.uncompressed_size; + compression_acct[2] += p.crc.compressed_size; } + } + + if (acc_replicas_key.replicas.nr_devs) { + ret = bch2_disk_accounting_mod(trans, &acc_replicas_key, &replicas_sectors, 1, gc); + if (ret) + return ret; + } + + if (acc_replicas_key.replicas.nr_devs && !level && k.k->p.snapshot) { + struct disk_accounting_pos acc_snapshot_key = { + .type = BCH_DISK_ACCOUNTING_snapshot, + .snapshot.id = k.k->p.snapshot, + }; + ret = bch2_disk_accounting_mod(trans, &acc_snapshot_key, &replicas_sectors, 1, gc); + if (ret) + return ret; + } + + if (acct_compression_key.compression.type) { + if (flags & BTREE_TRIGGER_overwrite) + bch2_u64s_neg(compression_acct, ARRAY_SIZE(compression_acct)); + + ret = bch2_disk_accounting_mod(trans, &acct_compression_key, compression_acct, + ARRAY_SIZE(compression_acct), gc); + if (ret) + return ret; + } + + if (level) { + struct disk_accounting_pos acc_btree_key = { + .type = BCH_DISK_ACCOUNTING_btree, + .btree.id = btree_id, + }; + ret = bch2_disk_accounting_mod(trans, &acc_btree_key, &replicas_sectors, 1, gc); + if (ret) + return ret; + } + + if (bch2_bkey_rebalance_opts(k)) { + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_rebalance_work, + }; + ret = bch2_disk_accounting_mod(trans, &acc, &replicas_sectors, 1, gc); if (ret) return ret; } @@ -1231,36 +827,18 @@ static int __trigger_reservation(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c k, enum btree_iter_update_trigger_flags flags) { - struct bch_fs *c = trans->c; - unsigned replicas = bkey_s_c_to_reservation(k).v->nr_replicas; - s64 sectors = (s64) k.k->size * replicas; + if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) { + s64 sectors = k.k->size; - if (flags & BTREE_TRIGGER_overwrite) - sectors = -sectors; - - if (flags & BTREE_TRIGGER_transactional) { - int ret = bch2_replicas_deltas_realloc(trans, 0); - if (ret) - return ret; + if (flags & BTREE_TRIGGER_overwrite) + sectors = -sectors; - struct replicas_delta_list *d = trans->fs_usage_deltas; - replicas = min(replicas, ARRAY_SIZE(d->persistent_reserved)); + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_persistent_reserved, + .persistent_reserved.nr_replicas = bkey_s_c_to_reservation(k).v->nr_replicas, + }; - d->persistent_reserved[replicas - 1] += sectors; - } - - if (flags & BTREE_TRIGGER_gc) { - percpu_down_read(&c->mark_lock); - preempt_disable(); - - struct bch_fs_usage *fs_usage = this_cpu_ptr(c->usage_gc); - - replicas = min(replicas, ARRAY_SIZE(fs_usage->persistent_reserved)); - fs_usage->b.reserved += sectors; - fs_usage->persistent_reserved[replicas - 1] += sectors; - - preempt_enable(); - percpu_up_read(&c->mark_lock); + return bch2_disk_accounting_mod(trans, &acc, §ors, 1, flags & BTREE_TRIGGER_gc); } return 0; @@ -1292,7 +870,7 @@ static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans, if (a->v.data_type && type && a->v.data_type != type) { bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, - BCH_FSCK_ERR_bucket_metadata_type_mismatch, + bucket_metadata_type_mismatch, "bucket %llu:%llu gen %u different types of data in same bucket: %s, %s\n" "while marking %s", iter.pos.inode, iter.pos.offset, a->v.gen, @@ -1314,10 +892,11 @@ err: return ret; } -static int bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca, +static int bch2_mark_metadata_bucket(struct btree_trans *trans, struct bch_dev *ca, u64 b, enum bch_data_type data_type, unsigned sectors, enum btree_iter_update_trigger_flags flags) { + struct bch_fs *c = trans->c; int ret = 0; percpu_down_read(&c->mark_lock); @@ -1347,10 +926,12 @@ static int bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca, g->data_type = data_type; g->dirty_sectors += sectors; struct bch_alloc_v4 new = bucket_m_to_alloc(*g); + bucket_unlock(g); + percpu_up_read(&c->mark_lock); + ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags); + return ret; err: bucket_unlock(g); - if (!ret) - bch2_dev_usage_update(c, ca, &old, &new, 0, true); percpu_up_read(&c->mark_lock); return ret; } @@ -1371,7 +952,7 @@ int bch2_trans_mark_metadata_bucket(struct btree_trans *trans, return 0; if (flags & BTREE_TRIGGER_gc) - return bch2_mark_metadata_bucket(trans->c, ca, b, type, sectors, flags); + return bch2_mark_metadata_bucket(trans, ca, b, type, sectors, flags); else if (flags & BTREE_TRIGGER_transactional) return commit_do(trans, NULL, NULL, 0, __bch2_trans_mark_metadata_bucket(trans, ca, b, type, sectors)); @@ -1486,7 +1067,7 @@ int __bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res, u64 sectors, int flags) { struct bch_fs_pcpu *pcpu; - u64 old, v, get; + u64 old, get; s64 sectors_available; int ret; @@ -1497,17 +1078,16 @@ int __bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res, if (sectors <= pcpu->sectors_available) goto out; - v = atomic64_read(&c->sectors_available); + old = atomic64_read(&c->sectors_available); do { - old = v; get = min((u64) sectors + SECTORS_CACHE, old); if (get < sectors) { preempt_enable(); goto recalculate; } - } while ((v = atomic64_cmpxchg(&c->sectors_available, - old, old - get)) != old); + } while (!atomic64_try_cmpxchg(&c->sectors_available, + &old, old - get)); pcpu->sectors_available += get; @@ -1597,7 +1177,6 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) bucket_gens->nbuckets = nbuckets; if (resize) { - down_write(&c->gc_lock); down_write(&ca->bucket_lock); percpu_down_write(&c->mark_lock); } @@ -1620,7 +1199,6 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) if (resize) { percpu_up_write(&c->mark_lock); up_write(&ca->bucket_lock); - up_write(&c->gc_lock); } ret = 0; @@ -1635,23 +1213,14 @@ void bch2_dev_buckets_free(struct bch_dev *ca) { kvfree(ca->buckets_nouse); kvfree(rcu_dereference_protected(ca->bucket_gens, 1)); - - for (unsigned i = 0; i < ARRAY_SIZE(ca->usage); i++) - free_percpu(ca->usage[i]); - kfree(ca->usage_base); + free_percpu(ca->usage); } int bch2_dev_buckets_alloc(struct bch_fs *c, struct bch_dev *ca) { - ca->usage_base = kzalloc(sizeof(struct bch_dev_usage), GFP_KERNEL); - if (!ca->usage_base) + ca->usage = alloc_percpu(struct bch_dev_usage); + if (!ca->usage) return -BCH_ERR_ENOMEM_usage_init; - for (unsigned i = 0; i < ARRAY_SIZE(ca->usage); i++) { - ca->usage[i] = alloc_percpu(struct bch_dev_usage); - if (!ca->usage[i]) - return -BCH_ERR_ENOMEM_usage_init; - } - return bch2_dev_buckets_resize(c, ca, ca->mi.nbuckets); } diff --git a/libbcachefs/buckets.h b/libbcachefs/buckets.h index 617ffde2..ff813dc6 100644 --- a/libbcachefs/buckets.h +++ b/libbcachefs/buckets.h @@ -85,7 +85,7 @@ static inline struct bucket_array *gc_bucket_array(struct bch_dev *ca) return rcu_dereference_check(ca->buckets_gc, !ca->fs || percpu_rwsem_is_held(&ca->fs->mark_lock) || - lockdep_is_held(&ca->fs->gc_lock) || + lockdep_is_held(&ca->fs->state_lock) || lockdep_is_held(&ca->bucket_lock)); } @@ -102,7 +102,7 @@ static inline struct bucket_gens *bucket_gens(struct bch_dev *ca) return rcu_dereference_check(ca->bucket_gens, !ca->fs || percpu_rwsem_is_held(&ca->fs->mark_lock) || - lockdep_is_held(&ca->fs->gc_lock) || + lockdep_is_held(&ca->fs->state_lock) || lockdep_is_held(&ca->bucket_lock)); } @@ -199,7 +199,6 @@ static inline struct bch_dev_usage bch2_dev_usage_read(struct bch_dev *ca) return ret; } -void bch2_dev_usage_init(struct bch_dev *); void bch2_dev_usage_to_text(struct printbuf *, struct bch_dev_usage *); static inline u64 bch2_dev_buckets_reserved(struct bch_dev *ca, enum bch_watermark watermark) @@ -261,73 +260,14 @@ static inline u64 dev_buckets_available(struct bch_dev *ca, /* Filesystem usage: */ -static inline unsigned __fs_usage_u64s(unsigned nr_replicas) -{ - return sizeof(struct bch_fs_usage) / sizeof(u64) + nr_replicas; -} - -static inline unsigned fs_usage_u64s(struct bch_fs *c) -{ - return __fs_usage_u64s(READ_ONCE(c->replicas.nr)); -} - -static inline unsigned __fs_usage_online_u64s(unsigned nr_replicas) -{ - return sizeof(struct bch_fs_usage_online) / sizeof(u64) + nr_replicas; -} - -static inline unsigned fs_usage_online_u64s(struct bch_fs *c) -{ - return __fs_usage_online_u64s(READ_ONCE(c->replicas.nr)); -} - static inline unsigned dev_usage_u64s(void) { return sizeof(struct bch_dev_usage) / sizeof(u64); } -u64 bch2_fs_usage_read_one(struct bch_fs *, u64 *); - -struct bch_fs_usage_online *bch2_fs_usage_read(struct bch_fs *); - -void bch2_fs_usage_acc_to_base(struct bch_fs *, unsigned); - -void bch2_fs_usage_to_text(struct printbuf *, - struct bch_fs *, struct bch_fs_usage_online *); - -u64 bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage_online *); - struct bch_fs_usage_short bch2_fs_usage_read_short(struct bch_fs *); -void bch2_dev_usage_update(struct bch_fs *, struct bch_dev *, - const struct bch_alloc_v4 *, - const struct bch_alloc_v4 *, u64, bool); - -/* key/bucket marking: */ - -static inline struct bch_fs_usage *fs_usage_ptr(struct bch_fs *c, - unsigned journal_seq, - bool gc) -{ - percpu_rwsem_assert_held(&c->mark_lock); - BUG_ON(!gc && !journal_seq); - - return this_cpu_ptr(gc - ? c->usage_gc - : c->usage[journal_seq & JOURNAL_BUF_MASK]); -} - -int bch2_update_replicas(struct bch_fs *, struct bkey_s_c, - struct bch_replicas_entry_v1 *, s64, - unsigned, bool); -int bch2_update_replicas_list(struct btree_trans *, - struct bch_replicas_entry_v1 *, s64); -int bch2_update_cached_sectors_list(struct btree_trans *, unsigned, s64); -int bch2_replicas_deltas_realloc(struct btree_trans *, unsigned); - -void bch2_fs_usage_initialize(struct bch_fs *); - int bch2_bucket_ref_update(struct btree_trans *, struct bch_dev *, struct bkey_s_c, const struct bch_extent_ptr *, s64, enum bch_data_type, u8, u8, u32 *); @@ -356,9 +296,6 @@ int bch2_trigger_reservation(struct btree_trans *, enum btree_id, unsigned, void bch2_trans_account_disk_usage_change(struct btree_trans *); -void bch2_trans_fs_usage_revert(struct btree_trans *, struct replicas_delta_list *); -int bch2_trans_fs_usage_apply(struct btree_trans *, struct replicas_delta_list *); - int bch2_trans_mark_metadata_bucket(struct btree_trans *, struct bch_dev *, u64, enum bch_data_type, unsigned, enum btree_iter_update_trigger_flags); @@ -419,13 +356,13 @@ static inline int bch2_disk_reservation_add(struct bch_fs *c, struct disk_reserv #ifdef __KERNEL__ u64 old, new; + old = this_cpu_read(c->pcpu->sectors_available); do { - old = this_cpu_read(c->pcpu->sectors_available); if (sectors > old) return __bch2_disk_reservation_add(c, res, sectors, flags); new = old - sectors; - } while (this_cpu_cmpxchg(c->pcpu->sectors_available, old, new) != old); + } while (!this_cpu_try_cmpxchg(c->pcpu->sectors_available, &old, new)); this_cpu_add(*c->online_reserved, sectors); res->sectors += sectors; diff --git a/libbcachefs/buckets_types.h b/libbcachefs/buckets_types.h index 6a317402..405c8533 100644 --- a/libbcachefs/buckets_types.h +++ b/libbcachefs/buckets_types.h @@ -16,7 +16,8 @@ struct bucket { u32 stripe; u32 dirty_sectors; u32 cached_sectors; -}; + u32 stripe_sectors; +} __aligned(sizeof(long)); struct bucket_array { struct rcu_head rcu; @@ -33,7 +34,7 @@ struct bucket_gens { }; struct bch_dev_usage { - struct { + struct bch_dev_usage_type { u64 buckets; u64 sectors; /* _compressed_ sectors: */ /* @@ -54,18 +55,6 @@ struct bch_fs_usage_base { u64 nr_inodes; }; -struct bch_fs_usage { - /* all fields are in units of 512 byte sectors: */ - struct bch_fs_usage_base b; - u64 persistent_reserved[BCH_REPLICAS_MAX]; - u64 replicas[]; -}; - -struct bch_fs_usage_online { - u64 online_reserved; - struct bch_fs_usage u; -}; - struct bch_fs_usage_short { u64 capacity; u64 used; diff --git a/libbcachefs/chardev.c b/libbcachefs/chardev.c index 9e54323f..70db3a73 100644 --- a/libbcachefs/chardev.c +++ b/libbcachefs/chardev.c @@ -5,6 +5,7 @@ #include "bcachefs_ioctl.h" #include "buckets.h" #include "chardev.h" +#include "disk_accounting.h" #include "journal.h" #include "move.h" #include "recovery_passes.h" @@ -213,9 +214,21 @@ static long bch2_ioctl_fsck_offline(struct bch_ioctl_fsck_offline __user *user_a if (arg.opts) { char *optstr = strndup_user((char __user *)(unsigned long) arg.opts, 1 << 16); + char *ro, *rest; + + /* + * If passed a "read_only" mount option, remove it because it is + * no longer a valid mount option, and the filesystem will be + * set "read_only" regardless. + */ + ro = strstr(optstr, "read_only"); + if (ro) { + rest = ro + strlen("read_only"); + memmove(ro, rest, strlen(rest) + 1); + } ret = PTR_ERR_OR_ZERO(optstr) ?: - bch2_parse_mount_opts(NULL, &thr->opts, optstr); + bch2_parse_mount_opts(NULL, &thr->opts, NULL, optstr); kfree(optstr); if (ret) @@ -223,6 +236,7 @@ static long bch2_ioctl_fsck_offline(struct bch_ioctl_fsck_offline __user *user_a } opt_set(thr->opts, stdio, (u64)(unsigned long)&thr->thr.stdio); + opt_set(thr->opts, read_only, 1); /* We need request_key() to be called before we punt to kthread: */ opt_set(thr->opts, nostart, true); @@ -501,11 +515,9 @@ static long bch2_ioctl_data(struct bch_fs *c, static long bch2_ioctl_fs_usage(struct bch_fs *c, struct bch_ioctl_fs_usage __user *user_arg) { - struct bch_ioctl_fs_usage *arg = NULL; - struct bch_replicas_usage *dst_e, *dst_end; - struct bch_fs_usage_online *src; + struct bch_ioctl_fs_usage arg; + darray_char replicas = {}; u32 replica_entries_bytes; - unsigned i; int ret = 0; if (!test_bit(BCH_FS_started, &c->flags)) @@ -514,62 +526,61 @@ static long bch2_ioctl_fs_usage(struct bch_fs *c, if (get_user(replica_entries_bytes, &user_arg->replica_entries_bytes)) return -EFAULT; - arg = kzalloc(size_add(sizeof(*arg), replica_entries_bytes), GFP_KERNEL); - if (!arg) - return -ENOMEM; - - src = bch2_fs_usage_read(c); - if (!src) { - ret = -ENOMEM; + ret = bch2_fs_replicas_usage_read(c, &replicas) ?: + (replica_entries_bytes < replicas.nr ? -ERANGE : 0) ?: + copy_to_user_errcode(&user_arg->replicas, replicas.data, replicas.nr); + if (ret) goto err; - } - - arg->capacity = c->capacity; - arg->used = bch2_fs_sectors_used(c, src); - arg->online_reserved = src->online_reserved; - - for (i = 0; i < BCH_REPLICAS_MAX; i++) - arg->persistent_reserved[i] = src->u.persistent_reserved[i]; - - dst_e = arg->replicas; - dst_end = (void *) arg->replicas + replica_entries_bytes; - - for (i = 0; i < c->replicas.nr; i++) { - struct bch_replicas_entry_v1 *src_e = - cpu_replicas_entry(&c->replicas, i); - - /* check that we have enough space for one replicas entry */ - if (dst_e + 1 > dst_end) { - ret = -ERANGE; - break; - } - - dst_e->sectors = src->u.replicas[i]; - dst_e->r = *src_e; - /* recheck after setting nr_devs: */ - if (replicas_usage_next(dst_e) > dst_end) { - ret = -ERANGE; - break; - } + struct bch_fs_usage_short u = bch2_fs_usage_read_short(c); + arg.capacity = c->capacity; + arg.used = u.used; + arg.online_reserved = percpu_u64_get(c->online_reserved); + arg.replica_entries_bytes = replicas.nr; - memcpy(dst_e->r.devs, src_e->devs, src_e->nr_devs); + for (unsigned i = 0; i < BCH_REPLICAS_MAX; i++) { + struct disk_accounting_pos k = { + .type = BCH_DISK_ACCOUNTING_persistent_reserved, + .persistent_reserved.nr_replicas = i, + }; - dst_e = replicas_usage_next(dst_e); + bch2_accounting_mem_read(c, + disk_accounting_pos_to_bpos(&k), + &arg.persistent_reserved[i], 1); } - arg->replica_entries_bytes = (void *) dst_e - (void *) arg->replicas; + ret = copy_to_user_errcode(user_arg, &arg, sizeof(arg)); +err: + darray_exit(&replicas); + return ret; +} + +static long bch2_ioctl_query_accounting(struct bch_fs *c, + struct bch_ioctl_query_accounting __user *user_arg) +{ + struct bch_ioctl_query_accounting arg; + darray_char accounting = {}; + int ret = 0; - percpu_up_read(&c->mark_lock); - kfree(src); + if (!test_bit(BCH_FS_started, &c->flags)) + return -EINVAL; + ret = copy_from_user_errcode(&arg, user_arg, sizeof(arg)) ?: + bch2_fs_accounting_read(c, &accounting, arg.accounting_types_mask) ?: + (arg.accounting_u64s * sizeof(u64) < accounting.nr ? -ERANGE : 0) ?: + copy_to_user_errcode(&user_arg->accounting, accounting.data, accounting.nr); if (ret) goto err; - ret = copy_to_user_errcode(user_arg, arg, - sizeof(*arg) + arg->replica_entries_bytes); + arg.capacity = c->capacity; + arg.used = bch2_fs_usage_read_short(c).used; + arg.online_reserved = percpu_u64_get(c->online_reserved); + arg.accounting_u64s = accounting.nr / sizeof(u64); + + ret = copy_to_user_errcode(user_arg, &arg, sizeof(arg)); err: - kfree(arg); + bch_err_fn(c, ret); + darray_exit(&accounting); return ret; } @@ -604,7 +615,7 @@ static long bch2_ioctl_dev_usage(struct bch_fs *c, arg.bucket_size = ca->mi.bucket_size; arg.nr_buckets = ca->mi.nbuckets - ca->mi.first_bucket; - for (i = 0; i < BCH_DATA_NR; i++) { + for (i = 0; i < ARRAY_SIZE(arg.d); i++) { arg.d[i].buckets = src.d[i].buckets; arg.d[i].sectors = src.d[i].sectors; arg.d[i].fragmented = src.d[i].fragmented; @@ -849,7 +860,7 @@ static long bch2_ioctl_fsck_online(struct bch_fs *c, char *optstr = strndup_user((char __user *)(unsigned long) arg.opts, 1 << 16); ret = PTR_ERR_OR_ZERO(optstr) ?: - bch2_parse_mount_opts(c, &thr->opts, optstr); + bch2_parse_mount_opts(c, &thr->opts, NULL, optstr); kfree(optstr); if (ret) @@ -925,6 +936,8 @@ long bch2_fs_ioctl(struct bch_fs *c, unsigned cmd, void __user *arg) BCH_IOCTL(disk_resize_journal, struct bch_ioctl_disk_resize_journal); case BCH_IOCTL_FSCK_ONLINE: BCH_IOCTL(fsck_online, struct bch_ioctl_fsck_online); + case BCH_IOCTL_QUERY_ACCOUNTING: + return bch2_ioctl_query_accounting(c, arg); default: return -ENOTTY; } diff --git a/libbcachefs/dirent.c b/libbcachefs/dirent.c index c67460d8..d743da89 100644 --- a/libbcachefs/dirent.c +++ b/libbcachefs/dirent.c @@ -534,6 +534,14 @@ int bch2_empty_dir_trans(struct btree_trans *trans, subvol_inum dir) static int bch2_dir_emit(struct dir_context *ctx, struct bkey_s_c_dirent d, subvol_inum target) { struct qstr name = bch2_dirent_get_name(d); + /* + * Although not required by the kernel code, updating ctx->pos is needed + * for the bcachefs FUSE driver. Without this update, the FUSE + * implementation will be stuck in an infinite loop when reading + * directories (via the bcachefs_fuse_readdir callback). + * In kernel space, ctx->pos is updated by the VFS code. + */ + ctx->pos = d.k->p.offset; bool ret = dir_emit(ctx, name.name, name.len, target.inum, diff --git a/libbcachefs/disk_accounting.c b/libbcachefs/disk_accounting.c new file mode 100644 index 00000000..48d68db2 --- /dev/null +++ b/libbcachefs/disk_accounting.c @@ -0,0 +1,742 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "bcachefs.h" +#include "bcachefs_ioctl.h" +#include "btree_cache.h" +#include "btree_update.h" +#include "btree_write_buffer.h" +#include "buckets.h" +#include "compress.h" +#include "disk_accounting.h" +#include "error.h" +#include "journal_io.h" +#include "replicas.h" + +/* + * Notes on disk accounting: + * + * We have two parallel sets of counters to be concerned with, and both must be + * kept in sync. + * + * - Persistent/on disk accounting, stored in the accounting btree and updated + * via btree write buffer updates that treat new accounting keys as deltas to + * apply to existing values. But reading from a write buffer btree is + * expensive, so we also have + * + * - In memory accounting, where accounting is stored as an array of percpu + * counters, indexed by an eytzinger array of disk acounting keys/bpos (which + * are the same thing, excepting byte swabbing on big endian). + * + * Cheap to read, but non persistent. + * + * Disk accounting updates are generated by transactional triggers; these run as + * keys enter and leave the btree, and can compare old and new versions of keys; + * the output of these triggers are deltas to the various counters. + * + * Disk accounting updates are done as btree write buffer updates, where the + * counters in the disk accounting key are deltas that will be applied to the + * counter in the btree when the key is flushed by the write buffer (or journal + * replay). + * + * To do a disk accounting update: + * - initialize a disk_accounting_pos, to specify which counter is being update + * - initialize counter deltas, as an array of 1-3 s64s + * - call bch2_disk_accounting_mod() + * + * This queues up the accounting update to be done at transaction commit time. + * Underneath, it's a normal btree write buffer update. + * + * The transaction commit path is responsible for propagating updates to the in + * memory counters, with bch2_accounting_mem_mod(). + * + * The commit path also assigns every disk accounting update a unique version + * number, based on the journal sequence number and offset within that journal + * buffer; this is used by journal replay to determine which updates have been + * done. + * + * The transaction commit path also ensures that replicas entry accounting + * updates are properly marked in the superblock (so that we know whether we can + * mount without data being unavailable); it will update the superblock if + * bch2_accounting_mem_mod() tells it to. + */ + +static const char * const disk_accounting_type_strs[] = { +#define x(t, n, ...) [n] = #t, + BCH_DISK_ACCOUNTING_TYPES() +#undef x + NULL +}; + +static inline void accounting_key_init(struct bkey_i *k, struct disk_accounting_pos pos, + s64 *d, unsigned nr) +{ + struct bkey_i_accounting *acc = bkey_accounting_init(k); + + acc->k.p = disk_accounting_pos_to_bpos(&pos); + set_bkey_val_u64s(&acc->k, sizeof(struct bch_accounting) / sizeof(u64) + nr); + + memcpy_u64s_small(acc->v.d, d, nr); +} + +int bch2_disk_accounting_mod(struct btree_trans *trans, + struct disk_accounting_pos *k, + s64 *d, unsigned nr, bool gc) +{ + /* Normalize: */ + switch (k->type) { + case BCH_DISK_ACCOUNTING_replicas: + bubble_sort(k->replicas.devs, k->replicas.nr_devs, u8_cmp); + break; + } + + BUG_ON(nr > BCH_ACCOUNTING_MAX_COUNTERS); + + struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i; + + accounting_key_init(&k_i.k, *k, d, nr); + + return likely(!gc) + ? bch2_trans_update_buffered(trans, BTREE_ID_accounting, &k_i.k) + : bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true); +} + +int bch2_mod_dev_cached_sectors(struct btree_trans *trans, + unsigned dev, s64 sectors, + bool gc) +{ + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_replicas, + }; + + bch2_replicas_entry_cached(&acc.replicas, dev); + + return bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc); +} + +int bch2_accounting_invalid(struct bch_fs *c, struct bkey_s_c k, + enum bch_validate_flags flags, + struct printbuf *err) +{ + return 0; +} + +void bch2_accounting_key_to_text(struct printbuf *out, struct disk_accounting_pos *k) +{ + if (k->type >= BCH_DISK_ACCOUNTING_TYPE_NR) { + prt_printf(out, "unknown type %u", k->type); + return; + } + + prt_str(out, disk_accounting_type_strs[k->type]); + prt_str(out, " "); + + switch (k->type) { + case BCH_DISK_ACCOUNTING_nr_inodes: + break; + case BCH_DISK_ACCOUNTING_persistent_reserved: + prt_printf(out, "replicas=%u", k->persistent_reserved.nr_replicas); + break; + case BCH_DISK_ACCOUNTING_replicas: + bch2_replicas_entry_to_text(out, &k->replicas); + break; + case BCH_DISK_ACCOUNTING_dev_data_type: + prt_printf(out, "dev=%u data_type=", k->dev_data_type.dev); + bch2_prt_data_type(out, k->dev_data_type.data_type); + break; + case BCH_DISK_ACCOUNTING_compression: + bch2_prt_compression_type(out, k->compression.type); + break; + case BCH_DISK_ACCOUNTING_snapshot: + prt_printf(out, "id=%u", k->snapshot.id); + break; + case BCH_DISK_ACCOUNTING_btree: + prt_printf(out, "btree=%s", bch2_btree_id_str(k->btree.id)); + break; + } +} + +void bch2_accounting_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k) +{ + struct bkey_s_c_accounting acc = bkey_s_c_to_accounting(k); + struct disk_accounting_pos acc_k; + bpos_to_disk_accounting_pos(&acc_k, k.k->p); + + bch2_accounting_key_to_text(out, &acc_k); + + for (unsigned i = 0; i < bch2_accounting_counters(k.k); i++) + prt_printf(out, " %lli", acc.v->d[i]); +} + +void bch2_accounting_swab(struct bkey_s k) +{ + for (u64 *p = (u64 *) k.v; + p < (u64 *) bkey_val_end(k); + p++) + *p = swab64(*p); +} + +static inline bool accounting_to_replicas(struct bch_replicas_entry_v1 *r, struct bpos p) +{ + struct disk_accounting_pos acc_k; + bpos_to_disk_accounting_pos(&acc_k, p); + + switch (acc_k.type) { + case BCH_DISK_ACCOUNTING_replicas: + memcpy(r, &acc_k.replicas, replicas_entry_bytes(&acc_k.replicas)); + return true; + default: + return false; + } +} + +static int bch2_accounting_update_sb_one(struct bch_fs *c, struct bpos p) +{ + struct bch_replicas_padded r; + return accounting_to_replicas(&r.e, p) + ? bch2_mark_replicas(c, &r.e) + : 0; +} + +/* + * Ensure accounting keys being updated are present in the superblock, when + * applicable (i.e. replicas updates) + */ +int bch2_accounting_update_sb(struct btree_trans *trans) +{ + for (struct jset_entry *i = trans->journal_entries; + i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); + i = vstruct_next(i)) + if (jset_entry_is_key(i) && i->start->k.type == KEY_TYPE_accounting) { + int ret = bch2_accounting_update_sb_one(trans->c, i->start->k.p); + if (ret) + return ret; + } + + return 0; +} + +static int __bch2_accounting_mem_mod_slowpath(struct bch_fs *c, struct bkey_s_c_accounting a, bool gc) +{ + struct bch_replicas_padded r; + + if (accounting_to_replicas(&r.e, a.k->p) && + !bch2_replicas_marked_locked(c, &r.e)) + return -BCH_ERR_btree_insert_need_mark_replicas; + + struct bch_accounting_mem *acc = &c->accounting[gc]; + unsigned new_nr_counters = acc->nr_counters + bch2_accounting_counters(a.k); + + u64 __percpu *new_counters = __alloc_percpu_gfp(new_nr_counters * sizeof(u64), + sizeof(u64), GFP_KERNEL); + if (!new_counters) + return -BCH_ERR_ENOMEM_disk_accounting; + + preempt_disable(); + memcpy(this_cpu_ptr(new_counters), + bch2_acc_percpu_u64s(acc->v, acc->nr_counters), + acc->nr_counters * sizeof(u64)); + preempt_enable(); + + struct accounting_pos_offset n = { + .pos = a.k->p, + .version = a.k->version, + .offset = acc->nr_counters, + .nr_counters = bch2_accounting_counters(a.k), + }; + if (darray_push(&acc->k, n)) { + free_percpu(new_counters); + return -BCH_ERR_ENOMEM_disk_accounting; + } + + eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), accounting_pos_cmp, NULL); + + free_percpu(acc->v); + acc->v = new_counters; + acc->nr_counters = new_nr_counters; + + for (unsigned i = 0; i < n.nr_counters; i++) + this_cpu_add(acc->v[n.offset + i], a.v->d[i]); + return 0; +} + +int bch2_accounting_mem_mod_slowpath(struct bch_fs *c, struct bkey_s_c_accounting a, bool gc) +{ + percpu_up_read(&c->mark_lock); + percpu_down_write(&c->mark_lock); + int ret = __bch2_accounting_mem_mod_slowpath(c, a, gc); + percpu_up_write(&c->mark_lock); + percpu_down_read(&c->mark_lock); + return ret; +} + +/* + * Read out accounting keys for replicas entries, as an array of + * bch_replicas_usage entries. + * + * Note: this may be deprecated/removed at smoe point in the future and replaced + * with something more general, it exists to support the ioctl used by the + * 'bcachefs fs usage' command. + */ +int bch2_fs_replicas_usage_read(struct bch_fs *c, darray_char *usage) +{ + struct bch_accounting_mem *acc = &c->accounting[0]; + int ret = 0; + + darray_init(usage); + + percpu_down_read(&c->mark_lock); + darray_for_each(acc->k, i) { + struct { + struct bch_replicas_usage r; + u8 pad[BCH_BKEY_PTRS_MAX]; + } u; + + if (!accounting_to_replicas(&u.r.r, i->pos)) + continue; + + bch2_accounting_mem_read(c, i->pos, &u.r.sectors, 1); + + ret = darray_make_room(usage, replicas_usage_bytes(&u.r)); + if (ret) + break; + + memcpy(&darray_top(*usage), &u.r, replicas_usage_bytes(&u.r)); + usage->nr += replicas_usage_bytes(&u.r); + } + percpu_up_read(&c->mark_lock); + + if (ret) + darray_exit(usage); + return ret; +} + +int bch2_fs_accounting_read(struct bch_fs *c, darray_char *out_buf, unsigned accounting_types_mask) +{ + + struct bch_accounting_mem *acc = &c->accounting[0]; + int ret = 0; + + darray_init(out_buf); + + percpu_down_read(&c->mark_lock); + darray_for_each(acc->k, i) { + struct disk_accounting_pos a_p; + bpos_to_disk_accounting_pos(&a_p, i->pos); + + if (!(accounting_types_mask & BIT(a_p.type))) + continue; + + ret = darray_make_room(out_buf, sizeof(struct bkey_i_accounting) + + sizeof(u64) * i->nr_counters); + if (ret) + break; + + struct bkey_i_accounting *a_out = + bkey_accounting_init((void *) &darray_top(*out_buf)); + set_bkey_val_u64s(&a_out->k, i->nr_counters); + a_out->k.p = i->pos; + bch2_accounting_mem_read(c, i->pos, a_out->v.d, i->nr_counters); + + if (!bch2_accounting_key_is_zero(accounting_i_to_s_c(a_out))) + out_buf->nr += bkey_bytes(&a_out->k); + } + + percpu_up_read(&c->mark_lock); + + if (ret) + darray_exit(out_buf); + return ret; +} + +void bch2_fs_accounting_to_text(struct printbuf *out, struct bch_fs *c) +{ + struct bch_accounting_mem *acc = &c->accounting[0]; + + percpu_down_read(&c->mark_lock); + out->atomic++; + + eytzinger0_for_each(i, acc->k.nr) { + struct disk_accounting_pos acc_k; + bpos_to_disk_accounting_pos(&acc_k, acc->k.data[i].pos); + + bch2_accounting_key_to_text(out, &acc_k); + + u64 v[BCH_ACCOUNTING_MAX_COUNTERS]; + bch2_accounting_mem_read_counters(c, i, v, ARRAY_SIZE(v), false); + + prt_str(out, ":"); + for (unsigned j = 0; j < acc->k.data[i].nr_counters; j++) + prt_printf(out, " %llu", v[j]); + prt_newline(out); + } + + --out->atomic; + percpu_up_read(&c->mark_lock); +} + +/* Ensures all counters in @src exist in @dst: */ +static int copy_counters(struct bch_accounting_mem *dst, + struct bch_accounting_mem *src) +{ + unsigned orig_dst_k_nr = dst->k.nr; + unsigned dst_counters = dst->nr_counters; + + darray_for_each(src->k, i) + if (eytzinger0_find(dst->k.data, orig_dst_k_nr, sizeof(dst->k.data[0]), + accounting_pos_cmp, &i->pos) >= orig_dst_k_nr) { + if (darray_push(&dst->k, ((struct accounting_pos_offset) { + .pos = i->pos, + .offset = dst_counters, + .nr_counters = i->nr_counters }))) + goto err; + + dst_counters += i->nr_counters; + } + + if (dst->k.nr == orig_dst_k_nr) + return 0; + + u64 __percpu *new_counters = __alloc_percpu_gfp(dst_counters * sizeof(u64), + sizeof(u64), GFP_KERNEL); + if (!new_counters) + goto err; + + preempt_disable(); + memcpy(this_cpu_ptr(new_counters), + bch2_acc_percpu_u64s(dst->v, dst->nr_counters), + dst->nr_counters * sizeof(u64)); + preempt_enable(); + + free_percpu(dst->v); + dst->v = new_counters; + dst->nr_counters = dst_counters; + + eytzinger0_sort(dst->k.data, dst->k.nr, sizeof(dst->k.data[0]), accounting_pos_cmp, NULL); + + return 0; +err: + dst->k.nr = orig_dst_k_nr; + return -BCH_ERR_ENOMEM_disk_accounting; +} + +int bch2_accounting_gc_done(struct bch_fs *c) +{ + struct bch_accounting_mem *dst = &c->accounting[0]; + struct bch_accounting_mem *src = &c->accounting[1]; + struct btree_trans *trans = bch2_trans_get(c); + struct printbuf buf = PRINTBUF; + int ret = 0; + + percpu_down_write(&c->mark_lock); + + ret = copy_counters(dst, src) ?: + copy_counters(src, dst); + if (ret) + goto err; + + BUG_ON(dst->k.nr != src->k.nr); + + for (unsigned i = 0; i < src->k.nr; i++) { + BUG_ON(src->k.data[i].nr_counters != dst->k.data[i].nr_counters); + BUG_ON(!bpos_eq(dst->k.data[i].pos, src->k.data[i].pos)); + + struct disk_accounting_pos acc_k; + bpos_to_disk_accounting_pos(&acc_k, src->k.data[i].pos); + + unsigned nr = src->k.data[i].nr_counters; + u64 src_v[BCH_ACCOUNTING_MAX_COUNTERS]; + u64 dst_v[BCH_ACCOUNTING_MAX_COUNTERS]; + + bch2_accounting_mem_read_counters(c, i, dst_v, nr, false); + bch2_accounting_mem_read_counters(c, i, src_v, nr, true); + + if (memcmp(dst_v, src_v, nr * sizeof(u64))) { + printbuf_reset(&buf); + prt_str(&buf, "accounting mismatch for "); + bch2_accounting_key_to_text(&buf, &acc_k); + + prt_str(&buf, ": got"); + for (unsigned j = 0; j < nr; j++) + prt_printf(&buf, " %llu", dst_v[j]); + + prt_str(&buf, " should be"); + for (unsigned j = 0; j < nr; j++) + prt_printf(&buf, " %llu", src_v[j]); + + for (unsigned j = 0; j < nr; j++) + src_v[j] -= dst_v[j]; + + if (fsck_err(trans, accounting_mismatch, "%s", buf.buf)) { + ret = commit_do(trans, NULL, NULL, 0, + bch2_disk_accounting_mod(trans, &acc_k, src_v, nr, false)); + if (ret) + goto err; + + if (!test_bit(BCH_FS_may_go_rw, &c->flags)) { + memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta)); + struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i; + + accounting_key_init(&k_i.k, acc_k, src_v, nr); + bch2_accounting_mem_mod_locked(trans, bkey_i_to_s_c_accounting(&k_i.k), false); + + preempt_disable(); + struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage); + struct bch_fs_usage_base *src = &trans->fs_usage_delta; + acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64)); + preempt_enable(); + } + } + } + } +err: +fsck_err: + percpu_up_write(&c->mark_lock); + printbuf_exit(&buf); + bch2_trans_put(trans); + bch_err_fn(c, ret); + return ret; +} + +static int accounting_read_key(struct bch_fs *c, struct btree_trans *trans, struct bkey_s_c k) +{ + struct printbuf buf = PRINTBUF; + + if (k.k->type != KEY_TYPE_accounting) + return 0; + + percpu_down_read(&c->mark_lock); + int ret = __bch2_accounting_mem_mod(c, bkey_s_c_to_accounting(k), false); + percpu_up_read(&c->mark_lock); + + if (bch2_accounting_key_is_zero(bkey_s_c_to_accounting(k)) && + ret == -BCH_ERR_btree_insert_need_mark_replicas) + ret = 0; + + struct disk_accounting_pos acc; + bpos_to_disk_accounting_pos(&acc, k.k->p); + + if (fsck_err_on(ret == -BCH_ERR_btree_insert_need_mark_replicas, + trans, accounting_replicas_not_marked, + "accounting not marked in superblock replicas\n %s", + (bch2_accounting_key_to_text(&buf, &acc), + buf.buf))) + ret = bch2_accounting_update_sb_one(c, k.k->p); +fsck_err: + printbuf_exit(&buf); + return ret; +} + +/* + * At startup time, initialize the in memory accounting from the btree (and + * journal) + */ +int bch2_accounting_read(struct bch_fs *c) +{ + struct bch_accounting_mem *acc = &c->accounting[0]; + + int ret = bch2_trans_run(c, + for_each_btree_key(trans, iter, + BTREE_ID_accounting, POS_MIN, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, ({ + struct bkey u; + struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, &iter), &u); + accounting_read_key(c, trans, k); + }))); + if (ret) + goto err; + + struct journal_keys *keys = &c->journal_keys; + move_gap(keys, keys->nr); + darray_for_each(*keys, i) { + if (i->k->k.type == KEY_TYPE_accounting) { + struct bkey_s_c k = bkey_i_to_s_c(i->k); + unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr, + sizeof(acc->k.data[0]), + accounting_pos_cmp, &k.k->p); + + bool applied = idx < acc->k.nr && + bversion_cmp(acc->k.data[idx].version, k.k->version) >= 0; + + if (applied) + continue; + + ret = accounting_read_key(c, NULL, k); + if (ret) + goto err; + } + } + + percpu_down_read(&c->mark_lock); + preempt_disable(); + struct bch_fs_usage_base *usage = this_cpu_ptr(c->usage); + + for (unsigned i = 0; i < acc->k.nr; i++) { + struct disk_accounting_pos k; + bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos); + + u64 v[BCH_ACCOUNTING_MAX_COUNTERS]; + bch2_accounting_mem_read_counters(c, i, v, ARRAY_SIZE(v), false); + + switch (k.type) { + case BCH_DISK_ACCOUNTING_persistent_reserved: + usage->reserved += v[0] * k.persistent_reserved.nr_replicas; + break; + case BCH_DISK_ACCOUNTING_replicas: + fs_usage_data_type_to_base(usage, k.replicas.data_type, v[0]); + break; + case BCH_DISK_ACCOUNTING_dev_data_type: + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, k.dev_data_type.dev); + if (ca) { + struct bch_dev_usage_type __percpu *d = &ca->usage->d[k.dev_data_type.data_type]; + percpu_u64_set(&d->buckets, v[0]); + percpu_u64_set(&d->sectors, v[1]); + percpu_u64_set(&d->fragmented, v[2]); + + if (k.dev_data_type.data_type == BCH_DATA_sb || + k.dev_data_type.data_type == BCH_DATA_journal) + usage->hidden += v[0] * ca->mi.bucket_size; + } + rcu_read_unlock(); + break; + } + } + preempt_enable(); + percpu_up_read(&c->mark_lock); +err: + bch_err_fn(c, ret); + return ret; +} + +int bch2_dev_usage_remove(struct bch_fs *c, unsigned dev) +{ + return bch2_trans_run(c, + bch2_btree_write_buffer_flush_sync(trans) ?: + for_each_btree_key_commit(trans, iter, BTREE_ID_accounting, POS_MIN, + BTREE_ITER_all_snapshots, k, NULL, NULL, 0, ({ + struct disk_accounting_pos acc; + bpos_to_disk_accounting_pos(&acc, k.k->p); + + acc.type == BCH_DISK_ACCOUNTING_dev_data_type && + acc.dev_data_type.dev == dev + ? bch2_btree_bit_mod_buffered(trans, BTREE_ID_accounting, k.k->p, 0) + : 0; + })) ?: + bch2_btree_write_buffer_flush_sync(trans)); +} + +int bch2_dev_usage_init(struct bch_dev *ca, bool gc) +{ + struct bch_fs *c = ca->fs; + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_dev_data_type, + .dev_data_type.dev = ca->dev_idx, + .dev_data_type.data_type = BCH_DATA_free, + }; + u64 v[3] = { ca->mi.nbuckets - ca->mi.first_bucket, 0, 0 }; + + int ret = bch2_trans_do(c, NULL, NULL, 0, + bch2_disk_accounting_mod(trans, &acc, v, ARRAY_SIZE(v), gc)); + bch_err_fn(c, ret); + return ret; +} + +void bch2_verify_accounting_clean(struct bch_fs *c) +{ + bool mismatch = false; + struct bch_fs_usage_base base = {}, base_inmem = {}; + + bch2_trans_run(c, + for_each_btree_key(trans, iter, + BTREE_ID_accounting, POS_MIN, + BTREE_ITER_all_snapshots, k, ({ + u64 v[BCH_ACCOUNTING_MAX_COUNTERS]; + struct bkey_s_c_accounting a = bkey_s_c_to_accounting(k); + unsigned nr = bch2_accounting_counters(k.k); + + bch2_accounting_mem_read(c, k.k->p, v, nr); + + if (memcmp(a.v->d, v, nr * sizeof(u64))) { + struct printbuf buf = PRINTBUF; + + bch2_bkey_val_to_text(&buf, c, k); + prt_str(&buf, " !="); + for (unsigned j = 0; j < nr; j++) + prt_printf(&buf, " %llu", v[j]); + + pr_err("%s", buf.buf); + printbuf_exit(&buf); + mismatch = true; + } + + struct disk_accounting_pos acc_k; + bpos_to_disk_accounting_pos(&acc_k, a.k->p); + + switch (acc_k.type) { + case BCH_DISK_ACCOUNTING_persistent_reserved: + base.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0]; + break; + case BCH_DISK_ACCOUNTING_replicas: + fs_usage_data_type_to_base(&base, acc_k.replicas.data_type, a.v->d[0]); + break; + case BCH_DISK_ACCOUNTING_dev_data_type: { + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, acc_k.dev_data_type.dev); + if (!ca) { + rcu_read_unlock(); + continue; + } + + v[0] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].buckets); + v[1] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].sectors); + v[2] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].fragmented); + rcu_read_unlock(); + + if (memcmp(a.v->d, v, 3 * sizeof(u64))) { + struct printbuf buf = PRINTBUF; + + bch2_bkey_val_to_text(&buf, c, k); + prt_str(&buf, " in mem"); + for (unsigned j = 0; j < nr; j++) + prt_printf(&buf, " %llu", v[j]); + + pr_err("dev accounting mismatch: %s", buf.buf); + printbuf_exit(&buf); + mismatch = true; + } + } + } + + 0; + }))); + + acc_u64s_percpu(&base_inmem.hidden, &c->usage->hidden, sizeof(base_inmem) / sizeof(u64)); + +#define check(x) \ + if (base.x != base_inmem.x) { \ + pr_err("fs_usage_base.%s mismatch: %llu != %llu", #x, base.x, base_inmem.x); \ + mismatch = true; \ + } + + //check(hidden); + check(btree); + check(data); + check(cached); + check(reserved); + check(nr_inodes); + + WARN_ON(mismatch); +} + +void bch2_accounting_free(struct bch_accounting_mem *acc) +{ + darray_exit(&acc->k); + free_percpu(acc->v); + acc->v = NULL; + acc->nr_counters = 0; +} + +void bch2_fs_accounting_exit(struct bch_fs *c) +{ + bch2_accounting_free(&c->accounting[0]); +} diff --git a/libbcachefs/disk_accounting.h b/libbcachefs/disk_accounting.h new file mode 100644 index 00000000..ab1f74cb --- /dev/null +++ b/libbcachefs/disk_accounting.h @@ -0,0 +1,211 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BCACHEFS_DISK_ACCOUNTING_H +#define _BCACHEFS_DISK_ACCOUNTING_H + +#include "eytzinger.h" +#include "sb-members.h" + +static inline void bch2_u64s_neg(u64 *v, unsigned nr) +{ + for (unsigned i = 0; i < nr; i++) + v[i] = -v[i]; +} + +static inline unsigned bch2_accounting_counters(const struct bkey *k) +{ + return bkey_val_u64s(k) - offsetof(struct bch_accounting, d) / sizeof(u64); +} + +static inline void bch2_accounting_neg(struct bkey_s_accounting a) +{ + bch2_u64s_neg(a.v->d, bch2_accounting_counters(a.k)); +} + +static inline bool bch2_accounting_key_is_zero(struct bkey_s_c_accounting a) +{ + for (unsigned i = 0; i < bch2_accounting_counters(a.k); i++) + if (a.v->d[i]) + return false; + return true; +} + +static inline void bch2_accounting_accumulate(struct bkey_i_accounting *dst, + struct bkey_s_c_accounting src) +{ + EBUG_ON(dst->k.u64s != src.k->u64s); + + for (unsigned i = 0; i < bch2_accounting_counters(&dst->k); i++) + dst->v.d[i] += src.v->d[i]; + if (bversion_cmp(dst->k.version, src.k->version) < 0) + dst->k.version = src.k->version; +} + +static inline void fs_usage_data_type_to_base(struct bch_fs_usage_base *fs_usage, + enum bch_data_type data_type, + s64 sectors) +{ + switch (data_type) { + case BCH_DATA_btree: + fs_usage->btree += sectors; + break; + case BCH_DATA_user: + case BCH_DATA_parity: + fs_usage->data += sectors; + break; + case BCH_DATA_cached: + fs_usage->cached += sectors; + break; + default: + break; + } +} + +static inline void bpos_to_disk_accounting_pos(struct disk_accounting_pos *acc, struct bpos p) +{ + acc->_pad = p; +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + bch2_bpos_swab(&acc->_pad); +#endif +} + +static inline struct bpos disk_accounting_pos_to_bpos(struct disk_accounting_pos *k) +{ + struct bpos ret = k->_pad; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + bch2_bpos_swab(&ret); +#endif + return ret; +} + +int bch2_disk_accounting_mod(struct btree_trans *, struct disk_accounting_pos *, + s64 *, unsigned, bool); +int bch2_mod_dev_cached_sectors(struct btree_trans *, unsigned, s64, bool); + +int bch2_accounting_invalid(struct bch_fs *, struct bkey_s_c, + enum bch_validate_flags, struct printbuf *); +void bch2_accounting_key_to_text(struct printbuf *, struct disk_accounting_pos *); +void bch2_accounting_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); +void bch2_accounting_swab(struct bkey_s); + +#define bch2_bkey_ops_accounting ((struct bkey_ops) { \ + .key_invalid = bch2_accounting_invalid, \ + .val_to_text = bch2_accounting_to_text, \ + .swab = bch2_accounting_swab, \ + .min_val_size = 8, \ +}) + +int bch2_accounting_update_sb(struct btree_trans *); + +static inline int accounting_pos_cmp(const void *_l, const void *_r) +{ + const struct bpos *l = _l, *r = _r; + + return bpos_cmp(*l, *r); +} + +int bch2_accounting_mem_mod_slowpath(struct bch_fs *, struct bkey_s_c_accounting, bool); + +static inline int __bch2_accounting_mem_mod(struct bch_fs *c, struct bkey_s_c_accounting a, bool gc) +{ + struct bch_accounting_mem *acc = &c->accounting[gc]; + unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), + accounting_pos_cmp, &a.k->p); + if (unlikely(idx >= acc->k.nr)) + return bch2_accounting_mem_mod_slowpath(c, a, gc); + + unsigned offset = acc->k.data[idx].offset; + + EBUG_ON(bch2_accounting_counters(a.k) != acc->k.data[idx].nr_counters); + + for (unsigned i = 0; i < bch2_accounting_counters(a.k); i++) + this_cpu_add(acc->v[offset + i], a.v->d[i]); + return 0; +} + +/* + * Update in memory counters so they match the btree update we're doing; called + * from transaction commit path + */ +static inline int bch2_accounting_mem_mod_locked(struct btree_trans *trans, struct bkey_s_c_accounting a, bool gc) +{ + struct bch_fs *c = trans->c; + + if (!gc) { + struct disk_accounting_pos acc_k; + bpos_to_disk_accounting_pos(&acc_k, a.k->p); + + switch (acc_k.type) { + case BCH_DISK_ACCOUNTING_persistent_reserved: + trans->fs_usage_delta.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0]; + break; + case BCH_DISK_ACCOUNTING_replicas: + fs_usage_data_type_to_base(&trans->fs_usage_delta, acc_k.replicas.data_type, a.v->d[0]); + break; + case BCH_DISK_ACCOUNTING_dev_data_type: + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, acc_k.dev_data_type.dev); + if (ca) { + this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].buckets, a.v->d[0]); + this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].sectors, a.v->d[1]); + this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].fragmented, a.v->d[2]); + } + rcu_read_unlock(); + break; + } + } + + return __bch2_accounting_mem_mod(c, a, gc); +} + +static inline int bch2_accounting_mem_add(struct btree_trans *trans, struct bkey_s_c_accounting a, bool gc) +{ + percpu_down_read(&trans->c->mark_lock); + int ret = bch2_accounting_mem_mod_locked(trans, a, gc); + percpu_up_read(&trans->c->mark_lock); + return ret; +} + +static inline void bch2_accounting_mem_read_counters(struct bch_fs *c, unsigned idx, + u64 *v, unsigned nr, bool gc) +{ + memset(v, 0, sizeof(*v) * nr); + + struct bch_accounting_mem *acc = &c->accounting[gc]; + if (unlikely(idx >= acc->k.nr)) + return; + + unsigned offset = acc->k.data[idx].offset; + nr = min_t(unsigned, nr, acc->k.data[idx].nr_counters); + + for (unsigned i = 0; i < nr; i++) + v[i] = percpu_u64_get(acc->v + offset + i); +} + +static inline void bch2_accounting_mem_read(struct bch_fs *c, struct bpos p, + u64 *v, unsigned nr) +{ + struct bch_accounting_mem *acc = &c->accounting[0]; + unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), + accounting_pos_cmp, &p); + + bch2_accounting_mem_read_counters(c, idx, v, nr, false); +} + +int bch2_fs_replicas_usage_read(struct bch_fs *, darray_char *); +int bch2_fs_accounting_read(struct bch_fs *, darray_char *, unsigned); +void bch2_fs_accounting_to_text(struct printbuf *, struct bch_fs *); + +int bch2_accounting_gc_done(struct bch_fs *); + +int bch2_accounting_read(struct bch_fs *); + +int bch2_dev_usage_remove(struct bch_fs *, unsigned); +int bch2_dev_usage_init(struct bch_dev *, bool); + +void bch2_verify_accounting_clean(struct bch_fs *c); + +void bch2_accounting_free(struct bch_accounting_mem *); +void bch2_fs_accounting_exit(struct bch_fs *); + +#endif /* _BCACHEFS_DISK_ACCOUNTING_H */ diff --git a/libbcachefs/disk_accounting_format.h b/libbcachefs/disk_accounting_format.h new file mode 100644 index 00000000..cba41706 --- /dev/null +++ b/libbcachefs/disk_accounting_format.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BCACHEFS_DISK_ACCOUNTING_FORMAT_H +#define _BCACHEFS_DISK_ACCOUNTING_FORMAT_H + +#include "replicas_format.h" + +/* + * Disk accounting - KEY_TYPE_accounting - on disk format: + * + * Here, the key has considerably more structure than a typical key (bpos); an + * accounting key is 'struct disk_accounting_pos', which is a union of bpos. + * + * More specifically: a key is just a muliword integer (where word endianness + * matches native byte order), so we're treating bpos as an opaque 20 byte + * integer and mapping bch_accounting_key to that. + * + * This is a type-tagged union of all our various subtypes; a disk accounting + * key can be device counters, replicas counters, et cetera - it's extensible. + * + * The value is a list of u64s or s64s; the number of counters is specific to a + * given accounting type. + * + * Unlike with other key types, updates are _deltas_, and the deltas are not + * resolved until the update to the underlying btree, done by btree write buffer + * flush or journal replay. + * + * Journal replay in particular requires special handling. The journal tracks a + * range of entries which may possibly have not yet been applied to the btree + * yet - it does not know definitively whether individual entries are dirty and + * still need to be applied. + * + * To handle this, we use the version field of struct bkey, and give every + * accounting update a unique version number - a total ordering in time; the + * version number is derived from the key's position in the journal. Then + * journal replay can compare the version number of the key from the journal + * with the version number of the key in the btree to determine if a key needs + * to be replayed. + * + * For this to work, we must maintain this strict time ordering of updates as + * they are flushed to the btree, both via write buffer flush and via journal + * replay. This has complications for the write buffer code while journal replay + * is still in progress; the write buffer cannot flush any accounting keys to + * the btree until journal replay has finished replaying its accounting keys, or + * the (newer) version number of the keys from the write buffer will cause + * updates from journal replay to be lost. + */ + +struct bch_accounting { + struct bch_val v; + __u64 d[]; +}; + +#define BCH_ACCOUNTING_MAX_COUNTERS 3 + +#define BCH_DATA_TYPES() \ + x(free, 0) \ + x(sb, 1) \ + x(journal, 2) \ + x(btree, 3) \ + x(user, 4) \ + x(cached, 5) \ + x(parity, 6) \ + x(stripe, 7) \ + x(need_gc_gens, 8) \ + x(need_discard, 9) \ + x(unstriped, 10) + +enum bch_data_type { +#define x(t, n) BCH_DATA_##t, + BCH_DATA_TYPES() +#undef x + BCH_DATA_NR +}; + +static inline bool data_type_is_empty(enum bch_data_type type) +{ + switch (type) { + case BCH_DATA_free: + case BCH_DATA_need_gc_gens: + case BCH_DATA_need_discard: + return true; + default: + return false; + } +} + +static inline bool data_type_is_hidden(enum bch_data_type type) +{ + switch (type) { + case BCH_DATA_sb: + case BCH_DATA_journal: + return true; + default: + return false; + } +} + +#define BCH_DISK_ACCOUNTING_TYPES() \ + x(nr_inodes, 0) \ + x(persistent_reserved, 1) \ + x(replicas, 2) \ + x(dev_data_type, 3) \ + x(compression, 4) \ + x(snapshot, 5) \ + x(btree, 6) \ + x(rebalance_work, 7) + +enum disk_accounting_type { +#define x(f, nr) BCH_DISK_ACCOUNTING_##f = nr, + BCH_DISK_ACCOUNTING_TYPES() +#undef x + BCH_DISK_ACCOUNTING_TYPE_NR, +}; + +struct bch_nr_inodes { +}; + +struct bch_persistent_reserved { + __u8 nr_replicas; +}; + +struct bch_dev_data_type { + __u8 dev; + __u8 data_type; +}; + +struct bch_dev_stripe_buckets { + __u8 dev; +}; + +struct bch_acct_compression { + __u8 type; +}; + +struct bch_acct_snapshot { + __u32 id; +}; + +struct bch_acct_btree { + __u32 id; +}; + +struct disk_accounting_pos { + union { + struct { + __u8 type; + union { + struct bch_nr_inodes nr_inodes; + struct bch_persistent_reserved persistent_reserved; + struct bch_replicas_entry_v1 replicas; + struct bch_dev_data_type dev_data_type; + struct bch_dev_stripe_buckets dev_stripe_buckets; + struct bch_acct_compression compression; + struct bch_acct_snapshot snapshot; + struct bch_acct_btree btree; + }; + }; + struct bpos _pad; + }; +}; + +#endif /* _BCACHEFS_DISK_ACCOUNTING_FORMAT_H */ diff --git a/libbcachefs/disk_accounting_types.h b/libbcachefs/disk_accounting_types.h new file mode 100644 index 00000000..5656ac54 --- /dev/null +++ b/libbcachefs/disk_accounting_types.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BCACHEFS_DISK_ACCOUNTING_TYPES_H +#define _BCACHEFS_DISK_ACCOUNTING_TYPES_H + +#include "darray.h" + +struct accounting_pos_offset { + struct bpos pos; + struct bversion version; + u32 offset:24, + nr_counters:8; +}; + +struct bch_accounting_mem { + DARRAY(struct accounting_pos_offset) k; + u64 __percpu *v; + unsigned nr_counters; +}; + +#endif /* _BCACHEFS_DISK_ACCOUNTING_TYPES_H */ diff --git a/libbcachefs/disk_groups.c b/libbcachefs/disk_groups.c index 521a86df..5df8de0b 100644 --- a/libbcachefs/disk_groups.c +++ b/libbcachefs/disk_groups.c @@ -511,7 +511,7 @@ int bch2_opt_target_parse(struct bch_fs *c, const char *val, u64 *res, return -EINVAL; if (!c) - return 0; + return -BCH_ERR_option_needs_open_fs; if (!strlen(val) || !strcmp(val, "none")) { *res = 0; diff --git a/libbcachefs/ec.c b/libbcachefs/ec.c index d8b9beca..f254a53c 100644 --- a/libbcachefs/ec.c +++ b/libbcachefs/ec.c @@ -13,6 +13,7 @@ #include "btree_write_buffer.h" #include "buckets.h" #include "checksum.h" +#include "disk_accounting.h" #include "disk_groups.h" #include "ec.h" #include "error.h" @@ -292,12 +293,11 @@ static int mark_stripe_bucket(struct btree_trans *trans, bucket_lock(g); struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old; ret = __mark_stripe_bucket(trans, ca, s, ptr_idx, deleting, bucket, &new, flags); - if (!ret) { - alloc_to_bucket(g, new); - bch2_dev_usage_update(c, ca, &old, &new, 0, true); - } + alloc_to_bucket(g, new); bucket_unlock(g); percpu_up_read(&c->mark_lock); + if (!ret) + ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags); } err: bch2_dev_put(ca); @@ -358,7 +358,12 @@ int bch2_trigger_stripe(struct btree_trans *trans, if (unlikely(flags & BTREE_TRIGGER_check_repair)) return bch2_check_fix_ptrs(trans, btree, level, _new.s_c, flags); - if (flags & BTREE_TRIGGER_transactional) { + BUG_ON(new_s && old_s && + (new_s->nr_blocks != old_s->nr_blocks || + new_s->nr_redundant != old_s->nr_redundant)); + + + if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) { /* * If the pointers aren't changing, we don't need to do anything: */ @@ -369,26 +374,58 @@ int bch2_trigger_stripe(struct btree_trans *trans, new_s->nr_blocks * sizeof(struct bch_extent_ptr))) return 0; - BUG_ON(new_s && old_s && - (new_s->nr_blocks != old_s->nr_blocks || - new_s->nr_redundant != old_s->nr_redundant)); + struct gc_stripe *gc = NULL; + if (flags & BTREE_TRIGGER_gc) { + gc = genradix_ptr_alloc(&c->gc_stripes, idx, GFP_KERNEL); + if (!gc) { + bch_err(c, "error allocating memory for gc_stripes, idx %llu", idx); + return -BCH_ERR_ENOMEM_mark_stripe; + } + + /* + * This will be wrong when we bring back runtime gc: we should + * be unmarking the old key and then marking the new key + * + * Also: when we bring back runtime gc, locking + */ + gc->alive = true; + gc->sectors = le16_to_cpu(new_s->sectors); + gc->nr_blocks = new_s->nr_blocks; + gc->nr_redundant = new_s->nr_redundant; + + for (unsigned i = 0; i < new_s->nr_blocks; i++) + gc->ptrs[i] = new_s->ptrs[i]; + + /* + * gc recalculates this field from stripe ptr + * references: + */ + memset(gc->block_sectors, 0, sizeof(gc->block_sectors)); + } if (new_s) { - s64 sectors = le16_to_cpu(new_s->sectors); + s64 sectors = (u64) le16_to_cpu(new_s->sectors) * new_s->nr_redundant; - struct bch_replicas_padded r; - bch2_bkey_to_replicas(&r.e, new); - int ret = bch2_update_replicas_list(trans, &r.e, sectors * new_s->nr_redundant); + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_replicas, + }; + bch2_bkey_to_replicas(&acc.replicas, new); + int ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc); if (ret) return ret; + + if (gc) + memcpy(&gc->r.e, &acc.replicas, replicas_entry_bytes(&acc.replicas)); } if (old_s) { - s64 sectors = -((s64) le16_to_cpu(old_s->sectors)); + s64 sectors = -((s64) le16_to_cpu(old_s->sectors)) * old_s->nr_redundant; - struct bch_replicas_padded r; - bch2_bkey_to_replicas(&r.e, old); - int ret = bch2_update_replicas_list(trans, &r.e, sectors * old_s->nr_redundant); + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_replicas, + }; + bch2_bkey_to_replicas(&acc.replicas, old); + int ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc); if (ret) return ret; } @@ -437,52 +474,6 @@ int bch2_trigger_stripe(struct btree_trans *trans, } } - if (flags & BTREE_TRIGGER_gc) { - struct gc_stripe *m = - genradix_ptr_alloc(&c->gc_stripes, idx, GFP_KERNEL); - - if (!m) { - bch_err(c, "error allocating memory for gc_stripes, idx %llu", - idx); - return -BCH_ERR_ENOMEM_mark_stripe; - } - /* - * This will be wrong when we bring back runtime gc: we should - * be unmarking the old key and then marking the new key - */ - m->alive = true; - m->sectors = le16_to_cpu(new_s->sectors); - m->nr_blocks = new_s->nr_blocks; - m->nr_redundant = new_s->nr_redundant; - - for (unsigned i = 0; i < new_s->nr_blocks; i++) - m->ptrs[i] = new_s->ptrs[i]; - - bch2_bkey_to_replicas(&m->r.e, new); - - /* - * gc recalculates this field from stripe ptr - * references: - */ - memset(m->block_sectors, 0, sizeof(m->block_sectors)); - - int ret = mark_stripe_buckets(trans, old, new, flags); - if (ret) - return ret; - - ret = bch2_update_replicas(c, new, &m->r.e, - ((s64) m->sectors * m->nr_redundant), - 0, true); - if (ret) { - struct printbuf buf = PRINTBUF; - - bch2_bkey_val_to_text(&buf, c, new); - bch2_fs_fatal_error(c, ": no replicas entry for %s", buf.buf); - printbuf_exit(&buf); - return ret; - } - } - return 0; } diff --git a/libbcachefs/errcode.h b/libbcachefs/errcode.h index dbe35b80..a268af3e 100644 --- a/libbcachefs/errcode.h +++ b/libbcachefs/errcode.h @@ -116,6 +116,9 @@ x(ENOENT, ENOENT_dev_idx_not_found) \ x(ENOTEMPTY, ENOTEMPTY_dir_not_empty) \ x(ENOTEMPTY, ENOTEMPTY_subvol_not_empty) \ + x(EEXIST, EEXIST_str_hash_set) \ + x(EEXIST, EEXIST_discard_in_flight_add) \ + x(EEXIST, EEXIST_subvolume_create) \ x(0, open_buckets_empty) \ x(0, freelist_empty) \ x(BCH_ERR_freelist_empty, no_buckets_found) \ @@ -254,7 +257,8 @@ x(BCH_ERR_nopromote, nopromote_no_writes) \ x(BCH_ERR_nopromote, nopromote_enomem) \ x(0, need_inode_lock) \ - x(0, invalid_snapshot_node) + x(0, invalid_snapshot_node) \ + x(0, option_needs_open_fs) enum bch_errcode { BCH_ERR_START = 2048, diff --git a/libbcachefs/error.c b/libbcachefs/error.c index c66eeffc..285b7251 100644 --- a/libbcachefs/error.c +++ b/libbcachefs/error.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include "bcachefs.h" +#include "btree_iter.h" #include "error.h" #include "journal.h" #include "recovery_passes.h" @@ -97,7 +98,7 @@ static enum ask_yn parse_yn_response(char *buf) } #ifdef __KERNEL__ -static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c) +static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans) { struct stdio_redirect *stdio = c->stdio; @@ -107,25 +108,44 @@ static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c) if (!stdio) return YN_NO; - char buf[100]; + if (trans) + bch2_trans_unlock(trans); + + unsigned long unlock_long_at = trans ? jiffies + HZ * 2 : 0; + darray_char line = {}; int ret; do { + unsigned long t; bch2_print(c, " (y,n, or Y,N for all errors of this type) "); +rewait: + t = unlock_long_at + ? max_t(long, unlock_long_at - jiffies, 0) + : MAX_SCHEDULE_TIMEOUT; + + int r = bch2_stdio_redirect_readline_timeout(stdio, &line, t); + if (r == -ETIME) { + bch2_trans_unlock_long(trans); + unlock_long_at = 0; + goto rewait; + } - int r = bch2_stdio_redirect_readline(stdio, buf, sizeof(buf) - 1); - if (r < 0) - return YN_NO; - buf[r] = '\0'; - } while ((ret = parse_yn_response(buf)) < 0); + if (r < 0) { + ret = YN_NO; + break; + } + darray_last(line) = '\0'; + } while ((ret = parse_yn_response(line.data)) < 0); + + darray_exit(&line); return ret; } #else #include "tools-util.h" -static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c) +static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans) { char *buf = NULL; size_t buflen = 0; @@ -191,7 +211,8 @@ static void prt_actioning(struct printbuf *out, const char *action) prt_str(out, "ing"); } -int bch2_fsck_err(struct bch_fs *c, +int __bch2_fsck_err(struct bch_fs *c, + struct btree_trans *trans, enum bch_fsck_flags flags, enum bch_sb_error_id err, const char *fmt, ...) @@ -203,6 +224,11 @@ int bch2_fsck_err(struct bch_fs *c, int ret = -BCH_ERR_fsck_ignore; const char *action_orig = "fix?", *action = action_orig; + if (!c) + c = trans->c; + + WARN_ON(!trans && bch2_current_has_btree_trans(c)); + if ((flags & FSCK_CAN_FIX) && test_bit(err, c->sb.errors_silent)) return -BCH_ERR_fsck_fix; @@ -297,7 +323,15 @@ int bch2_fsck_err(struct bch_fs *c, bch2_print_string_as_lines(KERN_ERR, out->buf); print = false; - int ask = bch2_fsck_ask_yn(c); + int ask = bch2_fsck_ask_yn(c, trans); + + if (trans) { + ret = bch2_trans_relock(trans); + if (ret) { + mutex_unlock(&c->fsck_error_msgs_lock); + goto err; + } + } if (ask >= YN_ALLNO && s) s->fix = ask == YN_ALLNO diff --git a/libbcachefs/error.h b/libbcachefs/error.h index 36caedf7..157ec3fa 100644 --- a/libbcachefs/error.h +++ b/libbcachefs/error.h @@ -117,18 +117,21 @@ enum bch_fsck_flags { #define fsck_err_count(_c, _err) bch2_sb_err_count(_c, BCH_FSCK_ERR_##_err) -__printf(4, 5) __cold -int bch2_fsck_err(struct bch_fs *, +__printf(5, 6) __cold +int __bch2_fsck_err(struct bch_fs *, struct btree_trans *, enum bch_fsck_flags, enum bch_sb_error_id, const char *, ...); +#define bch2_fsck_err(c, _flags, _err_type, ...) \ + __bch2_fsck_err(type_is(c, struct bch_fs *) ? (struct bch_fs *) c : NULL,\ + type_is(c, struct btree_trans *) ? (struct btree_trans *) c : NULL,\ + _flags, BCH_FSCK_ERR_##_err_type, __VA_ARGS__) + void bch2_flush_fsck_errs(struct bch_fs *); #define __fsck_err(c, _flags, _err_type, ...) \ ({ \ - int _ret = bch2_fsck_err(c, _flags, BCH_FSCK_ERR_##_err_type, \ - __VA_ARGS__); \ - \ + int _ret = bch2_fsck_err(c, _flags, _err_type, __VA_ARGS__); \ if (_ret != -BCH_ERR_fsck_fix && \ _ret != -BCH_ERR_fsck_ignore) { \ ret = _ret; \ @@ -143,7 +146,12 @@ void bch2_flush_fsck_errs(struct bch_fs *); /* XXX: mark in superblock that filesystem contains errors, if we ignore: */ #define __fsck_err_on(cond, c, _flags, _err_type, ...) \ - (unlikely(cond) ? __fsck_err(c, _flags, _err_type, __VA_ARGS__) : false) +({ \ + if (type_is(c, struct bch_fs *)) \ + WARN_ON(bch2_current_has_btree_trans((struct bch_fs *) c));\ + \ + (unlikely(cond) ? __fsck_err(c, _flags, _err_type, __VA_ARGS__) : false);\ +}) #define need_fsck_err_on(cond, c, _err_type, ...) \ __fsck_err_on(cond, c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, _err_type, __VA_ARGS__) diff --git a/libbcachefs/fs-io-direct.c b/libbcachefs/fs-io-direct.c index 09d21aef..049b61bc 100644 --- a/libbcachefs/fs-io-direct.c +++ b/libbcachefs/fs-io-direct.c @@ -609,8 +609,10 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) if (unlikely(ret)) goto err_put_write_ref; - if (unlikely((req->ki_pos|iter->count) & (block_bytes(c) - 1))) + if (unlikely((req->ki_pos|iter->count) & (block_bytes(c) - 1))) { + ret = -EINVAL; goto err_put_write_ref; + } inode_dio_begin(&inode->v); bch2_pagecache_block_get(inode); diff --git a/libbcachefs/fs-io.c b/libbcachefs/fs-io.c index ef20b640..189d408e 100644 --- a/libbcachefs/fs-io.c +++ b/libbcachefs/fs-io.c @@ -192,7 +192,9 @@ int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync) { struct bch_inode_info *inode = file_bch_inode(file); struct bch_fs *c = inode->v.i_sb->s_fs_info; - int ret; + int ret, err; + + trace_bch2_fsync(file, datasync); ret = file_write_and_wait_range(file, start, end); if (ret) @@ -205,6 +207,11 @@ out: ret = bch2_err_class(ret); if (ret == -EROFS) ret = -EIO; + + err = file_check_and_advance_wb_err(file); + if (!ret) + ret = err; + return ret; } @@ -860,9 +867,6 @@ loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src, if (remap_flags & ~(REMAP_FILE_DEDUP|REMAP_FILE_ADVISORY)) return -EINVAL; - if (remap_flags & REMAP_FILE_DEDUP) - return -EOPNOTSUPP; - if ((pos_src & (block_bytes(c) - 1)) || (pos_dst & (block_bytes(c) - 1))) return -EINVAL; @@ -895,7 +899,8 @@ loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src, if (ret) goto err; - file_update_time(file_dst); + if (!(remap_flags & REMAP_FILE_DEDUP)) + file_update_time(file_dst); bch2_mark_pagecache_unallocated(src, pos_src >> 9, (pos_src + aligned_len) >> 9); diff --git a/libbcachefs/fs-ioctl.c b/libbcachefs/fs-ioctl.c index 205a323f..23427a87 100644 --- a/libbcachefs/fs-ioctl.c +++ b/libbcachefs/fs-ioctl.c @@ -272,6 +272,70 @@ err1: return ret; } +static int bch2_ioc_getversion(struct bch_inode_info *inode, u32 __user *arg) +{ + return put_user(inode->v.i_generation, arg); +} + +static int bch2_ioc_getlabel(struct bch_fs *c, char __user *user_label) +{ + int ret; + size_t len; + char label[BCH_SB_LABEL_SIZE]; + + BUILD_BUG_ON(BCH_SB_LABEL_SIZE >= FSLABEL_MAX); + + mutex_lock(&c->sb_lock); + memcpy(label, c->disk_sb.sb->label, BCH_SB_LABEL_SIZE); + mutex_unlock(&c->sb_lock); + + len = strnlen(label, BCH_SB_LABEL_SIZE); + if (len == BCH_SB_LABEL_SIZE) { + bch_warn(c, + "label is too long, return the first %zu bytes", + --len); + } + + ret = copy_to_user(user_label, label, len); + + return ret ? -EFAULT : 0; +} + +static int bch2_ioc_setlabel(struct bch_fs *c, + struct file *file, + struct bch_inode_info *inode, + const char __user *user_label) +{ + int ret; + char label[BCH_SB_LABEL_SIZE]; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(label, user_label, sizeof(label))) + return -EFAULT; + + if (strnlen(label, BCH_SB_LABEL_SIZE) == BCH_SB_LABEL_SIZE) { + bch_err(c, + "unable to set label with more than %d bytes", + BCH_SB_LABEL_SIZE - 1); + return -EINVAL; + } + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + mutex_lock(&c->sb_lock); + strscpy(c->disk_sb.sb->label, label, BCH_SB_LABEL_SIZE); + mutex_unlock(&c->sb_lock); + + ret = bch2_write_super(c); + + mnt_drop_write_file(file); + return ret; +} + static int bch2_ioc_goingdown(struct bch_fs *c, u32 __user *arg) { u32 flags; @@ -373,7 +437,7 @@ retry: } if (dst_dentry->d_inode) { - error = -EEXIST; + error = -BCH_ERR_EEXIST_subvolume_create; goto err3; } @@ -506,13 +570,21 @@ long bch2_fs_file_ioctl(struct file *file, unsigned cmd, unsigned long arg) break; case FS_IOC_GETVERSION: - ret = -ENOTTY; + ret = bch2_ioc_getversion(inode, (u32 __user *) arg); break; case FS_IOC_SETVERSION: ret = -ENOTTY; break; + case FS_IOC_GETFSLABEL: + ret = bch2_ioc_getlabel(c, (void __user *) arg); + break; + + case FS_IOC_SETFSLABEL: + ret = bch2_ioc_setlabel(c, file, inode, (const void __user *) arg); + break; + case FS_IOC_GOINGDOWN: ret = bch2_ioc_goingdown(c, (u32 __user *) arg); break; @@ -554,6 +626,12 @@ long bch2_compat_fs_ioctl(struct file *file, unsigned cmd, unsigned long arg) case FS_IOC32_SETFLAGS: cmd = FS_IOC_SETFLAGS; break; + case FS_IOC32_GETVERSION: + cmd = FS_IOC_GETVERSION; + break; + case FS_IOC_GETFSLABEL: + case FS_IOC_SETFSLABEL: + break; default: return -ENOIOCTLCMD; } diff --git a/libbcachefs/fs.c b/libbcachefs/fs.c index 96040a95..514bf83e 100644 --- a/libbcachefs/fs.c +++ b/libbcachefs/fs.c @@ -26,11 +26,13 @@ #include "snapshot.h" #include "super.h" #include "xattr.h" +#include "trace.h" #include <linux/aio.h> #include <linux/backing-dev.h> #include <linux/exportfs.h> #include <linux/fiemap.h> +#include <linux/fs_context.h> #include <linux/module.h> #include <linux/pagemap.h> #include <linux/posix_acl.h> @@ -1680,6 +1682,8 @@ static int bch2_sync_fs(struct super_block *sb, int wait) struct bch_fs *c = sb->s_fs_info; int ret; + trace_bch2_sync_fs(sb, wait); + if (c->opts.journal_flush_disabled) return 0; @@ -1708,15 +1712,11 @@ static struct bch_fs *bch2_path_to_fs(const char *path) return c ?: ERR_PTR(-ENOENT); } -static int bch2_remount(struct super_block *sb, int *flags, char *data) +static int bch2_remount(struct super_block *sb, int *flags, + struct bch_opts opts) { struct bch_fs *c = sb->s_fs_info; - struct bch_opts opts = bch2_opts_empty(); - int ret; - - ret = bch2_parse_mount_opts(c, &opts, data); - if (ret) - goto err; + int ret = 0; opt_set(opts, read_only, (*flags & SB_RDONLY) != 0); @@ -1843,7 +1843,6 @@ static const struct super_operations bch_super_operations = { .statfs = bch2_statfs, .show_devname = bch2_show_devname, .show_options = bch2_show_options, - .remount_fs = bch2_remount, .put_super = bch2_put_super, .freeze_fs = bch2_freeze, .unfreeze_fs = bch2_unfreeze, @@ -1876,77 +1875,63 @@ static int bch2_test_super(struct super_block *s, void *data) return true; } -static struct dentry *bch2_mount(struct file_system_type *fs_type, - int flags, const char *dev_name, void *data) +static int bch2_fs_get_tree(struct fs_context *fc) { struct bch_fs *c; struct super_block *sb; struct inode *vinode; - struct bch_opts opts = bch2_opts_empty(); + struct bch2_opts_parse *opts_parse = fc->fs_private; + struct bch_opts opts = opts_parse->opts; + darray_str devs; + darray_fs devs_to_fs = {}; int ret; - opt_set(opts, read_only, (flags & SB_RDONLY) != 0); + opt_set(opts, read_only, (fc->sb_flags & SB_RDONLY) != 0); + opt_set(opts, nostart, true); - ret = bch2_parse_mount_opts(NULL, &opts, data); - if (ret) { - ret = bch2_err_class(ret); - return ERR_PTR(ret); - } - - if (!dev_name || strlen(dev_name) == 0) - return ERR_PTR(-EINVAL); + if (!fc->source || strlen(fc->source) == 0) + return -EINVAL; - darray_str devs; - ret = bch2_split_devs(dev_name, &devs); + ret = bch2_split_devs(fc->source, &devs); if (ret) - return ERR_PTR(ret); + return ret; - darray_fs devs_to_fs = {}; darray_for_each(devs, i) { ret = darray_push(&devs_to_fs, bch2_path_to_fs(*i)); - if (ret) { - sb = ERR_PTR(ret); - goto got_sb; - } + if (ret) + goto err; } - sb = sget(fs_type, bch2_test_super, bch2_noset_super, flags|SB_NOSEC, &devs_to_fs); + sb = sget(fc->fs_type, bch2_test_super, bch2_noset_super, fc->sb_flags|SB_NOSEC, &devs_to_fs); if (!IS_ERR(sb)) goto got_sb; c = bch2_fs_open(devs.data, devs.nr, opts); - if (IS_ERR(c)) { - sb = ERR_CAST(c); - goto got_sb; - } + ret = PTR_ERR_OR_ZERO(c); + if (ret) + goto err; /* Some options can't be parsed until after the fs is started: */ - ret = bch2_parse_mount_opts(c, &opts, data); - if (ret) { - bch2_fs_stop(c); - sb = ERR_PTR(ret); - goto got_sb; - } + opts = bch2_opts_empty(); + ret = bch2_parse_mount_opts(c, &opts, NULL, opts_parse->parse_later.buf); + if (ret) + goto err_stop_fs; bch2_opts_apply(&c->opts, opts); - sb = sget(fs_type, NULL, bch2_set_super, flags|SB_NOSEC, c); - if (IS_ERR(sb)) - bch2_fs_stop(c); -got_sb: - darray_exit(&devs_to_fs); - bch2_darray_str_exit(&devs); - - if (IS_ERR(sb)) { - ret = PTR_ERR(sb); - ret = bch2_err_class(ret); - return ERR_PTR(ret); - } + ret = bch2_fs_start(c); + if (ret) + goto err_stop_fs; + sb = sget(fc->fs_type, NULL, bch2_set_super, fc->sb_flags|SB_NOSEC, c); + ret = PTR_ERR_OR_ZERO(sb); + if (ret) + goto err_stop_fs; +got_sb: c = sb->s_fs_info; if (sb->s_root) { - if ((flags ^ sb->s_flags) & SB_RDONLY) { + if ((fc->sb_flags ^ sb->s_flags) & SB_RDONLY) { ret = -EBUSY; goto err_put_super; } @@ -2011,12 +1996,28 @@ got_sb: sb->s_flags |= SB_ACTIVE; out: - return dget(sb->s_root); + fc->root = dget(sb->s_root); +err: + darray_exit(&devs_to_fs); + bch2_darray_str_exit(&devs); + /* + * On an inconsistency error in recovery we might see an -EROFS derived + * errorcode (from the journal), but we don't want to return that to + * userspace as that causes util-linux to retry the mount RO - which is + * confusing: + */ + if (bch2_err_matches(ret, EROFS) && ret != -EROFS) + ret = -EIO; + return bch2_err_class(ret); + +err_stop_fs: + bch2_fs_stop(c); + goto err; err_put_super: __bch2_fs_stop(c); deactivate_locked_super(sb); - return ERR_PTR(bch2_err_class(ret)); + goto err; } static void bch2_kill_sb(struct super_block *sb) @@ -2027,12 +2028,76 @@ static void bch2_kill_sb(struct super_block *sb) bch2_fs_free(c); } +static void bch2_fs_context_free(struct fs_context *fc) +{ + struct bch2_opts_parse *opts = fc->fs_private; + + if (opts) { + printbuf_exit(&opts->parse_later); + kfree(opts); + } +} + +static int bch2_fs_parse_param(struct fs_context *fc, + struct fs_parameter *param) +{ + /* + * the "source" param, i.e., the name of the device(s) to mount, + * is handled by the VFS layer. + */ + if (!strcmp(param->key, "source")) + return -ENOPARAM; + + struct bch2_opts_parse *opts = fc->fs_private; + struct bch_fs *c = NULL; + + /* for reconfigure, we already have a struct bch_fs */ + if (fc->root) + c = fc->root->d_sb->s_fs_info; + + int ret = bch2_parse_one_mount_opt(c, &opts->opts, + &opts->parse_later, param->key, + param->string); + + return bch2_err_class(ret); +} + +static int bch2_fs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct bch2_opts_parse *opts = fc->fs_private; + + return bch2_remount(sb, &fc->sb_flags, opts->opts); +} + +static const struct fs_context_operations bch2_context_ops = { + .free = bch2_fs_context_free, + .parse_param = bch2_fs_parse_param, + .get_tree = bch2_fs_get_tree, + .reconfigure = bch2_fs_reconfigure, +}; + +static int bch2_init_fs_context(struct fs_context *fc) +{ + struct bch2_opts_parse *opts = kzalloc(sizeof(*opts), GFP_KERNEL); + + if (!opts) + return -ENOMEM; + + opts->parse_later = PRINTBUF; + + fc->ops = &bch2_context_ops; + fc->fs_private = opts; + + return 0; +} + static struct file_system_type bcache_fs_type = { - .owner = THIS_MODULE, - .name = "bcachefs", - .mount = bch2_mount, - .kill_sb = bch2_kill_sb, - .fs_flags = FS_REQUIRES_DEV, + .owner = THIS_MODULE, + .name = "bcachefs", + .init_fs_context = bch2_init_fs_context, + .kill_sb = bch2_kill_sb, + .fs_flags = FS_REQUIRES_DEV, }; MODULE_ALIAS_FS("bcachefs"); diff --git a/libbcachefs/fsck.c b/libbcachefs/fsck.c index fd277bd5..037b64c1 100644 --- a/libbcachefs/fsck.c +++ b/libbcachefs/fsck.c @@ -455,31 +455,42 @@ static int reconstruct_subvol(struct btree_trans *trans, u32 snapshotid, u32 sub return 0; } -static int reconstruct_inode(struct btree_trans *trans, u32 snapshot, u64 inum, u64 size, unsigned mode) +static int reconstruct_inode(struct btree_trans *trans, enum btree_id btree, u32 snapshot, u64 inum) { struct bch_fs *c = trans->c; - struct bch_inode_unpacked new_inode; + unsigned i_mode = S_IFREG; + u64 i_size = 0; - bch2_inode_init_early(c, &new_inode); - bch2_inode_init_late(&new_inode, bch2_current_time(c), 0, 0, mode|0755, 0, NULL); - new_inode.bi_size = size; - new_inode.bi_inum = inum; + switch (btree) { + case BTREE_ID_extents: { + struct btree_iter iter = {}; - return __bch2_fsck_write_inode(trans, &new_inode, snapshot); -} + bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, SPOS(inum, U64_MAX, snapshot), 0); + struct bkey_s_c k = bch2_btree_iter_peek_prev(&iter); + bch2_trans_iter_exit(trans, &iter); + int ret = bkey_err(k); + if (ret) + return ret; -static int reconstruct_reg_inode(struct btree_trans *trans, u32 snapshot, u64 inum) -{ - struct btree_iter iter = {}; + i_size = k.k->p.offset << 9; + break; + } + case BTREE_ID_dirents: + i_mode = S_IFDIR; + break; + case BTREE_ID_xattrs: + break; + default: + BUG(); + } - bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, SPOS(inum, U64_MAX, snapshot), 0); - struct bkey_s_c k = bch2_btree_iter_peek_prev(&iter); - bch2_trans_iter_exit(trans, &iter); - int ret = bkey_err(k); - if (ret) - return ret; + struct bch_inode_unpacked new_inode; + bch2_inode_init_early(c, &new_inode); + bch2_inode_init_late(&new_inode, bch2_current_time(c), 0, 0, i_mode|0600, 0, NULL); + new_inode.bi_size = i_size; + new_inode.bi_inum = inum; - return reconstruct_inode(trans, snapshot, inum, k.k->p.offset << 9, S_IFREG); + return __bch2_fsck_write_inode(trans, &new_inode, snapshot); } struct snapshots_seen { @@ -824,8 +835,8 @@ static int hash_check_key(struct btree_trans *trans, break; if (fsck_err_on(k.k->type == desc.key_type && - !desc.cmp_bkey(k, hash_k), c, - hash_table_key_duplicate, + !desc.cmp_bkey(k, hash_k), + trans, hash_table_key_duplicate, "duplicate hash table keys:\n%s", (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, hash_k), @@ -844,7 +855,7 @@ out: printbuf_exit(&buf); return ret; bad_hash: - if (fsck_err(c, hash_table_key_wrong_offset, + if (fsck_err(trans, hash_table_key_wrong_offset, "hash table key at wrong offset: btree %s inode %llu offset %llu, hashed to %llu\n%s", bch2_btree_id_str(desc.btree_id), hash_k.k->p.inode, hash_k.k->p.offset, hash, (printbuf_reset(&buf), @@ -919,11 +930,11 @@ static int check_inode_dirent_inode(struct btree_trans *trans, struct bkey_s_c i return ret; if (fsck_err_on(ret, - c, inode_points_to_missing_dirent, + trans, inode_points_to_missing_dirent, "inode points to missing dirent\n%s", (bch2_bkey_val_to_text(&buf, c, inode_k), buf.buf)) || fsck_err_on(!ret && !dirent_points_to_inode(d, inode), - c, inode_points_to_wrong_dirent, + trans, inode_points_to_wrong_dirent, "inode points to dirent that does not point back:\n%s", (bch2_bkey_val_to_text(&buf, c, inode_k), prt_newline(&buf), @@ -986,7 +997,7 @@ static int check_inode(struct btree_trans *trans, if (fsck_err_on(prev->bi_hash_seed != u.bi_hash_seed || inode_d_type(prev) != inode_d_type(&u), - c, inode_snapshot_mismatch, + trans, inode_snapshot_mismatch, "inodes in different snapshots don't match")) { bch_err(c, "repair not implemented yet"); return -BCH_ERR_fsck_repair_unimplemented; @@ -1018,7 +1029,8 @@ static int check_inode(struct btree_trans *trans, if (ret < 0) return ret; - fsck_err_on(!ret, c, unlinked_inode_not_on_deleted_list, + fsck_err_on(!ret, + trans, unlinked_inode_not_on_deleted_list, "inode %llu:%u unlinked, but not on deleted list", u.bi_inum, k.k->p.snapshot); ret = 0; @@ -1026,7 +1038,7 @@ static int check_inode(struct btree_trans *trans, if (u.bi_flags & BCH_INODE_unlinked && (!c->sb.clean || - fsck_err(c, inode_unlinked_but_clean, + fsck_err(trans, inode_unlinked_but_clean, "filesystem marked clean, but inode %llu unlinked", u.bi_inum))) { ret = bch2_inode_rm_snapshot(trans, u.bi_inum, iter->pos.snapshot); @@ -1036,7 +1048,7 @@ static int check_inode(struct btree_trans *trans, if (u.bi_flags & BCH_INODE_i_size_dirty && (!c->sb.clean || - fsck_err(c, inode_i_size_dirty_but_clean, + fsck_err(trans, inode_i_size_dirty_but_clean, "filesystem marked clean, but inode %llu has i_size dirty", u.bi_inum))) { bch_verbose(c, "truncating inode %llu", u.bi_inum); @@ -1066,7 +1078,7 @@ static int check_inode(struct btree_trans *trans, if (u.bi_flags & BCH_INODE_i_sectors_dirty && (!c->sb.clean || - fsck_err(c, inode_i_sectors_dirty_but_clean, + fsck_err(trans, inode_i_sectors_dirty_but_clean, "filesystem marked clean, but inode %llu has i_sectors dirty", u.bi_inum))) { s64 sectors; @@ -1101,7 +1113,7 @@ static int check_inode(struct btree_trans *trans, if (fsck_err_on(u.bi_parent_subvol && (u.bi_subvol == 0 || u.bi_subvol == BCACHEFS_ROOT_SUBVOL), - c, inode_bi_parent_nonzero, + trans, inode_bi_parent_nonzero, "inode %llu:%u has subvol %u but nonzero parent subvol %u", u.bi_inum, k.k->p.snapshot, u.bi_subvol, u.bi_parent_subvol)) { u.bi_parent_subvol = 0; @@ -1121,13 +1133,13 @@ static int check_inode(struct btree_trans *trans, } if (fsck_err_on(ret, - c, inode_bi_subvol_missing, + trans, inode_bi_subvol_missing, "inode %llu:%u bi_subvol points to missing subvolume %u", u.bi_inum, k.k->p.snapshot, u.bi_subvol) || fsck_err_on(le64_to_cpu(s.inode) != u.bi_inum || !bch2_snapshot_is_ancestor(c, le32_to_cpu(s.snapshot), k.k->p.snapshot), - c, inode_bi_subvol_wrong, + trans, inode_bi_subvol_wrong, "inode %llu:%u points to subvol %u, but subvol points to %llu:%u", u.bi_inum, k.k->p.snapshot, u.bi_subvol, le64_to_cpu(s.inode), @@ -1170,6 +1182,71 @@ int bch2_check_inodes(struct bch_fs *c) return ret; } +static inline bool btree_matches_i_mode(enum btree_id btree, unsigned mode) +{ + switch (btree) { + case BTREE_ID_extents: + return S_ISREG(mode) || S_ISLNK(mode); + case BTREE_ID_dirents: + return S_ISDIR(mode); + case BTREE_ID_xattrs: + return true; + default: + BUG(); + } +} + +static int check_key_has_inode(struct btree_trans *trans, + struct btree_iter *iter, + struct inode_walker *inode, + struct inode_walker_entry *i, + struct bkey_s_c k) +{ + struct bch_fs *c = trans->c; + struct printbuf buf = PRINTBUF; + int ret = PTR_ERR_OR_ZERO(i); + if (ret) + return ret; + + if (k.k->type == KEY_TYPE_whiteout) + goto out; + + if (!i && (c->sb.btrees_lost_data & BIT_ULL(BTREE_ID_inodes))) { + ret = reconstruct_inode(trans, iter->btree_id, k.k->p.snapshot, k.k->p.inode) ?: + bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); + if (ret) + goto err; + + inode->last_pos.inode--; + ret = -BCH_ERR_transaction_restart_nested; + goto err; + } + + if (fsck_err_on(!i, + trans, key_in_missing_inode, + "key in missing inode:\n %s", + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) + goto delete; + + if (fsck_err_on(i && !btree_matches_i_mode(iter->btree_id, i->inode.bi_mode), + trans, key_in_wrong_inode_type, + "key for wrong inode mode %o:\n %s", + i->inode.bi_mode, + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) + goto delete; +out: +err: +fsck_err: + printbuf_exit(&buf); + bch_err_fn(c, ret); + return ret; +delete: + ret = bch2_btree_delete_at(trans, iter, BTREE_UPDATE_internal_snapshot_node); + goto out; +} + static int check_i_sectors_notnested(struct btree_trans *trans, struct inode_walker *w) { struct bch_fs *c = trans->c; @@ -1192,7 +1269,7 @@ static int check_i_sectors_notnested(struct btree_trans *trans, struct inode_wal } if (fsck_err_on(!(i->inode.bi_flags & BCH_INODE_i_sectors_dirty), - c, inode_i_sectors_wrong, + trans, inode_i_sectors_wrong, "inode %llu:%u has incorrect i_sectors: got %llu, should be %llu", w->last_pos.inode, i->snapshot, i->inode.bi_sectors, i->count)) { @@ -1340,7 +1417,7 @@ static int overlapping_extents_found(struct btree_trans *trans, prt_printf(&buf, "\n overwriting %s extent", pos1.snapshot >= pos2.p.snapshot ? "first" : "second"); - if (fsck_err(c, extent_overlapping, + if (fsck_err(trans, extent_overlapping, "overlapping extents%s", buf.buf)) { struct btree_iter *old_iter = &iter1; struct disk_reservation res = { 0 }; @@ -1476,43 +1553,20 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter, goto err; } + ret = snapshots_seen_update(c, s, iter->btree_id, k.k->p); + if (ret) + goto err; + i = walk_inode(trans, inode, k); ret = PTR_ERR_OR_ZERO(i); if (ret) goto err; - ret = snapshots_seen_update(c, s, iter->btree_id, k.k->p); + ret = check_key_has_inode(trans, iter, inode, i, k); if (ret) goto err; if (k.k->type != KEY_TYPE_whiteout) { - if (!i && (c->sb.btrees_lost_data & BIT_ULL(BTREE_ID_inodes))) { - ret = reconstruct_reg_inode(trans, k.k->p.snapshot, k.k->p.inode) ?: - bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); - if (ret) - goto err; - - inode->last_pos.inode--; - ret = -BCH_ERR_transaction_restart_nested; - goto err; - } - - if (fsck_err_on(!i, c, extent_in_missing_inode, - "extent in missing inode:\n %s", - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, k), buf.buf))) - goto delete; - - if (fsck_err_on(i && - !S_ISREG(i->inode.bi_mode) && - !S_ISLNK(i->inode.bi_mode), - c, extent_in_non_reg_inode, - "extent in non regular inode mode %o:\n %s", - i->inode.bi_mode, - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, k), buf.buf))) - goto delete; - ret = check_overlapping_extents(trans, s, extent_ends, k, iter, &inode->recalculate_sums); if (ret) @@ -1525,7 +1579,7 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter, * didn't have one, iterate over all inodes: */ if (!i) - i = inode->inodes.data + inode->inodes.nr - 1; + i = &darray_last(inode->inodes); for (; inode->inodes.data && i >= inode->inodes.data; @@ -1538,7 +1592,7 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter, if (fsck_err_on(!(i->inode.bi_flags & BCH_INODE_i_size_dirty) && k.k->p.offset > round_up(i->inode.bi_size, block_bytes(c)) >> 9 && !bkey_extent_is_reservation(k), - c, extent_past_end_of_inode, + trans, extent_past_end_of_inode, "extent type past end of inode %llu:%u, i_size %llu\n %s", i->inode.bi_inum, i->snapshot, i->inode.bi_size, (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { @@ -1574,9 +1628,6 @@ fsck_err: printbuf_exit(&buf); bch_err_fn(c, ret); return ret; -delete: - ret = bch2_btree_delete_at(trans, iter, BTREE_UPDATE_internal_snapshot_node); - goto out; } /* @@ -1656,7 +1707,7 @@ static int check_subdir_count_notnested(struct btree_trans *trans, struct inode_ } if (fsck_err_on(i->inode.bi_nlink != i->count, - c, inode_dir_wrong_nlink, + trans, inode_dir_wrong_nlink, "directory %llu:%u with wrong i_nlink: got %u, should be %llu", w->last_pos.inode, i->snapshot, i->inode.bi_nlink, i->count)) { i->inode.bi_nlink = i->count; @@ -1691,7 +1742,7 @@ static int check_dirent_inode_dirent(struct btree_trans *trans, return 0; if (bch2_inode_should_have_bp(target) && - !fsck_err(c, inode_wrong_backpointer, + !fsck_err(trans, inode_wrong_backpointer, "dirent points to inode that does not point back:\n %s", (bch2_bkey_val_to_text(&buf, c, d.s_c), prt_printf(&buf, "\n "), @@ -1717,7 +1768,7 @@ static int check_dirent_inode_dirent(struct btree_trans *trans, ret = 0; if (fsck_err_on(!backpointer_exists, - c, inode_wrong_backpointer, + trans, inode_wrong_backpointer, "inode %llu:%u has wrong backpointer:\n" "got %llu:%llu\n" "should be %llu:%llu", @@ -1740,7 +1791,7 @@ static int check_dirent_inode_dirent(struct btree_trans *trans, if (fsck_err_on(backpointer_exists && (S_ISDIR(target->bi_mode) || target->bi_subvol), - c, inode_dir_multiple_links, + trans, inode_dir_multiple_links, "%s %llu:%u with multiple links\n%s", S_ISDIR(target->bi_mode) ? "directory" : "subvolume", target->bi_inum, target_snapshot, buf.buf)) { @@ -1754,7 +1805,7 @@ static int check_dirent_inode_dirent(struct btree_trans *trans, * it up, it ignores inodes with nlink 0 */ if (fsck_err_on(backpointer_exists && !target->bi_nlink, - c, inode_multiple_links_but_nlink_0, + trans, inode_multiple_links_but_nlink_0, "inode %llu:%u type %s has multiple links but i_nlink 0\n%s", target->bi_inum, target_snapshot, bch2_d_types[d.v->d_type], buf.buf)) { target->bi_nlink++; @@ -1789,7 +1840,7 @@ static int check_dirent_target(struct btree_trans *trans, goto err; if (fsck_err_on(d.v->d_type != inode_d_type(target), - c, dirent_d_type_wrong, + trans, dirent_d_type_wrong, "incorrect d_type: got %s, should be %s:\n%s", bch2_d_type_str(d.v->d_type), bch2_d_type_str(inode_d_type(target)), @@ -1886,11 +1937,12 @@ static int check_dirent_to_subvol(struct btree_trans *trans, struct btree_iter * parent_snapshot = d.k->p.snapshot; } - if (fsck_err_on(ret, c, dirent_to_missing_parent_subvol, + if (fsck_err_on(ret, + trans, dirent_to_missing_parent_subvol, "dirent parent_subvol points to missing subvolume\n%s", (bch2_bkey_val_to_text(&buf, c, d.s_c), buf.buf)) || fsck_err_on(!ret && !bch2_snapshot_is_ancestor(c, parent_snapshot, d.k->p.snapshot), - c, dirent_not_visible_in_parent_subvol, + trans, dirent_not_visible_in_parent_subvol, "dirent not visible in parent_subvol (not an ancestor of subvol snap %u)\n%s", parent_snapshot, (bch2_bkey_val_to_text(&buf, c, d.s_c), buf.buf))) { @@ -1916,7 +1968,7 @@ static int check_dirent_to_subvol(struct btree_trans *trans, struct btree_iter * return ret; if (ret) { - if (fsck_err(c, dirent_to_missing_subvol, + if (fsck_err(trans, dirent_to_missing_subvol, "dirent points to missing subvolume\n%s", (bch2_bkey_val_to_text(&buf, c, d.s_c), buf.buf))) return __remove_dirent(trans, d.k->p); @@ -1925,7 +1977,7 @@ static int check_dirent_to_subvol(struct btree_trans *trans, struct btree_iter * } if (fsck_err_on(le32_to_cpu(s.v->fs_path_parent) != parent_subvol, - c, subvol_fs_path_parent_wrong, + trans, subvol_fs_path_parent_wrong, "subvol with wrong fs_path_parent, should be be %u\n%s", parent_subvol, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { @@ -1953,7 +2005,7 @@ static int check_dirent_to_subvol(struct btree_trans *trans, struct btree_iter * } if (fsck_err_on(!ret && parent_subvol != subvol_root.bi_parent_subvol, - c, inode_bi_parent_wrong, + trans, inode_bi_parent_wrong, "subvol root %llu has wrong bi_parent_subvol: got %u, should be %u", target_inum, subvol_root.bi_parent_subvol, parent_subvol)) { @@ -2006,49 +2058,21 @@ static int check_dirent(struct btree_trans *trans, struct btree_iter *iter, goto err; } - BUG_ON(!btree_iter_path(trans, iter)->should_be_locked); - i = walk_inode(trans, dir, k); ret = PTR_ERR_OR_ZERO(i); if (ret < 0) goto err; - if (dir->first_this_inode && dir->inodes.nr) - *hash_info = bch2_hash_info_init(c, &dir->inodes.data[0].inode); - dir->first_this_inode = false; - - if (!i && (c->sb.btrees_lost_data & BIT_ULL(BTREE_ID_inodes))) { - ret = reconstruct_inode(trans, k.k->p.snapshot, k.k->p.inode, 0, S_IFDIR) ?: - bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); - if (ret) - goto err; - - dir->last_pos.inode--; - ret = -BCH_ERR_transaction_restart_nested; + ret = check_key_has_inode(trans, iter, dir, i, k); + if (ret) goto err; - } - - if (fsck_err_on(!i, c, dirent_in_missing_dir_inode, - "dirent in nonexisting directory:\n%s", - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { - ret = bch2_btree_delete_at(trans, iter, - BTREE_UPDATE_internal_snapshot_node); - goto out; - } if (!i) goto out; - if (fsck_err_on(!S_ISDIR(i->inode.bi_mode), - c, dirent_in_non_dir_inode, - "dirent in non directory inode type %s:\n%s", - bch2_d_type_str(inode_d_type(&i->inode)), - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { - ret = bch2_btree_delete_at(trans, iter, 0); - goto out; - } + if (dir->first_this_inode) + *hash_info = bch2_hash_info_init(c, &i->inode); + dir->first_this_inode = false; ret = hash_check_key(trans, bch2_dirent_hash_desc, hash_info, iter, k); if (ret < 0) @@ -2074,7 +2098,7 @@ static int check_dirent(struct btree_trans *trans, struct btree_iter *iter, goto err; if (fsck_err_on(!target->inodes.nr, - c, dirent_to_missing_inode, + trans, dirent_to_missing_inode, "dirent points to missing inode:\n%s", (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, k), @@ -2153,20 +2177,18 @@ static int check_xattr(struct btree_trans *trans, struct btree_iter *iter, if (ret) return ret; - if (inode->first_this_inode && inode->inodes.nr) - *hash_info = bch2_hash_info_init(c, &inode->inodes.data[0].inode); - inode->first_this_inode = false; - - if (fsck_err_on(!i, c, xattr_in_missing_inode, - "xattr for missing inode %llu", - k.k->p.inode)) - return bch2_btree_delete_at(trans, iter, 0); + ret = check_key_has_inode(trans, iter, inode, i, k); + if (ret) + return ret; if (!i) return 0; + if (inode->first_this_inode) + *hash_info = bch2_hash_info_init(c, &i->inode); + inode->first_this_inode = false; + ret = hash_check_key(trans, bch2_xattr_hash_desc, hash_info, iter, k); -fsck_err: bch_err_fn(c, ret); return ret; } @@ -2204,7 +2226,7 @@ static int check_root_trans(struct btree_trans *trans) if (ret && !bch2_err_matches(ret, ENOENT)) return ret; - if (mustfix_fsck_err_on(ret, c, root_subvol_missing, + if (mustfix_fsck_err_on(ret, trans, root_subvol_missing, "root subvol missing")) { struct bkey_i_subvolume *root_subvol = bch2_trans_kmalloc(trans, sizeof(*root_subvol)); @@ -2230,10 +2252,11 @@ static int check_root_trans(struct btree_trans *trans) if (ret && !bch2_err_matches(ret, ENOENT)) return ret; - if (mustfix_fsck_err_on(ret, c, root_dir_missing, + if (mustfix_fsck_err_on(ret, + trans, root_dir_missing, "root directory missing") || mustfix_fsck_err_on(!S_ISDIR(root_inode.bi_mode), - c, root_inode_not_dir, + trans, root_inode_not_dir, "root inode not a directory")) { bch2_inode_init(c, &root_inode, 0, 0, S_IFDIR|0755, 0, NULL); @@ -2305,7 +2328,7 @@ static int check_subvol_path(struct btree_trans *trans, struct btree_iter *iter, break; if (fsck_err_on(!ret, - c, subvol_unreachable, + trans, subvol_unreachable, "unreachable subvolume %s", (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { @@ -2330,7 +2353,7 @@ static int check_subvol_path(struct btree_trans *trans, struct btree_iter *iter, goto err; if (fsck_err_on(k.k->type != KEY_TYPE_subvolume, - c, subvol_unreachable, + trans, subvol_unreachable, "unreachable subvolume %s", (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { @@ -2409,7 +2432,7 @@ static int check_path(struct btree_trans *trans, pathbuf *p, struct bkey_s_c ino if (bch2_err_matches(ret, ENOENT)) { ret = 0; - if (fsck_err(c, inode_unreachable, + if (fsck_err(trans, inode_unreachable, "unreachable inode\n%s", (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, inode_k), @@ -2455,7 +2478,7 @@ static int check_path(struct btree_trans *trans, pathbuf *p, struct bkey_s_c ino pr_err("%llu:%u", i->inum, i->snapshot); pr_err("%llu:%u", inode.bi_inum, snapshot); - if (fsck_err(c, dir_loop, "directory structure loop")) { + if (fsck_err(trans, dir_loop, "directory structure loop")) { ret = remove_backpointer(trans, &inode); bch_err_msg(c, ret, "removing dirent"); if (ret) @@ -2661,7 +2684,6 @@ static int check_nlinks_update_inode(struct btree_trans *trans, struct btree_ite struct nlink_table *links, size_t *idx, u64 range_end) { - struct bch_fs *c = trans->c; struct bch_inode_unpacked u; struct nlink *link = &links->d[*idx]; int ret = 0; @@ -2687,7 +2709,7 @@ static int check_nlinks_update_inode(struct btree_trans *trans, struct btree_ite } if (fsck_err_on(bch2_inode_nlink_get(&u) != link->count, - c, inode_wrong_nlink, + trans, inode_wrong_nlink, "inode %llu type %s has wrong i_nlink (%u, should be %u)", u.bi_inum, bch2_d_types[mode_to_type(u.bi_mode)], bch2_inode_nlink_get(&u), link->count)) { diff --git a/libbcachefs/inode.c b/libbcachefs/inode.c index aafa79fa..6b807ecb 100644 --- a/libbcachefs/inode.c +++ b/libbcachefs/inode.c @@ -8,6 +8,7 @@ #include "buckets.h" #include "compress.h" #include "dirent.h" +#include "disk_accounting.h" #include "error.h" #include "extents.h" #include "extent_update.h" @@ -550,6 +551,8 @@ static void __bch2_inode_unpacked_to_text(struct printbuf *out, prt_printf(out, #_name "=%llu\n", (u64) inode->_name); BCH_INODE_FIELDS_v3() #undef x + + bch2_printbuf_strip_trailing_newline(out); printbuf_indent_sub(out, 2); } @@ -596,39 +599,26 @@ int bch2_trigger_inode(struct btree_trans *trans, struct bkey_s new, enum btree_iter_update_trigger_flags flags) { - s64 nr = (s64) bkey_is_inode(new.k) - (s64) bkey_is_inode(old.k); - - if (flags & BTREE_TRIGGER_transactional) { - if (nr) { - int ret = bch2_replicas_deltas_realloc(trans, 0); - if (ret) - return ret; - - trans->fs_usage_deltas->nr_inodes += nr; - } - - bool old_deleted = bkey_is_deleted_inode(old); - bool new_deleted = bkey_is_deleted_inode(new.s_c); - if (old_deleted != new_deleted) { - int ret = bch2_btree_bit_mod_buffered(trans, BTREE_ID_deleted_inodes, - new.k->p, new_deleted); - if (ret) - return ret; - } - } - if ((flags & BTREE_TRIGGER_atomic) && (flags & BTREE_TRIGGER_insert)) { BUG_ON(!trans->journal_res.seq); - bkey_s_to_inode_v3(new).v->bi_journal_seq = cpu_to_le64(trans->journal_res.seq); } - if (flags & BTREE_TRIGGER_gc) { - struct bch_fs *c = trans->c; + s64 nr = bkey_is_inode(new.k) - bkey_is_inode(old.k); + if ((flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) && nr) { + struct disk_accounting_pos acc = { .type = BCH_DISK_ACCOUNTING_nr_inodes }; + int ret = bch2_disk_accounting_mod(trans, &acc, &nr, 1, flags & BTREE_TRIGGER_gc); + if (ret) + return ret; + } - percpu_down_read(&c->mark_lock); - this_cpu_add(c->usage_gc->b.nr_inodes, nr); - percpu_up_read(&c->mark_lock); + int deleted_delta = (int) bkey_is_deleted_inode(new.s_c) - + (int) bkey_is_deleted_inode(old); + if ((flags & BTREE_TRIGGER_transactional) && deleted_delta) { + int ret = bch2_btree_bit_mod_buffered(trans, BTREE_ID_deleted_inodes, + new.k->p, deleted_delta > 0); + if (ret) + return ret; } return 0; @@ -1096,8 +1086,8 @@ static int may_delete_deleted_inode(struct btree_trans *trans, return ret; ret = bkey_is_inode(k.k) ? 0 : -BCH_ERR_ENOENT_inode; - if (fsck_err_on(!bkey_is_inode(k.k), c, - deleted_inode_missing, + if (fsck_err_on(!bkey_is_inode(k.k), + trans, deleted_inode_missing, "nonexistent inode %llu:%u in deleted_inodes btree", pos.offset, pos.snapshot)) goto delete; @@ -1109,7 +1099,7 @@ static int may_delete_deleted_inode(struct btree_trans *trans, if (S_ISDIR(inode.bi_mode)) { ret = bch2_empty_dir_snapshot(trans, pos.offset, 0, pos.snapshot); if (fsck_err_on(bch2_err_matches(ret, ENOTEMPTY), - c, deleted_inode_is_dir, + trans, deleted_inode_is_dir, "non empty directory %llu:%u in deleted_inodes btree", pos.offset, pos.snapshot)) goto delete; @@ -1117,15 +1107,14 @@ static int may_delete_deleted_inode(struct btree_trans *trans, goto out; } - if (fsck_err_on(!(inode.bi_flags & BCH_INODE_unlinked), c, - deleted_inode_not_unlinked, + if (fsck_err_on(!(inode.bi_flags & BCH_INODE_unlinked), + trans, deleted_inode_not_unlinked, "non-deleted inode %llu:%u in deleted_inodes btree", pos.offset, pos.snapshot)) goto delete; if (c->sb.clean && - !fsck_err(c, - deleted_inode_but_clean, + !fsck_err(trans, deleted_inode_but_clean, "filesystem marked as clean but have deleted inode %llu:%u", pos.offset, pos.snapshot)) { ret = 0; diff --git a/libbcachefs/io_write.c b/libbcachefs/io_write.c index 9401d13e..db9cda7b 100644 --- a/libbcachefs/io_write.c +++ b/libbcachefs/io_write.c @@ -69,11 +69,10 @@ void bch2_latency_acct(struct bch_dev *ca, u64 submit_time, int rw) u64 io_latency = time_after64(now, submit_time) ? now - submit_time : 0; - u64 old, new, v = atomic64_read(latency); + u64 old, new; + old = atomic64_read(latency); do { - old = v; - /* * If the io latency was reasonably close to the current * latency, skip doing the update and atomic operation - most of @@ -84,7 +83,7 @@ void bch2_latency_acct(struct bch_dev *ca, u64 submit_time, int rw) break; new = ewma_add(old, io_latency, 5); - } while ((v = atomic64_cmpxchg(latency, old, new)) != old); + } while (!atomic64_try_cmpxchg(latency, &old, new)); bch2_congested_acct(ca, io_latency, now, rw); diff --git a/libbcachefs/journal.c b/libbcachefs/journal.c index adec8e1e..b9577161 100644 --- a/libbcachefs/journal.c +++ b/libbcachefs/journal.c @@ -230,7 +230,6 @@ static void __journal_entry_close(struct journal *j, unsigned closed_val, bool t struct bch_fs *c = container_of(j, struct bch_fs, journal); struct journal_buf *buf = journal_cur_buf(j); union journal_res_state old, new; - u64 v = atomic64_read(&j->reservations.counter); unsigned sectors; BUG_ON(closed_val != JOURNAL_ENTRY_CLOSED_VAL && @@ -238,15 +237,16 @@ static void __journal_entry_close(struct journal *j, unsigned closed_val, bool t lockdep_assert_held(&j->lock); + old.v = atomic64_read(&j->reservations.counter); do { - old.v = new.v = v; + new.v = old.v; new.cur_entry_offset = closed_val; if (old.cur_entry_offset == JOURNAL_ENTRY_ERROR_VAL || old.cur_entry_offset == new.cur_entry_offset) return; - } while ((v = atomic64_cmpxchg(&j->reservations.counter, - old.v, new.v)) != old.v); + } while (!atomic64_try_cmpxchg(&j->reservations.counter, + &old.v, new.v)); if (!__journal_entry_is_open(old)) return; @@ -353,7 +353,6 @@ static int journal_entry_open(struct journal *j) ((journal_cur_seq(j) + 1) & JOURNAL_BUF_MASK); union journal_res_state old, new; int u64s; - u64 v; lockdep_assert_held(&j->lock); BUG_ON(journal_entry_is_open(j)); @@ -432,9 +431,9 @@ static int journal_entry_open(struct journal *j) */ j->cur_entry_u64s = u64s; - v = atomic64_read(&j->reservations.counter); + old.v = atomic64_read(&j->reservations.counter); do { - old.v = new.v = v; + new.v = old.v; BUG_ON(old.cur_entry_offset == JOURNAL_ENTRY_ERROR_VAL); @@ -446,8 +445,8 @@ static int journal_entry_open(struct journal *j) /* Handle any already added entries */ new.cur_entry_offset = le32_to_cpu(buf->data->u64s); - } while ((v = atomic64_cmpxchg(&j->reservations.counter, - old.v, new.v)) != old.v); + } while (!atomic64_try_cmpxchg(&j->reservations.counter, + &old.v, new.v)); if (nr_unwritten_journal_entries(j) == 1) mod_delayed_work(j->wq, diff --git a/libbcachefs/journal.h b/libbcachefs/journal.h index fd1f7cda..00d0f4fc 100644 --- a/libbcachefs/journal.h +++ b/libbcachefs/journal.h @@ -327,10 +327,10 @@ static inline int journal_res_get_fast(struct journal *j, unsigned flags) { union journal_res_state old, new; - u64 v = atomic64_read(&j->reservations.counter); + old.v = atomic64_read(&j->reservations.counter); do { - old.v = new.v = v; + new.v = old.v; /* * Check if there is still room in the current journal @@ -356,8 +356,8 @@ static inline int journal_res_get_fast(struct journal *j, if (flags & JOURNAL_RES_GET_CHECK) return 1; - } while ((v = atomic64_cmpxchg(&j->reservations.counter, - old.v, new.v)) != old.v); + } while (!atomic64_try_cmpxchg(&j->reservations.counter, + &old.v, new.v)); res->ref = true; res->idx = old.idx; diff --git a/libbcachefs/journal_io.c b/libbcachefs/journal_io.c index cdcb1ad4..5fda9a93 100644 --- a/libbcachefs/journal_io.c +++ b/libbcachefs/journal_io.c @@ -1583,7 +1583,7 @@ static CLOSURE_CALLBACK(journal_write_done) struct bch_fs *c = container_of(j, struct bch_fs, journal); struct bch_replicas_padded replicas; union journal_res_state old, new; - u64 v, seq = le64_to_cpu(w->data->seq); + u64 seq = le64_to_cpu(w->data->seq); int err = 0; bch2_time_stats_update(!JSET_NO_FLUSH(w->data) @@ -1642,14 +1642,15 @@ static CLOSURE_CALLBACK(journal_write_done) if (j->watermark != BCH_WATERMARK_stripe) journal_reclaim_kick(&c->journal); - v = atomic64_read(&j->reservations.counter); + old.v = atomic64_read(&j->reservations.counter); do { - old.v = new.v = v; + new.v = old.v; BUG_ON(journal_state_count(new, new.unwritten_idx)); BUG_ON(new.unwritten_idx != (seq & JOURNAL_BUF_MASK)); new.unwritten_idx++; - } while ((v = atomic64_cmpxchg(&j->reservations.counter, old.v, new.v)) != old.v); + } while (!atomic64_try_cmpxchg(&j->reservations.counter, + &old.v, new.v)); closure_wake_up(&w->wait); completed = true; @@ -1847,8 +1848,14 @@ static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w) } } - if (wb.wb) - bch2_journal_keys_to_write_buffer_end(c, &wb); + if (wb.wb) { + ret = bch2_journal_keys_to_write_buffer_end(c, &wb); + if (ret) { + bch2_fs_fatal_error(c, "error flushing journal keys to btree write buffer: %s", + bch2_err_str(ret)); + return ret; + } + } spin_lock(&c->journal.lock); w->need_flush_to_write_buffer = false; diff --git a/libbcachefs/lru.c b/libbcachefs/lru.c index a40d1162..c535ba27 100644 --- a/libbcachefs/lru.c +++ b/libbcachefs/lru.c @@ -94,8 +94,8 @@ static int bch2_check_lru_key(struct btree_trans *trans, u64 idx; int ret; - if (fsck_err_on(!bch2_dev_bucket_exists(c, alloc_pos), c, - lru_entry_to_invalid_bucket, + if (fsck_err_on(!bch2_dev_bucket_exists(c, alloc_pos), + trans, lru_entry_to_invalid_bucket, "lru key points to nonexistent device:bucket %llu:%llu", alloc_pos.inode, alloc_pos.offset)) return bch2_btree_delete_at(trans, lru_iter, 0); @@ -125,7 +125,7 @@ static int bch2_check_lru_key(struct btree_trans *trans, goto out; } - if (fsck_err(c, lru_entry_bad, + if (fsck_err(trans, lru_entry_bad, "incorrect lru entry: lru %s time %llu\n" " %s\n" " for %s", diff --git a/libbcachefs/move.c b/libbcachefs/move.c index 8171f947..60e4ef94 100644 --- a/libbcachefs/move.c +++ b/libbcachefs/move.c @@ -547,6 +547,7 @@ static int bch2_move_data_btree(struct moving_context *ctxt, ctxt->stats->pos = BBPOS(btree_id, start); } + bch2_trans_begin(trans); bch2_trans_iter_init(trans, &iter, btree_id, start, BTREE_ITER_prefetch| BTREE_ITER_all_snapshots); @@ -804,7 +805,7 @@ int bch2_evacuate_bucket(struct moving_context *ctxt, if (!b) goto next; - unsigned sectors = btree_ptr_sectors_written(&b->key); + unsigned sectors = btree_ptr_sectors_written(bkey_i_to_s_c(&b->key)); ret = bch2_btree_node_rewrite(trans, &iter, b, 0); bch2_trans_iter_exit(trans, &iter); @@ -920,7 +921,20 @@ static bool rereplicate_pred(struct bch_fs *c, void *arg, ? c->opts.metadata_replicas : io_opts->data_replicas; - if (!nr_good || nr_good >= replicas) + rcu_read_lock(); + struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); + unsigned i = 0; + bkey_for_each_ptr(ptrs, ptr) { + struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); + if (!ptr->cached && + (!ca || !ca->mi.durability)) + data_opts->kill_ptrs |= BIT(i); + i++; + } + rcu_read_unlock(); + + if (!data_opts->kill_ptrs && + (!nr_good || nr_good >= replicas)) return false; data_opts->target = 0; diff --git a/libbcachefs/opts.c b/libbcachefs/opts.c index bb068fd7..e10fc1da 100644 --- a/libbcachefs/opts.c +++ b/libbcachefs/opts.c @@ -378,6 +378,10 @@ int bch2_opt_parse(struct bch_fs *c, break; case BCH_OPT_FN: ret = opt->fn.parse(c, val, res, err); + + if (ret == -BCH_ERR_option_needs_open_fs) + return ret; + if (ret < 0) { if (err) prt_printf(err, "%s: parse error", @@ -460,14 +464,81 @@ int bch2_opts_check_may_set(struct bch_fs *c) return 0; } +int bch2_parse_one_mount_opt(struct bch_fs *c, struct bch_opts *opts, + struct printbuf *parse_later, + const char *name, const char *val) +{ + struct printbuf err = PRINTBUF; + u64 v; + int ret, id; + + id = bch2_mount_opt_lookup(name); + + /* Check for the form "noopt", negation of a boolean opt: */ + if (id < 0 && + !val && + !strncmp("no", name, 2)) { + id = bch2_mount_opt_lookup(name + 2); + val = "0"; + } + + /* Unknown options are ignored: */ + if (id < 0) + return 0; + + if (!(bch2_opt_table[id].flags & OPT_MOUNT)) + goto bad_opt; + + if (id == Opt_acl && + !IS_ENABLED(CONFIG_BCACHEFS_POSIX_ACL)) + goto bad_opt; + + if ((id == Opt_usrquota || + id == Opt_grpquota) && + !IS_ENABLED(CONFIG_BCACHEFS_QUOTA)) + goto bad_opt; + + ret = bch2_opt_parse(c, &bch2_opt_table[id], val, &v, &err); + if (ret == -BCH_ERR_option_needs_open_fs && parse_later) { + prt_printf(parse_later, "%s=%s,", name, val); + if (parse_later->allocation_failure) { + ret = -ENOMEM; + goto out; + } + + ret = 0; + goto out; + } + + if (ret < 0) + goto bad_val; + + if (opts) + bch2_opt_set_by_id(opts, id, v); + + ret = 0; + goto out; + +bad_opt: + pr_err("Bad mount option %s", name); + ret = -BCH_ERR_option_name; + goto out; + +bad_val: + pr_err("Invalid mount option %s", err.buf); + ret = -BCH_ERR_option_value; + +out: + printbuf_exit(&err); + return ret; +} + int bch2_parse_mount_opts(struct bch_fs *c, struct bch_opts *opts, - char *options) + struct printbuf *parse_later, char *options) { char *copied_opts, *copied_opts_start; char *opt, *name, *val; - int ret, id; - struct printbuf err = PRINTBUF; - u64 v; + int ret; if (!options) return 0; @@ -488,53 +559,16 @@ int bch2_parse_mount_opts(struct bch_fs *c, struct bch_opts *opts, name = strsep(&opt, "="); val = opt; - id = bch2_mount_opt_lookup(name); - - /* Check for the form "noopt", negation of a boolean opt: */ - if (id < 0 && - !val && - !strncmp("no", name, 2)) { - id = bch2_mount_opt_lookup(name + 2); - val = "0"; - } - - /* Unknown options are ignored: */ - if (id < 0) - continue; - - if (!(bch2_opt_table[id].flags & OPT_MOUNT)) - goto bad_opt; - - if (id == Opt_acl && - !IS_ENABLED(CONFIG_BCACHEFS_POSIX_ACL)) - goto bad_opt; - - if ((id == Opt_usrquota || - id == Opt_grpquota) && - !IS_ENABLED(CONFIG_BCACHEFS_QUOTA)) - goto bad_opt; - - ret = bch2_opt_parse(c, &bch2_opt_table[id], val, &v, &err); + ret = bch2_parse_one_mount_opt(c, opts, parse_later, name, val); if (ret < 0) - goto bad_val; - - bch2_opt_set_by_id(opts, id, v); + goto out; } ret = 0; goto out; -bad_opt: - pr_err("Bad mount option %s", name); - ret = -BCH_ERR_option_name; - goto out; -bad_val: - pr_err("Invalid mount option %s", err.buf); - ret = -BCH_ERR_option_value; - goto out; out: kfree(copied_opts_start); - printbuf_exit(&err); return ret; } diff --git a/libbcachefs/opts.h b/libbcachefs/opts.h index 25530e0b..cff35845 100644 --- a/libbcachefs/opts.h +++ b/libbcachefs/opts.h @@ -406,7 +406,7 @@ enum fsck_err_opts { BCH2_NO_SB_OPT, BCH_SB_SECTOR, \ "offset", "Sector offset of superblock") \ x(read_only, u8, \ - OPT_FS|OPT_MOUNT, \ + OPT_FS, \ OPT_BOOL(), \ BCH2_NO_SB_OPT, false, \ NULL, NULL) \ @@ -488,6 +488,13 @@ struct bch_opts { #undef x }; +struct bch2_opts_parse { + struct bch_opts opts; + + /* to save opts that can't be parsed before the FS is opened: */ + struct printbuf parse_later; +}; + static const __maybe_unused struct bch_opts bch2_opts_default = { #define x(_name, _bits, _mode, _type, _sb_opt, _default, ...) \ ._name##_defined = true, \ @@ -566,7 +573,10 @@ void bch2_opt_to_text(struct printbuf *, struct bch_fs *, struct bch_sb *, int bch2_opt_check_may_set(struct bch_fs *, int, u64); int bch2_opts_check_may_set(struct bch_fs *); -int bch2_parse_mount_opts(struct bch_fs *, struct bch_opts *, char *); +int bch2_parse_one_mount_opt(struct bch_fs *, struct bch_opts *, + struct printbuf *, const char *, const char *); +int bch2_parse_mount_opts(struct bch_fs *, struct bch_opts *, struct printbuf *, + char *); /* inode opts: */ diff --git a/libbcachefs/printbuf.c b/libbcachefs/printbuf.c index 9f529e4c..4cf5a2af 100644 --- a/libbcachefs/printbuf.c +++ b/libbcachefs/printbuf.c @@ -316,6 +316,20 @@ void bch2_prt_newline(struct printbuf *buf) buf->cur_tabstop = 0; } +void bch2_printbuf_strip_trailing_newline(struct printbuf *out) +{ + for (int p = out->pos - 1; p >= 0; --p) { + if (out->buf[p] == '\n') { + out->pos = p; + break; + } + if (out->buf[p] != ' ') + break; + } + + printbuf_nul_terminate_reserved(out); +} + static void __prt_tab(struct printbuf *out) { int spaces = max_t(int, 0, cur_tabstop(out) - printbuf_linelen(out)); diff --git a/libbcachefs/printbuf.h b/libbcachefs/printbuf.h index 9ecc56bc..1d570387 100644 --- a/libbcachefs/printbuf.h +++ b/libbcachefs/printbuf.h @@ -115,6 +115,7 @@ void bch2_printbuf_indent_add(struct printbuf *, unsigned); void bch2_printbuf_indent_sub(struct printbuf *, unsigned); void bch2_prt_newline(struct printbuf *); +void bch2_printbuf_strip_trailing_newline(struct printbuf *); void bch2_prt_tab(struct printbuf *); void bch2_prt_tab_rjust(struct printbuf *); diff --git a/libbcachefs/recovery.c b/libbcachefs/recovery.c index cf513fc7..3ebffd04 100644 --- a/libbcachefs/recovery.c +++ b/libbcachefs/recovery.c @@ -10,6 +10,7 @@ #include "btree_io.h" #include "buckets.h" #include "dirent.h" +#include "disk_accounting.h" #include "errcode.h" #include "error.h" #include "fs-common.h" @@ -90,6 +91,7 @@ static void bch2_reconstruct_alloc(struct bch_fs *c) __set_bit_le64(BCH_FSCK_ERR_freespace_hole_missing, ext->errors_silent); __set_bit_le64(BCH_FSCK_ERR_ptr_to_missing_backpointer, ext->errors_silent); __set_bit_le64(BCH_FSCK_ERR_lru_entry_bad, ext->errors_silent); + __set_bit_le64(BCH_FSCK_ERR_accounting_mismatch, ext->errors_silent); c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info); bch2_write_super(c); @@ -134,6 +136,45 @@ static void replay_now_at(struct journal *j, u64 seq) bch2_journal_pin_put(j, j->replay_journal_seq++); } +static int bch2_journal_replay_accounting_key(struct btree_trans *trans, + struct journal_key *k) +{ + struct btree_iter iter; + bch2_trans_node_iter_init(trans, &iter, k->btree_id, k->k->k.p, + BTREE_MAX_DEPTH, k->level, + BTREE_ITER_intent); + int ret = bch2_btree_iter_traverse(&iter); + if (ret) + goto out; + + struct bkey u; + struct bkey_s_c old = bch2_btree_path_peek_slot(btree_iter_path(trans, &iter), &u); + + /* Has this delta already been applied to the btree? */ + if (bversion_cmp(old.k->version, k->k->k.version) >= 0) { + ret = 0; + goto out; + } + + struct bkey_i *new = k->k; + if (old.k->type == KEY_TYPE_accounting) { + new = bch2_bkey_make_mut_noupdate(trans, bkey_i_to_s_c(k->k)); + ret = PTR_ERR_OR_ZERO(new); + if (ret) + goto out; + + bch2_accounting_accumulate(bkey_i_to_accounting(new), + bkey_s_c_to_accounting(old)); + } + + trans->journal_res.seq = k->journal_seq; + + ret = bch2_trans_update(trans, &iter, new, BTREE_TRIGGER_norun); +out: + bch2_trans_iter_exit(trans, &iter); + return ret; +} + static int bch2_journal_replay_key(struct btree_trans *trans, struct journal_key *k) { @@ -184,6 +225,11 @@ static int bch2_journal_replay_key(struct btree_trans *trans, if (k->overwritten) goto out; + if (k->k->k.type == KEY_TYPE_accounting) { + ret = bch2_trans_update_buffered(trans, BTREE_ID_accounting, k->k); + goto out; + } + ret = bch2_trans_update(trans, &iter, k->k, update_flags); out: bch2_trans_iter_exit(trans, &iter); @@ -222,6 +268,30 @@ int bch2_journal_replay(struct bch_fs *c) trans = bch2_trans_get(c); /* + * Replay accounting keys first: we can't allow the write buffer to + * flush accounting keys until we're done + */ + darray_for_each(*keys, k) { + if (!(k->k->k.type == KEY_TYPE_accounting && !k->allocated)) + continue; + + cond_resched(); + + ret = commit_do(trans, NULL, NULL, + BCH_TRANS_COMMIT_no_enospc| + BCH_TRANS_COMMIT_journal_reclaim| + BCH_TRANS_COMMIT_skip_accounting_apply| + BCH_TRANS_COMMIT_no_journal_res, + bch2_journal_replay_accounting_key(trans, k)); + if (bch2_fs_fatal_err_on(ret, c, "error replaying accounting; %s", bch2_err_str(ret))) + goto err; + + k->overwritten = true; + } + + set_bit(BCH_FS_accounting_replay_done, &c->flags); + + /* * First, attempt to replay keys in sorted order. This is more * efficient - better locality of btree access - but some might fail if * that would cause a journal deadlock. @@ -241,9 +311,10 @@ int bch2_journal_replay(struct bch_fs *c) commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc| BCH_TRANS_COMMIT_journal_reclaim| + BCH_TRANS_COMMIT_skip_accounting_apply| (!k->allocated ? BCH_TRANS_COMMIT_no_journal_res : 0), bch2_journal_replay_key(trans, k)); - BUG_ON(!ret && !k->overwritten); + BUG_ON(!ret && !k->overwritten && k->k->k.type != KEY_TYPE_accounting); if (ret) { ret = darray_push(&keys_sorted, k); if (ret) @@ -271,6 +342,7 @@ int bch2_journal_replay(struct bch_fs *c) ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc| + BCH_TRANS_COMMIT_skip_accounting_apply| (!k->allocated ? BCH_TRANS_COMMIT_no_journal_res|BCH_WATERMARK_reclaim : 0), @@ -280,7 +352,7 @@ int bch2_journal_replay(struct bch_fs *c) if (ret) goto err; - BUG_ON(!k->overwritten); + BUG_ON(k->btree_id != BTREE_ID_accounting && !k->overwritten); } /* @@ -349,45 +421,10 @@ static int journal_replay_entry_early(struct bch_fs *c, container_of(entry, struct jset_entry_usage, entry); switch (entry->btree_id) { - case BCH_FS_USAGE_reserved: - if (entry->level < BCH_REPLICAS_MAX) - c->usage_base->persistent_reserved[entry->level] = - le64_to_cpu(u->v); - break; - case BCH_FS_USAGE_inodes: - c->usage_base->b.nr_inodes = le64_to_cpu(u->v); - break; case BCH_FS_USAGE_key_version: - atomic64_set(&c->key_version, - le64_to_cpu(u->v)); + atomic64_set(&c->key_version, le64_to_cpu(u->v)); break; } - - break; - } - case BCH_JSET_ENTRY_data_usage: { - struct jset_entry_data_usage *u = - container_of(entry, struct jset_entry_data_usage, entry); - - ret = bch2_replicas_set_usage(c, &u->r, - le64_to_cpu(u->v)); - break; - } - case BCH_JSET_ENTRY_dev_usage: { - struct jset_entry_dev_usage *u = - container_of(entry, struct jset_entry_dev_usage, entry); - unsigned nr_types = jset_entry_dev_usage_nr_types(u); - - rcu_read_lock(); - struct bch_dev *ca = bch2_dev_rcu(c, le32_to_cpu(u->dev)); - if (ca) - for (unsigned i = 0; i < min_t(unsigned, nr_types, BCH_DATA_NR); i++) { - ca->usage_base->d[i].buckets = le64_to_cpu(u->d[i].buckets); - ca->usage_base->d[i].sectors = le64_to_cpu(u->d[i].sectors); - ca->usage_base->d[i].fragmented = le64_to_cpu(u->d[i].fragmented); - } - rcu_read_unlock(); - break; } case BCH_JSET_ENTRY_blacklist: { @@ -448,8 +485,6 @@ static int journal_replay_early(struct bch_fs *c, } } - bch2_fs_usage_initialize(c); - return 0; } @@ -804,6 +839,10 @@ use_clean: if (ret) goto err; + set_bit(BCH_FS_btree_running, &c->flags); + + ret = bch2_sb_set_upgrade_extra(c); + ret = bch2_run_recovery_passes(c); if (ret) goto err; @@ -963,14 +1002,12 @@ int bch2_fs_initialize(struct bch_fs *c) mutex_unlock(&c->sb_lock); c->curr_recovery_pass = BCH_RECOVERY_PASS_NR; + set_bit(BCH_FS_btree_running, &c->flags); set_bit(BCH_FS_may_go_rw, &c->flags); for (unsigned i = 0; i < BTREE_ID_NR; i++) bch2_btree_root_alloc_fake(c, i, 0); - for_each_member_device(c, ca) - bch2_dev_usage_init(ca); - ret = bch2_fs_journal_alloc(c); if (ret) goto err; @@ -980,12 +1017,21 @@ int bch2_fs_initialize(struct bch_fs *c) * set up the journal.pin FIFO and journal.cur pointer: */ bch2_fs_journal_start(&c->journal, 1); + set_bit(BCH_FS_accounting_replay_done, &c->flags); bch2_journal_set_replay_done(&c->journal); ret = bch2_fs_read_write_early(c); if (ret) goto err; + for_each_member_device(c, ca) { + ret = bch2_dev_usage_init(ca, false); + if (ret) { + bch2_dev_put(ca); + goto err; + } + } + /* * Write out the superblock and journal buckets, now that we can do * btree updates @@ -1019,7 +1065,7 @@ int bch2_fs_initialize(struct bch_fs *c) bch2_inode_pack(&packed_inode, &root_inode); packed_inode.inode.k.p.snapshot = U32_MAX; - ret = bch2_btree_insert(c, BTREE_ID_inodes, &packed_inode.inode.k_i, NULL, 0); + ret = bch2_btree_insert(c, BTREE_ID_inodes, &packed_inode.inode.k_i, NULL, 0, 0); bch_err_msg(c, ret, "creating root directory"); if (ret) goto err; diff --git a/libbcachefs/recovery_passes.c b/libbcachefs/recovery_passes.c index 4a9eb958..73339a0a 100644 --- a/libbcachefs/recovery_passes.c +++ b/libbcachefs/recovery_passes.c @@ -5,6 +5,7 @@ #include "backpointers.h" #include "btree_gc.h" #include "btree_node_scan.h" +#include "disk_accounting.h" #include "ec.h" #include "fsck.h" #include "inode.h" @@ -192,6 +193,8 @@ int bch2_run_online_recovery_passes(struct bch_fs *c) { int ret = 0; + down_read(&c->state_lock); + for (unsigned i = 0; i < ARRAY_SIZE(recovery_pass_fns); i++) { struct recovery_pass_fn *p = recovery_pass_fns + i; @@ -207,6 +210,8 @@ int bch2_run_online_recovery_passes(struct bch_fs *c) break; } + up_read(&c->state_lock); + return ret; } diff --git a/libbcachefs/recovery_passes_types.h b/libbcachefs/recovery_passes_types.h index 773aea9a..8c7dee59 100644 --- a/libbcachefs/recovery_passes_types.h +++ b/libbcachefs/recovery_passes_types.h @@ -15,6 +15,7 @@ #define BCH_RECOVERY_PASSES() \ x(scan_for_btree_nodes, 37, 0) \ x(check_topology, 4, 0) \ + x(accounting_read, 39, PASS_ALWAYS) \ x(alloc_read, 0, PASS_ALWAYS) \ x(stripes_read, 1, PASS_ALWAYS) \ x(initialize_subvolumes, 2, 0) \ diff --git a/libbcachefs/reflink.c b/libbcachefs/reflink.c index 9ac6cf21..5f92715e 100644 --- a/libbcachefs/reflink.c +++ b/libbcachefs/reflink.c @@ -171,7 +171,7 @@ static s64 gc_trigger_reflink_p_segment(struct btree_trans *trans, not_found: BUG_ON(!(flags & BTREE_TRIGGER_check_repair)); - if (fsck_err(c, reflink_p_to_missing_reflink_v, + if (fsck_err(trans, reflink_p_to_missing_reflink_v, "pointer to missing indirect extent\n" " %s\n" " missing range %llu-%llu", diff --git a/libbcachefs/replicas.c b/libbcachefs/replicas.c index 57a1f09c..06f6d48f 100644 --- a/libbcachefs/replicas.c +++ b/libbcachefs/replicas.c @@ -2,6 +2,7 @@ #include "bcachefs.h" #include "buckets.h" +#include "disk_accounting.h" #include "journal.h" #include "replicas.h" #include "super-io.h" @@ -243,145 +244,25 @@ static bool __replicas_has_entry(struct bch_replicas_cpu *r, return __replicas_entry_idx(r, search) >= 0; } -bool bch2_replicas_marked(struct bch_fs *c, +bool bch2_replicas_marked_locked(struct bch_fs *c, struct bch_replicas_entry_v1 *search) { - bool marked; - - if (!search->nr_devs) - return true; - verify_replicas_entry(search); - percpu_down_read(&c->mark_lock); - marked = __replicas_has_entry(&c->replicas, search) && - (likely((!c->replicas_gc.entries)) || - __replicas_has_entry(&c->replicas_gc, search)); - percpu_up_read(&c->mark_lock); - - return marked; + return !search->nr_devs || + (__replicas_has_entry(&c->replicas, search) && + (likely((!c->replicas_gc.entries)) || + __replicas_has_entry(&c->replicas_gc, search))); } -static void __replicas_table_update(struct bch_fs_usage *dst, - struct bch_replicas_cpu *dst_r, - struct bch_fs_usage *src, - struct bch_replicas_cpu *src_r) -{ - int src_idx, dst_idx; - - *dst = *src; - - for (src_idx = 0; src_idx < src_r->nr; src_idx++) { - if (!src->replicas[src_idx]) - continue; - - dst_idx = __replicas_entry_idx(dst_r, - cpu_replicas_entry(src_r, src_idx)); - BUG_ON(dst_idx < 0); - - dst->replicas[dst_idx] = src->replicas[src_idx]; - } -} - -static void __replicas_table_update_pcpu(struct bch_fs_usage __percpu *dst_p, - struct bch_replicas_cpu *dst_r, - struct bch_fs_usage __percpu *src_p, - struct bch_replicas_cpu *src_r) -{ - unsigned src_nr = sizeof(struct bch_fs_usage) / sizeof(u64) + src_r->nr; - struct bch_fs_usage *dst, *src = (void *) - bch2_acc_percpu_u64s((u64 __percpu *) src_p, src_nr); - - preempt_disable(); - dst = this_cpu_ptr(dst_p); - preempt_enable(); - - __replicas_table_update(dst, dst_r, src, src_r); -} - -/* - * Resize filesystem accounting: - */ -static int replicas_table_update(struct bch_fs *c, - struct bch_replicas_cpu *new_r) +bool bch2_replicas_marked(struct bch_fs *c, + struct bch_replicas_entry_v1 *search) { - struct bch_fs_usage __percpu *new_usage[JOURNAL_BUF_NR]; - struct bch_fs_usage_online *new_scratch = NULL; - struct bch_fs_usage __percpu *new_gc = NULL; - struct bch_fs_usage *new_base = NULL; - unsigned i, bytes = sizeof(struct bch_fs_usage) + - sizeof(u64) * new_r->nr; - unsigned scratch_bytes = sizeof(struct bch_fs_usage_online) + - sizeof(u64) * new_r->nr; - int ret = 0; - - memset(new_usage, 0, sizeof(new_usage)); - - for (i = 0; i < ARRAY_SIZE(new_usage); i++) - if (!(new_usage[i] = __alloc_percpu_gfp(bytes, - sizeof(u64), GFP_KERNEL))) - goto err; + percpu_down_read(&c->mark_lock); + bool ret = bch2_replicas_marked_locked(c, search); + percpu_up_read(&c->mark_lock); - if (!(new_base = kzalloc(bytes, GFP_KERNEL)) || - !(new_scratch = kmalloc(scratch_bytes, GFP_KERNEL)) || - (c->usage_gc && - !(new_gc = __alloc_percpu_gfp(bytes, sizeof(u64), GFP_KERNEL)))) - goto err; - - for (i = 0; i < ARRAY_SIZE(new_usage); i++) - if (c->usage[i]) - __replicas_table_update_pcpu(new_usage[i], new_r, - c->usage[i], &c->replicas); - if (c->usage_base) - __replicas_table_update(new_base, new_r, - c->usage_base, &c->replicas); - if (c->usage_gc) - __replicas_table_update_pcpu(new_gc, new_r, - c->usage_gc, &c->replicas); - - for (i = 0; i < ARRAY_SIZE(new_usage); i++) - swap(c->usage[i], new_usage[i]); - swap(c->usage_base, new_base); - swap(c->usage_scratch, new_scratch); - swap(c->usage_gc, new_gc); - swap(c->replicas, *new_r); -out: - free_percpu(new_gc); - kfree(new_scratch); - for (i = 0; i < ARRAY_SIZE(new_usage); i++) - free_percpu(new_usage[i]); - kfree(new_base); return ret; -err: - bch_err(c, "error updating replicas table: memory allocation failure"); - ret = -BCH_ERR_ENOMEM_replicas_table; - goto out; -} - -static unsigned reserve_journal_replicas(struct bch_fs *c, - struct bch_replicas_cpu *r) -{ - struct bch_replicas_entry_v1 *e; - unsigned journal_res_u64s = 0; - - /* nr_inodes: */ - journal_res_u64s += - DIV_ROUND_UP(sizeof(struct jset_entry_usage), sizeof(u64)); - - /* key_version: */ - journal_res_u64s += - DIV_ROUND_UP(sizeof(struct jset_entry_usage), sizeof(u64)); - - /* persistent_reserved: */ - journal_res_u64s += - DIV_ROUND_UP(sizeof(struct jset_entry_usage), sizeof(u64)) * - BCH_REPLICAS_MAX; - - for_each_cpu_replicas_entry(r, e) - journal_res_u64s += - DIV_ROUND_UP(sizeof(struct jset_entry_data_usage) + - e->nr_devs, sizeof(u64)); - return journal_res_u64s; } noinline @@ -417,10 +298,6 @@ static int bch2_mark_replicas_slowpath(struct bch_fs *c, ret = bch2_cpu_replicas_to_sb_replicas(c, &new_r); if (ret) goto err; - - bch2_journal_entry_res_resize(&c->journal, - &c->replicas_journal_res, - reserve_journal_replicas(c, &new_r)); } if (!new_r.entries && @@ -435,7 +312,7 @@ static int bch2_mark_replicas_slowpath(struct bch_fs *c, /* don't update in memory replicas until changes are persistent */ percpu_down_write(&c->mark_lock); if (new_r.entries) - ret = replicas_table_update(c, &new_r); + swap(c->replicas, new_r); if (new_gc.entries) swap(new_gc, c->replicas_gc); percpu_up_write(&c->mark_lock); @@ -457,20 +334,6 @@ int bch2_mark_replicas(struct bch_fs *c, struct bch_replicas_entry_v1 *r) ? 0 : bch2_mark_replicas_slowpath(c, r); } -/* replicas delta list: */ - -int bch2_replicas_delta_list_mark(struct bch_fs *c, - struct replicas_delta_list *r) -{ - struct replicas_delta *d = r->d; - struct replicas_delta *top = (void *) r->d + r->used; - int ret = 0; - - for (d = r->d; !ret && d != top; d = replicas_delta_next(d)) - ret = bch2_mark_replicas(c, &d->r); - return ret; -} - /* * Old replicas_gc mechanism: only used for journal replicas entries now, should * die at some point: @@ -484,8 +347,9 @@ int bch2_replicas_gc_end(struct bch_fs *c, int ret) percpu_down_write(&c->mark_lock); ret = ret ?: - bch2_cpu_replicas_to_sb_replicas(c, &c->replicas_gc) ?: - replicas_table_update(c, &c->replicas_gc); + bch2_cpu_replicas_to_sb_replicas(c, &c->replicas_gc); + if (!ret) + swap(c->replicas, c->replicas_gc); kfree(c->replicas_gc.entries); c->replicas_gc.entries = NULL; @@ -584,20 +448,26 @@ retry: struct bch_replicas_entry_v1 *e = cpu_replicas_entry(&c->replicas, i); - if (e->data_type == BCH_DATA_journal || - c->usage_base->replicas[i] || - percpu_u64_get(&c->usage[0]->replicas[i]) || - percpu_u64_get(&c->usage[1]->replicas[i]) || - percpu_u64_get(&c->usage[2]->replicas[i]) || - percpu_u64_get(&c->usage[3]->replicas[i])) + struct disk_accounting_pos k = { + .type = BCH_DISK_ACCOUNTING_replicas, + }; + + memcpy(&k.replicas, e, replicas_entry_bytes(e)); + + u64 v = 0; + bch2_accounting_mem_read(c, disk_accounting_pos_to_bpos(&k), &v, 1); + + if (e->data_type == BCH_DATA_journal || v) memcpy(cpu_replicas_entry(&new, new.nr++), e, new.entry_size); } bch2_cpu_replicas_sort(&new); - ret = bch2_cpu_replicas_to_sb_replicas(c, &new) ?: - replicas_table_update(c, &new); + ret = bch2_cpu_replicas_to_sb_replicas(c, &new); + + if (!ret) + swap(c->replicas, new); kfree(new.entries); @@ -611,34 +481,6 @@ retry: return ret; } -int bch2_replicas_set_usage(struct bch_fs *c, - struct bch_replicas_entry_v1 *r, - u64 sectors) -{ - int ret, idx = bch2_replicas_entry_idx(c, r); - - if (idx < 0) { - struct bch_replicas_cpu n; - - n = cpu_replicas_add_entry(c, &c->replicas, r); - if (!n.entries) - return -BCH_ERR_ENOMEM_cpu_replicas; - - ret = replicas_table_update(c, &n); - if (ret) - return ret; - - kfree(n.entries); - - idx = bch2_replicas_entry_idx(c, r); - BUG_ON(ret < 0); - } - - c->usage_base->replicas[idx] = sectors; - - return 0; -} - /* Replicas tracking - superblock: */ static int @@ -724,8 +566,7 @@ int bch2_sb_replicas_to_cpu_replicas(struct bch_fs *c) bch2_cpu_replicas_sort(&new_r); percpu_down_write(&c->mark_lock); - - ret = replicas_table_update(c, &new_r); + swap(c->replicas, new_r); percpu_up_write(&c->mark_lock); kfree(new_r.entries); @@ -1027,10 +868,8 @@ unsigned bch2_sb_dev_has_data(struct bch_sb *sb, unsigned dev) unsigned bch2_dev_has_data(struct bch_fs *c, struct bch_dev *ca) { - unsigned ret; - mutex_lock(&c->sb_lock); - ret = bch2_sb_dev_has_data(c->disk_sb.sb, ca->dev_idx); + unsigned ret = bch2_sb_dev_has_data(c->disk_sb.sb, ca->dev_idx); mutex_unlock(&c->sb_lock); return ret; @@ -1038,25 +877,6 @@ unsigned bch2_dev_has_data(struct bch_fs *c, struct bch_dev *ca) void bch2_fs_replicas_exit(struct bch_fs *c) { - unsigned i; - - kfree(c->usage_scratch); - for (i = 0; i < ARRAY_SIZE(c->usage); i++) - free_percpu(c->usage[i]); - kfree(c->usage_base); kfree(c->replicas.entries); kfree(c->replicas_gc.entries); - - mempool_exit(&c->replicas_delta_pool); -} - -int bch2_fs_replicas_init(struct bch_fs *c) -{ - bch2_journal_entry_res_resize(&c->journal, - &c->replicas_journal_res, - reserve_journal_replicas(c, &c->replicas)); - - return mempool_init_kmalloc_pool(&c->replicas_delta_pool, 1, - REPLICAS_DELTA_LIST_MAX) ?: - replicas_table_update(c, &c->replicas); } diff --git a/libbcachefs/replicas.h b/libbcachefs/replicas.h index 654a4b26..62248255 100644 --- a/libbcachefs/replicas.h +++ b/libbcachefs/replicas.h @@ -25,18 +25,13 @@ int bch2_replicas_entry_idx(struct bch_fs *, void bch2_devlist_to_replicas(struct bch_replicas_entry_v1 *, enum bch_data_type, struct bch_devs_list); + +bool bch2_replicas_marked_locked(struct bch_fs *, + struct bch_replicas_entry_v1 *); bool bch2_replicas_marked(struct bch_fs *, struct bch_replicas_entry_v1 *); int bch2_mark_replicas(struct bch_fs *, struct bch_replicas_entry_v1 *); -static inline struct replicas_delta * -replicas_delta_next(struct replicas_delta *d) -{ - return (void *) d + replicas_entry_bytes(&d->r) + 8; -} - -int bch2_replicas_delta_list_mark(struct bch_fs *, struct replicas_delta_list *); - void bch2_bkey_to_replicas(struct bch_replicas_entry_v1 *, struct bkey_s_c); static inline void bch2_replicas_entry_cached(struct bch_replicas_entry_v1 *e, @@ -58,10 +53,6 @@ int bch2_replicas_gc_end(struct bch_fs *, int); int bch2_replicas_gc_start(struct bch_fs *, unsigned); int bch2_replicas_gc2(struct bch_fs *); -int bch2_replicas_set_usage(struct bch_fs *, - struct bch_replicas_entry_v1 *, - u64); - #define for_each_cpu_replicas_entry(_r, _i) \ for (_i = (_r)->entries; \ (void *) (_i) < (void *) (_r)->entries + (_r)->nr * (_r)->entry_size;\ @@ -88,6 +79,5 @@ extern const struct bch_sb_field_ops bch_sb_field_ops_replicas; extern const struct bch_sb_field_ops bch_sb_field_ops_replicas_v0; void bch2_fs_replicas_exit(struct bch_fs *); -int bch2_fs_replicas_init(struct bch_fs *); #endif /* _BCACHEFS_REPLICAS_H */ diff --git a/libbcachefs/replicas_types.h b/libbcachefs/replicas_types.h index ac90d142..fed71c86 100644 --- a/libbcachefs/replicas_types.h +++ b/libbcachefs/replicas_types.h @@ -8,20 +8,4 @@ struct bch_replicas_cpu { struct bch_replicas_entry_v1 *entries; }; -struct replicas_delta { - s64 delta; - struct bch_replicas_entry_v1 r; -} __packed; - -struct replicas_delta_list { - unsigned size; - unsigned used; - - struct {} memset_start; - u64 nr_inodes; - u64 persistent_reserved[BCH_REPLICAS_MAX]; - struct {} memset_end; - struct replicas_delta d[]; -}; - #endif /* _BCACHEFS_REPLICAS_TYPES_H */ diff --git a/libbcachefs/sb-clean.c b/libbcachefs/sb-clean.c index 47f10ab5..c57d42bb 100644 --- a/libbcachefs/sb-clean.c +++ b/libbcachefs/sb-clean.c @@ -183,25 +183,6 @@ void bch2_journal_super_entries_add_common(struct bch_fs *c, struct jset_entry **end, u64 journal_seq) { - percpu_down_read(&c->mark_lock); - - if (!journal_seq) { - for (unsigned i = 0; i < ARRAY_SIZE(c->usage); i++) - bch2_fs_usage_acc_to_base(c, i); - } else { - bch2_fs_usage_acc_to_base(c, journal_seq & JOURNAL_BUF_MASK); - } - - { - struct jset_entry_usage *u = - container_of(jset_entry_init(end, sizeof(*u)), - struct jset_entry_usage, entry); - - u->entry.type = BCH_JSET_ENTRY_usage; - u->entry.btree_id = BCH_FS_USAGE_inodes; - u->v = cpu_to_le64(c->usage_base->b.nr_inodes); - } - { struct jset_entry_usage *u = container_of(jset_entry_init(end, sizeof(*u)), @@ -212,49 +193,6 @@ void bch2_journal_super_entries_add_common(struct bch_fs *c, u->v = cpu_to_le64(atomic64_read(&c->key_version)); } - for (unsigned i = 0; i < BCH_REPLICAS_MAX; i++) { - struct jset_entry_usage *u = - container_of(jset_entry_init(end, sizeof(*u)), - struct jset_entry_usage, entry); - - u->entry.type = BCH_JSET_ENTRY_usage; - u->entry.btree_id = BCH_FS_USAGE_reserved; - u->entry.level = i; - u->v = cpu_to_le64(c->usage_base->persistent_reserved[i]); - } - - for (unsigned i = 0; i < c->replicas.nr; i++) { - struct bch_replicas_entry_v1 *e = - cpu_replicas_entry(&c->replicas, i); - struct jset_entry_data_usage *u = - container_of(jset_entry_init(end, sizeof(*u) + e->nr_devs), - struct jset_entry_data_usage, entry); - - u->entry.type = BCH_JSET_ENTRY_data_usage; - u->v = cpu_to_le64(c->usage_base->replicas[i]); - unsafe_memcpy(&u->r, e, replicas_entry_bytes(e), - "embedded variable length struct"); - } - - for_each_member_device(c, ca) { - unsigned b = sizeof(struct jset_entry_dev_usage) + - sizeof(struct jset_entry_dev_usage_type) * BCH_DATA_NR; - struct jset_entry_dev_usage *u = - container_of(jset_entry_init(end, b), - struct jset_entry_dev_usage, entry); - - u->entry.type = BCH_JSET_ENTRY_dev_usage; - u->dev = cpu_to_le32(ca->dev_idx); - - for (unsigned i = 0; i < BCH_DATA_NR; i++) { - u->d[i].buckets = cpu_to_le64(ca->usage_base->d[i].buckets); - u->d[i].sectors = cpu_to_le64(ca->usage_base->d[i].sectors); - u->d[i].fragmented = cpu_to_le64(ca->usage_base->d[i].fragmented); - } - } - - percpu_up_read(&c->mark_lock); - for (unsigned i = 0; i < 2; i++) { struct jset_entry_clock *clock = container_of(jset_entry_init(end, sizeof(*clock)), diff --git a/libbcachefs/sb-downgrade.c b/libbcachefs/sb-downgrade.c index 3fb23e39..739919e2 100644 --- a/libbcachefs/sb-downgrade.c +++ b/libbcachefs/sb-downgrade.c @@ -54,9 +54,27 @@ BCH_FSCK_ERR_subvol_children_not_set) \ x(mi_btree_bitmap, \ BIT_ULL(BCH_RECOVERY_PASS_check_allocations), \ - BCH_FSCK_ERR_btree_bitmap_not_marked) - -#define DOWNGRADE_TABLE() + BCH_FSCK_ERR_btree_bitmap_not_marked) \ + x(disk_accounting_v2, \ + BIT_ULL(BCH_RECOVERY_PASS_check_allocations), \ + BCH_FSCK_ERR_accounting_mismatch) + +#define DOWNGRADE_TABLE() \ + x(bucket_stripe_sectors, \ + 0) \ + x(disk_accounting_v2, \ + BIT_ULL(BCH_RECOVERY_PASS_check_alloc_info), \ + BCH_FSCK_ERR_dev_usage_buckets_wrong, \ + BCH_FSCK_ERR_dev_usage_sectors_wrong, \ + BCH_FSCK_ERR_dev_usage_fragmented_wrong, \ + BCH_FSCK_ERR_fs_usage_hidden_wrong, \ + BCH_FSCK_ERR_fs_usage_btree_wrong, \ + BCH_FSCK_ERR_fs_usage_data_wrong, \ + BCH_FSCK_ERR_fs_usage_cached_wrong, \ + BCH_FSCK_ERR_fs_usage_reserved_wrong, \ + BCH_FSCK_ERR_fs_usage_nr_inodes_wrong, \ + BCH_FSCK_ERR_fs_usage_replicas_wrong, \ + BCH_FSCK_ERR_bkey_version_in_future) struct upgrade_downgrade_entry { u64 recovery_passes; @@ -80,6 +98,37 @@ UPGRADE_TABLE() #undef x }; +static int have_stripes(struct bch_fs *c) +{ + return !btree_node_fake(c->btree_roots_known[BTREE_ID_stripes].b); +} + +int bch2_sb_set_upgrade_extra(struct bch_fs *c) +{ + unsigned old_version = c->sb.version_upgrade_complete ?: c->sb.version; + unsigned new_version = c->sb.version; + bool write_sb = false; + int ret = 0; + + mutex_lock(&c->sb_lock); + struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext); + + if (old_version < bcachefs_metadata_version_bucket_stripe_sectors && + new_version >= bcachefs_metadata_version_bucket_stripe_sectors && + (ret = have_stripes(c) > 0)) { + __set_bit_le64(BCH_RECOVERY_PASS_STABLE_check_allocations, ext->recovery_passes_required); + __set_bit_le64(BCH_FSCK_ERR_alloc_key_dirty_sectors_wrong, ext->errors_silent); + __set_bit_le64(BCH_FSCK_ERR_alloc_key_stripe_sectors_wrong, ext->errors_silent); + write_sb = true; + } + + if (write_sb) + bch2_write_super(c); + mutex_unlock(&c->sb_lock); + + return ret < 0 ? ret : 0; +} + void bch2_sb_set_upgrade(struct bch_fs *c, unsigned old_version, unsigned new_version) @@ -101,16 +150,12 @@ void bch2_sb_set_upgrade(struct bch_fs *c, ext->recovery_passes_required[0] |= cpu_to_le64(bch2_recovery_passes_to_stable(passes)); - for (const u16 *e = i->errors; - e < i->errors + i->nr_errors; - e++) { - __set_bit(*e, c->sb.errors_silent); - ext->errors_silent[*e / 64] |= cpu_to_le64(BIT_ULL(*e % 64)); - } + for (const u16 *e = i->errors; e < i->errors + i->nr_errors; e++) + __set_bit_le64(*e, ext->errors_silent); } } -#define x(ver, passes, ...) static const u16 downgrade_ver_##errors[] = { __VA_ARGS__ }; +#define x(ver, passes, ...) static const u16 downgrade_##ver##_errors[] = { __VA_ARGS__ }; DOWNGRADE_TABLE() #undef x @@ -125,6 +170,33 @@ DOWNGRADE_TABLE() #undef x }; +static int downgrade_table_extra(struct bch_fs *c, darray_char *table) +{ + struct bch_sb_field_downgrade_entry *dst = (void *) &darray_top(*table); + unsigned bytes = sizeof(*dst) + sizeof(dst->errors[0]) * le16_to_cpu(dst->nr_errors); + int ret = 0; + + unsigned nr_errors = le16_to_cpu(dst->nr_errors); + + switch (le16_to_cpu(dst->version)) { + case bcachefs_metadata_version_bucket_stripe_sectors: + if (have_stripes(c)) { + bytes += sizeof(dst->errors[0]) * 2; + + ret = darray_make_room(table, bytes); + if (ret) + return ret; + + __set_bit_le64(BCH_RECOVERY_PASS_STABLE_check_allocations, dst->recovery_passes); + dst->errors[nr_errors++] = cpu_to_le16(BCH_FSCK_ERR_alloc_key_dirty_sectors_wrong); + } + break; + } + + dst->nr_errors = cpu_to_le16(nr_errors); + return ret; +} + static inline const struct bch_sb_field_downgrade_entry * downgrade_entry_next_c(const struct bch_sb_field_downgrade_entry *e) { @@ -210,6 +282,9 @@ const struct bch_sb_field_ops bch_sb_field_ops_downgrade = { int bch2_sb_downgrade_update(struct bch_fs *c) { + if (!test_bit(BCH_FS_btree_running, &c->flags)) + return 0; + darray_char table = {}; int ret = 0; @@ -234,7 +309,14 @@ int bch2_sb_downgrade_update(struct bch_fs *c) for (unsigned i = 0; i < src->nr_errors; i++) dst->errors[i] = cpu_to_le16(src->errors[i]); - table.nr += bytes; + downgrade_table_extra(c, &table); + + if (!dst->recovery_passes[0] && + !dst->recovery_passes[1] && + !dst->nr_errors) + continue; + + table.nr += sizeof(*dst) + sizeof(dst->errors[0]) * le16_to_cpu(dst->nr_errors); } struct bch_sb_field_downgrade *d = bch2_sb_field_get(c->disk_sb.sb, downgrade); diff --git a/libbcachefs/sb-downgrade.h b/libbcachefs/sb-downgrade.h index 57e6c916..095b7cc9 100644 --- a/libbcachefs/sb-downgrade.h +++ b/libbcachefs/sb-downgrade.h @@ -6,6 +6,7 @@ extern const struct bch_sb_field_ops bch_sb_field_ops_downgrade; int bch2_sb_downgrade_update(struct bch_fs *); void bch2_sb_set_upgrade(struct bch_fs *, unsigned, unsigned); +int bch2_sb_set_upgrade_extra(struct bch_fs *); void bch2_sb_set_downgrade(struct bch_fs *, unsigned, unsigned); #endif /* _BCACHEFS_SB_DOWNGRADE_H */ diff --git a/libbcachefs/sb-errors_format.h b/libbcachefs/sb-errors_format.h index 84d2763b..ced43190 100644 --- a/libbcachefs/sb-errors_format.h +++ b/libbcachefs/sb-errors_format.h @@ -219,8 +219,8 @@ x(deleted_inode_is_dir, 213) \ x(deleted_inode_not_unlinked, 214) \ x(extent_overlapping, 215) \ - x(extent_in_missing_inode, 216) \ - x(extent_in_non_reg_inode, 217) \ + x(key_in_missing_inode, 216) \ + x(key_in_wrong_inode_type, 217) \ x(extent_past_end_of_inode, 218) \ x(dirent_empty_name, 219) \ x(dirent_val_too_big, 220) \ @@ -273,7 +273,10 @@ x(sb_clean_entry_overrun, 267) \ x(btree_ptr_v2_written_0, 268) \ x(subvol_snapshot_bad, 269) \ - x(subvol_inode_bad, 270) + x(subvol_inode_bad, 270) \ + x(alloc_key_stripe_sectors_wrong, 271) \ + x(accounting_mismatch, 272) \ + x(accounting_replicas_not_marked, 273) enum bch_sb_error_id { #define x(t, n) BCH_FSCK_ERR_##t = n, diff --git a/libbcachefs/snapshot.c b/libbcachefs/snapshot.c index 51918acf..169fd458 100644 --- a/libbcachefs/snapshot.c +++ b/libbcachefs/snapshot.c @@ -549,7 +549,7 @@ static int check_snapshot_tree(struct btree_trans *trans, if (fsck_err_on(ret || root_id != bch2_snapshot_root(c, root_id) || st.k->p.offset != le32_to_cpu(s.tree), - c, snapshot_tree_to_missing_snapshot, + trans, snapshot_tree_to_missing_snapshot, "snapshot tree points to missing/incorrect snapshot:\n %s", (bch2_bkey_val_to_text(&buf, c, st.s_c), buf.buf))) { ret = bch2_btree_delete_at(trans, iter, 0); @@ -562,19 +562,19 @@ static int check_snapshot_tree(struct btree_trans *trans, goto err; if (fsck_err_on(ret, - c, snapshot_tree_to_missing_subvol, + trans, snapshot_tree_to_missing_subvol, "snapshot tree points to missing subvolume:\n %s", (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, st.s_c), buf.buf)) || fsck_err_on(!bch2_snapshot_is_ancestor(c, le32_to_cpu(subvol.snapshot), root_id), - c, snapshot_tree_to_wrong_subvol, + trans, snapshot_tree_to_wrong_subvol, "snapshot tree points to subvolume that does not point to snapshot in this tree:\n %s", (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, st.s_c), buf.buf)) || fsck_err_on(BCH_SUBVOLUME_SNAP(&subvol), - c, snapshot_tree_to_snapshot_subvol, + trans, snapshot_tree_to_snapshot_subvol, "snapshot tree points to snapshot subvolume:\n %s", (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, st.s_c), buf.buf))) { @@ -811,7 +811,7 @@ static int check_snapshot(struct btree_trans *trans, } } else { if (fsck_err_on(s.subvol, - c, snapshot_should_not_have_subvol, + trans, snapshot_should_not_have_subvol, "snapshot should not point to subvol:\n %s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { u = bch2_bkey_make_mut_typed(trans, iter, &k, 0, snapshot); @@ -828,7 +828,8 @@ static int check_snapshot(struct btree_trans *trans, if (ret < 0) goto err; - if (fsck_err_on(!ret, c, snapshot_to_bad_snapshot_tree, + if (fsck_err_on(!ret, + trans, snapshot_to_bad_snapshot_tree, "snapshot points to missing/incorrect tree:\n %s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { ret = snapshot_tree_ptr_repair(trans, iter, k, &s); @@ -840,7 +841,7 @@ static int check_snapshot(struct btree_trans *trans, real_depth = bch2_snapshot_depth(c, parent_id); if (fsck_err_on(le32_to_cpu(s.depth) != real_depth, - c, snapshot_bad_depth, + trans, snapshot_bad_depth, "snapshot with incorrect depth field, should be %u:\n %s", real_depth, (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { u = bch2_bkey_make_mut_typed(trans, iter, &k, 0, snapshot); @@ -856,7 +857,8 @@ static int check_snapshot(struct btree_trans *trans, if (ret < 0) goto err; - if (fsck_err_on(!ret, c, snapshot_bad_skiplist, + if (fsck_err_on(!ret, + trans, snapshot_bad_skiplist, "snapshot with bad skiplist field:\n %s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { u = bch2_bkey_make_mut_typed(trans, iter, &k, 0, snapshot); @@ -1018,7 +1020,7 @@ int bch2_reconstruct_snapshots(struct bch_fs *c) darray_for_each(*t, id) { if (fsck_err_on(!bch2_snapshot_equiv(c, *id), - c, snapshot_node_missing, + trans, snapshot_node_missing, "snapshot node %u from tree %s missing, recreate?", *id, buf.buf)) { if (t->nr > 1) { bch_err(c, "cannot reconstruct snapshot trees with multiple nodes"); @@ -1050,8 +1052,8 @@ int bch2_check_key_has_snapshot(struct btree_trans *trans, struct printbuf buf = PRINTBUF; int ret = 0; - if (fsck_err_on(!bch2_snapshot_equiv(c, k.k->p.snapshot), c, - bkey_in_missing_snapshot, + if (fsck_err_on(!bch2_snapshot_equiv(c, k.k->p.snapshot), + trans, bkey_in_missing_snapshot, "key in missing snapshot %s, delete?", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) ret = bch2_btree_delete_at(trans, iter, diff --git a/libbcachefs/str_hash.h b/libbcachefs/str_hash.h index cbad9b27..c8c266cb 100644 --- a/libbcachefs/str_hash.h +++ b/libbcachefs/str_hash.h @@ -300,7 +300,7 @@ not_found: if (!found && (flags & STR_HASH_must_replace)) { ret = -BCH_ERR_ENOENT_str_hash_set_must_replace; } else if (found && (flags & STR_HASH_must_create)) { - ret = -EEXIST; + ret = -BCH_ERR_EEXIST_str_hash_set; } else { if (!found && slot.path) swap(iter, slot); diff --git a/libbcachefs/subvolume.c b/libbcachefs/subvolume.c index dfc9cf30..f56720b5 100644 --- a/libbcachefs/subvolume.c +++ b/libbcachefs/subvolume.c @@ -57,7 +57,7 @@ static int check_subvol(struct btree_trans *trans, if (fsck_err_on(subvol.k->p.offset == BCACHEFS_ROOT_SUBVOL && subvol.v->fs_path_parent, - c, subvol_root_fs_path_parent_nonzero, + trans, subvol_root_fs_path_parent_nonzero, "root subvolume has nonzero fs_path_parent\n%s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { struct bkey_i_subvolume *n = @@ -80,7 +80,7 @@ static int check_subvol(struct btree_trans *trans, goto err; if (fsck_err_on(subvol_children_k.k->type != KEY_TYPE_set, - c, subvol_children_not_set, + trans, subvol_children_not_set, "subvolume not set in subvolume_children btree at %llu:%llu\n%s", pos.inode, pos.offset, (printbuf_reset(&buf), @@ -101,7 +101,8 @@ static int check_subvol(struct btree_trans *trans, if (ret && !bch2_err_matches(ret, ENOENT)) return ret; - if (fsck_err_on(ret, c, subvol_to_missing_root, + if (fsck_err_on(ret, + trans, subvol_to_missing_root, "subvolume %llu points to missing subvolume root %llu:%u", k.k->p.offset, le64_to_cpu(subvol.v->inode), le32_to_cpu(subvol.v->snapshot))) { @@ -111,7 +112,7 @@ static int check_subvol(struct btree_trans *trans, } if (fsck_err_on(inode.bi_subvol != subvol.k->p.offset, - c, subvol_root_wrong_bi_subvol, + trans, subvol_root_wrong_bi_subvol, "subvol root %llu:%u has wrong bi_subvol field: got %u, should be %llu", inode.bi_inum, inode_iter.k.p.snapshot, inode.bi_subvol, subvol.k->p.offset)) { @@ -139,7 +140,7 @@ static int check_subvol(struct btree_trans *trans, return ret; if (fsck_err_on(le32_to_cpu(st.master_subvol) != subvol.k->p.offset, - c, subvol_not_master_and_not_snapshot, + trans, subvol_not_master_and_not_snapshot, "subvolume %llu is not set as snapshot but is not master subvolume", k.k->p.offset)) { struct bkey_i_subvolume *s = @@ -173,7 +174,6 @@ static int check_subvol_child(struct btree_trans *trans, struct btree_iter *child_iter, struct bkey_s_c child_k) { - struct bch_fs *c = trans->c; struct bch_subvolume s; int ret = bch2_bkey_get_val_typed(trans, BTREE_ID_subvolumes, POS(0, child_k.k->p.offset), 0, subvolume, &s); @@ -182,7 +182,7 @@ static int check_subvol_child(struct btree_trans *trans, if (fsck_err_on(ret || le32_to_cpu(s.fs_path_parent) != child_k.k->p.inode, - c, subvol_children_bad, + trans, subvol_children_bad, "incorrect entry in subvolume_children btree %llu:%llu", child_k.k->p.inode, child_k.k->p.offset)) { ret = bch2_btree_delete_at(trans, child_iter, 0); @@ -630,9 +630,9 @@ int bch2_initialize_subvolumes(struct bch_fs *c) root_volume.v.snapshot = cpu_to_le32(U32_MAX); root_volume.v.inode = cpu_to_le64(BCACHEFS_ROOT_INO); - ret = bch2_btree_insert(c, BTREE_ID_snapshot_trees, &root_tree.k_i, NULL, 0) ?: - bch2_btree_insert(c, BTREE_ID_snapshots, &root_snapshot.k_i, NULL, 0) ?: - bch2_btree_insert(c, BTREE_ID_subvolumes, &root_volume.k_i, NULL, 0); + ret = bch2_btree_insert(c, BTREE_ID_snapshot_trees, &root_tree.k_i, NULL, 0, 0) ?: + bch2_btree_insert(c, BTREE_ID_snapshots, &root_snapshot.k_i, NULL, 0, 0) ?: + bch2_btree_insert(c, BTREE_ID_subvolumes, &root_volume.k_i, NULL, 0, 0); bch_err_fn(c, ret); return ret; } diff --git a/libbcachefs/super-io.c b/libbcachefs/super-io.c index d73a0222..055478d2 100644 --- a/libbcachefs/super-io.c +++ b/libbcachefs/super-io.c @@ -1310,15 +1310,15 @@ void bch2_sb_to_text(struct printbuf *out, struct bch_sb *sb, prt_printf(out, "Device index:\t%u\n", sb->dev_idx); - prt_str(out, "Label:\t"); + prt_printf(out, "Label:\t"); prt_printf(out, "%.*s", (int) sizeof(sb->label), sb->label); prt_newline(out); - prt_str(out, "Version:\t"); + prt_printf(out, "Version:\t"); bch2_version_to_text(out, le16_to_cpu(sb->version)); prt_newline(out); - prt_str(out, "Version upgrade complete:\t"); + prt_printf(out, "Version upgrade complete:\t"); bch2_version_to_text(out, BCH_SB_VERSION_UPGRADE_COMPLETE(sb)); prt_newline(out); diff --git a/libbcachefs/super.c b/libbcachefs/super.c index df2bea38..a7f0245f 100644 --- a/libbcachefs/super.c +++ b/libbcachefs/super.c @@ -25,6 +25,7 @@ #include "clock.h" #include "compress.h" #include "debug.h" +#include "disk_accounting.h" #include "disk_groups.h" #include "ec.h" #include "errcode.h" @@ -88,6 +89,19 @@ const char * const bch2_fs_flag_strs[] = { NULL }; +void bch2_print_str(struct bch_fs *c, const char *str) +{ +#ifdef __KERNEL__ + struct stdio_redirect *stdio = bch2_fs_stdio_redirect(c); + + if (unlikely(stdio)) { + bch2_stdio_redirect_printf(stdio, true, "%s", str); + return; + } +#endif + bch2_print_string_as_lines(KERN_ERR, str); +} + __printf(2, 0) static void bch2_print_maybe_redirect(struct stdio_redirect *stdio, const char *fmt, va_list args) { @@ -222,22 +236,6 @@ struct bch_fs *bch2_uuid_to_fs(__uuid_t uuid) return c; } -static void bch2_dev_usage_journal_reserve(struct bch_fs *c) -{ - unsigned nr = 0, u64s = - ((sizeof(struct jset_entry_dev_usage) + - sizeof(struct jset_entry_dev_usage_type) * BCH_DATA_NR)) / - sizeof(u64); - - rcu_read_lock(); - for_each_member_device_rcu(c, ca, NULL) - nr++; - rcu_read_unlock(); - - bch2_journal_entry_res_resize(&c->journal, - &c->dev_usage_journal_res, u64s * nr); -} - /* Filesystem RO/RW: */ /* @@ -376,6 +374,7 @@ void bch2_fs_read_only(struct bch_fs *c) BUG_ON(atomic_long_read(&c->btree_key_cache.nr_dirty)); BUG_ON(c->btree_write_buffer.inc.keys.nr); BUG_ON(c->btree_write_buffer.flushing.keys.nr); + bch2_verify_accounting_clean(c); bch_verbose(c, "marking filesystem clean"); bch2_fs_mark_clean(c); @@ -537,6 +536,7 @@ static void __bch2_fs_free(struct bch_fs *c) bch2_find_btree_nodes_exit(&c->found_btree_nodes); bch2_free_pending_node_rewrites(c); bch2_fs_allocator_background_exit(c); + bch2_fs_accounting_exit(c); bch2_fs_sb_errors_exit(c); bch2_fs_counters_exit(c); bch2_fs_snapshots_exit(c); @@ -569,6 +569,7 @@ static void __bch2_fs_free(struct bch_fs *c) darray_exit(&c->btree_roots_extra); free_percpu(c->pcpu); + free_percpu(c->usage); mempool_exit(&c->large_bkey_pool); mempool_exit(&c->btree_bounce_pool); bioset_exit(&c->btree_bio); @@ -762,7 +763,6 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) init_waitqueue_head(&c->ro_ref_wait); sema_init(&c->online_fsck_mutex, 1); - init_rwsem(&c->gc_lock); mutex_init(&c->gc_gens_lock); atomic_set(&c->journal_keys.ref, 1); c->journal_keys.initial_ref_held = true; @@ -785,8 +785,6 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) INIT_LIST_HEAD(&c->list); - mutex_init(&c->usage_scratch_lock); - mutex_init(&c->bio_bounce_pages_lock); mutex_init(&c->snapshot_table_lock); init_rwsem(&c->snapshot_create_lock); @@ -892,6 +890,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) offsetof(struct btree_write_bio, wbio.bio)), BIOSET_NEED_BVECS) || !(c->pcpu = alloc_percpu(struct bch_fs_pcpu)) || + !(c->usage = alloc_percpu(struct bch_fs_usage_base)) || !(c->online_reserved = alloc_percpu(u64)) || mempool_init_kvmalloc_pool(&c->btree_bounce_pool, 1, c->opts.btree_node_size) || @@ -907,7 +906,6 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) bch2_io_clock_init(&c->io_clock[READ]) ?: bch2_io_clock_init(&c->io_clock[WRITE]) ?: bch2_fs_journal_init(&c->journal) ?: - bch2_fs_replicas_init(c) ?: bch2_fs_btree_cache_init(c) ?: bch2_fs_btree_key_cache_init(&c->btree_key_cache) ?: bch2_fs_btree_iter_init(c) ?: @@ -927,17 +925,17 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) if (ret) goto err; - for (i = 0; i < c->sb.nr_devices; i++) - if (bch2_member_exists(c->disk_sb.sb, i) && - bch2_dev_alloc(c, i)) { - ret = -EEXIST; + for (i = 0; i < c->sb.nr_devices; i++) { + if (!bch2_member_exists(c->disk_sb.sb, i)) + continue; + ret = bch2_dev_alloc(c, i); + if (ret) goto err; - } + } bch2_journal_entry_res_resize(&c->journal, &c->btree_root_journal_res, BTREE_ID_NR * (JSET_KEYS_U64s + BKEY_BTREE_PTR_U64s_MAX)); - bch2_dev_usage_journal_reserve(c); bch2_journal_entry_res_resize(&c->journal, &c->clock_journal_res, (sizeof(struct jset_entry_clock) / sizeof(u64)) * 2); @@ -1603,7 +1601,8 @@ static int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca) bch2_btree_delete_range(c, BTREE_ID_alloc, start, end, BTREE_TRIGGER_norun, NULL) ?: bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end, - BTREE_TRIGGER_norun, NULL); + BTREE_TRIGGER_norun, NULL) ?: + bch2_dev_usage_remove(c, ca->dev_idx); bch_err_msg(c, ret, "removing dev alloc info"); return ret; } @@ -1640,6 +1639,16 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) if (ret) goto err; + /* + * We need to flush the entire journal to get rid of keys that reference + * the device being removed before removing the superblock entry + */ + bch2_journal_flush_all_pins(&c->journal); + + /* + * this is really just needed for the bch2_replicas_gc_(start|end) + * calls, and could be cleaned up: + */ ret = bch2_journal_flush_device_pins(&c->journal, ca->dev_idx); bch_err_msg(ca, ret, "bch2_journal_flush_device_pins()"); if (ret) @@ -1683,17 +1692,6 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) bch2_dev_free(ca); /* - * At this point the device object has been removed in-core, but the - * on-disk journal might still refer to the device index via sb device - * usage entries. Recovery fails if it sees usage information for an - * invalid device. Flush journal pins to push the back of the journal - * past now invalid device index references before we update the - * superblock, but after the device object has been removed so any - * further journal writes elide usage info for the device. - */ - bch2_journal_flush_all_pins(&c->journal); - - /* * Free this device's slot in the bch_member array - all pointers to * this device must be gone: */ @@ -1705,8 +1703,6 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) mutex_unlock(&c->sb_lock); up_write(&c->state_lock); - - bch2_dev_usage_journal_reserve(c); return 0; err: if (ca->mi.state == BCH_MEMBER_STATE_rw && @@ -1754,8 +1750,6 @@ int bch2_dev_add(struct bch_fs *c, const char *path) goto err; } - bch2_dev_usage_init(ca); - ret = __bch2_dev_attach_bdev(ca, &sb); if (ret) goto err; @@ -1837,7 +1831,9 @@ have_slot: bch2_write_super(c); mutex_unlock(&c->sb_lock); - bch2_dev_usage_journal_reserve(c); + ret = bch2_dev_usage_init(ca, false); + if (ret) + goto err_late; ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_transactional); bch_err_msg(ca, ret, "marking new superblock"); @@ -2009,15 +2005,18 @@ int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) mutex_unlock(&c->sb_lock); if (ca->mi.freespace_initialized) { - ret = bch2_dev_freespace_init(c, ca, old_nbuckets, nbuckets); + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_dev_data_type, + .dev_data_type.dev = ca->dev_idx, + .dev_data_type.data_type = BCH_DATA_free, + }; + u64 v[3] = { nbuckets - old_nbuckets, 0, 0 }; + + ret = bch2_trans_do(ca->fs, NULL, NULL, 0, + bch2_disk_accounting_mod(trans, &acc, v, ARRAY_SIZE(v), false)) ?: + bch2_dev_freespace_init(c, ca, old_nbuckets, nbuckets); if (ret) goto err; - - /* - * XXX: this is all wrong transactionally - we'll be able to do - * this correctly after the disk space accounting rewrite - */ - ca->usage_base->d[BCH_DATA_free].buckets += nbuckets - old_nbuckets; } bch2_recalc_capacity(c); @@ -2029,6 +2028,9 @@ err: /* return with ref on ca->ref: */ struct bch_dev *bch2_dev_lookup(struct bch_fs *c, const char *name) { + if (!strncmp(name, "/dev/", strlen("/dev/"))) + name += strlen("/dev/"); + for_each_member_device(c, ca) if (!strcmp(name, ca->name)) return ca; diff --git a/libbcachefs/sysfs.c b/libbcachefs/sysfs.c index 93ca74d1..8df52a23 100644 --- a/libbcachefs/sysfs.c +++ b/libbcachefs/sysfs.c @@ -21,6 +21,7 @@ #include "buckets.h" #include "clock.h" #include "compress.h" +#include "disk_accounting.h" #include "disk_groups.h" #include "ec.h" #include "inode.h" @@ -198,6 +199,8 @@ read_attribute(disk_groups); read_attribute(has_data); read_attribute(alloc_debug); +read_attribute(accounting); +read_attribute(usage_base); #define x(t, n, ...) read_attribute(t); BCH_PERSISTENT_COUNTERS() @@ -251,91 +254,42 @@ static size_t bch2_btree_cache_size(struct bch_fs *c) static int bch2_compression_stats_to_text(struct printbuf *out, struct bch_fs *c) { - struct btree_trans *trans; - enum btree_id id; - struct compression_type_stats { - u64 nr_extents; - u64 sectors_compressed; - u64 sectors_uncompressed; - } s[BCH_COMPRESSION_TYPE_NR]; - u64 compressed_incompressible = 0; - int ret = 0; - - memset(s, 0, sizeof(s)); - - if (!test_bit(BCH_FS_started, &c->flags)) - return -EPERM; - - trans = bch2_trans_get(c); - - for (id = 0; id < BTREE_ID_NR; id++) { - if (!btree_type_has_ptrs(id)) - continue; - - ret = for_each_btree_key(trans, iter, id, POS_MIN, - BTREE_ITER_all_snapshots, k, ({ - struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); - struct bch_extent_crc_unpacked crc; - const union bch_extent_entry *entry; - bool compressed = false, incompressible = false; - - bkey_for_each_crc(k.k, ptrs, crc, entry) { - incompressible |= crc.compression_type == BCH_COMPRESSION_TYPE_incompressible; - compressed |= crc_is_compressed(crc); - - if (crc_is_compressed(crc)) { - s[crc.compression_type].nr_extents++; - s[crc.compression_type].sectors_compressed += crc.compressed_size; - s[crc.compression_type].sectors_uncompressed += crc.uncompressed_size; - } - } - - compressed_incompressible += compressed && incompressible; - - if (!compressed) { - unsigned t = incompressible ? BCH_COMPRESSION_TYPE_incompressible : 0; - - s[t].nr_extents++; - s[t].sectors_compressed += k.k->size; - s[t].sectors_uncompressed += k.k->size; - } - 0; - })); - } - - bch2_trans_put(trans); - - if (ret) - return ret; - + prt_str(out, "type"); printbuf_tabstop_push(out, 12); printbuf_tabstop_push(out, 16); printbuf_tabstop_push(out, 16); printbuf_tabstop_push(out, 24); prt_printf(out, "type\tcompressed\runcompressed\raverage extent size\r\n"); - for (unsigned i = 0; i < ARRAY_SIZE(s); i++) { + for (unsigned i = 1; i < BCH_COMPRESSION_TYPE_NR; i++) { + struct disk_accounting_pos a = { + .type = BCH_DISK_ACCOUNTING_compression, + .compression.type = i, + }; + struct bpos p = disk_accounting_pos_to_bpos(&a); + u64 v[3]; + bch2_accounting_mem_read(c, p, v, ARRAY_SIZE(v)); + + u64 nr_extents = v[0]; + u64 sectors_uncompressed = v[1]; + u64 sectors_compressed = v[2]; + bch2_prt_compression_type(out, i); prt_tab(out); - prt_human_readable_u64(out, s[i].sectors_compressed << 9); + prt_human_readable_u64(out, sectors_compressed << 9); prt_tab_rjust(out); - prt_human_readable_u64(out, s[i].sectors_uncompressed << 9); + prt_human_readable_u64(out, sectors_uncompressed << 9); prt_tab_rjust(out); - prt_human_readable_u64(out, s[i].nr_extents - ? div_u64(s[i].sectors_uncompressed << 9, s[i].nr_extents) + prt_human_readable_u64(out, nr_extents + ? div_u64(sectors_uncompressed << 9, nr_extents) : 0); prt_tab_rjust(out); prt_newline(out); } - if (compressed_incompressible) { - prt_printf(out, "%llu compressed & incompressible extents", compressed_incompressible); - prt_newline(out); - } - return 0; } @@ -346,6 +300,20 @@ static void bch2_gc_gens_pos_to_text(struct printbuf *out, struct bch_fs *c) prt_printf(out, "\n"); } +static void bch2_fs_usage_base_to_text(struct printbuf *out, struct bch_fs *c) +{ + struct bch_fs_usage_base b = {}; + + acc_u64s_percpu(&b.hidden, &c->usage->hidden, sizeof(b) / sizeof(u64)); + + prt_printf(out, "hidden:\t\t%llu\n", b.hidden); + prt_printf(out, "btree:\t\t%llu\n", b.btree); + prt_printf(out, "data:\t\t%llu\n", b.data); + prt_printf(out, "cached:\t%llu\n", b.cached); + prt_printf(out, "reserved:\t\t%llu\n", b.reserved); + prt_printf(out, "nr_inodes:\t%llu\n", b.nr_inodes); +} + SHOW(bch2_fs) { struct bch_fs *c = container_of(kobj, struct bch_fs, kobj); @@ -429,6 +397,12 @@ SHOW(bch2_fs) if (attr == &sysfs_alloc_debug) bch2_fs_alloc_debug_to_text(out, c); + if (attr == &sysfs_accounting) + bch2_fs_accounting_to_text(out, c); + + if (attr == &sysfs_usage_base) + bch2_fs_usage_base_to_text(out, c); + return 0; } @@ -633,6 +607,8 @@ struct attribute *bch2_fs_internal_files[] = { &sysfs_disk_groups, &sysfs_alloc_debug, + &sysfs_accounting, + &sysfs_usage_base, NULL }; diff --git a/libbcachefs/tests.c b/libbcachefs/tests.c index 68104b20..01b768c9 100644 --- a/libbcachefs/tests.c +++ b/libbcachefs/tests.c @@ -121,7 +121,7 @@ static int test_iterate(struct bch_fs *c, u64 nr) ck.k.p.offset = i; ck.k.p.snapshot = U32_MAX; - ret = bch2_btree_insert(c, BTREE_ID_xattrs, &ck.k_i, NULL, 0); + ret = bch2_btree_insert(c, BTREE_ID_xattrs, &ck.k_i, NULL, 0, 0); bch_err_msg(c, ret, "insert error"); if (ret) return ret; @@ -176,7 +176,7 @@ static int test_iterate_extents(struct bch_fs *c, u64 nr) ck.k.p.snapshot = U32_MAX; ck.k.size = 8; - ret = bch2_btree_insert(c, BTREE_ID_extents, &ck.k_i, NULL, 0); + ret = bch2_btree_insert(c, BTREE_ID_extents, &ck.k_i, NULL, 0, 0); bch_err_msg(c, ret, "insert error"); if (ret) return ret; @@ -232,7 +232,7 @@ static int test_iterate_slots(struct bch_fs *c, u64 nr) ck.k.p.offset = i * 2; ck.k.p.snapshot = U32_MAX; - ret = bch2_btree_insert(c, BTREE_ID_xattrs, &ck.k_i, NULL, 0); + ret = bch2_btree_insert(c, BTREE_ID_xattrs, &ck.k_i, NULL, 0, 0); bch_err_msg(c, ret, "insert error"); if (ret) return ret; @@ -292,7 +292,7 @@ static int test_iterate_slots_extents(struct bch_fs *c, u64 nr) ck.k.p.snapshot = U32_MAX; ck.k.size = 8; - ret = bch2_btree_insert(c, BTREE_ID_extents, &ck.k_i, NULL, 0); + ret = bch2_btree_insert(c, BTREE_ID_extents, &ck.k_i, NULL, 0, 0); bch_err_msg(c, ret, "insert error"); if (ret) return ret; @@ -396,7 +396,7 @@ static int insert_test_extent(struct bch_fs *c, k.k_i.k.size = end - start; k.k_i.k.version.lo = test_version++; - ret = bch2_btree_insert(c, BTREE_ID_extents, &k.k_i, NULL, 0); + ret = bch2_btree_insert(c, BTREE_ID_extents, &k.k_i, NULL, 0, 0); bch_err_fn(c, ret); return ret; } @@ -481,7 +481,7 @@ static int test_snapshot_filter(struct bch_fs *c, u32 snapid_lo, u32 snapid_hi) bkey_cookie_init(&cookie.k_i); cookie.k.p.snapshot = snapid_hi; - ret = bch2_btree_insert(c, BTREE_ID_xattrs, &cookie.k_i, NULL, 0); + ret = bch2_btree_insert(c, BTREE_ID_xattrs, &cookie.k_i, NULL, 0, 0); if (ret) return ret; @@ -506,7 +506,7 @@ static int test_snapshots(struct bch_fs *c, u64 nr) bkey_cookie_init(&cookie.k_i); cookie.k.p.snapshot = U32_MAX; - ret = bch2_btree_insert(c, BTREE_ID_xattrs, &cookie.k_i, NULL, 0); + ret = bch2_btree_insert(c, BTREE_ID_xattrs, &cookie.k_i, NULL, 0, 0); if (ret) return ret; diff --git a/libbcachefs/thread_with_file.c b/libbcachefs/thread_with_file.c index b1af7ac4..0807ce9b 100644 --- a/libbcachefs/thread_with_file.c +++ b/libbcachefs/thread_with_file.c @@ -67,9 +67,14 @@ err: /* stdio_redirect */ +static bool stdio_redirect_has_more_input(struct stdio_redirect *stdio, size_t seen) +{ + return stdio->input.buf.nr > seen || stdio->done; +} + static bool stdio_redirect_has_input(struct stdio_redirect *stdio) { - return stdio->input.buf.nr || stdio->done; + return stdio_redirect_has_more_input(stdio, 0); } static bool stdio_redirect_has_output(struct stdio_redirect *stdio) @@ -181,9 +186,13 @@ static ssize_t thread_with_stdio_write(struct file *file, const char __user *ubu } spin_lock(&buf->lock); - if (buf->buf.nr < STDIO_REDIRECT_BUFSIZE) - darray_make_room_gfp(&buf->buf, - min(b, STDIO_REDIRECT_BUFSIZE - buf->buf.nr), GFP_NOWAIT); + size_t makeroom = b; + if (!buf->waiting_for_line || memchr(buf->buf.data, '\n', buf->buf.nr)) + makeroom = min_t(ssize_t, makeroom, + max_t(ssize_t, STDIO_REDIRECT_BUFSIZE - buf->buf.nr, + 0)); + darray_make_room_gfp(&buf->buf, makeroom, GFP_NOWAIT); + b = min(len, darray_room(buf->buf)); if (b && !copy_from_user_nofault(&darray_top(buf->buf), ubuf, b)) { @@ -355,43 +364,67 @@ int bch2_stdio_redirect_read(struct stdio_redirect *stdio, char *ubuf, size_t le return ret; } -int bch2_stdio_redirect_readline(struct stdio_redirect *stdio, char *ubuf, size_t len) +int bch2_stdio_redirect_readline_timeout(struct stdio_redirect *stdio, + darray_char *line, + unsigned long timeout) { + unsigned long until = jiffies + timeout, t; struct stdio_buf *buf = &stdio->input; - size_t copied = 0; - ssize_t ret = 0; + size_t seen = 0; again: - do { - wait_event_timeout(buf->wait, stdio_redirect_has_input(stdio), - sysctl_hung_task_timeout_secs * HZ / 2); - } while (!stdio_redirect_has_input(stdio)); + t = timeout != MAX_SCHEDULE_TIMEOUT + ? max_t(long, until - jiffies, 0) + : timeout; - if (stdio->done) { - ret = -1; - goto out; - } + t = min(t, sysctl_hung_task_timeout_secs * HZ / 2); + + wait_event_timeout(buf->wait, stdio_redirect_has_more_input(stdio, seen), t); + + if (stdio->done) + return -1; spin_lock(&buf->lock); - size_t b = min(len, buf->buf.nr); - char *n = memchr(buf->buf.data, '\n', b); - if (n) - b = min_t(size_t, b, n + 1 - buf->buf.data); + seen = buf->buf.nr; + char *n = memchr(buf->buf.data, '\n', seen); + + if (!n && timeout != MAX_SCHEDULE_TIMEOUT && jiffies >= until) { + spin_unlock(&buf->lock); + return -ETIME; + } + + if (!n) { + buf->waiting_for_line = true; + spin_unlock(&buf->lock); + goto again; + } + + size_t b = n + 1 - buf->buf.data; + if (b > line->size) { + spin_unlock(&buf->lock); + int ret = darray_resize(line, b); + if (ret) + return ret; + seen = 0; + goto again; + } + buf->buf.nr -= b; - memcpy(ubuf, buf->buf.data, b); + memcpy(line->data, buf->buf.data, b); memmove(buf->buf.data, buf->buf.data + b, buf->buf.nr); - ubuf += b; - len -= b; - copied += b; + line->nr = b; + + buf->waiting_for_line = false; spin_unlock(&buf->lock); wake_up(&buf->wait); + return 0; +} - if (!n && len) - goto again; -out: - return copied ?: ret; +int bch2_stdio_redirect_readline(struct stdio_redirect *stdio, darray_char *line) +{ + return bch2_stdio_redirect_readline_timeout(stdio, line, MAX_SCHEDULE_TIMEOUT); } __printf(3, 0) diff --git a/libbcachefs/thread_with_file.h b/libbcachefs/thread_with_file.h index 1d63d14d..72497b92 100644 --- a/libbcachefs/thread_with_file.h +++ b/libbcachefs/thread_with_file.h @@ -71,7 +71,9 @@ int bch2_run_thread_with_stdio(struct thread_with_stdio *, int bch2_run_thread_with_stdout(struct thread_with_stdio *, const struct thread_with_stdio_ops *); int bch2_stdio_redirect_read(struct stdio_redirect *, char *, size_t); -int bch2_stdio_redirect_readline(struct stdio_redirect *, char *, size_t); + +int bch2_stdio_redirect_readline_timeout(struct stdio_redirect *, darray_char *, unsigned long); +int bch2_stdio_redirect_readline(struct stdio_redirect *, darray_char *); __printf(3, 0) ssize_t bch2_stdio_redirect_vprintf(struct stdio_redirect *, bool, const char *, va_list); __printf(3, 4) ssize_t bch2_stdio_redirect_printf(struct stdio_redirect *, bool, const char *, ...); diff --git a/libbcachefs/thread_with_file_types.h b/libbcachefs/thread_with_file_types.h index e0daf4ee..f4d484d4 100644 --- a/libbcachefs/thread_with_file_types.h +++ b/libbcachefs/thread_with_file_types.h @@ -8,15 +8,12 @@ struct stdio_buf { spinlock_t lock; wait_queue_head_t wait; darray_char buf; + bool waiting_for_line; }; struct stdio_redirect { struct stdio_buf input; struct stdio_buf output; - - spinlock_t input_lock; - wait_queue_head_t input_wait; - darray_char input_buf; bool done; }; diff --git a/libbcachefs/trace.h b/libbcachefs/trace.h index 362e1fc7..d0e6b9de 100644 --- a/libbcachefs/trace.h +++ b/libbcachefs/trace.h @@ -43,7 +43,7 @@ DECLARE_EVENT_CLASS(fs_str, TP_fast_assign( __entry->dev = c->dev; - __assign_str(str, str); + __assign_str(str); ), TP_printk("%d,%d\n%s", MAJOR(__entry->dev), MINOR(__entry->dev), __get_str(str)) @@ -64,7 +64,7 @@ DECLARE_EVENT_CLASS(trans_str, __entry->dev = trans->c->dev; strscpy(__entry->trans_fn, trans->fn, sizeof(__entry->trans_fn)); __entry->caller_ip = caller_ip; - __assign_str(str, str); + __assign_str(str); ), TP_printk("%d,%d %s %pS %s", @@ -85,7 +85,7 @@ DECLARE_EVENT_CLASS(trans_str_nocaller, TP_fast_assign( __entry->dev = trans->c->dev; strscpy(__entry->trans_fn, trans->fn, sizeof(__entry->trans_fn)); - __assign_str(str, str); + __assign_str(str); ), TP_printk("%d,%d %s %s", @@ -200,6 +200,56 @@ DECLARE_EVENT_CLASS(bio, (unsigned long long)__entry->sector, __entry->nr_sector) ); +/* fs.c: */ +TRACE_EVENT(bch2_sync_fs, + TP_PROTO(struct super_block *sb, int wait), + + TP_ARGS(sb, wait), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( int, wait ) + + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->wait = wait; + ), + + TP_printk("dev %d,%d wait %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->wait) +); + +/* fs-io.c: */ +TRACE_EVENT(bch2_fsync, + TP_PROTO(struct file *file, int datasync), + + TP_ARGS(file, datasync), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( ino_t, ino ) + __field( ino_t, parent ) + __field( int, datasync ) + ), + + TP_fast_assign( + struct dentry *dentry = file->f_path.dentry; + + __entry->dev = dentry->d_sb->s_dev; + __entry->ino = d_inode(dentry)->i_ino; + __entry->parent = d_inode(dentry->d_parent)->i_ino; + __entry->datasync = datasync; + ), + + TP_printk("dev %d,%d ino %lu parent %lu datasync %d ", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long) __entry->ino, + (unsigned long) __entry->parent, __entry->datasync) +); + /* super-io.c: */ TRACE_EVENT(write_super, TP_PROTO(struct bch_fs *c, unsigned long ip), diff --git a/libbcachefs/two_state_shared_lock.h b/libbcachefs/two_state_shared_lock.h index 90580177..7f647846 100644 --- a/libbcachefs/two_state_shared_lock.h +++ b/libbcachefs/two_state_shared_lock.h @@ -36,15 +36,14 @@ static inline void bch2_two_state_unlock(two_state_lock_t *lock, int s) static inline bool bch2_two_state_trylock(two_state_lock_t *lock, int s) { long i = s ? 1 : -1; - long v = atomic_long_read(&lock->v), old; + long old; + old = atomic_long_read(&lock->v); do { - old = v; - - if (i > 0 ? v < 0 : v > 0) + if (i > 0 ? old < 0 : old > 0) return false; - } while ((v = atomic_long_cmpxchg_acquire(&lock->v, - old, old + i)) != old); + } while (!atomic_long_try_cmpxchg_acquire(&lock->v, &old, old + i)); + return true; } diff --git a/libbcachefs/util.h b/libbcachefs/util.h index 5cf885b0..f4dd09c4 100644 --- a/libbcachefs/util.h +++ b/libbcachefs/util.h @@ -445,11 +445,6 @@ static inline unsigned fract_exp_two(unsigned x, unsigned fract_bits) void bch2_bio_map(struct bio *bio, void *base, size_t); int bch2_bio_alloc_pages(struct bio *, size_t, gfp_t); -static inline sector_t bdev_sectors(struct block_device *bdev) -{ - return bdev->bd_inode->i_size >> 9; -} - #define closure_bio_submit(bio, cl) \ do { \ closure_get(cl); \ @@ -723,9 +718,7 @@ static inline void percpu_u64_set(u64 __percpu *dst, u64 src) static inline void acc_u64s(u64 *acc, const u64 *src, unsigned nr) { - unsigned i; - - for (i = 0; i < nr; i++) + for (unsigned i = 0; i < nr; i++) acc[i] += src[i]; } |