diff options
Diffstat (limited to 'libbcachefs/super-io.c')
| -rw-r--r-- | libbcachefs/super-io.c | 1195 |
1 files changed, 103 insertions, 1092 deletions
diff --git a/libbcachefs/super-io.c b/libbcachefs/super-io.c index 69101f3a..a2b981a3 100644 --- a/libbcachefs/super-io.c +++ b/libbcachefs/super-io.c @@ -1,8 +1,11 @@ #include "bcachefs.h" #include "checksum.h" +#include "disk_groups.h" #include "error.h" #include "io.h" +#include "replicas.h" +#include "quota.h" #include "super-io.h" #include "super.h" #include "vstructs.h" @@ -10,13 +13,6 @@ #include <linux/backing-dev.h> #include <linux/sort.h> -static int bch2_sb_replicas_to_cpu_replicas(struct bch_fs *); -static int bch2_cpu_replicas_to_sb_replicas(struct bch_fs *, - struct bch_replicas_cpu *); -static int bch2_sb_disk_groups_to_cpu(struct bch_fs *); - -/* superblock fields (optional/variable size sections: */ - const char * const bch2_sb_fields[] = { #define x(name, nr) #name, BCH_SB_FIELDS() @@ -24,34 +20,8 @@ const char * const bch2_sb_fields[] = { NULL }; -#define x(f, nr) \ -static const char *bch2_sb_validate_##f(struct bch_sb *, struct bch_sb_field *); - BCH_SB_FIELDS() -#undef x - -struct bch_sb_field_ops { - const char * (*validate)(struct bch_sb *, struct bch_sb_field *); -}; - -static const struct bch_sb_field_ops bch2_sb_field_ops[] = { -#define x(f, nr) \ - [BCH_SB_FIELD_##f] = { \ - .validate = bch2_sb_validate_##f, \ - }, - BCH_SB_FIELDS() -#undef x -}; - -static const char *bch2_sb_field_validate(struct bch_sb *sb, - struct bch_sb_field *f) - -{ - unsigned type = le32_to_cpu(f->type); - - return type < BCH_SB_FIELD_NR - ? bch2_sb_field_ops[type].validate(sb, f) - : NULL; -} +static const char *bch2_sb_field_validate(struct bch_sb *, + struct bch_sb_field *); struct bch_sb_field *bch2_sb_field_get(struct bch_sb *sb, enum bch_sb_field_type type) @@ -66,14 +36,18 @@ struct bch_sb_field *bch2_sb_field_get(struct bch_sb *sb, return NULL; } -static struct bch_sb_field *__bch2_sb_field_resize(struct bch_sb *sb, - struct bch_sb_field *f, - unsigned u64s) +static struct bch_sb_field *__bch2_sb_field_resize(struct bch_sb_handle *sb, + struct bch_sb_field *f, + unsigned u64s) { unsigned old_u64s = f ? le32_to_cpu(f->u64s) : 0; + unsigned sb_u64s = le32_to_cpu(sb->sb->u64s) + u64s - old_u64s; + + BUG_ON(get_order(__vstruct_bytes(struct bch_sb, sb_u64s)) > + sb->page_order); if (!f) { - f = vstruct_last(sb); + f = vstruct_last(sb->sb); memset(f, 0, sizeof(u64) * u64s); f->u64s = cpu_to_le32(u64s); f->type = 0; @@ -84,13 +58,13 @@ static struct bch_sb_field *__bch2_sb_field_resize(struct bch_sb *sb, f->u64s = cpu_to_le32(u64s); dst = vstruct_end(f); - memmove(dst, src, vstruct_end(sb) - src); + memmove(dst, src, vstruct_end(sb->sb) - src); if (dst > src) memset(src, 0, dst - src); } - le32_add_cpu(&sb->u64s, u64s - old_u64s); + sb->sb->u64s = cpu_to_le32(sb_u64s); return f; } @@ -108,26 +82,42 @@ void bch2_free_super(struct bch_sb_handle *sb) memset(sb, 0, sizeof(*sb)); } -static int __bch2_super_realloc(struct bch_sb_handle *sb, unsigned order) +int bch2_sb_realloc(struct bch_sb_handle *sb, unsigned u64s) { + size_t new_bytes = __vstruct_bytes(struct bch_sb, u64s); + unsigned order = get_order(new_bytes); struct bch_sb *new_sb; struct bio *bio; + if (sb->have_layout) { + u64 max_bytes = 512 << sb->sb->layout.sb_max_size_bits; + + if (new_bytes > max_bytes) { + char buf[BDEVNAME_SIZE]; + + pr_err("%s: superblock too big: want %zu but have %llu", + bdevname(sb->bdev, buf), new_bytes, max_bytes); + return -ENOSPC; + } + } + if (sb->page_order >= order && sb->sb) return 0; if (dynamic_fault("bcachefs:add:super_realloc")) return -ENOMEM; - bio = bio_kmalloc(GFP_KERNEL, 1 << order); - if (!bio) - return -ENOMEM; + if (sb->have_bio) { + bio = bio_kmalloc(GFP_KERNEL, 1 << order); + if (!bio) + return -ENOMEM; - if (sb->bio) - bio_put(sb->bio); - sb->bio = bio; + if (sb->bio) + bio_put(sb->bio); + sb->bio = bio; + } - new_sb = (void *) __get_free_pages(GFP_KERNEL, order); + new_sb = (void *) __get_free_pages(GFP_KERNEL|__GFP_ZERO, order); if (!new_sb) return -ENOMEM; @@ -142,45 +132,6 @@ static int __bch2_super_realloc(struct bch_sb_handle *sb, unsigned order) return 0; } -static int bch2_sb_realloc(struct bch_sb_handle *sb, unsigned u64s) -{ - u64 new_bytes = __vstruct_bytes(struct bch_sb, u64s); - u64 max_bytes = 512 << sb->sb->layout.sb_max_size_bits; - - if (new_bytes > max_bytes) { - char buf[BDEVNAME_SIZE]; - - pr_err("%s: superblock too big: want %llu but have %llu", - bdevname(sb->bdev, buf), new_bytes, max_bytes); - return -ENOSPC; - } - - return __bch2_super_realloc(sb, get_order(new_bytes)); -} - -static int bch2_fs_sb_realloc(struct bch_fs *c, unsigned u64s) -{ - u64 bytes = __vstruct_bytes(struct bch_sb, u64s); - struct bch_sb *sb; - unsigned order = get_order(bytes); - - if (c->disk_sb && order <= c->disk_sb_order) - return 0; - - sb = (void *) __get_free_pages(GFP_KERNEL|__GFP_ZERO, order); - if (!sb) - return -ENOMEM; - - if (c->disk_sb) - memcpy(sb, c->disk_sb, PAGE_SIZE << c->disk_sb_order); - - free_pages((unsigned long) c->disk_sb, c->disk_sb_order); - - c->disk_sb = sb; - c->disk_sb_order = order; - return 0; -} - struct bch_sb_field *bch2_sb_field_resize(struct bch_sb_handle *sb, enum bch_sb_field_type type, unsigned u64s) @@ -192,38 +143,26 @@ struct bch_sb_field *bch2_sb_field_resize(struct bch_sb_handle *sb, if (bch2_sb_realloc(sb, le32_to_cpu(sb->sb->u64s) + d)) return NULL; - f = __bch2_sb_field_resize(sb->sb, f, u64s); - f->type = cpu_to_le32(type); - return f; -} - -struct bch_sb_field *bch2_fs_sb_field_resize(struct bch_fs *c, - enum bch_sb_field_type type, - unsigned u64s) -{ - struct bch_sb_field *f = bch2_sb_field_get(c->disk_sb, type); - ssize_t old_u64s = f ? le32_to_cpu(f->u64s) : 0; - ssize_t d = -old_u64s + u64s; - struct bch_dev *ca; - unsigned i; - - lockdep_assert_held(&c->sb_lock); + if (sb->fs_sb) { + struct bch_fs *c = container_of(sb, struct bch_fs, disk_sb); + struct bch_dev *ca; + unsigned i; - if (bch2_fs_sb_realloc(c, le32_to_cpu(c->disk_sb->u64s) + d)) - return NULL; + lockdep_assert_held(&c->sb_lock); - /* XXX: we're not checking that offline device have enough space */ + /* XXX: we're not checking that offline device have enough space */ - for_each_online_member(ca, c, i) { - struct bch_sb_handle *sb = &ca->disk_sb; + for_each_online_member(ca, c, i) { + struct bch_sb_handle *sb = &ca->disk_sb; - if (bch2_sb_realloc(sb, le32_to_cpu(sb->sb->u64s) + d)) { - percpu_ref_put(&ca->ref); - return NULL; + if (bch2_sb_realloc(sb, le32_to_cpu(sb->sb->u64s) + d)) { + percpu_ref_put(&ca->ref); + return NULL; + } } } - f = __bch2_sb_field_resize(c->disk_sb, f, u64s); + f = __bch2_sb_field_resize(sb, f, u64s); f->type = cpu_to_le32(type); return f; } @@ -384,7 +323,7 @@ const char *bch2_sb_validate(struct bch_sb_handle *disk_sb) static void bch2_sb_update(struct bch_fs *c) { - struct bch_sb *src = c->disk_sb; + struct bch_sb *src = c->disk_sb.sb; struct bch_sb_field_members *mi = bch2_sb_get_members(src); struct bch_dev *ca; unsigned i; @@ -407,9 +346,10 @@ static void bch2_sb_update(struct bch_fs *c) } /* doesn't copy member info */ -static void __copy_super(struct bch_sb *dst, struct bch_sb *src) +static void __copy_super(struct bch_sb_handle *dst_handle, struct bch_sb *src) { struct bch_sb_field *src_f, *dst_f; + struct bch_sb *dst = dst_handle->sb; dst->version = src->version; dst->seq = src->seq; @@ -433,8 +373,8 @@ static void __copy_super(struct bch_sb *dst, struct bch_sb *src) continue; dst_f = bch2_sb_field_get(dst, le32_to_cpu(src_f->type)); - dst_f = __bch2_sb_field_resize(dst, dst_f, - le32_to_cpu(src_f->u64s)); + dst_f = __bch2_sb_field_resize(dst_handle, dst_f, + le32_to_cpu(src_f->u64s)); memcpy(dst_f, src_f, vstruct_bytes(src_f)); } @@ -451,11 +391,12 @@ int bch2_sb_to_fs(struct bch_fs *c, struct bch_sb *src) lockdep_assert_held(&c->sb_lock); - ret = bch2_fs_sb_realloc(c, le32_to_cpu(src->u64s) - journal_u64s); + ret = bch2_sb_realloc(&c->disk_sb, + le32_to_cpu(src->u64s) - journal_u64s); if (ret) return ret; - __copy_super(c->disk_sb, src); + __copy_super(&c->disk_sb, src); ret = bch2_sb_replicas_to_cpu_replicas(c); if (ret) @@ -471,7 +412,7 @@ int bch2_sb_to_fs(struct bch_fs *c, struct bch_sb *src) int bch2_sb_from_fs(struct bch_fs *c, struct bch_dev *ca) { - struct bch_sb *src = c->disk_sb, *dst = ca->disk_sb.sb; + struct bch_sb *src = c->disk_sb.sb, *dst = ca->disk_sb.sb; struct bch_sb_field_journal *journal_buckets = bch2_sb_get_journal(dst); unsigned journal_u64s = journal_buckets @@ -484,7 +425,7 @@ int bch2_sb_from_fs(struct bch_fs *c, struct bch_dev *ca) if (ret) return ret; - __copy_super(dst, src); + __copy_super(&ca->disk_sb, src); return 0; } @@ -494,7 +435,6 @@ static const char *read_one_super(struct bch_sb_handle *sb, u64 offset) { struct bch_csum csum; size_t bytes; - unsigned order; reread: bio_reset(sb->bio); bio_set_dev(sb->bio, sb->bdev); @@ -518,9 +458,8 @@ reread: if (bytes > 512 << sb->sb->layout.sb_max_size_bits) return "Bad superblock: too big"; - order = get_order(bytes); - if (order > sb->page_order) { - if (__bch2_super_realloc(sb, order)) + if (get_order(bytes) > sb->page_order) { + if (bch2_sb_realloc(sb, le32_to_cpu(sb->sb->u64s))) return "cannot allocate memory"; goto reread; } @@ -550,7 +489,8 @@ int bch2_read_super(const char *path, struct bch_opts *opts, pr_verbose_init(*opts, ""); memset(sb, 0, sizeof(*sb)); - sb->mode = FMODE_READ; + sb->mode = FMODE_READ; + sb->have_bio = true; if (!opt_get(*opts, noexcl)) sb->mode |= FMODE_EXCL; @@ -575,7 +515,7 @@ int bch2_read_super(const char *path, struct bch_opts *opts, } err = "cannot allocate memory"; - ret = __bch2_super_realloc(sb, 0); + ret = bch2_sb_realloc(sb, 0); if (ret) goto err; @@ -644,6 +584,7 @@ got_super: bdev_get_queue(sb->bdev)->backing_dev_info->capabilities |= BDI_CAP_STABLE_WRITES; ret = 0; + sb->have_layout = true; out: pr_verbose_init(*opts, "ret %i", ret); return ret; @@ -711,7 +652,7 @@ void bch2_write_super(struct bch_fs *c) closure_init_stack(cl); memset(&sb_written, 0, sizeof(sb_written)); - le64_add_cpu(&c->disk_sb->seq, 1); + le64_add_cpu(&c->disk_sb.sb->seq, 1); for_each_online_member(ca, c, i) bch2_sb_from_fs(c, ca); @@ -837,6 +778,10 @@ err: return err; } +static const struct bch_sb_field_ops bch_sb_field_ops_journal = { + .validate = bch2_sb_validate_journal, +}; + /* BCH_SB_FIELD_members: */ static const char *bch2_sb_validate_members(struct bch_sb *sb, @@ -880,6 +825,10 @@ static const char *bch2_sb_validate_members(struct bch_sb *sb, return NULL; } +static const struct bch_sb_field_ops bch_sb_field_ops_members = { + .validate = bch2_sb_validate_members, +}; + /* BCH_SB_FIELD_crypt: */ static const char *bch2_sb_validate_crypt(struct bch_sb *sb, @@ -896,980 +845,42 @@ static const char *bch2_sb_validate_crypt(struct bch_sb *sb, return NULL; } -/* BCH_SB_FIELD_replicas: */ - -/* Replicas tracking - in memory: */ - -#define for_each_cpu_replicas_entry(_r, _i) \ - for (_i = (_r)->entries; \ - (void *) (_i) < (void *) (_r)->entries + (_r)->nr * (_r)->entry_size;\ - _i = (void *) (_i) + (_r)->entry_size) - -static inline struct bch_replicas_cpu_entry * -cpu_replicas_entry(struct bch_replicas_cpu *r, unsigned i) -{ - return (void *) r->entries + r->entry_size * i; -} - -static void bch2_cpu_replicas_sort(struct bch_replicas_cpu *r) -{ - eytzinger0_sort(r->entries, r->nr, r->entry_size, memcmp, NULL); -} - -static inline bool replicas_test_dev(struct bch_replicas_cpu_entry *e, - unsigned dev) -{ - return (e->devs[dev >> 3] & (1 << (dev & 7))) != 0; -} - -static inline void replicas_set_dev(struct bch_replicas_cpu_entry *e, - unsigned dev) -{ - e->devs[dev >> 3] |= 1 << (dev & 7); -} - -static inline unsigned replicas_dev_slots(struct bch_replicas_cpu *r) -{ - return (r->entry_size - - offsetof(struct bch_replicas_cpu_entry, devs)) * 8; -} - -int bch2_cpu_replicas_to_text(struct bch_replicas_cpu *r, - char *buf, size_t size) -{ - char *out = buf, *end = out + size; - struct bch_replicas_cpu_entry *e; - bool first = true; - unsigned i; - - for_each_cpu_replicas_entry(r, e) { - bool first_e = true; - - if (!first) - out += scnprintf(out, end - out, " "); - first = false; - - out += scnprintf(out, end - out, "%u: [", e->data_type); - - for (i = 0; i < replicas_dev_slots(r); i++) - if (replicas_test_dev(e, i)) { - if (!first_e) - out += scnprintf(out, end - out, " "); - first_e = false; - out += scnprintf(out, end - out, "%u", i); - } - out += scnprintf(out, end - out, "]"); - } - - return out - buf; -} - -static inline unsigned bkey_to_replicas(struct bkey_s_c_extent e, - enum bch_data_type data_type, - struct bch_replicas_cpu_entry *r, - unsigned *max_dev) -{ - const struct bch_extent_ptr *ptr; - unsigned nr = 0; - - BUG_ON(!data_type || - data_type == BCH_DATA_SB || - data_type >= BCH_DATA_NR); - - memset(r, 0, sizeof(*r)); - r->data_type = data_type; - - *max_dev = 0; - - extent_for_each_ptr(e, ptr) - if (!ptr->cached) { - *max_dev = max_t(unsigned, *max_dev, ptr->dev); - replicas_set_dev(r, ptr->dev); - nr++; - } - return nr; -} - -static inline void devlist_to_replicas(struct bch_devs_list devs, - enum bch_data_type data_type, - struct bch_replicas_cpu_entry *r, - unsigned *max_dev) -{ - unsigned i; - - BUG_ON(!data_type || - data_type == BCH_DATA_SB || - data_type >= BCH_DATA_NR); - - memset(r, 0, sizeof(*r)); - r->data_type = data_type; - - *max_dev = 0; - - for (i = 0; i < devs.nr; i++) { - *max_dev = max_t(unsigned, *max_dev, devs.devs[i]); - replicas_set_dev(r, devs.devs[i]); - } -} - -static struct bch_replicas_cpu * -cpu_replicas_add_entry(struct bch_replicas_cpu *old, - struct bch_replicas_cpu_entry new_entry, - unsigned max_dev) -{ - struct bch_replicas_cpu *new; - unsigned i, nr, entry_size; - - entry_size = offsetof(struct bch_replicas_cpu_entry, devs) + - DIV_ROUND_UP(max_dev + 1, 8); - entry_size = max(entry_size, old->entry_size); - nr = old->nr + 1; - - new = kzalloc(sizeof(struct bch_replicas_cpu) + - nr * entry_size, GFP_NOIO); - if (!new) - return NULL; - - new->nr = nr; - new->entry_size = entry_size; - - for (i = 0; i < old->nr; i++) - memcpy(cpu_replicas_entry(new, i), - cpu_replicas_entry(old, i), - min(new->entry_size, old->entry_size)); - - memcpy(cpu_replicas_entry(new, old->nr), - &new_entry, - new->entry_size); - - bch2_cpu_replicas_sort(new); - return new; -} - -static bool replicas_has_entry(struct bch_replicas_cpu *r, - struct bch_replicas_cpu_entry search, - unsigned max_dev) -{ - return max_dev < replicas_dev_slots(r) && - eytzinger0_find(r->entries, r->nr, - r->entry_size, - memcmp, &search) < r->nr; -} - -noinline -static int bch2_mark_replicas_slowpath(struct bch_fs *c, - struct bch_replicas_cpu_entry new_entry, - unsigned max_dev) -{ - struct bch_replicas_cpu *old_gc, *new_gc = NULL, *old_r, *new_r = NULL; - int ret = -ENOMEM; - - mutex_lock(&c->sb_lock); - - old_gc = rcu_dereference_protected(c->replicas_gc, - lockdep_is_held(&c->sb_lock)); - if (old_gc && !replicas_has_entry(old_gc, new_entry, max_dev)) { - new_gc = cpu_replicas_add_entry(old_gc, new_entry, max_dev); - if (!new_gc) - goto err; - } - - old_r = rcu_dereference_protected(c->replicas, - lockdep_is_held(&c->sb_lock)); - if (!replicas_has_entry(old_r, new_entry, max_dev)) { - new_r = cpu_replicas_add_entry(old_r, new_entry, max_dev); - if (!new_r) - goto err; - - ret = bch2_cpu_replicas_to_sb_replicas(c, new_r); - if (ret) - goto err; - } - - /* allocations done, now commit: */ - - if (new_r) - bch2_write_super(c); - - /* don't update in memory replicas until changes are persistent */ - - if (new_gc) { - rcu_assign_pointer(c->replicas_gc, new_gc); - kfree_rcu(old_gc, rcu); - } - - if (new_r) { - rcu_assign_pointer(c->replicas, new_r); - kfree_rcu(old_r, rcu); - } - - mutex_unlock(&c->sb_lock); - return 0; -err: - mutex_unlock(&c->sb_lock); - if (new_gc) - kfree(new_gc); - if (new_r) - kfree(new_r); - return ret; -} - -int bch2_mark_replicas(struct bch_fs *c, - enum bch_data_type data_type, - struct bch_devs_list devs) -{ - struct bch_replicas_cpu_entry search; - struct bch_replicas_cpu *r, *gc_r; - unsigned max_dev; - bool marked; - - if (!devs.nr) - return 0; - - BUG_ON(devs.nr >= BCH_REPLICAS_MAX); - - devlist_to_replicas(devs, data_type, &search, &max_dev); - - rcu_read_lock(); - r = rcu_dereference(c->replicas); - gc_r = rcu_dereference(c->replicas_gc); - marked = replicas_has_entry(r, search, max_dev) && - (!likely(gc_r) || replicas_has_entry(gc_r, search, max_dev)); - rcu_read_unlock(); - - return likely(marked) ? 0 - : bch2_mark_replicas_slowpath(c, search, max_dev); -} - -int bch2_mark_bkey_replicas(struct bch_fs *c, - enum bch_data_type data_type, - struct bkey_s_c k) -{ - struct bch_devs_list cached = bch2_bkey_cached_devs(k); - unsigned i; - int ret; - - for (i = 0; i < cached.nr; i++) - if ((ret = bch2_mark_replicas(c, BCH_DATA_CACHED, - bch2_dev_list_single(cached.devs[i])))) - return ret; - - return bch2_mark_replicas(c, data_type, bch2_bkey_dirty_devs(k)); -} - -int bch2_replicas_gc_end(struct bch_fs *c, int err) -{ - struct bch_replicas_cpu *new_r, *old_r; - int ret = 0; - - lockdep_assert_held(&c->replicas_gc_lock); - - mutex_lock(&c->sb_lock); - - new_r = rcu_dereference_protected(c->replicas_gc, - lockdep_is_held(&c->sb_lock)); - - if (err) { - rcu_assign_pointer(c->replicas_gc, NULL); - kfree_rcu(new_r, rcu); - goto err; - } - - if (bch2_cpu_replicas_to_sb_replicas(c, new_r)) { - ret = -ENOSPC; - goto err; - } - - old_r = rcu_dereference_protected(c->replicas, - lockdep_is_held(&c->sb_lock)); - - rcu_assign_pointer(c->replicas, new_r); - rcu_assign_pointer(c->replicas_gc, NULL); - kfree_rcu(old_r, rcu); - - bch2_write_super(c); -err: - mutex_unlock(&c->sb_lock); - return ret; -} - -int bch2_replicas_gc_start(struct bch_fs *c, unsigned typemask) -{ - struct bch_replicas_cpu *dst, *src; - struct bch_replicas_cpu_entry *e; - - lockdep_assert_held(&c->replicas_gc_lock); - - mutex_lock(&c->sb_lock); - BUG_ON(c->replicas_gc); - - src = rcu_dereference_protected(c->replicas, - lockdep_is_held(&c->sb_lock)); - - dst = kzalloc(sizeof(struct bch_replicas_cpu) + - src->nr * src->entry_size, GFP_NOIO); - if (!dst) { - mutex_unlock(&c->sb_lock); - return -ENOMEM; - } - - dst->nr = 0; - dst->entry_size = src->entry_size; - - for_each_cpu_replicas_entry(src, e) - if (!((1 << e->data_type) & typemask)) - memcpy(cpu_replicas_entry(dst, dst->nr++), - e, dst->entry_size); - - bch2_cpu_replicas_sort(dst); - - rcu_assign_pointer(c->replicas_gc, dst); - mutex_unlock(&c->sb_lock); - - return 0; -} - -/* Replicas tracking - superblock: */ - -static void bch2_sb_replicas_nr_entries(struct bch_sb_field_replicas *r, - unsigned *nr, - unsigned *bytes, - unsigned *max_dev) -{ - struct bch_replicas_entry *i; - unsigned j; - - *nr = 0; - *bytes = sizeof(*r); - *max_dev = 0; - - if (!r) - return; - - for_each_replicas_entry(r, i) { - for (j = 0; j < i->nr; j++) - *max_dev = max_t(unsigned, *max_dev, i->devs[j]); - (*nr)++; - } - - *bytes = (void *) i - (void *) r; -} - -static struct bch_replicas_cpu * -__bch2_sb_replicas_to_cpu_replicas(struct bch_sb_field_replicas *sb_r) -{ - struct bch_replicas_cpu *cpu_r; - unsigned i, nr, bytes, max_dev, entry_size; - - bch2_sb_replicas_nr_entries(sb_r, &nr, &bytes, &max_dev); - - entry_size = offsetof(struct bch_replicas_cpu_entry, devs) + - DIV_ROUND_UP(max_dev + 1, 8); - - cpu_r = kzalloc(sizeof(struct bch_replicas_cpu) + - nr * entry_size, GFP_NOIO); - if (!cpu_r) - return NULL; - - cpu_r->nr = nr; - cpu_r->entry_size = entry_size; - - if (nr) { - struct bch_replicas_cpu_entry *dst = - cpu_replicas_entry(cpu_r, 0); - struct bch_replicas_entry *src = sb_r->entries; - - while (dst < cpu_replicas_entry(cpu_r, nr)) { - dst->data_type = src->data_type; - for (i = 0; i < src->nr; i++) - replicas_set_dev(dst, src->devs[i]); - - src = replicas_entry_next(src); - dst = (void *) dst + entry_size; - } - } - - bch2_cpu_replicas_sort(cpu_r); - return cpu_r; -} - -static int bch2_sb_replicas_to_cpu_replicas(struct bch_fs *c) -{ - struct bch_sb_field_replicas *sb_r; - struct bch_replicas_cpu *cpu_r, *old_r; - - sb_r = bch2_sb_get_replicas(c->disk_sb); - cpu_r = __bch2_sb_replicas_to_cpu_replicas(sb_r); - if (!cpu_r) - return -ENOMEM; - - old_r = rcu_dereference_check(c->replicas, lockdep_is_held(&c->sb_lock)); - rcu_assign_pointer(c->replicas, cpu_r); - if (old_r) - kfree_rcu(old_r, rcu); - - return 0; -} - -static int bch2_cpu_replicas_to_sb_replicas(struct bch_fs *c, - struct bch_replicas_cpu *r) -{ - struct bch_sb_field_replicas *sb_r; - struct bch_replicas_entry *sb_e; - struct bch_replicas_cpu_entry *e; - size_t i, bytes; - - bytes = sizeof(struct bch_sb_field_replicas); - - for_each_cpu_replicas_entry(r, e) { - bytes += sizeof(struct bch_replicas_entry); - for (i = 0; i < r->entry_size - 1; i++) - bytes += hweight8(e->devs[i]); - } - - sb_r = bch2_fs_sb_resize_replicas(c, - DIV_ROUND_UP(sizeof(*sb_r) + bytes, sizeof(u64))); - if (!sb_r) - return -ENOSPC; - - memset(&sb_r->entries, 0, - vstruct_end(&sb_r->field) - - (void *) &sb_r->entries); - - sb_e = sb_r->entries; - for_each_cpu_replicas_entry(r, e) { - sb_e->data_type = e->data_type; - - for (i = 0; i < replicas_dev_slots(r); i++) - if (replicas_test_dev(e, i)) - sb_e->devs[sb_e->nr++] = i; - - sb_e = replicas_entry_next(sb_e); - - BUG_ON((void *) sb_e > vstruct_end(&sb_r->field)); - } - - return 0; -} - -static const char *bch2_sb_validate_replicas(struct bch_sb *sb, - struct bch_sb_field *f) -{ - struct bch_sb_field_replicas *sb_r = field_to_type(f, replicas); - struct bch_sb_field_members *mi = bch2_sb_get_members(sb); - struct bch_replicas_cpu *cpu_r = NULL; - struct bch_replicas_entry *e; - const char *err; - unsigned i; - - for_each_replicas_entry(sb_r, e) { - err = "invalid replicas entry: invalid data type"; - if (e->data_type >= BCH_DATA_NR) - goto err; - - err = "invalid replicas entry: no devices"; - if (!e->nr) - goto err; - - err = "invalid replicas entry: too many devices"; - if (e->nr >= BCH_REPLICAS_MAX) - goto err; - - err = "invalid replicas entry: invalid device"; - for (i = 0; i < e->nr; i++) - if (!bch2_dev_exists(sb, mi, e->devs[i])) - goto err; - } - - err = "cannot allocate memory"; - cpu_r = __bch2_sb_replicas_to_cpu_replicas(sb_r); - if (!cpu_r) - goto err; - - sort_cmp_size(cpu_r->entries, - cpu_r->nr, - cpu_r->entry_size, - memcmp, NULL); - - for (i = 0; i + 1 < cpu_r->nr; i++) { - struct bch_replicas_cpu_entry *l = - cpu_replicas_entry(cpu_r, i); - struct bch_replicas_cpu_entry *r = - cpu_replicas_entry(cpu_r, i + 1); - - BUG_ON(memcmp(l, r, cpu_r->entry_size) > 0); - - err = "duplicate replicas entry"; - if (!memcmp(l, r, cpu_r->entry_size)) - goto err; - } - - err = NULL; -err: - kfree(cpu_r); - return err; -} - -int bch2_sb_replicas_to_text(struct bch_sb_field_replicas *r, char *buf, size_t size) -{ - char *out = buf, *end = out + size; - struct bch_replicas_entry *e; - bool first = true; - unsigned i; - - if (!r) { - out += scnprintf(out, end - out, "(no replicas section found)"); - return out - buf; - } - - for_each_replicas_entry(r, e) { - if (!first) - out += scnprintf(out, end - out, " "); - first = false; - - out += scnprintf(out, end - out, "%u: [", e->data_type); - - for (i = 0; i < e->nr; i++) - out += scnprintf(out, end - out, - i ? " %u" : "%u", e->devs[i]); - out += scnprintf(out, end - out, "]"); - } - - return out - buf; -} - -/* Query replicas: */ - -bool bch2_replicas_marked(struct bch_fs *c, - enum bch_data_type data_type, - struct bch_devs_list devs) -{ - struct bch_replicas_cpu_entry search; - unsigned max_dev; - bool ret; - - if (!devs.nr) - return true; - - devlist_to_replicas(devs, data_type, &search, &max_dev); - - rcu_read_lock(); - ret = replicas_has_entry(rcu_dereference(c->replicas), - search, max_dev); - rcu_read_unlock(); - - return ret; -} - -bool bch2_bkey_replicas_marked(struct bch_fs *c, - enum bch_data_type data_type, - struct bkey_s_c k) -{ - struct bch_devs_list cached = bch2_bkey_cached_devs(k); - unsigned i; - - for (i = 0; i < cached.nr; i++) - if (!bch2_replicas_marked(c, BCH_DATA_CACHED, - bch2_dev_list_single(cached.devs[i]))) - return false; - - return bch2_replicas_marked(c, data_type, bch2_bkey_dirty_devs(k)); -} - -struct replicas_status __bch2_replicas_status(struct bch_fs *c, - struct bch_devs_mask online_devs) -{ - struct bch_sb_field_members *mi; - struct bch_replicas_cpu_entry *e; - struct bch_replicas_cpu *r; - unsigned i, dev, dev_slots, nr_online, nr_offline; - struct replicas_status ret; - - memset(&ret, 0, sizeof(ret)); - - for (i = 0; i < ARRAY_SIZE(ret.replicas); i++) - ret.replicas[i].nr_online = UINT_MAX; - - mi = bch2_sb_get_members(c->disk_sb); - rcu_read_lock(); - - r = rcu_dereference(c->replicas); - dev_slots = replicas_dev_slots(r); - - for_each_cpu_replicas_entry(r, e) { - if (e->data_type >= ARRAY_SIZE(ret.replicas)) - panic("e %p data_type %u\n", e, e->data_type); - - nr_online = nr_offline = 0; - - for (dev = 0; dev < dev_slots; dev++) { - if (!replicas_test_dev(e, dev)) - continue; - - BUG_ON(!bch2_dev_exists(c->disk_sb, mi, dev)); - - if (test_bit(dev, online_devs.d)) - nr_online++; - else - nr_offline++; - } - - ret.replicas[e->data_type].nr_online = - min(ret.replicas[e->data_type].nr_online, - nr_online); - - ret.replicas[e->data_type].nr_offline = - max(ret.replicas[e->data_type].nr_offline, - nr_offline); - } - - rcu_read_unlock(); - - return ret; -} - -struct replicas_status bch2_replicas_status(struct bch_fs *c) -{ - return __bch2_replicas_status(c, bch2_online_devs(c)); -} - -static bool have_enough_devs(struct replicas_status s, - enum bch_data_type type, - bool force_if_degraded, - bool force_if_lost) -{ - return (!s.replicas[type].nr_offline || force_if_degraded) && - (s.replicas[type].nr_online || force_if_lost); -} - -bool bch2_have_enough_devs(struct replicas_status s, unsigned flags) -{ - return (have_enough_devs(s, BCH_DATA_JOURNAL, - flags & BCH_FORCE_IF_METADATA_DEGRADED, - flags & BCH_FORCE_IF_METADATA_LOST) && - have_enough_devs(s, BCH_DATA_BTREE, - flags & BCH_FORCE_IF_METADATA_DEGRADED, - flags & BCH_FORCE_IF_METADATA_LOST) && - have_enough_devs(s, BCH_DATA_USER, - flags & BCH_FORCE_IF_DATA_DEGRADED, - flags & BCH_FORCE_IF_DATA_LOST)); -} - -unsigned bch2_replicas_online(struct bch_fs *c, bool meta) -{ - struct replicas_status s = bch2_replicas_status(c); - - return meta - ? min(s.replicas[BCH_DATA_JOURNAL].nr_online, - s.replicas[BCH_DATA_BTREE].nr_online) - : s.replicas[BCH_DATA_USER].nr_online; -} - -unsigned bch2_dev_has_data(struct bch_fs *c, struct bch_dev *ca) -{ - struct bch_replicas_cpu_entry *e; - struct bch_replicas_cpu *r; - unsigned ret = 0; - - rcu_read_lock(); - r = rcu_dereference(c->replicas); - - if (ca->dev_idx >= replicas_dev_slots(r)) - goto out; - - for_each_cpu_replicas_entry(r, e) - if (replicas_test_dev(e, ca->dev_idx)) - ret |= 1 << e->data_type; -out: - rcu_read_unlock(); - - return ret; -} +static const struct bch_sb_field_ops bch_sb_field_ops_crypt = { + .validate = bch2_sb_validate_crypt, +}; -/* Quotas: */ +static const struct bch_sb_field_ops *bch2_sb_field_ops[] = { +#define x(f, nr) \ + [BCH_SB_FIELD_##f] = &bch_sb_field_ops_##f, + BCH_SB_FIELDS() +#undef x +}; -static const char *bch2_sb_validate_quota(struct bch_sb *sb, +static const char *bch2_sb_field_validate(struct bch_sb *sb, struct bch_sb_field *f) { - struct bch_sb_field_quota *q = field_to_type(f, quota); - - if (vstruct_bytes(&q->field) != sizeof(*q)) - return "invalid field quota: wrong size"; - - return NULL; -} - -/* Disk groups: */ - -static int strcmp_void(const void *l, const void *r) -{ - return strcmp(l, r); -} - -static const char *bch2_sb_validate_disk_groups(struct bch_sb *sb, - struct bch_sb_field *f) -{ - struct bch_sb_field_disk_groups *groups = - field_to_type(f, disk_groups); - struct bch_disk_group *g; - struct bch_sb_field_members *mi; - struct bch_member *m; - unsigned i, nr_groups, nr_live = 0, len; - char **labels, *l; - const char *err = NULL; - - mi = bch2_sb_get_members(sb); - groups = bch2_sb_get_disk_groups(sb); - nr_groups = disk_groups_nr(groups); - - for (m = mi->members; - m < mi->members + sb->nr_devices; - m++) { - unsigned g; - - if (!BCH_MEMBER_GROUP(m)) - continue; - - g = BCH_MEMBER_GROUP(m) - 1; - - if (g >= nr_groups || - BCH_GROUP_DELETED(&groups->entries[g])) - return "disk has invalid group"; - } - - if (!nr_groups) - return NULL; - - labels = kcalloc(nr_groups, sizeof(char *), GFP_KERNEL); - if (!labels) - return "cannot allocate memory"; - - for (g = groups->entries; - g < groups->entries + nr_groups; - g++) { - if (BCH_GROUP_DELETED(g)) - continue; - - len = strnlen(g->label, sizeof(g->label)); - - labels[nr_live++] = l = kmalloc(len + 1, GFP_KERNEL); - if (!l) { - err = "cannot allocate memory"; - goto err; - } - - memcpy(l, g->label, len); - l[len] = '\0'; - } - - sort(labels, nr_live, sizeof(labels[0]), strcmp_void, NULL); - - for (i = 0; i + 1 < nr_live; i++) - if (!strcmp(labels[i], labels[i + 1])) { - err = "duplicate group labels"; - goto err; - } - - err = NULL; -err: - for (i = 0; i < nr_live; i++) - kfree(labels[i]); - kfree(labels); - return err; -} - -static int bch2_sb_disk_groups_to_cpu(struct bch_fs *c) -{ - struct bch_sb_field_members *mi; - struct bch_sb_field_disk_groups *groups; - struct bch_disk_groups_cpu *cpu_g, *old_g; - unsigned i, nr_groups; - - lockdep_assert_held(&c->sb_lock); - - mi = bch2_sb_get_members(c->disk_sb); - groups = bch2_sb_get_disk_groups(c->disk_sb); - nr_groups = disk_groups_nr(groups); - - if (!groups) - return 0; - - cpu_g = kzalloc(sizeof(*cpu_g) + - sizeof(cpu_g->entries[0]) * nr_groups, GFP_KERNEL); - if (!cpu_g) - return -ENOMEM; - - cpu_g->nr = nr_groups; - - for (i = 0; i < nr_groups; i++) { - struct bch_disk_group *src = &groups->entries[i]; - struct bch_disk_group_cpu *dst = &cpu_g->entries[i]; - - dst->deleted = BCH_GROUP_DELETED(src); - } - - for (i = 0; i < c->disk_sb->nr_devices; i++) { - struct bch_member *m = mi->members + i; - struct bch_disk_group_cpu *dst = - &cpu_g->entries[BCH_MEMBER_GROUP(m)]; - - if (!bch2_member_exists(m)) - continue; - - dst = BCH_MEMBER_GROUP(m) - ? &cpu_g->entries[BCH_MEMBER_GROUP(m) - 1] - : NULL; - if (dst) - __set_bit(i, dst->devs.d); - } - - old_g = c->disk_groups; - rcu_assign_pointer(c->disk_groups, cpu_g); - if (old_g) - kfree_rcu(old_g, rcu); - - return 0; -} - -const struct bch_devs_mask *bch2_target_to_mask(struct bch_fs *c, unsigned target) -{ - struct target t = target_decode(target); - - switch (t.type) { - case TARGET_DEV: { - struct bch_dev *ca = t.dev < c->sb.nr_devices - ? rcu_dereference(c->devs[t.dev]) - : NULL; - return ca ? &ca->self : NULL; - } - case TARGET_GROUP: { - struct bch_disk_groups_cpu *g = rcu_dereference(c->disk_groups); - - return t.group < g->nr && !g->entries[t.group].deleted - ? &g->entries[t.group].devs - : NULL; - } - default: - BUG(); - } -} - -int __bch2_disk_group_find(struct bch_sb_field_disk_groups *groups, - const char *name) -{ - unsigned i, nr_groups = disk_groups_nr(groups); - unsigned len = strlen(name); - - for (i = 0; i < nr_groups; i++) { - struct bch_disk_group *g = groups->entries + i; - - if (BCH_GROUP_DELETED(g)) - continue; - - if (strnlen(g->label, sizeof(g->label)) == len && - !memcmp(name, g->label, len)) - return i; - } - - return -1; -} - -static int bch2_disk_group_find(struct bch_fs *c, const char *name) -{ - int ret; - - mutex_lock(&c->sb_lock); - ret = __bch2_disk_group_find(bch2_sb_get_disk_groups(c->disk_sb), name); - mutex_unlock(&c->sb_lock); + unsigned type = le32_to_cpu(f->type); - return ret; + return type < BCH_SB_FIELD_NR + ? bch2_sb_field_ops[type]->validate(sb, f) + : NULL; } -int bch2_opt_target_parse(struct bch_fs *c, const char *buf, u64 *v) +size_t bch2_sb_field_to_text(char *buf, size_t size, + struct bch_sb *sb, struct bch_sb_field *f) { - struct bch_dev *ca; - int g; - - if (!strlen(buf) || !strcmp(buf, "none")) { - *v = 0; - return 0; - } - - /* Is it a device? */ - ca = bch2_dev_lookup(c, buf); - if (!IS_ERR(ca)) { - *v = dev_to_target(ca->dev_idx); - percpu_ref_put(&ca->ref); - return 0; - } + unsigned type = le32_to_cpu(f->type); + size_t (*to_text)(char *, size_t, struct bch_sb *, + struct bch_sb_field *) = + type < BCH_SB_FIELD_NR + ? bch2_sb_field_ops[type]->to_text + : NULL; - g = bch2_disk_group_find(c, buf); - if (g >= 0) { - *v = group_to_target(g); + if (!to_text) { + if (size) + buf[0] = '\0'; return 0; } - return -EINVAL; -} - -int bch2_opt_target_print(struct bch_fs *c, char *buf, size_t len, u64 v) -{ - struct target t = target_decode(v); - int ret; - - switch (t.type) { - case TARGET_NULL: - return scnprintf(buf, len, "none"); - case TARGET_DEV: { - struct bch_dev *ca; - - rcu_read_lock(); - ca = t.dev < c->sb.nr_devices - ? rcu_dereference(c->devs[t.dev]) - : NULL; - - if (ca && percpu_ref_tryget(&ca->io_ref)) { - char b[BDEVNAME_SIZE]; - - ret = scnprintf(buf, len, "/dev/%s", - bdevname(ca->disk_sb.bdev, b)); - percpu_ref_put(&ca->io_ref); - } else if (ca) { - ret = scnprintf(buf, len, "offline device %u", t.dev); - } else { - ret = scnprintf(buf, len, "invalid device %u", t.dev); - } - - rcu_read_unlock(); - break; - } - case TARGET_GROUP: { - struct bch_sb_field_disk_groups *groups; - struct bch_disk_group *g; - - mutex_lock(&c->sb_lock); - groups = bch2_sb_get_disk_groups(c->disk_sb); - - g = t.group < disk_groups_nr(groups) - ? groups->entries + t.group - : NULL; - - if (g && !BCH_GROUP_DELETED(g)) { - ret = len ? min(len - 1, strnlen(g->label, sizeof(g->label))) : 0; - - memcpy(buf, g->label, ret); - if (len) - buf[ret] = '\0'; - } else { - ret = scnprintf(buf, len, "invalid group %u", t.group); - } - - mutex_unlock(&c->sb_lock); - break; - } - default: - BUG(); - } - - return ret; + return to_text(buf, size, sb, f); } |