From 2831b89a7c8ba9262a00ec6d70d1355c02f84a67 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 2 Apr 2017 19:43:35 -0800 Subject: More cmd_migrate improvements Factor out bch2_pick_bucket_size() from the format code, and pick the bucket size before picking the superblock location - that way we can ensure the superblock gets its own bucket and doesn't trigger warnings due to the allocation code noticing different types of data in the same bucket. --- cmd_device.c | 2 +- cmd_format.c | 8 ++-- cmd_key.c | 4 +- cmd_migrate.c | 29 +++++++++++---- crypto.c | 2 +- crypto.h | 2 +- libbcachefs.c | 117 ++++++++++++++++++++++++++++++---------------------------- libbcachefs.h | 11 +++--- 8 files changed, 98 insertions(+), 77 deletions(-) diff --git a/cmd_device.c b/cmd_device.c index f71ab238..ac586ffe 100644 --- a/cmd_device.c +++ b/cmd_device.c @@ -245,7 +245,7 @@ int cmd_device_add(int argc, char *argv[]) format_opts.btree_node_size = read_file_u64(fs.sysfs_fd, "btree_node_size_bytes") >> 9; - struct bch_sb *sb = bcache_format(format_opts, &dev_opts, 1); + struct bch_sb *sb = bch2_format(format_opts, &dev_opts, 1); free(sb); fsync(dev_opts.fd); close(dev_opts.fd); diff --git a/cmd_format.c b/cmd_format.c index fbf8bddd..ae6dd33d 100644 --- a/cmd_format.c +++ b/cmd_format.c @@ -263,10 +263,10 @@ int cmd_format(int argc, char *argv[]) dev->fd = open_for_format(dev->path, force); struct bch_sb *sb = - bcache_format(opts, devices.item, darray_size(devices)); + bch2_format(opts, devices.item, darray_size(devices)); if (!quiet) - bcache_super_print(sb, HUMAN_READABLE); + bch2_super_print(sb, HUMAN_READABLE); free(sb); if (opts.passphrase) { @@ -284,7 +284,7 @@ int cmd_show_super(int argc, char *argv[]) if (argc != 2) die("please supply a single device"); - sb = bcache_super_read(argv[1]); - bcache_super_print(sb, HUMAN_READABLE); + sb = bch2_super_read(argv[1]); + bch2_super_print(sb, HUMAN_READABLE); return 0; } diff --git a/cmd_key.c b/cmd_key.c index e9fc377b..b22de815 100644 --- a/cmd_key.c +++ b/cmd_key.c @@ -15,11 +15,11 @@ int cmd_unlock(int argc, char *argv[]) if (argc != 2) die("please supply a single device"); - sb = bcache_super_read(argv[1]); + sb = bch2_super_read(argv[1]); passphrase = read_passphrase("Enter passphrase: "); - add_bcache_key(sb, passphrase); + bch2_add_key(sb, passphrase); memzero_explicit(passphrase, strlen(passphrase)); free(passphrase); diff --git a/cmd_migrate.c b/cmd_migrate.c index 2810dc23..8ce84d65 100644 --- a/cmd_migrate.c +++ b/cmd_migrate.c @@ -390,6 +390,15 @@ static void copy_file(struct bch_fs *c, struct bch_inode_unpacked *dst, continue; } + if (e.fe_physical < 1 << 20) { + copy_data(c, dst, + src, + round_down(e.fe_logical, block_bytes(c)), + round_up(e.fe_logical + e.fe_length, + block_bytes(c))); + continue; + } + if ((e.fe_physical & (block_bytes(c) - 1))) die("Unaligned extent in %s - can't handle", src_path); @@ -598,11 +607,15 @@ static void copy_fs(struct bch_fs *c, int src_fd, const char *src_path, static void find_superblock_space(ranges extents, struct dev_opts *dev) { struct range *i; + darray_foreach(i, extents) { - u64 offset = max(256ULL << 10, i->start); + u64 start = round_up(max(256ULL << 10, i->start), + dev->bucket_size << 9); + u64 end = round_down(i->end, + dev->bucket_size << 9); - if (offset + (128 << 10) <= i->end) { - dev->sb_offset = offset >> 9; + if (start + (128 << 10) <= end) { + dev->sb_offset = start >> 9; dev->sb_end = dev->sb_offset + 256; return; } @@ -685,6 +698,8 @@ int cmd_migrate(int argc, char *argv[]) u64 bcachefs_inum; char *file_path = mprintf("%s/bcachefs", fs_path); + bch2_pick_bucket_size(format_opts, &dev); + ranges extents = reserve_new_fs_space(file_path, block_size, get_size(dev.path, dev.fd) / 5, &bcachefs_inum, stat.st_dev, force); @@ -710,11 +725,11 @@ int cmd_migrate(int argc, char *argv[]) } } - struct bch_sb *sb = bcache_format(format_opts, &dev, 1); + struct bch_sb *sb = bch2_format(format_opts, &dev, 1); u64 sb_offset = le64_to_cpu(sb->layout.sb_offset[0]); if (format_opts.passphrase) - add_bcache_key(sb, format_opts.passphrase); + bch2_add_key(sb, format_opts.passphrase); free(sb); @@ -810,7 +825,7 @@ int cmd_migrate_superblock(int argc, char *argv[]) die("Please specify offset of existing superblock"); int fd = xopen(dev, O_RDWR); - struct bch_sb *sb = __bcache_super_read(fd, offset); + struct bch_sb *sb = __bch2_super_read(fd, offset); if (sb->layout.nr_superblocks >= ARRAY_SIZE(sb->layout.sb_offset)) die("Can't add superblock: no space left in superblock layout"); @@ -826,7 +841,7 @@ int cmd_migrate_superblock(int argc, char *argv[]) sb->layout.sb_offset[0] = cpu_to_le64(BCH_SB_SECTOR); - bcache_super_write(fd, sb); + bch2_super_write(fd, sb); close(fd); return 0; diff --git a/crypto.c b/crypto.c index 5ef1c874..c692c59a 100644 --- a/crypto.c +++ b/crypto.c @@ -77,7 +77,7 @@ void derive_passphrase(struct bch_sb_field_crypt *crypt, } } -void add_bcache_key(struct bch_sb *sb, const char *passphrase) +void bch2_add_key(struct bch_sb *sb, const char *passphrase) { struct bch_sb_field_crypt *crypt = bch2_sb_get_crypt(sb); if (!crypt) diff --git a/crypto.h b/crypto.h index 91a8b9fc..1e66a433 100644 --- a/crypto.h +++ b/crypto.h @@ -10,7 +10,7 @@ struct bch_key; char *read_passphrase(const char *); void derive_passphrase(struct bch_sb_field_crypt *, struct bch_key *, const char *); -void add_bcache_key(struct bch_sb *, const char *); +void bch2_add_key(struct bch_sb *, const char *); void bch_sb_crypt_init(struct bch_sb *sb, struct bch_sb_field_crypt *, const char *); diff --git a/libbcachefs.c b/libbcachefs.c index 9e46ff9d..0fdf5da4 100644 --- a/libbcachefs.c +++ b/libbcachefs.c @@ -68,71 +68,76 @@ static void init_layout(struct bch_sb_layout *l, unsigned block_size, l->sb_offset[1] = cpu_to_le64(backup); } -struct bch_sb *bcache_format(struct format_opts opts, - struct dev_opts *devs, size_t nr_devs) +void bch2_pick_bucket_size(struct format_opts opts, struct dev_opts *dev) { - struct bch_sb *sb; - struct dev_opts *i; - struct bch_sb_field_members *mi; - unsigned u64s; + if (!dev->sb_offset) { + dev->sb_offset = BCH_SB_SECTOR; + dev->sb_end = BCH_SB_SECTOR + 256; + } - /* calculate block size: */ - if (!opts.block_size) - for (i = devs; i < devs + nr_devs; i++) - opts.block_size = max(opts.block_size, - get_blocksize(i->path, i->fd)); + if (!dev->size) + dev->size = get_size(dev->path, dev->fd) >> 9; - /* calculate bucket sizes: */ - for (i = devs; i < devs + nr_devs; i++) { - if (!i->sb_offset) { - i->sb_offset = BCH_SB_SECTOR; - i->sb_end = BCH_SB_SECTOR + 256; - } + if (!dev->bucket_size) { + if (dev->size < min_size(opts.block_size)) + die("cannot format %s, too small (%llu sectors, min %llu)", + dev->path, dev->size, min_size(opts.block_size)); - if (!i->size) - i->size = get_size(i->path, i->fd) >> 9; + /* Bucket size must be >= block size: */ + dev->bucket_size = opts.block_size; - if (!i->bucket_size) { - if (i->size < min_size(opts.block_size)) - die("cannot format %s, too small (%llu sectors, min %llu)", - i->path, i->size, min_size(opts.block_size)); + /* Bucket size must be >= btree node size: */ + dev->bucket_size = max(dev->bucket_size, opts.btree_node_size); - /* Bucket size must be >= block size: */ - i->bucket_size = opts.block_size; + /* Want a bucket size of at least 128k, if possible: */ + dev->bucket_size = max(dev->bucket_size, 256U); - /* Bucket size must be >= btree node size: */ - i->bucket_size = max(i->bucket_size, opts.btree_node_size); + if (dev->size >= min_size(dev->bucket_size)) { + unsigned scale = max(1, + ilog2(dev->size / min_size(dev->bucket_size)) / 4); - /* Want a bucket size of at least 128k, if possible: */ - i->bucket_size = max(i->bucket_size, 256U); + scale = rounddown_pow_of_two(scale); - if (i->size >= min_size(i->bucket_size)) { - unsigned scale = max(1, - ilog2(i->size / min_size(i->bucket_size)) / 4); + /* max bucket size 1 mb */ + dev->bucket_size = min(dev->bucket_size * scale, 1U << 11); + } else { + do { + dev->bucket_size /= 2; + } while (dev->size < min_size(dev->bucket_size)); + } + } - scale = rounddown_pow_of_two(scale); + dev->nbuckets = dev->size / dev->bucket_size; - /* max bucket size 1 mb */ - i->bucket_size = min(i->bucket_size * scale, 1U << 11); - } else { - do { - i->bucket_size /= 2; - } while (i->size < min_size(i->bucket_size)); - } - } + if (dev->bucket_size < opts.block_size) + die("Bucket size cannot be smaller than block size"); - i->nbuckets = i->size / i->bucket_size; + if (dev->bucket_size < opts.btree_node_size) + die("Bucket size cannot be smaller than btree node size"); - if (i->bucket_size < opts.block_size) - die("Bucket size cannot be smaller than block size"); + if (dev->nbuckets < BCH_MIN_NR_NBUCKETS) + die("Not enough buckets: %llu, need %u (bucket size %u)", + dev->nbuckets, BCH_MIN_NR_NBUCKETS, dev->bucket_size); - if (i->bucket_size < opts.btree_node_size) - die("Bucket size cannot be smaller than btree node size"); +} - if (i->nbuckets < BCH_MIN_NR_NBUCKETS) - die("Not enough buckets: %llu, need %u (bucket size %u)", - i->nbuckets, BCH_MIN_NR_NBUCKETS, i->bucket_size); - } +struct bch_sb *bch2_format(struct format_opts opts, + struct dev_opts *devs, size_t nr_devs) +{ + struct bch_sb *sb; + struct dev_opts *i; + struct bch_sb_field_members *mi; + unsigned u64s; + + /* calculate block size: */ + if (!opts.block_size) + for (i = devs; i < devs + nr_devs; i++) + opts.block_size = max(opts.block_size, + get_blocksize(i->path, i->fd)); + + /* calculate bucket sizes: */ + for (i = devs; i < devs + nr_devs; i++) + bch2_pick_bucket_size(opts, i); /* calculate btree node size: */ if (!opts.btree_node_size) { @@ -242,14 +247,14 @@ struct bch_sb *bcache_format(struct format_opts opts, xpwrite(i->fd, zeroes, BCH_SB_SECTOR << 9, 0); } - bcache_super_write(i->fd, sb); + bch2_super_write(i->fd, sb); close(i->fd); } return sb; } -void bcache_super_write(int fd, struct bch_sb *sb) +void bch2_super_write(int fd, struct bch_sb *sb) { struct nonce nonce = { 0 }; @@ -270,7 +275,7 @@ void bcache_super_write(int fd, struct bch_sb *sb) fsync(fd); } -struct bch_sb *__bcache_super_read(int fd, u64 sector) +struct bch_sb *__bch2_super_read(int fd, u64 sector) { struct bch_sb sb, *ret; @@ -288,15 +293,15 @@ struct bch_sb *__bcache_super_read(int fd, u64 sector) return ret; } -struct bch_sb *bcache_super_read(const char *path) +struct bch_sb *bch2_super_read(const char *path) { int fd = xopen(path, O_RDONLY); - struct bch_sb *sb = __bcache_super_read(fd, BCH_SB_SECTOR); + struct bch_sb *sb = __bch2_super_read(fd, BCH_SB_SECTOR); close(fd); return sb; } -void bcache_super_print(struct bch_sb *sb, int units) +void bch2_super_print(struct bch_sb *sb, int units) { struct bch_sb_field_members *mi; char user_uuid_str[40], internal_uuid_str[40], member_uuid_str[40]; diff --git a/libbcachefs.h b/libbcachefs.h index 2e677aef..e5f3b867 100644 --- a/libbcachefs.h +++ b/libbcachefs.h @@ -59,12 +59,13 @@ struct dev_opts { u64 sb_end; }; -struct bch_sb *bcache_format(struct format_opts, struct dev_opts *, size_t); +void bch2_pick_bucket_size(struct format_opts, struct dev_opts *); +struct bch_sb *bch2_format(struct format_opts, struct dev_opts *, size_t); -void bcache_super_write(int, struct bch_sb *); -struct bch_sb *__bcache_super_read(int, u64); -struct bch_sb *bcache_super_read(const char *); +void bch2_super_write(int, struct bch_sb *); +struct bch_sb *__bch2_super_read(int, u64); +struct bch_sb *bch2_super_read(const char *); -void bcache_super_print(struct bch_sb *, int); +void bch2_super_print(struct bch_sb *, int); #endif /* _LIBBCACHE_H */ -- cgit v1.2.3