diff options
Diffstat (limited to 'c_src/cmd_image.c')
-rw-r--r-- | c_src/cmd_image.c | 499 |
1 files changed, 414 insertions, 85 deletions
diff --git a/c_src/cmd_image.c b/c_src/cmd_image.c index 12478b1f..9da3ed34 100644 --- a/c_src/cmd_image.c +++ b/c_src/cmd_image.c @@ -26,8 +26,11 @@ #include "crypto.h" #include "libbcachefs/alloc_background.h" #include "libbcachefs/alloc_foreground.h" +#include "libbcachefs/btree_update.h" #include "libbcachefs/data_update.h" +#include "libbcachefs/disk_accounting.h" #include "libbcachefs/errcode.h" +#include "libbcachefs/journal_reclaim.h" #include "libbcachefs/move.h" #include "libbcachefs/opts.h" #include "libbcachefs/super-io.h" @@ -64,6 +67,22 @@ static u64 count_input_size(int dirfd) return bytes; } +static void set_data_allowed_for_image_update(struct bch_fs *c) +{ + mutex_lock(&c->sb_lock); + struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, 0); + SET_BCH_MEMBER_DATA_ALLOWED(m, BIT(BCH_DATA_user)); + + m = bch2_members_v2_get_mut(c->disk_sb.sb, 1); + SET_BCH_MEMBER_DATA_ALLOWED(m, BIT(BCH_DATA_journal)|BIT(BCH_DATA_btree)); + + bch2_write_super(c); + mutex_unlock(&c->sb_lock); + + bch2_dev_allocator_set_rw(c, c->devs[0], true); + bch2_dev_allocator_set_rw(c, c->devs[1], true); +} + struct move_btree_args { bool move_alloc; unsigned target; @@ -76,7 +95,6 @@ static bool move_btree_pred(struct bch_fs *c, void *_arg, { struct move_btree_args *args = _arg; - data_opts->target = dev_to_target(0); data_opts->target = args->target; if (k.k->type != KEY_TYPE_btree_ptr_v2) @@ -91,6 +109,12 @@ static bool move_btree_pred(struct bch_fs *c, void *_arg, static int move_btree(struct bch_fs *c, bool move_alloc, unsigned target_dev) { + /* + * Flush the key cache first, otherwise key cache flushing later will do + * btree updates to the wrong device + */ + bch2_journal_flush_all_pins(&c->journal); + struct move_btree_args args = { .move_alloc = move_alloc, .target = dev_to_target(target_dev), @@ -141,39 +165,223 @@ static void check_gaps(struct bch_fs *c) static int get_nbuckets_used(struct bch_fs *c, u64 *nbuckets) { - struct btree_trans *trans = bch2_trans_get(c); - struct btree_iter iter; - bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, POS(0, U64_MAX), 0); + CLASS(btree_trans, trans)(c); + CLASS(btree_iter, iter)(trans, BTREE_ID_alloc, POS(0, U64_MAX), 0); struct bkey_s_c k; - int ret = lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_prev(trans, &iter))); + int ret = lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_prev(&iter))); if (!ret && k.k->type != KEY_TYPE_alloc_v4) ret = -ENOENT; if (ret) { fprintf(stderr, "error looking up last alloc key: %s\n", bch2_err_str(ret)); - goto err; + return ret; } *nbuckets = (k.k->p.offset + 1); -err: - bch2_trans_iter_exit(trans, &iter); - bch2_trans_put(trans); - return ret; + return 0; } -static void print_dev_usage_all(struct bch_fs *c) +static void prt_sectors(struct printbuf *out, u64 v) +{ + prt_tab(out); + prt_human_readable_u64(out, v << 9); + prt_tab_rjust(out); + prt_newline(out); +} + +static void print_data_type_usage(struct printbuf *out, + struct bch_dev *ca, + struct bch_dev_usage_full u, + unsigned i) +{ + if (u.d[i].buckets) { + bch2_prt_data_type(out, i); + prt_sectors(out, bucket_to_sector(ca, u.d[i].buckets)); + } + + if (u.d[i].fragmented) { + bch2_prt_data_type(out, i); + prt_str(out, " fragmented"); + prt_sectors(out, u.d[i].fragmented); + } +} + +static void print_image_usage(struct bch_fs *c, bool keep_alloc, u64 nbuckets) { struct printbuf buf = PRINTBUF; + printbuf_tabstop_push(&buf, 24); + printbuf_tabstop_push(&buf, 16); + printbuf_tabstop_push(&buf, 16); + printbuf_tabstop_push(&buf, 8); + + struct bch_dev *ca = c->devs[0]; + struct bch_dev_usage_full dev_stats = bch2_dev_usage_full_read(ca); + + print_data_type_usage(&buf, ca, dev_stats, BCH_DATA_sb); + print_data_type_usage(&buf, ca, dev_stats, BCH_DATA_journal); + print_data_type_usage(&buf, ca, dev_stats, BCH_DATA_btree); + + printbuf_indent_add(&buf, 2); + + for (unsigned i = 0; i < BTREE_ID_NR; i++) { + if (btree_id_is_alloc(i) && !keep_alloc) + continue; + + struct disk_accounting_pos a; + disk_accounting_key_init(a, btree, i); + + u64 v = 0; + bch2_accounting_mem_read(c, disk_accounting_pos_to_bpos(&a), &v, 1); + + if (v) { + bch2_btree_id_to_text(&buf, i); + prt_sectors(&buf, v); + } + } + + printbuf_indent_sub(&buf, 2); + + struct disk_accounting_pos acc_replicas_key; + memset(&acc_replicas_key, 0, sizeof(acc_replicas_key)); + acc_replicas_key.type = BCH_DISK_ACCOUNTING_replicas; + acc_replicas_key.replicas.data_type = BCH_DATA_user; + acc_replicas_key.replicas.nr_devs = 0; + acc_replicas_key.replicas.nr_required = 1; + acc_replicas_key.replicas.nr_required = 1; + replicas_entry_add_dev(&acc_replicas_key.replicas, 0); + + u64 v = 0; + bch2_accounting_mem_read(c, disk_accounting_pos_to_bpos(&acc_replicas_key), &v, 1); + prt_printf(&buf, "user"); + prt_sectors(&buf, v); + + if (dev_stats.d[BCH_DATA_user].fragmented) { + prt_printf(&buf, "user fragmented"); + prt_sectors(&buf, dev_stats.d[BCH_DATA_user].fragmented); + } + + bool compression_header = false; + for (unsigned i = 1; i < BCH_COMPRESSION_TYPE_NR; i++) { + struct disk_accounting_pos a; + disk_accounting_key_init(a, compression, .type = i); + struct bpos p = disk_accounting_pos_to_bpos(&a); + u64 v[3]; + bch2_accounting_mem_read(c, p, v, ARRAY_SIZE(v)); + + if (!v[0]) + continue; + + if (!compression_header) { + prt_printf(&buf, "compression type\tcompressed\runcompressed\rratio\r\n"); + printbuf_indent_add(&buf, 2); + } + compression_header = true; - for_each_member_device(c, ca) { - struct bch_dev_usage_full stats = bch2_dev_usage_full_read(ca); - bch2_dev_usage_to_text(&buf, ca, &stats); + u64 sectors_uncompressed = v[1]; + u64 sectors_compressed = v[2]; + + bch2_prt_compression_type(&buf, i); + prt_tab(&buf); + + prt_human_readable_u64(&buf, sectors_compressed << 9); + prt_tab_rjust(&buf); + + if (i == BCH_COMPRESSION_TYPE_incompressible) { + prt_newline(&buf); + continue; + } + + prt_human_readable_u64(&buf, sectors_uncompressed << 9); + prt_printf(&buf, "\r%llu%%\r\n", + div64_u64(sectors_compressed * 100, + sectors_uncompressed)); } + if (compression_header) + printbuf_indent_sub(&buf, 2); + + prt_printf(&buf, "image size"); + prt_sectors(&buf, bucket_to_sector(c->devs[0], nbuckets)); + printf("%s", buf.buf); printbuf_exit(&buf); } +static int finish_image(struct bch_fs *c, + bool keep_alloc, + unsigned verbosity) +{ + if (verbosity > 1) + printf("moving %stree to primary device\n", + keep_alloc ? "" : "non-alloc "); + + mutex_lock(&c->sb_lock); + struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, 0); + SET_BCH_MEMBER_DATA_ALLOWED(m, BCH_MEMBER_DATA_ALLOWED(m)|BIT(BCH_DATA_btree)); + bch2_write_super(c); + mutex_unlock(&c->sb_lock); + + bch2_dev_allocator_set_rw(c, c->devs[0], true); + + int ret = move_btree(c, keep_alloc, 0); + bch_err_msg(c, ret, "migrating btree from temporary device"); + if (ret) + return ret; + + bch2_fs_read_only(c); + + if (0) + check_gaps(c); + + u64 nbuckets; + ret = get_nbuckets_used(c, &nbuckets); + if (ret) + return ret; + + if (verbosity) + print_image_usage(c, keep_alloc, nbuckets); + + if (ftruncate(c->devs[0]->disk_sb.bdev->bd_fd, nbuckets * bucket_bytes(c->devs[0]))) { + fprintf(stderr, "truncate error: %m\n"); + return -errno; + } + + mutex_lock(&c->sb_lock); + if (!keep_alloc) { + if (verbosity > 1) + printf("Stripping alloc info\n"); + strip_fs_alloc(c); + } + + rcu_assign_pointer(c->devs[1], NULL); + + m = bch2_members_v2_get_mut(c->disk_sb.sb, 0); + SET_BCH_MEMBER_DATA_ALLOWED(m, BCH_MEMBER_DATA_ALLOWED(m)|BIT(BCH_DATA_journal)); + + bch2_members_v2_get_mut(c->disk_sb.sb, 0)->nbuckets = cpu_to_le64(nbuckets); + + for_each_online_member(c, ca, 0) { + struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); + SET_BCH_MEMBER_RESIZE_ON_MOUNT(m, true); + } + + c->disk_sb.sb->features[0] |= cpu_to_le64(BIT_ULL(BCH_FEATURE_small_image)); + + /* + * sb->nr_devices must be 1 so that it can be mounted without UUID + * conflicts + */ + unsigned u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) + + sizeof(struct bch_member), sizeof(u64)); + bch2_sb_field_resize(&c->disk_sb, members_v2, u64s); + c->disk_sb.sb->nr_devices = 1; + SET_BCH_SB_MULTI_DEVICE(c->disk_sb.sb, false); + + bch2_write_super(c); + mutex_unlock(&c->sb_lock); + return 0; +} + /* * Build an image file: * @@ -188,9 +396,9 @@ static void print_dev_usage_all(struct bch_fs *c) * metadata device is dropped. */ static void image_create(struct bch_opt_strs fs_opt_strs, - struct bch_opts fs_opts, + struct bch_opts fs_opts, struct format_opts format_opts, - struct dev_opts dev_opts, + struct dev_opts dev_opts, const char *src_path, bool keep_alloc, unsigned verbosity) @@ -246,11 +454,9 @@ static void image_create(struct bch_opt_strs fs_opt_strs, struct bch_opts opts = bch2_opts_empty(); opt_set(opts, copygc_enabled, false); opt_set(opts, rebalance_enabled, false); + opt_set(opts, nostart, true); struct bch_fs *c = bch2_fs_open(&device_paths, &opts); - - unlink(device_paths.data[1]); - int ret = PTR_ERR_OR_ZERO(c); if (ret) { fprintf(stderr, "error opening %s: %s\n", @@ -258,76 +464,20 @@ static void image_create(struct bch_opt_strs fs_opt_strs, goto err; } - struct copy_fs_state s = {}; - copy_fs(c, src_fd, src_path, &s, 0); - - printf("moving non-alloc btree to primary device\n"); - - mutex_lock(&c->sb_lock); - struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, 0); - SET_BCH_MEMBER_DATA_ALLOWED(m, BCH_MEMBER_DATA_ALLOWED(m)|BIT(BCH_DATA_btree)); - bch2_write_super(c); - mutex_unlock(&c->sb_lock); + c->loglevel = 5 + max_t(int, 0, verbosity - 1); - bch2_dev_allocator_set_rw(c, c->devs[0], true); - - ret = move_btree(c, keep_alloc, 0); - if (ret) { - fprintf(stderr, "error migrating btree from temporary device: %s\n", - bch2_err_str(ret)); - goto err; - } - - bch2_fs_read_only(c); - - if (verbosity > 1) - print_dev_usage_all(c); - - if (0) - check_gaps(c); + unlink(device_paths.data[1]); - u64 nbuckets; - ret = get_nbuckets_used(c, &nbuckets); + ret = bch2_fs_start(c); + bch_err_msg(c, ret, "starting fs"); if (ret) goto err; - if (ftruncate(c->devs[0]->disk_sb.bdev->bd_fd, nbuckets * bucket_bytes(c->devs[0]))) { - fprintf(stderr, "truncate error: %m\n"); + struct copy_fs_state s = { .verbosity = verbosity }; + ret = copy_fs(c, &s, src_fd, src_path) ?: + finish_image(c, keep_alloc, verbosity); + if (ret) goto err; - } - - mutex_lock(&c->sb_lock); - if (!keep_alloc) { - printf("Stripping alloc info\n"); - strip_fs_alloc(c); - } - - rcu_assign_pointer(c->devs[1], NULL); - - m = bch2_members_v2_get_mut(c->disk_sb.sb, 0); - SET_BCH_MEMBER_DATA_ALLOWED(m, BCH_MEMBER_DATA_ALLOWED(m)|BIT(BCH_DATA_journal)); - - bch2_members_v2_get_mut(c->disk_sb.sb, 0)->nbuckets = cpu_to_le64(nbuckets); - - for_each_online_member(c, ca, 0) { - struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); - SET_BCH_MEMBER_RESIZE_ON_MOUNT(m, true); - } - - c->disk_sb.sb->features[0] |= cpu_to_le64(BIT_ULL(BCH_FEATURE_small_image)); - - /* - * sb->nr_devices must be 1 so that it can be mounted without UUID - * conflicts - */ - unsigned u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) + - sizeof(struct bch_member), sizeof(u64)); - bch2_sb_field_resize(&c->disk_sb, members_v2, u64s); - c->disk_sb.sb->nr_devices = 1; - SET_BCH_SB_MULTI_DEVICE(c->disk_sb.sb, false); - - bch2_write_super(c); - mutex_unlock(&c->sb_lock); bch2_fs_stop(c); darray_exit(&device_paths); @@ -354,6 +504,7 @@ static void image_create_usage(void) " --superblock_size=size\n" " --bucket_size=size\n" " --fs_size=size Expected size of device image will be used on, hint for bucket size\n" + " --version=version Create filesystem with specified on disk format version instead of the latest\n" " -f, --force\n" " -q, --quiet Only print errors\n" " -v, --verbose Verbose filesystem initialization\n" @@ -371,6 +522,7 @@ static int cmd_image_create(int argc, char *argv[]) { "fs_label", required_argument, NULL, 'L' }, { "uuid", required_argument, NULL, 'U' }, { "superblock_size", required_argument, NULL, 'S' }, + { "version", required_argument, NULL, 'V' }, { "force", no_argument, NULL, 'f' }, { "quiet", no_argument, NULL, 'q' }, { "verbose", no_argument, NULL, 'v' }, @@ -446,6 +598,9 @@ static int cmd_image_create(int argc, char *argv[]) case 'v': verbosity++; break; + case 'V': + opts.version = version_parse(optarg); + break; case 'h': image_create_usage(); exit(EXIT_SUCCESS); @@ -459,16 +614,187 @@ static int cmd_image_create(int argc, char *argv[]) } args_shift(optind); - if (argc != 1) + if (argc != 1) { + image_create_usage(); die("Please supply a filename for the new image"); + } dev_opts.path = argv[0]; - image_create(fs_opt_strs, fs_opts, opts, dev_opts, opts.source, keep_alloc, verbosity); + image_create(fs_opt_strs, fs_opts, opts, dev_opts, opts.source, + keep_alloc, verbosity); bch2_opt_strs_free(&fs_opt_strs); return 0; } +static int image_update(const char *src_path, const char *dst_image, + bool keep_alloc, + unsigned verbosity) +{ + int src_fd = xopen(src_path, O_RDONLY); + + if (!S_ISDIR(xfstat(src_fd).st_mode)) + die("%s is not a directory", src_path); + + u64 input_bytes = count_input_size(src_fd); + + if (truncate(dst_image, xstat(dst_image).st_size + input_bytes * 2)) + die("truncate error: %m"); + + darray_const_str device_paths = {}; + darray_push(&device_paths, dst_image); + + struct bch_opts opts = bch2_opts_empty(); + opt_set(opts, copygc_enabled, false); + opt_set(opts, rebalance_enabled, false); + opt_set(opts, nostart, true); + + struct bch_fs *c = bch2_fs_open(&device_paths, &opts); + int ret = PTR_ERR_OR_ZERO(c); + if (ret) + die("error opening image: %s", bch2_err_str(ret)); + + c->loglevel = 5 + max_t(int, 0, verbosity - 1); + + struct dev_opts dev_opts = dev_opts_default(); + dev_opts.path = mprintf("%s.metadata", dst_image); + + if (1) { + /* factor this out into a helper */ + + ret = open_for_format(&dev_opts, BLK_OPEN_CREAT, false); + if (ret) { + fprintf(stderr, "error opening %s: %m", dev_opts.path); + goto err; + } + + u64 metadata_dev_size = max(input_bytes, + c->opts.btree_node_size * BCH_MIN_NR_NBUCKETS); + + if (ftruncate(dev_opts.bdev->bd_fd, metadata_dev_size)) { + fprintf(stderr, "ftruncate error: %m"); + goto err; + } + + ret = bch2_format_for_device_add(&dev_opts, + c->opts.block_size, + c->opts.btree_node_size); + bch_err_msg(c, ret, "opening %s", dev_opts.path); + if (ret) + goto err; + + CLASS(printbuf, err)(); + ret = bch2_dev_add(c, dev_opts.path, &err); + if (ret) { + bch_err(c, "error adding metadata device: %s\n%s", + bch2_err_str(ret), err.buf); + goto err; + } + } + + set_data_allowed_for_image_update(c); + + ret = bch2_fs_start(c); + bch_err_msg(c, ret, "starting fs"); + if (ret) + goto err; + + bch_verbose(c, "Moving btree to temp device"); + ret = move_btree(c, true, 1); + bch_err_msg(c, ret, "migrating btree to temp device"); + if (ret) + goto err_stop; + + /* xattrs will be recreated */ + bch_verbose(c, "Deleting xattrs"); + ret = bch2_btree_delete_range(c, BTREE_ID_xattrs, POS_MIN, SPOS_MAX, + BTREE_ITER_all_snapshots, NULL); + bch_err_msg(c, ret, "deleting xattrs"); + if (ret) + goto err_stop; + + bch_verbose(c, "Syncing data"); + struct copy_fs_state s = { .verbosity = verbosity }; + + ret = copy_fs(c, &s, src_fd, src_path) ?: + finish_image(c, keep_alloc, verbosity); + if (ret) + goto err; + + bch2_fs_stop(c); + unlink(dev_opts.path); + xclose(src_fd); + return 0; +err_stop: + bch2_fs_stop(c); +err: + unlink(dev_opts.path); + exit(EXIT_FAILURE); +} + +static void image_update_usage(void) +{ + puts("bcachefs image update - update an image file, minimizing changes\n" + "Usage: bcachefs image update [OPTION]... <file>\n" + "\n" + "Options:\n" + " --source=path Source directory to be used as content for the new image\n" + " -a, --keep-alloc Include allocation info in the filesystem\n" + " 6.16+ regenerates alloc info on first rw mount\n" + " -q, --quiet Only print errors\n" + " -v, --verbose Verbose filesystem initialization\n" + " -h, --help Display this help and exit\n" + "\n" + "Report bugs to <linux-bcachefs@vger.kernel.org>"); +} + +static int cmd_image_update(int argc, char *argv[]) +{ + static const struct option longopts[] = { + { "source", required_argument, NULL, 's' }, + { "keep-alloc", no_argument, NULL, 'a' }, + { "quiet", no_argument, NULL, 'q' }, + { "verbose", no_argument, NULL, 'v' }, + { "help", no_argument, NULL, 'h' }, + { NULL } + }; + const char *source = NULL; + bool keep_alloc = false; + unsigned verbosity = 1; + int opt; + + while ((opt = getopt_long(argc, argv, "s:aqvh", + longopts, NULL)) != -1) + switch (opt) { + case 's': + source = optarg; + break; + case 'a': + keep_alloc = true; + break; + case 'q': + verbosity = 0; + break; + case 'v': + verbosity++; + break; + case 'h': + image_update_usage(); + exit(EXIT_SUCCESS); + default: + die("getopt ret %i %c", opt, opt); + } + args_shift(optind); + + if (argc != 1) { + image_update_usage(); + die("Please supply a filename"); + } + + return image_update(source, argv[0], + keep_alloc, verbosity); +} + static int image_usage(void) { puts("bcachefs image - commands for creating and updating image files\n" @@ -476,6 +802,7 @@ static int image_usage(void) "\n" "Commands:\n" " create Create a minimally-sized disk image\n" + " update Update a disk image, minimizing changes\n" "\n" "Report bugs to <linux-bcachefs@vger.kernel.org>"); return 0; @@ -489,6 +816,8 @@ int image_cmds(int argc, char *argv[]) return image_usage(); if (!strcmp(cmd, "create")) return cmd_image_create(argc, argv); + if (!strcmp(cmd, "update")) + return cmd_image_update(argc, argv); image_usage(); return -EINVAL; |