summaryrefslogtreecommitdiff
path: root/c_src/cmd_image.c
diff options
context:
space:
mode:
Diffstat (limited to 'c_src/cmd_image.c')
-rw-r--r--c_src/cmd_image.c499
1 files changed, 414 insertions, 85 deletions
diff --git a/c_src/cmd_image.c b/c_src/cmd_image.c
index 12478b1f..9da3ed34 100644
--- a/c_src/cmd_image.c
+++ b/c_src/cmd_image.c
@@ -26,8 +26,11 @@
#include "crypto.h"
#include "libbcachefs/alloc_background.h"
#include "libbcachefs/alloc_foreground.h"
+#include "libbcachefs/btree_update.h"
#include "libbcachefs/data_update.h"
+#include "libbcachefs/disk_accounting.h"
#include "libbcachefs/errcode.h"
+#include "libbcachefs/journal_reclaim.h"
#include "libbcachefs/move.h"
#include "libbcachefs/opts.h"
#include "libbcachefs/super-io.h"
@@ -64,6 +67,22 @@ static u64 count_input_size(int dirfd)
return bytes;
}
+static void set_data_allowed_for_image_update(struct bch_fs *c)
+{
+ mutex_lock(&c->sb_lock);
+ struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, 0);
+ SET_BCH_MEMBER_DATA_ALLOWED(m, BIT(BCH_DATA_user));
+
+ m = bch2_members_v2_get_mut(c->disk_sb.sb, 1);
+ SET_BCH_MEMBER_DATA_ALLOWED(m, BIT(BCH_DATA_journal)|BIT(BCH_DATA_btree));
+
+ bch2_write_super(c);
+ mutex_unlock(&c->sb_lock);
+
+ bch2_dev_allocator_set_rw(c, c->devs[0], true);
+ bch2_dev_allocator_set_rw(c, c->devs[1], true);
+}
+
struct move_btree_args {
bool move_alloc;
unsigned target;
@@ -76,7 +95,6 @@ static bool move_btree_pred(struct bch_fs *c, void *_arg,
{
struct move_btree_args *args = _arg;
- data_opts->target = dev_to_target(0);
data_opts->target = args->target;
if (k.k->type != KEY_TYPE_btree_ptr_v2)
@@ -91,6 +109,12 @@ static bool move_btree_pred(struct bch_fs *c, void *_arg,
static int move_btree(struct bch_fs *c, bool move_alloc, unsigned target_dev)
{
+ /*
+ * Flush the key cache first, otherwise key cache flushing later will do
+ * btree updates to the wrong device
+ */
+ bch2_journal_flush_all_pins(&c->journal);
+
struct move_btree_args args = {
.move_alloc = move_alloc,
.target = dev_to_target(target_dev),
@@ -141,39 +165,223 @@ static void check_gaps(struct bch_fs *c)
static int get_nbuckets_used(struct bch_fs *c, u64 *nbuckets)
{
- struct btree_trans *trans = bch2_trans_get(c);
- struct btree_iter iter;
- bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, POS(0, U64_MAX), 0);
+ CLASS(btree_trans, trans)(c);
+ CLASS(btree_iter, iter)(trans, BTREE_ID_alloc, POS(0, U64_MAX), 0);
struct bkey_s_c k;
- int ret = lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_prev(trans, &iter)));
+ int ret = lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_prev(&iter)));
if (!ret && k.k->type != KEY_TYPE_alloc_v4)
ret = -ENOENT;
if (ret) {
fprintf(stderr, "error looking up last alloc key: %s\n", bch2_err_str(ret));
- goto err;
+ return ret;
}
*nbuckets = (k.k->p.offset + 1);
-err:
- bch2_trans_iter_exit(trans, &iter);
- bch2_trans_put(trans);
- return ret;
+ return 0;
}
-static void print_dev_usage_all(struct bch_fs *c)
+static void prt_sectors(struct printbuf *out, u64 v)
+{
+ prt_tab(out);
+ prt_human_readable_u64(out, v << 9);
+ prt_tab_rjust(out);
+ prt_newline(out);
+}
+
+static void print_data_type_usage(struct printbuf *out,
+ struct bch_dev *ca,
+ struct bch_dev_usage_full u,
+ unsigned i)
+{
+ if (u.d[i].buckets) {
+ bch2_prt_data_type(out, i);
+ prt_sectors(out, bucket_to_sector(ca, u.d[i].buckets));
+ }
+
+ if (u.d[i].fragmented) {
+ bch2_prt_data_type(out, i);
+ prt_str(out, " fragmented");
+ prt_sectors(out, u.d[i].fragmented);
+ }
+}
+
+static void print_image_usage(struct bch_fs *c, bool keep_alloc, u64 nbuckets)
{
struct printbuf buf = PRINTBUF;
+ printbuf_tabstop_push(&buf, 24);
+ printbuf_tabstop_push(&buf, 16);
+ printbuf_tabstop_push(&buf, 16);
+ printbuf_tabstop_push(&buf, 8);
+
+ struct bch_dev *ca = c->devs[0];
+ struct bch_dev_usage_full dev_stats = bch2_dev_usage_full_read(ca);
+
+ print_data_type_usage(&buf, ca, dev_stats, BCH_DATA_sb);
+ print_data_type_usage(&buf, ca, dev_stats, BCH_DATA_journal);
+ print_data_type_usage(&buf, ca, dev_stats, BCH_DATA_btree);
+
+ printbuf_indent_add(&buf, 2);
+
+ for (unsigned i = 0; i < BTREE_ID_NR; i++) {
+ if (btree_id_is_alloc(i) && !keep_alloc)
+ continue;
+
+ struct disk_accounting_pos a;
+ disk_accounting_key_init(a, btree, i);
+
+ u64 v = 0;
+ bch2_accounting_mem_read(c, disk_accounting_pos_to_bpos(&a), &v, 1);
+
+ if (v) {
+ bch2_btree_id_to_text(&buf, i);
+ prt_sectors(&buf, v);
+ }
+ }
+
+ printbuf_indent_sub(&buf, 2);
+
+ struct disk_accounting_pos acc_replicas_key;
+ memset(&acc_replicas_key, 0, sizeof(acc_replicas_key));
+ acc_replicas_key.type = BCH_DISK_ACCOUNTING_replicas;
+ acc_replicas_key.replicas.data_type = BCH_DATA_user;
+ acc_replicas_key.replicas.nr_devs = 0;
+ acc_replicas_key.replicas.nr_required = 1;
+ acc_replicas_key.replicas.nr_required = 1;
+ replicas_entry_add_dev(&acc_replicas_key.replicas, 0);
+
+ u64 v = 0;
+ bch2_accounting_mem_read(c, disk_accounting_pos_to_bpos(&acc_replicas_key), &v, 1);
+ prt_printf(&buf, "user");
+ prt_sectors(&buf, v);
+
+ if (dev_stats.d[BCH_DATA_user].fragmented) {
+ prt_printf(&buf, "user fragmented");
+ prt_sectors(&buf, dev_stats.d[BCH_DATA_user].fragmented);
+ }
+
+ bool compression_header = false;
+ for (unsigned i = 1; i < BCH_COMPRESSION_TYPE_NR; i++) {
+ struct disk_accounting_pos a;
+ disk_accounting_key_init(a, compression, .type = i);
+ struct bpos p = disk_accounting_pos_to_bpos(&a);
+ u64 v[3];
+ bch2_accounting_mem_read(c, p, v, ARRAY_SIZE(v));
+
+ if (!v[0])
+ continue;
+
+ if (!compression_header) {
+ prt_printf(&buf, "compression type\tcompressed\runcompressed\rratio\r\n");
+ printbuf_indent_add(&buf, 2);
+ }
+ compression_header = true;
- for_each_member_device(c, ca) {
- struct bch_dev_usage_full stats = bch2_dev_usage_full_read(ca);
- bch2_dev_usage_to_text(&buf, ca, &stats);
+ u64 sectors_uncompressed = v[1];
+ u64 sectors_compressed = v[2];
+
+ bch2_prt_compression_type(&buf, i);
+ prt_tab(&buf);
+
+ prt_human_readable_u64(&buf, sectors_compressed << 9);
+ prt_tab_rjust(&buf);
+
+ if (i == BCH_COMPRESSION_TYPE_incompressible) {
+ prt_newline(&buf);
+ continue;
+ }
+
+ prt_human_readable_u64(&buf, sectors_uncompressed << 9);
+ prt_printf(&buf, "\r%llu%%\r\n",
+ div64_u64(sectors_compressed * 100,
+ sectors_uncompressed));
}
+ if (compression_header)
+ printbuf_indent_sub(&buf, 2);
+
+ prt_printf(&buf, "image size");
+ prt_sectors(&buf, bucket_to_sector(c->devs[0], nbuckets));
+
printf("%s", buf.buf);
printbuf_exit(&buf);
}
+static int finish_image(struct bch_fs *c,
+ bool keep_alloc,
+ unsigned verbosity)
+{
+ if (verbosity > 1)
+ printf("moving %stree to primary device\n",
+ keep_alloc ? "" : "non-alloc ");
+
+ mutex_lock(&c->sb_lock);
+ struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, 0);
+ SET_BCH_MEMBER_DATA_ALLOWED(m, BCH_MEMBER_DATA_ALLOWED(m)|BIT(BCH_DATA_btree));
+ bch2_write_super(c);
+ mutex_unlock(&c->sb_lock);
+
+ bch2_dev_allocator_set_rw(c, c->devs[0], true);
+
+ int ret = move_btree(c, keep_alloc, 0);
+ bch_err_msg(c, ret, "migrating btree from temporary device");
+ if (ret)
+ return ret;
+
+ bch2_fs_read_only(c);
+
+ if (0)
+ check_gaps(c);
+
+ u64 nbuckets;
+ ret = get_nbuckets_used(c, &nbuckets);
+ if (ret)
+ return ret;
+
+ if (verbosity)
+ print_image_usage(c, keep_alloc, nbuckets);
+
+ if (ftruncate(c->devs[0]->disk_sb.bdev->bd_fd, nbuckets * bucket_bytes(c->devs[0]))) {
+ fprintf(stderr, "truncate error: %m\n");
+ return -errno;
+ }
+
+ mutex_lock(&c->sb_lock);
+ if (!keep_alloc) {
+ if (verbosity > 1)
+ printf("Stripping alloc info\n");
+ strip_fs_alloc(c);
+ }
+
+ rcu_assign_pointer(c->devs[1], NULL);
+
+ m = bch2_members_v2_get_mut(c->disk_sb.sb, 0);
+ SET_BCH_MEMBER_DATA_ALLOWED(m, BCH_MEMBER_DATA_ALLOWED(m)|BIT(BCH_DATA_journal));
+
+ bch2_members_v2_get_mut(c->disk_sb.sb, 0)->nbuckets = cpu_to_le64(nbuckets);
+
+ for_each_online_member(c, ca, 0) {
+ struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx);
+ SET_BCH_MEMBER_RESIZE_ON_MOUNT(m, true);
+ }
+
+ c->disk_sb.sb->features[0] |= cpu_to_le64(BIT_ULL(BCH_FEATURE_small_image));
+
+ /*
+ * sb->nr_devices must be 1 so that it can be mounted without UUID
+ * conflicts
+ */
+ unsigned u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) +
+ sizeof(struct bch_member), sizeof(u64));
+ bch2_sb_field_resize(&c->disk_sb, members_v2, u64s);
+ c->disk_sb.sb->nr_devices = 1;
+ SET_BCH_SB_MULTI_DEVICE(c->disk_sb.sb, false);
+
+ bch2_write_super(c);
+ mutex_unlock(&c->sb_lock);
+ return 0;
+}
+
/*
* Build an image file:
*
@@ -188,9 +396,9 @@ static void print_dev_usage_all(struct bch_fs *c)
* metadata device is dropped.
*/
static void image_create(struct bch_opt_strs fs_opt_strs,
- struct bch_opts fs_opts,
+ struct bch_opts fs_opts,
struct format_opts format_opts,
- struct dev_opts dev_opts,
+ struct dev_opts dev_opts,
const char *src_path,
bool keep_alloc,
unsigned verbosity)
@@ -246,11 +454,9 @@ static void image_create(struct bch_opt_strs fs_opt_strs,
struct bch_opts opts = bch2_opts_empty();
opt_set(opts, copygc_enabled, false);
opt_set(opts, rebalance_enabled, false);
+ opt_set(opts, nostart, true);
struct bch_fs *c = bch2_fs_open(&device_paths, &opts);
-
- unlink(device_paths.data[1]);
-
int ret = PTR_ERR_OR_ZERO(c);
if (ret) {
fprintf(stderr, "error opening %s: %s\n",
@@ -258,76 +464,20 @@ static void image_create(struct bch_opt_strs fs_opt_strs,
goto err;
}
- struct copy_fs_state s = {};
- copy_fs(c, src_fd, src_path, &s, 0);
-
- printf("moving non-alloc btree to primary device\n");
-
- mutex_lock(&c->sb_lock);
- struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, 0);
- SET_BCH_MEMBER_DATA_ALLOWED(m, BCH_MEMBER_DATA_ALLOWED(m)|BIT(BCH_DATA_btree));
- bch2_write_super(c);
- mutex_unlock(&c->sb_lock);
+ c->loglevel = 5 + max_t(int, 0, verbosity - 1);
- bch2_dev_allocator_set_rw(c, c->devs[0], true);
-
- ret = move_btree(c, keep_alloc, 0);
- if (ret) {
- fprintf(stderr, "error migrating btree from temporary device: %s\n",
- bch2_err_str(ret));
- goto err;
- }
-
- bch2_fs_read_only(c);
-
- if (verbosity > 1)
- print_dev_usage_all(c);
-
- if (0)
- check_gaps(c);
+ unlink(device_paths.data[1]);
- u64 nbuckets;
- ret = get_nbuckets_used(c, &nbuckets);
+ ret = bch2_fs_start(c);
+ bch_err_msg(c, ret, "starting fs");
if (ret)
goto err;
- if (ftruncate(c->devs[0]->disk_sb.bdev->bd_fd, nbuckets * bucket_bytes(c->devs[0]))) {
- fprintf(stderr, "truncate error: %m\n");
+ struct copy_fs_state s = { .verbosity = verbosity };
+ ret = copy_fs(c, &s, src_fd, src_path) ?:
+ finish_image(c, keep_alloc, verbosity);
+ if (ret)
goto err;
- }
-
- mutex_lock(&c->sb_lock);
- if (!keep_alloc) {
- printf("Stripping alloc info\n");
- strip_fs_alloc(c);
- }
-
- rcu_assign_pointer(c->devs[1], NULL);
-
- m = bch2_members_v2_get_mut(c->disk_sb.sb, 0);
- SET_BCH_MEMBER_DATA_ALLOWED(m, BCH_MEMBER_DATA_ALLOWED(m)|BIT(BCH_DATA_journal));
-
- bch2_members_v2_get_mut(c->disk_sb.sb, 0)->nbuckets = cpu_to_le64(nbuckets);
-
- for_each_online_member(c, ca, 0) {
- struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx);
- SET_BCH_MEMBER_RESIZE_ON_MOUNT(m, true);
- }
-
- c->disk_sb.sb->features[0] |= cpu_to_le64(BIT_ULL(BCH_FEATURE_small_image));
-
- /*
- * sb->nr_devices must be 1 so that it can be mounted without UUID
- * conflicts
- */
- unsigned u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) +
- sizeof(struct bch_member), sizeof(u64));
- bch2_sb_field_resize(&c->disk_sb, members_v2, u64s);
- c->disk_sb.sb->nr_devices = 1;
- SET_BCH_SB_MULTI_DEVICE(c->disk_sb.sb, false);
-
- bch2_write_super(c);
- mutex_unlock(&c->sb_lock);
bch2_fs_stop(c);
darray_exit(&device_paths);
@@ -354,6 +504,7 @@ static void image_create_usage(void)
" --superblock_size=size\n"
" --bucket_size=size\n"
" --fs_size=size Expected size of device image will be used on, hint for bucket size\n"
+ " --version=version Create filesystem with specified on disk format version instead of the latest\n"
" -f, --force\n"
" -q, --quiet Only print errors\n"
" -v, --verbose Verbose filesystem initialization\n"
@@ -371,6 +522,7 @@ static int cmd_image_create(int argc, char *argv[])
{ "fs_label", required_argument, NULL, 'L' },
{ "uuid", required_argument, NULL, 'U' },
{ "superblock_size", required_argument, NULL, 'S' },
+ { "version", required_argument, NULL, 'V' },
{ "force", no_argument, NULL, 'f' },
{ "quiet", no_argument, NULL, 'q' },
{ "verbose", no_argument, NULL, 'v' },
@@ -446,6 +598,9 @@ static int cmd_image_create(int argc, char *argv[])
case 'v':
verbosity++;
break;
+ case 'V':
+ opts.version = version_parse(optarg);
+ break;
case 'h':
image_create_usage();
exit(EXIT_SUCCESS);
@@ -459,16 +614,187 @@ static int cmd_image_create(int argc, char *argv[])
}
args_shift(optind);
- if (argc != 1)
+ if (argc != 1) {
+ image_create_usage();
die("Please supply a filename for the new image");
+ }
dev_opts.path = argv[0];
- image_create(fs_opt_strs, fs_opts, opts, dev_opts, opts.source, keep_alloc, verbosity);
+ image_create(fs_opt_strs, fs_opts, opts, dev_opts, opts.source,
+ keep_alloc, verbosity);
bch2_opt_strs_free(&fs_opt_strs);
return 0;
}
+static int image_update(const char *src_path, const char *dst_image,
+ bool keep_alloc,
+ unsigned verbosity)
+{
+ int src_fd = xopen(src_path, O_RDONLY);
+
+ if (!S_ISDIR(xfstat(src_fd).st_mode))
+ die("%s is not a directory", src_path);
+
+ u64 input_bytes = count_input_size(src_fd);
+
+ if (truncate(dst_image, xstat(dst_image).st_size + input_bytes * 2))
+ die("truncate error: %m");
+
+ darray_const_str device_paths = {};
+ darray_push(&device_paths, dst_image);
+
+ struct bch_opts opts = bch2_opts_empty();
+ opt_set(opts, copygc_enabled, false);
+ opt_set(opts, rebalance_enabled, false);
+ opt_set(opts, nostart, true);
+
+ struct bch_fs *c = bch2_fs_open(&device_paths, &opts);
+ int ret = PTR_ERR_OR_ZERO(c);
+ if (ret)
+ die("error opening image: %s", bch2_err_str(ret));
+
+ c->loglevel = 5 + max_t(int, 0, verbosity - 1);
+
+ struct dev_opts dev_opts = dev_opts_default();
+ dev_opts.path = mprintf("%s.metadata", dst_image);
+
+ if (1) {
+ /* factor this out into a helper */
+
+ ret = open_for_format(&dev_opts, BLK_OPEN_CREAT, false);
+ if (ret) {
+ fprintf(stderr, "error opening %s: %m", dev_opts.path);
+ goto err;
+ }
+
+ u64 metadata_dev_size = max(input_bytes,
+ c->opts.btree_node_size * BCH_MIN_NR_NBUCKETS);
+
+ if (ftruncate(dev_opts.bdev->bd_fd, metadata_dev_size)) {
+ fprintf(stderr, "ftruncate error: %m");
+ goto err;
+ }
+
+ ret = bch2_format_for_device_add(&dev_opts,
+ c->opts.block_size,
+ c->opts.btree_node_size);
+ bch_err_msg(c, ret, "opening %s", dev_opts.path);
+ if (ret)
+ goto err;
+
+ CLASS(printbuf, err)();
+ ret = bch2_dev_add(c, dev_opts.path, &err);
+ if (ret) {
+ bch_err(c, "error adding metadata device: %s\n%s",
+ bch2_err_str(ret), err.buf);
+ goto err;
+ }
+ }
+
+ set_data_allowed_for_image_update(c);
+
+ ret = bch2_fs_start(c);
+ bch_err_msg(c, ret, "starting fs");
+ if (ret)
+ goto err;
+
+ bch_verbose(c, "Moving btree to temp device");
+ ret = move_btree(c, true, 1);
+ bch_err_msg(c, ret, "migrating btree to temp device");
+ if (ret)
+ goto err_stop;
+
+ /* xattrs will be recreated */
+ bch_verbose(c, "Deleting xattrs");
+ ret = bch2_btree_delete_range(c, BTREE_ID_xattrs, POS_MIN, SPOS_MAX,
+ BTREE_ITER_all_snapshots, NULL);
+ bch_err_msg(c, ret, "deleting xattrs");
+ if (ret)
+ goto err_stop;
+
+ bch_verbose(c, "Syncing data");
+ struct copy_fs_state s = { .verbosity = verbosity };
+
+ ret = copy_fs(c, &s, src_fd, src_path) ?:
+ finish_image(c, keep_alloc, verbosity);
+ if (ret)
+ goto err;
+
+ bch2_fs_stop(c);
+ unlink(dev_opts.path);
+ xclose(src_fd);
+ return 0;
+err_stop:
+ bch2_fs_stop(c);
+err:
+ unlink(dev_opts.path);
+ exit(EXIT_FAILURE);
+}
+
+static void image_update_usage(void)
+{
+ puts("bcachefs image update - update an image file, minimizing changes\n"
+ "Usage: bcachefs image update [OPTION]... <file>\n"
+ "\n"
+ "Options:\n"
+ " --source=path Source directory to be used as content for the new image\n"
+ " -a, --keep-alloc Include allocation info in the filesystem\n"
+ " 6.16+ regenerates alloc info on first rw mount\n"
+ " -q, --quiet Only print errors\n"
+ " -v, --verbose Verbose filesystem initialization\n"
+ " -h, --help Display this help and exit\n"
+ "\n"
+ "Report bugs to <linux-bcachefs@vger.kernel.org>");
+}
+
+static int cmd_image_update(int argc, char *argv[])
+{
+ static const struct option longopts[] = {
+ { "source", required_argument, NULL, 's' },
+ { "keep-alloc", no_argument, NULL, 'a' },
+ { "quiet", no_argument, NULL, 'q' },
+ { "verbose", no_argument, NULL, 'v' },
+ { "help", no_argument, NULL, 'h' },
+ { NULL }
+ };
+ const char *source = NULL;
+ bool keep_alloc = false;
+ unsigned verbosity = 1;
+ int opt;
+
+ while ((opt = getopt_long(argc, argv, "s:aqvh",
+ longopts, NULL)) != -1)
+ switch (opt) {
+ case 's':
+ source = optarg;
+ break;
+ case 'a':
+ keep_alloc = true;
+ break;
+ case 'q':
+ verbosity = 0;
+ break;
+ case 'v':
+ verbosity++;
+ break;
+ case 'h':
+ image_update_usage();
+ exit(EXIT_SUCCESS);
+ default:
+ die("getopt ret %i %c", opt, opt);
+ }
+ args_shift(optind);
+
+ if (argc != 1) {
+ image_update_usage();
+ die("Please supply a filename");
+ }
+
+ return image_update(source, argv[0],
+ keep_alloc, verbosity);
+}
+
static int image_usage(void)
{
puts("bcachefs image - commands for creating and updating image files\n"
@@ -476,6 +802,7 @@ static int image_usage(void)
"\n"
"Commands:\n"
" create Create a minimally-sized disk image\n"
+ " update Update a disk image, minimizing changes\n"
"\n"
"Report bugs to <linux-bcachefs@vger.kernel.org>");
return 0;
@@ -489,6 +816,8 @@ int image_cmds(int argc, char *argv[])
return image_usage();
if (!strcmp(cmd, "create"))
return cmd_image_create(argc, argv);
+ if (!strcmp(cmd, "update"))
+ return cmd_image_update(argc, argv);
image_usage();
return -EINVAL;