1 files changed, 87 insertions, 79 deletions

diff --git a/libbcachefs/sb-members.c b/libbcachefs/sb-members.c
index 6245e342..d26a0ca4 100644
--- a/libbcachefs/sb-members.c
+++ b/libbcachefs/sb-members.c
@@ -12,27 +12,36 @@
 
 int bch2_dev_missing_bkey(struct bch_fs *c, struct bkey_s_c k, unsigned dev)
 {
-	struct printbuf buf = PRINTBUF;
+	CLASS(printbuf, buf)();
 	bch2_log_msg_start(c, &buf);
 
-	prt_printf(&buf, "pointer to nonexistent device %u in key\n", dev);
+	bool removed = test_bit(dev, c->devs_removed.d);
+
+	prt_printf(&buf, "pointer to %s device %u in key\n",
+		   removed ? "removed" : "nonexistent", dev);
 	bch2_bkey_val_to_text(&buf, c, k);
+	prt_newline(&buf);
 
-	bool print = bch2_count_fsck_err(c, ptr_to_invalid_device, &buf);
+	bool print = removed
+		? bch2_count_fsck_err(c, ptr_to_removed_device, &buf)
+		: bch2_count_fsck_err(c, ptr_to_invalid_device, &buf);
 
 	int ret = bch2_run_explicit_recovery_pass(c, &buf,
 					BCH_RECOVERY_PASS_check_allocations, 0);
 
 	if (print)
 		bch2_print_str(c, KERN_ERR, buf.buf);
-	printbuf_exit(&buf);
 	return ret;
 }
 
 void bch2_dev_missing_atomic(struct bch_fs *c, unsigned dev)
 {
-	if (dev != BCH_SB_MEMBER_INVALID)
-		bch2_fs_inconsistent(c, "pointer to nonexistent device %u", dev);
+	if (dev != BCH_SB_MEMBER_INVALID) {
+		bch2_fs_inconsistent(c, "pointer to %s device %u",
+				     test_bit(dev, c->devs_removed.d)
+				     ? "removed" : "nonexistent", dev);
+		dump_stack();
+	}
 }
 
 void bch2_dev_bucket_missing(struct bch_dev *ca, u64 bucket)
@@ -61,34 +70,13 @@ struct bch_member *bch2_members_v2_get_mut(struct bch_sb *sb, int i)
 	return __bch2_members_v2_get_mut(bch2_sb_field_get(sb, members_v2), i);
 }
 
-static struct bch_member members_v2_get(struct bch_sb_field_members_v2 *mi, int i)
-{
-	struct bch_member ret, *p = __bch2_members_v2_get_mut(mi, i);
-	memset(&ret, 0, sizeof(ret));
-	memcpy(&ret, p, min_t(size_t, le16_to_cpu(mi->member_bytes), sizeof(ret)));
-	return ret;
-}
-
-static struct bch_member *members_v1_get_mut(struct bch_sb_field_members_v1 *mi, int i)
-{
-	return (void *) mi->_members + (i * BCH_MEMBER_V1_BYTES);
-}
-
-static struct bch_member members_v1_get(struct bch_sb_field_members_v1 *mi, int i)
-{
-	struct bch_member ret, *p = members_v1_get_mut(mi, i);
-	memset(&ret, 0, sizeof(ret));
-	memcpy(&ret, p, min_t(size_t, BCH_MEMBER_V1_BYTES, sizeof(ret)));
-	return ret;
-}
-
 struct bch_member bch2_sb_member_get(struct bch_sb *sb, int i)
 {
 	struct bch_sb_field_members_v2 *mi2 = bch2_sb_field_get(sb, members_v2);
 	if (mi2)
-		return members_v2_get(mi2, i);
+		return bch2_members_v2_get(mi2, i);
 	struct bch_sb_field_members_v1 *mi1 = bch2_sb_field_get(sb, members_v1);
-	return members_v1_get(mi1, i);
+	return bch2_members_v1_get(mi1, i);
 }
 
 static int sb_members_v2_resize_entries(struct bch_fs *c)
@@ -204,33 +192,25 @@ static int validate_member(struct printbuf *err,
 	return 0;
 }
 
-static void member_to_text(struct printbuf *out,
-			   struct bch_member m,
-			   struct bch_sb_field_disk_groups *gi,
-			   struct bch_sb *sb,
-			   int i)
+void bch2_member_to_text(struct printbuf *out,
+			 struct bch_member *m,
+			 struct bch_sb_field_disk_groups *gi,
+			 struct bch_sb *sb,
+			 unsigned idx)
 {
-	unsigned data_have = bch2_sb_dev_has_data(sb, i);
-	u64 bucket_size = le16_to_cpu(m.bucket_size);
-	u64 device_size = le64_to_cpu(m.nbuckets) * bucket_size;
-
-	if (!bch2_member_alive(&m))
-		return;
-
-	prt_printf(out, "Device:\t%u\n", i);
-
-	printbuf_indent_add(out, 2);
+	u64 bucket_size = le16_to_cpu(m->bucket_size);
+	u64 device_size = le64_to_cpu(m->nbuckets) * bucket_size;
 
 	prt_printf(out, "Label:\t");
-	if (BCH_MEMBER_GROUP(&m))
+	if (BCH_MEMBER_GROUP(m))
 		bch2_disk_path_to_text_sb(out, sb,
-				BCH_MEMBER_GROUP(&m) - 1);
+				BCH_MEMBER_GROUP(m) - 1);
 	else
 		prt_printf(out, "(none)");
 	prt_newline(out);
 
 	prt_printf(out, "UUID:\t");
-	pr_uuid(out, m.uuid.b);
+	pr_uuid(out, m->uuid.b);
 	prt_newline(out);
 
 	prt_printf(out, "Size:\t");
@@ -238,40 +218,41 @@ static void member_to_text(struct printbuf *out,
 	prt_newline(out);
 
 	for (unsigned i = 0; i < BCH_MEMBER_ERROR_NR; i++)
-		prt_printf(out, "%s errors:\t%llu\n", bch2_member_error_strs[i], le64_to_cpu(m.errors[i]));
+		prt_printf(out, "%s errors:\t%llu\n", bch2_member_error_strs[i], le64_to_cpu(m->errors[i]));
 
 	for (unsigned i = 0; i < BCH_IOPS_NR; i++)
-		prt_printf(out, "%s iops:\t%u\n", bch2_iops_measurements[i], le32_to_cpu(m.iops[i]));
+		prt_printf(out, "%s iops:\t%u\n", bch2_iops_measurements[i], le32_to_cpu(m->iops[i]));
 
 	prt_printf(out, "Bucket size:\t");
 	prt_units_u64(out, bucket_size << 9);
 	prt_newline(out);
 
-	prt_printf(out, "First bucket:\t%u\n", le16_to_cpu(m.first_bucket));
-	prt_printf(out, "Buckets:\t%llu\n", le64_to_cpu(m.nbuckets));
+	prt_printf(out, "First bucket:\t%u\n", le16_to_cpu(m->first_bucket));
+	prt_printf(out, "Buckets:\t%llu\n", le64_to_cpu(m->nbuckets));
 
 	prt_printf(out, "Last mount:\t");
-	if (m.last_mount)
-		bch2_prt_datetime(out, le64_to_cpu(m.last_mount));
+	if (m->last_mount)
+		bch2_prt_datetime(out, le64_to_cpu(m->last_mount));
 	else
 		prt_printf(out, "(never)");
 	prt_newline(out);
 
-	prt_printf(out, "Last superblock write:\t%llu\n", le64_to_cpu(m.seq));
+	prt_printf(out, "Last superblock write:\t%llu\n", le64_to_cpu(m->seq));
 
 	prt_printf(out, "State:\t%s\n",
-		   BCH_MEMBER_STATE(&m) < BCH_MEMBER_STATE_NR
-		   ? bch2_member_states[BCH_MEMBER_STATE(&m)]
+		   BCH_MEMBER_STATE(m) < BCH_MEMBER_STATE_NR
+		   ? bch2_member_states[BCH_MEMBER_STATE(m)]
 		   : "unknown");
 
 	prt_printf(out, "Data allowed:\t");
-	if (BCH_MEMBER_DATA_ALLOWED(&m))
-		prt_bitflags(out, __bch2_data_types, BCH_MEMBER_DATA_ALLOWED(&m));
+	if (BCH_MEMBER_DATA_ALLOWED(m))
+		prt_bitflags(out, __bch2_data_types, BCH_MEMBER_DATA_ALLOWED(m));
 	else
 		prt_printf(out, "(none)");
 	prt_newline(out);
 
 	prt_printf(out, "Has data:\t");
+	unsigned data_have = bch2_sb_dev_has_data(sb, idx);
 	if (data_have)
 		prt_bitflags(out, __bch2_data_types, data_have);
 	else
@@ -279,22 +260,36 @@ static void member_to_text(struct printbuf *out,
 	prt_newline(out);
 
 	prt_printf(out, "Btree allocated bitmap blocksize:\t");
-	if (m.btree_bitmap_shift < 64)
-		prt_units_u64(out, 1ULL << m.btree_bitmap_shift);
+	if (m->btree_bitmap_shift < 64)
+		prt_units_u64(out, 1ULL << m->btree_bitmap_shift);
 	else
-		prt_printf(out, "(invalid shift %u)", m.btree_bitmap_shift);
+		prt_printf(out, "(invalid shift %u)", m->btree_bitmap_shift);
 	prt_newline(out);
 
 	prt_printf(out, "Btree allocated bitmap:\t");
-	bch2_prt_u64_base2_nbits(out, le64_to_cpu(m.btree_allocated_bitmap), 64);
+	bch2_prt_u64_base2_nbits(out, le64_to_cpu(m->btree_allocated_bitmap), 64);
 	prt_newline(out);
 
-	prt_printf(out, "Durability:\t%llu\n", BCH_MEMBER_DURABILITY(&m) ? BCH_MEMBER_DURABILITY(&m) - 1 : 1);
+	prt_printf(out, "Durability:\t%llu\n", BCH_MEMBER_DURABILITY(m) ? BCH_MEMBER_DURABILITY(m) - 1 : 1);
+
+	prt_printf(out, "Discard:\t%llu\n", BCH_MEMBER_DISCARD(m));
+	prt_printf(out, "Freespace initialized:\t%llu\n", BCH_MEMBER_FREESPACE_INITIALIZED(m));
+	prt_printf(out, "Resize on mount:\t%llu\n", BCH_MEMBER_RESIZE_ON_MOUNT(m));
+}
 
-	prt_printf(out, "Discard:\t%llu\n", BCH_MEMBER_DISCARD(&m));
-	prt_printf(out, "Freespace initialized:\t%llu\n", BCH_MEMBER_FREESPACE_INITIALIZED(&m));
-	prt_printf(out, "Resize on mount:\t%llu\n", BCH_MEMBER_RESIZE_ON_MOUNT(&m));
+static void member_to_text(struct printbuf *out,
+			   struct bch_member m,
+			   struct bch_sb_field_disk_groups *gi,
+			   struct bch_sb *sb,
+			   unsigned idx)
+{
+	if (!bch2_member_alive(&m))
+		return;
 
+	prt_printf(out, "Device:\t%u\n", idx);
+
+	printbuf_indent_add(out, 2);
+	bch2_member_to_text(out, &m, gi, sb, idx);
 	printbuf_indent_sub(out, 2);
 }
 
@@ -310,7 +305,7 @@ static int bch2_sb_members_v1_validate(struct bch_sb *sb, struct bch_sb_field *f
 	}
 
 	for (i = 0; i < sb->nr_devices; i++) {
-		struct bch_member m = members_v1_get(mi, i);
+		struct bch_member m = bch2_members_v1_get(mi, i);
 
 		int ret = validate_member(err, m, sb, i);
 		if (ret)
@@ -336,7 +331,7 @@ static void bch2_sb_members_v1_to_text(struct printbuf *out, struct bch_sb *sb,
 		prt_printf(out, "nr_devices mismatch: have %i entries, should be %u", nr, sb->nr_devices);
 
 	for (unsigned i = 0; i < min(sb->nr_devices, nr); i++)
-		member_to_text(out, members_v1_get(mi, i), gi, sb, i);
+		member_to_text(out, bch2_members_v1_get(mi, i), gi, sb, i);
 }
 
 const struct bch_sb_field_ops bch_sb_field_ops_members_v1 = {
@@ -370,7 +365,7 @@ static void bch2_sb_members_v2_to_text(struct printbuf *out, struct bch_sb *sb,
 	 */
 
 	for (unsigned i = 0; i < min(sb->nr_devices, nr); i++)
-		member_to_text(out, members_v2_get(mi, i), gi, sb, i);
+		member_to_text(out, bch2_members_v2_get(mi, i), gi, sb, i);
 }
 
 static int bch2_sb_members_v2_validate(struct bch_sb *sb, struct bch_sb_field *f,
@@ -387,7 +382,7 @@ static int bch2_sb_members_v2_validate(struct bch_sb *sb, struct bch_sb_field *f
 	}
 
 	for (unsigned i = 0; i < sb->nr_devices; i++) {
-		int ret = validate_member(err, members_v2_get(mi, i), sb, i);
+		int ret = validate_member(err, bch2_members_v2_get(mi, i), sb, i);
 		if (ret)
 			return ret;
 	}
@@ -413,14 +408,29 @@ void bch2_sb_members_from_cpu(struct bch_fs *c)
 	}
 }
 
+void bch2_sb_members_to_cpu(struct bch_fs *c)
+{
+	for_each_member_device(c, ca) {
+		struct bch_member m = bch2_sb_member_get(c->disk_sb.sb, ca->dev_idx);
+		ca->mi = bch2_mi_to_cpu(&m);
+	}
+
+	struct bch_sb_field_members_v2 *mi2 = bch2_sb_field_get(c->disk_sb.sb, members_v2);
+	if (mi2)
+		for (unsigned i = 0; i < c->sb.nr_devices; i++) {
+			struct bch_member m = bch2_members_v2_get(mi2, i);
+			bool removed = uuid_equal(&m.uuid, &BCH_SB_MEMBER_DELETED_UUID);
+			mod_bit(i, c->devs_removed.d, removed);
+		}
+}
+
 void bch2_dev_io_errors_to_text(struct printbuf *out, struct bch_dev *ca)
 {
 	struct bch_fs *c = ca->fs;
 	struct bch_member m;
 
-	mutex_lock(&ca->fs->sb_lock);
-	m = bch2_sb_member_get(c->disk_sb.sb, ca->dev_idx);
-	mutex_unlock(&ca->fs->sb_lock);
+	scoped_guard(mutex, &ca->fs->sb_lock)
+		m = bch2_sb_member_get(c->disk_sb.sb, ca->dev_idx);
 
 	printbuf_tabstop_push(out, 12);
 
@@ -447,16 +457,15 @@ void bch2_dev_io_errors_to_text(struct printbuf *out, struct bch_dev *ca)
 void bch2_dev_errors_reset(struct bch_dev *ca)
 {
 	struct bch_fs *c = ca->fs;
-	struct bch_member *m;
 
-	mutex_lock(&c->sb_lock);
-	m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx);
+	guard(mutex)(&c->sb_lock);
+
+	struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx);
 	for (unsigned i = 0; i < ARRAY_SIZE(m->errors_at_reset); i++)
 		m->errors_at_reset[i] = cpu_to_le64(atomic64_read(&ca->errors[i]));
 	m->errors_reset_time = cpu_to_le64(ktime_get_real_seconds());
 
 	bch2_write_super(c);
-	mutex_unlock(&c->sb_lock);
 }
 
 /*
@@ -588,7 +597,7 @@ have_slot:
 
 void bch2_sb_members_clean_deleted(struct bch_fs *c)
 {
-	mutex_lock(&c->sb_lock);
+	guard(mutex)(&c->sb_lock);
 	bool write_sb = false;
 
 	for (unsigned i = 0; i < c->sb.nr_devices; i++) {
@@ -602,5 +611,4 @@ void bch2_sb_members_clean_deleted(struct bch_fs *c)
 
 	if (write_sb)
 		bch2_write_super(c);
-	mutex_unlock(&c->sb_lock);
 }