1 files changed, 154 insertions, 39 deletions

diff --git a/libbcachefs/fs-io.c b/libbcachefs/fs-io.c
index 8b41be87..298e3592 100644
--- a/libbcachefs/fs-io.c
+++ b/libbcachefs/fs-io.c
@@ -26,9 +26,67 @@
 #include <trace/events/bcachefs.h>
 #include <trace/events/writeback.h>
 
-struct bio_set *bch2_writepage_bioset;
-struct bio_set *bch2_dio_read_bioset;
-struct bio_set *bch2_dio_write_bioset;
+struct i_sectors_hook {
+	struct extent_insert_hook	hook;
+	s64				sectors;
+	struct bch_inode_info		*inode;
+};
+
+struct bchfs_write_op {
+	struct bch_inode_info		*inode;
+	s64				sectors_added;
+	bool				is_dio;
+	bool				unalloc;
+	u64				new_i_size;
+
+	/* must be last: */
+	struct bch_write_op		op;
+};
+
+static inline void bch2_fswrite_op_init(struct bchfs_write_op *op,
+					struct bch_inode_info *inode,
+					bool is_dio)
+{
+	op->inode		= inode;
+	op->sectors_added	= 0;
+	op->is_dio		= is_dio;
+	op->unalloc		= false;
+	op->new_i_size		= U64_MAX;
+}
+
+struct bch_writepage_io {
+	struct closure			cl;
+
+	/* must be last: */
+	struct bchfs_write_op		op;
+};
+
+struct dio_write {
+	struct closure			cl;
+	struct kiocb			*req;
+	struct bch_fs			*c;
+	long				written;
+	long				error;
+	loff_t				offset;
+
+	struct disk_reservation		res;
+
+	struct iovec			*iovec;
+	struct iovec			inline_vecs[UIO_FASTIOV];
+	struct iov_iter			iter;
+
+	struct task_struct		*task;
+
+	/* must be last: */
+	struct bchfs_write_op		iop;
+};
+
+struct dio_read {
+	struct closure			cl;
+	struct kiocb			*req;
+	long				ret;
+	struct bch_read_bio		rbio;
+};
 
 /* pagecache_block must be held */
 static int write_invalidate_inode_pages_range(struct address_space *mapping,
@@ -101,7 +159,7 @@ static inline void i_size_dirty_get(struct bch_inode_info *inode)
 
 /* i_sectors accounting: */
 
-static enum extent_insert_hook_ret
+static enum btree_insert_ret
 i_sectors_hook_fn(struct extent_insert_hook *hook,
 		  struct bpos committed_pos,
 		  struct bpos next_pos,
@@ -119,7 +177,7 @@ i_sectors_hook_fn(struct extent_insert_hook *hook,
 
 	h->sectors += sectors * sign;
 
-	return BTREE_HOOK_DO_INSERT;
+	return BTREE_INSERT_OK;
 }
 
 static int inode_set_i_sectors_dirty(struct bch_inode_info *inode,
@@ -208,7 +266,7 @@ struct bchfs_extent_trans_hook {
 	bool				need_inode_update;
 };
 
-static enum extent_insert_hook_ret
+static enum btree_insert_ret
 bchfs_extent_update_hook(struct extent_insert_hook *hook,
 			 struct bpos committed_pos,
 			 struct bpos next_pos,
@@ -224,6 +282,10 @@ bchfs_extent_update_hook(struct extent_insert_hook *hook,
 	u64 offset = min(next_pos.offset << 9, h->op->new_i_size);
 	bool do_pack = false;
 
+	if (h->op->unalloc &&
+	    !bch2_extent_is_fully_allocated(k))
+		return BTREE_INSERT_ENOSPC;
+
 	BUG_ON((next_pos.offset << 9) > round_up(offset, PAGE_SIZE));
 
 	/* XXX: inode->i_size locking */
@@ -232,7 +294,7 @@ bchfs_extent_update_hook(struct extent_insert_hook *hook,
 
 		if (!h->need_inode_update) {
 			h->need_inode_update = true;
-			return BTREE_HOOK_RESTART_TRANS;
+			return BTREE_INSERT_NEED_TRAVERSE;
 		}
 
 		h->inode_u.bi_size = offset;
@@ -247,7 +309,7 @@ bchfs_extent_update_hook(struct extent_insert_hook *hook,
 	if (sectors) {
 		if (!h->need_inode_update) {
 			h->need_inode_update = true;
-			return BTREE_HOOK_RESTART_TRANS;
+			return BTREE_INSERT_NEED_TRAVERSE;
 		}
 
 		h->inode_u.bi_sectors += sectors;
@@ -267,7 +329,7 @@ bchfs_extent_update_hook(struct extent_insert_hook *hook,
 	if (do_pack)
 		bch2_inode_pack(&h->inode_p, &h->inode_u);
 
-	return BTREE_HOOK_DO_INSERT;
+	return BTREE_INSERT_OK;
 }
 
 static int bchfs_write_index_update(struct bch_write_op *wop)
@@ -352,12 +414,16 @@ static int bchfs_write_index_update(struct bch_write_op *wop)
 					BTREE_INSERT_NOFAIL|BTREE_INSERT_ATOMIC,
 					BTREE_INSERT_ENTRY(&extent_iter, k));
 		}
+
+		BUG_ON(bkey_cmp(extent_iter.pos, bkey_start_pos(&k->k)));
+		BUG_ON(!ret != !k->k.size);
 err:
 		if (ret == -EINTR)
 			continue;
 		if (ret)
 			break;
 
+		BUG_ON(bkey_cmp(extent_iter.pos, k->k.p) < 0);
 		bch2_keylist_pop_front(keys);
 	} while (!bch2_keylist_empty(keys));
 
@@ -748,8 +814,7 @@ static void bchfs_read(struct bch_fs *c, struct btree_iter *iter,
 		if (bkey_extent_is_allocation(k.k))
 			bch2_add_page_sectors(bio, k);
 
-		if (!bkey_extent_is_allocation(k.k) ||
-		    bkey_extent_is_compressed(k))
+		if (!bch2_extent_is_fully_allocated(k))
 			bch2_mark_pages_unalloc(bio);
 
 		if (pick.ca) {
@@ -759,7 +824,8 @@ static void bchfs_read(struct bch_fs *c, struct btree_iter *iter,
 				trace_read_split(&rbio->bio);
 			}
 
-			bch2_read_extent(c, rbio, k, &pick, flags);
+			bch2_read_extent(c, rbio, bkey_s_c_to_extent(k),
+					 &pick, flags);
 		} else {
 			zero_fill_bio(bio);
 
@@ -963,22 +1029,20 @@ static void bch2_writepage_io_alloc(struct bch_fs *c,
 alloc_io:
 		w->io = container_of(bio_alloc_bioset(GFP_NOFS,
 						      BIO_MAX_PAGES,
-						      bch2_writepage_bioset),
+						      &c->writepage_bioset),
 				     struct bch_writepage_io, op.op.wbio.bio);
 
 		closure_init(&w->io->cl, NULL);
-		w->io->op.inode		= inode;
-		w->io->op.sectors_added	= 0;
-		w->io->op.is_dio	= false;
+		bch2_fswrite_op_init(&w->io->op, inode, false);
 		bch2_write_op_init(&w->io->op.op, c,
 				(struct disk_reservation) {
 					.nr_replicas = c->opts.data_replicas,
 				},
 				c->fastest_devs,
-				inode->ei_last_dirtied,
+				writepoint_hashed(inode->ei_last_dirtied),
 				POS(inum, 0),
 				&inode->ei_journal_seq,
-				BCH_WRITE_THROTTLE);
+				0);
 		w->io->op.op.index_update_fn = bchfs_write_index_update;
 	}
 
@@ -1409,7 +1473,7 @@ static int bch2_direct_IO_read(struct bch_fs *c, struct kiocb *req,
 
 	bio = bio_alloc_bioset(GFP_KERNEL,
 			       iov_iter_npages(iter, BIO_MAX_PAGES),
-			       bch2_dio_read_bioset);
+			       &c->dio_read_bioset);
 
 	bio->bi_end_io = bch2_direct_IO_read_endio;
 
@@ -1541,20 +1605,19 @@ static void bch2_do_direct_IO_write(struct dio_write *dio)
 		return;
 	}
 
-	dio->iop.inode		= inode;
 	dio->iop.sectors_added	= 0;
-	dio->iop.is_dio		= true;
-	dio->iop.new_i_size	= U64_MAX;
 	bch2_write_op_init(&dio->iop.op, dio->c, dio->res,
 			   dio->c->fastest_devs,
-			   (unsigned long) dio->task,
+			   writepoint_hashed((unsigned long) dio->task),
 			   POS(inode->v.i_ino, (dio->offset + dio->written) >> 9),
 			   &inode->ei_journal_seq,
-			   flags|BCH_WRITE_THROTTLE);
+			   flags);
 	dio->iop.op.index_update_fn = bchfs_write_index_update;
 
-	dio->res.sectors -= bio_sectors(bio);
-	dio->iop.op.res.sectors = bio_sectors(bio);
+	if (!dio->iop.unalloc) {
+		dio->res.sectors -= bio_sectors(bio);
+		dio->iop.op.res.sectors = bio_sectors(bio);
+	}
 
 	task_io_account_write(bio->bi_iter.bi_size);
 
@@ -1589,6 +1652,31 @@ static void bch2_dio_write_loop_async(struct closure *cl)
 	}
 }
 
+static int bch2_check_range_allocated(struct bch_fs *c, struct bpos pos,
+				      u64 size)
+{
+	struct btree_iter iter;
+	struct bpos end = pos;
+	struct bkey_s_c k;
+	int ret = 0;
+
+	end.offset += size;
+
+	for_each_btree_key(&iter, c, BTREE_ID_EXTENTS, pos,
+			     BTREE_ITER_WITH_HOLES, k) {
+		if (bkey_cmp(bkey_start_pos(k.k), end) >= 0)
+			break;
+
+		if (!bch2_extent_is_fully_allocated(k)) {
+			ret = -ENOSPC;
+			break;
+		}
+	}
+	bch2_btree_iter_unlock(&iter);
+
+	return ret;
+}
+
 static int bch2_direct_IO_write(struct bch_fs *c,
 				struct kiocb *req, struct file *file,
 				struct bch_inode_info *inode,
@@ -1610,17 +1698,18 @@ static int bch2_direct_IO_write(struct bch_fs *c,
 
 	bio = bio_alloc_bioset(GFP_KERNEL,
 			       iov_iter_npages(iter, BIO_MAX_PAGES),
-			       bch2_dio_write_bioset);
+			       &c->dio_write_bioset);
 	dio = container_of(bio, struct dio_write, iop.op.wbio.bio);
-	dio->req	= req;
-	dio->c		= c;
-	dio->written	= 0;
-	dio->error	= 0;
-	dio->offset	= offset;
-	dio->iovec	= NULL;
-	dio->iter	= *iter;
-	dio->task	= current;
 	closure_init(&dio->cl, NULL);
+	dio->req		= req;
+	dio->c			= c;
+	dio->written		= 0;
+	dio->error		= 0;
+	dio->offset		= offset;
+	dio->iovec		= NULL;
+	dio->iter		= *iter;
+	dio->task		= current;
+	bch2_fswrite_op_init(&dio->iop, inode, true);
 
 	if (offset + iter->count > inode->v.i_size)
 		sync = true;
@@ -1635,9 +1724,15 @@ static int bch2_direct_IO_write(struct bch_fs *c,
 	 */
 	ret = bch2_disk_reservation_get(c, &dio->res, iter->count >> 9, 0);
 	if (unlikely(ret)) {
-		closure_debug_destroy(&dio->cl);
-		bio_put(bio);
-		return ret;
+		if (bch2_check_range_allocated(c, POS(inode->v.i_ino,
+						      offset >> 9),
+					       iter->count >> 9)) {
+			closure_debug_destroy(&dio->cl);
+			bio_put(bio);
+			return ret;
+		}
+
+		dio->iop.unalloc = true;
 	}
 
 	inode_dio_begin(&inode->v);
@@ -2318,7 +2413,7 @@ static long bch2_fallocate(struct bch_inode_info *inode, int mode,
 		reservation.v.nr_replicas = bch2_extent_nr_dirty_ptrs(k);
 
 		if (reservation.v.nr_replicas < replicas ||
-		    bkey_extent_is_compressed(k)) {
+		    bch2_extent_is_compressed(k)) {
 			ret = bch2_disk_reservation_get(c, &disk_res,
 						       sectors, 0);
 			if (ret)
@@ -2564,4 +2659,24 @@ loff_t bch2_llseek(struct file *file, loff_t offset, int whence)
 	return -EINVAL;
 }
 
+void bch2_fs_fsio_exit(struct bch_fs *c)
+{
+	bioset_exit(&c->dio_write_bioset);
+	bioset_exit(&c->dio_read_bioset);
+	bioset_exit(&c->writepage_bioset);
+}
+
+int bch2_fs_fsio_init(struct bch_fs *c)
+{
+	if (bioset_init(&c->writepage_bioset,
+			4, offsetof(struct bch_writepage_io, op.op.wbio.bio)) ||
+	    bioset_init(&c->dio_read_bioset,
+			4, offsetof(struct dio_read, rbio.bio)) ||
+	    bioset_init(&c->dio_write_bioset,
+			4, offsetof(struct dio_write, iop.op.wbio.bio)))
+		return -ENOMEM;
+
+	return 0;
+}
+
 #endif /* NO_BCACHEFS_FS */