summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKent Overstreet <koverstreet@google.com>2013-05-09 15:36:07 -0700
committerKent Overstreet <koverstreet@google.com>2013-06-17 19:37:04 -0700
commit3625b85f263cdb4cc1329be0fa904e8b3fa998dd (patch)
tree2e2d864a503f0ee3dafe5a3a084bd307b9d4d7fe
parent32e633decc037104a0a07e74ccaf560a3cd615a3 (diff)
aio: Don't use ctx->tail unnecessarily
aio_complete() (arguably) needs to keep its own trusted copy of the tail pointer, but io_getevents() doesn't have to use it - it's already using the head pointer from the ring buffer. So convert it to use the tail from the ring buffer so it touches fewer cachelines and doesn't contend with the cacheline aio_complete() needs. Signed-off-by: Kent Overstreet <koverstreet@google.com> Cc: Zach Brown <zab@redhat.com> Cc: Felipe Balbi <balbi@ti.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Mark Fasheh <mfasheh@suse.com> Cc: Joel Becker <jlbec@evilplan.org> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Jens Axboe <axboe@kernel.dk> Cc: Asai Thambi S P <asamymuthupa@micron.com> Cc: Selvan Mani <smani@micron.com> Cc: Sam Bradshaw <sbradshaw@micron.com> Cc: Jeff Moyer <jmoyer@redhat.com> Cc: Al Viro <viro@zeniv.linux.org.uk> Cc: Benjamin LaHaise <bcrl@kvack.org>
-rw-r--r--fs/aio.c42
1 files changed, 23 insertions, 19 deletions
diff --git a/fs/aio.c b/fs/aio.c
index 0d5033e20445..d7160647ba2f 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -305,7 +305,8 @@ static void free_ioctx(struct work_struct *work)
struct kioctx *ctx = container_of(work, struct kioctx, free_work);
struct aio_ring *ring;
struct kiocb *req;
- unsigned cpu, head, avail;
+ unsigned cpu, avail;
+ DEFINE_WAIT(wait);
spin_lock_irq(&ctx->ctx_lock);
@@ -326,22 +327,24 @@ static void free_ioctx(struct work_struct *work)
kcpu->reqs_available = 0;
}
- ring = kmap_atomic(ctx->ring_pages[0]);
- head = ring->head;
- kunmap_atomic(ring);
+ while (1) {
+ prepare_to_wait(&ctx->wait, &wait, TASK_UNINTERRUPTIBLE);
- while (atomic_read(&ctx->reqs_available) < ctx->nr_events - 1) {
- wait_event(ctx->wait,
- (head != ctx->tail) ||
- (atomic_read(&ctx->reqs_available) >=
- ctx->nr_events - 1));
-
- avail = (head <= ctx->tail ? ctx->tail : ctx->nr_events) - head;
+ ring = kmap_atomic(ctx->ring_pages[0]);
+ avail = (ring->head <= ring->tail)
+ ? ring->tail - ring->head
+ : ctx->nr_events - ring->head + ring->tail;
atomic_add(avail, &ctx->reqs_available);
- head += avail;
- head %= ctx->nr_events;
+ ring->head = ring->tail;
+ kunmap_atomic(ring);
+
+ if (atomic_read(&ctx->reqs_available) >= ctx->nr_events - 1)
+ break;
+
+ schedule();
}
+ finish_wait(&ctx->wait, &wait);
WARN_ON(atomic_read(&ctx->reqs_available) > ctx->nr_events - 1);
@@ -766,7 +769,7 @@ static long aio_read_events_ring(struct kioctx *ctx,
struct io_event __user *event, long nr)
{
struct aio_ring *ring;
- unsigned head, pos;
+ unsigned head, tail, pos;
long ret = 0;
int copy_ret;
@@ -774,11 +777,12 @@ static long aio_read_events_ring(struct kioctx *ctx,
ring = kmap_atomic(ctx->ring_pages[0]);
head = ring->head;
+ tail = ring->tail;
kunmap_atomic(ring);
- pr_debug("h%u t%u m%u\n", head, ctx->tail, ctx->nr_events);
+ pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events);
- if (head == ctx->tail)
+ if (head == tail)
goto out;
while (ret < nr) {
@@ -786,8 +790,8 @@ static long aio_read_events_ring(struct kioctx *ctx,
struct io_event *ev;
struct page *page;
- avail = (head <= ctx->tail ? ctx->tail : ctx->nr_events) - head;
- if (head == ctx->tail)
+ avail = (head <= tail ? tail : ctx->nr_events) - head;
+ if (head == tail)
break;
avail = min(avail, nr - ret);
@@ -818,7 +822,7 @@ static long aio_read_events_ring(struct kioctx *ctx,
kunmap_atomic(ring);
flush_dcache_page(ctx->ring_pages[0]);
- pr_debug("%li h%u t%u\n", ret, head, ctx->tail);
+ pr_debug("%li h%u t%u\n", ret, head, tail);
put_reqs_available(ctx, ret);
out: