[PATCH] dio: centralize completion in dio_complete()

[powerpc.git] / fs / direct-io.c

diff --git a/fs/direct-io.c b/fs/direct-io.c
index 5981e17..b57b671 100644 (file)
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -27,6 +27,7 @@
 #include <linux/slab.h>
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
+#include <linux/task_io_accounting_ops.h>
 #include <linux/bio.h>
 #include <linux/wait.h>
 #include <linux/err.h>
@@ -209,19 +210,46 @@ static struct page *dio_get_page(struct dio *dio)
        return dio->pages[dio->head++];
 }
 
-/*
- * Called when all DIO BIO I/O has been completed - let the filesystem
- * know, if it registered an interest earlier via get_block.  Pass the
- * private field of the map buffer_head so that filesystems can use it
- * to hold additional state between get_block calls and dio_complete.
+/**
+ * dio_complete() - called when all DIO BIO I/O has been completed
+ * @offset: the byte offset in the file of the completed operation
+ *
+ * This releases locks as dictated by the locking type, lets interested parties
+ * know that a DIO operation has completed, and calculates the resulting return
+ * code for the operation.
+ *
+ * It lets the filesystem know if it registered an interest earlier via
+ * get_block.  Pass the private field of the map buffer_head so that
+ * filesystems can use it to hold additional state between get_block calls and
+ * dio_complete.
  */
-static void dio_complete(struct dio *dio, loff_t offset, ssize_t bytes)
+static int dio_complete(struct dio *dio, loff_t offset, int ret)
 {
+       ssize_t transferred = 0;
+
+       if (dio->result) {
+               transferred = dio->result;
+
+               /* Check for short read case */
+               if ((dio->rw == READ) && ((offset + transferred) > dio->i_size))
+                       transferred = dio->i_size - offset;
+       }
+
        if (dio->end_io && dio->result)
-               dio->end_io(dio->iocb, offset, bytes, dio->map_bh.b_private);
+               dio->end_io(dio->iocb, offset, transferred,
+                           dio->map_bh.b_private);
        if (dio->lock_type == DIO_LOCKING)
                /* lockdep: non-owner release */
                up_read_non_owner(&dio->inode->i_alloc_sem);
+
+       if (ret == 0)
+               ret = dio->page_errors;
+       if (ret == 0)
+               ret = dio->io_error;
+       if (ret == 0)
+               ret = transferred;
+
+       return ret;
 }
 
 /*
@@ -235,8 +263,7 @@ static void finished_one_bio(struct dio *dio)
        spin_lock_irqsave(&dio->bio_lock, flags);
        if (dio->bio_count == 1) {
                if (dio->is_async) {
-                       ssize_t transferred;
-                       loff_t offset;
+                       int ret;
 
                        /*
                         * Last reference to the dio is going away.
@@ -244,24 +271,12 @@ static void finished_one_bio(struct dio *dio)
                         */
                        spin_unlock_irqrestore(&dio->bio_lock, flags);
 
-                       /* Check for short read case */
-                       transferred = dio->result;
-                       offset = dio->iocb->ki_pos;
-
-                       if ((dio->rw == READ) &&
-                           ((offset + transferred) > dio->i_size))
-                               transferred = dio->i_size - offset;
-
-                       /* check for error in completion path */
-                       if (dio->io_error)
-                               transferred = dio->io_error;
-
-                       dio_complete(dio, offset, transferred);
+                       ret = dio_complete(dio, dio->iocb->ki_pos, 0);
 
                        /* Complete AIO later if falling back to buffered i/o */
                        if (dio->result == dio->size ||
                                ((dio->rw == READ) && dio->result)) {
-                               aio_complete(dio->iocb, transferred, 0);
+                               aio_complete(dio->iocb, ret, 0);
                                kfree(dio);
                                return;
                        } else {
@@ -433,10 +448,8 @@ static int dio_bio_complete(struct dio *dio, struct bio *bio)
 /*
  * Wait on and process all in-flight BIOs.
  */
-static int dio_await_completion(struct dio *dio)
+static void dio_await_completion(struct dio *dio)
 {
-       int ret = 0;
-
        if (dio->bio)
                dio_bio_submit(dio);
 
@@ -447,13 +460,9 @@ static int dio_await_completion(struct dio *dio)
         */
        while (dio->bio_count) {
                struct bio *bio = dio_await_one(dio);
-               int ret2;
-
-               ret2 = dio_bio_complete(dio, bio);
-               if (ret == 0)
-                       ret = ret2;
+               /* io errors are propogated through dio->io_error */
+               dio_bio_complete(dio, bio);
        }
-       return ret;
 }
 
 /*
@@ -675,6 +684,13 @@ submit_page_section(struct dio *dio, struct page *page,
 {
        int ret = 0;
 
+       if (dio->rw & WRITE) {
+               /*
+                * Read accounting is performed in submit_bio()
+                */
+               task_io_account_write(len);
+       }
+
        /*
         * Can we just grow the current page's presence in the dio?
         */
@@ -1119,28 +1135,10 @@ direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode,
                        kfree(dio);
                }
        } else {
-               ssize_t transferred = 0;
-
                finished_one_bio(dio);
-               ret2 = dio_await_completion(dio);
-               if (ret == 0)
-                       ret = ret2;
-               if (ret == 0)
-                       ret = dio->page_errors;
-               if (dio->result) {
-                       loff_t i_size = i_size_read(inode);
+               dio_await_completion(dio);
 
-                       transferred = dio->result;
-                       /*
-                        * Adjust the return value if the read crossed a
-                        * non-block-aligned EOF.
-                        */
-                       if (rw == READ && (offset + transferred > i_size))
-                               transferred = i_size - offset;
-               }
-               dio_complete(dio, offset, transferred);
-               if (ret == 0)
-                       ret = transferred;
+               ret = dio_complete(dio, offset, ret);
 
                /* We could have also come here on an AIO file extend */
                if (!is_sync_kiocb(iocb) && (rw & WRITE) &&