#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/idr.h>
+#include <linux/blktrace_api.h>
static const char *_name = DM_NAME;
int error;
struct bio *bio;
atomic_t io_count;
+ unsigned long start_time;
};
/*
mempool_free(tio, md->tio_pool);
}
+static void start_io_acct(struct dm_io *io)
+{
+ struct mapped_device *md = io->md;
+
+ io->start_time = jiffies;
+
+ preempt_disable();
+ disk_round_stats(dm_disk(md));
+ preempt_enable();
+ dm_disk(md)->in_flight = atomic_inc_return(&md->pending);
+}
+
+static int end_io_acct(struct dm_io *io)
+{
+ struct mapped_device *md = io->md;
+ struct bio *bio = io->bio;
+ unsigned long duration = jiffies - io->start_time;
+ int pending;
+ int rw = bio_data_dir(bio);
+
+ preempt_disable();
+ disk_round_stats(dm_disk(md));
+ preempt_enable();
+ dm_disk(md)->in_flight = pending = atomic_dec_return(&md->pending);
+
+ disk_stat_add(dm_disk(md), ticks[rw], duration);
+
+ return !pending;
+}
+
/*
* Add the bio to the list of deferred io.
*/
io->error = error;
if (atomic_dec_and_test(&io->io_count)) {
- if (atomic_dec_and_test(&io->md->pending))
+ if (end_io_acct(io))
/* nudge anyone waiting on suspend queue */
wake_up(&io->md->wait);
+ blk_add_trace_bio(io->md->queue, io->bio, BLK_TA_COMPLETE);
+
bio_endio(io->bio, io->bio->bi_size, io->error);
free_io(io->md, io);
}
struct target_io *tio)
{
int r;
+ sector_t sector;
/*
* Sanity checks.
* this io.
*/
atomic_inc(&tio->io->io_count);
+ sector = clone->bi_sector;
r = ti->type->map(ti, clone, &tio->info);
- if (r > 0)
+ if (r > 0) {
/* the bio has been remapped so dispatch it */
+
+ blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
+ tio->io->bio->bi_bdev->bd_dev, sector,
+ clone->bi_sector);
+
generic_make_request(clone);
+ }
else if (r < 0) {
/* error the io and bail out */
} else {
/*
- * Create two copy bios to deal with io that has
- * been split across a target.
+ * Handle a bvec that must be split between two or more targets.
*/
struct bio_vec *bv = bio->bi_io_vec + ci->idx;
+ sector_t remaining = to_sector(bv->bv_len);
+ unsigned int offset = 0;
- clone = split_bvec(bio, ci->sector, ci->idx,
- bv->bv_offset, max);
- __map_bio(ti, clone, tio);
+ do {
+ if (offset) {
+ ti = dm_table_find_target(ci->map, ci->sector);
+ max = max_io_len(ci->md, ci->sector, ti);
- ci->sector += max;
- ci->sector_count -= max;
- ti = dm_table_find_target(ci->map, ci->sector);
-
- len = to_sector(bv->bv_len) - max;
- clone = split_bvec(bio, ci->sector, ci->idx,
- bv->bv_offset + to_bytes(max), len);
- tio = alloc_tio(ci->md);
- tio->io = ci->io;
- tio->ti = ti;
- memset(&tio->info, 0, sizeof(tio->info));
- __map_bio(ti, clone, tio);
+ tio = alloc_tio(ci->md);
+ tio->io = ci->io;
+ tio->ti = ti;
+ memset(&tio->info, 0, sizeof(tio->info));
+ }
+
+ len = min(remaining, max);
+
+ clone = split_bvec(bio, ci->sector, ci->idx,
+ bv->bv_offset + offset, len);
+
+ __map_bio(ti, clone, tio);
+
+ ci->sector += len;
+ ci->sector_count -= len;
+ offset += to_bytes(len);
+ } while (remaining -= len);
- ci->sector += len;
- ci->sector_count -= len;
ci->idx++;
}
}
ci.sector_count = bio_sectors(bio);
ci.idx = bio->bi_idx;
- atomic_inc(&md->pending);
+ start_io_acct(ci.io);
while (ci.sector_count)
__clone_and_map(&ci);
bad3:
mempool_destroy(md->io_pool);
bad2:
- blk_put_queue(md->queue);
+ blk_cleanup_queue(md->queue);
free_minor(minor);
bad1:
kfree(md);
static void free_dev(struct mapped_device *md)
{
- free_minor(md->disk->first_minor);
+ unsigned int minor = md->disk->first_minor;
+
+ if (md->suspended_bdev) {
+ thaw_bdev(md->suspended_bdev, NULL);
+ bdput(md->suspended_bdev);
+ }
mempool_destroy(md->tio_pool);
mempool_destroy(md->io_pool);
del_gendisk(md->disk);
+ free_minor(minor);
put_disk(md->disk);
- blk_put_queue(md->queue);
+ blk_cleanup_queue(md->queue);
kfree(md);
}