/*
* initialize elevator private data (as_data).
*/
-static void *as_init_queue(request_queue_t *q, elevator_t *e)
+static void *as_init_queue(request_queue_t *q)
{
struct as_data *ad;
#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/debugfs.h>
+#include <linux/time.h>
#include <asm/uaccess.h>
static DEFINE_PER_CPU(unsigned long long, blk_trace_cpu_offset) = { 0, };
static unsigned int blktrace_seq __read_mostly = 1;
+/*
+ * Send out a notify message.
+ */
+static inline unsigned int trace_note(struct blk_trace *bt,
+ pid_t pid, int action,
+ const void *data, size_t len)
+{
+ struct blk_io_trace *t;
+ int cpu = smp_processor_id();
+
+ t = relay_reserve(bt->rchan, sizeof(*t) + len);
+ if (t == NULL)
+ return 0;
+
+ t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
+ t->time = sched_clock() - per_cpu(blk_trace_cpu_offset, cpu);
+ t->device = bt->dev;
+ t->action = action;
+ t->pid = pid;
+ t->cpu = cpu;
+ t->pdu_len = len;
+ memcpy((void *) t + sizeof(*t), data, len);
+ return blktrace_seq;
+}
+
/*
* Send out a notify for this process, if we haven't done so since a trace
* started
*/
static void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)
{
- struct blk_io_trace *t;
+ tsk->btrace_seq = trace_note(bt, tsk->pid,
+ BLK_TN_PROCESS,
+ tsk->comm, sizeof(tsk->comm));
+}
- t = relay_reserve(bt->rchan, sizeof(*t) + sizeof(tsk->comm));
- if (t) {
- t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
- t->device = bt->dev;
- t->action = BLK_TC_ACT(BLK_TC_NOTIFY);
- t->pid = tsk->pid;
- t->cpu = smp_processor_id();
- t->pdu_len = sizeof(tsk->comm);
- memcpy((void *) t + sizeof(*t), tsk->comm, t->pdu_len);
- tsk->btrace_seq = blktrace_seq;
- }
+static void trace_note_time(struct blk_trace *bt)
+{
+ struct timespec now;
+ unsigned long flags;
+ u32 words[2];
+
+ getnstimeofday(&now);
+ words[0] = now.tv_sec;
+ words[1] = now.tv_nsec;
+
+ local_irq_save(flags);
+ trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));
+ local_irq_restore(flags);
}
static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
blktrace_seq++;
smp_mb();
bt->trace_state = Blktrace_running;
+
+ trace_note_time(bt);
ret = 0;
}
} else {
}
static void
-cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_io_context *cic,
- struct request *rq)
+cfq_update_io_seektime(struct cfq_io_context *cic, struct request *rq)
{
sector_t sdist;
u64 total;
}
cfq_update_io_thinktime(cfqd, cic);
- cfq_update_io_seektime(cfqd, cic, rq);
+ cfq_update_io_seektime(cic, rq);
cfq_update_idle_window(cfqd, cfqq, cic);
cic->last_queue = jiffies;
/*
* queue lock held here
*/
-static void cfq_put_request(request_queue_t *q, struct request *rq)
+static void cfq_put_request(struct request *rq)
{
struct cfq_queue *cfqq = RQ_CFQQ(rq);
kfree(cfqd);
}
-static void *cfq_init_queue(request_queue_t *q, elevator_t *e)
+static void *cfq_init_queue(request_queue_t *q)
{
struct cfq_data *cfqd;
int i;
/*
* initialize elevator private data (deadline_data).
*/
-static void *deadline_init_queue(request_queue_t *q, elevator_t *e)
+static void *deadline_init_queue(request_queue_t *q)
{
struct deadline_data *dd;
static void *elevator_init_queue(request_queue_t *q, struct elevator_queue *eq)
{
- return eq->ops->elevator_init_fn(q, eq);
+ return eq->ops->elevator_init_fn(q);
}
static void elevator_attach(request_queue_t *q, struct elevator_queue *eq,
elevator_t *e = q->elevator;
if (e->ops->elevator_put_req_fn)
- e->ops->elevator_put_req_fn(q, rq);
+ e->ops->elevator_put_req_fn(rq);
}
int elv_may_queue(request_queue_t *q, int rw)
EXPORT_SYMBOL(blk_insert_request);
+static int __blk_rq_unmap_user(struct bio *bio)
+{
+ int ret = 0;
+
+ if (bio) {
+ if (bio_flagged(bio, BIO_USER_MAPPED))
+ bio_unmap_user(bio);
+ else
+ ret = bio_uncopy_user(bio);
+ }
+
+ return ret;
+}
+
+static int __blk_rq_map_user(request_queue_t *q, struct request *rq,
+ void __user *ubuf, unsigned int len)
+{
+ unsigned long uaddr;
+ struct bio *bio, *orig_bio;
+ int reading, ret;
+
+ reading = rq_data_dir(rq) == READ;
+
+ /*
+ * if alignment requirement is satisfied, map in user pages for
+ * direct dma. else, set up kernel bounce buffers
+ */
+ uaddr = (unsigned long) ubuf;
+ if (!(uaddr & queue_dma_alignment(q)) && !(len & queue_dma_alignment(q)))
+ bio = bio_map_user(q, NULL, uaddr, len, reading);
+ else
+ bio = bio_copy_user(q, uaddr, len, reading);
+
+ if (IS_ERR(bio)) {
+ return PTR_ERR(bio);
+ }
+
+ orig_bio = bio;
+ blk_queue_bounce(q, &bio);
+ /*
+ * We link the bounce buffer in and could have to traverse it
+ * later so we have to get a ref to prevent it from being freed
+ */
+ bio_get(bio);
+
+ /*
+ * for most (all? don't know of any) queues we could
+ * skip grabbing the queue lock here. only drivers with
+ * funky private ->back_merge_fn() function could be
+ * problematic.
+ */
+ spin_lock_irq(q->queue_lock);
+ if (!rq->bio)
+ blk_rq_bio_prep(q, rq, bio);
+ else if (!q->back_merge_fn(q, rq, bio)) {
+ ret = -EINVAL;
+ spin_unlock_irq(q->queue_lock);
+ goto unmap_bio;
+ } else {
+ rq->biotail->bi_next = bio;
+ rq->biotail = bio;
+
+ rq->nr_sectors += bio_sectors(bio);
+ rq->hard_nr_sectors = rq->nr_sectors;
+ rq->data_len += bio->bi_size;
+ }
+ spin_unlock_irq(q->queue_lock);
+
+ return bio->bi_size;
+
+unmap_bio:
+ /* if it was boucned we must call the end io function */
+ bio_endio(bio, bio->bi_size, 0);
+ __blk_rq_unmap_user(orig_bio);
+ bio_put(bio);
+ return ret;
+}
+
/**
* blk_rq_map_user - map user data to a request, for REQ_BLOCK_PC usage
* @q: request queue where request should be inserted
* unmapping.
*/
int blk_rq_map_user(request_queue_t *q, struct request *rq, void __user *ubuf,
- unsigned int len)
+ unsigned long len)
{
- unsigned long uaddr;
- struct bio *bio;
- int reading;
+ unsigned long bytes_read = 0;
+ int ret;
if (len > (q->max_hw_sectors << 9))
return -EINVAL;
if (!len || !ubuf)
return -EINVAL;
- reading = rq_data_dir(rq) == READ;
+ while (bytes_read != len) {
+ unsigned long map_len, end, start;
- /*
- * if alignment requirement is satisfied, map in user pages for
- * direct dma. else, set up kernel bounce buffers
- */
- uaddr = (unsigned long) ubuf;
- if (!(uaddr & queue_dma_alignment(q)) && !(len & queue_dma_alignment(q)))
- bio = bio_map_user(q, NULL, uaddr, len, reading);
- else
- bio = bio_copy_user(q, uaddr, len, reading);
+ map_len = min_t(unsigned long, len - bytes_read, BIO_MAX_SIZE);
+ end = ((unsigned long)ubuf + map_len + PAGE_SIZE - 1)
+ >> PAGE_SHIFT;
+ start = (unsigned long)ubuf >> PAGE_SHIFT;
- if (!IS_ERR(bio)) {
- rq->bio = rq->biotail = bio;
- blk_rq_bio_prep(q, rq, bio);
+ /*
+ * A bad offset could cause us to require BIO_MAX_PAGES + 1
+ * pages. If this happens we just lower the requested
+ * mapping len by a page so that we can fit
+ */
+ if (end - start > BIO_MAX_PAGES)
+ map_len -= PAGE_SIZE;
- rq->buffer = rq->data = NULL;
- rq->data_len = len;
- return 0;
+ ret = __blk_rq_map_user(q, rq, ubuf, map_len);
+ if (ret < 0)
+ goto unmap_rq;
+ bytes_read += ret;
+ ubuf += ret;
}
- /*
- * bio is the err-ptr
- */
- return PTR_ERR(bio);
+ rq->buffer = rq->data = NULL;
+ return 0;
+unmap_rq:
+ blk_rq_unmap_user(rq);
+ return ret;
}
EXPORT_SYMBOL(blk_rq_map_user);
* unmapping.
*/
int blk_rq_map_user_iov(request_queue_t *q, struct request *rq,
- struct sg_iovec *iov, int iov_count)
+ struct sg_iovec *iov, int iov_count, unsigned int len)
{
struct bio *bio;
if (IS_ERR(bio))
return PTR_ERR(bio);
- rq->bio = rq->biotail = bio;
+ if (bio->bi_size != len) {
+ bio_endio(bio, bio->bi_size, 0);
+ bio_unmap_user(bio);
+ return -EINVAL;
+ }
+
+ bio_get(bio);
blk_rq_bio_prep(q, rq, bio);
rq->buffer = rq->data = NULL;
- rq->data_len = bio->bi_size;
return 0;
}
/**
* blk_rq_unmap_user - unmap a request with user data
- * @bio: bio to be unmapped
- * @ulen: length of user buffer
+ * @rq: rq to be unmapped
*
* Description:
- * Unmap a bio previously mapped by blk_rq_map_user().
+ * Unmap a rq previously mapped by blk_rq_map_user().
+ * rq->bio must be set to the original head of the request.
*/
-int blk_rq_unmap_user(struct bio *bio, unsigned int ulen)
+int blk_rq_unmap_user(struct request *rq)
{
- int ret = 0;
+ struct bio *bio, *mapped_bio;
- if (bio) {
- if (bio_flagged(bio, BIO_USER_MAPPED))
- bio_unmap_user(bio);
+ while ((bio = rq->bio)) {
+ if (bio_flagged(bio, BIO_BOUNCED))
+ mapped_bio = bio->bi_private;
else
- ret = bio_uncopy_user(bio);
- }
+ mapped_bio = bio;
+ __blk_rq_unmap_user(mapped_bio);
+ rq->bio = bio->bi_next;
+ bio_put(bio);
+ }
return 0;
}
if (rq_data_dir(rq) == WRITE)
bio->bi_rw |= (1 << BIO_RW);
- rq->bio = rq->biotail = bio;
blk_rq_bio_prep(q, rq, bio);
-
rq->buffer = rq->data = NULL;
- rq->data_len = len;
return 0;
}
rq->hard_cur_sectors = rq->current_nr_sectors;
rq->hard_nr_sectors = rq->nr_sectors = bio_sectors(bio);
rq->buffer = bio_data(bio);
+ rq->data_len = bio->bi_size;
rq->bio = rq->biotail = bio;
}
return list_entry(rq->queuelist.next, struct request, queuelist);
}
-static void *noop_init_queue(request_queue_t *q, elevator_t *e)
+static void *noop_init_queue(request_queue_t *q)
{
struct noop_data *nd;
unsigned long start_time;
int writing = 0, ret = 0;
struct request *rq;
- struct bio *bio;
char sense[SCSI_SENSE_BUFFERSIZE];
unsigned char cmd[BLK_MAX_CDB];
if (!rq)
return -ENOMEM;
- if (hdr->iovec_count) {
- const int size = sizeof(struct sg_iovec) * hdr->iovec_count;
- struct sg_iovec *iov;
-
- iov = kmalloc(size, GFP_KERNEL);
- if (!iov) {
- ret = -ENOMEM;
- goto out;
- }
-
- if (copy_from_user(iov, hdr->dxferp, size)) {
- kfree(iov);
- ret = -EFAULT;
- goto out;
- }
-
- ret = blk_rq_map_user_iov(q, rq, iov, hdr->iovec_count);
- kfree(iov);
- } else if (hdr->dxfer_len)
- ret = blk_rq_map_user(q, rq, hdr->dxferp, hdr->dxfer_len);
-
- if (ret)
- goto out;
-
/*
* fill in request structure
*/
rq->sense_len = 0;
rq->cmd_type = REQ_TYPE_BLOCK_PC;
- bio = rq->bio;
/*
* bounce this after holding a reference to the original bio, it's
if (!rq->timeout)
rq->timeout = BLK_DEFAULT_TIMEOUT;
+ if (hdr->iovec_count) {
+ const int size = sizeof(struct sg_iovec) * hdr->iovec_count;
+ struct sg_iovec *iov;
+
+ iov = kmalloc(size, GFP_KERNEL);
+ if (!iov) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ if (copy_from_user(iov, hdr->dxferp, size)) {
+ kfree(iov);
+ ret = -EFAULT;
+ goto out;
+ }
+
+ ret = blk_rq_map_user_iov(q, rq, iov, hdr->iovec_count,
+ hdr->dxfer_len);
+ kfree(iov);
+ } else if (hdr->dxfer_len)
+ ret = blk_rq_map_user(q, rq, hdr->dxferp, hdr->dxfer_len);
+
+ if (ret)
+ goto out;
+
rq->retries = 0;
start_time = jiffies;
hdr->sb_len_wr = len;
}
- if (blk_rq_unmap_user(bio, hdr->dxfer_len))
+ if (blk_rq_unmap_user(rq))
ret = -EFAULT;
/* may not have succeeded, but output values written to control
rq->timeout = 60 * HZ;
bio = rq->bio;
- if (rq->bio)
- blk_queue_bounce(q, &rq->bio);
-
if (blk_execute_rq(q, cdi->disk, rq, 0)) {
struct request_sense *s = rq->sense;
ret = -EIO;
cdi->last_sense = s->sense_key;
}
- if (blk_rq_unmap_user(bio, len))
+ rq->bio = bio;
+ if (blk_rq_unmap_user(rq))
ret = -EFAULT;
if (ret)
break;
}
- if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) {
- ret = -EINVAL;
+ if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)
break;
- }
len -= bytes;
}
nr_pages += end - start;
/*
- * transfer and buffer must be aligned to at least hardsector
- * size for now, in the future we can relax this restriction
+ * buffer must be aligned to at least hardsector size for now
*/
- if ((uaddr & queue_dma_alignment(q)) || (len & queue_dma_alignment(q)))
+ if (uaddr & queue_dma_alignment(q))
return ERR_PTR(-EINVAL);
}
int write_to_vm)
{
struct bio *bio;
- int len = 0, i;
bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm);
*/
bio_get(bio);
- for (i = 0; i < iov_count; i++)
- len += iov[i].iov_len;
-
- if (bio->bi_size == len)
- return bio;
-
- /*
- * don't support partial mappings
- */
- bio_endio(bio, bio->bi_size, 0);
- bio_unmap_user(bio);
- return ERR_PTR(-EINVAL);
+ return bio;
}
static void __bio_unmap_user(struct bio *bio)
extern void blk_run_queue(request_queue_t *);
extern void blk_start_queueing(request_queue_t *);
extern void blk_queue_activity_fn(request_queue_t *, activity_fn *, void *);
-extern int blk_rq_map_user(request_queue_t *, struct request *, void __user *, unsigned int);
-extern int blk_rq_unmap_user(struct bio *, unsigned int);
+extern int blk_rq_map_user(request_queue_t *, struct request *, void __user *, unsigned long);
+extern int blk_rq_unmap_user(struct request *);
extern int blk_rq_map_kern(request_queue_t *, struct request *, void *, unsigned int, gfp_t);
-extern int blk_rq_map_user_iov(request_queue_t *, struct request *, struct sg_iovec *, int);
+extern int blk_rq_map_user_iov(request_queue_t *, struct request *,
+ struct sg_iovec *, int, unsigned int);
extern int blk_execute_rq(request_queue_t *, struct gendisk *,
struct request *, int);
extern void blk_execute_rq_nowait(request_queue_t *, struct gendisk *,
__BLK_TA_REMAP, /* bio was remapped */
};
+/*
+ * Notify events.
+ */
+enum blktrace_notify {
+ __BLK_TN_PROCESS = 0, /* establish pid/name mapping */
+ __BLK_TN_TIMESTAMP, /* include system clock */
+};
+
+
/*
* Trace actions in full. Additionally, read or write is masked
*/
#define BLK_TA_BOUNCE (__BLK_TA_BOUNCE)
#define BLK_TA_REMAP (__BLK_TA_REMAP | BLK_TC_ACT(BLK_TC_QUEUE))
+#define BLK_TN_PROCESS (__BLK_TN_PROCESS | BLK_TC_ACT(BLK_TC_NOTIFY))
+#define BLK_TN_TIMESTAMP (__BLK_TN_TIMESTAMP | BLK_TC_ACT(BLK_TC_NOTIFY))
+
#define BLK_IO_TRACE_MAGIC 0x65617400
#define BLK_IO_TRACE_VERSION 0x07
typedef int (elevator_may_queue_fn) (request_queue_t *, int);
typedef int (elevator_set_req_fn) (request_queue_t *, struct request *, gfp_t);
-typedef void (elevator_put_req_fn) (request_queue_t *, struct request *);
+typedef void (elevator_put_req_fn) (struct request *);
typedef void (elevator_activate_req_fn) (request_queue_t *, struct request *);
typedef void (elevator_deactivate_req_fn) (request_queue_t *, struct request *);
-typedef void *(elevator_init_fn) (request_queue_t *, elevator_t *);
+typedef void *(elevator_init_fn) (request_queue_t *);
typedef void (elevator_exit_fn) (elevator_t *);
struct elevator_ops
projects / powerpc.git / commitdiff