#include <linux/blkdev.h>
#include <linux/elevator.h>
#include <linux/bio.h>
-#include <linux/config.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/compiler.h>
-#include <linux/hash.h>
#include <linux/rbtree.h>
#include <linux/interrupt.h>
struct as_rq *next_arq[2]; /* next in sort order */
sector_t last_sector[2]; /* last REQ_SYNC & REQ_ASYNC sectors */
- struct list_head *hash; /* request hash */
unsigned long exit_prob; /* probability a task will exit while
being waited on */
struct io_context *io_context; /* The submitting task */
- /*
- * request hash, key is the ending offset (for back merge lookup)
- */
- struct list_head hash;
- unsigned int on_hash;
-
/*
* expire fifo
*/
put_io_context(arq->io_context);
}
-/*
- * the back merge hash support functions
- */
-static const int as_hash_shift = 6;
-#define AS_HASH_BLOCK(sec) ((sec) >> 3)
-#define AS_HASH_FN(sec) (hash_long(AS_HASH_BLOCK((sec)), as_hash_shift))
-#define AS_HASH_ENTRIES (1 << as_hash_shift)
-#define rq_hash_key(rq) ((rq)->sector + (rq)->nr_sectors)
-#define list_entry_hash(ptr) list_entry((ptr), struct as_rq, hash)
-
-static inline void __as_del_arq_hash(struct as_rq *arq)
-{
- arq->on_hash = 0;
- list_del_init(&arq->hash);
-}
-
-static inline void as_del_arq_hash(struct as_rq *arq)
-{
- if (arq->on_hash)
- __as_del_arq_hash(arq);
-}
-
-static void as_add_arq_hash(struct as_data *ad, struct as_rq *arq)
-{
- struct request *rq = arq->request;
-
- BUG_ON(arq->on_hash);
-
- arq->on_hash = 1;
- list_add(&arq->hash, &ad->hash[AS_HASH_FN(rq_hash_key(rq))]);
-}
-
-/*
- * move hot entry to front of chain
- */
-static inline void as_hot_arq_hash(struct as_data *ad, struct as_rq *arq)
-{
- struct request *rq = arq->request;
- struct list_head *head = &ad->hash[AS_HASH_FN(rq_hash_key(rq))];
-
- if (!arq->on_hash) {
- WARN_ON(1);
- return;
- }
-
- if (arq->hash.prev != head) {
- list_del(&arq->hash);
- list_add(&arq->hash, head);
- }
-}
-
-static struct request *as_find_arq_hash(struct as_data *ad, sector_t offset)
-{
- struct list_head *hash_list = &ad->hash[AS_HASH_FN(offset)];
- struct list_head *entry, *next = hash_list->next;
-
- while ((entry = next) != hash_list) {
- struct as_rq *arq = list_entry_hash(entry);
- struct request *__rq = arq->request;
-
- next = entry->next;
-
- BUG_ON(!arq->on_hash);
-
- if (!rq_mergeable(__rq)) {
- as_del_arq_hash(arq);
- continue;
- }
-
- if (rq_hash_key(__rq) == offset)
- return __rq;
- }
-
- return NULL;
-}
-
/*
* rb tree support functions
*/
-#define RB_EMPTY(root) ((root)->rb_node == NULL)
-#define ON_RB(node) (rb_parent(node) != node)
-#define RB_CLEAR(node) (rb_set_parent(node, node))
#define rb_entry_arq(node) rb_entry((node), struct as_rq, rb_node)
#define ARQ_RB_ROOT(ad, arq) (&(ad)->sort_list[(arq)->is_sync])
#define rq_rb_key(rq) (rq)->sector
static inline void as_del_arq_rb(struct as_data *ad, struct as_rq *arq)
{
- if (!ON_RB(&arq->rb_node)) {
+ if (!RB_EMPTY_NODE(&arq->rb_node)) {
WARN_ON(1);
return;
}
rb_erase(&arq->rb_node, ARQ_RB_ROOT(ad, arq));
- RB_CLEAR(&arq->rb_node);
+ RB_CLEAR_NODE(&arq->rb_node);
}
static struct request *
struct rb_node *rbprev = rb_prev(&last->rb_node);
struct as_rq *arq_next, *arq_prev;
- BUG_ON(!ON_RB(&last->rb_node));
+ BUG_ON(!RB_EMPTY_NODE(&last->rb_node));
if (rbprev)
arq_prev = rb_entry_arq(rbprev);
}
/*
- * as_can_anticipate indicates weather we should either run arq
+ * as_can_anticipate indicates whether we should either run arq
* or keep anticipating a better request.
*/
static int as_can_anticipate(struct as_data *ad, struct as_rq *arq)
ad->next_arq[data_dir] = as_find_next_arq(ad, arq);
list_del_init(&arq->fifo);
- as_del_arq_hash(arq);
as_del_arq_rb(ad, arq);
}
struct request *rq = arq->request;
const int data_dir = arq->is_sync;
- BUG_ON(!ON_RB(&arq->rb_node));
+ BUG_ON(!RB_EMPTY_NODE(&arq->rb_node));
as_antic_stop(ad);
ad->antic_status = ANTIC_OFF;
*/
if (reads) {
- BUG_ON(RB_EMPTY(&ad->sort_list[REQ_SYNC]));
+ BUG_ON(RB_EMPTY_ROOT(&ad->sort_list[REQ_SYNC]));
if (writes && ad->batch_data_dir == REQ_SYNC)
/*
if (writes) {
dispatch_writes:
- BUG_ON(RB_EMPTY(&ad->sort_list[REQ_ASYNC]));
+ BUG_ON(RB_EMPTY_ROOT(&ad->sort_list[REQ_ASYNC]));
if (ad->batch_data_dir == REQ_SYNC) {
ad->changed_batch = 1;
arq->state = AS_RQ_NEW;
if (rq_data_dir(arq->request) == READ
- || current->flags&PF_SYNCWRITE)
+ || (arq->request->cmd_flags & REQ_RW_SYNC))
arq->is_sync = 1;
else
arq->is_sync = 0;
}
as_add_arq_rb(ad, arq);
- if (rq_mergeable(arq->request))
- as_add_arq_hash(ad, arq);
/*
* set expire time (only used for reads) and add to fifo list
struct as_data *ad = q->elevator->elevator_data;
sector_t rb_key = bio->bi_sector + bio_sectors(bio);
struct request *__rq;
- int ret;
-
- /*
- * see if the merge hash can satisfy a back merge
- */
- __rq = as_find_arq_hash(ad, bio->bi_sector);
- if (__rq) {
- BUG_ON(__rq->sector + __rq->nr_sectors != bio->bi_sector);
-
- if (elv_rq_merge_ok(__rq, bio)) {
- ret = ELEVATOR_BACK_MERGE;
- goto out;
- }
- }
/*
* check for front merge
*/
__rq = as_find_arq_rb(ad, rb_key, bio_data_dir(bio));
- if (__rq) {
- BUG_ON(rb_key != rq_rb_key(__rq));
-
- if (elv_rq_merge_ok(__rq, bio)) {
- ret = ELEVATOR_FRONT_MERGE;
- goto out;
- }
+ if (__rq && elv_rq_merge_ok(__rq, bio)) {
+ *req = __rq;
+ return ELEVATOR_FRONT_MERGE;
}
return ELEVATOR_NO_MERGE;
-out:
- if (ret) {
- if (rq_mergeable(__rq))
- as_hot_arq_hash(ad, RQ_DATA(__rq));
- }
- *req = __rq;
- return ret;
}
static void as_merged_request(request_queue_t *q, struct request *req)
struct as_data *ad = q->elevator->elevator_data;
struct as_rq *arq = RQ_DATA(req);
- /*
- * hash always needs to be repositioned, key is end sector
- */
- as_del_arq_hash(arq);
- as_add_arq_hash(ad, arq);
-
/*
* if the merge was a front merge, we need to reposition request
*/
BUG_ON(!arq);
BUG_ON(!anext);
- /*
- * reposition arq (this is the merged request) in hash, and in rbtree
- * in case of a front merge
- */
- as_del_arq_hash(arq);
- as_add_arq_hash(ad, arq);
-
if (rq_rb_key(req) != arq->rb_key) {
as_del_arq_rb(ad, arq);
as_add_arq_rb(ad, arq);
if (arq) {
memset(arq, 0, sizeof(*arq));
- RB_CLEAR(&arq->rb_node);
+ RB_CLEAR_NODE(&arq->rb_node);
arq->request = rq;
arq->state = AS_RQ_PRESCHED;
arq->io_context = NULL;
- INIT_LIST_HEAD(&arq->hash);
- arq->on_hash = 0;
INIT_LIST_HEAD(&arq->fifo);
rq->elevator_private = arq;
return 0;
mempool_destroy(ad->arq_pool);
put_io_context(ad->io_context);
- kfree(ad->hash);
kfree(ad);
}
static void *as_init_queue(request_queue_t *q, elevator_t *e)
{
struct as_data *ad;
- int i;
if (!arq_pool)
return NULL;
ad->q = q; /* Identify what queue the data belongs to */
- ad->hash = kmalloc_node(sizeof(struct list_head)*AS_HASH_ENTRIES,
- GFP_KERNEL, q->node);
- if (!ad->hash) {
- kfree(ad);
- return NULL;
- }
-
ad->arq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
mempool_free_slab, arq_pool, q->node);
if (!ad->arq_pool) {
- kfree(ad->hash);
kfree(ad);
return NULL;
}
init_timer(&ad->antic_timer);
INIT_WORK(&ad->antic_work, as_work_handler, q);
- for (i = 0; i < AS_HASH_ENTRIES; i++)
- INIT_LIST_HEAD(&ad->hash[i]);
-
INIT_LIST_HEAD(&ad->fifo_list[REQ_SYNC]);
INIT_LIST_HEAD(&ad->fifo_list[REQ_ASYNC]);
ad->sort_list[REQ_SYNC] = RB_ROOT;
projects / powerpc.git / blobdiff