X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=drivers%2Fblock%2Fll_rw_blk.c;h=17c403ebd58bbf65cadfd323a1f00aa6b5f04732;hb=9bf4aaab3e101692164d49b7ca357651eb691cb6;hp=5baa11da0ac9cf24f9c6d4129d24d135ee73aed4;hpb=db216c3d5e4c040e557a50f8f5d35d5c415e8c1c;p=linux-2.6.git diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c index 5baa11da0..17c403ebd 100644 --- a/drivers/block/ll_rw_blk.c +++ b/drivers/block/ll_rw_blk.c @@ -632,6 +632,8 @@ int blk_queue_resize_tags(request_queue_t *q, int new_depth) return 0; } +EXPORT_SYMBOL(blk_queue_resize_tags); + /** * blk_queue_end_tag - end tag operations for a request * @q: the request queue for the device @@ -817,14 +819,14 @@ EXPORT_SYMBOL(blk_dump_rq_flags); void blk_recount_segments(request_queue_t *q, struct bio *bio) { struct bio_vec *bv, *bvprv = NULL; - int i, nr_phys_segs, nr_hw_segs, seg_size, cluster; + int i, nr_phys_segs, nr_hw_segs, seg_size, hw_seg_size, cluster; int high, highprv = 1; if (unlikely(!bio->bi_io_vec)) return; cluster = q->queue_flags & (1 << QUEUE_FLAG_CLUSTER); - seg_size = nr_phys_segs = nr_hw_segs = 0; + hw_seg_size = seg_size = nr_phys_segs = nr_hw_segs = 0; bio_for_each_segment(bv, bio, i) { /* * the trick here is making sure that a high page is never @@ -841,22 +843,35 @@ void blk_recount_segments(request_queue_t *q, struct bio *bio) goto new_segment; if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bv)) goto new_segment; + if (BIOVEC_VIRT_OVERSIZE(hw_seg_size + bv->bv_len)) + goto new_hw_segment; seg_size += bv->bv_len; + hw_seg_size += bv->bv_len; bvprv = bv; continue; } new_segment: - if (!BIOVEC_VIRT_MERGEABLE(bvprv, bv)) + if (BIOVEC_VIRT_MERGEABLE(bvprv, bv) && + !BIOVEC_VIRT_OVERSIZE(hw_seg_size + bv->bv_len)) { + hw_seg_size += bv->bv_len; + } else { new_hw_segment: + if (hw_seg_size > bio->bi_hw_front_size) + bio->bi_hw_front_size = hw_seg_size; + hw_seg_size = BIOVEC_VIRT_START_SIZE(bv) + bv->bv_len; nr_hw_segs++; + } nr_phys_segs++; bvprv = bv; seg_size = bv->bv_len; highprv = high; } - + if (hw_seg_size > bio->bi_hw_back_size) + bio->bi_hw_back_size = hw_seg_size; + if (nr_hw_segs == 1 && hw_seg_size > bio->bi_hw_front_size) + bio->bi_hw_front_size = hw_seg_size; bio->bi_phys_segments = nr_phys_segs; bio->bi_hw_segments = nr_hw_segs; bio->bi_flags |= (1 << BIO_SEG_VALID); @@ -889,22 +904,17 @@ EXPORT_SYMBOL(blk_phys_contig_segment); int blk_hw_contig_segment(request_queue_t *q, struct bio *bio, struct bio *nxt) { - if (!(q->queue_flags & (1 << QUEUE_FLAG_CLUSTER))) - return 0; - - if (!BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio), __BVEC_START(nxt))) + if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) + blk_recount_segments(q, bio); + if (unlikely(!bio_flagged(nxt, BIO_SEG_VALID))) + blk_recount_segments(q, nxt); + if (!BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio), __BVEC_START(nxt)) || + BIOVEC_VIRT_OVERSIZE(bio->bi_hw_front_size + bio->bi_hw_back_size)) return 0; if (bio->bi_size + nxt->bi_size > q->max_segment_size) return 0; - /* - * bio and nxt are contigous in memory, check if the queue allows - * these two to be merged into one - */ - if (BIO_SEG_BOUNDARY(q, bio, nxt)) - return 1; - - return 0; + return 1; } EXPORT_SYMBOL(blk_hw_contig_segment); @@ -974,7 +984,8 @@ static inline int ll_new_mergeable(request_queue_t *q, if (req->nr_phys_segments + nr_phys_segs > q->max_phys_segments) { req->flags |= REQ_NOMERGE; - q->last_merge = NULL; + if (req == q->last_merge) + q->last_merge = NULL; return 0; } @@ -996,7 +1007,8 @@ static inline int ll_new_hw_segment(request_queue_t *q, if (req->nr_hw_segments + nr_hw_segs > q->max_hw_segments || req->nr_phys_segments + nr_phys_segs > q->max_phys_segments) { req->flags |= REQ_NOMERGE; - q->last_merge = NULL; + if (req == q->last_merge) + q->last_merge = NULL; return 0; } @@ -1012,14 +1024,31 @@ static inline int ll_new_hw_segment(request_queue_t *q, static int ll_back_merge_fn(request_queue_t *q, struct request *req, struct bio *bio) { + int len; + if (req->nr_sectors + bio_sectors(bio) > q->max_sectors) { req->flags |= REQ_NOMERGE; - q->last_merge = NULL; + if (req == q->last_merge) + q->last_merge = NULL; return 0; } - - if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(req->biotail), __BVEC_START(bio))) - return ll_new_mergeable(q, req, bio); + if (unlikely(!bio_flagged(req->biotail, BIO_SEG_VALID))) + blk_recount_segments(q, req->biotail); + if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) + blk_recount_segments(q, bio); + len = req->biotail->bi_hw_back_size + bio->bi_hw_front_size; + if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(req->biotail), __BVEC_START(bio)) && + !BIOVEC_VIRT_OVERSIZE(len)) { + int mergeable = ll_new_mergeable(q, req, bio); + + if (mergeable) { + if (req->nr_hw_segments == 1) + req->bio->bi_hw_front_size = len; + if (bio->bi_hw_segments == 1) + bio->bi_hw_back_size = len; + } + return mergeable; + } return ll_new_hw_segment(q, req, bio); } @@ -1027,14 +1056,31 @@ static int ll_back_merge_fn(request_queue_t *q, struct request *req, static int ll_front_merge_fn(request_queue_t *q, struct request *req, struct bio *bio) { + int len; + if (req->nr_sectors + bio_sectors(bio) > q->max_sectors) { req->flags |= REQ_NOMERGE; - q->last_merge = NULL; + if (req == q->last_merge) + q->last_merge = NULL; return 0; } - - if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio), __BVEC_START(req->bio))) - return ll_new_mergeable(q, req, bio); + len = bio->bi_hw_back_size + req->bio->bi_hw_front_size; + if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) + blk_recount_segments(q, bio); + if (unlikely(!bio_flagged(req->bio, BIO_SEG_VALID))) + blk_recount_segments(q, req->bio); + if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio), __BVEC_START(req->bio)) && + !BIOVEC_VIRT_OVERSIZE(len)) { + int mergeable = ll_new_mergeable(q, req, bio); + + if (mergeable) { + if (bio->bi_hw_segments == 1) + bio->bi_hw_front_size = len; + if (req->nr_hw_segments == 1) + req->biotail->bi_hw_back_size = len; + } + return mergeable; + } return ll_new_hw_segment(q, req, bio); } @@ -1066,8 +1112,17 @@ static int ll_merge_requests_fn(request_queue_t *q, struct request *req, return 0; total_hw_segments = req->nr_hw_segments + next->nr_hw_segments; - if (blk_hw_contig_segment(q, req->biotail, next->bio)) + if (blk_hw_contig_segment(q, req->biotail, next->bio)) { + int len = req->biotail->bi_hw_back_size + next->bio->bi_hw_front_size; + /* + * propagate the combined length to the end of the requests + */ + if (req->nr_hw_segments == 1) + req->bio->bi_hw_front_size = len; + if (next->nr_hw_segments == 1) + next->biotail->bi_hw_back_size = len; total_hw_segments--; + } if (total_hw_segments > q->max_hw_segments) return 0; @@ -1123,7 +1178,7 @@ EXPORT_SYMBOL(blk_remove_plug); /* * remove the plug and let it rip.. */ -static inline void __generic_unplug_device(request_queue_t *q) +void __generic_unplug_device(request_queue_t *q) { if (test_bit(QUEUE_FLAG_STOPPED, &q->queue_flags)) return; @@ -1137,6 +1192,7 @@ static inline void __generic_unplug_device(request_queue_t *q) if (elv_next_request(q)) q->request_fn(q); } +EXPORT_SYMBOL(__generic_unplug_device); /** * generic_unplug_device - fire a request queue @@ -1411,9 +1467,6 @@ request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock) printk("Using %s io scheduler\n", chosen_elevator->elevator_name); } - if (elevator_init(q, chosen_elevator)) - goto out_elv; - q->request_fn = rfn; q->back_merge_fn = ll_back_merge_fn; q->front_merge_fn = ll_front_merge_fn; @@ -1431,8 +1484,12 @@ request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock) blk_queue_max_hw_segments(q, MAX_HW_SEGMENTS); blk_queue_max_phys_segments(q, MAX_PHYS_SEGMENTS); - return q; -out_elv: + /* + * all done + */ + if (!elevator_init(q, chosen_elevator)) + return q; + blk_cleanup_queue(q); out_init: kmem_cache_free(requestq_cachep, q); @@ -1759,54 +1816,53 @@ EXPORT_SYMBOL(blk_insert_request); * * A matching blk_rq_unmap_user() must be issued at the end of io, while * still in process context. + * + * Note: The mapped bio may need to be bounced through blk_queue_bounce() + * before being submitted to the device, as pages mapped may be out of + * reach. It's the callers responsibility to make sure this happens. The + * original bio must be passed back in to blk_rq_unmap_user() for proper + * unmapping. */ struct request *blk_rq_map_user(request_queue_t *q, int rw, void __user *ubuf, unsigned int len) { - struct request *rq = NULL; - char *buf = NULL; + unsigned long uaddr; + struct request *rq; struct bio *bio; - int ret; + + if (len > (q->max_sectors << 9)) + return ERR_PTR(-EINVAL); + if ((!len && ubuf) || (len && !ubuf)) + return ERR_PTR(-EINVAL); rq = blk_get_request(q, rw, __GFP_WAIT); if (!rq) return ERR_PTR(-ENOMEM); - bio = bio_map_user(q, NULL, (unsigned long) ubuf, len, rw == READ); - if (!bio) { - int bytes = (len + 511) & ~511; - - buf = kmalloc(bytes, q->bounce_gfp | GFP_USER); - if (!buf) { - ret = -ENOMEM; - goto fault; - } - - if (rw == WRITE) { - if (copy_from_user(buf, ubuf, len)) { - ret = -EFAULT; - goto fault; - } - } else - memset(buf, 0, len); - } + /* + * if alignment requirement is satisfied, map in user pages for + * direct dma. else, set up kernel bounce buffers + */ + uaddr = (unsigned long) ubuf; + if (!(uaddr & queue_dma_alignment(q)) && !(len & queue_dma_alignment(q))) + bio = bio_map_user(q, NULL, uaddr, len, rw == READ); + else + bio = bio_copy_user(q, uaddr, len, rw == READ); - rq->bio = rq->biotail = bio; - if (rq->bio) + if (!IS_ERR(bio)) { + rq->bio = rq->biotail = bio; blk_rq_bio_prep(q, rq, bio); - rq->buffer = rq->data = buf; - rq->data_len = len; - return rq; -fault: - if (buf) - kfree(buf); - if (bio) - bio_unmap_user(bio, 1); - if (rq) - blk_put_request(rq); + rq->buffer = rq->data = NULL; + rq->data_len = len; + return rq; + } - return ERR_PTR(ret); + /* + * bio is the err-ptr + */ + blk_put_request(rq); + return (struct request *) bio; } EXPORT_SYMBOL(blk_rq_map_user); @@ -1820,18 +1876,15 @@ EXPORT_SYMBOL(blk_rq_map_user); * Description: * Unmap a request previously mapped by blk_rq_map_user(). */ -int blk_rq_unmap_user(struct request *rq, void __user *ubuf, struct bio *bio, - unsigned int ulen) +int blk_rq_unmap_user(struct request *rq, struct bio *bio, unsigned int ulen) { - const int read = rq_data_dir(rq) == READ; int ret = 0; - if (bio) - bio_unmap_user(bio, read); - if (rq->buffer) { - if (read && copy_to_user(ubuf, rq->buffer, ulen)) - ret = -EFAULT; - kfree(rq->buffer); + if (bio) { + if (bio_flagged(bio, BIO_USER_MAPPED)) + bio_unmap_user(bio); + else + ret = bio_uncopy_user(bio); } blk_put_request(rq); @@ -2280,13 +2333,9 @@ get_rq: out: if (freereq) __blk_put_request(q, freereq); + if (bio_sync(bio)) + __generic_unplug_device(q); - if (blk_queue_plugged(q)) { - int nrq = q->rq.count[READ] + q->rq.count[WRITE] - q->in_flight; - - if (nrq == q->unplug_thresh || bio_sync(bio)) - __generic_unplug_device(q); - } spin_unlock_irq(q->queue_lock); return 0; @@ -2535,7 +2584,7 @@ EXPORT_SYMBOL(process_that_request_first); void blk_recalc_rq_segments(struct request *rq) { - struct bio *bio; + struct bio *bio, *prevbio = NULL; int nr_phys_segs, nr_hw_segs; if (!rq->bio) @@ -2548,6 +2597,13 @@ void blk_recalc_rq_segments(struct request *rq) nr_phys_segs += bio_phys_segments(rq->q, bio); nr_hw_segs += bio_hw_segments(rq->q, bio); + if (prevbio) { + if (blk_phys_contig_segment(rq->q, prevbio, bio)) + nr_phys_segs--; + if (blk_hw_contig_segment(rq->q, prevbio, bio)) + nr_hw_segs--; + } + prevbio = bio; } rq->nr_phys_segments = nr_phys_segs; @@ -2610,7 +2666,7 @@ static int __end_that_request_first(struct request *req, int uptodate, } total_bytes = bio_nbytes = 0; - while ((bio = req->bio)) { + while ((bio = req->bio) != NULL) { int nbytes; if (nr_bytes >= bio->bi_size) { @@ -2812,6 +2868,8 @@ int kblockd_schedule_work(struct work_struct *work) return queue_work(kblockd_workqueue, work); } +EXPORT_SYMBOL(kblockd_schedule_work); + void kblockd_flush(void) { flush_workqueue(kblockd_workqueue);