1 /******************************************************************************
2 * arch/xen/drivers/blkif/backend/main.c
4 * Back-end of the driver for virtual block devices. This portion of the
5 * driver exports a 'unified' block-device interface that can be accessed
6 * by any operating system that implements a compatible front end. A
7 * reference front-end implementation can be found in:
8 * arch/xen/drivers/blkif/frontend
10 * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
11 * Copyright (c) 2005, Christopher Clark
15 #include <asm-xen/evtchn.h>
16 #ifdef CONFIG_XEN_BLKDEV_GRANT
17 #include <asm-xen/xen-public/grant_table.h>
21 * These are rather arbitrary. They are fairly large because adjacent requests
22 * pulled from a communication ring are quite likely to end up being part of
23 * the same scatter/gather request at the disc.
25 * ** TRY INCREASING 'MAX_PENDING_REQS' IF WRITE SPEEDS SEEM TOO LOW **
26 * This will increase the chances of being able to write whole tracks.
27 * 64 should be enough to keep us competitive with Linux.
29 #define MAX_PENDING_REQS 64
30 #define BATCH_PER_DOMAIN 16
32 static unsigned long mmap_vstart;
34 (MAX_PENDING_REQS * BLKIF_MAX_SEGMENTS_PER_REQUEST)
35 #define MMAP_VADDR(_req,_seg) \
37 ((_req) * BLKIF_MAX_SEGMENTS_PER_REQUEST * PAGE_SIZE) + \
41 * Each outstanding request that we've passed to the lower device layers has a
42 * 'pending_req' allocated to it. Each buffer_head that completes decrements
43 * the pendcnt towards zero. When it hits zero, the specified domain has a
44 * response queued for it, with the saved 'id' passed back.
51 unsigned short operation;
56 * We can't allocate pending_req's in order, since they may complete out of
57 * order. We therefore maintain an allocation ring. This ring also indicates
58 * when enough work has been passed down -- at that point the allocation ring
61 static pending_req_t pending_reqs[MAX_PENDING_REQS];
62 static unsigned char pending_ring[MAX_PENDING_REQS];
63 static spinlock_t pend_prod_lock = SPIN_LOCK_UNLOCKED;
64 /* NB. We use a different index type to differentiate from shared blk rings. */
65 typedef unsigned int PEND_RING_IDX;
66 #define MASK_PEND_IDX(_i) ((_i)&(MAX_PENDING_REQS-1))
67 static PEND_RING_IDX pending_prod, pending_cons;
68 #define NR_PENDING_REQS (MAX_PENDING_REQS - pending_prod + pending_cons)
70 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
71 static kmem_cache_t *buffer_head_cachep;
73 static request_queue_t *plugged_queue;
74 static inline void flush_plugged_queue(void)
76 request_queue_t *q = plugged_queue;
79 if ( q->unplug_fn != NULL )
87 #ifdef CONFIG_XEN_BLKDEV_GRANT
88 /* When using grant tables to map a frame for device access then the
89 * handle returned must be used to unmap the frame. This is needed to
90 * drop the ref count on the frame.
92 static u16 pending_grant_handles[MMAP_PAGES];
93 #define pending_handle(_idx, _i) \
94 (pending_grant_handles[((_idx) * BLKIF_MAX_SEGMENTS_PER_REQUEST) + (_i)])
95 #define BLKBACK_INVALID_HANDLE (0xFFFF)
98 #ifdef CONFIG_XEN_BLKDEV_TAP_BE
100 * If the tap driver is used, we may get pages belonging to either the tap
101 * or (more likely) the real frontend. The backend must specify which domain
102 * a given page belongs to in update_va_mapping though. For the moment,
103 * the tap rewrites the ID field of the request to contain the request index
104 * and the id of the real front end domain.
106 #define BLKTAP_COOKIE 0xbeadfeed
107 static inline domid_t ID_TO_DOM(unsigned long id) { return (id >> 16); }
110 static int do_block_io_op(blkif_t *blkif, int max_to_do);
111 static void dispatch_probe(blkif_t *blkif, blkif_request_t *req);
112 static void dispatch_rw_block_io(blkif_t *blkif, blkif_request_t *req);
113 static void make_response(blkif_t *blkif, unsigned long id,
114 unsigned short op, int st);
116 static void fast_flush_area(int idx, int nr_pages)
118 #ifdef CONFIG_XEN_BLKDEV_GRANT
119 gnttab_op_t aop[BLKIF_MAX_SEGMENTS_PER_REQUEST];
120 unsigned int i, invcount = 0;
123 for ( i = 0; i < nr_pages; i++ )
125 if ( BLKBACK_INVALID_HANDLE != ( handle = pending_handle(idx, i) ) )
127 aop[i].u.unmap_grant_ref.host_virt_addr = MMAP_VADDR(idx, i);
128 aop[i].u.unmap_grant_ref.dev_bus_addr = 0;
129 aop[i].u.unmap_grant_ref.handle = handle;
130 pending_handle(idx, i) = BLKBACK_INVALID_HANDLE;
134 if ( unlikely(HYPERVISOR_grant_table_op(
135 GNTTABOP_unmap_grant_ref, aop, invcount)))
139 multicall_entry_t mcl[BLKIF_MAX_SEGMENTS_PER_REQUEST];
142 for ( i = 0; i < nr_pages; i++ )
144 mcl[i].op = __HYPERVISOR_update_va_mapping;
145 mcl[i].args[0] = MMAP_VADDR(idx, i);
150 mcl[nr_pages-1].args[2] = UVMF_TLB_FLUSH|UVMF_ALL;
151 if ( unlikely(HYPERVISOR_multicall(mcl, nr_pages) != 0) )
157 /******************************************************************
158 * BLOCK-DEVICE SCHEDULER LIST MAINTENANCE
161 static struct list_head blkio_schedule_list;
162 static spinlock_t blkio_schedule_list_lock;
164 static int __on_blkdev_list(blkif_t *blkif)
166 return blkif->blkdev_list.next != NULL;
169 static void remove_from_blkdev_list(blkif_t *blkif)
172 if ( !__on_blkdev_list(blkif) ) return;
173 spin_lock_irqsave(&blkio_schedule_list_lock, flags);
174 if ( __on_blkdev_list(blkif) )
176 list_del(&blkif->blkdev_list);
177 blkif->blkdev_list.next = NULL;
180 spin_unlock_irqrestore(&blkio_schedule_list_lock, flags);
183 static void add_to_blkdev_list_tail(blkif_t *blkif)
186 if ( __on_blkdev_list(blkif) ) return;
187 spin_lock_irqsave(&blkio_schedule_list_lock, flags);
188 if ( !__on_blkdev_list(blkif) && (blkif->status == CONNECTED) )
190 list_add_tail(&blkif->blkdev_list, &blkio_schedule_list);
193 spin_unlock_irqrestore(&blkio_schedule_list_lock, flags);
197 /******************************************************************
198 * SCHEDULER FUNCTIONS
201 static DECLARE_WAIT_QUEUE_HEAD(blkio_schedule_wait);
203 static int blkio_schedule(void *arg)
205 DECLARE_WAITQUEUE(wq, current);
208 struct list_head *ent;
211 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
218 /* Wait for work to do. */
219 add_wait_queue(&blkio_schedule_wait, &wq);
220 set_current_state(TASK_INTERRUPTIBLE);
221 if ( (NR_PENDING_REQS == MAX_PENDING_REQS) ||
222 list_empty(&blkio_schedule_list) )
224 __set_current_state(TASK_RUNNING);
225 remove_wait_queue(&blkio_schedule_wait, &wq);
227 /* Queue up a batch of requests. */
228 while ( (NR_PENDING_REQS < MAX_PENDING_REQS) &&
229 !list_empty(&blkio_schedule_list) )
231 ent = blkio_schedule_list.next;
232 blkif = list_entry(ent, blkif_t, blkdev_list);
234 remove_from_blkdev_list(blkif);
235 if ( do_block_io_op(blkif, BATCH_PER_DOMAIN) )
236 add_to_blkdev_list_tail(blkif);
240 /* Push the batch through to disc. */
241 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
242 run_task_queue(&tq_disk);
244 flush_plugged_queue();
249 static void maybe_trigger_blkio_schedule(void)
252 * Needed so that two processes, who together make the following predicate
253 * true, don't both read stale values and evaluate the predicate
254 * incorrectly. Incredibly unlikely to stall the scheduler on x86, but...
258 if ( (NR_PENDING_REQS < (MAX_PENDING_REQS/2)) &&
259 !list_empty(&blkio_schedule_list) )
260 wake_up(&blkio_schedule_wait);
265 /******************************************************************
266 * COMPLETION CALLBACK -- Called as bh->b_end_io()
269 static void __end_block_io_op(pending_req_t *pending_req, int uptodate)
273 /* An error fails the entire request. */
276 DPRINTK("Buffer not up-to-date at end of operation\n");
277 pending_req->status = BLKIF_RSP_ERROR;
280 if ( atomic_dec_and_test(&pending_req->pendcnt) )
282 int pending_idx = pending_req - pending_reqs;
283 fast_flush_area(pending_idx, pending_req->nr_pages);
284 make_response(pending_req->blkif, pending_req->id,
285 pending_req->operation, pending_req->status);
286 blkif_put(pending_req->blkif);
287 spin_lock_irqsave(&pend_prod_lock, flags);
288 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
289 spin_unlock_irqrestore(&pend_prod_lock, flags);
290 maybe_trigger_blkio_schedule();
294 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
295 static void end_block_io_op(struct buffer_head *bh, int uptodate)
297 __end_block_io_op(bh->b_private, uptodate);
298 kmem_cache_free(buffer_head_cachep, bh);
301 static int end_block_io_op(struct bio *bio, unsigned int done, int error)
304 __end_block_io_op(bio->bi_private, (done && !error));
311 /******************************************************************************
312 * NOTIFICATION FROM GUEST OS.
315 irqreturn_t blkif_be_int(int irq, void *dev_id, struct pt_regs *regs)
317 blkif_t *blkif = dev_id;
318 add_to_blkdev_list_tail(blkif);
319 maybe_trigger_blkio_schedule();
325 /******************************************************************
326 * DOWNWARD CALLS -- These interface with the block-device layer proper.
329 static int do_block_io_op(blkif_t *blkif, int max_to_do)
331 blkif_back_ring_t *blk_ring = &blkif->blk_ring;
332 blkif_request_t *req;
336 rp = blk_ring->sring->req_prod;
337 rmb(); /* Ensure we see queued requests up to 'rp'. */
339 for ( i = blk_ring->req_cons;
340 (i != rp) && !RING_REQUEST_CONS_OVERFLOW(blk_ring, i);
343 if ( (max_to_do-- == 0) || (NR_PENDING_REQS == MAX_PENDING_REQS) )
349 req = RING_GET_REQUEST(blk_ring, i);
350 switch ( req->operation )
354 dispatch_rw_block_io(blkif, req);
358 dispatch_probe(blkif, req);
362 DPRINTK("error: unknown block io operation [%d]\n",
364 make_response(blkif, req->id, req->operation, BLKIF_RSP_ERROR);
369 blk_ring->req_cons = i;
373 static void dispatch_probe(blkif_t *blkif, blkif_request_t *req)
375 int rsp = BLKIF_RSP_ERROR;
376 int pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)];
378 /* We expect one buffer only. */
379 if ( unlikely(req->nr_segments != 1) )
382 /* Make sure the buffer is page-sized. */
383 if ( (blkif_first_sect(req->frame_and_sects[0]) != 0) ||
384 (blkif_last_sect(req->frame_and_sects[0]) != 7) )
387 #ifdef CONFIG_XEN_BLKDEV_GRANT
391 op.u.map_grant_ref.host_virt_addr = MMAP_VADDR(pending_idx, 0);
392 op.u.map_grant_ref.flags = GNTMAP_host_map;
393 op.u.map_grant_ref.ref = blkif_gref_from_fas(req->frame_and_sects[0]);
394 op.u.map_grant_ref.dom = blkif->domid;
396 if ( unlikely(HYPERVISOR_grant_table_op(
397 GNTTABOP_map_grant_ref, &op, 1)))
400 if ( op.u.map_grant_ref.handle < 0 )
403 pending_handle(pending_idx, 0) = op.u.map_grant_ref.handle;
405 #else /* else CONFIG_XEN_BLKDEV_GRANT */
407 #ifdef CONFIG_XEN_BLKDEV_TAP_BE
408 /* Grab the real frontend out of the probe message. */
409 if (req->frame_and_sects[1] == BLKTAP_COOKIE)
410 blkif->is_blktap = 1;
414 #ifdef CONFIG_XEN_BLKDEV_TAP_BE
415 if ( HYPERVISOR_update_va_mapping_otherdomain(
416 MMAP_VADDR(pending_idx, 0),
417 (pte_t) { (req->frame_and_sects[0] & PAGE_MASK) | __PAGE_KERNEL },
418 0, (blkif->is_blktap ? ID_TO_DOM(req->id) : blkif->domid) ) )
422 if ( HYPERVISOR_update_va_mapping_otherdomain(
423 MMAP_VADDR(pending_idx, 0),
424 (pte_t) { (req->frame_and_sects[0] & PAGE_MASK) | __PAGE_KERNEL },
429 #endif /* endif CONFIG_XEN_BLKDEV_GRANT */
431 rsp = vbd_probe(blkif, (vdisk_t *)MMAP_VADDR(pending_idx, 0),
432 PAGE_SIZE / sizeof(vdisk_t));
435 fast_flush_area(pending_idx, 1);
436 make_response(blkif, req->id, req->operation, rsp);
439 static void dispatch_rw_block_io(blkif_t *blkif, blkif_request_t *req)
441 extern void ll_rw_block(int rw, int nr, struct buffer_head * bhs[]);
442 int operation = (req->operation == BLKIF_OP_WRITE) ? WRITE : READ;
443 unsigned long fas = 0;
444 int i, pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)];
445 pending_req_t *pending_req;
446 #ifdef CONFIG_XEN_BLKDEV_GRANT
447 gnttab_op_t aop[BLKIF_MAX_SEGMENTS_PER_REQUEST];
449 unsigned long remap_prot;
450 multicall_entry_t mcl[BLKIF_MAX_SEGMENTS_PER_REQUEST];
452 struct phys_req preq;
454 unsigned long buf; unsigned int nsec;
455 } seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
457 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
458 struct buffer_head *bh;
460 struct bio *bio = NULL, *biolist[BLKIF_MAX_SEGMENTS_PER_REQUEST];
465 /* Check that number of segments is sane. */
466 nseg = req->nr_segments;
467 if ( unlikely(nseg == 0) ||
468 unlikely(nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST) )
470 DPRINTK("Bad number of segments in request (%d)\n", nseg);
474 preq.dev = req->device;
475 preq.sector_number = req->sector_number;
478 #ifdef CONFIG_XEN_BLKDEV_GRANT
479 for ( i = 0; i < nseg; i++ )
481 fas = req->frame_and_sects[i];
482 seg[i].nsec = blkif_last_sect(fas) - blkif_first_sect(fas) + 1;
484 if ( seg[i].nsec <= 0 )
486 preq.nr_sects += seg[i].nsec;
488 aop[i].u.map_grant_ref.host_virt_addr = MMAP_VADDR(pending_idx, i);
490 aop[i].u.map_grant_ref.dom = blkif->domid;
491 aop[i].u.map_grant_ref.ref = blkif_gref_from_fas(fas);
492 aop[i].u.map_grant_ref.flags = ( GNTMAP_host_map |
493 ( ( operation == READ ) ?
494 0 : GNTMAP_readonly ) );
497 if ( unlikely(HYPERVISOR_grant_table_op(
498 GNTTABOP_map_grant_ref, aop, nseg)))
501 for ( i = 0; i < nseg; i++ )
503 if ( unlikely(aop[i].u.map_grant_ref.handle < 0) )
505 DPRINTK("invalid buffer -- could not remap it\n");
506 fast_flush_area(pending_idx, nseg);
510 phys_to_machine_mapping[__pa(MMAP_VADDR(pending_idx, i))>>PAGE_SHIFT] =
511 FOREIGN_FRAME(aop[i].u.map_grant_ref.dev_bus_addr);
513 pending_handle(pending_idx, i) = aop[i].u.map_grant_ref.handle;
517 for ( i = 0; i < nseg; i++ )
519 fas = req->frame_and_sects[i];
520 #ifdef CONFIG_XEN_BLKDEV_GRANT
521 seg[i].buf = (aop[i].u.map_grant_ref.dev_bus_addr << PAGE_SHIFT) |
522 (blkif_first_sect(fas) << 9);
524 seg[i].buf = (fas & PAGE_MASK) | (blkif_first_sect(fas) << 9);
525 seg[i].nsec = blkif_last_sect(fas) - blkif_first_sect(fas) + 1;
526 if ( seg[i].nsec <= 0 )
528 preq.nr_sects += seg[i].nsec;
532 if ( vbd_translate(&preq, blkif, operation) != 0 )
534 DPRINTK("access denied: %s of [%llu,%llu] on dev=%04x\n",
535 operation == READ ? "read" : "write", preq.sector_number,
536 preq.sector_number + preq.nr_sects, preq.dev);
540 #ifndef CONFIG_XEN_BLKDEV_GRANT
541 if ( operation == READ )
542 remap_prot = _PAGE_PRESENT|_PAGE_DIRTY|_PAGE_ACCESSED|_PAGE_RW;
544 remap_prot = _PAGE_PRESENT|_PAGE_DIRTY|_PAGE_ACCESSED;
546 for ( i = 0; i < nseg; i++ )
548 mcl[i].op = __HYPERVISOR_update_va_mapping_otherdomain;
549 mcl[i].args[0] = MMAP_VADDR(pending_idx, i);
550 mcl[i].args[1] = (seg[i].buf & PAGE_MASK) | remap_prot;
552 mcl[i].args[3] = blkif->domid;
553 #ifdef CONFIG_XEN_BLKDEV_TAP_BE
554 if ( blkif->is_blktap )
555 mcl[i].args[3] = ID_TO_DOM(req->id);
557 phys_to_machine_mapping[__pa(MMAP_VADDR(pending_idx, i))>>PAGE_SHIFT] =
558 FOREIGN_FRAME(seg[i].buf >> PAGE_SHIFT);
561 BUG_ON(HYPERVISOR_multicall(mcl, nseg) != 0);
563 for ( i = 0; i < nseg; i++ )
565 if ( unlikely(mcl[i].args[5] != 0) )
567 DPRINTK("invalid buffer -- could not remap it\n");
568 fast_flush_area(pending_idx, nseg);
572 #endif /* end ifndef CONFIG_XEN_BLKDEV_GRANT */
574 pending_req = &pending_reqs[pending_idx];
575 pending_req->blkif = blkif;
576 pending_req->id = req->id;
577 pending_req->operation = operation;
578 pending_req->status = BLKIF_RSP_OKAY;
579 pending_req->nr_pages = nseg;
581 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
583 atomic_set(&pending_req->pendcnt, nseg);
587 for ( i = 0; i < nseg; i++ )
589 bh = kmem_cache_alloc(buffer_head_cachep, GFP_KERNEL);
590 if ( unlikely(bh == NULL) )
592 __end_block_io_op(pending_req, 0);
596 memset(bh, 0, sizeof (struct buffer_head));
598 init_waitqueue_head(&bh->b_wait);
599 bh->b_size = seg[i].nsec << 9;
600 bh->b_dev = preq.dev;
601 bh->b_rdev = preq.dev;
602 bh->b_rsector = (unsigned long)preq.sector_number;
603 bh->b_data = (char *)MMAP_VADDR(pending_idx, i) +
604 (seg[i].buf & ~PAGE_MASK);
605 bh->b_page = virt_to_page(MMAP_VADDR(pending_idx, i));
606 bh->b_end_io = end_block_io_op;
607 bh->b_private = pending_req;
609 bh->b_state = (1 << BH_Mapped) | (1 << BH_Lock) |
610 (1 << BH_Req) | (1 << BH_Launder);
611 if ( operation == WRITE )
612 bh->b_state |= (1 << BH_JBD) | (1 << BH_Req) | (1 << BH_Uptodate);
614 atomic_set(&bh->b_count, 1);
616 /* Dispatch a single request. We'll flush it to disc later. */
617 generic_make_request(operation, bh);
619 preq.sector_number += seg[i].nsec;
624 for ( i = 0; i < nseg; i++ )
626 while ( (bio == NULL) ||
628 virt_to_page(MMAP_VADDR(pending_idx, i)),
630 seg[i].buf & ~PAGE_MASK) == 0) )
632 bio = biolist[nbio++] = bio_alloc(GFP_KERNEL, nseg-i);
633 if ( unlikely(bio == NULL) )
635 for ( i = 0; i < (nbio-1); i++ )
637 fast_flush_area(pending_idx, nseg);
641 bio->bi_bdev = preq.bdev;
642 bio->bi_private = pending_req;
643 bio->bi_end_io = end_block_io_op;
644 bio->bi_sector = preq.sector_number;
647 preq.sector_number += seg[i].nsec;
650 if ( (q = bdev_get_queue(bio->bi_bdev)) != plugged_queue )
652 flush_plugged_queue();
657 atomic_set(&pending_req->pendcnt, nbio);
661 for ( i = 0; i < nbio; i++ )
662 submit_bio(operation, biolist[i]);
669 make_response(blkif, req->id, req->operation, BLKIF_RSP_ERROR);
674 /******************************************************************
675 * MISCELLANEOUS SETUP / TEARDOWN / DEBUGGING
679 static void make_response(blkif_t *blkif, unsigned long id,
680 unsigned short op, int st)
682 blkif_response_t *resp;
684 blkif_back_ring_t *blk_ring = &blkif->blk_ring;
686 /* Place on the response ring for the relevant domain. */
687 spin_lock_irqsave(&blkif->blk_ring_lock, flags);
688 resp = RING_GET_RESPONSE(blk_ring, blk_ring->rsp_prod_pvt);
690 resp->operation = op;
692 wmb(); /* Ensure other side can see the response fields. */
693 blk_ring->rsp_prod_pvt++;
694 RING_PUSH_RESPONSES(blk_ring);
695 spin_unlock_irqrestore(&blkif->blk_ring_lock, flags);
697 /* Kick the relevant domain. */
698 notify_via_evtchn(blkif->evtchn);
701 void blkif_deschedule(blkif_t *blkif)
703 remove_from_blkdev_list(blkif);
706 static int __init blkif_init(void)
710 if ( !(xen_start_info.flags & SIF_INITDOMAIN) &&
711 !(xen_start_info.flags & SIF_BLK_BE_DOMAIN) )
714 blkif_interface_init();
716 if ( (mmap_vstart = allocate_empty_lowmem_region(MMAP_PAGES)) == 0 )
720 pending_prod = MAX_PENDING_REQS;
721 memset(pending_reqs, 0, sizeof(pending_reqs));
722 for ( i = 0; i < MAX_PENDING_REQS; i++ )
725 spin_lock_init(&blkio_schedule_list_lock);
726 INIT_LIST_HEAD(&blkio_schedule_list);
728 if ( kernel_thread(blkio_schedule, 0, CLONE_FS | CLONE_FILES) < 0 )
731 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
732 buffer_head_cachep = kmem_cache_create(
733 "buffer_head_cache", sizeof(struct buffer_head),
734 0, SLAB_HWCACHE_ALIGN, NULL, NULL);
739 #ifdef CONFIG_XEN_BLKDEV_GRANT
740 memset( pending_grant_handles, BLKBACK_INVALID_HANDLE, MMAP_PAGES );
741 printk(KERN_ALERT "Blkif backend is using grant tables.\n");
744 #ifdef CONFIG_XEN_BLKDEV_TAP_BE
745 printk(KERN_ALERT "NOTE: Blkif backend is running with tap support on!\n");
751 __initcall(blkif_init);