- int rsp = BLKIF_RSP_ERROR;
- int pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)];
-
- /* We expect one buffer only. */
- if ( unlikely(req->nr_segments != 1) )
- goto out;
-
- /* Make sure the buffer is page-sized. */
- if ( (blkif_first_sect(req->frame_and_sects[0]) != 0) ||
- (blkif_last_sect(req->frame_and_sects[0]) != 7) )
- goto out;
-
-#ifdef CONFIG_XEN_BLKDEV_GRANT
- {
- gnttab_op_t op;
-
- op.u.map_grant_ref.host_virt_addr = MMAP_VADDR(pending_idx, 0);
- op.u.map_grant_ref.flags = GNTMAP_host_map;
- op.u.map_grant_ref.ref = blkif_gref_from_fas(req->frame_and_sects[0]);
- op.u.map_grant_ref.dom = blkif->domid;
-
- if ( unlikely(HYPERVISOR_grant_table_op(
- GNTTABOP_map_grant_ref, &op, 1)))
- BUG();
-
- if ( op.u.map_grant_ref.handle < 0 )
- goto out;
-
- pending_handle(pending_idx, 0) = op.u.map_grant_ref.handle;
- }
-#else /* else CONFIG_XEN_BLKDEV_GRANT */
-
-#ifdef CONFIG_XEN_BLKDEV_TAP_BE
- /* Grab the real frontend out of the probe message. */
- if (req->frame_and_sects[1] == BLKTAP_COOKIE)
- blkif->is_blktap = 1;
-#endif
-
-
-#ifdef CONFIG_XEN_BLKDEV_TAP_BE
- if ( HYPERVISOR_update_va_mapping_otherdomain(
- MMAP_VADDR(pending_idx, 0),
- (pte_t) { (req->frame_and_sects[0] & PAGE_MASK) | __PAGE_KERNEL },
- 0, (blkif->is_blktap ? ID_TO_DOM(req->id) : blkif->domid) ) )
-
- goto out;
-#else
- if ( HYPERVISOR_update_va_mapping_otherdomain(
- MMAP_VADDR(pending_idx, 0),
- (pte_t) { (req->frame_and_sects[0] & PAGE_MASK) | __PAGE_KERNEL },
- 0, blkif->domid) )
-
- goto out;
-#endif
-#endif /* endif CONFIG_XEN_BLKDEV_GRANT */
-
- rsp = vbd_probe(blkif, (vdisk_t *)MMAP_VADDR(pending_idx, 0),
- PAGE_SIZE / sizeof(vdisk_t));
-
- out:
- fast_flush_area(pending_idx, 1);
- make_response(blkif, req->id, req->operation, rsp);
-}
-
-static void dispatch_rw_block_io(blkif_t *blkif, blkif_request_t *req)
-{
- extern void ll_rw_block(int rw, int nr, struct buffer_head * bhs[]);
- int operation = (req->operation == BLKIF_OP_WRITE) ? WRITE : READ;
- unsigned long fas = 0;
- int i, pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)];
- pending_req_t *pending_req;
-#ifdef CONFIG_XEN_BLKDEV_GRANT
- gnttab_op_t aop[BLKIF_MAX_SEGMENTS_PER_REQUEST];
-#else
- unsigned long remap_prot;
- multicall_entry_t mcl[BLKIF_MAX_SEGMENTS_PER_REQUEST];
-#endif
- struct phys_req preq;
- struct {
- unsigned long buf; unsigned int nsec;
- } seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
- unsigned int nseg;
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
- struct buffer_head *bh;
-#else
- struct bio *bio = NULL, *biolist[BLKIF_MAX_SEGMENTS_PER_REQUEST];
- int nbio = 0;
- request_queue_t *q;
-#endif
-
- /* Check that number of segments is sane. */
- nseg = req->nr_segments;
- if ( unlikely(nseg == 0) ||
- unlikely(nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST) )
- {
- DPRINTK("Bad number of segments in request (%d)\n", nseg);
- goto bad_descriptor;
- }
-
- preq.dev = req->device;
- preq.sector_number = req->sector_number;
- preq.nr_sects = 0;
-
-#ifdef CONFIG_XEN_BLKDEV_GRANT
- for ( i = 0; i < nseg; i++ )
- {
- fas = req->frame_and_sects[i];
- seg[i].nsec = blkif_last_sect(fas) - blkif_first_sect(fas) + 1;
-
- if ( seg[i].nsec <= 0 )
- goto bad_descriptor;
- preq.nr_sects += seg[i].nsec;
-
- aop[i].u.map_grant_ref.host_virt_addr = MMAP_VADDR(pending_idx, i);
-
- aop[i].u.map_grant_ref.dom = blkif->domid;
- aop[i].u.map_grant_ref.ref = blkif_gref_from_fas(fas);
- aop[i].u.map_grant_ref.flags = ( GNTMAP_host_map |
- ( ( operation == READ ) ?
- 0 : GNTMAP_readonly ) );
- }
-
- if ( unlikely(HYPERVISOR_grant_table_op(
- GNTTABOP_map_grant_ref, aop, nseg)))
- BUG();
-
- for ( i = 0; i < nseg; i++ )
- {
- if ( unlikely(aop[i].u.map_grant_ref.handle < 0) )
- {
- DPRINTK("invalid buffer -- could not remap it\n");
- fast_flush_area(pending_idx, nseg);
- goto bad_descriptor;
- }
-
- phys_to_machine_mapping[__pa(MMAP_VADDR(pending_idx, i))>>PAGE_SHIFT] =
- FOREIGN_FRAME(aop[i].u.map_grant_ref.dev_bus_addr);
-
- pending_handle(pending_idx, i) = aop[i].u.map_grant_ref.handle;
- }
-#endif
-
- for ( i = 0; i < nseg; i++ )
- {
- fas = req->frame_and_sects[i];
-#ifdef CONFIG_XEN_BLKDEV_GRANT
- seg[i].buf = (aop[i].u.map_grant_ref.dev_bus_addr << PAGE_SHIFT) |
- (blkif_first_sect(fas) << 9);
-#else
- seg[i].buf = (fas & PAGE_MASK) | (blkif_first_sect(fas) << 9);
- seg[i].nsec = blkif_last_sect(fas) - blkif_first_sect(fas) + 1;
- if ( seg[i].nsec <= 0 )
- goto bad_descriptor;
- preq.nr_sects += seg[i].nsec;
-#endif
- }
-
- if ( vbd_translate(&preq, blkif, operation) != 0 )
- {
- DPRINTK("access denied: %s of [%llu,%llu] on dev=%04x\n",
- operation == READ ? "read" : "write", preq.sector_number,
- preq.sector_number + preq.nr_sects, preq.dev);
- goto bad_descriptor;
- }
-
-#ifndef CONFIG_XEN_BLKDEV_GRANT
- if ( operation == READ )
- remap_prot = _PAGE_PRESENT|_PAGE_DIRTY|_PAGE_ACCESSED|_PAGE_RW;
- else
- remap_prot = _PAGE_PRESENT|_PAGE_DIRTY|_PAGE_ACCESSED;
-
- for ( i = 0; i < nseg; i++ )
- {
- mcl[i].op = __HYPERVISOR_update_va_mapping_otherdomain;
- mcl[i].args[0] = MMAP_VADDR(pending_idx, i);
- mcl[i].args[1] = (seg[i].buf & PAGE_MASK) | remap_prot;
- mcl[i].args[2] = 0;
- mcl[i].args[3] = blkif->domid;
-#ifdef CONFIG_XEN_BLKDEV_TAP_BE
- if ( blkif->is_blktap )
- mcl[i].args[3] = ID_TO_DOM(req->id);
-#endif
- phys_to_machine_mapping[__pa(MMAP_VADDR(pending_idx, i))>>PAGE_SHIFT] =
- FOREIGN_FRAME(seg[i].buf >> PAGE_SHIFT);
- }
-
- BUG_ON(HYPERVISOR_multicall(mcl, nseg) != 0);
-
- for ( i = 0; i < nseg; i++ )
- {
- if ( unlikely(mcl[i].args[5] != 0) )
- {
- DPRINTK("invalid buffer -- could not remap it\n");
- fast_flush_area(pending_idx, nseg);
- goto bad_descriptor;
- }
- }
-#endif /* end ifndef CONFIG_XEN_BLKDEV_GRANT */
-
- pending_req = &pending_reqs[pending_idx];
- pending_req->blkif = blkif;
- pending_req->id = req->id;
- pending_req->operation = operation;
- pending_req->status = BLKIF_RSP_OKAY;
- pending_req->nr_pages = nseg;
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
-
- atomic_set(&pending_req->pendcnt, nseg);
- pending_cons++;
- blkif_get(blkif);
-
- for ( i = 0; i < nseg; i++ )
- {
- bh = kmem_cache_alloc(buffer_head_cachep, GFP_KERNEL);
- if ( unlikely(bh == NULL) )
- {
- __end_block_io_op(pending_req, 0);
- continue;
- }
-
- memset(bh, 0, sizeof (struct buffer_head));
-
- init_waitqueue_head(&bh->b_wait);
- bh->b_size = seg[i].nsec << 9;
- bh->b_dev = preq.dev;
- bh->b_rdev = preq.dev;
- bh->b_rsector = (unsigned long)preq.sector_number;
- bh->b_data = (char *)MMAP_VADDR(pending_idx, i) +
- (seg[i].buf & ~PAGE_MASK);
- bh->b_page = virt_to_page(MMAP_VADDR(pending_idx, i));
- bh->b_end_io = end_block_io_op;
- bh->b_private = pending_req;
-
- bh->b_state = (1 << BH_Mapped) | (1 << BH_Lock) |
- (1 << BH_Req) | (1 << BH_Launder);
- if ( operation == WRITE )
- bh->b_state |= (1 << BH_JBD) | (1 << BH_Req) | (1 << BH_Uptodate);
-
- atomic_set(&bh->b_count, 1);
-
- /* Dispatch a single request. We'll flush it to disc later. */
- generic_make_request(operation, bh);
-
- preq.sector_number += seg[i].nsec;
- }
-
-#else
-
- for ( i = 0; i < nseg; i++ )
- {
- while ( (bio == NULL) ||
- (bio_add_page(bio,
- virt_to_page(MMAP_VADDR(pending_idx, i)),
- seg[i].nsec << 9,
- seg[i].buf & ~PAGE_MASK) == 0) )
- {
- bio = biolist[nbio++] = bio_alloc(GFP_KERNEL, nseg-i);
- if ( unlikely(bio == NULL) )
- {
- for ( i = 0; i < (nbio-1); i++ )
- bio_put(biolist[i]);
- fast_flush_area(pending_idx, nseg);
- goto bad_descriptor;
- }
-
- bio->bi_bdev = preq.bdev;
- bio->bi_private = pending_req;
- bio->bi_end_io = end_block_io_op;
- bio->bi_sector = preq.sector_number;
- }
-
- preq.sector_number += seg[i].nsec;
- }
-
- if ( (q = bdev_get_queue(bio->bi_bdev)) != plugged_queue )
- {
- flush_plugged_queue();
- blk_get_queue(q);
- plugged_queue = q;
- }
-
- atomic_set(&pending_req->pendcnt, nbio);
- pending_cons++;
- blkif_get(blkif);
-
- for ( i = 0; i < nbio; i++ )
- submit_bio(operation, biolist[i]);
-
-#endif
-
- return;
-
- bad_descriptor:
- make_response(blkif, req->id, req->operation, BLKIF_RSP_ERROR);
+ extern void ll_rw_block(int rw, int nr, struct buffer_head * bhs[]);
+ int operation = (req->operation == BLKIF_OP_WRITE) ? WRITE : READ;
+ struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+ struct phys_req preq;
+ struct {
+ unsigned long buf; unsigned int nsec;
+ } seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+ unsigned int nseg;
+ struct bio *bio = NULL, *biolist[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+ int ret, i, nbio = 0;
+
+ /* Check that number of segments is sane. */
+ nseg = req->nr_segments;
+ if (unlikely(nseg == 0) ||
+ unlikely(nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
+ DPRINTK("Bad number of segments in request (%d)\n", nseg);
+ goto fail_response;
+ }
+
+ preq.dev = req->handle;
+ preq.sector_number = req->sector_number;
+ preq.nr_sects = 0;
+
+ pending_req->blkif = blkif;
+ pending_req->id = req->id;
+ pending_req->operation = operation;
+ pending_req->status = BLKIF_RSP_OKAY;
+ pending_req->nr_pages = nseg;
+
+ for (i = 0; i < nseg; i++) {
+ uint32_t flags;
+
+ seg[i].nsec = req->seg[i].last_sect -
+ req->seg[i].first_sect + 1;
+
+ if ((req->seg[i].last_sect >= (PAGE_SIZE >> 9)) ||
+ (seg[i].nsec <= 0))
+ goto fail_response;
+ preq.nr_sects += seg[i].nsec;
+
+ flags = GNTMAP_host_map;
+ if ( operation == WRITE )
+ flags |= GNTMAP_readonly;
+ gnttab_set_map_op(&map[i], vaddr(pending_req, i), flags,
+ req->seg[i].gref, blkif->domid);
+ }
+
+ ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map, nseg);
+ BUG_ON(ret);
+
+ for (i = 0; i < nseg; i++) {
+ if (unlikely(map[i].status != 0)) {
+ DPRINTK("invalid buffer -- could not remap it\n");
+ goto fail_flush;
+ }
+
+ pending_handle(pending_req, i) = map[i].handle;
+ set_phys_to_machine(__pa(vaddr(
+ pending_req, i)) >> PAGE_SHIFT,
+ FOREIGN_FRAME(map[i].dev_bus_addr >> PAGE_SHIFT));
+ seg[i].buf = map[i].dev_bus_addr |
+ (req->seg[i].first_sect << 9);
+ }
+
+ if (vbd_translate(&preq, blkif, operation) != 0) {
+ DPRINTK("access denied: %s of [%llu,%llu] on dev=%04x\n",
+ operation == READ ? "read" : "write",
+ preq.sector_number,
+ preq.sector_number + preq.nr_sects, preq.dev);
+ goto fail_flush;
+ }
+
+ for (i = 0; i < nseg; i++) {
+ if (((int)preq.sector_number|(int)seg[i].nsec) &
+ ((bdev_hardsect_size(preq.bdev) >> 9) - 1)) {
+ DPRINTK("Misaligned I/O request from domain %d",
+ blkif->domid);
+ goto fail_put_bio;
+ }
+
+ while ((bio == NULL) ||
+ (bio_add_page(bio,
+ virt_to_page(vaddr(pending_req, i)),
+ seg[i].nsec << 9,
+ seg[i].buf & ~PAGE_MASK) == 0)) {
+ bio = biolist[nbio++] = bio_alloc(GFP_KERNEL, nseg-i);
+ if (unlikely(bio == NULL))
+ goto fail_put_bio;
+
+ bio->bi_bdev = preq.bdev;
+ bio->bi_private = pending_req;
+ bio->bi_end_io = end_block_io_op;
+ bio->bi_sector = preq.sector_number;
+ }
+
+ preq.sector_number += seg[i].nsec;
+ }
+
+ plug_queue(blkif, bio);
+ atomic_set(&pending_req->pendcnt, nbio);
+ blkif_get(blkif);
+
+ for (i = 0; i < nbio; i++)
+ submit_bio(operation, biolist[i]);
+
+ return;
+
+ fail_put_bio:
+ for (i = 0; i < (nbio-1); i++)
+ bio_put(biolist[i]);
+ fail_flush:
+ fast_flush_area(pending_req);
+ fail_response:
+ make_response(blkif, req->id, req->operation, BLKIF_RSP_ERROR);
+ free_req(pending_req);