drivers/scsi/scsi_lib.c

   1 /*
   2  *  scsi_lib.c Copyright (C) 1999 Eric Youngdale
   3  *
   4  *  SCSI queueing library.
   5  *      Initial versions: Eric Youngdale (eric@andante.org).
   6  *                        Based upon conversations with large numbers
   7  *                        of people at Linux Expo.
   8  */
   9
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/completion.h>
  13 #include <linux/kernel.h>
  14 #include <linux/mempool.h>
  15 #include <linux/slab.h>
  16 #include <linux/init.h>
  17 #include <linux/pci.h>
  18 #include <linux/delay.h>
  19 #include <linux/hardirq.h>
  20
  21 #include <scsi/scsi.h>
  22 #include <scsi/scsi_cmnd.h>
  23 #include <scsi/scsi_dbg.h>
  24 #include <scsi/scsi_device.h>
  25 #include <scsi/scsi_driver.h>
  26 #include <scsi/scsi_eh.h>
  27 #include <scsi/scsi_host.h>
  28
  29 #include "scsi_priv.h"
  30 #include "scsi_logging.h"
  31
  32
  33 #define SG_MEMPOOL_NR           ARRAY_SIZE(scsi_sg_pools)
  34 #define SG_MEMPOOL_SIZE         32
  35
  36 struct scsi_host_sg_pool {
  37         size_t          size;
  38         char            *name;
  39         kmem_cache_t    *slab;
  40         mempool_t       *pool;
  41 };
  42
  43 #if (SCSI_MAX_PHYS_SEGMENTS < 32)
  44 #error SCSI_MAX_PHYS_SEGMENTS is too small
  45 #endif
  46
  47 #define SP(x) { x, "sgpool-" #x }
  48 static struct scsi_host_sg_pool scsi_sg_pools[] = {
  49         SP(8),
  50         SP(16),
  51         SP(32),
  52 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
  53         SP(64),
  54 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
  55         SP(128),
  56 #if (SCSI_MAX_PHYS_SEGMENTS > 128)
  57         SP(256),
  58 #if (SCSI_MAX_PHYS_SEGMENTS > 256)
  59 #error SCSI_MAX_PHYS_SEGMENTS is too large
  60 #endif
  61 #endif
  62 #endif
  63 #endif
  64 };
  65 #undef SP
  66
  67 static void scsi_run_queue(struct request_queue *q);
  68
  69 /*
  70  * Function:    scsi_unprep_request()
  71  *
  72  * Purpose:     Remove all preparation done for a request, including its
  73  *              associated scsi_cmnd, so that it can be requeued.
  74  *
  75  * Arguments:   req     - request to unprepare
  76  *
  77  * Lock status: Assumed that no locks are held upon entry.
  78  *
  79  * Returns:     Nothing.
  80  */
  81 static void scsi_unprep_request(struct request *req)
  82 {
  83         struct scsi_cmnd *cmd = req->special;
  84
  85         req->flags &= ~REQ_DONTPREP;
  86         req->special = NULL;
  87
  88         scsi_put_command(cmd);
  89 }
  90
  91 /*
  92  * Function:    scsi_queue_insert()
  93  *
  94  * Purpose:     Insert a command in the midlevel queue.
  95  *
  96  * Arguments:   cmd    - command that we are adding to queue.
  97  *              reason - why we are inserting command to queue.
  98  *
  99  * Lock status: Assumed that lock is not held upon entry.
 100  *
 101  * Returns:     Nothing.
 102  *
 103  * Notes:       We do this for one of two cases.  Either the host is busy
 104  *              and it cannot accept any more commands for the time being,
 105  *              or the device returned QUEUE_FULL and can accept no more
 106  *              commands.
 107  * Notes:       This could be called either from an interrupt context or a
 108  *              normal process context.
 109  */
 110 int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
 111 {
 112         struct Scsi_Host *host = cmd->device->host;
 113         struct scsi_device *device = cmd->device;
 114         struct request_queue *q = device->request_queue;
 115         unsigned long flags;
 116
 117         SCSI_LOG_MLQUEUE(1,
 118                  printk("Inserting command %p into mlqueue\n", cmd));
 119
 120         /*
 121          * Set the appropriate busy bit for the device/host.
 122          *
 123          * If the host/device isn't busy, assume that something actually
 124          * completed, and that we should be able to queue a command now.
 125          *
 126          * Note that the prior mid-layer assumption that any host could
 127          * always queue at least one command is now broken.  The mid-layer
 128          * will implement a user specifiable stall (see
 129          * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
 130          * if a command is requeued with no other commands outstanding
 131          * either for the device or for the host.
 132          */
 133         if (reason == SCSI_MLQUEUE_HOST_BUSY)
 134                 host->host_blocked = host->max_host_blocked;
 135         else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
 136                 device->device_blocked = device->max_device_blocked;
 137
 138         /*
 139          * Decrement the counters, since these commands are no longer
 140          * active on the host/device.
 141          */
 142         scsi_device_unbusy(device);
 143
 144         /*
 145          * Requeue this command.  It will go before all other commands
 146          * that are already in the queue.
 147          *
 148          * NOTE: there is magic here about the way the queue is plugged if
 149          * we have no outstanding commands.
 150          *
 151          * Although we *don't* plug the queue, we call the request
 152          * function.  The SCSI request function detects the blocked condition
 153          * and plugs the queue appropriately.
 154          */
 155         spin_lock_irqsave(q->queue_lock, flags);
 156         blk_requeue_request(q, cmd->request);
 157         spin_unlock_irqrestore(q->queue_lock, flags);
 158
 159         scsi_run_queue(q);
 160
 161         return 0;
 162 }
 163
 164 /**
 165  * scsi_execute - insert request and wait for the result
 166  * @sdev:       scsi device
 167  * @cmd:        scsi command
 168  * @data_direction: data direction
 169  * @buffer:     data buffer
 170  * @bufflen:    len of buffer
 171  * @sense:      optional sense buffer
 172  * @timeout:    request timeout in seconds
 173  * @retries:    number of times to retry request
 174  * @flags:      or into request flags;
 175  *
 176  * returns the req->errors value which is the the scsi_cmnd result
 177  * field.
 178  **/
 179 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
 180                  int data_direction, void *buffer, unsigned bufflen,
 181                  unsigned char *sense, int timeout, int retries, int flags)
 182 {
 183         struct request *req;
 184         int write = (data_direction == DMA_TO_DEVICE);
 185         int ret = DRIVER_ERROR << 24;
 186
 187         req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
 188
 189         if (bufflen &&  blk_rq_map_kern(sdev->request_queue, req,
 190                                         buffer, bufflen, __GFP_WAIT))
 191                 goto out;
 192
 193         req->cmd_len = COMMAND_SIZE(cmd[0]);
 194         memcpy(req->cmd, cmd, req->cmd_len);
 195         req->sense = sense;
 196         req->sense_len = 0;
 197         req->retries = retries;
 198         req->timeout = timeout;
 199         req->flags |= flags | REQ_BLOCK_PC | REQ_SPECIAL | REQ_QUIET;
 200
 201         /*
 202          * head injection *required* here otherwise quiesce won't work
 203          */
 204         blk_execute_rq(req->q, NULL, req, 1);
 205
 206         ret = req->errors;
 207  out:
 208         blk_put_request(req);
 209
 210         return ret;
 211 }
 212 EXPORT_SYMBOL(scsi_execute);
 213
 214
 215 int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
 216                      int data_direction, void *buffer, unsigned bufflen,
 217                      struct scsi_sense_hdr *sshdr, int timeout, int retries)
 218 {
 219         char *sense = NULL;
 220         int result;
 221
 222         if (sshdr) {
 223                 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
 224                 if (!sense)
 225                         return DRIVER_ERROR << 24;
 226         }
 227         result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
 228                               sense, timeout, retries, 0);
 229         if (sshdr)
 230                 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
 231
 232         kfree(sense);
 233         return result;
 234 }
 235 EXPORT_SYMBOL(scsi_execute_req);
 236
 237 struct scsi_io_context {
 238         void *data;
 239         void (*done)(void *data, char *sense, int result, int resid);
 240         char sense[SCSI_SENSE_BUFFERSIZE];
 241 };
 242
 243 static kmem_cache_t *scsi_io_context_cache;
 244
 245 static void scsi_end_async(struct request *req, int uptodate)
 246 {
 247         struct scsi_io_context *sioc = req->end_io_data;
 248
 249         if (sioc->done)
 250                 sioc->done(sioc->data, sioc->sense, req->errors, req->data_len);
 251
 252         kmem_cache_free(scsi_io_context_cache, sioc);
 253         __blk_put_request(req->q, req);
 254 }
 255
 256 static int scsi_merge_bio(struct request *rq, struct bio *bio)
 257 {
 258         struct request_queue *q = rq->q;
 259
 260         bio->bi_flags &= ~(1 << BIO_SEG_VALID);
 261         if (rq_data_dir(rq) == WRITE)
 262                 bio->bi_rw |= (1 << BIO_RW);
 263         blk_queue_bounce(q, &bio);
 264
 265         if (!rq->bio)
 266                 blk_rq_bio_prep(q, rq, bio);
 267         else if (!q->back_merge_fn(q, rq, bio))
 268                 return -EINVAL;
 269         else {
 270                 rq->biotail->bi_next = bio;
 271                 rq->biotail = bio;
 272                 rq->hard_nr_sectors += bio_sectors(bio);
 273                 rq->nr_sectors = rq->hard_nr_sectors;
 274         }
 275
 276         return 0;
 277 }
 278
 279 static int scsi_bi_endio(struct bio *bio, unsigned int bytes_done, int error)
 280 {
 281         if (bio->bi_size)
 282                 return 1;
 283
 284         bio_put(bio);
 285         return 0;
 286 }
 287
 288 /**
 289  * scsi_req_map_sg - map a scatterlist into a request
 290  * @rq:         request to fill
 291  * @sg:         scatterlist
 292  * @nsegs:      number of elements
 293  * @bufflen:    len of buffer
 294  * @gfp:        memory allocation flags
 295  *
 296  * scsi_req_map_sg maps a scatterlist into a request so that the
 297  * request can be sent to the block layer. We do not trust the scatterlist
 298  * sent to use, as some ULDs use that struct to only organize the pages.
 299  */
 300 static int scsi_req_map_sg(struct request *rq, struct scatterlist *sgl,
 301                            int nsegs, unsigned bufflen, gfp_t gfp)
 302 {
 303         struct request_queue *q = rq->q;
 304         int nr_pages = (bufflen + sgl[0].offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
 305         unsigned int data_len = 0, len, bytes, off;
 306         struct page *page;
 307         struct bio *bio = NULL;
 308         int i, err, nr_vecs = 0;
 309
 310         for (i = 0; i < nsegs; i++) {
 311                 page = sgl[i].page;
 312                 off = sgl[i].offset;
 313                 len = sgl[i].length;
 314                 data_len += len;
 315
 316                 while (len > 0) {
 317                         bytes = min_t(unsigned int, len, PAGE_SIZE - off);
 318
 319                         if (!bio) {
 320                                 nr_vecs = min_t(int, BIO_MAX_PAGES, nr_pages);
 321                                 nr_pages -= nr_vecs;
 322
 323                                 bio = bio_alloc(gfp, nr_vecs);
 324                                 if (!bio) {
 325                                         err = -ENOMEM;
 326                                         goto free_bios;
 327                                 }
 328                                 bio->bi_end_io = scsi_bi_endio;
 329                         }
 330
 331                         if (bio_add_pc_page(q, bio, page, bytes, off) !=
 332                             bytes) {
 333                                 bio_put(bio);
 334                                 err = -EINVAL;
 335                                 goto free_bios;
 336                         }
 337
 338                         if (bio->bi_vcnt >= nr_vecs) {
 339                                 err = scsi_merge_bio(rq, bio);
 340                                 if (err) {
 341                                         bio_endio(bio, bio->bi_size, 0);
 342                                         goto free_bios;
 343                                 }
 344                                 bio = NULL;
 345                         }
 346
 347                         page++;
 348                         len -= bytes;
 349                         off = 0;
 350                 }
 351         }
 352
 353         rq->buffer = rq->data = NULL;
 354         rq->data_len = data_len;
 355         return 0;
 356
 357 free_bios:
 358         while ((bio = rq->bio) != NULL) {
 359                 rq->bio = bio->bi_next;
 360                 /*
 361                  * call endio instead of bio_put incase it was bounced
 362                  */
 363                 bio_endio(bio, bio->bi_size, 0);
 364         }
 365
 366         return err;
 367 }
 368
 369 /**
 370  * scsi_execute_async - insert request
 371  * @sdev:       scsi device
 372  * @cmd:        scsi command
 373  * @cmd_len:    length of scsi cdb
 374  * @data_direction: data direction
 375  * @buffer:     data buffer (this can be a kernel buffer or scatterlist)
 376  * @bufflen:    len of buffer
 377  * @use_sg:     if buffer is a scatterlist this is the number of elements
 378  * @timeout:    request timeout in seconds
 379  * @retries:    number of times to retry request
 380  * @flags:      or into request flags
 381  **/
 382 int scsi_execute_async(struct scsi_device *sdev, const unsigned char *cmd,
 383                        int cmd_len, int data_direction, void *buffer, unsigned bufflen,
 384                        int use_sg, int timeout, int retries, void *privdata,
 385                        void (*done)(void *, char *, int, int), gfp_t gfp)
 386 {
 387         struct request *req;
 388         struct scsi_io_context *sioc;
 389         int err = 0;
 390         int write = (data_direction == DMA_TO_DEVICE);
 391
 392         sioc = kmem_cache_alloc(scsi_io_context_cache, gfp);
 393         if (!sioc)
 394                 return DRIVER_ERROR << 24;
 395         memset(sioc, 0, sizeof(*sioc));
 396
 397         req = blk_get_request(sdev->request_queue, write, gfp);
 398         if (!req)
 399                 goto free_sense;
 400         req->flags |= REQ_BLOCK_PC | REQ_QUIET;
 401
 402         if (use_sg)
 403                 err = scsi_req_map_sg(req, buffer, use_sg, bufflen, gfp);
 404         else if (bufflen)
 405                 err = blk_rq_map_kern(req->q, req, buffer, bufflen, gfp);
 406
 407         if (err)
 408                 goto free_req;
 409
 410         req->cmd_len = cmd_len;
 411         memset(req->cmd, 0, BLK_MAX_CDB); /* ATAPI hates garbage after CDB */
 412         memcpy(req->cmd, cmd, req->cmd_len);
 413         req->sense = sioc->sense;
 414         req->sense_len = 0;
 415         req->timeout = timeout;
 416         req->retries = retries;
 417         req->end_io_data = sioc;
 418
 419         sioc->data = privdata;
 420         sioc->done = done;
 421
 422         blk_execute_rq_nowait(req->q, NULL, req, 1, scsi_end_async);
 423         return 0;
 424
 425 free_req:
 426         blk_put_request(req);
 427 free_sense:
 428         kfree(sioc);
 429         return DRIVER_ERROR << 24;
 430 }
 431 EXPORT_SYMBOL_GPL(scsi_execute_async);
 432
 433 /*
 434  * Function:    scsi_init_cmd_errh()
 435  *
 436  * Purpose:     Initialize cmd fields related to error handling.
 437  *
 438  * Arguments:   cmd     - command that is ready to be queued.
 439  *
 440  * Notes:       This function has the job of initializing a number of
 441  *              fields related to error handling.   Typically this will
 442  *              be called once for each command, as required.
 443  */
 444 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
 445 {
 446         cmd->serial_number = 0;
 447         memset(cmd->sense_buffer, 0, sizeof cmd->sense_buffer);
 448         if (cmd->cmd_len == 0)
 449                 cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]);
 450 }
 451
 452 void scsi_device_unbusy(struct scsi_device *sdev)
 453 {
 454         struct Scsi_Host *shost = sdev->host;
 455         unsigned long flags;
 456
 457         spin_lock_irqsave(shost->host_lock, flags);
 458         shost->host_busy--;
 459         if (unlikely(scsi_host_in_recovery(shost) &&
 460                      (shost->host_failed || shost->host_eh_scheduled)))
 461                 scsi_eh_wakeup(shost);
 462         spin_unlock(shost->host_lock);
 463         spin_lock(sdev->request_queue->queue_lock);
 464         sdev->device_busy--;
 465         spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
 466 }
 467
 468 /*
 469  * Called for single_lun devices on IO completion. Clear starget_sdev_user,
 470  * and call blk_run_queue for all the scsi_devices on the target -
 471  * including current_sdev first.
 472  *
 473  * Called with *no* scsi locks held.
 474  */
 475 static void scsi_single_lun_run(struct scsi_device *current_sdev)
 476 {
 477         struct Scsi_Host *shost = current_sdev->host;
 478         struct scsi_device *sdev, *tmp;
 479         struct scsi_target *starget = scsi_target(current_sdev);
 480         unsigned long flags;
 481
 482         spin_lock_irqsave(shost->host_lock, flags);
 483         starget->starget_sdev_user = NULL;
 484         spin_unlock_irqrestore(shost->host_lock, flags);
 485
 486         /*
 487          * Call blk_run_queue for all LUNs on the target, starting with
 488          * current_sdev. We race with others (to set starget_sdev_user),
 489          * but in most cases, we will be first. Ideally, each LU on the
 490          * target would get some limited time or requests on the target.
 491          */
 492         blk_run_queue(current_sdev->request_queue);
 493
 494         spin_lock_irqsave(shost->host_lock, flags);
 495         if (starget->starget_sdev_user)
 496                 goto out;
 497         list_for_each_entry_safe(sdev, tmp, &starget->devices,
 498                         same_target_siblings) {
 499                 if (sdev == current_sdev)
 500                         continue;
 501                 if (scsi_device_get(sdev))
 502                         continue;
 503
 504                 spin_unlock_irqrestore(shost->host_lock, flags);
 505                 blk_run_queue(sdev->request_queue);
 506                 spin_lock_irqsave(shost->host_lock, flags);
 507
 508                 scsi_device_put(sdev);
 509         }
 510  out:
 511         spin_unlock_irqrestore(shost->host_lock, flags);
 512 }
 513
 514 /*
 515  * Function:    scsi_run_queue()
 516  *
 517  * Purpose:     Select a proper request queue to serve next
 518  *
 519  * Arguments:   q       - last request's queue
 520  *
 521  * Returns:     Nothing
 522  *
 523  * Notes:       The previous command was completely finished, start
 524  *              a new one if possible.
 525  */
 526 static void scsi_run_queue(struct request_queue *q)
 527 {
 528         struct scsi_device *sdev = q->queuedata;
 529         struct Scsi_Host *shost = sdev->host;
 530         unsigned long flags;
 531
 532         if (sdev->single_lun)
 533                 scsi_single_lun_run(sdev);
 534
 535         spin_lock_irqsave(shost->host_lock, flags);
 536         while (!list_empty(&shost->starved_list) &&
 537                !shost->host_blocked && !shost->host_self_blocked &&
 538                 !((shost->can_queue > 0) &&
 539                   (shost->host_busy >= shost->can_queue))) {
 540                 /*
 541                  * As long as shost is accepting commands and we have
 542                  * starved queues, call blk_run_queue. scsi_request_fn
 543                  * drops the queue_lock and can add us back to the
 544                  * starved_list.
 545                  *
 546                  * host_lock protects the starved_list and starved_entry.
 547                  * scsi_request_fn must get the host_lock before checking
 548                  * or modifying starved_list or starved_entry.
 549                  */
 550                 sdev = list_entry(shost->starved_list.next,
 551                                           struct scsi_device, starved_entry);
 552                 list_del_init(&sdev->starved_entry);
 553                 spin_unlock_irqrestore(shost->host_lock, flags);
 554
 555                 blk_run_queue(sdev->request_queue);
 556
 557                 spin_lock_irqsave(shost->host_lock, flags);
 558                 if (unlikely(!list_empty(&sdev->starved_entry)))
 559                         /*
 560                          * sdev lost a race, and was put back on the
 561                          * starved list. This is unlikely but without this
 562                          * in theory we could loop forever.
 563                          */
 564                         break;
 565         }
 566         spin_unlock_irqrestore(shost->host_lock, flags);
 567
 568         blk_run_queue(q);
 569 }
 570
 571 /*
 572  * Function:    scsi_requeue_command()
 573  *
 574  * Purpose:     Handle post-processing of completed commands.
 575  *
 576  * Arguments:   q       - queue to operate on
 577  *              cmd     - command that may need to be requeued.
 578  *
 579  * Returns:     Nothing
 580  *
 581  * Notes:       After command completion, there may be blocks left
 582  *              over which weren't finished by the previous command
 583  *              this can be for a number of reasons - the main one is
 584  *              I/O errors in the middle of the request, in which case
 585  *              we need to request the blocks that come after the bad
 586  *              sector.
 587  * Notes:       Upon return, cmd is a stale pointer.
 588  */
 589 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
 590 {
 591         struct request *req = cmd->request;
 592         unsigned long flags;
 593
 594         scsi_unprep_request(req);
 595         spin_lock_irqsave(q->queue_lock, flags);
 596         blk_requeue_request(q, req);
 597         spin_unlock_irqrestore(q->queue_lock, flags);
 598
 599         scsi_run_queue(q);
 600 }
 601
 602 void scsi_next_command(struct scsi_cmnd *cmd)
 603 {
 604         struct scsi_device *sdev = cmd->device;
 605         struct request_queue *q = sdev->request_queue;
 606
 607         /* need to hold a reference on the device before we let go of the cmd */
 608         get_device(&sdev->sdev_gendev);
 609
 610         scsi_put_command(cmd);
 611         scsi_run_queue(q);
 612
 613         /* ok to remove device now */
 614         put_device(&sdev->sdev_gendev);
 615 }
 616
 617 void scsi_run_host_queues(struct Scsi_Host *shost)
 618 {
 619         struct scsi_device *sdev;
 620
 621         shost_for_each_device(sdev, shost)
 622                 scsi_run_queue(sdev->request_queue);
 623 }
 624
 625 /*
 626  * Function:    scsi_end_request()
 627  *
 628  * Purpose:     Post-processing of completed commands (usually invoked at end
 629  *              of upper level post-processing and scsi_io_completion).
 630  *
 631  * Arguments:   cmd      - command that is complete.
 632  *              uptodate - 1 if I/O indicates success, <= 0 for I/O error.
 633  *              bytes    - number of bytes of completed I/O
 634  *              requeue  - indicates whether we should requeue leftovers.
 635  *
 636  * Lock status: Assumed that lock is not held upon entry.
 637  *
 638  * Returns:     cmd if requeue required, NULL otherwise.
 639  *
 640  * Notes:       This is called for block device requests in order to
 641  *              mark some number of sectors as complete.
 642  *
 643  *              We are guaranteeing that the request queue will be goosed
 644  *              at some point during this call.
 645  * Notes:       If cmd was requeued, upon return it will be a stale pointer.
 646  */
 647 static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
 648                                           int bytes, int requeue)
 649 {
 650         request_queue_t *q = cmd->device->request_queue;
 651         struct request *req = cmd->request;
 652         unsigned long flags;
 653
 654         /*
 655          * If there are blocks left over at the end, set up the command
 656          * to queue the remainder of them.
 657          */
 658         if (end_that_request_chunk(req, uptodate, bytes)) {
 659                 int leftover = (req->hard_nr_sectors << 9);
 660
 661                 if (blk_pc_request(req))
 662                         leftover = req->data_len;
 663
 664                 /* kill remainder if no retrys */
 665                 if (!uptodate && blk_noretry_request(req))
 666                         end_that_request_chunk(req, 0, leftover);
 667                 else {
 668                         if (requeue) {
 669                                 /*
 670                                  * Bleah.  Leftovers again.  Stick the
 671                                  * leftovers in the front of the
 672                                  * queue, and goose the queue again.
 673                                  */
 674                                 scsi_requeue_command(q, cmd);
 675                                 cmd = NULL;
 676                         }
 677                         return cmd;
 678                 }
 679         }
 680
 681         add_disk_randomness(req->rq_disk);
 682
 683         spin_lock_irqsave(q->queue_lock, flags);
 684         if (blk_rq_tagged(req))
 685                 blk_queue_end_tag(q, req);
 686         end_that_request_last(req, uptodate);
 687         spin_unlock_irqrestore(q->queue_lock, flags);
 688
 689         /*
 690          * This will goose the queue request function at the end, so we don't
 691          * need to worry about launching another command.
 692          */
 693         scsi_next_command(cmd);
 694         return NULL;
 695 }
 696
 697 static struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, gfp_t gfp_mask)
 698 {
 699         struct scsi_host_sg_pool *sgp;
 700         struct scatterlist *sgl;
 701
 702         BUG_ON(!cmd->use_sg);
 703
 704         switch (cmd->use_sg) {
 705         case 1 ... 8:
 706                 cmd->sglist_len = 0;
 707                 break;
 708         case 9 ... 16:
 709                 cmd->sglist_len = 1;
 710                 break;
 711         case 17 ... 32:
 712                 cmd->sglist_len = 2;
 713                 break;
 714 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
 715         case 33 ... 64:
 716                 cmd->sglist_len = 3;
 717                 break;
 718 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
 719         case 65 ... 128:
 720                 cmd->sglist_len = 4;
 721                 break;
 722 #if (SCSI_MAX_PHYS_SEGMENTS  > 128)
 723         case 129 ... 256:
 724                 cmd->sglist_len = 5;
 725                 break;
 726 #endif
 727 #endif
 728 #endif
 729         default:
 730                 return NULL;
 731         }
 732
 733         sgp = scsi_sg_pools + cmd->sglist_len;
 734         sgl = mempool_alloc(sgp->pool, gfp_mask);
 735         return sgl;
 736 }
 737
 738 static void scsi_free_sgtable(struct scatterlist *sgl, int index)
 739 {
 740         struct scsi_host_sg_pool *sgp;
 741
 742         BUG_ON(index >= SG_MEMPOOL_NR);
 743
 744         sgp = scsi_sg_pools + index;
 745         mempool_free(sgl, sgp->pool);
 746 }
 747
 748 /*
 749  * Function:    scsi_release_buffers()
 750  *
 751  * Purpose:     Completion processing for block device I/O requests.
 752  *
 753  * Arguments:   cmd     - command that we are bailing.
 754  *
 755  * Lock status: Assumed that no lock is held upon entry.
 756  *
 757  * Returns:     Nothing
 758  *
 759  * Notes:       In the event that an upper level driver rejects a
 760  *              command, we must release resources allocated during
 761  *              the __init_io() function.  Primarily this would involve
 762  *              the scatter-gather table, and potentially any bounce
 763  *              buffers.
 764  */
 765 static void scsi_release_buffers(struct scsi_cmnd *cmd)
 766 {
 767         if (cmd->use_sg)
 768                 scsi_free_sgtable(cmd->request_buffer, cmd->sglist_len);
 769
 770         /*
 771          * Zero these out.  They now point to freed memory, and it is
 772          * dangerous to hang onto the pointers.
 773          */
 774         cmd->request_buffer = NULL;
 775         cmd->request_bufflen = 0;
 776 }
 777
 778 /*
 779  * Function:    scsi_io_completion()
 780  *
 781  * Purpose:     Completion processing for block device I/O requests.
 782  *
 783  * Arguments:   cmd   - command that is finished.
 784  *
 785  * Lock status: Assumed that no lock is held upon entry.
 786  *
 787  * Returns:     Nothing
 788  *
 789  * Notes:       This function is matched in terms of capabilities to
 790  *              the function that created the scatter-gather list.
 791  *              In other words, if there are no bounce buffers
 792  *              (the normal case for most drivers), we don't need
 793  *              the logic to deal with cleaning up afterwards.
 794  *
 795  *              We must do one of several things here:
 796  *
 797  *              a) Call scsi_end_request.  This will finish off the
 798  *                 specified number of sectors.  If we are done, the
 799  *                 command block will be released, and the queue
 800  *                 function will be goosed.  If we are not done, then
 801  *                 scsi_end_request will directly goose the queue.
 802  *
 803  *              b) We can just use scsi_requeue_command() here.  This would
 804  *                 be used if we just wanted to retry, for example.
 805  */
 806 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 807 {
 808         int result = cmd->result;
 809         int this_count = cmd->request_bufflen;
 810         request_queue_t *q = cmd->device->request_queue;
 811         struct request *req = cmd->request;
 812         int clear_errors = 1;
 813         struct scsi_sense_hdr sshdr;
 814         int sense_valid = 0;
 815         int sense_deferred = 0;
 816
 817         scsi_release_buffers(cmd);
 818
 819         if (result) {
 820                 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
 821                 if (sense_valid)
 822                         sense_deferred = scsi_sense_is_deferred(&sshdr);
 823         }
 824
 825         if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
 826                 req->errors = result;
 827                 if (result) {
 828                         clear_errors = 0;
 829                         if (sense_valid && req->sense) {
 830                                 /*
 831                                  * SG_IO wants current and deferred errors
 832                                  */
 833                                 int len = 8 + cmd->sense_buffer[7];
 834
 835                                 if (len > SCSI_SENSE_BUFFERSIZE)
 836                                         len = SCSI_SENSE_BUFFERSIZE;
 837                                 memcpy(req->sense, cmd->sense_buffer,  len);
 838                                 req->sense_len = len;
 839                         }
 840                 } else
 841                         req->data_len = cmd->resid;
 842         }
 843
 844         /*
 845          * Next deal with any sectors which we were able to correctly
 846          * handle.
 847          */
 848         SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, "
 849                                       "%d bytes done.\n",
 850                                       req->nr_sectors, good_bytes));
 851         SCSI_LOG_HLCOMPLETE(1, printk("use_sg is %d\n", cmd->use_sg));
 852
 853         if (clear_errors)
 854                 req->errors = 0;
 855
 856         /* A number of bytes were successfully read.  If there
 857          * are leftovers and there is some kind of error
 858          * (result != 0), retry the rest.
 859          */
 860         if (scsi_end_request(cmd, 1, good_bytes, result == 0) == NULL)
 861                 return;
 862
 863         /* good_bytes = 0, or (inclusive) there were leftovers and
 864          * result = 0, so scsi_end_request couldn't retry.
 865          */
 866         if (sense_valid && !sense_deferred) {
 867                 switch (sshdr.sense_key) {
 868                 case UNIT_ATTENTION:
 869                         if (cmd->device->removable) {
 870                                 /* Detected disc change.  Set a bit
 871                                  * and quietly refuse further access.
 872                                  */
 873                                 cmd->device->changed = 1;
 874                                 scsi_end_request(cmd, 0, this_count, 1);
 875                                 return;
 876                         } else {
 877                                 /* Must have been a power glitch, or a
 878                                  * bus reset.  Could not have been a
 879                                  * media change, so we just retry the
 880                                  * request and see what happens.
 881                                  */
 882                                 scsi_requeue_command(q, cmd);
 883                                 return;
 884                         }
 885                         break;
 886                 case ILLEGAL_REQUEST:
 887                         /* If we had an ILLEGAL REQUEST returned, then
 888                          * we may have performed an unsupported
 889                          * command.  The only thing this should be
 890                          * would be a ten byte read where only a six
 891                          * byte read was supported.  Also, on a system
 892                          * where READ CAPACITY failed, we may have
 893                          * read past the end of the disk.
 894                          */
 895                         if ((cmd->device->use_10_for_rw &&
 896                             sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
 897                             (cmd->cmnd[0] == READ_10 ||
 898                              cmd->cmnd[0] == WRITE_10)) {
 899                                 cmd->device->use_10_for_rw = 0;
 900                                 /* This will cause a retry with a
 901                                  * 6-byte command.
 902                                  */
 903                                 scsi_requeue_command(q, cmd);
 904                                 return;
 905                         } else {
 906                                 scsi_end_request(cmd, 0, this_count, 1);
 907                                 return;
 908                         }
 909                         break;
 910                 case NOT_READY:
 911                         /* If the device is in the process of becoming
 912                          * ready, or has a temporary blockage, retry.
 913                          */
 914                         if (sshdr.asc == 0x04) {
 915                                 switch (sshdr.ascq) {
 916                                 case 0x01: /* becoming ready */
 917                                 case 0x04: /* format in progress */
 918                                 case 0x05: /* rebuild in progress */
 919                                 case 0x06: /* recalculation in progress */
 920                                 case 0x07: /* operation in progress */
 921                                 case 0x08: /* Long write in progress */
 922                                 case 0x09: /* self test in progress */
 923                                         scsi_requeue_command(q, cmd);
 924                                         return;
 925                                 default:
 926                                         break;
 927                                 }
 928                         }
 929                         if (!(req->flags & REQ_QUIET)) {
 930                                 scmd_printk(KERN_INFO, cmd,
 931                                             "Device not ready: ");
 932                                 scsi_print_sense_hdr("", &sshdr);
 933                         }
 934                         scsi_end_request(cmd, 0, this_count, 1);
 935                         return;
 936                 case VOLUME_OVERFLOW:
 937                         if (!(req->flags & REQ_QUIET)) {
 938                                 scmd_printk(KERN_INFO, cmd,
 939                                             "Volume overflow, CDB: ");
 940                                 __scsi_print_command(cmd->cmnd);
 941                                 scsi_print_sense("", cmd);
 942                         }
 943                         /* See SSC3rXX or current. */
 944                         scsi_end_request(cmd, 0, this_count, 1);
 945                         return;
 946                 default:
 947                         break;
 948                 }
 949         }
 950         if (host_byte(result) == DID_RESET) {
 951                 /* Third party bus reset or reset for error recovery
 952                  * reasons.  Just retry the request and see what
 953                  * happens.
 954                  */
 955                 scsi_requeue_command(q, cmd);
 956                 return;
 957         }
 958         if (result) {
 959                 if (!(req->flags & REQ_QUIET)) {
 960                         scmd_printk(KERN_INFO, cmd,
 961                                     "SCSI error: return code = 0x%08x\n",
 962                                     result);
 963                         if (driver_byte(result) & DRIVER_SENSE)
 964                                 scsi_print_sense("", cmd);
 965                 }
 966         }
 967         scsi_end_request(cmd, 0, this_count, !result);
 968 }
 969 EXPORT_SYMBOL(scsi_io_completion);
 970
 971 /*
 972  * Function:    scsi_init_io()
 973  *
 974  * Purpose:     SCSI I/O initialize function.
 975  *
 976  * Arguments:   cmd   - Command descriptor we wish to initialize
 977  *
 978  * Returns:     0 on success
 979  *              BLKPREP_DEFER if the failure is retryable
 980  *              BLKPREP_KILL if the failure is fatal
 981  */
 982 static int scsi_init_io(struct scsi_cmnd *cmd)
 983 {
 984         struct request     *req = cmd->request;
 985         struct scatterlist *sgpnt;
 986         int                count;
 987
 988         /*
 989          * if this is a rq->data based REQ_BLOCK_PC, setup for a non-sg xfer
 990          */
 991         if ((req->flags & REQ_BLOCK_PC) && !req->bio) {
 992                 cmd->request_bufflen = req->data_len;
 993                 cmd->request_buffer = req->data;
 994                 req->buffer = req->data;
 995                 cmd->use_sg = 0;
 996                 return 0;
 997         }
 998
 999         /*
1000          * we used to not use scatter-gather for single segment request,
1001          * but now we do (it makes highmem I/O easier to support without
1002          * kmapping pages)
1003          */
1004         cmd->use_sg = req->nr_phys_segments;
1005
1006         /*
1007          * if sg table allocation fails, requeue request later.
1008          */
1009         sgpnt = scsi_alloc_sgtable(cmd, GFP_ATOMIC);
1010         if (unlikely(!sgpnt)) {
1011                 scsi_unprep_request(req);
1012                 return BLKPREP_DEFER;
1013         }
1014
1015         cmd->request_buffer = (char *) sgpnt;
1016         cmd->request_bufflen = req->nr_sectors << 9;
1017         if (blk_pc_request(req))
1018                 cmd->request_bufflen = req->data_len;
1019         req->buffer = NULL;
1020
1021         /*
1022          * Next, walk the list, and fill in the addresses and sizes of
1023          * each segment.
1024          */
1025         count = blk_rq_map_sg(req->q, req, cmd->request_buffer);
1026
1027         /*
1028          * mapped well, send it off
1029          */
1030         if (likely(count <= cmd->use_sg)) {
1031                 cmd->use_sg = count;
1032                 return 0;
1033         }
1034
1035         printk(KERN_ERR "Incorrect number of segments after building list\n");
1036         printk(KERN_ERR "counted %d, received %d\n", count, cmd->use_sg);
1037         printk(KERN_ERR "req nr_sec %lu, cur_nr_sec %u\n", req->nr_sectors,
1038                         req->current_nr_sectors);
1039
1040         /* release the command and kill it */
1041         scsi_release_buffers(cmd);
1042         scsi_put_command(cmd);
1043         return BLKPREP_KILL;
1044 }
1045
1046 static int scsi_issue_flush_fn(request_queue_t *q, struct gendisk *disk,
1047                                sector_t *error_sector)
1048 {
1049         struct scsi_device *sdev = q->queuedata;
1050         struct scsi_driver *drv;
1051
1052         if (sdev->sdev_state != SDEV_RUNNING)
1053                 return -ENXIO;
1054
1055         drv = *(struct scsi_driver **) disk->private_data;
1056         if (drv->issue_flush)
1057                 return drv->issue_flush(&sdev->sdev_gendev, error_sector);
1058
1059         return -EOPNOTSUPP;
1060 }
1061
1062 static void scsi_blk_pc_done(struct scsi_cmnd *cmd)
1063 {
1064         BUG_ON(!blk_pc_request(cmd->request));
1065         /*
1066          * This will complete the whole command with uptodate=1 so
1067          * as far as the block layer is concerned the command completed
1068          * successfully. Since this is a REQ_BLOCK_PC command the
1069          * caller should check the request's errors value
1070          */
1071         scsi_io_completion(cmd, cmd->request_bufflen);
1072 }
1073
1074 static void scsi_setup_blk_pc_cmnd(struct scsi_cmnd *cmd)
1075 {
1076         struct request *req = cmd->request;
1077
1078         BUG_ON(sizeof(req->cmd) > sizeof(cmd->cmnd));
1079         memcpy(cmd->cmnd, req->cmd, sizeof(cmd->cmnd));
1080         cmd->cmd_len = req->cmd_len;
1081         if (!req->data_len)
1082                 cmd->sc_data_direction = DMA_NONE;
1083         else if (rq_data_dir(req) == WRITE)
1084                 cmd->sc_data_direction = DMA_TO_DEVICE;
1085         else
1086                 cmd->sc_data_direction = DMA_FROM_DEVICE;
1087
1088         cmd->transfersize = req->data_len;
1089         cmd->allowed = req->retries;
1090         cmd->timeout_per_command = req->timeout;
1091         cmd->done = scsi_blk_pc_done;
1092 }
1093
1094 static int scsi_prep_fn(struct request_queue *q, struct request *req)
1095 {
1096         struct scsi_device *sdev = q->queuedata;
1097         struct scsi_cmnd *cmd;
1098         int specials_only = 0;
1099
1100         /*
1101          * Just check to see if the device is online.  If it isn't, we
1102          * refuse to process any commands.  The device must be brought
1103          * online before trying any recovery commands
1104          */
1105         if (unlikely(!scsi_device_online(sdev))) {
1106                 sdev_printk(KERN_ERR, sdev,
1107                             "rejecting I/O to offline device\n");
1108                 goto kill;
1109         }
1110         if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1111                 /* OK, we're not in a running state don't prep
1112                  * user commands */
1113                 if (sdev->sdev_state == SDEV_DEL) {
1114                         /* Device is fully deleted, no commands
1115                          * at all allowed down */
1116                         sdev_printk(KERN_ERR, sdev,
1117                                     "rejecting I/O to dead device\n");
1118                         goto kill;
1119                 }
1120                 /* OK, we only allow special commands (i.e. not
1121                  * user initiated ones */
1122                 specials_only = sdev->sdev_state;
1123         }
1124
1125         /*
1126          * Find the actual device driver associated with this command.
1127          * The SPECIAL requests are things like character device or
1128          * ioctls, which did not originate from ll_rw_blk.  Note that
1129          * the special field is also used to indicate the cmd for
1130          * the remainder of a partially fulfilled request that can
1131          * come up when there is a medium error.  We have to treat
1132          * these two cases differently.  We differentiate by looking
1133          * at request->cmd, as this tells us the real story.
1134          */
1135         if (req->flags & REQ_SPECIAL && req->special) {
1136                 cmd = req->special;
1137         } else if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1138
1139                 if(unlikely(specials_only) && !(req->flags & REQ_SPECIAL)) {
1140                         if(specials_only == SDEV_QUIESCE ||
1141                                         specials_only == SDEV_BLOCK)
1142                                 goto defer;
1143
1144                         sdev_printk(KERN_ERR, sdev,
1145                                     "rejecting I/O to device being removed\n");
1146                         goto kill;
1147                 }
1148
1149
1150                 /*
1151                  * Now try and find a command block that we can use.
1152                  */
1153                 if (!req->special) {
1154                         cmd = scsi_get_command(sdev, GFP_ATOMIC);
1155                         if (unlikely(!cmd))
1156                                 goto defer;
1157                 } else
1158                         cmd = req->special;
1159
1160                 /* pull a tag out of the request if we have one */
1161                 cmd->tag = req->tag;
1162         } else {
1163                 blk_dump_rq_flags(req, "SCSI bad req");
1164                 goto kill;
1165         }
1166
1167         /* note the overloading of req->special.  When the tag
1168          * is active it always means cmd.  If the tag goes
1169          * back for re-queueing, it may be reset */
1170         req->special = cmd;
1171         cmd->request = req;
1172
1173         /*
1174          * FIXME: drop the lock here because the functions below
1175          * expect to be called without the queue lock held.  Also,
1176          * previously, we dequeued the request before dropping the
1177          * lock.  We hope REQ_STARTED prevents anything untoward from
1178          * happening now.
1179          */
1180         if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1181                 int ret;
1182
1183                 /*
1184                  * This will do a couple of things:
1185                  *  1) Fill in the actual SCSI command.
1186                  *  2) Fill in any other upper-level specific fields
1187                  * (timeout).
1188                  *
1189                  * If this returns 0, it means that the request failed
1190                  * (reading past end of disk, reading offline device,
1191                  * etc).   This won't actually talk to the device, but
1192                  * some kinds of consistency checking may cause the
1193                  * request to be rejected immediately.
1194                  */
1195
1196                 /*
1197                  * This sets up the scatter-gather table (allocating if
1198                  * required).
1199                  */
1200                 ret = scsi_init_io(cmd);
1201                 switch(ret) {
1202                         /* For BLKPREP_KILL/DEFER the cmd was released */
1203                 case BLKPREP_KILL:
1204                         goto kill;
1205                 case BLKPREP_DEFER:
1206                         goto defer;
1207                 }
1208
1209                 /*
1210                  * Initialize the actual SCSI command for this request.
1211                  */
1212                 if (req->flags & REQ_BLOCK_PC) {
1213                         scsi_setup_blk_pc_cmnd(cmd);
1214                 } else if (req->rq_disk) {
1215                         struct scsi_driver *drv;
1216
1217                         drv = *(struct scsi_driver **)req->rq_disk->private_data;
1218                         if (unlikely(!drv->init_command(cmd))) {
1219                                 scsi_release_buffers(cmd);
1220                                 scsi_put_command(cmd);
1221                                 goto kill;
1222                         }
1223                 }
1224         }
1225
1226         /*
1227          * The request is now prepped, no need to come back here
1228          */
1229         req->flags |= REQ_DONTPREP;
1230         return BLKPREP_OK;
1231
1232  defer:
1233         /* If we defer, the elv_next_request() returns NULL, but the
1234          * queue must be restarted, so we plug here if no returning
1235          * command will automatically do that. */
1236         if (sdev->device_busy == 0)
1237                 blk_plug_device(q);
1238         return BLKPREP_DEFER;
1239  kill:
1240         req->errors = DID_NO_CONNECT << 16;
1241         return BLKPREP_KILL;
1242 }
1243
1244 /*
1245  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1246  * return 0.
1247  *
1248  * Called with the queue_lock held.
1249  */
1250 static inline int scsi_dev_queue_ready(struct request_queue *q,
1251                                   struct scsi_device *sdev)
1252 {
1253         if (sdev->device_busy >= sdev->queue_depth)
1254                 return 0;
1255         if (sdev->device_busy == 0 && sdev->device_blocked) {
1256                 /*
1257                  * unblock after device_blocked iterates to zero
1258                  */
1259                 if (--sdev->device_blocked == 0) {
1260                         SCSI_LOG_MLQUEUE(3,
1261                                    sdev_printk(KERN_INFO, sdev,
1262                                    "unblocking device at zero depth\n"));
1263                 } else {
1264                         blk_plug_device(q);
1265                         return 0;
1266                 }
1267         }
1268         if (sdev->device_blocked)
1269                 return 0;
1270
1271         return 1;
1272 }
1273
1274 /*
1275  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1276  * return 0. We must end up running the queue again whenever 0 is
1277  * returned, else IO can hang.
1278  *
1279  * Called with host_lock held.
1280  */
1281 static inline int scsi_host_queue_ready(struct request_queue *q,
1282                                    struct Scsi_Host *shost,
1283                                    struct scsi_device *sdev)
1284 {
1285         if (scsi_host_in_recovery(shost))
1286                 return 0;
1287         if (shost->host_busy == 0 && shost->host_blocked) {
1288                 /*
1289                  * unblock after host_blocked iterates to zero
1290                  */
1291                 if (--shost->host_blocked == 0) {
1292                         SCSI_LOG_MLQUEUE(3,
1293                                 printk("scsi%d unblocking host at zero depth\n",
1294                                         shost->host_no));
1295                 } else {
1296                         blk_plug_device(q);
1297                         return 0;
1298                 }
1299         }
1300         if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
1301             shost->host_blocked || shost->host_self_blocked) {
1302                 if (list_empty(&sdev->starved_entry))
1303                         list_add_tail(&sdev->starved_entry, &shost->starved_list);
1304                 return 0;
1305         }
1306
1307         /* We're OK to process the command, so we can't be starved */
1308         if (!list_empty(&sdev->starved_entry))
1309                 list_del_init(&sdev->starved_entry);
1310
1311         return 1;
1312 }
1313
1314 /*
1315  * Kill a request for a dead device
1316  */
1317 static void scsi_kill_request(struct request *req, request_queue_t *q)
1318 {
1319         struct scsi_cmnd *cmd = req->special;
1320         struct scsi_device *sdev = cmd->device;
1321         struct Scsi_Host *shost = sdev->host;
1322
1323         blkdev_dequeue_request(req);
1324
1325         if (unlikely(cmd == NULL)) {
1326                 printk(KERN_CRIT "impossible request in %s.\n",
1327                                  __FUNCTION__);
1328                 BUG();
1329         }
1330
1331         scsi_init_cmd_errh(cmd);
1332         cmd->result = DID_NO_CONNECT << 16;
1333         atomic_inc(&cmd->device->iorequest_cnt);
1334
1335         /*
1336          * SCSI request completion path will do scsi_device_unbusy(),
1337          * bump busy counts.  To bump the counters, we need to dance
1338          * with the locks as normal issue path does.
1339          */
1340         sdev->device_busy++;
1341         spin_unlock(sdev->request_queue->queue_lock);
1342         spin_lock(shost->host_lock);
1343         shost->host_busy++;
1344         spin_unlock(shost->host_lock);
1345         spin_lock(sdev->request_queue->queue_lock);
1346
1347         __scsi_done(cmd);
1348 }
1349
1350 static void scsi_softirq_done(struct request *rq)
1351 {
1352         struct scsi_cmnd *cmd = rq->completion_data;
1353         unsigned long wait_for = (cmd->allowed + 1) * cmd->timeout_per_command;
1354         int disposition;
1355
1356         INIT_LIST_HEAD(&cmd->eh_entry);
1357
1358         disposition = scsi_decide_disposition(cmd);
1359         if (disposition != SUCCESS &&
1360             time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
1361                 sdev_printk(KERN_ERR, cmd->device,
1362                             "timing out command, waited %lus\n",
1363                             wait_for/HZ);
1364                 disposition = SUCCESS;
1365         }
1366
1367         scsi_log_completion(cmd, disposition);
1368
1369         switch (disposition) {
1370                 case SUCCESS:
1371                         scsi_finish_command(cmd);
1372                         break;
1373                 case NEEDS_RETRY:
1374                         scsi_retry_command(cmd);
1375                         break;
1376                 case ADD_TO_MLQUEUE:
1377                         scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
1378                         break;
1379                 default:
1380                         if (!scsi_eh_scmd_add(cmd, 0))
1381                                 scsi_finish_command(cmd);
1382         }
1383 }
1384
1385 /*
1386  * Function:    scsi_request_fn()
1387  *
1388  * Purpose:     Main strategy routine for SCSI.
1389  *
1390  * Arguments:   q       - Pointer to actual queue.
1391  *
1392  * Returns:     Nothing
1393  *
1394  * Lock status: IO request lock assumed to be held when called.
1395  */
1396 static void scsi_request_fn(struct request_queue *q)
1397 {
1398         struct scsi_device *sdev = q->queuedata;
1399         struct Scsi_Host *shost;
1400         struct scsi_cmnd *cmd;
1401         struct request *req;
1402
1403         if (!sdev) {
1404                 printk("scsi: killing requests for dead queue\n");
1405                 while ((req = elv_next_request(q)) != NULL)
1406                         scsi_kill_request(req, q);
1407                 return;
1408         }
1409
1410         if(!get_device(&sdev->sdev_gendev))
1411                 /* We must be tearing the block queue down already */
1412                 return;
1413
1414         /*
1415          * To start with, we keep looping until the queue is empty, or until
1416          * the host is no longer able to accept any more requests.
1417          */
1418         shost = sdev->host;
1419         while (!blk_queue_plugged(q)) {
1420                 int rtn;
1421                 /*
1422                  * get next queueable request.  We do this early to make sure
1423                  * that the request is fully prepared even if we cannot
1424                  * accept it.
1425                  */
1426                 req = elv_next_request(q);
1427                 if (!req || !scsi_dev_queue_ready(q, sdev))
1428                         break;
1429
1430                 if (unlikely(!scsi_device_online(sdev))) {
1431                         sdev_printk(KERN_ERR, sdev,
1432                                     "rejecting I/O to offline device\n");
1433                         scsi_kill_request(req, q);
1434                         continue;
1435                 }
1436
1437
1438                 /*
1439                  * Remove the request from the request list.
1440                  */
1441                 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1442                         blkdev_dequeue_request(req);
1443                 sdev->device_busy++;
1444
1445                 spin_unlock(q->queue_lock);
1446                 cmd = req->special;
1447                 if (unlikely(cmd == NULL)) {
1448                         printk(KERN_CRIT "impossible request in %s.\n"
1449                                          "please mail a stack trace to "
1450                                          "linux-scsi@vger.kernel.org",
1451                                          __FUNCTION__);
1452                         BUG();
1453                 }
1454                 spin_lock(shost->host_lock);
1455
1456                 if (!scsi_host_queue_ready(q, shost, sdev))
1457                         goto not_ready;
1458                 if (sdev->single_lun) {
1459                         if (scsi_target(sdev)->starget_sdev_user &&
1460                             scsi_target(sdev)->starget_sdev_user != sdev)
1461                                 goto not_ready;
1462                         scsi_target(sdev)->starget_sdev_user = sdev;
1463                 }
1464                 shost->host_busy++;
1465
1466                 /*
1467                  * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
1468                  *              take the lock again.
1469                  */
1470                 spin_unlock_irq(shost->host_lock);
1471
1472                 /*
1473                  * Finally, initialize any error handling parameters, and set up
1474                  * the timers for timeouts.
1475                  */
1476                 scsi_init_cmd_errh(cmd);
1477
1478                 /*
1479                  * Dispatch the command to the low-level driver.
1480                  */
1481                 rtn = scsi_dispatch_cmd(cmd);
1482                 spin_lock_irq(q->queue_lock);
1483                 if(rtn) {
1484                         /* we're refusing the command; because of
1485                          * the way locks get dropped, we need to
1486                          * check here if plugging is required */
1487                         if(sdev->device_busy == 0)
1488                                 blk_plug_device(q);
1489
1490                         break;
1491                 }
1492         }
1493
1494         goto out;
1495
1496  not_ready:
1497         spin_unlock_irq(shost->host_lock);
1498
1499         /*
1500          * lock q, handle tag, requeue req, and decrement device_busy. We
1501          * must return with queue_lock held.
1502          *
1503          * Decrementing device_busy without checking it is OK, as all such
1504          * cases (host limits or settings) should run the queue at some
1505          * later time.
1506          */
1507         spin_lock_irq(q->queue_lock);
1508         blk_requeue_request(q, req);
1509         sdev->device_busy--;
1510         if(sdev->device_busy == 0)
1511                 blk_plug_device(q);
1512  out:
1513         /* must be careful here...if we trigger the ->remove() function
1514          * we cannot be holding the q lock */
1515         spin_unlock_irq(q->queue_lock);
1516         put_device(&sdev->sdev_gendev);
1517         spin_lock_irq(q->queue_lock);
1518 }
1519
1520 u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
1521 {
1522         struct device *host_dev;
1523         u64 bounce_limit = 0xffffffff;
1524
1525         if (shost->unchecked_isa_dma)
1526                 return BLK_BOUNCE_ISA;
1527         /*
1528          * Platforms with virtual-DMA translation
1529          * hardware have no practical limit.
1530          */
1531         if (!PCI_DMA_BUS_IS_PHYS)
1532                 return BLK_BOUNCE_ANY;
1533
1534         host_dev = scsi_get_device(shost);
1535         if (host_dev && host_dev->dma_mask)
1536                 bounce_limit = *host_dev->dma_mask;
1537
1538         return bounce_limit;
1539 }
1540 EXPORT_SYMBOL(scsi_calculate_bounce_limit);
1541
1542 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
1543 {
1544         struct Scsi_Host *shost = sdev->host;
1545         struct request_queue *q;
1546
1547         q = blk_init_queue(scsi_request_fn, NULL);
1548         if (!q)
1549                 return NULL;
1550
1551         blk_queue_prep_rq(q, scsi_prep_fn);
1552
1553         blk_queue_max_hw_segments(q, shost->sg_tablesize);
1554         blk_queue_max_phys_segments(q, SCSI_MAX_PHYS_SEGMENTS);
1555         blk_queue_max_sectors(q, shost->max_sectors);
1556         blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
1557         blk_queue_segment_boundary(q, shost->dma_boundary);
1558         blk_queue_issue_flush_fn(q, scsi_issue_flush_fn);
1559         blk_queue_softirq_done(q, scsi_softirq_done);
1560
1561         if (!shost->use_clustering)
1562                 clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
1563         return q;
1564 }
1565
1566 void scsi_free_queue(struct request_queue *q)
1567 {
1568         blk_cleanup_queue(q);
1569 }
1570
1571 /*
1572  * Function:    scsi_block_requests()
1573  *
1574  * Purpose:     Utility function used by low-level drivers to prevent further
1575  *              commands from being queued to the device.
1576  *
1577  * Arguments:   shost       - Host in question
1578  *
1579  * Returns:     Nothing
1580  *
1581  * Lock status: No locks are assumed held.
1582  *
1583  * Notes:       There is no timer nor any other means by which the requests
1584  *              get unblocked other than the low-level driver calling
1585  *              scsi_unblock_requests().
1586  */
1587 void scsi_block_requests(struct Scsi_Host *shost)
1588 {
1589         shost->host_self_blocked = 1;
1590 }
1591 EXPORT_SYMBOL(scsi_block_requests);
1592
1593 /*
1594  * Function:    scsi_unblock_requests()
1595  *
1596  * Purpose:     Utility function used by low-level drivers to allow further
1597  *              commands from being queued to the device.
1598  *
1599  * Arguments:   shost       - Host in question
1600  *
1601  * Returns:     Nothing
1602  *
1603  * Lock status: No locks are assumed held.
1604  *
1605  * Notes:       There is no timer nor any other means by which the requests
1606  *              get unblocked other than the low-level driver calling
1607  *              scsi_unblock_requests().
1608  *
1609  *              This is done as an API function so that changes to the
1610  *              internals of the scsi mid-layer won't require wholesale
1611  *              changes to drivers that use this feature.
1612  */
1613 void scsi_unblock_requests(struct Scsi_Host *shost)
1614 {
1615         shost->host_self_blocked = 0;
1616         scsi_run_host_queues(shost);
1617 }
1618 EXPORT_SYMBOL(scsi_unblock_requests);
1619
1620 int __init scsi_init_queue(void)
1621 {
1622         int i;
1623
1624         scsi_io_context_cache = kmem_cache_create("scsi_io_context",
1625                                         sizeof(struct scsi_io_context),
1626                                         0, 0, NULL, NULL);
1627         if (!scsi_io_context_cache) {
1628                 printk(KERN_ERR "SCSI: can't init scsi io context cache\n");
1629                 return -ENOMEM;
1630         }
1631
1632         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1633                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1634                 int size = sgp->size * sizeof(struct scatterlist);
1635
1636                 sgp->slab = kmem_cache_create(sgp->name, size, 0,
1637                                 SLAB_HWCACHE_ALIGN, NULL, NULL);
1638                 if (!sgp->slab) {
1639                         printk(KERN_ERR "SCSI: can't init sg slab %s\n",
1640                                         sgp->name);
1641                 }
1642
1643                 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
1644                                                      sgp->slab);
1645                 if (!sgp->pool) {
1646                         printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
1647                                         sgp->name);
1648                 }
1649         }
1650
1651         return 0;
1652 }
1653
1654 void scsi_exit_queue(void)
1655 {
1656         int i;
1657
1658         kmem_cache_destroy(scsi_io_context_cache);
1659
1660         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1661                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1662                 mempool_destroy(sgp->pool);
1663                 kmem_cache_destroy(sgp->slab);
1664         }
1665 }
1666
1667 /**
1668  *      scsi_mode_select - issue a mode select
1669  *      @sdev:  SCSI device to be queried
1670  *      @pf:    Page format bit (1 == standard, 0 == vendor specific)
1671  *      @sp:    Save page bit (0 == don't save, 1 == save)
1672  *      @modepage: mode page being requested
1673  *      @buffer: request buffer (may not be smaller than eight bytes)
1674  *      @len:   length of request buffer.
1675  *      @timeout: command timeout
1676  *      @retries: number of retries before failing
1677  *      @data: returns a structure abstracting the mode header data
1678  *      @sense: place to put sense data (or NULL if no sense to be collected).
1679  *              must be SCSI_SENSE_BUFFERSIZE big.
1680  *
1681  *      Returns zero if successful; negative error number or scsi
1682  *      status on error
1683  *
1684  */
1685 int
1686 scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
1687                  unsigned char *buffer, int len, int timeout, int retries,
1688                  struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
1689 {
1690         unsigned char cmd[10];
1691         unsigned char *real_buffer;
1692         int ret;
1693
1694         memset(cmd, 0, sizeof(cmd));
1695         cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);
1696
1697         if (sdev->use_10_for_ms) {
1698                 if (len > 65535)
1699                         return -EINVAL;
1700                 real_buffer = kmalloc(8 + len, GFP_KERNEL);
1701                 if (!real_buffer)
1702                         return -ENOMEM;
1703                 memcpy(real_buffer + 8, buffer, len);
1704                 len += 8;
1705                 real_buffer[0] = 0;
1706                 real_buffer[1] = 0;
1707                 real_buffer[2] = data->medium_type;
1708                 real_buffer[3] = data->device_specific;
1709                 real_buffer[4] = data->longlba ? 0x01 : 0;
1710                 real_buffer[5] = 0;
1711                 real_buffer[6] = data->block_descriptor_length >> 8;
1712                 real_buffer[7] = data->block_descriptor_length;
1713
1714                 cmd[0] = MODE_SELECT_10;
1715                 cmd[7] = len >> 8;
1716                 cmd[8] = len;
1717         } else {
1718                 if (len > 255 || data->block_descriptor_length > 255 ||
1719                     data->longlba)
1720                         return -EINVAL;
1721
1722                 real_buffer = kmalloc(4 + len, GFP_KERNEL);
1723                 if (!real_buffer)
1724                         return -ENOMEM;
1725                 memcpy(real_buffer + 4, buffer, len);
1726                 len += 4;
1727                 real_buffer[0] = 0;
1728                 real_buffer[1] = data->medium_type;
1729                 real_buffer[2] = data->device_specific;
1730                 real_buffer[3] = data->block_descriptor_length;
1731
1732
1733                 cmd[0] = MODE_SELECT;
1734                 cmd[4] = len;
1735         }
1736
1737         ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
1738                                sshdr, timeout, retries);
1739         kfree(real_buffer);
1740         return ret;
1741 }
1742 EXPORT_SYMBOL_GPL(scsi_mode_select);
1743
1744 /**
1745  *      scsi_mode_sense - issue a mode sense, falling back from 10 to
1746  *              six bytes if necessary.
1747  *      @sdev:  SCSI device to be queried
1748  *      @dbd:   set if mode sense will allow block descriptors to be returned
1749  *      @modepage: mode page being requested
1750  *      @buffer: request buffer (may not be smaller than eight bytes)
1751  *      @len:   length of request buffer.
1752  *      @timeout: command timeout
1753  *      @retries: number of retries before failing
1754  *      @data: returns a structure abstracting the mode header data
1755  *      @sense: place to put sense data (or NULL if no sense to be collected).
1756  *              must be SCSI_SENSE_BUFFERSIZE big.
1757  *
1758  *      Returns zero if unsuccessful, or the header offset (either 4
1759  *      or 8 depending on whether a six or ten byte command was
1760  *      issued) if successful.
1761  **/
1762 int
1763 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
1764                   unsigned char *buffer, int len, int timeout, int retries,
1765                   struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
1766 {
1767         unsigned char cmd[12];
1768         int use_10_for_ms;
1769         int header_length;
1770         int result;
1771         struct scsi_sense_hdr my_sshdr;
1772
1773         memset(data, 0, sizeof(*data));
1774         memset(&cmd[0], 0, 12);
1775         cmd[1] = dbd & 0x18;    /* allows DBD and LLBA bits */
1776         cmd[2] = modepage;
1777
1778         /* caller might not be interested in sense, but we need it */
1779         if (!sshdr)
1780                 sshdr = &my_sshdr;
1781
1782  retry:
1783         use_10_for_ms = sdev->use_10_for_ms;
1784
1785         if (use_10_for_ms) {
1786                 if (len < 8)
1787                         len = 8;
1788
1789                 cmd[0] = MODE_SENSE_10;
1790                 cmd[8] = len;
1791                 header_length = 8;
1792         } else {
1793                 if (len < 4)
1794                         len = 4;
1795
1796                 cmd[0] = MODE_SENSE;
1797                 cmd[4] = len;
1798                 header_length = 4;
1799         }
1800
1801         memset(buffer, 0, len);
1802
1803         result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
1804                                   sshdr, timeout, retries);
1805
1806         /* This code looks awful: what it's doing is making sure an
1807          * ILLEGAL REQUEST sense return identifies the actual command
1808          * byte as the problem.  MODE_SENSE commands can return
1809          * ILLEGAL REQUEST if the code page isn't supported */
1810
1811         if (use_10_for_ms && !scsi_status_is_good(result) &&
1812             (driver_byte(result) & DRIVER_SENSE)) {
1813                 if (scsi_sense_valid(sshdr)) {
1814                         if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
1815                             (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
1816                                 /*
1817                                  * Invalid command operation code
1818                                  */
1819                                 sdev->use_10_for_ms = 0;
1820                                 goto retry;
1821                         }
1822                 }
1823         }
1824
1825         if(scsi_status_is_good(result)) {
1826                 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
1827                              (modepage == 6 || modepage == 8))) {
1828                         /* Initio breakage? */
1829                         header_length = 0;
1830                         data->length = 13;
1831                         data->medium_type = 0;
1832                         data->device_specific = 0;
1833                         data->longlba = 0;
1834                         data->block_descriptor_length = 0;
1835                 } else if(use_10_for_ms) {
1836                         data->length = buffer[0]*256 + buffer[1] + 2;
1837                         data->medium_type = buffer[2];
1838                         data->device_specific = buffer[3];
1839                         data->longlba = buffer[4] & 0x01;
1840                         data->block_descriptor_length = buffer[6]*256
1841                                 + buffer[7];
1842                 } else {
1843                         data->length = buffer[0] + 1;
1844                         data->medium_type = buffer[1];
1845                         data->device_specific = buffer[2];
1846                         data->block_descriptor_length = buffer[3];
1847                 }
1848                 data->header_length = header_length;
1849         }
1850
1851         return result;
1852 }
1853 EXPORT_SYMBOL(scsi_mode_sense);
1854
1855 int
1856 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries)
1857 {
1858         char cmd[] = {
1859                 TEST_UNIT_READY, 0, 0, 0, 0, 0,
1860         };
1861         struct scsi_sense_hdr sshdr;
1862         int result;
1863
1864         result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, &sshdr,
1865                                   timeout, retries);
1866
1867         if ((driver_byte(result) & DRIVER_SENSE) && sdev->removable) {
1868
1869                 if ((scsi_sense_valid(&sshdr)) &&
1870                     ((sshdr.sense_key == UNIT_ATTENTION) ||
1871                      (sshdr.sense_key == NOT_READY))) {
1872                         sdev->changed = 1;
1873                         result = 0;
1874                 }
1875         }
1876         return result;
1877 }
1878 EXPORT_SYMBOL(scsi_test_unit_ready);
1879
1880 /**
1881  *      scsi_device_set_state - Take the given device through the device
1882  *              state model.
1883  *      @sdev:  scsi device to change the state of.
1884  *      @state: state to change to.
1885  *
1886  *      Returns zero if unsuccessful or an error if the requested
1887  *      transition is illegal.
1888  **/
1889 int
1890 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
1891 {
1892         enum scsi_device_state oldstate = sdev->sdev_state;
1893
1894         if (state == oldstate)
1895                 return 0;
1896
1897         switch (state) {
1898         case SDEV_CREATED:
1899                 /* There are no legal states that come back to
1900                  * created.  This is the manually initialised start
1901                  * state */
1902                 goto illegal;
1903
1904         case SDEV_RUNNING:
1905                 switch (oldstate) {
1906                 case SDEV_CREATED:
1907                 case SDEV_OFFLINE:
1908                 case SDEV_QUIESCE:
1909                 case SDEV_BLOCK:
1910                         break;
1911                 default:
1912                         goto illegal;
1913                 }
1914                 break;
1915
1916         case SDEV_QUIESCE:
1917                 switch (oldstate) {
1918                 case SDEV_RUNNING:
1919                 case SDEV_OFFLINE:
1920                         break;
1921                 default:
1922                         goto illegal;
1923                 }
1924                 break;
1925
1926         case SDEV_OFFLINE:
1927                 switch (oldstate) {
1928                 case SDEV_CREATED:
1929                 case SDEV_RUNNING:
1930                 case SDEV_QUIESCE:
1931                 case SDEV_BLOCK:
1932                         break;
1933                 default:
1934                         goto illegal;
1935                 }
1936                 break;
1937
1938         case SDEV_BLOCK:
1939                 switch (oldstate) {
1940                 case SDEV_CREATED:
1941                 case SDEV_RUNNING:
1942                         break;
1943                 default:
1944                         goto illegal;
1945                 }
1946                 break;
1947
1948         case SDEV_CANCEL:
1949                 switch (oldstate) {
1950                 case SDEV_CREATED:
1951                 case SDEV_RUNNING:
1952                 case SDEV_QUIESCE:
1953                 case SDEV_OFFLINE:
1954                 case SDEV_BLOCK:
1955                         break;
1956                 default:
1957                         goto illegal;
1958                 }
1959                 break;
1960
1961         case SDEV_DEL:
1962                 switch (oldstate) {
1963                 case SDEV_CREATED:
1964                 case SDEV_RUNNING:
1965                 case SDEV_OFFLINE:
1966                 case SDEV_CANCEL:
1967                         break;
1968                 default:
1969                         goto illegal;
1970                 }
1971                 break;
1972
1973         }
1974         sdev->sdev_state = state;
1975         return 0;
1976
1977  illegal:
1978         SCSI_LOG_ERROR_RECOVERY(1,
1979                                 sdev_printk(KERN_ERR, sdev,
1980                                             "Illegal state transition %s->%s\n",
1981                                             scsi_device_state_name(oldstate),
1982                                             scsi_device_state_name(state))
1983                                 );
1984         return -EINVAL;
1985 }
1986 EXPORT_SYMBOL(scsi_device_set_state);
1987
1988 /**
1989  *      scsi_device_quiesce - Block user issued commands.
1990  *      @sdev:  scsi device to quiesce.
1991  *
1992  *      This works by trying to transition to the SDEV_QUIESCE state
1993  *      (which must be a legal transition).  When the device is in this
1994  *      state, only special requests will be accepted, all others will
1995  *      be deferred.  Since special requests may also be requeued requests,
1996  *      a successful return doesn't guarantee the device will be
1997  *      totally quiescent.
1998  *
1999  *      Must be called with user context, may sleep.
2000  *
2001  *      Returns zero if unsuccessful or an error if not.
2002  **/
2003 int
2004 scsi_device_quiesce(struct scsi_device *sdev)
2005 {
2006         int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
2007         if (err)
2008                 return err;
2009
2010         scsi_run_queue(sdev->request_queue);
2011         while (sdev->device_busy) {
2012                 msleep_interruptible(200);
2013                 scsi_run_queue(sdev->request_queue);
2014         }
2015         return 0;
2016 }
2017 EXPORT_SYMBOL(scsi_device_quiesce);
2018
2019 /**
2020  *      scsi_device_resume - Restart user issued commands to a quiesced device.
2021  *      @sdev:  scsi device to resume.
2022  *
2023  *      Moves the device from quiesced back to running and restarts the
2024  *      queues.
2025  *
2026  *      Must be called with user context, may sleep.
2027  **/
2028 void
2029 scsi_device_resume(struct scsi_device *sdev)
2030 {
2031         if(scsi_device_set_state(sdev, SDEV_RUNNING))
2032                 return;
2033         scsi_run_queue(sdev->request_queue);
2034 }
2035 EXPORT_SYMBOL(scsi_device_resume);
2036
2037 static void
2038 device_quiesce_fn(struct scsi_device *sdev, void *data)
2039 {
2040         scsi_device_quiesce(sdev);
2041 }
2042
2043 void
2044 scsi_target_quiesce(struct scsi_target *starget)
2045 {
2046         starget_for_each_device(starget, NULL, device_quiesce_fn);
2047 }
2048 EXPORT_SYMBOL(scsi_target_quiesce);
2049
2050 static void
2051 device_resume_fn(struct scsi_device *sdev, void *data)
2052 {
2053         scsi_device_resume(sdev);
2054 }
2055
2056 void
2057 scsi_target_resume(struct scsi_target *starget)
2058 {
2059         starget_for_each_device(starget, NULL, device_resume_fn);
2060 }
2061 EXPORT_SYMBOL(scsi_target_resume);
2062
2063 /**
2064  * scsi_internal_device_block - internal function to put a device
2065  *                              temporarily into the SDEV_BLOCK state
2066  * @sdev:       device to block
2067  *
2068  * Block request made by scsi lld's to temporarily stop all
2069  * scsi commands on the specified device.  Called from interrupt
2070  * or normal process context.
2071  *
2072  * Returns zero if successful or error if not
2073  *
2074  * Notes:
2075  *      This routine transitions the device to the SDEV_BLOCK state
2076  *      (which must be a legal transition).  When the device is in this
2077  *      state, all commands are deferred until the scsi lld reenables
2078  *      the device with scsi_device_unblock or device_block_tmo fires.
2079  *      This routine assumes the host_lock is held on entry.
2080  **/
2081 int
2082 scsi_internal_device_block(struct scsi_device *sdev)
2083 {
2084         request_queue_t *q = sdev->request_queue;
2085         unsigned long flags;
2086         int err = 0;
2087
2088         err = scsi_device_set_state(sdev, SDEV_BLOCK);
2089         if (err)
2090                 return err;
2091
2092         /*
2093          * The device has transitioned to SDEV_BLOCK.  Stop the
2094          * block layer from calling the midlayer with this device's
2095          * request queue.
2096          */
2097         spin_lock_irqsave(q->queue_lock, flags);
2098         blk_stop_queue(q);
2099         spin_unlock_irqrestore(q->queue_lock, flags);
2100
2101         return 0;
2102 }
2103 EXPORT_SYMBOL_GPL(scsi_internal_device_block);
2104
2105 /**
2106  * scsi_internal_device_unblock - resume a device after a block request
2107  * @sdev:       device to resume
2108  *
2109  * Called by scsi lld's or the midlayer to restart the device queue
2110  * for the previously suspended scsi device.  Called from interrupt or
2111  * normal process context.
2112  *
2113  * Returns zero if successful or error if not.
2114  *
2115  * Notes:
2116  *      This routine transitions the device to the SDEV_RUNNING state
2117  *      (which must be a legal transition) allowing the midlayer to
2118  *      goose the queue for this device.  This routine assumes the
2119  *      host_lock is held upon entry.
2120  **/
2121 int
2122 scsi_internal_device_unblock(struct scsi_device *sdev)
2123 {
2124         request_queue_t *q = sdev->request_queue;
2125         int err;
2126         unsigned long flags;
2127
2128         /*
2129          * Try to transition the scsi device to SDEV_RUNNING
2130          * and goose the device queue if successful.
2131          */
2132         err = scsi_device_set_state(sdev, SDEV_RUNNING);
2133         if (err)
2134                 return err;
2135
2136         spin_lock_irqsave(q->queue_lock, flags);
2137         blk_start_queue(q);
2138         spin_unlock_irqrestore(q->queue_lock, flags);
2139
2140         return 0;
2141 }
2142 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
2143
2144 static void
2145 device_block(struct scsi_device *sdev, void *data)
2146 {
2147         scsi_internal_device_block(sdev);
2148 }
2149
2150 static int
2151 target_block(struct device *dev, void *data)
2152 {
2153         if (scsi_is_target_device(dev))
2154                 starget_for_each_device(to_scsi_target(dev), NULL,
2155                                         device_block);
2156         return 0;
2157 }
2158
2159 void
2160 scsi_target_block(struct device *dev)
2161 {
2162         if (scsi_is_target_device(dev))
2163                 starget_for_each_device(to_scsi_target(dev), NULL,
2164                                         device_block);
2165         else
2166                 device_for_each_child(dev, NULL, target_block);
2167 }
2168 EXPORT_SYMBOL_GPL(scsi_target_block);
2169
2170 static void
2171 device_unblock(struct scsi_device *sdev, void *data)
2172 {
2173         scsi_internal_device_unblock(sdev);
2174 }
2175
2176 static int
2177 target_unblock(struct device *dev, void *data)
2178 {
2179         if (scsi_is_target_device(dev))
2180                 starget_for_each_device(to_scsi_target(dev), NULL,
2181                                         device_unblock);
2182         return 0;
2183 }
2184
2185 void
2186 scsi_target_unblock(struct device *dev)
2187 {
2188         if (scsi_is_target_device(dev))
2189                 starget_for_each_device(to_scsi_target(dev), NULL,
2190                                         device_unblock);
2191         else
2192                 device_for_each_child(dev, NULL, target_unblock);
2193 }
2194 EXPORT_SYMBOL_GPL(scsi_target_unblock);
2195
2196 /**
2197  * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
2198  * @sg:         scatter-gather list
2199  * @sg_count:   number of segments in sg
2200  * @offset:     offset in bytes into sg, on return offset into the mapped area
2201  * @len:        bytes to map, on return number of bytes mapped
2202  *
2203  * Returns virtual address of the start of the mapped page
2204  */
2205 void *scsi_kmap_atomic_sg(struct scatterlist *sg, int sg_count,
2206                           size_t *offset, size_t *len)
2207 {
2208         int i;
2209         size_t sg_len = 0, len_complete = 0;
2210         struct page *page;
2211
2212         for (i = 0; i < sg_count; i++) {
2213                 len_complete = sg_len; /* Complete sg-entries */
2214                 sg_len += sg[i].length;
2215                 if (sg_len > *offset)
2216                         break;
2217         }
2218
2219         if (unlikely(i == sg_count)) {
2220                 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
2221                         "elements %d\n",
2222                        __FUNCTION__, sg_len, *offset, sg_count);
2223                 WARN_ON(1);
2224                 return NULL;
2225         }
2226
2227         /* Offset starting from the beginning of first page in this sg-entry */
2228         *offset = *offset - len_complete + sg[i].offset;
2229
2230         /* Assumption: contiguous pages can be accessed as "page + i" */
2231         page = nth_page(sg[i].page, (*offset >> PAGE_SHIFT));
2232         *offset &= ~PAGE_MASK;
2233
2234         /* Bytes in this sg-entry from *offset to the end of the page */
2235         sg_len = PAGE_SIZE - *offset;
2236         if (*len > sg_len)
2237                 *len = sg_len;
2238
2239         return kmap_atomic(page, KM_BIO_SRC_IRQ);
2240 }
2241 EXPORT_SYMBOL(scsi_kmap_atomic_sg);
2242
2243 /**
2244  * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously
2245  *                         mapped with scsi_kmap_atomic_sg
2246  * @virt:       virtual address to be unmapped
2247  */
2248 void scsi_kunmap_atomic_sg(void *virt)
2249 {
2250         kunmap_atomic(virt, KM_BIO_SRC_IRQ);
2251 }
2252 EXPORT_SYMBOL(scsi_kunmap_atomic_sg);