drivers/ide/ide-io.c

   1 /*
   2  *      IDE I/O functions
   3  *
   4  *      Basic PIO and command management functionality.
   5  *
   6  * This code was split off from ide.c. See ide.c for history and original
   7  * copyrights.
   8  *
   9  * This program is free software; you can redistribute it and/or modify it
  10  * under the terms of the GNU General Public License as published by the
  11  * Free Software Foundation; either version 2, or (at your option) any
  12  * later version.
  13  *
  14  * This program is distributed in the hope that it will be useful, but
  15  * WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * General Public License for more details.
  18  *
  19  * For the avoidance of doubt the "preferred form" of this code is one which
  20  * is in an open non patent encumbered format. Where cryptographic key signing
  21  * forms part of the process of creating an executable the information
  22  * including keys needed to generate an equivalently functional executable
  23  * are deemed to be part of the source code.
  24  */
  25
  26
  27 #include <linux/config.h>
  28 #include <linux/module.h>
  29 #include <linux/types.h>
  30 #include <linux/string.h>
  31 #include <linux/kernel.h>
  32 #include <linux/timer.h>
  33 #include <linux/mm.h>
  34 #include <linux/interrupt.h>
  35 #include <linux/major.h>
  36 #include <linux/errno.h>
  37 #include <linux/genhd.h>
  38 #include <linux/blkpg.h>
  39 #include <linux/slab.h>
  40 #include <linux/init.h>
  41 #include <linux/pci.h>
  42 #include <linux/delay.h>
  43 #include <linux/ide.h>
  44 #include <linux/completion.h>
  45 #include <linux/reboot.h>
  46 #include <linux/cdrom.h>
  47 #include <linux/seq_file.h>
  48 #include <linux/device.h>
  49 #include <linux/kmod.h>
  50 #include <linux/scatterlist.h>
  51
  52 #include <asm/byteorder.h>
  53 #include <asm/irq.h>
  54 #include <asm/uaccess.h>
  55 #include <asm/io.h>
  56 #include <asm/bitops.h>
  57
  58 int __ide_end_request(ide_drive_t *drive, struct request *rq, int uptodate,
  59                       int nr_sectors)
  60 {
  61         int ret = 1;
  62
  63         BUG_ON(!(rq->flags & REQ_STARTED));
  64
  65         /*
  66          * if failfast is set on a request, override number of sectors and
  67          * complete the whole request right now
  68          */
  69         if (blk_noretry_request(rq) && end_io_error(uptodate))
  70                 nr_sectors = rq->hard_nr_sectors;
  71
  72         if (!blk_fs_request(rq) && end_io_error(uptodate) && !rq->errors)
  73                 rq->errors = -EIO;
  74
  75         /*
  76          * decide whether to reenable DMA -- 3 is a random magic for now,
  77          * if we DMA timeout more than 3 times, just stay in PIO
  78          */
  79         if (drive->state == DMA_PIO_RETRY && drive->retry_pio <= 3) {
  80                 drive->state = 0;
  81                 HWGROUP(drive)->hwif->ide_dma_on(drive);
  82         }
  83
  84         if (!end_that_request_first(rq, uptodate, nr_sectors)) {
  85                 add_disk_randomness(rq->rq_disk);
  86
  87                 if (blk_rq_tagged(rq))
  88                         blk_queue_end_tag(drive->queue, rq);
  89
  90                 blkdev_dequeue_request(rq);
  91                 HWGROUP(drive)->rq = NULL;
  92                 end_that_request_last(rq);
  93                 ret = 0;
  94         }
  95         return ret;
  96 }
  97 EXPORT_SYMBOL(__ide_end_request);
  98
  99 /**
 100  *      ide_end_request         -       complete an IDE I/O
 101  *      @drive: IDE device for the I/O
 102  *      @uptodate:
 103  *      @nr_sectors: number of sectors completed
 104  *
 105  *      This is our end_request wrapper function. We complete the I/O
 106  *      update random number input and dequeue the request, which if
 107  *      it was tagged may be out of order.
 108  */
 109
 110 int ide_end_request (ide_drive_t *drive, int uptodate, int nr_sectors)
 111 {
 112         struct request *rq;
 113         unsigned long flags;
 114         int ret = 1;
 115
 116         spin_lock_irqsave(&ide_lock, flags);
 117         rq = HWGROUP(drive)->rq;
 118
 119         if (!nr_sectors)
 120                 nr_sectors = rq->hard_cur_sectors;
 121
 122         if (blk_complete_barrier_rq_locked(drive->queue, rq, nr_sectors))
 123                 ret = rq->nr_sectors != 0;
 124         else
 125                 ret = __ide_end_request(drive, rq, uptodate, nr_sectors);
 126
 127         spin_unlock_irqrestore(&ide_lock, flags);
 128         return ret;
 129 }
 130 EXPORT_SYMBOL(ide_end_request);
 131
 132 /*
 133  * Power Management state machine. This one is rather trivial for now,
 134  * we should probably add more, like switching back to PIO on suspend
 135  * to help some BIOSes, re-do the door locking on resume, etc...
 136  */
 137
 138 enum {
 139         ide_pm_flush_cache      = ide_pm_state_start_suspend,
 140         idedisk_pm_standby,
 141
 142         idedisk_pm_idle         = ide_pm_state_start_resume,
 143         ide_pm_restore_dma,
 144 };
 145
 146 static void ide_complete_power_step(ide_drive_t *drive, struct request *rq, u8 stat, u8 error)
 147 {
 148         if (drive->media != ide_disk)
 149                 return;
 150
 151         switch (rq->pm->pm_step) {
 152         case ide_pm_flush_cache:        /* Suspend step 1 (flush cache) complete */
 153                 if (rq->pm->pm_state == 4)
 154                         rq->pm->pm_step = ide_pm_state_completed;
 155                 else
 156                         rq->pm->pm_step = idedisk_pm_standby;
 157                 break;
 158         case idedisk_pm_standby:        /* Suspend step 2 (standby) complete */
 159                 rq->pm->pm_step = ide_pm_state_completed;
 160                 break;
 161         case idedisk_pm_idle:           /* Resume step 1 (idle) complete */
 162                 rq->pm->pm_step = ide_pm_restore_dma;
 163                 break;
 164         }
 165 }
 166
 167 static ide_startstop_t ide_start_power_step(ide_drive_t *drive, struct request *rq)
 168 {
 169         ide_task_t *args = rq->special;
 170
 171         memset(args, 0, sizeof(*args));
 172
 173         if (drive->media != ide_disk) {
 174                 /* skip idedisk_pm_idle for ATAPI devices */
 175                 if (rq->pm->pm_step == idedisk_pm_idle)
 176                         rq->pm->pm_step = ide_pm_restore_dma;
 177         }
 178
 179         switch (rq->pm->pm_step) {
 180         case ide_pm_flush_cache:        /* Suspend step 1 (flush cache) */
 181                 if (drive->media != ide_disk)
 182                         break;
 183                 /* Not supported? Switch to next step now. */
 184                 if (!drive->wcache || !ide_id_has_flush_cache(drive->id)) {
 185                         ide_complete_power_step(drive, rq, 0, 0);
 186                         return ide_stopped;
 187                 }
 188                 if (ide_id_has_flush_cache_ext(drive->id))
 189                         args->tfRegister[IDE_COMMAND_OFFSET] = WIN_FLUSH_CACHE_EXT;
 190                 else
 191                         args->tfRegister[IDE_COMMAND_OFFSET] = WIN_FLUSH_CACHE;
 192                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
 193                 args->handler      = &task_no_data_intr;
 194                 return do_rw_taskfile(drive, args);
 195
 196         case idedisk_pm_standby:        /* Suspend step 2 (standby) */
 197                 args->tfRegister[IDE_COMMAND_OFFSET] = WIN_STANDBYNOW1;
 198                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
 199                 args->handler      = &task_no_data_intr;
 200                 return do_rw_taskfile(drive, args);
 201
 202         case idedisk_pm_idle:           /* Resume step 1 (idle) */
 203                 args->tfRegister[IDE_COMMAND_OFFSET] = WIN_IDLEIMMEDIATE;
 204                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
 205                 args->handler = task_no_data_intr;
 206                 return do_rw_taskfile(drive, args);
 207
 208         case ide_pm_restore_dma:        /* Resume step 2 (restore DMA) */
 209                 /*
 210                  * Right now, all we do is call hwif->ide_dma_check(drive),
 211                  * we could be smarter and check for current xfer_speed
 212                  * in struct drive etc...
 213                  */
 214                 if ((drive->id->capability & 1) == 0)
 215                         break;
 216                 if (drive->hwif->ide_dma_check == NULL)
 217                         break;
 218                 drive->hwif->ide_dma_check(drive);
 219                 break;
 220         }
 221         rq->pm->pm_step = ide_pm_state_completed;
 222         return ide_stopped;
 223 }
 224
 225 /**
 226  *      ide_end_dequeued_request        -       complete an IDE I/O
 227  *      @drive: IDE device for the I/O
 228  *      @uptodate:
 229  *      @nr_sectors: number of sectors completed
 230  *
 231  *      Complete an I/O that is no longer on the request queue. This
 232  *      typically occurs when we pull the request and issue a REQUEST_SENSE.
 233  *      We must still finish the old request but we must not tamper with the
 234  *      queue in the meantime.
 235  *
 236  *      NOTE: This path does not handle barrier, but barrier is not supported
 237  *      on ide-cd anyway.
 238  */
 239
 240 int ide_end_dequeued_request(ide_drive_t *drive, struct request *rq,
 241                              int uptodate, int nr_sectors)
 242 {
 243         unsigned long flags;
 244         int ret = 1;
 245
 246         spin_lock_irqsave(&ide_lock, flags);
 247
 248         BUG_ON(!(rq->flags & REQ_STARTED));
 249
 250         /*
 251          * if failfast is set on a request, override number of sectors and
 252          * complete the whole request right now
 253          */
 254         if (blk_noretry_request(rq) && end_io_error(uptodate))
 255                 nr_sectors = rq->hard_nr_sectors;
 256
 257         if (!blk_fs_request(rq) && end_io_error(uptodate) && !rq->errors)
 258                 rq->errors = -EIO;
 259
 260         /*
 261          * decide whether to reenable DMA -- 3 is a random magic for now,
 262          * if we DMA timeout more than 3 times, just stay in PIO
 263          */
 264         if (drive->state == DMA_PIO_RETRY && drive->retry_pio <= 3) {
 265                 drive->state = 0;
 266                 HWGROUP(drive)->hwif->ide_dma_on(drive);
 267         }
 268
 269         if (!end_that_request_first(rq, uptodate, nr_sectors)) {
 270                 add_disk_randomness(rq->rq_disk);
 271                 if (blk_rq_tagged(rq))
 272                         blk_queue_end_tag(drive->queue, rq);
 273                 end_that_request_last(rq);
 274                 ret = 0;
 275         }
 276         spin_unlock_irqrestore(&ide_lock, flags);
 277         return ret;
 278 }
 279 EXPORT_SYMBOL_GPL(ide_end_dequeued_request);
 280
 281
 282 /**
 283  *      ide_complete_pm_request - end the current Power Management request
 284  *      @drive: target drive
 285  *      @rq: request
 286  *
 287  *      This function cleans up the current PM request and stops the queue
 288  *      if necessary.
 289  */
 290 static void ide_complete_pm_request (ide_drive_t *drive, struct request *rq)
 291 {
 292         unsigned long flags;
 293
 294 #ifdef DEBUG_PM
 295         printk("%s: completing PM request, %s\n", drive->name,
 296                blk_pm_suspend_request(rq) ? "suspend" : "resume");
 297 #endif
 298         spin_lock_irqsave(&ide_lock, flags);
 299         if (blk_pm_suspend_request(rq)) {
 300                 blk_stop_queue(drive->queue);
 301         } else {
 302                 drive->blocked = 0;
 303                 blk_start_queue(drive->queue);
 304         }
 305         blkdev_dequeue_request(rq);
 306         HWGROUP(drive)->rq = NULL;
 307         end_that_request_last(rq);
 308         spin_unlock_irqrestore(&ide_lock, flags);
 309 }
 310
 311 /*
 312  * FIXME: probably move this somewhere else, name is bad too :)
 313  */
 314 u64 ide_get_error_location(ide_drive_t *drive, char *args)
 315 {
 316         u32 high, low;
 317         u8 hcyl, lcyl, sect;
 318         u64 sector;
 319
 320         high = 0;
 321         hcyl = args[5];
 322         lcyl = args[4];
 323         sect = args[3];
 324
 325         if (ide_id_has_flush_cache_ext(drive->id)) {
 326                 low = (hcyl << 16) | (lcyl << 8) | sect;
 327                 HWIF(drive)->OUTB(drive->ctl|0x80, IDE_CONTROL_REG);
 328                 high = ide_read_24(drive);
 329         } else {
 330                 u8 cur = HWIF(drive)->INB(IDE_SELECT_REG);
 331                 if (cur & 0x40) {
 332                         high = cur & 0xf;
 333                         low = (hcyl << 16) | (lcyl << 8) | sect;
 334                 } else {
 335                         low = hcyl * drive->head * drive->sect;
 336                         low += lcyl * drive->sect;
 337                         low += sect - 1;
 338                 }
 339         }
 340
 341         sector = ((u64) high << 24) | low;
 342         return sector;
 343 }
 344 EXPORT_SYMBOL(ide_get_error_location);
 345
 346 /**
 347  *      ide_end_drive_cmd       -       end an explicit drive command
 348  *      @drive: command
 349  *      @stat: status bits
 350  *      @err: error bits
 351  *
 352  *      Clean up after success/failure of an explicit drive command.
 353  *      These get thrown onto the queue so they are synchronized with
 354  *      real I/O operations on the drive.
 355  *
 356  *      In LBA48 mode we have to read the register set twice to get
 357  *      all the extra information out.
 358  */
 359
 360 void ide_end_drive_cmd (ide_drive_t *drive, u8 stat, u8 err)
 361 {
 362         ide_hwif_t *hwif = HWIF(drive);
 363         unsigned long flags;
 364         struct request *rq;
 365
 366         spin_lock_irqsave(&ide_lock, flags);
 367         rq = HWGROUP(drive)->rq;
 368         spin_unlock_irqrestore(&ide_lock, flags);
 369
 370         if (rq->flags & REQ_DRIVE_CMD) {
 371                 u8 *args = (u8 *) rq->buffer;
 372                 if (rq->errors == 0)
 373                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
 374
 375                 if (args) {
 376                         args[0] = stat;
 377                         args[1] = err;
 378                         args[2] = hwif->INB(IDE_NSECTOR_REG);
 379                 }
 380         } else if (rq->flags & REQ_DRIVE_TASK) {
 381                 u8 *args = (u8 *) rq->buffer;
 382                 if (rq->errors == 0)
 383                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
 384
 385                 if (args) {
 386                         args[0] = stat;
 387                         args[1] = err;
 388                         args[2] = hwif->INB(IDE_NSECTOR_REG);
 389                         args[3] = hwif->INB(IDE_SECTOR_REG);
 390                         args[4] = hwif->INB(IDE_LCYL_REG);
 391                         args[5] = hwif->INB(IDE_HCYL_REG);
 392                         args[6] = hwif->INB(IDE_SELECT_REG);
 393                 }
 394         } else if (rq->flags & REQ_DRIVE_TASKFILE) {
 395                 ide_task_t *args = (ide_task_t *) rq->special;
 396                 if (rq->errors == 0)
 397                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
 398
 399                 if (args) {
 400                         if (args->tf_in_flags.b.data) {
 401                                 u16 data                                = hwif->INW(IDE_DATA_REG);
 402                                 args->tfRegister[IDE_DATA_OFFSET]       = (data) & 0xFF;
 403                                 args->hobRegister[IDE_DATA_OFFSET]      = (data >> 8) & 0xFF;
 404                         }
 405                         args->tfRegister[IDE_ERROR_OFFSET]   = err;
 406                         /* be sure we're looking at the low order bits */
 407                         hwif->OUTB(drive->ctl & ~0x80, IDE_CONTROL_REG);
 408                         args->tfRegister[IDE_NSECTOR_OFFSET] = hwif->INB(IDE_NSECTOR_REG);
 409                         args->tfRegister[IDE_SECTOR_OFFSET]  = hwif->INB(IDE_SECTOR_REG);
 410                         args->tfRegister[IDE_LCYL_OFFSET]    = hwif->INB(IDE_LCYL_REG);
 411                         args->tfRegister[IDE_HCYL_OFFSET]    = hwif->INB(IDE_HCYL_REG);
 412                         args->tfRegister[IDE_SELECT_OFFSET]  = hwif->INB(IDE_SELECT_REG);
 413                         args->tfRegister[IDE_STATUS_OFFSET]  = stat;
 414
 415                         if (drive->addressing == 1) {
 416                                 hwif->OUTB(drive->ctl|0x80, IDE_CONTROL_REG);
 417                                 args->hobRegister[IDE_FEATURE_OFFSET]   = hwif->INB(IDE_FEATURE_REG);
 418                                 args->hobRegister[IDE_NSECTOR_OFFSET]   = hwif->INB(IDE_NSECTOR_REG);
 419                                 args->hobRegister[IDE_SECTOR_OFFSET]    = hwif->INB(IDE_SECTOR_REG);
 420                                 args->hobRegister[IDE_LCYL_OFFSET]      = hwif->INB(IDE_LCYL_REG);
 421                                 args->hobRegister[IDE_HCYL_OFFSET]      = hwif->INB(IDE_HCYL_REG);
 422                         }
 423                 }
 424         } else if (blk_pm_request(rq)) {
 425 #ifdef DEBUG_PM
 426                 printk("%s: complete_power_step(step: %d, stat: %x, err: %x)\n",
 427                         drive->name, rq->pm->pm_step, stat, err);
 428 #endif
 429                 ide_complete_power_step(drive, rq, stat, err);
 430                 if (rq->pm->pm_step == ide_pm_state_completed)
 431                         ide_complete_pm_request(drive, rq);
 432                 return;
 433         }
 434
 435         spin_lock_irqsave(&ide_lock, flags);
 436         blkdev_dequeue_request(rq);
 437         HWGROUP(drive)->rq = NULL;
 438         rq->errors = err;
 439         end_that_request_last(rq);
 440         spin_unlock_irqrestore(&ide_lock, flags);
 441 }
 442
 443 EXPORT_SYMBOL(ide_end_drive_cmd);
 444
 445 /**
 446  *      try_to_flush_leftover_data      -       flush junk
 447  *      @drive: drive to flush
 448  *
 449  *      try_to_flush_leftover_data() is invoked in response to a drive
 450  *      unexpectedly having its DRQ_STAT bit set.  As an alternative to
 451  *      resetting the drive, this routine tries to clear the condition
 452  *      by read a sector's worth of data from the drive.  Of course,
 453  *      this may not help if the drive is *waiting* for data from *us*.
 454  */
 455 static void try_to_flush_leftover_data (ide_drive_t *drive)
 456 {
 457         int i = (drive->mult_count ? drive->mult_count : 1) * SECTOR_WORDS;
 458
 459         if (drive->media != ide_disk)
 460                 return;
 461         while (i > 0) {
 462                 u32 buffer[16];
 463                 u32 wcount = (i > 16) ? 16 : i;
 464
 465                 i -= wcount;
 466                 HWIF(drive)->ata_input_data(drive, buffer, wcount);
 467         }
 468 }
 469
 470 static void ide_kill_rq(ide_drive_t *drive, struct request *rq)
 471 {
 472         if (rq->rq_disk) {
 473                 ide_driver_t *drv;
 474
 475                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
 476                 drv->end_request(drive, 0, 0);
 477         } else
 478                 ide_end_request(drive, 0, 0);
 479 }
 480
 481 static ide_startstop_t ide_ata_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
 482 {
 483         ide_hwif_t *hwif = drive->hwif;
 484
 485         if (stat & BUSY_STAT || ((stat & WRERR_STAT) && !drive->nowerr)) {
 486                 /* other bits are useless when BUSY */
 487                 rq->errors |= ERROR_RESET;
 488         } else if (stat & ERR_STAT) {
 489                 /* err has different meaning on cdrom and tape */
 490                 if (err == ABRT_ERR) {
 491                         if (drive->select.b.lba &&
 492                             /* some newer drives don't support WIN_SPECIFY */
 493                             hwif->INB(IDE_COMMAND_REG) == WIN_SPECIFY)
 494                                 return ide_stopped;
 495                 } else if ((err & BAD_CRC) == BAD_CRC) {
 496                         /* UDMA crc error, just retry the operation */
 497                         drive->crc_count++;
 498                 } else if (err & (BBD_ERR | ECC_ERR)) {
 499                         /* retries won't help these */
 500                         rq->errors = ERROR_MAX;
 501                 } else if (err & TRK0_ERR) {
 502                         /* help it find track zero */
 503                         rq->errors |= ERROR_RECAL;
 504                 }
 505         }
 506
 507         if ((stat & DRQ_STAT) && rq_data_dir(rq) == READ)
 508                 try_to_flush_leftover_data(drive);
 509
 510         if (hwif->INB(IDE_STATUS_REG) & (BUSY_STAT|DRQ_STAT))
 511                 /* force an abort */
 512                 hwif->OUTB(WIN_IDLEIMMEDIATE, IDE_COMMAND_REG);
 513
 514         if (rq->errors >= ERROR_MAX || blk_noretry_request(rq))
 515                 ide_kill_rq(drive, rq);
 516         else {
 517                 if ((rq->errors & ERROR_RESET) == ERROR_RESET) {
 518                         ++rq->errors;
 519                         return ide_do_reset(drive);
 520                 }
 521                 if ((rq->errors & ERROR_RECAL) == ERROR_RECAL)
 522                         drive->special.b.recalibrate = 1;
 523                 ++rq->errors;
 524         }
 525         return ide_stopped;
 526 }
 527
 528 static ide_startstop_t ide_atapi_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
 529 {
 530         ide_hwif_t *hwif = drive->hwif;
 531
 532         if (stat & BUSY_STAT || ((stat & WRERR_STAT) && !drive->nowerr)) {
 533                 /* other bits are useless when BUSY */
 534                 rq->errors |= ERROR_RESET;
 535         } else {
 536                 /* add decoding error stuff */
 537         }
 538
 539         if (hwif->INB(IDE_STATUS_REG) & (BUSY_STAT|DRQ_STAT))
 540                 /* force an abort */
 541                 hwif->OUTB(WIN_IDLEIMMEDIATE, IDE_COMMAND_REG);
 542
 543         if (rq->errors >= ERROR_MAX) {
 544                 ide_kill_rq(drive, rq);
 545         } else {
 546                 if ((rq->errors & ERROR_RESET) == ERROR_RESET) {
 547                         ++rq->errors;
 548                         return ide_do_reset(drive);
 549                 }
 550                 ++rq->errors;
 551         }
 552
 553         return ide_stopped;
 554 }
 555
 556 ide_startstop_t
 557 __ide_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
 558 {
 559         if (drive->media == ide_disk)
 560                 return ide_ata_error(drive, rq, stat, err);
 561         return ide_atapi_error(drive, rq, stat, err);
 562 }
 563
 564 EXPORT_SYMBOL_GPL(__ide_error);
 565
 566 /**
 567  *      ide_error       -       handle an error on the IDE
 568  *      @drive: drive the error occurred on
 569  *      @msg: message to report
 570  *      @stat: status bits
 571  *
 572  *      ide_error() takes action based on the error returned by the drive.
 573  *      For normal I/O that may well include retries. We deal with
 574  *      both new-style (taskfile) and old style command handling here.
 575  *      In the case of taskfile command handling there is work left to
 576  *      do
 577  */
 578
 579 ide_startstop_t ide_error (ide_drive_t *drive, const char *msg, u8 stat)
 580 {
 581         struct request *rq;
 582         u8 err;
 583
 584         err = ide_dump_status(drive, msg, stat);
 585
 586         if ((rq = HWGROUP(drive)->rq) == NULL)
 587                 return ide_stopped;
 588
 589         /* retry only "normal" I/O: */
 590         if (rq->flags & (REQ_DRIVE_CMD | REQ_DRIVE_TASK | REQ_DRIVE_TASKFILE)) {
 591                 rq->errors = 1;
 592                 ide_end_drive_cmd(drive, stat, err);
 593                 return ide_stopped;
 594         }
 595
 596         if (rq->rq_disk) {
 597                 ide_driver_t *drv;
 598
 599                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
 600                 return drv->error(drive, rq, stat, err);
 601         } else
 602                 return __ide_error(drive, rq, stat, err);
 603 }
 604
 605 EXPORT_SYMBOL_GPL(ide_error);
 606
 607 ide_startstop_t __ide_abort(ide_drive_t *drive, struct request *rq)
 608 {
 609         if (drive->media != ide_disk)
 610                 rq->errors |= ERROR_RESET;
 611
 612         ide_kill_rq(drive, rq);
 613
 614         return ide_stopped;
 615 }
 616
 617 EXPORT_SYMBOL_GPL(__ide_abort);
 618
 619 /**
 620  *      ide_abort       -       abort pending IDE operatins
 621  *      @drive: drive the error occurred on
 622  *      @msg: message to report
 623  *
 624  *      ide_abort kills and cleans up when we are about to do a
 625  *      host initiated reset on active commands. Longer term we
 626  *      want handlers to have sensible abort handling themselves
 627  *
 628  *      This differs fundamentally from ide_error because in
 629  *      this case the command is doing just fine when we
 630  *      blow it away.
 631  */
 632
 633 ide_startstop_t ide_abort(ide_drive_t *drive, const char *msg)
 634 {
 635         struct request *rq;
 636
 637         if (drive == NULL || (rq = HWGROUP(drive)->rq) == NULL)
 638                 return ide_stopped;
 639
 640         /* retry only "normal" I/O: */
 641         if (rq->flags & (REQ_DRIVE_CMD | REQ_DRIVE_TASK | REQ_DRIVE_TASKFILE)) {
 642                 rq->errors = 1;
 643                 ide_end_drive_cmd(drive, BUSY_STAT, 0);
 644                 return ide_stopped;
 645         }
 646
 647         if (rq->rq_disk) {
 648                 ide_driver_t *drv;
 649
 650                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
 651                 return drv->abort(drive, rq);
 652         } else
 653                 return __ide_abort(drive, rq);
 654 }
 655
 656 /**
 657  *      ide_cmd         -       issue a simple drive command
 658  *      @drive: drive the command is for
 659  *      @cmd: command byte
 660  *      @nsect: sector byte
 661  *      @handler: handler for the command completion
 662  *
 663  *      Issue a simple drive command with interrupts.
 664  *      The drive must be selected beforehand.
 665  */
 666
 667 static void ide_cmd (ide_drive_t *drive, u8 cmd, u8 nsect,
 668                 ide_handler_t *handler)
 669 {
 670         ide_hwif_t *hwif = HWIF(drive);
 671         if (IDE_CONTROL_REG)
 672                 hwif->OUTB(drive->ctl,IDE_CONTROL_REG); /* clear nIEN */
 673         SELECT_MASK(drive,0);
 674         hwif->OUTB(nsect,IDE_NSECTOR_REG);
 675         ide_execute_command(drive, cmd, handler, WAIT_CMD, NULL);
 676 }
 677
 678 /**
 679  *      drive_cmd_intr          -       drive command completion interrupt
 680  *      @drive: drive the completion interrupt occurred on
 681  *
 682  *      drive_cmd_intr() is invoked on completion of a special DRIVE_CMD.
 683  *      We do any necessary daya reading and then wait for the drive to
 684  *      go non busy. At that point we may read the error data and complete
 685  *      the request
 686  */
 687
 688 static ide_startstop_t drive_cmd_intr (ide_drive_t *drive)
 689 {
 690         struct request *rq = HWGROUP(drive)->rq;
 691         ide_hwif_t *hwif = HWIF(drive);
 692         u8 *args = (u8 *) rq->buffer;
 693         u8 stat = hwif->INB(IDE_STATUS_REG);
 694         int retries = 10;
 695
 696         local_irq_enable();
 697         if ((stat & DRQ_STAT) && args && args[3]) {
 698                 u8 io_32bit = drive->io_32bit;
 699                 drive->io_32bit = 0;
 700                 hwif->ata_input_data(drive, &args[4], args[3] * SECTOR_WORDS);
 701                 drive->io_32bit = io_32bit;
 702                 while (((stat = hwif->INB(IDE_STATUS_REG)) & BUSY_STAT) && retries--)
 703                         udelay(100);
 704         }
 705
 706         if (!OK_STAT(stat, READY_STAT, BAD_STAT))
 707                 return ide_error(drive, "drive_cmd", stat);
 708                 /* calls ide_end_drive_cmd */
 709         ide_end_drive_cmd(drive, stat, hwif->INB(IDE_ERROR_REG));
 710         return ide_stopped;
 711 }
 712
 713 static void ide_init_specify_cmd(ide_drive_t *drive, ide_task_t *task)
 714 {
 715         task->tfRegister[IDE_NSECTOR_OFFSET] = drive->sect;
 716         task->tfRegister[IDE_SECTOR_OFFSET]  = drive->sect;
 717         task->tfRegister[IDE_LCYL_OFFSET]    = drive->cyl;
 718         task->tfRegister[IDE_HCYL_OFFSET]    = drive->cyl>>8;
 719         task->tfRegister[IDE_SELECT_OFFSET]  = ((drive->head-1)|drive->select.all)&0xBF;
 720         task->tfRegister[IDE_COMMAND_OFFSET] = WIN_SPECIFY;
 721
 722         task->handler = &set_geometry_intr;
 723 }
 724
 725 static void ide_init_restore_cmd(ide_drive_t *drive, ide_task_t *task)
 726 {
 727         task->tfRegister[IDE_NSECTOR_OFFSET] = drive->sect;
 728         task->tfRegister[IDE_COMMAND_OFFSET] = WIN_RESTORE;
 729
 730         task->handler = &recal_intr;
 731 }
 732
 733 static void ide_init_setmult_cmd(ide_drive_t *drive, ide_task_t *task)
 734 {
 735         task->tfRegister[IDE_NSECTOR_OFFSET] = drive->mult_req;
 736         task->tfRegister[IDE_COMMAND_OFFSET] = WIN_SETMULT;
 737
 738         task->handler = &set_multmode_intr;
 739 }
 740
 741 static ide_startstop_t ide_disk_special(ide_drive_t *drive)
 742 {
 743         special_t *s = &drive->special;
 744         ide_task_t args;
 745
 746         memset(&args, 0, sizeof(ide_task_t));
 747         args.command_type = IDE_DRIVE_TASK_NO_DATA;
 748
 749         if (s->b.set_geometry) {
 750                 s->b.set_geometry = 0;
 751                 ide_init_specify_cmd(drive, &args);
 752         } else if (s->b.recalibrate) {
 753                 s->b.recalibrate = 0;
 754                 ide_init_restore_cmd(drive, &args);
 755         } else if (s->b.set_multmode) {
 756                 s->b.set_multmode = 0;
 757                 if (drive->mult_req > drive->id->max_multsect)
 758                         drive->mult_req = drive->id->max_multsect;
 759                 ide_init_setmult_cmd(drive, &args);
 760         } else if (s->all) {
 761                 int special = s->all;
 762                 s->all = 0;
 763                 printk(KERN_ERR "%s: bad special flag: 0x%02x\n", drive->name, special);
 764                 return ide_stopped;
 765         }
 766
 767         do_rw_taskfile(drive, &args);
 768
 769         return ide_started;
 770 }
 771
 772 /**
 773  *      do_special              -       issue some special commands
 774  *      @drive: drive the command is for
 775  *
 776  *      do_special() is used to issue WIN_SPECIFY, WIN_RESTORE, and WIN_SETMULT
 777  *      commands to a drive.  It used to do much more, but has been scaled
 778  *      back.
 779  */
 780
 781 static ide_startstop_t do_special (ide_drive_t *drive)
 782 {
 783         special_t *s = &drive->special;
 784
 785 #ifdef DEBUG
 786         printk("%s: do_special: 0x%02x\n", drive->name, s->all);
 787 #endif
 788         if (s->b.set_tune) {
 789                 s->b.set_tune = 0;
 790                 if (HWIF(drive)->tuneproc != NULL)
 791                         HWIF(drive)->tuneproc(drive, drive->tune_req);
 792                 return ide_stopped;
 793         } else {
 794                 if (drive->media == ide_disk)
 795                         return ide_disk_special(drive);
 796
 797                 s->all = 0;
 798                 drive->mult_req = 0;
 799                 return ide_stopped;
 800         }
 801 }
 802
 803 void ide_map_sg(ide_drive_t *drive, struct request *rq)
 804 {
 805         ide_hwif_t *hwif = drive->hwif;
 806         struct scatterlist *sg = hwif->sg_table;
 807
 808         if (hwif->sg_mapped)    /* needed by ide-scsi */
 809                 return;
 810
 811         if ((rq->flags & REQ_DRIVE_TASKFILE) == 0) {
 812                 hwif->sg_nents = blk_rq_map_sg(drive->queue, rq, sg);
 813         } else {
 814                 sg_init_one(sg, rq->buffer, rq->nr_sectors * SECTOR_SIZE);
 815                 hwif->sg_nents = 1;
 816         }
 817 }
 818
 819 EXPORT_SYMBOL_GPL(ide_map_sg);
 820
 821 void ide_init_sg_cmd(ide_drive_t *drive, struct request *rq)
 822 {
 823         ide_hwif_t *hwif = drive->hwif;
 824
 825         hwif->nsect = hwif->nleft = rq->nr_sectors;
 826         hwif->cursg = hwif->cursg_ofs = 0;
 827 }
 828
 829 EXPORT_SYMBOL_GPL(ide_init_sg_cmd);
 830
 831 /**
 832  *      execute_drive_command   -       issue special drive command
 833  *      @drive: the drive to issue th command on
 834  *      @rq: the request structure holding the command
 835  *
 836  *      execute_drive_cmd() issues a special drive command,  usually
 837  *      initiated by ioctl() from the external hdparm program. The
 838  *      command can be a drive command, drive task or taskfile
 839  *      operation. Weirdly you can call it with NULL to wait for
 840  *      all commands to finish. Don't do this as that is due to change
 841  */
 842
 843 static ide_startstop_t execute_drive_cmd (ide_drive_t *drive,
 844                 struct request *rq)
 845 {
 846         ide_hwif_t *hwif = HWIF(drive);
 847         if (rq->flags & REQ_DRIVE_TASKFILE) {
 848                 ide_task_t *args = rq->special;
 849
 850                 if (!args)
 851                         goto done;
 852
 853                 hwif->data_phase = args->data_phase;
 854
 855                 switch (hwif->data_phase) {
 856                 case TASKFILE_MULTI_OUT:
 857                 case TASKFILE_OUT:
 858                 case TASKFILE_MULTI_IN:
 859                 case TASKFILE_IN:
 860                         ide_init_sg_cmd(drive, rq);
 861                         ide_map_sg(drive, rq);
 862                 default:
 863                         break;
 864                 }
 865
 866                 if (args->tf_out_flags.all != 0)
 867                         return flagged_taskfile(drive, args);
 868                 return do_rw_taskfile(drive, args);
 869         } else if (rq->flags & REQ_DRIVE_TASK) {
 870                 u8 *args = rq->buffer;
 871                 u8 sel;
 872
 873                 if (!args)
 874                         goto done;
 875 #ifdef DEBUG
 876                 printk("%s: DRIVE_TASK_CMD ", drive->name);
 877                 printk("cmd=0x%02x ", args[0]);
 878                 printk("fr=0x%02x ", args[1]);
 879                 printk("ns=0x%02x ", args[2]);
 880                 printk("sc=0x%02x ", args[3]);
 881                 printk("lcyl=0x%02x ", args[4]);
 882                 printk("hcyl=0x%02x ", args[5]);
 883                 printk("sel=0x%02x\n", args[6]);
 884 #endif
 885                 hwif->OUTB(args[1], IDE_FEATURE_REG);
 886                 hwif->OUTB(args[3], IDE_SECTOR_REG);
 887                 hwif->OUTB(args[4], IDE_LCYL_REG);
 888                 hwif->OUTB(args[5], IDE_HCYL_REG);
 889                 sel = (args[6] & ~0x10);
 890                 if (drive->select.b.unit)
 891                         sel |= 0x10;
 892                 hwif->OUTB(sel, IDE_SELECT_REG);
 893                 ide_cmd(drive, args[0], args[2], &drive_cmd_intr);
 894                 return ide_started;
 895         } else if (rq->flags & REQ_DRIVE_CMD) {
 896                 u8 *args = rq->buffer;
 897
 898                 if (!args)
 899                         goto done;
 900 #ifdef DEBUG
 901                 printk("%s: DRIVE_CMD ", drive->name);
 902                 printk("cmd=0x%02x ", args[0]);
 903                 printk("sc=0x%02x ", args[1]);
 904                 printk("fr=0x%02x ", args[2]);
 905                 printk("xx=0x%02x\n", args[3]);
 906 #endif
 907                 if (args[0] == WIN_SMART) {
 908                         hwif->OUTB(0x4f, IDE_LCYL_REG);
 909                         hwif->OUTB(0xc2, IDE_HCYL_REG);
 910                         hwif->OUTB(args[2],IDE_FEATURE_REG);
 911                         hwif->OUTB(args[1],IDE_SECTOR_REG);
 912                         ide_cmd(drive, args[0], args[3], &drive_cmd_intr);
 913                         return ide_started;
 914                 }
 915                 hwif->OUTB(args[2],IDE_FEATURE_REG);
 916                 ide_cmd(drive, args[0], args[1], &drive_cmd_intr);
 917                 return ide_started;
 918         }
 919
 920 done:
 921         /*
 922          * NULL is actually a valid way of waiting for
 923          * all current requests to be flushed from the queue.
 924          */
 925 #ifdef DEBUG
 926         printk("%s: DRIVE_CMD (null)\n", drive->name);
 927 #endif
 928         ide_end_drive_cmd(drive,
 929                         hwif->INB(IDE_STATUS_REG),
 930                         hwif->INB(IDE_ERROR_REG));
 931         return ide_stopped;
 932 }
 933
 934 /**
 935  *      start_request   -       start of I/O and command issuing for IDE
 936  *
 937  *      start_request() initiates handling of a new I/O request. It
 938  *      accepts commands and I/O (read/write) requests. It also does
 939  *      the final remapping for weird stuff like EZDrive. Once
 940  *      device mapper can work sector level the EZDrive stuff can go away
 941  *
 942  *      FIXME: this function needs a rename
 943  */
 944
 945 static ide_startstop_t start_request (ide_drive_t *drive, struct request *rq)
 946 {
 947         ide_startstop_t startstop;
 948         sector_t block;
 949
 950         BUG_ON(!(rq->flags & REQ_STARTED));
 951
 952 #ifdef DEBUG
 953         printk("%s: start_request: current=0x%08lx\n",
 954                 HWIF(drive)->name, (unsigned long) rq);
 955 #endif
 956
 957         /* bail early if we've exceeded max_failures */
 958         if (drive->max_failures && (drive->failures > drive->max_failures)) {
 959                 goto kill_rq;
 960         }
 961
 962         block    = rq->sector;
 963         if (blk_fs_request(rq) &&
 964             (drive->media == ide_disk || drive->media == ide_floppy)) {
 965                 block += drive->sect0;
 966         }
 967         /* Yecch - this will shift the entire interval,
 968            possibly killing some innocent following sector */
 969         if (block == 0 && drive->remap_0_to_1 == 1)
 970                 block = 1;  /* redirect MBR access to EZ-Drive partn table */
 971
 972         if (blk_pm_suspend_request(rq) &&
 973             rq->pm->pm_step == ide_pm_state_start_suspend)
 974                 /* Mark drive blocked when starting the suspend sequence. */
 975                 drive->blocked = 1;
 976         else if (blk_pm_resume_request(rq) &&
 977                  rq->pm->pm_step == ide_pm_state_start_resume) {
 978                 /*
 979                  * The first thing we do on wakeup is to wait for BSY bit to
 980                  * go away (with a looong timeout) as a drive on this hwif may
 981                  * just be POSTing itself.
 982                  * We do that before even selecting as the "other" device on
 983                  * the bus may be broken enough to walk on our toes at this
 984                  * point.
 985                  */
 986                 int rc;
 987 #ifdef DEBUG_PM
 988                 printk("%s: Wakeup request inited, waiting for !BSY...\n", drive->name);
 989 #endif
 990                 rc = ide_wait_not_busy(HWIF(drive), 35000);
 991                 if (rc)
 992                         printk(KERN_WARNING "%s: bus not ready on wakeup\n", drive->name);
 993                 SELECT_DRIVE(drive);
 994                 HWIF(drive)->OUTB(8, HWIF(drive)->io_ports[IDE_CONTROL_OFFSET]);
 995                 rc = ide_wait_not_busy(HWIF(drive), 10000);
 996                 if (rc)
 997                         printk(KERN_WARNING "%s: drive not ready on wakeup\n", drive->name);
 998         }
 999
1000         SELECT_DRIVE(drive);
1001         if (ide_wait_stat(&startstop, drive, drive->ready_stat, BUSY_STAT|DRQ_STAT, WAIT_READY)) {
1002                 printk(KERN_ERR "%s: drive not ready for command\n", drive->name);
1003                 return startstop;
1004         }
1005         if (!drive->special.all) {
1006                 ide_driver_t *drv;
1007
1008                 if (rq->flags & (REQ_DRIVE_CMD | REQ_DRIVE_TASK))
1009                         return execute_drive_cmd(drive, rq);
1010                 else if (rq->flags & REQ_DRIVE_TASKFILE)
1011                         return execute_drive_cmd(drive, rq);
1012                 else if (blk_pm_request(rq)) {
1013 #ifdef DEBUG_PM
1014                         printk("%s: start_power_step(step: %d)\n",
1015                                 drive->name, rq->pm->pm_step);
1016 #endif
1017                         startstop = ide_start_power_step(drive, rq);
1018                         if (startstop == ide_stopped &&
1019                             rq->pm->pm_step == ide_pm_state_completed)
1020                                 ide_complete_pm_request(drive, rq);
1021                         return startstop;
1022                 }
1023
1024                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
1025                 return drv->do_request(drive, rq, block);
1026         }
1027         return do_special(drive);
1028 kill_rq:
1029         ide_kill_rq(drive, rq);
1030         return ide_stopped;
1031 }
1032
1033 /**
1034  *      ide_stall_queue         -       pause an IDE device
1035  *      @drive: drive to stall
1036  *      @timeout: time to stall for (jiffies)
1037  *
1038  *      ide_stall_queue() can be used by a drive to give excess bandwidth back
1039  *      to the hwgroup by sleeping for timeout jiffies.
1040  */
1041
1042 void ide_stall_queue (ide_drive_t *drive, unsigned long timeout)
1043 {
1044         if (timeout > WAIT_WORSTCASE)
1045                 timeout = WAIT_WORSTCASE;
1046         drive->sleep = timeout + jiffies;
1047         drive->sleeping = 1;
1048 }
1049
1050 EXPORT_SYMBOL(ide_stall_queue);
1051
1052 #define WAKEUP(drive)   ((drive)->service_start + 2 * (drive)->service_time)
1053
1054 /**
1055  *      choose_drive            -       select a drive to service
1056  *      @hwgroup: hardware group to select on
1057  *
1058  *      choose_drive() selects the next drive which will be serviced.
1059  *      This is necessary because the IDE layer can't issue commands
1060  *      to both drives on the same cable, unlike SCSI.
1061  */
1062
1063 static inline ide_drive_t *choose_drive (ide_hwgroup_t *hwgroup)
1064 {
1065         ide_drive_t *drive, *best;
1066
1067 repeat:
1068         best = NULL;
1069         drive = hwgroup->drive;
1070
1071         /*
1072          * drive is doing pre-flush, ordered write, post-flush sequence. even
1073          * though that is 3 requests, it must be seen as a single transaction.
1074          * we must not preempt this drive until that is complete
1075          */
1076         if (blk_queue_flushing(drive->queue)) {
1077                 /*
1078                  * small race where queue could get replugged during
1079                  * the 3-request flush cycle, just yank the plug since
1080                  * we want it to finish asap
1081                  */
1082                 blk_remove_plug(drive->queue);
1083                 return drive;
1084         }
1085
1086         do {
1087                 if ((!drive->sleeping || time_after_eq(jiffies, drive->sleep))
1088                     && !elv_queue_empty(drive->queue)) {
1089                         if (!best
1090                          || (drive->sleeping && (!best->sleeping || time_before(drive->sleep, best->sleep)))
1091                          || (!best->sleeping && time_before(WAKEUP(drive), WAKEUP(best))))
1092                         {
1093                                 if (!blk_queue_plugged(drive->queue))
1094                                         best = drive;
1095                         }
1096                 }
1097         } while ((drive = drive->next) != hwgroup->drive);
1098         if (best && best->nice1 && !best->sleeping && best != hwgroup->drive && best->service_time > WAIT_MIN_SLEEP) {
1099                 long t = (signed long)(WAKEUP(best) - jiffies);
1100                 if (t >= WAIT_MIN_SLEEP) {
1101                 /*
1102                  * We *may* have some time to spare, but first let's see if
1103                  * someone can potentially benefit from our nice mood today..
1104                  */
1105                         drive = best->next;
1106                         do {
1107                                 if (!drive->sleeping
1108                                  && time_before(jiffies - best->service_time, WAKEUP(drive))
1109                                  && time_before(WAKEUP(drive), jiffies + t))
1110                                 {
1111                                         ide_stall_queue(best, min_t(long, t, 10 * WAIT_MIN_SLEEP));
1112                                         goto repeat;
1113                                 }
1114                         } while ((drive = drive->next) != best);
1115                 }
1116         }
1117         return best;
1118 }
1119
1120 /*
1121  * Issue a new request to a drive from hwgroup
1122  * Caller must have already done spin_lock_irqsave(&ide_lock, ..);
1123  *
1124  * A hwgroup is a serialized group of IDE interfaces.  Usually there is
1125  * exactly one hwif (interface) per hwgroup, but buggy controllers (eg. CMD640)
1126  * may have both interfaces in a single hwgroup to "serialize" access.
1127  * Or possibly multiple ISA interfaces can share a common IRQ by being grouped
1128  * together into one hwgroup for serialized access.
1129  *
1130  * Note also that several hwgroups can end up sharing a single IRQ,
1131  * possibly along with many other devices.  This is especially common in
1132  * PCI-based systems with off-board IDE controller cards.
1133  *
1134  * The IDE driver uses the single global ide_lock spinlock to protect
1135  * access to the request queues, and to protect the hwgroup->busy flag.
1136  *
1137  * The first thread into the driver for a particular hwgroup sets the
1138  * hwgroup->busy flag to indicate that this hwgroup is now active,
1139  * and then initiates processing of the top request from the request queue.
1140  *
1141  * Other threads attempting entry notice the busy setting, and will simply
1142  * queue their new requests and exit immediately.  Note that hwgroup->busy
1143  * remains set even when the driver is merely awaiting the next interrupt.
1144  * Thus, the meaning is "this hwgroup is busy processing a request".
1145  *
1146  * When processing of a request completes, the completing thread or IRQ-handler
1147  * will start the next request from the queue.  If no more work remains,
1148  * the driver will clear the hwgroup->busy flag and exit.
1149  *
1150  * The ide_lock (spinlock) is used to protect all access to the
1151  * hwgroup->busy flag, but is otherwise not needed for most processing in
1152  * the driver.  This makes the driver much more friendlier to shared IRQs
1153  * than previous designs, while remaining 100% (?) SMP safe and capable.
1154  */
1155 static void ide_do_request (ide_hwgroup_t *hwgroup, int masked_irq)
1156 {
1157         ide_drive_t     *drive;
1158         ide_hwif_t      *hwif;
1159         struct request  *rq;
1160         ide_startstop_t startstop;
1161
1162         /* for atari only: POSSIBLY BROKEN HERE(?) */
1163         ide_get_lock(ide_intr, hwgroup);
1164
1165         /* caller must own ide_lock */
1166         BUG_ON(!irqs_disabled());
1167
1168         while (!hwgroup->busy) {
1169                 hwgroup->busy = 1;
1170                 drive = choose_drive(hwgroup);
1171                 if (drive == NULL) {
1172                         int sleeping = 0;
1173                         unsigned long sleep = 0; /* shut up, gcc */
1174                         hwgroup->rq = NULL;
1175                         drive = hwgroup->drive;
1176                         do {
1177                                 if (drive->sleeping && (!sleeping || time_before(drive->sleep, sleep))) {
1178                                         sleeping = 1;
1179                                         sleep = drive->sleep;
1180                                 }
1181                         } while ((drive = drive->next) != hwgroup->drive);
1182                         if (sleeping) {
1183                 /*
1184                  * Take a short snooze, and then wake up this hwgroup again.
1185                  * This gives other hwgroups on the same a chance to
1186                  * play fairly with us, just in case there are big differences
1187                  * in relative throughputs.. don't want to hog the cpu too much.
1188                  */
1189                                 if (time_before(sleep, jiffies + WAIT_MIN_SLEEP))
1190                                         sleep = jiffies + WAIT_MIN_SLEEP;
1191 #if 1
1192                                 if (timer_pending(&hwgroup->timer))
1193                                         printk(KERN_CRIT "ide_set_handler: timer already active\n");
1194 #endif
1195                                 /* so that ide_timer_expiry knows what to do */
1196                                 hwgroup->sleeping = 1;
1197                                 mod_timer(&hwgroup->timer, sleep);
1198                                 /* we purposely leave hwgroup->busy==1
1199                                  * while sleeping */
1200                         } else {
1201                                 /* Ugly, but how can we sleep for the lock
1202                                  * otherwise? perhaps from tq_disk?
1203                                  */
1204
1205                                 /* for atari only */
1206                                 ide_release_lock();
1207                                 hwgroup->busy = 0;
1208                         }
1209
1210                         /* no more work for this hwgroup (for now) */
1211                         return;
1212                 }
1213                 hwif = HWIF(drive);
1214                 if (hwgroup->hwif->sharing_irq &&
1215                     hwif != hwgroup->hwif &&
1216                     hwif->io_ports[IDE_CONTROL_OFFSET]) {
1217                         /* set nIEN for previous hwif */
1218                         SELECT_INTERRUPT(drive);
1219                 }
1220                 hwgroup->hwif = hwif;
1221                 hwgroup->drive = drive;
1222                 drive->sleeping = 0;
1223                 drive->service_start = jiffies;
1224
1225                 if (blk_queue_plugged(drive->queue)) {
1226                         printk(KERN_ERR "ide: huh? queue was plugged!\n");
1227                         break;
1228                 }
1229
1230                 /*
1231                  * we know that the queue isn't empty, but this can happen
1232                  * if the q->prep_rq_fn() decides to kill a request
1233                  */
1234                 rq = elv_next_request(drive->queue);
1235                 if (!rq) {
1236                         hwgroup->busy = 0;
1237                         break;
1238                 }
1239
1240                 /*
1241                  * Sanity: don't accept a request that isn't a PM request
1242                  * if we are currently power managed. This is very important as
1243                  * blk_stop_queue() doesn't prevent the elv_next_request()
1244                  * above to return us whatever is in the queue. Since we call
1245                  * ide_do_request() ourselves, we end up taking requests while
1246                  * the queue is blocked...
1247                  *
1248                  * We let requests forced at head of queue with ide-preempt
1249                  * though. I hope that doesn't happen too much, hopefully not
1250                  * unless the subdriver triggers such a thing in its own PM
1251                  * state machine.
1252                  */
1253                 if (drive->blocked && !blk_pm_request(rq) && !(rq->flags & REQ_PREEMPT)) {
1254                         /* We clear busy, there should be no pending ATA command at this point. */
1255                         hwgroup->busy = 0;
1256                         break;
1257                 }
1258
1259                 hwgroup->rq = rq;
1260
1261                 /*
1262                  * Some systems have trouble with IDE IRQs arriving while
1263                  * the driver is still setting things up.  So, here we disable
1264                  * the IRQ used by this interface while the request is being started.
1265                  * This may look bad at first, but pretty much the same thing
1266                  * happens anyway when any interrupt comes in, IDE or otherwise
1267                  *  -- the kernel masks the IRQ while it is being handled.
1268                  */
1269                 if (masked_irq != IDE_NO_IRQ && hwif->irq != masked_irq)
1270                         disable_irq_nosync(hwif->irq);
1271                 spin_unlock(&ide_lock);
1272                 local_irq_enable();
1273                         /* allow other IRQs while we start this request */
1274                 startstop = start_request(drive, rq);
1275                 spin_lock_irq(&ide_lock);
1276                 if (masked_irq != IDE_NO_IRQ && hwif->irq != masked_irq)
1277                         enable_irq(hwif->irq);
1278                 if (startstop == ide_stopped)
1279                         hwgroup->busy = 0;
1280         }
1281 }
1282
1283 /*
1284  * Passes the stuff to ide_do_request
1285  */
1286 void do_ide_request(request_queue_t *q)
1287 {
1288         ide_drive_t *drive = q->queuedata;
1289
1290         ide_do_request(HWGROUP(drive), IDE_NO_IRQ);
1291 }
1292
1293 /*
1294  * un-busy the hwgroup etc, and clear any pending DMA status. we want to
1295  * retry the current request in pio mode instead of risking tossing it
1296  * all away
1297  */
1298 static ide_startstop_t ide_dma_timeout_retry(ide_drive_t *drive, int error)
1299 {
1300         ide_hwif_t *hwif = HWIF(drive);
1301         struct request *rq;
1302         ide_startstop_t ret = ide_stopped;
1303
1304         /*
1305          * end current dma transaction
1306          */
1307
1308         if (error < 0) {
1309                 printk(KERN_WARNING "%s: DMA timeout error\n", drive->name);
1310                 (void)HWIF(drive)->ide_dma_end(drive);
1311                 ret = ide_error(drive, "dma timeout error",
1312                                                 hwif->INB(IDE_STATUS_REG));
1313         } else {
1314                 printk(KERN_WARNING "%s: DMA timeout retry\n", drive->name);
1315                 (void) hwif->ide_dma_timeout(drive);
1316         }
1317
1318         /*
1319          * disable dma for now, but remember that we did so because of
1320          * a timeout -- we'll reenable after we finish this next request
1321          * (or rather the first chunk of it) in pio.
1322          */
1323         drive->retry_pio++;
1324         drive->state = DMA_PIO_RETRY;
1325         (void) hwif->ide_dma_off_quietly(drive);
1326
1327         /*
1328          * un-busy drive etc (hwgroup->busy is cleared on return) and
1329          * make sure request is sane
1330          */
1331         rq = HWGROUP(drive)->rq;
1332         HWGROUP(drive)->rq = NULL;
1333
1334         rq->errors = 0;
1335
1336         if (!rq->bio)
1337                 goto out;
1338
1339         rq->sector = rq->bio->bi_sector;
1340         rq->current_nr_sectors = bio_iovec(rq->bio)->bv_len >> 9;
1341         rq->hard_cur_sectors = rq->current_nr_sectors;
1342         rq->buffer = bio_data(rq->bio);
1343 out:
1344         return ret;
1345 }
1346
1347 /**
1348  *      ide_timer_expiry        -       handle lack of an IDE interrupt
1349  *      @data: timer callback magic (hwgroup)
1350  *
1351  *      An IDE command has timed out before the expected drive return
1352  *      occurred. At this point we attempt to clean up the current
1353  *      mess. If the current handler includes an expiry handler then
1354  *      we invoke the expiry handler, and providing it is happy the
1355  *      work is done. If that fails we apply generic recovery rules
1356  *      invoking the handler and checking the drive DMA status. We
1357  *      have an excessively incestuous relationship with the DMA
1358  *      logic that wants cleaning up.
1359  */
1360
1361 void ide_timer_expiry (unsigned long data)
1362 {
1363         ide_hwgroup_t   *hwgroup = (ide_hwgroup_t *) data;
1364         ide_handler_t   *handler;
1365         ide_expiry_t    *expiry;
1366         unsigned long   flags;
1367         unsigned long   wait = -1;
1368
1369         spin_lock_irqsave(&ide_lock, flags);
1370
1371         if ((handler = hwgroup->handler) == NULL) {
1372                 /*
1373                  * Either a marginal timeout occurred
1374                  * (got the interrupt just as timer expired),
1375                  * or we were "sleeping" to give other devices a chance.
1376                  * Either way, we don't really want to complain about anything.
1377                  */
1378                 if (hwgroup->sleeping) {
1379                         hwgroup->sleeping = 0;
1380                         hwgroup->busy = 0;
1381                 }
1382         } else {
1383                 ide_drive_t *drive = hwgroup->drive;
1384                 if (!drive) {
1385                         printk(KERN_ERR "ide_timer_expiry: hwgroup->drive was NULL\n");
1386                         hwgroup->handler = NULL;
1387                 } else {
1388                         ide_hwif_t *hwif;
1389                         ide_startstop_t startstop = ide_stopped;
1390                         if (!hwgroup->busy) {
1391                                 hwgroup->busy = 1;      /* paranoia */
1392                                 printk(KERN_ERR "%s: ide_timer_expiry: hwgroup->busy was 0 ??\n", drive->name);
1393                         }
1394                         if ((expiry = hwgroup->expiry) != NULL) {
1395                                 /* continue */
1396                                 if ((wait = expiry(drive)) > 0) {
1397                                         /* reset timer */
1398                                         hwgroup->timer.expires  = jiffies + wait;
1399                                         add_timer(&hwgroup->timer);
1400                                         spin_unlock_irqrestore(&ide_lock, flags);
1401                                         return;
1402                                 }
1403                         }
1404                         hwgroup->handler = NULL;
1405                         /*
1406                          * We need to simulate a real interrupt when invoking
1407                          * the handler() function, which means we need to
1408                          * globally mask the specific IRQ:
1409                          */
1410                         spin_unlock(&ide_lock);
1411                         hwif  = HWIF(drive);
1412 #if DISABLE_IRQ_NOSYNC
1413                         disable_irq_nosync(hwif->irq);
1414 #else
1415                         /* disable_irq_nosync ?? */
1416                         disable_irq(hwif->irq);
1417 #endif /* DISABLE_IRQ_NOSYNC */
1418                         /* local CPU only,
1419                          * as if we were handling an interrupt */
1420                         local_irq_disable();
1421                         if (hwgroup->polling) {
1422                                 startstop = handler(drive);
1423                         } else if (drive_is_ready(drive)) {
1424                                 if (drive->waiting_for_dma)
1425                                         (void) hwgroup->hwif->ide_dma_lostirq(drive);
1426                                 (void)ide_ack_intr(hwif);
1427                                 printk(KERN_WARNING "%s: lost interrupt\n", drive->name);
1428                                 startstop = handler(drive);
1429                         } else {
1430                                 if (drive->waiting_for_dma) {
1431                                         startstop = ide_dma_timeout_retry(drive, wait);
1432                                 } else
1433                                         startstop =
1434                                         ide_error(drive, "irq timeout", hwif->INB(IDE_STATUS_REG));
1435                         }
1436                         drive->service_time = jiffies - drive->service_start;
1437                         spin_lock_irq(&ide_lock);
1438                         enable_irq(hwif->irq);
1439                         if (startstop == ide_stopped)
1440                                 hwgroup->busy = 0;
1441                 }
1442         }
1443         ide_do_request(hwgroup, IDE_NO_IRQ);
1444         spin_unlock_irqrestore(&ide_lock, flags);
1445 }
1446
1447 /**
1448  *      unexpected_intr         -       handle an unexpected IDE interrupt
1449  *      @irq: interrupt line
1450  *      @hwgroup: hwgroup being processed
1451  *
1452  *      There's nothing really useful we can do with an unexpected interrupt,
1453  *      other than reading the status register (to clear it), and logging it.
1454  *      There should be no way that an irq can happen before we're ready for it,
1455  *      so we needn't worry much about losing an "important" interrupt here.
1456  *
1457  *      On laptops (and "green" PCs), an unexpected interrupt occurs whenever
1458  *      the drive enters "idle", "standby", or "sleep" mode, so if the status
1459  *      looks "good", we just ignore the interrupt completely.
1460  *
1461  *      This routine assumes __cli() is in effect when called.
1462  *
1463  *      If an unexpected interrupt happens on irq15 while we are handling irq14
1464  *      and if the two interfaces are "serialized" (CMD640), then it looks like
1465  *      we could screw up by interfering with a new request being set up for
1466  *      irq15.
1467  *
1468  *      In reality, this is a non-issue.  The new command is not sent unless
1469  *      the drive is ready to accept one, in which case we know the drive is
1470  *      not trying to interrupt us.  And ide_set_handler() is always invoked
1471  *      before completing the issuance of any new drive command, so we will not
1472  *      be accidentally invoked as a result of any valid command completion
1473  *      interrupt.
1474  *
1475  *      Note that we must walk the entire hwgroup here. We know which hwif
1476  *      is doing the current command, but we don't know which hwif burped
1477  *      mysteriously.
1478  */
1479
1480 static void unexpected_intr (int irq, ide_hwgroup_t *hwgroup)
1481 {
1482         u8 stat;
1483         ide_hwif_t *hwif = hwgroup->hwif;
1484
1485         /*
1486          * handle the unexpected interrupt
1487          */
1488         do {
1489                 if (hwif->irq == irq) {
1490                         stat = hwif->INB(hwif->io_ports[IDE_STATUS_OFFSET]);
1491                         if (!OK_STAT(stat, READY_STAT, BAD_STAT)) {
1492                                 /* Try to not flood the console with msgs */
1493                                 static unsigned long last_msgtime, count;
1494                                 ++count;
1495                                 if (time_after(jiffies, last_msgtime + HZ)) {
1496                                         last_msgtime = jiffies;
1497                                         printk(KERN_ERR "%s%s: unexpected interrupt, "
1498                                                 "status=0x%02x, count=%ld\n",
1499                                                 hwif->name,
1500                                                 (hwif->next==hwgroup->hwif) ? "" : "(?)", stat, count);
1501                                 }
1502                         }
1503                 }
1504         } while ((hwif = hwif->next) != hwgroup->hwif);
1505 }
1506
1507 /**
1508  *      ide_intr        -       default IDE interrupt handler
1509  *      @irq: interrupt number
1510  *      @dev_id: hwif group
1511  *      @regs: unused weirdness from the kernel irq layer
1512  *
1513  *      This is the default IRQ handler for the IDE layer. You should
1514  *      not need to override it. If you do be aware it is subtle in
1515  *      places
1516  *
1517  *      hwgroup->hwif is the interface in the group currently performing
1518  *      a command. hwgroup->drive is the drive and hwgroup->handler is
1519  *      the IRQ handler to call. As we issue a command the handlers
1520  *      step through multiple states, reassigning the handler to the
1521  *      next step in the process. Unlike a smart SCSI controller IDE
1522  *      expects the main processor to sequence the various transfer
1523  *      stages. We also manage a poll timer to catch up with most
1524  *      timeout situations. There are still a few where the handlers
1525  *      don't ever decide to give up.
1526  *
1527  *      The handler eventually returns ide_stopped to indicate the
1528  *      request completed. At this point we issue the next request
1529  *      on the hwgroup and the process begins again.
1530  */
1531
1532 irqreturn_t ide_intr (int irq, void *dev_id, struct pt_regs *regs)
1533 {
1534         unsigned long flags;
1535         ide_hwgroup_t *hwgroup = (ide_hwgroup_t *)dev_id;
1536         ide_hwif_t *hwif;
1537         ide_drive_t *drive;
1538         ide_handler_t *handler;
1539         ide_startstop_t startstop;
1540
1541         spin_lock_irqsave(&ide_lock, flags);
1542         hwif = hwgroup->hwif;
1543
1544         if (!ide_ack_intr(hwif)) {
1545                 spin_unlock_irqrestore(&ide_lock, flags);
1546                 return IRQ_NONE;
1547         }
1548         if (hwif->polling) {
1549                 /* We took an interrupt during a polled drive retune.
1550                    This should go away eventually when that code uses
1551                    the polling logic like do_reset1 */
1552                 spin_unlock_irqrestore(&ide_lock, flags);
1553                 return IRQ_HANDLED;
1554         }
1555         if ((handler = hwgroup->handler) == NULL || hwgroup->polling) {
1556                 /*
1557                  * Not expecting an interrupt from this drive.
1558                  * That means this could be:
1559                  *      (1) an interrupt from another PCI device
1560                  *      sharing the same PCI INT# as us.
1561                  * or   (2) a drive just entered sleep or standby mode,
1562                  *      and is interrupting to let us know.
1563                  * or   (3) a spurious interrupt of unknown origin.
1564                  *
1565                  * For PCI, we cannot tell the difference,
1566                  * so in that case we just ignore it and hope it goes away.
1567                  *
1568                  * FIXME: unexpected_intr should be hwif-> then we can
1569                  * remove all the ifdef PCI crap
1570                  */
1571 #ifdef CONFIG_BLK_DEV_IDEPCI
1572                 if (hwif->pci_dev && !hwif->pci_dev->vendor)
1573 #endif  /* CONFIG_BLK_DEV_IDEPCI */
1574                 {
1575                         /*
1576                          * Probably not a shared PCI interrupt,
1577                          * so we can safely try to do something about it:
1578                          */
1579                         unexpected_intr(irq, hwgroup);
1580 #ifdef CONFIG_BLK_DEV_IDEPCI
1581                 } else {
1582                         /*
1583                          * Whack the status register, just in case
1584                          * we have a leftover pending IRQ.
1585                          */
1586                         (void) hwif->INB(hwif->io_ports[IDE_STATUS_OFFSET]);
1587 #endif /* CONFIG_BLK_DEV_IDEPCI */
1588                 }
1589                 spin_unlock_irqrestore(&ide_lock, flags);
1590                 return IRQ_NONE;
1591         }
1592         drive = hwgroup->drive;
1593         if (!drive) {
1594                 /*
1595                  * This should NEVER happen, and there isn't much
1596                  * we could do about it here.
1597                  *
1598                  * [Note - this can occur if the drive is hot unplugged]
1599                  */
1600                 spin_unlock_irqrestore(&ide_lock, flags);
1601                 return IRQ_HANDLED;
1602         }
1603         if (!drive_is_ready(drive)) {
1604                 /*
1605                  * This happens regularly when we share a PCI IRQ with
1606                  * another device.  Unfortunately, it can also happen
1607                  * with some buggy drives that trigger the IRQ before
1608                  * their status register is up to date.  Hopefully we have
1609                  * enough advance overhead that the latter isn't a problem.
1610                  */
1611                 spin_unlock_irqrestore(&ide_lock, flags);
1612                 return IRQ_NONE;
1613         }
1614         if (!hwgroup->busy) {
1615                 hwgroup->busy = 1;      /* paranoia */
1616                 printk(KERN_ERR "%s: ide_intr: hwgroup->busy was 0 ??\n", drive->name);
1617         }
1618         hwgroup->handler = NULL;
1619         del_timer(&hwgroup->timer);
1620         spin_unlock(&ide_lock);
1621
1622         if (drive->unmask)
1623                 local_irq_enable();
1624         /* service this interrupt, may set handler for next interrupt */
1625         startstop = handler(drive);
1626         spin_lock_irq(&ide_lock);
1627
1628         /*
1629          * Note that handler() may have set things up for another
1630          * interrupt to occur soon, but it cannot happen until
1631          * we exit from this routine, because it will be the
1632          * same irq as is currently being serviced here, and Linux
1633          * won't allow another of the same (on any CPU) until we return.
1634          */
1635         drive->service_time = jiffies - drive->service_start;
1636         if (startstop == ide_stopped) {
1637                 if (hwgroup->handler == NULL) { /* paranoia */
1638                         hwgroup->busy = 0;
1639                         ide_do_request(hwgroup, hwif->irq);
1640                 } else {
1641                         printk(KERN_ERR "%s: ide_intr: huh? expected NULL handler "
1642                                 "on exit\n", drive->name);
1643                 }
1644         }
1645         spin_unlock_irqrestore(&ide_lock, flags);
1646         return IRQ_HANDLED;
1647 }
1648
1649 /**
1650  *      ide_init_drive_cmd      -       initialize a drive command request
1651  *      @rq: request object
1652  *
1653  *      Initialize a request before we fill it in and send it down to
1654  *      ide_do_drive_cmd. Commands must be set up by this function. Right
1655  *      now it doesn't do a lot, but if that changes abusers will have a
1656  *      nasty suprise.
1657  */
1658
1659 void ide_init_drive_cmd (struct request *rq)
1660 {
1661         memset(rq, 0, sizeof(*rq));
1662         rq->flags = REQ_DRIVE_CMD;
1663         rq->ref_count = 1;
1664 }
1665
1666 EXPORT_SYMBOL(ide_init_drive_cmd);
1667
1668 /**
1669  *      ide_do_drive_cmd        -       issue IDE special command
1670  *      @drive: device to issue command
1671  *      @rq: request to issue
1672  *      @action: action for processing
1673  *
1674  *      This function issues a special IDE device request
1675  *      onto the request queue.
1676  *
1677  *      If action is ide_wait, then the rq is queued at the end of the
1678  *      request queue, and the function sleeps until it has been processed.
1679  *      This is for use when invoked from an ioctl handler.
1680  *
1681  *      If action is ide_preempt, then the rq is queued at the head of
1682  *      the request queue, displacing the currently-being-processed
1683  *      request and this function returns immediately without waiting
1684  *      for the new rq to be completed.  This is VERY DANGEROUS, and is
1685  *      intended for careful use by the ATAPI tape/cdrom driver code.
1686  *
1687  *      If action is ide_next, then the rq is queued immediately after
1688  *      the currently-being-processed-request (if any), and the function
1689  *      returns without waiting for the new rq to be completed.  As above,
1690  *      This is VERY DANGEROUS, and is intended for careful use by the
1691  *      ATAPI tape/cdrom driver code.
1692  *
1693  *      If action is ide_end, then the rq is queued at the end of the
1694  *      request queue, and the function returns immediately without waiting
1695  *      for the new rq to be completed. This is again intended for careful
1696  *      use by the ATAPI tape/cdrom driver code.
1697  */
1698
1699 int ide_do_drive_cmd (ide_drive_t *drive, struct request *rq, ide_action_t action)
1700 {
1701         unsigned long flags;
1702         ide_hwgroup_t *hwgroup = HWGROUP(drive);
1703         DECLARE_COMPLETION(wait);
1704         int where = ELEVATOR_INSERT_BACK, err;
1705         int must_wait = (action == ide_wait || action == ide_head_wait);
1706
1707         rq->errors = 0;
1708         rq->rq_status = RQ_ACTIVE;
1709
1710         /*
1711          * we need to hold an extra reference to request for safe inspection
1712          * after completion
1713          */
1714         if (must_wait) {
1715                 rq->ref_count++;
1716                 rq->waiting = &wait;
1717                 rq->end_io = blk_end_sync_rq;
1718         }
1719
1720         spin_lock_irqsave(&ide_lock, flags);
1721         if (action == ide_preempt)
1722                 hwgroup->rq = NULL;
1723         if (action == ide_preempt || action == ide_head_wait) {
1724                 where = ELEVATOR_INSERT_FRONT;
1725                 rq->flags |= REQ_PREEMPT;
1726         }
1727         __elv_add_request(drive->queue, rq, where, 0);
1728         ide_do_request(hwgroup, IDE_NO_IRQ);
1729         spin_unlock_irqrestore(&ide_lock, flags);
1730
1731         err = 0;
1732         if (must_wait) {
1733                 wait_for_completion(&wait);
1734                 rq->waiting = NULL;
1735                 if (rq->errors)
1736                         err = -EIO;
1737
1738                 blk_put_request(rq);
1739         }
1740
1741         return err;
1742 }
1743
1744 EXPORT_SYMBOL(ide_do_drive_cmd);