X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=drivers%2Fata%2Fsata_nv.c;fp=drivers%2Fata%2Fsata_nv.c;h=91f24b1a2dc3419db10f9d73842a94a6141d190a;hb=76828883507a47dae78837ab5dec5a5b4513c667;hp=0000000000000000000000000000000000000000;hpb=64ba3f394c830ec48a1c31b53dcae312c56f1604;p=linux-2.6.git diff --git a/drivers/ata/sata_nv.c b/drivers/ata/sata_nv.c new file mode 100644 index 000000000..91f24b1a2 --- /dev/null +++ b/drivers/ata/sata_nv.c @@ -0,0 +1,1552 @@ +/* + * sata_nv.c - NVIDIA nForce SATA + * + * Copyright 2004 NVIDIA Corp. All rights reserved. + * Copyright 2004 Andrew Chew + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * + * + * libata documentation is available via 'make {ps|pdf}docs', + * as Documentation/DocBook/libata.* + * + * No hardware documentation available outside of NVIDIA. + * This driver programs the NVIDIA SATA controller in a similar + * fashion as with other PCI IDE BMDMA controllers, with a few + * NV-specific details such as register offsets, SATA phy location, + * hotplug info, etc. + * + * CK804/MCP04 controllers support an alternate programming interface + * similar to the ADMA specification (with some modifications). + * This allows the use of NCQ. Non-DMA-mapped ATA commands are still + * sent through the legacy interface. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "sata_nv" +#define DRV_VERSION "3.2" + +#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL + +enum { + NV_PORTS = 2, + NV_PIO_MASK = 0x1f, + NV_MWDMA_MASK = 0x07, + NV_UDMA_MASK = 0x7f, + NV_PORT0_SCR_REG_OFFSET = 0x00, + NV_PORT1_SCR_REG_OFFSET = 0x40, + + /* INT_STATUS/ENABLE */ + NV_INT_STATUS = 0x10, + NV_INT_ENABLE = 0x11, + NV_INT_STATUS_CK804 = 0x440, + NV_INT_ENABLE_CK804 = 0x441, + + /* INT_STATUS/ENABLE bits */ + NV_INT_DEV = 0x01, + NV_INT_PM = 0x02, + NV_INT_ADDED = 0x04, + NV_INT_REMOVED = 0x08, + + NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */ + + NV_INT_ALL = 0x0f, + NV_INT_MASK = NV_INT_DEV | + NV_INT_ADDED | NV_INT_REMOVED, + + /* INT_CONFIG */ + NV_INT_CONFIG = 0x12, + NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI + + // For PCI config register 20 + NV_MCP_SATA_CFG_20 = 0x50, + NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04, + NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17), + NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16), + NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14), + NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12), + + NV_ADMA_MAX_CPBS = 32, + NV_ADMA_CPB_SZ = 128, + NV_ADMA_APRD_SZ = 16, + NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) / + NV_ADMA_APRD_SZ, + NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5, + NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ, + NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS * + (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ), + + /* BAR5 offset to ADMA general registers */ + NV_ADMA_GEN = 0x400, + NV_ADMA_GEN_CTL = 0x00, + NV_ADMA_NOTIFIER_CLEAR = 0x30, + + /* BAR5 offset to ADMA ports */ + NV_ADMA_PORT = 0x480, + + /* size of ADMA port register space */ + NV_ADMA_PORT_SIZE = 0x100, + + /* ADMA port registers */ + NV_ADMA_CTL = 0x40, + NV_ADMA_CPB_COUNT = 0x42, + NV_ADMA_NEXT_CPB_IDX = 0x43, + NV_ADMA_STAT = 0x44, + NV_ADMA_CPB_BASE_LOW = 0x48, + NV_ADMA_CPB_BASE_HIGH = 0x4C, + NV_ADMA_APPEND = 0x50, + NV_ADMA_NOTIFIER = 0x68, + NV_ADMA_NOTIFIER_ERROR = 0x6C, + + /* NV_ADMA_CTL register bits */ + NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0), + NV_ADMA_CTL_CHANNEL_RESET = (1 << 5), + NV_ADMA_CTL_GO = (1 << 7), + NV_ADMA_CTL_AIEN = (1 << 8), + NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11), + NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12), + + /* CPB response flag bits */ + NV_CPB_RESP_DONE = (1 << 0), + NV_CPB_RESP_ATA_ERR = (1 << 3), + NV_CPB_RESP_CMD_ERR = (1 << 4), + NV_CPB_RESP_CPB_ERR = (1 << 7), + + /* CPB control flag bits */ + NV_CPB_CTL_CPB_VALID = (1 << 0), + NV_CPB_CTL_QUEUE = (1 << 1), + NV_CPB_CTL_APRD_VALID = (1 << 2), + NV_CPB_CTL_IEN = (1 << 3), + NV_CPB_CTL_FPDMA = (1 << 4), + + /* APRD flags */ + NV_APRD_WRITE = (1 << 1), + NV_APRD_END = (1 << 2), + NV_APRD_CONT = (1 << 3), + + /* NV_ADMA_STAT flags */ + NV_ADMA_STAT_TIMEOUT = (1 << 0), + NV_ADMA_STAT_HOTUNPLUG = (1 << 1), + NV_ADMA_STAT_HOTPLUG = (1 << 2), + NV_ADMA_STAT_CPBERR = (1 << 4), + NV_ADMA_STAT_SERROR = (1 << 5), + NV_ADMA_STAT_CMD_COMPLETE = (1 << 6), + NV_ADMA_STAT_IDLE = (1 << 8), + NV_ADMA_STAT_LEGACY = (1 << 9), + NV_ADMA_STAT_STOPPED = (1 << 10), + NV_ADMA_STAT_DONE = (1 << 12), + NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR | + NV_ADMA_STAT_TIMEOUT, + + /* port flags */ + NV_ADMA_PORT_REGISTER_MODE = (1 << 0), + NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1), + +}; + +/* ADMA Physical Region Descriptor - one SG segment */ +struct nv_adma_prd { + __le64 addr; + __le32 len; + u8 flags; + u8 packet_len; + __le16 reserved; +}; + +enum nv_adma_regbits { + CMDEND = (1 << 15), /* end of command list */ + WNB = (1 << 14), /* wait-not-BSY */ + IGN = (1 << 13), /* ignore this entry */ + CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */ + DA2 = (1 << (2 + 8)), + DA1 = (1 << (1 + 8)), + DA0 = (1 << (0 + 8)), +}; + +/* ADMA Command Parameter Block + The first 5 SG segments are stored inside the Command Parameter Block itself. + If there are more than 5 segments the remainder are stored in a separate + memory area indicated by next_aprd. */ +struct nv_adma_cpb { + u8 resp_flags; /* 0 */ + u8 reserved1; /* 1 */ + u8 ctl_flags; /* 2 */ + /* len is length of taskfile in 64 bit words */ + u8 len; /* 3 */ + u8 tag; /* 4 */ + u8 next_cpb_idx; /* 5 */ + __le16 reserved2; /* 6-7 */ + __le16 tf[12]; /* 8-31 */ + struct nv_adma_prd aprd[5]; /* 32-111 */ + __le64 next_aprd; /* 112-119 */ + __le64 reserved3; /* 120-127 */ +}; + + +struct nv_adma_port_priv { + struct nv_adma_cpb *cpb; + dma_addr_t cpb_dma; + struct nv_adma_prd *aprd; + dma_addr_t aprd_dma; + u8 flags; + int last_issue_ncq; +}; + +#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & ( 1 << (19 + (12 * (PORT))))) + +static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent); +static void nv_ck804_host_stop(struct ata_host *host); +static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance); +static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance); +static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance); +static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg); +static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val); + +static void nv_nf2_freeze(struct ata_port *ap); +static void nv_nf2_thaw(struct ata_port *ap); +static void nv_ck804_freeze(struct ata_port *ap); +static void nv_ck804_thaw(struct ata_port *ap); +static void nv_error_handler(struct ata_port *ap); +static int nv_adma_slave_config(struct scsi_device *sdev); +static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc); +static void nv_adma_qc_prep(struct ata_queued_cmd *qc); +static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc); +static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance); +static void nv_adma_irq_clear(struct ata_port *ap); +static int nv_adma_port_start(struct ata_port *ap); +static void nv_adma_port_stop(struct ata_port *ap); +static void nv_adma_error_handler(struct ata_port *ap); +static void nv_adma_host_stop(struct ata_host *host); +static void nv_adma_bmdma_setup(struct ata_queued_cmd *qc); +static void nv_adma_bmdma_start(struct ata_queued_cmd *qc); +static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc); +static u8 nv_adma_bmdma_status(struct ata_port *ap); + +enum nv_host_type +{ + GENERIC, + NFORCE2, + NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */ + CK804, + ADMA +}; + +static const struct pci_device_id nv_pci_tbl[] = { + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), GENERIC }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), GENERIC }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), GENERIC }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), GENERIC }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC }, + { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC }, + { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, + PCI_ANY_ID, PCI_ANY_ID, + PCI_CLASS_STORAGE_IDE<<8, 0xffff00, GENERIC }, + { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, + PCI_ANY_ID, PCI_ANY_ID, + PCI_CLASS_STORAGE_RAID<<8, 0xffff00, GENERIC }, + + { } /* terminate list */ +}; + +static struct pci_driver nv_pci_driver = { + .name = DRV_NAME, + .id_table = nv_pci_tbl, + .probe = nv_init_one, + .remove = ata_pci_remove_one, +}; + +static struct scsi_host_template nv_sht = { + .module = THIS_MODULE, + .name = DRV_NAME, + .ioctl = ata_scsi_ioctl, + .queuecommand = ata_scsi_queuecmd, + .can_queue = ATA_DEF_QUEUE, + .this_id = ATA_SHT_THIS_ID, + .sg_tablesize = LIBATA_MAX_PRD, + .cmd_per_lun = ATA_SHT_CMD_PER_LUN, + .emulated = ATA_SHT_EMULATED, + .use_clustering = ATA_SHT_USE_CLUSTERING, + .proc_name = DRV_NAME, + .dma_boundary = ATA_DMA_BOUNDARY, + .slave_configure = ata_scsi_slave_config, + .slave_destroy = ata_scsi_slave_destroy, + .bios_param = ata_std_bios_param, +}; + +static struct scsi_host_template nv_adma_sht = { + .module = THIS_MODULE, + .name = DRV_NAME, + .ioctl = ata_scsi_ioctl, + .queuecommand = ata_scsi_queuecmd, + .can_queue = NV_ADMA_MAX_CPBS, + .this_id = ATA_SHT_THIS_ID, + .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN, + .cmd_per_lun = ATA_SHT_CMD_PER_LUN, + .emulated = ATA_SHT_EMULATED, + .use_clustering = ATA_SHT_USE_CLUSTERING, + .proc_name = DRV_NAME, + .dma_boundary = NV_ADMA_DMA_BOUNDARY, + .slave_configure = nv_adma_slave_config, + .slave_destroy = ata_scsi_slave_destroy, + .bios_param = ata_std_bios_param, +}; + +static const struct ata_port_operations nv_generic_ops = { + .port_disable = ata_port_disable, + .tf_load = ata_tf_load, + .tf_read = ata_tf_read, + .exec_command = ata_exec_command, + .check_status = ata_check_status, + .dev_select = ata_std_dev_select, + .bmdma_setup = ata_bmdma_setup, + .bmdma_start = ata_bmdma_start, + .bmdma_stop = ata_bmdma_stop, + .bmdma_status = ata_bmdma_status, + .qc_prep = ata_qc_prep, + .qc_issue = ata_qc_issue_prot, + .freeze = ata_bmdma_freeze, + .thaw = ata_bmdma_thaw, + .error_handler = nv_error_handler, + .post_internal_cmd = ata_bmdma_post_internal_cmd, + .data_xfer = ata_pio_data_xfer, + .irq_handler = nv_generic_interrupt, + .irq_clear = ata_bmdma_irq_clear, + .scr_read = nv_scr_read, + .scr_write = nv_scr_write, + .port_start = ata_port_start, + .port_stop = ata_port_stop, + .host_stop = ata_pci_host_stop, +}; + +static const struct ata_port_operations nv_nf2_ops = { + .port_disable = ata_port_disable, + .tf_load = ata_tf_load, + .tf_read = ata_tf_read, + .exec_command = ata_exec_command, + .check_status = ata_check_status, + .dev_select = ata_std_dev_select, + .bmdma_setup = ata_bmdma_setup, + .bmdma_start = ata_bmdma_start, + .bmdma_stop = ata_bmdma_stop, + .bmdma_status = ata_bmdma_status, + .qc_prep = ata_qc_prep, + .qc_issue = ata_qc_issue_prot, + .freeze = nv_nf2_freeze, + .thaw = nv_nf2_thaw, + .error_handler = nv_error_handler, + .post_internal_cmd = ata_bmdma_post_internal_cmd, + .data_xfer = ata_pio_data_xfer, + .irq_handler = nv_nf2_interrupt, + .irq_clear = ata_bmdma_irq_clear, + .scr_read = nv_scr_read, + .scr_write = nv_scr_write, + .port_start = ata_port_start, + .port_stop = ata_port_stop, + .host_stop = ata_pci_host_stop, +}; + +static const struct ata_port_operations nv_ck804_ops = { + .port_disable = ata_port_disable, + .tf_load = ata_tf_load, + .tf_read = ata_tf_read, + .exec_command = ata_exec_command, + .check_status = ata_check_status, + .dev_select = ata_std_dev_select, + .bmdma_setup = ata_bmdma_setup, + .bmdma_start = ata_bmdma_start, + .bmdma_stop = ata_bmdma_stop, + .bmdma_status = ata_bmdma_status, + .qc_prep = ata_qc_prep, + .qc_issue = ata_qc_issue_prot, + .freeze = nv_ck804_freeze, + .thaw = nv_ck804_thaw, + .error_handler = nv_error_handler, + .post_internal_cmd = ata_bmdma_post_internal_cmd, + .data_xfer = ata_pio_data_xfer, + .irq_handler = nv_ck804_interrupt, + .irq_clear = ata_bmdma_irq_clear, + .scr_read = nv_scr_read, + .scr_write = nv_scr_write, + .port_start = ata_port_start, + .port_stop = ata_port_stop, + .host_stop = nv_ck804_host_stop, +}; + +static const struct ata_port_operations nv_adma_ops = { + .port_disable = ata_port_disable, + .tf_load = ata_tf_load, + .tf_read = ata_tf_read, + .check_atapi_dma = nv_adma_check_atapi_dma, + .exec_command = ata_exec_command, + .check_status = ata_check_status, + .dev_select = ata_std_dev_select, + .bmdma_setup = nv_adma_bmdma_setup, + .bmdma_start = nv_adma_bmdma_start, + .bmdma_stop = nv_adma_bmdma_stop, + .bmdma_status = nv_adma_bmdma_status, + .qc_prep = nv_adma_qc_prep, + .qc_issue = nv_adma_qc_issue, + .freeze = nv_ck804_freeze, + .thaw = nv_ck804_thaw, + .error_handler = nv_adma_error_handler, + .post_internal_cmd = nv_adma_bmdma_stop, + .data_xfer = ata_mmio_data_xfer, + .irq_handler = nv_adma_interrupt, + .irq_clear = nv_adma_irq_clear, + .scr_read = nv_scr_read, + .scr_write = nv_scr_write, + .port_start = nv_adma_port_start, + .port_stop = nv_adma_port_stop, + .host_stop = nv_adma_host_stop, +}; + +static struct ata_port_info nv_port_info[] = { + /* generic */ + { + .sht = &nv_sht, + .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY | + ATA_FLAG_HRST_TO_RESUME, + .pio_mask = NV_PIO_MASK, + .mwdma_mask = NV_MWDMA_MASK, + .udma_mask = NV_UDMA_MASK, + .port_ops = &nv_generic_ops, + }, + /* nforce2/3 */ + { + .sht = &nv_sht, + .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY | + ATA_FLAG_HRST_TO_RESUME, + .pio_mask = NV_PIO_MASK, + .mwdma_mask = NV_MWDMA_MASK, + .udma_mask = NV_UDMA_MASK, + .port_ops = &nv_nf2_ops, + }, + /* ck804 */ + { + .sht = &nv_sht, + .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY | + ATA_FLAG_HRST_TO_RESUME, + .pio_mask = NV_PIO_MASK, + .mwdma_mask = NV_MWDMA_MASK, + .udma_mask = NV_UDMA_MASK, + .port_ops = &nv_ck804_ops, + }, + /* ADMA */ + { + .sht = &nv_adma_sht, + .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY | + ATA_FLAG_MMIO | ATA_FLAG_NCQ, + .pio_mask = NV_PIO_MASK, + .mwdma_mask = NV_MWDMA_MASK, + .udma_mask = NV_UDMA_MASK, + .port_ops = &nv_adma_ops, + }, +}; + +MODULE_AUTHOR("NVIDIA"); +MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller"); +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(pci, nv_pci_tbl); +MODULE_VERSION(DRV_VERSION); + +static int adma_enabled = 1; + +static inline void __iomem *__nv_adma_ctl_block(void __iomem *mmio, + unsigned int port_no) +{ + mmio += NV_ADMA_PORT + port_no * NV_ADMA_PORT_SIZE; + return mmio; +} + +static inline void __iomem *nv_adma_ctl_block(struct ata_port *ap) +{ + return __nv_adma_ctl_block(ap->host->mmio_base, ap->port_no); +} + +static inline void __iomem *nv_adma_gen_block(struct ata_port *ap) +{ + return (ap->host->mmio_base + NV_ADMA_GEN); +} + +static inline void __iomem *nv_adma_notifier_clear_block(struct ata_port *ap) +{ + return (nv_adma_gen_block(ap) + NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no)); +} + +static void nv_adma_register_mode(struct ata_port *ap) +{ + void __iomem *mmio = nv_adma_ctl_block(ap); + struct nv_adma_port_priv *pp = ap->private_data; + u16 tmp; + + if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) + return; + + tmp = readw(mmio + NV_ADMA_CTL); + writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL); + + pp->flags |= NV_ADMA_PORT_REGISTER_MODE; +} + +static void nv_adma_mode(struct ata_port *ap) +{ + void __iomem *mmio = nv_adma_ctl_block(ap); + struct nv_adma_port_priv *pp = ap->private_data; + u16 tmp; + + if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) + return; + + WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE); + + tmp = readw(mmio + NV_ADMA_CTL); + writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL); + + pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE; +} + +static int nv_adma_slave_config(struct scsi_device *sdev) +{ + struct ata_port *ap = ata_shost_to_port(sdev->host); + struct nv_adma_port_priv *pp = ap->private_data; + struct pci_dev *pdev = to_pci_dev(ap->host->dev); + u64 bounce_limit; + unsigned long segment_boundary; + unsigned short sg_tablesize; + int rc; + int adma_enable; + u32 current_reg, new_reg, config_mask; + + rc = ata_scsi_slave_config(sdev); + + if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun) + /* Not a proper libata device, ignore */ + return rc; + + if (ap->device[sdev->id].class == ATA_DEV_ATAPI) { + /* + * NVIDIA reports that ADMA mode does not support ATAPI commands. + * Therefore ATAPI commands are sent through the legacy interface. + * However, the legacy interface only supports 32-bit DMA. + * Restrict DMA parameters as required by the legacy interface + * when an ATAPI device is connected. + */ + bounce_limit = ATA_DMA_MASK; + segment_boundary = ATA_DMA_BOUNDARY; + /* Subtract 1 since an extra entry may be needed for padding, see + libata-scsi.c */ + sg_tablesize = LIBATA_MAX_PRD - 1; + + /* Since the legacy DMA engine is in use, we need to disable ADMA + on the port. */ + adma_enable = 0; + nv_adma_register_mode(ap); + } + else { + bounce_limit = *ap->dev->dma_mask; + segment_boundary = NV_ADMA_DMA_BOUNDARY; + sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN; + adma_enable = 1; + } + + pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, ¤t_reg); + + if(ap->port_no == 1) + config_mask = NV_MCP_SATA_CFG_20_PORT1_EN | + NV_MCP_SATA_CFG_20_PORT1_PWB_EN; + else + config_mask = NV_MCP_SATA_CFG_20_PORT0_EN | + NV_MCP_SATA_CFG_20_PORT0_PWB_EN; + + if(adma_enable) { + new_reg = current_reg | config_mask; + pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE; + } + else { + new_reg = current_reg & ~config_mask; + pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE; + } + + if(current_reg != new_reg) + pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg); + + blk_queue_bounce_limit(sdev->request_queue, bounce_limit); + blk_queue_segment_boundary(sdev->request_queue, segment_boundary); + blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize); + ata_port_printk(ap, KERN_INFO, + "bounce limit 0x%llX, segment boundary 0x%lX, hw segs %hu\n", + (unsigned long long)bounce_limit, segment_boundary, sg_tablesize); + return rc; +} + +static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc) +{ + struct nv_adma_port_priv *pp = qc->ap->private_data; + return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE); +} + +static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb) +{ + unsigned int idx = 0; + + cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device | WNB); + + if ((tf->flags & ATA_TFLAG_LBA48) == 0) { + cpb[idx++] = cpu_to_le16(IGN); + cpb[idx++] = cpu_to_le16(IGN); + cpb[idx++] = cpu_to_le16(IGN); + cpb[idx++] = cpu_to_le16(IGN); + cpb[idx++] = cpu_to_le16(IGN); + } + else { + cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature); + cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect); + cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal); + cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam); + cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah); + } + cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature); + cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect); + cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal); + cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam); + cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah); + + cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND); + + return idx; +} + +static void nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err) +{ + struct nv_adma_port_priv *pp = ap->private_data; + int complete = 0, have_err = 0; + u8 flags = pp->cpb[cpb_num].resp_flags; + + VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags); + + if (flags & NV_CPB_RESP_DONE) { + VPRINTK("CPB flags done, flags=0x%x\n", flags); + complete = 1; + } + if (flags & NV_CPB_RESP_ATA_ERR) { + ata_port_printk(ap, KERN_ERR, "CPB flags ATA err, flags=0x%x\n", flags); + have_err = 1; + complete = 1; + } + if (flags & NV_CPB_RESP_CMD_ERR) { + ata_port_printk(ap, KERN_ERR, "CPB flags CMD err, flags=0x%x\n", flags); + have_err = 1; + complete = 1; + } + if (flags & NV_CPB_RESP_CPB_ERR) { + ata_port_printk(ap, KERN_ERR, "CPB flags CPB err, flags=0x%x\n", flags); + have_err = 1; + complete = 1; + } + if(complete || force_err) + { + struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num); + if(likely(qc)) { + u8 ata_status = 0; + /* Only use the ATA port status for non-NCQ commands. + For NCQ commands the current status may have nothing to do with + the command just completed. */ + if(qc->tf.protocol != ATA_PROT_NCQ) + ata_status = readb(nv_adma_ctl_block(ap) + (ATA_REG_STATUS * 4)); + + if(have_err || force_err) + ata_status |= ATA_ERR; + + qc->err_mask |= ac_err_mask(ata_status); + DPRINTK("Completing qc from tag %d with err_mask %u\n",cpb_num, + qc->err_mask); + ata_qc_complete(qc); + } + } +} + +static int nv_host_intr(struct ata_port *ap, u8 irq_stat) +{ + struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->active_tag); + + /* freeze if hotplugged */ + if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) { + ata_port_freeze(ap); + return 1; + } + + /* bail out if not our interrupt */ + if (!(irq_stat & NV_INT_DEV)) + return 0; + + /* DEV interrupt w/ no active qc? */ + if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) { + ata_check_status(ap); + return 1; + } + + /* handle interrupt */ + return ata_host_intr(ap, qc); +} + +static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance) +{ + struct ata_host *host = dev_instance; + int i, handled = 0; + u32 notifier_clears[2]; + + spin_lock(&host->lock); + + for (i = 0; i < host->n_ports; i++) { + struct ata_port *ap = host->ports[i]; + notifier_clears[i] = 0; + + if (ap && !(ap->flags & ATA_FLAG_DISABLED)) { + struct nv_adma_port_priv *pp = ap->private_data; + void __iomem *mmio = nv_adma_ctl_block(ap); + u16 status; + u32 gen_ctl; + int have_global_err = 0; + u32 notifier, notifier_error; + + /* if in ATA register mode, use standard ata interrupt handler */ + if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) { + u8 irq_stat = readb(host->mmio_base + NV_INT_STATUS_CK804) + >> (NV_INT_PORT_SHIFT * i); + if(ata_tag_valid(ap->active_tag)) + /** NV_INT_DEV indication seems unreliable at times + at least in ADMA mode. Force it on always when a + command is active, to prevent losing interrupts. */ + irq_stat |= NV_INT_DEV; + handled += nv_host_intr(ap, irq_stat); + continue; + } + + notifier = readl(mmio + NV_ADMA_NOTIFIER); + notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR); + notifier_clears[i] = notifier | notifier_error; + + gen_ctl = readl(nv_adma_gen_block(ap) + NV_ADMA_GEN_CTL); + + if( !NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier && + !notifier_error) + /* Nothing to do */ + continue; + + status = readw(mmio + NV_ADMA_STAT); + + /* Clear status. Ensure the controller sees the clearing before we start + looking at any of the CPB statuses, so that any CPB completions after + this point in the handler will raise another interrupt. */ + writew(status, mmio + NV_ADMA_STAT); + readw(mmio + NV_ADMA_STAT); /* flush posted write */ + rmb(); + + /* freeze if hotplugged */ + if (unlikely(status & (NV_ADMA_STAT_HOTPLUG | NV_ADMA_STAT_HOTUNPLUG))) { + ata_port_printk(ap, KERN_NOTICE, "Hotplug event, freezing\n"); + ata_port_freeze(ap); + handled++; + continue; + } + + if (status & NV_ADMA_STAT_TIMEOUT) { + ata_port_printk(ap, KERN_ERR, "timeout, stat=0x%x\n", status); + have_global_err = 1; + } + if (status & NV_ADMA_STAT_CPBERR) { + ata_port_printk(ap, KERN_ERR, "CPB error, stat=0x%x\n", status); + have_global_err = 1; + } + if ((status & NV_ADMA_STAT_DONE) || have_global_err) { + /** Check CPBs for completed commands */ + + if(ata_tag_valid(ap->active_tag)) + /* Non-NCQ command */ + nv_adma_check_cpb(ap, ap->active_tag, have_global_err || + (notifier_error & (1 << ap->active_tag))); + else { + int pos; + u32 active = ap->sactive; + while( (pos = ffs(active)) ) { + pos--; + nv_adma_check_cpb(ap, pos, have_global_err || + (notifier_error & (1 << pos)) ); + active &= ~(1 << pos ); + } + } + } + + handled++; /* irq handled if we got here */ + } + } + + if(notifier_clears[0] || notifier_clears[1]) { + /* Note: Both notifier clear registers must be written + if either is set, even if one is zero, according to NVIDIA. */ + writel(notifier_clears[0], + nv_adma_notifier_clear_block(host->ports[0])); + writel(notifier_clears[1], + nv_adma_notifier_clear_block(host->ports[1])); + } + + spin_unlock(&host->lock); + + return IRQ_RETVAL(handled); +} + +static void nv_adma_irq_clear(struct ata_port *ap) +{ + void __iomem *mmio = nv_adma_ctl_block(ap); + u16 status = readw(mmio + NV_ADMA_STAT); + u32 notifier = readl(mmio + NV_ADMA_NOTIFIER); + u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR); + unsigned long dma_stat_addr = ap->ioaddr.bmdma_addr + ATA_DMA_STATUS; + + /* clear ADMA status */ + writew(status, mmio + NV_ADMA_STAT); + writel(notifier | notifier_error, + nv_adma_notifier_clear_block(ap)); + + /** clear legacy status */ + outb(inb(dma_stat_addr), dma_stat_addr); +} + +static void nv_adma_bmdma_setup(struct ata_queued_cmd *qc) +{ + struct ata_port *ap = qc->ap; + unsigned int rw = (qc->tf.flags & ATA_TFLAG_WRITE); + struct nv_adma_port_priv *pp = ap->private_data; + u8 dmactl; + + if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) { + WARN_ON(1); + return; + } + + /* load PRD table addr. */ + outl(ap->prd_dma, ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS); + + /* specify data direction, triple-check start bit is clear */ + dmactl = inb(ap->ioaddr.bmdma_addr + ATA_DMA_CMD); + dmactl &= ~(ATA_DMA_WR | ATA_DMA_START); + if (!rw) + dmactl |= ATA_DMA_WR; + + outb(dmactl, ap->ioaddr.bmdma_addr + ATA_DMA_CMD); + + /* issue r/w command */ + ata_exec_command(ap, &qc->tf); +} + +static void nv_adma_bmdma_start(struct ata_queued_cmd *qc) +{ + struct ata_port *ap = qc->ap; + struct nv_adma_port_priv *pp = ap->private_data; + u8 dmactl; + + if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) { + WARN_ON(1); + return; + } + + /* start host DMA transaction */ + dmactl = inb(ap->ioaddr.bmdma_addr + ATA_DMA_CMD); + outb(dmactl | ATA_DMA_START, + ap->ioaddr.bmdma_addr + ATA_DMA_CMD); +} + +static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc) +{ + struct ata_port *ap = qc->ap; + struct nv_adma_port_priv *pp = ap->private_data; + + if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) + return; + + /* clear start/stop bit */ + outb(inb(ap->ioaddr.bmdma_addr + ATA_DMA_CMD) & ~ATA_DMA_START, + ap->ioaddr.bmdma_addr + ATA_DMA_CMD); + + /* one-PIO-cycle guaranteed wait, per spec, for HDMA1:0 transition */ + ata_altstatus(ap); /* dummy read */ +} + +static u8 nv_adma_bmdma_status(struct ata_port *ap) +{ + struct nv_adma_port_priv *pp = ap->private_data; + + WARN_ON(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)); + + return inb(ap->ioaddr.bmdma_addr + ATA_DMA_STATUS); +} + +static int nv_adma_port_start(struct ata_port *ap) +{ + struct device *dev = ap->host->dev; + struct nv_adma_port_priv *pp; + int rc; + void *mem; + dma_addr_t mem_dma; + void __iomem *mmio = nv_adma_ctl_block(ap); + u16 tmp; + + VPRINTK("ENTER\n"); + + rc = ata_port_start(ap); + if (rc) + return rc; + + pp = kzalloc(sizeof(*pp), GFP_KERNEL); + if (!pp) { + rc = -ENOMEM; + goto err_out; + } + + mem = dma_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ, + &mem_dma, GFP_KERNEL); + + if (!mem) { + rc = -ENOMEM; + goto err_out_kfree; + } + memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ); + + /* + * First item in chunk of DMA memory: + * 128-byte command parameter block (CPB) + * one for each command tag + */ + pp->cpb = mem; + pp->cpb_dma = mem_dma; + + writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW); + writel((mem_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH); + + mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ; + mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ; + + /* + * Second item: block of ADMA_SGTBL_LEN s/g entries + */ + pp->aprd = mem; + pp->aprd_dma = mem_dma; + + ap->private_data = pp; + + /* clear any outstanding interrupt conditions */ + writew(0xffff, mmio + NV_ADMA_STAT); + + /* initialize port variables */ + pp->flags = NV_ADMA_PORT_REGISTER_MODE; + + /* clear CPB fetch count */ + writew(0, mmio + NV_ADMA_CPB_COUNT); + + /* clear GO for register mode */ + tmp = readw(mmio + NV_ADMA_CTL); + writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL); + + tmp = readw(mmio + NV_ADMA_CTL); + writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); + readl( mmio + NV_ADMA_CTL ); /* flush posted write */ + udelay(1); + writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); + readl( mmio + NV_ADMA_CTL ); /* flush posted write */ + + return 0; + +err_out_kfree: + kfree(pp); +err_out: + ata_port_stop(ap); + return rc; +} + +static void nv_adma_port_stop(struct ata_port *ap) +{ + struct device *dev = ap->host->dev; + struct nv_adma_port_priv *pp = ap->private_data; + void __iomem *mmio = nv_adma_ctl_block(ap); + + VPRINTK("ENTER\n"); + + writew(0, mmio + NV_ADMA_CTL); + + ap->private_data = NULL; + dma_free_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ, pp->cpb, pp->cpb_dma); + kfree(pp); + ata_port_stop(ap); +} + + +static void nv_adma_setup_port(struct ata_probe_ent *probe_ent, unsigned int port) +{ + void __iomem *mmio = probe_ent->mmio_base; + struct ata_ioports *ioport = &probe_ent->port[port]; + + VPRINTK("ENTER\n"); + + mmio += NV_ADMA_PORT + port * NV_ADMA_PORT_SIZE; + + ioport->cmd_addr = (unsigned long) mmio; + ioport->data_addr = (unsigned long) mmio + (ATA_REG_DATA * 4); + ioport->error_addr = + ioport->feature_addr = (unsigned long) mmio + (ATA_REG_ERR * 4); + ioport->nsect_addr = (unsigned long) mmio + (ATA_REG_NSECT * 4); + ioport->lbal_addr = (unsigned long) mmio + (ATA_REG_LBAL * 4); + ioport->lbam_addr = (unsigned long) mmio + (ATA_REG_LBAM * 4); + ioport->lbah_addr = (unsigned long) mmio + (ATA_REG_LBAH * 4); + ioport->device_addr = (unsigned long) mmio + (ATA_REG_DEVICE * 4); + ioport->status_addr = + ioport->command_addr = (unsigned long) mmio + (ATA_REG_STATUS * 4); + ioport->altstatus_addr = + ioport->ctl_addr = (unsigned long) mmio + 0x20; +} + +static int nv_adma_host_init(struct ata_probe_ent *probe_ent) +{ + struct pci_dev *pdev = to_pci_dev(probe_ent->dev); + unsigned int i; + u32 tmp32; + + VPRINTK("ENTER\n"); + + /* enable ADMA on the ports */ + pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32); + tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN | + NV_MCP_SATA_CFG_20_PORT0_PWB_EN | + NV_MCP_SATA_CFG_20_PORT1_EN | + NV_MCP_SATA_CFG_20_PORT1_PWB_EN; + + pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32); + + for (i = 0; i < probe_ent->n_ports; i++) + nv_adma_setup_port(probe_ent, i); + + for (i = 0; i < probe_ent->n_ports; i++) { + void __iomem *mmio = __nv_adma_ctl_block(probe_ent->mmio_base, i); + u16 tmp; + + /* enable interrupt, clear reset if not already clear */ + tmp = readw(mmio + NV_ADMA_CTL); + writew(tmp | NV_ADMA_CTL_AIEN, mmio + NV_ADMA_CTL); + } + + return 0; +} + +static void nv_adma_fill_aprd(struct ata_queued_cmd *qc, + struct scatterlist *sg, + int idx, + struct nv_adma_prd *aprd) +{ + u8 flags; + + memset(aprd, 0, sizeof(struct nv_adma_prd)); + + flags = 0; + if (qc->tf.flags & ATA_TFLAG_WRITE) + flags |= NV_APRD_WRITE; + if (idx == qc->n_elem - 1) + flags |= NV_APRD_END; + else if (idx != 4) + flags |= NV_APRD_CONT; + + aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg))); + aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */ + aprd->flags = flags; +} + +static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb) +{ + struct nv_adma_port_priv *pp = qc->ap->private_data; + unsigned int idx; + struct nv_adma_prd *aprd; + struct scatterlist *sg; + + VPRINTK("ENTER\n"); + + idx = 0; + + ata_for_each_sg(sg, qc) { + aprd = (idx < 5) ? &cpb->aprd[idx] : &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (idx-5)]; + nv_adma_fill_aprd(qc, sg, idx, aprd); + idx++; + } + if (idx > 5) + cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag))); +} + +static void nv_adma_qc_prep(struct ata_queued_cmd *qc) +{ + struct nv_adma_port_priv *pp = qc->ap->private_data; + struct nv_adma_cpb *cpb = &pp->cpb[qc->tag]; + u8 ctl_flags = NV_CPB_CTL_CPB_VALID | + NV_CPB_CTL_APRD_VALID | + NV_CPB_CTL_IEN; + + VPRINTK("qc->flags = 0x%lx\n", qc->flags); + + if (!(qc->flags & ATA_QCFLAG_DMAMAP) || + (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) { + nv_adma_register_mode(qc->ap); + ata_qc_prep(qc); + return; + } + + memset(cpb, 0, sizeof(struct nv_adma_cpb)); + + cpb->len = 3; + cpb->tag = qc->tag; + cpb->next_cpb_idx = 0; + + /* turn on NCQ flags for NCQ commands */ + if (qc->tf.protocol == ATA_PROT_NCQ) + ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA; + + nv_adma_tf_to_cpb(&qc->tf, cpb->tf); + + nv_adma_fill_sg(qc, cpb); + + /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID until we are + finished filling in all of the contents */ + wmb(); + cpb->ctl_flags = ctl_flags; +} + +static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc) +{ + struct nv_adma_port_priv *pp = qc->ap->private_data; + void __iomem *mmio = nv_adma_ctl_block(qc->ap); + int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ); + + VPRINTK("ENTER\n"); + + if (!(qc->flags & ATA_QCFLAG_DMAMAP) || + (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) { + /* use ATA register mode */ + VPRINTK("no dmamap or ATAPI, using ATA register mode: 0x%lx\n", qc->flags); + nv_adma_register_mode(qc->ap); + return ata_qc_issue_prot(qc); + } else + nv_adma_mode(qc->ap); + + /* write append register, command tag in lower 8 bits + and (number of cpbs to append -1) in top 8 bits */ + wmb(); + + if(curr_ncq != pp->last_issue_ncq) { + /* Seems to need some delay before switching between NCQ and non-NCQ + commands, else we get command timeouts and such. */ + udelay(20); + pp->last_issue_ncq = curr_ncq; + } + + writew(qc->tag, mmio + NV_ADMA_APPEND); + + DPRINTK("Issued tag %u\n",qc->tag); + + return 0; +} + +static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance) +{ + struct ata_host *host = dev_instance; + unsigned int i; + unsigned int handled = 0; + unsigned long flags; + + spin_lock_irqsave(&host->lock, flags); + + for (i = 0; i < host->n_ports; i++) { + struct ata_port *ap; + + ap = host->ports[i]; + if (ap && + !(ap->flags & ATA_FLAG_DISABLED)) { + struct ata_queued_cmd *qc; + + qc = ata_qc_from_tag(ap, ap->active_tag); + if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) + handled += ata_host_intr(ap, qc); + else + // No request pending? Clear interrupt status + // anyway, in case there's one pending. + ap->ops->check_status(ap); + } + + } + + spin_unlock_irqrestore(&host->lock, flags); + + return IRQ_RETVAL(handled); +} + +static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat) +{ + int i, handled = 0; + + for (i = 0; i < host->n_ports; i++) { + struct ata_port *ap = host->ports[i]; + + if (ap && !(ap->flags & ATA_FLAG_DISABLED)) + handled += nv_host_intr(ap, irq_stat); + + irq_stat >>= NV_INT_PORT_SHIFT; + } + + return IRQ_RETVAL(handled); +} + +static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance) +{ + struct ata_host *host = dev_instance; + u8 irq_stat; + irqreturn_t ret; + + spin_lock(&host->lock); + irq_stat = inb(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS); + ret = nv_do_interrupt(host, irq_stat); + spin_unlock(&host->lock); + + return ret; +} + +static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance) +{ + struct ata_host *host = dev_instance; + u8 irq_stat; + irqreturn_t ret; + + spin_lock(&host->lock); + irq_stat = readb(host->mmio_base + NV_INT_STATUS_CK804); + ret = nv_do_interrupt(host, irq_stat); + spin_unlock(&host->lock); + + return ret; +} + +static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg) +{ + if (sc_reg > SCR_CONTROL) + return 0xffffffffU; + + return ioread32((void __iomem *)ap->ioaddr.scr_addr + (sc_reg * 4)); +} + +static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val) +{ + if (sc_reg > SCR_CONTROL) + return; + + iowrite32(val, (void __iomem *)ap->ioaddr.scr_addr + (sc_reg * 4)); +} + +static void nv_nf2_freeze(struct ata_port *ap) +{ + unsigned long scr_addr = ap->host->ports[0]->ioaddr.scr_addr; + int shift = ap->port_no * NV_INT_PORT_SHIFT; + u8 mask; + + mask = inb(scr_addr + NV_INT_ENABLE); + mask &= ~(NV_INT_ALL << shift); + outb(mask, scr_addr + NV_INT_ENABLE); +} + +static void nv_nf2_thaw(struct ata_port *ap) +{ + unsigned long scr_addr = ap->host->ports[0]->ioaddr.scr_addr; + int shift = ap->port_no * NV_INT_PORT_SHIFT; + u8 mask; + + outb(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS); + + mask = inb(scr_addr + NV_INT_ENABLE); + mask |= (NV_INT_MASK << shift); + outb(mask, scr_addr + NV_INT_ENABLE); +} + +static void nv_ck804_freeze(struct ata_port *ap) +{ + void __iomem *mmio_base = ap->host->mmio_base; + int shift = ap->port_no * NV_INT_PORT_SHIFT; + u8 mask; + + mask = readb(mmio_base + NV_INT_ENABLE_CK804); + mask &= ~(NV_INT_ALL << shift); + writeb(mask, mmio_base + NV_INT_ENABLE_CK804); +} + +static void nv_ck804_thaw(struct ata_port *ap) +{ + void __iomem *mmio_base = ap->host->mmio_base; + int shift = ap->port_no * NV_INT_PORT_SHIFT; + u8 mask; + + writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804); + + mask = readb(mmio_base + NV_INT_ENABLE_CK804); + mask |= (NV_INT_MASK << shift); + writeb(mask, mmio_base + NV_INT_ENABLE_CK804); +} + +static int nv_hardreset(struct ata_port *ap, unsigned int *class) +{ + unsigned int dummy; + + /* SATA hardreset fails to retrieve proper device signature on + * some controllers. Don't classify on hardreset. For more + * info, see http://bugme.osdl.org/show_bug.cgi?id=3352 + */ + return sata_std_hardreset(ap, &dummy); +} + +static void nv_error_handler(struct ata_port *ap) +{ + ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset, + nv_hardreset, ata_std_postreset); +} + +static void nv_adma_error_handler(struct ata_port *ap) +{ + struct nv_adma_port_priv *pp = ap->private_data; + if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) { + void __iomem *mmio = nv_adma_ctl_block(ap); + int i; + u16 tmp; + + u32 notifier = readl(mmio + NV_ADMA_NOTIFIER); + u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR); + u32 gen_ctl = readl(nv_adma_gen_block(ap) + NV_ADMA_GEN_CTL); + u32 status = readw(mmio + NV_ADMA_STAT); + + ata_port_printk(ap, KERN_ERR, "EH in ADMA mode, notifier 0x%X " + "notifier_error 0x%X gen_ctl 0x%X status 0x%X\n", + notifier, notifier_error, gen_ctl, status); + + for( i=0;icpb[i]; + if( cpb->ctl_flags || cpb->resp_flags ) + ata_port_printk(ap, KERN_ERR, + "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n", + i, cpb->ctl_flags, cpb->resp_flags); + } + + /* Push us back into port register mode for error handling. */ + nv_adma_register_mode(ap); + + ata_port_printk(ap, KERN_ERR, "Resetting port\n"); + + /* Mark all of the CPBs as invalid to prevent them from being executed */ + for( i=0;icpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID; + + /* clear CPB fetch count */ + writew(0, mmio + NV_ADMA_CPB_COUNT); + + /* Reset channel */ + tmp = readw(mmio + NV_ADMA_CTL); + writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); + readl( mmio + NV_ADMA_CTL ); /* flush posted write */ + udelay(1); + writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); + readl( mmio + NV_ADMA_CTL ); /* flush posted write */ + } + + ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset, + nv_hardreset, ata_std_postreset); +} + +static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent) +{ + static int printed_version = 0; + struct ata_port_info *ppi[2]; + struct ata_probe_ent *probe_ent; + int pci_dev_busy = 0; + int rc; + u32 bar; + unsigned long base; + unsigned long type = ent->driver_data; + int mask_set = 0; + + // Make sure this is a SATA controller by counting the number of bars + // (NVIDIA SATA controllers will always have six bars). Otherwise, + // it's an IDE controller and we ignore it. + for (bar=0; bar<6; bar++) + if (pci_resource_start(pdev, bar) == 0) + return -ENODEV; + + if ( !printed_version++) + dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n"); + + rc = pci_enable_device(pdev); + if (rc) + goto err_out; + + rc = pci_request_regions(pdev, DRV_NAME); + if (rc) { + pci_dev_busy = 1; + goto err_out_disable; + } + + if(type >= CK804 && adma_enabled) { + dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n"); + type = ADMA; + if(!pci_set_dma_mask(pdev, DMA_64BIT_MASK) && + !pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK)) + mask_set = 1; + } + + if(!mask_set) { + rc = pci_set_dma_mask(pdev, ATA_DMA_MASK); + if (rc) + goto err_out_regions; + rc = pci_set_consistent_dma_mask(pdev, ATA_DMA_MASK); + if (rc) + goto err_out_regions; + } + + rc = -ENOMEM; + + ppi[0] = ppi[1] = &nv_port_info[type]; + probe_ent = ata_pci_init_native_mode(pdev, ppi, ATA_PORT_PRIMARY | ATA_PORT_SECONDARY); + if (!probe_ent) + goto err_out_regions; + + probe_ent->mmio_base = pci_iomap(pdev, 5, 0); + if (!probe_ent->mmio_base) { + rc = -EIO; + goto err_out_free_ent; + } + + base = (unsigned long)probe_ent->mmio_base; + + probe_ent->port[0].scr_addr = base + NV_PORT0_SCR_REG_OFFSET; + probe_ent->port[1].scr_addr = base + NV_PORT1_SCR_REG_OFFSET; + + /* enable SATA space for CK804 */ + if (type >= CK804) { + u8 regval; + + pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val); + regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN; + pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval); + } + + pci_set_master(pdev); + + if (type == ADMA) { + rc = nv_adma_host_init(probe_ent); + if (rc) + goto err_out_iounmap; + } + + rc = ata_device_add(probe_ent); + if (rc != NV_PORTS) + goto err_out_iounmap; + + kfree(probe_ent); + + return 0; + +err_out_iounmap: + pci_iounmap(pdev, probe_ent->mmio_base); +err_out_free_ent: + kfree(probe_ent); +err_out_regions: + pci_release_regions(pdev); +err_out_disable: + if (!pci_dev_busy) + pci_disable_device(pdev); +err_out: + return rc; +} + +static void nv_ck804_host_stop(struct ata_host *host) +{ + struct pci_dev *pdev = to_pci_dev(host->dev); + u8 regval; + + /* disable SATA space for CK804 */ + pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val); + regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN; + pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval); + + ata_pci_host_stop(host); +} + +static void nv_adma_host_stop(struct ata_host *host) +{ + struct pci_dev *pdev = to_pci_dev(host->dev); + int i; + u32 tmp32; + + for (i = 0; i < host->n_ports; i++) { + void __iomem *mmio = __nv_adma_ctl_block(host->mmio_base, i); + u16 tmp; + + /* disable interrupt */ + tmp = readw(mmio + NV_ADMA_CTL); + writew(tmp & ~NV_ADMA_CTL_AIEN, mmio + NV_ADMA_CTL); + } + + /* disable ADMA on the ports */ + pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32); + tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN | + NV_MCP_SATA_CFG_20_PORT0_PWB_EN | + NV_MCP_SATA_CFG_20_PORT1_EN | + NV_MCP_SATA_CFG_20_PORT1_PWB_EN); + + pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32); + + nv_ck804_host_stop(host); +} + +static int __init nv_init(void) +{ + return pci_register_driver(&nv_pci_driver); +} + +static void __exit nv_exit(void) +{ + pci_unregister_driver(&nv_pci_driver); +} + +module_init(nv_init); +module_exit(nv_exit); +module_param_named(adma, adma_enabled, bool, 0444); +MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");