drivers/net/ixgb/ixgb_main.c

   1 /*******************************************************************************
   2
   3
   4   Copyright(c) 1999 - 2004 Intel Corporation. All rights reserved.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms of the GNU General Public License as published by the Free
   8   Software Foundation; either version 2 of the License, or (at your option)
   9   any later version.
  10
  11   This program is distributed in the hope that it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   You should have received a copy of the GNU General Public License along with
  17   this program; if not, write to the Free Software Foundation, Inc., 59
  18   Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19
  20   The full GNU General Public License is included in this distribution in the
  21   file called LICENSE.
  22
  23   Contact Information:
  24   Linux NICS <linux.nics@intel.com>
  25   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  26
  27 *******************************************************************************/
  28
  29 #include "ixgb.h"
  30
  31 char ixgb_driver_name[] = "ixgb";
  32 char ixgb_driver_string[] = "Intel(R) PRO/10GbE Network Driver";
  33 char ixgb_driver_version[] = "1.0.66";
  34 char ixgb_copyright[] = "Copyright (c) 2001-2004 Intel Corporation.";
  35
  36 /* ixgb_pci_tbl - PCI Device ID Table
  37  *
  38  * Wildcard entries (PCI_ANY_ID) should come last
  39  * Last entry must be all 0s
  40  *
  41  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  42  *   Class, Class Mask, private data (not used) }
  43  */
  44 static struct pci_device_id ixgb_pci_tbl[] = {
  45         {INTEL_VENDOR_ID, IXGB_DEVICE_ID_82597EX,
  46          PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
  47         {INTEL_VENDOR_ID, IXGB_DEVICE_ID_82597EX_SR,
  48          PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
  49
  50         /* required last entry */
  51         {0,}
  52 };
  53
  54 MODULE_DEVICE_TABLE(pci, ixgb_pci_tbl);
  55
  56 /* Local Function Prototypes */
  57
  58 int ixgb_up(struct ixgb_adapter *adapter);
  59 void ixgb_down(struct ixgb_adapter *adapter, boolean_t kill_watchdog);
  60 void ixgb_reset(struct ixgb_adapter *adapter);
  61
  62 static int ixgb_init_module(void);
  63 static void ixgb_exit_module(void);
  64 static int ixgb_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
  65 static void __devexit ixgb_remove(struct pci_dev *pdev);
  66 static int ixgb_sw_init(struct ixgb_adapter *adapter);
  67 static int ixgb_open(struct net_device *netdev);
  68 static int ixgb_close(struct net_device *netdev);
  69 static int ixgb_setup_tx_resources(struct ixgb_adapter *adapter);
  70 static int ixgb_setup_rx_resources(struct ixgb_adapter *adapter);
  71 static void ixgb_configure_tx(struct ixgb_adapter *adapter);
  72 static void ixgb_configure_rx(struct ixgb_adapter *adapter);
  73 static void ixgb_setup_rctl(struct ixgb_adapter *adapter);
  74 static void ixgb_clean_tx_ring(struct ixgb_adapter *adapter);
  75 static void ixgb_clean_rx_ring(struct ixgb_adapter *adapter);
  76 static void ixgb_free_tx_resources(struct ixgb_adapter *adapter);
  77 static void ixgb_free_rx_resources(struct ixgb_adapter *adapter);
  78 static void ixgb_set_multi(struct net_device *netdev);
  79 static void ixgb_watchdog(unsigned long data);
  80 static int ixgb_xmit_frame(struct sk_buff *skb, struct net_device *netdev);
  81 static struct net_device_stats *ixgb_get_stats(struct net_device *netdev);
  82 static int ixgb_change_mtu(struct net_device *netdev, int new_mtu);
  83 static int ixgb_set_mac(struct net_device *netdev, void *p);
  84 static void ixgb_update_stats(struct ixgb_adapter *adapter);
  85 static inline void ixgb_irq_disable(struct ixgb_adapter *adapter);
  86 static inline void ixgb_irq_enable(struct ixgb_adapter *adapter);
  87 static irqreturn_t ixgb_intr(int irq, void *data, struct pt_regs *regs);
  88 static boolean_t ixgb_clean_tx_irq(struct ixgb_adapter *adapter);
  89 #ifdef CONFIG_IXGB_NAPI
  90 static int ixgb_clean(struct net_device *netdev, int *budget);
  91 static boolean_t ixgb_clean_rx_irq(struct ixgb_adapter *adapter,
  92                                    int *work_done, int work_to_do);
  93 #else
  94 static boolean_t ixgb_clean_rx_irq(struct ixgb_adapter *adapter);
  95 #endif
  96 static void ixgb_alloc_rx_buffers(struct ixgb_adapter *adapter);
  97 static int ixgb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
  98 static inline void ixgb_rx_checksum(struct ixgb_adapter *adapter,
  99                                     struct ixgb_rx_desc *rx_desc,
 100                                     struct sk_buff *skb);
 101 static void ixgb_tx_timeout(struct net_device *dev);
 102 static void ixgb_tx_timeout_task(struct net_device *dev);
 103 static void ixgb_vlan_rx_register(struct net_device *netdev,
 104                                   struct vlan_group *grp);
 105 static void ixgb_vlan_rx_add_vid(struct net_device *netdev, uint16_t vid);
 106 static void ixgb_vlan_rx_kill_vid(struct net_device *netdev, uint16_t vid);
 107 static void ixgb_restore_vlan(struct ixgb_adapter *adapter);
 108
 109 static int ixgb_notify_reboot(struct notifier_block *, unsigned long event,
 110                               void *ptr);
 111 static int ixgb_suspend(struct pci_dev *pdev, uint32_t state);
 112
 113 #ifdef CONFIG_NET_POLL_CONTROLLER
 114 /* for netdump / net console */
 115 static void ixgb_netpoll(struct net_device *dev);
 116 #endif
 117
 118 struct notifier_block ixgb_notifier_reboot = {
 119         .notifier_call = ixgb_notify_reboot,
 120         .next = NULL,
 121         .priority = 0
 122 };
 123
 124 /* Exported from other modules */
 125
 126 extern void ixgb_check_options(struct ixgb_adapter *adapter);
 127 extern int ixgb_ethtool_ioctl(struct net_device *netdev, struct ifreq *ifr);
 128
 129 static struct pci_driver ixgb_driver = {
 130         .name = ixgb_driver_name,
 131         .id_table = ixgb_pci_tbl,
 132         .probe = ixgb_probe,
 133         .remove = __devexit_p(ixgb_remove),
 134         /* Power Managment Hooks */
 135         .suspend = NULL,
 136         .resume = NULL
 137 };
 138
 139 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 140 MODULE_DESCRIPTION("Intel(R) PRO/10GbE Network Driver");
 141 MODULE_LICENSE("GPL");
 142
 143 /* some defines for controlling descriptor fetches in h/w */
 144 #define RXDCTL_PTHRESH_DEFAULT 128      /* chip considers prefech below this */
 145 #define RXDCTL_HTHRESH_DEFAULT 16       /* chip will only prefetch if tail is
 146                                            pushed this many descriptors from head */
 147 #define RXDCTL_WTHRESH_DEFAULT 16       /* chip writes back at this many or RXT0 */
 148
 149 /**
 150  * ixgb_init_module - Driver Registration Routine
 151  *
 152  * ixgb_init_module is the first routine called when the driver is
 153  * loaded. All it does is register with the PCI subsystem.
 154  **/
 155
 156 static int __init ixgb_init_module(void)
 157 {
 158         int ret;
 159         printk(KERN_INFO "%s - version %s\n",
 160                ixgb_driver_string, ixgb_driver_version);
 161
 162         printk(KERN_INFO "%s\n", ixgb_copyright);
 163
 164         ret = pci_module_init(&ixgb_driver);
 165         if (ret >= 0) {
 166                 register_reboot_notifier(&ixgb_notifier_reboot);
 167         }
 168         return ret;
 169 }
 170
 171 module_init(ixgb_init_module);
 172
 173 /**
 174  * ixgb_exit_module - Driver Exit Cleanup Routine
 175  *
 176  * ixgb_exit_module is called just before the driver is removed
 177  * from memory.
 178  **/
 179
 180 static void __exit ixgb_exit_module(void)
 181 {
 182         unregister_reboot_notifier(&ixgb_notifier_reboot);
 183         pci_unregister_driver(&ixgb_driver);
 184 }
 185
 186 module_exit(ixgb_exit_module);
 187
 188 int ixgb_up(struct ixgb_adapter *adapter)
 189 {
 190         struct net_device *netdev = adapter->netdev;
 191         int err;
 192         int max_frame = netdev->mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH;
 193         struct ixgb_hw *hw = &adapter->hw;
 194
 195         /* hardware has been reset, we need to reload some things */
 196
 197         ixgb_set_multi(netdev);
 198
 199         ixgb_restore_vlan(adapter);
 200
 201         ixgb_configure_tx(adapter);
 202         ixgb_setup_rctl(adapter);
 203         ixgb_configure_rx(adapter);
 204         ixgb_alloc_rx_buffers(adapter);
 205
 206         if ((err = request_irq(adapter->pdev->irq, &ixgb_intr,
 207                                SA_SHIRQ | SA_SAMPLE_RANDOM,
 208                                netdev->name, netdev)))
 209                 return err;
 210
 211         /* disable interrupts and get the hardware into a known state */
 212         IXGB_WRITE_REG(&adapter->hw, IMC, 0xffffffff);
 213
 214         if ((hw->max_frame_size != max_frame) ||
 215             (hw->max_frame_size !=
 216              (IXGB_READ_REG(hw, MFS) >> IXGB_MFS_SHIFT))) {
 217
 218                 hw->max_frame_size = max_frame;
 219
 220                 IXGB_WRITE_REG(hw, MFS, hw->max_frame_size << IXGB_MFS_SHIFT);
 221
 222                 if (hw->max_frame_size >
 223                     IXGB_MAX_ENET_FRAME_SIZE_WITHOUT_FCS + ENET_FCS_LENGTH) {
 224                         uint32_t ctrl0 = IXGB_READ_REG(hw, CTRL0);
 225
 226                         if (!(ctrl0 & IXGB_CTRL0_JFE)) {
 227                                 ctrl0 |= IXGB_CTRL0_JFE;
 228                                 IXGB_WRITE_REG(hw, CTRL0, ctrl0);
 229                         }
 230                 }
 231         }
 232
 233         mod_timer(&adapter->watchdog_timer, jiffies);
 234         ixgb_irq_enable(adapter);
 235
 236         return 0;
 237 }
 238
 239 void ixgb_down(struct ixgb_adapter *adapter, boolean_t kill_watchdog)
 240 {
 241         struct net_device *netdev = adapter->netdev;
 242
 243         ixgb_irq_disable(adapter);
 244         free_irq(adapter->pdev->irq, netdev);
 245         if (kill_watchdog)
 246                 del_timer_sync(&adapter->watchdog_timer);
 247         adapter->link_speed = 0;
 248         adapter->link_duplex = 0;
 249         netif_carrier_off(netdev);
 250         netif_stop_queue(netdev);
 251
 252         ixgb_reset(adapter);
 253         ixgb_clean_tx_ring(adapter);
 254         ixgb_clean_rx_ring(adapter);
 255 }
 256
 257 void ixgb_reset(struct ixgb_adapter *adapter)
 258 {
 259
 260         ixgb_adapter_stop(&adapter->hw);
 261         if (!ixgb_init_hw(&adapter->hw))
 262                 IXGB_DBG("ixgb_init_hw failed.\n");
 263 }
 264
 265 /**
 266  * ixgb_probe - Device Initialization Routine
 267  * @pdev: PCI device information struct
 268  * @ent: entry in ixgb_pci_tbl
 269  *
 270  * Returns 0 on success, negative on failure
 271  *
 272  * ixgb_probe initializes an adapter identified by a pci_dev structure.
 273  * The OS initialization, configuring of the adapter private structure,
 274  * and a hardware reset occur.
 275  **/
 276
 277 static int __devinit
 278 ixgb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 279 {
 280         struct net_device *netdev = NULL;
 281         struct ixgb_adapter *adapter;
 282         static int cards_found = 0;
 283         unsigned long mmio_start;
 284         int mmio_len;
 285         int pci_using_dac;
 286         int i;
 287         int err;
 288
 289         if ((err = pci_enable_device(pdev)))
 290                 return err;
 291
 292         if (!(err = pci_set_dma_mask(pdev, DMA_64BIT_MASK))) {
 293                 pci_using_dac = 1;
 294         } else {
 295                 if ((err = pci_set_dma_mask(pdev, DMA_32BIT_MASK))) {
 296                         IXGB_ERR("No usable DMA configuration, aborting\n");
 297                         return err;
 298                 }
 299                 pci_using_dac = 0;
 300         }
 301
 302         if ((err = pci_request_regions(pdev, ixgb_driver_name)))
 303                 return err;
 304
 305         pci_set_master(pdev);
 306
 307         netdev = alloc_etherdev(sizeof(struct ixgb_adapter));
 308         if (!netdev) {
 309                 err = -ENOMEM;
 310                 goto err_alloc_etherdev;
 311         }
 312
 313         SET_MODULE_OWNER(netdev);
 314         SET_NETDEV_DEV(netdev, &pdev->dev);
 315
 316         pci_set_drvdata(pdev, netdev);
 317         adapter = netdev->priv;
 318         adapter->netdev = netdev;
 319         adapter->pdev = pdev;
 320         adapter->hw.back = adapter;
 321
 322         mmio_start = pci_resource_start(pdev, BAR_0);
 323         mmio_len = pci_resource_len(pdev, BAR_0);
 324
 325         adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
 326         if (!adapter->hw.hw_addr) {
 327                 err = -EIO;
 328                 goto err_ioremap;
 329         }
 330
 331         for (i = BAR_1; i <= BAR_5; i++) {
 332                 if (pci_resource_len(pdev, i) == 0)
 333                         continue;
 334                 if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
 335                         adapter->hw.io_base = pci_resource_start(pdev, i);
 336                         break;
 337                 }
 338         }
 339
 340         netdev->open = &ixgb_open;
 341         netdev->stop = &ixgb_close;
 342         netdev->hard_start_xmit = &ixgb_xmit_frame;
 343         netdev->get_stats = &ixgb_get_stats;
 344         netdev->set_multicast_list = &ixgb_set_multi;
 345         netdev->set_mac_address = &ixgb_set_mac;
 346         netdev->change_mtu = &ixgb_change_mtu;
 347         netdev->do_ioctl = &ixgb_ioctl;
 348         netdev->tx_timeout = &ixgb_tx_timeout;
 349         netdev->watchdog_timeo = HZ;
 350 #ifdef CONFIG_IXGB_NAPI
 351         netdev->poll = &ixgb_clean;
 352         netdev->weight = 64;
 353 #endif
 354         netdev->vlan_rx_register = ixgb_vlan_rx_register;
 355         netdev->vlan_rx_add_vid = ixgb_vlan_rx_add_vid;
 356         netdev->vlan_rx_kill_vid = ixgb_vlan_rx_kill_vid;
 357 #ifdef CONFIG_NET_POLL_CONTROLLER
 358         netdev->poll_controller = ixgb_netpoll;
 359 #endif
 360
 361         netdev->mem_start = mmio_start;
 362         netdev->mem_end = mmio_start + mmio_len;
 363         netdev->base_addr = adapter->hw.io_base;
 364
 365         adapter->bd_number = cards_found;
 366         adapter->link_speed = 0;
 367         adapter->link_duplex = 0;
 368
 369         /* setup the private structure */
 370
 371         if ((err = ixgb_sw_init(adapter)))
 372                 goto err_sw_init;
 373
 374         netdev->features = NETIF_F_SG |
 375             NETIF_F_HW_CSUM |
 376             NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_FILTER;
 377 #ifdef NETIF_F_TSO
 378         netdev->features |= NETIF_F_TSO;
 379 #endif
 380
 381         if (pci_using_dac)
 382                 netdev->features |= NETIF_F_HIGHDMA;
 383
 384         /* make sure the EEPROM is good */
 385
 386         if (!ixgb_validate_eeprom_checksum(&adapter->hw)) {
 387                 printk(KERN_ERR "The EEPROM Checksum Is Not Valid\n");
 388                 err = -EIO;
 389                 goto err_eeprom;
 390         }
 391
 392         ixgb_get_ee_mac_addr(&adapter->hw, netdev->dev_addr);
 393
 394         if (!is_valid_ether_addr(netdev->dev_addr)) {
 395                 err = -EIO;
 396                 goto err_eeprom;
 397         }
 398
 399         adapter->part_num = ixgb_get_ee_pba_number(&adapter->hw);
 400
 401         init_timer(&adapter->watchdog_timer);
 402         adapter->watchdog_timer.function = &ixgb_watchdog;
 403         adapter->watchdog_timer.data = (unsigned long)adapter;
 404
 405         INIT_WORK(&adapter->tx_timeout_task,
 406                   (void (*)(void *))ixgb_tx_timeout_task, netdev);
 407
 408         if ((err = register_netdev(netdev)))
 409                 goto err_register;
 410
 411         /* we're going to reset, so assume we have no link for now */
 412
 413         netif_carrier_off(netdev);
 414         netif_stop_queue(netdev);
 415
 416         printk(KERN_INFO "%s: Intel(R) PRO/10GbE Network Connection\n",
 417                netdev->name);
 418         ixgb_check_options(adapter);
 419         /* reset the hardware with the new settings */
 420
 421         ixgb_reset(adapter);
 422
 423         cards_found++;
 424         return 0;
 425
 426       err_register:
 427       err_sw_init:
 428       err_eeprom:
 429         iounmap(adapter->hw.hw_addr);
 430       err_ioremap:
 431         free_netdev(netdev);
 432       err_alloc_etherdev:
 433         pci_release_regions(pdev);
 434         return err;
 435 }
 436
 437 /**
 438  * ixgb_remove - Device Removal Routine
 439  * @pdev: PCI device information struct
 440  *
 441  * ixgb_remove is called by the PCI subsystem to alert the driver
 442  * that it should release a PCI device.  The could be caused by a
 443  * Hot-Plug event, or because the driver is going to be removed from
 444  * memory.
 445  **/
 446
 447 static void __devexit ixgb_remove(struct pci_dev *pdev)
 448 {
 449         struct net_device *netdev = pci_get_drvdata(pdev);
 450         struct ixgb_adapter *adapter = netdev->priv;
 451
 452         unregister_netdev(netdev);
 453
 454         iounmap(adapter->hw.hw_addr);
 455         pci_release_regions(pdev);
 456
 457         free_netdev(netdev);
 458 }
 459
 460 /**
 461  * ixgb_sw_init - Initialize general software structures (struct ixgb_adapter)
 462  * @adapter: board private structure to initialize
 463  *
 464  * ixgb_sw_init initializes the Adapter private data structure.
 465  * Fields are initialized based on PCI device information and
 466  * OS network device settings (MTU size).
 467  **/
 468
 469 static int __devinit ixgb_sw_init(struct ixgb_adapter *adapter)
 470 {
 471         struct ixgb_hw *hw = &adapter->hw;
 472         struct net_device *netdev = adapter->netdev;
 473         struct pci_dev *pdev = adapter->pdev;
 474
 475         /* PCI config space info */
 476
 477         hw->vendor_id = pdev->vendor;
 478         hw->device_id = pdev->device;
 479         hw->subsystem_vendor_id = pdev->subsystem_vendor;
 480         hw->subsystem_id = pdev->subsystem_device;
 481
 482         adapter->rx_buffer_len = IXGB_RXBUFFER_2048;
 483
 484         hw->max_frame_size = netdev->mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH;
 485
 486         if ((hw->device_id == IXGB_DEVICE_ID_82597EX)
 487             || (hw->device_id == IXGB_DEVICE_ID_82597EX_SR))
 488                 hw->mac_type = ixgb_82597;
 489         else {
 490                 /* should never have loaded on this device */
 491                 printk(KERN_ERR "ixgb: unsupported device id\n");
 492         }
 493
 494         /* enable flow control to be programmed */
 495         hw->fc.send_xon = 1;
 496
 497         atomic_set(&adapter->irq_sem, 1);
 498         spin_lock_init(&adapter->tx_lock);
 499
 500         return 0;
 501 }
 502
 503 /**
 504  * ixgb_open - Called when a network interface is made active
 505  * @netdev: network interface device structure
 506  *
 507  * Returns 0 on success, negative value on failure
 508  *
 509  * The open entry point is called when a network interface is made
 510  * active by the system (IFF_UP).  At this point all resources needed
 511  * for transmit and receive operations are allocated, the interrupt
 512  * handler is registered with the OS, the watchdog timer is started,
 513  * and the stack is notified that the interface is ready.
 514  **/
 515
 516 static int ixgb_open(struct net_device *netdev)
 517 {
 518         struct ixgb_adapter *adapter = netdev->priv;
 519         int err;
 520
 521         /* allocate transmit descriptors */
 522
 523         if ((err = ixgb_setup_tx_resources(adapter)))
 524                 goto err_setup_tx;
 525
 526         /* allocate receive descriptors */
 527
 528         if ((err = ixgb_setup_rx_resources(adapter)))
 529                 goto err_setup_rx;
 530
 531         if ((err = ixgb_up(adapter)))
 532                 goto err_up;
 533
 534         return 0;
 535
 536       err_up:
 537         ixgb_free_rx_resources(adapter);
 538       err_setup_rx:
 539         ixgb_free_tx_resources(adapter);
 540       err_setup_tx:
 541         ixgb_reset(adapter);
 542
 543         return err;
 544 }
 545
 546 /**
 547  * ixgb_close - Disables a network interface
 548  * @netdev: network interface device structure
 549  *
 550  * Returns 0, this is not allowed to fail
 551  *
 552  * The close entry point is called when an interface is de-activated
 553  * by the OS.  The hardware is still under the drivers control, but
 554  * needs to be disabled.  A global MAC reset is issued to stop the
 555  * hardware, and all transmit and receive resources are freed.
 556  **/
 557
 558 static int ixgb_close(struct net_device *netdev)
 559 {
 560         struct ixgb_adapter *adapter = netdev->priv;
 561
 562         ixgb_down(adapter, TRUE);
 563
 564         ixgb_free_tx_resources(adapter);
 565         ixgb_free_rx_resources(adapter);
 566
 567         return 0;
 568 }
 569
 570 /**
 571  * ixgb_setup_tx_resources - allocate Tx resources (Descriptors)
 572  * @adapter: board private structure
 573  *
 574  * Return 0 on success, negative on failure
 575  **/
 576
 577 static int ixgb_setup_tx_resources(struct ixgb_adapter *adapter)
 578 {
 579         struct ixgb_desc_ring *txdr = &adapter->tx_ring;
 580         struct pci_dev *pdev = adapter->pdev;
 581         int size;
 582
 583         size = sizeof(struct ixgb_buffer) * txdr->count;
 584         txdr->buffer_info = kmalloc(size, GFP_KERNEL);
 585         if (!txdr->buffer_info) {
 586                 return -ENOMEM;
 587         }
 588         memset(txdr->buffer_info, 0, size);
 589
 590         /* round up to nearest 4K */
 591
 592         txdr->size = txdr->count * sizeof(struct ixgb_tx_desc);
 593         IXGB_ROUNDUP(txdr->size, 4096);
 594
 595         txdr->desc = pci_alloc_consistent(pdev, txdr->size, &txdr->dma);
 596         if (!txdr->desc) {
 597                 kfree(txdr->buffer_info);
 598                 return -ENOMEM;
 599         }
 600         memset(txdr->desc, 0, txdr->size);
 601
 602         txdr->next_to_use = 0;
 603         txdr->next_to_clean = 0;
 604
 605         return 0;
 606 }
 607
 608 /**
 609  * ixgb_configure_tx - Configure 82597 Transmit Unit after Reset.
 610  * @adapter: board private structure
 611  *
 612  * Configure the Tx unit of the MAC after a reset.
 613  **/
 614
 615 static void ixgb_configure_tx(struct ixgb_adapter *adapter)
 616 {
 617         uint64_t tdba = adapter->tx_ring.dma;
 618         uint32_t tdlen = adapter->tx_ring.count * sizeof(struct ixgb_tx_desc);
 619         uint32_t tctl;
 620         struct ixgb_hw *hw = &adapter->hw;
 621
 622         /* Setup the Base and Length of the Tx Descriptor Ring
 623          * tx_ring.dma can be either a 32 or 64 bit value
 624          */
 625
 626         IXGB_WRITE_REG(hw, TDBAL, (tdba & 0x00000000ffffffffULL));
 627         IXGB_WRITE_REG(hw, TDBAH, (tdba >> 32));
 628
 629         IXGB_WRITE_REG(hw, TDLEN, tdlen);
 630
 631         /* Setup the HW Tx Head and Tail descriptor pointers */
 632
 633         IXGB_WRITE_REG(hw, TDH, 0);
 634         IXGB_WRITE_REG(hw, TDT, 0);
 635
 636         /* don't set up txdctl, it induces performance problems if
 637          * configured incorrectly
 638          txdctl  = TXDCTL_PTHRESH_DEFAULT; // prefetch txds below this threshold
 639          txdctl |= (TXDCTL_HTHRESH_DEFAULT // only prefetch if there are this many ready
 640          << IXGB_TXDCTL_HTHRESH_SHIFT);
 641          IXGB_WRITE_REG (hw, TXDCTL, txdctl);
 642          */
 643
 644         /* Set the Tx Interrupt Delay register */
 645
 646         IXGB_WRITE_REG(hw, TIDV, adapter->tx_int_delay);
 647
 648         /* Program the Transmit Control Register */
 649
 650         tctl = IXGB_TCTL_TCE | IXGB_TCTL_TXEN | IXGB_TCTL_TPDE;
 651         IXGB_WRITE_REG(hw, TCTL, tctl);
 652
 653         /* Setup Transmit Descriptor Settings for this adapter */
 654         adapter->tx_cmd_type =
 655             IXGB_TX_DESC_TYPE
 656             | (adapter->tx_int_delay_enable ? IXGB_TX_DESC_CMD_IDE : 0);
 657 }
 658
 659 /**
 660  * ixgb_setup_rx_resources - allocate Rx resources (Descriptors)
 661  * @adapter: board private structure
 662  *
 663  * Returns 0 on success, negative on failure
 664  **/
 665
 666 static int ixgb_setup_rx_resources(struct ixgb_adapter *adapter)
 667 {
 668         struct ixgb_desc_ring *rxdr = &adapter->rx_ring;
 669         struct pci_dev *pdev = adapter->pdev;
 670         int size;
 671
 672         size = sizeof(struct ixgb_buffer) * rxdr->count;
 673         rxdr->buffer_info = kmalloc(size, GFP_KERNEL);
 674         if (!rxdr->buffer_info) {
 675                 return -ENOMEM;
 676         }
 677         memset(rxdr->buffer_info, 0, size);
 678
 679         /* Round up to nearest 4K */
 680
 681         rxdr->size = rxdr->count * sizeof(struct ixgb_rx_desc);
 682         IXGB_ROUNDUP(rxdr->size, 4096);
 683
 684         rxdr->desc = pci_alloc_consistent(pdev, rxdr->size, &rxdr->dma);
 685
 686         if (!rxdr->desc) {
 687                 kfree(rxdr->buffer_info);
 688                 return -ENOMEM;
 689         }
 690         memset(rxdr->desc, 0, rxdr->size);
 691
 692         rxdr->next_to_clean = 0;
 693         rxdr->next_to_use = 0;
 694
 695         return 0;
 696 }
 697
 698 /**
 699  * ixgb_setup_rctl - configure the receive control register
 700  * @adapter: Board private structure
 701  **/
 702
 703 static void ixgb_setup_rctl(struct ixgb_adapter *adapter)
 704 {
 705         uint32_t rctl;
 706
 707         rctl = IXGB_READ_REG(&adapter->hw, RCTL);
 708
 709         rctl &= ~(3 << IXGB_RCTL_MO_SHIFT);
 710
 711         rctl |=
 712             IXGB_RCTL_BAM | IXGB_RCTL_RDMTS_1_2 |
 713             IXGB_RCTL_RXEN | IXGB_RCTL_CFF |
 714             (adapter->hw.mc_filter_type << IXGB_RCTL_MO_SHIFT);
 715
 716         rctl |= IXGB_RCTL_SECRC;
 717
 718         switch (adapter->rx_buffer_len) {
 719         case IXGB_RXBUFFER_2048:
 720         default:
 721                 rctl |= IXGB_RCTL_BSIZE_2048;
 722                 break;
 723         case IXGB_RXBUFFER_4096:
 724                 rctl |= IXGB_RCTL_BSIZE_4096;
 725                 break;
 726         case IXGB_RXBUFFER_8192:
 727                 rctl |= IXGB_RCTL_BSIZE_8192;
 728                 break;
 729         case IXGB_RXBUFFER_16384:
 730                 rctl |= IXGB_RCTL_BSIZE_16384;
 731                 break;
 732         }
 733
 734         IXGB_WRITE_REG(&adapter->hw, RCTL, rctl);
 735 }
 736
 737 /**
 738  * ixgb_configure_rx - Configure 82597 Receive Unit after Reset.
 739  * @adapter: board private structure
 740  *
 741  * Configure the Rx unit of the MAC after a reset.
 742  **/
 743
 744 static void ixgb_configure_rx(struct ixgb_adapter *adapter)
 745 {
 746         uint64_t rdba = adapter->rx_ring.dma;
 747         uint32_t rdlen = adapter->rx_ring.count * sizeof(struct ixgb_rx_desc);
 748         struct ixgb_hw *hw = &adapter->hw;
 749         uint32_t rctl;
 750         uint32_t rxcsum;
 751         uint32_t rxdctl;
 752
 753         /* make sure receives are disabled while setting up the descriptors */
 754
 755         rctl = IXGB_READ_REG(hw, RCTL);
 756         IXGB_WRITE_REG(hw, RCTL, rctl & ~IXGB_RCTL_RXEN);
 757
 758         /* set the Receive Delay Timer Register */
 759
 760         IXGB_WRITE_REG(hw, RDTR, adapter->rx_int_delay);
 761
 762         /* Setup the Base and Length of the Rx Descriptor Ring */
 763
 764         IXGB_WRITE_REG(hw, RDBAL, (rdba & 0x00000000ffffffffULL));
 765         IXGB_WRITE_REG(hw, RDBAH, (rdba >> 32));
 766
 767         IXGB_WRITE_REG(hw, RDLEN, rdlen);
 768
 769         /* Setup the HW Rx Head and Tail Descriptor Pointers */
 770         IXGB_WRITE_REG(hw, RDH, 0);
 771         IXGB_WRITE_REG(hw, RDT, 0);
 772
 773         /* burst 16 or burst when RXT0 */
 774         rxdctl = RXDCTL_WTHRESH_DEFAULT << IXGB_RXDCTL_WTHRESH_SHIFT
 775             | RXDCTL_HTHRESH_DEFAULT << IXGB_RXDCTL_HTHRESH_SHIFT
 776             | RXDCTL_PTHRESH_DEFAULT << IXGB_RXDCTL_PTHRESH_SHIFT;
 777         IXGB_WRITE_REG(hw, RXDCTL, rxdctl);
 778
 779         if (adapter->raidc) {
 780                 uint32_t raidc;
 781                 uint8_t poll_threshold;
 782
 783                 /* Poll every rx_int_delay period, if RBD exists
 784                  * Receive Backlog Detection is set to <threshold>
 785                  * Rx Descriptors
 786                  * max is 0x3F == set to poll when 504 RxDesc left
 787                  * min is 0 */
 788
 789                 /* polling times are 1 == 0.8192us
 790                    2 == 1.6384us
 791                    3 == 3.2768us etc
 792                    ...
 793                    511 == 418 us
 794                  */
 795 #define IXGB_RAIDC_POLL_DEFAULT 122     /* set to poll every ~100 us under load
 796                                            also known as 10000 interrupts / sec */
 797
 798                 /* divide this by 2^3 (8) to get a register size count */
 799                 poll_threshold = ((adapter->rx_ring.count - 1) >> 3);
 800                 /* poll at half of that size */
 801                 poll_threshold >>= 1;
 802                 /* make sure its not bigger than our max */
 803                 poll_threshold &= 0x3F;
 804
 805                 raidc = IXGB_RAIDC_EN | /* turn on raidc style moderation */
 806                     IXGB_RAIDC_RXT_GATE |       /* don't interrupt with rxt0 while
 807                                                    in RBD mode (polling) */
 808                     (IXGB_RAIDC_POLL_DEFAULT << IXGB_RAIDC_POLL_SHIFT) |
 809                     /* this sets the regular "min interrupt delay" */
 810                     (adapter->rx_int_delay << IXGB_RAIDC_DELAY_SHIFT) |
 811                     poll_threshold;
 812
 813                 IXGB_WRITE_REG(hw, RAIDC, raidc);
 814         }
 815
 816         /* Enable Receive Checksum Offload for TCP and UDP */
 817         if (adapter->rx_csum == TRUE) {
 818                 rxcsum = IXGB_READ_REG(hw, RXCSUM);
 819                 rxcsum |= IXGB_RXCSUM_TUOFL;
 820                 IXGB_WRITE_REG(hw, RXCSUM, rxcsum);
 821         }
 822
 823         /* Enable Receives */
 824
 825         IXGB_WRITE_REG(hw, RCTL, rctl);
 826 }
 827
 828 /**
 829  * ixgb_free_tx_resources - Free Tx Resources
 830  * @adapter: board private structure
 831  *
 832  * Free all transmit software resources
 833  **/
 834
 835 static void ixgb_free_tx_resources(struct ixgb_adapter *adapter)
 836 {
 837         struct pci_dev *pdev = adapter->pdev;
 838
 839         ixgb_clean_tx_ring(adapter);
 840
 841         kfree(adapter->tx_ring.buffer_info);
 842         adapter->tx_ring.buffer_info = NULL;
 843
 844         pci_free_consistent(pdev, adapter->tx_ring.size,
 845                             adapter->tx_ring.desc, adapter->tx_ring.dma);
 846
 847         adapter->tx_ring.desc = NULL;
 848 }
 849
 850 /**
 851  * ixgb_clean_tx_ring - Free Tx Buffers
 852  * @adapter: board private structure
 853  **/
 854
 855 static void ixgb_clean_tx_ring(struct ixgb_adapter *adapter)
 856 {
 857         struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
 858         struct ixgb_buffer *buffer_info;
 859         struct pci_dev *pdev = adapter->pdev;
 860         unsigned long size;
 861         unsigned int i;
 862
 863         /* Free all the Tx ring sk_buffs */
 864
 865         for (i = 0; i < tx_ring->count; i++) {
 866                 buffer_info = &tx_ring->buffer_info[i];
 867                 if (buffer_info->skb) {
 868
 869                         pci_unmap_page(pdev,
 870                                        buffer_info->dma,
 871                                        buffer_info->length, PCI_DMA_TODEVICE);
 872
 873                         dev_kfree_skb(buffer_info->skb);
 874
 875                         buffer_info->skb = NULL;
 876                 }
 877         }
 878
 879         size = sizeof(struct ixgb_buffer) * tx_ring->count;
 880         memset(tx_ring->buffer_info, 0, size);
 881
 882         /* Zero out the descriptor ring */
 883
 884         memset(tx_ring->desc, 0, tx_ring->size);
 885
 886         tx_ring->next_to_use = 0;
 887         tx_ring->next_to_clean = 0;
 888
 889         IXGB_WRITE_REG(&adapter->hw, TDH, 0);
 890         IXGB_WRITE_REG(&adapter->hw, TDT, 0);
 891 }
 892
 893 /**
 894  * ixgb_free_rx_resources - Free Rx Resources
 895  * @adapter: board private structure
 896  *
 897  * Free all receive software resources
 898  **/
 899
 900 static void ixgb_free_rx_resources(struct ixgb_adapter *adapter)
 901 {
 902         struct ixgb_desc_ring *rx_ring = &adapter->rx_ring;
 903         struct pci_dev *pdev = adapter->pdev;
 904
 905         ixgb_clean_rx_ring(adapter);
 906
 907         kfree(rx_ring->buffer_info);
 908         rx_ring->buffer_info = NULL;
 909
 910         pci_free_consistent(pdev, rx_ring->size, rx_ring->desc, rx_ring->dma);
 911
 912         rx_ring->desc = NULL;
 913 }
 914
 915 /**
 916  * ixgb_clean_rx_ring - Free Rx Buffers
 917  * @adapter: board private structure
 918  **/
 919
 920 static void ixgb_clean_rx_ring(struct ixgb_adapter *adapter)
 921 {
 922         struct ixgb_desc_ring *rx_ring = &adapter->rx_ring;
 923         struct ixgb_buffer *buffer_info;
 924         struct pci_dev *pdev = adapter->pdev;
 925         unsigned long size;
 926         unsigned int i;
 927
 928         /* Free all the Rx ring sk_buffs */
 929
 930         for (i = 0; i < rx_ring->count; i++) {
 931                 buffer_info = &rx_ring->buffer_info[i];
 932                 if (buffer_info->skb) {
 933
 934                         pci_unmap_single(pdev,
 935                                          buffer_info->dma,
 936                                          buffer_info->length,
 937                                          PCI_DMA_FROMDEVICE);
 938
 939                         dev_kfree_skb(buffer_info->skb);
 940
 941                         buffer_info->skb = NULL;
 942                 }
 943         }
 944
 945         size = sizeof(struct ixgb_buffer) * rx_ring->count;
 946         memset(rx_ring->buffer_info, 0, size);
 947
 948         /* Zero out the descriptor ring */
 949
 950         memset(rx_ring->desc, 0, rx_ring->size);
 951
 952         rx_ring->next_to_clean = 0;
 953         rx_ring->next_to_use = 0;
 954
 955         IXGB_WRITE_REG(&adapter->hw, RDH, 0);
 956         IXGB_WRITE_REG(&adapter->hw, RDT, 0);
 957 }
 958
 959 /**
 960  * ixgb_set_mac - Change the Ethernet Address of the NIC
 961  * @netdev: network interface device structure
 962  * @p: pointer to an address structure
 963  *
 964  * Returns 0 on success, negative on failure
 965  **/
 966
 967 static int ixgb_set_mac(struct net_device *netdev, void *p)
 968 {
 969         struct ixgb_adapter *adapter = netdev->priv;
 970         struct sockaddr *addr = p;
 971
 972         if (!is_valid_ether_addr(addr->sa_data))
 973                 return -EADDRNOTAVAIL;
 974
 975         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 976
 977         ixgb_rar_set(&adapter->hw, addr->sa_data, 0);
 978
 979         return 0;
 980 }
 981
 982 /**
 983  * ixgb_set_multi - Multicast and Promiscuous mode set
 984  * @netdev: network interface device structure
 985  *
 986  * The set_multi entry point is called whenever the multicast address
 987  * list or the network interface flags are updated.  This routine is
 988  * responsible for configuring the hardware for proper multicast,
 989  * promiscuous mode, and all-multi behavior.
 990  **/
 991
 992 static void ixgb_set_multi(struct net_device *netdev)
 993 {
 994         struct ixgb_adapter *adapter = netdev->priv;
 995         struct ixgb_hw *hw = &adapter->hw;
 996         struct dev_mc_list *mc_ptr;
 997         uint32_t rctl;
 998         int i;
 999
1000         /* Check for Promiscuous and All Multicast modes */
1001
1002         rctl = IXGB_READ_REG(hw, RCTL);
1003
1004         if (netdev->flags & IFF_PROMISC) {
1005                 rctl |= (IXGB_RCTL_UPE | IXGB_RCTL_MPE);
1006         } else if (netdev->flags & IFF_ALLMULTI) {
1007                 rctl |= IXGB_RCTL_MPE;
1008                 rctl &= ~IXGB_RCTL_UPE;
1009         } else {
1010                 rctl &= ~(IXGB_RCTL_UPE | IXGB_RCTL_MPE);
1011         }
1012
1013         if (netdev->mc_count > IXGB_MAX_NUM_MULTICAST_ADDRESSES) {
1014                 rctl |= IXGB_RCTL_MPE;
1015                 IXGB_WRITE_REG(hw, RCTL, rctl);
1016         } else {
1017                 uint8_t mta[netdev->mc_count * IXGB_ETH_LENGTH_OF_ADDRESS];
1018
1019                 IXGB_WRITE_REG(hw, RCTL, rctl);
1020
1021                 for (i = 0, mc_ptr = netdev->mc_list; mc_ptr;
1022                      i++, mc_ptr = mc_ptr->next)
1023                         memcpy(&mta[i * IXGB_ETH_LENGTH_OF_ADDRESS],
1024                                mc_ptr->dmi_addr, IXGB_ETH_LENGTH_OF_ADDRESS);
1025
1026                 ixgb_mc_addr_list_update(hw, mta, netdev->mc_count, 0);
1027         }
1028 }
1029
1030 /**
1031  * ixgb_watchdog - Timer Call-back
1032  * @data: pointer to netdev cast into an unsigned long
1033  **/
1034
1035 static void ixgb_watchdog(unsigned long data)
1036 {
1037         struct ixgb_adapter *adapter = (struct ixgb_adapter *)data;
1038         struct net_device *netdev = adapter->netdev;
1039         struct ixgb_desc_ring *txdr = &adapter->tx_ring;
1040         unsigned int i;
1041
1042         ixgb_check_for_link(&adapter->hw);
1043
1044         if (ixgb_check_for_bad_link(&adapter->hw)) {
1045                 /* force the reset path */
1046                 netif_stop_queue(netdev);
1047         }
1048
1049         if (adapter->hw.link_up) {
1050                 if (!netif_carrier_ok(netdev)) {
1051                         printk(KERN_INFO "ixgb: %s NIC Link is Up %d Mbps %s\n",
1052                                netdev->name, 10000, "Full Duplex");
1053                         adapter->link_speed = 10000;
1054                         adapter->link_duplex = FULL_DUPLEX;
1055                         netif_carrier_on(netdev);
1056                         netif_wake_queue(netdev);
1057                 }
1058         } else {
1059                 if (netif_carrier_ok(netdev)) {
1060                         adapter->link_speed = 0;
1061                         adapter->link_duplex = 0;
1062                         printk(KERN_INFO
1063                                "ixgb: %s NIC Link is Down\n", netdev->name);
1064                         netif_carrier_off(netdev);
1065                         netif_stop_queue(netdev);
1066
1067                 }
1068         }
1069
1070         ixgb_update_stats(adapter);
1071
1072         if (!netif_carrier_ok(netdev)) {
1073                 if (IXGB_DESC_UNUSED(txdr) + 1 < txdr->count) {
1074                         /* We've lost link, so the controller stops DMA,
1075                          * but we've got queued Tx work that's never going
1076                          * to get done, so reset controller to flush Tx.
1077                          * (Do the reset outside of interrupt context). */
1078                         schedule_work(&adapter->tx_timeout_task);
1079                 }
1080         }
1081
1082         /* Early detection of hung controller */
1083         i = txdr->next_to_clean;
1084         if (txdr->buffer_info[i].dma &&
1085             time_after(jiffies, txdr->buffer_info[i].time_stamp + HZ) &&
1086             !(IXGB_READ_REG(&adapter->hw, STATUS) & IXGB_STATUS_TXOFF))
1087                 netif_stop_queue(netdev);
1088
1089         /* generate an interrupt to force clean up of any stragglers */
1090         IXGB_WRITE_REG(&adapter->hw, ICS, IXGB_INT_TXDW);
1091
1092         /* Reset the timer */
1093         mod_timer(&adapter->watchdog_timer, jiffies + 2 * HZ);
1094 }
1095
1096 #define IXGB_TX_FLAGS_CSUM              0x00000001
1097 #define IXGB_TX_FLAGS_VLAN              0x00000002
1098 #define IXGB_TX_FLAGS_TSO               0x00000004
1099
1100 static inline boolean_t
1101 ixgb_tso(struct ixgb_adapter *adapter, struct sk_buff *skb)
1102 {
1103 #ifdef NETIF_F_TSO
1104         struct ixgb_context_desc *context_desc;
1105         unsigned int i;
1106         uint8_t ipcss, ipcso, tucss, tucso, hdr_len;
1107         uint16_t ipcse, tucse, mss;
1108
1109         if (likely(skb_shinfo(skb)->tso_size)) {
1110                 hdr_len = ((skb->h.raw - skb->data) + (skb->h.th->doff << 2));
1111                 mss = skb_shinfo(skb)->tso_size;
1112                 skb->nh.iph->tot_len = 0;
1113                 skb->nh.iph->check = 0;
1114                 skb->h.th->check = ~csum_tcpudp_magic(skb->nh.iph->saddr,
1115                                                       skb->nh.iph->daddr,
1116                                                       0, IPPROTO_TCP, 0);
1117                 ipcss = skb->nh.raw - skb->data;
1118                 ipcso = (void *)&(skb->nh.iph->check) - (void *)skb->data;
1119                 ipcse = skb->h.raw - skb->data - 1;
1120                 tucss = skb->h.raw - skb->data;
1121                 tucso = (void *)&(skb->h.th->check) - (void *)skb->data;
1122                 tucse = 0;
1123
1124                 i = adapter->tx_ring.next_to_use;
1125                 context_desc = IXGB_CONTEXT_DESC(adapter->tx_ring, i);
1126
1127                 context_desc->ipcss = ipcss;
1128                 context_desc->ipcso = ipcso;
1129                 context_desc->ipcse = cpu_to_le16(ipcse);
1130                 context_desc->tucss = tucss;
1131                 context_desc->tucso = tucso;
1132                 context_desc->tucse = cpu_to_le16(tucse);
1133                 context_desc->mss = cpu_to_le16(mss);
1134                 context_desc->hdr_len = hdr_len;
1135                 context_desc->status = 0;
1136                 context_desc->cmd_type_len = cpu_to_le32(IXGB_CONTEXT_DESC_TYPE
1137                                                          |
1138                                                          IXGB_CONTEXT_DESC_CMD_TSE
1139                                                          |
1140                                                          IXGB_CONTEXT_DESC_CMD_IP
1141                                                          |
1142                                                          IXGB_CONTEXT_DESC_CMD_TCP
1143                                                          |
1144                                                          IXGB_CONTEXT_DESC_CMD_RS
1145                                                          |
1146                                                          IXGB_CONTEXT_DESC_CMD_IDE
1147                                                          | (skb->len -
1148                                                             (hdr_len)));
1149
1150                 if (++i == adapter->tx_ring.count)
1151                         i = 0;
1152                 adapter->tx_ring.next_to_use = i;
1153
1154                 return TRUE;
1155         }
1156 #endif
1157
1158         return FALSE;
1159 }
1160
1161 static inline boolean_t
1162 ixgb_tx_csum(struct ixgb_adapter *adapter, struct sk_buff *skb)
1163 {
1164         struct ixgb_context_desc *context_desc;
1165         unsigned int i;
1166         uint8_t css, cso;
1167
1168         if (likely(skb->ip_summed == CHECKSUM_HW)) {
1169                 css = skb->h.raw - skb->data;
1170                 cso = (skb->h.raw + skb->csum) - skb->data;
1171
1172                 i = adapter->tx_ring.next_to_use;
1173                 context_desc = IXGB_CONTEXT_DESC(adapter->tx_ring, i);
1174
1175                 context_desc->tucss = css;
1176                 context_desc->tucso = cso;
1177                 context_desc->tucse = 0;
1178                 /* zero out any previously existing data in one instruction */
1179                 *(uint32_t *) & (context_desc->ipcss) = 0;
1180                 context_desc->status = 0;
1181                 context_desc->hdr_len = 0;
1182                 context_desc->mss = 0;
1183                 context_desc->cmd_type_len =
1184                     cpu_to_le32(IXGB_CONTEXT_DESC_TYPE
1185                                 | IXGB_TX_DESC_CMD_RS | IXGB_TX_DESC_CMD_IDE);
1186
1187                 if (++i == adapter->tx_ring.count)
1188                         i = 0;
1189                 adapter->tx_ring.next_to_use = i;
1190
1191                 return TRUE;
1192         }
1193
1194         return FALSE;
1195 }
1196
1197 #define IXGB_MAX_TXD_PWR        14
1198 #define IXGB_MAX_DATA_PER_TXD   (1<<IXGB_MAX_TXD_PWR)
1199
1200 static inline int
1201 ixgb_tx_map(struct ixgb_adapter *adapter, struct sk_buff *skb,
1202             unsigned int first)
1203 {
1204         struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
1205         struct ixgb_buffer *buffer_info;
1206         int len = skb->len;
1207         unsigned int offset = 0, size, count = 0, i;
1208
1209         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
1210         unsigned int f;
1211         len -= skb->data_len;
1212
1213         i = tx_ring->next_to_use;
1214
1215         while (len) {
1216                 buffer_info = &tx_ring->buffer_info[i];
1217                 size = min(len, IXGB_MAX_JUMBO_FRAME_SIZE);
1218                 buffer_info->length = size;
1219                 buffer_info->dma =
1220                     pci_map_single(adapter->pdev,
1221                                    skb->data + offset, size, PCI_DMA_TODEVICE);
1222                 buffer_info->time_stamp = jiffies;
1223
1224                 len -= size;
1225                 offset += size;
1226                 count++;
1227                 if (++i == tx_ring->count)
1228                         i = 0;
1229         }
1230
1231         for (f = 0; f < nr_frags; f++) {
1232                 struct skb_frag_struct *frag;
1233
1234                 frag = &skb_shinfo(skb)->frags[f];
1235                 len = frag->size;
1236                 offset = 0;
1237
1238                 while (len) {
1239                         buffer_info = &tx_ring->buffer_info[i];
1240                         size = min(len, IXGB_MAX_JUMBO_FRAME_SIZE);
1241                         buffer_info->length = size;
1242                         buffer_info->dma =
1243                             pci_map_page(adapter->pdev,
1244                                          frag->page,
1245                                          frag->page_offset + offset,
1246                                          size, PCI_DMA_TODEVICE);
1247                         buffer_info->time_stamp = jiffies;
1248
1249                         len -= size;
1250                         offset += size;
1251                         count++;
1252                         if (++i == tx_ring->count)
1253                                 i = 0;
1254                 }
1255         }
1256         i = (i == 0) ? tx_ring->count - 1 : i - 1;
1257         tx_ring->buffer_info[i].skb = skb;
1258         tx_ring->buffer_info[first].next_to_watch = i;
1259
1260         return count;
1261 }
1262
1263 static inline void
1264 ixgb_tx_queue(struct ixgb_adapter *adapter, int count, int vlan_id,
1265               int tx_flags)
1266 {
1267         struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
1268         struct ixgb_tx_desc *tx_desc = NULL;
1269         struct ixgb_buffer *buffer_info;
1270         uint32_t cmd_type_len = adapter->tx_cmd_type;
1271         uint8_t status = 0;
1272         uint8_t popts = 0;
1273         unsigned int i;
1274
1275         if (tx_flags & IXGB_TX_FLAGS_TSO) {
1276                 cmd_type_len |= IXGB_TX_DESC_CMD_TSE;
1277                 popts |= (IXGB_TX_DESC_POPTS_IXSM | IXGB_TX_DESC_POPTS_TXSM);
1278         }
1279
1280         if (tx_flags & IXGB_TX_FLAGS_CSUM)
1281                 popts |= IXGB_TX_DESC_POPTS_TXSM;
1282
1283         if (tx_flags & IXGB_TX_FLAGS_VLAN) {
1284                 cmd_type_len |= IXGB_TX_DESC_CMD_VLE;
1285         }
1286
1287         i = tx_ring->next_to_use;
1288
1289         while (count--) {
1290                 buffer_info = &tx_ring->buffer_info[i];
1291                 tx_desc = IXGB_TX_DESC(*tx_ring, i);
1292                 tx_desc->buff_addr = cpu_to_le64(buffer_info->dma);
1293                 tx_desc->cmd_type_len =
1294                     cpu_to_le32(cmd_type_len | buffer_info->length);
1295                 tx_desc->status = status;
1296                 tx_desc->popts = popts;
1297                 tx_desc->vlan = cpu_to_le16(vlan_id);
1298
1299                 if (++i == tx_ring->count)
1300                         i = 0;
1301         }
1302
1303         tx_desc->cmd_type_len |= cpu_to_le32(IXGB_TX_DESC_CMD_EOP
1304                                              | IXGB_TX_DESC_CMD_RS);
1305
1306         /* Force memory writes to complete before letting h/w
1307          * know there are new descriptors to fetch.  (Only
1308          * applicable for weak-ordered memory model archs,
1309          * such as IA-64). */
1310         wmb();
1311
1312         tx_ring->next_to_use = i;
1313         IXGB_WRITE_REG(&adapter->hw, TDT, i);
1314 }
1315
1316 /* Tx Descriptors needed, worst case */
1317 #define TXD_USE_COUNT(S) (((S) >> IXGB_MAX_TXD_PWR) + \
1318                          (((S) & (IXGB_MAX_DATA_PER_TXD - 1)) ? 1 : 0))
1319 #define DESC_NEEDED TXD_USE_COUNT(IXGB_MAX_DATA_PER_TXD) + \
1320         MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1
1321
1322 static int ixgb_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1323 {
1324         struct ixgb_adapter *adapter = netdev->priv;
1325         unsigned int first;
1326         unsigned int tx_flags = 0;
1327         unsigned long flags;
1328         int vlan_id = 0;
1329
1330         if (skb->len <= 0) {
1331                 dev_kfree_skb_any(skb);
1332                 return 0;
1333         }
1334
1335         spin_lock_irqsave(&adapter->tx_lock, flags);
1336         if (unlikely(IXGB_DESC_UNUSED(&adapter->tx_ring) < DESC_NEEDED)) {
1337                 netif_stop_queue(netdev);
1338                 spin_unlock_irqrestore(&adapter->tx_lock, flags);
1339                 return 1;
1340         }
1341         spin_unlock_irqrestore(&adapter->tx_lock, flags);
1342
1343         if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
1344                 tx_flags |= IXGB_TX_FLAGS_VLAN;
1345                 vlan_id = vlan_tx_tag_get(skb);
1346         }
1347
1348         first = adapter->tx_ring.next_to_use;
1349
1350         if (ixgb_tso(adapter, skb))
1351                 tx_flags |= IXGB_TX_FLAGS_TSO;
1352         else if (ixgb_tx_csum(adapter, skb))
1353                 tx_flags |= IXGB_TX_FLAGS_CSUM;
1354
1355         ixgb_tx_queue(adapter, ixgb_tx_map(adapter, skb, first), vlan_id,
1356                       tx_flags);
1357
1358         netdev->trans_start = jiffies;
1359
1360         return 0;
1361 }
1362
1363 /**
1364  * ixgb_tx_timeout - Respond to a Tx Hang
1365  * @netdev: network interface device structure
1366  **/
1367
1368 static void ixgb_tx_timeout(struct net_device *netdev)
1369 {
1370         struct ixgb_adapter *adapter = netdev->priv;
1371
1372         /* Do the reset outside of interrupt context */
1373         schedule_work(&adapter->tx_timeout_task);
1374 }
1375
1376 static void ixgb_tx_timeout_task(struct net_device *netdev)
1377 {
1378         struct ixgb_adapter *adapter = netdev->priv;
1379
1380         netif_device_detach(netdev);
1381         ixgb_down(adapter, TRUE);
1382         ixgb_up(adapter);
1383         netif_device_attach(netdev);
1384 }
1385
1386 /**
1387  * ixgb_get_stats - Get System Network Statistics
1388  * @netdev: network interface device structure
1389  *
1390  * Returns the address of the device statistics structure.
1391  * The statistics are actually updated from the timer callback.
1392  **/
1393
1394 static struct net_device_stats *ixgb_get_stats(struct net_device *netdev)
1395 {
1396         struct ixgb_adapter *adapter = netdev->priv;
1397
1398         return &adapter->net_stats;
1399 }
1400
1401 /**
1402  * ixgb_change_mtu - Change the Maximum Transfer Unit
1403  * @netdev: network interface device structure
1404  * @new_mtu: new value for maximum frame size
1405  *
1406  * Returns 0 on success, negative on failure
1407  **/
1408
1409 static int ixgb_change_mtu(struct net_device *netdev, int new_mtu)
1410 {
1411         struct ixgb_adapter *adapter = netdev->priv;
1412         uint32_t old_mtu = adapter->rx_buffer_len;
1413         int max_frame = new_mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH;
1414
1415         if ((max_frame < IXGB_MIN_ENET_FRAME_SIZE_WITHOUT_FCS + ENET_FCS_LENGTH)
1416             || (max_frame > IXGB_MAX_JUMBO_FRAME_SIZE + ENET_FCS_LENGTH)) {
1417                 IXGB_ERR("Invalid MTU setting\n");
1418                 return -EINVAL;
1419         }
1420
1421         if ((max_frame <=
1422              IXGB_MAX_ENET_FRAME_SIZE_WITHOUT_FCS + ENET_FCS_LENGTH)
1423             || (max_frame <= IXGB_RXBUFFER_2048)) {
1424                 adapter->rx_buffer_len = IXGB_RXBUFFER_2048;
1425
1426         } else if (max_frame <= IXGB_RXBUFFER_4096) {
1427                 adapter->rx_buffer_len = IXGB_RXBUFFER_4096;
1428
1429         } else if (max_frame <= IXGB_RXBUFFER_8192) {
1430                 adapter->rx_buffer_len = IXGB_RXBUFFER_8192;
1431
1432         } else {
1433                 adapter->rx_buffer_len = IXGB_RXBUFFER_16384;
1434         }
1435
1436         netdev->mtu = new_mtu;
1437
1438         if (old_mtu != adapter->rx_buffer_len && netif_running(netdev)) {
1439
1440                 ixgb_down(adapter, TRUE);
1441                 ixgb_up(adapter);
1442         }
1443
1444         return 0;
1445 }
1446
1447 /**
1448  * ixgb_update_stats - Update the board statistics counters.
1449  * @adapter: board private structure
1450  **/
1451
1452 static void ixgb_update_stats(struct ixgb_adapter *adapter)
1453 {
1454         adapter->stats.tprl += IXGB_READ_REG(&adapter->hw, TPRL);
1455         adapter->stats.tprh += IXGB_READ_REG(&adapter->hw, TPRH);
1456         adapter->stats.gprcl += IXGB_READ_REG(&adapter->hw, GPRCL);
1457         adapter->stats.gprch += IXGB_READ_REG(&adapter->hw, GPRCH);
1458         adapter->stats.bprcl += IXGB_READ_REG(&adapter->hw, BPRCL);
1459         adapter->stats.bprch += IXGB_READ_REG(&adapter->hw, BPRCH);
1460         adapter->stats.mprcl += IXGB_READ_REG(&adapter->hw, MPRCL);
1461         adapter->stats.mprch += IXGB_READ_REG(&adapter->hw, MPRCH);
1462         adapter->stats.uprcl += IXGB_READ_REG(&adapter->hw, UPRCL);
1463         adapter->stats.uprch += IXGB_READ_REG(&adapter->hw, UPRCH);
1464         adapter->stats.vprcl += IXGB_READ_REG(&adapter->hw, VPRCL);
1465         adapter->stats.vprch += IXGB_READ_REG(&adapter->hw, VPRCH);
1466         adapter->stats.jprcl += IXGB_READ_REG(&adapter->hw, JPRCL);
1467         adapter->stats.jprch += IXGB_READ_REG(&adapter->hw, JPRCH);
1468         adapter->stats.gorcl += IXGB_READ_REG(&adapter->hw, GORCL);
1469         adapter->stats.gorch += IXGB_READ_REG(&adapter->hw, GORCH);
1470         adapter->stats.torl += IXGB_READ_REG(&adapter->hw, TORL);
1471         adapter->stats.torh += IXGB_READ_REG(&adapter->hw, TORH);
1472         adapter->stats.rnbc += IXGB_READ_REG(&adapter->hw, RNBC);
1473         adapter->stats.ruc += IXGB_READ_REG(&adapter->hw, RUC);
1474         adapter->stats.roc += IXGB_READ_REG(&adapter->hw, ROC);
1475         adapter->stats.rlec += IXGB_READ_REG(&adapter->hw, RLEC);
1476         adapter->stats.crcerrs += IXGB_READ_REG(&adapter->hw, CRCERRS);
1477         adapter->stats.icbc += IXGB_READ_REG(&adapter->hw, ICBC);
1478         adapter->stats.ecbc += IXGB_READ_REG(&adapter->hw, ECBC);
1479         adapter->stats.mpc += IXGB_READ_REG(&adapter->hw, MPC);
1480         adapter->stats.tptl += IXGB_READ_REG(&adapter->hw, TPTL);
1481         adapter->stats.tpth += IXGB_READ_REG(&adapter->hw, TPTH);
1482         adapter->stats.gptcl += IXGB_READ_REG(&adapter->hw, GPTCL);
1483         adapter->stats.gptch += IXGB_READ_REG(&adapter->hw, GPTCH);
1484         adapter->stats.bptcl += IXGB_READ_REG(&adapter->hw, BPTCL);
1485         adapter->stats.bptch += IXGB_READ_REG(&adapter->hw, BPTCH);
1486         adapter->stats.mptcl += IXGB_READ_REG(&adapter->hw, MPTCL);
1487         adapter->stats.mptch += IXGB_READ_REG(&adapter->hw, MPTCH);
1488         adapter->stats.uptcl += IXGB_READ_REG(&adapter->hw, UPTCL);
1489         adapter->stats.uptch += IXGB_READ_REG(&adapter->hw, UPTCH);
1490         adapter->stats.vptcl += IXGB_READ_REG(&adapter->hw, VPTCL);
1491         adapter->stats.vptch += IXGB_READ_REG(&adapter->hw, VPTCH);
1492         adapter->stats.jptcl += IXGB_READ_REG(&adapter->hw, JPTCL);
1493         adapter->stats.jptch += IXGB_READ_REG(&adapter->hw, JPTCH);
1494         adapter->stats.gotcl += IXGB_READ_REG(&adapter->hw, GOTCL);
1495         adapter->stats.gotch += IXGB_READ_REG(&adapter->hw, GOTCH);
1496         adapter->stats.totl += IXGB_READ_REG(&adapter->hw, TOTL);
1497         adapter->stats.toth += IXGB_READ_REG(&adapter->hw, TOTH);
1498         adapter->stats.dc += IXGB_READ_REG(&adapter->hw, DC);
1499         adapter->stats.plt64c += IXGB_READ_REG(&adapter->hw, PLT64C);
1500         adapter->stats.tsctc += IXGB_READ_REG(&adapter->hw, TSCTC);
1501         adapter->stats.tsctfc += IXGB_READ_REG(&adapter->hw, TSCTFC);
1502         adapter->stats.ibic += IXGB_READ_REG(&adapter->hw, IBIC);
1503         adapter->stats.rfc += IXGB_READ_REG(&adapter->hw, RFC);
1504         adapter->stats.lfc += IXGB_READ_REG(&adapter->hw, LFC);
1505         adapter->stats.pfrc += IXGB_READ_REG(&adapter->hw, PFRC);
1506         adapter->stats.pftc += IXGB_READ_REG(&adapter->hw, PFTC);
1507         adapter->stats.mcfrc += IXGB_READ_REG(&adapter->hw, MCFRC);
1508         adapter->stats.mcftc += IXGB_READ_REG(&adapter->hw, MCFTC);
1509         adapter->stats.xonrxc += IXGB_READ_REG(&adapter->hw, XONRXC);
1510         adapter->stats.xontxc += IXGB_READ_REG(&adapter->hw, XONTXC);
1511         adapter->stats.xoffrxc += IXGB_READ_REG(&adapter->hw, XOFFRXC);
1512         adapter->stats.xofftxc += IXGB_READ_REG(&adapter->hw, XOFFTXC);
1513         adapter->stats.rjc += IXGB_READ_REG(&adapter->hw, RJC);
1514
1515         /* Fill out the OS statistics structure */
1516
1517         adapter->net_stats.rx_packets = adapter->stats.gprcl;
1518         adapter->net_stats.tx_packets = adapter->stats.gptcl;
1519         adapter->net_stats.rx_bytes = adapter->stats.gorcl;
1520         adapter->net_stats.tx_bytes = adapter->stats.gotcl;
1521         adapter->net_stats.multicast = adapter->stats.mprcl;
1522         adapter->net_stats.collisions = 0;
1523
1524         /* ignore RLEC as it reports errors for padded (<64bytes) frames
1525          * with a length in the type/len field */
1526         adapter->net_stats.rx_errors =
1527             /* adapter->stats.rnbc + */ adapter->stats.crcerrs +
1528             adapter->stats.ruc +
1529             adapter->stats.roc /*+ adapter->stats.rlec */  +
1530             adapter->stats.icbc +
1531             adapter->stats.ecbc + adapter->stats.mpc;
1532
1533         adapter->net_stats.rx_dropped = adapter->stats.mpc;
1534
1535         /* see above
1536          * adapter->net_stats.rx_length_errors = adapter->stats.rlec;
1537          */
1538
1539         adapter->net_stats.rx_crc_errors = adapter->stats.crcerrs;
1540         adapter->net_stats.rx_fifo_errors = adapter->stats.mpc;
1541         adapter->net_stats.rx_missed_errors = adapter->stats.mpc;
1542         adapter->net_stats.rx_over_errors = adapter->stats.mpc;
1543
1544         adapter->net_stats.tx_errors = 0;
1545         adapter->net_stats.rx_frame_errors = 0;
1546         adapter->net_stats.tx_aborted_errors = 0;
1547         adapter->net_stats.tx_carrier_errors = 0;
1548         adapter->net_stats.tx_fifo_errors = 0;
1549         adapter->net_stats.tx_heartbeat_errors = 0;
1550         adapter->net_stats.tx_window_errors = 0;
1551 }
1552
1553 /**
1554  * ixgb_irq_disable - Mask off interrupt generation on the NIC
1555  * @adapter: board private structure
1556  **/
1557
1558 static inline void ixgb_irq_disable(struct ixgb_adapter *adapter)
1559 {
1560         atomic_inc(&adapter->irq_sem);
1561         IXGB_WRITE_REG(&adapter->hw, IMC, ~0);
1562         IXGB_WRITE_FLUSH(&adapter->hw);
1563         synchronize_irq(adapter->pdev->irq);
1564 }
1565
1566 /**
1567  * ixgb_irq_enable - Enable default interrupt generation settings
1568  * @adapter: board private structure
1569  **/
1570
1571 static inline void ixgb_irq_enable(struct ixgb_adapter *adapter)
1572 {
1573         if (atomic_dec_and_test(&adapter->irq_sem)) {
1574                 IXGB_WRITE_REG(&adapter->hw, IMS,
1575                                IXGB_INT_RXT0 | IXGB_INT_RXDMT0 | IXGB_INT_TXDW |
1576                                IXGB_INT_RXO | IXGB_INT_LSC);
1577                 IXGB_WRITE_FLUSH(&adapter->hw);
1578         }
1579 }
1580
1581 #define IXGB_MAX_INTR 10
1582 /**
1583  * ixgb_intr - Interrupt Handler
1584  * @irq: interrupt number
1585  * @data: pointer to a network interface device structure
1586  * @pt_regs: CPU registers structure
1587  **/
1588
1589 static irqreturn_t ixgb_intr(int irq, void *data, struct pt_regs *regs)
1590 {
1591         struct net_device *netdev = data;
1592         struct ixgb_adapter *adapter = netdev->priv;
1593         struct ixgb_hw *hw = &adapter->hw;
1594         uint32_t icr = IXGB_READ_REG(&adapter->hw, ICR);
1595 #ifndef CONFIG_IXGB_NAPI
1596         unsigned int i;
1597 #endif
1598
1599         if (unlikely(!icr))
1600                 return IRQ_NONE;        /* Not our interrupt */
1601
1602         if (unlikely(icr & (IXGB_INT_RXSEQ | IXGB_INT_LSC))) {
1603                 mod_timer(&adapter->watchdog_timer, jiffies);
1604         }
1605 #ifdef CONFIG_IXGB_NAPI
1606         if (netif_rx_schedule_prep(netdev)) {
1607
1608                 /* Disable interrupts and register for poll. The flush
1609                    of the posted write is intentionally left out.
1610                  */
1611
1612                 atomic_inc(&adapter->irq_sem);
1613                 IXGB_WRITE_REG(hw, IMC, ~0);
1614                 __netif_rx_schedule(netdev);
1615         }
1616 #else
1617         for (i = 0; i < IXGB_MAX_INTR; i++)
1618                 if (!ixgb_clean_rx_irq(adapter) & !ixgb_clean_tx_irq(adapter))
1619                         break;
1620         /* if RAIDC:EN == 1 and ICR:RXDMT0 == 1, we need to
1621          * set IMS:RXDMT0 to 1 to restart the RBD timer (POLL)
1622          */
1623         if ((icr & IXGB_INT_RXDMT0) && adapter->raidc) {
1624                 /* ready the timer by writing the clear reg */
1625                 IXGB_WRITE_REG(hw, IMC, IXGB_INT_RXDMT0);
1626                 /* now restart it, h/w will decide if its necessary */
1627                 IXGB_WRITE_REG(hw, IMS, IXGB_INT_RXDMT0);
1628         }
1629 #endif
1630         return IRQ_HANDLED;
1631 }
1632
1633 #ifdef CONFIG_IXGB_NAPI
1634 /**
1635  * ixgb_clean - NAPI Rx polling callback
1636  * @adapter: board private structure
1637  **/
1638
1639 static int ixgb_clean(struct net_device *netdev, int *budget)
1640 {
1641         struct ixgb_adapter *adapter = netdev->priv;
1642         int work_to_do = min(*budget, netdev->quota);
1643         int work_done = 0;
1644
1645         ixgb_clean_tx_irq(adapter);
1646         ixgb_clean_rx_irq(adapter, &work_done, work_to_do);
1647
1648         *budget -= work_done;
1649         netdev->quota -= work_done;
1650
1651         if (work_done < work_to_do || !netif_running(netdev)) {
1652                 netif_rx_complete(netdev);
1653                 /* RAIDC will be automatically restarted by irq_enable */
1654                 ixgb_irq_enable(adapter);
1655         }
1656
1657         return (work_done >= work_to_do);
1658 }
1659 #endif
1660
1661 /**
1662  * ixgb_clean_tx_irq - Reclaim resources after transmit completes
1663  * @adapter: board private structure
1664  **/
1665
1666 static boolean_t ixgb_clean_tx_irq(struct ixgb_adapter *adapter)
1667 {
1668         struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
1669         struct net_device *netdev = adapter->netdev;
1670         struct pci_dev *pdev = adapter->pdev;
1671         struct ixgb_tx_desc *tx_desc, *eop_desc;
1672         struct ixgb_buffer *buffer_info;
1673         unsigned int i, eop;
1674         boolean_t cleaned = FALSE;
1675
1676         i = tx_ring->next_to_clean;
1677         eop = tx_ring->buffer_info[i].next_to_watch;
1678         eop_desc = IXGB_TX_DESC(*tx_ring, eop);
1679
1680         while (eop_desc->status & cpu_to_le32(IXGB_TX_DESC_STATUS_DD)) {
1681
1682                 for (cleaned = FALSE; !cleaned;) {
1683                         tx_desc = IXGB_TX_DESC(*tx_ring, i);
1684                         buffer_info = &tx_ring->buffer_info[i];
1685
1686                         if (tx_desc->popts
1687                             & (IXGB_TX_DESC_POPTS_TXSM |
1688                                IXGB_TX_DESC_POPTS_IXSM))
1689                                 adapter->hw_csum_tx_good++;
1690
1691                         if (buffer_info->dma) {
1692
1693                                 pci_unmap_page(pdev,
1694                                                buffer_info->dma,
1695                                                buffer_info->length,
1696                                                PCI_DMA_TODEVICE);
1697
1698                                 buffer_info->dma = 0;
1699                         }
1700
1701                         if (buffer_info->skb) {
1702
1703                                 dev_kfree_skb_any(buffer_info->skb);
1704
1705                                 buffer_info->skb = NULL;
1706                         }
1707
1708                         *(uint32_t *) & (tx_desc->status) = 0;
1709
1710                         cleaned = (i == eop);
1711                         if (++i == tx_ring->count)
1712                                 i = 0;
1713                 }
1714
1715                 eop = tx_ring->buffer_info[i].next_to_watch;
1716                 eop_desc = IXGB_TX_DESC(*tx_ring, eop);
1717         }
1718
1719         tx_ring->next_to_clean = i;
1720
1721         spin_lock(&adapter->tx_lock);
1722         if (cleaned && netif_queue_stopped(netdev) && netif_carrier_ok(netdev)
1723             && (IXGB_DESC_UNUSED(tx_ring) > IXGB_TX_QUEUE_WAKE)) {
1724
1725                 netif_wake_queue(netdev);
1726         }
1727         spin_unlock(&adapter->tx_lock);
1728
1729         return cleaned;
1730 }
1731
1732 /**
1733  * ixgb_clean_rx_irq - Send received data up the network stack,
1734  * @adapter: board private structure
1735  **/
1736
1737 static boolean_t
1738 #ifdef CONFIG_IXGB_NAPI
1739 ixgb_clean_rx_irq(struct ixgb_adapter *adapter, int *work_done, int work_to_do)
1740 #else
1741 ixgb_clean_rx_irq(struct ixgb_adapter *adapter)
1742 #endif
1743 {
1744         struct ixgb_desc_ring *rx_ring = &adapter->rx_ring;
1745         struct net_device *netdev = adapter->netdev;
1746         struct pci_dev *pdev = adapter->pdev;
1747         struct ixgb_rx_desc *rx_desc, *next_rxd;
1748         struct ixgb_buffer *buffer_info, *next_buffer, *next2_buffer;
1749         struct sk_buff *skb, *next_skb;
1750         uint32_t length;
1751         unsigned int i, j;
1752         boolean_t cleaned = FALSE;
1753
1754         i = rx_ring->next_to_clean;
1755         rx_desc = IXGB_RX_DESC(*rx_ring, i);
1756         buffer_info = &rx_ring->buffer_info[i];
1757
1758         while (rx_desc->status & IXGB_RX_DESC_STATUS_DD) {
1759
1760                 skb = buffer_info->skb;
1761                 prefetch(skb->data);
1762
1763                 if (++i == rx_ring->count)
1764                         i = 0;
1765                 next_rxd = IXGB_RX_DESC(*rx_ring, i);
1766                 prefetch(next_rxd);
1767
1768                 if ((j = i + 1) == rx_ring->count)
1769                         j = 0;
1770                 next2_buffer = &rx_ring->buffer_info[j];
1771                 prefetch(next2_buffer);
1772
1773                 next_buffer = &rx_ring->buffer_info[i];
1774                 next_skb = next_buffer->skb;
1775                 prefetch(next_skb);
1776
1777 #ifdef CONFIG_IXGB_NAPI
1778                 if (*work_done >= work_to_do)
1779                         break;
1780
1781                 (*work_done)++;
1782 #endif
1783
1784                 cleaned = TRUE;
1785
1786                 pci_unmap_single(pdev,
1787                                  buffer_info->dma,
1788                                  buffer_info->length, PCI_DMA_FROMDEVICE);
1789
1790                 length = le16_to_cpu(rx_desc->length);
1791
1792                 if (unlikely(!(rx_desc->status & IXGB_RX_DESC_STATUS_EOP))) {
1793
1794                         /* All receives must fit into a single buffer */
1795
1796                         IXGB_DBG("Receive packet consumed multiple buffers "
1797                                  "length<%x>\n", length);
1798
1799                         dev_kfree_skb_irq(skb);
1800                         rx_desc->status = 0;
1801                         buffer_info->skb = NULL;
1802
1803                         rx_desc = next_rxd;
1804                         buffer_info = next_buffer;
1805                         continue;
1806                 }
1807
1808                 if (unlikely(rx_desc->errors
1809                              & (IXGB_RX_DESC_ERRORS_CE | IXGB_RX_DESC_ERRORS_SE
1810                                 | IXGB_RX_DESC_ERRORS_P |
1811                                 IXGB_RX_DESC_ERRORS_RXE))) {
1812
1813                         dev_kfree_skb_irq(skb);
1814                         rx_desc->status = 0;
1815                         buffer_info->skb = NULL;
1816
1817                         rx_desc = next_rxd;
1818                         buffer_info = next_buffer;
1819                         continue;
1820                 }
1821
1822                 /* Good Receive */
1823                 skb_put(skb, length);
1824
1825                 /* Receive Checksum Offload */
1826                 ixgb_rx_checksum(adapter, rx_desc, skb);
1827
1828                 skb->protocol = eth_type_trans(skb, netdev);
1829 #ifdef CONFIG_IXGB_NAPI
1830                 if (adapter->vlgrp
1831                     && (rx_desc->status & IXGB_RX_DESC_STATUS_VP)) {
1832                         vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
1833                                                  le16_to_cpu(rx_desc->
1834                                                              special &
1835                                                              IXGB_RX_DESC_SPECIAL_VLAN_MASK));
1836                 } else {
1837                         netif_receive_skb(skb);
1838                 }
1839 #else                           /* CONFIG_IXGB_NAPI */
1840                 if (adapter->vlgrp
1841                     && (rx_desc->status & IXGB_RX_DESC_STATUS_VP)) {
1842                         vlan_hwaccel_rx(skb, adapter->vlgrp,
1843                                         le16_to_cpu(rx_desc->
1844                                                     special &
1845                                                     IXGB_RX_DESC_SPECIAL_VLAN_MASK));
1846                 } else {
1847                         netif_rx(skb);
1848                 }
1849 #endif                          /* CONFIG_IXGB_NAPI */
1850                 netdev->last_rx = jiffies;
1851
1852                 rx_desc->status = 0;
1853                 buffer_info->skb = NULL;
1854
1855                 rx_desc = next_rxd;
1856                 buffer_info = next_buffer;
1857         }
1858
1859         rx_ring->next_to_clean = i;
1860
1861         ixgb_alloc_rx_buffers(adapter);
1862
1863         return cleaned;
1864 }
1865
1866 /**
1867  * ixgb_alloc_rx_buffers - Replace used receive buffers
1868  * @adapter: address of board private structure
1869  **/
1870
1871 static void ixgb_alloc_rx_buffers(struct ixgb_adapter *adapter)
1872 {
1873         struct ixgb_desc_ring *rx_ring = &adapter->rx_ring;
1874         struct net_device *netdev = adapter->netdev;
1875         struct pci_dev *pdev = adapter->pdev;
1876         struct ixgb_rx_desc *rx_desc;
1877         struct ixgb_buffer *buffer_info;
1878         struct sk_buff *skb;
1879         int reserve_len = 2;
1880         unsigned int i;
1881         int num_group_tail_writes;
1882         long cleancount;
1883
1884         i = rx_ring->next_to_use;
1885         buffer_info = &rx_ring->buffer_info[i];
1886         cleancount = IXGB_DESC_UNUSED(rx_ring);
1887
1888         /* lessen this to 4 if we're
1889          * in the midst of raidc and rbd is occuring
1890          * because we don't want to delay returning buffers when low
1891          */
1892         num_group_tail_writes = adapter->raidc ? 4 : IXGB_RX_BUFFER_WRITE;
1893
1894         /* leave one descriptor unused */
1895         while (--cleancount > 0) {
1896                 rx_desc = IXGB_RX_DESC(*rx_ring, i);
1897
1898                 skb = dev_alloc_skb(adapter->rx_buffer_len + reserve_len);
1899
1900                 if (unlikely(!skb)) {
1901                         /* Better luck next round */
1902                         break;
1903                 }
1904
1905                 /* Make buffer alignment 2 beyond a 16 byte boundary
1906                  * this will result in a 16 byte aligned IP header after
1907                  * the 14 byte MAC header is removed
1908                  */
1909                 skb_reserve(skb, reserve_len);
1910
1911                 skb->dev = netdev;
1912
1913                 buffer_info->skb = skb;
1914                 buffer_info->length = adapter->rx_buffer_len;
1915                 buffer_info->dma =
1916                     pci_map_single(pdev,
1917                                    skb->data,
1918                                    adapter->rx_buffer_len, PCI_DMA_FROMDEVICE);
1919
1920                 rx_desc->buff_addr = cpu_to_le64(buffer_info->dma);
1921
1922                 if ((i & ~(num_group_tail_writes - 1)) == i) {
1923                         /* Force memory writes to complete before letting h/w
1924                          * know there are new descriptors to fetch.  (Only
1925                          * applicable for weak-ordered memory model archs,
1926                          * such as IA-64). */
1927                         wmb();
1928
1929                         IXGB_WRITE_REG(&adapter->hw, RDT, i);
1930                 }
1931
1932                 if (++i == rx_ring->count)
1933                         i = 0;
1934                 buffer_info = &rx_ring->buffer_info[i];
1935         }
1936
1937         rx_ring->next_to_use = i;
1938 }
1939
1940 /**
1941  * ixgb_ioctl - perform a command - e.g: ethtool:get_driver_info.
1942  * @param netdev network interface device structure
1943  * @param ifr data to be used/filled in by the ioctl command
1944  * @param cmd ioctl command to execute
1945  **/
1946
1947 static int ixgb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
1948 {
1949         switch (cmd) {
1950         case SIOCETHTOOL:
1951                 return ixgb_ethtool_ioctl(netdev, ifr);
1952         default:
1953                 return -EOPNOTSUPP;
1954         }
1955
1956         return 0;
1957 }
1958
1959 /**
1960  * ixgb_rx_checksum - Receive Checksum Offload for 82597.
1961  * @adapter: board private structure
1962  * @rx_desc: receive descriptor
1963  * @sk_buff: socket buffer with received data
1964  **/
1965
1966 static inline void
1967 ixgb_rx_checksum(struct ixgb_adapter *adapter,
1968                  struct ixgb_rx_desc *rx_desc, struct sk_buff *skb)
1969 {
1970         /* Ignore Checksum bit is set OR
1971          * TCP Checksum has not been calculated
1972          */
1973         if ((rx_desc->status & IXGB_RX_DESC_STATUS_IXSM) ||
1974             (!(rx_desc->status & IXGB_RX_DESC_STATUS_TCPCS))) {
1975                 skb->ip_summed = CHECKSUM_NONE;
1976                 return;
1977         }
1978
1979         /* At this point we know the hardware did the TCP checksum */
1980         /* now look at the TCP checksum error bit */
1981         if (rx_desc->errors & IXGB_RX_DESC_ERRORS_TCPE) {
1982                 /* let the stack verify checksum errors */
1983                 skb->ip_summed = CHECKSUM_NONE;
1984                 adapter->hw_csum_rx_error++;
1985         } else {
1986                 /* TCP checksum is good */
1987                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1988                 adapter->hw_csum_rx_good++;
1989         }
1990 }
1991
1992 /**
1993  * ixgb_vlan_rx_register - enables or disables vlan tagging/stripping.
1994  *
1995  * @param netdev network interface device structure
1996  * @param grp indicates to enable or disable tagging/stripping
1997  **/
1998 static void
1999 ixgb_vlan_rx_register(struct net_device *netdev, struct vlan_group *grp)
2000 {
2001         struct ixgb_adapter *adapter = netdev->priv;
2002         uint32_t ctrl, rctl;
2003
2004         ixgb_irq_disable(adapter);
2005         adapter->vlgrp = grp;
2006
2007         if (grp) {
2008                 /* enable VLAN tag insert/strip */
2009                 ctrl = IXGB_READ_REG(&adapter->hw, CTRL0);
2010                 ctrl |= IXGB_CTRL0_VME;
2011                 IXGB_WRITE_REG(&adapter->hw, CTRL0, ctrl);
2012
2013                 /* enable VLAN receive filtering */
2014
2015                 rctl = IXGB_READ_REG(&adapter->hw, RCTL);
2016                 rctl |= IXGB_RCTL_VFE;
2017                 rctl &= ~IXGB_RCTL_CFIEN;
2018                 IXGB_WRITE_REG(&adapter->hw, RCTL, rctl);
2019         } else {
2020                 /* disable VLAN tag insert/strip */
2021
2022                 ctrl = IXGB_READ_REG(&adapter->hw, CTRL0);
2023                 ctrl &= ~IXGB_CTRL0_VME;
2024                 IXGB_WRITE_REG(&adapter->hw, CTRL0, ctrl);
2025
2026                 /* disable VLAN filtering */
2027
2028                 rctl = IXGB_READ_REG(&adapter->hw, RCTL);
2029                 rctl &= ~IXGB_RCTL_VFE;
2030                 IXGB_WRITE_REG(&adapter->hw, RCTL, rctl);
2031         }
2032
2033         ixgb_irq_enable(adapter);
2034 }
2035
2036 static void ixgb_vlan_rx_add_vid(struct net_device *netdev, uint16_t vid)
2037 {
2038         struct ixgb_adapter *adapter = netdev->priv;
2039         uint32_t vfta, index;
2040
2041         /* add VID to filter table */
2042
2043         index = (vid >> 5) & 0x7F;
2044         vfta = IXGB_READ_REG_ARRAY(&adapter->hw, VFTA, index);
2045         vfta |= (1 << (vid & 0x1F));
2046         ixgb_write_vfta(&adapter->hw, index, vfta);
2047 }
2048
2049 static void ixgb_vlan_rx_kill_vid(struct net_device *netdev, uint16_t vid)
2050 {
2051         struct ixgb_adapter *adapter = netdev->priv;
2052         uint32_t vfta, index;
2053
2054         ixgb_irq_disable(adapter);
2055
2056         if (adapter->vlgrp)
2057                 adapter->vlgrp->vlan_devices[vid] = NULL;
2058
2059         ixgb_irq_enable(adapter);
2060
2061         /* remove VID from filter table */
2062
2063         index = (vid >> 5) & 0x7F;
2064         vfta = IXGB_READ_REG_ARRAY(&adapter->hw, VFTA, index);
2065         vfta &= ~(1 << (vid & 0x1F));
2066         ixgb_write_vfta(&adapter->hw, index, vfta);
2067 }
2068
2069 static void ixgb_restore_vlan(struct ixgb_adapter *adapter)
2070 {
2071         ixgb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
2072
2073         if (adapter->vlgrp) {
2074                 uint16_t vid;
2075                 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
2076                         if (!adapter->vlgrp->vlan_devices[vid])
2077                                 continue;
2078                         ixgb_vlan_rx_add_vid(adapter->netdev, vid);
2079                 }
2080         }
2081 }
2082
2083 /**
2084  * ixgb_notify_reboot - handles OS notification of reboot event.
2085  * @param nb notifier block, unused
2086  * @param event Event being passed to driver to act upon
2087  * @param p A pointer to our net device
2088  **/
2089 static int
2090 ixgb_notify_reboot(struct notifier_block *nb, unsigned long event, void *p)
2091 {
2092         struct pci_dev *pdev = NULL;
2093
2094         switch (event) {
2095         case SYS_DOWN:
2096         case SYS_HALT:
2097         case SYS_POWER_OFF:
2098                 while ((pdev = pci_find_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) {
2099                         if (pci_dev_driver(pdev) == &ixgb_driver)
2100                                 ixgb_suspend(pdev, 3);
2101                 }
2102         }
2103         return NOTIFY_DONE;
2104 }
2105
2106 /**
2107  * ixgb_suspend - driver suspend function called from notify.
2108  * @param pdev pci driver structure used for passing to
2109  * @param state power state to enter
2110  **/
2111 static int ixgb_suspend(struct pci_dev *pdev, uint32_t state)
2112 {
2113         struct net_device *netdev = pci_get_drvdata(pdev);
2114         struct ixgb_adapter *adapter = netdev->priv;
2115
2116         netif_device_detach(netdev);
2117
2118         if (netif_running(netdev))
2119                 ixgb_down(adapter, TRUE);
2120
2121         pci_save_state(pdev, adapter->pci_state);
2122
2123         state = (state > 0) ? 3 : 0;
2124         pci_set_power_state(pdev, state);
2125         msec_delay(200);
2126
2127         return 0;
2128 }
2129
2130 #ifdef CONFIG_NET_POLL_CONTROLLER
2131 /*
2132  * Polling 'interrupt' - used by things like netconsole to send skbs
2133  * without having to re-enable interrupts. It's not called while
2134  * the interrupt routine is executing.
2135  */
2136
2137 static void ixgb_netpoll(struct net_device *dev)
2138 {
2139         struct ixgb_adapter *adapter = dev->priv;
2140         disable_irq(adapter->pdev->irq);
2141         ixgb_intr(adapter->pdev->irq, dev, NULL);
2142         enable_irq(adapter->pdev->irq);
2143 }
2144 #endif
2145
2146 /* ixgb_main.c */