2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * $Id: ipoib_ib.c 1386 2004-12-27 16:23:17Z roland $
35 #include <linux/delay.h>
36 #include <linux/dma-mapping.h>
42 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG_DATA
45 module_param(data_debug_level, int, 0644);
46 MODULE_PARM_DESC(data_debug_level,
47 "Enable data path debug tracing if > 0");
50 #define IPOIB_OP_RECV (1ul << 31)
52 static DECLARE_MUTEX(pkey_sem);
54 struct ipoib_ah *ipoib_create_ah(struct net_device *dev,
55 struct ib_pd *pd, struct ib_ah_attr *attr)
59 ah = kmalloc(sizeof *ah, GFP_KERNEL);
67 ah->ah = ib_create_ah(pd, attr);
72 ipoib_dbg(netdev_priv(dev), "Created ah %p\n", ah->ah);
77 void ipoib_free_ah(struct kref *kref)
79 struct ipoib_ah *ah = container_of(kref, struct ipoib_ah, ref);
80 struct ipoib_dev_priv *priv = netdev_priv(ah->dev);
84 if (ah->last_send <= priv->tx_tail) {
85 ipoib_dbg(priv, "Freeing ah %p\n", ah->ah);
86 ib_destroy_ah(ah->ah);
89 spin_lock_irqsave(&priv->lock, flags);
90 list_add_tail(&ah->list, &priv->dead_ahs);
91 spin_unlock_irqrestore(&priv->lock, flags);
95 static inline int ipoib_ib_receive(struct ipoib_dev_priv *priv,
99 struct ib_sge list = {
101 .length = IPOIB_BUF_SIZE,
102 .lkey = priv->mr->lkey,
104 struct ib_recv_wr param = {
105 .wr_id = wr_id | IPOIB_OP_RECV,
108 .recv_flags = IB_RECV_SIGNALED
110 struct ib_recv_wr *bad_wr;
112 return ib_post_recv(priv->qp, ¶m, &bad_wr);
115 static int ipoib_ib_post_receive(struct net_device *dev, int id)
117 struct ipoib_dev_priv *priv = netdev_priv(dev);
122 skb = dev_alloc_skb(IPOIB_BUF_SIZE + 4);
124 ipoib_warn(priv, "failed to allocate receive buffer\n");
126 priv->rx_ring[id].skb = NULL;
129 skb_reserve(skb, 4); /* 16 byte align IP header */
130 priv->rx_ring[id].skb = skb;
131 addr = dma_map_single(priv->ca->dma_device,
132 skb->data, IPOIB_BUF_SIZE,
134 pci_unmap_addr_set(&priv->rx_ring[id], mapping, addr);
136 ret = ipoib_ib_receive(priv, id, addr);
138 ipoib_warn(priv, "ipoib_ib_receive failed for buf %d (%d)\n",
140 priv->rx_ring[id].skb = NULL;
146 static int ipoib_ib_post_receives(struct net_device *dev)
148 struct ipoib_dev_priv *priv = netdev_priv(dev);
151 for (i = 0; i < IPOIB_RX_RING_SIZE; ++i) {
152 if (ipoib_ib_post_receive(dev, i)) {
153 ipoib_warn(priv, "ipoib_ib_post_receive failed for buf %d\n", i);
161 static void ipoib_ib_handle_wc(struct net_device *dev,
164 struct ipoib_dev_priv *priv = netdev_priv(dev);
165 unsigned int wr_id = wc->wr_id;
167 ipoib_dbg_data(priv, "called: id %d, op %d, status: %d\n",
168 wr_id, wc->opcode, wc->status);
170 if (wr_id & IPOIB_OP_RECV) {
171 wr_id &= ~IPOIB_OP_RECV;
173 if (wr_id < IPOIB_RX_RING_SIZE) {
174 struct sk_buff *skb = priv->rx_ring[wr_id].skb;
176 priv->rx_ring[wr_id].skb = NULL;
178 dma_unmap_single(priv->ca->dma_device,
179 pci_unmap_addr(&priv->rx_ring[wr_id],
184 if (wc->status != IB_WC_SUCCESS) {
185 if (wc->status != IB_WC_WR_FLUSH_ERR)
186 ipoib_warn(priv, "failed recv event "
187 "(status=%d, wrid=%d vend_err %x)\n",
188 wc->status, wr_id, wc->vendor_err);
189 dev_kfree_skb_any(skb);
193 ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n",
194 wc->byte_len, wc->slid);
196 skb_put(skb, wc->byte_len);
197 skb_pull(skb, IB_GRH_BYTES);
199 if (wc->slid != priv->local_lid ||
200 wc->src_qp != priv->qp->qp_num) {
201 skb->protocol = ((struct ipoib_header *) skb->data)->proto;
203 skb_pull(skb, IPOIB_ENCAP_LEN);
205 dev->last_rx = jiffies;
206 ++priv->stats.rx_packets;
207 priv->stats.rx_bytes += skb->len;
210 /* XXX get correct PACKET_ type here */
211 skb->pkt_type = PACKET_HOST;
214 ipoib_dbg_data(priv, "dropping loopback packet\n");
215 dev_kfree_skb_any(skb);
219 if (ipoib_ib_post_receive(dev, wr_id))
220 ipoib_warn(priv, "ipoib_ib_post_receive failed "
221 "for buf %d\n", wr_id);
223 ipoib_warn(priv, "completion event with wrid %d\n",
227 struct ipoib_buf *tx_req;
230 if (wr_id >= IPOIB_TX_RING_SIZE) {
231 ipoib_warn(priv, "completion event with wrid %d (> %d)\n",
232 wr_id, IPOIB_TX_RING_SIZE);
236 ipoib_dbg_data(priv, "send complete, wrid %d\n", wr_id);
238 tx_req = &priv->tx_ring[wr_id];
240 dma_unmap_single(priv->ca->dma_device,
241 pci_unmap_addr(tx_req, mapping),
245 ++priv->stats.tx_packets;
246 priv->stats.tx_bytes += tx_req->skb->len;
248 dev_kfree_skb_any(tx_req->skb);
250 spin_lock_irqsave(&priv->tx_lock, flags);
252 if (netif_queue_stopped(dev) &&
253 priv->tx_head - priv->tx_tail <= IPOIB_TX_RING_SIZE / 2)
254 netif_wake_queue(dev);
255 spin_unlock_irqrestore(&priv->tx_lock, flags);
257 if (wc->status != IB_WC_SUCCESS &&
258 wc->status != IB_WC_WR_FLUSH_ERR)
259 ipoib_warn(priv, "failed send event "
260 "(status=%d, wrid=%d vend_err %x)\n",
261 wc->status, wr_id, wc->vendor_err);
265 void ipoib_ib_completion(struct ib_cq *cq, void *dev_ptr)
267 struct net_device *dev = (struct net_device *) dev_ptr;
268 struct ipoib_dev_priv *priv = netdev_priv(dev);
271 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
273 n = ib_poll_cq(cq, IPOIB_NUM_WC, priv->ibwc);
274 for (i = 0; i < n; ++i)
275 ipoib_ib_handle_wc(dev, priv->ibwc + i);
276 } while (n == IPOIB_NUM_WC);
279 static inline int post_send(struct ipoib_dev_priv *priv,
281 struct ib_ah *address, u32 qpn,
282 dma_addr_t addr, int len)
284 struct ib_send_wr *bad_wr;
286 priv->tx_sge.addr = addr;
287 priv->tx_sge.length = len;
289 priv->tx_wr.wr_id = wr_id;
290 priv->tx_wr.wr.ud.remote_qpn = qpn;
291 priv->tx_wr.wr.ud.ah = address;
293 return ib_post_send(priv->qp, &priv->tx_wr, &bad_wr);
296 void ipoib_send(struct net_device *dev, struct sk_buff *skb,
297 struct ipoib_ah *address, u32 qpn)
299 struct ipoib_dev_priv *priv = netdev_priv(dev);
300 struct ipoib_buf *tx_req;
303 if (skb->len > dev->mtu + INFINIBAND_ALEN) {
304 ipoib_warn(priv, "packet len %d (> %d) too long to send, dropping\n",
305 skb->len, dev->mtu + INFINIBAND_ALEN);
306 ++priv->stats.tx_dropped;
307 ++priv->stats.tx_errors;
308 dev_kfree_skb_any(skb);
312 ipoib_dbg_data(priv, "sending packet, length=%d address=%p qpn=0x%06x\n",
313 skb->len, address, qpn);
316 * We put the skb into the tx_ring _before_ we call post_send()
317 * because it's entirely possible that the completion handler will
318 * run before we execute anything after the post_send(). That
319 * means we have to make sure everything is properly recorded and
320 * our state is consistent before we call post_send().
322 tx_req = &priv->tx_ring[priv->tx_head & (IPOIB_TX_RING_SIZE - 1)];
324 addr = dma_map_single(priv->ca->dma_device, skb->data, skb->len,
326 pci_unmap_addr_set(tx_req, mapping, addr);
328 if (unlikely(post_send(priv, priv->tx_head & (IPOIB_TX_RING_SIZE - 1),
329 address->ah, qpn, addr, skb->len))) {
330 ipoib_warn(priv, "post_send failed\n");
331 ++priv->stats.tx_errors;
332 dma_unmap_single(priv->ca->dma_device, addr, skb->len,
334 dev_kfree_skb_any(skb);
336 dev->trans_start = jiffies;
338 address->last_send = priv->tx_head;
341 if (priv->tx_head - priv->tx_tail == IPOIB_TX_RING_SIZE) {
342 ipoib_dbg(priv, "TX ring full, stopping kernel net queue\n");
343 netif_stop_queue(dev);
348 static void __ipoib_reap_ah(struct net_device *dev)
350 struct ipoib_dev_priv *priv = netdev_priv(dev);
351 struct ipoib_ah *ah, *tah;
352 LIST_HEAD(remove_list);
354 spin_lock_irq(&priv->lock);
355 list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list)
356 if (ah->last_send <= priv->tx_tail) {
358 list_add_tail(&ah->list, &remove_list);
360 spin_unlock_irq(&priv->lock);
362 list_for_each_entry_safe(ah, tah, &remove_list, list) {
363 ipoib_dbg(priv, "Reaping ah %p\n", ah->ah);
364 ib_destroy_ah(ah->ah);
369 void ipoib_reap_ah(void *dev_ptr)
371 struct net_device *dev = dev_ptr;
372 struct ipoib_dev_priv *priv = netdev_priv(dev);
374 __ipoib_reap_ah(dev);
376 if (!test_bit(IPOIB_STOP_REAPER, &priv->flags))
377 queue_delayed_work(ipoib_workqueue, &priv->ah_reap_task, HZ);
380 int ipoib_ib_dev_open(struct net_device *dev)
382 struct ipoib_dev_priv *priv = netdev_priv(dev);
385 ret = ipoib_qp_create(dev);
387 ipoib_warn(priv, "ipoib_qp_create returned %d\n", ret);
391 ret = ipoib_ib_post_receives(dev);
393 ipoib_warn(priv, "ipoib_ib_post_receives returned %d\n", ret);
397 clear_bit(IPOIB_STOP_REAPER, &priv->flags);
398 queue_delayed_work(ipoib_workqueue, &priv->ah_reap_task, HZ);
403 int ipoib_ib_dev_up(struct net_device *dev)
405 struct ipoib_dev_priv *priv = netdev_priv(dev);
407 set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
409 return ipoib_mcast_start_thread(dev);
412 int ipoib_ib_dev_down(struct net_device *dev)
414 struct ipoib_dev_priv *priv = netdev_priv(dev);
416 ipoib_dbg(priv, "downing ib_dev\n");
418 clear_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
419 netif_carrier_off(dev);
421 /* Shutdown the P_Key thread if still active */
422 if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
424 set_bit(IPOIB_PKEY_STOP, &priv->flags);
425 cancel_delayed_work(&priv->pkey_task);
427 flush_workqueue(ipoib_workqueue);
430 ipoib_mcast_stop_thread(dev);
433 * Flush the multicast groups first so we stop any multicast joins. The
434 * completion thread may have already died and we may deadlock waiting
435 * for the completion thread to finish some multicast joins.
437 ipoib_mcast_dev_flush(dev);
439 /* Delete broadcast and local addresses since they will be recreated */
440 ipoib_mcast_dev_down(dev);
442 ipoib_flush_paths(dev);
447 static int recvs_pending(struct net_device *dev)
449 struct ipoib_dev_priv *priv = netdev_priv(dev);
453 for (i = 0; i < IPOIB_RX_RING_SIZE; ++i)
454 if (priv->rx_ring[i].skb)
460 int ipoib_ib_dev_stop(struct net_device *dev)
462 struct ipoib_dev_priv *priv = netdev_priv(dev);
463 struct ib_qp_attr qp_attr;
466 struct ipoib_buf *tx_req;
469 /* Kill the existing QP and allocate a new one */
470 qp_attr.qp_state = IB_QPS_ERR;
471 attr_mask = IB_QP_STATE;
472 if (ib_modify_qp(priv->qp, &qp_attr, attr_mask))
473 ipoib_warn(priv, "Failed to modify QP to ERROR state\n");
475 /* Wait for all sends and receives to complete */
478 while (priv->tx_head != priv->tx_tail || recvs_pending(dev)) {
479 if (time_after(jiffies, begin + 5 * HZ)) {
480 ipoib_warn(priv, "timing out; %d sends %d receives not completed\n",
481 priv->tx_head - priv->tx_tail, recvs_pending(dev));
484 * assume the HW is wedged and just free up
485 * all our pending work requests.
487 while (priv->tx_tail < priv->tx_head) {
488 tx_req = &priv->tx_ring[priv->tx_tail &
489 (IPOIB_TX_RING_SIZE - 1)];
490 dma_unmap_single(priv->ca->dma_device,
491 pci_unmap_addr(tx_req, mapping),
494 dev_kfree_skb_any(tx_req->skb);
498 for (i = 0; i < IPOIB_RX_RING_SIZE; ++i)
499 if (priv->rx_ring[i].skb) {
500 dma_unmap_single(priv->ca->dma_device,
501 pci_unmap_addr(&priv->rx_ring[i],
505 dev_kfree_skb_any(priv->rx_ring[i].skb);
506 priv->rx_ring[i].skb = NULL;
515 ipoib_dbg(priv, "All sends and receives done.\n");
518 qp_attr.qp_state = IB_QPS_RESET;
519 attr_mask = IB_QP_STATE;
520 if (ib_modify_qp(priv->qp, &qp_attr, attr_mask))
521 ipoib_warn(priv, "Failed to modify QP to RESET state\n");
523 /* Wait for all AHs to be reaped */
524 set_bit(IPOIB_STOP_REAPER, &priv->flags);
525 cancel_delayed_work(&priv->ah_reap_task);
526 flush_workqueue(ipoib_workqueue);
530 while (!list_empty(&priv->dead_ahs)) {
531 __ipoib_reap_ah(dev);
533 if (time_after(jiffies, begin + HZ)) {
534 ipoib_warn(priv, "timing out; will leak address handles\n");
544 int ipoib_ib_dev_init(struct net_device *dev, struct ib_device *ca, int port)
546 struct ipoib_dev_priv *priv = netdev_priv(dev);
552 if (ipoib_transport_dev_init(dev, ca)) {
553 printk(KERN_WARNING "%s: ipoib_transport_dev_init failed\n", ca->name);
557 if (dev->flags & IFF_UP) {
558 if (ipoib_ib_dev_open(dev)) {
559 ipoib_transport_dev_cleanup(dev);
567 void ipoib_ib_dev_flush(void *_dev)
569 struct net_device *dev = (struct net_device *)_dev;
570 struct ipoib_dev_priv *priv = netdev_priv(dev), *cpriv;
572 if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
575 ipoib_dbg(priv, "flushing\n");
577 ipoib_ib_dev_down(dev);
580 * The device could have been brought down between the start and when
581 * we get here, don't bring it back up if it's not configured up
583 if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
584 ipoib_ib_dev_up(dev);
586 /* Flush any child interfaces too */
587 list_for_each_entry(cpriv, &priv->child_intfs, list)
588 ipoib_ib_dev_flush(&cpriv->dev);
591 void ipoib_ib_dev_cleanup(struct net_device *dev)
593 struct ipoib_dev_priv *priv = netdev_priv(dev);
595 ipoib_dbg(priv, "cleaning up ib_dev\n");
597 ipoib_mcast_stop_thread(dev);
599 /* Delete the broadcast address and the local address */
600 ipoib_mcast_dev_down(dev);
602 ipoib_transport_dev_cleanup(dev);
606 * Delayed P_Key Assigment Interim Support
608 * The following is initial implementation of delayed P_Key assigment
609 * mechanism. It is using the same approach implemented for the multicast
610 * group join. The single goal of this implementation is to quickly address
611 * Bug #2507. This implementation will probably be removed when the P_Key
612 * change async notification is available.
614 int ipoib_open(struct net_device *dev);
616 static void ipoib_pkey_dev_check_presence(struct net_device *dev)
618 struct ipoib_dev_priv *priv = netdev_priv(dev);
621 if (ib_find_cached_pkey(priv->ca, priv->port, priv->pkey, &pkey_index))
622 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
624 set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
627 void ipoib_pkey_poll(void *dev_ptr)
629 struct net_device *dev = dev_ptr;
630 struct ipoib_dev_priv *priv = netdev_priv(dev);
632 ipoib_pkey_dev_check_presence(dev);
634 if (test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
638 if (!test_bit(IPOIB_PKEY_STOP, &priv->flags))
639 queue_delayed_work(ipoib_workqueue,
646 int ipoib_pkey_dev_delay_open(struct net_device *dev)
648 struct ipoib_dev_priv *priv = netdev_priv(dev);
650 /* Look for the interface pkey value in the IB Port P_Key table and */
651 /* set the interface pkey assigment flag */
652 ipoib_pkey_dev_check_presence(dev);
654 /* P_Key value not assigned yet - start polling */
655 if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
657 clear_bit(IPOIB_PKEY_STOP, &priv->flags);
658 queue_delayed_work(ipoib_workqueue,