/*
* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
- * Copyright (c) 2005, 2006 Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2005 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2005 Mellanox Technologies. All rights reserved.
* Copyright (c) 2004 Voltaire, Inc. All rights reserved.
*
#include <linux/init.h>
#include <linux/hardirq.h>
-#include <asm/io.h>
-
#include <rdma/ib_pack.h>
#include "mthca_dev.h"
#define MTHCA_ARBEL_CQ_DB_REQ_NOT (2 << 24)
#define MTHCA_ARBEL_CQ_DB_REQ_NOT_MULT (3 << 24)
-static inline struct mthca_cqe *get_cqe_from_buf(struct mthca_cq_buf *buf,
- int entry)
+static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry)
{
- if (buf->is_direct)
- return buf->queue.direct.buf + (entry * MTHCA_CQ_ENTRY_SIZE);
+ if (cq->is_direct)
+ return cq->queue.direct.buf + (entry * MTHCA_CQ_ENTRY_SIZE);
else
- return buf->queue.page_list[entry * MTHCA_CQ_ENTRY_SIZE / PAGE_SIZE].buf
+ return cq->queue.page_list[entry * MTHCA_CQ_ENTRY_SIZE / PAGE_SIZE].buf
+ (entry * MTHCA_CQ_ENTRY_SIZE) % PAGE_SIZE;
}
-static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry)
-{
- return get_cqe_from_buf(&cq->buf, entry);
-}
-
-static inline struct mthca_cqe *cqe_sw(struct mthca_cqe *cqe)
+static inline struct mthca_cqe *cqe_sw(struct mthca_cq *cq, int i)
{
+ struct mthca_cqe *cqe = get_cqe(cq, i);
return MTHCA_CQ_ENTRY_OWNER_HW & cqe->owner ? NULL : cqe;
}
static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq)
{
- return cqe_sw(get_cqe(cq, cq->cons_index & cq->ibcq.cqe));
+ return cqe_sw(cq, cq->cons_index & cq->ibcq.cqe);
}
static inline void set_cqe_hw(struct mthca_cqe *cqe)
mthca_write64(doorbell,
dev->kar + MTHCA_CQ_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
- /*
- * Make sure doorbells don't leak out of CQ spinlock
- * and reach the HCA out of order:
- */
- mmiowb();
}
}
spin_lock(&dev->cq_table.lock);
cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1));
- if (cq)
- ++cq->refcount;
+ if (cq)
+ atomic_inc(&cq->refcount);
spin_unlock(&dev->cq_table.lock);
if (!cq) {
if (cq->ibcq.event_handler)
cq->ibcq.event_handler(&event, cq->ibcq.cq_context);
- spin_lock(&dev->cq_table.lock);
- if (!--cq->refcount)
+ if (atomic_dec_and_test(&cq->refcount))
wake_up(&cq->wait);
- spin_unlock(&dev->cq_table.lock);
}
static inline int is_recv_cqe(struct mthca_cqe *cqe)
return !(cqe->is_send & 0x80);
}
-void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn,
+void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn,
struct mthca_srq *srq)
{
+ struct mthca_cq *cq;
struct mthca_cqe *cqe;
u32 prod_index;
int nfreed = 0;
+ spin_lock_irq(&dev->cq_table.lock);
+ cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1));
+ if (cq)
+ atomic_inc(&cq->refcount);
+ spin_unlock_irq(&dev->cq_table.lock);
+
+ if (!cq)
+ return;
+
spin_lock_irq(&cq->lock);
/*
* from our QP and therefore don't need to be checked.
*/
for (prod_index = cq->cons_index;
- cqe_sw(get_cqe(cq, prod_index & cq->ibcq.cqe));
+ cqe_sw(cq, prod_index & cq->ibcq.cqe);
++prod_index)
if (prod_index == cq->cons_index + cq->ibcq.cqe)
break;
if (0)
mthca_dbg(dev, "Cleaning QPN %06x from CQN %06x; ci %d, pi %d\n",
- qpn, cq->cqn, cq->cons_index, prod_index);
+ qpn, cqn, cq->cons_index, prod_index);
/*
* Now sweep backwards through the CQ, removing CQ entries
}
spin_unlock_irq(&cq->lock);
+ if (atomic_dec_and_test(&cq->refcount))
+ wake_up(&cq->wait);
}
-void mthca_cq_resize_copy_cqes(struct mthca_cq *cq)
-{
- int i;
-
- /*
- * In Tavor mode, the hardware keeps the consumer and producer
- * indices mod the CQ size. Since we might be making the CQ
- * bigger, we need to deal with the case where the producer
- * index wrapped around before the CQ was resized.
- */
- if (!mthca_is_memfree(to_mdev(cq->ibcq.device)) &&
- cq->ibcq.cqe < cq->resize_buf->cqe) {
- cq->cons_index &= cq->ibcq.cqe;
- if (cqe_sw(get_cqe(cq, cq->ibcq.cqe)))
- cq->cons_index -= cq->ibcq.cqe + 1;
- }
-
- for (i = cq->cons_index; cqe_sw(get_cqe(cq, i & cq->ibcq.cqe)); ++i)
- memcpy(get_cqe_from_buf(&cq->resize_buf->buf,
- i & cq->resize_buf->cqe),
- get_cqe(cq, i & cq->ibcq.cqe), MTHCA_CQ_ENTRY_SIZE);
-}
-
-int mthca_alloc_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int nent)
-{
- int ret;
- int i;
-
- ret = mthca_buf_alloc(dev, nent * MTHCA_CQ_ENTRY_SIZE,
- MTHCA_MAX_DIRECT_CQ_SIZE,
- &buf->queue, &buf->is_direct,
- &dev->driver_pd, 1, &buf->mr);
- if (ret)
- return ret;
-
- for (i = 0; i < nent; ++i)
- set_cqe_hw(get_cqe_from_buf(buf, i));
-
- return 0;
-}
-
-void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int cqe)
-{
- mthca_buf_free(dev, (cqe + 1) * MTHCA_CQ_ENTRY_SIZE, &buf->queue,
- buf->is_direct, &buf->mr);
-}
-
-static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,
- struct mthca_qp *qp, int wqe_index, int is_send,
- struct mthca_err_cqe *cqe,
- struct ib_wc *entry, int *free_cqe)
+static int handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,
+ struct mthca_qp *qp, int wqe_index, int is_send,
+ struct mthca_err_cqe *cqe,
+ struct ib_wc *entry, int *free_cqe)
{
+ int err;
int dbd;
__be32 new_wqe;
* error case, so we don't have to check the doorbell count, etc.
*/
if (mthca_is_memfree(dev))
- return;
+ return 0;
- mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe);
+ err = mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe);
+ if (err)
+ return err;
/*
* If we're at the end of the WQE chain, or we've used up our
* the next poll operation.
*/
if (!(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd))
- return;
+ return 0;
cqe->db_cnt = cpu_to_be16(be16_to_cpu(cqe->db_cnt) - dbd);
cqe->wqe = new_wqe;
cqe->syndrome = SYNDROME_WR_FLUSH_ERR;
*free_cqe = 0;
+
+ return 0;
}
static inline int mthca_poll_one(struct mthca_dev *dev,
entry->wr_id = srq->wrid[wqe_index];
mthca_free_srq_wqe(srq, wqe);
} else {
- s32 wqe;
wq = &(*cur_qp)->rq;
- wqe = be32_to_cpu(cqe->wqe);
- wqe_index = wqe >> wq->wqe_shift;
- /*
- * WQE addr == base - 1 might be reported in receive completion
- * with error instead of (rq size - 1) by Sinai FW 1.0.800 and
- * Arbel FW 5.1.400. This bug should be fixed in later FW revs.
- */
- if (unlikely(wqe_index < 0))
- wqe_index = wq->max - 1;
+ wqe_index = be32_to_cpu(cqe->wqe) >> wq->wqe_shift;
entry->wr_id = (*cur_qp)->wrid[wqe_index];
}
}
if (is_error) {
- handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send,
- (struct mthca_err_cqe *) cqe,
- entry, &free_cqe);
+ err = handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send,
+ (struct mthca_err_cqe *) cqe,
+ entry, &free_cqe);
goto out;
}
spin_lock_irqsave(&cq->lock, flags);
- npolled = 0;
-repoll:
- while (npolled < num_entries) {
+ for (npolled = 0; npolled < num_entries; ++npolled) {
err = mthca_poll_one(dev, cq, &qp,
&freed, entry + npolled);
if (err)
break;
- ++npolled;
}
if (freed) {
update_cons_index(dev, cq, freed);
}
- /*
- * If a CQ resize is in progress and we discovered that the
- * old buffer is empty, then peek in the new buffer, and if
- * it's not empty, switch to the new buffer and continue
- * polling there.
- */
- if (unlikely(err == -EAGAIN && cq->resize_buf &&
- cq->resize_buf->state == CQ_RESIZE_READY)) {
- /*
- * In Tavor mode, the hardware keeps the producer
- * index modulo the CQ size. Since we might be making
- * the CQ bigger, we need to mask our consumer index
- * using the size of the old CQ buffer before looking
- * in the new CQ buffer.
- */
- if (!mthca_is_memfree(dev))
- cq->cons_index &= cq->ibcq.cqe;
-
- if (cqe_sw(get_cqe_from_buf(&cq->resize_buf->buf,
- cq->cons_index & cq->resize_buf->cqe))) {
- struct mthca_cq_buf tbuf;
- int tcqe;
-
- tbuf = cq->buf;
- tcqe = cq->ibcq.cqe;
- cq->buf = cq->resize_buf->buf;
- cq->ibcq.cqe = cq->resize_buf->cqe;
-
- cq->resize_buf->buf = tbuf;
- cq->resize_buf->cqe = tcqe;
- cq->resize_buf->state = CQ_RESIZE_SWAPPED;
-
- goto repoll;
- }
- }
-
spin_unlock_irqrestore(&cq->lock, flags);
return err == 0 || err == -EAGAIN ? npolled : err;
return 0;
}
+static void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq *cq)
+{
+ mthca_buf_free(dev, (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE,
+ &cq->queue, cq->is_direct, &cq->mr);
+}
+
int mthca_init_cq(struct mthca_dev *dev, int nent,
struct mthca_ucontext *ctx, u32 pdn,
struct mthca_cq *cq)
{
+ int size = nent * MTHCA_CQ_ENTRY_SIZE;
struct mthca_mailbox *mailbox;
struct mthca_cq_context *cq_context;
int err = -ENOMEM;
u8 status;
+ int i;
+
+ might_sleep();
cq->ibcq.cqe = nent - 1;
cq->is_kernel = !ctx;
cq_context = mailbox->buf;
if (cq->is_kernel) {
- err = mthca_alloc_cq_buf(dev, &cq->buf, nent);
+ err = mthca_buf_alloc(dev, size, MTHCA_MAX_DIRECT_CQ_SIZE,
+ &cq->queue, &cq->is_direct,
+ &dev->driver_pd, 1, &cq->mr);
if (err)
goto err_out_mailbox;
+
+ for (i = 0; i < nent; ++i)
+ set_cqe_hw(get_cqe(cq, i));
}
spin_lock_init(&cq->lock);
- cq->refcount = 1;
+ atomic_set(&cq->refcount, 1);
init_waitqueue_head(&cq->wait);
- mutex_init(&cq->mutex);
memset(cq_context, 0, sizeof *cq_context);
cq_context->flags = cpu_to_be32(MTHCA_CQ_STATUS_OK |
cq_context->error_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn);
cq_context->comp_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_COMP].eqn);
cq_context->pd = cpu_to_be32(pdn);
- cq_context->lkey = cpu_to_be32(cq->buf.mr.ibmr.lkey);
+ cq_context->lkey = cpu_to_be32(cq->mr.ibmr.lkey);
cq_context->cqn = cpu_to_be32(cq->cqn);
if (mthca_is_memfree(dev)) {
err_out_free_mr:
if (cq->is_kernel)
- mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
+ mthca_free_cq_buf(dev, cq);
err_out_mailbox:
mthca_free_mailbox(dev, mailbox);
return err;
}
-static inline int get_cq_refcount(struct mthca_dev *dev, struct mthca_cq *cq)
-{
- int c;
-
- spin_lock_irq(&dev->cq_table.lock);
- c = cq->refcount;
- spin_unlock_irq(&dev->cq_table.lock);
-
- return c;
-}
-
void mthca_free_cq(struct mthca_dev *dev,
struct mthca_cq *cq)
{
int err;
u8 status;
+ might_sleep();
+
mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
if (IS_ERR(mailbox)) {
mthca_warn(dev, "No memory for mailbox to free CQ.\n");
spin_lock_irq(&dev->cq_table.lock);
mthca_array_clear(&dev->cq_table.cq,
cq->cqn & (dev->limits.num_cqs - 1));
- --cq->refcount;
spin_unlock_irq(&dev->cq_table.lock);
if (dev->mthca_flags & MTHCA_FLAG_MSI_X)
else
synchronize_irq(dev->pdev->irq);
- wait_event(cq->wait, !get_cq_refcount(dev, cq));
+ atomic_dec(&cq->refcount);
+ wait_event(cq->wait, !atomic_read(&cq->refcount));
if (cq->is_kernel) {
- mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
+ mthca_free_cq_buf(dev, cq);
if (mthca_is_memfree(dev)) {
mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index);
mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
return err;
}
-void mthca_cleanup_cq_table(struct mthca_dev *dev)
+void __devexit mthca_cleanup_cq_table(struct mthca_dev *dev)
{
mthca_array_cleanup(&dev->cq_table.cq, dev->limits.num_cqs);
mthca_alloc_cleanup(&dev->cq_table.alloc);
git://git.onelab.eu / linux-2.6.git / blobdiff