X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=drivers%2Finfiniband%2Fhw%2Fmthca%2Fmthca_cq.c;h=96f1a86bf04950f54df57b36522ae008bfff00d5;hb=987b0145d94eecf292d8b301228356f44611ab7c;hp=205854e9c662809b49576b1ea8c3eb213e44ce72;hpb=f7ed79d23a47594e7834d66a8f14449796d4f3e6;p=linux-2.6.git diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c index 205854e9c..96f1a86bf 100644 --- a/drivers/infiniband/hw/mthca/mthca_cq.c +++ b/drivers/infiniband/hw/mthca/mthca_cq.c @@ -1,7 +1,7 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005, 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2005 Mellanox Technologies. All rights reserved. * Copyright (c) 2004 Voltaire, Inc. All rights reserved. * @@ -150,29 +150,24 @@ struct mthca_err_cqe { #define MTHCA_ARBEL_CQ_DB_REQ_NOT (2 << 24) #define MTHCA_ARBEL_CQ_DB_REQ_NOT_MULT (3 << 24) -static inline struct mthca_cqe *get_cqe_from_buf(struct mthca_cq_buf *buf, - int entry) +static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry) { - if (buf->is_direct) - return buf->queue.direct.buf + (entry * MTHCA_CQ_ENTRY_SIZE); + if (cq->is_direct) + return cq->queue.direct.buf + (entry * MTHCA_CQ_ENTRY_SIZE); else - return buf->queue.page_list[entry * MTHCA_CQ_ENTRY_SIZE / PAGE_SIZE].buf + return cq->queue.page_list[entry * MTHCA_CQ_ENTRY_SIZE / PAGE_SIZE].buf + (entry * MTHCA_CQ_ENTRY_SIZE) % PAGE_SIZE; } -static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry) -{ - return get_cqe_from_buf(&cq->buf, entry); -} - -static inline struct mthca_cqe *cqe_sw(struct mthca_cqe *cqe) +static inline struct mthca_cqe *cqe_sw(struct mthca_cq *cq, int i) { + struct mthca_cqe *cqe = get_cqe(cq, i); return MTHCA_CQ_ENTRY_OWNER_HW & cqe->owner ? NULL : cqe; } static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq) { - return cqe_sw(get_cqe(cq, cq->cons_index & cq->ibcq.cqe)); + return cqe_sw(cq, cq->cons_index & cq->ibcq.cqe); } static inline void set_cqe_hw(struct mthca_cqe *cqe) @@ -238,9 +233,9 @@ void mthca_cq_event(struct mthca_dev *dev, u32 cqn, spin_lock(&dev->cq_table.lock); cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); - if (cq) - ++cq->refcount; + if (cq) + atomic_inc(&cq->refcount); spin_unlock(&dev->cq_table.lock); if (!cq) { @@ -254,10 +249,8 @@ void mthca_cq_event(struct mthca_dev *dev, u32 cqn, if (cq->ibcq.event_handler) cq->ibcq.event_handler(&event, cq->ibcq.cq_context); - spin_lock(&dev->cq_table.lock); - if (!--cq->refcount) + if (atomic_dec_and_test(&cq->refcount)) wake_up(&cq->wait); - spin_unlock(&dev->cq_table.lock); } static inline int is_recv_cqe(struct mthca_cqe *cqe) @@ -269,13 +262,23 @@ static inline int is_recv_cqe(struct mthca_cqe *cqe) return !(cqe->is_send & 0x80); } -void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, +void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn, struct mthca_srq *srq) { + struct mthca_cq *cq; struct mthca_cqe *cqe; u32 prod_index; int nfreed = 0; + spin_lock_irq(&dev->cq_table.lock); + cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); + if (cq) + atomic_inc(&cq->refcount); + spin_unlock_irq(&dev->cq_table.lock); + + if (!cq) + return; + spin_lock_irq(&cq->lock); /* @@ -286,14 +289,14 @@ void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, * from our QP and therefore don't need to be checked. */ for (prod_index = cq->cons_index; - cqe_sw(get_cqe(cq, prod_index & cq->ibcq.cqe)); + cqe_sw(cq, prod_index & cq->ibcq.cqe); ++prod_index) if (prod_index == cq->cons_index + cq->ibcq.cqe) break; if (0) mthca_dbg(dev, "Cleaning QPN %06x from CQN %06x; ci %d, pi %d\n", - qpn, cq->cqn, cq->cons_index, prod_index); + qpn, cqn, cq->cons_index, prod_index); /* * Now sweep backwards through the CQ, removing CQ entries @@ -317,60 +320,16 @@ void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, } spin_unlock_irq(&cq->lock); + if (atomic_dec_and_test(&cq->refcount)) + wake_up(&cq->wait); } -void mthca_cq_resize_copy_cqes(struct mthca_cq *cq) -{ - int i; - - /* - * In Tavor mode, the hardware keeps the consumer and producer - * indices mod the CQ size. Since we might be making the CQ - * bigger, we need to deal with the case where the producer - * index wrapped around before the CQ was resized. - */ - if (!mthca_is_memfree(to_mdev(cq->ibcq.device)) && - cq->ibcq.cqe < cq->resize_buf->cqe) { - cq->cons_index &= cq->ibcq.cqe; - if (cqe_sw(get_cqe(cq, cq->ibcq.cqe))) - cq->cons_index -= cq->ibcq.cqe + 1; - } - - for (i = cq->cons_index; cqe_sw(get_cqe(cq, i & cq->ibcq.cqe)); ++i) - memcpy(get_cqe_from_buf(&cq->resize_buf->buf, - i & cq->resize_buf->cqe), - get_cqe(cq, i & cq->ibcq.cqe), MTHCA_CQ_ENTRY_SIZE); -} - -int mthca_alloc_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int nent) -{ - int ret; - int i; - - ret = mthca_buf_alloc(dev, nent * MTHCA_CQ_ENTRY_SIZE, - MTHCA_MAX_DIRECT_CQ_SIZE, - &buf->queue, &buf->is_direct, - &dev->driver_pd, 1, &buf->mr); - if (ret) - return ret; - - for (i = 0; i < nent; ++i) - set_cqe_hw(get_cqe_from_buf(buf, i)); - - return 0; -} - -void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int cqe) -{ - mthca_buf_free(dev, (cqe + 1) * MTHCA_CQ_ENTRY_SIZE, &buf->queue, - buf->is_direct, &buf->mr); -} - -static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, - struct mthca_qp *qp, int wqe_index, int is_send, - struct mthca_err_cqe *cqe, - struct ib_wc *entry, int *free_cqe) +static int handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, + struct mthca_qp *qp, int wqe_index, int is_send, + struct mthca_err_cqe *cqe, + struct ib_wc *entry, int *free_cqe) { + int err; int dbd; __be32 new_wqe; @@ -453,9 +412,11 @@ static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, * error case, so we don't have to check the doorbell count, etc. */ if (mthca_is_memfree(dev)) - return; + return 0; - mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe); + err = mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe); + if (err) + return err; /* * If we're at the end of the WQE chain, or we've used up our @@ -463,13 +424,15 @@ static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, * the next poll operation. */ if (!(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd)) - return; + return 0; cqe->db_cnt = cpu_to_be16(be16_to_cpu(cqe->db_cnt) - dbd); cqe->wqe = new_wqe; cqe->syndrome = SYNDROME_WR_FLUSH_ERR; *free_cqe = 0; + + return 0; } static inline int mthca_poll_one(struct mthca_dev *dev, @@ -555,9 +518,9 @@ static inline int mthca_poll_one(struct mthca_dev *dev, } if (is_error) { - handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send, - (struct mthca_err_cqe *) cqe, - entry, &free_cqe); + err = handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send, + (struct mthca_err_cqe *) cqe, + entry, &free_cqe); goto out; } @@ -651,14 +614,11 @@ int mthca_poll_cq(struct ib_cq *ibcq, int num_entries, spin_lock_irqsave(&cq->lock, flags); - npolled = 0; -repoll: - while (npolled < num_entries) { + for (npolled = 0; npolled < num_entries; ++npolled) { err = mthca_poll_one(dev, cq, &qp, &freed, entry + npolled); if (err) break; - ++npolled; } if (freed) { @@ -666,42 +626,6 @@ repoll: update_cons_index(dev, cq, freed); } - /* - * If a CQ resize is in progress and we discovered that the - * old buffer is empty, then peek in the new buffer, and if - * it's not empty, switch to the new buffer and continue - * polling there. - */ - if (unlikely(err == -EAGAIN && cq->resize_buf && - cq->resize_buf->state == CQ_RESIZE_READY)) { - /* - * In Tavor mode, the hardware keeps the producer - * index modulo the CQ size. Since we might be making - * the CQ bigger, we need to mask our consumer index - * using the size of the old CQ buffer before looking - * in the new CQ buffer. - */ - if (!mthca_is_memfree(dev)) - cq->cons_index &= cq->ibcq.cqe; - - if (cqe_sw(get_cqe_from_buf(&cq->resize_buf->buf, - cq->cons_index & cq->resize_buf->cqe))) { - struct mthca_cq_buf tbuf; - int tcqe; - - tbuf = cq->buf; - tcqe = cq->ibcq.cqe; - cq->buf = cq->resize_buf->buf; - cq->ibcq.cqe = cq->resize_buf->cqe; - - cq->resize_buf->buf = tbuf; - cq->resize_buf->cqe = tcqe; - cq->resize_buf->state = CQ_RESIZE_SWAPPED; - - goto repoll; - } - } - spin_unlock_irqrestore(&cq->lock, flags); return err == 0 || err == -EAGAIN ? npolled : err; @@ -760,14 +684,24 @@ int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify) return 0; } +static void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq *cq) +{ + mthca_buf_free(dev, (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE, + &cq->queue, cq->is_direct, &cq->mr); +} + int mthca_init_cq(struct mthca_dev *dev, int nent, struct mthca_ucontext *ctx, u32 pdn, struct mthca_cq *cq) { + int size = nent * MTHCA_CQ_ENTRY_SIZE; struct mthca_mailbox *mailbox; struct mthca_cq_context *cq_context; int err = -ENOMEM; u8 status; + int i; + + might_sleep(); cq->ibcq.cqe = nent - 1; cq->is_kernel = !ctx; @@ -805,13 +739,18 @@ int mthca_init_cq(struct mthca_dev *dev, int nent, cq_context = mailbox->buf; if (cq->is_kernel) { - err = mthca_alloc_cq_buf(dev, &cq->buf, nent); + err = mthca_buf_alloc(dev, size, MTHCA_MAX_DIRECT_CQ_SIZE, + &cq->queue, &cq->is_direct, + &dev->driver_pd, 1, &cq->mr); if (err) goto err_out_mailbox; + + for (i = 0; i < nent; ++i) + set_cqe_hw(get_cqe(cq, i)); } spin_lock_init(&cq->lock); - cq->refcount = 1; + atomic_set(&cq->refcount, 1); init_waitqueue_head(&cq->wait); memset(cq_context, 0, sizeof *cq_context); @@ -826,7 +765,7 @@ int mthca_init_cq(struct mthca_dev *dev, int nent, cq_context->error_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn); cq_context->comp_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_COMP].eqn); cq_context->pd = cpu_to_be32(pdn); - cq_context->lkey = cpu_to_be32(cq->buf.mr.ibmr.lkey); + cq_context->lkey = cpu_to_be32(cq->mr.ibmr.lkey); cq_context->cqn = cpu_to_be32(cq->cqn); if (mthca_is_memfree(dev)) { @@ -864,7 +803,7 @@ int mthca_init_cq(struct mthca_dev *dev, int nent, err_out_free_mr: if (cq->is_kernel) - mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); + mthca_free_cq_buf(dev, cq); err_out_mailbox: mthca_free_mailbox(dev, mailbox); @@ -886,17 +825,6 @@ err_out: return err; } -static inline int get_cq_refcount(struct mthca_dev *dev, struct mthca_cq *cq) -{ - int c; - - spin_lock_irq(&dev->cq_table.lock); - c = cq->refcount; - spin_unlock_irq(&dev->cq_table.lock); - - return c; -} - void mthca_free_cq(struct mthca_dev *dev, struct mthca_cq *cq) { @@ -904,6 +832,8 @@ void mthca_free_cq(struct mthca_dev *dev, int err; u8 status; + might_sleep(); + mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) { mthca_warn(dev, "No memory for mailbox to free CQ.\n"); @@ -930,7 +860,6 @@ void mthca_free_cq(struct mthca_dev *dev, spin_lock_irq(&dev->cq_table.lock); mthca_array_clear(&dev->cq_table.cq, cq->cqn & (dev->limits.num_cqs - 1)); - --cq->refcount; spin_unlock_irq(&dev->cq_table.lock); if (dev->mthca_flags & MTHCA_FLAG_MSI_X) @@ -938,10 +867,11 @@ void mthca_free_cq(struct mthca_dev *dev, else synchronize_irq(dev->pdev->irq); - wait_event(cq->wait, !get_cq_refcount(dev, cq)); + atomic_dec(&cq->refcount); + wait_event(cq->wait, !atomic_read(&cq->refcount)); if (cq->is_kernel) { - mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); + mthca_free_cq_buf(dev, cq); if (mthca_is_memfree(dev)) { mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); @@ -974,7 +904,7 @@ int __devinit mthca_init_cq_table(struct mthca_dev *dev) return err; } -void mthca_cleanup_cq_table(struct mthca_dev *dev) +void __devexit mthca_cleanup_cq_table(struct mthca_dev *dev) { mthca_array_cleanup(&dev->cq_table.cq, dev->limits.num_cqs); mthca_alloc_cleanup(&dev->cq_table.alloc);