Lines Matching refs:cq
169 static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry) in get_cqe() argument
171 return get_cqe_from_buf(&cq->buf, entry); in get_cqe()
179 static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq) in next_cqe_sw() argument
181 return cqe_sw(get_cqe(cq, cq->cons_index & cq->ibcq.cqe)); in next_cqe_sw()
204 static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq, in update_cons_index() argument
208 *cq->set_ci_db = cpu_to_be32(cq->cons_index); in update_cons_index()
211 mthca_write64(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn, incr - 1, in update_cons_index()
224 struct mthca_cq *cq; in mthca_cq_completion() local
226 cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); in mthca_cq_completion()
228 if (!cq) { in mthca_cq_completion()
233 ++cq->arm_sn; in mthca_cq_completion()
235 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); in mthca_cq_completion()
241 struct mthca_cq *cq; in mthca_cq_event() local
246 cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); in mthca_cq_event()
247 if (cq) in mthca_cq_event()
248 ++cq->refcount; in mthca_cq_event()
252 if (!cq) { in mthca_cq_event()
259 event.element.cq = &cq->ibcq; in mthca_cq_event()
260 if (cq->ibcq.event_handler) in mthca_cq_event()
261 cq->ibcq.event_handler(&event, cq->ibcq.cq_context); in mthca_cq_event()
264 if (!--cq->refcount) in mthca_cq_event()
265 wake_up(&cq->wait); in mthca_cq_event()
278 void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, in mthca_cq_clean() argument
285 spin_lock_irq(&cq->lock); in mthca_cq_clean()
294 for (prod_index = cq->cons_index; in mthca_cq_clean()
295 cqe_sw(get_cqe(cq, prod_index & cq->ibcq.cqe)); in mthca_cq_clean()
297 if (prod_index == cq->cons_index + cq->ibcq.cqe) in mthca_cq_clean()
302 qpn, cq->cqn, cq->cons_index, prod_index); in mthca_cq_clean()
308 while ((int) --prod_index - (int) cq->cons_index >= 0) { in mthca_cq_clean()
309 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); in mthca_cq_clean()
315 memcpy(get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe), in mthca_cq_clean()
321 set_cqe_hw(get_cqe(cq, (cq->cons_index + i) & cq->ibcq.cqe)); in mthca_cq_clean()
323 cq->cons_index += nfreed; in mthca_cq_clean()
324 update_cons_index(dev, cq, nfreed); in mthca_cq_clean()
327 spin_unlock_irq(&cq->lock); in mthca_cq_clean()
330 void mthca_cq_resize_copy_cqes(struct mthca_cq *cq) in mthca_cq_resize_copy_cqes() argument
340 if (!mthca_is_memfree(to_mdev(cq->ibcq.device)) && in mthca_cq_resize_copy_cqes()
341 cq->ibcq.cqe < cq->resize_buf->cqe) { in mthca_cq_resize_copy_cqes()
342 cq->cons_index &= cq->ibcq.cqe; in mthca_cq_resize_copy_cqes()
343 if (cqe_sw(get_cqe(cq, cq->ibcq.cqe))) in mthca_cq_resize_copy_cqes()
344 cq->cons_index -= cq->ibcq.cqe + 1; in mthca_cq_resize_copy_cqes()
347 for (i = cq->cons_index; cqe_sw(get_cqe(cq, i & cq->ibcq.cqe)); ++i) in mthca_cq_resize_copy_cqes()
348 memcpy(get_cqe_from_buf(&cq->resize_buf->buf, in mthca_cq_resize_copy_cqes()
349 i & cq->resize_buf->cqe), in mthca_cq_resize_copy_cqes()
350 get_cqe(cq, i & cq->ibcq.cqe), MTHCA_CQ_ENTRY_SIZE); in mthca_cq_resize_copy_cqes()
377 static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, in handle_error_cqe() argument
389 cq->cqn, cq->cons_index); in handle_error_cqe()
484 struct mthca_cq *cq, in mthca_poll_one() argument
498 cqe = next_cqe_sw(cq); in mthca_poll_one()
510 cq->cqn, cq->cons_index, be32_to_cpu(cqe->my_qpn), in mthca_poll_one()
576 handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send, in mthca_poll_one()
656 ++cq->cons_index; in mthca_poll_one()
666 struct mthca_cq *cq = to_mcq(ibcq); in mthca_poll_cq() local
673 spin_lock_irqsave(&cq->lock, flags); in mthca_poll_cq()
678 err = mthca_poll_one(dev, cq, &qp, in mthca_poll_cq()
687 update_cons_index(dev, cq, freed); in mthca_poll_cq()
696 if (unlikely(err == -EAGAIN && cq->resize_buf && in mthca_poll_cq()
697 cq->resize_buf->state == CQ_RESIZE_READY)) { in mthca_poll_cq()
706 cq->cons_index &= cq->ibcq.cqe; in mthca_poll_cq()
708 if (cqe_sw(get_cqe_from_buf(&cq->resize_buf->buf, in mthca_poll_cq()
709 cq->cons_index & cq->resize_buf->cqe))) { in mthca_poll_cq()
713 tbuf = cq->buf; in mthca_poll_cq()
714 tcqe = cq->ibcq.cqe; in mthca_poll_cq()
715 cq->buf = cq->resize_buf->buf; in mthca_poll_cq()
716 cq->ibcq.cqe = cq->resize_buf->cqe; in mthca_poll_cq()
718 cq->resize_buf->buf = tbuf; in mthca_poll_cq()
719 cq->resize_buf->cqe = tcqe; in mthca_poll_cq()
720 cq->resize_buf->state = CQ_RESIZE_SWAPPED; in mthca_poll_cq()
726 spin_unlock_irqrestore(&cq->lock, flags); in mthca_poll_cq()
731 int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags) in mthca_tavor_arm_cq() argument
736 to_mcq(cq)->cqn; in mthca_tavor_arm_cq()
738 mthca_write64(dbhi, 0xffffffff, to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL, in mthca_tavor_arm_cq()
739 MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq->device)->doorbell_lock)); in mthca_tavor_arm_cq()
746 struct mthca_cq *cq = to_mcq(ibcq); in mthca_arbel_arm_cq() local
749 u32 sn = cq->arm_sn & 3; in mthca_arbel_arm_cq()
751 db_rec[0] = cpu_to_be32(cq->cons_index); in mthca_arbel_arm_cq()
752 db_rec[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) | in mthca_arbel_arm_cq()
756 mthca_write_db_rec(db_rec, cq->arm_db); in mthca_arbel_arm_cq()
767 MTHCA_ARBEL_CQ_DB_REQ_NOT) | cq->cqn; in mthca_arbel_arm_cq()
769 mthca_write64(dbhi, cq->cons_index, in mthca_arbel_arm_cq()
778 struct mthca_cq *cq) in mthca_init_cq() argument
784 cq->ibcq.cqe = nent - 1; in mthca_init_cq()
785 cq->is_kernel = !ctx; in mthca_init_cq()
787 cq->cqn = mthca_alloc(&dev->cq_table.alloc); in mthca_init_cq()
788 if (cq->cqn == -1) in mthca_init_cq()
792 err = mthca_table_get(dev, dev->cq_table.table, cq->cqn); in mthca_init_cq()
796 if (cq->is_kernel) { in mthca_init_cq()
797 cq->arm_sn = 1; in mthca_init_cq()
801 cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, in mthca_init_cq()
802 cq->cqn, &cq->set_ci_db); in mthca_init_cq()
803 if (cq->set_ci_db_index < 0) in mthca_init_cq()
806 cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM, in mthca_init_cq()
807 cq->cqn, &cq->arm_db); in mthca_init_cq()
808 if (cq->arm_db_index < 0) in mthca_init_cq()
819 if (cq->is_kernel) { in mthca_init_cq()
820 err = mthca_alloc_cq_buf(dev, &cq->buf, nent); in mthca_init_cq()
825 spin_lock_init(&cq->lock); in mthca_init_cq()
826 cq->refcount = 1; in mthca_init_cq()
827 init_waitqueue_head(&cq->wait); in mthca_init_cq()
828 mutex_init(&cq->mutex); in mthca_init_cq()
842 cq_context->lkey = cpu_to_be32(cq->buf.mr.ibmr.lkey); in mthca_init_cq()
843 cq_context->cqn = cpu_to_be32(cq->cqn); in mthca_init_cq()
846 cq_context->ci_db = cpu_to_be32(cq->set_ci_db_index); in mthca_init_cq()
847 cq_context->state_db = cpu_to_be32(cq->arm_db_index); in mthca_init_cq()
850 err = mthca_SW2HW_CQ(dev, mailbox, cq->cqn); in mthca_init_cq()
857 if (mthca_array_set(&dev->cq_table.cq, in mthca_init_cq()
858 cq->cqn & (dev->limits.num_cqs - 1), in mthca_init_cq()
859 cq)) { in mthca_init_cq()
865 cq->cons_index = 0; in mthca_init_cq()
872 if (cq->is_kernel) in mthca_init_cq()
873 mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mthca_init_cq()
879 if (cq->is_kernel && mthca_is_memfree(dev)) in mthca_init_cq()
880 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); in mthca_init_cq()
883 if (cq->is_kernel && mthca_is_memfree(dev)) in mthca_init_cq()
884 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); in mthca_init_cq()
887 mthca_table_put(dev, dev->cq_table.table, cq->cqn); in mthca_init_cq()
890 mthca_free(&dev->cq_table.alloc, cq->cqn); in mthca_init_cq()
895 static inline int get_cq_refcount(struct mthca_dev *dev, struct mthca_cq *cq) in get_cq_refcount() argument
900 c = cq->refcount; in get_cq_refcount()
907 struct mthca_cq *cq) in mthca_free_cq() argument
918 err = mthca_HW2SW_CQ(dev, mailbox, cq->cqn); in mthca_free_cq()
927 cq->cqn, cq->cons_index, in mthca_free_cq()
928 cq->is_kernel ? !!next_cqe_sw(cq) : 0); in mthca_free_cq()
934 mthca_array_clear(&dev->cq_table.cq, in mthca_free_cq()
935 cq->cqn & (dev->limits.num_cqs - 1)); in mthca_free_cq()
936 --cq->refcount; in mthca_free_cq()
944 wait_event(cq->wait, !get_cq_refcount(dev, cq)); in mthca_free_cq()
946 if (cq->is_kernel) { in mthca_free_cq()
947 mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mthca_free_cq()
949 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); in mthca_free_cq()
950 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); in mthca_free_cq()
954 mthca_table_put(dev, dev->cq_table.table, cq->cqn); in mthca_free_cq()
955 mthca_free(&dev->cq_table.alloc, cq->cqn); in mthca_free_cq()
972 err = mthca_array_init(&dev->cq_table.cq, in mthca_init_cq_table()
982 mthca_array_cleanup(&dev->cq_table.cq, dev->limits.num_cqs); in mthca_cleanup_cq_table()