107 	    (dev->mq.cq.va + (dev->mq.cq.tail * sizeof(struct ocrdma_mcqe)));  in ocrdma_get_mcqe()
116 	dev->mq.cq.tail = (dev->mq.cq.tail + 1) & (OCRDMA_MQ_CQ_LEN - 1);  in ocrdma_mcq_inc_tail()
505 				   struct ocrdma_queue_info *cq,  in ocrdma_mbx_mq_cq_create()  argument
517 	cmd->pgsz_pgcnt = (cq->size / OCRDMA_MIN_Q_PAGE_SIZE) <<  in ocrdma_mbx_mq_cq_create()
519 	cmd->pgsz_pgcnt |= PAGES_4K_SPANNED(cq->va, cq->size);  in ocrdma_mbx_mq_cq_create()
523 	cmd->pdid_cqecnt = cq->size / sizeof(struct ocrdma_mcqe);  in ocrdma_mbx_mq_cq_create()
525 	ocrdma_build_q_pages(&cmd->pa[0], cq->size / OCRDMA_MIN_Q_PAGE_SIZE,  in ocrdma_mbx_mq_cq_create()
526 			     cq->dma, PAGE_SIZE_4K);  in ocrdma_mbx_mq_cq_create()
530 		cq->id = (u16) (rsp->cq_id & OCRDMA_CREATE_CQ_RSP_CQ_ID_MASK);  in ocrdma_mbx_mq_cq_create()
531 		cq->created = true;  in ocrdma_mbx_mq_cq_create()
547 				struct ocrdma_queue_info *cq)  in ocrdma_mbx_create_mq()  argument
561 	cmd->cqid_pages |= (cq->id << OCRDMA_CREATE_MQ_CQ_ID_SHIFT);  in ocrdma_mbx_create_mq()
567 	cmd->async_cqid_ringsize = cq->id;  in ocrdma_mbx_create_mq()
588 	status = ocrdma_alloc_q(dev, &dev->mq.cq, OCRDMA_MQ_CQ_LEN,  in ocrdma_create_mq()
594 	status = ocrdma_mbx_mq_cq_create(dev, &dev->mq.cq, &dev->eq_tbl[0].q);  in ocrdma_create_mq()
607 	status = ocrdma_mbx_create_mq(dev, &dev->mq.sq, &dev->mq.cq);  in ocrdma_create_mq()
610 	ocrdma_ring_cq_db(dev, dev->mq.cq.id, true, false, 0);  in ocrdma_create_mq()
616 	ocrdma_mbx_delete_q(dev, &dev->mq.cq, QTYPE_CQ);  in ocrdma_create_mq()
618 	ocrdma_free_q(dev, &dev->mq.cq);  in ocrdma_create_mq()
625 	struct ocrdma_queue_info *mbxq, *cq;  in ocrdma_destroy_mq()  local
636 	cq = &dev->mq.cq;  in ocrdma_destroy_mq()
637 	if (cq->created) {  in ocrdma_destroy_mq()
638 		ocrdma_mbx_delete_q(dev, cq, QTYPE_CQ);  in ocrdma_destroy_mq()
639 		ocrdma_free_q(dev, cq);  in ocrdma_destroy_mq()
658 	struct ocrdma_cq *cq = NULL;  in ocrdma_dispatch_ibevent()  local
670 		cq = dev->cq_tbl[cqe->cqvalid_cqid & OCRDMA_AE_MCQE_CQID_MASK];  in ocrdma_dispatch_ibevent()
678 		ib_evt.element.cq = &cq->ibcq;  in ocrdma_dispatch_ibevent()
684 		ib_evt.element.cq = &cq->ibcq;  in ocrdma_dispatch_ibevent()
744 		if (cq->ibcq.event_handler)  in ocrdma_dispatch_ibevent()
745 			cq->ibcq.event_handler(&ib_evt, cq->ibcq.cq_context);  in ocrdma_dispatch_ibevent()
833 	ocrdma_ring_cq_db(dev, dev->mq.cq.id, true, false, cqe_popped);  in ocrdma_mq_cq_handler()
838 				struct ocrdma_cq *cq, bool sq)  in _ocrdma_qp_buddy_cq_handler()  argument
843 	struct list_head *head = sq?(&cq->sq_head):(&cq->rq_head);  in _ocrdma_qp_buddy_cq_handler()
861 		if (qp->sq_cq == cq)  in _ocrdma_qp_buddy_cq_handler()
871 				       struct ocrdma_cq *cq)  in ocrdma_qp_buddy_cq_handler()  argument
887 	bcq = _ocrdma_qp_buddy_cq_handler(dev, cq, true);  in ocrdma_qp_buddy_cq_handler()
889 		bcq = _ocrdma_qp_buddy_cq_handler(dev, cq, false);  in ocrdma_qp_buddy_cq_handler()
903 	struct ocrdma_cq *cq;  in ocrdma_qp_cq_handler()  local
908 	cq = dev->cq_tbl[cq_idx];  in ocrdma_qp_cq_handler()
909 	if (cq == NULL)  in ocrdma_qp_cq_handler()
912 	if (cq->ibcq.comp_handler) {  in ocrdma_qp_cq_handler()
913 		spin_lock_irqsave(&cq->comp_handler_lock, flags);  in ocrdma_qp_cq_handler()
914 		(*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context);  in ocrdma_qp_cq_handler()
915 		spin_unlock_irqrestore(&cq->comp_handler_lock, flags);  in ocrdma_qp_cq_handler()
917 	ocrdma_qp_buddy_cq_handler(dev, cq);  in ocrdma_qp_cq_handler()
923 	if (cq_id == dev->mq.cq.id)  in ocrdma_cq_handler()
1714 int ocrdma_mbx_create_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq,  in ocrdma_mbx_create_cq()  argument
1732 		cq->max_hw_cqe = 1;  in ocrdma_mbx_create_cq()
1737 		cq->max_hw_cqe = dev->attr.max_cqe;  in ocrdma_mbx_create_cq()
1743 	cq->len = roundup(max_hw_cqe * cqe_size, OCRDMA_MIN_Q_PAGE_SIZE);  in ocrdma_mbx_create_cq()
1750 	cq->va = dma_alloc_coherent(&pdev->dev, cq->len, &cq->pa, GFP_KERNEL);  in ocrdma_mbx_create_cq()
1751 	if (!cq->va) {  in ocrdma_mbx_create_cq()
1755 	memset(cq->va, 0, cq->len);  in ocrdma_mbx_create_cq()
1756 	page_size = cq->len / hw_pages;  in ocrdma_mbx_create_cq()
1762 	cq->eqn = ocrdma_bind_eq(dev);  in ocrdma_mbx_create_cq()
1764 	cqe_count = cq->len / cqe_size;  in ocrdma_mbx_create_cq()
1765 	cq->cqe_cnt = cqe_count;  in ocrdma_mbx_create_cq()
1787 	cmd->cmd.eqn = cq->eqn;  in ocrdma_mbx_create_cq()
1792 		cq->phase_change = false;  in ocrdma_mbx_create_cq()
1793 		cmd->cmd.pdid_cqecnt = (cq->len / cqe_size);  in ocrdma_mbx_create_cq()
1795 		cmd->cmd.pdid_cqecnt = (cq->len / cqe_size) - 1;  in ocrdma_mbx_create_cq()
1797 		cq->phase_change = true;  in ocrdma_mbx_create_cq()
1803 	ocrdma_build_q_pages(&cmd->cmd.pa[0], hw_pages, cq->pa, page_size);  in ocrdma_mbx_create_cq()
1809 	cq->id = (u16) (rsp->rsp.cq_id & OCRDMA_CREATE_CQ_RSP_CQ_ID_MASK);  in ocrdma_mbx_create_cq()
1813 	ocrdma_unbind_eq(dev, cq->eqn);  in ocrdma_mbx_create_cq()
1814 	dma_free_coherent(&pdev->dev, cq->len, cq->va, cq->pa);  in ocrdma_mbx_create_cq()
1820 int ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq)  in ocrdma_mbx_destroy_cq()  argument
1832 	    (cq->id << OCRDMA_DESTROY_CQ_QID_SHIFT) &  in ocrdma_mbx_destroy_cq()
1836 	ocrdma_unbind_eq(dev, cq->eqn);  in ocrdma_mbx_destroy_cq()
1837 	dma_free_coherent(&dev->nic_info.pdev->dev, cq->len, cq->va, cq->pa);  in ocrdma_mbx_destroy_cq()
2019 bool ocrdma_is_qp_in_sq_flushlist(struct ocrdma_cq *cq, struct ocrdma_qp *qp)  in ocrdma_is_qp_in_sq_flushlist()  argument
2023 	list_for_each_entry(tmp, &cq->sq_head, sq_entry) {  in ocrdma_is_qp_in_sq_flushlist()
2032 bool ocrdma_is_qp_in_rq_flushlist(struct ocrdma_cq *cq, struct ocrdma_qp *qp)  in ocrdma_is_qp_in_rq_flushlist()  argument
2036 	list_for_each_entry(tmp, &cq->rq_head, rq_entry) {  in ocrdma_is_qp_in_rq_flushlist()
2317 	struct ocrdma_cq *cq;  in ocrdma_mbx_create_qp()  local
2372 	cq = get_ocrdma_cq(attrs->send_cq);  in ocrdma_mbx_create_qp()
2373 	cmd->wq_rq_cqid |= (cq->id << OCRDMA_CREATE_QP_REQ_WQ_CQID_SHIFT) &  in ocrdma_mbx_create_qp()
2375 	qp->sq_cq = cq;  in ocrdma_mbx_create_qp()
2376 	cq = get_ocrdma_cq(attrs->recv_cq);  in ocrdma_mbx_create_qp()
2377 	cmd->wq_rq_cqid |= (cq->id << OCRDMA_CREATE_QP_REQ_RQ_CQID_SHIFT) &  in ocrdma_mbx_create_qp()
2379 	qp->rq_cq = cq;  in ocrdma_mbx_create_qp()