966 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,  in ocrdma_copy_cq_uresp()  argument
975 	uresp.cq_id = cq->id;  in ocrdma_copy_cq_uresp()
976 	uresp.page_size = PAGE_ALIGN(cq->len);  in ocrdma_copy_cq_uresp()
978 	uresp.max_hw_cqe = cq->max_hw_cqe;  in ocrdma_copy_cq_uresp()
979 	uresp.page_addr[0] = virt_to_phys(cq->va);  in ocrdma_copy_cq_uresp()
982 	uresp.phase_change = cq->phase_change ? 1 : 0;  in ocrdma_copy_cq_uresp()
986 		       __func__, dev->id, cq->id);  in ocrdma_copy_cq_uresp()
997 	cq->ucontext = uctx;  in ocrdma_copy_cq_uresp()
1006 	struct ocrdma_cq *cq;  in ocrdma_create_cq()  local
1018 	cq = kzalloc(sizeof(*cq), GFP_KERNEL);  in ocrdma_create_cq()
1019 	if (!cq)  in ocrdma_create_cq()
1022 	spin_lock_init(&cq->cq_lock);  in ocrdma_create_cq()
1023 	spin_lock_init(&cq->comp_handler_lock);  in ocrdma_create_cq()
1024 	INIT_LIST_HEAD(&cq->sq_head);  in ocrdma_create_cq()
1025 	INIT_LIST_HEAD(&cq->rq_head);  in ocrdma_create_cq()
1026 	cq->first_arm = true;  in ocrdma_create_cq()
1033 	status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id);  in ocrdma_create_cq()
1035 		kfree(cq);  in ocrdma_create_cq()
1039 		status = ocrdma_copy_cq_uresp(dev, cq, udata, ib_ctx);  in ocrdma_create_cq()
1043 	cq->phase = OCRDMA_CQE_VALID;  in ocrdma_create_cq()
1044 	dev->cq_tbl[cq->id] = cq;  in ocrdma_create_cq()
1045 	return &cq->ibcq;  in ocrdma_create_cq()
1048 	ocrdma_mbx_destroy_cq(dev, cq);  in ocrdma_create_cq()
1049 	kfree(cq);  in ocrdma_create_cq()
1057 	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);  in ocrdma_resize_cq()  local
1059 	if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) {  in ocrdma_resize_cq()
1067 static void ocrdma_flush_cq(struct ocrdma_cq *cq)  in ocrdma_flush_cq()  argument
1073 	struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);  in ocrdma_flush_cq()
1076 	cqe = cq->va;  in ocrdma_flush_cq()
1077 	cqe_cnt = cq->cqe_cnt;  in ocrdma_flush_cq()
1082 	spin_lock_irqsave(&cq->cq_lock, flags);  in ocrdma_flush_cq()
1084 		if (is_cqe_valid(cq, cqe))  in ocrdma_flush_cq()
1089 	ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count);  in ocrdma_flush_cq()
1090 	spin_unlock_irqrestore(&cq->cq_lock, flags);  in ocrdma_flush_cq()
1095 	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);  in ocrdma_destroy_cq()  local
1101 	dev->cq_tbl[cq->id] = NULL;  in ocrdma_destroy_cq()
1102 	indx = ocrdma_get_eq_table_index(dev, cq->eqn);  in ocrdma_destroy_cq()
1109 	ocrdma_flush_cq(cq);  in ocrdma_destroy_cq()
1111 	(void)ocrdma_mbx_destroy_cq(dev, cq);  in ocrdma_destroy_cq()
1112 	if (cq->ucontext) {  in ocrdma_destroy_cq()
1113 		pdid = cq->ucontext->cntxt_pd->id;  in ocrdma_destroy_cq()
1114 		ocrdma_del_mmap(cq->ucontext, (u64) cq->pa,  in ocrdma_destroy_cq()
1115 				PAGE_ALIGN(cq->len));  in ocrdma_destroy_cq()
1116 		ocrdma_del_mmap(cq->ucontext,  in ocrdma_destroy_cq()
1121 	kfree(cq);  in ocrdma_destroy_cq()
1633 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)  in ocrdma_discard_cqes()  argument
1642 	spin_lock_irqsave(&cq->cq_lock, cq_flags);  in ocrdma_discard_cqes()
1651 	cur_getp = cq->getp;  in ocrdma_discard_cqes()
1658 		cqe = cq->va + cur_getp;  in ocrdma_discard_cqes()
1693 		cur_getp = (cur_getp + 1) % cq->max_hw_cqe;  in ocrdma_discard_cqes()
1695 	spin_unlock_irqrestore(&cq->cq_lock, cq_flags);  in ocrdma_discard_cqes()
2802 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,  in ocrdma_change_cq_phase()  argument
2805 	if (cq->phase_change) {  in ocrdma_change_cq_phase()
2807 			cq->phase = (~cq->phase & OCRDMA_CQE_VALID);  in ocrdma_change_cq_phase()
2814 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,  in ocrdma_poll_hwcq()  argument
2822 	struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);  in ocrdma_poll_hwcq()
2826 	cur_getp = cq->getp;  in ocrdma_poll_hwcq()
2828 		cqe = cq->va + cur_getp;  in ocrdma_poll_hwcq()
2830 		if (!is_cqe_valid(cq, cqe))  in ocrdma_poll_hwcq()
2854 		cur_getp = (cur_getp + 1) % cq->max_hw_cqe;  in ocrdma_poll_hwcq()
2855 		ocrdma_change_cq_phase(cq, cqe, cur_getp);  in ocrdma_poll_hwcq()
2865 	cq->getp = cur_getp;  in ocrdma_poll_hwcq()
2866 	if (cq->deferred_arm) {  in ocrdma_poll_hwcq()
2867 		ocrdma_ring_cq_db(dev, cq->id, true, cq->deferred_sol,  in ocrdma_poll_hwcq()
2869 		cq->deferred_arm = false;  in ocrdma_poll_hwcq()
2870 		cq->deferred_sol = false;  in ocrdma_poll_hwcq()
2873 		ocrdma_ring_cq_db(dev, cq->id, false, cq->deferred_sol,  in ocrdma_poll_hwcq()
2875 		cq->deferred_sol = false;  in ocrdma_poll_hwcq()
2882 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,  in ocrdma_add_err_cqe()  argument
2890 		if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {  in ocrdma_add_err_cqe()
2893 		} else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {  in ocrdma_add_err_cqe()
2911 	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);  in ocrdma_poll_cq()  local
2918 	spin_lock_irqsave(&cq->cq_lock, flags);  in ocrdma_poll_cq()
2919 	num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc);  in ocrdma_poll_cq()
2920 	spin_unlock_irqrestore(&cq->cq_lock, flags);  in ocrdma_poll_cq()
2931 		list_for_each_entry(qp, &cq->sq_head, sq_entry) {  in ocrdma_poll_cq()
2934 			err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);  in ocrdma_poll_cq()
2946 	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);  in ocrdma_arm_cq()  local
2952 	cq_id = cq->id;  in ocrdma_arm_cq()
2954 	spin_lock_irqsave(&cq->cq_lock, flags);  in ocrdma_arm_cq()
2960 	if (cq->first_arm) {  in ocrdma_arm_cq()
2962 		cq->first_arm = false;  in ocrdma_arm_cq()
2965 	cq->deferred_arm = true;  in ocrdma_arm_cq()
2966 	cq->deferred_sol = sol_needed;  in ocrdma_arm_cq()
2967 	spin_unlock_irqrestore(&cq->cq_lock, flags);  in ocrdma_arm_cq()