71 int cxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq,  in cxio_hal_cq_op()  argument
79 	setup.id = cq->cqid;  in cxio_hal_cq_op()
92 	if (Q_PTR2IDX((cq->rptr), cq->size_log2) != ret) {  in cxio_hal_cq_op()
95 		rptr = cq->rptr;  in cxio_hal_cq_op()
101 		while (Q_PTR2IDX((rptr+1), cq->size_log2) != ret)  in cxio_hal_cq_op()
109 		cqe = cq->queue + Q_PTR2IDX(rptr, cq->size_log2);  in cxio_hal_cq_op()
110 		while (!CQ_VLD_ENTRY(rptr, cq->size_log2, cqe)) {  in cxio_hal_cq_op()
158 int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel)  in cxio_create_cq()  argument
161 	int size = (1UL << (cq->size_log2)) * sizeof(struct t3_cqe);  in cxio_create_cq()
164 	cq->cqid = cxio_hal_get_cqid(rdev_p->rscp);  in cxio_create_cq()
165 	if (!cq->cqid)  in cxio_create_cq()
168 		cq->sw_queue = kzalloc(size, GFP_KERNEL);  in cxio_create_cq()
169 		if (!cq->sw_queue)  in cxio_create_cq()
172 	cq->queue = dma_alloc_coherent(&(rdev_p->rnic_info.pdev->dev), size,  in cxio_create_cq()
173 					     &(cq->dma_addr), GFP_KERNEL);  in cxio_create_cq()
174 	if (!cq->queue) {  in cxio_create_cq()
175 		kfree(cq->sw_queue);  in cxio_create_cq()
178 	dma_unmap_addr_set(cq, mapping, cq->dma_addr);  in cxio_create_cq()
179 	memset(cq->queue, 0, size);  in cxio_create_cq()
180 	setup.id = cq->cqid;  in cxio_create_cq()
181 	setup.base_addr = (u64) (cq->dma_addr);  in cxio_create_cq()
182 	setup.size = 1UL << cq->size_log2;  in cxio_create_cq()
193 int cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)  in cxio_resize_cq()  argument
196 	setup.id = cq->cqid;  in cxio_resize_cq()
197 	setup.base_addr = (u64) (cq->dma_addr);  in cxio_resize_cq()
198 	setup.size = 1UL << cq->size_log2;  in cxio_resize_cq()
323 int cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)  in cxio_destroy_cq()  argument
326 	err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);  in cxio_destroy_cq()
327 	kfree(cq->sw_queue);  in cxio_destroy_cq()
329 			  (1UL << (cq->size_log2))  in cxio_destroy_cq()
330 			  * sizeof(struct t3_cqe), cq->queue,  in cxio_destroy_cq()
331 			  dma_unmap_addr(cq, mapping));  in cxio_destroy_cq()
332 	cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);  in cxio_destroy_cq()
350 static void insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)  in insert_recv_cqe()  argument
355 	     wq, cq, cq->sw_rptr, cq->sw_wptr);  in insert_recv_cqe()
362 				 V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,  in insert_recv_cqe()
363 						       cq->size_log2)));  in insert_recv_cqe()
364 	*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;  in insert_recv_cqe()
365 	cq->sw_wptr++;  in insert_recv_cqe()
368 int cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)  in cxio_flush_rq()  argument
373 	PDBG("%s wq %p cq %p\n", __func__, wq, cq);  in cxio_flush_rq()
380 		insert_recv_cqe(wq, cq);  in cxio_flush_rq()
386 static void insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,  in insert_sq_cqe()  argument
392 	     wq, cq, cq->sw_rptr, cq->sw_wptr);  in insert_sq_cqe()
399 			         V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,  in insert_sq_cqe()
400 						       cq->size_log2)));  in insert_sq_cqe()
403 	*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;  in insert_sq_cqe()
404 	cq->sw_wptr++;  in insert_sq_cqe()
407 int cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)  in cxio_flush_sq()  argument
417 		insert_sq_cqe(wq, cq, sqp);  in cxio_flush_sq()
428 void cxio_flush_hw_cq(struct t3_cq *cq)  in cxio_flush_hw_cq()  argument
432 	PDBG("%s cq %p cqid 0x%x\n", __func__, cq, cq->cqid);  in cxio_flush_hw_cq()
433 	cqe = cxio_next_hw_cqe(cq);  in cxio_flush_hw_cq()
436 		     __func__, cq->rptr, cq->sw_wptr);  in cxio_flush_hw_cq()
437 		swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2);  in cxio_flush_hw_cq()
440 		cq->sw_wptr++;  in cxio_flush_hw_cq()
441 		cq->rptr++;  in cxio_flush_hw_cq()
442 		cqe = cxio_next_hw_cqe(cq);  in cxio_flush_hw_cq()
464 void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)  in cxio_count_scqes()  argument
470 	ptr = cq->sw_rptr;  in cxio_count_scqes()
471 	while (!Q_EMPTY(ptr, cq->sw_wptr)) {  in cxio_count_scqes()
472 		cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));  in cxio_count_scqes()
479 	PDBG("%s cq %p count %d\n", __func__, cq, *count);  in cxio_count_scqes()
482 void cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)  in cxio_count_rcqes()  argument
489 	ptr = cq->sw_rptr;  in cxio_count_rcqes()
490 	while (!Q_EMPTY(ptr, cq->sw_wptr)) {  in cxio_count_rcqes()
491 		cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));  in cxio_count_rcqes()
497 	PDBG("%s cq %p count %d\n", __func__, cq, *count);  in cxio_count_rcqes()
1073 static void flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)  in flush_completed_wrs()  argument
1091 			     Q_PTR2IDX(cq->sw_wptr, cq->size_log2));  in flush_completed_wrs()
1093 			*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2))  in flush_completed_wrs()
1095 			cq->sw_wptr++;  in flush_completed_wrs()
1147 int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,  in cxio_poll_cq()  argument
1155 	hw_cqe = cxio_next_cqe(cq);  in cxio_poll_cq()
1321 	flush_completed_wrs(wq, cq);  in cxio_poll_cq()
1326 		     __func__, cq, cq->cqid, cq->sw_rptr);  in cxio_poll_cq()
1327 		++cq->sw_rptr;  in cxio_poll_cq()
1330 		     __func__, cq, cq->cqid, cq->rptr);  in cxio_poll_cq()
1331 		++cq->rptr;  in cxio_poll_cq()
1336 		if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1)))  in cxio_poll_cq()
1337 		    || ((cq->rptr - cq->wptr) >= 128)) {  in cxio_poll_cq()
1338 			*credit = cq->rptr - cq->wptr;  in cxio_poll_cq()
1339 			cq->wptr = cq->rptr;  in cxio_poll_cq()