185 static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq)  in insert_recv_cqe()  argument
190 	     wq, cq, cq->sw_cidx, cq->sw_pidx);  in insert_recv_cqe()
196 				 CQE_QPID_V(wq->sq.qid));  in insert_recv_cqe()
202 int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count)  in c4iw_flush_rq()  argument
205 	int in_use = wq->rq.in_use - count;  in c4iw_flush_rq()
209 	     wq, cq, wq->rq.in_use, count);  in c4iw_flush_rq()
211 		insert_recv_cqe(wq, cq);  in c4iw_flush_rq()
217 static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,  in insert_sq_cqe()  argument
223 	     wq, cq, cq->sw_cidx, cq->sw_pidx);  in insert_sq_cqe()
229 				 CQE_QPID_V(wq->sq.qid));  in insert_sq_cqe()
236 static void advance_oldest_read(struct t4_wq *wq);
241 	struct t4_wq *wq = &qhp->wq;  in c4iw_flush_sq()  local
247 	if (wq->sq.flush_cidx == -1)  in c4iw_flush_sq()
248 		wq->sq.flush_cidx = wq->sq.cidx;  in c4iw_flush_sq()
249 	idx = wq->sq.flush_cidx;  in c4iw_flush_sq()
250 	BUG_ON(idx >= wq->sq.size);  in c4iw_flush_sq()
251 	while (idx != wq->sq.pidx) {  in c4iw_flush_sq()
252 		swsqe = &wq->sq.sw_sq[idx];  in c4iw_flush_sq()
255 		insert_sq_cqe(wq, cq, swsqe);  in c4iw_flush_sq()
256 		if (wq->sq.oldest_read == swsqe) {  in c4iw_flush_sq()
258 			advance_oldest_read(wq);  in c4iw_flush_sq()
261 		if (++idx == wq->sq.size)  in c4iw_flush_sq()
264 	wq->sq.flush_cidx += flushed;  in c4iw_flush_sq()
265 	if (wq->sq.flush_cidx >= wq->sq.size)  in c4iw_flush_sq()
266 		wq->sq.flush_cidx -= wq->sq.size;  in c4iw_flush_sq()
270 static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)  in flush_completed_wrs()  argument
275 	if (wq->sq.flush_cidx == -1)  in flush_completed_wrs()
276 		wq->sq.flush_cidx = wq->sq.cidx;  in flush_completed_wrs()
277 	cidx = wq->sq.flush_cidx;  in flush_completed_wrs()
278 	BUG_ON(cidx > wq->sq.size);  in flush_completed_wrs()
280 	while (cidx != wq->sq.pidx) {  in flush_completed_wrs()
281 		swsqe = &wq->sq.sw_sq[cidx];  in flush_completed_wrs()
283 			if (++cidx == wq->sq.size)  in flush_completed_wrs()
298 			if (++cidx == wq->sq.size)  in flush_completed_wrs()
300 			wq->sq.flush_cidx = cidx;  in flush_completed_wrs()
306 static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,  in create_read_req_cqe()  argument
309 	read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;  in create_read_req_cqe()
310 	read_cqe->len = htonl(wq->sq.oldest_read->read_len);  in create_read_req_cqe()
318 static void advance_oldest_read(struct t4_wq *wq)  in advance_oldest_read()  argument
321 	u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;  in advance_oldest_read()
323 	if (rptr == wq->sq.size)  in advance_oldest_read()
325 	while (rptr != wq->sq.pidx) {  in advance_oldest_read()
326 		wq->sq.oldest_read = &wq->sq.sw_sq[rptr];  in advance_oldest_read()
328 		if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)  in advance_oldest_read()
330 		if (++rptr == wq->sq.size)  in advance_oldest_read()
333 	wq->sq.oldest_read = NULL;  in advance_oldest_read()
385 			if (!qhp->wq.sq.oldest_read->signaled) {  in c4iw_flush_hw_cq()
386 				advance_oldest_read(&qhp->wq);  in c4iw_flush_hw_cq()
394 			create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe);  in c4iw_flush_hw_cq()
396 			advance_oldest_read(&qhp->wq);  in c4iw_flush_hw_cq()
403 			swsqe = &qhp->wq.sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];  in c4iw_flush_hw_cq()
406 			flush_completed_wrs(&qhp->wq, &chp->cq);  in c4iw_flush_hw_cq()
419 static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)  in cqe_completes_wr()  argument
430 	if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq))  in cqe_completes_wr()
435 void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)  in c4iw_count_rcqes()  argument
446 		    (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq))  in c4iw_count_rcqes()
470 static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,  in poll_cq()  argument
492 	if (wq == NULL) {  in poll_cq()
500 	if (wq->flushed && !SW_CQE(hw_cqe)) {  in poll_cq()
528 				t4_set_wq_in_error(wq);  in poll_cq()
539 				t4_set_wq_in_error(wq);  in poll_cq()
547 		if (!wq->sq.oldest_read->signaled) {  in poll_cq()
548 			advance_oldest_read(wq);  in poll_cq()
557 		create_read_req_cqe(wq, hw_cqe, &read_cqe);  in poll_cq()
559 		advance_oldest_read(wq);  in poll_cq()
562 	if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) {  in poll_cq()
564 		t4_set_wq_in_error(wq);  in poll_cq()
579 		if (t4_rq_empty(wq)) {  in poll_cq()
580 			t4_set_wq_in_error(wq);  in poll_cq()
584 		if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) {  in poll_cq()
585 			t4_set_wq_in_error(wq);  in poll_cq()
603 	if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) {  in poll_cq()
608 		swsqe = &wq->sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];  in poll_cq()
624 		BUG_ON(idx >= wq->sq.size);  in poll_cq()
634 		if (idx < wq->sq.cidx)  in poll_cq()
635 			wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx;  in poll_cq()
637 			wq->sq.in_use -= idx - wq->sq.cidx;  in poll_cq()
638 		BUG_ON(wq->sq.in_use <= 0 && wq->sq.in_use >= wq->sq.size);  in poll_cq()
640 		wq->sq.cidx = (uint16_t)idx;  in poll_cq()
641 		PDBG("%s completing sq idx %u\n", __func__, wq->sq.cidx);  in poll_cq()
642 		*cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id;  in poll_cq()
644 			c4iw_log_wr_stats(wq, hw_cqe);  in poll_cq()
645 		t4_sq_consume(wq);  in poll_cq()
647 		PDBG("%s completing rq idx %u\n", __func__, wq->rq.cidx);  in poll_cq()
648 		*cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id;  in poll_cq()
649 		BUG_ON(t4_rq_empty(wq));  in poll_cq()
651 			c4iw_log_wr_stats(wq, hw_cqe);  in poll_cq()
652 		t4_rq_consume(wq);  in poll_cq()
660 	flush_completed_wrs(wq, cq);  in poll_cq()
688 	struct t4_wq *wq;  in c4iw_poll_cq_one()  local
701 		wq = NULL;  in c4iw_poll_cq_one()
704 		wq = &(qhp->wq);  in c4iw_poll_cq_one()
706 	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit);  in c4iw_poll_cq_one()
823 	if (wq)  in c4iw_poll_cq_one()