344 static inline int t4_rqes_posted(struct t4_wq *wq)  in t4_rqes_posted()  argument
346 	return wq->rq.in_use;  in t4_rqes_posted()
349 static inline int t4_rq_empty(struct t4_wq *wq)  in t4_rq_empty()  argument
351 	return wq->rq.in_use == 0;  in t4_rq_empty()
354 static inline int t4_rq_full(struct t4_wq *wq)  in t4_rq_full()  argument
356 	return wq->rq.in_use == (wq->rq.size - 1);  in t4_rq_full()
359 static inline u32 t4_rq_avail(struct t4_wq *wq)  in t4_rq_avail()  argument
361 	return wq->rq.size - 1 - wq->rq.in_use;  in t4_rq_avail()
364 static inline void t4_rq_produce(struct t4_wq *wq, u8 len16)  in t4_rq_produce()  argument
366 	wq->rq.in_use++;  in t4_rq_produce()
367 	if (++wq->rq.pidx == wq->rq.size)  in t4_rq_produce()
368 		wq->rq.pidx = 0;  in t4_rq_produce()
369 	wq->rq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_rq_produce()
370 	if (wq->rq.wq_pidx >= wq->rq.size * T4_RQ_NUM_SLOTS)  in t4_rq_produce()
371 		wq->rq.wq_pidx %= wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_produce()
374 static inline void t4_rq_consume(struct t4_wq *wq)  in t4_rq_consume()  argument
376 	wq->rq.in_use--;  in t4_rq_consume()
377 	wq->rq.msn++;  in t4_rq_consume()
378 	if (++wq->rq.cidx == wq->rq.size)  in t4_rq_consume()
379 		wq->rq.cidx = 0;  in t4_rq_consume()
382 static inline u16 t4_rq_host_wq_pidx(struct t4_wq *wq)  in t4_rq_host_wq_pidx()  argument
384 	return wq->rq.queue[wq->rq.size].status.host_wq_pidx;  in t4_rq_host_wq_pidx()
387 static inline u16 t4_rq_wq_size(struct t4_wq *wq)  in t4_rq_wq_size()  argument
389 		return wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_wq_size()
397 static inline int t4_sq_empty(struct t4_wq *wq)  in t4_sq_empty()  argument
399 	return wq->sq.in_use == 0;  in t4_sq_empty()
402 static inline int t4_sq_full(struct t4_wq *wq)  in t4_sq_full()  argument
404 	return wq->sq.in_use == (wq->sq.size - 1);  in t4_sq_full()
407 static inline u32 t4_sq_avail(struct t4_wq *wq)  in t4_sq_avail()  argument
409 	return wq->sq.size - 1 - wq->sq.in_use;  in t4_sq_avail()
412 static inline void t4_sq_produce(struct t4_wq *wq, u8 len16)  in t4_sq_produce()  argument
414 	wq->sq.in_use++;  in t4_sq_produce()
415 	if (++wq->sq.pidx == wq->sq.size)  in t4_sq_produce()
416 		wq->sq.pidx = 0;  in t4_sq_produce()
417 	wq->sq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_sq_produce()
418 	if (wq->sq.wq_pidx >= wq->sq.size * T4_SQ_NUM_SLOTS)  in t4_sq_produce()
419 		wq->sq.wq_pidx %= wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_produce()
422 static inline void t4_sq_consume(struct t4_wq *wq)  in t4_sq_consume()  argument
424 	BUG_ON(wq->sq.in_use < 1);  in t4_sq_consume()
425 	if (wq->sq.cidx == wq->sq.flush_cidx)  in t4_sq_consume()
426 		wq->sq.flush_cidx = -1;  in t4_sq_consume()
427 	wq->sq.in_use--;  in t4_sq_consume()
428 	if (++wq->sq.cidx == wq->sq.size)  in t4_sq_consume()
429 		wq->sq.cidx = 0;  in t4_sq_consume()
432 static inline u16 t4_sq_host_wq_pidx(struct t4_wq *wq)  in t4_sq_host_wq_pidx()  argument
434 	return wq->sq.queue[wq->sq.size].status.host_wq_pidx;  in t4_sq_host_wq_pidx()
437 static inline u16 t4_sq_wq_size(struct t4_wq *wq)  in t4_sq_wq_size()  argument
439 		return wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_wq_size()
458 static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, union t4_wr *wqe)  in t4_ring_sq_db()  argument
463 	if (wq->sq.bar2_va) {  in t4_ring_sq_db()
464 		if (inc == 1 && wq->sq.bar2_qid == 0 && wqe) {  in t4_ring_sq_db()
466 			     __func__, wq->sq.pidx);  in t4_ring_sq_db()
468 				 (wq->sq.bar2_va + SGE_UDB_WCDOORBELL),  in t4_ring_sq_db()
472 			     __func__, wq->sq.pidx);  in t4_ring_sq_db()
473 			writel(PIDX_T5_V(inc) | QID_V(wq->sq.bar2_qid),  in t4_ring_sq_db()
474 			       wq->sq.bar2_va + SGE_UDB_KDOORBELL);  in t4_ring_sq_db()
481 	writel(QID_V(wq->sq.qid) | PIDX_V(inc), wq->db);  in t4_ring_sq_db()
484 static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc,  in t4_ring_rq_db()  argument
490 	if (wq->rq.bar2_va) {  in t4_ring_rq_db()
491 		if (inc == 1 && wq->rq.bar2_qid == 0 && wqe) {  in t4_ring_rq_db()
493 			     __func__, wq->rq.pidx);  in t4_ring_rq_db()
495 				 (wq->rq.bar2_va + SGE_UDB_WCDOORBELL),  in t4_ring_rq_db()
499 			     __func__, wq->rq.pidx);  in t4_ring_rq_db()
500 			writel(PIDX_T5_V(inc) | QID_V(wq->rq.bar2_qid),  in t4_ring_rq_db()
501 			       wq->rq.bar2_va + SGE_UDB_KDOORBELL);  in t4_ring_rq_db()
508 	writel(QID_V(wq->rq.qid) | PIDX_V(inc), wq->db);  in t4_ring_rq_db()
511 static inline int t4_wq_in_error(struct t4_wq *wq)  in t4_wq_in_error()  argument
513 	return wq->rq.queue[wq->rq.size].status.qp_err;  in t4_wq_in_error()
516 static inline void t4_set_wq_in_error(struct t4_wq *wq)  in t4_set_wq_in_error()  argument
518 	wq->rq.queue[wq->rq.size].status.qp_err = 1;  in t4_set_wq_in_error()
521 static inline void t4_disable_wq_db(struct t4_wq *wq)  in t4_disable_wq_db()  argument
523 	wq->rq.queue[wq->rq.size].status.db_off = 1;  in t4_disable_wq_db()
526 static inline void t4_enable_wq_db(struct t4_wq *wq)  in t4_enable_wq_db()  argument
528 	wq->rq.queue[wq->rq.size].status.db_off = 0;  in t4_enable_wq_db()
531 static inline int t4_wq_db_enabled(struct t4_wq *wq)  in t4_wq_db_enabled()  argument
533 	return !wq->rq.queue[wq->rq.size].status.db_off;  in t4_wq_db_enabled()