340 static inline int t4_rqes_posted(struct t4_wq *wq)  in t4_rqes_posted()  argument
342 	return wq->rq.in_use;  in t4_rqes_posted()
345 static inline int t4_rq_empty(struct t4_wq *wq)  in t4_rq_empty()  argument
347 	return wq->rq.in_use == 0;  in t4_rq_empty()
350 static inline int t4_rq_full(struct t4_wq *wq)  in t4_rq_full()  argument
352 	return wq->rq.in_use == (wq->rq.size - 1);  in t4_rq_full()
355 static inline u32 t4_rq_avail(struct t4_wq *wq)  in t4_rq_avail()  argument
357 	return wq->rq.size - 1 - wq->rq.in_use;  in t4_rq_avail()
360 static inline void t4_rq_produce(struct t4_wq *wq, u8 len16)  in t4_rq_produce()  argument
362 	wq->rq.in_use++;  in t4_rq_produce()
363 	if (++wq->rq.pidx == wq->rq.size)  in t4_rq_produce()
364 		wq->rq.pidx = 0;  in t4_rq_produce()
365 	wq->rq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_rq_produce()
366 	if (wq->rq.wq_pidx >= wq->rq.size * T4_RQ_NUM_SLOTS)  in t4_rq_produce()
367 		wq->rq.wq_pidx %= wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_produce()
370 static inline void t4_rq_consume(struct t4_wq *wq)  in t4_rq_consume()  argument
372 	wq->rq.in_use--;  in t4_rq_consume()
373 	wq->rq.msn++;  in t4_rq_consume()
374 	if (++wq->rq.cidx == wq->rq.size)  in t4_rq_consume()
375 		wq->rq.cidx = 0;  in t4_rq_consume()
378 static inline u16 t4_rq_host_wq_pidx(struct t4_wq *wq)  in t4_rq_host_wq_pidx()  argument
380 	return wq->rq.queue[wq->rq.size].status.host_wq_pidx;  in t4_rq_host_wq_pidx()
383 static inline u16 t4_rq_wq_size(struct t4_wq *wq)  in t4_rq_wq_size()  argument
385 		return wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_wq_size()
393 static inline int t4_sq_empty(struct t4_wq *wq)  in t4_sq_empty()  argument
395 	return wq->sq.in_use == 0;  in t4_sq_empty()
398 static inline int t4_sq_full(struct t4_wq *wq)  in t4_sq_full()  argument
400 	return wq->sq.in_use == (wq->sq.size - 1);  in t4_sq_full()
403 static inline u32 t4_sq_avail(struct t4_wq *wq)  in t4_sq_avail()  argument
405 	return wq->sq.size - 1 - wq->sq.in_use;  in t4_sq_avail()
408 static inline void t4_sq_produce(struct t4_wq *wq, u8 len16)  in t4_sq_produce()  argument
410 	wq->sq.in_use++;  in t4_sq_produce()
411 	if (++wq->sq.pidx == wq->sq.size)  in t4_sq_produce()
412 		wq->sq.pidx = 0;  in t4_sq_produce()
413 	wq->sq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_sq_produce()
414 	if (wq->sq.wq_pidx >= wq->sq.size * T4_SQ_NUM_SLOTS)  in t4_sq_produce()
415 		wq->sq.wq_pidx %= wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_produce()
418 static inline void t4_sq_consume(struct t4_wq *wq)  in t4_sq_consume()  argument
420 	BUG_ON(wq->sq.in_use < 1);  in t4_sq_consume()
421 	if (wq->sq.cidx == wq->sq.flush_cidx)  in t4_sq_consume()
422 		wq->sq.flush_cidx = -1;  in t4_sq_consume()
423 	wq->sq.in_use--;  in t4_sq_consume()
424 	if (++wq->sq.cidx == wq->sq.size)  in t4_sq_consume()
425 		wq->sq.cidx = 0;  in t4_sq_consume()
428 static inline u16 t4_sq_host_wq_pidx(struct t4_wq *wq)  in t4_sq_host_wq_pidx()  argument
430 	return wq->sq.queue[wq->sq.size].status.host_wq_pidx;  in t4_sq_host_wq_pidx()
433 static inline u16 t4_sq_wq_size(struct t4_wq *wq)  in t4_sq_wq_size()  argument
435 		return wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_wq_size()
454 static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, u8 t5,  in t4_ring_sq_db()  argument
463 			     __func__, wq->sq.pidx);  in t4_ring_sq_db()
464 			pio_copy(wq->sq.udb + 7, (void *)wqe);  in t4_ring_sq_db()
467 			     __func__, wq->sq.pidx);  in t4_ring_sq_db()
468 			writel(PIDX_T5_V(inc), wq->sq.udb);  in t4_ring_sq_db()
475 	writel(QID_V(wq->sq.qid) | PIDX_V(inc), wq->db);  in t4_ring_sq_db()
478 static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc, u8 t5,  in t4_ring_rq_db()  argument
487 			     __func__, wq->rq.pidx);  in t4_ring_rq_db()
488 			pio_copy(wq->rq.udb + 7, (void *)wqe);  in t4_ring_rq_db()
491 			     __func__, wq->rq.pidx);  in t4_ring_rq_db()
492 			writel(PIDX_T5_V(inc), wq->rq.udb);  in t4_ring_rq_db()
499 	writel(QID_V(wq->rq.qid) | PIDX_V(inc), wq->db);  in t4_ring_rq_db()
502 static inline int t4_wq_in_error(struct t4_wq *wq)  in t4_wq_in_error()  argument
504 	return wq->rq.queue[wq->rq.size].status.qp_err;  in t4_wq_in_error()
507 static inline void t4_set_wq_in_error(struct t4_wq *wq)  in t4_set_wq_in_error()  argument
509 	wq->rq.queue[wq->rq.size].status.qp_err = 1;  in t4_set_wq_in_error()
512 static inline void t4_disable_wq_db(struct t4_wq *wq)  in t4_disable_wq_db()  argument
514 	wq->rq.queue[wq->rq.size].status.db_off = 1;  in t4_disable_wq_db()
517 static inline void t4_enable_wq_db(struct t4_wq *wq)  in t4_enable_wq_db()  argument
519 	wq->rq.queue[wq->rq.size].status.db_off = 0;  in t4_enable_wq_db()
522 static inline int t4_wq_db_enabled(struct t4_wq *wq)  in t4_wq_db_enabled()  argument
524 	return !wq->rq.queue[wq->rq.size].status.db_off;  in t4_wq_db_enabled()