89 				struct qede_rx_queue *rxq);
699 static bool qede_has_rx_work(struct qede_rx_queue *rxq)  in qede_has_rx_work()  argument
706 	hw_comp_cons = le16_to_cpu(*rxq->hw_cons_ptr);  in qede_has_rx_work()
707 	sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring);  in qede_has_rx_work()
725 static void qede_reuse_rx_data(struct qede_rx_queue *rxq)  in qede_reuse_rx_data()  argument
727 	struct eth_rx_bd *rx_bd_cons = qed_chain_consume(&rxq->rx_bd_ring);  in qede_reuse_rx_data()
728 	struct eth_rx_bd *rx_bd_prod = qed_chain_produce(&rxq->rx_bd_ring);  in qede_reuse_rx_data()
730 		&rxq->sw_rx_ring[rxq->sw_rx_cons & NUM_RX_BDS_MAX];  in qede_reuse_rx_data()
732 		&rxq->sw_rx_ring[rxq->sw_rx_prod & NUM_RX_BDS_MAX];  in qede_reuse_rx_data()
740 	rxq->sw_rx_cons++;  in qede_reuse_rx_data()
741 	rxq->sw_rx_prod++;  in qede_reuse_rx_data()
745 				       struct qede_rx_queue *rxq)  in qede_update_rx_prod()  argument
747 	u16 bd_prod = qed_chain_get_prod_idx(&rxq->rx_bd_ring);  in qede_update_rx_prod()
748 	u16 cqe_prod = qed_chain_get_prod_idx(&rxq->rx_comp_ring);  in qede_update_rx_prod()
761 	internal_ram_wr(rxq->hw_rxq_prod_addr, sizeof(rx_prods),  in qede_update_rx_prod()
836 	struct qede_rx_queue *rxq = fp->rxq;  in qede_rx_int()  local
842 	hw_comp_cons = le16_to_cpu(*rxq->hw_cons_ptr);  in qede_rx_int()
843 	sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring);  in qede_rx_int()
866 			qed_chain_consume(&rxq->rx_comp_ring);  in qede_rx_int()
877 		sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS_MAX;  in qede_rx_int()
878 		sw_rx_data = &rxq->sw_rx_ring[sw_rx_index];  in qede_rx_int()
891 		if (likely(qede_alloc_rx_buffer(edev, rxq) == 0)) {  in qede_rx_int()
894 					 rxq->rx_buf_size, DMA_FROM_DEVICE);  in qede_rx_int()
904 				rxq->rx_hw_errors++;  in qede_rx_int()
915 				rxq->rx_alloc_errors++;  in qede_rx_int()
924 			qede_reuse_rx_data(rxq);  in qede_rx_int()
925 			rxq->rx_alloc_errors++;  in qede_rx_int()
947 		qed_chain_consume(&rxq->rx_bd_ring);  in qede_rx_int()
950 		rxq->sw_rx_cons++;  in qede_rx_int()
954 		qed_chain_recycle_consumed(&rxq->rx_comp_ring);  in qede_rx_int()
955 		sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring);  in qede_rx_int()
964 	qede_update_rx_prod(edev, rxq);  in qede_rx_int()
983 		if (qede_has_rx_work(fp->rxq)) {  in qede_poll()
992 		if (!(qede_has_rx_work(fp->rxq) || qede_has_tx_work(fp))) {  in qede_poll()
1009 			if (!(qede_has_rx_work(fp->rxq) ||  in qede_poll()
1298 			kfree(fp->rxq);  in qede_free_fp_array()
1327 		fp->rxq = kcalloc(1, sizeof(*fp->rxq), GFP_KERNEL);  in qede_alloc_fp_array()
1328 		if (!fp->rxq) {  in qede_alloc_fp_array()
1560 				 struct qede_rx_queue *rxq)  in qede_free_rx_buffers()  argument
1564 	for (i = rxq->sw_rx_cons; i != rxq->sw_rx_prod; i++) {  in qede_free_rx_buffers()
1568 		rx_buf = &rxq->sw_rx_ring[i & NUM_RX_BDS_MAX];  in qede_free_rx_buffers()
1573 				 rxq->rx_buf_size, DMA_FROM_DEVICE);  in qede_free_rx_buffers()
1581 			      struct qede_rx_queue *rxq)  in qede_free_mem_rxq()  argument
1584 	qede_free_rx_buffers(edev, rxq);  in qede_free_mem_rxq()
1587 	kfree(rxq->sw_rx_ring);  in qede_free_mem_rxq()
1590 	edev->ops->common->chain_free(edev->cdev, &rxq->rx_bd_ring);  in qede_free_mem_rxq()
1591 	edev->ops->common->chain_free(edev->cdev, &rxq->rx_comp_ring);  in qede_free_mem_rxq()
1595 				struct qede_rx_queue *rxq)  in qede_alloc_rx_buffer()  argument
1603 	rx_buf_size = rxq->rx_buf_size;  in qede_alloc_rx_buffer()
1619 	sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_prod & NUM_RX_BDS_MAX];  in qede_alloc_rx_buffer()
1625 	rx_bd = (struct eth_rx_bd *)qed_chain_produce(&rxq->rx_bd_ring);  in qede_alloc_rx_buffer()
1630 	rxq->sw_rx_prod++;  in qede_alloc_rx_buffer()
1637 			      struct qede_rx_queue *rxq)  in qede_alloc_mem_rxq()  argument
1641 	rxq->num_rx_buffers = edev->q_num_rx_buffers;  in qede_alloc_mem_rxq()
1643 	rxq->rx_buf_size = NET_IP_ALIGN +  in qede_alloc_mem_rxq()
1649 	size = sizeof(*rxq->sw_rx_ring) * NUM_RX_BDS_MAX;  in qede_alloc_mem_rxq()
1650 	rxq->sw_rx_ring = kzalloc(size, GFP_KERNEL);  in qede_alloc_mem_rxq()
1651 	if (!rxq->sw_rx_ring) {  in qede_alloc_mem_rxq()
1662 					    &rxq->rx_bd_ring);  in qede_alloc_mem_rxq()
1673 					    &rxq->rx_comp_ring);  in qede_alloc_mem_rxq()
1678 	for (i = 0; i < rxq->num_rx_buffers; i++) {  in qede_alloc_mem_rxq()
1679 		rc = qede_alloc_rx_buffer(edev, rxq);  in qede_alloc_mem_rxq()
1687 	} else if (num_allocated < rxq->num_rx_buffers) {  in qede_alloc_mem_rxq()
1696 	qede_free_mem_rxq(edev, rxq);  in qede_alloc_mem_rxq()
1751 	qede_free_mem_rxq(edev, fp->rxq);  in qede_free_mem_fp()
1769 	rc = qede_alloc_mem_rxq(edev, fp->rxq);  in qede_alloc_mem_fp()
1843 		memset((void *)fp->rxq, 0, sizeof(*fp->rxq));  in qede_init_fp()
1844 		fp->rxq->rxq_id = rss_id;  in qede_init_fp()
2116 		dma_addr_t phys_table = fp->rxq->rx_comp_ring.pbl.p_phys_table;  in qede_start_queues()
2126 					   fp->rxq->rx_buf_size,  in qede_start_queues()
2127 					   fp->rxq->rx_bd_ring.p_phys_addr,  in qede_start_queues()
2129 					   fp->rxq->rx_comp_ring.page_cnt,  in qede_start_queues()
2130 					   &fp->rxq->hw_rxq_prod_addr);  in qede_start_queues()
2136 		fp->rxq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[RX_PI];  in qede_start_queues()
2138 		qede_update_rx_prod(edev, fp->rxq);  in qede_start_queues()