201 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_txq_notify()  argument
206 	val |= txo->q.id & DB_TXULP_RING_ID_MASK;  in be_txq_notify()
210 	iowrite32(val, adapter->db + txo->db_offset);  in be_txq_notify()
590 	struct be_tx_obj *txo;  in be_get_stats64()  local
610 	for_all_tx_queues(adapter, txo, i) {  in be_get_stats64()
611 		const struct be_tx_stats *tx_stats = tx_stats(txo);  in be_get_stats64()
615 			pkts = tx_stats(txo)->tx_pkts;  in be_get_stats64()
616 			bytes = tx_stats(txo)->tx_bytes;  in be_get_stats64()
667 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)  in be_tx_stats_update()  argument
669 	struct be_tx_stats *stats = tx_stats(txo);  in be_tx_stats_update()
733 static inline bool be_is_txq_full(struct be_tx_obj *txo)  in be_is_txq_full()  argument
735 	return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len;  in be_is_txq_full()
738 static inline bool be_can_txq_wake(struct be_tx_obj *txo)  in be_can_txq_wake()  argument
740 	return atomic_read(&txo->q.used) < txo->q.len / 2;  in be_can_txq_wake()
743 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo)  in be_is_tx_compl_pending()  argument
745 	return atomic_read(&txo->q.used) > txo->pend_wrb_cnt;  in be_is_tx_compl_pending()
833 static u16 be_tx_get_wrb_hdr(struct be_tx_obj *txo)  in be_tx_get_wrb_hdr()  argument
835 	u16 head = txo->q.head;  in be_tx_get_wrb_hdr()
837 	queue_head_inc(&txo->q);  in be_tx_get_wrb_hdr()
843 				struct be_tx_obj *txo,  in be_tx_setup_wrb_hdr()  argument
848 	struct be_queue_info *txq = &txo->q;  in be_tx_setup_wrb_hdr()
854 	BUG_ON(txo->sent_skb_list[head]);  in be_tx_setup_wrb_hdr()
855 	txo->sent_skb_list[head] = skb;  in be_tx_setup_wrb_hdr()
856 	txo->last_req_hdr = head;  in be_tx_setup_wrb_hdr()
858 	txo->last_req_wrb_cnt = num_frags;  in be_tx_setup_wrb_hdr()
859 	txo->pend_wrb_cnt += num_frags;  in be_tx_setup_wrb_hdr()
863 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr,  in be_tx_setup_wrb_frag()  argument
867 	struct be_queue_info *txq = &txo->q;  in be_tx_setup_wrb_frag()
879 			    struct be_tx_obj *txo, u16 head, bool map_single,  in be_xmit_restore()  argument
884 	struct be_queue_info *txq = &txo->q;  in be_xmit_restore()
906 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_xmit_enqueue()  argument
912 	struct be_queue_info *txq = &txo->q;  in be_xmit_enqueue()
918 	head = be_tx_get_wrb_hdr(txo);  in be_xmit_enqueue()
927 		be_tx_setup_wrb_frag(txo, busaddr, len);  in be_xmit_enqueue()
938 		be_tx_setup_wrb_frag(txo, busaddr, len);  in be_xmit_enqueue()
942 	be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head);  in be_xmit_enqueue()
944 	be_tx_stats_update(txo, skb);  in be_xmit_enqueue()
949 	be_xmit_restore(adapter, txo, head, map_single, copied);  in be_xmit_enqueue()
1125 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)  in be_xmit_flush()  argument
1127 	struct be_queue_info *txq = &txo->q;  in be_xmit_flush()
1128 	struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);  in be_xmit_flush()
1135 	if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {  in be_xmit_flush()
1139 		txo->pend_wrb_cnt++;  in be_xmit_flush()
1142 		hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<  in be_xmit_flush()
1145 	be_txq_notify(adapter, txo, txo->pend_wrb_cnt);  in be_xmit_flush()
1146 	txo->pend_wrb_cnt = 0;  in be_xmit_flush()
1153 	struct be_tx_obj *txo = &adapter->tx_obj[q_idx];  in be_xmit()  local
1164 	wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);  in be_xmit()
1170 	if (be_is_txq_full(txo)) {  in be_xmit()
1172 		tx_stats(txo)->tx_stops++;  in be_xmit()
1176 		be_xmit_flush(adapter, txo);  in be_xmit()
1180 	tx_stats(txo)->tx_drv_drops++;  in be_xmit()
1182 	if (flush && txo->pend_wrb_cnt)  in be_xmit()
1183 		be_xmit_flush(adapter, txo);  in be_xmit()
1660 	struct be_tx_obj *txo;  in be_eqd_update()  local
1680 		txo = &adapter->tx_obj[eqo->idx];  in be_eqd_update()
1682 			start = u64_stats_fetch_begin_irq(&txo->stats.sync);  in be_eqd_update()
1683 			tx_pkts = txo->stats.tx_reqs;  in be_eqd_update()
1684 		} while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));  in be_eqd_update()
2135 static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo)  in be_tx_compl_get()  argument
2137 	struct be_queue_info *tx_cq = &txo->cq;  in be_tx_compl_get()
2138 	struct be_tx_compl_info *txcp = &txo->txcp;  in be_tx_compl_get()
2157 			       struct be_tx_obj *txo, u16 last_index)  in be_tx_compl_process()  argument
2159 	struct sk_buff **sent_skbs = txo->sent_skb_list;  in be_tx_compl_process()
2160 	struct be_queue_info *txq = &txo->q;  in be_tx_compl_process()
2274 	struct be_tx_obj *txo;  in be_tx_compl_clean()  local
2281 		for_all_tx_queues(adapter, txo, i) {  in be_tx_compl_clean()
2284 			txq = &txo->q;  in be_tx_compl_clean()
2285 			while ((txcp = be_tx_compl_get(txo))) {  in be_tx_compl_clean()
2287 					be_tx_compl_process(adapter, txo,  in be_tx_compl_clean()
2292 				be_cq_notify(adapter, txo->cq.id, false, cmpl);  in be_tx_compl_clean()
2296 			if (!be_is_tx_compl_pending(txo))  in be_tx_compl_clean()
2307 	for_all_tx_queues(adapter, txo, i) {  in be_tx_compl_clean()
2308 		txq = &txo->q;  in be_tx_compl_clean()
2320 			num_wrbs = be_tx_compl_process(adapter, txo, end_idx);  in be_tx_compl_clean()
2323 			txo->pend_wrb_cnt = 0;  in be_tx_compl_clean()
2439 	struct be_tx_obj *txo;  in be_tx_queues_destroy()  local
2442 	for_all_tx_queues(adapter, txo, i) {  in be_tx_queues_destroy()
2443 		q = &txo->q;  in be_tx_queues_destroy()
2448 		q = &txo->cq;  in be_tx_queues_destroy()
2458 	struct be_tx_obj *txo;  in be_tx_qs_create()  local
2464 	for_all_tx_queues(adapter, txo, i) {  in be_tx_qs_create()
2465 		cq = &txo->cq;  in be_tx_qs_create()
2471 		u64_stats_init(&txo->stats.sync);  in be_tx_qs_create()
2472 		u64_stats_init(&txo->stats.sync_compl);  in be_tx_qs_create()
2482 		status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,  in be_tx_qs_create()
2487 		status = be_cmd_txq_create(adapter, txo);  in be_tx_qs_create()
2662 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)  in be_update_tx_err()  argument
2666 		tx_stats(txo)->tx_hdr_parse_err++;  in be_update_tx_err()
2669 		tx_stats(txo)->tx_dma_err++;  in be_update_tx_err()
2672 		tx_stats(txo)->tx_spoof_check_err++;  in be_update_tx_err()
2677 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)  in lancer_update_tx_err()  argument
2681 		tx_stats(txo)->tx_tso_err++;  in lancer_update_tx_err()
2685 		tx_stats(txo)->tx_spoof_check_err++;  in lancer_update_tx_err()
2688 		tx_stats(txo)->tx_qinq_err++;  in lancer_update_tx_err()
2691 		tx_stats(txo)->tx_internal_parity_err++;  in lancer_update_tx_err()
2694 		tx_stats(txo)->tx_dma_err++;  in lancer_update_tx_err()
2699 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,  in be_process_tx()  argument
2705 	while ((txcp = be_tx_compl_get(txo))) {  in be_process_tx()
2706 		num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index);  in be_process_tx()
2711 				lancer_update_tx_err(txo, txcp->status);  in be_process_tx()
2713 				be_update_tx_err(txo, txcp->status);  in be_process_tx()
2718 		be_cq_notify(adapter, txo->cq.id, true, work_done);  in be_process_tx()
2719 		atomic_sub(num_wrbs, &txo->q.used);  in be_process_tx()
2724 		    be_can_txq_wake(txo)) {  in be_process_tx()
2728 		u64_stats_update_begin(&tx_stats(txo)->sync_compl);  in be_process_tx()
2729 		tx_stats(txo)->tx_compl += work_done;  in be_process_tx()
2730 		u64_stats_update_end(&tx_stats(txo)->sync_compl);  in be_process_tx()
2840 	struct be_tx_obj *txo;  in be_poll()  local
2844 	for_all_tx_queues_on_eq(adapter, eqo, txo, i)  in be_poll()
2845 		be_process_tx(adapter, txo, i);  in be_poll()
3243 	struct be_tx_obj *txo;  in be_open()  local
3258 	for_all_tx_queues(adapter, txo, i)  in be_open()
3259 		be_cq_notify(adapter, txo->cq.id, true, 0);  in be_open()