165 	struct mlxsw_pci_queue *q;  member
196 static void mlxsw_pci_queue_tasklet_schedule(struct mlxsw_pci_queue *q)  in mlxsw_pci_queue_tasklet_schedule()  argument
198 	tasklet_schedule(&q->tasklet);  in mlxsw_pci_queue_tasklet_schedule()
201 static char *__mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q,  in __mlxsw_pci_queue_elem_get()  argument
204 	return q->mem_item.buf + (elem_size * elem_index);  in __mlxsw_pci_queue_elem_get()
208 mlxsw_pci_queue_elem_info_get(struct mlxsw_pci_queue *q, int elem_index)  in mlxsw_pci_queue_elem_info_get()  argument
210 	return &q->elem_info[elem_index];  in mlxsw_pci_queue_elem_info_get()
214 mlxsw_pci_queue_elem_info_producer_get(struct mlxsw_pci_queue *q)  in mlxsw_pci_queue_elem_info_producer_get()  argument
216 	int index = q->producer_counter & (q->count - 1);  in mlxsw_pci_queue_elem_info_producer_get()
218 	if ((q->producer_counter - q->consumer_counter) == q->count)  in mlxsw_pci_queue_elem_info_producer_get()
220 	return mlxsw_pci_queue_elem_info_get(q, index);  in mlxsw_pci_queue_elem_info_producer_get()
224 mlxsw_pci_queue_elem_info_consumer_get(struct mlxsw_pci_queue *q)  in mlxsw_pci_queue_elem_info_consumer_get()  argument
226 	int index = q->consumer_counter & (q->count - 1);  in mlxsw_pci_queue_elem_info_consumer_get()
228 	return mlxsw_pci_queue_elem_info_get(q, index);  in mlxsw_pci_queue_elem_info_consumer_get()
231 static char *mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q, int elem_index)  in mlxsw_pci_queue_elem_get()  argument
233 	return mlxsw_pci_queue_elem_info_get(q, elem_index)->elem;  in mlxsw_pci_queue_elem_get()
236 static bool mlxsw_pci_elem_hw_owned(struct mlxsw_pci_queue *q, bool owner_bit)  in mlxsw_pci_elem_hw_owned()  argument
238 	return owner_bit != !!(q->consumer_counter & q->count);  in mlxsw_pci_elem_hw_owned()
241 static char *mlxsw_pci_queue_sw_elem_get(struct mlxsw_pci_queue *q,  in mlxsw_pci_queue_sw_elem_get()  argument
248 	elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);  in mlxsw_pci_queue_sw_elem_get()
251 	if (mlxsw_pci_elem_hw_owned(q, owner_bit))  in mlxsw_pci_queue_sw_elem_get()
253 	q->consumer_counter++;  in mlxsw_pci_queue_sw_elem_get()
298 	return &mlxsw_pci->queues[q_type].q[q_num];  in __mlxsw_pci_queue_get()
328 					   struct mlxsw_pci_queue *q,  in __mlxsw_pci_queue_doorbell_set()  argument
333 				   mlxsw_pci_doorbell_type_offset[q->type],  in __mlxsw_pci_queue_doorbell_set()
334 				   q->num), val);  in __mlxsw_pci_queue_doorbell_set()
338 					       struct mlxsw_pci_queue *q,  in __mlxsw_pci_queue_doorbell_arm_set()  argument
343 				   mlxsw_pci_doorbell_arm_type_offset[q->type],  in __mlxsw_pci_queue_doorbell_arm_set()
344 				   q->num), val);  in __mlxsw_pci_queue_doorbell_arm_set()
348 						   struct mlxsw_pci_queue *q)  in mlxsw_pci_queue_doorbell_producer_ring()  argument
351 	__mlxsw_pci_queue_doorbell_set(mlxsw_pci, q, q->producer_counter);  in mlxsw_pci_queue_doorbell_producer_ring()
355 						   struct mlxsw_pci_queue *q)  in mlxsw_pci_queue_doorbell_consumer_ring()  argument
358 	__mlxsw_pci_queue_doorbell_set(mlxsw_pci, q,  in mlxsw_pci_queue_doorbell_consumer_ring()
359 				       q->consumer_counter + q->count);  in mlxsw_pci_queue_doorbell_consumer_ring()
364 					   struct mlxsw_pci_queue *q)  in mlxsw_pci_queue_doorbell_arm_consumer_ring()  argument
367 	__mlxsw_pci_queue_doorbell_arm_set(mlxsw_pci, q, q->consumer_counter);  in mlxsw_pci_queue_doorbell_arm_consumer_ring()
370 static dma_addr_t __mlxsw_pci_queue_page_get(struct mlxsw_pci_queue *q,  in __mlxsw_pci_queue_page_get()  argument
373 	return q->mem_item.mapaddr + MLXSW_PCI_PAGE_SIZE * page_index;  in __mlxsw_pci_queue_page_get()
377 			      struct mlxsw_pci_queue *q)  in mlxsw_pci_sdq_init()  argument
382 	q->producer_counter = 0;  in mlxsw_pci_sdq_init()
383 	q->consumer_counter = 0;  in mlxsw_pci_sdq_init()
386 	mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, q->num);  in mlxsw_pci_sdq_init()
390 		dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);  in mlxsw_pci_sdq_init()
395 	err = mlxsw_cmd_sw2hw_sdq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_sdq_init()
398 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_sdq_init()
403 			       struct mlxsw_pci_queue *q)  in mlxsw_pci_sdq_fini()  argument
405 	mlxsw_cmd_hw2sw_sdq(mlxsw_pci->core, q->num);  in mlxsw_pci_sdq_fini()
411 	struct mlxsw_pci_queue *q;  in mlxsw_pci_sdq_dbg_read()  local
418 		q = mlxsw_pci_sdq_get(mlxsw_pci, i);  in mlxsw_pci_sdq_dbg_read()
419 		spin_lock_bh(&q->lock);  in mlxsw_pci_sdq_dbg_read()
421 			   i, q->producer_counter, q->consumer_counter,  in mlxsw_pci_sdq_dbg_read()
422 			   q->count);  in mlxsw_pci_sdq_dbg_read()
423 		spin_unlock_bh(&q->lock);  in mlxsw_pci_sdq_dbg_read()
499 			      struct mlxsw_pci_queue *q)  in mlxsw_pci_rdq_init()  argument
506 	q->producer_counter = 0;  in mlxsw_pci_rdq_init()
507 	q->consumer_counter = 0;  in mlxsw_pci_rdq_init()
512 	mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, sdq_count + q->num);  in mlxsw_pci_rdq_init()
515 		dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);  in mlxsw_pci_rdq_init()
520 	err = mlxsw_cmd_sw2hw_rdq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_rdq_init()
524 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_rdq_init()
526 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_rdq_init()
527 		elem_info = mlxsw_pci_queue_elem_info_producer_get(q);  in mlxsw_pci_rdq_init()
533 		q->producer_counter++;  in mlxsw_pci_rdq_init()
534 		mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_rdq_init()
541 		elem_info = mlxsw_pci_queue_elem_info_get(q, i);  in mlxsw_pci_rdq_init()
544 	mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);  in mlxsw_pci_rdq_init()
550 			       struct mlxsw_pci_queue *q)  in mlxsw_pci_rdq_fini()  argument
555 	mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);  in mlxsw_pci_rdq_fini()
556 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_rdq_fini()
557 		elem_info = mlxsw_pci_queue_elem_info_get(q, i);  in mlxsw_pci_rdq_fini()
565 	struct mlxsw_pci_queue *q;  in mlxsw_pci_rdq_dbg_read()  local
572 		q = mlxsw_pci_rdq_get(mlxsw_pci, i);  in mlxsw_pci_rdq_dbg_read()
573 		spin_lock_bh(&q->lock);  in mlxsw_pci_rdq_dbg_read()
575 			   i, q->producer_counter, q->consumer_counter,  in mlxsw_pci_rdq_dbg_read()
576 			   q->count);  in mlxsw_pci_rdq_dbg_read()
577 		spin_unlock_bh(&q->lock);  in mlxsw_pci_rdq_dbg_read()
583 			     struct mlxsw_pci_queue *q)  in mlxsw_pci_cq_init()  argument
588 	q->consumer_counter = 0;  in mlxsw_pci_cq_init()
590 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_cq_init()
591 		char *elem = mlxsw_pci_queue_elem_get(q, i);  in mlxsw_pci_cq_init()
600 	mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count));  in mlxsw_pci_cq_init()
602 		dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);  in mlxsw_pci_cq_init()
606 	err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_cq_init()
609 	mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_cq_init()
610 	mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_cq_init()
615 			      struct mlxsw_pci_queue *q)  in mlxsw_pci_cq_fini()  argument
617 	mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num);  in mlxsw_pci_cq_fini()
624 	struct mlxsw_pci_queue *q;  in mlxsw_pci_cq_dbg_read()  local
631 		q = mlxsw_pci_cq_get(mlxsw_pci, i);  in mlxsw_pci_cq_dbg_read()
632 		spin_lock_bh(&q->lock);  in mlxsw_pci_cq_dbg_read()
634 			   i, q->consumer_counter, q->u.cq.comp_sdq_count,  in mlxsw_pci_cq_dbg_read()
635 			   q->u.cq.comp_rdq_count, q->count);  in mlxsw_pci_cq_dbg_read()
636 		spin_unlock_bh(&q->lock);  in mlxsw_pci_cq_dbg_read()
642 				     struct mlxsw_pci_queue *q,  in mlxsw_pci_cqe_sdq_handle()  argument
652 	spin_lock(&q->lock);  in mlxsw_pci_cqe_sdq_handle()
653 	elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);  in mlxsw_pci_cqe_sdq_handle()
661 	if (q->consumer_counter++ != consumer_counter_limit)  in mlxsw_pci_cqe_sdq_handle()
663 	spin_unlock(&q->lock);  in mlxsw_pci_cqe_sdq_handle()
667 				     struct mlxsw_pci_queue *q,  in mlxsw_pci_cqe_rdq_handle()  argument
679 	elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);  in mlxsw_pci_cqe_rdq_handle()
686 	if (q->consumer_counter++ != consumer_counter_limit)  in mlxsw_pci_cqe_rdq_handle()
703 	memset(wqe, 0, q->elem_size);  in mlxsw_pci_cqe_rdq_handle()
708 	q->producer_counter++;  in mlxsw_pci_cqe_rdq_handle()
709 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_cqe_rdq_handle()
717 static char *mlxsw_pci_cq_sw_cqe_get(struct mlxsw_pci_queue *q)  in mlxsw_pci_cq_sw_cqe_get()  argument
719 	return mlxsw_pci_queue_sw_elem_get(q, mlxsw_pci_cqe_owner_get);  in mlxsw_pci_cq_sw_cqe_get()
724 	struct mlxsw_pci_queue *q = (struct mlxsw_pci_queue *) data;  in mlxsw_pci_cq_tasklet()  local
725 	struct mlxsw_pci *mlxsw_pci = q->pci;  in mlxsw_pci_cq_tasklet()
728 	int credits = q->count >> 1;  in mlxsw_pci_cq_tasklet()
730 	while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) {  in mlxsw_pci_cq_tasklet()
741 			q->u.cq.comp_sdq_count++;  in mlxsw_pci_cq_tasklet()
748 			q->u.cq.comp_rdq_count++;  in mlxsw_pci_cq_tasklet()
754 		mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_cq_tasklet()
755 		mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_cq_tasklet()
760 			     struct mlxsw_pci_queue *q)  in mlxsw_pci_eq_init()  argument
765 	q->consumer_counter = 0;  in mlxsw_pci_eq_init()
767 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_eq_init()
768 		char *elem = mlxsw_pci_queue_elem_get(q, i);  in mlxsw_pci_eq_init()
776 	mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count));  in mlxsw_pci_eq_init()
778 		dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);  in mlxsw_pci_eq_init()
782 	err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_eq_init()
785 	mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_eq_init()
786 	mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_eq_init()
791 			      struct mlxsw_pci_queue *q)  in mlxsw_pci_eq_fini()  argument
793 	mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num);  in mlxsw_pci_eq_fini()
799 	struct mlxsw_pci_queue *q;  in mlxsw_pci_eq_dbg_read()  local
806 		q = mlxsw_pci_eq_get(mlxsw_pci, i);  in mlxsw_pci_eq_dbg_read()
807 		spin_lock_bh(&q->lock);  in mlxsw_pci_eq_dbg_read()
809 			   i, q->consumer_counter, q->u.eq.ev_cmd_count,  in mlxsw_pci_eq_dbg_read()
810 			   q->u.eq.ev_comp_count, q->u.eq.ev_other_count,  in mlxsw_pci_eq_dbg_read()
811 			   q->count);  in mlxsw_pci_eq_dbg_read()
812 		spin_unlock_bh(&q->lock);  in mlxsw_pci_eq_dbg_read()
827 static char *mlxsw_pci_eq_sw_eqe_get(struct mlxsw_pci_queue *q)  in mlxsw_pci_eq_sw_eqe_get()  argument
829 	return mlxsw_pci_queue_sw_elem_get(q, mlxsw_pci_eqe_owner_get);  in mlxsw_pci_eq_sw_eqe_get()
834 	struct mlxsw_pci_queue *q = (struct mlxsw_pci_queue *) data;  in mlxsw_pci_eq_tasklet()  local
835 	struct mlxsw_pci *mlxsw_pci = q->pci;  in mlxsw_pci_eq_tasklet()
842 	int credits = q->count >> 1;  in mlxsw_pci_eq_tasklet()
846 	while ((eqe = mlxsw_pci_eq_sw_eqe_get(q))) {  in mlxsw_pci_eq_tasklet()
852 			q->u.eq.ev_cmd_count++;  in mlxsw_pci_eq_tasklet()
858 			q->u.eq.ev_comp_count++;  in mlxsw_pci_eq_tasklet()
861 			q->u.eq.ev_other_count++;  in mlxsw_pci_eq_tasklet()
867 		mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_eq_tasklet()
868 		mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_eq_tasklet()
874 		q = mlxsw_pci_cq_get(mlxsw_pci, cqn);  in mlxsw_pci_eq_tasklet()
875 		mlxsw_pci_queue_tasklet_schedule(q);  in mlxsw_pci_eq_tasklet()
883 		    struct mlxsw_pci_queue *q);
885 		     struct mlxsw_pci_queue *q);
932 				struct mlxsw_pci_queue *q, u8 q_num)  in mlxsw_pci_queue_init()  argument
934 	struct mlxsw_pci_mem_item *mem_item = &q->mem_item;  in mlxsw_pci_queue_init()
938 	spin_lock_init(&q->lock);  in mlxsw_pci_queue_init()
939 	q->num = q_num;  in mlxsw_pci_queue_init()
940 	q->count = q_ops->elem_count;  in mlxsw_pci_queue_init()
941 	q->elem_size = q_ops->elem_size;  in mlxsw_pci_queue_init()
942 	q->type = q_ops->type;  in mlxsw_pci_queue_init()
943 	q->pci = mlxsw_pci;  in mlxsw_pci_queue_init()
946 		tasklet_init(&q->tasklet, q_ops->tasklet, (unsigned long) q);  in mlxsw_pci_queue_init()
956 	q->elem_info = kcalloc(q->count, sizeof(*q->elem_info), GFP_KERNEL);  in mlxsw_pci_queue_init()
957 	if (!q->elem_info) {  in mlxsw_pci_queue_init()
965 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_queue_init()
968 		elem_info = mlxsw_pci_queue_elem_info_get(q, i);  in mlxsw_pci_queue_init()
970 			__mlxsw_pci_queue_elem_get(q, q_ops->elem_size, i);  in mlxsw_pci_queue_init()
974 	err = q_ops->init(mlxsw_pci, mbox, q);  in mlxsw_pci_queue_init()
980 	kfree(q->elem_info);  in mlxsw_pci_queue_init()
989 				 struct mlxsw_pci_queue *q)  in mlxsw_pci_queue_fini()  argument
991 	struct mlxsw_pci_mem_item *mem_item = &q->mem_item;  in mlxsw_pci_queue_fini()
993 	q_ops->fini(mlxsw_pci, q);  in mlxsw_pci_queue_fini()
994 	kfree(q->elem_info);  in mlxsw_pci_queue_fini()
1010 	queue_group->q = kcalloc(num_qs, sizeof(*queue_group->q), GFP_KERNEL);  in mlxsw_pci_queue_group_init()
1011 	if (!queue_group->q)  in mlxsw_pci_queue_group_init()
1016 					   &queue_group->q[i], i);  in mlxsw_pci_queue_group_init()
1030 		mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);  in mlxsw_pci_queue_group_init()
1031 	kfree(queue_group->q);  in mlxsw_pci_queue_group_init()
1043 		mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);  in mlxsw_pci_queue_group_fini()
1044 	kfree(queue_group->q);  in mlxsw_pci_queue_group_fini()
1357 	struct mlxsw_pci_queue *q;  in mlxsw_pci_eq_irq_handler()  local
1361 		q = mlxsw_pci_eq_get(mlxsw_pci, i);  in mlxsw_pci_eq_irq_handler()
1362 		mlxsw_pci_queue_tasklet_schedule(q);  in mlxsw_pci_eq_irq_handler()
1513 	struct mlxsw_pci_queue *q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);  in mlxsw_pci_skb_transmit_busy()  local
1515 	return !mlxsw_pci_queue_elem_info_producer_get(q);  in mlxsw_pci_skb_transmit_busy()
1522 	struct mlxsw_pci_queue *q;  in mlxsw_pci_skb_transmit()  local
1534 	q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);  in mlxsw_pci_skb_transmit()
1535 	spin_lock_bh(&q->lock);  in mlxsw_pci_skb_transmit()
1536 	elem_info = mlxsw_pci_queue_elem_info_producer_get(q);  in mlxsw_pci_skb_transmit()
1570 	q->producer_counter++;  in mlxsw_pci_skb_transmit()
1571 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_skb_transmit()
1579 	spin_unlock_bh(&q->lock);  in mlxsw_pci_skb_transmit()