96 	struct cppi41_dd *cdd;  member
238 static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc)  in desc_to_chan()  argument
246 	if (!((desc >= cdd->descs_phys) &&  in desc_to_chan()
247 			(desc < (cdd->descs_phys + descs_size)))) {  in desc_to_chan()
251 	desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc);  in desc_to_chan()
253 	c = cdd->chan_busy[desc_num];  in desc_to_chan()
254 	cdd->chan_busy[desc_num] = NULL;  in desc_to_chan()
273 static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num)  in cppi41_pop_desc()  argument
277 	desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num));  in cppi41_pop_desc()
284 	struct cppi41_dd *cdd = data;  in cppi41_irq()  local
289 	status = cppi_readl(cdd->usbss_mem + USBSS_IRQ_STATUS);  in cppi41_irq()
292 	cppi_writel(status, cdd->usbss_mem + USBSS_IRQ_STATUS);  in cppi41_irq()
299 		val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i));  in cppi41_irq()
319 			desc = cppi41_pop_desc(cdd, q_num);  in cppi41_irq()
320 			c = desc_to_chan(cdd, desc);  in cppi41_irq()
384 	struct cppi41_dd *cdd = c->cdd;  in push_desc_queue()  local
390 	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);  in push_desc_queue()
391 	WARN_ON(cdd->chan_busy[desc_num]);  in push_desc_queue()
392 	cdd->chan_busy[desc_num] = c;  in push_desc_queue()
396 	cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));  in push_desc_queue()
535 	struct cppi41_dd *cdd = c->cdd;  in cppi41_tear_down_chan()  local
541 	td = cdd->cd;  in cppi41_tear_down_chan()
542 	td += cdd->first_td_desc;  in cppi41_tear_down_chan()
544 	td_desc_phys = cdd->descs_phys;  in cppi41_tear_down_chan()
545 	td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc);  in cppi41_tear_down_chan()
553 		cppi_writel(reg, cdd->qmgr_mem +  in cppi41_tear_down_chan()
554 				QMGR_QUEUE_D(cdd->td_queue.submit));  in cppi41_tear_down_chan()
570 		desc_phys = cppi41_pop_desc(cdd, cdd->td_queue.complete);  in cppi41_tear_down_chan()
572 			desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);  in cppi41_tear_down_chan()
608 		desc_phys = cppi41_pop_desc(cdd, c->q_num);  in cppi41_tear_down_chan()
610 			desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);  in cppi41_tear_down_chan()
624 	struct cppi41_dd *cdd = c->cdd;  in cppi41_stop_chan()  local
630 	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);  in cppi41_stop_chan()
631 	if (!cdd->chan_busy[desc_num])  in cppi41_stop_chan()
638 	WARN_ON(!cdd->chan_busy[desc_num]);  in cppi41_stop_chan()
639 	cdd->chan_busy[desc_num] = NULL;  in cppi41_stop_chan()
644 static void cleanup_chans(struct cppi41_dd *cdd)  in cleanup_chans()  argument
646 	while (!list_empty(&cdd->ddev.channels)) {  in cleanup_chans()
649 		cchan = list_first_entry(&cdd->ddev.channels,  in cleanup_chans()
656 static int cppi41_add_chans(struct device *dev, struct cppi41_dd *cdd)  in cppi41_add_chans()  argument
678 		cchan->cdd = cdd;  in cppi41_add_chans()
680 			cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1);  in cppi41_add_chans()
683 			cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1);  in cppi41_add_chans()
687 		cchan->desc = &cdd->cd[i];  in cppi41_add_chans()
688 		cchan->desc_phys = cdd->descs_phys;  in cppi41_add_chans()
690 		cchan->chan.device = &cdd->ddev;  in cppi41_add_chans()
691 		list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels);  in cppi41_add_chans()
693 	cdd->first_td_desc = n_chans;  in cppi41_add_chans()
697 	cleanup_chans(cdd);  in cppi41_add_chans()
701 static void purge_descs(struct device *dev, struct cppi41_dd *cdd)  in purge_descs()  argument
710 		cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i));  in purge_descs()
711 		cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i));  in purge_descs()
713 		dma_free_coherent(dev, mem_decs, cdd->cd,  in purge_descs()
714 				cdd->descs_phys);  in purge_descs()
718 static void disable_sched(struct cppi41_dd *cdd)  in disable_sched()  argument
720 	cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);  in disable_sched()
723 static void deinit_cppi41(struct device *dev, struct cppi41_dd *cdd)  in deinit_cppi41()  argument
725 	disable_sched(cdd);  in deinit_cppi41()
727 	purge_descs(dev, cdd);  in deinit_cppi41()
729 	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);  in deinit_cppi41()
730 	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);  in deinit_cppi41()
731 	dma_free_coherent(dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch,  in deinit_cppi41()
732 			cdd->scratch_phys);  in deinit_cppi41()
735 static int init_descs(struct device *dev, struct cppi41_dd *cdd)  in init_descs()  argument
759 		cdd->cd = dma_alloc_coherent(dev, mem_decs,  in init_descs()
760 				&cdd->descs_phys, GFP_KERNEL);  in init_descs()
761 		if (!cdd->cd)  in init_descs()
764 		cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));  in init_descs()
765 		cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i));  in init_descs()
772 static void init_sched(struct cppi41_dd *cdd)  in init_sched()  argument
779 	cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);  in init_sched()
787 		cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word));  in init_sched()
792 	cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL);  in init_sched()
795 static int init_cppi41(struct device *dev, struct cppi41_dd *cdd)  in init_cppi41()  argument
800 	cdd->qmgr_scratch = dma_alloc_coherent(dev, QMGR_SCRATCH_SIZE,  in init_cppi41()
801 			&cdd->scratch_phys, GFP_KERNEL);  in init_cppi41()
802 	if (!cdd->qmgr_scratch)  in init_cppi41()
805 	cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);  in init_cppi41()
806 	cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);  in init_cppi41()
807 	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);  in init_cppi41()
809 	ret = init_descs(dev, cdd);  in init_cppi41()
813 	cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ);  in init_cppi41()
814 	init_sched(cdd);  in init_cppi41()
817 	deinit_cppi41(dev, cdd);  in init_cppi41()
834 	struct cppi41_dd *cdd;  in cpp41_dma_filter_fn()  local
848 	cdd = cchan->cdd;  in cpp41_dma_filter_fn()
850 		queues = cdd->queues_tx;  in cpp41_dma_filter_fn()
852 		queues = cdd->queues_rx;  in cpp41_dma_filter_fn()
913 	struct cppi41_dd *cdd;  in cppi41_dma_probe()  local
923 	cdd = devm_kzalloc(&pdev->dev, sizeof(*cdd), GFP_KERNEL);  in cppi41_dma_probe()
924 	if (!cdd)  in cppi41_dma_probe()
927 	dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask);  in cppi41_dma_probe()
928 	cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources;  in cppi41_dma_probe()
929 	cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources;  in cppi41_dma_probe()
930 	cdd->ddev.device_tx_status = cppi41_dma_tx_status;  in cppi41_dma_probe()
931 	cdd->ddev.device_issue_pending = cppi41_dma_issue_pending;  in cppi41_dma_probe()
932 	cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg;  in cppi41_dma_probe()
933 	cdd->ddev.device_terminate_all = cppi41_stop_chan;  in cppi41_dma_probe()
934 	cdd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in cppi41_dma_probe()
935 	cdd->ddev.src_addr_widths = CPPI41_DMA_BUSWIDTHS;  in cppi41_dma_probe()
936 	cdd->ddev.dst_addr_widths = CPPI41_DMA_BUSWIDTHS;  in cppi41_dma_probe()
937 	cdd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in cppi41_dma_probe()
938 	cdd->ddev.dev = dev;  in cppi41_dma_probe()
939 	INIT_LIST_HEAD(&cdd->ddev.channels);  in cppi41_dma_probe()
940 	cpp41_dma_info.dma_cap = cdd->ddev.cap_mask;  in cppi41_dma_probe()
942 	cdd->usbss_mem = of_iomap(dev->of_node, 0);  in cppi41_dma_probe()
943 	cdd->ctrl_mem = of_iomap(dev->of_node, 1);  in cppi41_dma_probe()
944 	cdd->sched_mem = of_iomap(dev->of_node, 2);  in cppi41_dma_probe()
945 	cdd->qmgr_mem = of_iomap(dev->of_node, 3);  in cppi41_dma_probe()
947 	if (!cdd->usbss_mem || !cdd->ctrl_mem || !cdd->sched_mem ||  in cppi41_dma_probe()
948 			!cdd->qmgr_mem)  in cppi41_dma_probe()
956 	cdd->queues_rx = glue_info->queues_rx;  in cppi41_dma_probe()
957 	cdd->queues_tx = glue_info->queues_tx;  in cppi41_dma_probe()
958 	cdd->td_queue = glue_info->td_queue;  in cppi41_dma_probe()
960 	ret = init_cppi41(dev, cdd);  in cppi41_dma_probe()
964 	ret = cppi41_add_chans(dev, cdd);  in cppi41_dma_probe()
974 	cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER);  in cppi41_dma_probe()
977 			dev_name(dev), cdd);  in cppi41_dma_probe()
980 	cdd->irq = irq;  in cppi41_dma_probe()
982 	ret = dma_async_device_register(&cdd->ddev);  in cppi41_dma_probe()
991 	platform_set_drvdata(pdev, cdd);  in cppi41_dma_probe()
994 	dma_async_device_unregister(&cdd->ddev);  in cppi41_dma_probe()
997 	cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);  in cppi41_dma_probe()
998 	cleanup_chans(cdd);  in cppi41_dma_probe()
1000 	deinit_cppi41(dev, cdd);  in cppi41_dma_probe()
1005 	iounmap(cdd->usbss_mem);  in cppi41_dma_probe()
1006 	iounmap(cdd->ctrl_mem);  in cppi41_dma_probe()
1007 	iounmap(cdd->sched_mem);  in cppi41_dma_probe()
1008 	iounmap(cdd->qmgr_mem);  in cppi41_dma_probe()
1014 	struct cppi41_dd *cdd = platform_get_drvdata(pdev);  in cppi41_dma_remove()  local
1017 	dma_async_device_unregister(&cdd->ddev);  in cppi41_dma_remove()
1019 	cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);  in cppi41_dma_remove()
1020 	devm_free_irq(&pdev->dev, cdd->irq, cdd);  in cppi41_dma_remove()
1021 	cleanup_chans(cdd);  in cppi41_dma_remove()
1022 	deinit_cppi41(&pdev->dev, cdd);  in cppi41_dma_remove()
1023 	iounmap(cdd->usbss_mem);  in cppi41_dma_remove()
1024 	iounmap(cdd->ctrl_mem);  in cppi41_dma_remove()
1025 	iounmap(cdd->sched_mem);  in cppi41_dma_remove()
1026 	iounmap(cdd->qmgr_mem);  in cppi41_dma_remove()
1035 	struct cppi41_dd *cdd = dev_get_drvdata(dev);  in cppi41_suspend()  local
1037 	cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ);  in cppi41_suspend()
1038 	cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);  in cppi41_suspend()
1039 	disable_sched(cdd);  in cppi41_suspend()
1046 	struct cppi41_dd *cdd = dev_get_drvdata(dev);  in cppi41_resume()  local
1051 		cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));  in cppi41_resume()
1053 	list_for_each_entry(c, &cdd->ddev.channels, chan.device_node)  in cppi41_resume()
1057 	init_sched(cdd);  in cppi41_resume()
1059 	cppi_writel(cdd->dma_tdfdq, cdd->ctrl_mem + DMA_TDFDQ);  in cppi41_resume()
1060 	cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);  in cppi41_resume()
1061 	cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);  in cppi41_resume()
1062 	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);  in cppi41_resume()
1064 	cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER);  in cppi41_resume()