63 static void atc_issue_pending(struct dma_chan *chan);
105 static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,  in atc_alloc_descriptor()  argument
109 	struct at_dma	*atdma = to_at_dma(chan->device);  in atc_alloc_descriptor()
116 		dma_async_tx_descriptor_init(&desc->txd, chan);  in atc_alloc_descriptor()
313 static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie)  in atc_get_bytes_left()  argument
315 	struct at_dma_chan      *atchan = to_at_dma_chan(chan);  in atc_get_bytes_left()
581 			atchan = &atdma->chan[i];  in at_dma_interrupt()
614 	struct at_dma_chan	*atchan = to_at_dma_chan(tx->chan);  in atc_tx_submit()
622 		dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",  in atc_tx_submit()
627 		dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",  in atc_tx_submit()
646 atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,  in atc_prep_dma_memcpy()  argument
649 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_dma_memcpy()
660 	dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d0x%x s0x%x l0x%zx f0x%lx\n",  in atc_prep_dma_memcpy()
664 		dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");  in atc_prep_dma_memcpy()
732 atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,  in atc_prep_slave_sg()  argument
736 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_slave_sg()
737 	struct at_dma_slave	*atslave = chan->private;  in atc_prep_slave_sg()
750 	dev_vdbg(chan2dev(chan), "prep_slave_sg (%d): %s f0x%lx\n",  in atc_prep_slave_sg()
756 		dev_dbg(chan2dev(chan), "prep_slave_sg: sg length is zero!\n");  in atc_prep_slave_sg()
785 				dev_dbg(chan2dev(chan),  in atc_prep_slave_sg()
826 				dev_dbg(chan2dev(chan),  in atc_prep_slave_sg()
867 	dev_err(chan2dev(chan), "not enough descriptors available\n");  in atc_prep_slave_sg()
883 atc_prep_dma_sg(struct dma_chan *chan,  in atc_prep_dma_sg()  argument
888 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_dma_sg()
1036 atc_dma_cyclic_fill_desc(struct dma_chan *chan, struct at_desc *desc,  in atc_dma_cyclic_fill_desc()  argument
1041 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_dma_cyclic_fill_desc()
1094 atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,  in atc_prep_dma_cyclic()  argument
1098 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_dma_cyclic()
1099 	struct at_dma_slave	*atslave = chan->private;  in atc_prep_dma_cyclic()
1108 	dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@0x%08x - %d (%d/%d)\n",  in atc_prep_dma_cyclic()
1114 		dev_dbg(chan2dev(chan), "prep_dma_cyclic: length is zero!\n");  in atc_prep_dma_cyclic()
1120 		dev_dbg(chan2dev(chan), "prep_dma_cyclic: channel in use!\n");  in atc_prep_dma_cyclic()
1144 		if (atc_dma_cyclic_fill_desc(chan, desc, i, buf_addr,  in atc_prep_dma_cyclic()
1162 	dev_err(chan2dev(chan), "not enough descriptors available\n");  in atc_prep_dma_cyclic()
1169 static int atc_config(struct dma_chan *chan,  in atc_config()  argument
1172 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_config()
1174 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in atc_config()
1177 	if (!chan->private)  in atc_config()
1188 static int atc_pause(struct dma_chan *chan)  in atc_pause()  argument
1190 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_pause()
1191 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_pause()
1197 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in atc_pause()
1209 static int atc_resume(struct dma_chan *chan)  in atc_resume()  argument
1211 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_resume()
1212 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_resume()
1218 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in atc_resume()
1233 static int atc_terminate_all(struct dma_chan *chan)  in atc_terminate_all()  argument
1235 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_terminate_all()
1236 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_terminate_all()
1243 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in atc_terminate_all()
1288 atc_tx_status(struct dma_chan *chan,  in atc_tx_status()  argument
1292 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_tx_status()
1297 	ret = dma_cookie_status(chan, cookie, txstate);  in atc_tx_status()
1310 	bytes = atc_get_bytes_left(chan, cookie);  in atc_tx_status()
1315 		dev_vdbg(chan2dev(chan), "get residual bytes error\n");  in atc_tx_status()
1321 	dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %d\n",  in atc_tx_status()
1331 static void atc_issue_pending(struct dma_chan *chan)  in atc_issue_pending()  argument
1333 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_issue_pending()
1336 	dev_vdbg(chan2dev(chan), "issue_pending\n");  in atc_issue_pending()
1354 static int atc_alloc_chan_resources(struct dma_chan *chan)  in atc_alloc_chan_resources()  argument
1356 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_alloc_chan_resources()
1357 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_alloc_chan_resources()
1365 	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");  in atc_alloc_chan_resources()
1369 		dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");  in atc_alloc_chan_resources()
1375 	atslave = chan->private;  in atc_alloc_chan_resources()
1395 		desc = atc_alloc_descriptor(chan, GFP_KERNEL);  in atc_alloc_chan_resources()
1407 	dma_cookie_init(chan);  in atc_alloc_chan_resources()
1413 	dev_dbg(chan2dev(chan),  in atc_alloc_chan_resources()
1424 static void atc_free_chan_resources(struct dma_chan *chan)  in atc_free_chan_resources()  argument
1426 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_free_chan_resources()
1427 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_free_chan_resources()
1431 	dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n",  in atc_free_chan_resources()
1440 		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);  in atc_free_chan_resources()
1449 	dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");  in atc_free_chan_resources()
1453 static bool at_dma_filter(struct dma_chan *chan, void *slave)  in at_dma_filter()  argument
1457 	if (atslave->dma_dev == chan->device->dev) {  in at_dma_filter()
1458 		chan->private = atslave;  in at_dma_filter()
1468 	struct dma_chan *chan;  in at_dma_xlate()  local
1513 	chan = dma_request_channel(mask, at_dma_filter, atslave);  in at_dma_xlate()
1514 	if (!chan)  in at_dma_xlate()
1517 	atchan = to_at_dma_chan(chan);  in at_dma_xlate()
1521 	return chan;  in at_dma_xlate()
1686 		struct at_dma_chan	*atchan = &atdma->chan[i];  in at_dma_probe()
1785 	struct dma_chan		*chan, *_chan;  in at_dma_remove()  local
1794 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_remove()
1796 		struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in at_dma_remove()
1799 		atc_disable_chan_irq(atdma, chan->chan_id);  in at_dma_remove()
1802 		list_del(&chan->device_node);  in at_dma_remove()
1831 	struct dma_chan *chan, *_chan;  in at_dma_prepare()  local
1833 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_prepare()
1835 		struct at_dma_chan *atchan = to_at_dma_chan(chan);  in at_dma_prepare()
1845 	struct dma_chan	*chan = &atchan->chan_common;  in atc_suspend_cyclic()  local
1850 		dev_warn(chan2dev(chan),  in atc_suspend_cyclic()
1852 		atc_pause(chan);  in atc_suspend_cyclic()
1866 	struct dma_chan *chan, *_chan;  in at_dma_suspend_noirq()  local
1869 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_suspend_noirq()
1871 		struct at_dma_chan *atchan = to_at_dma_chan(chan);  in at_dma_suspend_noirq()
1908 	struct dma_chan *chan, *_chan;  in at_dma_resume_noirq()  local
1920 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_resume_noirq()
1922 		struct at_dma_chan *atchan = to_at_dma_chan(chan);  in at_dma_resume_noirq()