195 	struct dma_chan			chan;  member
226 	struct at_xdmac_chan	chan[0];  member
268 	return container_of(dchan, struct at_xdmac_chan, chan);  in to_at_xdmac_chan()
271 static struct device *chan2dev(struct dma_chan *chan)  in chan2dev()  argument
273 	return &chan->dev->device;  in chan2dev()
338 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()
341 	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);  in at_xdmac_start_xfer()
375 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
397 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
402 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
416 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(tx->chan);  in at_xdmac_tx_submit()
423 	dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n",  in at_xdmac_tx_submit()
433 static struct at_xdmac_desc *at_xdmac_alloc_desc(struct dma_chan *chan,  in at_xdmac_alloc_desc()  argument
437 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_alloc_desc()
444 		dma_async_tx_descriptor_init(&desc->tx_dma_desc, chan);  in at_xdmac_alloc_desc()
458 		desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT);  in at_xdmac_get_desc()
474 	struct dma_chan		*chan;  in at_xdmac_xlate()  local
482 	chan = dma_get_any_slave_channel(&atxdmac->dma);  in at_xdmac_xlate()
483 	if (!chan) {  in at_xdmac_xlate()
488 	atchan = to_at_xdmac_chan(chan);  in at_xdmac_xlate()
495 	return chan;  in at_xdmac_xlate()
498 static int at_xdmac_compute_chan_conf(struct dma_chan *chan,  in at_xdmac_compute_chan_conf()  argument
501 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_compute_chan_conf()
517 			dev_err(chan2dev(chan), "invalid src maxburst value\n");  in at_xdmac_compute_chan_conf()
523 			dev_err(chan2dev(chan), "invalid src addr width value\n");  in at_xdmac_compute_chan_conf()
540 			dev_err(chan2dev(chan), "invalid src maxburst value\n");  in at_xdmac_compute_chan_conf()
546 			dev_err(chan2dev(chan), "invalid dst addr width value\n");  in at_xdmac_compute_chan_conf()
552 	dev_dbg(chan2dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);  in at_xdmac_compute_chan_conf()
575 static int at_xdmac_set_slave_config(struct dma_chan *chan,  in at_xdmac_set_slave_config()  argument
578 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_set_slave_config()
581 		dev_err(chan2dev(chan), "invalid slave configuration\n");  in at_xdmac_set_slave_config()
591 at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,  in at_xdmac_prep_slave_sg()  argument
595 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_slave_sg()
607 		dev_err(chan2dev(chan), "invalid DMA direction\n");  in at_xdmac_prep_slave_sg()
611 	dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n",  in at_xdmac_prep_slave_sg()
619 	if (at_xdmac_compute_chan_conf(chan, direction))  in at_xdmac_prep_slave_sg()
630 			dev_err(chan2dev(chan), "sg data length is zero\n");  in at_xdmac_prep_slave_sg()
633 		dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n",  in at_xdmac_prep_slave_sg()
638 			dev_err(chan2dev(chan), "can't get descriptor\n");  in at_xdmac_prep_slave_sg()
663 		dev_dbg(chan2dev(chan),  in at_xdmac_prep_slave_sg()
670 			dev_dbg(chan2dev(chan),  in at_xdmac_prep_slave_sg()
679 		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",  in at_xdmac_prep_slave_sg()
697 at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,  in at_xdmac_prep_dma_cyclic()  argument
702 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_cyclic()
708 	dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n",  in at_xdmac_prep_dma_cyclic()
713 		dev_err(chan2dev(chan), "invalid DMA direction\n");  in at_xdmac_prep_dma_cyclic()
718 		dev_err(chan2dev(chan), "channel currently used\n");  in at_xdmac_prep_dma_cyclic()
722 	if (at_xdmac_compute_chan_conf(chan, direction))  in at_xdmac_prep_dma_cyclic()
731 			dev_err(chan2dev(chan), "can't get descriptor\n");  in at_xdmac_prep_dma_cyclic()
738 		dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_cyclic()
756 		dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_cyclic()
763 			dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_cyclic()
772 		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",  in at_xdmac_prep_dma_cyclic()
778 	dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_cyclic()
789 at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,  in at_xdmac_prep_dma_memcpy()  argument
792 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memcpy()
813 	dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n",  in at_xdmac_prep_dma_memcpy()
826 		dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__);  in at_xdmac_prep_dma_memcpy()
829 		dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__);  in at_xdmac_prep_dma_memcpy()
832 		dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__);  in at_xdmac_prep_dma_memcpy()
835 		dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__);  in at_xdmac_prep_dma_memcpy()
842 		dev_dbg(chan2dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size);  in at_xdmac_prep_dma_memcpy()
848 			dev_err(chan2dev(chan), "can't get descriptor\n");  in at_xdmac_prep_dma_memcpy()
863 		dev_dbg(chan2dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size);  in at_xdmac_prep_dma_memcpy()
868 			dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__);  in at_xdmac_prep_dma_memcpy()
871 			dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__);  in at_xdmac_prep_dma_memcpy()
874 			dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__);  in at_xdmac_prep_dma_memcpy()
877 			dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__);  in at_xdmac_prep_dma_memcpy()
893 		dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_memcpy()
900 			dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_memcpy()
909 		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",  in at_xdmac_prep_dma_memcpy()
921 at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,  in at_xdmac_tx_status()  argument
924 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_tx_status()
925 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()
934 	ret = dma_cookie_status(chan, cookie, txstate);  in at_xdmac_tx_status()
1019 	dev_dbg(chan2dev(chan),  in at_xdmac_tx_status()
1032 	dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_remove_xfer()
1057 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_advance_work()
1083 	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08lx\n",  in at_xdmac_tasklet()
1097 			dev_err(chan2dev(&atchan->chan), "read bus error!!!");  in at_xdmac_tasklet()
1099 			dev_err(chan2dev(&atchan->chan), "write bus error!!!");  in at_xdmac_tasklet()
1101 			dev_err(chan2dev(&atchan->chan), "request overflow error!!!");  in at_xdmac_tasklet()
1107 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_tasklet()
1152 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1159 			dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_interrupt()
1181 static void at_xdmac_issue_pending(struct dma_chan *chan)  in at_xdmac_issue_pending()  argument
1183 	struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);  in at_xdmac_issue_pending()
1185 	dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);  in at_xdmac_issue_pending()
1193 static int at_xdmac_device_config(struct dma_chan *chan,  in at_xdmac_device_config()  argument
1196 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_config()
1200 	dev_dbg(chan2dev(chan), "%s\n", __func__);  in at_xdmac_device_config()
1203 	ret = at_xdmac_set_slave_config(chan, config);  in at_xdmac_device_config()
1209 static int at_xdmac_device_pause(struct dma_chan *chan)  in at_xdmac_device_pause()  argument
1211 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_pause()
1212 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()
1215 	dev_dbg(chan2dev(chan), "%s\n", __func__);  in at_xdmac_device_pause()
1230 static int at_xdmac_device_resume(struct dma_chan *chan)  in at_xdmac_device_resume()  argument
1232 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_resume()
1233 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()
1236 	dev_dbg(chan2dev(chan), "%s\n", __func__);  in at_xdmac_device_resume()
1251 static int at_xdmac_device_terminate_all(struct dma_chan *chan)  in at_xdmac_device_terminate_all()  argument
1254 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_terminate_all()
1255 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()
1258 	dev_dbg(chan2dev(chan), "%s\n", __func__);  in at_xdmac_device_terminate_all()
1276 static int at_xdmac_alloc_chan_resources(struct dma_chan *chan)  in at_xdmac_alloc_chan_resources()  argument
1278 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_alloc_chan_resources()
1286 		dev_err(chan2dev(chan),  in at_xdmac_alloc_chan_resources()
1293 		dev_err(chan2dev(chan),  in at_xdmac_alloc_chan_resources()
1300 		desc = at_xdmac_alloc_desc(chan, GFP_ATOMIC);  in at_xdmac_alloc_chan_resources()
1302 			dev_warn(chan2dev(chan),  in at_xdmac_alloc_chan_resources()
1309 	dma_cookie_init(chan);  in at_xdmac_alloc_chan_resources()
1311 	dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);  in at_xdmac_alloc_chan_resources()
1318 static void at_xdmac_free_chan_resources(struct dma_chan *chan)  in at_xdmac_free_chan_resources()  argument
1320 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_free_chan_resources()
1321 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_free_chan_resources()
1325 		dev_dbg(chan2dev(chan), "%s: freeing descriptor %p\n", __func__, desc);  in at_xdmac_free_chan_resources()
1338 	struct dma_chan		*chan, *_chan;  in atmel_xdmac_prepare()  local
1340 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_prepare()
1341 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_prepare()
1358 	struct dma_chan		*chan, *_chan;  in atmel_xdmac_suspend()  local
1360 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_suspend()
1361 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_suspend()
1366 				at_xdmac_device_pause(chan);  in atmel_xdmac_suspend()
1384 	struct dma_chan		*chan, *_chan;  in atmel_xdmac_resume()  local
1391 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1398 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_resume()
1399 		atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_resume()
1403 				at_xdmac_device_resume(chan);  in atmel_xdmac_resume()
1518 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()
1520 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
1521 		list_add_tail(&atchan->chan.device_node,  in at_xdmac_probe()
1581 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()
1584 		at_xdmac_free_chan_resources(&atchan->chan);  in at_xdmac_remove()