85 	struct dw_desc *desc, *_desc;  in dwc_desc_get()  local
91 	list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {  in dwc_desc_get()
93 		if (async_tx_test_ack(&desc->txd)) {  in dwc_desc_get()
94 			list_del(&desc->desc_node);  in dwc_desc_get()
95 			ret = desc;  in dwc_desc_get()
98 		dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);  in dwc_desc_get()
111 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)  in dwc_desc_put()  argument
115 	if (desc) {  in dwc_desc_put()
119 		list_for_each_entry(child, &desc->tx_list, desc_node)  in dwc_desc_put()
123 		list_splice_init(&desc->tx_list, &dwc->free_list);  in dwc_desc_put()
124 		dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);  in dwc_desc_put()
125 		list_add(&desc->desc_node, &dwc->free_list);  in dwc_desc_put()
191 				       struct dw_desc *desc)  in dwc_do_single_block()  argument
200 	ctllo = desc->lli.ctllo | DWC_CTLL_INT_EN;  in dwc_do_single_block()
202 	channel_writel(dwc, SAR, desc->lli.sar);  in dwc_do_single_block()
203 	channel_writel(dwc, DAR, desc->lli.dar);  in dwc_do_single_block()
205 	channel_writel(dwc, CTL_HI, desc->lli.ctlhi);  in dwc_do_single_block()
260 	struct dw_desc *desc;  in dwc_dostart_first_queued()  local
266 	desc = dwc_first_active(dwc);  in dwc_dostart_first_queued()
267 	dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie);  in dwc_dostart_first_queued()
268 	dwc_dostart(dwc, desc);  in dwc_dostart_first_queued()
274 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,  in dwc_descriptor_complete()  argument
279 	struct dma_async_tx_descriptor	*txd = &desc->txd;  in dwc_descriptor_complete()
293 	list_for_each_entry(child, &desc->tx_list, desc_node)  in dwc_descriptor_complete()
295 	async_tx_ack(&desc->txd);  in dwc_descriptor_complete()
297 	list_splice_init(&desc->tx_list, &dwc->free_list);  in dwc_descriptor_complete()
298 	list_move(&desc->desc_node, &dwc->free_list);  in dwc_descriptor_complete()
309 	struct dw_desc *desc, *_desc;  in dwc_complete_all()  local
331 	list_for_each_entry_safe(desc, _desc, &list, desc_node)  in dwc_complete_all()
332 		dwc_descriptor_complete(dwc, desc, true);  in dwc_complete_all()
347 	struct dw_desc *desc, *_desc;  in dwc_scan_descriptors()  local
367 			desc = dwc_first_active(dwc);  in dwc_scan_descriptors()
369 			head = &desc->tx_list;  in dwc_scan_descriptors()
373 					desc = to_dw_desc(active->prev);  in dwc_scan_descriptors()
375 				dwc->residue -= desc->len;  in dwc_scan_descriptors()
412 	list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {  in dwc_scan_descriptors()
414 		dwc->residue = desc->total_len;  in dwc_scan_descriptors()
417 		if (desc->txd.phys == llp) {  in dwc_scan_descriptors()
423 		if (desc->lli.llp == llp) {  in dwc_scan_descriptors()
430 		dwc->residue -= desc->len;  in dwc_scan_descriptors()
431 		list_for_each_entry(child, &desc->tx_list, desc_node) {  in dwc_scan_descriptors()
446 		dwc_descriptor_complete(dwc, desc, true);  in dwc_scan_descriptors()
574 			dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);  in dwc_handle_cyclic()
658 	struct dw_desc		*desc = txd_to_dw_desc(tx);  in dwc_tx_submit()  local
672 	dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", __func__, desc->txd.cookie);  in dwc_tx_submit()
673 	list_add_tail(&desc->desc_node, &dwc->queue);  in dwc_tx_submit()
686 	struct dw_desc		*desc;  in dwc_prep_dma_memcpy()  local
725 		desc = dwc_desc_get(dwc);  in dwc_prep_dma_memcpy()
726 		if (!desc)  in dwc_prep_dma_memcpy()
729 		desc->lli.sar = src + offset;  in dwc_prep_dma_memcpy()
730 		desc->lli.dar = dest + offset;  in dwc_prep_dma_memcpy()
731 		desc->lli.ctllo = ctllo;  in dwc_prep_dma_memcpy()
732 		desc->lli.ctlhi = xfer_count;  in dwc_prep_dma_memcpy()
733 		desc->len = xfer_count << src_width;  in dwc_prep_dma_memcpy()
736 			first = desc;  in dwc_prep_dma_memcpy()
738 			prev->lli.llp = desc->txd.phys;  in dwc_prep_dma_memcpy()
739 			list_add_tail(&desc->desc_node,  in dwc_prep_dma_memcpy()
742 		prev = desc;  in dwc_prep_dma_memcpy()
803 			struct dw_desc	*desc;  in dwc_prep_slave_sg()  local
813 			desc = dwc_desc_get(dwc);  in dwc_prep_slave_sg()
814 			if (!desc)  in dwc_prep_slave_sg()
817 			desc->lli.sar = mem;  in dwc_prep_slave_sg()
818 			desc->lli.dar = reg;  in dwc_prep_slave_sg()
819 			desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);  in dwc_prep_slave_sg()
829 			desc->lli.ctlhi = dlen >> mem_width;  in dwc_prep_slave_sg()
830 			desc->len = dlen;  in dwc_prep_slave_sg()
833 				first = desc;  in dwc_prep_slave_sg()
835 				prev->lli.llp = desc->txd.phys;  in dwc_prep_slave_sg()
836 				list_add_tail(&desc->desc_node,  in dwc_prep_slave_sg()
839 			prev = desc;  in dwc_prep_slave_sg()
860 			struct dw_desc	*desc;  in dwc_prep_slave_sg()  local
870 			desc = dwc_desc_get(dwc);  in dwc_prep_slave_sg()
871 			if (!desc)  in dwc_prep_slave_sg()
874 			desc->lli.sar = reg;  in dwc_prep_slave_sg()
875 			desc->lli.dar = mem;  in dwc_prep_slave_sg()
876 			desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);  in dwc_prep_slave_sg()
885 			desc->lli.ctlhi = dlen >> reg_width;  in dwc_prep_slave_sg()
886 			desc->len = dlen;  in dwc_prep_slave_sg()
889 				first = desc;  in dwc_prep_slave_sg()
891 				prev->lli.llp = desc->txd.phys;  in dwc_prep_slave_sg()
892 				list_add_tail(&desc->desc_node,  in dwc_prep_slave_sg()
895 			prev = desc;  in dwc_prep_slave_sg()
1026 	struct dw_desc		*desc, *_desc;  in dwc_terminate_all()  local
1045 	list_for_each_entry_safe(desc, _desc, &list, desc_node)  in dwc_terminate_all()
1046 		dwc_descriptor_complete(dwc, desc, false);  in dwc_terminate_all()
1131 	struct dw_desc		*desc;  in dwc_alloc_chan_resources()  local
1171 		desc = dma_pool_alloc(dw->desc_pool, GFP_ATOMIC, &phys);  in dwc_alloc_chan_resources()
1172 		if (!desc)  in dwc_alloc_chan_resources()
1175 		memset(desc, 0, sizeof(struct dw_desc));  in dwc_alloc_chan_resources()
1177 		INIT_LIST_HEAD(&desc->tx_list);  in dwc_alloc_chan_resources()
1178 		dma_async_tx_descriptor_init(&desc->txd, chan);  in dwc_alloc_chan_resources()
1179 		desc->txd.tx_submit = dwc_tx_submit;  in dwc_alloc_chan_resources()
1180 		desc->txd.flags = DMA_CTRL_ACK;  in dwc_alloc_chan_resources()
1181 		desc->txd.phys = phys;  in dwc_alloc_chan_resources()
1183 		dwc_desc_put(dwc, desc);  in dwc_alloc_chan_resources()
1205 	struct dw_desc		*desc, *_desc;  in dwc_free_chan_resources()  local
1242 	list_for_each_entry_safe(desc, _desc, &list, desc_node) {  in dwc_free_chan_resources()
1243 		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);  in dwc_free_chan_resources()
1244 		dma_pool_free(dw->desc_pool, desc, desc->txd.phys);  in dwc_free_chan_resources()
1275 	dwc_dostart(dwc, dwc->cdesc->desc[0]);  in dw_dma_cyclic_start()
1322 	struct dw_desc			*desc;  in dw_dma_cyclic_prep()  local
1384 	cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL);  in dw_dma_cyclic_prep()
1385 	if (!cdesc->desc)  in dw_dma_cyclic_prep()
1389 		desc = dwc_desc_get(dwc);  in dw_dma_cyclic_prep()
1390 		if (!desc)  in dw_dma_cyclic_prep()
1395 			desc->lli.dar = sconfig->dst_addr;  in dw_dma_cyclic_prep()
1396 			desc->lli.sar = buf_addr + (period_len * i);  in dw_dma_cyclic_prep()
1397 			desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)  in dw_dma_cyclic_prep()
1404 			desc->lli.ctllo |= sconfig->device_fc ?  in dw_dma_cyclic_prep()
1410 			desc->lli.dar = buf_addr + (period_len * i);  in dw_dma_cyclic_prep()
1411 			desc->lli.sar = sconfig->src_addr;  in dw_dma_cyclic_prep()
1412 			desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)  in dw_dma_cyclic_prep()
1419 			desc->lli.ctllo |= sconfig->device_fc ?  in dw_dma_cyclic_prep()
1428 		desc->lli.ctlhi = (period_len >> reg_width);  in dw_dma_cyclic_prep()
1429 		cdesc->desc[i] = desc;  in dw_dma_cyclic_prep()
1432 			last->lli.llp = desc->txd.phys;  in dw_dma_cyclic_prep()
1434 		last = desc;  in dw_dma_cyclic_prep()
1438 	last->lli.llp = cdesc->desc[0]->txd.phys;  in dw_dma_cyclic_prep()
1451 		dwc_desc_put(dwc, cdesc->desc[i]);  in dw_dma_cyclic_prep()
1488 		dwc_desc_put(dwc, cdesc->desc[i]);  in dw_dma_cyclic_free()
1490 	kfree(cdesc->desc);  in dw_dma_cyclic_free()