108 	struct at_desc	*desc = NULL;  in atc_alloc_descriptor()  local
112 	desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);  in atc_alloc_descriptor()
113 	if (desc) {  in atc_alloc_descriptor()
114 		memset(desc, 0, sizeof(struct at_desc));  in atc_alloc_descriptor()
115 		INIT_LIST_HEAD(&desc->tx_list);  in atc_alloc_descriptor()
116 		dma_async_tx_descriptor_init(&desc->txd, chan);  in atc_alloc_descriptor()
118 		desc->txd.flags = DMA_CTRL_ACK;  in atc_alloc_descriptor()
119 		desc->txd.tx_submit = atc_tx_submit;  in atc_alloc_descriptor()
120 		desc->txd.phys = phys;  in atc_alloc_descriptor()
123 	return desc;  in atc_alloc_descriptor()
132 	struct at_desc *desc, *_desc;  in atc_desc_get()  local
139 	list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {  in atc_desc_get()
141 		if (async_tx_test_ack(&desc->txd)) {  in atc_desc_get()
142 			list_del(&desc->desc_node);  in atc_desc_get()
143 			ret = desc;  in atc_desc_get()
147 				"desc %p not ACKed\n", desc);  in atc_desc_get()
174 static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)  in atc_desc_put()  argument
176 	if (desc) {  in atc_desc_put()
181 		list_for_each_entry(child, &desc->tx_list, desc_node)  in atc_desc_put()
185 		list_splice_init(&desc->tx_list, &atchan->free_list);  in atc_desc_put()
187 			 "moving desc %p to freelist\n", desc);  in atc_desc_put()
188 		list_add(&desc->desc_node, &atchan->free_list);  in atc_desc_put()
202 			   struct at_desc *desc)  in atc_desc_chain()  argument
205 		*first = desc;  in atc_desc_chain()
208 		(*prev)->lli.dscr = desc->txd.phys;  in atc_desc_chain()
210 		list_add_tail(&desc->desc_node,  in atc_desc_chain()
213 	*prev = desc;  in atc_desc_chain()
263 	struct at_desc *desc, *_desc;  in atc_get_desc_by_cookie()  local
265 	list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) {  in atc_get_desc_by_cookie()
266 		if (desc->txd.cookie == cookie)  in atc_get_desc_by_cookie()
267 			return desc;  in atc_get_desc_by_cookie()
270 	list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) {  in atc_get_desc_by_cookie()
271 		if (desc->txd.cookie == cookie)  in atc_get_desc_by_cookie()
272 			return desc;  in atc_get_desc_by_cookie()
287 					struct at_desc *desc)  in atc_calc_bytes_left()  argument
289 	return current_len - ((ctrla & ATC_BTSIZE_MAX) << desc->tx_width);  in atc_calc_bytes_left()
301 			struct at_dma_chan *atchan, struct at_desc *desc)  in atc_calc_bytes_left_from_reg()  argument
305 	return atc_calc_bytes_left(current_len, ctrla, desc);  in atc_calc_bytes_left_from_reg()
317 	struct at_desc *desc;  in atc_get_bytes_left()  local
326 	desc = atc_get_desc_by_cookie(atchan, cookie);  in atc_get_bytes_left()
327 	if (desc == NULL)  in atc_get_bytes_left()
329 	else if (desc != desc_first)  in atc_get_bytes_left()
330 		return desc->total_len;  in atc_get_bytes_left()
356 		list_for_each_entry(desc, &desc_first->tx_list, desc_node) {  in atc_get_bytes_left()
357 			if (desc->lli.dscr == dscr)  in atc_get_bytes_left()
360 			ret -= desc->len;  in atc_get_bytes_left()
369 		if (!desc->lli.dscr)  in atc_get_bytes_left()
370 			ret = atc_calc_bytes_left_from_reg(ret, atchan, desc);  in atc_get_bytes_left()
386 atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)  in atc_chain_complete()  argument
388 	struct dma_async_tx_descriptor	*txd = &desc->txd;  in atc_chain_complete()
398 	list_splice_init(&desc->tx_list, &atchan->free_list);  in atc_chain_complete()
400 	list_move(&desc->desc_node, &atchan->free_list);  in atc_chain_complete()
431 	struct at_desc *desc, *_desc;  in atc_complete_all()  local
447 	list_for_each_entry_safe(desc, _desc, &list, desc_node)  in atc_complete_all()
448 		atc_chain_complete(atchan, desc);  in atc_complete_all()
613 	struct at_desc		*desc = txd_to_at_desc(tx);  in atc_tx_submit()  local
623 				desc->txd.cookie);  in atc_tx_submit()
624 		atc_dostart(atchan, desc);  in atc_tx_submit()
625 		list_add_tail(&desc->desc_node, &atchan->active_list);  in atc_tx_submit()
628 				desc->txd.cookie);  in atc_tx_submit()
629 		list_add_tail(&desc->desc_node, &atchan->queue);  in atc_tx_submit()
650 	struct at_desc		*desc = NULL;  in atc_prep_dma_memcpy()  local
686 		desc = atc_desc_get(atchan);  in atc_prep_dma_memcpy()
687 		if (!desc)  in atc_prep_dma_memcpy()
690 		desc->lli.saddr = src + offset;  in atc_prep_dma_memcpy()
691 		desc->lli.daddr = dest + offset;  in atc_prep_dma_memcpy()
692 		desc->lli.ctrla = ctrla | xfer_count;  in atc_prep_dma_memcpy()
693 		desc->lli.ctrlb = ctrlb;  in atc_prep_dma_memcpy()
695 		desc->txd.cookie = 0;  in atc_prep_dma_memcpy()
696 		desc->len = xfer_count << src_width;  in atc_prep_dma_memcpy()
698 		atc_desc_chain(&first, &prev, desc);  in atc_prep_dma_memcpy()
710 	set_desc_eol(desc);  in atc_prep_dma_memcpy()
774 			struct at_desc	*desc;  in atc_prep_slave_sg()  local
778 			desc = atc_desc_get(atchan);  in atc_prep_slave_sg()
779 			if (!desc)  in atc_prep_slave_sg()
793 			desc->lli.saddr = mem;  in atc_prep_slave_sg()
794 			desc->lli.daddr = reg;  in atc_prep_slave_sg()
795 			desc->lli.ctrla = ctrla  in atc_prep_slave_sg()
798 			desc->lli.ctrlb = ctrlb;  in atc_prep_slave_sg()
799 			desc->len = len;  in atc_prep_slave_sg()
801 			atc_desc_chain(&first, &prev, desc);  in atc_prep_slave_sg()
815 			struct at_desc	*desc;  in atc_prep_slave_sg()  local
819 			desc = atc_desc_get(atchan);  in atc_prep_slave_sg()
820 			if (!desc)  in atc_prep_slave_sg()
834 			desc->lli.saddr = reg;  in atc_prep_slave_sg()
835 			desc->lli.daddr = mem;  in atc_prep_slave_sg()
836 			desc->lli.ctrla = ctrla  in atc_prep_slave_sg()
839 			desc->lli.ctrlb = ctrlb;  in atc_prep_slave_sg()
840 			desc->len = len;  in atc_prep_slave_sg()
842 			atc_desc_chain(&first, &prev, desc);  in atc_prep_slave_sg()
889 	struct at_desc		*desc = NULL;  in atc_prep_dma_sg()  local
966 		desc = atc_desc_get(atchan);  in atc_prep_dma_sg()
967 		if (!desc)  in atc_prep_dma_sg()
970 		desc->lli.saddr = src;  in atc_prep_dma_sg()
971 		desc->lli.daddr = dst;  in atc_prep_dma_sg()
972 		desc->lli.ctrla = ctrla | xfer_count;  in atc_prep_dma_sg()
973 		desc->lli.ctrlb = ctrlb;  in atc_prep_dma_sg()
975 		desc->txd.cookie = 0;  in atc_prep_dma_sg()
976 		desc->len = len;  in atc_prep_dma_sg()
982 		desc->tx_width = src_width;  in atc_prep_dma_sg()
984 		atc_desc_chain(&first, &prev, desc);  in atc_prep_dma_sg()
1000 	set_desc_eol(desc);  in atc_prep_dma_sg()
1036 atc_dma_cyclic_fill_desc(struct dma_chan *chan, struct at_desc *desc,  in atc_dma_cyclic_fill_desc()  argument
1054 		desc->lli.saddr = buf_addr + (period_len * period_index);  in atc_dma_cyclic_fill_desc()
1055 		desc->lli.daddr = sconfig->dst_addr;  in atc_dma_cyclic_fill_desc()
1056 		desc->lli.ctrla = ctrla;  in atc_dma_cyclic_fill_desc()
1057 		desc->lli.ctrlb = ATC_DST_ADDR_MODE_FIXED  in atc_dma_cyclic_fill_desc()
1062 		desc->len = period_len;  in atc_dma_cyclic_fill_desc()
1066 		desc->lli.saddr = sconfig->src_addr;  in atc_dma_cyclic_fill_desc()
1067 		desc->lli.daddr = buf_addr + (period_len * period_index);  in atc_dma_cyclic_fill_desc()
1068 		desc->lli.ctrla = ctrla;  in atc_dma_cyclic_fill_desc()
1069 		desc->lli.ctrlb = ATC_DST_ADDR_MODE_INCR  in atc_dma_cyclic_fill_desc()
1074 		desc->len = period_len;  in atc_dma_cyclic_fill_desc()
1138 		struct at_desc	*desc;  in atc_prep_dma_cyclic()  local
1140 		desc = atc_desc_get(atchan);  in atc_prep_dma_cyclic()
1141 		if (!desc)  in atc_prep_dma_cyclic()
1144 		if (atc_dma_cyclic_fill_desc(chan, desc, i, buf_addr,  in atc_prep_dma_cyclic()
1148 		atc_desc_chain(&first, &prev, desc);  in atc_prep_dma_cyclic()
1238 	struct at_desc		*desc, *_desc;  in atc_terminate_all()  local
1265 	list_for_each_entry_safe(desc, _desc, &list, desc_node)  in atc_terminate_all()
1266 		atc_chain_complete(atchan, desc);  in atc_terminate_all()
1358 	struct at_desc		*desc;  in atc_alloc_chan_resources()  local
1395 		desc = atc_alloc_descriptor(chan, GFP_KERNEL);  in atc_alloc_chan_resources()
1396 		if (!desc) {  in atc_alloc_chan_resources()
1401 		list_add_tail(&desc->desc_node, &tmp_list);  in atc_alloc_chan_resources()
1428 	struct at_desc		*desc, *_desc;  in atc_free_chan_resources()  local
1439 	list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {  in atc_free_chan_resources()
1440 		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);  in atc_free_chan_resources()
1441 		list_del(&desc->desc_node);  in atc_free_chan_resources()
1443 		dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);  in atc_free_chan_resources()