88 	struct fsl_re_desc *desc;  in fsl_re_tx_submit()  local
93 	desc = to_fsl_re_dma_desc(tx);  in fsl_re_tx_submit()
98 	list_add_tail(&desc->node, &re_chan->submit_q);  in fsl_re_tx_submit()
109 	struct fsl_re_desc *desc, *_desc;  in fsl_re_issue_pending()  local
118 	list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) {  in fsl_re_issue_pending()
122 		list_move_tail(&desc->node, &re_chan->active_q);  in fsl_re_issue_pending()
125 		       &desc->hwdesc, sizeof(struct fsl_re_hw_desc));  in fsl_re_issue_pending()
135 static void fsl_re_desc_done(struct fsl_re_desc *desc)  in fsl_re_desc_done()  argument
140 	dma_cookie_complete(&desc->async_tx);  in fsl_re_desc_done()
142 	callback = desc->async_tx.callback;  in fsl_re_desc_done()
143 	callback_param = desc->async_tx.callback_param;  in fsl_re_desc_done()
147 	dma_descriptor_unmap(&desc->async_tx);  in fsl_re_desc_done()
152 	struct fsl_re_desc *desc, *_desc;  in fsl_re_cleanup_descs()  local
156 	list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) {  in fsl_re_cleanup_descs()
157 		if (async_tx_test_ack(&desc->async_tx))  in fsl_re_cleanup_descs()
158 			list_move_tail(&desc->node, &re_chan->free_q);  in fsl_re_cleanup_descs()
168 	struct fsl_re_desc *desc, *_desc;  in fsl_re_dequeue()  local
183 		list_for_each_entry_safe(desc, _desc, &re_chan->active_q,  in fsl_re_dequeue()
186 			if (desc->hwdesc.lbea32 == hwdesc->lbea32 &&  in fsl_re_dequeue()
187 			    desc->hwdesc.addr_low == hwdesc->addr_low) {  in fsl_re_dequeue()
194 			fsl_re_desc_done(desc);  in fsl_re_dequeue()
195 			list_move_tail(&desc->node, &re_chan->ack_q);  in fsl_re_dequeue()
260 					    struct fsl_re_desc *desc,  in fsl_re_init_desc()  argument
263 	desc->re_chan = re_chan;  in fsl_re_init_desc()
264 	desc->async_tx.tx_submit = fsl_re_tx_submit;  in fsl_re_init_desc()
265 	dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan);  in fsl_re_init_desc()
266 	INIT_LIST_HEAD(&desc->node);  in fsl_re_init_desc()
268 	desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT;  in fsl_re_init_desc()
269 	desc->hwdesc.lbea32 = upper_32_bits(paddr);  in fsl_re_init_desc()
270 	desc->hwdesc.addr_low = lower_32_bits(paddr);  in fsl_re_init_desc()
271 	desc->cf_addr = cf;  in fsl_re_init_desc()
272 	desc->cf_paddr = paddr;  in fsl_re_init_desc()
274 	desc->cdb_addr = (void *)(cf + FSL_RE_CF_DESC_SIZE);  in fsl_re_init_desc()
275 	desc->cdb_paddr = paddr + FSL_RE_CF_DESC_SIZE;  in fsl_re_init_desc()
277 	return desc;  in fsl_re_init_desc()
283 	struct fsl_re_desc *desc = NULL;  in fsl_re_chan_alloc_desc()  local
293 		desc = list_first_entry(&re_chan->free_q,  in fsl_re_chan_alloc_desc()
295 		list_del(&desc->node);  in fsl_re_chan_alloc_desc()
297 		desc->async_tx.flags = flags;  in fsl_re_chan_alloc_desc()
301 	if (!desc) {  in fsl_re_chan_alloc_desc()
302 		desc = kzalloc(sizeof(*desc), GFP_NOWAIT);  in fsl_re_chan_alloc_desc()
303 		if (!desc)  in fsl_re_chan_alloc_desc()
309 			kfree(desc);  in fsl_re_chan_alloc_desc()
313 		desc = fsl_re_init_desc(re_chan, desc, cf, paddr);  in fsl_re_chan_alloc_desc()
314 		desc->async_tx.flags = flags;  in fsl_re_chan_alloc_desc()
321 	return desc;  in fsl_re_chan_alloc_desc()
330 	struct fsl_re_desc *desc;  in fsl_re_prep_dma_genq()  local
345 	desc = fsl_re_chan_alloc_desc(re_chan, flags);  in fsl_re_prep_dma_genq()
346 	if (desc <= 0)  in fsl_re_prep_dma_genq()
360 	xor = desc->cdb_addr;  in fsl_re_prep_dma_genq()
376 	cf = desc->cf_addr;  in fsl_re_prep_dma_genq()
377 	fill_cfd_frame(cf, 0, sizeof(*xor), desc->cdb_paddr, 0);  in fsl_re_prep_dma_genq()
392 	return &desc->async_tx;  in fsl_re_prep_dma_genq()
417 	struct fsl_re_desc *desc;  in fsl_re_prep_dma_pq()  local
449 			desc = to_fsl_re_dma_desc(tx);  in fsl_re_prep_dma_pq()
469 	desc = fsl_re_chan_alloc_desc(re_chan, flags);  in fsl_re_prep_dma_pq()
470 	if (desc <= 0)  in fsl_re_prep_dma_pq()
480 	pq = desc->cdb_addr;  in fsl_re_prep_dma_pq()
497 	cf = desc->cf_addr;  in fsl_re_prep_dma_pq()
498 	fill_cfd_frame(cf, 0, sizeof(struct fsl_re_pq_cdb), desc->cdb_paddr, 0);  in fsl_re_prep_dma_pq()
526 	return &desc->async_tx;  in fsl_re_prep_dma_pq()
539 	struct fsl_re_desc *desc;  in fsl_re_prep_dma_memcpy()  local
553 	desc = fsl_re_chan_alloc_desc(re_chan, flags);  in fsl_re_prep_dma_memcpy()
554 	if (desc <= 0)  in fsl_re_prep_dma_memcpy()
563 	move = desc->cdb_addr;  in fsl_re_prep_dma_memcpy()
567 	cf = desc->cf_addr;  in fsl_re_prep_dma_memcpy()
568 	fill_cfd_frame(cf, 0, sizeof(*move), desc->cdb_paddr, 0);  in fsl_re_prep_dma_memcpy()
578 	return &desc->async_tx;  in fsl_re_prep_dma_memcpy()
584 	struct fsl_re_desc *desc;  in fsl_re_alloc_chan_resources()  local
591 		desc = kzalloc(sizeof(*desc), GFP_KERNEL);  in fsl_re_alloc_chan_resources()
592 		if (!desc)  in fsl_re_alloc_chan_resources()
598 			kfree(desc);  in fsl_re_alloc_chan_resources()
602 		INIT_LIST_HEAD(&desc->node);  in fsl_re_alloc_chan_resources()
603 		fsl_re_init_desc(re_chan, desc, cf, paddr);  in fsl_re_alloc_chan_resources()
605 		list_add_tail(&desc->node, &re_chan->free_q);  in fsl_re_alloc_chan_resources()
614 	struct fsl_re_desc *desc;  in fsl_re_free_chan_resources()  local
618 		desc = list_first_entry(&re_chan->free_q,  in fsl_re_free_chan_resources()
622 		list_del(&desc->node);  in fsl_re_free_chan_resources()
623 		dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr,  in fsl_re_free_chan_resources()
624 			      desc->cf_paddr);  in fsl_re_free_chan_resources()
625 		kfree(desc);  in fsl_re_free_chan_resources()