89 	struct fsl_re_chan *re_chan;  in fsl_re_tx_submit()  local
94 	re_chan = container_of(tx->chan, struct fsl_re_chan, chan);  in fsl_re_tx_submit()
96 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_tx_submit()
98 	list_add_tail(&desc->node, &re_chan->submit_q);  in fsl_re_tx_submit()
99 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_tx_submit()
107 	struct fsl_re_chan *re_chan;  in fsl_re_issue_pending()  local
112 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_issue_pending()
114 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_issue_pending()
116 		in_be32(&re_chan->jrregs->inbring_slot_avail));  in fsl_re_issue_pending()
118 	list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) {  in fsl_re_issue_pending()
122 		list_move_tail(&desc->node, &re_chan->active_q);  in fsl_re_issue_pending()
124 		memcpy(&re_chan->inb_ring_virt_addr[re_chan->inb_count],  in fsl_re_issue_pending()
127 		re_chan->inb_count = (re_chan->inb_count + 1) &  in fsl_re_issue_pending()
129 		out_be32(&re_chan->jrregs->inbring_add_job, FSL_RE_ADD_JOB(1));  in fsl_re_issue_pending()
132 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_issue_pending()
150 static void fsl_re_cleanup_descs(struct fsl_re_chan *re_chan)  in fsl_re_cleanup_descs()  argument
155 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_cleanup_descs()
156 	list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) {  in fsl_re_cleanup_descs()
158 			list_move_tail(&desc->node, &re_chan->free_q);  in fsl_re_cleanup_descs()
160 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_cleanup_descs()
162 	fsl_re_issue_pending(&re_chan->chan);  in fsl_re_cleanup_descs()
167 	struct fsl_re_chan *re_chan;  in fsl_re_dequeue()  local
174 	re_chan = dev_get_drvdata((struct device *)data);  in fsl_re_dequeue()
176 	fsl_re_cleanup_descs(re_chan);  in fsl_re_dequeue()
178 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_dequeue()
179 	count =	FSL_RE_SLOT_FULL(in_be32(&re_chan->jrregs->oubring_slot_full));  in fsl_re_dequeue()
182 		hwdesc = &re_chan->oub_ring_virt_addr[re_chan->oub_count];  in fsl_re_dequeue()
183 		list_for_each_entry_safe(desc, _desc, &re_chan->active_q,  in fsl_re_dequeue()
195 			list_move_tail(&desc->node, &re_chan->ack_q);  in fsl_re_dequeue()
197 			dev_err(re_chan->dev,  in fsl_re_dequeue()
201 		oub_count = (re_chan->oub_count + 1) & FSL_RE_RING_SIZE_MASK;  in fsl_re_dequeue()
202 		re_chan->oub_count = oub_count;  in fsl_re_dequeue()
204 		out_be32(&re_chan->jrregs->oubring_job_rmvd,  in fsl_re_dequeue()
207 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_dequeue()
213 	struct fsl_re_chan *re_chan;  in fsl_re_isr()  local
216 	re_chan = dev_get_drvdata((struct device *)data);  in fsl_re_isr()
218 	irqstate = in_be32(&re_chan->jrregs->jr_interrupt_status);  in fsl_re_isr()
228 		status = in_be32(&re_chan->jrregs->jr_status);  in fsl_re_isr()
229 		dev_err(re_chan->dev, "chan error irqstate: %x, status: %x\n",  in fsl_re_isr()
234 	out_be32(&re_chan->jrregs->jr_interrupt_status, FSL_RE_CLR_INTR);  in fsl_re_isr()
236 	tasklet_schedule(&re_chan->irqtask);  in fsl_re_isr()
259 static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan,  in fsl_re_init_desc()  argument
263 	desc->re_chan = re_chan;  in fsl_re_init_desc()
265 	dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan);  in fsl_re_init_desc()
280 static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan,  in fsl_re_chan_alloc_desc()  argument
288 	fsl_re_cleanup_descs(re_chan);  in fsl_re_chan_alloc_desc()
290 	spin_lock_irqsave(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
291 	if (!list_empty(&re_chan->free_q)) {  in fsl_re_chan_alloc_desc()
293 		desc = list_first_entry(&re_chan->free_q,  in fsl_re_chan_alloc_desc()
299 	spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
306 		cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT,  in fsl_re_chan_alloc_desc()
313 		desc = fsl_re_init_desc(re_chan, desc, cf, paddr);  in fsl_re_chan_alloc_desc()
316 		spin_lock_irqsave(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
317 		re_chan->alloc_count++;  in fsl_re_chan_alloc_desc()
318 		spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
329 	struct fsl_re_chan *re_chan;  in fsl_re_prep_dma_genq()  local
338 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_prep_dma_genq()
340 		dev_err(re_chan->dev, "genq tx length %lu, max length %d\n",  in fsl_re_prep_dma_genq()
345 	desc = fsl_re_chan_alloc_desc(re_chan, flags);  in fsl_re_prep_dma_genq()
416 	struct fsl_re_chan *re_chan;  in fsl_re_prep_dma_pq()  local
425 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_prep_dma_pq()
427 		dev_err(re_chan->dev, "pq tx length is %lu, max length is %d\n",  in fsl_re_prep_dma_pq()
469 	desc = fsl_re_chan_alloc_desc(re_chan, flags);  in fsl_re_prep_dma_pq()
518 			dev_err(re_chan->dev, "PQ tx continuation error!\n");  in fsl_re_prep_dma_pq()
538 	struct fsl_re_chan *re_chan;  in fsl_re_prep_dma_memcpy()  local
545 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_prep_dma_memcpy()
548 		dev_err(re_chan->dev, "cp tx length is %lu, max length is %d\n",  in fsl_re_prep_dma_memcpy()
553 	desc = fsl_re_chan_alloc_desc(re_chan, flags);  in fsl_re_prep_dma_memcpy()
583 	struct fsl_re_chan *re_chan;  in fsl_re_alloc_chan_resources()  local
589 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_alloc_chan_resources()
595 		cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL,  in fsl_re_alloc_chan_resources()
603 		fsl_re_init_desc(re_chan, desc, cf, paddr);  in fsl_re_alloc_chan_resources()
605 		list_add_tail(&desc->node, &re_chan->free_q);  in fsl_re_alloc_chan_resources()
606 		re_chan->alloc_count++;  in fsl_re_alloc_chan_resources()
608 	return re_chan->alloc_count;  in fsl_re_alloc_chan_resources()
613 	struct fsl_re_chan *re_chan;  in fsl_re_free_chan_resources()  local
616 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_free_chan_resources()
617 	while (re_chan->alloc_count--) {  in fsl_re_free_chan_resources()
618 		desc = list_first_entry(&re_chan->free_q,  in fsl_re_free_chan_resources()
623 		dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr,  in fsl_re_free_chan_resources()
628 	if (!list_empty(&re_chan->free_q))  in fsl_re_free_chan_resources()
629 		dev_err(re_chan->dev, "chan resource cannot be cleaned!\n");  in fsl_re_free_chan_resources()