108 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(c);  in dma_chan_to_sirfsoc_dma()  local
109 	return container_of(schan, struct sirfsoc_dma, channels[c->chan_id]);  in dma_chan_to_sirfsoc_dma()
113 static void sirfsoc_dma_execute(struct sirfsoc_dma_chan *schan)  in sirfsoc_dma_execute()  argument
115 	struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan);  in sirfsoc_dma_execute()
116 	int cid = schan->chan.chan_id;  in sirfsoc_dma_execute()
124 	sdesc = list_first_entry(&schan->queued, struct sirfsoc_dma_desc,  in sirfsoc_dma_execute()
127 	list_move_tail(&sdesc->node, &schan->active);  in sirfsoc_dma_execute()
132 	writel_relaxed(cid | (schan->mode << SIRFSOC_DMA_MODE_CTRL_BIT) |  in sirfsoc_dma_execute()
152 		schan->happened_cyclic = schan->completed_cyclic = 0;  in sirfsoc_dma_execute()
160 	struct sirfsoc_dma_chan *schan;  in sirfsoc_dma_irq()  local
169 		schan = &sdma->channels[ch];  in sirfsoc_dma_irq()
171 		spin_lock(&schan->lock);  in sirfsoc_dma_irq()
173 		sdesc = list_first_entry(&schan->active, struct sirfsoc_dma_desc,  in sirfsoc_dma_irq()
177 			list_splice_tail_init(&schan->active, &schan->completed);  in sirfsoc_dma_irq()
178 			if (!list_empty(&schan->queued))  in sirfsoc_dma_irq()
179 				sirfsoc_dma_execute(schan);  in sirfsoc_dma_irq()
181 			schan->happened_cyclic++;  in sirfsoc_dma_irq()
183 		spin_unlock(&schan->lock);  in sirfsoc_dma_irq()
196 	struct sirfsoc_dma_chan *schan;  in sirfsoc_dma_process_completed()  local
205 		schan = &sdma->channels[i];  in sirfsoc_dma_process_completed()
208 		spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_process_completed()
209 		if (!list_empty(&schan->completed)) {  in sirfsoc_dma_process_completed()
210 			list_splice_tail_init(&schan->completed, &list);  in sirfsoc_dma_process_completed()
211 			spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_process_completed()
225 			spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_process_completed()
226 			list_splice_tail_init(&list, &schan->free);  in sirfsoc_dma_process_completed()
227 			schan->chan.completed_cookie = last_cookie;  in sirfsoc_dma_process_completed()
228 			spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_process_completed()
231 			sdesc = list_first_entry(&schan->active, struct sirfsoc_dma_desc,  in sirfsoc_dma_process_completed()
236 				spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_process_completed()
241 			happened_cyclic = schan->happened_cyclic;  in sirfsoc_dma_process_completed()
242 			spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_process_completed()
245 			while (happened_cyclic != schan->completed_cyclic) {  in sirfsoc_dma_process_completed()
248 				schan->completed_cyclic++;  in sirfsoc_dma_process_completed()
265 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(txd->chan);  in sirfsoc_dma_tx_submit()  local
272 	spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_tx_submit()
275 	list_move_tail(&sdesc->node, &schan->queued);  in sirfsoc_dma_tx_submit()
279 	spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_tx_submit()
287 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_slave_config()  local
294 	spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_slave_config()
295 	schan->mode = (config->src_maxburst == 4 ? 1 : 0);  in sirfsoc_dma_slave_config()
296 	spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_slave_config()
303 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_terminate_all()  local
304 	struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan);  in sirfsoc_dma_terminate_all()
305 	int cid = schan->chan.chan_id;  in sirfsoc_dma_terminate_all()
308 	spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_terminate_all()
324 	list_splice_tail_init(&schan->active, &schan->free);  in sirfsoc_dma_terminate_all()
325 	list_splice_tail_init(&schan->queued, &schan->free);  in sirfsoc_dma_terminate_all()
327 	spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_terminate_all()
334 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_pause_chan()  local
335 	struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan);  in sirfsoc_dma_pause_chan()
336 	int cid = schan->chan.chan_id;  in sirfsoc_dma_pause_chan()
339 	spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_pause_chan()
349 	spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_pause_chan()
356 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_resume_chan()  local
357 	struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan);  in sirfsoc_dma_resume_chan()
358 	int cid = schan->chan.chan_id;  in sirfsoc_dma_resume_chan()
361 	spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_resume_chan()
371 	spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_resume_chan()
380 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_alloc_chan_resources()  local
408 	spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_alloc_chan_resources()
410 	list_splice_tail_init(&descs, &schan->free);  in sirfsoc_dma_alloc_chan_resources()
411 	spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_alloc_chan_resources()
419 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_free_chan_resources()  local
425 	spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_free_chan_resources()
428 	BUG_ON(!list_empty(&schan->prepared));  in sirfsoc_dma_free_chan_resources()
429 	BUG_ON(!list_empty(&schan->queued));  in sirfsoc_dma_free_chan_resources()
430 	BUG_ON(!list_empty(&schan->active));  in sirfsoc_dma_free_chan_resources()
431 	BUG_ON(!list_empty(&schan->completed));  in sirfsoc_dma_free_chan_resources()
434 	list_splice_tail_init(&schan->free, &descs);  in sirfsoc_dma_free_chan_resources()
436 	spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_free_chan_resources()
448 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_issue_pending()  local
451 	spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_issue_pending()
453 	if (list_empty(&schan->active) && !list_empty(&schan->queued))  in sirfsoc_dma_issue_pending()
454 		sirfsoc_dma_execute(schan);  in sirfsoc_dma_issue_pending()
456 	spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_issue_pending()
465 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_tx_status()  local
469 	int cid = schan->chan.chan_id;  in sirfsoc_dma_tx_status()
474 	spin_lock_irqsave(&schan->lock, flags);  in sirfsoc_dma_tx_status()
476 	sdesc = list_first_entry(&schan->active, struct sirfsoc_dma_desc,  in sirfsoc_dma_tx_status()
487 	spin_unlock_irqrestore(&schan->lock, flags);  in sirfsoc_dma_tx_status()
497 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_prep_interleaved()  local
508 	spin_lock_irqsave(&schan->lock, iflags);  in sirfsoc_dma_prep_interleaved()
509 	if (!list_empty(&schan->free)) {  in sirfsoc_dma_prep_interleaved()
510 		sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc,  in sirfsoc_dma_prep_interleaved()
514 	spin_unlock_irqrestore(&schan->lock, iflags);  in sirfsoc_dma_prep_interleaved()
524 	spin_lock_irqsave(&schan->lock, iflags);  in sirfsoc_dma_prep_interleaved()
544 		list_add_tail(&sdesc->node, &schan->prepared);  in sirfsoc_dma_prep_interleaved()
550 	spin_unlock_irqrestore(&schan->lock, iflags);  in sirfsoc_dma_prep_interleaved()
554 	spin_unlock_irqrestore(&schan->lock, iflags);  in sirfsoc_dma_prep_interleaved()
565 	struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);  in sirfsoc_dma_prep_cyclic()  local
584 	spin_lock_irqsave(&schan->lock, iflags);  in sirfsoc_dma_prep_cyclic()
585 	if (!list_empty(&schan->free)) {  in sirfsoc_dma_prep_cyclic()
586 		sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc,  in sirfsoc_dma_prep_cyclic()
590 	spin_unlock_irqrestore(&schan->lock, iflags);  in sirfsoc_dma_prep_cyclic()
596 	spin_lock_irqsave(&schan->lock, iflags);  in sirfsoc_dma_prep_cyclic()
602 	list_add_tail(&sdesc->node, &schan->prepared);  in sirfsoc_dma_prep_cyclic()
603 	spin_unlock_irqrestore(&schan->lock, iflags);  in sirfsoc_dma_prep_cyclic()
649 	struct sirfsoc_dma_chan *schan;  in sirfsoc_dma_probe()  local
728 		schan = &sdma->channels[i];  in sirfsoc_dma_probe()
730 		schan->chan.device = dma;  in sirfsoc_dma_probe()
731 		dma_cookie_init(&schan->chan);  in sirfsoc_dma_probe()
733 		INIT_LIST_HEAD(&schan->free);  in sirfsoc_dma_probe()
734 		INIT_LIST_HEAD(&schan->prepared);  in sirfsoc_dma_probe()
735 		INIT_LIST_HEAD(&schan->queued);  in sirfsoc_dma_probe()
736 		INIT_LIST_HEAD(&schan->active);  in sirfsoc_dma_probe()
737 		INIT_LIST_HEAD(&schan->completed);  in sirfsoc_dma_probe()
739 		spin_lock_init(&schan->lock);  in sirfsoc_dma_probe()
740 		list_add_tail(&schan->chan.device_node, &dma->channels);  in sirfsoc_dma_probe()
816 	struct sirfsoc_dma_chan *schan;  in sirfsoc_dma_pm_suspend()  local
835 		schan = &sdma->channels[ch];  in sirfsoc_dma_pm_suspend()
836 		if (list_empty(&schan->active))  in sirfsoc_dma_pm_suspend()
838 		sdesc = list_first_entry(&schan->active,  in sirfsoc_dma_pm_suspend()
857 	struct sirfsoc_dma_chan *schan;  in sirfsoc_dma_pm_resume()  local
868 		schan = &sdma->channels[ch];  in sirfsoc_dma_pm_resume()
869 		if (list_empty(&schan->active))  in sirfsoc_dma_pm_resume()
871 		sdesc = list_first_entry(&schan->active,  in sirfsoc_dma_pm_resume()