111 static struct timb_dma *tdchantotd(struct timb_dma_chan *td_chan)  in tdchantotd()  argument
113 	int id = td_chan->chan.chan_id;  in tdchantotd()
114 	return (struct timb_dma *)((u8 *)td_chan -  in tdchantotd()
119 static void __td_enable_chan_irq(struct timb_dma_chan *td_chan)  in __td_enable_chan_irq()  argument
121 	int id = td_chan->chan.chan_id;  in __td_enable_chan_irq()
122 	struct timb_dma *td = tdchantotd(td_chan);  in __td_enable_chan_irq()
128 	dev_dbg(chan2dev(&td_chan->chan), "Enabling irq: %d, IER: 0x%x\n", id,  in __td_enable_chan_irq()
134 static bool __td_dma_done_ack(struct timb_dma_chan *td_chan)  in __td_dma_done_ack()  argument
136 	int id = td_chan->chan.chan_id;  in __td_dma_done_ack()
137 	struct timb_dma *td = (struct timb_dma *)((u8 *)td_chan -  in __td_dma_done_ack()
142 	dev_dbg(chan2dev(&td_chan->chan), "Checking irq: %d, td: %p\n", id, td);  in __td_dma_done_ack()
153 static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc,  in td_fill_desc()  argument
157 		dev_err(chan2dev(&td_chan->chan), "Too big sg element\n");  in td_fill_desc()
163 		dev_err(chan2dev(&td_chan->chan), "Incorrect length: %d\n",  in td_fill_desc()
168 	dev_dbg(chan2dev(&td_chan->chan), "desc: %p, addr: 0x%llx\n",  in td_fill_desc()
186 static void __td_start_dma(struct timb_dma_chan *td_chan)  in __td_start_dma()  argument
190 	if (td_chan->ongoing) {  in __td_start_dma()
191 		dev_err(chan2dev(&td_chan->chan),  in __td_start_dma()
196 	td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc,  in __td_start_dma()
199 	dev_dbg(chan2dev(&td_chan->chan),  in __td_start_dma()
201 		td_chan, td_chan->chan.chan_id, td_chan->membase);  in __td_start_dma()
203 	if (td_chan->direction == DMA_DEV_TO_MEM) {  in __td_start_dma()
206 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_DHAR);  in __td_start_dma()
207 		iowrite32(td_desc->txd.phys, td_chan->membase +  in __td_start_dma()
210 		iowrite32(td_chan->bytes_per_line, td_chan->membase +  in __td_start_dma()
213 		iowrite32(TIMBDMA_RX_EN, td_chan->membase + TIMBDMA_OFFS_RX_ER);  in __td_start_dma()
216 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_TX_DHAR);  in __td_start_dma()
217 		iowrite32(td_desc->txd.phys, td_chan->membase +  in __td_start_dma()
221 	td_chan->ongoing = true;  in __td_start_dma()
224 		__td_enable_chan_irq(td_chan);  in __td_start_dma()
227 static void __td_finish(struct timb_dma_chan *td_chan)  in __td_finish()  argument
235 	if (list_empty(&td_chan->active_list))  in __td_finish()
238 	td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc,  in __td_finish()
242 	dev_dbg(chan2dev(&td_chan->chan), "descriptor %u complete\n",  in __td_finish()
246 	if (td_chan->direction == DMA_DEV_TO_MEM)  in __td_finish()
247 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_ER);  in __td_finish()
253 	td_chan->ongoing = false;  in __td_finish()
258 	list_move(&td_desc->desc_node, &td_chan->free_list);  in __td_finish()
275 		struct timb_dma_chan *td_chan = td->channels + i;  in __td_ier_mask()  local
276 		if (td_chan->ongoing) {  in __td_ier_mask()
278 				list_entry(td_chan->active_list.next,  in __td_ier_mask()
288 static void __td_start_next(struct timb_dma_chan *td_chan)  in __td_start_next()  argument
292 	BUG_ON(list_empty(&td_chan->queue));  in __td_start_next()
293 	BUG_ON(td_chan->ongoing);  in __td_start_next()
295 	td_desc = list_entry(td_chan->queue.next, struct timb_dma_desc,  in __td_start_next()
298 	dev_dbg(chan2dev(&td_chan->chan), "%s: started %u\n",  in __td_start_next()
301 	list_move(&td_desc->desc_node, &td_chan->active_list);  in __td_start_next()
302 	__td_start_dma(td_chan);  in __td_start_next()
309 	struct timb_dma_chan *td_chan = container_of(txd->chan,  in td_tx_submit()  local
313 	spin_lock_bh(&td_chan->lock);  in td_tx_submit()
316 	if (list_empty(&td_chan->active_list)) {  in td_tx_submit()
319 		list_add_tail(&td_desc->desc_node, &td_chan->active_list);  in td_tx_submit()
320 		__td_start_dma(td_chan);  in td_tx_submit()
325 		list_add_tail(&td_desc->desc_node, &td_chan->queue);  in td_tx_submit()
328 	spin_unlock_bh(&td_chan->lock);  in td_tx_submit()
333 static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan)  in td_alloc_init_desc()  argument
335 	struct dma_chan *chan = &td_chan->chan;  in td_alloc_init_desc()
345 	td_desc->desc_list_len = td_chan->desc_elems * TIMB_DMA_DESC_SIZE;  in td_alloc_init_desc()
385 static void td_desc_put(struct timb_dma_chan *td_chan,  in td_desc_put()  argument
388 	dev_dbg(chan2dev(&td_chan->chan), "Putting desc: %p\n", td_desc);  in td_desc_put()
390 	spin_lock_bh(&td_chan->lock);  in td_desc_put()
391 	list_add(&td_desc->desc_node, &td_chan->free_list);  in td_desc_put()
392 	spin_unlock_bh(&td_chan->lock);  in td_desc_put()
395 static struct timb_dma_desc *td_desc_get(struct timb_dma_chan *td_chan)  in td_desc_get()  argument
400 	spin_lock_bh(&td_chan->lock);  in td_desc_get()
401 	list_for_each_entry_safe(td_desc, _td_desc, &td_chan->free_list,  in td_desc_get()
408 		dev_dbg(chan2dev(&td_chan->chan), "desc %p not ACKed\n",  in td_desc_get()
411 	spin_unlock_bh(&td_chan->lock);  in td_desc_get()
418 	struct timb_dma_chan *td_chan =  in td_alloc_chan_resources()  local
424 	BUG_ON(!list_empty(&td_chan->free_list));  in td_alloc_chan_resources()
425 	for (i = 0; i < td_chan->descs; i++) {  in td_alloc_chan_resources()
426 		struct timb_dma_desc *td_desc = td_alloc_init_desc(td_chan);  in td_alloc_chan_resources()
437 		td_desc_put(td_chan, td_desc);  in td_alloc_chan_resources()
440 	spin_lock_bh(&td_chan->lock);  in td_alloc_chan_resources()
442 	spin_unlock_bh(&td_chan->lock);  in td_alloc_chan_resources()
449 	struct timb_dma_chan *td_chan =  in td_free_chan_resources()  local
457 	BUG_ON(!list_empty(&td_chan->active_list));  in td_free_chan_resources()
458 	BUG_ON(!list_empty(&td_chan->queue));  in td_free_chan_resources()
460 	spin_lock_bh(&td_chan->lock);  in td_free_chan_resources()
461 	list_splice_init(&td_chan->free_list, &list);  in td_free_chan_resources()
462 	spin_unlock_bh(&td_chan->lock);  in td_free_chan_resources()
487 	struct timb_dma_chan *td_chan =  in td_issue_pending()  local
491 	spin_lock_bh(&td_chan->lock);  in td_issue_pending()
493 	if (!list_empty(&td_chan->active_list))  in td_issue_pending()
495 		if (__td_dma_done_ack(td_chan))  in td_issue_pending()
496 			__td_finish(td_chan);  in td_issue_pending()
498 	if (list_empty(&td_chan->active_list) && !list_empty(&td_chan->queue))  in td_issue_pending()
499 		__td_start_next(td_chan);  in td_issue_pending()
501 	spin_unlock_bh(&td_chan->lock);  in td_issue_pending()
509 	struct timb_dma_chan *td_chan =  in td_prep_slave_sg()  local
522 	if (td_chan->direction != direction) {  in td_prep_slave_sg()
528 	td_desc = td_desc_get(td_chan);  in td_prep_slave_sg()
543 		err = td_fill_desc(td_chan, td_desc->desc_list + desc_usage, sg,  in td_prep_slave_sg()
548 			td_desc_put(td_chan, td_desc);  in td_prep_slave_sg()
562 	struct timb_dma_chan *td_chan =  in td_terminate_all()  local
569 	spin_lock_bh(&td_chan->lock);  in td_terminate_all()
570 	list_for_each_entry_safe(td_desc, _td_desc, &td_chan->queue,  in td_terminate_all()
572 		list_move(&td_desc->desc_node, &td_chan->free_list);  in td_terminate_all()
575 	__td_finish(td_chan);  in td_terminate_all()
576 	spin_unlock_bh(&td_chan->lock);  in td_terminate_all()
597 			struct timb_dma_chan *td_chan = td->channels + i;  in td_tasklet()  local
598 			spin_lock(&td_chan->lock);  in td_tasklet()
599 			__td_finish(td_chan);  in td_tasklet()
600 			if (!list_empty(&td_chan->queue))  in td_tasklet()
601 				__td_start_next(td_chan);  in td_tasklet()
602 			spin_unlock(&td_chan->lock);  in td_tasklet()
699 		struct timb_dma_chan *td_chan = &td->channels[i];  in td_probe()  local
710 		td_chan->chan.device = &td->dma;  in td_probe()
711 		dma_cookie_init(&td_chan->chan);  in td_probe()
712 		spin_lock_init(&td_chan->lock);  in td_probe()
713 		INIT_LIST_HEAD(&td_chan->active_list);  in td_probe()
714 		INIT_LIST_HEAD(&td_chan->queue);  in td_probe()
715 		INIT_LIST_HEAD(&td_chan->free_list);  in td_probe()
717 		td_chan->descs = pchan->descriptors;  in td_probe()
718 		td_chan->desc_elems = pchan->descriptor_elements;  in td_probe()
719 		td_chan->bytes_per_line = pchan->bytes_per_line;  in td_probe()
720 		td_chan->direction = pchan->rx ? DMA_DEV_TO_MEM :  in td_probe()
723 		td_chan->membase = td->membase +  in td_probe()
728 			i, td_chan->membase);  in td_probe()
730 		list_add_tail(&td_chan->chan.device_node, &td->dma.channels);  in td_probe()