Lines Matching refs:atchan
83 static struct at_desc *atc_first_active(struct at_dma_chan *atchan) in atc_first_active() argument
85 return list_first_entry(&atchan->active_list, in atc_first_active()
89 static struct at_desc *atc_first_queued(struct at_dma_chan *atchan) in atc_first_queued() argument
91 return list_first_entry(&atchan->queue, in atc_first_queued()
130 static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) in atc_desc_get() argument
138 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
139 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_desc_get()
146 dev_dbg(chan2dev(&atchan->chan_common), in atc_desc_get()
149 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
150 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_get()
155 ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); in atc_desc_get()
157 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
158 atchan->descs_allocated++; in atc_desc_get()
159 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
161 dev_err(chan2dev(&atchan->chan_common), in atc_desc_get()
174 static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) in atc_desc_put() argument
180 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_put()
182 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
185 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_desc_put()
186 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
188 list_add(&desc->desc_node, &atchan->free_list); in atc_desc_put()
189 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_put()
223 static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) in atc_dostart() argument
225 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_dostart()
228 if (atc_chan_is_enabled(atchan)) { in atc_dostart()
229 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
231 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
233 channel_readl(atchan, SADDR), in atc_dostart()
234 channel_readl(atchan, DADDR), in atc_dostart()
235 channel_readl(atchan, CTRLA), in atc_dostart()
236 channel_readl(atchan, CTRLB), in atc_dostart()
237 channel_readl(atchan, DSCR)); in atc_dostart()
243 vdbg_dump_regs(atchan); in atc_dostart()
245 channel_writel(atchan, SADDR, 0); in atc_dostart()
246 channel_writel(atchan, DADDR, 0); in atc_dostart()
247 channel_writel(atchan, CTRLA, 0); in atc_dostart()
248 channel_writel(atchan, CTRLB, 0); in atc_dostart()
249 channel_writel(atchan, DSCR, first->txd.phys); in atc_dostart()
250 dma_writel(atdma, CHER, atchan->mask); in atc_dostart()
252 vdbg_dump_regs(atchan); in atc_dostart()
260 static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan, in atc_get_desc_by_cookie() argument
265 list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) { in atc_get_desc_by_cookie()
270 list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { in atc_get_desc_by_cookie()
301 struct at_dma_chan *atchan, struct at_desc *desc) in atc_calc_bytes_left_from_reg() argument
303 u32 ctrla = channel_readl(atchan, CTRLA); in atc_calc_bytes_left_from_reg()
315 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_get_bytes_left() local
316 struct at_desc *desc_first = atc_first_active(atchan); in atc_get_bytes_left()
326 desc = atc_get_desc_by_cookie(atchan, cookie); in atc_get_bytes_left()
347 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
349 dscr = channel_readl(atchan, DSCR); in atc_get_bytes_left()
370 ret = atc_calc_bytes_left_from_reg(ret, atchan, desc); in atc_get_bytes_left()
373 ret = atc_calc_bytes_left_from_reg(ret, atchan, desc_first); in atc_get_bytes_left()
386 atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) in atc_chain_complete() argument
390 dev_vdbg(chan2dev(&atchan->chan_common), in atc_chain_complete()
394 if (!atc_chan_is_cyclic(atchan)) in atc_chain_complete()
398 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_chain_complete()
400 list_move(&desc->desc_node, &atchan->free_list); in atc_chain_complete()
405 if (!atc_chan_is_cyclic(atchan)) { in atc_chain_complete()
429 static void atc_complete_all(struct at_dma_chan *atchan) in atc_complete_all() argument
434 dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); in atc_complete_all()
440 if (!list_empty(&atchan->queue)) in atc_complete_all()
441 atc_dostart(atchan, atc_first_queued(atchan)); in atc_complete_all()
443 list_splice_init(&atchan->active_list, &list); in atc_complete_all()
445 list_splice_init(&atchan->queue, &atchan->active_list); in atc_complete_all()
448 atc_chain_complete(atchan, desc); in atc_complete_all()
457 static void atc_advance_work(struct at_dma_chan *atchan) in atc_advance_work() argument
459 dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); in atc_advance_work()
461 if (atc_chan_is_enabled(atchan)) in atc_advance_work()
464 if (list_empty(&atchan->active_list) || in atc_advance_work()
465 list_is_singular(&atchan->active_list)) { in atc_advance_work()
466 atc_complete_all(atchan); in atc_advance_work()
468 atc_chain_complete(atchan, atc_first_active(atchan)); in atc_advance_work()
470 atc_dostart(atchan, atc_first_active(atchan)); in atc_advance_work()
481 static void atc_handle_error(struct at_dma_chan *atchan) in atc_handle_error() argument
491 bad_desc = atc_first_active(atchan); in atc_handle_error()
496 list_splice_init(&atchan->queue, atchan->active_list.prev); in atc_handle_error()
499 if (!list_empty(&atchan->active_list)) in atc_handle_error()
500 atc_dostart(atchan, atc_first_active(atchan)); in atc_handle_error()
509 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
511 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
513 atc_dump_lli(atchan, &bad_desc->lli); in atc_handle_error()
515 atc_dump_lli(atchan, &child->lli); in atc_handle_error()
518 atc_chain_complete(atchan, bad_desc); in atc_handle_error()
527 static void atc_handle_cyclic(struct at_dma_chan *atchan) in atc_handle_cyclic() argument
529 struct at_desc *first = atc_first_active(atchan); in atc_handle_cyclic()
534 dev_vdbg(chan2dev(&atchan->chan_common), in atc_handle_cyclic()
536 channel_readl(atchan, DSCR)); in atc_handle_cyclic()
546 struct at_dma_chan *atchan = (struct at_dma_chan *)data; in atc_tasklet() local
549 spin_lock_irqsave(&atchan->lock, flags); in atc_tasklet()
550 if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) in atc_tasklet()
551 atc_handle_error(atchan); in atc_tasklet()
552 else if (atc_chan_is_cyclic(atchan)) in atc_tasklet()
553 atc_handle_cyclic(atchan); in atc_tasklet()
555 atc_advance_work(atchan); in atc_tasklet()
557 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tasklet()
563 struct at_dma_chan *atchan; in at_dma_interrupt() local
581 atchan = &atdma->chan[i]; in at_dma_interrupt()
586 AT_DMA_RES(i) | atchan->mask); in at_dma_interrupt()
588 set_bit(ATC_IS_ERROR, &atchan->status); in at_dma_interrupt()
590 tasklet_schedule(&atchan->tasklet); in at_dma_interrupt()
614 struct at_dma_chan *atchan = to_at_dma_chan(tx->chan); in atc_tx_submit() local
618 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_submit()
621 if (list_empty(&atchan->active_list)) { in atc_tx_submit()
624 atc_dostart(atchan, desc); in atc_tx_submit()
625 list_add_tail(&desc->desc_node, &atchan->active_list); in atc_tx_submit()
629 list_add_tail(&desc->desc_node, &atchan->queue); in atc_tx_submit()
632 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_submit()
649 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_memcpy() local
686 desc = atc_desc_get(atchan); in atc_prep_dma_memcpy()
717 atc_desc_put(atchan, first); in atc_prep_dma_memcpy()
736 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_slave_sg() local
738 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_slave_sg()
771 | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if); in atc_prep_slave_sg()
778 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
811 | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if); in atc_prep_slave_sg()
819 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
869 atc_desc_put(atchan, first); in atc_prep_slave_sg()
888 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_sg() local
966 desc = atc_desc_get(atchan); in atc_prep_dma_sg()
1007 atc_desc_put(atchan, first); in atc_prep_dma_sg()
1041 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_dma_cyclic_fill_desc() local
1042 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_dma_cyclic_fill_desc()
1060 | ATC_SIF(atchan->mem_if) in atc_dma_cyclic_fill_desc()
1061 | ATC_DIF(atchan->per_if); in atc_dma_cyclic_fill_desc()
1072 | ATC_SIF(atchan->per_if) in atc_dma_cyclic_fill_desc()
1073 | ATC_DIF(atchan->mem_if); in atc_dma_cyclic_fill_desc()
1098 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_cyclic() local
1100 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_dma_cyclic()
1118 was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
1140 desc = atc_desc_get(atchan); in atc_prep_dma_cyclic()
1163 atc_desc_put(atchan, first); in atc_prep_dma_cyclic()
1165 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
1172 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_config() local
1180 memcpy(&atchan->dma_sconfig, sconfig, sizeof(*sconfig)); in atc_config()
1182 convert_burst(&atchan->dma_sconfig.src_maxburst); in atc_config()
1183 convert_burst(&atchan->dma_sconfig.dst_maxburst); in atc_config()
1190 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_pause() local
1192 int chan_id = atchan->chan_common.chan_id; in atc_pause()
1199 spin_lock_irqsave(&atchan->lock, flags); in atc_pause()
1202 set_bit(ATC_IS_PAUSED, &atchan->status); in atc_pause()
1204 spin_unlock_irqrestore(&atchan->lock, flags); in atc_pause()
1211 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_resume() local
1213 int chan_id = atchan->chan_common.chan_id; in atc_resume()
1220 if (!atc_chan_is_paused(atchan)) in atc_resume()
1223 spin_lock_irqsave(&atchan->lock, flags); in atc_resume()
1226 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_resume()
1228 spin_unlock_irqrestore(&atchan->lock, flags); in atc_resume()
1235 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_terminate_all() local
1237 int chan_id = atchan->chan_common.chan_id; in atc_terminate_all()
1251 spin_lock_irqsave(&atchan->lock, flags); in atc_terminate_all()
1254 dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); in atc_terminate_all()
1257 while (dma_readl(atdma, CHSR) & atchan->mask) in atc_terminate_all()
1261 list_splice_init(&atchan->queue, &list); in atc_terminate_all()
1262 list_splice_init(&atchan->active_list, &list); in atc_terminate_all()
1266 atc_chain_complete(atchan, desc); in atc_terminate_all()
1268 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_terminate_all()
1270 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_terminate_all()
1272 spin_unlock_irqrestore(&atchan->lock, flags); in atc_terminate_all()
1292 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_tx_status() local
1307 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_status()
1312 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_status()
1333 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_issue_pending() local
1339 if (atc_chan_is_cyclic(atchan)) in atc_issue_pending()
1342 spin_lock_irqsave(&atchan->lock, flags); in atc_issue_pending()
1343 atc_advance_work(atchan); in atc_issue_pending()
1344 spin_unlock_irqrestore(&atchan->lock, flags); in atc_issue_pending()
1356 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_alloc_chan_resources() local
1368 if (atc_chan_is_enabled(atchan)) { in atc_alloc_chan_resources()
1390 if (!list_empty(&atchan->free_list)) in atc_alloc_chan_resources()
1391 return atchan->descs_allocated; in atc_alloc_chan_resources()
1404 spin_lock_irqsave(&atchan->lock, flags); in atc_alloc_chan_resources()
1405 atchan->descs_allocated = i; in atc_alloc_chan_resources()
1406 list_splice(&tmp_list, &atchan->free_list); in atc_alloc_chan_resources()
1408 spin_unlock_irqrestore(&atchan->lock, flags); in atc_alloc_chan_resources()
1411 channel_writel(atchan, CFG, cfg); in atc_alloc_chan_resources()
1415 atchan->descs_allocated); in atc_alloc_chan_resources()
1417 return atchan->descs_allocated; in atc_alloc_chan_resources()
1426 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_free_chan_resources() local
1432 atchan->descs_allocated); in atc_free_chan_resources()
1435 BUG_ON(!list_empty(&atchan->active_list)); in atc_free_chan_resources()
1436 BUG_ON(!list_empty(&atchan->queue)); in atc_free_chan_resources()
1437 BUG_ON(atc_chan_is_enabled(atchan)); in atc_free_chan_resources()
1439 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_free_chan_resources()
1445 list_splice_init(&atchan->free_list, &list); in atc_free_chan_resources()
1446 atchan->descs_allocated = 0; in atc_free_chan_resources()
1447 atchan->status = 0; in atc_free_chan_resources()
1469 struct at_dma_chan *atchan; in at_dma_xlate() local
1517 atchan = to_at_dma_chan(chan); in at_dma_xlate()
1518 atchan->per_if = dma_spec->args[0] & 0xff; in at_dma_xlate()
1519 atchan->mem_if = (dma_spec->args[0] >> 16) & 0xff; in at_dma_xlate()
1686 struct at_dma_chan *atchan = &atdma->chan[i]; in at_dma_probe() local
1688 atchan->mem_if = AT_DMA_MEM_IF; in at_dma_probe()
1689 atchan->per_if = AT_DMA_PER_IF; in at_dma_probe()
1690 atchan->chan_common.device = &atdma->dma_common; in at_dma_probe()
1691 dma_cookie_init(&atchan->chan_common); in at_dma_probe()
1692 list_add_tail(&atchan->chan_common.device_node, in at_dma_probe()
1695 atchan->ch_regs = atdma->regs + ch_regs(i); in at_dma_probe()
1696 spin_lock_init(&atchan->lock); in at_dma_probe()
1697 atchan->mask = 1 << i; in at_dma_probe()
1699 INIT_LIST_HEAD(&atchan->active_list); in at_dma_probe()
1700 INIT_LIST_HEAD(&atchan->queue); in at_dma_probe()
1701 INIT_LIST_HEAD(&atchan->free_list); in at_dma_probe()
1703 tasklet_init(&atchan->tasklet, atc_tasklet, in at_dma_probe()
1704 (unsigned long)atchan); in at_dma_probe()
1796 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_remove() local
1801 tasklet_kill(&atchan->tasklet); in at_dma_remove()
1835 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_prepare() local
1837 if (atc_chan_is_enabled(atchan) && !atc_chan_is_cyclic(atchan)) in at_dma_prepare()
1843 static void atc_suspend_cyclic(struct at_dma_chan *atchan) in atc_suspend_cyclic() argument
1845 struct dma_chan *chan = &atchan->chan_common; in atc_suspend_cyclic()
1849 if (!atc_chan_is_paused(atchan)) { in atc_suspend_cyclic()
1857 atchan->save_dscr = channel_readl(atchan, DSCR); in atc_suspend_cyclic()
1859 vdbg_dump_regs(atchan); in atc_suspend_cyclic()
1871 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_suspend_noirq() local
1873 if (atc_chan_is_cyclic(atchan)) in at_dma_suspend_noirq()
1874 atc_suspend_cyclic(atchan); in at_dma_suspend_noirq()
1875 atchan->save_cfg = channel_readl(atchan, CFG); in at_dma_suspend_noirq()
1885 static void atc_resume_cyclic(struct at_dma_chan *atchan) in atc_resume_cyclic() argument
1887 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_resume_cyclic()
1891 channel_writel(atchan, SADDR, 0); in atc_resume_cyclic()
1892 channel_writel(atchan, DADDR, 0); in atc_resume_cyclic()
1893 channel_writel(atchan, CTRLA, 0); in atc_resume_cyclic()
1894 channel_writel(atchan, CTRLB, 0); in atc_resume_cyclic()
1895 channel_writel(atchan, DSCR, atchan->save_dscr); in atc_resume_cyclic()
1896 dma_writel(atdma, CHER, atchan->mask); in atc_resume_cyclic()
1901 vdbg_dump_regs(atchan); in atc_resume_cyclic()
1922 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_resume_noirq() local
1924 channel_writel(atchan, CFG, atchan->save_cfg); in at_dma_resume_noirq()
1925 if (atc_chan_is_cyclic(atchan)) in at_dma_resume_noirq()
1926 atc_resume_cyclic(atchan); in at_dma_resume_noirq()