215 static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac)  in chan2dev()  argument
217 	return &edmac->chan.dev->device;  in chan2dev()
236 static void ep93xx_dma_set_active(struct ep93xx_dma_chan *edmac,  in ep93xx_dma_set_active()  argument
239 	BUG_ON(!list_empty(&edmac->active));  in ep93xx_dma_set_active()
241 	list_add_tail(&desc->node, &edmac->active);  in ep93xx_dma_set_active()
257 		list_move_tail(&d->node, &edmac->active);  in ep93xx_dma_set_active()
263 ep93xx_dma_get_active(struct ep93xx_dma_chan *edmac)  in ep93xx_dma_get_active()  argument
265 	if (list_empty(&edmac->active))  in ep93xx_dma_get_active()
268 	return list_first_entry(&edmac->active, struct ep93xx_dma_desc, node);  in ep93xx_dma_get_active()
283 static bool ep93xx_dma_advance_active(struct ep93xx_dma_chan *edmac)  in ep93xx_dma_advance_active()  argument
287 	list_rotate_left(&edmac->active);  in ep93xx_dma_advance_active()
289 	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))  in ep93xx_dma_advance_active()
292 	desc = ep93xx_dma_get_active(edmac);  in ep93xx_dma_advance_active()
307 static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control)  in m2p_set_control()  argument
309 	writel(control, edmac->regs + M2P_CONTROL);  in m2p_set_control()
314 	readl(edmac->regs + M2P_CONTROL);  in m2p_set_control()
317 static int m2p_hw_setup(struct ep93xx_dma_chan *edmac)  in m2p_hw_setup()  argument
319 	struct ep93xx_dma_data *data = edmac->chan.private;  in m2p_hw_setup()
322 	writel(data->port & 0xf, edmac->regs + M2P_PPALLOC);  in m2p_hw_setup()
326 	m2p_set_control(edmac, control);  in m2p_hw_setup()
331 static inline u32 m2p_channel_state(struct ep93xx_dma_chan *edmac)  in m2p_channel_state()  argument
333 	return (readl(edmac->regs + M2P_STATUS) >> 4) & 0x3;  in m2p_channel_state()
336 static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac)  in m2p_hw_shutdown()  argument
340 	control = readl(edmac->regs + M2P_CONTROL);  in m2p_hw_shutdown()
342 	m2p_set_control(edmac, control);  in m2p_hw_shutdown()
344 	while (m2p_channel_state(edmac) >= M2P_STATE_ON)  in m2p_hw_shutdown()
347 	m2p_set_control(edmac, 0);  in m2p_hw_shutdown()
349 	while (m2p_channel_state(edmac) == M2P_STATE_STALL)  in m2p_hw_shutdown()
353 static void m2p_fill_desc(struct ep93xx_dma_chan *edmac)  in m2p_fill_desc()  argument
358 	desc = ep93xx_dma_get_active(edmac);  in m2p_fill_desc()
360 		dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n");  in m2p_fill_desc()
364 	if (ep93xx_dma_chan_direction(&edmac->chan) == DMA_MEM_TO_DEV)  in m2p_fill_desc()
369 	if (edmac->buffer == 0) {  in m2p_fill_desc()
370 		writel(desc->size, edmac->regs + M2P_MAXCNT0);  in m2p_fill_desc()
371 		writel(bus_addr, edmac->regs + M2P_BASE0);  in m2p_fill_desc()
373 		writel(desc->size, edmac->regs + M2P_MAXCNT1);  in m2p_fill_desc()
374 		writel(bus_addr, edmac->regs + M2P_BASE1);  in m2p_fill_desc()
377 	edmac->buffer ^= 1;  in m2p_fill_desc()
380 static void m2p_hw_submit(struct ep93xx_dma_chan *edmac)  in m2p_hw_submit()  argument
382 	u32 control = readl(edmac->regs + M2P_CONTROL);  in m2p_hw_submit()
384 	m2p_fill_desc(edmac);  in m2p_hw_submit()
387 	if (ep93xx_dma_advance_active(edmac)) {  in m2p_hw_submit()
388 		m2p_fill_desc(edmac);  in m2p_hw_submit()
392 	m2p_set_control(edmac, control);  in m2p_hw_submit()
395 static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac)  in m2p_hw_interrupt()  argument
397 	u32 irq_status = readl(edmac->regs + M2P_INTERRUPT);  in m2p_hw_interrupt()
401 		struct ep93xx_dma_desc *desc = ep93xx_dma_get_active(edmac);  in m2p_hw_interrupt()
404 		writel(1, edmac->regs + M2P_INTERRUPT);  in m2p_hw_interrupt()
414 		dev_err(chan2dev(edmac),  in m2p_hw_interrupt()
427 		control = readl(edmac->regs + M2P_CONTROL);  in m2p_hw_interrupt()
429 		m2p_set_control(edmac, control);  in m2p_hw_interrupt()
434 		if (ep93xx_dma_advance_active(edmac))  in m2p_hw_interrupt()
435 			m2p_fill_desc(edmac);  in m2p_hw_interrupt()
447 static int m2m_hw_setup(struct ep93xx_dma_chan *edmac)  in m2m_hw_setup()  argument
449 	const struct ep93xx_dma_data *data = edmac->chan.private;  in m2m_hw_setup()
454 		writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_setup()
504 	writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_setup()
508 static void m2m_hw_shutdown(struct ep93xx_dma_chan *edmac)  in m2m_hw_shutdown()  argument
511 	writel(0, edmac->regs + M2M_CONTROL);  in m2m_hw_shutdown()
514 static void m2m_fill_desc(struct ep93xx_dma_chan *edmac)  in m2m_fill_desc()  argument
518 	desc = ep93xx_dma_get_active(edmac);  in m2m_fill_desc()
520 		dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n");  in m2m_fill_desc()
524 	if (edmac->buffer == 0) {  in m2m_fill_desc()
525 		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE0);  in m2m_fill_desc()
526 		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE0);  in m2m_fill_desc()
527 		writel(desc->size, edmac->regs + M2M_BCR0);  in m2m_fill_desc()
529 		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE1);  in m2m_fill_desc()
530 		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE1);  in m2m_fill_desc()
531 		writel(desc->size, edmac->regs + M2M_BCR1);  in m2m_fill_desc()
534 	edmac->buffer ^= 1;  in m2m_fill_desc()
537 static void m2m_hw_submit(struct ep93xx_dma_chan *edmac)  in m2m_hw_submit()  argument
539 	struct ep93xx_dma_data *data = edmac->chan.private;  in m2m_hw_submit()
540 	u32 control = readl(edmac->regs + M2M_CONTROL);  in m2m_hw_submit()
548 	control |= edmac->runtime_ctrl;  in m2m_hw_submit()
550 	m2m_fill_desc(edmac);  in m2m_hw_submit()
553 	if (ep93xx_dma_advance_active(edmac)) {  in m2m_hw_submit()
554 		m2m_fill_desc(edmac);  in m2m_hw_submit()
563 	writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_submit()
571 		writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_submit()
585 static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac)  in m2m_hw_interrupt()  argument
587 	u32 status = readl(edmac->regs + M2M_STATUS);  in m2m_hw_interrupt()
596 	if (!(readl(edmac->regs + M2M_INTERRUPT) & M2M_INTERRUPT_MASK))  in m2m_hw_interrupt()
601 		writel(0, edmac->regs + M2M_INTERRUPT);  in m2m_hw_interrupt()
608 	desc = ep93xx_dma_get_active(edmac);  in m2m_hw_interrupt()
625 		if (ep93xx_dma_advance_active(edmac)) {  in m2m_hw_interrupt()
626 			m2m_fill_desc(edmac);  in m2m_hw_interrupt()
627 			if (done && !edmac->chan.private) {  in m2m_hw_interrupt()
629 				control = readl(edmac->regs + M2M_CONTROL);  in m2m_hw_interrupt()
631 				writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_interrupt()
647 		control = readl(edmac->regs + M2M_CONTROL);  in m2m_hw_interrupt()
650 		writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_interrupt()
665 ep93xx_dma_desc_get(struct ep93xx_dma_chan *edmac)  in ep93xx_dma_desc_get()  argument
671 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_desc_get()
672 	list_for_each_entry_safe(desc, _desc, &edmac->free_list, node) {  in ep93xx_dma_desc_get()
689 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_desc_get()
693 static void ep93xx_dma_desc_put(struct ep93xx_dma_chan *edmac,  in ep93xx_dma_desc_put()  argument
699 		spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_desc_put()
700 		list_splice_init(&desc->tx_list, &edmac->free_list);  in ep93xx_dma_desc_put()
701 		list_add(&desc->node, &edmac->free_list);  in ep93xx_dma_desc_put()
702 		spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_desc_put()
714 static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac)  in ep93xx_dma_advance_work()  argument
719 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_advance_work()
720 	if (!list_empty(&edmac->active) || list_empty(&edmac->queue)) {  in ep93xx_dma_advance_work()
721 		spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_advance_work()
726 	new = list_first_entry(&edmac->queue, struct ep93xx_dma_desc, node);  in ep93xx_dma_advance_work()
729 	ep93xx_dma_set_active(edmac, new);  in ep93xx_dma_advance_work()
732 	edmac->edma->hw_submit(edmac);  in ep93xx_dma_advance_work()
733 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_advance_work()
738 	struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data;  in ep93xx_dma_tasklet()  local
744 	spin_lock_irq(&edmac->lock);  in ep93xx_dma_tasklet()
750 	desc = ep93xx_dma_get_active(edmac);  in ep93xx_dma_tasklet()
754 			if (!test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))  in ep93xx_dma_tasklet()
756 			list_splice_init(&edmac->active, &list);  in ep93xx_dma_tasklet()
761 	spin_unlock_irq(&edmac->lock);  in ep93xx_dma_tasklet()
764 	ep93xx_dma_advance_work(edmac);  in ep93xx_dma_tasklet()
769 		ep93xx_dma_desc_put(edmac, desc);  in ep93xx_dma_tasklet()
778 	struct ep93xx_dma_chan *edmac = dev_id;  in ep93xx_dma_interrupt()  local
782 	spin_lock(&edmac->lock);  in ep93xx_dma_interrupt()
784 	desc = ep93xx_dma_get_active(edmac);  in ep93xx_dma_interrupt()
786 		dev_warn(chan2dev(edmac),  in ep93xx_dma_interrupt()
788 		spin_unlock(&edmac->lock);  in ep93xx_dma_interrupt()
792 	switch (edmac->edma->hw_interrupt(edmac)) {  in ep93xx_dma_interrupt()
795 		tasklet_schedule(&edmac->tasklet);  in ep93xx_dma_interrupt()
799 		if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))  in ep93xx_dma_interrupt()
800 			tasklet_schedule(&edmac->tasklet);  in ep93xx_dma_interrupt()
804 		dev_warn(chan2dev(edmac), "unknown interrupt!\n");  in ep93xx_dma_interrupt()
809 	spin_unlock(&edmac->lock);  in ep93xx_dma_interrupt()
823 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(tx->chan);  in ep93xx_dma_tx_submit()  local
828 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_tx_submit()
838 	if (list_empty(&edmac->active)) {  in ep93xx_dma_tx_submit()
839 		ep93xx_dma_set_active(edmac, desc);  in ep93xx_dma_tx_submit()
840 		edmac->edma->hw_submit(edmac);  in ep93xx_dma_tx_submit()
842 		list_add_tail(&desc->node, &edmac->queue);  in ep93xx_dma_tx_submit()
845 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_tx_submit()
859 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_alloc_chan_resources()  local
865 	if (!edmac->edma->m2m) {  in ep93xx_dma_alloc_chan_resources()
890 	ret = clk_enable(edmac->clk);  in ep93xx_dma_alloc_chan_resources()
894 	ret = request_irq(edmac->irq, ep93xx_dma_interrupt, 0, name, edmac);  in ep93xx_dma_alloc_chan_resources()
898 	spin_lock_irq(&edmac->lock);  in ep93xx_dma_alloc_chan_resources()
899 	dma_cookie_init(&edmac->chan);  in ep93xx_dma_alloc_chan_resources()
900 	ret = edmac->edma->hw_setup(edmac);  in ep93xx_dma_alloc_chan_resources()
901 	spin_unlock_irq(&edmac->lock);  in ep93xx_dma_alloc_chan_resources()
911 			dev_warn(chan2dev(edmac), "not enough descriptors\n");  in ep93xx_dma_alloc_chan_resources()
921 		ep93xx_dma_desc_put(edmac, desc);  in ep93xx_dma_alloc_chan_resources()
927 	free_irq(edmac->irq, edmac);  in ep93xx_dma_alloc_chan_resources()
929 	clk_disable(edmac->clk);  in ep93xx_dma_alloc_chan_resources()
943 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_free_chan_resources()  local
948 	BUG_ON(!list_empty(&edmac->active));  in ep93xx_dma_free_chan_resources()
949 	BUG_ON(!list_empty(&edmac->queue));  in ep93xx_dma_free_chan_resources()
951 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_free_chan_resources()
952 	edmac->edma->hw_shutdown(edmac);  in ep93xx_dma_free_chan_resources()
953 	edmac->runtime_addr = 0;  in ep93xx_dma_free_chan_resources()
954 	edmac->runtime_ctrl = 0;  in ep93xx_dma_free_chan_resources()
955 	edmac->buffer = 0;  in ep93xx_dma_free_chan_resources()
956 	list_splice_init(&edmac->free_list, &list);  in ep93xx_dma_free_chan_resources()
957 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_free_chan_resources()
962 	clk_disable(edmac->clk);  in ep93xx_dma_free_chan_resources()
963 	free_irq(edmac->irq, edmac);  in ep93xx_dma_free_chan_resources()
980 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_prep_dma_memcpy()  local
986 		desc = ep93xx_dma_desc_get(edmac);  in ep93xx_dma_prep_dma_memcpy()
988 			dev_warn(chan2dev(edmac), "couln't get descriptor\n");  in ep93xx_dma_prep_dma_memcpy()
1009 	ep93xx_dma_desc_put(edmac, first);  in ep93xx_dma_prep_dma_memcpy()
1029 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_prep_slave_sg()  local
1034 	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {  in ep93xx_dma_prep_slave_sg()
1035 		dev_warn(chan2dev(edmac),  in ep93xx_dma_prep_slave_sg()
1040 	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {  in ep93xx_dma_prep_slave_sg()
1041 		dev_warn(chan2dev(edmac),  in ep93xx_dma_prep_slave_sg()
1051 			dev_warn(chan2dev(edmac), "too big transfer size %d\n",  in ep93xx_dma_prep_slave_sg()
1056 		desc = ep93xx_dma_desc_get(edmac);  in ep93xx_dma_prep_slave_sg()
1058 			dev_warn(chan2dev(edmac), "couln't get descriptor\n");  in ep93xx_dma_prep_slave_sg()
1064 			desc->dst_addr = edmac->runtime_addr;  in ep93xx_dma_prep_slave_sg()
1066 			desc->src_addr = edmac->runtime_addr;  in ep93xx_dma_prep_slave_sg()
1083 	ep93xx_dma_desc_put(edmac, first);  in ep93xx_dma_prep_slave_sg()
1109 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_prep_dma_cyclic()  local
1113 	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {  in ep93xx_dma_prep_dma_cyclic()
1114 		dev_warn(chan2dev(edmac),  in ep93xx_dma_prep_dma_cyclic()
1119 	if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {  in ep93xx_dma_prep_dma_cyclic()
1120 		dev_warn(chan2dev(edmac),  in ep93xx_dma_prep_dma_cyclic()
1126 		dev_warn(chan2dev(edmac), "too big period length %d\n",  in ep93xx_dma_prep_dma_cyclic()
1134 		desc = ep93xx_dma_desc_get(edmac);  in ep93xx_dma_prep_dma_cyclic()
1136 			dev_warn(chan2dev(edmac), "couln't get descriptor\n");  in ep93xx_dma_prep_dma_cyclic()
1142 			desc->dst_addr = edmac->runtime_addr;  in ep93xx_dma_prep_dma_cyclic()
1144 			desc->src_addr = edmac->runtime_addr;  in ep93xx_dma_prep_dma_cyclic()
1161 	ep93xx_dma_desc_put(edmac, first);  in ep93xx_dma_prep_dma_cyclic()
1174 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_terminate_all()  local
1179 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_terminate_all()
1181 	edmac->edma->hw_shutdown(edmac);  in ep93xx_dma_terminate_all()
1182 	clear_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags);  in ep93xx_dma_terminate_all()
1183 	list_splice_init(&edmac->active, &list);  in ep93xx_dma_terminate_all()
1184 	list_splice_init(&edmac->queue, &list);  in ep93xx_dma_terminate_all()
1189 	edmac->edma->hw_setup(edmac);  in ep93xx_dma_terminate_all()
1190 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_terminate_all()
1193 		ep93xx_dma_desc_put(edmac, desc);  in ep93xx_dma_terminate_all()
1201 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_slave_config()  local
1206 	if (!edmac->edma->m2m)  in ep93xx_dma_slave_config()
1238 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_slave_config()
1239 	edmac->runtime_addr = addr;  in ep93xx_dma_slave_config()
1240 	edmac->runtime_ctrl = ctrl;  in ep93xx_dma_slave_config()
1241 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_slave_config()
1293 		struct ep93xx_dma_chan *edmac = &edma->channels[i];  in ep93xx_dma_probe()  local
1295 		edmac->chan.device = dma_dev;  in ep93xx_dma_probe()
1296 		edmac->regs = cdata->base;  in ep93xx_dma_probe()
1297 		edmac->irq = cdata->irq;  in ep93xx_dma_probe()
1298 		edmac->edma = edma;  in ep93xx_dma_probe()
1300 		edmac->clk = clk_get(NULL, cdata->name);  in ep93xx_dma_probe()
1301 		if (IS_ERR(edmac->clk)) {  in ep93xx_dma_probe()
1307 		spin_lock_init(&edmac->lock);  in ep93xx_dma_probe()
1308 		INIT_LIST_HEAD(&edmac->active);  in ep93xx_dma_probe()
1309 		INIT_LIST_HEAD(&edmac->queue);  in ep93xx_dma_probe()
1310 		INIT_LIST_HEAD(&edmac->free_list);  in ep93xx_dma_probe()
1311 		tasklet_init(&edmac->tasklet, ep93xx_dma_tasklet,  in ep93xx_dma_probe()
1312 			     (unsigned long)edmac);  in ep93xx_dma_probe()
1314 		list_add_tail(&edmac->chan.device_node,  in ep93xx_dma_probe()
1354 			struct ep93xx_dma_chan *edmac = &edma->channels[i];  in ep93xx_dma_probe()  local
1355 			if (!IS_ERR_OR_NULL(edmac->clk))  in ep93xx_dma_probe()
1356 				clk_put(edmac->clk);  in ep93xx_dma_probe()