141 	struct dma_chan chan;  member
167 #define to_rcar_dmac_chan(c)	container_of(c, struct rcar_dmac_chan, chan)
290 static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg)  in rcar_dmac_chan_read()  argument
293 		return readw(chan->iomem + reg);  in rcar_dmac_chan_read()
295 		return readl(chan->iomem + reg);  in rcar_dmac_chan_read()
298 static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data)  in rcar_dmac_chan_write()  argument
301 		writew(data, chan->iomem + reg);  in rcar_dmac_chan_write()
303 		writel(data, chan->iomem + reg);  in rcar_dmac_chan_write()
310 static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan)  in rcar_dmac_chan_is_busy()  argument
312 	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);  in rcar_dmac_chan_is_busy()
317 static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan)  in rcar_dmac_chan_start_xfer()  argument
319 	struct rcar_dmac_desc *desc = chan->desc.running;  in rcar_dmac_chan_start_xfer()
322 	WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan));  in rcar_dmac_chan_start_xfer()
324 	if (chan->mid_rid >= 0)  in rcar_dmac_chan_start_xfer()
325 		rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid);  in rcar_dmac_chan_start_xfer()
330 		dev_dbg(chan->chan.device->dev,  in rcar_dmac_chan_start_xfer()
332 			chan->index, desc, desc->nchunks, &desc->hwdescs.dma);  in rcar_dmac_chan_start_xfer()
335 		rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE,  in rcar_dmac_chan_start_xfer()
338 		rcar_dmac_chan_write(chan, RCAR_DMADPBASE,  in rcar_dmac_chan_start_xfer()
341 		rcar_dmac_chan_write(chan, RCAR_DMACHCRB,  in rcar_dmac_chan_start_xfer()
354 		rcar_dmac_chan_write(chan, RCAR_DMADAR,  in rcar_dmac_chan_start_xfer()
361 		rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1));  in rcar_dmac_chan_start_xfer()
387 		dev_dbg(chan->chan.device->dev,  in rcar_dmac_chan_start_xfer()
389 			chan->index, chunk, chunk->size, &chunk->src_addr,  in rcar_dmac_chan_start_xfer()
393 		rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,  in rcar_dmac_chan_start_xfer()
395 		rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,  in rcar_dmac_chan_start_xfer()
398 		rcar_dmac_chan_write(chan, RCAR_DMASAR,  in rcar_dmac_chan_start_xfer()
400 		rcar_dmac_chan_write(chan, RCAR_DMADAR,  in rcar_dmac_chan_start_xfer()
402 		rcar_dmac_chan_write(chan, RCAR_DMATCR,  in rcar_dmac_chan_start_xfer()
408 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr | RCAR_DMACHCR_DE);  in rcar_dmac_chan_start_xfer()
435 	struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan);  in rcar_dmac_tx_submit()  local
440 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_tx_submit()
444 	dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n",  in rcar_dmac_tx_submit()
445 		chan->index, tx->cookie, desc);  in rcar_dmac_tx_submit()
447 	list_add_tail(&desc->node, &chan->desc.pending);  in rcar_dmac_tx_submit()
451 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_tx_submit()
465 static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)  in rcar_dmac_desc_alloc()  argument
478 		dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);  in rcar_dmac_desc_alloc()
485 	spin_lock_irq(&chan->lock);  in rcar_dmac_desc_alloc()
486 	list_splice_tail(&list, &chan->desc.free);  in rcar_dmac_desc_alloc()
487 	list_add_tail(&page->node, &chan->desc.pages);  in rcar_dmac_desc_alloc()
488 	spin_unlock_irq(&chan->lock);  in rcar_dmac_desc_alloc()
505 static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan,  in rcar_dmac_desc_put()  argument
510 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_desc_put()
511 	list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free);  in rcar_dmac_desc_put()
512 	list_add_tail(&desc->node, &chan->desc.free);  in rcar_dmac_desc_put()
513 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_desc_put()
516 static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan)  in rcar_dmac_desc_recycle_acked()  argument
527 	spin_lock_irq(&chan->lock);  in rcar_dmac_desc_recycle_acked()
528 	list_splice_init(&chan->desc.wait, &list);  in rcar_dmac_desc_recycle_acked()
529 	spin_unlock_irq(&chan->lock);  in rcar_dmac_desc_recycle_acked()
534 			rcar_dmac_desc_put(chan, desc);  in rcar_dmac_desc_recycle_acked()
542 	spin_lock_irq(&chan->lock);  in rcar_dmac_desc_recycle_acked()
543 	list_splice(&list, &chan->desc.wait);  in rcar_dmac_desc_recycle_acked()
544 	spin_unlock_irq(&chan->lock);  in rcar_dmac_desc_recycle_acked()
556 static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan)  in rcar_dmac_desc_get()  argument
562 	rcar_dmac_desc_recycle_acked(chan);  in rcar_dmac_desc_get()
564 	spin_lock_irq(&chan->lock);  in rcar_dmac_desc_get()
566 	while (list_empty(&chan->desc.free)) {  in rcar_dmac_desc_get()
573 		spin_unlock_irq(&chan->lock);  in rcar_dmac_desc_get()
574 		ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT);  in rcar_dmac_desc_get()
577 		spin_lock_irq(&chan->lock);  in rcar_dmac_desc_get()
580 	desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node);  in rcar_dmac_desc_get()
583 	spin_unlock_irq(&chan->lock);  in rcar_dmac_desc_get()
593 static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)  in rcar_dmac_xfer_chunk_alloc()  argument
609 	spin_lock_irq(&chan->lock);  in rcar_dmac_xfer_chunk_alloc()
610 	list_splice_tail(&list, &chan->desc.chunks_free);  in rcar_dmac_xfer_chunk_alloc()
611 	list_add_tail(&page->node, &chan->desc.pages);  in rcar_dmac_xfer_chunk_alloc()
612 	spin_unlock_irq(&chan->lock);  in rcar_dmac_xfer_chunk_alloc()
627 rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan)  in rcar_dmac_xfer_chunk_get()  argument
632 	spin_lock_irq(&chan->lock);  in rcar_dmac_xfer_chunk_get()
634 	while (list_empty(&chan->desc.chunks_free)) {  in rcar_dmac_xfer_chunk_get()
641 		spin_unlock_irq(&chan->lock);  in rcar_dmac_xfer_chunk_get()
642 		ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT);  in rcar_dmac_xfer_chunk_get()
645 		spin_lock_irq(&chan->lock);  in rcar_dmac_xfer_chunk_get()
648 	chunk = list_first_entry(&chan->desc.chunks_free,  in rcar_dmac_xfer_chunk_get()
652 	spin_unlock_irq(&chan->lock);  in rcar_dmac_xfer_chunk_get()
657 static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan,  in rcar_dmac_realloc_hwdesc()  argument
672 		dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size,  in rcar_dmac_realloc_hwdesc()
681 	desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size,  in rcar_dmac_realloc_hwdesc()
689 static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan,  in rcar_dmac_fill_hwdesc()  argument
695 	rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc));  in rcar_dmac_fill_hwdesc()
715 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)  in rcar_dmac_chan_halt()  argument
717 	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);  in rcar_dmac_chan_halt()
721 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr);  in rcar_dmac_chan_halt()
724 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan)  in rcar_dmac_chan_reinit()  argument
730 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_chan_reinit()
733 	list_splice_init(&chan->desc.pending, &descs);  in rcar_dmac_chan_reinit()
734 	list_splice_init(&chan->desc.active, &descs);  in rcar_dmac_chan_reinit()
735 	list_splice_init(&chan->desc.done, &descs);  in rcar_dmac_chan_reinit()
736 	list_splice_init(&chan->desc.wait, &descs);  in rcar_dmac_chan_reinit()
738 	chan->desc.running = NULL;  in rcar_dmac_chan_reinit()
740 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_chan_reinit()
744 		rcar_dmac_desc_put(chan, desc);  in rcar_dmac_chan_reinit()
759 		struct rcar_dmac_chan *chan = &dmac->channels[i];  in rcar_dmac_abort()  local
762 		spin_lock(&chan->lock);  in rcar_dmac_abort()
763 		rcar_dmac_chan_halt(chan);  in rcar_dmac_abort()
764 		spin_unlock(&chan->lock);  in rcar_dmac_abort()
766 		rcar_dmac_chan_reinit(chan);  in rcar_dmac_abort()
774 static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan,  in rcar_dmac_chan_configure_desc()  argument
791 		xfer_size = chan->src_xfer_size;  in rcar_dmac_chan_configure_desc()
797 		xfer_size = chan->dst_xfer_size;  in rcar_dmac_chan_configure_desc()
823 rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl,  in rcar_dmac_chan_prep_sg()  argument
837 	desc = rcar_dmac_desc_get(chan);  in rcar_dmac_chan_prep_sg()
847 	rcar_dmac_chan_configure_desc(chan, desc);  in rcar_dmac_chan_prep_sg()
883 			chunk = rcar_dmac_xfer_chunk_get(chan);  in rcar_dmac_chan_prep_sg()
885 				rcar_dmac_desc_put(chan, desc);  in rcar_dmac_chan_prep_sg()
899 			dev_dbg(chan->chan.device->dev,  in rcar_dmac_chan_prep_sg()
901 				chan->index, chunk, desc, i, sg, size, len,  in rcar_dmac_chan_prep_sg()
930 		if (rcar_dmac_fill_hwdesc(chan, desc) < 0)  in rcar_dmac_chan_prep_sg()
941 static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan)  in rcar_dmac_alloc_chan_resources()  argument
943 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_alloc_chan_resources()
958 	return pm_runtime_get_sync(chan->device->dev);  in rcar_dmac_alloc_chan_resources()
961 static void rcar_dmac_free_chan_resources(struct dma_chan *chan)  in rcar_dmac_free_chan_resources()  argument
963 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_free_chan_resources()
964 	struct rcar_dmac *dmac = to_rcar_dmac(chan->device);  in rcar_dmac_free_chan_resources()
996 	pm_runtime_put(chan->device->dev);  in rcar_dmac_free_chan_resources()
1000 rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,  in rcar_dmac_prep_dma_memcpy()  argument
1003 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_prep_dma_memcpy()
1020 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,  in rcar_dmac_prep_slave_sg()  argument
1024 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_prep_slave_sg()
1029 		dev_warn(chan->device->dev,  in rcar_dmac_prep_slave_sg()
1044 rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,  in rcar_dmac_prep_dma_cyclic()  argument
1048 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_prep_dma_cyclic()
1057 		dev_warn(chan->device->dev,  in rcar_dmac_prep_dma_cyclic()
1065 		dev_err(chan->device->dev,  in rcar_dmac_prep_dma_cyclic()
1099 static int rcar_dmac_device_config(struct dma_chan *chan,  in rcar_dmac_device_config()  argument
1102 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_device_config()
1116 static int rcar_dmac_chan_terminate_all(struct dma_chan *chan)  in rcar_dmac_chan_terminate_all()  argument
1118 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_chan_terminate_all()
1135 static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan,  in rcar_dmac_chan_get_residue()  argument
1138 	struct rcar_dmac_desc *desc = chan->desc.running;  in rcar_dmac_chan_get_residue()
1162 		dptr = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &  in rcar_dmac_chan_get_residue()
1178 	residue += rcar_dmac_chan_read(chan, RCAR_DMATCR) << desc->xfer_shift;  in rcar_dmac_chan_get_residue()
1183 static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan,  in rcar_dmac_tx_status()  argument
1187 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_tx_status()
1192 	status = dma_cookie_status(chan, cookie, txstate);  in rcar_dmac_tx_status()
1205 static void rcar_dmac_issue_pending(struct dma_chan *chan)  in rcar_dmac_issue_pending()  argument
1207 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_issue_pending()
1240 static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan)  in rcar_dmac_isr_desc_stage_end()  argument
1242 	struct rcar_dmac_desc *desc = chan->desc.running;  in rcar_dmac_isr_desc_stage_end()
1255 	stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &  in rcar_dmac_isr_desc_stage_end()
1257 	rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage));  in rcar_dmac_isr_desc_stage_end()
1262 static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan)  in rcar_dmac_isr_transfer_end()  argument
1264 	struct rcar_dmac_desc *desc = chan->desc.running;  in rcar_dmac_isr_transfer_end()
1308 	list_move_tail(&desc->node, &chan->desc.done);  in rcar_dmac_isr_transfer_end()
1311 	if (!list_empty(&chan->desc.active))  in rcar_dmac_isr_transfer_end()
1312 		chan->desc.running = list_first_entry(&chan->desc.active,  in rcar_dmac_isr_transfer_end()
1316 		chan->desc.running = NULL;  in rcar_dmac_isr_transfer_end()
1319 	if (chan->desc.running)  in rcar_dmac_isr_transfer_end()
1320 		rcar_dmac_chan_start_xfer(chan);  in rcar_dmac_isr_transfer_end()
1328 	struct rcar_dmac_chan *chan = dev;  in rcar_dmac_isr_channel()  local
1332 	spin_lock(&chan->lock);  in rcar_dmac_isr_channel()
1334 	chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);  in rcar_dmac_isr_channel()
1337 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask);  in rcar_dmac_isr_channel()
1340 		ret |= rcar_dmac_isr_desc_stage_end(chan);  in rcar_dmac_isr_channel()
1343 		ret |= rcar_dmac_isr_transfer_end(chan);  in rcar_dmac_isr_channel()
1345 	spin_unlock(&chan->lock);  in rcar_dmac_isr_channel()
1352 	struct rcar_dmac_chan *chan = dev;  in rcar_dmac_isr_channel_thread()  local
1355 	spin_lock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1358 	if (chan->desc.running && chan->desc.running->cyclic) {  in rcar_dmac_isr_channel_thread()
1362 		desc = chan->desc.running;  in rcar_dmac_isr_channel_thread()
1367 			spin_unlock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1369 			spin_lock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1377 	while (!list_empty(&chan->desc.done)) {  in rcar_dmac_isr_channel_thread()
1378 		desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc,  in rcar_dmac_isr_channel_thread()
1384 			spin_unlock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1391 			spin_lock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1394 		list_add_tail(&desc->node, &chan->desc.wait);  in rcar_dmac_isr_channel_thread()
1397 	spin_unlock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1400 	rcar_dmac_desc_recycle_acked(chan);  in rcar_dmac_isr_channel_thread()
1427 static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg)  in rcar_dmac_chan_filter()  argument
1429 	struct rcar_dmac *dmac = to_rcar_dmac(chan->device);  in rcar_dmac_chan_filter()
1439 	if (chan->device->device_config != rcar_dmac_device_config ||  in rcar_dmac_chan_filter()
1440 	    dma_spec->np != chan->device->dev->of_node)  in rcar_dmac_chan_filter()
1450 	struct dma_chan *chan;  in rcar_dmac_of_xlate()  local
1460 	chan = dma_request_channel(mask, rcar_dmac_chan_filter, dma_spec);  in rcar_dmac_of_xlate()
1461 	if (!chan)  in rcar_dmac_of_xlate()
1464 	rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_of_xlate()
1467 	return chan;  in rcar_dmac_of_xlate()
1519 	struct dma_chan *chan = &rchan->chan;  in rcar_dmac_chan_probe()  local
1562 	chan->device = &dmac->engine;  in rcar_dmac_chan_probe()
1563 	dma_cookie_init(chan);  in rcar_dmac_chan_probe()
1565 	list_add_tail(&chan->device_node, &dmac->engine.channels);  in rcar_dmac_chan_probe()