33 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,
605 static int ipu_enable_channel(struct idmac *idmac, struct idmac_channel *ichan)  in ipu_enable_channel()  argument
608 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_enable_channel()
616 	ichan->active_buffer = 0;  in ipu_enable_channel()
617 	ichan->status = IPU_CHANNEL_ENABLED;  in ipu_enable_channel()
653 static int ipu_init_channel_buffer(struct idmac_channel *ichan,  in ipu_init_channel_buffer()  argument
660 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_init_channel_buffer()
661 	struct idmac *idmac = to_idmac(ichan->dma_chan.device);  in ipu_init_channel_buffer()
701 	ichan->status = IPU_CHANNEL_READY;  in ipu_init_channel_buffer()
732 static void ipu_update_channel_buffer(struct idmac_channel *ichan,  in ipu_update_channel_buffer()  argument
735 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_update_channel_buffer()
745 			ichan->status = IPU_CHANNEL_READY;  in ipu_update_channel_buffer()
756 			ichan->status = IPU_CHANNEL_READY;  in ipu_update_channel_buffer()
776 static int ipu_submit_buffer(struct idmac_channel *ichan,  in ipu_submit_buffer()  argument
779 	unsigned int chan_id = ichan->dma_chan.chan_id;  in ipu_submit_buffer()
780 	struct device *dev = &ichan->dma_chan.dev->device;  in ipu_submit_buffer()
791 	ipu_update_channel_buffer(ichan, buf_idx, sg_dma_address(sg));  in ipu_submit_buffer()
801 static int ipu_submit_channel_buffers(struct idmac_channel *ichan,  in ipu_submit_channel_buffers()  argument
808 		if (!ichan->sg[i]) {  in ipu_submit_channel_buffers()
809 			ichan->sg[i] = sg;  in ipu_submit_channel_buffers()
811 			ret = ipu_submit_buffer(ichan, desc, sg, i);  in ipu_submit_channel_buffers()
825 	struct idmac_channel *ichan = to_idmac_chan(tx->chan);  in idmac_tx_submit()  local
828 	struct device *dev = &ichan->dma_chan.dev->device;  in idmac_tx_submit()
840 	mutex_lock(&ichan->chan_mutex);  in idmac_tx_submit()
844 	if (ichan->status < IPU_CHANNEL_READY) {  in idmac_tx_submit()
845 		struct idmac_video_param *video = &ichan->params.video;  in idmac_tx_submit()
853 		WARN_ON(ichan->sg[0] || ichan->sg[1]);  in idmac_tx_submit()
855 		cookie = ipu_init_channel_buffer(ichan,  in idmac_tx_submit()
872 	spin_lock_irqsave(&ichan->lock, flags);  in idmac_tx_submit()
874 	list_add_tail(&desc->list, &ichan->queue);  in idmac_tx_submit()
876 	ret = ipu_submit_channel_buffers(ichan, desc);  in idmac_tx_submit()
878 	spin_unlock_irqrestore(&ichan->lock, flags);  in idmac_tx_submit()
885 	if (ichan->status < IPU_CHANNEL_ENABLED) {  in idmac_tx_submit()
886 		ret = ipu_enable_channel(idmac, ichan);  in idmac_tx_submit()
897 		spin_lock_irqsave(&ichan->lock, flags);  in idmac_tx_submit()
899 		spin_unlock_irqrestore(&ichan->lock, flags);  in idmac_tx_submit()
901 		ichan->dma_chan.cookie = cookie;  in idmac_tx_submit()
905 	mutex_unlock(&ichan->chan_mutex);  in idmac_tx_submit()
911 static int idmac_desc_alloc(struct idmac_channel *ichan, int n)  in idmac_desc_alloc()  argument
914 	struct idmac *idmac = to_idmac(ichan->dma_chan.device);  in idmac_desc_alloc()
922 	ichan->n_tx_desc = n;  in idmac_desc_alloc()
923 	ichan->desc = desc;  in idmac_desc_alloc()
924 	INIT_LIST_HEAD(&ichan->queue);  in idmac_desc_alloc()
925 	INIT_LIST_HEAD(&ichan->free_list);  in idmac_desc_alloc()
931 		dma_async_tx_descriptor_init(txd, &ichan->dma_chan);  in idmac_desc_alloc()
934 		list_add(&desc->list, &ichan->free_list);  in idmac_desc_alloc()
950 static int ipu_init_channel(struct idmac *idmac, struct idmac_channel *ichan)  in ipu_init_channel()  argument
952 	union ipu_channel_param *params = &ichan->params;  in ipu_init_channel()
954 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_init_channel()
996 	if (n_desc && !ichan->desc)  in ipu_init_channel()
997 		ret = idmac_desc_alloc(ichan, n_desc);  in ipu_init_channel()
1009 static void ipu_uninit_channel(struct idmac *idmac, struct idmac_channel *ichan)  in ipu_uninit_channel()  argument
1011 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_uninit_channel()
1031 	ichan->sec_chan_en = false;  in ipu_uninit_channel()
1058 	ichan->n_tx_desc = 0;  in ipu_uninit_channel()
1059 	vfree(ichan->desc);  in ipu_uninit_channel()
1060 	ichan->desc = NULL;  in ipu_uninit_channel()
1071 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,  in ipu_disable_channel()  argument
1074 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_disable_channel()
1107 		     timeout && !ipu_irq_status(ichan->eof_irq); timeout--)  in ipu_disable_channel()
1125 static struct scatterlist *idmac_sg_next(struct idmac_channel *ichan,  in idmac_sg_next()  argument
1134 	if ((*desc)->list.next == &ichan->queue)  in idmac_sg_next()
1157 	struct idmac_channel *ichan = dev_id;  in idmac_interrupt()  local
1158 	struct device *dev = &ichan->dma_chan.dev->device;  in idmac_interrupt()
1159 	unsigned int chan_id = ichan->dma_chan.chan_id;  in idmac_interrupt()
1171 	dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer);  in idmac_interrupt()
1197 	spin_lock(&ichan->lock);  in idmac_interrupt()
1198 	if (unlikely((ichan->active_buffer && (ready1 >> chan_id) & 1) ||  in idmac_interrupt()
1199 		     (!ichan->active_buffer && (ready0 >> chan_id) & 1)  in idmac_interrupt()
1201 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1205 			ichan->active_buffer, ready0, ready1);  in idmac_interrupt()
1209 	if (unlikely(list_empty(&ichan->queue))) {  in idmac_interrupt()
1210 		ichan->sg[ichan->active_buffer] = NULL;  in idmac_interrupt()
1211 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1215 			ichan->active_buffer, ready0, ready1);  in idmac_interrupt()
1224 	sg = &ichan->sg[ichan->active_buffer];  in idmac_interrupt()
1225 	sgnext = ichan->sg[!ichan->active_buffer];  in idmac_interrupt()
1228 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1232 	desc = list_entry(ichan->queue.next, struct idmac_tx_desc, list);  in idmac_interrupt()
1238 		ichan->active_buffer, curbuf);  in idmac_interrupt()
1241 	sgnew = idmac_sg_next(ichan, &descnew, *sg);  in idmac_interrupt()
1252 		ichan->sg[!ichan->active_buffer] = sgnew;  in idmac_interrupt()
1255 			ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer);  in idmac_interrupt()
1260 			ichan->status = IPU_CHANNEL_READY;  in idmac_interrupt()
1266 	sgnew = idmac_sg_next(ichan, &descnew, sgnew);  in idmac_interrupt()
1280 	    ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) {  in idmac_interrupt()
1284 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1287 		spin_lock(&ichan->lock);  in idmac_interrupt()
1291 	ichan->active_buffer = !ichan->active_buffer;  in idmac_interrupt()
1298 	spin_unlock(&ichan->lock);  in idmac_interrupt()
1312 		struct idmac_channel *ichan = ipu->channel + i;  in ipu_gc_tasklet()  local
1318 		for (j = 0; j < ichan->n_tx_desc; j++) {  in ipu_gc_tasklet()
1319 			desc = ichan->desc + j;  in ipu_gc_tasklet()
1320 			spin_lock_irqsave(&ichan->lock, flags);  in ipu_gc_tasklet()
1322 				list_move(&desc->list, &ichan->free_list);  in ipu_gc_tasklet()
1324 					if (ichan->sg[0] == sg)  in ipu_gc_tasklet()
1325 						ichan->sg[0] = NULL;  in ipu_gc_tasklet()
1326 					else if (ichan->sg[1] == sg)  in ipu_gc_tasklet()
1327 						ichan->sg[1] = NULL;  in ipu_gc_tasklet()
1331 			spin_unlock_irqrestore(&ichan->lock, flags);  in ipu_gc_tasklet()
1342 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_prep_slave_sg()  local
1357 	mutex_lock(&ichan->chan_mutex);  in idmac_prep_slave_sg()
1359 	spin_lock_irqsave(&ichan->lock, flags);  in idmac_prep_slave_sg()
1360 	if (!list_empty(&ichan->free_list)) {  in idmac_prep_slave_sg()
1361 		desc = list_entry(ichan->free_list.next,  in idmac_prep_slave_sg()
1371 	spin_unlock_irqrestore(&ichan->lock, flags);  in idmac_prep_slave_sg()
1373 	mutex_unlock(&ichan->chan_mutex);  in idmac_prep_slave_sg()
1383 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_issue_pending()  local
1390 	ipu_select_buffer(chan->chan_id, ichan->active_buffer);  in idmac_issue_pending()
1403 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_pause()  local
1409 	mutex_lock(&ichan->chan_mutex);  in idmac_pause()
1415 	list_for_each_safe(list, tmp, &ichan->queue)  in idmac_pause()
1418 	ichan->sg[0] = NULL;  in idmac_pause()
1419 	ichan->sg[1] = NULL;  in idmac_pause()
1423 	ichan->status = IPU_CHANNEL_INITIALIZED;  in idmac_pause()
1425 	mutex_unlock(&ichan->chan_mutex);  in idmac_pause()
1432 	struct idmac_channel *ichan = to_idmac_chan(chan);  in __idmac_terminate_all()  local
1438 	ipu_disable_channel(idmac, ichan,  in __idmac_terminate_all()
1439 			    ichan->status >= IPU_CHANNEL_ENABLED);  in __idmac_terminate_all()
1444 	spin_lock_irqsave(&ichan->lock, flags);  in __idmac_terminate_all()
1445 	list_splice_init(&ichan->queue, &ichan->free_list);  in __idmac_terminate_all()
1447 	if (ichan->desc)  in __idmac_terminate_all()
1448 		for (i = 0; i < ichan->n_tx_desc; i++) {  in __idmac_terminate_all()
1449 			struct idmac_tx_desc *desc = ichan->desc + i;  in __idmac_terminate_all()
1452 				list_add(&desc->list, &ichan->free_list);  in __idmac_terminate_all()
1457 	ichan->sg[0] = NULL;  in __idmac_terminate_all()
1458 	ichan->sg[1] = NULL;  in __idmac_terminate_all()
1459 	spin_unlock_irqrestore(&ichan->lock, flags);  in __idmac_terminate_all()
1463 	ichan->status = IPU_CHANNEL_INITIALIZED;  in __idmac_terminate_all()
1470 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_terminate_all()  local
1473 	mutex_lock(&ichan->chan_mutex);  in idmac_terminate_all()
1477 	mutex_unlock(&ichan->chan_mutex);  in idmac_terminate_all()
1485 	struct idmac_channel *ichan = dev_id;  in ic_sof_irq()  local
1487 	       irq, ichan->dma_chan.chan_id);  in ic_sof_irq()
1494 	struct idmac_channel *ichan = dev_id;  in ic_eof_irq()  local
1496 	       irq, ichan->dma_chan.chan_id);  in ic_eof_irq()
1506 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_alloc_chan_resources()  local
1512 	WARN_ON(ichan->status != IPU_CHANNEL_FREE);  in idmac_alloc_chan_resources()
1520 	ichan->eof_irq = ret;  in idmac_alloc_chan_resources()
1526 	ipu_disable_channel(idmac, ichan, true);  in idmac_alloc_chan_resources()
1528 	ret = ipu_init_channel(idmac, ichan);  in idmac_alloc_chan_resources()
1532 	ret = request_irq(ichan->eof_irq, idmac_interrupt, 0,  in idmac_alloc_chan_resources()
1533 			  ichan->eof_name, ichan);  in idmac_alloc_chan_resources()
1541 			ret = request_irq(ic_sof, ic_sof_irq, 0, "IC SOF", ichan);  in idmac_alloc_chan_resources()
1547 			ret = request_irq(ic_eof, ic_eof_irq, 0, "IC EOF", ichan);  in idmac_alloc_chan_resources()
1554 	ichan->status = IPU_CHANNEL_INITIALIZED;  in idmac_alloc_chan_resources()
1557 		chan->chan_id, ichan->eof_irq);  in idmac_alloc_chan_resources()
1562 	ipu_uninit_channel(idmac, ichan);  in idmac_alloc_chan_resources()
1571 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_free_chan_resources()  local
1574 	mutex_lock(&ichan->chan_mutex);  in idmac_free_chan_resources()
1578 	if (ichan->status > IPU_CHANNEL_FREE) {  in idmac_free_chan_resources()
1582 				free_irq(ic_sof, ichan);  in idmac_free_chan_resources()
1587 				free_irq(ic_eof, ichan);  in idmac_free_chan_resources()
1593 		free_irq(ichan->eof_irq, ichan);  in idmac_free_chan_resources()
1597 	ichan->status = IPU_CHANNEL_FREE;  in idmac_free_chan_resources()
1599 	ipu_uninit_channel(idmac, ichan);  in idmac_free_chan_resources()
1601 	mutex_unlock(&ichan->chan_mutex);  in idmac_free_chan_resources()
1635 		struct idmac_channel *ichan = ipu->channel + i;  in ipu_idmac_init()  local
1636 		struct dma_chan *dma_chan = &ichan->dma_chan;  in ipu_idmac_init()
1638 		spin_lock_init(&ichan->lock);  in ipu_idmac_init()
1639 		mutex_init(&ichan->chan_mutex);  in ipu_idmac_init()
1641 		ichan->status		= IPU_CHANNEL_FREE;  in ipu_idmac_init()
1642 		ichan->sec_chan_en	= false;  in ipu_idmac_init()
1643 		snprintf(ichan->eof_name, sizeof(ichan->eof_name), "IDMAC EOF %d", i);  in ipu_idmac_init()
1662 		struct idmac_channel *ichan = ipu->channel + i;  in ipu_idmac_exit()  local
1664 		idmac_terminate_all(&ichan->dma_chan);  in ipu_idmac_exit()