Lines Matching refs:chan
235 #define chan_dbg(chan, fmt, arg...) \ argument
236 dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)
237 #define chan_err(chan, fmt, arg...) \ argument
238 dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
351 struct xgene_dma_chan chan[XGENE_DMA_MAX_CHANNEL]; member
457 static void xgene_dma_prep_cpy_desc(struct xgene_dma_chan *chan, in xgene_dma_prep_cpy_desc() argument
467 xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num); in xgene_dma_prep_cpy_desc()
507 static void xgene_dma_prep_xor_desc(struct xgene_dma_chan *chan, in xgene_dma_prep_xor_desc() argument
521 xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num); in xgene_dma_prep_xor_desc()
557 struct xgene_dma_chan *chan; in xgene_dma_tx_submit() local
563 chan = to_dma_chan(tx->chan); in xgene_dma_tx_submit()
566 spin_lock_bh(&chan->lock); in xgene_dma_tx_submit()
571 list_splice_tail_init(&desc->tx_list, &chan->ld_pending); in xgene_dma_tx_submit()
573 spin_unlock_bh(&chan->lock); in xgene_dma_tx_submit()
578 static void xgene_dma_clean_descriptor(struct xgene_dma_chan *chan, in xgene_dma_clean_descriptor() argument
582 chan_dbg(chan, "LD %p free\n", desc); in xgene_dma_clean_descriptor()
583 dma_pool_free(chan->desc_pool, desc, desc->tx.phys); in xgene_dma_clean_descriptor()
587 struct xgene_dma_chan *chan) in xgene_dma_alloc_descriptor() argument
592 desc = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &phys); in xgene_dma_alloc_descriptor()
594 chan_err(chan, "Failed to allocate LDs\n"); in xgene_dma_alloc_descriptor()
603 dma_async_tx_descriptor_init(&desc->tx, &chan->dma_chan); in xgene_dma_alloc_descriptor()
605 chan_dbg(chan, "LD %p allocated\n", desc); in xgene_dma_alloc_descriptor()
617 static void xgene_dma_clean_completed_descriptor(struct xgene_dma_chan *chan) in xgene_dma_clean_completed_descriptor() argument
622 list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node) { in xgene_dma_clean_completed_descriptor()
624 xgene_dma_clean_descriptor(chan, desc); in xgene_dma_clean_completed_descriptor()
636 static void xgene_dma_run_tx_complete_actions(struct xgene_dma_chan *chan, in xgene_dma_run_tx_complete_actions() argument
672 static void xgene_dma_clean_running_descriptor(struct xgene_dma_chan *chan, in xgene_dma_clean_running_descriptor() argument
687 list_add_tail(&desc->node, &chan->ld_completed); in xgene_dma_clean_running_descriptor()
691 chan_dbg(chan, "LD %p free\n", desc); in xgene_dma_clean_running_descriptor()
692 dma_pool_free(chan->desc_pool, desc, desc->tx.phys); in xgene_dma_clean_running_descriptor()
743 static void xgene_chan_xfer_ld_pending(struct xgene_dma_chan *chan) in xgene_chan_xfer_ld_pending() argument
752 if (list_empty(&chan->ld_pending)) { in xgene_chan_xfer_ld_pending()
753 chan_dbg(chan, "No pending LDs\n"); in xgene_chan_xfer_ld_pending()
761 list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_pending, node) { in xgene_chan_xfer_ld_pending()
768 if (chan->pending >= chan->max_outstanding) in xgene_chan_xfer_ld_pending()
771 ret = xgene_chan_xfer_request(&chan->tx_ring, desc_sw); in xgene_chan_xfer_ld_pending()
779 list_move_tail(&desc_sw->node, &chan->ld_running); in xgene_chan_xfer_ld_pending()
782 chan->pending++; in xgene_chan_xfer_ld_pending()
795 static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan) in xgene_dma_cleanup_descriptors() argument
797 struct xgene_dma_ring *ring = &chan->rx_ring; in xgene_dma_cleanup_descriptors()
803 xgene_dma_clean_completed_descriptor(chan); in xgene_dma_cleanup_descriptors()
806 list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_running, node) { in xgene_dma_cleanup_descriptors()
825 chan_err(chan, "%s\n", xgene_dma_desc_err[status]); in xgene_dma_cleanup_descriptors()
847 xgene_dma_run_tx_complete_actions(chan, desc_sw); in xgene_dma_cleanup_descriptors()
849 xgene_dma_clean_running_descriptor(chan, desc_sw); in xgene_dma_cleanup_descriptors()
855 chan->pending--; in xgene_dma_cleanup_descriptors()
863 xgene_chan_xfer_ld_pending(chan); in xgene_dma_cleanup_descriptors()
868 struct xgene_dma_chan *chan = to_dma_chan(dchan); in xgene_dma_alloc_chan_resources() local
871 if (chan->desc_pool) in xgene_dma_alloc_chan_resources()
874 chan->desc_pool = dma_pool_create(chan->name, chan->dev, in xgene_dma_alloc_chan_resources()
877 if (!chan->desc_pool) { in xgene_dma_alloc_chan_resources()
878 chan_err(chan, "Failed to allocate descriptor pool\n"); in xgene_dma_alloc_chan_resources()
882 chan_dbg(chan, "Allocate descripto pool\n"); in xgene_dma_alloc_chan_resources()
894 static void xgene_dma_free_desc_list(struct xgene_dma_chan *chan, in xgene_dma_free_desc_list() argument
900 xgene_dma_clean_descriptor(chan, desc); in xgene_dma_free_desc_list()
903 static void xgene_dma_free_tx_desc_list(struct xgene_dma_chan *chan, in xgene_dma_free_tx_desc_list() argument
909 xgene_dma_clean_descriptor(chan, desc); in xgene_dma_free_tx_desc_list()
914 struct xgene_dma_chan *chan = to_dma_chan(dchan); in xgene_dma_free_chan_resources() local
916 chan_dbg(chan, "Free all resources\n"); in xgene_dma_free_chan_resources()
918 if (!chan->desc_pool) in xgene_dma_free_chan_resources()
921 spin_lock_bh(&chan->lock); in xgene_dma_free_chan_resources()
924 xgene_dma_cleanup_descriptors(chan); in xgene_dma_free_chan_resources()
927 xgene_dma_free_desc_list(chan, &chan->ld_pending); in xgene_dma_free_chan_resources()
928 xgene_dma_free_desc_list(chan, &chan->ld_running); in xgene_dma_free_chan_resources()
929 xgene_dma_free_desc_list(chan, &chan->ld_completed); in xgene_dma_free_chan_resources()
931 spin_unlock_bh(&chan->lock); in xgene_dma_free_chan_resources()
934 dma_pool_destroy(chan->desc_pool); in xgene_dma_free_chan_resources()
935 chan->desc_pool = NULL; in xgene_dma_free_chan_resources()
943 struct xgene_dma_chan *chan; in xgene_dma_prep_memcpy() local
949 chan = to_dma_chan(dchan); in xgene_dma_prep_memcpy()
953 new = xgene_dma_alloc_descriptor(chan); in xgene_dma_prep_memcpy()
961 xgene_dma_prep_cpy_desc(chan, new, dst, src, copy); in xgene_dma_prep_memcpy()
988 xgene_dma_free_tx_desc_list(chan, &first->tx_list); in xgene_dma_prep_memcpy()
998 struct xgene_dma_chan *chan; in xgene_dma_prep_sg() local
1012 chan = to_dma_chan(dchan); in xgene_dma_prep_sg()
1032 new = xgene_dma_alloc_descriptor(chan); in xgene_dma_prep_sg()
1037 xgene_dma_prep_cpy_desc(chan, new, dst, src, len); in xgene_dma_prep_sg()
1096 xgene_dma_free_tx_desc_list(chan, &first->tx_list); in xgene_dma_prep_sg()
1105 struct xgene_dma_chan *chan; in xgene_dma_prep_xor() local
1112 chan = to_dma_chan(dchan); in xgene_dma_prep_xor()
1116 new = xgene_dma_alloc_descriptor(chan); in xgene_dma_prep_xor()
1121 xgene_dma_prep_xor_desc(chan, new, &dst, src, in xgene_dma_prep_xor()
1144 xgene_dma_free_tx_desc_list(chan, &first->tx_list); in xgene_dma_prep_xor()
1153 struct xgene_dma_chan *chan; in xgene_dma_prep_pq() local
1161 chan = to_dma_chan(dchan); in xgene_dma_prep_pq()
1178 new = xgene_dma_alloc_descriptor(chan); in xgene_dma_prep_pq()
1196 xgene_dma_prep_xor_desc(chan, new, &dst[0], src, in xgene_dma_prep_pq()
1206 xgene_dma_prep_xor_desc(chan, new, &dst[1], _src, in xgene_dma_prep_pq()
1221 xgene_dma_free_tx_desc_list(chan, &first->tx_list); in xgene_dma_prep_pq()
1227 struct xgene_dma_chan *chan = to_dma_chan(dchan); in xgene_dma_issue_pending() local
1229 spin_lock_bh(&chan->lock); in xgene_dma_issue_pending()
1230 xgene_chan_xfer_ld_pending(chan); in xgene_dma_issue_pending()
1231 spin_unlock_bh(&chan->lock); in xgene_dma_issue_pending()
1243 struct xgene_dma_chan *chan = (struct xgene_dma_chan *)data; in xgene_dma_tasklet_cb() local
1245 spin_lock_bh(&chan->lock); in xgene_dma_tasklet_cb()
1248 xgene_dma_cleanup_descriptors(chan); in xgene_dma_tasklet_cb()
1251 enable_irq(chan->rx_irq); in xgene_dma_tasklet_cb()
1253 spin_unlock_bh(&chan->lock); in xgene_dma_tasklet_cb()
1258 struct xgene_dma_chan *chan = (struct xgene_dma_chan *)id; in xgene_dma_chan_ring_isr() local
1260 BUG_ON(!chan); in xgene_dma_chan_ring_isr()
1266 disable_irq_nosync(chan->rx_irq); in xgene_dma_chan_ring_isr()
1273 tasklet_schedule(&chan->tasklet); in xgene_dma_chan_ring_isr()
1401 static int xgene_dma_get_ring_size(struct xgene_dma_chan *chan, in xgene_dma_get_ring_size() argument
1423 chan_err(chan, "Unsupported cfg ring size %d\n", cfgsize); in xgene_dma_get_ring_size()
1443 static void xgene_dma_delete_chan_rings(struct xgene_dma_chan *chan) in xgene_dma_delete_chan_rings() argument
1445 xgene_dma_delete_ring_one(&chan->rx_ring); in xgene_dma_delete_chan_rings()
1446 xgene_dma_delete_ring_one(&chan->tx_ring); in xgene_dma_delete_chan_rings()
1449 static int xgene_dma_create_ring_one(struct xgene_dma_chan *chan, in xgene_dma_create_ring_one() argument
1454 ring->pdma = chan->pdma; in xgene_dma_create_ring_one()
1456 ring->num = chan->pdma->ring_num++; in xgene_dma_create_ring_one()
1459 ring->size = xgene_dma_get_ring_size(chan, cfgsize); in xgene_dma_create_ring_one()
1464 ring->desc_vaddr = dma_zalloc_coherent(chan->dev, ring->size, in xgene_dma_create_ring_one()
1467 chan_err(chan, "Failed to allocate ring desc\n"); in xgene_dma_create_ring_one()
1478 static int xgene_dma_create_chan_rings(struct xgene_dma_chan *chan) in xgene_dma_create_chan_rings() argument
1480 struct xgene_dma_ring *rx_ring = &chan->rx_ring; in xgene_dma_create_chan_rings()
1481 struct xgene_dma_ring *tx_ring = &chan->tx_ring; in xgene_dma_create_chan_rings()
1486 rx_ring->buf_num = XGENE_DMA_CPU_BUFNUM + chan->id; in xgene_dma_create_chan_rings()
1488 ret = xgene_dma_create_ring_one(chan, rx_ring, in xgene_dma_create_chan_rings()
1493 chan_dbg(chan, "Rx ring id 0x%X num %d desc 0x%p\n", in xgene_dma_create_chan_rings()
1498 tx_ring->buf_num = XGENE_DMA_BUFNUM + chan->id; in xgene_dma_create_chan_rings()
1500 ret = xgene_dma_create_ring_one(chan, tx_ring, in xgene_dma_create_chan_rings()
1509 chan_dbg(chan, in xgene_dma_create_chan_rings()
1514 chan->max_outstanding = rx_ring->slots; in xgene_dma_create_chan_rings()
1524 ret = xgene_dma_create_chan_rings(&pdma->chan[i]); in xgene_dma_init_rings()
1527 xgene_dma_delete_chan_rings(&pdma->chan[j]); in xgene_dma_init_rings()
1695 struct xgene_dma_chan *chan; in xgene_dma_request_irqs() local
1709 chan = &pdma->chan[i]; in xgene_dma_request_irqs()
1710 ret = devm_request_irq(chan->dev, chan->rx_irq, in xgene_dma_request_irqs()
1712 0, chan->name, chan); in xgene_dma_request_irqs()
1714 chan_err(chan, "Failed to register Rx IRQ %d\n", in xgene_dma_request_irqs()
1715 chan->rx_irq); in xgene_dma_request_irqs()
1719 chan = &pdma->chan[i]; in xgene_dma_request_irqs()
1720 devm_free_irq(chan->dev, chan->rx_irq, chan); in xgene_dma_request_irqs()
1732 struct xgene_dma_chan *chan; in xgene_dma_free_irqs() local
1739 chan = &pdma->chan[i]; in xgene_dma_free_irqs()
1740 devm_free_irq(chan->dev, chan->rx_irq, chan); in xgene_dma_free_irqs()
1744 static void xgene_dma_set_caps(struct xgene_dma_chan *chan, in xgene_dma_set_caps() argument
1763 if ((chan->id == XGENE_DMA_PQ_CHANNEL) && in xgene_dma_set_caps()
1764 is_pq_enabled(chan->pdma)) { in xgene_dma_set_caps()
1767 } else if ((chan->id == XGENE_DMA_XOR_CHANNEL) && in xgene_dma_set_caps()
1768 !is_pq_enabled(chan->pdma)) { in xgene_dma_set_caps()
1773 dma_dev->dev = chan->dev; in xgene_dma_set_caps()
1796 struct xgene_dma_chan *chan = &pdma->chan[id]; in xgene_dma_async_register() local
1800 chan->dma_chan.device = dma_dev; in xgene_dma_async_register()
1802 spin_lock_init(&chan->lock); in xgene_dma_async_register()
1803 INIT_LIST_HEAD(&chan->ld_pending); in xgene_dma_async_register()
1804 INIT_LIST_HEAD(&chan->ld_running); in xgene_dma_async_register()
1805 INIT_LIST_HEAD(&chan->ld_completed); in xgene_dma_async_register()
1806 tasklet_init(&chan->tasklet, xgene_dma_tasklet_cb, in xgene_dma_async_register()
1807 (unsigned long)chan); in xgene_dma_async_register()
1809 chan->pending = 0; in xgene_dma_async_register()
1810 chan->desc_pool = NULL; in xgene_dma_async_register()
1811 dma_cookie_init(&chan->dma_chan); in xgene_dma_async_register()
1814 xgene_dma_set_caps(chan, dma_dev); in xgene_dma_async_register()
1818 list_add_tail(&chan->dma_chan.device_node, &dma_dev->channels); in xgene_dma_async_register()
1823 chan_err(chan, "Failed to register async device %d", ret); in xgene_dma_async_register()
1824 tasklet_kill(&chan->tasklet); in xgene_dma_async_register()
1831 "%s: CAPABILITY ( %s%s%s%s)\n", dma_chan_name(&chan->dma_chan), in xgene_dma_async_register()
1849 tasklet_kill(&pdma->chan[j].tasklet); in xgene_dma_init_async()
1869 struct xgene_dma_chan *chan; in xgene_dma_init_channels() local
1875 chan = &pdma->chan[i]; in xgene_dma_init_channels()
1876 chan->dev = pdma->dev; in xgene_dma_init_channels()
1877 chan->pdma = pdma; in xgene_dma_init_channels()
1878 chan->id = i; in xgene_dma_init_channels()
1879 snprintf(chan->name, sizeof(chan->name), "dmachan%d", chan->id); in xgene_dma_init_channels()
1962 pdma->chan[i - 1].rx_irq = irq; in xgene_dma_get_resources()
2035 xgene_dma_delete_chan_rings(&pdma->chan[i]); in xgene_dma_probe()
2047 struct xgene_dma_chan *chan; in xgene_dma_remove() local
2058 chan = &pdma->chan[i]; in xgene_dma_remove()
2059 tasklet_kill(&chan->tasklet); in xgene_dma_remove()
2060 xgene_dma_delete_chan_rings(chan); in xgene_dma_remove()