Lines Matching refs:chan
58 struct ioat_chan_common *chan; in ioat_dma_do_interrupt() local
75 chan = ioat_chan_by_index(instance, bit); in ioat_dma_do_interrupt()
76 if (test_bit(IOAT_RUN, &chan->state)) in ioat_dma_do_interrupt()
77 tasklet_schedule(&chan->cleanup_task); in ioat_dma_do_interrupt()
91 struct ioat_chan_common *chan = data; in ioat_dma_do_interrupt_msix() local
93 if (test_bit(IOAT_RUN, &chan->state)) in ioat_dma_do_interrupt_msix()
94 tasklet_schedule(&chan->cleanup_task); in ioat_dma_do_interrupt_msix()
100 void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx) in ioat_init_channel() argument
103 struct dma_chan *c = &chan->common; in ioat_init_channel()
106 chan->device = device; in ioat_init_channel()
107 chan->reg_base = device->reg_base + (0x80 * (idx + 1)); in ioat_init_channel()
108 spin_lock_init(&chan->cleanup_lock); in ioat_init_channel()
109 chan->common.device = dma; in ioat_init_channel()
110 dma_cookie_init(&chan->common); in ioat_init_channel()
111 list_add_tail(&chan->common.device_node, &dma->channels); in ioat_init_channel()
112 device->idx[idx] = chan; in ioat_init_channel()
113 init_timer(&chan->timer); in ioat_init_channel()
114 chan->timer.function = device->timer_fn; in ioat_init_channel()
115 chan->timer.data = data; in ioat_init_channel()
116 tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); in ioat_init_channel()
180 static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan) in ioat1_dma_memcpy_issue_pending() argument
182 struct ioat_dma_chan *ioat = to_ioat_chan(chan); in ioat1_dma_memcpy_issue_pending()
197 struct ioat_chan_common *chan = &ioat->base; in ioat1_reset_channel() local
198 void __iomem *reg_base = chan->reg_base; in ioat1_reset_channel()
201 dev_warn(to_dev(chan), "reset\n"); in ioat1_reset_channel()
203 chansts = *chan->completion & IOAT_CHANSTS_STATUS; in ioat1_reset_channel()
205 dev_err(to_dev(chan), in ioat1_reset_channel()
207 chan_num(chan), chansts, chanerr); in ioat1_reset_channel()
221 reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); in ioat1_reset_channel()
222 set_bit(IOAT_RESET_PENDING, &chan->state); in ioat1_reset_channel()
223 mod_timer(&chan->timer, jiffies + RESET_DELAY); in ioat1_reset_channel()
228 struct dma_chan *c = tx->chan; in ioat1_tx_submit()
231 struct ioat_chan_common *chan = &ioat->base; in ioat1_tx_submit() local
251 if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state)) in ioat1_tx_submit()
252 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); in ioat1_tx_submit()
310 struct ioat_chan_common *chan = &ioat->base; in ioat1_dma_alloc_chan_resources() local
321 writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET); in ioat1_dma_alloc_chan_resources()
323 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); in ioat1_dma_alloc_chan_resources()
325 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr); in ioat1_dma_alloc_chan_resources()
326 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); in ioat1_dma_alloc_chan_resources()
333 dev_err(to_dev(chan), "Only %d initial descriptors\n", i); in ioat1_dma_alloc_chan_resources()
346 chan->completion = pci_pool_alloc(chan->device->completion_pool, in ioat1_dma_alloc_chan_resources()
347 GFP_KERNEL, &chan->completion_dma); in ioat1_dma_alloc_chan_resources()
348 memset(chan->completion, 0, sizeof(*chan->completion)); in ioat1_dma_alloc_chan_resources()
349 writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF, in ioat1_dma_alloc_chan_resources()
350 chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); in ioat1_dma_alloc_chan_resources()
351 writel(((u64) chan->completion_dma) >> 32, in ioat1_dma_alloc_chan_resources()
352 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); in ioat1_dma_alloc_chan_resources()
354 set_bit(IOAT_RUN, &chan->state); in ioat1_dma_alloc_chan_resources()
356 dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n", in ioat1_dma_alloc_chan_resources()
361 void ioat_stop(struct ioat_chan_common *chan) in ioat_stop() argument
363 struct ioatdma_device *device = chan->device; in ioat_stop()
365 int chan_id = chan_num(chan); in ioat_stop()
371 clear_bit(IOAT_RUN, &chan->state); in ioat_stop()
388 del_timer_sync(&chan->timer); in ioat_stop()
391 tasklet_kill(&chan->cleanup_task); in ioat_stop()
394 device->cleanup_fn((unsigned long) &chan->common); in ioat_stop()
404 struct ioat_chan_common *chan = &ioat->base; in ioat1_dma_free_chan_resources() local
405 struct ioatdma_device *ioatdma_device = chan->device; in ioat1_dma_free_chan_resources()
415 ioat_stop(chan); in ioat1_dma_free_chan_resources()
421 chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); in ioat1_dma_free_chan_resources()
426 dev_dbg(to_dev(chan), "%s: freeing %d from used list\n", in ioat1_dma_free_chan_resources()
445 chan->completion, in ioat1_dma_free_chan_resources()
446 chan->completion_dma); in ioat1_dma_free_chan_resources()
450 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n", in ioat1_dma_free_chan_resources()
453 chan->last_completion = 0; in ioat1_dma_free_chan_resources()
454 chan->completion_dma = 0; in ioat1_dma_free_chan_resources()
536 struct ioat_chan_common *chan = &ioat->base; in ioat1_dma_prep_memcpy() local
538 dev_err(to_dev(chan), in ioat1_dma_prep_memcpy()
539 "chan%d - get_next_desc failed\n", chan_num(chan)); in ioat1_dma_prep_memcpy()
560 struct ioat_chan_common *chan = &ioat->base; in ioat1_cleanup_event() local
563 if (!test_bit(IOAT_RUN, &chan->state)) in ioat1_cleanup_event()
568 dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) in ioat_get_current_completion() argument
573 completion = *chan->completion; in ioat_get_current_completion()
576 dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, in ioat_get_current_completion()
580 u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); in ioat_get_current_completion()
581 dev_err(to_dev(chan), "Channel halted, chanerr = %x\n", in ioat_get_current_completion()
590 bool ioat_cleanup_preamble(struct ioat_chan_common *chan, in ioat_cleanup_preamble() argument
593 *phys_complete = ioat_get_current_completion(chan); in ioat_cleanup_preamble()
594 if (*phys_complete == chan->last_completion) in ioat_cleanup_preamble()
596 clear_bit(IOAT_COMPLETION_ACK, &chan->state); in ioat_cleanup_preamble()
597 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); in ioat_cleanup_preamble()
604 struct ioat_chan_common *chan = &ioat->base; in __cleanup() local
608 dev_dbg(to_dev(chan), "%s: phys_complete: %llx\n", in __cleanup()
649 dev_dbg(to_dev(chan), in __cleanup()
652 clear_bit(IOAT_COMPLETION_PENDING, &chan->state); in __cleanup()
660 chan->last_completion = phys_complete; in __cleanup()
673 struct ioat_chan_common *chan = &ioat->base; in ioat1_cleanup() local
676 prefetch(chan->completion); in ioat1_cleanup()
678 if (!spin_trylock_bh(&chan->cleanup_lock)) in ioat1_cleanup()
681 if (!ioat_cleanup_preamble(chan, &phys_complete)) { in ioat1_cleanup()
682 spin_unlock_bh(&chan->cleanup_lock); in ioat1_cleanup()
687 spin_unlock_bh(&chan->cleanup_lock); in ioat1_cleanup()
694 spin_unlock_bh(&chan->cleanup_lock); in ioat1_cleanup()
700 struct ioat_chan_common *chan = &ioat->base; in ioat1_timer_event() local
702 dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state); in ioat1_timer_event()
704 spin_lock_bh(&chan->cleanup_lock); in ioat1_timer_event()
705 if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) { in ioat1_timer_event()
713 ioat_start(chan); in ioat1_timer_event()
716 set_bit(IOAT_COMPLETION_PENDING, &chan->state); in ioat1_timer_event()
717 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); in ioat1_timer_event()
719 } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) { in ioat1_timer_event()
727 if (ioat_cleanup_preamble(chan, &phys_complete)) in ioat1_timer_event()
729 else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) in ioat1_timer_event()
732 u64 status = ioat_chansts(chan); in ioat1_timer_event()
736 *chan->completion = status; in ioat1_timer_event()
738 set_bit(IOAT_COMPLETION_ACK, &chan->state); in ioat1_timer_event()
739 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); in ioat1_timer_event()
743 spin_unlock_bh(&chan->cleanup_lock); in ioat1_timer_event()
750 struct ioat_chan_common *chan = to_chan_common(c); in ioat_dma_tx_status() local
751 struct ioatdma_device *device = chan->device; in ioat_dma_tx_status()
765 struct ioat_chan_common *chan = &ioat->base; in ioat1_dma_start_null_desc() local
774 dev_err(to_dev(chan), in ioat1_dma_start_null_desc()
795 ioat_start(chan); in ioat1_dma_start_null_desc()
923 struct ioat_chan_common *chan; in ioat_dma_setup_interrupts() local
952 chan = ioat_chan_by_index(device, i); in ioat_dma_setup_interrupts()
955 "ioat-msix", chan); in ioat_dma_setup_interrupts()
959 chan = ioat_chan_by_index(device, j); in ioat_dma_setup_interrupts()
960 devm_free_irq(dev, msix->vector, chan); in ioat_dma_setup_interrupts()
1146 struct ioat_chan_common *chan; in ioat_attr_show() local
1149 chan = container_of(kobj, struct ioat_chan_common, kobj); in ioat_attr_show()
1153 return entry->show(&chan->common, page); in ioat_attr_show()
1171 struct ioat_chan_common *chan = to_chan_common(c); in ioat_kobject_add() local
1175 err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata"); in ioat_kobject_add()
1177 dev_warn(to_dev(chan), in ioat_kobject_add()
1179 kobject_put(&chan->kobj); in ioat_kobject_add()
1180 set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state); in ioat_kobject_add()
1191 struct ioat_chan_common *chan = to_chan_common(c); in ioat_kobject_del() local
1193 if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) { in ioat_kobject_del()
1194 kobject_del(&chan->kobj); in ioat_kobject_del()
1195 kobject_put(&chan->kobj); in ioat_kobject_del()