Lines Matching refs:chan
307 static u64 ioat3_get_current_completion(struct ioat_chan_common *chan) in ioat3_get_current_completion() argument
312 completion = *chan->completion; in ioat3_get_current_completion()
315 dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, in ioat3_get_current_completion()
321 static bool ioat3_cleanup_preamble(struct ioat_chan_common *chan, in ioat3_cleanup_preamble() argument
324 *phys_complete = ioat3_get_current_completion(chan); in ioat3_cleanup_preamble()
325 if (*phys_complete == chan->last_completion) in ioat3_cleanup_preamble()
328 clear_bit(IOAT_COMPLETION_ACK, &chan->state); in ioat3_cleanup_preamble()
329 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); in ioat3_cleanup_preamble()
373 struct ioat_chan_common *chan = &ioat->base; in __cleanup() local
374 struct ioatdma_device *device = chan->device; in __cleanup()
380 dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n", in __cleanup()
434 chan->last_completion = phys_complete; in __cleanup()
437 dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", in __cleanup()
439 clear_bit(IOAT_COMPLETION_PENDING, &chan->state); in __cleanup()
440 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); in __cleanup()
444 chan->device->reg_base + IOAT_INTRDELAY_OFFSET); in __cleanup()
449 struct ioat_chan_common *chan = &ioat->base; in ioat3_cleanup() local
452 spin_lock_bh(&chan->cleanup_lock); in ioat3_cleanup()
454 if (ioat3_cleanup_preamble(chan, &phys_complete)) in ioat3_cleanup()
457 if (is_ioat_halted(*chan->completion)) { in ioat3_cleanup()
458 u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); in ioat3_cleanup()
461 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); in ioat3_cleanup()
466 spin_unlock_bh(&chan->cleanup_lock); in ioat3_cleanup()
472 struct ioat_chan_common *chan = &ioat->base; in ioat3_cleanup_event() local
475 if (!test_bit(IOAT_RUN, &chan->state)) in ioat3_cleanup_event()
482 struct ioat_chan_common *chan = &ioat->base; in ioat3_restart_channel() local
485 ioat2_quiesce(chan, 0); in ioat3_restart_channel()
486 if (ioat3_cleanup_preamble(chan, &phys_complete)) in ioat3_restart_channel()
494 struct ioat_chan_common *chan = &ioat->base; in ioat3_eh() local
495 struct pci_dev *pdev = to_pdev(chan); in ioat3_eh()
505 if (ioat3_cleanup_preamble(chan, &phys_complete)) in ioat3_eh()
508 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); in ioat3_eh()
511 dev_dbg(to_dev(chan), "%s: error = %x:%x\n", in ioat3_eh()
540 dev_err(to_dev(chan), "%s: fatal error (%x:%x)\n", in ioat3_eh()
555 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); in ioat3_eh()
559 *chan->completion = desc->txd.phys; in ioat3_eh()
568 struct ioat_chan_common *chan = &ioat->base; in check_active() local
571 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); in check_active()
575 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &chan->state)) in check_active()
576 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); in check_active()
587 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); in check_active()
595 struct ioat_chan_common *chan = &ioat->base; in ioat3_timer_event() local
599 status = ioat_chansts(chan); in ioat3_timer_event()
607 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); in ioat3_timer_event()
608 dev_err(to_dev(chan), "%s: Channel halted (%x)\n", in ioat3_timer_event()
610 if (test_bit(IOAT_RUN, &chan->state)) in ioat3_timer_event()
620 spin_lock_bh(&chan->cleanup_lock); in ioat3_timer_event()
621 if (ioat_cleanup_preamble(chan, &phys_complete)) in ioat3_timer_event()
623 else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) { in ioat3_timer_event()
627 spin_unlock_bh(&chan->cleanup_lock); in ioat3_timer_event()
630 set_bit(IOAT_COMPLETION_ACK, &chan->state); in ioat3_timer_event()
631 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); in ioat3_timer_event()
636 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); in ioat3_timer_event()
642 spin_unlock_bh(&chan->cleanup_lock); in ioat3_timer_event()
753 ioat3_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, in ioat3_prep_xor() argument
756 return __ioat3_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags); in ioat3_prep_xor()
760 ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src, in ioat3_prep_xor_val() argument
769 return __ioat3_prep_xor_lock(chan, result, src[0], &src[1], in ioat3_prep_xor_val()
842 struct ioat_chan_common *chan = &ioat->base; in __ioat3_prep_pq_lock() local
843 struct ioatdma_device *device = chan->device; in __ioat3_prep_pq_lock()
856 dev_dbg(to_dev(chan), "%s\n", __func__); in __ioat3_prep_pq_lock()
966 struct ioat_chan_common *chan = &ioat->base; in __ioat3_prep_pq16_lock() local
967 struct ioatdma_device *device = chan->device; in __ioat3_prep_pq16_lock()
978 dev_dbg(to_dev(chan), "%s\n", __func__); in __ioat3_prep_pq16_lock()
1004 dev_err(to_dev(chan), in __ioat3_prep_pq16_lock()
1071 ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, in ioat3_prep_pq() argument
1095 __ioat3_prep_pq16_lock(chan, NULL, dst, single_source, in ioat3_prep_pq()
1098 __ioat3_prep_pq_lock(chan, NULL, dst, single_source, 2, in ioat3_prep_pq()
1103 __ioat3_prep_pq16_lock(chan, NULL, dst, src, src_cnt, in ioat3_prep_pq()
1105 __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, in ioat3_prep_pq()
1111 ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, in ioat3_prep_pq_val() argument
1127 __ioat3_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len, in ioat3_prep_pq_val()
1129 __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len, in ioat3_prep_pq_val()
1134 ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, in ioat3_prep_pqxor() argument
1146 __ioat3_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len, in ioat3_prep_pqxor()
1148 __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, in ioat3_prep_pqxor()
1153 ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, in ioat3_prep_pqxor_val() argument
1171 __ioat3_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1, in ioat3_prep_pqxor_val()
1173 __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, in ioat3_prep_pqxor_val()
1507 struct ioat_chan_common *chan; in ioat3_irq_reinit() local
1509 chan = ioat_chan_by_index(device, i); in ioat3_irq_reinit()
1510 devm_free_irq(&pdev->dev, msix->vector, chan); in ioat3_irq_reinit()
1529 static int ioat3_reset_hw(struct ioat_chan_common *chan) in ioat3_reset_hw() argument
1534 struct ioatdma_device *device = chan->device; in ioat3_reset_hw()
1540 ioat2_quiesce(chan, msecs_to_jiffies(100)); in ioat3_reset_hw()
1542 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); in ioat3_reset_hw()
1543 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); in ioat3_reset_hw()
1568 err = ioat2_reset_sync(chan, msecs_to_jiffies(200)); in ioat3_reset_hw()
1582 struct ioat_chan_common *chan; in ioat3_intr_quirk() local
1593 chan = to_chan_common(c); in ioat3_intr_quirk()
1594 errmask = readl(chan->reg_base + in ioat3_intr_quirk()
1598 writel(errmask, chan->reg_base + in ioat3_intr_quirk()
1610 struct ioat_chan_common *chan; in ioat3_dma_probe() local
1702 chan = to_chan_common(c); in ioat3_dma_probe()
1704 chan->reg_base + IOAT_DCACTRL_OFFSET); in ioat3_dma_probe()