105 static dma_addr_t pq16_get_src(struct ioat_raw_descriptor *desc[3], int idx)  in pq16_get_src()
107 	struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];  in pq16_get_src()
240 static void pq16_set_src(struct ioat_raw_descriptor *desc[3],  in pq16_set_src()
243 	struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0];  in pq16_set_src()
245 		(struct ioat_pq16a_descriptor *)desc[1];  in pq16_set_src()
246 	struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];  in pq16_set_src()
286 static bool desc_has_ext(struct ioat_ring_ent *desc)  in desc_has_ext()  argument
288 	struct ioat_dma_descriptor *hw = desc->hw;  in desc_has_ext()
292 		struct ioat_xor_descriptor *xor = desc->xor;  in desc_has_ext()
298 		struct ioat_pq_descriptor *pq = desc->pq;  in desc_has_ext()
335 desc_get_errstat(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc)  in desc_get_errstat()  argument
337 	struct ioat_dma_descriptor *hw = desc->hw;  in desc_get_errstat()
343 		struct ioat_pq_descriptor *pq = desc->pq;  in desc_get_errstat()
352 			*desc->result |= SUM_CHECK_P_RESULT;  in desc_get_errstat()
355 			*desc->result |= SUM_CHECK_Q_RESULT;  in desc_get_errstat()
375 	struct ioat_ring_ent *desc;  in __cleanup()  local
399 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __cleanup()
400 		dump_desc_dbg(ioat, desc);  in __cleanup()
404 			desc_get_errstat(ioat, desc);  in __cleanup()
406 		tx = &desc->txd;  in __cleanup()
420 		if (desc_has_ext(desc)) {  in __cleanup()
426 		if (desc->sed) {  in __cleanup()
427 			ioat3_free_sed(device, desc->sed);  in __cleanup()
428 			desc->sed = NULL;  in __cleanup()
499 	struct ioat_ring_ent *desc;  in ioat3_eh()  local
514 	desc = ioat2_get_ring_ent(ioat, ioat->tail);  in ioat3_eh()
515 	hw = desc->hw;  in ioat3_eh()
516 	dump_desc_dbg(ioat, desc);  in ioat3_eh()
521 			*desc->result |= SUM_CHECK_P_RESULT;  in ioat3_eh()
528 			*desc->result |= SUM_CHECK_P_RESULT;  in ioat3_eh()
532 			*desc->result |= SUM_CHECK_Q_RESULT;  in ioat3_eh()
544 		tx = &desc->txd;  in ioat3_eh()
559 	*chan->completion = desc->txd.phys;  in ioat3_eh()
668 	struct ioat_ring_ent *desc;  in __ioat3_prep_xor_lock()  local
705 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_xor_lock()
706 		xor = desc->xor;  in __ioat3_prep_xor_lock()
727 		dump_desc_dbg(ioat, desc);  in __ioat3_prep_xor_lock()
731 	desc->txd.flags = flags;  in __ioat3_prep_xor_lock()
732 	desc->len = total_len;  in __ioat3_prep_xor_lock()
734 		desc->result = result;  in __ioat3_prep_xor_lock()
774 dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext)  in dump_pq_desc_dbg()  argument
777 	struct ioat_pq_descriptor *pq = desc->pq;  in dump_pq_desc_dbg()
786 		desc_id(desc), (unsigned long long) desc->txd.phys,  in dump_pq_desc_dbg()
788 		desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, pq->ctl_f.int_en,  in dump_pq_desc_dbg()
801 			       struct ioat_ring_ent *desc)  in dump_pq16_desc_dbg()  argument
804 	struct ioat_pq_descriptor *pq = desc->pq;  in dump_pq16_desc_dbg()
811 	if (desc->sed) {  in dump_pq16_desc_dbg()
812 		descs[1] = (void *)desc->sed->hw;  in dump_pq16_desc_dbg()
813 		descs[2] = (void *)desc->sed->hw + 64;  in dump_pq16_desc_dbg()
819 		desc_id(desc), (unsigned long long) desc->txd.phys,  in dump_pq16_desc_dbg()
821 		desc->txd.flags, pq->size, pq->ctl,  in dump_pq16_desc_dbg()
845 	struct ioat_ring_ent *desc;  in __ioat3_prep_pq_lock()  local
889 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_pq_lock()
890 		pq = desc->pq;  in __ioat3_prep_pq_lock()
930 	desc->txd.flags = flags;  in __ioat3_prep_pq_lock()
931 	desc->len = total_len;  in __ioat3_prep_pq_lock()
933 		desc->result = result;  in __ioat3_prep_pq_lock()
935 	dump_pq_desc_dbg(ioat, desc, ext);  in __ioat3_prep_pq_lock()
940 		compl_desc = desc;  in __ioat3_prep_pq_lock()
968 	struct ioat_ring_ent *desc;  in __ioat3_prep_pq16_lock()  local
997 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_pq16_lock()
998 		pq = desc->pq;  in __ioat3_prep_pq16_lock()
1002 		desc->sed = ioat3_alloc_sed(device, (src_cnt-2) >> 3);  in __ioat3_prep_pq16_lock()
1003 		if (!desc->sed) {  in __ioat3_prep_pq16_lock()
1009 		pq->sed_addr = desc->sed->dma;  in __ioat3_prep_pq16_lock()
1010 		desc->sed->parent = desc;  in __ioat3_prep_pq16_lock()
1012 		descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw;  in __ioat3_prep_pq16_lock()
1044 	desc->txd.flags = flags;  in __ioat3_prep_pq16_lock()
1045 	desc->len = total_len;  in __ioat3_prep_pq16_lock()
1047 		desc->result = result;  in __ioat3_prep_pq16_lock()
1054 	dump_pq16_desc_dbg(ioat, desc);  in __ioat3_prep_pq16_lock()
1057 	return &desc->txd;  in __ioat3_prep_pq16_lock()
1181 	struct ioat_ring_ent *desc;  in ioat3_prep_interrupt_lock()  local
1185 		desc = ioat2_get_ring_ent(ioat, ioat->head);  in ioat3_prep_interrupt_lock()
1189 	hw = desc->hw;  in ioat3_prep_interrupt_lock()
1199 	desc->txd.flags = flags;  in ioat3_prep_interrupt_lock()
1200 	desc->len = 1;  in ioat3_prep_interrupt_lock()
1202 	dump_desc_dbg(ioat, desc);  in ioat3_prep_interrupt_lock()
1205 	return &desc->txd;  in ioat3_prep_interrupt_lock()