1914 	struct ppc440spe_adma_desc_slot *sw_desc;  in ppc440spe_adma_tx_submit()  local
1921 	sw_desc = tx_to_ppc440spe_adma_slot(tx);  in ppc440spe_adma_tx_submit()
1923 	group_start = sw_desc->group_head;  in ppc440spe_adma_tx_submit()
1932 		list_splice_init(&sw_desc->group_list, &chan->chain);  in ppc440spe_adma_tx_submit()
1939 		list_splice_init(&sw_desc->group_list,  in ppc440spe_adma_tx_submit()
1953 		sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);  in ppc440spe_adma_tx_submit()
1965 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;  in ppc440spe_adma_prep_dma_interrupt()  local
1976 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,  in ppc440spe_adma_prep_dma_interrupt()
1978 	if (sw_desc) {  in ppc440spe_adma_prep_dma_interrupt()
1979 		group_start = sw_desc->group_head;  in ppc440spe_adma_prep_dma_interrupt()
1982 		sw_desc->async_tx.flags = flags;  in ppc440spe_adma_prep_dma_interrupt()
1986 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_interrupt()
1997 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;  in ppc440spe_adma_prep_dma_memcpy()  local
2014 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,  in ppc440spe_adma_prep_dma_memcpy()
2016 	if (sw_desc) {  in ppc440spe_adma_prep_dma_memcpy()
2017 		group_start = sw_desc->group_head;  in ppc440spe_adma_prep_dma_memcpy()
2022 		sw_desc->unmap_len = len;  in ppc440spe_adma_prep_dma_memcpy()
2023 		sw_desc->async_tx.flags = flags;  in ppc440spe_adma_prep_dma_memcpy()
2027 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_memcpy()
2039 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;  in ppc440spe_adma_prep_dma_xor()  local
2057 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,  in ppc440spe_adma_prep_dma_xor()
2059 	if (sw_desc) {  in ppc440spe_adma_prep_dma_xor()
2060 		group_start = sw_desc->group_head;  in ppc440spe_adma_prep_dma_xor()
2067 		sw_desc->unmap_len = len;  in ppc440spe_adma_prep_dma_xor()
2068 		sw_desc->async_tx.flags = flags;  in ppc440spe_adma_prep_dma_xor()
2072 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_xor()
2105 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;  in ppc440spe_dma01_prep_mult()  local
2115 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);  in ppc440spe_dma01_prep_mult()
2116 	if (sw_desc) {  in ppc440spe_dma01_prep_mult()
2121 		chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_dma01_prep_mult()
2122 		set_bits(op, &sw_desc->flags);  in ppc440spe_dma01_prep_mult()
2123 		sw_desc->src_cnt = src_cnt;  in ppc440spe_dma01_prep_mult()
2124 		sw_desc->dst_cnt = dst_cnt;  in ppc440spe_dma01_prep_mult()
2128 		iter = list_first_entry(&sw_desc->group_list,  in ppc440spe_dma01_prep_mult()
2173 		sw_desc->async_tx.flags = flags;  in ppc440spe_dma01_prep_mult()
2178 	return sw_desc;  in ppc440spe_dma01_prep_mult()
2191 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;  in ppc440spe_dma01_prep_sum_product()  local
2201 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);  in ppc440spe_dma01_prep_sum_product()
2202 	if (sw_desc) {  in ppc440spe_dma01_prep_sum_product()
2207 		chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_dma01_prep_sum_product()
2208 		set_bits(op, &sw_desc->flags);  in ppc440spe_dma01_prep_sum_product()
2209 		sw_desc->src_cnt = src_cnt;  in ppc440spe_dma01_prep_sum_product()
2210 		sw_desc->dst_cnt = 1;  in ppc440spe_dma01_prep_sum_product()
2212 		iter = list_first_entry(&sw_desc->group_list,  in ppc440spe_dma01_prep_sum_product()
2282 		sw_desc->async_tx.flags = flags;  in ppc440spe_dma01_prep_sum_product()
2287 	return sw_desc;  in ppc440spe_dma01_prep_sum_product()
2296 	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;  in ppc440spe_dma01_prep_pq()  local
2396 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);  in ppc440spe_dma01_prep_pq()
2397 	if (sw_desc) {  in ppc440spe_dma01_prep_pq()
2398 		ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,  in ppc440spe_dma01_prep_pq()
2404 		ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);  in ppc440spe_dma01_prep_pq()
2406 			ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],  in ppc440spe_dma01_prep_pq()
2417 			ppc440spe_adma_pq_set_src_mult(sw_desc,  in ppc440spe_dma01_prep_pq()
2422 		sw_desc->async_tx.flags = flags;  in ppc440spe_dma01_prep_pq()
2423 		list_for_each_entry(iter, &sw_desc->group_list,  in ppc440spe_dma01_prep_pq()
2432 	return sw_desc;  in ppc440spe_dma01_prep_pq()
2441 	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;  in ppc440spe_dma2_prep_pq()  local
2459 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);  in ppc440spe_dma2_prep_pq()
2460 	if (sw_desc) {  in ppc440spe_dma2_prep_pq()
2462 		sw_desc->async_tx.flags = flags;  in ppc440spe_dma2_prep_pq()
2463 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {  in ppc440spe_dma2_prep_pq()
2475 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {  in ppc440spe_dma2_prep_pq()
2481 						 &sw_desc->group_list))) {  in ppc440spe_dma2_prep_pq()
2495 		sw_desc->dst_cnt = dst_cnt;  in ppc440spe_dma2_prep_pq()
2497 			set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);  in ppc440spe_dma2_prep_pq()
2499 			set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);  in ppc440spe_dma2_prep_pq()
2502 		ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);  in ppc440spe_dma2_prep_pq()
2508 			ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],  in ppc440spe_dma2_prep_pq()
2512 			ppc440spe_adma_pq_set_src_mult(sw_desc,  in ppc440spe_dma2_prep_pq()
2518 	return sw_desc;  in ppc440spe_dma2_prep_pq()
2530 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;  in ppc440spe_adma_prep_dma_pq()  local
2548 		sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,  in ppc440spe_adma_prep_dma_pq()
2550 		return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_pq()
2554 		sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,  in ppc440spe_adma_prep_dma_pq()
2556 		return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_pq()
2581 		sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,  in ppc440spe_adma_prep_dma_pq()
2587 		sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,  in ppc440spe_adma_prep_dma_pq()
2593 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_pq()
2606 	struct ppc440spe_adma_desc_slot *sw_desc, *iter;  in ppc440spe_adma_prep_dma_pqzero_sum()  local
2637 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,  in ppc440spe_adma_prep_dma_pqzero_sum()
2639 	if (sw_desc) {  in ppc440spe_adma_prep_dma_pqzero_sum()
2640 		ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);  in ppc440spe_adma_prep_dma_pqzero_sum()
2643 		sw_desc->async_tx.flags = flags;  in ppc440spe_adma_prep_dma_pqzero_sum()
2644 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {  in ppc440spe_adma_prep_dma_pqzero_sum()
2654 			iter = sw_desc->group_head;  in ppc440spe_adma_prep_dma_pqzero_sum()
2677 			iter = list_first_entry(&sw_desc->group_list,  in ppc440spe_adma_prep_dma_pqzero_sum()
2707 		ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);  in ppc440spe_adma_prep_dma_pqzero_sum()
2711 		list_for_each_entry_reverse(iter, &sw_desc->group_list,  in ppc440spe_adma_prep_dma_pqzero_sum()
2750 		list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,  in ppc440spe_adma_prep_dma_pqzero_sum()
2772 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_pqzero_sum()
2800 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_set_dest()  argument
2805 	BUG_ON(index >= sw_desc->dst_cnt);  in ppc440spe_adma_set_dest()
2807 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_set_dest()
2815 		ppc440spe_desc_set_dest_addr(sw_desc->group_head,  in ppc440spe_adma_set_dest()
2819 		sw_desc = ppc440spe_get_group_entry(sw_desc, index);  in ppc440spe_adma_set_dest()
2820 		ppc440spe_desc_set_dest_addr(sw_desc,  in ppc440spe_adma_set_dest()
2847 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_pq_set_dest()  argument
2856 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_pq_set_dest()
2877 		if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {  in ppc440spe_adma_pq_set_dest()
2879 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))  in ppc440spe_adma_pq_set_dest()
2881 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))  in ppc440spe_adma_pq_set_dest()
2884 			iter = ppc440spe_get_group_entry(sw_desc, index);  in ppc440spe_adma_pq_set_dest()
2888 					&sw_desc->group_list, chain_node)  in ppc440spe_adma_pq_set_dest()
2894 					&sw_desc->group_list, chain_node) {  in ppc440spe_adma_pq_set_dest()
2908 						&sw_desc->flags)) {  in ppc440spe_adma_pq_set_dest()
2910 							sw_desc, index++);  in ppc440spe_adma_pq_set_dest()
2916 						&sw_desc->flags)) {  in ppc440spe_adma_pq_set_dest()
2918 							sw_desc, index++);  in ppc440spe_adma_pq_set_dest()
2931 			ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?  in ppc440spe_adma_pq_set_dest()
2935 			qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?  in ppc440spe_adma_pq_set_dest()
2941 			iter = ppc440spe_get_group_entry(sw_desc, index++);  in ppc440spe_adma_pq_set_dest()
2947 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_dest()
2953 			if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {  in ppc440spe_adma_pq_set_dest()
2957 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_dest()
2962 					    &sw_desc->group_list,  in ppc440spe_adma_pq_set_dest()
2972 					    &sw_desc->group_list,  in ppc440spe_adma_pq_set_dest()
2995 		ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?  in ppc440spe_adma_pq_set_dest()
3000 		qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?  in ppc440spe_adma_pq_set_dest()
3005 		iter = ppc440spe_get_group_entry(sw_desc, 0);  in ppc440spe_adma_pq_set_dest()
3006 		for (i = 0; i < sw_desc->descs_per_op; i++) {  in ppc440spe_adma_pq_set_dest()
3017 			iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_dest()
3018 				sw_desc->descs_per_op);  in ppc440spe_adma_pq_set_dest()
3019 			for (i = 0; i < sw_desc->descs_per_op; i++) {  in ppc440spe_adma_pq_set_dest()
3037 		struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_pqzero_sum_set_dest()  argument
3045 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_pqzero_sum_set_dest()
3052 	list_for_each_entry_reverse(end, &sw_desc->group_list,  in ppc440spe_adma_pqzero_sum_set_dest()
3059 	iter = ppc440spe_get_group_entry(sw_desc, idx);  in ppc440spe_adma_pqzero_sum_set_dest()
3063 		list_for_each_entry_from(iter, &sw_desc->group_list,  in ppc440spe_adma_pqzero_sum_set_dest()
3075 		list_for_each_entry_from(iter, &sw_desc->group_list,  in ppc440spe_adma_pqzero_sum_set_dest()
3113 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_pq_set_src()  argument
3120 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_pq_set_src()
3127 		if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {  in ppc440spe_adma_pq_set_src()
3130 				&sw_desc->flags) ?  2 : 3;  in ppc440spe_adma_pq_set_src()
3138 						&sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3142 				    &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3146 				    &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3150 				    &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3156 				iter = ppc440spe_get_group_entry(sw_desc, 0);  in ppc440spe_adma_pq_set_src()
3168 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src()
3169 				    index - iskip + sw_desc->dst_cnt);  in ppc440spe_adma_pq_set_src()
3177 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3179 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3183 			iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src()
3191 			    test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&  in ppc440spe_adma_pq_set_src()
3192 			    sw_desc->dst_cnt == 2) {  in ppc440spe_adma_pq_set_src()
3196 				iter = ppc440spe_get_group_entry(sw_desc, 1);  in ppc440spe_adma_pq_set_src()
3205 		iter = sw_desc->group_head;  in ppc440spe_adma_pq_set_src()
3211 			iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src()
3212 				sw_desc->descs_per_op);  in ppc440spe_adma_pq_set_src()
3223 		struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_memcpy_xor_set_src()  argument
3228 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_memcpy_xor_set_src()
3229 	sw_desc = sw_desc->group_head;  in ppc440spe_adma_memcpy_xor_set_src()
3231 	if (likely(sw_desc))  in ppc440spe_adma_memcpy_xor_set_src()
3232 		ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);  in ppc440spe_adma_memcpy_xor_set_src()
3465 		struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_pq_set_src_mult()  argument
3472 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_pq_set_src_mult()
3477 		if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {  in ppc440spe_adma_pq_set_src_mult()
3479 					&sw_desc->flags) ? 2 : 3;  in ppc440spe_adma_pq_set_src_mult()
3483 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src_mult()
3484 					sw_desc->dst_cnt - 1);  in ppc440spe_adma_pq_set_src_mult()
3485 				if (sw_desc->dst_cnt == 2)  in ppc440spe_adma_pq_set_src_mult()
3487 							sw_desc, 0);  in ppc440spe_adma_pq_set_src_mult()
3493 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src_mult()
3495 							sw_desc->dst_cnt);  in ppc440spe_adma_pq_set_src_mult()
3507 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))  in ppc440spe_adma_pq_set_src_mult()
3509 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))  in ppc440spe_adma_pq_set_src_mult()
3512 			iter = ppc440spe_get_group_entry(sw_desc, index + znum);  in ppc440spe_adma_pq_set_src_mult()
3533 		iter = sw_desc->group_head;  in ppc440spe_adma_pq_set_src_mult()
3534 		if (sw_desc->dst_cnt == 2) {  in ppc440spe_adma_pq_set_src_mult()
3539 			iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src_mult()
3540 			       sw_desc->descs_per_op);  in ppc440spe_adma_pq_set_src_mult()
3673 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;  in ppc440spe_chan_start_null_xor()  local
3682 	sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);  in ppc440spe_chan_start_null_xor()
3683 	if (sw_desc) {  in ppc440spe_chan_start_null_xor()
3684 		group_start = sw_desc->group_head;  in ppc440spe_chan_start_null_xor()
3685 		list_splice_init(&sw_desc->group_list, &chan->chain);  in ppc440spe_chan_start_null_xor()
3686 		async_tx_ack(&sw_desc->async_tx);  in ppc440spe_chan_start_null_xor()
3689 		cookie = dma_cookie_assign(&sw_desc->async_tx);  in ppc440spe_chan_start_null_xor()
3700 		ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);  in ppc440spe_chan_start_null_xor()
3719 	struct ppc440spe_adma_desc_slot *sw_desc, *iter;  in ppc440spe_test_raid6()  local
3733 	sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);  in ppc440spe_test_raid6()
3734 	if (sw_desc) {  in ppc440spe_test_raid6()
3736 		ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);  in ppc440spe_test_raid6()
3737 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {  in ppc440spe_test_raid6()
3754 	ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);  in ppc440spe_test_raid6()
3755 	ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);  in ppc440spe_test_raid6()
3758 	ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);  in ppc440spe_test_raid6()
3760 	async_tx_ack(&sw_desc->async_tx);  in ppc440spe_test_raid6()
3761 	sw_desc->async_tx.callback = ppc440spe_test_callback;  in ppc440spe_test_raid6()
3762 	sw_desc->async_tx.callback_param = NULL;  in ppc440spe_test_raid6()
3766 	ppc440spe_adma_tx_submit(&sw_desc->async_tx);  in ppc440spe_test_raid6()