2154 		       struct aead_edesc *edesc,  in aead_unmap()  argument
2160 	dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents,  in aead_unmap()
2161 			     DMA_TO_DEVICE, edesc->assoc_chained);  in aead_unmap()
2164 		   edesc->src_nents, edesc->src_chained, edesc->dst_nents,  in aead_unmap()
2165 		   edesc->dst_chained, edesc->iv_dma, ivsize,  in aead_unmap()
2166 		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);  in aead_unmap()
2170 			     struct ablkcipher_edesc *edesc,  in ablkcipher_unmap()  argument
2177 		   edesc->src_nents, edesc->src_chained, edesc->dst_nents,  in ablkcipher_unmap()
2178 		   edesc->dst_chained, edesc->iv_dma, ivsize,  in ablkcipher_unmap()
2179 		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);  in ablkcipher_unmap()
2186 	struct aead_edesc *edesc;  in aead_encrypt_done()  local
2195 	edesc = (struct aead_edesc *)((char *)desc -  in aead_encrypt_done()
2201 	aead_unmap(jrdev, edesc, req);  in aead_encrypt_done()
2209 		       edesc->src_nents ? 100 : ivsize, 1);  in aead_encrypt_done()
2212 		       edesc->src_nents ? 100 : req->cryptlen +  in aead_encrypt_done()
2216 	kfree(edesc);  in aead_encrypt_done()
2225 	struct aead_edesc *edesc;  in aead_decrypt_done()  local
2234 	edesc = (struct aead_edesc *)((char *)desc -  in aead_decrypt_done()
2249 	aead_unmap(jrdev, edesc, req);  in aead_decrypt_done()
2264 	if (!err && edesc->sec4_sg_bytes) {  in aead_decrypt_done()
2265 		struct scatterlist *sg = sg_last(req->src, edesc->src_nents);  in aead_decrypt_done()
2272 	kfree(edesc);  in aead_decrypt_done()
2281 	struct ablkcipher_edesc *edesc;  in ablkcipher_encrypt_done()  local
2289 	edesc = (struct ablkcipher_edesc *)((char *)desc -  in ablkcipher_encrypt_done()
2298 		       edesc->src_nents > 1 ? 100 : ivsize, 1);  in ablkcipher_encrypt_done()
2301 		       edesc->dst_nents > 1 ? 100 : req->nbytes, 1);  in ablkcipher_encrypt_done()
2304 	ablkcipher_unmap(jrdev, edesc, req);  in ablkcipher_encrypt_done()
2305 	kfree(edesc);  in ablkcipher_encrypt_done()
2314 	struct ablkcipher_edesc *edesc;  in ablkcipher_decrypt_done()  local
2322 	edesc = (struct ablkcipher_edesc *)((char *)desc -  in ablkcipher_decrypt_done()
2333 		       edesc->dst_nents > 1 ? 100 : req->nbytes, 1);  in ablkcipher_decrypt_done()
2336 	ablkcipher_unmap(jrdev, edesc, req);  in ablkcipher_decrypt_done()
2337 	kfree(edesc);  in ablkcipher_decrypt_done()
2346 			  struct aead_edesc *edesc,  in init_aead_job()  argument
2354 	u32 *desc = edesc->hw_desc;  in init_aead_job()
2368 		       edesc->src_nents ? 100 : ivsize, 1);  in init_aead_job()
2371 			edesc->src_nents ? 100 : req->cryptlen, 1);  in init_aead_job()
2387 			src_dma = edesc->iv_dma;  in init_aead_job()
2392 		src_dma = edesc->sec4_sg_dma;  in init_aead_job()
2393 		sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 +  in init_aead_job()
2394 				 (edesc->src_nents ? : 1);  in init_aead_job()
2406 				  ((edesc->assoc_nents ? : 1) + 1);  in init_aead_job()
2410 		if (!edesc->dst_nents) {  in init_aead_job()
2413 			dst_dma = edesc->sec4_sg_dma +  in init_aead_job()
2431 			      struct aead_edesc *edesc,  in init_aead_giv_job()  argument
2439 	u32 *desc = edesc->hw_desc;  in init_aead_giv_job()
2455 			edesc->src_nents > 1 ? 100 : req->cryptlen, 1);  in init_aead_giv_job()
2471 			src_dma = edesc->iv_dma;  in init_aead_giv_job()
2476 		src_dma = edesc->sec4_sg_dma;  in init_aead_giv_job()
2477 		sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents;  in init_aead_giv_job()
2484 		dst_dma = edesc->iv_dma;  in init_aead_giv_job()
2488 				  (edesc->assoc_nents +  in init_aead_giv_job()
2489 				   (is_gcm ? 1 + edesc->src_nents : 0));  in init_aead_giv_job()
2492 			dst_dma = edesc->sec4_sg_dma +  in init_aead_giv_job()
2507 				struct ablkcipher_edesc *edesc,  in init_ablkcipher_job()  argument
2513 	u32 *desc = edesc->hw_desc;  in init_ablkcipher_job()
2524 		       edesc->src_nents ? 100 : req->nbytes, 1);  in init_ablkcipher_job()
2531 		src_dma = edesc->iv_dma;  in init_ablkcipher_job()
2534 		src_dma = edesc->sec4_sg_dma;  in init_ablkcipher_job()
2535 		sec4_sg_index += edesc->src_nents + 1;  in init_ablkcipher_job()
2541 		if (!edesc->src_nents && iv_contig) {  in init_ablkcipher_job()
2544 			dst_dma = edesc->sec4_sg_dma +  in init_ablkcipher_job()
2549 		if (!edesc->dst_nents) {  in init_ablkcipher_job()
2552 			dst_dma = edesc->sec4_sg_dma +  in init_ablkcipher_job()
2564 				    struct ablkcipher_edesc *edesc,  in init_ablkcipher_giv_job()  argument
2570 	u32 *desc = edesc->hw_desc;  in init_ablkcipher_giv_job()
2581 		       edesc->src_nents ? 100 : req->nbytes, 1);  in init_ablkcipher_giv_job()
2587 	if (!edesc->src_nents) {  in init_ablkcipher_giv_job()
2591 		src_dma = edesc->sec4_sg_dma;  in init_ablkcipher_giv_job()
2592 		sec4_sg_index += edesc->src_nents;  in init_ablkcipher_giv_job()
2598 		dst_dma = edesc->iv_dma;  in init_ablkcipher_giv_job()
2601 		dst_dma = edesc->sec4_sg_dma +  in init_ablkcipher_giv_job()
2621 	struct aead_edesc *edesc;  in aead_edesc_alloc()  local
2694 	edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +  in aead_edesc_alloc()
2696 	if (!edesc) {  in aead_edesc_alloc()
2701 	edesc->assoc_nents = assoc_nents;  in aead_edesc_alloc()
2702 	edesc->assoc_chained = assoc_chained;  in aead_edesc_alloc()
2703 	edesc->src_nents = src_nents;  in aead_edesc_alloc()
2704 	edesc->src_chained = src_chained;  in aead_edesc_alloc()
2705 	edesc->dst_nents = dst_nents;  in aead_edesc_alloc()
2706 	edesc->dst_chained = dst_chained;  in aead_edesc_alloc()
2707 	edesc->iv_dma = iv_dma;  in aead_edesc_alloc()
2708 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in aead_edesc_alloc()
2709 	edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +  in aead_edesc_alloc()
2718 				      edesc->sec4_sg +  in aead_edesc_alloc()
2723 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,  in aead_edesc_alloc()
2730 				      edesc->sec4_sg +  in aead_edesc_alloc()
2737 				   edesc->sec4_sg +  in aead_edesc_alloc()
2743 				   edesc->sec4_sg + sec4_sg_index, 0);  in aead_edesc_alloc()
2745 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in aead_edesc_alloc()
2747 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in aead_edesc_alloc()
2752 	return edesc;  in aead_edesc_alloc()
2757 	struct aead_edesc *edesc;  in aead_encrypt()  local
2766 	edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *  in aead_encrypt()
2768 	if (IS_ERR(edesc))  in aead_encrypt()
2769 		return PTR_ERR(edesc);  in aead_encrypt()
2772 	init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,  in aead_encrypt()
2776 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in aead_encrypt()
2777 		       desc_bytes(edesc->hw_desc), 1);  in aead_encrypt()
2780 	desc = edesc->hw_desc;  in aead_encrypt()
2785 		aead_unmap(jrdev, edesc, req);  in aead_encrypt()
2786 		kfree(edesc);  in aead_encrypt()
2794 	struct aead_edesc *edesc;  in aead_decrypt()  local
2803 	edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *  in aead_decrypt()
2805 	if (IS_ERR(edesc))  in aead_decrypt()
2806 		return PTR_ERR(edesc);  in aead_decrypt()
2816 		      ctx->sh_desc_dec_dma, edesc, req, all_contig, false);  in aead_decrypt()
2819 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in aead_decrypt()
2820 		       desc_bytes(edesc->hw_desc), 1);  in aead_decrypt()
2823 	desc = edesc->hw_desc;  in aead_decrypt()
2828 		aead_unmap(jrdev, edesc, req);  in aead_decrypt()
2829 		kfree(edesc);  in aead_decrypt()
2849 	struct aead_edesc *edesc;  in aead_giv_edesc_alloc()  local
2934 	edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +  in aead_giv_edesc_alloc()
2936 	if (!edesc) {  in aead_giv_edesc_alloc()
2941 	edesc->assoc_nents = assoc_nents;  in aead_giv_edesc_alloc()
2942 	edesc->assoc_chained = assoc_chained;  in aead_giv_edesc_alloc()
2943 	edesc->src_nents = src_nents;  in aead_giv_edesc_alloc()
2944 	edesc->src_chained = src_chained;  in aead_giv_edesc_alloc()
2945 	edesc->dst_nents = dst_nents;  in aead_giv_edesc_alloc()
2946 	edesc->dst_chained = dst_chained;  in aead_giv_edesc_alloc()
2947 	edesc->iv_dma = iv_dma;  in aead_giv_edesc_alloc()
2948 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in aead_giv_edesc_alloc()
2949 	edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +  in aead_giv_edesc_alloc()
2957 				      edesc->sec4_sg + sec4_sg_index, 0);  in aead_giv_edesc_alloc()
2961 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,  in aead_giv_edesc_alloc()
2967 				      edesc->sec4_sg + sec4_sg_index, 0);  in aead_giv_edesc_alloc()
2972 				   edesc->sec4_sg +  in aead_giv_edesc_alloc()
2978 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,  in aead_giv_edesc_alloc()
2982 				   edesc->sec4_sg + sec4_sg_index, 0);  in aead_giv_edesc_alloc()
2986 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,  in aead_giv_edesc_alloc()
2990 				   edesc->sec4_sg + sec4_sg_index, 0);  in aead_giv_edesc_alloc()
2992 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in aead_giv_edesc_alloc()
2994 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in aead_giv_edesc_alloc()
2999 	return edesc;  in aead_giv_edesc_alloc()
3005 	struct aead_edesc *edesc;  in aead_givencrypt()  local
3014 	edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN *  in aead_givencrypt()
3017 	if (IS_ERR(edesc))  in aead_givencrypt()
3018 		return PTR_ERR(edesc);  in aead_givencrypt()
3028 			  ctx->sh_desc_givenc_dma, edesc, req, contig);  in aead_givencrypt()
3031 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in aead_givencrypt()
3032 		       desc_bytes(edesc->hw_desc), 1);  in aead_givencrypt()
3035 	desc = edesc->hw_desc;  in aead_givencrypt()
3040 		aead_unmap(jrdev, edesc, req);  in aead_givencrypt()
3041 		kfree(edesc);  in aead_givencrypt()
3066 	struct ablkcipher_edesc *edesc;  in ablkcipher_edesc_alloc()  local
3107 	edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes +  in ablkcipher_edesc_alloc()
3109 	if (!edesc) {  in ablkcipher_edesc_alloc()
3114 	edesc->src_nents = src_nents;  in ablkcipher_edesc_alloc()
3115 	edesc->src_chained = src_chained;  in ablkcipher_edesc_alloc()
3116 	edesc->dst_nents = dst_nents;  in ablkcipher_edesc_alloc()
3117 	edesc->dst_chained = dst_chained;  in ablkcipher_edesc_alloc()
3118 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ablkcipher_edesc_alloc()
3119 	edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +  in ablkcipher_edesc_alloc()
3124 		dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);  in ablkcipher_edesc_alloc()
3126 				   edesc->sec4_sg + 1, 0);  in ablkcipher_edesc_alloc()
3132 			edesc->sec4_sg + sec4_sg_index, 0);  in ablkcipher_edesc_alloc()
3135 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ablkcipher_edesc_alloc()
3137 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ablkcipher_edesc_alloc()
3142 	edesc->iv_dma = iv_dma;  in ablkcipher_edesc_alloc()
3146 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,  in ablkcipher_edesc_alloc()
3151 	return edesc;  in ablkcipher_edesc_alloc()
3156 	struct ablkcipher_edesc *edesc;  in ablkcipher_encrypt()  local
3165 	edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *  in ablkcipher_encrypt()
3167 	if (IS_ERR(edesc))  in ablkcipher_encrypt()
3168 		return PTR_ERR(edesc);  in ablkcipher_encrypt()
3172 		ctx->sh_desc_enc_dma, edesc, req, iv_contig);  in ablkcipher_encrypt()
3175 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in ablkcipher_encrypt()
3176 		       desc_bytes(edesc->hw_desc), 1);  in ablkcipher_encrypt()
3178 	desc = edesc->hw_desc;  in ablkcipher_encrypt()
3184 		ablkcipher_unmap(jrdev, edesc, req);  in ablkcipher_encrypt()
3185 		kfree(edesc);  in ablkcipher_encrypt()
3193 	struct ablkcipher_edesc *edesc;  in ablkcipher_decrypt()  local
3202 	edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *  in ablkcipher_decrypt()
3204 	if (IS_ERR(edesc))  in ablkcipher_decrypt()
3205 		return PTR_ERR(edesc);  in ablkcipher_decrypt()
3209 		ctx->sh_desc_dec_dma, edesc, req, iv_contig);  in ablkcipher_decrypt()
3210 	desc = edesc->hw_desc;  in ablkcipher_decrypt()
3213 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in ablkcipher_decrypt()
3214 		       desc_bytes(edesc->hw_desc), 1);  in ablkcipher_decrypt()
3221 		ablkcipher_unmap(jrdev, edesc, req);  in ablkcipher_decrypt()
3222 		kfree(edesc);  in ablkcipher_decrypt()
3245 	struct ablkcipher_edesc *edesc;  in ablkcipher_giv_edesc_alloc()  local
3286 	edesc = kmalloc(sizeof(*edesc) + desc_bytes +  in ablkcipher_giv_edesc_alloc()
3288 	if (!edesc) {  in ablkcipher_giv_edesc_alloc()
3293 	edesc->src_nents = src_nents;  in ablkcipher_giv_edesc_alloc()
3294 	edesc->src_chained = src_chained;  in ablkcipher_giv_edesc_alloc()
3295 	edesc->dst_nents = dst_nents;  in ablkcipher_giv_edesc_alloc()
3296 	edesc->dst_chained = dst_chained;  in ablkcipher_giv_edesc_alloc()
3297 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ablkcipher_giv_edesc_alloc()
3298 	edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +  in ablkcipher_giv_edesc_alloc()
3303 		sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);  in ablkcipher_giv_edesc_alloc()
3308 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,  in ablkcipher_giv_edesc_alloc()
3312 				   edesc->sec4_sg + sec4_sg_index, 0);  in ablkcipher_giv_edesc_alloc()
3315 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ablkcipher_giv_edesc_alloc()
3317 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ablkcipher_giv_edesc_alloc()
3321 	edesc->iv_dma = iv_dma;  in ablkcipher_giv_edesc_alloc()
3326 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,  in ablkcipher_giv_edesc_alloc()
3331 	return edesc;  in ablkcipher_giv_edesc_alloc()
3337 	struct ablkcipher_edesc *edesc;  in ablkcipher_givencrypt()  local
3346 	edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *  in ablkcipher_givencrypt()
3348 	if (IS_ERR(edesc))  in ablkcipher_givencrypt()
3349 		return PTR_ERR(edesc);  in ablkcipher_givencrypt()
3353 				edesc, req, iv_contig);  in ablkcipher_givencrypt()
3357 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in ablkcipher_givencrypt()
3358 		       desc_bytes(edesc->hw_desc), 1);  in ablkcipher_givencrypt()
3360 	desc = edesc->hw_desc;  in ablkcipher_givencrypt()
3366 		ablkcipher_unmap(jrdev, edesc, req);  in ablkcipher_givencrypt()
3367 		kfree(edesc);  in ablkcipher_givencrypt()