791 			    struct aead_request *areq)  in ipsec_esp_unmap()  argument
799 		talitos_unmap_sg_chain(dev, areq->assoc, DMA_TO_DEVICE);  in ipsec_esp_unmap()
800 	else if (areq->assoclen)  in ipsec_esp_unmap()
802 		dma_unmap_sg(dev, areq->assoc,  in ipsec_esp_unmap()
806 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst);  in ipsec_esp_unmap()
820 	struct aead_request *areq = context;  in ipsec_esp_encrypt_done()  local
821 	struct crypto_aead *authenc = crypto_aead_reqtfm(areq);  in ipsec_esp_encrypt_done()
829 	ipsec_esp_unmap(dev, edesc, areq);  in ipsec_esp_encrypt_done()
836 		sg = sg_last(areq->dst, edesc->dst_nents);  in ipsec_esp_encrypt_done()
843 	aead_request_complete(areq, err);  in ipsec_esp_encrypt_done()
940 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,  in ipsec_esp()  argument
945 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);  in ipsec_esp()
949 	unsigned int cryptlen = areq->cryptlen;  in ipsec_esp()
960 	desc->ptr[1].len = cpu_to_be16(areq->assoclen + ivsize);  in ipsec_esp()
970 		sg_count = sg_to_link_tbl(areq->assoc, edesc->assoc_nents - 1,  in ipsec_esp()
971 					  areq->assoclen, tbl_ptr);  in ipsec_esp()
984 		if (areq->assoclen)  in ipsec_esp()
986 				       sg_dma_address(areq->assoc));  in ipsec_esp()
1013 	sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,  in ipsec_esp()
1014 				  (areq->src == areq->dst) ? DMA_BIDIRECTIONAL  in ipsec_esp()
1019 		to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src));  in ipsec_esp()
1026 		sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len,  in ipsec_esp()
1037 				       sg_dma_address(areq->src));  in ipsec_esp()
1045 	if (areq->src != areq->dst)  in ipsec_esp()
1046 		sg_count = talitos_map_sg(dev, areq->dst,  in ipsec_esp()
1051 		to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst));  in ipsec_esp()
1058 		sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,  in ipsec_esp()
1082 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);  in ipsec_esp()
1084 		ipsec_esp_unmap(dev, edesc, areq);  in ipsec_esp()
1217 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,  in aead_edesc_alloc()  argument
1220 	struct crypto_aead *authenc = crypto_aead_reqtfm(areq);  in aead_edesc_alloc()
1224 	return talitos_edesc_alloc(ctx->dev, areq->assoc, areq->src, areq->dst,  in aead_edesc_alloc()
1225 				   iv, areq->assoclen, areq->cryptlen,  in aead_edesc_alloc()
1227 				   areq->base.flags, encrypt);  in aead_edesc_alloc()
1300 	struct aead_request *areq = &req->areq;  in aead_givencrypt()  local
1301 	struct crypto_aead *authenc = crypto_aead_reqtfm(areq);  in aead_givencrypt()
1306 	edesc = aead_edesc_alloc(areq, req->giv, 0, true);  in aead_givencrypt()
1317 	return ipsec_esp(edesc, areq, req->seq, ipsec_esp_encrypt_done);  in aead_givencrypt()
1333 				  struct ablkcipher_request *areq)  in common_nonsnoop_unmap()  argument
1339 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst);  in common_nonsnoop_unmap()
1350 	struct ablkcipher_request *areq = context;  in ablkcipher_done()  local
1355 	common_nonsnoop_unmap(dev, edesc, areq);  in ablkcipher_done()
1359 	areq->base.complete(&areq->base, err);  in ablkcipher_done()
1363 			   struct ablkcipher_request *areq,  in common_nonsnoop()  argument
1368 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);  in common_nonsnoop()
1372 	unsigned int cryptlen = areq->nbytes;  in common_nonsnoop()
1396 	sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,  in common_nonsnoop()
1397 				  (areq->src == areq->dst) ? DMA_BIDIRECTIONAL  in common_nonsnoop()
1402 		to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src));  in common_nonsnoop()
1404 		sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,  in common_nonsnoop()
1415 				       sg_dma_address(areq->src));  in common_nonsnoop()
1423 	if (areq->src != areq->dst)  in common_nonsnoop()
1424 		sg_count = talitos_map_sg(dev, areq->dst,  in common_nonsnoop()
1429 		to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst));  in common_nonsnoop()
1438 		sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,  in common_nonsnoop()
1453 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);  in common_nonsnoop()
1455 		common_nonsnoop_unmap(dev, edesc, areq);  in common_nonsnoop()
1462 						    areq, bool encrypt)  in ablkcipher_edesc_alloc()
1464 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);  in ablkcipher_edesc_alloc()
1468 	return talitos_edesc_alloc(ctx->dev, NULL, areq->src, areq->dst,  in ablkcipher_edesc_alloc()
1469 				   areq->info, 0, areq->nbytes, 0, ivsize, 0,  in ablkcipher_edesc_alloc()
1470 				   areq->base.flags, encrypt);  in ablkcipher_edesc_alloc()
1473 static int ablkcipher_encrypt(struct ablkcipher_request *areq)  in ablkcipher_encrypt()  argument
1475 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);  in ablkcipher_encrypt()
1480 	edesc = ablkcipher_edesc_alloc(areq, true);  in ablkcipher_encrypt()
1487 	return common_nonsnoop(edesc, areq, ablkcipher_done);  in ablkcipher_encrypt()
1490 static int ablkcipher_decrypt(struct ablkcipher_request *areq)  in ablkcipher_decrypt()  argument
1492 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);  in ablkcipher_decrypt()
1497 	edesc = ablkcipher_edesc_alloc(areq, false);  in ablkcipher_decrypt()
1503 	return common_nonsnoop(edesc, areq, ablkcipher_done);  in ablkcipher_decrypt()
1508 				       struct ahash_request *areq)  in common_nonsnoop_hash_unmap()  argument
1510 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in common_nonsnoop_hash_unmap()
1535 	struct ahash_request *areq = context;  in ahash_done()  local
1538 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in ahash_done()
1545 	common_nonsnoop_hash_unmap(dev, edesc, areq);  in ahash_done()
1549 	areq->base.complete(&areq->base, err);  in ahash_done()
1553 				struct ahash_request *areq, unsigned int length,  in common_nonsnoop_hash()  argument
1558 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in common_nonsnoop_hash()
1560 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in common_nonsnoop_hash()
1624 				       areq->result, 0, DMA_FROM_DEVICE);  in common_nonsnoop_hash()
1633 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);  in common_nonsnoop_hash()
1635 		common_nonsnoop_hash_unmap(dev, edesc, areq);  in common_nonsnoop_hash()
1641 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,  in ahash_edesc_alloc()  argument
1644 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in ahash_edesc_alloc()
1646 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in ahash_edesc_alloc()
1649 				   nbytes, 0, 0, 0, areq->base.flags, false);  in ahash_edesc_alloc()
1652 static int ahash_init(struct ahash_request *areq)  in ahash_init()  argument
1654 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in ahash_init()
1655 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in ahash_init()
1673 static int ahash_init_sha224_swinit(struct ahash_request *areq)  in ahash_init_sha224_swinit()  argument
1675 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in ahash_init_sha224_swinit()
1677 	ahash_init(areq);  in ahash_init_sha224_swinit()
1696 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)  in ahash_process_req()  argument
1698 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in ahash_process_req()
1700 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in ahash_process_req()
1711 		sg_copy_to_buffer(areq->src,  in ahash_process_req()
1712 				  sg_count(areq->src, nbytes, &chained),  in ahash_process_req()
1739 			scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src);  in ahash_process_req()
1742 		req_ctx->psrc = areq->src;  in ahash_process_req()
1745 		int nents = sg_count(areq->src, nbytes, &chained);  in ahash_process_req()
1746 		sg_pcopy_to_buffer(areq->src, nents,  in ahash_process_req()
1754 	edesc = ahash_edesc_alloc(areq, nbytes_to_hash);  in ahash_process_req()
1776 	return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,  in ahash_process_req()
1780 static int ahash_update(struct ahash_request *areq)  in ahash_update()  argument
1782 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in ahash_update()
1786 	return ahash_process_req(areq, areq->nbytes);  in ahash_update()
1789 static int ahash_final(struct ahash_request *areq)  in ahash_final()  argument
1791 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in ahash_final()
1795 	return ahash_process_req(areq, 0);  in ahash_final()
1798 static int ahash_finup(struct ahash_request *areq)  in ahash_finup()  argument
1800 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in ahash_finup()
1804 	return ahash_process_req(areq, areq->nbytes);  in ahash_finup()
1807 static int ahash_digest(struct ahash_request *areq)  in ahash_digest()  argument
1809 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);  in ahash_digest()
1810 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);  in ahash_digest()
1812 	ahash->init(areq);  in ahash_digest()
1815 	return ahash_process_req(areq, areq->nbytes);  in ahash_digest()