195 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_crypt()  local
199 	memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));  in crypto_gcm_init_crypt()
202 	sg_init_table(pctx->src, 2);  in crypto_gcm_init_crypt()
203 	sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));  in crypto_gcm_init_crypt()
204 	scatterwalk_sg_chain(pctx->src, 2, req->src);  in crypto_gcm_init_crypt()
206 	dst = pctx->src;  in crypto_gcm_init_crypt()
208 		sg_init_table(pctx->dst, 2);  in crypto_gcm_init_crypt()
209 		sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));  in crypto_gcm_init_crypt()
210 		scatterwalk_sg_chain(pctx->dst, 2, req->dst);  in crypto_gcm_init_crypt()
211 		dst = pctx->dst;  in crypto_gcm_init_crypt()
215 	ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,  in crypto_gcm_init_crypt()
216 				     cryptlen + sizeof(pctx->auth_tag),  in crypto_gcm_init_crypt()
230 			   struct crypto_gcm_req_priv_ctx *pctx,  in gcm_hash_update()  argument
235 	struct ahash_request *ahreq = &pctx->u.ahreq;  in gcm_hash_update()
245 			   struct crypto_gcm_req_priv_ctx *pctx,  in gcm_hash_remain()  argument
249 	struct ahash_request *ahreq = &pctx->u.ahreq;  in gcm_hash_remain()
253 	sg_init_one(pctx->src, gcm_zeroes, remain);  in gcm_hash_remain()
254 	ahash_request_set_crypt(ahreq, pctx->src, NULL, remain);  in gcm_hash_remain()
260 			struct crypto_gcm_req_priv_ctx *pctx)  in gcm_hash_len()  argument
262 	struct ahash_request *ahreq = &pctx->u.ahreq;  in gcm_hash_len()
263 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;  in gcm_hash_len()
268 	memcpy(pctx->iauth_tag, &lengths, 16);  in gcm_hash_len()
269 	sg_init_one(pctx->src, pctx->iauth_tag, 16);  in gcm_hash_len()
272 	ahash_request_set_crypt(ahreq, pctx->src,  in gcm_hash_len()
279 			  struct crypto_gcm_req_priv_ctx *pctx)  in gcm_hash_final()  argument
281 	struct ahash_request *ahreq = &pctx->u.ahreq;  in gcm_hash_final()
285 	ahash_request_set_crypt(ahreq, NULL, pctx->iauth_tag, 0);  in gcm_hash_final()
292 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_final_done()  local
293 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;  in __gcm_hash_final_done()
296 		crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);  in __gcm_hash_final_done()
310 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_len_done()  local
313 		err = gcm_hash_final(req, pctx);  in __gcm_hash_len_done()
330 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_crypt_remain_done()  local
333 		err = gcm_hash_len(req, pctx);  in __gcm_hash_crypt_remain_done()
351 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_crypt_done()  local
352 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;  in __gcm_hash_crypt_done()
358 		err = gcm_hash_remain(req, pctx, remain,  in __gcm_hash_crypt_done()
376 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_assoc_remain_done()  local
377 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;  in __gcm_hash_assoc_remain_done()
385 		err = gcm_hash_update(req, pctx, compl,  in __gcm_hash_assoc_remain_done()
407 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_assoc_done()  local
413 		err = gcm_hash_remain(req, pctx, remain,  in __gcm_hash_assoc_done()
431 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_init_done()  local
439 		err = gcm_hash_update(req, pctx, compl,  in __gcm_hash_init_done()
459 		    struct crypto_gcm_req_priv_ctx *pctx)  in gcm_hash()  argument
461 	struct ahash_request *ahreq = &pctx->u.ahreq;  in gcm_hash()
462 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;  in gcm_hash()
477 	err = gcm_hash_update(req, pctx, compl, req->assoc, req->assoclen);  in gcm_hash()
481 		err = gcm_hash_remain(req, pctx, remain,  in gcm_hash()
488 	err = gcm_hash_update(req, pctx, compl, gctx->src, gctx->cryptlen);  in gcm_hash()
492 		err = gcm_hash_remain(req, pctx, remain,  in gcm_hash()
497 	err = gcm_hash_len(req, pctx);  in gcm_hash()
500 	err = gcm_hash_final(req, pctx);  in gcm_hash()
508 			      struct crypto_gcm_req_priv_ctx *pctx)  in gcm_enc_copy_hash()  argument
511 	u8 *auth_tag = pctx->auth_tag;  in gcm_enc_copy_hash()
519 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_enc_hash_done()  local
522 		gcm_enc_copy_hash(req, pctx);  in gcm_enc_hash_done()
530 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_encrypt_done()  local
533 		err = gcm_hash(req, pctx);  in gcm_encrypt_done()
537 			crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);  in gcm_encrypt_done()
538 			gcm_enc_copy_hash(req, pctx);  in gcm_encrypt_done()
547 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_encrypt()  local
548 	struct ablkcipher_request *abreq = &pctx->u.abreq;  in crypto_gcm_encrypt()
549 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;  in crypto_gcm_encrypt()
564 	err = gcm_hash(req, pctx);  in crypto_gcm_encrypt()
568 	crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);  in crypto_gcm_encrypt()
569 	gcm_enc_copy_hash(req, pctx);  in crypto_gcm_encrypt()
575 			     struct crypto_gcm_req_priv_ctx *pctx)  in crypto_gcm_verify()  argument
578 	u8 *auth_tag = pctx->auth_tag;  in crypto_gcm_verify()
579 	u8 *iauth_tag = pctx->iauth_tag;  in crypto_gcm_verify()
591 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_decrypt_done()  local
594 		err = crypto_gcm_verify(req, pctx);  in gcm_decrypt_done()
601 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_dec_hash_done()  local
602 	struct ablkcipher_request *abreq = &pctx->u.abreq;  in gcm_dec_hash_done()
603 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;  in gcm_dec_hash_done()
613 			err = crypto_gcm_verify(req, pctx);  in gcm_dec_hash_done()
622 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_decrypt()  local
623 	struct ablkcipher_request *abreq = &pctx->u.abreq;  in crypto_gcm_decrypt()
624 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;  in crypto_gcm_decrypt()
637 	err = gcm_hash(req, pctx);  in crypto_gcm_decrypt()
648 	return crypto_gcm_verify(req, pctx);  in crypto_gcm_decrypt()