29 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_req_iter_init()  local
30 	unsigned int len = req->nbytes + creq->cache_ptr;  in mv_cesa_ahash_req_iter_init()
32 	if (!creq->last_req)  in mv_cesa_ahash_req_iter_init()
37 	iter->src.op_offset = creq->cache_ptr;  in mv_cesa_ahash_req_iter_init()
48 static inline int mv_cesa_ahash_dma_alloc_cache(struct mv_cesa_ahash_req *creq,  in mv_cesa_ahash_dma_alloc_cache()  argument
51 	struct mv_cesa_ahash_dma_req *dreq = &creq->req.dma;  in mv_cesa_ahash_dma_alloc_cache()
53 	creq->cache = dma_pool_alloc(cesa_dev->dma->cache_pool, flags,  in mv_cesa_ahash_dma_alloc_cache()
55 	if (!creq->cache)  in mv_cesa_ahash_dma_alloc_cache()
61 static inline int mv_cesa_ahash_std_alloc_cache(struct mv_cesa_ahash_req *creq,  in mv_cesa_ahash_std_alloc_cache()  argument
64 	creq->cache = kzalloc(CESA_MAX_HASH_BLOCK_SIZE, flags);  in mv_cesa_ahash_std_alloc_cache()
65 	if (!creq->cache)  in mv_cesa_ahash_std_alloc_cache()
73 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_alloc_cache()  local
78 	if (creq->cache)  in mv_cesa_ahash_alloc_cache()
81 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_alloc_cache()
82 		ret = mv_cesa_ahash_dma_alloc_cache(creq, flags);  in mv_cesa_ahash_alloc_cache()
84 		ret = mv_cesa_ahash_std_alloc_cache(creq, flags);  in mv_cesa_ahash_alloc_cache()
89 static inline void mv_cesa_ahash_dma_free_cache(struct mv_cesa_ahash_req *creq)  in mv_cesa_ahash_dma_free_cache()  argument
91 	dma_pool_free(cesa_dev->dma->cache_pool, creq->cache,  in mv_cesa_ahash_dma_free_cache()
92 		      creq->req.dma.cache_dma);  in mv_cesa_ahash_dma_free_cache()
95 static inline void mv_cesa_ahash_std_free_cache(struct mv_cesa_ahash_req *creq)  in mv_cesa_ahash_std_free_cache()  argument
97 	kfree(creq->cache);  in mv_cesa_ahash_std_free_cache()
100 static void mv_cesa_ahash_free_cache(struct mv_cesa_ahash_req *creq)  in mv_cesa_ahash_free_cache()  argument
102 	if (!creq->cache)  in mv_cesa_ahash_free_cache()
105 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_free_cache()
106 		mv_cesa_ahash_dma_free_cache(creq);  in mv_cesa_ahash_free_cache()
108 		mv_cesa_ahash_std_free_cache(creq);  in mv_cesa_ahash_free_cache()
110 	creq->cache = NULL;  in mv_cesa_ahash_free_cache()
139 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_last_cleanup()  local
141 	mv_cesa_ahash_dma_free_padding(&creq->req.dma);  in mv_cesa_ahash_dma_last_cleanup()
146 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_cleanup()  local
148 	dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE);  in mv_cesa_ahash_dma_cleanup()
149 	mv_cesa_dma_cleanup(&creq->req.dma.base);  in mv_cesa_ahash_dma_cleanup()
154 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_cleanup()  local
156 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_cleanup()
162 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_last_cleanup()  local
164 	mv_cesa_ahash_free_cache(creq);  in mv_cesa_ahash_last_cleanup()
166 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_last_cleanup()
170 static int mv_cesa_ahash_pad_len(struct mv_cesa_ahash_req *creq)  in mv_cesa_ahash_pad_len()  argument
174 	index = creq->len & CESA_HASH_BLOCK_SIZE_MSK;  in mv_cesa_ahash_pad_len()
180 static int mv_cesa_ahash_pad_req(struct mv_cesa_ahash_req *creq, u8 *buf)  in mv_cesa_ahash_pad_req()  argument
186 	index = creq->len & CESA_HASH_BLOCK_SIZE_MSK;  in mv_cesa_ahash_pad_req()
187 	padlen = mv_cesa_ahash_pad_len(creq);  in mv_cesa_ahash_pad_req()
190 	if (creq->algo_le) {  in mv_cesa_ahash_pad_req()
191 		__le64 bits = cpu_to_le64(creq->len << 3);  in mv_cesa_ahash_pad_req()
194 		__be64 bits = cpu_to_be64(creq->len << 3);  in mv_cesa_ahash_pad_req()
203 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_step()  local
204 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_step()
211 	if (creq->cache_ptr)  in mv_cesa_ahash_std_step()
213 			    creq->cache, creq->cache_ptr);  in mv_cesa_ahash_std_step()
215 	len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset,  in mv_cesa_ahash_std_step()
218 	if (!creq->last_req) {  in mv_cesa_ahash_std_step()
223 	if (len - creq->cache_ptr)  in mv_cesa_ahash_std_step()
224 		sreq->offset += sg_pcopy_to_buffer(req->src, creq->src_nents,  in mv_cesa_ahash_std_step()
227 						   creq->cache_ptr,  in mv_cesa_ahash_std_step()
228 						   len - creq->cache_ptr,  in mv_cesa_ahash_std_step()
231 	op = &creq->op_tmpl;  in mv_cesa_ahash_std_step()
235 	if (creq->last_req && sreq->offset == req->nbytes &&  in mv_cesa_ahash_std_step()
236 	    creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) {  in mv_cesa_ahash_std_step()
246 		    creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) {  in mv_cesa_ahash_std_step()
247 			mv_cesa_set_mac_op_total_len(op, creq->len);  in mv_cesa_ahash_std_step()
249 			int trailerlen = mv_cesa_ahash_pad_len(creq) + 8;  in mv_cesa_ahash_std_step()
254 				memcpy_fromio(creq->cache,  in mv_cesa_ahash_std_step()
259 				len += mv_cesa_ahash_pad_req(creq,  in mv_cesa_ahash_std_step()
281 	creq->cache_ptr = new_cache_ptr;  in mv_cesa_ahash_std_step()
290 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_process()  local
291 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_process()
293 	if (sreq->offset < (req->nbytes - creq->cache_ptr))  in mv_cesa_ahash_std_process()
301 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_prepare()  local
302 	struct mv_cesa_tdma_req *dreq = &creq->req.dma.base;  in mv_cesa_ahash_dma_prepare()
309 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_prepare()  local
310 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_prepare()
314 	mv_cesa_adjust_op(engine, &creq->op_tmpl);  in mv_cesa_ahash_std_prepare()
315 	memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl));  in mv_cesa_ahash_std_prepare()
321 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq);  in mv_cesa_ahash_step()  local
323 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_step()
324 		mv_cesa_dma_step(&creq->req.dma.base);  in mv_cesa_ahash_step()
332 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq);  in mv_cesa_ahash_process()  local
333 	struct mv_cesa_engine *engine = creq->req.base.engine;  in mv_cesa_ahash_process()
337 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_process()
338 		ret = mv_cesa_dma_process(&creq->req.dma.base, status);  in mv_cesa_ahash_process()
347 		creq->state[i] = readl_relaxed(engine->regs + CESA_IVDIG(i));  in mv_cesa_ahash_process()
349 	if (creq->cache_ptr)  in mv_cesa_ahash_process()
350 		sg_pcopy_to_buffer(ahashreq->src, creq->src_nents,  in mv_cesa_ahash_process()
351 				   creq->cache,  in mv_cesa_ahash_process()
352 				   creq->cache_ptr,  in mv_cesa_ahash_process()
353 				   ahashreq->nbytes - creq->cache_ptr);  in mv_cesa_ahash_process()
355 	if (creq->last_req) {  in mv_cesa_ahash_process()
360 		if (creq->algo_le) {  in mv_cesa_ahash_process()
364 				result[i] = cpu_to_le32(creq->state[i]);  in mv_cesa_ahash_process()
369 				result[i] = cpu_to_be32(creq->state[i]);  in mv_cesa_ahash_process()
380 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq);  in mv_cesa_ahash_prepare()  local
384 	creq->req.base.engine = engine;  in mv_cesa_ahash_prepare()
386 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_prepare()
393 		writel_relaxed(creq->state[i], engine->regs + CESA_IVDIG(i));  in mv_cesa_ahash_prepare()
399 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq);  in mv_cesa_ahash_req_cleanup()  local
401 	if (creq->last_req)  in mv_cesa_ahash_req_cleanup()
417 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_init()  local
419 	memset(creq, 0, sizeof(*creq));  in mv_cesa_ahash_init()
427 	creq->op_tmpl = *tmpl;  in mv_cesa_ahash_init()
428 	creq->len = 0;  in mv_cesa_ahash_init()
429 	creq->algo_le = algo_le;  in mv_cesa_ahash_init()
447 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_cache_req()  local
450 	if (((creq->cache_ptr + req->nbytes) & CESA_HASH_BLOCK_SIZE_MSK) &&  in mv_cesa_ahash_cache_req()
451 	    !creq->last_req) {  in mv_cesa_ahash_cache_req()
457 	if (creq->cache_ptr + req->nbytes < 64 && !creq->last_req) {  in mv_cesa_ahash_cache_req()
463 		sg_pcopy_to_buffer(req->src, creq->src_nents,  in mv_cesa_ahash_cache_req()
464 				   creq->cache + creq->cache_ptr,  in mv_cesa_ahash_cache_req()
467 		creq->cache_ptr += req->nbytes;  in mv_cesa_ahash_cache_req()
504 			    struct mv_cesa_ahash_req *creq,  in mv_cesa_ahash_dma_add_cache()  argument
507 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_add_cache()
509 	if (!creq->cache_ptr)  in mv_cesa_ahash_dma_add_cache()
515 					     creq->cache_ptr,  in mv_cesa_ahash_dma_add_cache()
523 			   struct mv_cesa_ahash_req *creq,  in mv_cesa_ahash_dma_last_req()  argument
526 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_last_req()
535 	if (creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX && frag_len) {  in mv_cesa_ahash_dma_last_req()
536 		op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len,  in mv_cesa_ahash_dma_last_req()
541 		mv_cesa_set_mac_op_total_len(op, creq->len);  in mv_cesa_ahash_dma_last_req()
559 	trailerlen = mv_cesa_ahash_pad_req(creq, ahashdreq->padding);  in mv_cesa_ahash_dma_last_req()
572 		op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len + len,  in mv_cesa_ahash_dma_last_req()
593 	return mv_cesa_dma_add_frag(chain, &creq->op_tmpl, trailerlen - padoff,  in mv_cesa_ahash_dma_last_req()
599 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_req_init()  local
602 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_req_init()
612 	if (creq->src_nents) {  in mv_cesa_ahash_dma_req_init()
613 		ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents,  in mv_cesa_ahash_dma_req_init()
628 	ret = mv_cesa_ahash_dma_add_cache(&dreq->chain, &iter, creq, flags);  in mv_cesa_ahash_dma_req_init()
650 			op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl,  in mv_cesa_ahash_dma_req_init()
667 	if (creq->last_req)  in mv_cesa_ahash_dma_req_init()
668 		op = mv_cesa_ahash_dma_last_req(&dreq->chain, &iter, creq,  in mv_cesa_ahash_dma_req_init()
671 		op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl,  in mv_cesa_ahash_dma_req_init()
686 	if (!creq->last_req)  in mv_cesa_ahash_dma_req_init()
687 		creq->cache_ptr = req->nbytes + creq->cache_ptr -  in mv_cesa_ahash_dma_req_init()
690 		creq->cache_ptr = 0;  in mv_cesa_ahash_dma_req_init()
696 	dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE);  in mv_cesa_ahash_dma_req_init()
706 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_req_init()  local
710 		creq->req.base.type = CESA_DMA_REQ;  in mv_cesa_ahash_req_init()
712 		creq->req.base.type = CESA_STD_REQ;  in mv_cesa_ahash_req_init()
714 	creq->src_nents = sg_nents_for_len(req->src, req->nbytes);  in mv_cesa_ahash_req_init()
723 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_req_init()
731 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_update()  local
735 	creq->len += req->nbytes;  in mv_cesa_ahash_update()
752 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_final()  local
753 	struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl;  in mv_cesa_ahash_final()
757 	mv_cesa_set_mac_op_total_len(tmpl, creq->len);  in mv_cesa_ahash_final()
758 	creq->last_req = true;  in mv_cesa_ahash_final()
777 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_finup()  local
778 	struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl;  in mv_cesa_ahash_finup()
782 	creq->len += req->nbytes;  in mv_cesa_ahash_finup()
783 	mv_cesa_set_mac_op_total_len(tmpl, creq->len);  in mv_cesa_ahash_finup()
784 	creq->last_req = true;  in mv_cesa_ahash_finup()
804 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_export()  local
810 	*len = creq->len;  in mv_cesa_ahash_export()
811 	memcpy(hash, creq->state, digsize);  in mv_cesa_ahash_export()
813 	if (creq->cache)  in mv_cesa_ahash_export()
814 		memcpy(cache, creq->cache, creq->cache_ptr);  in mv_cesa_ahash_export()
823 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_import()  local
835 		mv_cesa_update_op_cfg(&creq->op_tmpl,  in mv_cesa_ahash_import()
839 	creq->len = len;  in mv_cesa_ahash_import()
840 	memcpy(creq->state, hash, digsize);  in mv_cesa_ahash_import()
841 	creq->cache_ptr = 0;  in mv_cesa_ahash_import()
851 	memcpy(creq->cache, cache, cache_ptr);  in mv_cesa_ahash_import()
852 	creq->cache_ptr = cache_ptr;  in mv_cesa_ahash_import()