27 			    struct ahash_request *req)  in mv_cesa_ahash_req_iter_init()  argument
29 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_req_iter_init()
30 	unsigned int len = req->nbytes + creq->cache_ptr;  in mv_cesa_ahash_req_iter_init()
36 	mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE);  in mv_cesa_ahash_req_iter_init()
51 	struct mv_cesa_ahash_dma_req *dreq = &creq->req.dma;  in mv_cesa_ahash_dma_alloc_cache()
71 static int mv_cesa_ahash_alloc_cache(struct ahash_request *req)  in mv_cesa_ahash_alloc_cache()  argument
73 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_alloc_cache()
74 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?  in mv_cesa_ahash_alloc_cache()
81 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_alloc_cache()
92 		      creq->req.dma.cache_dma);  in mv_cesa_ahash_dma_free_cache()
105 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_free_cache()
113 static int mv_cesa_ahash_dma_alloc_padding(struct mv_cesa_ahash_dma_req *req,  in mv_cesa_ahash_dma_alloc_padding()  argument
116 	if (req->padding)  in mv_cesa_ahash_dma_alloc_padding()
119 	req->padding = dma_pool_alloc(cesa_dev->dma->padding_pool, flags,  in mv_cesa_ahash_dma_alloc_padding()
120 				      &req->padding_dma);  in mv_cesa_ahash_dma_alloc_padding()
121 	if (!req->padding)  in mv_cesa_ahash_dma_alloc_padding()
127 static void mv_cesa_ahash_dma_free_padding(struct mv_cesa_ahash_dma_req *req)  in mv_cesa_ahash_dma_free_padding()  argument
129 	if (!req->padding)  in mv_cesa_ahash_dma_free_padding()
132 	dma_pool_free(cesa_dev->dma->padding_pool, req->padding,  in mv_cesa_ahash_dma_free_padding()
133 		      req->padding_dma);  in mv_cesa_ahash_dma_free_padding()
134 	req->padding = NULL;  in mv_cesa_ahash_dma_free_padding()
137 static inline void mv_cesa_ahash_dma_last_cleanup(struct ahash_request *req)  in mv_cesa_ahash_dma_last_cleanup()  argument
139 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_last_cleanup()
141 	mv_cesa_ahash_dma_free_padding(&creq->req.dma);  in mv_cesa_ahash_dma_last_cleanup()
144 static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req)  in mv_cesa_ahash_dma_cleanup()  argument
146 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_cleanup()
148 	dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE);  in mv_cesa_ahash_dma_cleanup()
149 	mv_cesa_dma_cleanup(&creq->req.dma.base);  in mv_cesa_ahash_dma_cleanup()
152 static inline void mv_cesa_ahash_cleanup(struct ahash_request *req)  in mv_cesa_ahash_cleanup()  argument
154 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_cleanup()
156 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_cleanup()
157 		mv_cesa_ahash_dma_cleanup(req);  in mv_cesa_ahash_cleanup()
160 static void mv_cesa_ahash_last_cleanup(struct ahash_request *req)  in mv_cesa_ahash_last_cleanup()  argument
162 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_last_cleanup()
166 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_last_cleanup()
167 		mv_cesa_ahash_dma_last_cleanup(req);  in mv_cesa_ahash_last_cleanup()
201 static void mv_cesa_ahash_std_step(struct ahash_request *req)  in mv_cesa_ahash_std_step()  argument
203 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_step()
204 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_step()
215 	len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset,  in mv_cesa_ahash_std_step()
224 		sreq->offset += sg_pcopy_to_buffer(req->src, creq->src_nents,  in mv_cesa_ahash_std_step()
235 	if (creq->last_req && sreq->offset == req->nbytes &&  in mv_cesa_ahash_std_step()
288 static int mv_cesa_ahash_std_process(struct ahash_request *req, u32 status)  in mv_cesa_ahash_std_process()  argument
290 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_process()
291 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_process()
293 	if (sreq->offset < (req->nbytes - creq->cache_ptr))  in mv_cesa_ahash_std_process()
299 static inline void mv_cesa_ahash_dma_prepare(struct ahash_request *req)  in mv_cesa_ahash_dma_prepare()  argument
301 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_prepare()
302 	struct mv_cesa_tdma_req *dreq = &creq->req.dma.base;  in mv_cesa_ahash_dma_prepare()
307 static void mv_cesa_ahash_std_prepare(struct ahash_request *req)  in mv_cesa_ahash_std_prepare()  argument
309 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_prepare()
310 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_prepare()
318 static void mv_cesa_ahash_step(struct crypto_async_request *req)  in mv_cesa_ahash_step()  argument
320 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_step()
323 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_step()
324 		mv_cesa_dma_step(&creq->req.dma.base);  in mv_cesa_ahash_step()
329 static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status)  in mv_cesa_ahash_process()  argument
331 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_process()
333 	struct mv_cesa_engine *engine = creq->req.base.engine;  in mv_cesa_ahash_process()
337 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_process()
338 		ret = mv_cesa_dma_process(&creq->req.dma.base, status);  in mv_cesa_ahash_process()
376 static void mv_cesa_ahash_prepare(struct crypto_async_request *req,  in mv_cesa_ahash_prepare()  argument
379 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_prepare()
384 	creq->req.base.engine = engine;  in mv_cesa_ahash_prepare()
386 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_prepare()
396 static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req)  in mv_cesa_ahash_req_cleanup()  argument
398 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_req_cleanup()
414 static int mv_cesa_ahash_init(struct ahash_request *req,  in mv_cesa_ahash_init()  argument
417 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_init()
445 static int mv_cesa_ahash_cache_req(struct ahash_request *req, bool *cached)  in mv_cesa_ahash_cache_req()  argument
447 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_cache_req()
450 	if (((creq->cache_ptr + req->nbytes) & CESA_HASH_BLOCK_SIZE_MSK) &&  in mv_cesa_ahash_cache_req()
452 		ret = mv_cesa_ahash_alloc_cache(req);  in mv_cesa_ahash_cache_req()
457 	if (creq->cache_ptr + req->nbytes < 64 && !creq->last_req) {  in mv_cesa_ahash_cache_req()
460 		if (!req->nbytes)  in mv_cesa_ahash_cache_req()
463 		sg_pcopy_to_buffer(req->src, creq->src_nents,  in mv_cesa_ahash_cache_req()
465 				   req->nbytes, 0);  in mv_cesa_ahash_cache_req()
467 		creq->cache_ptr += req->nbytes;  in mv_cesa_ahash_cache_req()
507 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_add_cache()
526 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_last_req()
597 static int mv_cesa_ahash_dma_req_init(struct ahash_request *req)  in mv_cesa_ahash_dma_req_init()  argument
599 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_req_init()
600 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?  in mv_cesa_ahash_dma_req_init()
602 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_req_init()
613 		ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents,  in mv_cesa_ahash_dma_req_init()
622 	mv_cesa_ahash_req_iter_init(&iter, req);  in mv_cesa_ahash_dma_req_init()
687 		creq->cache_ptr = req->nbytes + creq->cache_ptr -  in mv_cesa_ahash_dma_req_init()
696 	dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE);  in mv_cesa_ahash_dma_req_init()
699 	mv_cesa_ahash_last_cleanup(req);  in mv_cesa_ahash_dma_req_init()
704 static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached)  in mv_cesa_ahash_req_init()  argument
706 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_req_init()
710 		creq->req.base.type = CESA_DMA_REQ;  in mv_cesa_ahash_req_init()
712 		creq->req.base.type = CESA_STD_REQ;  in mv_cesa_ahash_req_init()
714 	creq->src_nents = sg_nents_for_len(req->src, req->nbytes);  in mv_cesa_ahash_req_init()
716 	ret = mv_cesa_ahash_cache_req(req, cached);  in mv_cesa_ahash_req_init()
723 	if (creq->req.base.type == CESA_DMA_REQ)  in mv_cesa_ahash_req_init()
724 		ret = mv_cesa_ahash_dma_req_init(req);  in mv_cesa_ahash_req_init()
729 static int mv_cesa_ahash_update(struct ahash_request *req)  in mv_cesa_ahash_update()  argument
731 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_update()
735 	creq->len += req->nbytes;  in mv_cesa_ahash_update()
736 	ret = mv_cesa_ahash_req_init(req, &cached);  in mv_cesa_ahash_update()
743 	ret = mv_cesa_queue_req(&req->base);  in mv_cesa_ahash_update()
744 	if (mv_cesa_req_needs_cleanup(&req->base, ret))  in mv_cesa_ahash_update()
745 		mv_cesa_ahash_cleanup(req);  in mv_cesa_ahash_update()
750 static int mv_cesa_ahash_final(struct ahash_request *req)  in mv_cesa_ahash_final()  argument
752 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_final()
759 	req->nbytes = 0;  in mv_cesa_ahash_final()
761 	ret = mv_cesa_ahash_req_init(req, &cached);  in mv_cesa_ahash_final()
768 	ret = mv_cesa_queue_req(&req->base);  in mv_cesa_ahash_final()
769 	if (mv_cesa_req_needs_cleanup(&req->base, ret))  in mv_cesa_ahash_final()
770 		mv_cesa_ahash_cleanup(req);  in mv_cesa_ahash_final()
775 static int mv_cesa_ahash_finup(struct ahash_request *req)  in mv_cesa_ahash_finup()  argument
777 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_finup()
782 	creq->len += req->nbytes;  in mv_cesa_ahash_finup()
786 	ret = mv_cesa_ahash_req_init(req, &cached);  in mv_cesa_ahash_finup()
793 	ret = mv_cesa_queue_req(&req->base);  in mv_cesa_ahash_finup()
794 	if (mv_cesa_req_needs_cleanup(&req->base, ret))  in mv_cesa_ahash_finup()
795 		mv_cesa_ahash_cleanup(req);  in mv_cesa_ahash_finup()
800 static int mv_cesa_ahash_export(struct ahash_request *req, void *hash,  in mv_cesa_ahash_export()  argument
803 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);  in mv_cesa_ahash_export()
804 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_export()
819 static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash,  in mv_cesa_ahash_import()  argument
822 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);  in mv_cesa_ahash_import()
823 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_import()
829 	ret = crypto_ahash_init(req);  in mv_cesa_ahash_import()
847 	ret = mv_cesa_ahash_alloc_cache(req);  in mv_cesa_ahash_import()
857 static int mv_cesa_md5_init(struct ahash_request *req)  in mv_cesa_md5_init()  argument
863 	mv_cesa_ahash_init(req, &tmpl, true);  in mv_cesa_md5_init()
868 static int mv_cesa_md5_export(struct ahash_request *req, void *out)  in mv_cesa_md5_export()  argument
872 	return mv_cesa_ahash_export(req, out_state->hash,  in mv_cesa_md5_export()
876 static int mv_cesa_md5_import(struct ahash_request *req, const void *in)  in mv_cesa_md5_import()  argument
880 	return mv_cesa_ahash_import(req, in_state->hash, in_state->byte_count,  in mv_cesa_md5_import()
884 static int mv_cesa_md5_digest(struct ahash_request *req)  in mv_cesa_md5_digest()  argument
888 	ret = mv_cesa_md5_init(req);  in mv_cesa_md5_digest()
892 	return mv_cesa_ahash_finup(req);  in mv_cesa_md5_digest()
920 static int mv_cesa_sha1_init(struct ahash_request *req)  in mv_cesa_sha1_init()  argument
926 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_sha1_init()
931 static int mv_cesa_sha1_export(struct ahash_request *req, void *out)  in mv_cesa_sha1_export()  argument
935 	return mv_cesa_ahash_export(req, out_state->state, &out_state->count,  in mv_cesa_sha1_export()
939 static int mv_cesa_sha1_import(struct ahash_request *req, const void *in)  in mv_cesa_sha1_import()  argument
943 	return mv_cesa_ahash_import(req, in_state->state, in_state->count,  in mv_cesa_sha1_import()
947 static int mv_cesa_sha1_digest(struct ahash_request *req)  in mv_cesa_sha1_digest()  argument
951 	ret = mv_cesa_sha1_init(req);  in mv_cesa_sha1_digest()
955 	return mv_cesa_ahash_finup(req);  in mv_cesa_sha1_digest()
983 static int mv_cesa_sha256_init(struct ahash_request *req)  in mv_cesa_sha256_init()  argument
989 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_sha256_init()
994 static int mv_cesa_sha256_digest(struct ahash_request *req)  in mv_cesa_sha256_digest()  argument
998 	ret = mv_cesa_sha256_init(req);  in mv_cesa_sha256_digest()
1002 	return mv_cesa_ahash_finup(req);  in mv_cesa_sha256_digest()
1005 static int mv_cesa_sha256_export(struct ahash_request *req, void *out)  in mv_cesa_sha256_export()  argument
1009 	return mv_cesa_ahash_export(req, out_state->state, &out_state->count,  in mv_cesa_sha256_export()
1013 static int mv_cesa_sha256_import(struct ahash_request *req, const void *in)  in mv_cesa_sha256_import()  argument
1017 	return mv_cesa_ahash_import(req, in_state->state, in_state->count,  in mv_cesa_sha256_import()
1051 static void mv_cesa_hmac_ahash_complete(struct crypto_async_request *req,  in mv_cesa_hmac_ahash_complete()  argument
1054 	struct mv_cesa_ahash_result *result = req->data;  in mv_cesa_hmac_ahash_complete()
1063 static int mv_cesa_ahmac_iv_state_init(struct ahash_request *req, u8 *pad,  in mv_cesa_ahmac_iv_state_init()  argument
1070 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in mv_cesa_ahmac_iv_state_init()
1073 	ahash_request_set_crypt(req, &sg, pad, blocksize);  in mv_cesa_ahmac_iv_state_init()
1076 	ret = crypto_ahash_init(req);  in mv_cesa_ahmac_iv_state_init()
1080 	ret = crypto_ahash_update(req);  in mv_cesa_ahmac_iv_state_init()
1088 	ret = crypto_ahash_export(req, state);  in mv_cesa_ahmac_iv_state_init()
1095 static int mv_cesa_ahmac_pad_init(struct ahash_request *req,  in mv_cesa_ahmac_pad_init()  argument
1113 		ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in mv_cesa_ahmac_pad_init()
1117 		ahash_request_set_crypt(req, &sg, ipad, keylen);  in mv_cesa_ahmac_pad_init()
1120 		ret = crypto_ahash_digest(req);  in mv_cesa_ahmac_pad_init()
1133 		keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));  in mv_cesa_ahmac_pad_init()
1151 	struct ahash_request *req;  in mv_cesa_ahmac_setkey()  local
1163 	req = ahash_request_alloc(tfm, GFP_KERNEL);  in mv_cesa_ahmac_setkey()
1164 	if (!req) {  in mv_cesa_ahmac_setkey()
1181 	ret = mv_cesa_ahmac_pad_init(req, key, keylen, ipad, opad, blocksize);  in mv_cesa_ahmac_setkey()
1185 	ret = mv_cesa_ahmac_iv_state_init(req, ipad, istate, blocksize);  in mv_cesa_ahmac_setkey()
1189 	ret = mv_cesa_ahmac_iv_state_init(req, opad, ostate, blocksize);  in mv_cesa_ahmac_setkey()
1194 	ahash_request_free(req);  in mv_cesa_ahmac_setkey()
1212 static int mv_cesa_ahmac_md5_init(struct ahash_request *req)  in mv_cesa_ahmac_md5_init()  argument
1214 	struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in mv_cesa_ahmac_md5_init()
1220 	mv_cesa_ahash_init(req, &tmpl, true);  in mv_cesa_ahmac_md5_init()
1245 static int mv_cesa_ahmac_md5_digest(struct ahash_request *req)  in mv_cesa_ahmac_md5_digest()  argument
1249 	ret = mv_cesa_ahmac_md5_init(req);  in mv_cesa_ahmac_md5_digest()
1253 	return mv_cesa_ahash_finup(req);  in mv_cesa_ahmac_md5_digest()
1282 static int mv_cesa_ahmac_sha1_init(struct ahash_request *req)  in mv_cesa_ahmac_sha1_init()  argument
1284 	struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in mv_cesa_ahmac_sha1_init()
1290 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_ahmac_sha1_init()
1315 static int mv_cesa_ahmac_sha1_digest(struct ahash_request *req)  in mv_cesa_ahmac_sha1_digest()  argument
1319 	ret = mv_cesa_ahmac_sha1_init(req);  in mv_cesa_ahmac_sha1_digest()
1323 	return mv_cesa_ahash_finup(req);  in mv_cesa_ahmac_sha1_digest()
1372 static int mv_cesa_ahmac_sha256_init(struct ahash_request *req)  in mv_cesa_ahmac_sha256_init()  argument
1374 	struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in mv_cesa_ahmac_sha256_init()
1380 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_ahmac_sha256_init()
1385 static int mv_cesa_ahmac_sha256_digest(struct ahash_request *req)  in mv_cesa_ahmac_sha256_digest()  argument
1389 	ret = mv_cesa_ahmac_sha256_init(req);  in mv_cesa_ahmac_sha256_digest()
1393 	return mv_cesa_ahash_finup(req);  in mv_cesa_ahmac_sha256_digest()