95 	struct crypto_async_request	*req;  member
101 	void				(*complete)(struct spacc_req *req);
187 static int spacc_ablk_submit(struct spacc_req *req);
327 	struct spacc_req *req = aead_request_ctx(areq);  in spacc_aead_make_ddts()  local
328 	struct spacc_engine *engine = req->engine;  in spacc_aead_make_ddts()
336 	if (req->is_encrypt)  in spacc_aead_make_ddts()
350 	src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr);  in spacc_aead_make_ddts()
354 	dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr);  in spacc_aead_make_ddts()
358 	req->src_ddt = src_ddt;  in spacc_aead_make_ddts()
359 	req->dst_ddt = dst_ddt;  in spacc_aead_make_ddts()
391 	total = req->is_encrypt ? 0 : areq->assoclen;  in spacc_aead_make_ddts()
409 	dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr);  in spacc_aead_make_ddts()
411 	dma_pool_free(engine->req_pool, src_ddt, req->src_addr);  in spacc_aead_make_ddts()
416 static void spacc_aead_free_ddts(struct spacc_req *req)  in spacc_aead_free_ddts()  argument
418 	struct aead_request *areq = container_of(req->req, struct aead_request,  in spacc_aead_free_ddts()
422 			 (req->is_encrypt ? crypto_aead_authsize(aead) : 0);  in spacc_aead_free_ddts()
435 	dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr);  in spacc_aead_free_ddts()
436 	dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr);  in spacc_aead_free_ddts()
439 static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt,  in spacc_free_ddt()  argument
445 	dma_unmap_sg(req->engine->dev, payload, nents, dir);  in spacc_free_ddt()
446 	dma_pool_free(req->engine->req_pool, ddt, ddt_addr);  in spacc_free_ddt()
521 static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type,  in spacc_aead_do_fallback()  argument
524 	struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req));  in spacc_aead_do_fallback()
526 	struct aead_request *subreq = aead_request_ctx(req);  in spacc_aead_do_fallback()
529 	aead_request_set_callback(subreq, req->base.flags,  in spacc_aead_do_fallback()
530 				  req->base.complete, req->base.data);  in spacc_aead_do_fallback()
531 	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,  in spacc_aead_do_fallback()
532 			       req->iv);  in spacc_aead_do_fallback()
533 	aead_request_set_ad(subreq, req->assoclen);  in spacc_aead_do_fallback()
539 static void spacc_aead_complete(struct spacc_req *req)  in spacc_aead_complete()  argument
541 	spacc_aead_free_ddts(req);  in spacc_aead_complete()
542 	req->req->complete(req->req, req->result);  in spacc_aead_complete()
545 static int spacc_aead_submit(struct spacc_req *req)  in spacc_aead_submit()  argument
548 		container_of(req->req, struct aead_request, base);  in spacc_aead_submit()
557 	req->result = -EINPROGRESS;  in spacc_aead_submit()
558 	req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key,  in spacc_aead_submit()
563 	writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);  in spacc_aead_submit()
564 	writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);  in spacc_aead_submit()
574 	if (!req->is_encrypt)  in spacc_aead_submit()
583 	ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |  in spacc_aead_submit()
585 	if (req->is_encrypt)  in spacc_aead_submit()
597 static int spacc_req_submit(struct spacc_req *req);
601 	struct spacc_req *req;  in spacc_push()  local
607 		req = list_first_entry(&engine->pending, struct spacc_req,  in spacc_push()
609 		list_move_tail(&req->list, &engine->in_progress);  in spacc_push()
611 		req->result = spacc_req_submit(req);  in spacc_push()
619 static int spacc_aead_setup(struct aead_request *req,  in spacc_aead_setup()  argument
622 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in spacc_aead_setup()
625 	struct spacc_req *dev_req = aead_request_ctx(req);  in spacc_aead_setup()
629 	dev_req->req		= &req->base;  in spacc_aead_setup()
635 	if (unlikely(spacc_aead_need_fallback(req) ||  in spacc_aead_setup()
636 		     ((err = spacc_aead_make_ddts(req)) == -E2BIG)))  in spacc_aead_setup()
637 		return spacc_aead_do_fallback(req, alg_type, is_encrypt);  in spacc_aead_setup()
646 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {  in spacc_aead_setup()
666 static int spacc_aead_encrypt(struct aead_request *req)  in spacc_aead_encrypt()  argument
668 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in spacc_aead_encrypt()
671 	return spacc_aead_setup(req, alg->type, 1);  in spacc_aead_encrypt()
674 static int spacc_aead_decrypt(struct aead_request *req)  in spacc_aead_decrypt()  argument
676 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in spacc_aead_decrypt()
679 	return spacc_aead_setup(req, alg->type, 0);  in spacc_aead_decrypt()
821 static int spacc_ablk_need_fallback(struct spacc_req *req)  in spacc_ablk_need_fallback()  argument
824 	struct crypto_tfm *tfm = req->req->tfm;  in spacc_ablk_need_fallback()
825 	struct crypto_alg *alg = req->req->tfm->__crt_alg;  in spacc_ablk_need_fallback()
836 static void spacc_ablk_complete(struct spacc_req *req)  in spacc_ablk_complete()  argument
839 		container_of(req->req, struct ablkcipher_request, base);  in spacc_ablk_complete()
842 		spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src,  in spacc_ablk_complete()
844 		spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,  in spacc_ablk_complete()
847 		spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,  in spacc_ablk_complete()
850 	req->req->complete(req->req, req->result);  in spacc_ablk_complete()
853 static int spacc_ablk_submit(struct spacc_req *req)  in spacc_ablk_submit()  argument
855 	struct crypto_tfm *tfm = req->req->tfm;  in spacc_ablk_submit()
857 	struct ablkcipher_request *ablk_req = ablkcipher_request_cast(req->req);  in spacc_ablk_submit()
858 	struct crypto_alg *alg = req->req->tfm->__crt_alg;  in spacc_ablk_submit()
863 	req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->key,  in spacc_ablk_submit()
867 	writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);  in spacc_ablk_submit()
868 	writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);  in spacc_ablk_submit()
876 	ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |  in spacc_ablk_submit()
877 		(req->is_encrypt ? (1 << SPA_CTRL_ENCRYPT_IDX) :  in spacc_ablk_submit()
887 static int spacc_ablk_do_fallback(struct ablkcipher_request *req,  in spacc_ablk_do_fallback()  argument
891 	    crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));  in spacc_ablk_do_fallback()
903 	ablkcipher_request_set_tfm(req, ctx->sw_cipher);  in spacc_ablk_do_fallback()
904 	err = is_encrypt ? crypto_ablkcipher_encrypt(req) :  in spacc_ablk_do_fallback()
905 		crypto_ablkcipher_decrypt(req);  in spacc_ablk_do_fallback()
906 	ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(old_tfm));  in spacc_ablk_do_fallback()
911 static int spacc_ablk_setup(struct ablkcipher_request *req, unsigned alg_type,  in spacc_ablk_setup()  argument
914 	struct crypto_alg *alg = req->base.tfm->__crt_alg;  in spacc_ablk_setup()
916 	struct spacc_req *dev_req = ablkcipher_request_ctx(req);  in spacc_ablk_setup()
920 	dev_req->req		= &req->base;  in spacc_ablk_setup()
927 		return spacc_ablk_do_fallback(req, alg_type, is_encrypt);  in spacc_ablk_setup()
933 	if (req->src != req->dst) {  in spacc_ablk_setup()
934 		dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src,  in spacc_ablk_setup()
935 			req->nbytes, DMA_TO_DEVICE, &dev_req->src_addr);  in spacc_ablk_setup()
939 		dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,  in spacc_ablk_setup()
940 			req->nbytes, DMA_FROM_DEVICE, &dev_req->dst_addr);  in spacc_ablk_setup()
944 		dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,  in spacc_ablk_setup()
945 			req->nbytes, DMA_BIDIRECTIONAL, &dev_req->dst_addr);  in spacc_ablk_setup()
962 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {  in spacc_ablk_setup()
977 	spacc_free_ddt(dev_req, dev_req->dst_ddt, dev_req->dst_addr, req->dst,  in spacc_ablk_setup()
978 		       req->nbytes, req->src == req->dst ?  in spacc_ablk_setup()
981 	if (req->src != req->dst)  in spacc_ablk_setup()
983 			       req->src, req->nbytes, DMA_TO_DEVICE);  in spacc_ablk_setup()
1023 static int spacc_ablk_encrypt(struct ablkcipher_request *req)  in spacc_ablk_encrypt()  argument
1025 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);  in spacc_ablk_encrypt()
1029 	return spacc_ablk_setup(req, alg->type, 1);  in spacc_ablk_encrypt()
1032 static int spacc_ablk_decrypt(struct ablkcipher_request *req)  in spacc_ablk_decrypt()  argument
1034 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);  in spacc_ablk_decrypt()
1038 	return spacc_ablk_setup(req, alg->type, 0);  in spacc_ablk_decrypt()
1049 	struct spacc_req *req;  in spacc_process_done()  local
1055 		req = list_first_entry(&engine->in_progress, struct spacc_req,  in spacc_process_done()
1057 		list_move_tail(&req->list, &engine->completed);  in spacc_process_done()
1062 		req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) &  in spacc_process_done()
1069 		if (unlikely(req->result)) {  in spacc_process_done()
1070 			switch (req->result) {  in spacc_process_done()
1072 				req->result = -EBADMSG;  in spacc_process_done()
1078 				req->result = -EFAULT;  in spacc_process_done()
1084 				req->result = -EIO;  in spacc_process_done()
1113 static int spacc_req_submit(struct spacc_req *req)  in spacc_req_submit()  argument
1115 	struct crypto_alg *alg = req->req->tfm->__crt_alg;  in spacc_req_submit()
1118 		return spacc_aead_submit(req);  in spacc_req_submit()
1120 		return spacc_ablk_submit(req);  in spacc_req_submit()
1126 	struct spacc_req *req, *tmp;  in spacc_spacc_complete()  local
1139 	list_for_each_entry_safe(req, tmp, &completed, list) {  in spacc_spacc_complete()
1140 		list_del(&req->list);  in spacc_spacc_complete()
1141 		req->complete(req);  in spacc_spacc_complete()