94 	struct crypto_async_request	*req;  member
100 	void				(*complete)(struct spacc_req *req);
180 static int spacc_ablk_submit(struct spacc_req *req);
321 static int spacc_aead_make_ddts(struct spacc_req *req, u8 *giv)  in spacc_aead_make_ddts()  argument
323 	struct aead_request *areq = container_of(req->req, struct aead_request,  in spacc_aead_make_ddts()
325 	struct spacc_engine *engine = req->engine;  in spacc_aead_make_ddts()
334 	src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr);  in spacc_aead_make_ddts()
338 	dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr);  in spacc_aead_make_ddts()
340 		dma_pool_free(engine->req_pool, src_ddt, req->src_addr);  in spacc_aead_make_ddts()
344 	req->src_ddt = src_ddt;  in spacc_aead_make_ddts()
345 	req->dst_ddt = dst_ddt;  in spacc_aead_make_ddts()
366 	req->giv_pa = iv_addr;  in spacc_aead_make_ddts()
374 		if (req->is_encrypt)  in spacc_aead_make_ddts()
380 	if (giv || req->is_encrypt)  in spacc_aead_make_ddts()
404 static void spacc_aead_free_ddts(struct spacc_req *req)  in spacc_aead_free_ddts()  argument
406 	struct aead_request *areq = container_of(req->req, struct aead_request,  in spacc_aead_free_ddts()
408 	struct spacc_alg *alg = to_spacc_alg(req->req->tfm->__crt_alg);  in spacc_aead_free_ddts()
409 	struct spacc_ablk_ctx *aead_ctx = crypto_tfm_ctx(req->req->tfm);  in spacc_aead_free_ddts()
425 	dma_unmap_single(engine->dev, req->giv_pa, ivsize, DMA_BIDIRECTIONAL);  in spacc_aead_free_ddts()
427 	dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr);  in spacc_aead_free_ddts()
428 	dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr);  in spacc_aead_free_ddts()
431 static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt,  in spacc_free_ddt()  argument
437 	dma_unmap_sg(req->engine->dev, payload, nents, dir);  in spacc_free_ddt()
438 	dma_pool_free(req->engine->req_pool, ddt, ddt_addr);  in spacc_free_ddt()
544 static int spacc_aead_need_fallback(struct spacc_req *req)  in spacc_aead_need_fallback()  argument
547 	struct crypto_tfm *tfm = req->req->tfm;  in spacc_aead_need_fallback()
548 	struct crypto_alg *alg = req->req->tfm->__crt_alg;  in spacc_aead_need_fallback()
552 	aead_req = container_of(req->req, struct aead_request, base);  in spacc_aead_need_fallback()
566 static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type,  in spacc_aead_do_fallback()  argument
569 	struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req));  in spacc_aead_do_fallback()
579 		aead_request_set_tfm(req, ctx->sw_cipher);  in spacc_aead_do_fallback()
580 		err = is_encrypt ? crypto_aead_encrypt(req) :  in spacc_aead_do_fallback()
581 		    crypto_aead_decrypt(req);  in spacc_aead_do_fallback()
582 		aead_request_set_tfm(req, __crypto_aead_cast(old_tfm));  in spacc_aead_do_fallback()
589 static void spacc_aead_complete(struct spacc_req *req)  in spacc_aead_complete()  argument
591 	spacc_aead_free_ddts(req);  in spacc_aead_complete()
592 	req->req->complete(req->req, req->result);  in spacc_aead_complete()
595 static int spacc_aead_submit(struct spacc_req *req)  in spacc_aead_submit()  argument
597 	struct crypto_tfm *tfm = req->req->tfm;  in spacc_aead_submit()
599 	struct crypto_alg *alg = req->req->tfm->__crt_alg;  in spacc_aead_submit()
604 		container_of(req->req, struct aead_request, base);  in spacc_aead_submit()
606 	req->result = -EINPROGRESS;  in spacc_aead_submit()
607 	req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key,  in spacc_aead_submit()
612 	writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);  in spacc_aead_submit()
613 	writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);  in spacc_aead_submit()
623 	if (!req->giv) {  in spacc_aead_submit()
627 		proc_len += req->giv_len;  in spacc_aead_submit()
633 	if (!req->is_encrypt)  in spacc_aead_submit()
642 	ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |  in spacc_aead_submit()
644 	if (req->is_encrypt)  in spacc_aead_submit()
656 static int spacc_req_submit(struct spacc_req *req);
660 	struct spacc_req *req;  in spacc_push()  local
666 		req = list_first_entry(&engine->pending, struct spacc_req,  in spacc_push()
668 		list_move_tail(&req->list, &engine->in_progress);  in spacc_push()
670 		req->result = spacc_req_submit(req);  in spacc_push()
681 static int spacc_aead_setup(struct aead_request *req, u8 *giv,  in spacc_aead_setup()  argument
684 	struct crypto_alg *alg = req->base.tfm->__crt_alg;  in spacc_aead_setup()
686 	struct spacc_req *dev_req = aead_request_ctx(req);  in spacc_aead_setup()
689 	unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));  in spacc_aead_setup()
693 	dev_req->req		= &req->base;  in spacc_aead_setup()
700 		return spacc_aead_do_fallback(req, alg_type, is_encrypt);  in spacc_aead_setup()
708 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {  in spacc_aead_setup()
728 static int spacc_aead_encrypt(struct aead_request *req)  in spacc_aead_encrypt()  argument
730 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in spacc_aead_encrypt()
734 	return spacc_aead_setup(req, NULL, alg->type, 1);  in spacc_aead_encrypt()
737 static int spacc_aead_givencrypt(struct aead_givcrypt_request *req)  in spacc_aead_givencrypt()  argument
739 	struct crypto_aead *tfm = aead_givcrypt_reqtfm(req);  in spacc_aead_givencrypt()
746 	memcpy(req->areq.iv, ctx->salt, ivsize);  in spacc_aead_givencrypt()
749 		memset(req->giv, 0, ivsize - sizeof(u64));  in spacc_aead_givencrypt()
752 	seq = cpu_to_be64(req->seq);  in spacc_aead_givencrypt()
753 	memcpy(req->giv + ivsize - len, &seq, len);  in spacc_aead_givencrypt()
755 	return spacc_aead_setup(&req->areq, req->giv, alg->type, 1);  in spacc_aead_givencrypt()
758 static int spacc_aead_decrypt(struct aead_request *req)  in spacc_aead_decrypt()  argument
760 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in spacc_aead_decrypt()
764 	return spacc_aead_setup(req, NULL, alg->type, 0);  in spacc_aead_decrypt()
910 static int spacc_ablk_need_fallback(struct spacc_req *req)  in spacc_ablk_need_fallback()  argument
913 	struct crypto_tfm *tfm = req->req->tfm;  in spacc_ablk_need_fallback()
914 	struct crypto_alg *alg = req->req->tfm->__crt_alg;  in spacc_ablk_need_fallback()
925 static void spacc_ablk_complete(struct spacc_req *req)  in spacc_ablk_complete()  argument
928 		container_of(req->req, struct ablkcipher_request, base);  in spacc_ablk_complete()
931 		spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src,  in spacc_ablk_complete()
933 		spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,  in spacc_ablk_complete()
936 		spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,  in spacc_ablk_complete()
939 	req->req->complete(req->req, req->result);  in spacc_ablk_complete()
942 static int spacc_ablk_submit(struct spacc_req *req)  in spacc_ablk_submit()  argument
944 	struct crypto_tfm *tfm = req->req->tfm;  in spacc_ablk_submit()
946 	struct ablkcipher_request *ablk_req = ablkcipher_request_cast(req->req);  in spacc_ablk_submit()
947 	struct crypto_alg *alg = req->req->tfm->__crt_alg;  in spacc_ablk_submit()
952 	req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->key,  in spacc_ablk_submit()
956 	writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);  in spacc_ablk_submit()
957 	writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);  in spacc_ablk_submit()
965 	ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |  in spacc_ablk_submit()
966 		(req->is_encrypt ? (1 << SPA_CTRL_ENCRYPT_IDX) :  in spacc_ablk_submit()
976 static int spacc_ablk_do_fallback(struct ablkcipher_request *req,  in spacc_ablk_do_fallback()  argument
980 	    crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));  in spacc_ablk_do_fallback()
992 	ablkcipher_request_set_tfm(req, ctx->sw_cipher);  in spacc_ablk_do_fallback()
993 	err = is_encrypt ? crypto_ablkcipher_encrypt(req) :  in spacc_ablk_do_fallback()
994 		crypto_ablkcipher_decrypt(req);  in spacc_ablk_do_fallback()
995 	ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(old_tfm));  in spacc_ablk_do_fallback()
1000 static int spacc_ablk_setup(struct ablkcipher_request *req, unsigned alg_type,  in spacc_ablk_setup()  argument
1003 	struct crypto_alg *alg = req->base.tfm->__crt_alg;  in spacc_ablk_setup()
1005 	struct spacc_req *dev_req = ablkcipher_request_ctx(req);  in spacc_ablk_setup()
1009 	dev_req->req		= &req->base;  in spacc_ablk_setup()
1016 		return spacc_ablk_do_fallback(req, alg_type, is_encrypt);  in spacc_ablk_setup()
1022 	if (req->src != req->dst) {  in spacc_ablk_setup()
1023 		dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src,  in spacc_ablk_setup()
1024 			req->nbytes, DMA_TO_DEVICE, &dev_req->src_addr);  in spacc_ablk_setup()
1028 		dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,  in spacc_ablk_setup()
1029 			req->nbytes, DMA_FROM_DEVICE, &dev_req->dst_addr);  in spacc_ablk_setup()
1033 		dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,  in spacc_ablk_setup()
1034 			req->nbytes, DMA_BIDIRECTIONAL, &dev_req->dst_addr);  in spacc_ablk_setup()
1051 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {  in spacc_ablk_setup()
1066 	spacc_free_ddt(dev_req, dev_req->dst_ddt, dev_req->dst_addr, req->dst,  in spacc_ablk_setup()
1067 		       req->nbytes, req->src == req->dst ?  in spacc_ablk_setup()
1070 	if (req->src != req->dst)  in spacc_ablk_setup()
1072 			       req->src, req->nbytes, DMA_TO_DEVICE);  in spacc_ablk_setup()
1112 static int spacc_ablk_encrypt(struct ablkcipher_request *req)  in spacc_ablk_encrypt()  argument
1114 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);  in spacc_ablk_encrypt()
1118 	return spacc_ablk_setup(req, alg->type, 1);  in spacc_ablk_encrypt()
1121 static int spacc_ablk_decrypt(struct ablkcipher_request *req)  in spacc_ablk_decrypt()  argument
1123 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);  in spacc_ablk_decrypt()
1127 	return spacc_ablk_setup(req, alg->type, 0);  in spacc_ablk_decrypt()
1138 	struct spacc_req *req;  in spacc_process_done()  local
1144 		req = list_first_entry(&engine->in_progress, struct spacc_req,  in spacc_process_done()
1146 		list_move_tail(&req->list, &engine->completed);  in spacc_process_done()
1151 		req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) &  in spacc_process_done()
1158 		if (unlikely(req->result)) {  in spacc_process_done()
1159 			switch (req->result) {  in spacc_process_done()
1161 				req->result = -EBADMSG;  in spacc_process_done()
1167 				req->result = -EFAULT;  in spacc_process_done()
1173 				req->result = -EIO;  in spacc_process_done()
1202 static int spacc_req_submit(struct spacc_req *req)  in spacc_req_submit()  argument
1204 	struct crypto_alg *alg = req->req->tfm->__crt_alg;  in spacc_req_submit()
1207 		return spacc_aead_submit(req);  in spacc_req_submit()
1209 		return spacc_ablk_submit(req);  in spacc_req_submit()
1215 	struct spacc_req *req, *tmp;  in spacc_spacc_complete()  local
1228 	list_for_each_entry_safe(req, tmp, &completed, list) {  in spacc_spacc_complete()
1229 		list_del(&req->list);  in spacc_spacc_complete()
1230 		req->complete(req);  in spacc_spacc_complete()