145 				  struct qat_alg_aead_ctx *ctx,  in qat_alg_do_precomputes()  argument
149 	SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);  in qat_alg_do_precomputes()
153 	int block_size = crypto_shash_blocksize(ctx->hash_tfm);  in qat_alg_do_precomputes()
154 	int digest_size = crypto_shash_digestsize(ctx->hash_tfm);  in qat_alg_do_precomputes()
163 	shash->tfm = ctx->hash_tfm;  in qat_alg_do_precomputes()
194 	switch (ctx->qat_hash_alg) {  in qat_alg_do_precomputes()
223 	offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);  in qat_alg_do_precomputes()
227 	switch (ctx->qat_hash_alg) {  in qat_alg_do_precomputes()
277 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);  in qat_alg_aead_init_enc_session()  local
279 	struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;  in qat_alg_aead_init_enc_session()
284 	struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;  in qat_alg_aead_init_enc_session()
296 					     ctx->qat_hash_alg, digestsize);  in qat_alg_aead_init_enc_session()
298 		cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));  in qat_alg_aead_init_enc_session()
300 	if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))  in qat_alg_aead_init_enc_session()
312 	cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;  in qat_alg_aead_init_enc_session()
327 	switch (ctx->qat_hash_alg) {  in qat_alg_aead_init_enc_session()
358 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);  in qat_alg_aead_init_dec_session()  local
360 	struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;  in qat_alg_aead_init_dec_session()
365 		roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);  in qat_alg_aead_init_dec_session()
366 	struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;  in qat_alg_aead_init_dec_session()
382 					     ctx->qat_hash_alg,  in qat_alg_aead_init_dec_session()
385 		cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));  in qat_alg_aead_init_dec_session()
387 	if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))  in qat_alg_aead_init_dec_session()
399 	cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;  in qat_alg_aead_init_dec_session()
407 		 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;  in qat_alg_aead_init_dec_session()
417 	switch (ctx->qat_hash_alg) {  in qat_alg_aead_init_dec_session()
445 static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,  in qat_alg_ablkcipher_init_com()  argument
467 static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,  in qat_alg_ablkcipher_init_enc()  argument
471 	struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;  in qat_alg_ablkcipher_init_enc()
472 	struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;  in qat_alg_ablkcipher_init_enc()
475 	qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);  in qat_alg_ablkcipher_init_enc()
476 	cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;  in qat_alg_ablkcipher_init_enc()
480 static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,  in qat_alg_ablkcipher_init_dec()  argument
484 	struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;  in qat_alg_ablkcipher_init_dec()
485 	struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;  in qat_alg_ablkcipher_init_dec()
488 	qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);  in qat_alg_ablkcipher_init_dec()
489 	cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;  in qat_alg_ablkcipher_init_dec()
556 static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,  in qat_alg_ablkcipher_init_sessions()  argument
566 	qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen, mode);  in qat_alg_ablkcipher_init_sessions()
567 	qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen, mode);  in qat_alg_ablkcipher_init_sessions()
570 	crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);  in qat_alg_ablkcipher_init_sessions()
577 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);  in qat_alg_aead_setkey()  local
580 	if (ctx->enc_cd) {  in qat_alg_aead_setkey()
582 		dev = &GET_DEV(ctx->inst->accel_dev);  in qat_alg_aead_setkey()
583 		memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));  in qat_alg_aead_setkey()
584 		memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));  in qat_alg_aead_setkey()
585 		memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));  in qat_alg_aead_setkey()
586 		memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));  in qat_alg_aead_setkey()
597 		ctx->inst = inst;  in qat_alg_aead_setkey()
598 		ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),  in qat_alg_aead_setkey()
599 						  &ctx->enc_cd_paddr,  in qat_alg_aead_setkey()
601 		if (!ctx->enc_cd) {  in qat_alg_aead_setkey()
604 		ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),  in qat_alg_aead_setkey()
605 						  &ctx->dec_cd_paddr,  in qat_alg_aead_setkey()
607 		if (!ctx->dec_cd) {  in qat_alg_aead_setkey()
618 	memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));  in qat_alg_aead_setkey()
620 			  ctx->dec_cd, ctx->dec_cd_paddr);  in qat_alg_aead_setkey()
621 	ctx->dec_cd = NULL;  in qat_alg_aead_setkey()
623 	memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));  in qat_alg_aead_setkey()
625 			  ctx->enc_cd, ctx->enc_cd_paddr);  in qat_alg_aead_setkey()
626 	ctx->enc_cd = NULL;  in qat_alg_aead_setkey()
778 	struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;  in qat_aead_alg_callback()  local
779 	struct qat_crypto_instance *inst = ctx->inst;  in qat_aead_alg_callback()
793 	struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx;  in qat_ablkcipher_alg_callback()  local
794 	struct qat_crypto_instance *inst = ctx->inst;  in qat_ablkcipher_alg_callback()
818 	struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_aead_dec()  local
826 	ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);  in qat_alg_aead_dec()
831 	*msg = ctx->dec_fw_req;  in qat_alg_aead_dec()
832 	qat_req->aead_ctx = ctx;  in qat_alg_aead_dec()
846 		ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);  in qat_alg_aead_dec()
850 		qat_alg_free_bufl(ctx->inst, qat_req);  in qat_alg_aead_dec()
860 	struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_aead_enc()  local
868 	ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);  in qat_alg_aead_enc()
873 	*msg = ctx->enc_fw_req;  in qat_alg_aead_enc()
874 	qat_req->aead_ctx = ctx;  in qat_alg_aead_enc()
891 		ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);  in qat_alg_aead_enc()
895 		qat_alg_free_bufl(ctx->inst, qat_req);  in qat_alg_aead_enc()
905 	struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);  in qat_alg_ablkcipher_setkey()  local
908 	spin_lock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
909 	if (ctx->enc_cd) {  in qat_alg_ablkcipher_setkey()
911 		dev = &GET_DEV(ctx->inst->accel_dev);  in qat_alg_ablkcipher_setkey()
912 		memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));  in qat_alg_ablkcipher_setkey()
913 		memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));  in qat_alg_ablkcipher_setkey()
914 		memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));  in qat_alg_ablkcipher_setkey()
915 		memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));  in qat_alg_ablkcipher_setkey()
922 			spin_unlock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
927 		ctx->inst = inst;  in qat_alg_ablkcipher_setkey()
928 		ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),  in qat_alg_ablkcipher_setkey()
929 						  &ctx->enc_cd_paddr,  in qat_alg_ablkcipher_setkey()
931 		if (!ctx->enc_cd) {  in qat_alg_ablkcipher_setkey()
932 			spin_unlock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
935 		ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),  in qat_alg_ablkcipher_setkey()
936 						  &ctx->dec_cd_paddr,  in qat_alg_ablkcipher_setkey()
938 		if (!ctx->dec_cd) {  in qat_alg_ablkcipher_setkey()
939 			spin_unlock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
943 	spin_unlock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
944 	if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode))  in qat_alg_ablkcipher_setkey()
950 	memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd));  in qat_alg_ablkcipher_setkey()
951 	dma_free_coherent(dev, sizeof(*ctx->enc_cd),  in qat_alg_ablkcipher_setkey()
952 			  ctx->dec_cd, ctx->dec_cd_paddr);  in qat_alg_ablkcipher_setkey()
953 	ctx->dec_cd = NULL;  in qat_alg_ablkcipher_setkey()
955 	memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd));  in qat_alg_ablkcipher_setkey()
956 	dma_free_coherent(dev, sizeof(*ctx->dec_cd),  in qat_alg_ablkcipher_setkey()
957 			  ctx->enc_cd, ctx->enc_cd_paddr);  in qat_alg_ablkcipher_setkey()
958 	ctx->enc_cd = NULL;  in qat_alg_ablkcipher_setkey()
987 	struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_ablkcipher_encrypt()  local
993 	ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);  in qat_alg_ablkcipher_encrypt()
998 	*msg = ctx->enc_fw_req;  in qat_alg_ablkcipher_encrypt()
999 	qat_req->ablkcipher_ctx = ctx;  in qat_alg_ablkcipher_encrypt()
1010 		ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);  in qat_alg_ablkcipher_encrypt()
1014 		qat_alg_free_bufl(ctx->inst, qat_req);  in qat_alg_ablkcipher_encrypt()
1024 	struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_ablkcipher_decrypt()  local
1030 	ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);  in qat_alg_ablkcipher_decrypt()
1035 	*msg = ctx->dec_fw_req;  in qat_alg_ablkcipher_decrypt()
1036 	qat_req->ablkcipher_ctx = ctx;  in qat_alg_ablkcipher_decrypt()
1047 		ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);  in qat_alg_ablkcipher_decrypt()
1051 		qat_alg_free_bufl(ctx->inst, qat_req);  in qat_alg_ablkcipher_decrypt()
1061 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);  in qat_alg_aead_init()  local
1063 	ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);  in qat_alg_aead_init()
1064 	if (IS_ERR(ctx->hash_tfm))  in qat_alg_aead_init()
1065 		return PTR_ERR(ctx->hash_tfm);  in qat_alg_aead_init()
1066 	ctx->qat_hash_alg = hash;  in qat_alg_aead_init()
1089 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);  in qat_alg_aead_exit()  local
1090 	struct qat_crypto_instance *inst = ctx->inst;  in qat_alg_aead_exit()
1093 	crypto_free_shash(ctx->hash_tfm);  in qat_alg_aead_exit()
1099 	if (ctx->enc_cd) {  in qat_alg_aead_exit()
1100 		memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));  in qat_alg_aead_exit()
1102 				  ctx->enc_cd, ctx->enc_cd_paddr);  in qat_alg_aead_exit()
1104 	if (ctx->dec_cd) {  in qat_alg_aead_exit()
1105 		memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));  in qat_alg_aead_exit()
1107 				  ctx->dec_cd, ctx->dec_cd_paddr);  in qat_alg_aead_exit()
1114 	struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_ablkcipher_init()  local
1116 	spin_lock_init(&ctx->lock);  in qat_alg_ablkcipher_init()
1119 	ctx->tfm = tfm;  in qat_alg_ablkcipher_init()
1125 	struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_ablkcipher_exit()  local
1126 	struct qat_crypto_instance *inst = ctx->inst;  in qat_alg_ablkcipher_exit()
1133 	if (ctx->enc_cd) {  in qat_alg_ablkcipher_exit()
1134 		memset(ctx->enc_cd, 0,  in qat_alg_ablkcipher_exit()
1138 				  ctx->enc_cd, ctx->enc_cd_paddr);  in qat_alg_ablkcipher_exit()
1140 	if (ctx->dec_cd) {  in qat_alg_ablkcipher_exit()
1141 		memset(ctx->dec_cd, 0,  in qat_alg_ablkcipher_exit()
1145 				  ctx->dec_cd, ctx->dec_cd_paddr);  in qat_alg_ablkcipher_exit()