154 				  struct qat_alg_aead_ctx *ctx,  in qat_alg_do_precomputes()  argument
158 	SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);  in qat_alg_do_precomputes()
162 	int block_size = crypto_shash_blocksize(ctx->hash_tfm);  in qat_alg_do_precomputes()
163 	int digest_size = crypto_shash_digestsize(ctx->hash_tfm);  in qat_alg_do_precomputes()
172 	shash->tfm = ctx->hash_tfm;  in qat_alg_do_precomputes()
203 	switch (ctx->qat_hash_alg) {  in qat_alg_do_precomputes()
232 	offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);  in qat_alg_do_precomputes()
236 	switch (ctx->qat_hash_alg) {  in qat_alg_do_precomputes()
281 static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx,  in qat_alg_aead_init_enc_session()  argument
285 	struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);  in qat_alg_aead_init_enc_session()
287 	struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;  in qat_alg_aead_init_enc_session()
292 	struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;  in qat_alg_aead_init_enc_session()
304 					     ctx->qat_hash_alg, digestsize);  in qat_alg_aead_init_enc_session()
306 		cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));  in qat_alg_aead_init_enc_session()
308 	if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))  in qat_alg_aead_init_enc_session()
320 	cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;  in qat_alg_aead_init_enc_session()
335 	switch (ctx->qat_hash_alg) {  in qat_alg_aead_init_enc_session()
361 static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx,  in qat_alg_aead_init_dec_session()  argument
365 	struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);  in qat_alg_aead_init_dec_session()
367 	struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;  in qat_alg_aead_init_dec_session()
372 		roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);  in qat_alg_aead_init_dec_session()
373 	struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;  in qat_alg_aead_init_dec_session()
389 					     ctx->qat_hash_alg,  in qat_alg_aead_init_dec_session()
392 		cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));  in qat_alg_aead_init_dec_session()
394 	if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))  in qat_alg_aead_init_dec_session()
406 	cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;  in qat_alg_aead_init_dec_session()
414 		 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;  in qat_alg_aead_init_dec_session()
424 	switch (ctx->qat_hash_alg) {  in qat_alg_aead_init_dec_session()
452 static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,  in qat_alg_ablkcipher_init_com()  argument
474 static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,  in qat_alg_ablkcipher_init_enc()  argument
478 	struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;  in qat_alg_ablkcipher_init_enc()
479 	struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;  in qat_alg_ablkcipher_init_enc()
482 	qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);  in qat_alg_ablkcipher_init_enc()
483 	cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;  in qat_alg_ablkcipher_init_enc()
487 static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,  in qat_alg_ablkcipher_init_dec()  argument
491 	struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;  in qat_alg_ablkcipher_init_dec()
492 	struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;  in qat_alg_ablkcipher_init_dec()
495 	qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);  in qat_alg_ablkcipher_init_dec()
496 	cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;  in qat_alg_ablkcipher_init_dec()
518 static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx,  in qat_alg_aead_init_sessions()  argument
524 	if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE))  in qat_alg_aead_init_sessions()
533 	if (qat_alg_aead_init_enc_session(ctx, alg, &keys))  in qat_alg_aead_init_sessions()
536 	if (qat_alg_aead_init_dec_session(ctx, alg, &keys))  in qat_alg_aead_init_sessions()
541 	crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);  in qat_alg_aead_init_sessions()
547 static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,  in qat_alg_ablkcipher_init_sessions()  argument
556 	qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen);  in qat_alg_ablkcipher_init_sessions()
557 	qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen);  in qat_alg_ablkcipher_init_sessions()
560 	crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);  in qat_alg_ablkcipher_init_sessions()
567 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);  in qat_alg_aead_setkey()  local
570 	spin_lock(&ctx->lock);  in qat_alg_aead_setkey()
571 	if (ctx->enc_cd) {  in qat_alg_aead_setkey()
573 		dev = &GET_DEV(ctx->inst->accel_dev);  in qat_alg_aead_setkey()
574 		memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));  in qat_alg_aead_setkey()
575 		memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));  in qat_alg_aead_setkey()
576 		memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));  in qat_alg_aead_setkey()
577 		memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));  in qat_alg_aead_setkey()
584 			spin_unlock(&ctx->lock);  in qat_alg_aead_setkey()
589 		ctx->inst = inst;  in qat_alg_aead_setkey()
590 		ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),  in qat_alg_aead_setkey()
591 						  &ctx->enc_cd_paddr,  in qat_alg_aead_setkey()
593 		if (!ctx->enc_cd) {  in qat_alg_aead_setkey()
594 			spin_unlock(&ctx->lock);  in qat_alg_aead_setkey()
597 		ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),  in qat_alg_aead_setkey()
598 						  &ctx->dec_cd_paddr,  in qat_alg_aead_setkey()
600 		if (!ctx->dec_cd) {  in qat_alg_aead_setkey()
601 			spin_unlock(&ctx->lock);  in qat_alg_aead_setkey()
605 	spin_unlock(&ctx->lock);  in qat_alg_aead_setkey()
606 	if (qat_alg_aead_init_sessions(ctx, key, keylen))  in qat_alg_aead_setkey()
612 	memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));  in qat_alg_aead_setkey()
614 			  ctx->dec_cd, ctx->dec_cd_paddr);  in qat_alg_aead_setkey()
615 	ctx->dec_cd = NULL;  in qat_alg_aead_setkey()
617 	memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));  in qat_alg_aead_setkey()
619 			  ctx->enc_cd, ctx->enc_cd_paddr);  in qat_alg_aead_setkey()
620 	ctx->enc_cd = NULL;  in qat_alg_aead_setkey()
801 	struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;  in qat_aead_alg_callback()  local
802 	struct qat_crypto_instance *inst = ctx->inst;  in qat_aead_alg_callback()
816 	struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx;  in qat_ablkcipher_alg_callback()  local
817 	struct qat_crypto_instance *inst = ctx->inst;  in qat_ablkcipher_alg_callback()
841 	struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_aead_dec()  local
849 	ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst,  in qat_alg_aead_dec()
855 	*msg = ctx->dec_fw_req;  in qat_alg_aead_dec()
856 	qat_req->aead_ctx = ctx;  in qat_alg_aead_dec()
871 		ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);  in qat_alg_aead_dec()
875 		qat_alg_free_bufl(ctx->inst, qat_req);  in qat_alg_aead_dec()
886 	struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_aead_enc_internal()  local
893 	ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst,  in qat_alg_aead_enc_internal()
899 	*msg = ctx->enc_fw_req;  in qat_alg_aead_enc_internal()
900 	qat_req->aead_ctx = ctx;  in qat_alg_aead_enc_internal()
921 		ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);  in qat_alg_aead_enc_internal()
925 		qat_alg_free_bufl(ctx->inst, qat_req);  in qat_alg_aead_enc_internal()
940 	struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_aead_genivenc()  local
943 	memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE);  in qat_alg_aead_genivenc()
954 	struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);  in qat_alg_ablkcipher_setkey()  local
957 	spin_lock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
958 	if (ctx->enc_cd) {  in qat_alg_ablkcipher_setkey()
960 		dev = &GET_DEV(ctx->inst->accel_dev);  in qat_alg_ablkcipher_setkey()
961 		memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));  in qat_alg_ablkcipher_setkey()
962 		memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));  in qat_alg_ablkcipher_setkey()
963 		memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));  in qat_alg_ablkcipher_setkey()
964 		memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));  in qat_alg_ablkcipher_setkey()
971 			spin_unlock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
976 		ctx->inst = inst;  in qat_alg_ablkcipher_setkey()
977 		ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),  in qat_alg_ablkcipher_setkey()
978 						  &ctx->enc_cd_paddr,  in qat_alg_ablkcipher_setkey()
980 		if (!ctx->enc_cd) {  in qat_alg_ablkcipher_setkey()
981 			spin_unlock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
984 		ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),  in qat_alg_ablkcipher_setkey()
985 						  &ctx->dec_cd_paddr,  in qat_alg_ablkcipher_setkey()
987 		if (!ctx->dec_cd) {  in qat_alg_ablkcipher_setkey()
988 			spin_unlock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
992 	spin_unlock(&ctx->lock);  in qat_alg_ablkcipher_setkey()
993 	if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen))  in qat_alg_ablkcipher_setkey()
999 	memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd));  in qat_alg_ablkcipher_setkey()
1000 	dma_free_coherent(dev, sizeof(*ctx->enc_cd),  in qat_alg_ablkcipher_setkey()
1001 			  ctx->dec_cd, ctx->dec_cd_paddr);  in qat_alg_ablkcipher_setkey()
1002 	ctx->dec_cd = NULL;  in qat_alg_ablkcipher_setkey()
1004 	memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd));  in qat_alg_ablkcipher_setkey()
1005 	dma_free_coherent(dev, sizeof(*ctx->dec_cd),  in qat_alg_ablkcipher_setkey()
1006 			  ctx->enc_cd, ctx->enc_cd_paddr);  in qat_alg_ablkcipher_setkey()
1007 	ctx->enc_cd = NULL;  in qat_alg_ablkcipher_setkey()
1015 	struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_ablkcipher_encrypt()  local
1021 	ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst,  in qat_alg_ablkcipher_encrypt()
1027 	*msg = ctx->enc_fw_req;  in qat_alg_ablkcipher_encrypt()
1028 	qat_req->ablkcipher_ctx = ctx;  in qat_alg_ablkcipher_encrypt()
1039 		ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);  in qat_alg_ablkcipher_encrypt()
1043 		qat_alg_free_bufl(ctx->inst, qat_req);  in qat_alg_ablkcipher_encrypt()
1053 	struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_ablkcipher_decrypt()  local
1059 	ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst,  in qat_alg_ablkcipher_decrypt()
1065 	*msg = ctx->dec_fw_req;  in qat_alg_ablkcipher_decrypt()
1066 	qat_req->ablkcipher_ctx = ctx;  in qat_alg_ablkcipher_decrypt()
1077 		ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);  in qat_alg_ablkcipher_decrypt()
1081 		qat_alg_free_bufl(ctx->inst, qat_req);  in qat_alg_ablkcipher_decrypt()
1091 	struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_aead_init()  local
1093 	ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);  in qat_alg_aead_init()
1094 	if (IS_ERR(ctx->hash_tfm))  in qat_alg_aead_init()
1096 	spin_lock_init(&ctx->lock);  in qat_alg_aead_init()
1097 	ctx->qat_hash_alg = hash;  in qat_alg_aead_init()
1100 	ctx->tfm = tfm;  in qat_alg_aead_init()
1121 	struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_aead_exit()  local
1122 	struct qat_crypto_instance *inst = ctx->inst;  in qat_alg_aead_exit()
1125 	if (!IS_ERR(ctx->hash_tfm))  in qat_alg_aead_exit()
1126 		crypto_free_shash(ctx->hash_tfm);  in qat_alg_aead_exit()
1132 	if (ctx->enc_cd) {  in qat_alg_aead_exit()
1133 		memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));  in qat_alg_aead_exit()
1135 				  ctx->enc_cd, ctx->enc_cd_paddr);  in qat_alg_aead_exit()
1137 	if (ctx->dec_cd) {  in qat_alg_aead_exit()
1138 		memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));  in qat_alg_aead_exit()
1140 				  ctx->dec_cd, ctx->dec_cd_paddr);  in qat_alg_aead_exit()
1147 	struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_ablkcipher_init()  local
1149 	spin_lock_init(&ctx->lock);  in qat_alg_ablkcipher_init()
1152 	ctx->tfm = tfm;  in qat_alg_ablkcipher_init()
1158 	struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);  in qat_alg_ablkcipher_exit()  local
1159 	struct qat_crypto_instance *inst = ctx->inst;  in qat_alg_ablkcipher_exit()
1166 	if (ctx->enc_cd) {  in qat_alg_ablkcipher_exit()
1167 		memset(ctx->enc_cd, 0,  in qat_alg_ablkcipher_exit()
1171 				  ctx->enc_cd, ctx->enc_cd_paddr);  in qat_alg_ablkcipher_exit()
1173 	if (ctx->dec_cd) {  in qat_alg_ablkcipher_exit()
1174 		memset(ctx->dec_cd, 0,  in qat_alg_ablkcipher_exit()
1178 				  ctx->dec_cd, ctx->dec_cd_paddr);  in qat_alg_ablkcipher_exit()