334 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)  in aes_encrypt()  argument
339 		crypto_aes_encrypt_x86(ctx, dst, src);  in aes_encrypt()
342 		aesni_enc(ctx, dst, src);  in aes_encrypt()
347 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)  in aes_decrypt()  argument
352 		crypto_aes_decrypt_x86(ctx, dst, src);  in aes_decrypt()
355 		aesni_dec(ctx, dst, src);  in aes_decrypt()
360 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)  in __aes_encrypt()  argument
364 	aesni_enc(ctx, dst, src);  in __aes_encrypt()
367 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)  in __aes_decrypt()  argument
371 	aesni_dec(ctx, dst, src);  in __aes_decrypt()
375 		       struct scatterlist *dst, struct scatterlist *src,  in ecb_encrypt()  argument
382 	blkcipher_walk_init(&walk, dst, src, nbytes);  in ecb_encrypt()
388 		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in ecb_encrypt()
399 		       struct scatterlist *dst, struct scatterlist *src,  in ecb_decrypt()  argument
406 	blkcipher_walk_init(&walk, dst, src, nbytes);  in ecb_decrypt()
412 		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in ecb_decrypt()
423 		       struct scatterlist *dst, struct scatterlist *src,  in cbc_encrypt()  argument
430 	blkcipher_walk_init(&walk, dst, src, nbytes);  in cbc_encrypt()
436 		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in cbc_encrypt()
447 		       struct scatterlist *dst, struct scatterlist *src,  in cbc_decrypt()  argument
454 	blkcipher_walk_init(&walk, dst, src, nbytes);  in cbc_decrypt()
460 		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in cbc_decrypt()
476 	u8 *src = walk->src.virt.addr;  in ctr_crypt_final()  local
481 	crypto_xor(keystream, src, nbytes);  in ctr_crypt_final()
506 		     struct scatterlist *dst, struct scatterlist *src,  in ctr_crypt()  argument
513 	blkcipher_walk_init(&walk, dst, src, nbytes);  in ctr_crypt()
519 		aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in ctr_crypt()
591 		       struct scatterlist *src, unsigned int nbytes)  in lrw_encrypt()  argument
608 	ret = lrw_crypt(desc, dst, src, nbytes, &req);  in lrw_encrypt()
615 		       struct scatterlist *src, unsigned int nbytes)  in lrw_decrypt()  argument
632 	ret = lrw_crypt(desc, dst, src, nbytes, &req);  in lrw_decrypt()
671 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_enc()  argument
673 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));  in aesni_xts_enc()
676 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_dec()  argument
678 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));  in aesni_xts_dec()
681 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_enc8()  argument
683 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);  in aesni_xts_enc8()
686 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_dec8()  argument
688 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);  in aesni_xts_dec8()
718 		       struct scatterlist *src, unsigned int nbytes)  in xts_encrypt()  argument
722 	return glue_xts_crypt_128bit(&aesni_enc_xts, desc, dst, src, nbytes,  in xts_encrypt()
729 		       struct scatterlist *src, unsigned int nbytes)  in xts_decrypt()  argument
733 	return glue_xts_crypt_128bit(&aesni_dec_xts, desc, dst, src, nbytes,  in xts_decrypt()
742 		       struct scatterlist *src, unsigned int nbytes)  in xts_encrypt()  argument
760 	ret = xts_crypt(desc, dst, src, nbytes, &req);  in xts_encrypt()
767 		       struct scatterlist *src, unsigned int nbytes)  in xts_decrypt()  argument
785 	ret = xts_crypt(desc, dst, src, nbytes, &req);  in xts_decrypt()
942 	u8 *src, *dst, *assoc;  in helper_rfc4106_encrypt()  local
966 	if (sg_is_last(req->src) &&  in helper_rfc4106_encrypt()
967 	    req->src->offset + req->src->length <= PAGE_SIZE &&  in helper_rfc4106_encrypt()
971 		scatterwalk_start(&src_sg_walk, req->src);  in helper_rfc4106_encrypt()
973 		src = assoc + req->assoclen;  in helper_rfc4106_encrypt()
974 		dst = src;  in helper_rfc4106_encrypt()
975 		if (unlikely(req->src != req->dst)) {  in helper_rfc4106_encrypt()
985 		scatterwalk_map_and_copy(assoc, req->src, 0,  in helper_rfc4106_encrypt()
987 		src = assoc + req->assoclen;  in helper_rfc4106_encrypt()
988 		dst = src;  in helper_rfc4106_encrypt()
992 	aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,  in helper_rfc4106_encrypt()
1000 		if (unlikely(req->src != req->dst)) {  in helper_rfc4106_encrypt()
1006 		scatterwalk_advance(&src_sg_walk, req->src->length);  in helper_rfc4106_encrypt()
1007 		scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);  in helper_rfc4106_encrypt()
1019 	u8 *src, *dst, *assoc;  in helper_rfc4106_decrypt()  local
1048 	if (sg_is_last(req->src) &&  in helper_rfc4106_decrypt()
1049 	    req->src->offset + req->src->length <= PAGE_SIZE &&  in helper_rfc4106_decrypt()
1053 		scatterwalk_start(&src_sg_walk, req->src);  in helper_rfc4106_decrypt()
1055 		src = assoc + req->assoclen;  in helper_rfc4106_decrypt()
1056 		dst = src;  in helper_rfc4106_decrypt()
1057 		if (unlikely(req->src != req->dst)) {  in helper_rfc4106_decrypt()
1067 		scatterwalk_map_and_copy(assoc, req->src, 0,  in helper_rfc4106_decrypt()
1069 		src = assoc + req->assoclen;  in helper_rfc4106_decrypt()
1070 		dst = src;  in helper_rfc4106_decrypt()
1074 	aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,  in helper_rfc4106_decrypt()
1080 	retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?  in helper_rfc4106_decrypt()
1084 		if (unlikely(req->src != req->dst)) {  in helper_rfc4106_decrypt()
1090 		scatterwalk_advance(&src_sg_walk, req->src->length);  in helper_rfc4106_decrypt()
1091 		scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);  in helper_rfc4106_decrypt()