83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
106 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
108 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
111 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
130 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
151 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
171 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
176 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
181 static void aesni_gcm_enc_avx(void *ctx, u8 *out,  in aesni_gcm_enc_avx()  argument
186         struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;  in aesni_gcm_enc_avx()
188 		aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,  in aesni_gcm_enc_avx()
191 		aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);  in aesni_gcm_enc_avx()
192 		aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,  in aesni_gcm_enc_avx()
197 static void aesni_gcm_dec_avx(void *ctx, u8 *out,  in aesni_gcm_dec_avx()  argument
202         struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;  in aesni_gcm_dec_avx()
204 		aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,  in aesni_gcm_dec_avx()
207 		aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);  in aesni_gcm_dec_avx()
208 		aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,  in aesni_gcm_dec_avx()
222 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
227 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
232 static void aesni_gcm_enc_avx2(void *ctx, u8 *out,  in aesni_gcm_enc_avx2()  argument
237        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;  in aesni_gcm_enc_avx2()
239 		aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,  in aesni_gcm_enc_avx2()
242 		aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);  in aesni_gcm_enc_avx2()
243 		aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,  in aesni_gcm_enc_avx2()
246 		aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);  in aesni_gcm_enc_avx2()
247 		aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,  in aesni_gcm_enc_avx2()
252 static void aesni_gcm_dec_avx2(void *ctx, u8 *out,  in aesni_gcm_dec_avx2()  argument
257        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;  in aesni_gcm_dec_avx2()
259 		aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,  in aesni_gcm_dec_avx2()
262 		aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);  in aesni_gcm_dec_avx2()
263 		aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,  in aesni_gcm_dec_avx2()
266 		aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);  in aesni_gcm_dec_avx2()
267 		aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,  in aesni_gcm_dec_avx2()
273 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out,
278 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
307 	struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);  in aes_set_key_common()  local
318 		err = crypto_aes_expand_key(ctx, in_key, key_len);  in aes_set_key_common()
321 		err = aesni_set_key(ctx, in_key, key_len);  in aes_set_key_common()
336 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));  in aes_encrypt()  local
339 		crypto_aes_encrypt_x86(ctx, dst, src);  in aes_encrypt()
342 		aesni_enc(ctx, dst, src);  in aes_encrypt()
349 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));  in aes_decrypt()  local
352 		crypto_aes_decrypt_x86(ctx, dst, src);  in aes_decrypt()
355 		aesni_dec(ctx, dst, src);  in aes_decrypt()
362 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));  in __aes_encrypt()  local
364 	aesni_enc(ctx, dst, src);  in __aes_encrypt()
369 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));  in __aes_decrypt()  local
371 	aesni_dec(ctx, dst, src);  in __aes_decrypt()
378 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in ecb_encrypt()  local
388 		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in ecb_encrypt()
402 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in ecb_decrypt()  local
412 		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in ecb_decrypt()
426 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in cbc_encrypt()  local
436 		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in cbc_encrypt()
450 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in cbc_decrypt()  local
460 		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in cbc_decrypt()
471 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,  in ctr_crypt_final()  argument
480 	aesni_enc(ctx, keystream, ctrblk);  in ctr_crypt_final()
487 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,  in aesni_ctr_enc_avx_tfm()  argument
496 	if (ctx->key_length == AES_KEYSIZE_128)  in aesni_ctr_enc_avx_tfm()
497 		aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);  in aesni_ctr_enc_avx_tfm()
498 	else if (ctx->key_length == AES_KEYSIZE_192)  in aesni_ctr_enc_avx_tfm()
499 		aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);  in aesni_ctr_enc_avx_tfm()
501 		aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);  in aesni_ctr_enc_avx_tfm()
509 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in ctr_crypt()  local
519 		aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in ctr_crypt()
525 		ctr_crypt_final(ctx, &walk);  in ctr_crypt()
559 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)  in lrw_xts_encrypt_callback()  argument
561 	aesni_ecb_enc(ctx, blks, blks, nbytes);  in lrw_xts_encrypt_callback()
564 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)  in lrw_xts_decrypt_callback()  argument
566 	aesni_ecb_dec(ctx, blks, blks, nbytes);  in lrw_xts_decrypt_callback()
572 	struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);  in lrw_aesni_setkey()  local
575 	err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key,  in lrw_aesni_setkey()
580 	return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE);  in lrw_aesni_setkey()
585 	struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);  in lrw_aesni_exit_tfm()  local
587 	lrw_free_table(&ctx->lrw_table);  in lrw_aesni_exit_tfm()
593 	struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in lrw_encrypt()  local
599 		.table_ctx = &ctx->lrw_table,  in lrw_encrypt()
600 		.crypt_ctx = aes_ctx(ctx->raw_aes_ctx),  in lrw_encrypt()
617 	struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in lrw_decrypt()  local
623 		.table_ctx = &ctx->lrw_table,  in lrw_decrypt()
624 		.crypt_ctx = aes_ctx(ctx->raw_aes_ctx),  in lrw_decrypt()
641 	struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm);  in xts_aesni_setkey()  local
654 	err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2);  in xts_aesni_setkey()
659 	return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2,  in xts_aesni_setkey()
664 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)  in aesni_xts_tweak()  argument
666 	aesni_enc(ctx, out, in);  in aesni_xts_tweak()
671 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_enc()  argument
673 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));  in aesni_xts_enc()
676 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_dec()  argument
678 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));  in aesni_xts_dec()
681 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_enc8()  argument
683 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);  in aesni_xts_enc8()
686 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_dec8()  argument
688 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);  in aesni_xts_dec8()
720 	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in xts_encrypt()  local
724 				     aes_ctx(ctx->raw_tweak_ctx),  in xts_encrypt()
725 				     aes_ctx(ctx->raw_crypt_ctx));  in xts_encrypt()
731 	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in xts_decrypt()  local
735 				     aes_ctx(ctx->raw_tweak_ctx),  in xts_decrypt()
736 				     aes_ctx(ctx->raw_crypt_ctx));  in xts_decrypt()
744 	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in xts_encrypt()  local
750 		.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),  in xts_encrypt()
752 		.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),  in xts_encrypt()
769 	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in xts_decrypt()  local
775 		.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),  in xts_decrypt()
777 		.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),  in xts_decrypt()
797 	struct cryptd_aead **ctx = crypto_aead_ctx(aead);  in rfc4106_init()  local
805 	*ctx = cryptd_tfm;  in rfc4106_init()
812 	struct cryptd_aead **ctx = crypto_aead_ctx(aead);  in rfc4106_exit()  local
814 	cryptd_free_aead(*ctx);  in rfc4106_exit()
888 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);  in common_rfc4106_set_key()  local
897 	memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));  in common_rfc4106_set_key()
900 				  &ctx->aes_key_expanded, key, key_len) ?:  in common_rfc4106_set_key()
901 	       rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);  in common_rfc4106_set_key()
907 	struct cryptd_aead **ctx = crypto_aead_ctx(parent);  in rfc4106_set_key()  local
908 	struct cryptd_aead *cryptd_tfm = *ctx;  in rfc4106_set_key()
933 	struct cryptd_aead **ctx = crypto_aead_ctx(parent);  in rfc4106_set_authsize()  local
934 	struct cryptd_aead *cryptd_tfm = *ctx;  in rfc4106_set_authsize()
945 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);  in helper_rfc4106_encrypt()  local
946 	void *aes_ctx = &(ctx->aes_key_expanded);  in helper_rfc4106_encrypt()
961 		*(iv+i) = ctx->nonce[i];  in helper_rfc4106_encrypt()
993 			  ctx->hash_subkey, assoc, req->assoclen - 8,  in helper_rfc4106_encrypt()
1024 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);  in helper_rfc4106_decrypt()  local
1025 	void *aes_ctx = &(ctx->aes_key_expanded);  in helper_rfc4106_decrypt()
1043 		*(iv+i) = ctx->nonce[i];  in helper_rfc4106_decrypt()
1075 			  ctx->hash_subkey, assoc, req->assoclen - 8,  in helper_rfc4106_decrypt()
1103 	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);  in rfc4106_encrypt()  local
1104 	struct cryptd_aead *cryptd_tfm = *ctx;  in rfc4106_encrypt()
1116 	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);  in rfc4106_decrypt()  local
1117 	struct cryptd_aead *cryptd_tfm = *ctx;  in rfc4106_decrypt()