83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
106 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
108 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
111 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
130 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
151 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
171 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
176 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
181 static void aesni_gcm_enc_avx(void *ctx, u8 *out,  in aesni_gcm_enc_avx()  argument
186         struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;  in aesni_gcm_enc_avx()
188 		aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,  in aesni_gcm_enc_avx()
191 		aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);  in aesni_gcm_enc_avx()
192 		aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,  in aesni_gcm_enc_avx()
197 static void aesni_gcm_dec_avx(void *ctx, u8 *out,  in aesni_gcm_dec_avx()  argument
202         struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;  in aesni_gcm_dec_avx()
204 		aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,  in aesni_gcm_dec_avx()
207 		aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);  in aesni_gcm_dec_avx()
208 		aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,  in aesni_gcm_dec_avx()
222 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
227 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
232 static void aesni_gcm_enc_avx2(void *ctx, u8 *out,  in aesni_gcm_enc_avx2()  argument
237        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;  in aesni_gcm_enc_avx2()
239 		aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,  in aesni_gcm_enc_avx2()
242 		aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);  in aesni_gcm_enc_avx2()
243 		aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,  in aesni_gcm_enc_avx2()
246 		aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);  in aesni_gcm_enc_avx2()
247 		aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,  in aesni_gcm_enc_avx2()
252 static void aesni_gcm_dec_avx2(void *ctx, u8 *out,  in aesni_gcm_dec_avx2()  argument
257        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;  in aesni_gcm_dec_avx2()
259 		aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,  in aesni_gcm_dec_avx2()
262 		aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);  in aesni_gcm_dec_avx2()
263 		aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,  in aesni_gcm_dec_avx2()
266 		aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);  in aesni_gcm_dec_avx2()
267 		aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,  in aesni_gcm_dec_avx2()
273 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out,
278 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
306 	struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);  in aes_set_key_common()  local
317 		err = crypto_aes_expand_key(ctx, in_key, key_len);  in aes_set_key_common()
320 		err = aesni_set_key(ctx, in_key, key_len);  in aes_set_key_common()
335 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));  in aes_encrypt()  local
338 		crypto_aes_encrypt_x86(ctx, dst, src);  in aes_encrypt()
341 		aesni_enc(ctx, dst, src);  in aes_encrypt()
348 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));  in aes_decrypt()  local
351 		crypto_aes_decrypt_x86(ctx, dst, src);  in aes_decrypt()
354 		aesni_dec(ctx, dst, src);  in aes_decrypt()
361 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));  in __aes_encrypt()  local
363 	aesni_enc(ctx, dst, src);  in __aes_encrypt()
368 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));  in __aes_decrypt()  local
370 	aesni_dec(ctx, dst, src);  in __aes_decrypt()
377 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in ecb_encrypt()  local
387 		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in ecb_encrypt()
401 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in ecb_decrypt()  local
411 		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in ecb_decrypt()
425 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in cbc_encrypt()  local
435 		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in cbc_encrypt()
449 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in cbc_decrypt()  local
459 		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in cbc_decrypt()
470 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,  in ctr_crypt_final()  argument
479 	aesni_enc(ctx, keystream, ctrblk);  in ctr_crypt_final()
486 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,  in aesni_ctr_enc_avx_tfm()  argument
495 	if (ctx->key_length == AES_KEYSIZE_128)  in aesni_ctr_enc_avx_tfm()
496 		aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);  in aesni_ctr_enc_avx_tfm()
497 	else if (ctx->key_length == AES_KEYSIZE_192)  in aesni_ctr_enc_avx_tfm()
498 		aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);  in aesni_ctr_enc_avx_tfm()
500 		aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);  in aesni_ctr_enc_avx_tfm()
508 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));  in ctr_crypt()  local
518 		aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,  in ctr_crypt()
524 		ctr_crypt_final(ctx, &walk);  in ctr_crypt()
558 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)  in lrw_xts_encrypt_callback()  argument
560 	aesni_ecb_enc(ctx, blks, blks, nbytes);  in lrw_xts_encrypt_callback()
563 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)  in lrw_xts_decrypt_callback()  argument
565 	aesni_ecb_dec(ctx, blks, blks, nbytes);  in lrw_xts_decrypt_callback()
571 	struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);  in lrw_aesni_setkey()  local
574 	err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key,  in lrw_aesni_setkey()
579 	return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE);  in lrw_aesni_setkey()
584 	struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);  in lrw_aesni_exit_tfm()  local
586 	lrw_free_table(&ctx->lrw_table);  in lrw_aesni_exit_tfm()
592 	struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in lrw_encrypt()  local
598 		.table_ctx = &ctx->lrw_table,  in lrw_encrypt()
599 		.crypt_ctx = aes_ctx(ctx->raw_aes_ctx),  in lrw_encrypt()
616 	struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in lrw_decrypt()  local
622 		.table_ctx = &ctx->lrw_table,  in lrw_decrypt()
623 		.crypt_ctx = aes_ctx(ctx->raw_aes_ctx),  in lrw_decrypt()
640 	struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm);  in xts_aesni_setkey()  local
653 	err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2);  in xts_aesni_setkey()
658 	return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2,  in xts_aesni_setkey()
663 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)  in aesni_xts_tweak()  argument
665 	aesni_enc(ctx, out, in);  in aesni_xts_tweak()
670 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_enc()  argument
672 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));  in aesni_xts_enc()
675 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_dec()  argument
677 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));  in aesni_xts_dec()
680 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_enc8()  argument
682 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);  in aesni_xts_enc8()
685 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)  in aesni_xts_dec8()  argument
687 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);  in aesni_xts_dec8()
719 	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in xts_encrypt()  local
723 				     aes_ctx(ctx->raw_tweak_ctx),  in xts_encrypt()
724 				     aes_ctx(ctx->raw_crypt_ctx));  in xts_encrypt()
730 	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in xts_decrypt()  local
734 				     aes_ctx(ctx->raw_tweak_ctx),  in xts_decrypt()
735 				     aes_ctx(ctx->raw_crypt_ctx));  in xts_decrypt()
743 	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in xts_encrypt()  local
749 		.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),  in xts_encrypt()
751 		.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),  in xts_encrypt()
768 	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);  in xts_decrypt()  local
774 		.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),  in xts_decrypt()
776 		.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),  in xts_decrypt()
796 	struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)  in rfc4106_init()  local
808 	memcpy(child_ctx, ctx, sizeof(*ctx));  in rfc4106_init()
809 	ctx->cryptd_tfm = cryptd_tfm;  in rfc4106_init()
817 	struct aesni_rfc4106_gcm_ctx *ctx =  in rfc4106_exit()  local
820 	if (!IS_ERR(ctx->cryptd_tfm))  in rfc4106_exit()
821 		cryptd_free_aead(ctx->cryptd_tfm);  in rfc4106_exit()
900 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);  in common_rfc4106_set_key()  local
915 	memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));  in common_rfc4106_set_key()
917 	if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)  in common_rfc4106_set_key()
932 		ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),  in common_rfc4106_set_key()
936 		ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);  in common_rfc4106_set_key()
940 	if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {  in common_rfc4106_set_key()
944 	ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);  in common_rfc4106_set_key()
953 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);  in rfc4106_set_key()  local
954 	struct crypto_aead *child = cryptd_aead_child(ctx->cryptd_tfm);  in rfc4106_set_key()
956 	struct cryptd_aead *cryptd_tfm = ctx->cryptd_tfm;  in rfc4106_set_key()
961 		memcpy(ctx, c_ctx, sizeof(*ctx));  in rfc4106_set_key()
962 		ctx->cryptd_tfm = cryptd_tfm;  in rfc4106_set_key()
987 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);  in rfc4106_set_authsize()  local
988 	struct crypto_aead *child = cryptd_aead_child(ctx->cryptd_tfm);  in rfc4106_set_authsize()
1003 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);  in __driver_rfc4106_encrypt()  local
1004 	u32 key_len = ctx->aes_key_expanded.key_length;  in __driver_rfc4106_encrypt()
1005 	void *aes_ctx = &(ctx->aes_key_expanded);  in __driver_rfc4106_encrypt()
1028 		*(iv+i) = ctx->nonce[i];  in __driver_rfc4106_encrypt()
1059 		ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst  in __driver_rfc4106_encrypt()
1089 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);  in __driver_rfc4106_decrypt()  local
1090 	u32 key_len = ctx->aes_key_expanded.key_length;  in __driver_rfc4106_decrypt()
1091 	void *aes_ctx = &(ctx->aes_key_expanded);  in __driver_rfc4106_decrypt()
1118 		*(iv+i) = ctx->nonce[i];  in __driver_rfc4106_decrypt()
1148 		ctx->hash_subkey, assoc, (unsigned long)req->assoclen,  in __driver_rfc4106_decrypt()
1175 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);  in rfc4106_encrypt()  local
1182 		aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);  in rfc4106_encrypt()
1196 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);  in rfc4106_decrypt()  local
1203 		aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);  in rfc4106_decrypt()