1/* 2 * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia 3 * 4 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 * 11 */ 12 13#include <linux/module.h> 14#include <linux/types.h> 15#include <linux/crypto.h> 16#include <linux/err.h> 17#include <crypto/ablk_helper.h> 18#include <crypto/algapi.h> 19#include <crypto/ctr.h> 20#include <crypto/lrw.h> 21#include <crypto/xts.h> 22#include <asm/xcr.h> 23#include <asm/xsave.h> 24#include <asm/crypto/camellia.h> 25#include <asm/crypto/glue_helper.h> 26 27#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 28#define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32 29 30/* 32-way AVX2/AES-NI parallel cipher functions */ 31asmlinkage void camellia_ecb_enc_32way(struct camellia_ctx *ctx, u8 *dst, 32 const u8 *src); 33asmlinkage void camellia_ecb_dec_32way(struct camellia_ctx *ctx, u8 *dst, 34 const u8 *src); 35 36asmlinkage void camellia_cbc_dec_32way(struct camellia_ctx *ctx, u8 *dst, 37 const u8 *src); 38asmlinkage void camellia_ctr_32way(struct camellia_ctx *ctx, u8 *dst, 39 const u8 *src, le128 *iv); 40 41asmlinkage void camellia_xts_enc_32way(struct camellia_ctx *ctx, u8 *dst, 42 const u8 *src, le128 *iv); 43asmlinkage void camellia_xts_dec_32way(struct camellia_ctx *ctx, u8 *dst, 44 const u8 *src, le128 *iv); 45 46static const struct common_glue_ctx camellia_enc = { 47 .num_funcs = 4, 48 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 49 50 .funcs = { { 51 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 52 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_32way) } 53 }, { 54 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 55 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) } 56 }, { 57 .num_blocks = 2, 58 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) } 59 }, { 60 .num_blocks = 1, 61 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) } 62 } } 63}; 64 65static const struct common_glue_ctx camellia_ctr = { 66 .num_funcs = 4, 67 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 68 69 .funcs = { { 70 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 71 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_32way) } 72 }, { 73 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 74 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) } 75 }, { 76 .num_blocks = 2, 77 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) } 78 }, { 79 .num_blocks = 1, 80 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) } 81 } } 82}; 83 84static const struct common_glue_ctx camellia_enc_xts = { 85 .num_funcs = 3, 86 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 87 88 .funcs = { { 89 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 90 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_32way) } 91 }, { 92 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 93 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) } 94 }, { 95 .num_blocks = 1, 96 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) } 97 } } 98}; 99 100static const struct common_glue_ctx camellia_dec = { 101 .num_funcs = 4, 102 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 103 104 .funcs = { { 105 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 106 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_32way) } 107 }, { 108 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 109 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) } 110 }, { 111 .num_blocks = 2, 112 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) } 113 }, { 114 .num_blocks = 1, 115 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) } 116 } } 117}; 118 119static const struct common_glue_ctx camellia_dec_cbc = { 120 .num_funcs = 4, 121 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 122 123 .funcs = { { 124 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 125 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_32way) } 126 }, { 127 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 128 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) } 129 }, { 130 .num_blocks = 2, 131 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) } 132 }, { 133 .num_blocks = 1, 134 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) } 135 } } 136}; 137 138static const struct common_glue_ctx camellia_dec_xts = { 139 .num_funcs = 3, 140 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 141 142 .funcs = { { 143 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 144 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_32way) } 145 }, { 146 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 147 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) } 148 }, { 149 .num_blocks = 1, 150 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) } 151 } } 152}; 153 154static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 155 struct scatterlist *src, unsigned int nbytes) 156{ 157 return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes); 158} 159 160static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 161 struct scatterlist *src, unsigned int nbytes) 162{ 163 return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes); 164} 165 166static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 167 struct scatterlist *src, unsigned int nbytes) 168{ 169 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc, 170 dst, src, nbytes); 171} 172 173static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 174 struct scatterlist *src, unsigned int nbytes) 175{ 176 return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src, 177 nbytes); 178} 179 180static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 181 struct scatterlist *src, unsigned int nbytes) 182{ 183 return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes); 184} 185 186static inline bool camellia_fpu_begin(bool fpu_enabled, unsigned int nbytes) 187{ 188 return glue_fpu_begin(CAMELLIA_BLOCK_SIZE, 189 CAMELLIA_AESNI_PARALLEL_BLOCKS, NULL, fpu_enabled, 190 nbytes); 191} 192 193static inline void camellia_fpu_end(bool fpu_enabled) 194{ 195 glue_fpu_end(fpu_enabled); 196} 197 198static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key, 199 unsigned int key_len) 200{ 201 return __camellia_setkey(crypto_tfm_ctx(tfm), in_key, key_len, 202 &tfm->crt_flags); 203} 204 205struct crypt_priv { 206 struct camellia_ctx *ctx; 207 bool fpu_enabled; 208}; 209 210static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 211{ 212 const unsigned int bsize = CAMELLIA_BLOCK_SIZE; 213 struct crypt_priv *ctx = priv; 214 int i; 215 216 ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes); 217 218 if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) { 219 camellia_ecb_enc_32way(ctx->ctx, srcdst, srcdst); 220 srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS; 221 nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS; 222 } 223 224 if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) { 225 camellia_ecb_enc_16way(ctx->ctx, srcdst, srcdst); 226 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS; 227 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS; 228 } 229 230 while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) { 231 camellia_enc_blk_2way(ctx->ctx, srcdst, srcdst); 232 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS; 233 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS; 234 } 235 236 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 237 camellia_enc_blk(ctx->ctx, srcdst, srcdst); 238} 239 240static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 241{ 242 const unsigned int bsize = CAMELLIA_BLOCK_SIZE; 243 struct crypt_priv *ctx = priv; 244 int i; 245 246 ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes); 247 248 if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) { 249 camellia_ecb_dec_32way(ctx->ctx, srcdst, srcdst); 250 srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS; 251 nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS; 252 } 253 254 if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) { 255 camellia_ecb_dec_16way(ctx->ctx, srcdst, srcdst); 256 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS; 257 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS; 258 } 259 260 while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) { 261 camellia_dec_blk_2way(ctx->ctx, srcdst, srcdst); 262 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS; 263 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS; 264 } 265 266 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 267 camellia_dec_blk(ctx->ctx, srcdst, srcdst); 268} 269 270static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 271 struct scatterlist *src, unsigned int nbytes) 272{ 273 struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 274 be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS]; 275 struct crypt_priv crypt_ctx = { 276 .ctx = &ctx->camellia_ctx, 277 .fpu_enabled = false, 278 }; 279 struct lrw_crypt_req req = { 280 .tbuf = buf, 281 .tbuflen = sizeof(buf), 282 283 .table_ctx = &ctx->lrw_table, 284 .crypt_ctx = &crypt_ctx, 285 .crypt_fn = encrypt_callback, 286 }; 287 int ret; 288 289 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 290 ret = lrw_crypt(desc, dst, src, nbytes, &req); 291 camellia_fpu_end(crypt_ctx.fpu_enabled); 292 293 return ret; 294} 295 296static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 297 struct scatterlist *src, unsigned int nbytes) 298{ 299 struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 300 be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS]; 301 struct crypt_priv crypt_ctx = { 302 .ctx = &ctx->camellia_ctx, 303 .fpu_enabled = false, 304 }; 305 struct lrw_crypt_req req = { 306 .tbuf = buf, 307 .tbuflen = sizeof(buf), 308 309 .table_ctx = &ctx->lrw_table, 310 .crypt_ctx = &crypt_ctx, 311 .crypt_fn = decrypt_callback, 312 }; 313 int ret; 314 315 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 316 ret = lrw_crypt(desc, dst, src, nbytes, &req); 317 camellia_fpu_end(crypt_ctx.fpu_enabled); 318 319 return ret; 320} 321 322static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 323 struct scatterlist *src, unsigned int nbytes) 324{ 325 struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 326 327 return glue_xts_crypt_128bit(&camellia_enc_xts, desc, dst, src, nbytes, 328 XTS_TWEAK_CAST(camellia_enc_blk), 329 &ctx->tweak_ctx, &ctx->crypt_ctx); 330} 331 332static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 333 struct scatterlist *src, unsigned int nbytes) 334{ 335 struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 336 337 return glue_xts_crypt_128bit(&camellia_dec_xts, desc, dst, src, nbytes, 338 XTS_TWEAK_CAST(camellia_enc_blk), 339 &ctx->tweak_ctx, &ctx->crypt_ctx); 340} 341 342static struct crypto_alg cmll_algs[10] = { { 343 .cra_name = "__ecb-camellia-aesni-avx2", 344 .cra_driver_name = "__driver-ecb-camellia-aesni-avx2", 345 .cra_priority = 0, 346 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 347 CRYPTO_ALG_INTERNAL, 348 .cra_blocksize = CAMELLIA_BLOCK_SIZE, 349 .cra_ctxsize = sizeof(struct camellia_ctx), 350 .cra_alignmask = 0, 351 .cra_type = &crypto_blkcipher_type, 352 .cra_module = THIS_MODULE, 353 .cra_u = { 354 .blkcipher = { 355 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 356 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 357 .setkey = camellia_setkey, 358 .encrypt = ecb_encrypt, 359 .decrypt = ecb_decrypt, 360 }, 361 }, 362}, { 363 .cra_name = "__cbc-camellia-aesni-avx2", 364 .cra_driver_name = "__driver-cbc-camellia-aesni-avx2", 365 .cra_priority = 0, 366 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 367 CRYPTO_ALG_INTERNAL, 368 .cra_blocksize = CAMELLIA_BLOCK_SIZE, 369 .cra_ctxsize = sizeof(struct camellia_ctx), 370 .cra_alignmask = 0, 371 .cra_type = &crypto_blkcipher_type, 372 .cra_module = THIS_MODULE, 373 .cra_u = { 374 .blkcipher = { 375 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 376 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 377 .setkey = camellia_setkey, 378 .encrypt = cbc_encrypt, 379 .decrypt = cbc_decrypt, 380 }, 381 }, 382}, { 383 .cra_name = "__ctr-camellia-aesni-avx2", 384 .cra_driver_name = "__driver-ctr-camellia-aesni-avx2", 385 .cra_priority = 0, 386 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 387 CRYPTO_ALG_INTERNAL, 388 .cra_blocksize = 1, 389 .cra_ctxsize = sizeof(struct camellia_ctx), 390 .cra_alignmask = 0, 391 .cra_type = &crypto_blkcipher_type, 392 .cra_module = THIS_MODULE, 393 .cra_u = { 394 .blkcipher = { 395 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 396 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 397 .ivsize = CAMELLIA_BLOCK_SIZE, 398 .setkey = camellia_setkey, 399 .encrypt = ctr_crypt, 400 .decrypt = ctr_crypt, 401 }, 402 }, 403}, { 404 .cra_name = "__lrw-camellia-aesni-avx2", 405 .cra_driver_name = "__driver-lrw-camellia-aesni-avx2", 406 .cra_priority = 0, 407 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 408 CRYPTO_ALG_INTERNAL, 409 .cra_blocksize = CAMELLIA_BLOCK_SIZE, 410 .cra_ctxsize = sizeof(struct camellia_lrw_ctx), 411 .cra_alignmask = 0, 412 .cra_type = &crypto_blkcipher_type, 413 .cra_module = THIS_MODULE, 414 .cra_exit = lrw_camellia_exit_tfm, 415 .cra_u = { 416 .blkcipher = { 417 .min_keysize = CAMELLIA_MIN_KEY_SIZE + 418 CAMELLIA_BLOCK_SIZE, 419 .max_keysize = CAMELLIA_MAX_KEY_SIZE + 420 CAMELLIA_BLOCK_SIZE, 421 .ivsize = CAMELLIA_BLOCK_SIZE, 422 .setkey = lrw_camellia_setkey, 423 .encrypt = lrw_encrypt, 424 .decrypt = lrw_decrypt, 425 }, 426 }, 427}, { 428 .cra_name = "__xts-camellia-aesni-avx2", 429 .cra_driver_name = "__driver-xts-camellia-aesni-avx2", 430 .cra_priority = 0, 431 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 432 CRYPTO_ALG_INTERNAL, 433 .cra_blocksize = CAMELLIA_BLOCK_SIZE, 434 .cra_ctxsize = sizeof(struct camellia_xts_ctx), 435 .cra_alignmask = 0, 436 .cra_type = &crypto_blkcipher_type, 437 .cra_module = THIS_MODULE, 438 .cra_u = { 439 .blkcipher = { 440 .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2, 441 .max_keysize = CAMELLIA_MAX_KEY_SIZE * 2, 442 .ivsize = CAMELLIA_BLOCK_SIZE, 443 .setkey = xts_camellia_setkey, 444 .encrypt = xts_encrypt, 445 .decrypt = xts_decrypt, 446 }, 447 }, 448}, { 449 .cra_name = "ecb(camellia)", 450 .cra_driver_name = "ecb-camellia-aesni-avx2", 451 .cra_priority = 500, 452 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 453 .cra_blocksize = CAMELLIA_BLOCK_SIZE, 454 .cra_ctxsize = sizeof(struct async_helper_ctx), 455 .cra_alignmask = 0, 456 .cra_type = &crypto_ablkcipher_type, 457 .cra_module = THIS_MODULE, 458 .cra_init = ablk_init, 459 .cra_exit = ablk_exit, 460 .cra_u = { 461 .ablkcipher = { 462 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 463 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 464 .setkey = ablk_set_key, 465 .encrypt = ablk_encrypt, 466 .decrypt = ablk_decrypt, 467 }, 468 }, 469}, { 470 .cra_name = "cbc(camellia)", 471 .cra_driver_name = "cbc-camellia-aesni-avx2", 472 .cra_priority = 500, 473 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 474 .cra_blocksize = CAMELLIA_BLOCK_SIZE, 475 .cra_ctxsize = sizeof(struct async_helper_ctx), 476 .cra_alignmask = 0, 477 .cra_type = &crypto_ablkcipher_type, 478 .cra_module = THIS_MODULE, 479 .cra_init = ablk_init, 480 .cra_exit = ablk_exit, 481 .cra_u = { 482 .ablkcipher = { 483 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 484 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 485 .ivsize = CAMELLIA_BLOCK_SIZE, 486 .setkey = ablk_set_key, 487 .encrypt = __ablk_encrypt, 488 .decrypt = ablk_decrypt, 489 }, 490 }, 491}, { 492 .cra_name = "ctr(camellia)", 493 .cra_driver_name = "ctr-camellia-aesni-avx2", 494 .cra_priority = 500, 495 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 496 .cra_blocksize = 1, 497 .cra_ctxsize = sizeof(struct async_helper_ctx), 498 .cra_alignmask = 0, 499 .cra_type = &crypto_ablkcipher_type, 500 .cra_module = THIS_MODULE, 501 .cra_init = ablk_init, 502 .cra_exit = ablk_exit, 503 .cra_u = { 504 .ablkcipher = { 505 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 506 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 507 .ivsize = CAMELLIA_BLOCK_SIZE, 508 .setkey = ablk_set_key, 509 .encrypt = ablk_encrypt, 510 .decrypt = ablk_encrypt, 511 .geniv = "chainiv", 512 }, 513 }, 514}, { 515 .cra_name = "lrw(camellia)", 516 .cra_driver_name = "lrw-camellia-aesni-avx2", 517 .cra_priority = 500, 518 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 519 .cra_blocksize = CAMELLIA_BLOCK_SIZE, 520 .cra_ctxsize = sizeof(struct async_helper_ctx), 521 .cra_alignmask = 0, 522 .cra_type = &crypto_ablkcipher_type, 523 .cra_module = THIS_MODULE, 524 .cra_init = ablk_init, 525 .cra_exit = ablk_exit, 526 .cra_u = { 527 .ablkcipher = { 528 .min_keysize = CAMELLIA_MIN_KEY_SIZE + 529 CAMELLIA_BLOCK_SIZE, 530 .max_keysize = CAMELLIA_MAX_KEY_SIZE + 531 CAMELLIA_BLOCK_SIZE, 532 .ivsize = CAMELLIA_BLOCK_SIZE, 533 .setkey = ablk_set_key, 534 .encrypt = ablk_encrypt, 535 .decrypt = ablk_decrypt, 536 }, 537 }, 538}, { 539 .cra_name = "xts(camellia)", 540 .cra_driver_name = "xts-camellia-aesni-avx2", 541 .cra_priority = 500, 542 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 543 .cra_blocksize = CAMELLIA_BLOCK_SIZE, 544 .cra_ctxsize = sizeof(struct async_helper_ctx), 545 .cra_alignmask = 0, 546 .cra_type = &crypto_ablkcipher_type, 547 .cra_module = THIS_MODULE, 548 .cra_init = ablk_init, 549 .cra_exit = ablk_exit, 550 .cra_u = { 551 .ablkcipher = { 552 .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2, 553 .max_keysize = CAMELLIA_MAX_KEY_SIZE * 2, 554 .ivsize = CAMELLIA_BLOCK_SIZE, 555 .setkey = ablk_set_key, 556 .encrypt = ablk_encrypt, 557 .decrypt = ablk_decrypt, 558 }, 559 }, 560} }; 561 562static int __init camellia_aesni_init(void) 563{ 564 u64 xcr0; 565 566 if (!cpu_has_avx2 || !cpu_has_avx || !cpu_has_aes || !cpu_has_osxsave) { 567 pr_info("AVX2 or AES-NI instructions are not detected.\n"); 568 return -ENODEV; 569 } 570 571 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 572 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { 573 pr_info("AVX2 detected but unusable.\n"); 574 return -ENODEV; 575 } 576 577 return crypto_register_algs(cmll_algs, ARRAY_SIZE(cmll_algs)); 578} 579 580static void __exit camellia_aesni_fini(void) 581{ 582 crypto_unregister_algs(cmll_algs, ARRAY_SIZE(cmll_algs)); 583} 584 585module_init(camellia_aesni_init); 586module_exit(camellia_aesni_fini); 587 588MODULE_LICENSE("GPL"); 589MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized"); 590MODULE_ALIAS_CRYPTO("camellia"); 591MODULE_ALIAS_CRYPTO("camellia-asm"); 592