1/*
2 * Glue Code for AVX assembler version of Twofish Cipher
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
22 * USA
23 *
24 */
25
26#include <linux/module.h>
27#include <linux/hardirq.h>
28#include <linux/types.h>
29#include <linux/crypto.h>
30#include <linux/err.h>
31#include <crypto/ablk_helper.h>
32#include <crypto/algapi.h>
33#include <crypto/twofish.h>
34#include <crypto/cryptd.h>
35#include <crypto/b128ops.h>
36#include <crypto/ctr.h>
37#include <crypto/lrw.h>
38#include <crypto/xts.h>
39#include <asm/i387.h>
40#include <asm/xcr.h>
41#include <asm/xsave.h>
42#include <asm/crypto/twofish.h>
43#include <asm/crypto/glue_helper.h>
44#include <crypto/scatterwalk.h>
45#include <linux/workqueue.h>
46#include <linux/spinlock.h>
47
48#define TWOFISH_PARALLEL_BLOCKS 8
49
50/* 8-way parallel cipher functions */
51asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
52				     const u8 *src);
53asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
54				     const u8 *src);
55
56asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
57				     const u8 *src);
58asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
59				 const u8 *src, le128 *iv);
60
61asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
62				     const u8 *src, le128 *iv);
63asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
64				     const u8 *src, le128 *iv);
65
66static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
67					const u8 *src)
68{
69	__twofish_enc_blk_3way(ctx, dst, src, false);
70}
71
72static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
73{
74	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
75				  GLUE_FUNC_CAST(twofish_enc_blk));
76}
77
78static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
79{
80	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
81				  GLUE_FUNC_CAST(twofish_dec_blk));
82}
83
84
85static const struct common_glue_ctx twofish_enc = {
86	.num_funcs = 3,
87	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
88
89	.funcs = { {
90		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
91		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
92	}, {
93		.num_blocks = 3,
94		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
95	}, {
96		.num_blocks = 1,
97		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
98	} }
99};
100
101static const struct common_glue_ctx twofish_ctr = {
102	.num_funcs = 3,
103	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
104
105	.funcs = { {
106		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
107		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
108	}, {
109		.num_blocks = 3,
110		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
111	}, {
112		.num_blocks = 1,
113		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
114	} }
115};
116
117static const struct common_glue_ctx twofish_enc_xts = {
118	.num_funcs = 2,
119	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
120
121	.funcs = { {
122		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
123		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
124	}, {
125		.num_blocks = 1,
126		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
127	} }
128};
129
130static const struct common_glue_ctx twofish_dec = {
131	.num_funcs = 3,
132	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
133
134	.funcs = { {
135		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
136		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
137	}, {
138		.num_blocks = 3,
139		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
140	}, {
141		.num_blocks = 1,
142		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
143	} }
144};
145
146static const struct common_glue_ctx twofish_dec_cbc = {
147	.num_funcs = 3,
148	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
149
150	.funcs = { {
151		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
152		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
153	}, {
154		.num_blocks = 3,
155		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
156	}, {
157		.num_blocks = 1,
158		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
159	} }
160};
161
162static const struct common_glue_ctx twofish_dec_xts = {
163	.num_funcs = 2,
164	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
165
166	.funcs = { {
167		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
168		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
169	}, {
170		.num_blocks = 1,
171		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
172	} }
173};
174
175static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
176		       struct scatterlist *src, unsigned int nbytes)
177{
178	return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
179}
180
181static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
182		       struct scatterlist *src, unsigned int nbytes)
183{
184	return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
185}
186
187static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
188		       struct scatterlist *src, unsigned int nbytes)
189{
190	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
191				       dst, src, nbytes);
192}
193
194static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
195		       struct scatterlist *src, unsigned int nbytes)
196{
197	return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
198				       nbytes);
199}
200
201static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
202		     struct scatterlist *src, unsigned int nbytes)
203{
204	return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
205}
206
207static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
208{
209	return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
210			      fpu_enabled, nbytes);
211}
212
213static inline void twofish_fpu_end(bool fpu_enabled)
214{
215	glue_fpu_end(fpu_enabled);
216}
217
218struct crypt_priv {
219	struct twofish_ctx *ctx;
220	bool fpu_enabled;
221};
222
223static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
224{
225	const unsigned int bsize = TF_BLOCK_SIZE;
226	struct crypt_priv *ctx = priv;
227	int i;
228
229	ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
230
231	if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
232		twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
233		return;
234	}
235
236	for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
237		twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
238
239	nbytes %= bsize * 3;
240
241	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
242		twofish_enc_blk(ctx->ctx, srcdst, srcdst);
243}
244
245static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
246{
247	const unsigned int bsize = TF_BLOCK_SIZE;
248	struct crypt_priv *ctx = priv;
249	int i;
250
251	ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
252
253	if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
254		twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
255		return;
256	}
257
258	for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
259		twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
260
261	nbytes %= bsize * 3;
262
263	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
264		twofish_dec_blk(ctx->ctx, srcdst, srcdst);
265}
266
267static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
268		       struct scatterlist *src, unsigned int nbytes)
269{
270	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
271	be128 buf[TWOFISH_PARALLEL_BLOCKS];
272	struct crypt_priv crypt_ctx = {
273		.ctx = &ctx->twofish_ctx,
274		.fpu_enabled = false,
275	};
276	struct lrw_crypt_req req = {
277		.tbuf = buf,
278		.tbuflen = sizeof(buf),
279
280		.table_ctx = &ctx->lrw_table,
281		.crypt_ctx = &crypt_ctx,
282		.crypt_fn = encrypt_callback,
283	};
284	int ret;
285
286	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
287	ret = lrw_crypt(desc, dst, src, nbytes, &req);
288	twofish_fpu_end(crypt_ctx.fpu_enabled);
289
290	return ret;
291}
292
293static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
294		       struct scatterlist *src, unsigned int nbytes)
295{
296	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
297	be128 buf[TWOFISH_PARALLEL_BLOCKS];
298	struct crypt_priv crypt_ctx = {
299		.ctx = &ctx->twofish_ctx,
300		.fpu_enabled = false,
301	};
302	struct lrw_crypt_req req = {
303		.tbuf = buf,
304		.tbuflen = sizeof(buf),
305
306		.table_ctx = &ctx->lrw_table,
307		.crypt_ctx = &crypt_ctx,
308		.crypt_fn = decrypt_callback,
309	};
310	int ret;
311
312	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
313	ret = lrw_crypt(desc, dst, src, nbytes, &req);
314	twofish_fpu_end(crypt_ctx.fpu_enabled);
315
316	return ret;
317}
318
319static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
320		       struct scatterlist *src, unsigned int nbytes)
321{
322	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
323
324	return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
325				     XTS_TWEAK_CAST(twofish_enc_blk),
326				     &ctx->tweak_ctx, &ctx->crypt_ctx);
327}
328
329static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
330		       struct scatterlist *src, unsigned int nbytes)
331{
332	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
333
334	return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
335				     XTS_TWEAK_CAST(twofish_enc_blk),
336				     &ctx->tweak_ctx, &ctx->crypt_ctx);
337}
338
339static struct crypto_alg twofish_algs[10] = { {
340	.cra_name		= "__ecb-twofish-avx",
341	.cra_driver_name	= "__driver-ecb-twofish-avx",
342	.cra_priority		= 0,
343	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
344				  CRYPTO_ALG_INTERNAL,
345	.cra_blocksize		= TF_BLOCK_SIZE,
346	.cra_ctxsize		= sizeof(struct twofish_ctx),
347	.cra_alignmask		= 0,
348	.cra_type		= &crypto_blkcipher_type,
349	.cra_module		= THIS_MODULE,
350	.cra_u = {
351		.blkcipher = {
352			.min_keysize	= TF_MIN_KEY_SIZE,
353			.max_keysize	= TF_MAX_KEY_SIZE,
354			.setkey		= twofish_setkey,
355			.encrypt	= ecb_encrypt,
356			.decrypt	= ecb_decrypt,
357		},
358	},
359}, {
360	.cra_name		= "__cbc-twofish-avx",
361	.cra_driver_name	= "__driver-cbc-twofish-avx",
362	.cra_priority		= 0,
363	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
364				  CRYPTO_ALG_INTERNAL,
365	.cra_blocksize		= TF_BLOCK_SIZE,
366	.cra_ctxsize		= sizeof(struct twofish_ctx),
367	.cra_alignmask		= 0,
368	.cra_type		= &crypto_blkcipher_type,
369	.cra_module		= THIS_MODULE,
370	.cra_u = {
371		.blkcipher = {
372			.min_keysize	= TF_MIN_KEY_SIZE,
373			.max_keysize	= TF_MAX_KEY_SIZE,
374			.setkey		= twofish_setkey,
375			.encrypt	= cbc_encrypt,
376			.decrypt	= cbc_decrypt,
377		},
378	},
379}, {
380	.cra_name		= "__ctr-twofish-avx",
381	.cra_driver_name	= "__driver-ctr-twofish-avx",
382	.cra_priority		= 0,
383	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
384				  CRYPTO_ALG_INTERNAL,
385	.cra_blocksize		= 1,
386	.cra_ctxsize		= sizeof(struct twofish_ctx),
387	.cra_alignmask		= 0,
388	.cra_type		= &crypto_blkcipher_type,
389	.cra_module		= THIS_MODULE,
390	.cra_u = {
391		.blkcipher = {
392			.min_keysize	= TF_MIN_KEY_SIZE,
393			.max_keysize	= TF_MAX_KEY_SIZE,
394			.ivsize		= TF_BLOCK_SIZE,
395			.setkey		= twofish_setkey,
396			.encrypt	= ctr_crypt,
397			.decrypt	= ctr_crypt,
398		},
399	},
400}, {
401	.cra_name		= "__lrw-twofish-avx",
402	.cra_driver_name	= "__driver-lrw-twofish-avx",
403	.cra_priority		= 0,
404	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
405				  CRYPTO_ALG_INTERNAL,
406	.cra_blocksize		= TF_BLOCK_SIZE,
407	.cra_ctxsize		= sizeof(struct twofish_lrw_ctx),
408	.cra_alignmask		= 0,
409	.cra_type		= &crypto_blkcipher_type,
410	.cra_module		= THIS_MODULE,
411	.cra_exit		= lrw_twofish_exit_tfm,
412	.cra_u = {
413		.blkcipher = {
414			.min_keysize	= TF_MIN_KEY_SIZE +
415					  TF_BLOCK_SIZE,
416			.max_keysize	= TF_MAX_KEY_SIZE +
417					  TF_BLOCK_SIZE,
418			.ivsize		= TF_BLOCK_SIZE,
419			.setkey		= lrw_twofish_setkey,
420			.encrypt	= lrw_encrypt,
421			.decrypt	= lrw_decrypt,
422		},
423	},
424}, {
425	.cra_name		= "__xts-twofish-avx",
426	.cra_driver_name	= "__driver-xts-twofish-avx",
427	.cra_priority		= 0,
428	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
429				  CRYPTO_ALG_INTERNAL,
430	.cra_blocksize		= TF_BLOCK_SIZE,
431	.cra_ctxsize		= sizeof(struct twofish_xts_ctx),
432	.cra_alignmask		= 0,
433	.cra_type		= &crypto_blkcipher_type,
434	.cra_module		= THIS_MODULE,
435	.cra_u = {
436		.blkcipher = {
437			.min_keysize	= TF_MIN_KEY_SIZE * 2,
438			.max_keysize	= TF_MAX_KEY_SIZE * 2,
439			.ivsize		= TF_BLOCK_SIZE,
440			.setkey		= xts_twofish_setkey,
441			.encrypt	= xts_encrypt,
442			.decrypt	= xts_decrypt,
443		},
444	},
445}, {
446	.cra_name		= "ecb(twofish)",
447	.cra_driver_name	= "ecb-twofish-avx",
448	.cra_priority		= 400,
449	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
450	.cra_blocksize		= TF_BLOCK_SIZE,
451	.cra_ctxsize		= sizeof(struct async_helper_ctx),
452	.cra_alignmask		= 0,
453	.cra_type		= &crypto_ablkcipher_type,
454	.cra_module		= THIS_MODULE,
455	.cra_init		= ablk_init,
456	.cra_exit		= ablk_exit,
457	.cra_u = {
458		.ablkcipher = {
459			.min_keysize	= TF_MIN_KEY_SIZE,
460			.max_keysize	= TF_MAX_KEY_SIZE,
461			.setkey		= ablk_set_key,
462			.encrypt	= ablk_encrypt,
463			.decrypt	= ablk_decrypt,
464		},
465	},
466}, {
467	.cra_name		= "cbc(twofish)",
468	.cra_driver_name	= "cbc-twofish-avx",
469	.cra_priority		= 400,
470	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
471	.cra_blocksize		= TF_BLOCK_SIZE,
472	.cra_ctxsize		= sizeof(struct async_helper_ctx),
473	.cra_alignmask		= 0,
474	.cra_type		= &crypto_ablkcipher_type,
475	.cra_module		= THIS_MODULE,
476	.cra_init		= ablk_init,
477	.cra_exit		= ablk_exit,
478	.cra_u = {
479		.ablkcipher = {
480			.min_keysize	= TF_MIN_KEY_SIZE,
481			.max_keysize	= TF_MAX_KEY_SIZE,
482			.ivsize		= TF_BLOCK_SIZE,
483			.setkey		= ablk_set_key,
484			.encrypt	= __ablk_encrypt,
485			.decrypt	= ablk_decrypt,
486		},
487	},
488}, {
489	.cra_name		= "ctr(twofish)",
490	.cra_driver_name	= "ctr-twofish-avx",
491	.cra_priority		= 400,
492	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
493	.cra_blocksize		= 1,
494	.cra_ctxsize		= sizeof(struct async_helper_ctx),
495	.cra_alignmask		= 0,
496	.cra_type		= &crypto_ablkcipher_type,
497	.cra_module		= THIS_MODULE,
498	.cra_init		= ablk_init,
499	.cra_exit		= ablk_exit,
500	.cra_u = {
501		.ablkcipher = {
502			.min_keysize	= TF_MIN_KEY_SIZE,
503			.max_keysize	= TF_MAX_KEY_SIZE,
504			.ivsize		= TF_BLOCK_SIZE,
505			.setkey		= ablk_set_key,
506			.encrypt	= ablk_encrypt,
507			.decrypt	= ablk_encrypt,
508			.geniv		= "chainiv",
509		},
510	},
511}, {
512	.cra_name		= "lrw(twofish)",
513	.cra_driver_name	= "lrw-twofish-avx",
514	.cra_priority		= 400,
515	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
516	.cra_blocksize		= TF_BLOCK_SIZE,
517	.cra_ctxsize		= sizeof(struct async_helper_ctx),
518	.cra_alignmask		= 0,
519	.cra_type		= &crypto_ablkcipher_type,
520	.cra_module		= THIS_MODULE,
521	.cra_init		= ablk_init,
522	.cra_exit		= ablk_exit,
523	.cra_u = {
524		.ablkcipher = {
525			.min_keysize	= TF_MIN_KEY_SIZE +
526					  TF_BLOCK_SIZE,
527			.max_keysize	= TF_MAX_KEY_SIZE +
528					  TF_BLOCK_SIZE,
529			.ivsize		= TF_BLOCK_SIZE,
530			.setkey		= ablk_set_key,
531			.encrypt	= ablk_encrypt,
532			.decrypt	= ablk_decrypt,
533		},
534	},
535}, {
536	.cra_name		= "xts(twofish)",
537	.cra_driver_name	= "xts-twofish-avx",
538	.cra_priority		= 400,
539	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
540	.cra_blocksize		= TF_BLOCK_SIZE,
541	.cra_ctxsize		= sizeof(struct async_helper_ctx),
542	.cra_alignmask		= 0,
543	.cra_type		= &crypto_ablkcipher_type,
544	.cra_module		= THIS_MODULE,
545	.cra_init		= ablk_init,
546	.cra_exit		= ablk_exit,
547	.cra_u = {
548		.ablkcipher = {
549			.min_keysize	= TF_MIN_KEY_SIZE * 2,
550			.max_keysize	= TF_MAX_KEY_SIZE * 2,
551			.ivsize		= TF_BLOCK_SIZE,
552			.setkey		= ablk_set_key,
553			.encrypt	= ablk_encrypt,
554			.decrypt	= ablk_decrypt,
555		},
556	},
557} };
558
559static int __init twofish_init(void)
560{
561	u64 xcr0;
562
563	if (!cpu_has_avx || !cpu_has_osxsave) {
564		printk(KERN_INFO "AVX instructions are not detected.\n");
565		return -ENODEV;
566	}
567
568	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
569	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
570		printk(KERN_INFO "AVX detected but unusable.\n");
571		return -ENODEV;
572	}
573
574	return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
575}
576
577static void __exit twofish_exit(void)
578{
579	crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
580}
581
582module_init(twofish_init);
583module_exit(twofish_exit);
584
585MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
586MODULE_LICENSE("GPL");
587MODULE_ALIAS_CRYPTO("twofish");
588