This source file includes following definitions.
- nitrox_aes_gcm_setkey
- nitrox_aead_setauthsize
- alloc_src_sglist
- alloc_dst_sglist
- free_src_sglist
- free_dst_sglist
- nitrox_set_creq
- nitrox_aead_callback
- nitrox_aes_gcm_enc
- nitrox_aes_gcm_dec
- nitrox_aead_init
- nitrox_gcm_common_init
- nitrox_aes_gcm_init
- nitrox_aead_exit
- nitrox_rfc4106_setkey
- nitrox_rfc4106_setauthsize
- nitrox_rfc4106_set_aead_rctx_sglist
- nitrox_rfc4106_callback
- nitrox_rfc4106_enc
- nitrox_rfc4106_dec
- nitrox_rfc4106_init
- nitrox_register_aeads
- nitrox_unregister_aeads
1
2 #include <linux/kernel.h>
3 #include <linux/printk.h>
4 #include <linux/crypto.h>
5 #include <linux/rtnetlink.h>
6
7 #include <crypto/aead.h>
8 #include <crypto/authenc.h>
9 #include <crypto/des.h>
10 #include <crypto/sha.h>
11 #include <crypto/internal/aead.h>
12 #include <crypto/scatterwalk.h>
13 #include <crypto/gcm.h>
14
15 #include "nitrox_dev.h"
16 #include "nitrox_common.h"
17 #include "nitrox_req.h"
18
19 #define GCM_AES_SALT_SIZE 4
20
21 union gph_p3 {
22 struct {
23 #ifdef __BIG_ENDIAN_BITFIELD
24 u16 iv_offset : 8;
25 u16 auth_offset : 8;
26 #else
27 u16 auth_offset : 8;
28 u16 iv_offset : 8;
29 #endif
30 };
31 u16 param;
32 };
33
34 static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
35 unsigned int keylen)
36 {
37 int aes_keylen;
38 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
39 struct flexi_crypto_context *fctx;
40 union fc_ctx_flags flags;
41
42 aes_keylen = flexi_aes_keylen(keylen);
43 if (aes_keylen < 0) {
44 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
45 return -EINVAL;
46 }
47
48
49 fctx = nctx->u.fctx;
50 flags.f = be64_to_cpu(fctx->flags.f);
51 flags.w0.aes_keylen = aes_keylen;
52 fctx->flags.f = cpu_to_be64(flags.f);
53
54
55 memset(&fctx->crypto, 0, sizeof(fctx->crypto));
56 memcpy(fctx->crypto.u.key, key, keylen);
57
58 return 0;
59 }
60
61 static int nitrox_aead_setauthsize(struct crypto_aead *aead,
62 unsigned int authsize)
63 {
64 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
65 struct flexi_crypto_context *fctx = nctx->u.fctx;
66 union fc_ctx_flags flags;
67
68 flags.f = be64_to_cpu(fctx->flags.f);
69 flags.w0.mac_len = authsize;
70 fctx->flags.f = cpu_to_be64(flags.f);
71
72 aead->authsize = authsize;
73
74 return 0;
75 }
76
77 static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
78 struct scatterlist *src, char *iv, int ivsize,
79 int buflen)
80 {
81 int nents = sg_nents_for_len(src, buflen);
82 int ret;
83
84 if (nents < 0)
85 return nents;
86
87
88 nents += 1;
89
90 ret = alloc_src_req_buf(nkreq, nents, ivsize);
91 if (ret)
92 return ret;
93
94 nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
95 nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);
96
97 return 0;
98 }
99
100 static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
101 struct scatterlist *dst, int ivsize, int buflen)
102 {
103 int nents = sg_nents_for_len(dst, buflen);
104 int ret;
105
106 if (nents < 0)
107 return nents;
108
109
110 nents += 3;
111
112
113
114 ret = alloc_dst_req_buf(nkreq, nents);
115 if (ret)
116 return ret;
117
118 nitrox_creq_set_orh(nkreq);
119 nitrox_creq_set_comp(nkreq);
120 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);
121
122 return 0;
123 }
124
125 static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
126 {
127 kfree(nkreq->src);
128 }
129
130 static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
131 {
132 kfree(nkreq->dst);
133 }
134
135 static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
136 {
137 struct se_crypto_request *creq = &rctx->nkreq.creq;
138 union gph_p3 param3;
139 int ret;
140
141 creq->flags = rctx->flags;
142 creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
143 GFP_ATOMIC;
144
145 creq->ctrl.value = 0;
146 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
147 creq->ctrl.s.arg = rctx->ctrl_arg;
148
149 creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
150 creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
151 creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
152 param3.iv_offset = 0;
153 param3.auth_offset = rctx->ivsize;
154 creq->gph.param3 = cpu_to_be16(param3.param);
155
156 creq->ctx_handle = rctx->ctx_handle;
157 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
158
159 ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
160 rctx->srclen);
161 if (ret)
162 return ret;
163
164 ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
165 rctx->dstlen);
166 if (ret) {
167 free_src_sglist(&rctx->nkreq);
168 return ret;
169 }
170
171 return 0;
172 }
173
174 static void nitrox_aead_callback(void *arg, int err)
175 {
176 struct aead_request *areq = arg;
177 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
178
179 free_src_sglist(&rctx->nkreq);
180 free_dst_sglist(&rctx->nkreq);
181 if (err) {
182 pr_err_ratelimited("request failed status 0x%0x\n", err);
183 err = -EINVAL;
184 }
185
186 areq->base.complete(&areq->base, err);
187 }
188
189 static int nitrox_aes_gcm_enc(struct aead_request *areq)
190 {
191 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
192 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
193 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
194 struct se_crypto_request *creq = &rctx->nkreq.creq;
195 struct flexi_crypto_context *fctx = nctx->u.fctx;
196 int ret;
197
198 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
199
200 rctx->cryptlen = areq->cryptlen;
201 rctx->assoclen = areq->assoclen;
202 rctx->srclen = areq->assoclen + areq->cryptlen;
203 rctx->dstlen = rctx->srclen + aead->authsize;
204 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
205 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
206 rctx->flags = areq->base.flags;
207 rctx->ctx_handle = nctx->u.ctx_handle;
208 rctx->src = areq->src;
209 rctx->dst = areq->dst;
210 rctx->ctrl_arg = ENCRYPT;
211 ret = nitrox_set_creq(rctx);
212 if (ret)
213 return ret;
214
215
216 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
217 areq);
218 }
219
220 static int nitrox_aes_gcm_dec(struct aead_request *areq)
221 {
222 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
223 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
224 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
225 struct se_crypto_request *creq = &rctx->nkreq.creq;
226 struct flexi_crypto_context *fctx = nctx->u.fctx;
227 int ret;
228
229 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
230
231 rctx->cryptlen = areq->cryptlen - aead->authsize;
232 rctx->assoclen = areq->assoclen;
233 rctx->srclen = areq->cryptlen + areq->assoclen;
234 rctx->dstlen = rctx->srclen - aead->authsize;
235 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
236 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
237 rctx->flags = areq->base.flags;
238 rctx->ctx_handle = nctx->u.ctx_handle;
239 rctx->src = areq->src;
240 rctx->dst = areq->dst;
241 rctx->ctrl_arg = DECRYPT;
242 ret = nitrox_set_creq(rctx);
243 if (ret)
244 return ret;
245
246
247 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
248 areq);
249 }
250
251 static int nitrox_aead_init(struct crypto_aead *aead)
252 {
253 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
254 struct crypto_ctx_hdr *chdr;
255
256
257 nctx->ndev = nitrox_get_first_device();
258 if (!nctx->ndev)
259 return -ENODEV;
260
261
262 chdr = crypto_alloc_context(nctx->ndev);
263 if (!chdr) {
264 nitrox_put_device(nctx->ndev);
265 return -ENOMEM;
266 }
267 nctx->chdr = chdr;
268 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
269 sizeof(struct ctx_hdr));
270 nctx->u.fctx->flags.f = 0;
271
272 return 0;
273 }
274
275 static int nitrox_gcm_common_init(struct crypto_aead *aead)
276 {
277 int ret;
278 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
279 union fc_ctx_flags *flags;
280
281 ret = nitrox_aead_init(aead);
282 if (ret)
283 return ret;
284
285 flags = &nctx->u.fctx->flags;
286 flags->w0.cipher_type = CIPHER_AES_GCM;
287 flags->w0.hash_type = AUTH_NULL;
288 flags->w0.iv_source = IV_FROM_DPTR;
289
290 flags->w0.auth_input_type = 1;
291 flags->f = be64_to_cpu(flags->f);
292
293 return 0;
294 }
295
296 static int nitrox_aes_gcm_init(struct crypto_aead *aead)
297 {
298 int ret;
299
300 ret = nitrox_gcm_common_init(aead);
301 if (ret)
302 return ret;
303
304 crypto_aead_set_reqsize(aead,
305 sizeof(struct aead_request) +
306 sizeof(struct nitrox_aead_rctx));
307
308 return 0;
309 }
310
311 static void nitrox_aead_exit(struct crypto_aead *aead)
312 {
313 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
314
315
316 if (nctx->u.ctx_handle) {
317 struct flexi_crypto_context *fctx = nctx->u.fctx;
318
319 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
320 memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
321 crypto_free_context((void *)nctx->chdr);
322 }
323 nitrox_put_device(nctx->ndev);
324
325 nctx->u.ctx_handle = 0;
326 nctx->ndev = NULL;
327 }
328
329 static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key,
330 unsigned int keylen)
331 {
332 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
333 struct flexi_crypto_context *fctx = nctx->u.fctx;
334 int ret;
335
336 if (keylen < GCM_AES_SALT_SIZE)
337 return -EINVAL;
338
339 keylen -= GCM_AES_SALT_SIZE;
340 ret = nitrox_aes_gcm_setkey(aead, key, keylen);
341 if (ret)
342 return ret;
343
344 memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
345 return 0;
346 }
347
348 static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
349 unsigned int authsize)
350 {
351 switch (authsize) {
352 case 8:
353 case 12:
354 case 16:
355 break;
356 default:
357 return -EINVAL;
358 }
359
360 return nitrox_aead_setauthsize(aead, authsize);
361 }
362
363 static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
364 {
365 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
366 struct nitrox_aead_rctx *aead_rctx = &rctx->base;
367 unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
368 struct scatterlist *sg;
369
370 if (areq->assoclen != 16 && areq->assoclen != 20)
371 return -EINVAL;
372
373 scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
374 sg_init_table(rctx->src, 3);
375 sg_set_buf(rctx->src, rctx->assoc, assoclen);
376 sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
377 if (sg != rctx->src + 1)
378 sg_chain(rctx->src, 2, sg);
379
380 if (areq->src != areq->dst) {
381 sg_init_table(rctx->dst, 3);
382 sg_set_buf(rctx->dst, rctx->assoc, assoclen);
383 sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
384 if (sg != rctx->dst + 1)
385 sg_chain(rctx->dst, 2, sg);
386 }
387
388 aead_rctx->src = rctx->src;
389 aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;
390
391 return 0;
392 }
393
394 static void nitrox_rfc4106_callback(void *arg, int err)
395 {
396 struct aead_request *areq = arg;
397 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
398 struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;
399
400 free_src_sglist(nkreq);
401 free_dst_sglist(nkreq);
402 if (err) {
403 pr_err_ratelimited("request failed status 0x%0x\n", err);
404 err = -EINVAL;
405 }
406
407 areq->base.complete(&areq->base, err);
408 }
409
410 static int nitrox_rfc4106_enc(struct aead_request *areq)
411 {
412 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
413 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
414 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
415 struct nitrox_aead_rctx *aead_rctx = &rctx->base;
416 struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
417 int ret;
418
419 aead_rctx->cryptlen = areq->cryptlen;
420 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
421 aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
422 aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
423 aead_rctx->iv = areq->iv;
424 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
425 aead_rctx->flags = areq->base.flags;
426 aead_rctx->ctx_handle = nctx->u.ctx_handle;
427 aead_rctx->ctrl_arg = ENCRYPT;
428
429 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
430 if (ret)
431 return ret;
432
433 ret = nitrox_set_creq(aead_rctx);
434 if (ret)
435 return ret;
436
437
438 return nitrox_process_se_request(nctx->ndev, creq,
439 nitrox_rfc4106_callback, areq);
440 }
441
442 static int nitrox_rfc4106_dec(struct aead_request *areq)
443 {
444 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
445 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
446 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
447 struct nitrox_aead_rctx *aead_rctx = &rctx->base;
448 struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
449 int ret;
450
451 aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
452 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
453 aead_rctx->srclen =
454 areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
455 aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
456 aead_rctx->iv = areq->iv;
457 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
458 aead_rctx->flags = areq->base.flags;
459 aead_rctx->ctx_handle = nctx->u.ctx_handle;
460 aead_rctx->ctrl_arg = DECRYPT;
461
462 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
463 if (ret)
464 return ret;
465
466 ret = nitrox_set_creq(aead_rctx);
467 if (ret)
468 return ret;
469
470
471 return nitrox_process_se_request(nctx->ndev, creq,
472 nitrox_rfc4106_callback, areq);
473 }
474
475 static int nitrox_rfc4106_init(struct crypto_aead *aead)
476 {
477 int ret;
478
479 ret = nitrox_gcm_common_init(aead);
480 if (ret)
481 return ret;
482
483 crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +
484 sizeof(struct nitrox_rfc4106_rctx));
485
486 return 0;
487 }
488
489 static struct aead_alg nitrox_aeads[] = { {
490 .base = {
491 .cra_name = "gcm(aes)",
492 .cra_driver_name = "n5_aes_gcm",
493 .cra_priority = PRIO,
494 .cra_flags = CRYPTO_ALG_ASYNC,
495 .cra_blocksize = AES_BLOCK_SIZE,
496 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
497 .cra_alignmask = 0,
498 .cra_module = THIS_MODULE,
499 },
500 .setkey = nitrox_aes_gcm_setkey,
501 .setauthsize = nitrox_aead_setauthsize,
502 .encrypt = nitrox_aes_gcm_enc,
503 .decrypt = nitrox_aes_gcm_dec,
504 .init = nitrox_aes_gcm_init,
505 .exit = nitrox_aead_exit,
506 .ivsize = GCM_AES_IV_SIZE,
507 .maxauthsize = AES_BLOCK_SIZE,
508 }, {
509 .base = {
510 .cra_name = "rfc4106(gcm(aes))",
511 .cra_driver_name = "n5_rfc4106",
512 .cra_priority = PRIO,
513 .cra_flags = CRYPTO_ALG_ASYNC,
514 .cra_blocksize = AES_BLOCK_SIZE,
515 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
516 .cra_alignmask = 0,
517 .cra_module = THIS_MODULE,
518 },
519 .setkey = nitrox_rfc4106_setkey,
520 .setauthsize = nitrox_rfc4106_setauthsize,
521 .encrypt = nitrox_rfc4106_enc,
522 .decrypt = nitrox_rfc4106_dec,
523 .init = nitrox_rfc4106_init,
524 .exit = nitrox_aead_exit,
525 .ivsize = GCM_RFC4106_IV_SIZE,
526 .maxauthsize = AES_BLOCK_SIZE,
527 } };
528
529 int nitrox_register_aeads(void)
530 {
531 return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
532 }
533
534 void nitrox_unregister_aeads(void)
535 {
536 crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
537 }