This source file includes following definitions.
- sec_alg_skcipher_init_template
- sec_alg_skcipher_init_context
- sec_free_hw_sgl
- sec_alloc_and_fill_hw_sgl
- sec_alg_skcipher_setkey
- sec_alg_skcipher_setkey_aes_ecb
- sec_alg_skcipher_setkey_aes_cbc
- sec_alg_skcipher_setkey_aes_ctr
- sec_alg_skcipher_setkey_aes_xts
- sec_alg_skcipher_setkey_des_ecb
- sec_alg_skcipher_setkey_des_cbc
- sec_alg_skcipher_setkey_3des_ecb
- sec_alg_skcipher_setkey_3des_cbc
- sec_alg_free_el
- sec_send_request
- sec_skcipher_alg_callback
- sec_alg_callback
- sec_alg_alloc_and_calc_split_sizes
- sec_map_and_split_sg
- sec_unmap_sg_on_err
- sec_alg_alloc_and_fill_el
- sec_alg_skcipher_crypto
- sec_alg_skcipher_encrypt
- sec_alg_skcipher_decrypt
- sec_alg_skcipher_init
- sec_alg_skcipher_exit
- sec_alg_skcipher_init_with_queue
- sec_alg_skcipher_exit_with_queue
- sec_algs_register
- sec_algs_unregister
1
2
3 #include <linux/crypto.h>
4 #include <linux/dma-mapping.h>
5 #include <linux/dmapool.h>
6 #include <linux/module.h>
7 #include <linux/mutex.h>
8 #include <linux/slab.h>
9
10 #include <crypto/aes.h>
11 #include <crypto/algapi.h>
12 #include <crypto/internal/des.h>
13 #include <crypto/skcipher.h>
14 #include <crypto/xts.h>
15 #include <crypto/internal/skcipher.h>
16
17 #include "sec_drv.h"
18
19 #define SEC_MAX_CIPHER_KEY 64
20 #define SEC_REQ_LIMIT SZ_32M
21
22 struct sec_c_alg_cfg {
23 unsigned c_alg : 3;
24 unsigned c_mode : 3;
25 unsigned key_len : 2;
26 unsigned c_width : 2;
27 };
28
29 static const struct sec_c_alg_cfg sec_c_alg_cfgs[] = {
30 [SEC_C_DES_ECB_64] = {
31 .c_alg = SEC_C_ALG_DES,
32 .c_mode = SEC_C_MODE_ECB,
33 .key_len = SEC_KEY_LEN_DES,
34 },
35 [SEC_C_DES_CBC_64] = {
36 .c_alg = SEC_C_ALG_DES,
37 .c_mode = SEC_C_MODE_CBC,
38 .key_len = SEC_KEY_LEN_DES,
39 },
40 [SEC_C_3DES_ECB_192_3KEY] = {
41 .c_alg = SEC_C_ALG_3DES,
42 .c_mode = SEC_C_MODE_ECB,
43 .key_len = SEC_KEY_LEN_3DES_3_KEY,
44 },
45 [SEC_C_3DES_ECB_192_2KEY] = {
46 .c_alg = SEC_C_ALG_3DES,
47 .c_mode = SEC_C_MODE_ECB,
48 .key_len = SEC_KEY_LEN_3DES_2_KEY,
49 },
50 [SEC_C_3DES_CBC_192_3KEY] = {
51 .c_alg = SEC_C_ALG_3DES,
52 .c_mode = SEC_C_MODE_CBC,
53 .key_len = SEC_KEY_LEN_3DES_3_KEY,
54 },
55 [SEC_C_3DES_CBC_192_2KEY] = {
56 .c_alg = SEC_C_ALG_3DES,
57 .c_mode = SEC_C_MODE_CBC,
58 .key_len = SEC_KEY_LEN_3DES_2_KEY,
59 },
60 [SEC_C_AES_ECB_128] = {
61 .c_alg = SEC_C_ALG_AES,
62 .c_mode = SEC_C_MODE_ECB,
63 .key_len = SEC_KEY_LEN_AES_128,
64 },
65 [SEC_C_AES_ECB_192] = {
66 .c_alg = SEC_C_ALG_AES,
67 .c_mode = SEC_C_MODE_ECB,
68 .key_len = SEC_KEY_LEN_AES_192,
69 },
70 [SEC_C_AES_ECB_256] = {
71 .c_alg = SEC_C_ALG_AES,
72 .c_mode = SEC_C_MODE_ECB,
73 .key_len = SEC_KEY_LEN_AES_256,
74 },
75 [SEC_C_AES_CBC_128] = {
76 .c_alg = SEC_C_ALG_AES,
77 .c_mode = SEC_C_MODE_CBC,
78 .key_len = SEC_KEY_LEN_AES_128,
79 },
80 [SEC_C_AES_CBC_192] = {
81 .c_alg = SEC_C_ALG_AES,
82 .c_mode = SEC_C_MODE_CBC,
83 .key_len = SEC_KEY_LEN_AES_192,
84 },
85 [SEC_C_AES_CBC_256] = {
86 .c_alg = SEC_C_ALG_AES,
87 .c_mode = SEC_C_MODE_CBC,
88 .key_len = SEC_KEY_LEN_AES_256,
89 },
90 [SEC_C_AES_CTR_128] = {
91 .c_alg = SEC_C_ALG_AES,
92 .c_mode = SEC_C_MODE_CTR,
93 .key_len = SEC_KEY_LEN_AES_128,
94 },
95 [SEC_C_AES_CTR_192] = {
96 .c_alg = SEC_C_ALG_AES,
97 .c_mode = SEC_C_MODE_CTR,
98 .key_len = SEC_KEY_LEN_AES_192,
99 },
100 [SEC_C_AES_CTR_256] = {
101 .c_alg = SEC_C_ALG_AES,
102 .c_mode = SEC_C_MODE_CTR,
103 .key_len = SEC_KEY_LEN_AES_256,
104 },
105 [SEC_C_AES_XTS_128] = {
106 .c_alg = SEC_C_ALG_AES,
107 .c_mode = SEC_C_MODE_XTS,
108 .key_len = SEC_KEY_LEN_AES_128,
109 },
110 [SEC_C_AES_XTS_256] = {
111 .c_alg = SEC_C_ALG_AES,
112 .c_mode = SEC_C_MODE_XTS,
113 .key_len = SEC_KEY_LEN_AES_256,
114 },
115 [SEC_C_NULL] = {
116 },
117 };
118
119
120
121
122
123 static DEFINE_MUTEX(algs_lock);
124 static unsigned int active_devs;
125
126 static void sec_alg_skcipher_init_template(struct sec_alg_tfm_ctx *ctx,
127 struct sec_bd_info *req,
128 enum sec_cipher_alg alg)
129 {
130 const struct sec_c_alg_cfg *cfg = &sec_c_alg_cfgs[alg];
131
132 memset(req, 0, sizeof(*req));
133 req->w0 |= cfg->c_mode << SEC_BD_W0_C_MODE_S;
134 req->w1 |= cfg->c_alg << SEC_BD_W1_C_ALG_S;
135 req->w3 |= cfg->key_len << SEC_BD_W3_C_KEY_LEN_S;
136 req->w0 |= cfg->c_width << SEC_BD_W0_C_WIDTH_S;
137
138 req->cipher_key_addr_lo = lower_32_bits(ctx->pkey);
139 req->cipher_key_addr_hi = upper_32_bits(ctx->pkey);
140 }
141
142 static void sec_alg_skcipher_init_context(struct crypto_skcipher *atfm,
143 const u8 *key,
144 unsigned int keylen,
145 enum sec_cipher_alg alg)
146 {
147 struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
148 struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
149
150 ctx->cipher_alg = alg;
151 memcpy(ctx->key, key, keylen);
152 sec_alg_skcipher_init_template(ctx, &ctx->req_template,
153 ctx->cipher_alg);
154 }
155
156 static void sec_free_hw_sgl(struct sec_hw_sgl *hw_sgl,
157 dma_addr_t psec_sgl, struct sec_dev_info *info)
158 {
159 struct sec_hw_sgl *sgl_current, *sgl_next;
160 dma_addr_t sgl_next_dma;
161
162 sgl_current = hw_sgl;
163 while (sgl_current) {
164 sgl_next = sgl_current->next;
165 sgl_next_dma = sgl_current->next_sgl;
166
167 dma_pool_free(info->hw_sgl_pool, sgl_current, psec_sgl);
168
169 sgl_current = sgl_next;
170 psec_sgl = sgl_next_dma;
171 }
172 }
173
174 static int sec_alloc_and_fill_hw_sgl(struct sec_hw_sgl **sec_sgl,
175 dma_addr_t *psec_sgl,
176 struct scatterlist *sgl,
177 int count,
178 struct sec_dev_info *info)
179 {
180 struct sec_hw_sgl *sgl_current = NULL;
181 struct sec_hw_sgl *sgl_next;
182 dma_addr_t sgl_next_dma;
183 struct scatterlist *sg;
184 int ret, sge_index, i;
185
186 if (!count)
187 return -EINVAL;
188
189 for_each_sg(sgl, sg, count, i) {
190 sge_index = i % SEC_MAX_SGE_NUM;
191 if (sge_index == 0) {
192 sgl_next = dma_pool_zalloc(info->hw_sgl_pool,
193 GFP_KERNEL, &sgl_next_dma);
194 if (!sgl_next) {
195 ret = -ENOMEM;
196 goto err_free_hw_sgls;
197 }
198
199 if (!sgl_current) {
200 *psec_sgl = sgl_next_dma;
201 *sec_sgl = sgl_next;
202 } else {
203 sgl_current->entry_sum_in_sgl = SEC_MAX_SGE_NUM;
204 sgl_current->next_sgl = sgl_next_dma;
205 sgl_current->next = sgl_next;
206 }
207 sgl_current = sgl_next;
208 }
209 sgl_current->sge_entries[sge_index].buf = sg_dma_address(sg);
210 sgl_current->sge_entries[sge_index].len = sg_dma_len(sg);
211 sgl_current->data_bytes_in_sgl += sg_dma_len(sg);
212 }
213 sgl_current->entry_sum_in_sgl = count % SEC_MAX_SGE_NUM;
214 sgl_current->next_sgl = 0;
215 (*sec_sgl)->entry_sum_in_chain = count;
216
217 return 0;
218
219 err_free_hw_sgls:
220 sec_free_hw_sgl(*sec_sgl, *psec_sgl, info);
221 *psec_sgl = 0;
222
223 return ret;
224 }
225
226 static int sec_alg_skcipher_setkey(struct crypto_skcipher *tfm,
227 const u8 *key, unsigned int keylen,
228 enum sec_cipher_alg alg)
229 {
230 struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
231 struct device *dev = ctx->queue->dev_info->dev;
232
233 mutex_lock(&ctx->lock);
234 if (ctx->key) {
235
236 memset(ctx->key, 0, SEC_MAX_CIPHER_KEY);
237 } else {
238
239 ctx->key = dma_alloc_coherent(dev, SEC_MAX_CIPHER_KEY,
240 &ctx->pkey, GFP_KERNEL);
241 if (!ctx->key) {
242 mutex_unlock(&ctx->lock);
243 return -ENOMEM;
244 }
245 }
246 mutex_unlock(&ctx->lock);
247 sec_alg_skcipher_init_context(tfm, key, keylen, alg);
248
249 return 0;
250 }
251
252 static int sec_alg_skcipher_setkey_aes_ecb(struct crypto_skcipher *tfm,
253 const u8 *key, unsigned int keylen)
254 {
255 enum sec_cipher_alg alg;
256
257 switch (keylen) {
258 case AES_KEYSIZE_128:
259 alg = SEC_C_AES_ECB_128;
260 break;
261 case AES_KEYSIZE_192:
262 alg = SEC_C_AES_ECB_192;
263 break;
264 case AES_KEYSIZE_256:
265 alg = SEC_C_AES_ECB_256;
266 break;
267 default:
268 return -EINVAL;
269 }
270
271 return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
272 }
273
274 static int sec_alg_skcipher_setkey_aes_cbc(struct crypto_skcipher *tfm,
275 const u8 *key, unsigned int keylen)
276 {
277 enum sec_cipher_alg alg;
278
279 switch (keylen) {
280 case AES_KEYSIZE_128:
281 alg = SEC_C_AES_CBC_128;
282 break;
283 case AES_KEYSIZE_192:
284 alg = SEC_C_AES_CBC_192;
285 break;
286 case AES_KEYSIZE_256:
287 alg = SEC_C_AES_CBC_256;
288 break;
289 default:
290 return -EINVAL;
291 }
292
293 return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
294 }
295
296 static int sec_alg_skcipher_setkey_aes_ctr(struct crypto_skcipher *tfm,
297 const u8 *key, unsigned int keylen)
298 {
299 enum sec_cipher_alg alg;
300
301 switch (keylen) {
302 case AES_KEYSIZE_128:
303 alg = SEC_C_AES_CTR_128;
304 break;
305 case AES_KEYSIZE_192:
306 alg = SEC_C_AES_CTR_192;
307 break;
308 case AES_KEYSIZE_256:
309 alg = SEC_C_AES_CTR_256;
310 break;
311 default:
312 return -EINVAL;
313 }
314
315 return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
316 }
317
318 static int sec_alg_skcipher_setkey_aes_xts(struct crypto_skcipher *tfm,
319 const u8 *key, unsigned int keylen)
320 {
321 enum sec_cipher_alg alg;
322 int ret;
323
324 ret = xts_verify_key(tfm, key, keylen);
325 if (ret)
326 return ret;
327
328 switch (keylen) {
329 case AES_KEYSIZE_128 * 2:
330 alg = SEC_C_AES_XTS_128;
331 break;
332 case AES_KEYSIZE_256 * 2:
333 alg = SEC_C_AES_XTS_256;
334 break;
335 default:
336 return -EINVAL;
337 }
338
339 return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
340 }
341
342 static int sec_alg_skcipher_setkey_des_ecb(struct crypto_skcipher *tfm,
343 const u8 *key, unsigned int keylen)
344 {
345 return verify_skcipher_des_key(tfm, key) ?:
346 sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64);
347 }
348
349 static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm,
350 const u8 *key, unsigned int keylen)
351 {
352 return verify_skcipher_des_key(tfm, key) ?:
353 sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64);
354 }
355
356 static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
357 const u8 *key, unsigned int keylen)
358 {
359 return verify_skcipher_des3_key(tfm, key) ?:
360 sec_alg_skcipher_setkey(tfm, key, keylen,
361 SEC_C_3DES_ECB_192_3KEY);
362 }
363
364 static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm,
365 const u8 *key, unsigned int keylen)
366 {
367 return verify_skcipher_des3_key(tfm, key) ?:
368 sec_alg_skcipher_setkey(tfm, key, keylen,
369 SEC_C_3DES_CBC_192_3KEY);
370 }
371
372 static void sec_alg_free_el(struct sec_request_el *el,
373 struct sec_dev_info *info)
374 {
375 sec_free_hw_sgl(el->out, el->dma_out, info);
376 sec_free_hw_sgl(el->in, el->dma_in, info);
377 kfree(el->sgl_in);
378 kfree(el->sgl_out);
379 kfree(el);
380 }
381
382
383 static int sec_send_request(struct sec_request *sec_req, struct sec_queue *queue)
384 {
385 struct sec_request_el *el, *temp;
386 int ret = 0;
387
388 mutex_lock(&sec_req->lock);
389 list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
390
391
392
393
394
395
396
397
398
399
400 if (!queue->havesoftqueue ||
401 (kfifo_is_empty(&queue->softqueue) &&
402 sec_queue_empty(queue))) {
403 ret = sec_queue_send(queue, &el->req, sec_req);
404 if (ret == -EAGAIN) {
405
406
407 ret = -EBUSY;
408 goto err_unlock;
409 }
410 } else {
411 kfifo_put(&queue->softqueue, el);
412 }
413 }
414 err_unlock:
415 mutex_unlock(&sec_req->lock);
416
417 return ret;
418 }
419
420 static void sec_skcipher_alg_callback(struct sec_bd_info *sec_resp,
421 struct crypto_async_request *req_base)
422 {
423 struct skcipher_request *skreq = container_of(req_base,
424 struct skcipher_request,
425 base);
426 struct sec_request *sec_req = skcipher_request_ctx(skreq);
427 struct sec_request *backlog_req;
428 struct sec_request_el *sec_req_el, *nextrequest;
429 struct sec_alg_tfm_ctx *ctx = sec_req->tfm_ctx;
430 struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
431 struct device *dev = ctx->queue->dev_info->dev;
432 int icv_or_skey_en, ret;
433 bool done;
434
435 sec_req_el = list_first_entry(&sec_req->elements, struct sec_request_el,
436 head);
437 icv_or_skey_en = (sec_resp->w0 & SEC_BD_W0_ICV_OR_SKEY_EN_M) >>
438 SEC_BD_W0_ICV_OR_SKEY_EN_S;
439 if (sec_resp->w1 & SEC_BD_W1_BD_INVALID || icv_or_skey_en == 3) {
440 dev_err(dev, "Got an invalid answer %lu %d\n",
441 sec_resp->w1 & SEC_BD_W1_BD_INVALID,
442 icv_or_skey_en);
443 sec_req->err = -EINVAL;
444
445
446
447
448
449 }
450
451 mutex_lock(&ctx->queue->queuelock);
452
453 switch (ctx->cipher_alg) {
454 case SEC_C_AES_CBC_128:
455 case SEC_C_AES_CBC_192:
456 case SEC_C_AES_CBC_256:
457 if (sec_req_el->req.w0 & SEC_BD_W0_DE)
458 sg_pcopy_to_buffer(sec_req_el->sgl_out,
459 sg_nents(sec_req_el->sgl_out),
460 skreq->iv,
461 crypto_skcipher_ivsize(atfm),
462 sec_req_el->el_length -
463 crypto_skcipher_ivsize(atfm));
464 else
465 sg_pcopy_to_buffer(sec_req_el->sgl_in,
466 sg_nents(sec_req_el->sgl_in),
467 skreq->iv,
468 crypto_skcipher_ivsize(atfm),
469 sec_req_el->el_length -
470 crypto_skcipher_ivsize(atfm));
471
472 break;
473 case SEC_C_AES_CTR_128:
474 case SEC_C_AES_CTR_192:
475 case SEC_C_AES_CTR_256:
476 crypto_inc(skreq->iv, 16);
477 break;
478 default:
479
480 break;
481 }
482
483 if (ctx->queue->havesoftqueue &&
484 !kfifo_is_empty(&ctx->queue->softqueue) &&
485 sec_queue_empty(ctx->queue)) {
486 ret = kfifo_get(&ctx->queue->softqueue, &nextrequest);
487 if (ret <= 0)
488 dev_err(dev,
489 "Error getting next element from kfifo %d\n",
490 ret);
491 else
492
493 sec_queue_send(ctx->queue, &nextrequest->req,
494 nextrequest->sec_req);
495 } else if (!list_empty(&ctx->backlog)) {
496
497 backlog_req = list_first_entry(&ctx->backlog,
498 typeof(*backlog_req),
499 backlog_head);
500 if (sec_queue_can_enqueue(ctx->queue,
501 backlog_req->num_elements) ||
502 (ctx->queue->havesoftqueue &&
503 kfifo_avail(&ctx->queue->softqueue) >
504 backlog_req->num_elements)) {
505 sec_send_request(backlog_req, ctx->queue);
506 backlog_req->req_base->complete(backlog_req->req_base,
507 -EINPROGRESS);
508 list_del(&backlog_req->backlog_head);
509 }
510 }
511 mutex_unlock(&ctx->queue->queuelock);
512
513 mutex_lock(&sec_req->lock);
514 list_del(&sec_req_el->head);
515 mutex_unlock(&sec_req->lock);
516 sec_alg_free_el(sec_req_el, ctx->queue->dev_info);
517
518
519
520
521
522 mutex_lock(&sec_req->lock);
523 done = list_empty(&sec_req->elements);
524 mutex_unlock(&sec_req->lock);
525 if (done) {
526 if (crypto_skcipher_ivsize(atfm)) {
527 dma_unmap_single(dev, sec_req->dma_iv,
528 crypto_skcipher_ivsize(atfm),
529 DMA_TO_DEVICE);
530 }
531 dma_unmap_sg(dev, skreq->src, sec_req->len_in,
532 DMA_BIDIRECTIONAL);
533 if (skreq->src != skreq->dst)
534 dma_unmap_sg(dev, skreq->dst, sec_req->len_out,
535 DMA_BIDIRECTIONAL);
536 skreq->base.complete(&skreq->base, sec_req->err);
537 }
538 }
539
540 void sec_alg_callback(struct sec_bd_info *resp, void *shadow)
541 {
542 struct sec_request *sec_req = shadow;
543
544 sec_req->cb(resp, sec_req->req_base);
545 }
546
547 static int sec_alg_alloc_and_calc_split_sizes(int length, size_t **split_sizes,
548 int *steps)
549 {
550 size_t *sizes;
551 int i;
552
553
554 *steps = roundup(length, SEC_REQ_LIMIT) / SEC_REQ_LIMIT;
555 sizes = kcalloc(*steps, sizeof(*sizes), GFP_KERNEL);
556 if (!sizes)
557 return -ENOMEM;
558
559 for (i = 0; i < *steps - 1; i++)
560 sizes[i] = SEC_REQ_LIMIT;
561 sizes[*steps - 1] = length - SEC_REQ_LIMIT * (*steps - 1);
562 *split_sizes = sizes;
563
564 return 0;
565 }
566
567 static int sec_map_and_split_sg(struct scatterlist *sgl, size_t *split_sizes,
568 int steps, struct scatterlist ***splits,
569 int **splits_nents,
570 int sgl_len_in,
571 struct device *dev)
572 {
573 int ret, count;
574
575 count = dma_map_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
576 if (!count)
577 return -EINVAL;
578
579 *splits = kcalloc(steps, sizeof(struct scatterlist *), GFP_KERNEL);
580 if (!*splits) {
581 ret = -ENOMEM;
582 goto err_unmap_sg;
583 }
584 *splits_nents = kcalloc(steps, sizeof(int), GFP_KERNEL);
585 if (!*splits_nents) {
586 ret = -ENOMEM;
587 goto err_free_splits;
588 }
589
590
591 ret = sg_split(sgl, count, 0, steps, split_sizes,
592 *splits, *splits_nents, GFP_KERNEL);
593 if (ret) {
594 ret = -ENOMEM;
595 goto err_free_splits_nents;
596 }
597
598 return 0;
599
600 err_free_splits_nents:
601 kfree(*splits_nents);
602 err_free_splits:
603 kfree(*splits);
604 err_unmap_sg:
605 dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
606
607 return ret;
608 }
609
610
611
612
613
614 static void sec_unmap_sg_on_err(struct scatterlist *sgl, int steps,
615 struct scatterlist **splits, int *splits_nents,
616 int sgl_len_in, struct device *dev)
617 {
618 int i;
619
620 for (i = 0; i < steps; i++)
621 kfree(splits[i]);
622 kfree(splits_nents);
623 kfree(splits);
624
625 dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
626 }
627
628 static struct sec_request_el
629 *sec_alg_alloc_and_fill_el(struct sec_bd_info *template, int encrypt,
630 int el_size, bool different_dest,
631 struct scatterlist *sgl_in, int n_ents_in,
632 struct scatterlist *sgl_out, int n_ents_out,
633 struct sec_dev_info *info)
634 {
635 struct sec_request_el *el;
636 struct sec_bd_info *req;
637 int ret;
638
639 el = kzalloc(sizeof(*el), GFP_KERNEL);
640 if (!el)
641 return ERR_PTR(-ENOMEM);
642 el->el_length = el_size;
643 req = &el->req;
644 memcpy(req, template, sizeof(*req));
645
646 req->w0 &= ~SEC_BD_W0_CIPHER_M;
647 if (encrypt)
648 req->w0 |= SEC_CIPHER_ENCRYPT << SEC_BD_W0_CIPHER_S;
649 else
650 req->w0 |= SEC_CIPHER_DECRYPT << SEC_BD_W0_CIPHER_S;
651
652 req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_19_16_M;
653 req->w0 |= ((el_size >> 16) << SEC_BD_W0_C_GRAN_SIZE_19_16_S) &
654 SEC_BD_W0_C_GRAN_SIZE_19_16_M;
655
656 req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_21_20_M;
657 req->w0 |= ((el_size >> 20) << SEC_BD_W0_C_GRAN_SIZE_21_20_S) &
658 SEC_BD_W0_C_GRAN_SIZE_21_20_M;
659
660
661 req->w2 = ((1 << SEC_BD_W2_GRAN_NUM_S) & SEC_BD_W2_GRAN_NUM_M) |
662 ((el_size << SEC_BD_W2_C_GRAN_SIZE_15_0_S) &
663 SEC_BD_W2_C_GRAN_SIZE_15_0_M);
664
665 req->w3 &= ~SEC_BD_W3_CIPHER_LEN_OFFSET_M;
666 req->w1 |= SEC_BD_W1_ADDR_TYPE;
667
668 el->sgl_in = sgl_in;
669
670 ret = sec_alloc_and_fill_hw_sgl(&el->in, &el->dma_in, el->sgl_in,
671 n_ents_in, info);
672 if (ret)
673 goto err_free_el;
674
675 req->data_addr_lo = lower_32_bits(el->dma_in);
676 req->data_addr_hi = upper_32_bits(el->dma_in);
677
678 if (different_dest) {
679 el->sgl_out = sgl_out;
680 ret = sec_alloc_and_fill_hw_sgl(&el->out, &el->dma_out,
681 el->sgl_out,
682 n_ents_out, info);
683 if (ret)
684 goto err_free_hw_sgl_in;
685
686 req->w0 |= SEC_BD_W0_DE;
687 req->cipher_destin_addr_lo = lower_32_bits(el->dma_out);
688 req->cipher_destin_addr_hi = upper_32_bits(el->dma_out);
689
690 } else {
691 req->w0 &= ~SEC_BD_W0_DE;
692 req->cipher_destin_addr_lo = lower_32_bits(el->dma_in);
693 req->cipher_destin_addr_hi = upper_32_bits(el->dma_in);
694 }
695
696 return el;
697
698 err_free_hw_sgl_in:
699 sec_free_hw_sgl(el->in, el->dma_in, info);
700 err_free_el:
701 kfree(el);
702
703 return ERR_PTR(ret);
704 }
705
706 static int sec_alg_skcipher_crypto(struct skcipher_request *skreq,
707 bool encrypt)
708 {
709 struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
710 struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
711 struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
712 struct sec_queue *queue = ctx->queue;
713 struct sec_request *sec_req = skcipher_request_ctx(skreq);
714 struct sec_dev_info *info = queue->dev_info;
715 int i, ret, steps;
716 size_t *split_sizes;
717 struct scatterlist **splits_in;
718 struct scatterlist **splits_out = NULL;
719 int *splits_in_nents;
720 int *splits_out_nents = NULL;
721 struct sec_request_el *el, *temp;
722 bool split = skreq->src != skreq->dst;
723
724 mutex_init(&sec_req->lock);
725 sec_req->req_base = &skreq->base;
726 sec_req->err = 0;
727
728 sec_req->len_in = sg_nents(skreq->src);
729
730 ret = sec_alg_alloc_and_calc_split_sizes(skreq->cryptlen, &split_sizes,
731 &steps);
732 if (ret)
733 return ret;
734 sec_req->num_elements = steps;
735 ret = sec_map_and_split_sg(skreq->src, split_sizes, steps, &splits_in,
736 &splits_in_nents, sec_req->len_in,
737 info->dev);
738 if (ret)
739 goto err_free_split_sizes;
740
741 if (split) {
742 sec_req->len_out = sg_nents(skreq->dst);
743 ret = sec_map_and_split_sg(skreq->dst, split_sizes, steps,
744 &splits_out, &splits_out_nents,
745 sec_req->len_out, info->dev);
746 if (ret)
747 goto err_unmap_in_sg;
748 }
749
750 sec_req->tfm_ctx = ctx;
751 sec_req->cb = sec_skcipher_alg_callback;
752 INIT_LIST_HEAD(&sec_req->elements);
753
754
755
756
757
758
759 if (crypto_skcipher_ivsize(atfm)) {
760 sec_req->dma_iv = dma_map_single(info->dev, skreq->iv,
761 crypto_skcipher_ivsize(atfm),
762 DMA_TO_DEVICE);
763 if (dma_mapping_error(info->dev, sec_req->dma_iv)) {
764 ret = -ENOMEM;
765 goto err_unmap_out_sg;
766 }
767 }
768
769
770 for (i = 0; i < steps; i++) {
771 el = sec_alg_alloc_and_fill_el(&ctx->req_template,
772 encrypt ? 1 : 0,
773 split_sizes[i],
774 skreq->src != skreq->dst,
775 splits_in[i], splits_in_nents[i],
776 split ? splits_out[i] : NULL,
777 split ? splits_out_nents[i] : 0,
778 info);
779 if (IS_ERR(el)) {
780 ret = PTR_ERR(el);
781 goto err_free_elements;
782 }
783 el->req.cipher_iv_addr_lo = lower_32_bits(sec_req->dma_iv);
784 el->req.cipher_iv_addr_hi = upper_32_bits(sec_req->dma_iv);
785 el->sec_req = sec_req;
786 list_add_tail(&el->head, &sec_req->elements);
787 }
788
789
790
791
792
793
794
795
796
797
798
799 mutex_lock(&queue->queuelock);
800
801
802
803
804
805
806
807
808 if ((!sec_queue_can_enqueue(queue, steps) &&
809 (!queue->havesoftqueue ||
810 kfifo_avail(&queue->softqueue) > steps)) ||
811 !list_empty(&ctx->backlog)) {
812 ret = -EBUSY;
813 if ((skreq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
814 list_add_tail(&sec_req->backlog_head, &ctx->backlog);
815 mutex_unlock(&queue->queuelock);
816 goto out;
817 }
818
819 mutex_unlock(&queue->queuelock);
820 goto err_free_elements;
821 }
822 ret = sec_send_request(sec_req, queue);
823 mutex_unlock(&queue->queuelock);
824 if (ret)
825 goto err_free_elements;
826
827 ret = -EINPROGRESS;
828 out:
829
830 kfree(splits_in_nents);
831 kfree(splits_in);
832 kfree(splits_out_nents);
833 kfree(splits_out);
834 kfree(split_sizes);
835 return ret;
836
837 err_free_elements:
838 list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
839 list_del(&el->head);
840 sec_alg_free_el(el, info);
841 }
842 if (crypto_skcipher_ivsize(atfm))
843 dma_unmap_single(info->dev, sec_req->dma_iv,
844 crypto_skcipher_ivsize(atfm),
845 DMA_BIDIRECTIONAL);
846 err_unmap_out_sg:
847 if (split)
848 sec_unmap_sg_on_err(skreq->dst, steps, splits_out,
849 splits_out_nents, sec_req->len_out,
850 info->dev);
851 err_unmap_in_sg:
852 sec_unmap_sg_on_err(skreq->src, steps, splits_in, splits_in_nents,
853 sec_req->len_in, info->dev);
854 err_free_split_sizes:
855 kfree(split_sizes);
856
857 return ret;
858 }
859
860 static int sec_alg_skcipher_encrypt(struct skcipher_request *req)
861 {
862 return sec_alg_skcipher_crypto(req, true);
863 }
864
865 static int sec_alg_skcipher_decrypt(struct skcipher_request *req)
866 {
867 return sec_alg_skcipher_crypto(req, false);
868 }
869
870 static int sec_alg_skcipher_init(struct crypto_skcipher *tfm)
871 {
872 struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
873
874 mutex_init(&ctx->lock);
875 INIT_LIST_HEAD(&ctx->backlog);
876 crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_request));
877
878 ctx->queue = sec_queue_alloc_start_safe();
879 if (IS_ERR(ctx->queue))
880 return PTR_ERR(ctx->queue);
881
882 mutex_init(&ctx->queue->queuelock);
883 ctx->queue->havesoftqueue = false;
884
885 return 0;
886 }
887
888 static void sec_alg_skcipher_exit(struct crypto_skcipher *tfm)
889 {
890 struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
891 struct device *dev = ctx->queue->dev_info->dev;
892
893 if (ctx->key) {
894 memzero_explicit(ctx->key, SEC_MAX_CIPHER_KEY);
895 dma_free_coherent(dev, SEC_MAX_CIPHER_KEY, ctx->key,
896 ctx->pkey);
897 }
898 sec_queue_stop_release(ctx->queue);
899 }
900
901 static int sec_alg_skcipher_init_with_queue(struct crypto_skcipher *tfm)
902 {
903 struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
904 int ret;
905
906 ret = sec_alg_skcipher_init(tfm);
907 if (ret)
908 return ret;
909
910 INIT_KFIFO(ctx->queue->softqueue);
911 ret = kfifo_alloc(&ctx->queue->softqueue, 512, GFP_KERNEL);
912 if (ret) {
913 sec_alg_skcipher_exit(tfm);
914 return ret;
915 }
916 ctx->queue->havesoftqueue = true;
917
918 return 0;
919 }
920
921 static void sec_alg_skcipher_exit_with_queue(struct crypto_skcipher *tfm)
922 {
923 struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
924
925 kfifo_free(&ctx->queue->softqueue);
926 sec_alg_skcipher_exit(tfm);
927 }
928
929 static struct skcipher_alg sec_algs[] = {
930 {
931 .base = {
932 .cra_name = "ecb(aes)",
933 .cra_driver_name = "hisi_sec_aes_ecb",
934 .cra_priority = 4001,
935 .cra_flags = CRYPTO_ALG_ASYNC,
936 .cra_blocksize = AES_BLOCK_SIZE,
937 .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
938 .cra_alignmask = 0,
939 .cra_module = THIS_MODULE,
940 },
941 .init = sec_alg_skcipher_init,
942 .exit = sec_alg_skcipher_exit,
943 .setkey = sec_alg_skcipher_setkey_aes_ecb,
944 .decrypt = sec_alg_skcipher_decrypt,
945 .encrypt = sec_alg_skcipher_encrypt,
946 .min_keysize = AES_MIN_KEY_SIZE,
947 .max_keysize = AES_MAX_KEY_SIZE,
948 .ivsize = 0,
949 }, {
950 .base = {
951 .cra_name = "cbc(aes)",
952 .cra_driver_name = "hisi_sec_aes_cbc",
953 .cra_priority = 4001,
954 .cra_flags = CRYPTO_ALG_ASYNC,
955 .cra_blocksize = AES_BLOCK_SIZE,
956 .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
957 .cra_alignmask = 0,
958 .cra_module = THIS_MODULE,
959 },
960 .init = sec_alg_skcipher_init_with_queue,
961 .exit = sec_alg_skcipher_exit_with_queue,
962 .setkey = sec_alg_skcipher_setkey_aes_cbc,
963 .decrypt = sec_alg_skcipher_decrypt,
964 .encrypt = sec_alg_skcipher_encrypt,
965 .min_keysize = AES_MIN_KEY_SIZE,
966 .max_keysize = AES_MAX_KEY_SIZE,
967 .ivsize = AES_BLOCK_SIZE,
968 }, {
969 .base = {
970 .cra_name = "ctr(aes)",
971 .cra_driver_name = "hisi_sec_aes_ctr",
972 .cra_priority = 4001,
973 .cra_flags = CRYPTO_ALG_ASYNC,
974 .cra_blocksize = AES_BLOCK_SIZE,
975 .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
976 .cra_alignmask = 0,
977 .cra_module = THIS_MODULE,
978 },
979 .init = sec_alg_skcipher_init_with_queue,
980 .exit = sec_alg_skcipher_exit_with_queue,
981 .setkey = sec_alg_skcipher_setkey_aes_ctr,
982 .decrypt = sec_alg_skcipher_decrypt,
983 .encrypt = sec_alg_skcipher_encrypt,
984 .min_keysize = AES_MIN_KEY_SIZE,
985 .max_keysize = AES_MAX_KEY_SIZE,
986 .ivsize = AES_BLOCK_SIZE,
987 }, {
988 .base = {
989 .cra_name = "xts(aes)",
990 .cra_driver_name = "hisi_sec_aes_xts",
991 .cra_priority = 4001,
992 .cra_flags = CRYPTO_ALG_ASYNC,
993 .cra_blocksize = AES_BLOCK_SIZE,
994 .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
995 .cra_alignmask = 0,
996 .cra_module = THIS_MODULE,
997 },
998 .init = sec_alg_skcipher_init,
999 .exit = sec_alg_skcipher_exit,
1000 .setkey = sec_alg_skcipher_setkey_aes_xts,
1001 .decrypt = sec_alg_skcipher_decrypt,
1002 .encrypt = sec_alg_skcipher_encrypt,
1003 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1004 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1005 .ivsize = AES_BLOCK_SIZE,
1006 }, {
1007
1008 .base = {
1009 .cra_name = "ecb(des)",
1010 .cra_driver_name = "hisi_sec_des_ecb",
1011 .cra_priority = 4001,
1012 .cra_flags = CRYPTO_ALG_ASYNC,
1013 .cra_blocksize = DES_BLOCK_SIZE,
1014 .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1015 .cra_alignmask = 0,
1016 .cra_module = THIS_MODULE,
1017 },
1018 .init = sec_alg_skcipher_init,
1019 .exit = sec_alg_skcipher_exit,
1020 .setkey = sec_alg_skcipher_setkey_des_ecb,
1021 .decrypt = sec_alg_skcipher_decrypt,
1022 .encrypt = sec_alg_skcipher_encrypt,
1023 .min_keysize = DES_KEY_SIZE,
1024 .max_keysize = DES_KEY_SIZE,
1025 .ivsize = 0,
1026 }, {
1027 .base = {
1028 .cra_name = "cbc(des)",
1029 .cra_driver_name = "hisi_sec_des_cbc",
1030 .cra_priority = 4001,
1031 .cra_flags = CRYPTO_ALG_ASYNC,
1032 .cra_blocksize = DES_BLOCK_SIZE,
1033 .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1034 .cra_alignmask = 0,
1035 .cra_module = THIS_MODULE,
1036 },
1037 .init = sec_alg_skcipher_init_with_queue,
1038 .exit = sec_alg_skcipher_exit_with_queue,
1039 .setkey = sec_alg_skcipher_setkey_des_cbc,
1040 .decrypt = sec_alg_skcipher_decrypt,
1041 .encrypt = sec_alg_skcipher_encrypt,
1042 .min_keysize = DES_KEY_SIZE,
1043 .max_keysize = DES_KEY_SIZE,
1044 .ivsize = DES_BLOCK_SIZE,
1045 }, {
1046 .base = {
1047 .cra_name = "cbc(des3_ede)",
1048 .cra_driver_name = "hisi_sec_3des_cbc",
1049 .cra_priority = 4001,
1050 .cra_flags = CRYPTO_ALG_ASYNC,
1051 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1052 .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1053 .cra_alignmask = 0,
1054 .cra_module = THIS_MODULE,
1055 },
1056 .init = sec_alg_skcipher_init_with_queue,
1057 .exit = sec_alg_skcipher_exit_with_queue,
1058 .setkey = sec_alg_skcipher_setkey_3des_cbc,
1059 .decrypt = sec_alg_skcipher_decrypt,
1060 .encrypt = sec_alg_skcipher_encrypt,
1061 .min_keysize = DES3_EDE_KEY_SIZE,
1062 .max_keysize = DES3_EDE_KEY_SIZE,
1063 .ivsize = DES3_EDE_BLOCK_SIZE,
1064 }, {
1065 .base = {
1066 .cra_name = "ecb(des3_ede)",
1067 .cra_driver_name = "hisi_sec_3des_ecb",
1068 .cra_priority = 4001,
1069 .cra_flags = CRYPTO_ALG_ASYNC,
1070 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1071 .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1072 .cra_alignmask = 0,
1073 .cra_module = THIS_MODULE,
1074 },
1075 .init = sec_alg_skcipher_init,
1076 .exit = sec_alg_skcipher_exit,
1077 .setkey = sec_alg_skcipher_setkey_3des_ecb,
1078 .decrypt = sec_alg_skcipher_decrypt,
1079 .encrypt = sec_alg_skcipher_encrypt,
1080 .min_keysize = DES3_EDE_KEY_SIZE,
1081 .max_keysize = DES3_EDE_KEY_SIZE,
1082 .ivsize = 0,
1083 }
1084 };
1085
1086 int sec_algs_register(void)
1087 {
1088 int ret = 0;
1089
1090 mutex_lock(&algs_lock);
1091 if (++active_devs != 1)
1092 goto unlock;
1093
1094 ret = crypto_register_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
1095 if (ret)
1096 --active_devs;
1097 unlock:
1098 mutex_unlock(&algs_lock);
1099
1100 return ret;
1101 }
1102
1103 void sec_algs_unregister(void)
1104 {
1105 mutex_lock(&algs_lock);
1106 if (--active_devs != 0)
1107 goto unlock;
1108 crypto_unregister_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
1109
1110 unlock:
1111 mutex_unlock(&algs_lock);
1112 }