This source file includes following definitions.
- cc_cpp_int_mask
- cc_req_mgr_fini
- cc_req_mgr_init
- enqueue_seq
- request_mgr_complete
- cc_queues_status
- cc_do_send_request
- cc_enqueue_backlog
- cc_proc_backlog
- cc_send_request
- cc_send_sync_request
- send_request_init
- complete_request
- comp_work_handler
- proc_completions
- cc_axi_comp_count
- comp_handler
1
2
3
4 #include <linux/kernel.h>
5 #include <linux/nospec.h>
6 #include "cc_driver.h"
7 #include "cc_buffer_mgr.h"
8 #include "cc_request_mgr.h"
9 #include "cc_pm.h"
10
11 #define CC_MAX_POLL_ITER 10
12
13 #define CC_MAX_DESC_SEQ_LEN 23
14
15 struct cc_req_mgr_handle {
16
17 unsigned int hw_queue_size;
18 unsigned int min_free_hw_slots;
19 unsigned int max_used_sw_slots;
20 struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];
21 u32 req_queue_head;
22 u32 req_queue_tail;
23 u32 axi_completed;
24 u32 q_free_slots;
25
26
27
28 spinlock_t hw_lock;
29 struct cc_hw_desc compl_desc;
30 u8 *dummy_comp_buff;
31 dma_addr_t dummy_comp_buff_dma;
32
33
34 struct list_head backlog;
35 unsigned int bl_len;
36 spinlock_t bl_lock;
37
38 #ifdef COMP_IN_WQ
39 struct workqueue_struct *workq;
40 struct delayed_work compwork;
41 #else
42 struct tasklet_struct comptask;
43 #endif
44 };
45
46 struct cc_bl_item {
47 struct cc_crypto_req creq;
48 struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN];
49 unsigned int len;
50 struct list_head list;
51 bool notif;
52 };
53
54 static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = {
55 { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT),
56 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT),
57 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT),
58 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT),
59 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT),
60 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT),
61 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT),
62 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) },
63 { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT),
64 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT),
65 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT),
66 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT),
67 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT),
68 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT),
69 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT),
70 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) }
71 };
72
73 static void comp_handler(unsigned long devarg);
74 #ifdef COMP_IN_WQ
75 static void comp_work_handler(struct work_struct *work);
76 #endif
77
78 static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot)
79 {
80 alg = array_index_nospec(alg, CC_CPP_NUM_ALGS);
81 slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS);
82
83 return cc_cpp_int_masks[alg][slot];
84 }
85
86 void cc_req_mgr_fini(struct cc_drvdata *drvdata)
87 {
88 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
89 struct device *dev = drvdata_to_dev(drvdata);
90
91 if (!req_mgr_h)
92 return;
93
94 if (req_mgr_h->dummy_comp_buff_dma) {
95 dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff,
96 req_mgr_h->dummy_comp_buff_dma);
97 }
98
99 dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -
100 req_mgr_h->min_free_hw_slots));
101 dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
102
103 #ifdef COMP_IN_WQ
104 flush_workqueue(req_mgr_h->workq);
105 destroy_workqueue(req_mgr_h->workq);
106 #else
107
108 tasklet_kill(&req_mgr_h->comptask);
109 #endif
110 kzfree(req_mgr_h);
111 drvdata->request_mgr_handle = NULL;
112 }
113
114 int cc_req_mgr_init(struct cc_drvdata *drvdata)
115 {
116 struct cc_req_mgr_handle *req_mgr_h;
117 struct device *dev = drvdata_to_dev(drvdata);
118 int rc = 0;
119
120 req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL);
121 if (!req_mgr_h) {
122 rc = -ENOMEM;
123 goto req_mgr_init_err;
124 }
125
126 drvdata->request_mgr_handle = req_mgr_h;
127
128 spin_lock_init(&req_mgr_h->hw_lock);
129 spin_lock_init(&req_mgr_h->bl_lock);
130 INIT_LIST_HEAD(&req_mgr_h->backlog);
131
132 #ifdef COMP_IN_WQ
133 dev_dbg(dev, "Initializing completion workqueue\n");
134 req_mgr_h->workq = create_singlethread_workqueue("ccree");
135 if (!req_mgr_h->workq) {
136 dev_err(dev, "Failed creating work queue\n");
137 rc = -ENOMEM;
138 goto req_mgr_init_err;
139 }
140 INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);
141 #else
142 dev_dbg(dev, "Initializing completion tasklet\n");
143 tasklet_init(&req_mgr_h->comptask, comp_handler,
144 (unsigned long)drvdata);
145 #endif
146 req_mgr_h->hw_queue_size = cc_ioread(drvdata,
147 CC_REG(DSCRPTR_QUEUE_SRAM_SIZE));
148 dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
149 if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
150 dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n",
151 req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
152 rc = -ENOMEM;
153 goto req_mgr_init_err;
154 }
155 req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;
156 req_mgr_h->max_used_sw_slots = 0;
157
158
159 req_mgr_h->dummy_comp_buff =
160 dma_alloc_coherent(dev, sizeof(u32),
161 &req_mgr_h->dummy_comp_buff_dma,
162 GFP_KERNEL);
163 if (!req_mgr_h->dummy_comp_buff) {
164 dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n",
165 sizeof(u32));
166 rc = -ENOMEM;
167 goto req_mgr_init_err;
168 }
169
170
171 hw_desc_init(&req_mgr_h->compl_desc);
172 set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32));
173 set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma,
174 sizeof(u32), NS_BIT, 1);
175 set_flow_mode(&req_mgr_h->compl_desc, BYPASS);
176 set_queue_last_ind(drvdata, &req_mgr_h->compl_desc);
177
178 return 0;
179
180 req_mgr_init_err:
181 cc_req_mgr_fini(drvdata);
182 return rc;
183 }
184
185 static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[],
186 unsigned int seq_len)
187 {
188 int i, w;
189 void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0);
190 struct device *dev = drvdata_to_dev(drvdata);
191
192
193
194
195
196
197 for (i = 0; i < seq_len; i++) {
198 for (w = 0; w <= 5; w++)
199 writel_relaxed(seq[i].word[w], reg);
200
201 if (cc_dump_desc)
202 dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
203 i, seq[i].word[0], seq[i].word[1],
204 seq[i].word[2], seq[i].word[3],
205 seq[i].word[4], seq[i].word[5]);
206 }
207 }
208
209
210
211
212
213
214
215
216 static void request_mgr_complete(struct device *dev, void *dx_compl_h,
217 int dummy)
218 {
219 struct completion *this_compl = dx_compl_h;
220
221 complete(this_compl);
222 }
223
224 static int cc_queues_status(struct cc_drvdata *drvdata,
225 struct cc_req_mgr_handle *req_mgr_h,
226 unsigned int total_seq_len)
227 {
228 unsigned long poll_queue;
229 struct device *dev = drvdata_to_dev(drvdata);
230
231
232
233
234
235 if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) ==
236 req_mgr_h->req_queue_tail) {
237 dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
238 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
239 return -ENOSPC;
240 }
241
242 if (req_mgr_h->q_free_slots >= total_seq_len)
243 return 0;
244
245
246 for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) {
247 req_mgr_h->q_free_slots =
248 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
249 if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)
250 req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;
251
252 if (req_mgr_h->q_free_slots >= total_seq_len) {
253
254 return 0;
255 }
256
257 dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
258 req_mgr_h->q_free_slots, total_seq_len);
259 }
260
261 dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",
262 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE,
263 req_mgr_h->q_free_slots, total_seq_len);
264 return -ENOSPC;
265 }
266
267
268
269
270
271
272
273
274
275
276
277
278
279 static int cc_do_send_request(struct cc_drvdata *drvdata,
280 struct cc_crypto_req *cc_req,
281 struct cc_hw_desc *desc, unsigned int len,
282 bool add_comp)
283 {
284 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
285 unsigned int used_sw_slots;
286 unsigned int total_seq_len = len;
287 struct device *dev = drvdata_to_dev(drvdata);
288
289 used_sw_slots = ((req_mgr_h->req_queue_head -
290 req_mgr_h->req_queue_tail) &
291 (MAX_REQUEST_QUEUE_SIZE - 1));
292 if (used_sw_slots > req_mgr_h->max_used_sw_slots)
293 req_mgr_h->max_used_sw_slots = used_sw_slots;
294
295
296 req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req;
297 req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) &
298 (MAX_REQUEST_QUEUE_SIZE - 1);
299
300
301 dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head);
302
303
304
305
306
307
308 wmb();
309
310
311
312 enqueue_seq(drvdata, desc, len);
313
314 if (add_comp) {
315 enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1);
316 total_seq_len++;
317 }
318
319 if (req_mgr_h->q_free_slots < total_seq_len) {
320
321
322
323
324 dev_err(dev, "HW free slot count mismatch.");
325 req_mgr_h->q_free_slots = 0;
326 } else {
327
328 req_mgr_h->q_free_slots -= total_seq_len;
329 }
330
331
332 return -EINPROGRESS;
333 }
334
335 static void cc_enqueue_backlog(struct cc_drvdata *drvdata,
336 struct cc_bl_item *bli)
337 {
338 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
339 struct device *dev = drvdata_to_dev(drvdata);
340
341 spin_lock_bh(&mgr->bl_lock);
342 list_add_tail(&bli->list, &mgr->backlog);
343 ++mgr->bl_len;
344 dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len);
345 spin_unlock_bh(&mgr->bl_lock);
346 tasklet_schedule(&mgr->comptask);
347 }
348
349 static void cc_proc_backlog(struct cc_drvdata *drvdata)
350 {
351 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
352 struct cc_bl_item *bli;
353 struct cc_crypto_req *creq;
354 void *req;
355 struct device *dev = drvdata_to_dev(drvdata);
356 int rc;
357
358 spin_lock(&mgr->bl_lock);
359
360 while (mgr->bl_len) {
361 bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list);
362 dev_dbg(dev, "---bl len: %d\n", mgr->bl_len);
363
364 spin_unlock(&mgr->bl_lock);
365
366
367 creq = &bli->creq;
368 req = creq->user_arg;
369
370
371
372
373
374 if (!bli->notif) {
375 creq->user_cb(dev, req, -EINPROGRESS);
376 bli->notif = true;
377 }
378
379 spin_lock(&mgr->hw_lock);
380
381 rc = cc_queues_status(drvdata, mgr, bli->len);
382 if (rc) {
383
384
385
386
387
388 spin_unlock(&mgr->hw_lock);
389 return;
390 }
391
392 rc = cc_do_send_request(drvdata, &bli->creq, bli->desc,
393 bli->len, false);
394
395 spin_unlock(&mgr->hw_lock);
396
397 if (rc != -EINPROGRESS) {
398 cc_pm_put_suspend(dev);
399 creq->user_cb(dev, req, rc);
400 }
401
402
403 spin_lock(&mgr->bl_lock);
404 list_del(&bli->list);
405 --mgr->bl_len;
406 kfree(bli);
407 }
408
409 spin_unlock(&mgr->bl_lock);
410 }
411
412 int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
413 struct cc_hw_desc *desc, unsigned int len,
414 struct crypto_async_request *req)
415 {
416 int rc;
417 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
418 struct device *dev = drvdata_to_dev(drvdata);
419 bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
420 gfp_t flags = cc_gfp_flags(req);
421 struct cc_bl_item *bli;
422
423 rc = cc_pm_get(dev);
424 if (rc) {
425 dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc);
426 return rc;
427 }
428
429 spin_lock_bh(&mgr->hw_lock);
430 rc = cc_queues_status(drvdata, mgr, len);
431
432 #ifdef CC_DEBUG_FORCE_BACKLOG
433 if (backlog_ok)
434 rc = -ENOSPC;
435 #endif
436
437 if (rc == -ENOSPC && backlog_ok) {
438 spin_unlock_bh(&mgr->hw_lock);
439
440 bli = kmalloc(sizeof(*bli), flags);
441 if (!bli) {
442 cc_pm_put_suspend(dev);
443 return -ENOMEM;
444 }
445
446 memcpy(&bli->creq, cc_req, sizeof(*cc_req));
447 memcpy(&bli->desc, desc, len * sizeof(*desc));
448 bli->len = len;
449 bli->notif = false;
450 cc_enqueue_backlog(drvdata, bli);
451 return -EBUSY;
452 }
453
454 if (!rc)
455 rc = cc_do_send_request(drvdata, cc_req, desc, len, false);
456
457 spin_unlock_bh(&mgr->hw_lock);
458 return rc;
459 }
460
461 int cc_send_sync_request(struct cc_drvdata *drvdata,
462 struct cc_crypto_req *cc_req, struct cc_hw_desc *desc,
463 unsigned int len)
464 {
465 int rc;
466 struct device *dev = drvdata_to_dev(drvdata);
467 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
468
469 init_completion(&cc_req->seq_compl);
470 cc_req->user_cb = request_mgr_complete;
471 cc_req->user_arg = &cc_req->seq_compl;
472
473 rc = cc_pm_get(dev);
474 if (rc) {
475 dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc);
476 return rc;
477 }
478
479 while (true) {
480 spin_lock_bh(&mgr->hw_lock);
481 rc = cc_queues_status(drvdata, mgr, len + 1);
482
483 if (!rc)
484 break;
485
486 spin_unlock_bh(&mgr->hw_lock);
487 if (rc != -EAGAIN) {
488 cc_pm_put_suspend(dev);
489 return rc;
490 }
491 wait_for_completion_interruptible(&drvdata->hw_queue_avail);
492 reinit_completion(&drvdata->hw_queue_avail);
493 }
494
495 rc = cc_do_send_request(drvdata, cc_req, desc, len, true);
496 spin_unlock_bh(&mgr->hw_lock);
497
498 if (rc != -EINPROGRESS) {
499 cc_pm_put_suspend(dev);
500 return rc;
501 }
502
503 wait_for_completion(&cc_req->seq_compl);
504 return 0;
505 }
506
507
508
509
510
511
512
513
514
515
516
517
518 int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc,
519 unsigned int len)
520 {
521 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
522 unsigned int total_seq_len = len;
523 int rc = 0;
524
525
526
527 rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len);
528 if (rc)
529 return rc;
530
531 set_queue_last_ind(drvdata, &desc[(len - 1)]);
532
533
534
535
536
537
538 wmb();
539 enqueue_seq(drvdata, desc, len);
540
541
542 req_mgr_h->q_free_slots =
543 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
544
545 return 0;
546 }
547
548 void complete_request(struct cc_drvdata *drvdata)
549 {
550 struct cc_req_mgr_handle *request_mgr_handle =
551 drvdata->request_mgr_handle;
552
553 complete(&drvdata->hw_queue_avail);
554 #ifdef COMP_IN_WQ
555 queue_delayed_work(request_mgr_handle->workq,
556 &request_mgr_handle->compwork, 0);
557 #else
558 tasklet_schedule(&request_mgr_handle->comptask);
559 #endif
560 }
561
562 #ifdef COMP_IN_WQ
563 static void comp_work_handler(struct work_struct *work)
564 {
565 struct cc_drvdata *drvdata =
566 container_of(work, struct cc_drvdata, compwork.work);
567
568 comp_handler((unsigned long)drvdata);
569 }
570 #endif
571
572 static void proc_completions(struct cc_drvdata *drvdata)
573 {
574 struct cc_crypto_req *cc_req;
575 struct device *dev = drvdata_to_dev(drvdata);
576 struct cc_req_mgr_handle *request_mgr_handle =
577 drvdata->request_mgr_handle;
578 unsigned int *tail = &request_mgr_handle->req_queue_tail;
579 unsigned int *head = &request_mgr_handle->req_queue_head;
580 int rc;
581 u32 mask;
582
583 while (request_mgr_handle->axi_completed) {
584 request_mgr_handle->axi_completed--;
585
586
587 if (*head == *tail) {
588
589
590
591
592 dev_err(dev, "Request queue is empty head == tail %u\n",
593 *head);
594 break;
595 }
596
597 cc_req = &request_mgr_handle->req_queue[*tail];
598
599 if (cc_req->cpp.is_cpp) {
600
601 dev_dbg(dev, "CPP request completion slot: %d alg:%d\n",
602 cc_req->cpp.slot, cc_req->cpp.alg);
603 mask = cc_cpp_int_mask(cc_req->cpp.alg,
604 cc_req->cpp.slot);
605 rc = (drvdata->irq & mask ? -EPERM : 0);
606 dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask,
607 drvdata->irq, rc);
608 } else {
609 dev_dbg(dev, "None CPP request completion\n");
610 rc = 0;
611 }
612
613 if (cc_req->user_cb)
614 cc_req->user_cb(dev, cc_req->user_arg, rc);
615 *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
616 dev_dbg(dev, "Dequeue request tail=%u\n", *tail);
617 dev_dbg(dev, "Request completed. axi_completed=%d\n",
618 request_mgr_handle->axi_completed);
619 cc_pm_put_suspend(dev);
620 }
621 }
622
623 static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata)
624 {
625 return FIELD_GET(AXIM_MON_COMP_VALUE,
626 cc_ioread(drvdata, drvdata->axim_mon_offset));
627 }
628
629
630 static void comp_handler(unsigned long devarg)
631 {
632 struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg;
633 struct cc_req_mgr_handle *request_mgr_handle =
634 drvdata->request_mgr_handle;
635 struct device *dev = drvdata_to_dev(drvdata);
636 u32 irq;
637
638 dev_dbg(dev, "Completion handler called!\n");
639 irq = (drvdata->irq & drvdata->comp_mask);
640
641
642
643
644 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
645
646
647
648 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
649
650 dev_dbg(dev, "AXI completion after updated: %d\n",
651 request_mgr_handle->axi_completed);
652
653 while (request_mgr_handle->axi_completed) {
654 do {
655 drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR));
656 irq = (drvdata->irq & drvdata->comp_mask);
657 proc_completions(drvdata);
658
659
660
661
662 request_mgr_handle->axi_completed +=
663 cc_axi_comp_count(drvdata);
664 } while (request_mgr_handle->axi_completed > 0);
665
666 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
667
668 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
669 }
670
671
672
673
674 cc_iowrite(drvdata, CC_REG(HOST_IMR),
675 cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask);
676
677 cc_proc_backlog(drvdata);
678 dev_dbg(dev, "Comp. handler done.\n");
679 }