This source file includes following definitions.
- register_algs
- unregister_algs
- caam_reset_hw_jr
- caam_jr_shutdown
- caam_jr_remove
- caam_jr_interrupt
- caam_jr_dequeue
- caam_jr_alloc
- caam_jr_free
- caam_jr_enqueue
- caam_jr_init
- caam_jr_irq_dispose_mapping
- caam_jr_probe
- jr_driver_init
- jr_driver_exit
1
2
3
4
5
6
7
8
9
10 #include <linux/of_irq.h>
11 #include <linux/of_address.h>
12
13 #include "compat.h"
14 #include "ctrl.h"
15 #include "regs.h"
16 #include "jr.h"
17 #include "desc.h"
18 #include "intern.h"
19
20 struct jr_driver_data {
21
22 struct list_head jr_list;
23 spinlock_t jr_alloc_lock;
24 } ____cacheline_aligned;
25
26 static struct jr_driver_data driver_data;
27 static DEFINE_MUTEX(algs_lock);
28 static unsigned int active_devs;
29
30 static void register_algs(struct device *dev)
31 {
32 mutex_lock(&algs_lock);
33
34 if (++active_devs != 1)
35 goto algs_unlock;
36
37 caam_algapi_init(dev);
38 caam_algapi_hash_init(dev);
39 caam_pkc_init(dev);
40 caam_rng_init(dev);
41 caam_qi_algapi_init(dev);
42
43 algs_unlock:
44 mutex_unlock(&algs_lock);
45 }
46
47 static void unregister_algs(void)
48 {
49 mutex_lock(&algs_lock);
50
51 if (--active_devs != 0)
52 goto algs_unlock;
53
54 caam_qi_algapi_exit();
55
56 caam_rng_exit();
57 caam_pkc_exit();
58 caam_algapi_hash_exit();
59 caam_algapi_exit();
60
61 algs_unlock:
62 mutex_unlock(&algs_lock);
63 }
64
65 static int caam_reset_hw_jr(struct device *dev)
66 {
67 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
68 unsigned int timeout = 100000;
69
70
71
72
73
74 clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
75
76
77 wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
78 while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
79 JRINT_ERR_HALT_INPROGRESS) && --timeout)
80 cpu_relax();
81
82 if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
83 JRINT_ERR_HALT_COMPLETE || timeout == 0) {
84 dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
85 return -EIO;
86 }
87
88
89 timeout = 100000;
90 wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
91 while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
92 cpu_relax();
93
94 if (timeout == 0) {
95 dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
96 return -EIO;
97 }
98
99
100 clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
101
102 return 0;
103 }
104
105
106
107
108 static int caam_jr_shutdown(struct device *dev)
109 {
110 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
111 int ret;
112
113 ret = caam_reset_hw_jr(dev);
114
115 tasklet_kill(&jrp->irqtask);
116
117 return ret;
118 }
119
120 static int caam_jr_remove(struct platform_device *pdev)
121 {
122 int ret;
123 struct device *jrdev;
124 struct caam_drv_private_jr *jrpriv;
125
126 jrdev = &pdev->dev;
127 jrpriv = dev_get_drvdata(jrdev);
128
129
130
131
132 if (atomic_read(&jrpriv->tfm_count)) {
133 dev_err(jrdev, "Device is busy\n");
134 return -EBUSY;
135 }
136
137
138 unregister_algs();
139
140
141 spin_lock(&driver_data.jr_alloc_lock);
142 list_del(&jrpriv->list_node);
143 spin_unlock(&driver_data.jr_alloc_lock);
144
145
146 ret = caam_jr_shutdown(jrdev);
147 if (ret)
148 dev_err(jrdev, "Failed to shut down job ring\n");
149
150 return ret;
151 }
152
153
154 static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
155 {
156 struct device *dev = st_dev;
157 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
158 u32 irqstate;
159
160
161
162
163
164 irqstate = rd_reg32(&jrp->rregs->jrintstatus);
165 if (!irqstate)
166 return IRQ_NONE;
167
168
169
170
171
172
173 if (irqstate & JRINT_JR_ERROR) {
174 dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
175 BUG();
176 }
177
178
179 clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
180
181
182 wr_reg32(&jrp->rregs->jrintstatus, irqstate);
183
184 preempt_disable();
185 tasklet_schedule(&jrp->irqtask);
186 preempt_enable();
187
188 return IRQ_HANDLED;
189 }
190
191
192 static void caam_jr_dequeue(unsigned long devarg)
193 {
194 int hw_idx, sw_idx, i, head, tail;
195 struct device *dev = (struct device *)devarg;
196 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
197 void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
198 u32 *userdesc, userstatus;
199 void *userarg;
200 u32 outring_used = 0;
201
202 while (outring_used ||
203 (outring_used = rd_reg32(&jrp->rregs->outring_used))) {
204
205 head = READ_ONCE(jrp->head);
206
207 sw_idx = tail = jrp->tail;
208 hw_idx = jrp->out_ring_read_index;
209
210 for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
211 sw_idx = (tail + i) & (JOBR_DEPTH - 1);
212
213 if (jr_outentry_desc(jrp->outring, hw_idx) ==
214 caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma))
215 break;
216 }
217
218 BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
219
220
221 dma_unmap_single(dev,
222 caam_dma_to_cpu(jr_outentry_desc(jrp->outring,
223 hw_idx)),
224 jrp->entinfo[sw_idx].desc_size,
225 DMA_TO_DEVICE);
226
227
228 jrp->entinfo[sw_idx].desc_addr_dma = 0;
229
230
231 usercall = jrp->entinfo[sw_idx].callbk;
232 userarg = jrp->entinfo[sw_idx].cbkarg;
233 userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
234 userstatus = caam32_to_cpu(jr_outentry_jrstatus(jrp->outring,
235 hw_idx));
236
237
238
239
240
241
242 mb();
243
244
245 wr_reg32(&jrp->rregs->outring_rmvd, 1);
246
247 jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
248 (JOBR_DEPTH - 1);
249
250
251
252
253
254
255 if (sw_idx == tail) {
256 do {
257 tail = (tail + 1) & (JOBR_DEPTH - 1);
258 } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
259 jrp->entinfo[tail].desc_addr_dma == 0);
260
261 jrp->tail = tail;
262 }
263
264
265 usercall(dev, userdesc, userstatus, userarg);
266 outring_used--;
267 }
268
269
270 clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
271 }
272
273
274
275
276
277
278
279 struct device *caam_jr_alloc(void)
280 {
281 struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
282 struct device *dev = ERR_PTR(-ENODEV);
283 int min_tfm_cnt = INT_MAX;
284 int tfm_cnt;
285
286 spin_lock(&driver_data.jr_alloc_lock);
287
288 if (list_empty(&driver_data.jr_list)) {
289 spin_unlock(&driver_data.jr_alloc_lock);
290 return ERR_PTR(-ENODEV);
291 }
292
293 list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
294 tfm_cnt = atomic_read(&jrpriv->tfm_count);
295 if (tfm_cnt < min_tfm_cnt) {
296 min_tfm_cnt = tfm_cnt;
297 min_jrpriv = jrpriv;
298 }
299 if (!min_tfm_cnt)
300 break;
301 }
302
303 if (min_jrpriv) {
304 atomic_inc(&min_jrpriv->tfm_count);
305 dev = min_jrpriv->dev;
306 }
307 spin_unlock(&driver_data.jr_alloc_lock);
308
309 return dev;
310 }
311 EXPORT_SYMBOL(caam_jr_alloc);
312
313
314
315
316
317
318 void caam_jr_free(struct device *rdev)
319 {
320 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
321
322 atomic_dec(&jrpriv->tfm_count);
323 }
324 EXPORT_SYMBOL(caam_jr_free);
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354 int caam_jr_enqueue(struct device *dev, u32 *desc,
355 void (*cbk)(struct device *dev, u32 *desc,
356 u32 status, void *areq),
357 void *areq)
358 {
359 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
360 struct caam_jrentry_info *head_entry;
361 int head, tail, desc_size;
362 dma_addr_t desc_dma;
363
364 desc_size = (caam32_to_cpu(*desc) & HDR_JD_LENGTH_MASK) * sizeof(u32);
365 desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
366 if (dma_mapping_error(dev, desc_dma)) {
367 dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
368 return -EIO;
369 }
370
371 spin_lock_bh(&jrp->inplock);
372
373 head = jrp->head;
374 tail = READ_ONCE(jrp->tail);
375
376 if (!jrp->inpring_avail ||
377 CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
378 spin_unlock_bh(&jrp->inplock);
379 dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
380 return -EBUSY;
381 }
382
383 head_entry = &jrp->entinfo[head];
384 head_entry->desc_addr_virt = desc;
385 head_entry->desc_size = desc_size;
386 head_entry->callbk = (void *)cbk;
387 head_entry->cbkarg = areq;
388 head_entry->desc_addr_dma = desc_dma;
389
390 jr_inpentry_set(jrp->inpring, head, cpu_to_caam_dma(desc_dma));
391
392
393
394
395
396
397 smp_wmb();
398
399 jrp->head = (head + 1) & (JOBR_DEPTH - 1);
400
401
402
403
404
405
406
407
408
409 wr_reg32(&jrp->rregs->inpring_jobadd, 1);
410
411 jrp->inpring_avail--;
412 if (!jrp->inpring_avail)
413 jrp->inpring_avail = rd_reg32(&jrp->rregs->inpring_avail);
414
415 spin_unlock_bh(&jrp->inplock);
416
417 return 0;
418 }
419 EXPORT_SYMBOL(caam_jr_enqueue);
420
421
422
423
424 static int caam_jr_init(struct device *dev)
425 {
426 struct caam_drv_private_jr *jrp;
427 dma_addr_t inpbusaddr, outbusaddr;
428 int i, error;
429
430 jrp = dev_get_drvdata(dev);
431
432 error = caam_reset_hw_jr(dev);
433 if (error)
434 return error;
435
436 jrp->inpring = dmam_alloc_coherent(dev, SIZEOF_JR_INPENTRY *
437 JOBR_DEPTH, &inpbusaddr,
438 GFP_KERNEL);
439 if (!jrp->inpring)
440 return -ENOMEM;
441
442 jrp->outring = dmam_alloc_coherent(dev, SIZEOF_JR_OUTENTRY *
443 JOBR_DEPTH, &outbusaddr,
444 GFP_KERNEL);
445 if (!jrp->outring)
446 return -ENOMEM;
447
448 jrp->entinfo = devm_kcalloc(dev, JOBR_DEPTH, sizeof(*jrp->entinfo),
449 GFP_KERNEL);
450 if (!jrp->entinfo)
451 return -ENOMEM;
452
453 for (i = 0; i < JOBR_DEPTH; i++)
454 jrp->entinfo[i].desc_addr_dma = !0;
455
456
457 jrp->out_ring_read_index = 0;
458 jrp->head = 0;
459 jrp->tail = 0;
460
461 wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
462 wr_reg64(&jrp->rregs->outring_base, outbusaddr);
463 wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
464 wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
465
466 jrp->inpring_avail = JOBR_DEPTH;
467
468 spin_lock_init(&jrp->inplock);
469
470
471 clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC |
472 (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
473 (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
474
475 tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
476
477
478 error = devm_request_irq(dev, jrp->irq, caam_jr_interrupt, IRQF_SHARED,
479 dev_name(dev), dev);
480 if (error) {
481 dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
482 jrp->ridx, jrp->irq);
483 tasklet_kill(&jrp->irqtask);
484 }
485
486 return error;
487 }
488
489 static void caam_jr_irq_dispose_mapping(void *data)
490 {
491 irq_dispose_mapping((unsigned long)data);
492 }
493
494
495
496
497 static int caam_jr_probe(struct platform_device *pdev)
498 {
499 struct device *jrdev;
500 struct device_node *nprop;
501 struct caam_job_ring __iomem *ctrl;
502 struct caam_drv_private_jr *jrpriv;
503 static int total_jobrs;
504 struct resource *r;
505 int error;
506
507 jrdev = &pdev->dev;
508 jrpriv = devm_kmalloc(jrdev, sizeof(*jrpriv), GFP_KERNEL);
509 if (!jrpriv)
510 return -ENOMEM;
511
512 dev_set_drvdata(jrdev, jrpriv);
513
514
515 jrpriv->ridx = total_jobrs++;
516
517 nprop = pdev->dev.of_node;
518
519
520 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
521 if (!r) {
522 dev_err(jrdev, "platform_get_resource() failed\n");
523 return -ENOMEM;
524 }
525
526 ctrl = devm_ioremap(jrdev, r->start, resource_size(r));
527 if (!ctrl) {
528 dev_err(jrdev, "devm_ioremap() failed\n");
529 return -ENOMEM;
530 }
531
532 jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
533
534 error = dma_set_mask_and_coherent(jrdev, caam_get_dma_mask(jrdev));
535 if (error) {
536 dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n",
537 error);
538 return error;
539 }
540
541
542 jrpriv->irq = irq_of_parse_and_map(nprop, 0);
543 if (!jrpriv->irq) {
544 dev_err(jrdev, "irq_of_parse_and_map failed\n");
545 return -EINVAL;
546 }
547
548 error = devm_add_action_or_reset(jrdev, caam_jr_irq_dispose_mapping,
549 (void *)(unsigned long)jrpriv->irq);
550 if (error)
551 return error;
552
553
554 error = caam_jr_init(jrdev);
555 if (error)
556 return error;
557
558 jrpriv->dev = jrdev;
559 spin_lock(&driver_data.jr_alloc_lock);
560 list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
561 spin_unlock(&driver_data.jr_alloc_lock);
562
563 atomic_set(&jrpriv->tfm_count, 0);
564
565 register_algs(jrdev->parent);
566
567 return 0;
568 }
569
570 static const struct of_device_id caam_jr_match[] = {
571 {
572 .compatible = "fsl,sec-v4.0-job-ring",
573 },
574 {
575 .compatible = "fsl,sec4.0-job-ring",
576 },
577 {},
578 };
579 MODULE_DEVICE_TABLE(of, caam_jr_match);
580
581 static struct platform_driver caam_jr_driver = {
582 .driver = {
583 .name = "caam_jr",
584 .of_match_table = caam_jr_match,
585 },
586 .probe = caam_jr_probe,
587 .remove = caam_jr_remove,
588 };
589
590 static int __init jr_driver_init(void)
591 {
592 spin_lock_init(&driver_data.jr_alloc_lock);
593 INIT_LIST_HEAD(&driver_data.jr_list);
594 return platform_driver_register(&caam_jr_driver);
595 }
596
597 static void __exit jr_driver_exit(void)
598 {
599 platform_driver_unregister(&caam_jr_driver);
600 }
601
602 module_init(jr_driver_init);
603 module_exit(jr_driver_exit);
604
605 MODULE_LICENSE("GPL");
606 MODULE_DESCRIPTION("FSL CAAM JR request backend");
607 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");