This source file includes following definitions.
- queue_empty
- queue_enqueued_ddcbs
- queue_free_ddcbs
- ddcb_mark_tapped
- ddcb_mark_appended
- ddcb_mark_cleared
- ddcb_mark_finished
- ddcb_mark_unused
- genwqe_crc16
- print_ddcb_info
- ddcb_requ_alloc
- ddcb_requ_free
- ddcb_requ_get_state
- ddcb_requ_set_state
- ddcb_requ_collect_debug_data
- ddcb_requ_finished
- enqueue_ddcb
- copy_ddcb_results
- genwqe_check_ddcb_queue
- __genwqe_wait_ddcb
- get_next_ddcb
- __genwqe_purge_ddcb
- genwqe_init_debug_data
- __genwqe_enqueue_ddcb
- __genwqe_execute_raw_ddcb
- genwqe_next_ddcb_ready
- genwqe_ddcbs_in_flight
- setup_ddcb_queue
- ddcb_queue_initialized
- free_ddcb_queue
- genwqe_pf_isr
- genwqe_vf_isr
- genwqe_card_thread
- genwqe_setup_service_layer
- queue_wake_up_all
- genwqe_finish_queue
- genwqe_release_service_layer
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 #include <linux/types.h>
22 #include <linux/sched.h>
23 #include <linux/wait.h>
24 #include <linux/pci.h>
25 #include <linux/string.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/delay.h>
28 #include <linux/module.h>
29 #include <linux/interrupt.h>
30 #include <linux/crc-itu-t.h>
31
32 #include "card_base.h"
33 #include "card_ddcb.h"
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82 static int queue_empty(struct ddcb_queue *queue)
83 {
84 return queue->ddcb_next == queue->ddcb_act;
85 }
86
87 static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
88 {
89 if (queue->ddcb_next >= queue->ddcb_act)
90 return queue->ddcb_next - queue->ddcb_act;
91
92 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
93 }
94
95 static int queue_free_ddcbs(struct ddcb_queue *queue)
96 {
97 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
98
99 if (WARN_ON_ONCE(free_ddcbs < 0)) {
100 return 0;
101 }
102 return free_ddcbs;
103 }
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121 static inline void ddcb_mark_tapped(struct ddcb *pddcb)
122 {
123 pddcb->priv[7] = 0xbb;
124 }
125
126 static inline void ddcb_mark_appended(struct ddcb *pddcb)
127 {
128 pddcb->priv[7] = 0xaa;
129 }
130
131 static inline void ddcb_mark_cleared(struct ddcb *pddcb)
132 {
133 pddcb->priv[6] = 0xcc;
134 }
135
136 static inline void ddcb_mark_finished(struct ddcb *pddcb)
137 {
138 pddcb->priv[6] = 0xff;
139 }
140
141 static inline void ddcb_mark_unused(struct ddcb *pddcb)
142 {
143 pddcb->priv_64 = cpu_to_be64(0);
144 }
145
146
147
148
149
150
151
152
153
154
155
156
157
158 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
159 {
160 return crc_itu_t(init, buff, len);
161 }
162
163 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
164 {
165 int i;
166 struct ddcb *pddcb;
167 unsigned long flags;
168 struct pci_dev *pci_dev = cd->pci_dev;
169
170 spin_lock_irqsave(&cd->print_lock, flags);
171
172 dev_info(&pci_dev->dev,
173 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
174 cd->card_idx, queue->ddcb_act, queue->ddcb_next);
175
176 pddcb = queue->ddcb_vaddr;
177 for (i = 0; i < queue->ddcb_max; i++) {
178 dev_err(&pci_dev->dev,
179 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
180 i == queue->ddcb_act ? '>' : ' ',
181 i,
182 be16_to_cpu(pddcb->retc_16),
183 be16_to_cpu(pddcb->seqnum_16),
184 pddcb->hsi,
185 pddcb->shi,
186 be64_to_cpu(pddcb->priv_64),
187 pddcb->cmd);
188 pddcb++;
189 }
190 spin_unlock_irqrestore(&cd->print_lock, flags);
191 }
192
193 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
194 {
195 struct ddcb_requ *req;
196
197 req = kzalloc(sizeof(*req), GFP_KERNEL);
198 if (!req)
199 return NULL;
200
201 return &req->cmd;
202 }
203
204 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
205 {
206 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
207
208 kfree(req);
209 }
210
211 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
212 {
213 return req->req_state;
214 }
215
216 static inline void ddcb_requ_set_state(struct ddcb_requ *req,
217 enum genwqe_requ_state new_state)
218 {
219 req->req_state = new_state;
220 }
221
222 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
223 {
224 return req->cmd.ddata_addr != 0x0;
225 }
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
242 {
243 return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
244 (cd->card_state != GENWQE_CARD_USED);
245 }
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262 #define RET_DDCB_APPENDED 1
263 #define RET_DDCB_TAPPED 2
264
265 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
266 struct ddcb *pddcb, int ddcb_no)
267 {
268 unsigned int try;
269 int prev_no;
270 struct ddcb *prev_ddcb;
271 __be32 old, new, icrc_hsi_shi;
272 u64 num;
273
274
275
276
277
278
279 ddcb_mark_unused(pddcb);
280
281
282 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
283 prev_ddcb = &queue->ddcb_vaddr[prev_no];
284
285
286
287
288
289
290 ddcb_mark_appended(pddcb);
291 for (try = 0; try < 2; try++) {
292 old = prev_ddcb->icrc_hsi_shi_32;
293
294
295 if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
296 break;
297
298 new = (old | DDCB_NEXT_BE32);
299
300 wmb();
301 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
302
303 if (icrc_hsi_shi == old)
304 return RET_DDCB_APPENDED;
305 }
306
307
308 ddcb_mark_tapped(pddcb);
309 num = (u64)ddcb_no << 8;
310
311 wmb();
312 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num);
313
314 return RET_DDCB_TAPPED;
315 }
316
317
318
319
320
321
322
323
324
325
326
327
328 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
329 {
330 struct ddcb_queue *queue = req->queue;
331 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
332
333 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
334
335
336 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16);
337 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
338 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
339
340 req->cmd.attn = be16_to_cpu(pddcb->attn_16);
341 req->cmd.progress = be32_to_cpu(pddcb->progress_32);
342 req->cmd.retc = be16_to_cpu(pddcb->retc_16);
343
344 if (ddcb_requ_collect_debug_data(req)) {
345 int prev_no = (ddcb_no == 0) ?
346 queue->ddcb_max - 1 : ddcb_no - 1;
347 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
348
349 memcpy(&req->debug_data.ddcb_finished, pddcb,
350 sizeof(req->debug_data.ddcb_finished));
351 memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
352 sizeof(req->debug_data.ddcb_prev));
353 }
354 }
355
356
357
358
359
360
361
362 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
363 struct ddcb_queue *queue)
364 {
365 unsigned long flags;
366 int ddcbs_finished = 0;
367 struct pci_dev *pci_dev = cd->pci_dev;
368
369 spin_lock_irqsave(&queue->ddcb_lock, flags);
370
371
372 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
373
374 struct ddcb *pddcb;
375 struct ddcb_requ *req;
376 u16 vcrc, vcrc_16, retc_16;
377
378 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
379
380 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
381 0x00000000)
382 goto go_home;
383
384 wmb();
385
386
387 req = queue->ddcb_req[queue->ddcb_act];
388 if (req == NULL) {
389
390
391 goto pick_next_one;
392 }
393
394
395
396
397
398
399
400
401
402 retc_16 = be16_to_cpu(pddcb->retc_16);
403 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
404 u64 errcnts, status;
405 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
406
407 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
408 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
409
410 dev_err(&pci_dev->dev,
411 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
412 __func__, be16_to_cpu(pddcb->seqnum_16),
413 pddcb->hsi, retc_16, errcnts, status,
414 queue->ddcb_daddr + ddcb_offs);
415 }
416
417 copy_ddcb_results(req, queue->ddcb_act);
418 queue->ddcb_req[queue->ddcb_act] = NULL;
419
420 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
421 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
422
423 ddcb_mark_finished(pddcb);
424
425
426 vcrc = genwqe_crc16(pddcb->asv,
427 VCRC_LENGTH(req->cmd.asv_length),
428 0xffff);
429 vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
430 if (vcrc != vcrc_16) {
431 printk_ratelimited(KERN_ERR
432 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
433 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
434 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
435 vcrc, vcrc_16);
436 }
437
438 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
439 queue->ddcbs_completed++;
440 queue->ddcbs_in_flight--;
441
442
443
444 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
445 wake_up_interruptible(&queue->busy_waitq);
446
447 pick_next_one:
448 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
449 ddcbs_finished++;
450 }
451
452 go_home:
453 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
454 return ddcbs_finished;
455 }
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
475 {
476 int rc;
477 unsigned int ddcb_no;
478 struct ddcb_queue *queue;
479 struct pci_dev *pci_dev = cd->pci_dev;
480
481 if (req == NULL)
482 return -EINVAL;
483
484 queue = req->queue;
485 if (queue == NULL)
486 return -EINVAL;
487
488 ddcb_no = req->num;
489 if (ddcb_no >= queue->ddcb_max)
490 return -EINVAL;
491
492 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
493 ddcb_requ_finished(cd, req),
494 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
495
496
497
498
499
500
501
502 if (rc == 0) {
503 struct ddcb_queue *queue = req->queue;
504 struct ddcb *pddcb;
505
506
507
508
509
510
511 genwqe_check_ddcb_queue(cd, req->queue);
512 if (ddcb_requ_finished(cd, req))
513 return rc;
514
515 dev_err(&pci_dev->dev,
516 "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
517 __func__, req->num, rc, ddcb_requ_get_state(req),
518 req);
519 dev_err(&pci_dev->dev,
520 "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__,
521 __genwqe_readq(cd, queue->IO_QUEUE_STATUS));
522
523 pddcb = &queue->ddcb_vaddr[req->num];
524 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
525
526 print_ddcb_info(cd, req->queue);
527 return -ETIMEDOUT;
528
529 } else if (rc == -ERESTARTSYS) {
530 return rc;
531
532
533
534
535
536 } else if (rc < 0) {
537 dev_err(&pci_dev->dev,
538 "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
539 __func__, req->num, rc, ddcb_requ_get_state(req));
540 return -EINVAL;
541 }
542
543
544 if (cd->card_state != GENWQE_CARD_USED) {
545 dev_err(&pci_dev->dev,
546 "[%s] err: DDCB#%d forced to stop (rc=%d)\n",
547 __func__, req->num, rc);
548 return -EIO;
549 }
550 return rc;
551 }
552
553
554
555
556
557
558
559
560
561
562 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
563 struct ddcb_queue *queue,
564 int *num)
565 {
566 u64 *pu64;
567 struct ddcb *pddcb;
568
569 if (queue_free_ddcbs(queue) == 0)
570 return NULL;
571
572
573 pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
574
575
576
577 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
578 return NULL;
579
580 *num = queue->ddcb_next;
581 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
582
583
584 pu64 = (u64 *)pddcb;
585 pu64[0] = 0ULL;
586 pu64[1] = 0ULL;
587
588
589 pu64[0x80/8] = 0ULL;
590 pu64[0x88/8] = 0ULL;
591 pu64[0x90/8] = 0ULL;
592 pu64[0x98/8] = 0ULL;
593 pu64[0xd0/8] = 0ULL;
594
595 pddcb->pre = DDCB_PRESET_PRE;
596 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
597 return pddcb;
598 }
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
616 {
617 struct ddcb *pddcb = NULL;
618 unsigned int t;
619 unsigned long flags;
620 struct ddcb_queue *queue = req->queue;
621 struct pci_dev *pci_dev = cd->pci_dev;
622 u64 queue_status;
623 __be32 icrc_hsi_shi = 0x0000;
624 __be32 old, new;
625
626
627 if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
628 dev_err(&pci_dev->dev,
629 "[%s] err: software timeout is not set!\n", __func__);
630 return -EFAULT;
631 }
632
633 pddcb = &queue->ddcb_vaddr[req->num];
634
635 for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
636
637 spin_lock_irqsave(&queue->ddcb_lock, flags);
638
639
640 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
641 goto go_home;
642
643
644 old = pddcb->icrc_hsi_shi_32;
645 if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
646
647 new = (old | DDCB_PURGE_BE32);
648 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
649 old, new);
650 if (icrc_hsi_shi == old)
651 goto finish_ddcb;
652 }
653
654
655 barrier();
656 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
657 if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
658 goto finish_ddcb;
659
660 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
661
662
663
664
665
666
667
668
669 copy_ddcb_results(req, req->num);
670 msleep(100);
671 continue;
672
673 finish_ddcb:
674 copy_ddcb_results(req, req->num);
675 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
676 queue->ddcbs_in_flight--;
677 queue->ddcb_req[req->num] = NULL;
678 ddcb_mark_cleared(pddcb);
679
680
681
682
683
684
685
686
687
688
689
690 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
691 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
692 (queue->ddcb_act == req->num)) {
693 queue->ddcb_act = ((queue->ddcb_act + 1) %
694 queue->ddcb_max);
695 }
696 go_home:
697 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
698 return 0;
699 }
700
701
702
703
704
705 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
706
707 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
708 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
709
710 dev_err(&pci_dev->dev,
711 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
712 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
713 queue_status);
714
715 print_ddcb_info(cd, req->queue);
716
717 return -EFAULT;
718 }
719
720 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
721 {
722 int len;
723 struct pci_dev *pci_dev = cd->pci_dev;
724
725 if (d == NULL) {
726 dev_err(&pci_dev->dev,
727 "[%s] err: invalid memory for debug data!\n",
728 __func__);
729 return -EFAULT;
730 }
731
732 len = sizeof(d->driver_version);
733 snprintf(d->driver_version, len, "%s", DRV_VERSION);
734 d->slu_unitcfg = cd->slu_unitcfg;
735 d->app_unitcfg = cd->app_unitcfg;
736 return 0;
737 }
738
739
740
741
742
743
744
745
746
747
748
749 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
750 unsigned int f_flags)
751 {
752 struct ddcb *pddcb;
753 unsigned long flags;
754 struct ddcb_queue *queue;
755 struct pci_dev *pci_dev = cd->pci_dev;
756 u16 icrc;
757
758 retry:
759 if (cd->card_state != GENWQE_CARD_USED) {
760 printk_ratelimited(KERN_ERR
761 "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
762 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
763 __func__, req->num);
764 return -EIO;
765 }
766
767 queue = req->queue = &cd->queue;
768
769
770
771
772 if (GENWQE_POLLING_ENABLED)
773 genwqe_check_ddcb_queue(cd, queue);
774
775
776
777
778
779
780 spin_lock_irqsave(&queue->ddcb_lock, flags);
781
782 pddcb = get_next_ddcb(cd, queue, &req->num);
783 if (pddcb == NULL) {
784 int rc;
785
786 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
787
788 if (f_flags & O_NONBLOCK) {
789 queue->return_on_busy++;
790 return -EBUSY;
791 }
792
793 queue->wait_on_busy++;
794 rc = wait_event_interruptible(queue->busy_waitq,
795 queue_free_ddcbs(queue) != 0);
796 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
797 __func__, rc);
798 if (rc == -ERESTARTSYS)
799 return rc;
800
801 goto retry;
802 }
803
804 if (queue->ddcb_req[req->num] != NULL) {
805 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
806
807 dev_err(&pci_dev->dev,
808 "[%s] picked DDCB %d with req=%p still in use!!\n",
809 __func__, req->num, req);
810 return -EFAULT;
811 }
812 ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
813 queue->ddcb_req[req->num] = req;
814
815 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
816 pddcb->cmd = req->cmd.cmd;
817 pddcb->acfunc = req->cmd.acfunc;
818
819
820
821
822
823
824
825
826
827 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
828 pddcb->xdir = 0x1;
829 else
830 pddcb->xdir = 0x0;
831
832
833 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
834 ((req->cmd.asv_length / 8)));
835 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850 if (genwqe_get_slu_id(cd) <= 0x2) {
851 memcpy(&pddcb->__asiv[0],
852 &req->cmd.__asiv[0],
853 DDCB_ASIV_LENGTH);
854 } else {
855 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
856 memcpy(&pddcb->n.asiv[0],
857 &req->cmd.asiv[0],
858 DDCB_ASIV_LENGTH_ATS);
859 }
860
861 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000);
862
863
864
865
866
867 icrc = genwqe_crc16((const u8 *)pddcb,
868 ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
869 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
870
871
872 if (!GENWQE_POLLING_ENABLED)
873 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
874
875 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
876 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
877
878 if (ddcb_requ_collect_debug_data(req)) {
879
880
881
882 genwqe_init_debug_data(cd, &req->debug_data);
883 memcpy(&req->debug_data.ddcb_before, pddcb,
884 sizeof(req->debug_data.ddcb_before));
885 }
886
887 enqueue_ddcb(cd, queue, pddcb, req->num);
888 queue->ddcbs_in_flight++;
889
890 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
891 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
892
893 ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
894 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
895 wake_up_interruptible(&cd->queue_waitq);
896
897 return 0;
898 }
899
900
901
902
903
904
905
906 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
907 struct genwqe_ddcb_cmd *cmd,
908 unsigned int f_flags)
909 {
910 int rc = 0;
911 struct pci_dev *pci_dev = cd->pci_dev;
912 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
913
914 if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
915 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
916 __func__, cmd->asiv_length);
917 return -EINVAL;
918 }
919 if (cmd->asv_length > DDCB_ASV_LENGTH) {
920 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
921 __func__, cmd->asiv_length);
922 return -EINVAL;
923 }
924 rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
925 if (rc != 0)
926 return rc;
927
928 rc = __genwqe_wait_ddcb(cd, req);
929 if (rc < 0)
930 goto err_exit;
931
932 if (ddcb_requ_collect_debug_data(req)) {
933 if (copy_to_user((struct genwqe_debug_data __user *)
934 (unsigned long)cmd->ddata_addr,
935 &req->debug_data,
936 sizeof(struct genwqe_debug_data)))
937 return -EFAULT;
938 }
939
940
941
942
943
944
945 if (cmd->retc != DDCB_RETC_COMPLETE) {
946
947
948 rc = -EBADMSG;
949 }
950
951 return rc;
952
953 err_exit:
954 __genwqe_purge_ddcb(cd, req);
955
956 if (ddcb_requ_collect_debug_data(req)) {
957 if (copy_to_user((struct genwqe_debug_data __user *)
958 (unsigned long)cmd->ddata_addr,
959 &req->debug_data,
960 sizeof(struct genwqe_debug_data)))
961 return -EFAULT;
962 }
963 return rc;
964 }
965
966
967
968
969
970
971 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
972 {
973 unsigned long flags;
974 struct ddcb *pddcb;
975 struct ddcb_queue *queue = &cd->queue;
976
977 spin_lock_irqsave(&queue->ddcb_lock, flags);
978
979 if (queue_empty(queue)) {
980 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
981 return 0;
982 }
983
984 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
985 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) {
986 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
987 return 1;
988 }
989
990 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
991 return 0;
992 }
993
994
995
996
997
998
999
1000
1001 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1002 {
1003 unsigned long flags;
1004 int ddcbs_in_flight = 0;
1005 struct ddcb_queue *queue = &cd->queue;
1006
1007 spin_lock_irqsave(&queue->ddcb_lock, flags);
1008 ddcbs_in_flight += queue->ddcbs_in_flight;
1009 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1010
1011 return ddcbs_in_flight;
1012 }
1013
1014 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1015 {
1016 int rc, i;
1017 struct ddcb *pddcb;
1018 u64 val64;
1019 unsigned int queue_size;
1020 struct pci_dev *pci_dev = cd->pci_dev;
1021
1022 if (GENWQE_DDCB_MAX < 2)
1023 return -EINVAL;
1024
1025 queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
1026
1027 queue->ddcbs_in_flight = 0;
1028 queue->ddcbs_max_in_flight = 0;
1029 queue->ddcbs_completed = 0;
1030 queue->return_on_busy = 0;
1031 queue->wait_on_busy = 0;
1032
1033 queue->ddcb_seq = 0x100;
1034 queue->ddcb_max = GENWQE_DDCB_MAX;
1035 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1036 &queue->ddcb_daddr);
1037 if (queue->ddcb_vaddr == NULL) {
1038 dev_err(&pci_dev->dev,
1039 "[%s] **err: could not allocate DDCB **\n", __func__);
1040 return -ENOMEM;
1041 }
1042 queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
1043 GFP_KERNEL);
1044 if (!queue->ddcb_req) {
1045 rc = -ENOMEM;
1046 goto free_ddcbs;
1047 }
1048
1049 queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
1050 sizeof(wait_queue_head_t),
1051 GFP_KERNEL);
1052 if (!queue->ddcb_waitqs) {
1053 rc = -ENOMEM;
1054 goto free_requs;
1055 }
1056
1057 for (i = 0; i < queue->ddcb_max; i++) {
1058 pddcb = &queue->ddcb_vaddr[i];
1059 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1060 pddcb->retc_16 = cpu_to_be16(0xfff);
1061
1062 queue->ddcb_req[i] = NULL;
1063 init_waitqueue_head(&queue->ddcb_waitqs[i]);
1064 }
1065
1066 queue->ddcb_act = 0;
1067 queue->ddcb_next = 0;
1068
1069 spin_lock_init(&queue->ddcb_lock);
1070 init_waitqueue_head(&queue->busy_waitq);
1071
1072 val64 = ((u64)(queue->ddcb_max - 1) << 8);
1073 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07);
1074 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1075 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1076 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64);
1077 return 0;
1078
1079 free_requs:
1080 kfree(queue->ddcb_req);
1081 queue->ddcb_req = NULL;
1082 free_ddcbs:
1083 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1084 queue->ddcb_daddr);
1085 queue->ddcb_vaddr = NULL;
1086 queue->ddcb_daddr = 0ull;
1087 return -ENODEV;
1088
1089 }
1090
1091 static int ddcb_queue_initialized(struct ddcb_queue *queue)
1092 {
1093 return queue->ddcb_vaddr != NULL;
1094 }
1095
1096 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1097 {
1098 unsigned int queue_size;
1099
1100 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1101
1102 kfree(queue->ddcb_req);
1103 queue->ddcb_req = NULL;
1104
1105 if (queue->ddcb_vaddr) {
1106 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1107 queue->ddcb_daddr);
1108 queue->ddcb_vaddr = NULL;
1109 queue->ddcb_daddr = 0ull;
1110 }
1111 }
1112
1113 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1114 {
1115 u64 gfir;
1116 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1117 struct pci_dev *pci_dev = cd->pci_dev;
1118
1119
1120
1121
1122
1123 cd->irqs_processed++;
1124 wake_up_interruptible(&cd->queue_waitq);
1125
1126
1127
1128
1129
1130 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1131 if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1132 !pci_channel_offline(pci_dev)) {
1133
1134 if (cd->use_platform_recovery) {
1135
1136
1137
1138
1139
1140 readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1141
1142
1143 if (pci_channel_offline(pci_dev))
1144 goto exit;
1145 }
1146
1147 wake_up_interruptible(&cd->health_waitq);
1148
1149
1150
1151
1152
1153 dev_err_ratelimited(&pci_dev->dev,
1154 "[%s] GFIR=%016llx\n",
1155 __func__, gfir);
1156 }
1157
1158 exit:
1159 return IRQ_HANDLED;
1160 }
1161
1162 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1163 {
1164 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1165
1166 cd->irqs_processed++;
1167 wake_up_interruptible(&cd->queue_waitq);
1168
1169 return IRQ_HANDLED;
1170 }
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180 static int genwqe_card_thread(void *data)
1181 {
1182 int should_stop = 0, rc = 0;
1183 struct genwqe_dev *cd = (struct genwqe_dev *)data;
1184
1185 while (!kthread_should_stop()) {
1186
1187 genwqe_check_ddcb_queue(cd, &cd->queue);
1188
1189 if (GENWQE_POLLING_ENABLED) {
1190 rc = wait_event_interruptible_timeout(
1191 cd->queue_waitq,
1192 genwqe_ddcbs_in_flight(cd) ||
1193 (should_stop = kthread_should_stop()), 1);
1194 } else {
1195 rc = wait_event_interruptible_timeout(
1196 cd->queue_waitq,
1197 genwqe_next_ddcb_ready(cd) ||
1198 (should_stop = kthread_should_stop()), HZ);
1199 }
1200 if (should_stop)
1201 break;
1202
1203
1204
1205
1206
1207 cond_resched();
1208 }
1209 return 0;
1210 }
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220 int genwqe_setup_service_layer(struct genwqe_dev *cd)
1221 {
1222 int rc;
1223 struct ddcb_queue *queue;
1224 struct pci_dev *pci_dev = cd->pci_dev;
1225
1226 if (genwqe_is_privileged(cd)) {
1227 rc = genwqe_card_reset(cd);
1228 if (rc < 0) {
1229 dev_err(&pci_dev->dev,
1230 "[%s] err: reset failed.\n", __func__);
1231 return rc;
1232 }
1233 genwqe_read_softreset(cd);
1234 }
1235
1236 queue = &cd->queue;
1237 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG;
1238 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS;
1239 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1240 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1241 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET;
1242 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP;
1243 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME;
1244 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1245 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW;
1246
1247 rc = setup_ddcb_queue(cd, queue);
1248 if (rc != 0) {
1249 rc = -ENODEV;
1250 goto err_out;
1251 }
1252
1253 init_waitqueue_head(&cd->queue_waitq);
1254 cd->card_thread = kthread_run(genwqe_card_thread, cd,
1255 GENWQE_DEVNAME "%d_thread",
1256 cd->card_idx);
1257 if (IS_ERR(cd->card_thread)) {
1258 rc = PTR_ERR(cd->card_thread);
1259 cd->card_thread = NULL;
1260 goto stop_free_queue;
1261 }
1262
1263 rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1264 if (rc)
1265 goto stop_kthread;
1266
1267
1268
1269
1270
1271
1272 init_waitqueue_head(&cd->health_waitq);
1273
1274 if (genwqe_is_privileged(cd)) {
1275 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1276 GENWQE_DEVNAME, cd);
1277 } else {
1278 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1279 GENWQE_DEVNAME, cd);
1280 }
1281 if (rc < 0) {
1282 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1283 goto stop_irq_cap;
1284 }
1285
1286 cd->card_state = GENWQE_CARD_USED;
1287 return 0;
1288
1289 stop_irq_cap:
1290 genwqe_reset_interrupt_capability(cd);
1291 stop_kthread:
1292 kthread_stop(cd->card_thread);
1293 cd->card_thread = NULL;
1294 stop_free_queue:
1295 free_ddcb_queue(cd, queue);
1296 err_out:
1297 return rc;
1298 }
1299
1300
1301
1302
1303
1304
1305
1306
1307 static int queue_wake_up_all(struct genwqe_dev *cd)
1308 {
1309 unsigned int i;
1310 unsigned long flags;
1311 struct ddcb_queue *queue = &cd->queue;
1312
1313 spin_lock_irqsave(&queue->ddcb_lock, flags);
1314
1315 for (i = 0; i < queue->ddcb_max; i++)
1316 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1317
1318 wake_up_interruptible(&queue->busy_waitq);
1319 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1320
1321 return 0;
1322 }
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332 int genwqe_finish_queue(struct genwqe_dev *cd)
1333 {
1334 int i, rc = 0, in_flight;
1335 int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
1336 struct pci_dev *pci_dev = cd->pci_dev;
1337 struct ddcb_queue *queue = &cd->queue;
1338
1339 if (!ddcb_queue_initialized(queue))
1340 return 0;
1341
1342
1343 if (cd->card_state == GENWQE_CARD_USED)
1344 cd->card_state = GENWQE_CARD_UNUSED;
1345
1346
1347
1348 queue_wake_up_all(cd);
1349
1350
1351 for (i = 0; i < waitmax; i++) {
1352 in_flight = genwqe_ddcbs_in_flight(cd);
1353
1354 if (in_flight == 0)
1355 break;
1356
1357 dev_dbg(&pci_dev->dev,
1358 " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1359 i, waitmax, in_flight);
1360
1361
1362
1363
1364
1365
1366
1367
1368 msleep(1000);
1369 }
1370 if (i == waitmax) {
1371 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n",
1372 __func__);
1373 rc = -EIO;
1374 }
1375 return rc;
1376 }
1377
1378
1379
1380
1381
1382
1383
1384 int genwqe_release_service_layer(struct genwqe_dev *cd)
1385 {
1386 struct pci_dev *pci_dev = cd->pci_dev;
1387
1388 if (!ddcb_queue_initialized(&cd->queue))
1389 return 1;
1390
1391 free_irq(pci_dev->irq, cd);
1392 genwqe_reset_interrupt_capability(cd);
1393
1394 if (cd->card_thread != NULL) {
1395 kthread_stop(cd->card_thread);
1396 cd->card_thread = NULL;
1397 }
1398
1399 free_ddcb_queue(cd, &cd->queue);
1400 return 0;
1401 }