This source file includes following definitions.
- periodic_next_shadow
- shadow_next_periodic
- periodic_unlink
- find_tt
- drop_tt
- bandwidth_dbg
- reserve_release_intr_bandwidth
- compute_tt_budget
- same_tt
- tt_start_uframe
- carryover_tt_bandwidth
- tt_available
- tt_no_collision
- enable_periodic
- disable_periodic
- qh_link_periodic
- qh_unlink_periodic
- cancel_unlink_wait_intr
- start_unlink_intr
- start_unlink_intr_wait
- end_unlink_intr
- check_period
- check_intr_schedule
- qh_schedule
- intr_submit
- scan_intr
- iso_stream_alloc
- iso_stream_init
- iso_stream_find
- iso_sched_alloc
- itd_sched_init
- iso_sched_free
- itd_urb_transaction
- reserve_release_iso_bandwidth
- itd_slot_ok
- sitd_slot_ok
- iso_stream_schedule
- itd_init
- itd_patch
- itd_link
- itd_link_urb
- itd_complete
- itd_submit
- sitd_sched_init
- sitd_urb_transaction
- sitd_patch
- sitd_link
- sitd_link_urb
- sitd_complete
- sitd_submit
- scan_isoc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 static int ehci_get_frame(struct usb_hcd *hcd);
25
26
27
28
29
30
31 static union ehci_shadow *
32 periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
33 __hc32 tag)
34 {
35 switch (hc32_to_cpu(ehci, tag)) {
36 case Q_TYPE_QH:
37 return &periodic->qh->qh_next;
38 case Q_TYPE_FSTN:
39 return &periodic->fstn->fstn_next;
40 case Q_TYPE_ITD:
41 return &periodic->itd->itd_next;
42
43 default:
44 return &periodic->sitd->sitd_next;
45 }
46 }
47
48 static __hc32 *
49 shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
50 __hc32 tag)
51 {
52 switch (hc32_to_cpu(ehci, tag)) {
53
54 case Q_TYPE_QH:
55 return &periodic->qh->hw->hw_next;
56
57 default:
58 return periodic->hw_next;
59 }
60 }
61
62
63 static void periodic_unlink(struct ehci_hcd *ehci, unsigned frame, void *ptr)
64 {
65 union ehci_shadow *prev_p = &ehci->pshadow[frame];
66 __hc32 *hw_p = &ehci->periodic[frame];
67 union ehci_shadow here = *prev_p;
68
69
70 while (here.ptr && here.ptr != ptr) {
71 prev_p = periodic_next_shadow(ehci, prev_p,
72 Q_NEXT_TYPE(ehci, *hw_p));
73 hw_p = shadow_next_periodic(ehci, &here,
74 Q_NEXT_TYPE(ehci, *hw_p));
75 here = *prev_p;
76 }
77
78 if (!here.ptr)
79 return;
80
81
82
83
84 *prev_p = *periodic_next_shadow(ehci, &here,
85 Q_NEXT_TYPE(ehci, *hw_p));
86
87 if (!ehci->use_dummy_qh ||
88 *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p))
89 != EHCI_LIST_END(ehci))
90 *hw_p = *shadow_next_periodic(ehci, &here,
91 Q_NEXT_TYPE(ehci, *hw_p));
92 else
93 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
94 }
95
96
97
98
99
100
101 static struct ehci_tt *find_tt(struct usb_device *udev)
102 {
103 struct usb_tt *utt = udev->tt;
104 struct ehci_tt *tt, **tt_index, **ptt;
105 unsigned port;
106 bool allocated_index = false;
107
108 if (!utt)
109 return NULL;
110
111
112
113
114
115
116 tt_index = NULL;
117 if (utt->multi) {
118 tt_index = utt->hcpriv;
119 if (!tt_index) {
120 tt_index = kcalloc(utt->hub->maxchild,
121 sizeof(*tt_index),
122 GFP_ATOMIC);
123 if (!tt_index)
124 return ERR_PTR(-ENOMEM);
125 utt->hcpriv = tt_index;
126 allocated_index = true;
127 }
128 port = udev->ttport - 1;
129 ptt = &tt_index[port];
130 } else {
131 port = 0;
132 ptt = (struct ehci_tt **) &utt->hcpriv;
133 }
134
135 tt = *ptt;
136 if (!tt) {
137 struct ehci_hcd *ehci =
138 hcd_to_ehci(bus_to_hcd(udev->bus));
139
140 tt = kzalloc(sizeof(*tt), GFP_ATOMIC);
141 if (!tt) {
142 if (allocated_index) {
143 utt->hcpriv = NULL;
144 kfree(tt_index);
145 }
146 return ERR_PTR(-ENOMEM);
147 }
148 list_add_tail(&tt->tt_list, &ehci->tt_list);
149 INIT_LIST_HEAD(&tt->ps_list);
150 tt->usb_tt = utt;
151 tt->tt_port = port;
152 *ptt = tt;
153 }
154
155 return tt;
156 }
157
158
159 static void drop_tt(struct usb_device *udev)
160 {
161 struct usb_tt *utt = udev->tt;
162 struct ehci_tt *tt, **tt_index, **ptt;
163 int cnt, i;
164
165 if (!utt || !utt->hcpriv)
166 return;
167
168 cnt = 0;
169 if (utt->multi) {
170 tt_index = utt->hcpriv;
171 ptt = &tt_index[udev->ttport - 1];
172
173
174 for (i = 0; i < utt->hub->maxchild; ++i)
175 cnt += !!tt_index[i];
176 } else {
177 tt_index = NULL;
178 ptt = (struct ehci_tt **) &utt->hcpriv;
179 }
180
181 tt = *ptt;
182 if (!tt || !list_empty(&tt->ps_list))
183 return;
184
185 list_del(&tt->tt_list);
186 *ptt = NULL;
187 kfree(tt);
188 if (cnt == 1) {
189 utt->hcpriv = NULL;
190 kfree(tt_index);
191 }
192 }
193
194 static void bandwidth_dbg(struct ehci_hcd *ehci, int sign, char *type,
195 struct ehci_per_sched *ps)
196 {
197 dev_dbg(&ps->udev->dev,
198 "ep %02x: %s %s @ %u+%u (%u.%u+%u) [%u/%u us] mask %04x\n",
199 ps->ep->desc.bEndpointAddress,
200 (sign >= 0 ? "reserve" : "release"), type,
201 (ps->bw_phase << 3) + ps->phase_uf, ps->bw_uperiod,
202 ps->phase, ps->phase_uf, ps->period,
203 ps->usecs, ps->c_usecs, ps->cs_mask);
204 }
205
206 static void reserve_release_intr_bandwidth(struct ehci_hcd *ehci,
207 struct ehci_qh *qh, int sign)
208 {
209 unsigned start_uf;
210 unsigned i, j, m;
211 int usecs = qh->ps.usecs;
212 int c_usecs = qh->ps.c_usecs;
213 int tt_usecs = qh->ps.tt_usecs;
214 struct ehci_tt *tt;
215
216 if (qh->ps.phase == NO_FRAME)
217 return;
218 start_uf = qh->ps.bw_phase << 3;
219
220 bandwidth_dbg(ehci, sign, "intr", &qh->ps);
221
222 if (sign < 0) {
223 usecs = -usecs;
224 c_usecs = -c_usecs;
225 tt_usecs = -tt_usecs;
226 }
227
228
229 for (i = start_uf + qh->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE;
230 i += qh->ps.bw_uperiod)
231 ehci->bandwidth[i] += usecs;
232
233
234 if (qh->ps.c_usecs) {
235
236 for (i = start_uf; i < EHCI_BANDWIDTH_SIZE;
237 i += qh->ps.bw_uperiod) {
238 for ((j = 2, m = 1 << (j+8)); j < 8; (++j, m <<= 1)) {
239 if (qh->ps.cs_mask & m)
240 ehci->bandwidth[i+j] += c_usecs;
241 }
242 }
243 }
244
245
246 if (tt_usecs) {
247 tt = find_tt(qh->ps.udev);
248 if (sign > 0)
249 list_add_tail(&qh->ps.ps_list, &tt->ps_list);
250 else
251 list_del(&qh->ps.ps_list);
252
253 for (i = start_uf >> 3; i < EHCI_BANDWIDTH_FRAMES;
254 i += qh->ps.bw_period)
255 tt->bandwidth[i] += tt_usecs;
256 }
257 }
258
259
260
261 static void compute_tt_budget(u8 budget_table[EHCI_BANDWIDTH_SIZE],
262 struct ehci_tt *tt)
263 {
264 struct ehci_per_sched *ps;
265 unsigned uframe, uf, x;
266 u8 *budget_line;
267
268 if (!tt)
269 return;
270 memset(budget_table, 0, EHCI_BANDWIDTH_SIZE);
271
272
273 list_for_each_entry(ps, &tt->ps_list, ps_list) {
274 for (uframe = ps->bw_phase << 3; uframe < EHCI_BANDWIDTH_SIZE;
275 uframe += ps->bw_uperiod) {
276 budget_line = &budget_table[uframe];
277 x = ps->tt_usecs;
278
279
280 for (uf = ps->phase_uf; uf < 8; ++uf) {
281 x += budget_line[uf];
282
283
284 if (x <= 125) {
285 budget_line[uf] = x;
286 break;
287 }
288 budget_line[uf] = 125;
289 x -= 125;
290 }
291 }
292 }
293 }
294
295 static int __maybe_unused same_tt(struct usb_device *dev1,
296 struct usb_device *dev2)
297 {
298 if (!dev1->tt || !dev2->tt)
299 return 0;
300 if (dev1->tt != dev2->tt)
301 return 0;
302 if (dev1->tt->multi)
303 return dev1->ttport == dev2->ttport;
304 else
305 return 1;
306 }
307
308 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
309
310
311
312
313
314
315
316
317
318 static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
319 {
320 unsigned char smask = hc32_to_cpu(ehci, mask) & QH_SMASK;
321
322 if (!smask) {
323 ehci_err(ehci, "invalid empty smask!\n");
324
325 return 7;
326 }
327 return ffs(smask) - 1;
328 }
329
330 static const unsigned char
331 max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
332
333
334 static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
335 {
336 int i;
337
338 for (i = 0; i < 7; i++) {
339 if (max_tt_usecs[i] < tt_usecs[i]) {
340 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
341 tt_usecs[i] = max_tt_usecs[i];
342 }
343 }
344 }
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367 static int tt_available(
368 struct ehci_hcd *ehci,
369 struct ehci_per_sched *ps,
370 struct ehci_tt *tt,
371 unsigned frame,
372 unsigned uframe
373 )
374 {
375 unsigned period = ps->bw_period;
376 unsigned usecs = ps->tt_usecs;
377
378 if ((period == 0) || (uframe >= 7))
379 return 0;
380
381 for (frame &= period - 1; frame < EHCI_BANDWIDTH_FRAMES;
382 frame += period) {
383 unsigned i, uf;
384 unsigned short tt_usecs[8];
385
386 if (tt->bandwidth[frame] + usecs > 900)
387 return 0;
388
389 uf = frame << 3;
390 for (i = 0; i < 8; (++i, ++uf))
391 tt_usecs[i] = ehci->tt_budget[uf];
392
393 if (max_tt_usecs[uframe] <= tt_usecs[uframe])
394 return 0;
395
396
397
398
399
400
401 if (usecs > 125) {
402 int ufs = (usecs / 125);
403
404 for (i = uframe; i < (uframe + ufs) && i < 8; i++)
405 if (tt_usecs[i] > 0)
406 return 0;
407 }
408
409 tt_usecs[uframe] += usecs;
410
411 carryover_tt_bandwidth(tt_usecs);
412
413
414 if (max_tt_usecs[7] < tt_usecs[7])
415 return 0;
416 }
417
418 return 1;
419 }
420
421 #else
422
423
424
425
426
427 static int tt_no_collision(
428 struct ehci_hcd *ehci,
429 unsigned period,
430 struct usb_device *dev,
431 unsigned frame,
432 u32 uf_mask
433 )
434 {
435 if (period == 0)
436 return 0;
437
438
439
440
441
442 for (; frame < ehci->periodic_size; frame += period) {
443 union ehci_shadow here;
444 __hc32 type;
445 struct ehci_qh_hw *hw;
446
447 here = ehci->pshadow[frame];
448 type = Q_NEXT_TYPE(ehci, ehci->periodic[frame]);
449 while (here.ptr) {
450 switch (hc32_to_cpu(ehci, type)) {
451 case Q_TYPE_ITD:
452 type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
453 here = here.itd->itd_next;
454 continue;
455 case Q_TYPE_QH:
456 hw = here.qh->hw;
457 if (same_tt(dev, here.qh->ps.udev)) {
458 u32 mask;
459
460 mask = hc32_to_cpu(ehci,
461 hw->hw_info2);
462
463 mask |= mask >> 8;
464 if (mask & uf_mask)
465 break;
466 }
467 type = Q_NEXT_TYPE(ehci, hw->hw_next);
468 here = here.qh->qh_next;
469 continue;
470 case Q_TYPE_SITD:
471 if (same_tt(dev, here.sitd->urb->dev)) {
472 u16 mask;
473
474 mask = hc32_to_cpu(ehci, here.sitd
475 ->hw_uframe);
476
477 mask |= mask >> 8;
478 if (mask & uf_mask)
479 break;
480 }
481 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
482 here = here.sitd->sitd_next;
483 continue;
484
485 default:
486 ehci_dbg(ehci,
487 "periodic frame %d bogus type %d\n",
488 frame, type);
489 }
490
491
492 return 0;
493 }
494 }
495
496
497 return 1;
498 }
499
500 #endif
501
502
503
504 static void enable_periodic(struct ehci_hcd *ehci)
505 {
506 if (ehci->periodic_count++)
507 return;
508
509
510 ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_PERIODIC);
511
512
513 ehci_poll_PSS(ehci);
514 turn_on_io_watchdog(ehci);
515 }
516
517 static void disable_periodic(struct ehci_hcd *ehci)
518 {
519 if (--ehci->periodic_count)
520 return;
521
522
523 ehci_poll_PSS(ehci);
524 }
525
526
527
528
529
530
531
532
533
534 static void qh_link_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
535 {
536 unsigned i;
537 unsigned period = qh->ps.period;
538
539 dev_dbg(&qh->ps.udev->dev,
540 "link qh%d-%04x/%p start %d [%d/%d us]\n",
541 period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
542 & (QH_CMASK | QH_SMASK),
543 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs);
544
545
546 if (period == 0)
547 period = 1;
548
549 for (i = qh->ps.phase; i < ehci->periodic_size; i += period) {
550 union ehci_shadow *prev = &ehci->pshadow[i];
551 __hc32 *hw_p = &ehci->periodic[i];
552 union ehci_shadow here = *prev;
553 __hc32 type = 0;
554
555
556 while (here.ptr) {
557 type = Q_NEXT_TYPE(ehci, *hw_p);
558 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
559 break;
560 prev = periodic_next_shadow(ehci, prev, type);
561 hw_p = shadow_next_periodic(ehci, &here, type);
562 here = *prev;
563 }
564
565
566
567
568 while (here.ptr && qh != here.qh) {
569 if (qh->ps.period > here.qh->ps.period)
570 break;
571 prev = &here.qh->qh_next;
572 hw_p = &here.qh->hw->hw_next;
573 here = *prev;
574 }
575
576 if (qh != here.qh) {
577 qh->qh_next = here;
578 if (here.qh)
579 qh->hw->hw_next = *hw_p;
580 wmb();
581 prev->qh = qh;
582 *hw_p = QH_NEXT(ehci, qh->qh_dma);
583 }
584 }
585 qh->qh_state = QH_STATE_LINKED;
586 qh->xacterrs = 0;
587 qh->unlink_reason = 0;
588
589
590 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->ps.bw_period
591 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period)
592 : (qh->ps.usecs * 8);
593
594 list_add(&qh->intr_node, &ehci->intr_qh_list);
595
596
597 ++ehci->intr_count;
598 enable_periodic(ehci);
599 }
600
601 static void qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
602 {
603 unsigned i;
604 unsigned period;
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622 period = qh->ps.period ? : 1;
623
624 for (i = qh->ps.phase; i < ehci->periodic_size; i += period)
625 periodic_unlink(ehci, i, qh);
626
627
628 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->ps.bw_period
629 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period)
630 : (qh->ps.usecs * 8);
631
632 dev_dbg(&qh->ps.udev->dev,
633 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
634 qh->ps.period,
635 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
636 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs);
637
638
639 qh->qh_state = QH_STATE_UNLINK;
640 qh->qh_next.ptr = NULL;
641
642 if (ehci->qh_scan_next == qh)
643 ehci->qh_scan_next = list_entry(qh->intr_node.next,
644 struct ehci_qh, intr_node);
645 list_del(&qh->intr_node);
646 }
647
648 static void cancel_unlink_wait_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
649 {
650 if (qh->qh_state != QH_STATE_LINKED ||
651 list_empty(&qh->unlink_node))
652 return;
653
654 list_del_init(&qh->unlink_node);
655
656
657
658
659
660 }
661
662 static void start_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
663 {
664
665 if (qh->qh_state != QH_STATE_LINKED)
666 return;
667
668
669 cancel_unlink_wait_intr(ehci, qh);
670
671 qh_unlink_periodic(ehci, qh);
672
673
674 wmb();
675
676
677
678
679
680
681 qh->unlink_cycle = ehci->intr_unlink_cycle;
682
683
684 list_add_tail(&qh->unlink_node, &ehci->intr_unlink);
685
686 if (ehci->intr_unlinking)
687 ;
688 else if (ehci->rh_state < EHCI_RH_RUNNING)
689 ehci_handle_intr_unlinks(ehci);
690 else if (ehci->intr_unlink.next == &qh->unlink_node) {
691 ehci_enable_event(ehci, EHCI_HRTIMER_UNLINK_INTR, true);
692 ++ehci->intr_unlink_cycle;
693 }
694 }
695
696
697
698
699
700
701 static void start_unlink_intr_wait(struct ehci_hcd *ehci,
702 struct ehci_qh *qh)
703 {
704 qh->unlink_cycle = ehci->intr_unlink_wait_cycle;
705
706
707 list_add_tail(&qh->unlink_node, &ehci->intr_unlink_wait);
708
709 if (ehci->rh_state < EHCI_RH_RUNNING)
710 ehci_handle_start_intr_unlinks(ehci);
711 else if (ehci->intr_unlink_wait.next == &qh->unlink_node) {
712 ehci_enable_event(ehci, EHCI_HRTIMER_START_UNLINK_INTR, true);
713 ++ehci->intr_unlink_wait_cycle;
714 }
715 }
716
717 static void end_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
718 {
719 struct ehci_qh_hw *hw = qh->hw;
720 int rc;
721
722 qh->qh_state = QH_STATE_IDLE;
723 hw->hw_next = EHCI_LIST_END(ehci);
724
725 if (!list_empty(&qh->qtd_list))
726 qh_completions(ehci, qh);
727
728
729 if (!list_empty(&qh->qtd_list) && ehci->rh_state == EHCI_RH_RUNNING) {
730 rc = qh_schedule(ehci, qh);
731 if (rc == 0) {
732 qh_refresh(ehci, qh);
733 qh_link_periodic(ehci, qh);
734 }
735
736
737
738
739
740
741
742 else {
743 ehci_err(ehci, "can't reschedule qh %p, err %d\n",
744 qh, rc);
745 }
746 }
747
748
749 --ehci->intr_count;
750 disable_periodic(ehci);
751 }
752
753
754
755 static int check_period(
756 struct ehci_hcd *ehci,
757 unsigned frame,
758 unsigned uframe,
759 unsigned uperiod,
760 unsigned usecs
761 ) {
762
763
764
765 if (uframe >= 8)
766 return 0;
767
768
769 usecs = ehci->uframe_periodic_max - usecs;
770
771 for (uframe += frame << 3; uframe < EHCI_BANDWIDTH_SIZE;
772 uframe += uperiod) {
773 if (ehci->bandwidth[uframe] > usecs)
774 return 0;
775 }
776
777
778 return 1;
779 }
780
781 static int check_intr_schedule(
782 struct ehci_hcd *ehci,
783 unsigned frame,
784 unsigned uframe,
785 struct ehci_qh *qh,
786 unsigned *c_maskp,
787 struct ehci_tt *tt
788 )
789 {
790 int retval = -ENOSPC;
791 u8 mask = 0;
792
793 if (qh->ps.c_usecs && uframe >= 6)
794 goto done;
795
796 if (!check_period(ehci, frame, uframe, qh->ps.bw_uperiod, qh->ps.usecs))
797 goto done;
798 if (!qh->ps.c_usecs) {
799 retval = 0;
800 *c_maskp = 0;
801 goto done;
802 }
803
804 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
805 if (tt_available(ehci, &qh->ps, tt, frame, uframe)) {
806 unsigned i;
807
808
809 for (i = uframe+2; i < 8 && i <= uframe+4; i++)
810 if (!check_period(ehci, frame, i,
811 qh->ps.bw_uperiod, qh->ps.c_usecs))
812 goto done;
813 else
814 mask |= 1 << i;
815
816 retval = 0;
817
818 *c_maskp = mask;
819 }
820 #else
821
822
823
824
825
826
827
828 mask = 0x03 << (uframe + qh->gap_uf);
829 *c_maskp = mask;
830
831 mask |= 1 << uframe;
832 if (tt_no_collision(ehci, qh->ps.bw_period, qh->ps.udev, frame, mask)) {
833 if (!check_period(ehci, frame, uframe + qh->gap_uf + 1,
834 qh->ps.bw_uperiod, qh->ps.c_usecs))
835 goto done;
836 if (!check_period(ehci, frame, uframe + qh->gap_uf,
837 qh->ps.bw_uperiod, qh->ps.c_usecs))
838 goto done;
839 retval = 0;
840 }
841 #endif
842 done:
843 return retval;
844 }
845
846
847
848
849 static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
850 {
851 int status = 0;
852 unsigned uframe;
853 unsigned c_mask;
854 struct ehci_qh_hw *hw = qh->hw;
855 struct ehci_tt *tt;
856
857 hw->hw_next = EHCI_LIST_END(ehci);
858
859
860 if (qh->ps.phase != NO_FRAME) {
861 ehci_dbg(ehci, "reused qh %p schedule\n", qh);
862 return 0;
863 }
864
865 uframe = 0;
866 c_mask = 0;
867 tt = find_tt(qh->ps.udev);
868 if (IS_ERR(tt)) {
869 status = PTR_ERR(tt);
870 goto done;
871 }
872 compute_tt_budget(ehci->tt_budget, tt);
873
874
875
876
877
878 if (qh->ps.bw_period) {
879 int i;
880 unsigned frame;
881
882 for (i = qh->ps.bw_period; i > 0; --i) {
883 frame = ++ehci->random_frame & (qh->ps.bw_period - 1);
884 for (uframe = 0; uframe < 8; uframe++) {
885 status = check_intr_schedule(ehci,
886 frame, uframe, qh, &c_mask, tt);
887 if (status == 0)
888 goto got_it;
889 }
890 }
891
892
893 } else {
894 status = check_intr_schedule(ehci, 0, 0, qh, &c_mask, tt);
895 }
896 if (status)
897 goto done;
898
899 got_it:
900 qh->ps.phase = (qh->ps.period ? ehci->random_frame &
901 (qh->ps.period - 1) : 0);
902 qh->ps.bw_phase = qh->ps.phase & (qh->ps.bw_period - 1);
903 qh->ps.phase_uf = uframe;
904 qh->ps.cs_mask = qh->ps.period ?
905 (c_mask << 8) | (1 << uframe) :
906 QH_SMASK;
907
908
909 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
910 hw->hw_info2 |= cpu_to_hc32(ehci, qh->ps.cs_mask);
911 reserve_release_intr_bandwidth(ehci, qh, 1);
912
913 done:
914 return status;
915 }
916
917 static int intr_submit(
918 struct ehci_hcd *ehci,
919 struct urb *urb,
920 struct list_head *qtd_list,
921 gfp_t mem_flags
922 ) {
923 unsigned epnum;
924 unsigned long flags;
925 struct ehci_qh *qh;
926 int status;
927 struct list_head empty;
928
929
930 epnum = urb->ep->desc.bEndpointAddress;
931
932 spin_lock_irqsave(&ehci->lock, flags);
933
934 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
935 status = -ESHUTDOWN;
936 goto done_not_linked;
937 }
938 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
939 if (unlikely(status))
940 goto done_not_linked;
941
942
943 INIT_LIST_HEAD(&empty);
944 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
945 if (qh == NULL) {
946 status = -ENOMEM;
947 goto done;
948 }
949 if (qh->qh_state == QH_STATE_IDLE) {
950 status = qh_schedule(ehci, qh);
951 if (status)
952 goto done;
953 }
954
955
956 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
957 BUG_ON(qh == NULL);
958
959
960 if (qh->qh_state == QH_STATE_IDLE) {
961 qh_refresh(ehci, qh);
962 qh_link_periodic(ehci, qh);
963 } else {
964
965 cancel_unlink_wait_intr(ehci, qh);
966 }
967
968
969 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
970
971 done:
972 if (unlikely(status))
973 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
974 done_not_linked:
975 spin_unlock_irqrestore(&ehci->lock, flags);
976 if (status)
977 qtd_list_free(ehci, urb, qtd_list);
978
979 return status;
980 }
981
982 static void scan_intr(struct ehci_hcd *ehci)
983 {
984 struct ehci_qh *qh;
985
986 list_for_each_entry_safe(qh, ehci->qh_scan_next, &ehci->intr_qh_list,
987 intr_node) {
988
989
990 if (!list_empty(&qh->qtd_list)) {
991 int temp;
992
993
994
995
996
997
998
999
1000 temp = qh_completions(ehci, qh);
1001 if (unlikely(temp))
1002 start_unlink_intr(ehci, qh);
1003 else if (unlikely(list_empty(&qh->qtd_list) &&
1004 qh->qh_state == QH_STATE_LINKED))
1005 start_unlink_intr_wait(ehci, qh);
1006 }
1007 }
1008 }
1009
1010
1011
1012
1013
1014 static struct ehci_iso_stream *
1015 iso_stream_alloc(gfp_t mem_flags)
1016 {
1017 struct ehci_iso_stream *stream;
1018
1019 stream = kzalloc(sizeof(*stream), mem_flags);
1020 if (likely(stream != NULL)) {
1021 INIT_LIST_HEAD(&stream->td_list);
1022 INIT_LIST_HEAD(&stream->free_list);
1023 stream->next_uframe = NO_FRAME;
1024 stream->ps.phase = NO_FRAME;
1025 }
1026 return stream;
1027 }
1028
1029 static void
1030 iso_stream_init(
1031 struct ehci_hcd *ehci,
1032 struct ehci_iso_stream *stream,
1033 struct urb *urb
1034 )
1035 {
1036 static const u8 smask_out[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
1037
1038 struct usb_device *dev = urb->dev;
1039 u32 buf1;
1040 unsigned epnum, maxp;
1041 int is_input;
1042 unsigned tmp;
1043
1044
1045
1046
1047
1048 epnum = usb_pipeendpoint(urb->pipe);
1049 is_input = usb_pipein(urb->pipe) ? USB_DIR_IN : 0;
1050 maxp = usb_endpoint_maxp(&urb->ep->desc);
1051 buf1 = is_input ? 1 << 11 : 0;
1052
1053
1054 if (dev->speed == USB_SPEED_HIGH) {
1055 unsigned multi = usb_endpoint_maxp_mult(&urb->ep->desc);
1056
1057 stream->highspeed = 1;
1058
1059 buf1 |= maxp;
1060 maxp *= multi;
1061
1062 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
1063 stream->buf1 = cpu_to_hc32(ehci, buf1);
1064 stream->buf2 = cpu_to_hc32(ehci, multi);
1065
1066
1067
1068
1069 stream->ps.usecs = HS_USECS_ISO(maxp);
1070
1071
1072 tmp = min_t(unsigned, EHCI_BANDWIDTH_SIZE,
1073 1 << (urb->ep->desc.bInterval - 1));
1074
1075
1076 stream->ps.bw_uperiod = min_t(unsigned, tmp, urb->interval);
1077
1078 stream->uperiod = urb->interval;
1079 stream->ps.period = urb->interval >> 3;
1080 stream->bandwidth = stream->ps.usecs * 8 /
1081 stream->ps.bw_uperiod;
1082
1083 } else {
1084 u32 addr;
1085 int think_time;
1086 int hs_transfers;
1087
1088 addr = dev->ttport << 24;
1089 if (!ehci_is_TDI(ehci)
1090 || (dev->tt->hub !=
1091 ehci_to_hcd(ehci)->self.root_hub))
1092 addr |= dev->tt->hub->devnum << 16;
1093 addr |= epnum << 8;
1094 addr |= dev->devnum;
1095 stream->ps.usecs = HS_USECS_ISO(maxp);
1096 think_time = dev->tt->think_time;
1097 stream->ps.tt_usecs = NS_TO_US(think_time + usb_calc_bus_time(
1098 dev->speed, is_input, 1, maxp));
1099 hs_transfers = max(1u, (maxp + 187) / 188);
1100 if (is_input) {
1101 u32 tmp;
1102
1103 addr |= 1 << 31;
1104 stream->ps.c_usecs = stream->ps.usecs;
1105 stream->ps.usecs = HS_USECS_ISO(1);
1106 stream->ps.cs_mask = 1;
1107
1108
1109 tmp = (1 << (hs_transfers + 2)) - 1;
1110 stream->ps.cs_mask |= tmp << (8 + 2);
1111 } else
1112 stream->ps.cs_mask = smask_out[hs_transfers - 1];
1113
1114
1115 tmp = min_t(unsigned, EHCI_BANDWIDTH_FRAMES,
1116 1 << (urb->ep->desc.bInterval - 1));
1117
1118
1119 stream->ps.bw_period = min_t(unsigned, tmp, urb->interval);
1120 stream->ps.bw_uperiod = stream->ps.bw_period << 3;
1121
1122 stream->ps.period = urb->interval;
1123 stream->uperiod = urb->interval << 3;
1124 stream->bandwidth = (stream->ps.usecs + stream->ps.c_usecs) /
1125 stream->ps.bw_period;
1126
1127
1128 stream->address = cpu_to_hc32(ehci, addr);
1129 }
1130
1131 stream->ps.udev = dev;
1132 stream->ps.ep = urb->ep;
1133
1134 stream->bEndpointAddress = is_input | epnum;
1135 stream->maxp = maxp;
1136 }
1137
1138 static struct ehci_iso_stream *
1139 iso_stream_find(struct ehci_hcd *ehci, struct urb *urb)
1140 {
1141 unsigned epnum;
1142 struct ehci_iso_stream *stream;
1143 struct usb_host_endpoint *ep;
1144 unsigned long flags;
1145
1146 epnum = usb_pipeendpoint (urb->pipe);
1147 if (usb_pipein(urb->pipe))
1148 ep = urb->dev->ep_in[epnum];
1149 else
1150 ep = urb->dev->ep_out[epnum];
1151
1152 spin_lock_irqsave(&ehci->lock, flags);
1153 stream = ep->hcpriv;
1154
1155 if (unlikely(stream == NULL)) {
1156 stream = iso_stream_alloc(GFP_ATOMIC);
1157 if (likely(stream != NULL)) {
1158 ep->hcpriv = stream;
1159 iso_stream_init(ehci, stream, urb);
1160 }
1161
1162
1163 } else if (unlikely(stream->hw != NULL)) {
1164 ehci_dbg(ehci, "dev %s ep%d%s, not iso??\n",
1165 urb->dev->devpath, epnum,
1166 usb_pipein(urb->pipe) ? "in" : "out");
1167 stream = NULL;
1168 }
1169
1170 spin_unlock_irqrestore(&ehci->lock, flags);
1171 return stream;
1172 }
1173
1174
1175
1176
1177
1178 static struct ehci_iso_sched *
1179 iso_sched_alloc(unsigned packets, gfp_t mem_flags)
1180 {
1181 struct ehci_iso_sched *iso_sched;
1182 int size = sizeof(*iso_sched);
1183
1184 size += packets * sizeof(struct ehci_iso_packet);
1185 iso_sched = kzalloc(size, mem_flags);
1186 if (likely(iso_sched != NULL))
1187 INIT_LIST_HEAD(&iso_sched->td_list);
1188
1189 return iso_sched;
1190 }
1191
1192 static inline void
1193 itd_sched_init(
1194 struct ehci_hcd *ehci,
1195 struct ehci_iso_sched *iso_sched,
1196 struct ehci_iso_stream *stream,
1197 struct urb *urb
1198 )
1199 {
1200 unsigned i;
1201 dma_addr_t dma = urb->transfer_dma;
1202
1203
1204 iso_sched->span = urb->number_of_packets * stream->uperiod;
1205
1206
1207
1208
1209 for (i = 0; i < urb->number_of_packets; i++) {
1210 struct ehci_iso_packet *uframe = &iso_sched->packet[i];
1211 unsigned length;
1212 dma_addr_t buf;
1213 u32 trans;
1214
1215 length = urb->iso_frame_desc[i].length;
1216 buf = dma + urb->iso_frame_desc[i].offset;
1217
1218 trans = EHCI_ISOC_ACTIVE;
1219 trans |= buf & 0x0fff;
1220 if (unlikely(((i + 1) == urb->number_of_packets))
1221 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1222 trans |= EHCI_ITD_IOC;
1223 trans |= length << 16;
1224 uframe->transaction = cpu_to_hc32(ehci, trans);
1225
1226
1227 uframe->bufp = (buf & ~(u64)0x0fff);
1228 buf += length;
1229 if (unlikely((uframe->bufp != (buf & ~(u64)0x0fff))))
1230 uframe->cross = 1;
1231 }
1232 }
1233
1234 static void
1235 iso_sched_free(
1236 struct ehci_iso_stream *stream,
1237 struct ehci_iso_sched *iso_sched
1238 )
1239 {
1240 if (!iso_sched)
1241 return;
1242
1243 list_splice(&iso_sched->td_list, &stream->free_list);
1244 kfree(iso_sched);
1245 }
1246
1247 static int
1248 itd_urb_transaction(
1249 struct ehci_iso_stream *stream,
1250 struct ehci_hcd *ehci,
1251 struct urb *urb,
1252 gfp_t mem_flags
1253 )
1254 {
1255 struct ehci_itd *itd;
1256 dma_addr_t itd_dma;
1257 int i;
1258 unsigned num_itds;
1259 struct ehci_iso_sched *sched;
1260 unsigned long flags;
1261
1262 sched = iso_sched_alloc(urb->number_of_packets, mem_flags);
1263 if (unlikely(sched == NULL))
1264 return -ENOMEM;
1265
1266 itd_sched_init(ehci, sched, stream, urb);
1267
1268 if (urb->interval < 8)
1269 num_itds = 1 + (sched->span + 7) / 8;
1270 else
1271 num_itds = urb->number_of_packets;
1272
1273
1274 spin_lock_irqsave(&ehci->lock, flags);
1275 for (i = 0; i < num_itds; i++) {
1276
1277
1278
1279
1280
1281 if (likely(!list_empty(&stream->free_list))) {
1282 itd = list_first_entry(&stream->free_list,
1283 struct ehci_itd, itd_list);
1284 if (itd->frame == ehci->now_frame)
1285 goto alloc_itd;
1286 list_del(&itd->itd_list);
1287 itd_dma = itd->itd_dma;
1288 } else {
1289 alloc_itd:
1290 spin_unlock_irqrestore(&ehci->lock, flags);
1291 itd = dma_pool_alloc(ehci->itd_pool, mem_flags,
1292 &itd_dma);
1293 spin_lock_irqsave(&ehci->lock, flags);
1294 if (!itd) {
1295 iso_sched_free(stream, sched);
1296 spin_unlock_irqrestore(&ehci->lock, flags);
1297 return -ENOMEM;
1298 }
1299 }
1300
1301 memset(itd, 0, sizeof(*itd));
1302 itd->itd_dma = itd_dma;
1303 itd->frame = NO_FRAME;
1304 list_add(&itd->itd_list, &sched->td_list);
1305 }
1306 spin_unlock_irqrestore(&ehci->lock, flags);
1307
1308
1309 urb->hcpriv = sched;
1310 urb->error_count = 0;
1311 return 0;
1312 }
1313
1314
1315
1316 static void reserve_release_iso_bandwidth(struct ehci_hcd *ehci,
1317 struct ehci_iso_stream *stream, int sign)
1318 {
1319 unsigned uframe;
1320 unsigned i, j;
1321 unsigned s_mask, c_mask, m;
1322 int usecs = stream->ps.usecs;
1323 int c_usecs = stream->ps.c_usecs;
1324 int tt_usecs = stream->ps.tt_usecs;
1325 struct ehci_tt *tt;
1326
1327 if (stream->ps.phase == NO_FRAME)
1328 return;
1329 uframe = stream->ps.bw_phase << 3;
1330
1331 bandwidth_dbg(ehci, sign, "iso", &stream->ps);
1332
1333 if (sign < 0) {
1334 usecs = -usecs;
1335 c_usecs = -c_usecs;
1336 tt_usecs = -tt_usecs;
1337 }
1338
1339 if (!stream->splits) {
1340 for (i = uframe + stream->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE;
1341 i += stream->ps.bw_uperiod)
1342 ehci->bandwidth[i] += usecs;
1343
1344 } else {
1345 s_mask = stream->ps.cs_mask;
1346 c_mask = s_mask >> 8;
1347
1348
1349 for (i = uframe; i < EHCI_BANDWIDTH_SIZE;
1350 i += stream->ps.bw_uperiod) {
1351 for ((j = stream->ps.phase_uf, m = 1 << j); j < 8;
1352 (++j, m <<= 1)) {
1353 if (s_mask & m)
1354 ehci->bandwidth[i+j] += usecs;
1355 else if (c_mask & m)
1356 ehci->bandwidth[i+j] += c_usecs;
1357 }
1358 }
1359
1360 tt = find_tt(stream->ps.udev);
1361 if (sign > 0)
1362 list_add_tail(&stream->ps.ps_list, &tt->ps_list);
1363 else
1364 list_del(&stream->ps.ps_list);
1365
1366 for (i = uframe >> 3; i < EHCI_BANDWIDTH_FRAMES;
1367 i += stream->ps.bw_period)
1368 tt->bandwidth[i] += tt_usecs;
1369 }
1370 }
1371
1372 static inline int
1373 itd_slot_ok(
1374 struct ehci_hcd *ehci,
1375 struct ehci_iso_stream *stream,
1376 unsigned uframe
1377 )
1378 {
1379 unsigned usecs;
1380
1381
1382 usecs = ehci->uframe_periodic_max - stream->ps.usecs;
1383
1384 for (uframe &= stream->ps.bw_uperiod - 1; uframe < EHCI_BANDWIDTH_SIZE;
1385 uframe += stream->ps.bw_uperiod) {
1386 if (ehci->bandwidth[uframe] > usecs)
1387 return 0;
1388 }
1389 return 1;
1390 }
1391
1392 static inline int
1393 sitd_slot_ok(
1394 struct ehci_hcd *ehci,
1395 struct ehci_iso_stream *stream,
1396 unsigned uframe,
1397 struct ehci_iso_sched *sched,
1398 struct ehci_tt *tt
1399 )
1400 {
1401 unsigned mask, tmp;
1402 unsigned frame, uf;
1403
1404 mask = stream->ps.cs_mask << (uframe & 7);
1405
1406
1407 if (((stream->ps.cs_mask & 0xff) << (uframe & 7)) >= (1 << 7))
1408 return 0;
1409
1410
1411 if (mask & ~0xffff)
1412 return 0;
1413
1414
1415 uframe &= stream->ps.bw_uperiod - 1;
1416 frame = uframe >> 3;
1417
1418 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1419
1420
1421
1422 uf = uframe & 7;
1423 if (!tt_available(ehci, &stream->ps, tt, frame, uf))
1424 return 0;
1425 #else
1426
1427
1428
1429 if (!tt_no_collision(ehci, stream->ps.bw_period,
1430 stream->ps.udev, frame, mask))
1431 return 0;
1432 #endif
1433
1434 do {
1435 unsigned max_used;
1436 unsigned i;
1437
1438
1439 uf = uframe;
1440 max_used = ehci->uframe_periodic_max - stream->ps.usecs;
1441 for (tmp = stream->ps.cs_mask & 0xff; tmp; tmp >>= 1, uf++) {
1442 if (ehci->bandwidth[uf] > max_used)
1443 return 0;
1444 }
1445
1446
1447 if (stream->ps.c_usecs) {
1448 max_used = ehci->uframe_periodic_max -
1449 stream->ps.c_usecs;
1450 uf = uframe & ~7;
1451 tmp = 1 << (2+8);
1452 for (i = (uframe & 7) + 2; i < 8; (++i, tmp <<= 1)) {
1453 if ((stream->ps.cs_mask & tmp) == 0)
1454 continue;
1455 if (ehci->bandwidth[uf+i] > max_used)
1456 return 0;
1457 }
1458 }
1459
1460 uframe += stream->ps.bw_uperiod;
1461 } while (uframe < EHCI_BANDWIDTH_SIZE);
1462
1463 stream->ps.cs_mask <<= uframe & 7;
1464 stream->splits = cpu_to_hc32(ehci, stream->ps.cs_mask);
1465 return 1;
1466 }
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479 static int
1480 iso_stream_schedule(
1481 struct ehci_hcd *ehci,
1482 struct urb *urb,
1483 struct ehci_iso_stream *stream
1484 )
1485 {
1486 u32 now, base, next, start, period, span, now2;
1487 u32 wrap = 0, skip = 0;
1488 int status = 0;
1489 unsigned mod = ehci->periodic_size << 3;
1490 struct ehci_iso_sched *sched = urb->hcpriv;
1491 bool empty = list_empty(&stream->td_list);
1492 bool new_stream = false;
1493
1494 period = stream->uperiod;
1495 span = sched->span;
1496 if (!stream->highspeed)
1497 span <<= 3;
1498
1499
1500 if (unlikely(empty && !hcd_periodic_completion_in_progress(
1501 ehci_to_hcd(ehci), urb->ep))) {
1502
1503
1504 if (stream->ps.phase == NO_FRAME) {
1505 int done = 0;
1506 struct ehci_tt *tt = find_tt(stream->ps.udev);
1507
1508 if (IS_ERR(tt)) {
1509 status = PTR_ERR(tt);
1510 goto fail;
1511 }
1512 compute_tt_budget(ehci->tt_budget, tt);
1513
1514 start = ((-(++ehci->random_frame)) << 3) & (period - 1);
1515
1516
1517
1518
1519
1520
1521 next = start;
1522 start += period;
1523 do {
1524 start--;
1525
1526 if (stream->highspeed) {
1527 if (itd_slot_ok(ehci, stream, start))
1528 done = 1;
1529 } else {
1530 if ((start % 8) >= 6)
1531 continue;
1532 if (sitd_slot_ok(ehci, stream, start,
1533 sched, tt))
1534 done = 1;
1535 }
1536 } while (start > next && !done);
1537
1538
1539 if (!done) {
1540 ehci_dbg(ehci, "iso sched full %p", urb);
1541 status = -ENOSPC;
1542 goto fail;
1543 }
1544 stream->ps.phase = (start >> 3) &
1545 (stream->ps.period - 1);
1546 stream->ps.bw_phase = stream->ps.phase &
1547 (stream->ps.bw_period - 1);
1548 stream->ps.phase_uf = start & 7;
1549 reserve_release_iso_bandwidth(ehci, stream, 1);
1550 }
1551
1552
1553 else {
1554 start = (stream->ps.phase << 3) + stream->ps.phase_uf;
1555 }
1556
1557 stream->next_uframe = start;
1558 new_stream = true;
1559 }
1560
1561 now = ehci_read_frame_index(ehci) & (mod - 1);
1562
1563
1564 if (ehci->i_thresh)
1565 next = now + ehci->i_thresh;
1566 else
1567 next = (now + 2 + 7) & ~0x07;
1568
1569
1570 if (ehci->isoc_count == 0)
1571 ehci->last_iso_frame = now >> 3;
1572
1573
1574
1575
1576
1577 base = ehci->last_iso_frame << 3;
1578 next = (next - base) & (mod - 1);
1579 start = (stream->next_uframe - base) & (mod - 1);
1580
1581 if (unlikely(new_stream))
1582 goto do_ASAP;
1583
1584
1585
1586
1587
1588
1589
1590 now2 = (now - base) & (mod - 1);
1591
1592
1593 if (unlikely(!empty && start < period)) {
1594 ehci_dbg(ehci, "request %p would overflow (%u-%u < %u mod %u)\n",
1595 urb, stream->next_uframe, base, period, mod);
1596 status = -EFBIG;
1597 goto fail;
1598 }
1599
1600
1601 if (likely(!empty || start <= now2 + period)) {
1602
1603
1604 if (unlikely(start < next &&
1605 (urb->transfer_flags & URB_ISO_ASAP)))
1606 goto do_ASAP;
1607
1608
1609 if (likely(start >= now2))
1610 goto use_start;
1611
1612
1613 } else {
1614 if (urb->transfer_flags & URB_ISO_ASAP)
1615 goto do_ASAP;
1616 wrap = mod;
1617 now2 += mod;
1618 }
1619
1620
1621 skip = (now2 - start + period - 1) & -period;
1622 if (skip >= span) {
1623 ehci_dbg(ehci, "iso underrun %p (%u+%u < %u) [%u]\n",
1624 urb, start + base, span - period, now2 + base,
1625 base);
1626
1627
1628 skip = span - period;
1629
1630
1631 if (empty) {
1632 skip = span;
1633 status = 1;
1634 iso_sched_free(stream, sched);
1635 sched = NULL;
1636 }
1637 }
1638 urb->error_count = skip / period;
1639 if (sched)
1640 sched->first_packet = urb->error_count;
1641 goto use_start;
1642
1643 do_ASAP:
1644
1645 start = next + ((start - next) & (period - 1));
1646
1647 use_start:
1648
1649 if (unlikely(start + span - period >= mod + wrap)) {
1650 ehci_dbg(ehci, "request %p would overflow (%u+%u >= %u)\n",
1651 urb, start, span - period, mod + wrap);
1652 status = -EFBIG;
1653 goto fail;
1654 }
1655
1656 start += base;
1657 stream->next_uframe = (start + skip) & (mod - 1);
1658
1659
1660 urb->start_frame = start & (mod - 1);
1661 if (!stream->highspeed)
1662 urb->start_frame >>= 3;
1663 return status;
1664
1665 fail:
1666 iso_sched_free(stream, sched);
1667 urb->hcpriv = NULL;
1668 return status;
1669 }
1670
1671
1672
1673 static inline void
1674 itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1675 struct ehci_itd *itd)
1676 {
1677 int i;
1678
1679
1680 itd->hw_next = EHCI_LIST_END(ehci);
1681 itd->hw_bufp[0] = stream->buf0;
1682 itd->hw_bufp[1] = stream->buf1;
1683 itd->hw_bufp[2] = stream->buf2;
1684
1685 for (i = 0; i < 8; i++)
1686 itd->index[i] = -1;
1687
1688
1689 }
1690
1691 static inline void
1692 itd_patch(
1693 struct ehci_hcd *ehci,
1694 struct ehci_itd *itd,
1695 struct ehci_iso_sched *iso_sched,
1696 unsigned index,
1697 u16 uframe
1698 )
1699 {
1700 struct ehci_iso_packet *uf = &iso_sched->packet[index];
1701 unsigned pg = itd->pg;
1702
1703
1704
1705 uframe &= 0x07;
1706 itd->index[uframe] = index;
1707
1708 itd->hw_transaction[uframe] = uf->transaction;
1709 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1710 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1711 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1712
1713
1714 if (unlikely(uf->cross)) {
1715 u64 bufp = uf->bufp + 4096;
1716
1717 itd->pg = ++pg;
1718 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1719 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1720 }
1721 }
1722
1723 static inline void
1724 itd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1725 {
1726 union ehci_shadow *prev = &ehci->pshadow[frame];
1727 __hc32 *hw_p = &ehci->periodic[frame];
1728 union ehci_shadow here = *prev;
1729 __hc32 type = 0;
1730
1731
1732 while (here.ptr) {
1733 type = Q_NEXT_TYPE(ehci, *hw_p);
1734 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
1735 break;
1736 prev = periodic_next_shadow(ehci, prev, type);
1737 hw_p = shadow_next_periodic(ehci, &here, type);
1738 here = *prev;
1739 }
1740
1741 itd->itd_next = here;
1742 itd->hw_next = *hw_p;
1743 prev->itd = itd;
1744 itd->frame = frame;
1745 wmb();
1746 *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1747 }
1748
1749
1750 static void itd_link_urb(
1751 struct ehci_hcd *ehci,
1752 struct urb *urb,
1753 unsigned mod,
1754 struct ehci_iso_stream *stream
1755 )
1756 {
1757 int packet;
1758 unsigned next_uframe, uframe, frame;
1759 struct ehci_iso_sched *iso_sched = urb->hcpriv;
1760 struct ehci_itd *itd;
1761
1762 next_uframe = stream->next_uframe & (mod - 1);
1763
1764 if (unlikely(list_empty(&stream->td_list)))
1765 ehci_to_hcd(ehci)->self.bandwidth_allocated
1766 += stream->bandwidth;
1767
1768 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1769 if (ehci->amd_pll_fix == 1)
1770 usb_amd_quirk_pll_disable();
1771 }
1772
1773 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1774
1775
1776 for (packet = iso_sched->first_packet, itd = NULL;
1777 packet < urb->number_of_packets;) {
1778 if (itd == NULL) {
1779
1780
1781
1782
1783
1784 itd = list_entry(iso_sched->td_list.next,
1785 struct ehci_itd, itd_list);
1786 list_move_tail(&itd->itd_list, &stream->td_list);
1787 itd->stream = stream;
1788 itd->urb = urb;
1789 itd_init(ehci, stream, itd);
1790 }
1791
1792 uframe = next_uframe & 0x07;
1793 frame = next_uframe >> 3;
1794
1795 itd_patch(ehci, itd, iso_sched, packet, uframe);
1796
1797 next_uframe += stream->uperiod;
1798 next_uframe &= mod - 1;
1799 packet++;
1800
1801
1802 if (((next_uframe >> 3) != frame)
1803 || packet == urb->number_of_packets) {
1804 itd_link(ehci, frame & (ehci->periodic_size - 1), itd);
1805 itd = NULL;
1806 }
1807 }
1808 stream->next_uframe = next_uframe;
1809
1810
1811 iso_sched_free(stream, iso_sched);
1812 urb->hcpriv = stream;
1813
1814 ++ehci->isoc_count;
1815 enable_periodic(ehci);
1816 }
1817
1818 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830 static bool itd_complete(struct ehci_hcd *ehci, struct ehci_itd *itd)
1831 {
1832 struct urb *urb = itd->urb;
1833 struct usb_iso_packet_descriptor *desc;
1834 u32 t;
1835 unsigned uframe;
1836 int urb_index = -1;
1837 struct ehci_iso_stream *stream = itd->stream;
1838 bool retval = false;
1839
1840
1841 for (uframe = 0; uframe < 8; uframe++) {
1842 if (likely(itd->index[uframe] == -1))
1843 continue;
1844 urb_index = itd->index[uframe];
1845 desc = &urb->iso_frame_desc[urb_index];
1846
1847 t = hc32_to_cpup(ehci, &itd->hw_transaction[uframe]);
1848 itd->hw_transaction[uframe] = 0;
1849
1850
1851 if (unlikely(t & ISO_ERRS)) {
1852 urb->error_count++;
1853 if (t & EHCI_ISOC_BUF_ERR)
1854 desc->status = usb_pipein(urb->pipe)
1855 ? -ENOSR
1856 : -ECOMM;
1857 else if (t & EHCI_ISOC_BABBLE)
1858 desc->status = -EOVERFLOW;
1859 else
1860 desc->status = -EPROTO;
1861
1862
1863 if (!(t & EHCI_ISOC_BABBLE)) {
1864 desc->actual_length = EHCI_ITD_LENGTH(t);
1865 urb->actual_length += desc->actual_length;
1866 }
1867 } else if (likely((t & EHCI_ISOC_ACTIVE) == 0)) {
1868 desc->status = 0;
1869 desc->actual_length = EHCI_ITD_LENGTH(t);
1870 urb->actual_length += desc->actual_length;
1871 } else {
1872
1873 urb->error_count++;
1874 }
1875 }
1876
1877
1878 if (likely((urb_index + 1) != urb->number_of_packets))
1879 goto done;
1880
1881
1882
1883
1884
1885
1886
1887
1888 ehci_urb_done(ehci, urb, 0);
1889 retval = true;
1890 urb = NULL;
1891
1892 --ehci->isoc_count;
1893 disable_periodic(ehci);
1894
1895 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1896 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1897 if (ehci->amd_pll_fix == 1)
1898 usb_amd_quirk_pll_enable();
1899 }
1900
1901 if (unlikely(list_is_singular(&stream->td_list)))
1902 ehci_to_hcd(ehci)->self.bandwidth_allocated
1903 -= stream->bandwidth;
1904
1905 done:
1906 itd->urb = NULL;
1907
1908
1909 list_move_tail(&itd->itd_list, &stream->free_list);
1910
1911
1912 if (list_empty(&stream->td_list)) {
1913 list_splice_tail_init(&stream->free_list,
1914 &ehci->cached_itd_list);
1915 start_free_itds(ehci);
1916 }
1917
1918 return retval;
1919 }
1920
1921
1922
1923 static int itd_submit(struct ehci_hcd *ehci, struct urb *urb,
1924 gfp_t mem_flags)
1925 {
1926 int status = -EINVAL;
1927 unsigned long flags;
1928 struct ehci_iso_stream *stream;
1929
1930
1931 stream = iso_stream_find(ehci, urb);
1932 if (unlikely(stream == NULL)) {
1933 ehci_dbg(ehci, "can't get iso stream\n");
1934 return -ENOMEM;
1935 }
1936 if (unlikely(urb->interval != stream->uperiod)) {
1937 ehci_dbg(ehci, "can't change iso interval %d --> %d\n",
1938 stream->uperiod, urb->interval);
1939 goto done;
1940 }
1941
1942 #ifdef EHCI_URB_TRACE
1943 ehci_dbg(ehci,
1944 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1945 __func__, urb->dev->devpath, urb,
1946 usb_pipeendpoint(urb->pipe),
1947 usb_pipein(urb->pipe) ? "in" : "out",
1948 urb->transfer_buffer_length,
1949 urb->number_of_packets, urb->interval,
1950 stream);
1951 #endif
1952
1953
1954 status = itd_urb_transaction(stream, ehci, urb, mem_flags);
1955 if (unlikely(status < 0)) {
1956 ehci_dbg(ehci, "can't init itds\n");
1957 goto done;
1958 }
1959
1960
1961 spin_lock_irqsave(&ehci->lock, flags);
1962 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1963 status = -ESHUTDOWN;
1964 goto done_not_linked;
1965 }
1966 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1967 if (unlikely(status))
1968 goto done_not_linked;
1969 status = iso_stream_schedule(ehci, urb, stream);
1970 if (likely(status == 0)) {
1971 itd_link_urb(ehci, urb, ehci->periodic_size << 3, stream);
1972 } else if (status > 0) {
1973 status = 0;
1974 ehci_urb_done(ehci, urb, 0);
1975 } else {
1976 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1977 }
1978 done_not_linked:
1979 spin_unlock_irqrestore(&ehci->lock, flags);
1980 done:
1981 return status;
1982 }
1983
1984
1985
1986
1987
1988
1989
1990
1991 static inline void
1992 sitd_sched_init(
1993 struct ehci_hcd *ehci,
1994 struct ehci_iso_sched *iso_sched,
1995 struct ehci_iso_stream *stream,
1996 struct urb *urb
1997 )
1998 {
1999 unsigned i;
2000 dma_addr_t dma = urb->transfer_dma;
2001
2002
2003 iso_sched->span = urb->number_of_packets * stream->ps.period;
2004
2005
2006
2007
2008 for (i = 0; i < urb->number_of_packets; i++) {
2009 struct ehci_iso_packet *packet = &iso_sched->packet[i];
2010 unsigned length;
2011 dma_addr_t buf;
2012 u32 trans;
2013
2014 length = urb->iso_frame_desc[i].length & 0x03ff;
2015 buf = dma + urb->iso_frame_desc[i].offset;
2016
2017 trans = SITD_STS_ACTIVE;
2018 if (((i + 1) == urb->number_of_packets)
2019 && !(urb->transfer_flags & URB_NO_INTERRUPT))
2020 trans |= SITD_IOC;
2021 trans |= length << 16;
2022 packet->transaction = cpu_to_hc32(ehci, trans);
2023
2024
2025 packet->bufp = buf;
2026 packet->buf1 = (buf + length) & ~0x0fff;
2027 if (packet->buf1 != (buf & ~(u64)0x0fff))
2028 packet->cross = 1;
2029
2030
2031 if (stream->bEndpointAddress & USB_DIR_IN)
2032 continue;
2033 length = (length + 187) / 188;
2034 if (length > 1)
2035 length |= 1 << 3;
2036 packet->buf1 |= length;
2037 }
2038 }
2039
2040 static int
2041 sitd_urb_transaction(
2042 struct ehci_iso_stream *stream,
2043 struct ehci_hcd *ehci,
2044 struct urb *urb,
2045 gfp_t mem_flags
2046 )
2047 {
2048 struct ehci_sitd *sitd;
2049 dma_addr_t sitd_dma;
2050 int i;
2051 struct ehci_iso_sched *iso_sched;
2052 unsigned long flags;
2053
2054 iso_sched = iso_sched_alloc(urb->number_of_packets, mem_flags);
2055 if (iso_sched == NULL)
2056 return -ENOMEM;
2057
2058 sitd_sched_init(ehci, iso_sched, stream, urb);
2059
2060
2061 spin_lock_irqsave(&ehci->lock, flags);
2062 for (i = 0; i < urb->number_of_packets; i++) {
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073 if (likely(!list_empty(&stream->free_list))) {
2074 sitd = list_first_entry(&stream->free_list,
2075 struct ehci_sitd, sitd_list);
2076 if (sitd->frame == ehci->now_frame)
2077 goto alloc_sitd;
2078 list_del(&sitd->sitd_list);
2079 sitd_dma = sitd->sitd_dma;
2080 } else {
2081 alloc_sitd:
2082 spin_unlock_irqrestore(&ehci->lock, flags);
2083 sitd = dma_pool_alloc(ehci->sitd_pool, mem_flags,
2084 &sitd_dma);
2085 spin_lock_irqsave(&ehci->lock, flags);
2086 if (!sitd) {
2087 iso_sched_free(stream, iso_sched);
2088 spin_unlock_irqrestore(&ehci->lock, flags);
2089 return -ENOMEM;
2090 }
2091 }
2092
2093 memset(sitd, 0, sizeof(*sitd));
2094 sitd->sitd_dma = sitd_dma;
2095 sitd->frame = NO_FRAME;
2096 list_add(&sitd->sitd_list, &iso_sched->td_list);
2097 }
2098
2099
2100 urb->hcpriv = iso_sched;
2101 urb->error_count = 0;
2102
2103 spin_unlock_irqrestore(&ehci->lock, flags);
2104 return 0;
2105 }
2106
2107
2108
2109 static inline void
2110 sitd_patch(
2111 struct ehci_hcd *ehci,
2112 struct ehci_iso_stream *stream,
2113 struct ehci_sitd *sitd,
2114 struct ehci_iso_sched *iso_sched,
2115 unsigned index
2116 )
2117 {
2118 struct ehci_iso_packet *uf = &iso_sched->packet[index];
2119 u64 bufp;
2120
2121 sitd->hw_next = EHCI_LIST_END(ehci);
2122 sitd->hw_fullspeed_ep = stream->address;
2123 sitd->hw_uframe = stream->splits;
2124 sitd->hw_results = uf->transaction;
2125 sitd->hw_backpointer = EHCI_LIST_END(ehci);
2126
2127 bufp = uf->bufp;
2128 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
2129 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
2130
2131 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
2132 if (uf->cross)
2133 bufp += 4096;
2134 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
2135 sitd->index = index;
2136 }
2137
2138 static inline void
2139 sitd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
2140 {
2141
2142 sitd->sitd_next = ehci->pshadow[frame];
2143 sitd->hw_next = ehci->periodic[frame];
2144 ehci->pshadow[frame].sitd = sitd;
2145 sitd->frame = frame;
2146 wmb();
2147 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
2148 }
2149
2150
2151 static void sitd_link_urb(
2152 struct ehci_hcd *ehci,
2153 struct urb *urb,
2154 unsigned mod,
2155 struct ehci_iso_stream *stream
2156 )
2157 {
2158 int packet;
2159 unsigned next_uframe;
2160 struct ehci_iso_sched *sched = urb->hcpriv;
2161 struct ehci_sitd *sitd;
2162
2163 next_uframe = stream->next_uframe;
2164
2165 if (list_empty(&stream->td_list))
2166
2167 ehci_to_hcd(ehci)->self.bandwidth_allocated
2168 += stream->bandwidth;
2169
2170 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2171 if (ehci->amd_pll_fix == 1)
2172 usb_amd_quirk_pll_disable();
2173 }
2174
2175 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
2176
2177
2178 for (packet = sched->first_packet, sitd = NULL;
2179 packet < urb->number_of_packets;
2180 packet++) {
2181
2182
2183 BUG_ON(list_empty(&sched->td_list));
2184
2185
2186
2187 sitd = list_entry(sched->td_list.next,
2188 struct ehci_sitd, sitd_list);
2189 list_move_tail(&sitd->sitd_list, &stream->td_list);
2190 sitd->stream = stream;
2191 sitd->urb = urb;
2192
2193 sitd_patch(ehci, stream, sitd, sched, packet);
2194 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1),
2195 sitd);
2196
2197 next_uframe += stream->uperiod;
2198 }
2199 stream->next_uframe = next_uframe & (mod - 1);
2200
2201
2202 iso_sched_free(stream, sched);
2203 urb->hcpriv = stream;
2204
2205 ++ehci->isoc_count;
2206 enable_periodic(ehci);
2207 }
2208
2209
2210
2211 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2212 | SITD_STS_XACT | SITD_STS_MMF)
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224 static bool sitd_complete(struct ehci_hcd *ehci, struct ehci_sitd *sitd)
2225 {
2226 struct urb *urb = sitd->urb;
2227 struct usb_iso_packet_descriptor *desc;
2228 u32 t;
2229 int urb_index;
2230 struct ehci_iso_stream *stream = sitd->stream;
2231 bool retval = false;
2232
2233 urb_index = sitd->index;
2234 desc = &urb->iso_frame_desc[urb_index];
2235 t = hc32_to_cpup(ehci, &sitd->hw_results);
2236
2237
2238 if (unlikely(t & SITD_ERRS)) {
2239 urb->error_count++;
2240 if (t & SITD_STS_DBE)
2241 desc->status = usb_pipein(urb->pipe)
2242 ? -ENOSR
2243 : -ECOMM;
2244 else if (t & SITD_STS_BABBLE)
2245 desc->status = -EOVERFLOW;
2246 else
2247 desc->status = -EPROTO;
2248 } else if (unlikely(t & SITD_STS_ACTIVE)) {
2249
2250 urb->error_count++;
2251 } else {
2252 desc->status = 0;
2253 desc->actual_length = desc->length - SITD_LENGTH(t);
2254 urb->actual_length += desc->actual_length;
2255 }
2256
2257
2258 if ((urb_index + 1) != urb->number_of_packets)
2259 goto done;
2260
2261
2262
2263
2264
2265
2266
2267
2268 ehci_urb_done(ehci, urb, 0);
2269 retval = true;
2270 urb = NULL;
2271
2272 --ehci->isoc_count;
2273 disable_periodic(ehci);
2274
2275 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2276 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2277 if (ehci->amd_pll_fix == 1)
2278 usb_amd_quirk_pll_enable();
2279 }
2280
2281 if (list_is_singular(&stream->td_list))
2282 ehci_to_hcd(ehci)->self.bandwidth_allocated
2283 -= stream->bandwidth;
2284
2285 done:
2286 sitd->urb = NULL;
2287
2288
2289 list_move_tail(&sitd->sitd_list, &stream->free_list);
2290
2291
2292 if (list_empty(&stream->td_list)) {
2293 list_splice_tail_init(&stream->free_list,
2294 &ehci->cached_sitd_list);
2295 start_free_itds(ehci);
2296 }
2297
2298 return retval;
2299 }
2300
2301
2302 static int sitd_submit(struct ehci_hcd *ehci, struct urb *urb,
2303 gfp_t mem_flags)
2304 {
2305 int status = -EINVAL;
2306 unsigned long flags;
2307 struct ehci_iso_stream *stream;
2308
2309
2310 stream = iso_stream_find(ehci, urb);
2311 if (stream == NULL) {
2312 ehci_dbg(ehci, "can't get iso stream\n");
2313 return -ENOMEM;
2314 }
2315 if (urb->interval != stream->ps.period) {
2316 ehci_dbg(ehci, "can't change iso interval %d --> %d\n",
2317 stream->ps.period, urb->interval);
2318 goto done;
2319 }
2320
2321 #ifdef EHCI_URB_TRACE
2322 ehci_dbg(ehci,
2323 "submit %p dev%s ep%d%s-iso len %d\n",
2324 urb, urb->dev->devpath,
2325 usb_pipeendpoint(urb->pipe),
2326 usb_pipein(urb->pipe) ? "in" : "out",
2327 urb->transfer_buffer_length);
2328 #endif
2329
2330
2331 status = sitd_urb_transaction(stream, ehci, urb, mem_flags);
2332 if (status < 0) {
2333 ehci_dbg(ehci, "can't init sitds\n");
2334 goto done;
2335 }
2336
2337
2338 spin_lock_irqsave(&ehci->lock, flags);
2339 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
2340 status = -ESHUTDOWN;
2341 goto done_not_linked;
2342 }
2343 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2344 if (unlikely(status))
2345 goto done_not_linked;
2346 status = iso_stream_schedule(ehci, urb, stream);
2347 if (likely(status == 0)) {
2348 sitd_link_urb(ehci, urb, ehci->periodic_size << 3, stream);
2349 } else if (status > 0) {
2350 status = 0;
2351 ehci_urb_done(ehci, urb, 0);
2352 } else {
2353 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2354 }
2355 done_not_linked:
2356 spin_unlock_irqrestore(&ehci->lock, flags);
2357 done:
2358 return status;
2359 }
2360
2361
2362
2363 static void scan_isoc(struct ehci_hcd *ehci)
2364 {
2365 unsigned uf, now_frame, frame;
2366 unsigned fmask = ehci->periodic_size - 1;
2367 bool modified, live;
2368 union ehci_shadow q, *q_p;
2369 __hc32 type, *hw_p;
2370
2371
2372
2373
2374
2375
2376 if (ehci->rh_state >= EHCI_RH_RUNNING) {
2377 uf = ehci_read_frame_index(ehci);
2378 now_frame = (uf >> 3) & fmask;
2379 live = true;
2380 } else {
2381 now_frame = (ehci->last_iso_frame - 1) & fmask;
2382 live = false;
2383 }
2384 ehci->now_frame = now_frame;
2385
2386 frame = ehci->last_iso_frame;
2387
2388 restart:
2389
2390 q_p = &ehci->pshadow[frame];
2391 hw_p = &ehci->periodic[frame];
2392 q.ptr = q_p->ptr;
2393 type = Q_NEXT_TYPE(ehci, *hw_p);
2394 modified = false;
2395
2396 while (q.ptr != NULL) {
2397 switch (hc32_to_cpu(ehci, type)) {
2398 case Q_TYPE_ITD:
2399
2400
2401
2402
2403
2404
2405 if (frame == now_frame && live) {
2406 rmb();
2407 for (uf = 0; uf < 8; uf++) {
2408 if (q.itd->hw_transaction[uf] &
2409 ITD_ACTIVE(ehci))
2410 break;
2411 }
2412 if (uf < 8) {
2413 q_p = &q.itd->itd_next;
2414 hw_p = &q.itd->hw_next;
2415 type = Q_NEXT_TYPE(ehci,
2416 q.itd->hw_next);
2417 q = *q_p;
2418 break;
2419 }
2420 }
2421
2422
2423
2424
2425
2426
2427
2428 *q_p = q.itd->itd_next;
2429 if (!ehci->use_dummy_qh ||
2430 q.itd->hw_next != EHCI_LIST_END(ehci))
2431 *hw_p = q.itd->hw_next;
2432 else
2433 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
2434 type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2435 wmb();
2436 modified = itd_complete(ehci, q.itd);
2437 q = *q_p;
2438 break;
2439 case Q_TYPE_SITD:
2440
2441
2442
2443
2444
2445
2446 if (((frame == now_frame) ||
2447 (((frame + 1) & fmask) == now_frame))
2448 && live
2449 && (q.sitd->hw_results & SITD_ACTIVE(ehci))) {
2450
2451 q_p = &q.sitd->sitd_next;
2452 hw_p = &q.sitd->hw_next;
2453 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2454 q = *q_p;
2455 break;
2456 }
2457
2458
2459
2460
2461
2462
2463 *q_p = q.sitd->sitd_next;
2464 if (!ehci->use_dummy_qh ||
2465 q.sitd->hw_next != EHCI_LIST_END(ehci))
2466 *hw_p = q.sitd->hw_next;
2467 else
2468 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
2469 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2470 wmb();
2471 modified = sitd_complete(ehci, q.sitd);
2472 q = *q_p;
2473 break;
2474 default:
2475 ehci_dbg(ehci, "corrupt type %d frame %d shadow %p\n",
2476 type, frame, q.ptr);
2477
2478
2479 case Q_TYPE_QH:
2480 case Q_TYPE_FSTN:
2481
2482 q.ptr = NULL;
2483 break;
2484 }
2485
2486
2487 if (unlikely(modified && ehci->isoc_count > 0))
2488 goto restart;
2489 }
2490
2491
2492 if (frame == now_frame)
2493 return;
2494
2495
2496 ehci->last_iso_frame = frame;
2497 frame = (frame + 1) & fmask;
2498
2499 goto restart;
2500 }