root/drivers/usb/host/xhci-ring.c

DEFINITIONS

1. xhci_trb_virt_to_dma
2. trb_is_noop
3. trb_is_link
4. last_trb_on_seg
5. last_trb_on_ring
6. link_trb_toggles_cycle
7. last_td_in_urb
8. inc_td_cnt
9. trb_to_noop
10. next_trb
11. inc_deq
12. inc_enq
13. room_on_ring
14. xhci_ring_cmd_db
15. xhci_mod_cmd_timer
16. xhci_next_queued_cmd
17. xhci_handle_stopped_cmd_ring
18. xhci_abort_cmd_ring
19. xhci_ring_ep_doorbell
20. ring_doorbell_for_active_rings
21. xhci_ring_doorbell_for_active_rings
22. xhci_triad_to_transfer_ring
23. xhci_get_hw_deq
24. xhci_find_new_dequeue_state
25. td_to_noop
26. xhci_stop_watchdog_timer_in_irq
27. xhci_giveback_urb_in_irq
28. xhci_unmap_td_bounce_buffer
29. xhci_handle_cmd_stop_ep
30. xhci_kill_ring_urbs
31. xhci_kill_endpoint_urbs
32. xhci_hc_died
33. xhci_stop_endpoint_command_watchdog
34. update_ring_for_set_deq_completion
35. xhci_handle_cmd_set_deq
36. xhci_handle_cmd_reset_ep
37. xhci_handle_cmd_enable_slot
38. xhci_handle_cmd_disable_slot
39. xhci_handle_cmd_config_ep
40. xhci_handle_cmd_addr_dev
41. xhci_handle_cmd_reset_dev
42. xhci_handle_cmd_nec_get_fw
43. xhci_complete_del_and_free_cmd
44. xhci_cleanup_command_queue
45. xhci_handle_command_timeout
46. handle_cmd_completion
47. handle_vendor_event
48. handle_device_notification
49. xhci_cavium_reset_phy_quirk
50. handle_port_status
51. trb_in_td
52. xhci_clear_hub_tt_buffer
53. xhci_cleanup_halted_endpoint
54. xhci_requires_manual_halt_cleanup
55. xhci_is_vendor_info_code
56. xhci_td_cleanup
57. finish_td
58. sum_trb_lengths
59. process_ctrl_td
60. process_isoc_td
61. skip_isoc_td
62. process_bulk_intr_td
63. handle_tx_event
64. xhci_handle_event
65. xhci_update_erst_dequeue
66. xhci_irq
67. xhci_msi_irq
68. queue_trb
69. prepare_ring
70. prepare_transfer
71. count_trbs
72. count_trbs_needed
73. count_sg_trbs_needed
74. count_isoc_trbs_needed
75. check_trb_math
76. giveback_first_trb
77. check_interval
78. xhci_queue_intr_tx
79. xhci_td_remainder
80. xhci_align_td
81. xhci_queue_bulk_tx
82. xhci_queue_ctrl_tx
83. xhci_get_burst_count
84. xhci_get_last_burst_packet_count
85. xhci_get_isoc_frame_id
86. xhci_queue_isoc_tx
87. xhci_queue_isoc_tx_prepare
88. queue_command
89. xhci_queue_slot_control
90. xhci_queue_address_device
91. xhci_queue_vendor_command
92. xhci_queue_reset_device
93. xhci_queue_configure_endpoint
94. xhci_queue_evaluate_context
95. xhci_queue_stop_endpoint
96. xhci_queue_new_dequeue_state
97. xhci_queue_reset_ep

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * xHCI host controller driver
   4  *
   5  * Copyright (C) 2008 Intel Corp.
   6  *
   7  * Author: Sarah Sharp
   8  * Some code borrowed from the Linux EHCI driver.
   9  */
  10 
  11 /*
  12  * Ring initialization rules:
  13  * 1. Each segment is initialized to zero, except for link TRBs.
  14  * 2. Ring cycle state = 0.  This represents Producer Cycle State (PCS) or
  15  *    Consumer Cycle State (CCS), depending on ring function.
  16  * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment.
  17  *
  18  * Ring behavior rules:
  19  * 1. A ring is empty if enqueue == dequeue.  This means there will always be at
  20  *    least one free TRB in the ring.  This is useful if you want to turn that
  21  *    into a link TRB and expand the ring.
  22  * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a
  23  *    link TRB, then load the pointer with the address in the link TRB.  If the
  24  *    link TRB had its toggle bit set, you may need to update the ring cycle
  25  *    state (see cycle bit rules).  You may have to do this multiple times
  26  *    until you reach a non-link TRB.
  27  * 3. A ring is full if enqueue++ (for the definition of increment above)
  28  *    equals the dequeue pointer.
  29  *
  30  * Cycle bit rules:
  31  * 1. When a consumer increments a dequeue pointer and encounters a toggle bit
  32  *    in a link TRB, it must toggle the ring cycle state.
  33  * 2. When a producer increments an enqueue pointer and encounters a toggle bit
  34  *    in a link TRB, it must toggle the ring cycle state.
  35  *
  36  * Producer rules:
  37  * 1. Check if ring is full before you enqueue.
  38  * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing.
  39  *    Update enqueue pointer between each write (which may update the ring
  40  *    cycle state).
  41  * 3. Notify consumer.  If SW is producer, it rings the doorbell for command
  42  *    and endpoint rings.  If HC is the producer for the event ring,
  43  *    and it generates an interrupt according to interrupt modulation rules.
  44  *
  45  * Consumer rules:
  46  * 1. Check if TRB belongs to you.  If the cycle bit == your ring cycle state,
  47  *    the TRB is owned by the consumer.
  48  * 2. Update dequeue pointer (which may update the ring cycle state) and
  49  *    continue processing TRBs until you reach a TRB which is not owned by you.
  50  * 3. Notify the producer.  SW is the consumer for the event ring, and it
  51  *   updates event ring dequeue pointer.  HC is the consumer for the command and
  52  *   endpoint rings; it generates events on the event ring for these.
  53  */
  54 
  55 #include <linux/scatterlist.h>
  56 #include <linux/slab.h>
  57 #include <linux/dma-mapping.h>
  58 #include "xhci.h"
  59 #include "xhci-trace.h"
  60 #include "xhci-mtk.h"
  61 
  62 /*
  63  * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
  64  * address of the TRB.
  65  */
  66 dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg,
  67                 union xhci_trb *trb)
  68 {
  69         unsigned long segment_offset;
  70 
  71         if (!seg || !trb || trb < seg->trbs)
  72                 return 0;
  73         /* offset in TRBs */
  74         segment_offset = trb - seg->trbs;
  75         if (segment_offset >= TRBS_PER_SEGMENT)
  76                 return 0;
  77         return seg->dma + (segment_offset * sizeof(*trb));
  78 }
  79 
  80 static bool trb_is_noop(union xhci_trb *trb)
  81 {
  82         return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
  83 }
  84 
  85 static bool trb_is_link(union xhci_trb *trb)
  86 {
  87         return TRB_TYPE_LINK_LE32(trb->link.control);
  88 }
  89 
  90 static bool last_trb_on_seg(struct xhci_segment *seg, union xhci_trb *trb)
  91 {
  92         return trb == &seg->trbs[TRBS_PER_SEGMENT - 1];
  93 }
  94 
  95 static bool last_trb_on_ring(struct xhci_ring *ring,
  96                         struct xhci_segment *seg, union xhci_trb *trb)
  97 {
  98         return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg);
  99 }
 100 
 101 static bool link_trb_toggles_cycle(union xhci_trb *trb)
 102 {
 103         return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
 104 }
 105 
 106 static bool last_td_in_urb(struct xhci_td *td)
 107 {
 108         struct urb_priv *urb_priv = td->urb->hcpriv;
 109 
 110         return urb_priv->num_tds_done == urb_priv->num_tds;
 111 }
 112 
 113 static void inc_td_cnt(struct urb *urb)
 114 {
 115         struct urb_priv *urb_priv = urb->hcpriv;
 116 
 117         urb_priv->num_tds_done++;
 118 }
 119 
 120 static void trb_to_noop(union xhci_trb *trb, u32 noop_type)
 121 {
 122         if (trb_is_link(trb)) {
 123                 /* unchain chained link TRBs */
 124                 trb->link.control &= cpu_to_le32(~TRB_CHAIN);
 125         } else {
 126                 trb->generic.field[0] = 0;
 127                 trb->generic.field[1] = 0;
 128                 trb->generic.field[2] = 0;
 129                 /* Preserve only the cycle bit of this TRB */
 130                 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
 131                 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type));
 132         }
 133 }
 134 
 135 /* Updates trb to point to the next TRB in the ring, and updates seg if the next
 136  * TRB is in a new segment.  This does not skip over link TRBs, and it does not
 137  * effect the ring dequeue or enqueue pointers.
 138  */
 139 static void next_trb(struct xhci_hcd *xhci,
 140                 struct xhci_ring *ring,
 141                 struct xhci_segment **seg,
 142                 union xhci_trb **trb)
 143 {
 144         if (trb_is_link(*trb)) {
 145                 *seg = (*seg)->next;
 146                 *trb = ((*seg)->trbs);
 147         } else {
 148                 (*trb)++;
 149         }
 150 }
 151 
 152 /*
 153  * See Cycle bit rules. SW is the consumer for the event ring only.
 154  * Don't make a ring full of link TRBs.  That would be dumb and this would loop.
 155  */
 156 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring)
 157 {
 158         /* event ring doesn't have link trbs, check for last trb */
 159         if (ring->type == TYPE_EVENT) {
 160                 if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
 161                         ring->dequeue++;
 162                         goto out;
 163                 }
 164                 if (last_trb_on_ring(ring, ring->deq_seg, ring->dequeue))
 165                         ring->cycle_state ^= 1;
 166                 ring->deq_seg = ring->deq_seg->next;
 167                 ring->dequeue = ring->deq_seg->trbs;
 168                 goto out;
 169         }
 170 
 171         /* All other rings have link trbs */
 172         if (!trb_is_link(ring->dequeue)) {
 173                 ring->dequeue++;
 174                 ring->num_trbs_free++;
 175         }
 176         while (trb_is_link(ring->dequeue)) {
 177                 ring->deq_seg = ring->deq_seg->next;
 178                 ring->dequeue = ring->deq_seg->trbs;
 179         }
 180 
 181 out:
 182         trace_xhci_inc_deq(ring);
 183 
 184         return;
 185 }
 186 
 187 /*
 188  * See Cycle bit rules. SW is the consumer for the event ring only.
 189  * Don't make a ring full of link TRBs.  That would be dumb and this would loop.
 190  *
 191  * If we've just enqueued a TRB that is in the middle of a TD (meaning the
 192  * chain bit is set), then set the chain bit in all the following link TRBs.
 193  * If we've enqueued the last TRB in a TD, make sure the following link TRBs
 194  * have their chain bit cleared (so that each Link TRB is a separate TD).
 195  *
 196  * Section 6.4.4.1 of the 0.95 spec says link TRBs cannot have the chain bit
 197  * set, but other sections talk about dealing with the chain bit set.  This was
 198  * fixed in the 0.96 specification errata, but we have to assume that all 0.95
 199  * xHCI hardware can't handle the chain bit being cleared on a link TRB.
 200  *
 201  * @more_trbs_coming:   Will you enqueue more TRBs before calling
 202  *                      prepare_transfer()?
 203  */
 204 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
 205                         bool more_trbs_coming)
 206 {
 207         u32 chain;
 208         union xhci_trb *next;
 209 
 210         chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
 211         /* If this is not event ring, there is one less usable TRB */
 212         if (!trb_is_link(ring->enqueue))
 213                 ring->num_trbs_free--;
 214         next = ++(ring->enqueue);
 215 
 216         /* Update the dequeue pointer further if that was a link TRB */
 217         while (trb_is_link(next)) {
 218 
 219                 /*
 220                  * If the caller doesn't plan on enqueueing more TDs before
 221                  * ringing the doorbell, then we don't want to give the link TRB
 222                  * to the hardware just yet. We'll give the link TRB back in
 223                  * prepare_ring() just before we enqueue the TD at the top of
 224                  * the ring.
 225                  */
 226                 if (!chain && !more_trbs_coming)
 227                         break;
 228 
 229                 /* If we're not dealing with 0.95 hardware or isoc rings on
 230                  * AMD 0.96 host, carry over the chain bit of the previous TRB
 231                  * (which may mean the chain bit is cleared).
 232                  */
 233                 if (!(ring->type == TYPE_ISOC &&
 234                       (xhci->quirks & XHCI_AMD_0x96_HOST)) &&
 235                     !xhci_link_trb_quirk(xhci)) {
 236                         next->link.control &= cpu_to_le32(~TRB_CHAIN);
 237                         next->link.control |= cpu_to_le32(chain);
 238                 }
 239                 /* Give this link TRB to the hardware */
 240                 wmb();
 241                 next->link.control ^= cpu_to_le32(TRB_CYCLE);
 242 
 243                 /* Toggle the cycle bit after the last ring segment. */
 244                 if (link_trb_toggles_cycle(next))
 245                         ring->cycle_state ^= 1;
 246 
 247                 ring->enq_seg = ring->enq_seg->next;
 248                 ring->enqueue = ring->enq_seg->trbs;
 249                 next = ring->enqueue;
 250         }
 251 
 252         trace_xhci_inc_enq(ring);
 253 }
 254 
 255 /*
 256  * Check to see if there's room to enqueue num_trbs on the ring and make sure
 257  * enqueue pointer will not advance into dequeue segment. See rules above.
 258  */
 259 static inline int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring,
 260                 unsigned int num_trbs)
 261 {
 262         int num_trbs_in_deq_seg;
 263 
 264         if (ring->num_trbs_free < num_trbs)
 265                 return 0;
 266 
 267         if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) {
 268                 num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs;
 269                 if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg)
 270                         return 0;
 271         }
 272 
 273         return 1;
 274 }
 275 
 276 /* Ring the host controller doorbell after placing a command on the ring */
 277 void xhci_ring_cmd_db(struct xhci_hcd *xhci)
 278 {
 279         if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING))
 280                 return;
 281 
 282         xhci_dbg(xhci, "// Ding dong!\n");
 283         writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]);
 284         /* Flush PCI posted writes */
 285         readl(&xhci->dba->doorbell[0]);
 286 }
 287 
 288 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay)
 289 {
 290         return mod_delayed_work(system_wq, &xhci->cmd_timer, delay);
 291 }
 292 
 293 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci)
 294 {
 295         return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command,
 296                                         cmd_list);
 297 }
 298 
 299 /*
 300  * Turn all commands on command ring with status set to "aborted" to no-op trbs.
 301  * If there are other commands waiting then restart the ring and kick the timer.
 302  * This must be called with command ring stopped and xhci->lock held.
 303  */
 304 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci,
 305                                          struct xhci_command *cur_cmd)
 306 {
 307         struct xhci_command *i_cmd;
 308 
 309         /* Turn all aborted commands in list to no-ops, then restart */
 310         list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) {
 311 
 312                 if (i_cmd->status != COMP_COMMAND_ABORTED)
 313                         continue;
 314 
 315                 i_cmd->status = COMP_COMMAND_RING_STOPPED;
 316 
 317                 xhci_dbg(xhci, "Turn aborted command %p to no-op\n",
 318                          i_cmd->command_trb);
 319 
 320                 trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP);
 321 
 322                 /*
 323                  * caller waiting for completion is called when command
 324                  *  completion event is received for these no-op commands
 325                  */
 326         }
 327 
 328         xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
 329 
 330         /* ring command ring doorbell to restart the command ring */
 331         if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) &&
 332             !(xhci->xhc_state & XHCI_STATE_DYING)) {
 333                 xhci->current_cmd = cur_cmd;
 334                 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
 335                 xhci_ring_cmd_db(xhci);
 336         }
 337 }
 338 
 339 /* Must be called with xhci->lock held, releases and aquires lock back */
 340 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags)
 341 {
 342         u64 temp_64;
 343         int ret;
 344 
 345         xhci_dbg(xhci, "Abort command ring\n");
 346 
 347         reinit_completion(&xhci->cmd_ring_stop_completion);
 348 
 349         temp_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
 350         xhci_write_64(xhci, temp_64 | CMD_RING_ABORT,
 351                         &xhci->op_regs->cmd_ring);
 352 
 353         /* Section 4.6.1.2 of xHCI 1.0 spec says software should also time the
 354          * completion of the Command Abort operation. If CRR is not negated in 5
 355          * seconds then driver handles it as if host died (-ENODEV).
 356          * In the future we should distinguish between -ENODEV and -ETIMEDOUT
 357          * and try to recover a -ETIMEDOUT with a host controller reset.
 358          */
 359         ret = xhci_handshake(&xhci->op_regs->cmd_ring,
 360                         CMD_RING_RUNNING, 0, 5 * 1000 * 1000);
 361         if (ret < 0) {
 362                 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret);
 363                 xhci_halt(xhci);
 364                 xhci_hc_died(xhci);
 365                 return ret;
 366         }
 367         /*
 368          * Writing the CMD_RING_ABORT bit should cause a cmd completion event,
 369          * however on some host hw the CMD_RING_RUNNING bit is correctly cleared
 370          * but the completion event in never sent. Wait 2 secs (arbitrary
 371          * number) to handle those cases after negation of CMD_RING_RUNNING.
 372          */
 373         spin_unlock_irqrestore(&xhci->lock, flags);
 374         ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion,
 375                                           msecs_to_jiffies(2000));
 376         spin_lock_irqsave(&xhci->lock, flags);
 377         if (!ret) {
 378                 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n");
 379                 xhci_cleanup_command_queue(xhci);
 380         } else {
 381                 xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci));
 382         }
 383         return 0;
 384 }
 385 
 386 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci,
 387                 unsigned int slot_id,
 388                 unsigned int ep_index,
 389                 unsigned int stream_id)
 390 {
 391         __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
 392         struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
 393         unsigned int ep_state = ep->ep_state;
 394 
 395         /* Don't ring the doorbell for this endpoint if there are pending
 396          * cancellations because we don't want to interrupt processing.
 397          * We don't want to restart any stream rings if there's a set dequeue
 398          * pointer command pending because the device can choose to start any
 399          * stream once the endpoint is on the HW schedule.
 400          */
 401         if ((ep_state & EP_STOP_CMD_PENDING) || (ep_state & SET_DEQ_PENDING) ||
 402             (ep_state & EP_HALTED) || (ep_state & EP_CLEARING_TT))
 403                 return;
 404         writel(DB_VALUE(ep_index, stream_id), db_addr);
 405         /* The CPU has better things to do at this point than wait for a
 406          * write-posting flush.  It'll get there soon enough.
 407          */
 408 }
 409 
 410 /* Ring the doorbell for any rings with pending URBs */
 411 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
 412                 unsigned int slot_id,
 413                 unsigned int ep_index)
 414 {
 415         unsigned int stream_id;
 416         struct xhci_virt_ep *ep;
 417 
 418         ep = &xhci->devs[slot_id]->eps[ep_index];
 419 
 420         /* A ring has pending URBs if its TD list is not empty */
 421         if (!(ep->ep_state & EP_HAS_STREAMS)) {
 422                 if (ep->ring && !(list_empty(&ep->ring->td_list)))
 423                         xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0);
 424                 return;
 425         }
 426 
 427         for (stream_id = 1; stream_id < ep->stream_info->num_streams;
 428                         stream_id++) {
 429                 struct xhci_stream_info *stream_info = ep->stream_info;
 430                 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list))
 431                         xhci_ring_ep_doorbell(xhci, slot_id, ep_index,
 432                                                 stream_id);
 433         }
 434 }
 435 
 436 void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
 437                 unsigned int slot_id,
 438                 unsigned int ep_index)
 439 {
 440         ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
 441 }
 442 
 443 /* Get the right ring for the given slot_id, ep_index and stream_id.
 444  * If the endpoint supports streams, boundary check the URB's stream ID.
 445  * If the endpoint doesn't support streams, return the singular endpoint ring.
 446  */
 447 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci,
 448                 unsigned int slot_id, unsigned int ep_index,
 449                 unsigned int stream_id)
 450 {
 451         struct xhci_virt_ep *ep;
 452 
 453         ep = &xhci->devs[slot_id]->eps[ep_index];
 454         /* Common case: no streams */
 455         if (!(ep->ep_state & EP_HAS_STREAMS))
 456                 return ep->ring;
 457 
 458         if (stream_id == 0) {
 459                 xhci_warn(xhci,
 460                                 "WARN: Slot ID %u, ep index %u has streams, "
 461                                 "but URB has no stream ID.\n",
 462                                 slot_id, ep_index);
 463                 return NULL;
 464         }
 465 
 466         if (stream_id < ep->stream_info->num_streams)
 467                 return ep->stream_info->stream_rings[stream_id];
 468 
 469         xhci_warn(xhci,
 470                         "WARN: Slot ID %u, ep index %u has "
 471                         "stream IDs 1 to %u allocated, "
 472                         "but stream ID %u is requested.\n",
 473                         slot_id, ep_index,
 474                         ep->stream_info->num_streams - 1,
 475                         stream_id);
 476         return NULL;
 477 }
 478 
 479 
 480 /*
 481  * Get the hw dequeue pointer xHC stopped on, either directly from the
 482  * endpoint context, or if streams are in use from the stream context.
 483  * The returned hw_dequeue contains the lowest four bits with cycle state
 484  * and possbile stream context type.
 485  */
 486 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev,
 487                            unsigned int ep_index, unsigned int stream_id)
 488 {
 489         struct xhci_ep_ctx *ep_ctx;
 490         struct xhci_stream_ctx *st_ctx;
 491         struct xhci_virt_ep *ep;
 492 
 493         ep = &vdev->eps[ep_index];
 494 
 495         if (ep->ep_state & EP_HAS_STREAMS) {
 496                 st_ctx = &ep->stream_info->stream_ctx_array[stream_id];
 497                 return le64_to_cpu(st_ctx->stream_ring);
 498         }
 499         ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
 500         return le64_to_cpu(ep_ctx->deq);
 501 }
 502 
 503 /*
 504  * Move the xHC's endpoint ring dequeue pointer past cur_td.
 505  * Record the new state of the xHC's endpoint ring dequeue segment,
 506  * dequeue pointer, stream id, and new consumer cycle state in state.
 507  * Update our internal representation of the ring's dequeue pointer.
 508  *
 509  * We do this in three jumps:
 510  *  - First we update our new ring state to be the same as when the xHC stopped.
 511  *  - Then we traverse the ring to find the segment that contains
 512  *    the last TRB in the TD.  We toggle the xHC's new cycle state when we pass
 513  *    any link TRBs with the toggle cycle bit set.
 514  *  - Finally we move the dequeue state one TRB further, toggling the cycle bit
 515  *    if we've moved it past a link TRB with the toggle cycle bit set.
 516  *
 517  * Some of the uses of xhci_generic_trb are grotty, but if they're done
 518  * with correct __le32 accesses they should work fine.  Only users of this are
 519  * in here.
 520  */
 521 void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
 522                 unsigned int slot_id, unsigned int ep_index,
 523                 unsigned int stream_id, struct xhci_td *cur_td,
 524                 struct xhci_dequeue_state *state)
 525 {
 526         struct xhci_virt_device *dev = xhci->devs[slot_id];
 527         struct xhci_virt_ep *ep = &dev->eps[ep_index];
 528         struct xhci_ring *ep_ring;
 529         struct xhci_segment *new_seg;
 530         union xhci_trb *new_deq;
 531         dma_addr_t addr;
 532         u64 hw_dequeue;
 533         bool cycle_found = false;
 534         bool td_last_trb_found = false;
 535 
 536         ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
 537                         ep_index, stream_id);
 538         if (!ep_ring) {
 539                 xhci_warn(xhci, "WARN can't find new dequeue state "
 540                                 "for invalid stream ID %u.\n",
 541                                 stream_id);
 542                 return;
 543         }
 544         /*
 545          * A cancelled TD can complete with a stall if HW cached the trb.
 546          * In this case driver can't find cur_td, but if the ring is empty we
 547          * can move the dequeue pointer to the current enqueue position.
 548          */
 549         if (!cur_td) {
 550                 if (list_empty(&ep_ring->td_list)) {
 551                         state->new_deq_seg = ep_ring->enq_seg;
 552                         state->new_deq_ptr = ep_ring->enqueue;
 553                         state->new_cycle_state = ep_ring->cycle_state;
 554                         goto done;
 555                 } else {
 556                         xhci_warn(xhci, "Can't find new dequeue state, missing cur_td\n");
 557                         return;
 558                 }
 559         }
 560 
 561         /* Dig out the cycle state saved by the xHC during the stop ep cmd */
 562         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 563                         "Finding endpoint context");
 564 
 565         hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id);
 566         new_seg = ep_ring->deq_seg;
 567         new_deq = ep_ring->dequeue;
 568         state->new_cycle_state = hw_dequeue & 0x1;
 569         state->stream_id = stream_id;
 570 
 571         /*
 572          * We want to find the pointer, segment and cycle state of the new trb
 573          * (the one after current TD's last_trb). We know the cycle state at
 574          * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are
 575          * found.
 576          */
 577         do {
 578                 if (!cycle_found && xhci_trb_virt_to_dma(new_seg, new_deq)
 579                     == (dma_addr_t)(hw_dequeue & ~0xf)) {
 580                         cycle_found = true;
 581                         if (td_last_trb_found)
 582                                 break;
 583                 }
 584                 if (new_deq == cur_td->last_trb)
 585                         td_last_trb_found = true;
 586 
 587                 if (cycle_found && trb_is_link(new_deq) &&
 588                     link_trb_toggles_cycle(new_deq))
 589                         state->new_cycle_state ^= 0x1;
 590 
 591                 next_trb(xhci, ep_ring, &new_seg, &new_deq);
 592 
 593                 /* Search wrapped around, bail out */
 594                 if (new_deq == ep->ring->dequeue) {
 595                         xhci_err(xhci, "Error: Failed finding new dequeue state\n");
 596                         state->new_deq_seg = NULL;
 597                         state->new_deq_ptr = NULL;
 598                         return;
 599                 }
 600 
 601         } while (!cycle_found || !td_last_trb_found);
 602 
 603         state->new_deq_seg = new_seg;
 604         state->new_deq_ptr = new_deq;
 605 
 606 done:
 607         /* Don't update the ring cycle state for the producer (us). */
 608         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 609                         "Cycle state = 0x%x", state->new_cycle_state);
 610 
 611         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 612                         "New dequeue segment = %p (virtual)",
 613                         state->new_deq_seg);
 614         addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr);
 615         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 616                         "New dequeue pointer = 0x%llx (DMA)",
 617                         (unsigned long long) addr);
 618 }
 619 
 620 /* flip_cycle means flip the cycle bit of all but the first and last TRB.
 621  * (The last TRB actually points to the ring enqueue pointer, which is not part
 622  * of this TD.)  This is used to remove partially enqueued isoc TDs from a ring.
 623  */
 624 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
 625                        struct xhci_td *td, bool flip_cycle)
 626 {
 627         struct xhci_segment *seg        = td->start_seg;
 628         union xhci_trb *trb             = td->first_trb;
 629 
 630         while (1) {
 631                 trb_to_noop(trb, TRB_TR_NOOP);
 632 
 633                 /* flip cycle if asked to */
 634                 if (flip_cycle && trb != td->first_trb && trb != td->last_trb)
 635                         trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
 636 
 637                 if (trb == td->last_trb)
 638                         break;
 639 
 640                 next_trb(xhci, ep_ring, &seg, &trb);
 641         }
 642 }
 643 
 644 static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci,
 645                 struct xhci_virt_ep *ep)
 646 {
 647         ep->ep_state &= ~EP_STOP_CMD_PENDING;
 648         /* Can't del_timer_sync in interrupt */
 649         del_timer(&ep->stop_cmd_timer);
 650 }
 651 
 652 /*
 653  * Must be called with xhci->lock held in interrupt context,
 654  * releases and re-acquires xhci->lock
 655  */
 656 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
 657                                      struct xhci_td *cur_td, int status)
 658 {
 659         struct urb      *urb            = cur_td->urb;
 660         struct urb_priv *urb_priv       = urb->hcpriv;
 661         struct usb_hcd  *hcd            = bus_to_hcd(urb->dev->bus);
 662 
 663         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
 664                 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
 665                 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
 666                         if (xhci->quirks & XHCI_AMD_PLL_FIX)
 667                                 usb_amd_quirk_pll_enable();
 668                 }
 669         }
 670         xhci_urb_free_priv(urb_priv);
 671         usb_hcd_unlink_urb_from_ep(hcd, urb);
 672         spin_unlock(&xhci->lock);
 673         trace_xhci_urb_giveback(urb);
 674         usb_hcd_giveback_urb(hcd, urb, status);
 675         spin_lock(&xhci->lock);
 676 }
 677 
 678 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
 679                 struct xhci_ring *ring, struct xhci_td *td)
 680 {
 681         struct device *dev = xhci_to_hcd(xhci)->self.controller;
 682         struct xhci_segment *seg = td->bounce_seg;
 683         struct urb *urb = td->urb;
 684         size_t len;
 685 
 686         if (!ring || !seg || !urb)
 687                 return;
 688 
 689         if (usb_urb_dir_out(urb)) {
 690                 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
 691                                  DMA_TO_DEVICE);
 692                 return;
 693         }
 694 
 695         dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
 696                          DMA_FROM_DEVICE);
 697         /* for in tranfers we need to copy the data from bounce to sg */
 698         len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs, seg->bounce_buf,
 699                              seg->bounce_len, seg->bounce_offs);
 700         if (len != seg->bounce_len)
 701                 xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n",
 702                                 len, seg->bounce_len);
 703         seg->bounce_len = 0;
 704         seg->bounce_offs = 0;
 705 }
 706 
 707 /*
 708  * When we get a command completion for a Stop Endpoint Command, we need to
 709  * unlink any cancelled TDs from the ring.  There are two ways to do that:
 710  *
 711  *  1. If the HW was in the middle of processing the TD that needs to be
 712  *     cancelled, then we must move the ring's dequeue pointer past the last TRB
 713  *     in the TD with a Set Dequeue Pointer Command.
 714  *  2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain
 715  *     bit cleared) so that the HW will skip over them.
 716  */
 717 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
 718                 union xhci_trb *trb, struct xhci_event_cmd *event)
 719 {
 720         unsigned int ep_index;
 721         struct xhci_ring *ep_ring;
 722         struct xhci_virt_ep *ep;
 723         struct xhci_td *cur_td = NULL;
 724         struct xhci_td *last_unlinked_td;
 725         struct xhci_ep_ctx *ep_ctx;
 726         struct xhci_virt_device *vdev;
 727         u64 hw_deq;
 728         struct xhci_dequeue_state deq_state;
 729 
 730         if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) {
 731                 if (!xhci->devs[slot_id])
 732                         xhci_warn(xhci, "Stop endpoint command "
 733                                 "completion for disabled slot %u\n",
 734                                 slot_id);
 735                 return;
 736         }
 737 
 738         memset(&deq_state, 0, sizeof(deq_state));
 739         ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
 740 
 741         vdev = xhci->devs[slot_id];
 742         ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
 743         trace_xhci_handle_cmd_stop_ep(ep_ctx);
 744 
 745         ep = &xhci->devs[slot_id]->eps[ep_index];
 746         last_unlinked_td = list_last_entry(&ep->cancelled_td_list,
 747                         struct xhci_td, cancelled_td_list);
 748 
 749         if (list_empty(&ep->cancelled_td_list)) {
 750                 xhci_stop_watchdog_timer_in_irq(xhci, ep);
 751                 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
 752                 return;
 753         }
 754 
 755         /* Fix up the ep ring first, so HW stops executing cancelled TDs.
 756          * We have the xHCI lock, so nothing can modify this list until we drop
 757          * it.  We're also in the event handler, so we can't get re-interrupted
 758          * if another Stop Endpoint command completes
 759          */
 760         list_for_each_entry(cur_td, &ep->cancelled_td_list, cancelled_td_list) {
 761                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 762                                 "Removing canceled TD starting at 0x%llx (dma).",
 763                                 (unsigned long long)xhci_trb_virt_to_dma(
 764                                         cur_td->start_seg, cur_td->first_trb));
 765                 ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
 766                 if (!ep_ring) {
 767                         /* This shouldn't happen unless a driver is mucking
 768                          * with the stream ID after submission.  This will
 769                          * leave the TD on the hardware ring, and the hardware
 770                          * will try to execute it, and may access a buffer
 771                          * that has already been freed.  In the best case, the
 772                          * hardware will execute it, and the event handler will
 773                          * ignore the completion event for that TD, since it was
 774                          * removed from the td_list for that endpoint.  In
 775                          * short, don't muck with the stream ID after
 776                          * submission.
 777                          */
 778                         xhci_warn(xhci, "WARN Cancelled URB %p "
 779                                         "has invalid stream ID %u.\n",
 780                                         cur_td->urb,
 781                                         cur_td->urb->stream_id);
 782                         goto remove_finished_td;
 783                 }
 784                 /*
 785                  * If we stopped on the TD we need to cancel, then we have to
 786                  * move the xHC endpoint ring dequeue pointer past this TD.
 787                  */
 788                 hw_deq = xhci_get_hw_deq(xhci, vdev, ep_index,
 789                                          cur_td->urb->stream_id);
 790                 hw_deq &= ~0xf;
 791 
 792                 if (trb_in_td(xhci, cur_td->start_seg, cur_td->first_trb,
 793                               cur_td->last_trb, hw_deq, false)) {
 794                         xhci_find_new_dequeue_state(xhci, slot_id, ep_index,
 795                                                     cur_td->urb->stream_id,
 796                                                     cur_td, &deq_state);
 797                 } else {
 798                         td_to_noop(xhci, ep_ring, cur_td, false);
 799                 }
 800 
 801 remove_finished_td:
 802                 /*
 803                  * The event handler won't see a completion for this TD anymore,
 804                  * so remove it from the endpoint ring's TD list.  Keep it in
 805                  * the cancelled TD list for URB completion later.
 806                  */
 807                 list_del_init(&cur_td->td_list);
 808         }
 809 
 810         xhci_stop_watchdog_timer_in_irq(xhci, ep);
 811 
 812         /* If necessary, queue a Set Transfer Ring Dequeue Pointer command */
 813         if (deq_state.new_deq_ptr && deq_state.new_deq_seg) {
 814                 xhci_queue_new_dequeue_state(xhci, slot_id, ep_index,
 815                                              &deq_state);
 816                 xhci_ring_cmd_db(xhci);
 817         } else {
 818                 /* Otherwise ring the doorbell(s) to restart queued transfers */
 819                 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
 820         }
 821 
 822         /*
 823          * Drop the lock and complete the URBs in the cancelled TD list.
 824          * New TDs to be cancelled might be added to the end of the list before
 825          * we can complete all the URBs for the TDs we already unlinked.
 826          * So stop when we've completed the URB for the last TD we unlinked.
 827          */
 828         do {
 829                 cur_td = list_first_entry(&ep->cancelled_td_list,
 830                                 struct xhci_td, cancelled_td_list);
 831                 list_del_init(&cur_td->cancelled_td_list);
 832 
 833                 /* Clean up the cancelled URB */
 834                 /* Doesn't matter what we pass for status, since the core will
 835                  * just overwrite it (because the URB has been unlinked).
 836                  */
 837                 ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
 838                 xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td);
 839                 inc_td_cnt(cur_td->urb);
 840                 if (last_td_in_urb(cur_td))
 841                         xhci_giveback_urb_in_irq(xhci, cur_td, 0);
 842 
 843                 /* Stop processing the cancelled list if the watchdog timer is
 844                  * running.
 845                  */
 846                 if (xhci->xhc_state & XHCI_STATE_DYING)
 847                         return;
 848         } while (cur_td != last_unlinked_td);
 849 
 850         /* Return to the event handler with xhci->lock re-acquired */
 851 }
 852 
 853 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring)
 854 {
 855         struct xhci_td *cur_td;
 856         struct xhci_td *tmp;
 857 
 858         list_for_each_entry_safe(cur_td, tmp, &ring->td_list, td_list) {
 859                 list_del_init(&cur_td->td_list);
 860 
 861                 if (!list_empty(&cur_td->cancelled_td_list))
 862                         list_del_init(&cur_td->cancelled_td_list);
 863 
 864                 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td);
 865 
 866                 inc_td_cnt(cur_td->urb);
 867                 if (last_td_in_urb(cur_td))
 868                         xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
 869         }
 870 }
 871 
 872 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci,
 873                 int slot_id, int ep_index)
 874 {
 875         struct xhci_td *cur_td;
 876         struct xhci_td *tmp;
 877         struct xhci_virt_ep *ep;
 878         struct xhci_ring *ring;
 879 
 880         ep = &xhci->devs[slot_id]->eps[ep_index];
 881         if ((ep->ep_state & EP_HAS_STREAMS) ||
 882                         (ep->ep_state & EP_GETTING_NO_STREAMS)) {
 883                 int stream_id;
 884 
 885                 for (stream_id = 1; stream_id < ep->stream_info->num_streams;
 886                                 stream_id++) {
 887                         ring = ep->stream_info->stream_rings[stream_id];
 888                         if (!ring)
 889                                 continue;
 890 
 891                         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 892                                         "Killing URBs for slot ID %u, ep index %u, stream %u",
 893                                         slot_id, ep_index, stream_id);
 894                         xhci_kill_ring_urbs(xhci, ring);
 895                 }
 896         } else {
 897                 ring = ep->ring;
 898                 if (!ring)
 899                         return;
 900                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 901                                 "Killing URBs for slot ID %u, ep index %u",
 902                                 slot_id, ep_index);
 903                 xhci_kill_ring_urbs(xhci, ring);
 904         }
 905 
 906         list_for_each_entry_safe(cur_td, tmp, &ep->cancelled_td_list,
 907                         cancelled_td_list) {
 908                 list_del_init(&cur_td->cancelled_td_list);
 909                 inc_td_cnt(cur_td->urb);
 910 
 911                 if (last_td_in_urb(cur_td))
 912                         xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
 913         }
 914 }
 915 
 916 /*
 917  * host controller died, register read returns 0xffffffff
 918  * Complete pending commands, mark them ABORTED.
 919  * URBs need to be given back as usb core might be waiting with device locks
 920  * held for the URBs to finish during device disconnect, blocking host remove.
 921  *
 922  * Call with xhci->lock held.
 923  * lock is relased and re-acquired while giving back urb.
 924  */
 925 void xhci_hc_died(struct xhci_hcd *xhci)
 926 {
 927         int i, j;
 928 
 929         if (xhci->xhc_state & XHCI_STATE_DYING)
 930                 return;
 931 
 932         xhci_err(xhci, "xHCI host controller not responding, assume dead\n");
 933         xhci->xhc_state |= XHCI_STATE_DYING;
 934 
 935         xhci_cleanup_command_queue(xhci);
 936 
 937         /* return any pending urbs, remove may be waiting for them */
 938         for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) {
 939                 if (!xhci->devs[i])
 940                         continue;
 941                 for (j = 0; j < 31; j++)
 942                         xhci_kill_endpoint_urbs(xhci, i, j);
 943         }
 944 
 945         /* inform usb core hc died if PCI remove isn't already handling it */
 946         if (!(xhci->xhc_state & XHCI_STATE_REMOVING))
 947                 usb_hc_died(xhci_to_hcd(xhci));
 948 }
 949 
 950 /* Watchdog timer function for when a stop endpoint command fails to complete.
 951  * In this case, we assume the host controller is broken or dying or dead.  The
 952  * host may still be completing some other events, so we have to be careful to
 953  * let the event ring handler and the URB dequeueing/enqueueing functions know
 954  * through xhci->state.
 955  *
 956  * The timer may also fire if the host takes a very long time to respond to the
 957  * command, and the stop endpoint command completion handler cannot delete the
 958  * timer before the timer function is called.  Another endpoint cancellation may
 959  * sneak in before the timer function can grab the lock, and that may queue
 960  * another stop endpoint command and add the timer back.  So we cannot use a
 961  * simple flag to say whether there is a pending stop endpoint command for a
 962  * particular endpoint.
 963  *
 964  * Instead we use a combination of that flag and checking if a new timer is
 965  * pending.
 966  */
 967 void xhci_stop_endpoint_command_watchdog(struct timer_list *t)
 968 {
 969         struct xhci_virt_ep *ep = from_timer(ep, t, stop_cmd_timer);
 970         struct xhci_hcd *xhci = ep->xhci;
 971         unsigned long flags;
 972 
 973         spin_lock_irqsave(&xhci->lock, flags);
 974 
 975         /* bail out if cmd completed but raced with stop ep watchdog timer.*/
 976         if (!(ep->ep_state & EP_STOP_CMD_PENDING) ||
 977             timer_pending(&ep->stop_cmd_timer)) {
 978                 spin_unlock_irqrestore(&xhci->lock, flags);
 979                 xhci_dbg(xhci, "Stop EP timer raced with cmd completion, exit");
 980                 return;
 981         }
 982 
 983         xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n");
 984         ep->ep_state &= ~EP_STOP_CMD_PENDING;
 985 
 986         xhci_halt(xhci);
 987 
 988         /*
 989          * handle a stop endpoint cmd timeout as if host died (-ENODEV).
 990          * In the future we could distinguish between -ENODEV and -ETIMEDOUT
 991          * and try to recover a -ETIMEDOUT with a host controller reset
 992          */
 993         xhci_hc_died(xhci);
 994 
 995         spin_unlock_irqrestore(&xhci->lock, flags);
 996         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 997                         "xHCI host controller is dead.");
 998 }
 999 
1000 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci,
1001                 struct xhci_virt_device *dev,
1002                 struct xhci_ring *ep_ring,
1003                 unsigned int ep_index)
1004 {
1005         union xhci_trb *dequeue_temp;
1006         int num_trbs_free_temp;
1007         bool revert = false;
1008 
1009         num_trbs_free_temp = ep_ring->num_trbs_free;
1010         dequeue_temp = ep_ring->dequeue;
1011 
1012         /* If we get two back-to-back stalls, and the first stalled transfer
1013          * ends just before a link TRB, the dequeue pointer will be left on
1014          * the link TRB by the code in the while loop.  So we have to update
1015          * the dequeue pointer one segment further, or we'll jump off
1016          * the segment into la-la-land.
1017          */
1018         if (trb_is_link(ep_ring->dequeue)) {
1019                 ep_ring->deq_seg = ep_ring->deq_seg->next;
1020                 ep_ring->dequeue = ep_ring->deq_seg->trbs;
1021         }
1022 
1023         while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {
1024                 /* We have more usable TRBs */
1025                 ep_ring->num_trbs_free++;
1026                 ep_ring->dequeue++;
1027                 if (trb_is_link(ep_ring->dequeue)) {
1028                         if (ep_ring->dequeue ==
1029                                         dev->eps[ep_index].queued_deq_ptr)
1030                                 break;
1031                         ep_ring->deq_seg = ep_ring->deq_seg->next;
1032                         ep_ring->dequeue = ep_ring->deq_seg->trbs;
1033                 }
1034                 if (ep_ring->dequeue == dequeue_temp) {
1035                         revert = true;
1036                         break;
1037                 }
1038         }
1039 
1040         if (revert) {
1041                 xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
1042                 ep_ring->num_trbs_free = num_trbs_free_temp;
1043         }
1044 }
1045 
1046 /*
1047  * When we get a completion for a Set Transfer Ring Dequeue Pointer command,
1048  * we need to clear the set deq pending flag in the endpoint ring state, so that
1049  * the TD queueing code can ring the doorbell again.  We also need to ring the
1050  * endpoint doorbell to restart the ring, but only if there aren't more
1051  * cancellations pending.
1052  */
1053 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
1054                 union xhci_trb *trb, u32 cmd_comp_code)
1055 {
1056         unsigned int ep_index;
1057         unsigned int stream_id;
1058         struct xhci_ring *ep_ring;
1059         struct xhci_virt_device *dev;
1060         struct xhci_virt_ep *ep;
1061         struct xhci_ep_ctx *ep_ctx;
1062         struct xhci_slot_ctx *slot_ctx;
1063 
1064         ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1065         stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
1066         dev = xhci->devs[slot_id];
1067         ep = &dev->eps[ep_index];
1068 
1069         ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id);
1070         if (!ep_ring) {
1071                 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
1072                                 stream_id);
1073                 /* XXX: Harmless??? */
1074                 goto cleanup;
1075         }
1076 
1077         ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
1078         slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
1079         trace_xhci_handle_cmd_set_deq(slot_ctx);
1080         trace_xhci_handle_cmd_set_deq_ep(ep_ctx);
1081 
1082         if (cmd_comp_code != COMP_SUCCESS) {
1083                 unsigned int ep_state;
1084                 unsigned int slot_state;
1085 
1086                 switch (cmd_comp_code) {
1087                 case COMP_TRB_ERROR:
1088                         xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n");
1089                         break;
1090                 case COMP_CONTEXT_STATE_ERROR:
1091                         xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n");
1092                         ep_state = GET_EP_CTX_STATE(ep_ctx);
1093                         slot_state = le32_to_cpu(slot_ctx->dev_state);
1094                         slot_state = GET_SLOT_STATE(slot_state);
1095                         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1096                                         "Slot state = %u, EP state = %u",
1097                                         slot_state, ep_state);
1098                         break;
1099                 case COMP_SLOT_NOT_ENABLED_ERROR:
1100                         xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n",
1101                                         slot_id);
1102                         break;
1103                 default:
1104                         xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n",
1105                                         cmd_comp_code);
1106                         break;
1107                 }
1108                 /* OK what do we do now?  The endpoint state is hosed, and we
1109                  * should never get to this point if the synchronization between
1110                  * queueing, and endpoint state are correct.  This might happen
1111                  * if the device gets disconnected after we've finished
1112                  * cancelling URBs, which might not be an error...
1113                  */
1114         } else {
1115                 u64 deq;
1116                 /* 4.6.10 deq ptr is written to the stream ctx for streams */
1117                 if (ep->ep_state & EP_HAS_STREAMS) {
1118                         struct xhci_stream_ctx *ctx =
1119                                 &ep->stream_info->stream_ctx_array[stream_id];
1120                         deq = le64_to_cpu(ctx->stream_ring) & SCTX_DEQ_MASK;
1121                 } else {
1122                         deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK;
1123                 }
1124                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1125                         "Successful Set TR Deq Ptr cmd, deq = @%08llx", deq);
1126                 if (xhci_trb_virt_to_dma(ep->queued_deq_seg,
1127                                          ep->queued_deq_ptr) == deq) {
1128                         /* Update the ring's dequeue segment and dequeue pointer
1129                          * to reflect the new position.
1130                          */
1131                         update_ring_for_set_deq_completion(xhci, dev,
1132                                 ep_ring, ep_index);
1133                 } else {
1134                         xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
1135                         xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n",
1136                                   ep->queued_deq_seg, ep->queued_deq_ptr);
1137                 }
1138         }
1139 
1140 cleanup:
1141         dev->eps[ep_index].ep_state &= ~SET_DEQ_PENDING;
1142         dev->eps[ep_index].queued_deq_seg = NULL;
1143         dev->eps[ep_index].queued_deq_ptr = NULL;
1144         /* Restart any rings with pending URBs */
1145         ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1146 }
1147 
1148 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
1149                 union xhci_trb *trb, u32 cmd_comp_code)
1150 {
1151         struct xhci_virt_device *vdev;
1152         struct xhci_ep_ctx *ep_ctx;
1153         unsigned int ep_index;
1154 
1155         ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1156         vdev = xhci->devs[slot_id];
1157         ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
1158         trace_xhci_handle_cmd_reset_ep(ep_ctx);
1159 
1160         /* This command will only fail if the endpoint wasn't halted,
1161          * but we don't care.
1162          */
1163         xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
1164                 "Ignoring reset ep completion code of %u", cmd_comp_code);
1165 
1166         /* HW with the reset endpoint quirk needs to have a configure endpoint
1167          * command complete before the endpoint can be used.  Queue that here
1168          * because the HW can't handle two commands being queued in a row.
1169          */
1170         if (xhci->quirks & XHCI_RESET_EP_QUIRK) {
1171                 struct xhci_command *command;
1172 
1173                 command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1174                 if (!command)
1175                         return;
1176 
1177                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1178                                 "Queueing configure endpoint command");
1179                 xhci_queue_configure_endpoint(xhci, command,
1180                                 xhci->devs[slot_id]->in_ctx->dma, slot_id,
1181                                 false);
1182                 xhci_ring_cmd_db(xhci);
1183         } else {
1184                 /* Clear our internal halted state */
1185                 xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_HALTED;
1186         }
1187 
1188         /* if this was a soft reset, then restart */
1189         if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP)
1190                 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1191 }
1192 
1193 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
1194                 struct xhci_command *command, u32 cmd_comp_code)
1195 {
1196         if (cmd_comp_code == COMP_SUCCESS)
1197                 command->slot_id = slot_id;
1198         else
1199                 command->slot_id = 0;
1200 }
1201 
1202 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
1203 {
1204         struct xhci_virt_device *virt_dev;
1205         struct xhci_slot_ctx *slot_ctx;
1206 
1207         virt_dev = xhci->devs[slot_id];
1208         if (!virt_dev)
1209                 return;
1210 
1211         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
1212         trace_xhci_handle_cmd_disable_slot(slot_ctx);
1213 
1214         if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
1215                 /* Delete default control endpoint resources */
1216                 xhci_free_device_endpoint_resources(xhci, virt_dev, true);
1217         xhci_free_virt_device(xhci, slot_id);
1218 }
1219 
1220 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
1221                 struct xhci_event_cmd *event, u32 cmd_comp_code)
1222 {
1223         struct xhci_virt_device *virt_dev;
1224         struct xhci_input_control_ctx *ctrl_ctx;
1225         struct xhci_ep_ctx *ep_ctx;
1226         unsigned int ep_index;
1227         unsigned int ep_state;
1228         u32 add_flags, drop_flags;
1229 
1230         /*
1231          * Configure endpoint commands can come from the USB core
1232          * configuration or alt setting changes, or because the HW
1233          * needed an extra configure endpoint command after a reset
1234          * endpoint command or streams were being configured.
1235          * If the command was for a halted endpoint, the xHCI driver
1236          * is not waiting on the configure endpoint command.
1237          */
1238         virt_dev = xhci->devs[slot_id];
1239         ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
1240         if (!ctrl_ctx) {
1241                 xhci_warn(xhci, "Could not get input context, bad type.\n");
1242                 return;
1243         }
1244 
1245         add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1246         drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1247         /* Input ctx add_flags are the endpoint index plus one */
1248         ep_index = xhci_last_valid_endpoint(add_flags) - 1;
1249 
1250         ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index);
1251         trace_xhci_handle_cmd_config_ep(ep_ctx);
1252 
1253         /* A usb_set_interface() call directly after clearing a halted
1254          * condition may race on this quirky hardware.  Not worth
1255          * worrying about, since this is prototype hardware.  Not sure
1256          * if this will work for streams, but streams support was
1257          * untested on this prototype.
1258          */
1259         if (xhci->quirks & XHCI_RESET_EP_QUIRK &&
1260                         ep_index != (unsigned int) -1 &&
1261                         add_flags - SLOT_FLAG == drop_flags) {
1262                 ep_state = virt_dev->eps[ep_index].ep_state;
1263                 if (!(ep_state & EP_HALTED))
1264                         return;
1265                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1266                                 "Completed config ep cmd - "
1267                                 "last ep index = %d, state = %d",
1268                                 ep_index, ep_state);
1269                 /* Clear internal halted state and restart ring(s) */
1270                 virt_dev->eps[ep_index].ep_state &= ~EP_HALTED;
1271                 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1272                 return;
1273         }
1274         return;
1275 }
1276 
1277 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id)
1278 {
1279         struct xhci_virt_device *vdev;
1280         struct xhci_slot_ctx *slot_ctx;
1281 
1282         vdev = xhci->devs[slot_id];
1283         slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1284         trace_xhci_handle_cmd_addr_dev(slot_ctx);
1285 }
1286 
1287 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
1288                 struct xhci_event_cmd *event)
1289 {
1290         struct xhci_virt_device *vdev;
1291         struct xhci_slot_ctx *slot_ctx;
1292 
1293         vdev = xhci->devs[slot_id];
1294         slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1295         trace_xhci_handle_cmd_reset_dev(slot_ctx);
1296 
1297         xhci_dbg(xhci, "Completed reset device command.\n");
1298         if (!xhci->devs[slot_id])
1299                 xhci_warn(xhci, "Reset device command completion "
1300                                 "for disabled slot %u\n", slot_id);
1301 }
1302 
1303 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
1304                 struct xhci_event_cmd *event)
1305 {
1306         if (!(xhci->quirks & XHCI_NEC_HOST)) {
1307                 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n");
1308                 return;
1309         }
1310         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1311                         "NEC firmware version %2x.%02x",
1312                         NEC_FW_MAJOR(le32_to_cpu(event->status)),
1313                         NEC_FW_MINOR(le32_to_cpu(event->status)));
1314 }
1315 
1316 static void xhci_complete_del_and_free_cmd(struct xhci_command *cmd, u32 status)
1317 {
1318         list_del(&cmd->cmd_list);
1319 
1320         if (cmd->completion) {
1321                 cmd->status = status;
1322                 complete(cmd->completion);
1323         } else {
1324                 kfree(cmd);
1325         }
1326 }
1327 
1328 void xhci_cleanup_command_queue(struct xhci_hcd *xhci)
1329 {
1330         struct xhci_command *cur_cmd, *tmp_cmd;
1331         xhci->current_cmd = NULL;
1332         list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list)
1333                 xhci_complete_del_and_free_cmd(cur_cmd, COMP_COMMAND_ABORTED);
1334 }
1335 
1336 void xhci_handle_command_timeout(struct work_struct *work)
1337 {
1338         struct xhci_hcd *xhci;
1339         unsigned long flags;
1340         u64 hw_ring_state;
1341 
1342         xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer);
1343 
1344         spin_lock_irqsave(&xhci->lock, flags);
1345 
1346         /*
1347          * If timeout work is pending, or current_cmd is NULL, it means we
1348          * raced with command completion. Command is handled so just return.
1349          */
1350         if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) {
1351                 spin_unlock_irqrestore(&xhci->lock, flags);
1352                 return;
1353         }
1354         /* mark this command to be cancelled */
1355         xhci->current_cmd->status = COMP_COMMAND_ABORTED;
1356 
1357         /* Make sure command ring is running before aborting it */
1358         hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
1359         if (hw_ring_state == ~(u64)0) {
1360                 xhci_hc_died(xhci);
1361                 goto time_out_completed;
1362         }
1363 
1364         if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) &&
1365             (hw_ring_state & CMD_RING_RUNNING))  {
1366                 /* Prevent new doorbell, and start command abort */
1367                 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED;
1368                 xhci_dbg(xhci, "Command timeout\n");
1369                 xhci_abort_cmd_ring(xhci, flags);
1370                 goto time_out_completed;
1371         }
1372 
1373         /* host removed. Bail out */
1374         if (xhci->xhc_state & XHCI_STATE_REMOVING) {
1375                 xhci_dbg(xhci, "host removed, ring start fail?\n");
1376                 xhci_cleanup_command_queue(xhci);
1377 
1378                 goto time_out_completed;
1379         }
1380 
1381         /* command timeout on stopped ring, ring can't be aborted */
1382         xhci_dbg(xhci, "Command timeout on stopped ring\n");
1383         xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd);
1384 
1385 time_out_completed:
1386         spin_unlock_irqrestore(&xhci->lock, flags);
1387         return;
1388 }
1389 
1390 static void handle_cmd_completion(struct xhci_hcd *xhci,
1391                 struct xhci_event_cmd *event)
1392 {
1393         int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1394         u64 cmd_dma;
1395         dma_addr_t cmd_dequeue_dma;
1396         u32 cmd_comp_code;
1397         union xhci_trb *cmd_trb;
1398         struct xhci_command *cmd;
1399         u32 cmd_type;
1400 
1401         cmd_dma = le64_to_cpu(event->cmd_trb);
1402         cmd_trb = xhci->cmd_ring->dequeue;
1403 
1404         trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic);
1405 
1406         cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
1407                         cmd_trb);
1408         /*
1409          * Check whether the completion event is for our internal kept
1410          * command.
1411          */
1412         if (!cmd_dequeue_dma || cmd_dma != (u64)cmd_dequeue_dma) {
1413                 xhci_warn(xhci,
1414                           "ERROR mismatched command completion event\n");
1415                 return;
1416         }
1417 
1418         cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list);
1419 
1420         cancel_delayed_work(&xhci->cmd_timer);
1421 
1422         cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status));
1423 
1424         /* If CMD ring stopped we own the trbs between enqueue and dequeue */
1425         if (cmd_comp_code == COMP_COMMAND_RING_STOPPED) {
1426                 complete_all(&xhci->cmd_ring_stop_completion);
1427                 return;
1428         }
1429 
1430         if (cmd->command_trb != xhci->cmd_ring->dequeue) {
1431                 xhci_err(xhci,
1432                          "Command completion event does not match command\n");
1433                 return;
1434         }
1435 
1436         /*
1437          * Host aborted the command ring, check if the current command was
1438          * supposed to be aborted, otherwise continue normally.
1439          * The command ring is stopped now, but the xHC will issue a Command
1440          * Ring Stopped event which will cause us to restart it.
1441          */
1442         if (cmd_comp_code == COMP_COMMAND_ABORTED) {
1443                 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
1444                 if (cmd->status == COMP_COMMAND_ABORTED) {
1445                         if (xhci->current_cmd == cmd)
1446                                 xhci->current_cmd = NULL;
1447                         goto event_handled;
1448                 }
1449         }
1450 
1451         cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3]));
1452         switch (cmd_type) {
1453         case TRB_ENABLE_SLOT:
1454                 xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code);
1455                 break;
1456         case TRB_DISABLE_SLOT:
1457                 xhci_handle_cmd_disable_slot(xhci, slot_id);
1458                 break;
1459         case TRB_CONFIG_EP:
1460                 if (!cmd->completion)
1461                         xhci_handle_cmd_config_ep(xhci, slot_id, event,
1462                                                   cmd_comp_code);
1463                 break;
1464         case TRB_EVAL_CONTEXT:
1465                 break;
1466         case TRB_ADDR_DEV:
1467                 xhci_handle_cmd_addr_dev(xhci, slot_id);
1468                 break;
1469         case TRB_STOP_RING:
1470                 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1471                                 le32_to_cpu(cmd_trb->generic.field[3])));
1472                 if (!cmd->completion)
1473                         xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event);
1474                 break;
1475         case TRB_SET_DEQ:
1476                 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1477                                 le32_to_cpu(cmd_trb->generic.field[3])));
1478                 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code);
1479                 break;
1480         case TRB_CMD_NOOP:
1481                 /* Is this an aborted command turned to NO-OP? */
1482                 if (cmd->status == COMP_COMMAND_RING_STOPPED)
1483                         cmd_comp_code = COMP_COMMAND_RING_STOPPED;
1484                 break;
1485         case TRB_RESET_EP:
1486                 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1487                                 le32_to_cpu(cmd_trb->generic.field[3])));
1488                 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code);
1489                 break;
1490         case TRB_RESET_DEV:
1491                 /* SLOT_ID field in reset device cmd completion event TRB is 0.
1492                  * Use the SLOT_ID from the command TRB instead (xhci 4.6.11)
1493                  */
1494                 slot_id = TRB_TO_SLOT_ID(
1495                                 le32_to_cpu(cmd_trb->generic.field[3]));
1496                 xhci_handle_cmd_reset_dev(xhci, slot_id, event);
1497                 break;
1498         case TRB_NEC_GET_FW:
1499                 xhci_handle_cmd_nec_get_fw(xhci, event);
1500                 break;
1501         default:
1502                 /* Skip over unknown commands on the event ring */
1503                 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type);
1504                 break;
1505         }
1506 
1507         /* restart timer if this wasn't the last command */
1508         if (!list_is_singular(&xhci->cmd_list)) {
1509                 xhci->current_cmd = list_first_entry(&cmd->cmd_list,
1510                                                 struct xhci_command, cmd_list);
1511                 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
1512         } else if (xhci->current_cmd == cmd) {
1513                 xhci->current_cmd = NULL;
1514         }
1515 
1516 event_handled:
1517         xhci_complete_del_and_free_cmd(cmd, cmd_comp_code);
1518 
1519         inc_deq(xhci, xhci->cmd_ring);
1520 }
1521 
1522 static void handle_vendor_event(struct xhci_hcd *xhci,
1523                 union xhci_trb *event)
1524 {
1525         u32 trb_type;
1526 
1527         trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3]));
1528         xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1529         if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1530                 handle_cmd_completion(xhci, &event->event_cmd);
1531 }
1532 
1533 static void handle_device_notification(struct xhci_hcd *xhci,
1534                 union xhci_trb *event)
1535 {
1536         u32 slot_id;
1537         struct usb_device *udev;
1538 
1539         slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3]));
1540         if (!xhci->devs[slot_id]) {
1541                 xhci_warn(xhci, "Device Notification event for "
1542                                 "unused slot %u\n", slot_id);
1543                 return;
1544         }
1545 
1546         xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n",
1547                         slot_id);
1548         udev = xhci->devs[slot_id]->udev;
1549         if (udev && udev->parent)
1550                 usb_wakeup_notification(udev->parent, udev->portnum);
1551 }
1552 
1553 /*
1554  * Quirk hanlder for errata seen on Cavium ThunderX2 processor XHCI
1555  * Controller.
1556  * As per ThunderX2errata-129 USB 2 device may come up as USB 1
1557  * If a connection to a USB 1 device is followed by another connection
1558  * to a USB 2 device.
1559  *
1560  * Reset the PHY after the USB device is disconnected if device speed
1561  * is less than HCD_USB3.
1562  * Retry the reset sequence max of 4 times checking the PLL lock status.
1563  *
1564  */
1565 static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci)
1566 {
1567         struct usb_hcd *hcd = xhci_to_hcd(xhci);
1568         u32 pll_lock_check;
1569         u32 retry_count = 4;
1570 
1571         do {
1572                 /* Assert PHY reset */
1573                 writel(0x6F, hcd->regs + 0x1048);
1574                 udelay(10);
1575                 /* De-assert the PHY reset */
1576                 writel(0x7F, hcd->regs + 0x1048);
1577                 udelay(200);
1578                 pll_lock_check = readl(hcd->regs + 0x1070);
1579         } while (!(pll_lock_check & 0x1) && --retry_count);
1580 }
1581 
1582 static void handle_port_status(struct xhci_hcd *xhci,
1583                 union xhci_trb *event)
1584 {
1585         struct usb_hcd *hcd;
1586         u32 port_id;
1587         u32 portsc, cmd_reg;
1588         int max_ports;
1589         int slot_id;
1590         unsigned int hcd_portnum;
1591         struct xhci_bus_state *bus_state;
1592         bool bogus_port_status = false;
1593         struct xhci_port *port;
1594 
1595         /* Port status change events always have a successful completion code */
1596         if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS)
1597                 xhci_warn(xhci,
1598                           "WARN: xHC returned failed port status event\n");
1599 
1600         port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
1601         max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1602 
1603         if ((port_id <= 0) || (port_id > max_ports)) {
1604                 xhci_warn(xhci, "Port change event with invalid port ID %d\n",
1605                           port_id);
1606                 inc_deq(xhci, xhci->event_ring);
1607                 return;
1608         }
1609 
1610         port = &xhci->hw_ports[port_id - 1];
1611         if (!port || !port->rhub || port->hcd_portnum == DUPLICATE_ENTRY) {
1612                 xhci_warn(xhci, "Port change event, no port for port ID %u\n",
1613                           port_id);
1614                 bogus_port_status = true;
1615                 goto cleanup;
1616         }
1617 
1618         /* We might get interrupts after shared_hcd is removed */
1619         if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) {
1620                 xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n");
1621                 bogus_port_status = true;
1622                 goto cleanup;
1623         }
1624 
1625         hcd = port->rhub->hcd;
1626         bus_state = &port->rhub->bus_state;
1627         hcd_portnum = port->hcd_portnum;
1628         portsc = readl(port->addr);
1629 
1630         xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n",
1631                  hcd->self.busnum, hcd_portnum + 1, port_id, portsc);
1632 
1633         trace_xhci_handle_port_status(hcd_portnum, portsc);
1634 
1635         if (hcd->state == HC_STATE_SUSPENDED) {
1636                 xhci_dbg(xhci, "resume root hub\n");
1637                 usb_hcd_resume_root_hub(hcd);
1638         }
1639 
1640         if (hcd->speed >= HCD_USB3 &&
1641             (portsc & PORT_PLS_MASK) == XDEV_INACTIVE) {
1642                 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1);
1643                 if (slot_id && xhci->devs[slot_id])
1644                         xhci->devs[slot_id]->flags |= VDEV_PORT_ERROR;
1645         }
1646 
1647         if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) {
1648                 xhci_dbg(xhci, "port resume event for port %d\n", port_id);
1649 
1650                 cmd_reg = readl(&xhci->op_regs->command);
1651                 if (!(cmd_reg & CMD_RUN)) {
1652                         xhci_warn(xhci, "xHC is not running.\n");
1653                         goto cleanup;
1654                 }
1655 
1656                 if (DEV_SUPERSPEED_ANY(portsc)) {
1657                         xhci_dbg(xhci, "remote wake SS port %d\n", port_id);
1658                         /* Set a flag to say the port signaled remote wakeup,
1659                          * so we can tell the difference between the end of
1660                          * device and host initiated resume.
1661                          */
1662                         bus_state->port_remote_wakeup |= 1 << hcd_portnum;
1663                         xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1664                         usb_hcd_start_port_resume(&hcd->self, hcd_portnum);
1665                         xhci_set_link_state(xhci, port, XDEV_U0);
1666                         /* Need to wait until the next link state change
1667                          * indicates the device is actually in U0.
1668                          */
1669                         bogus_port_status = true;
1670                         goto cleanup;
1671                 } else if (!test_bit(hcd_portnum, &bus_state->resuming_ports)) {
1672                         xhci_dbg(xhci, "resume HS port %d\n", port_id);
1673                         bus_state->resume_done[hcd_portnum] = jiffies +
1674                                 msecs_to_jiffies(USB_RESUME_TIMEOUT);
1675                         set_bit(hcd_portnum, &bus_state->resuming_ports);
1676                         /* Do the rest in GetPortStatus after resume time delay.
1677                          * Avoid polling roothub status before that so that a
1678                          * usb device auto-resume latency around ~40ms.
1679                          */
1680                         set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1681                         mod_timer(&hcd->rh_timer,
1682                                   bus_state->resume_done[hcd_portnum]);
1683                         usb_hcd_start_port_resume(&hcd->self, hcd_portnum);
1684                         bogus_port_status = true;
1685                 }
1686         }
1687 
1688         if ((portsc & PORT_PLC) &&
1689             DEV_SUPERSPEED_ANY(portsc) &&
1690             ((portsc & PORT_PLS_MASK) == XDEV_U0 ||
1691              (portsc & PORT_PLS_MASK) == XDEV_U1 ||
1692              (portsc & PORT_PLS_MASK) == XDEV_U2)) {
1693                 xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
1694                 complete(&bus_state->u3exit_done[hcd_portnum]);
1695                 /* We've just brought the device into U0/1/2 through either the
1696                  * Resume state after a device remote wakeup, or through the
1697                  * U3Exit state after a host-initiated resume.  If it's a device
1698                  * initiated remote wake, don't pass up the link state change,
1699                  * so the roothub behavior is consistent with external
1700                  * USB 3.0 hub behavior.
1701                  */
1702                 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1);
1703                 if (slot_id && xhci->devs[slot_id])
1704                         xhci_ring_device(xhci, slot_id);
1705                 if (bus_state->port_remote_wakeup & (1 << hcd_portnum)) {
1706                         xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1707                         usb_wakeup_notification(hcd->self.root_hub,
1708                                         hcd_portnum + 1);
1709                         bogus_port_status = true;
1710                         goto cleanup;
1711                 }
1712         }
1713 
1714         /*
1715          * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or
1716          * RExit to a disconnect state).  If so, let the the driver know it's
1717          * out of the RExit state.
1718          */
1719         if (!DEV_SUPERSPEED_ANY(portsc) && hcd->speed < HCD_USB3 &&
1720                         test_and_clear_bit(hcd_portnum,
1721                                 &bus_state->rexit_ports)) {
1722                 complete(&bus_state->rexit_done[hcd_portnum]);
1723                 bogus_port_status = true;
1724                 goto cleanup;
1725         }
1726 
1727         if (hcd->speed < HCD_USB3) {
1728                 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1729                 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) &&
1730                     (portsc & PORT_CSC) && !(portsc & PORT_CONNECT))
1731                         xhci_cavium_reset_phy_quirk(xhci);
1732         }
1733 
1734 cleanup:
1735         /* Update event ring dequeue pointer before dropping the lock */
1736         inc_deq(xhci, xhci->event_ring);
1737 
1738         /* Don't make the USB core poll the roothub if we got a bad port status
1739          * change event.  Besides, at that point we can't tell which roothub
1740          * (USB 2.0 or USB 3.0) to kick.
1741          */
1742         if (bogus_port_status)
1743                 return;
1744 
1745         /*
1746          * xHCI port-status-change events occur when the "or" of all the
1747          * status-change bits in the portsc register changes from 0 to 1.
1748          * New status changes won't cause an event if any other change
1749          * bits are still set.  When an event occurs, switch over to
1750          * polling to avoid losing status changes.
1751          */
1752         xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1753         set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1754         spin_unlock(&xhci->lock);
1755         /* Pass this up to the core */
1756         usb_hcd_poll_rh_status(hcd);
1757         spin_lock(&xhci->lock);
1758 }
1759 
1760 /*
1761  * This TD is defined by the TRBs starting at start_trb in start_seg and ending
1762  * at end_trb, which may be in another segment.  If the suspect DMA address is a
1763  * TRB in this TD, this function returns that TRB's segment.  Otherwise it
1764  * returns 0.
1765  */
1766 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci,
1767                 struct xhci_segment *start_seg,
1768                 union xhci_trb  *start_trb,
1769                 union xhci_trb  *end_trb,
1770                 dma_addr_t      suspect_dma,
1771                 bool            debug)
1772 {
1773         dma_addr_t start_dma;
1774         dma_addr_t end_seg_dma;
1775         dma_addr_t end_trb_dma;
1776         struct xhci_segment *cur_seg;
1777 
1778         start_dma = xhci_trb_virt_to_dma(start_seg, start_trb);
1779         cur_seg = start_seg;
1780 
1781         do {
1782                 if (start_dma == 0)
1783                         return NULL;
1784                 /* We may get an event for a Link TRB in the middle of a TD */
1785                 end_seg_dma = xhci_trb_virt_to_dma(cur_seg,
1786                                 &cur_seg->trbs[TRBS_PER_SEGMENT - 1]);
1787                 /* If the end TRB isn't in this segment, this is set to 0 */
1788                 end_trb_dma = xhci_trb_virt_to_dma(cur_seg, end_trb);
1789 
1790                 if (debug)
1791                         xhci_warn(xhci,
1792                                 "Looking for event-dma %016llx trb-start %016llx trb-end %016llx seg-start %016llx seg-end %016llx\n",
1793                                 (unsigned long long)suspect_dma,
1794                                 (unsigned long long)start_dma,
1795                                 (unsigned long long)end_trb_dma,
1796                                 (unsigned long long)cur_seg->dma,
1797                                 (unsigned long long)end_seg_dma);
1798 
1799                 if (end_trb_dma > 0) {
1800                         /* The end TRB is in this segment, so suspect should be here */
1801                         if (start_dma <= end_trb_dma) {
1802                                 if (suspect_dma >= start_dma && suspect_dma <= end_trb_dma)
1803                                         return cur_seg;
1804                         } else {
1805                                 /* Case for one segment with
1806                                  * a TD wrapped around to the top
1807                                  */
1808                                 if ((suspect_dma >= start_dma &&
1809                                                         suspect_dma <= end_seg_dma) ||
1810                                                 (suspect_dma >= cur_seg->dma &&
1811                                                  suspect_dma <= end_trb_dma))
1812                                         return cur_seg;
1813                         }
1814                         return NULL;
1815                 } else {
1816                         /* Might still be somewhere in this segment */
1817                         if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma)
1818                                 return cur_seg;
1819                 }
1820                 cur_seg = cur_seg->next;
1821                 start_dma = xhci_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]);
1822         } while (cur_seg != start_seg);
1823 
1824         return NULL;
1825 }
1826 
1827 static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td,
1828                 struct xhci_virt_ep *ep)
1829 {
1830         /*
1831          * As part of low/full-speed endpoint-halt processing
1832          * we must clear the TT buffer (USB 2.0 specification 11.17.5).
1833          */
1834         if (td->urb->dev->tt && !usb_pipeint(td->urb->pipe) &&
1835             (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) &&
1836             !(ep->ep_state & EP_CLEARING_TT)) {
1837                 ep->ep_state |= EP_CLEARING_TT;
1838                 td->urb->ep->hcpriv = td->urb->dev;
1839                 if (usb_hub_clear_tt_buffer(td->urb))
1840                         ep->ep_state &= ~EP_CLEARING_TT;
1841         }
1842 }
1843 
1844 static void xhci_cleanup_halted_endpoint(struct xhci_hcd *xhci,
1845                 unsigned int slot_id, unsigned int ep_index,
1846                 unsigned int stream_id, struct xhci_td *td,
1847                 enum xhci_ep_reset_type reset_type)
1848 {
1849         struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
1850         struct xhci_command *command;
1851 
1852         /*
1853          * Avoid resetting endpoint if link is inactive. Can cause host hang.
1854          * Device will be reset soon to recover the link so don't do anything
1855          */
1856         if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR)
1857                 return;
1858 
1859         command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1860         if (!command)
1861                 return;
1862 
1863         ep->ep_state |= EP_HALTED;
1864 
1865         xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type);
1866 
1867         if (reset_type == EP_HARD_RESET) {
1868                 ep->ep_state |= EP_HARD_CLEAR_TOGGLE;
1869                 xhci_cleanup_stalled_ring(xhci, slot_id, ep_index, stream_id,
1870                                           td);
1871         }
1872         xhci_ring_cmd_db(xhci);
1873 }
1874 
1875 /* Check if an error has halted the endpoint ring.  The class driver will
1876  * cleanup the halt for a non-default control endpoint if we indicate a stall.
1877  * However, a babble and other errors also halt the endpoint ring, and the class
1878  * driver won't clear the halt in that case, so we need to issue a Set Transfer
1879  * Ring Dequeue Pointer command manually.
1880  */
1881 static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
1882                 struct xhci_ep_ctx *ep_ctx,
1883                 unsigned int trb_comp_code)
1884 {
1885         /* TRB completion codes that may require a manual halt cleanup */
1886         if (trb_comp_code == COMP_USB_TRANSACTION_ERROR ||
1887                         trb_comp_code == COMP_BABBLE_DETECTED_ERROR ||
1888                         trb_comp_code == COMP_SPLIT_TRANSACTION_ERROR)
1889                 /* The 0.95 spec says a babbling control endpoint
1890                  * is not halted. The 0.96 spec says it is.  Some HW
1891                  * claims to be 0.95 compliant, but it halts the control
1892                  * endpoint anyway.  Check if a babble halted the
1893                  * endpoint.
1894                  */
1895                 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_HALTED)
1896                         return 1;
1897 
1898         return 0;
1899 }
1900 
1901 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code)
1902 {
1903         if (trb_comp_code >= 224 && trb_comp_code <= 255) {
1904                 /* Vendor defined "informational" completion code,
1905                  * treat as not-an-error.
1906                  */
1907                 xhci_dbg(xhci, "Vendor defined info completion code %u\n",
1908                                 trb_comp_code);
1909                 xhci_dbg(xhci, "Treating code as success.\n");
1910                 return 1;
1911         }
1912         return 0;
1913 }
1914 
1915 static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td,
1916                 struct xhci_ring *ep_ring, int *status)
1917 {
1918         struct urb *urb = NULL;
1919 
1920         /* Clean up the endpoint's TD list */
1921         urb = td->urb;
1922 
1923         /* if a bounce buffer was used to align this td then unmap it */
1924         xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
1925 
1926         /* Do one last check of the actual transfer length.
1927          * If the host controller said we transferred more data than the buffer
1928          * length, urb->actual_length will be a very big number (since it's
1929          * unsigned).  Play it safe and say we didn't transfer anything.
1930          */
1931         if (urb->actual_length > urb->transfer_buffer_length) {
1932                 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
1933                           urb->transfer_buffer_length, urb->actual_length);
1934                 urb->actual_length = 0;
1935                 *status = 0;
1936         }
1937         list_del_init(&td->td_list);
1938         /* Was this TD slated to be cancelled but completed anyway? */
1939         if (!list_empty(&td->cancelled_td_list))
1940                 list_del_init(&td->cancelled_td_list);
1941 
1942         inc_td_cnt(urb);
1943         /* Giveback the urb when all the tds are completed */
1944         if (last_td_in_urb(td)) {
1945                 if ((urb->actual_length != urb->transfer_buffer_length &&
1946                      (urb->transfer_flags & URB_SHORT_NOT_OK)) ||
1947                     (*status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc)))
1948                         xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
1949                                  urb, urb->actual_length,
1950                                  urb->transfer_buffer_length, *status);
1951 
1952                 /* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */
1953                 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
1954                         *status = 0;
1955                 xhci_giveback_urb_in_irq(xhci, td, *status);
1956         }
1957 
1958         return 0;
1959 }
1960 
1961 static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
1962         struct xhci_transfer_event *event,
1963         struct xhci_virt_ep *ep, int *status)
1964 {
1965         struct xhci_virt_device *xdev;
1966         struct xhci_ep_ctx *ep_ctx;
1967         struct xhci_ring *ep_ring;
1968         unsigned int slot_id;
1969         u32 trb_comp_code;
1970         int ep_index;
1971 
1972         slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1973         xdev = xhci->devs[slot_id];
1974         ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1975         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1976         ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1977         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1978 
1979         if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
1980                         trb_comp_code == COMP_STOPPED ||
1981                         trb_comp_code == COMP_STOPPED_SHORT_PACKET) {
1982                 /* The Endpoint Stop Command completion will take care of any
1983                  * stopped TDs.  A stopped TD may be restarted, so don't update
1984                  * the ring dequeue pointer or take this TD off any lists yet.
1985                  */
1986                 return 0;
1987         }
1988         if (trb_comp_code == COMP_STALL_ERROR ||
1989                 xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
1990                                                 trb_comp_code)) {
1991                 /*
1992                  * xhci internal endpoint state will go to a "halt" state for
1993                  * any stall, including default control pipe protocol stall.
1994                  * To clear the host side halt we need to issue a reset endpoint
1995                  * command, followed by a set dequeue command to move past the
1996                  * TD.
1997                  * Class drivers clear the device side halt from a functional
1998                  * stall later. Hub TT buffer should only be cleared for FS/LS
1999                  * devices behind HS hubs for functional stalls.
2000                  */
2001                 if ((ep_index != 0) || (trb_comp_code != COMP_STALL_ERROR))
2002                         xhci_clear_hub_tt_buffer(xhci, td, ep);
2003                 xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
2004                                         ep_ring->stream_id, td, EP_HARD_RESET);
2005         } else {
2006                 /* Update ring dequeue pointer */
2007                 while (ep_ring->dequeue != td->last_trb)
2008                         inc_deq(xhci, ep_ring);
2009                 inc_deq(xhci, ep_ring);
2010         }
2011 
2012         return xhci_td_cleanup(xhci, td, ep_ring, status);
2013 }
2014 
2015 /* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */
2016 static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring,
2017                            union xhci_trb *stop_trb)
2018 {
2019         u32 sum;
2020         union xhci_trb *trb = ring->dequeue;
2021         struct xhci_segment *seg = ring->deq_seg;
2022 
2023         for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) {
2024                 if (!trb_is_noop(trb) && !trb_is_link(trb))
2025                         sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
2026         }
2027         return sum;
2028 }
2029 
2030 /*
2031  * Process control tds, update urb status and actual_length.
2032  */
2033 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
2034         union xhci_trb *ep_trb, struct xhci_transfer_event *event,
2035         struct xhci_virt_ep *ep, int *status)
2036 {
2037         struct xhci_virt_device *xdev;
2038         unsigned int slot_id;
2039         int ep_index;
2040         struct xhci_ep_ctx *ep_ctx;
2041         u32 trb_comp_code;
2042         u32 remaining, requested;
2043         u32 trb_type;
2044 
2045         trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3]));
2046         slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2047         xdev = xhci->devs[slot_id];
2048         ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2049         ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2050         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2051         requested = td->urb->transfer_buffer_length;
2052         remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2053 
2054         switch (trb_comp_code) {
2055         case COMP_SUCCESS:
2056                 if (trb_type != TRB_STATUS) {
2057                         xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
2058                                   (trb_type == TRB_DATA) ? "data" : "setup");
2059                         *status = -ESHUTDOWN;
2060                         break;
2061                 }
2062                 *status = 0;
2063                 break;
2064         case COMP_SHORT_PACKET:
2065                 *status = 0;
2066                 break;
2067         case COMP_STOPPED_SHORT_PACKET:
2068                 if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
2069                         td->urb->actual_length = remaining;
2070                 else
2071                         xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
2072                 goto finish_td;
2073         case COMP_STOPPED:
2074                 switch (trb_type) {
2075                 case TRB_SETUP:
2076                         td->urb->actual_length = 0;
2077                         goto finish_td;
2078                 case TRB_DATA:
2079                 case TRB_NORMAL:
2080                         td->urb->actual_length = requested - remaining;
2081                         goto finish_td;
2082                 case TRB_STATUS:
2083                         td->urb->actual_length = requested;
2084                         goto finish_td;
2085                 default:
2086                         xhci_warn(xhci, "WARN: unexpected TRB Type %d\n",
2087                                   trb_type);
2088                         goto finish_td;
2089                 }
2090         case COMP_STOPPED_LENGTH_INVALID:
2091                 goto finish_td;
2092         default:
2093                 if (!xhci_requires_manual_halt_cleanup(xhci,
2094                                                        ep_ctx, trb_comp_code))
2095                         break;
2096                 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
2097                          trb_comp_code, ep_index);
2098                 /* else fall through */
2099         case COMP_STALL_ERROR:
2100                 /* Did we transfer part of the data (middle) phase? */
2101                 if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
2102                         td->urb->actual_length = requested - remaining;
2103                 else if (!td->urb_length_set)
2104                         td->urb->actual_length = 0;
2105                 goto finish_td;
2106         }
2107 
2108         /* stopped at setup stage, no data transferred */
2109         if (trb_type == TRB_SETUP)
2110                 goto finish_td;
2111 
2112         /*
2113          * if on data stage then update the actual_length of the URB and flag it
2114          * as set, so it won't be overwritten in the event for the last TRB.
2115          */
2116         if (trb_type == TRB_DATA ||
2117                 trb_type == TRB_NORMAL) {
2118                 td->urb_length_set = true;
2119                 td->urb->actual_length = requested - remaining;
2120                 xhci_dbg(xhci, "Waiting for status stage event\n");
2121                 return 0;
2122         }
2123 
2124         /* at status stage */
2125         if (!td->urb_length_set)
2126                 td->urb->actual_length = requested;
2127 
2128 finish_td:
2129         return finish_td(xhci, td, event, ep, status);
2130 }
2131 
2132 /*
2133  * Process isochronous tds, update urb packet status and actual_length.
2134  */
2135 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2136         union xhci_trb *ep_trb, struct xhci_transfer_event *event,
2137         struct xhci_virt_ep *ep, int *status)
2138 {
2139         struct xhci_ring *ep_ring;
2140         struct urb_priv *urb_priv;
2141         int idx;
2142         struct usb_iso_packet_descriptor *frame;
2143         u32 trb_comp_code;
2144         bool sum_trbs_for_length = false;
2145         u32 remaining, requested, ep_trb_len;
2146         int short_framestatus;
2147 
2148         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2149         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2150         urb_priv = td->urb->hcpriv;
2151         idx = urb_priv->num_tds_done;
2152         frame = &td->urb->iso_frame_desc[idx];
2153         requested = frame->length;
2154         remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2155         ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
2156         short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
2157                 -EREMOTEIO : 0;
2158 
2159         /* handle completion code */
2160         switch (trb_comp_code) {
2161         case COMP_SUCCESS:
2162                 if (remaining) {
2163                         frame->status = short_framestatus;
2164                         if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
2165                                 sum_trbs_for_length = true;
2166                         break;
2167                 }
2168                 frame->status = 0;
2169                 break;
2170         case COMP_SHORT_PACKET:
2171                 frame->status = short_framestatus;
2172                 sum_trbs_for_length = true;
2173                 break;
2174         case COMP_BANDWIDTH_OVERRUN_ERROR:
2175                 frame->status = -ECOMM;
2176                 break;
2177         case COMP_ISOCH_BUFFER_OVERRUN:
2178         case COMP_BABBLE_DETECTED_ERROR:
2179                 frame->status = -EOVERFLOW;
2180                 break;
2181         case COMP_INCOMPATIBLE_DEVICE_ERROR:
2182         case COMP_STALL_ERROR:
2183                 frame->status = -EPROTO;
2184                 break;
2185         case COMP_USB_TRANSACTION_ERROR:
2186                 frame->status = -EPROTO;
2187                 if (ep_trb != td->last_trb)
2188                         return 0;
2189                 break;
2190         case COMP_STOPPED:
2191                 sum_trbs_for_length = true;
2192                 break;
2193         case COMP_STOPPED_SHORT_PACKET:
2194                 /* field normally containing residue now contains tranferred */
2195                 frame->status = short_framestatus;
2196                 requested = remaining;
2197                 break;
2198         case COMP_STOPPED_LENGTH_INVALID:
2199                 requested = 0;
2200                 remaining = 0;
2201                 break;
2202         default:
2203                 sum_trbs_for_length = true;
2204                 frame->status = -1;
2205                 break;
2206         }
2207 
2208         if (sum_trbs_for_length)
2209                 frame->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb) +
2210                         ep_trb_len - remaining;
2211         else
2212                 frame->actual_length = requested;
2213 
2214         td->urb->actual_length += frame->actual_length;
2215 
2216         return finish_td(xhci, td, event, ep, status);
2217 }
2218 
2219 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2220                         struct xhci_transfer_event *event,
2221                         struct xhci_virt_ep *ep, int *status)
2222 {
2223         struct xhci_ring *ep_ring;
2224         struct urb_priv *urb_priv;
2225         struct usb_iso_packet_descriptor *frame;
2226         int idx;
2227 
2228         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2229         urb_priv = td->urb->hcpriv;
2230         idx = urb_priv->num_tds_done;
2231         frame = &td->urb->iso_frame_desc[idx];
2232 
2233         /* The transfer is partly done. */
2234         frame->status = -EXDEV;
2235 
2236         /* calc actual length */
2237         frame->actual_length = 0;
2238 
2239         /* Update ring dequeue pointer */
2240         while (ep_ring->dequeue != td->last_trb)
2241                 inc_deq(xhci, ep_ring);
2242         inc_deq(xhci, ep_ring);
2243 
2244         return xhci_td_cleanup(xhci, td, ep_ring, status);
2245 }
2246 
2247 /*
2248  * Process bulk and interrupt tds, update urb status and actual_length.
2249  */
2250 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
2251         union xhci_trb *ep_trb, struct xhci_transfer_event *event,
2252         struct xhci_virt_ep *ep, int *status)
2253 {
2254         struct xhci_slot_ctx *slot_ctx;
2255         struct xhci_ring *ep_ring;
2256         u32 trb_comp_code;
2257         u32 remaining, requested, ep_trb_len;
2258         unsigned int slot_id;
2259         int ep_index;
2260 
2261         slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2262         slot_ctx = xhci_get_slot_ctx(xhci, xhci->devs[slot_id]->out_ctx);
2263         ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2264         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2265         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2266         remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2267         ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
2268         requested = td->urb->transfer_buffer_length;
2269 
2270         switch (trb_comp_code) {
2271         case COMP_SUCCESS:
2272                 ep_ring->err_count = 0;
2273                 /* handle success with untransferred data as short packet */
2274                 if (ep_trb != td->last_trb || remaining) {
2275                         xhci_warn(xhci, "WARN Successful completion on short TX\n");
2276                         xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2277                                  td->urb->ep->desc.bEndpointAddress,
2278                                  requested, remaining);
2279                 }
2280                 *status = 0;
2281                 break;
2282         case COMP_SHORT_PACKET:
2283                 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2284                          td->urb->ep->desc.bEndpointAddress,
2285                          requested, remaining);
2286                 *status = 0;
2287                 break;
2288         case COMP_STOPPED_SHORT_PACKET:
2289                 td->urb->actual_length = remaining;
2290                 goto finish_td;
2291         case COMP_STOPPED_LENGTH_INVALID:
2292                 /* stopped on ep trb with invalid length, exclude it */
2293                 ep_trb_len      = 0;
2294                 remaining       = 0;
2295                 break;
2296         case COMP_USB_TRANSACTION_ERROR:
2297                 if ((ep_ring->err_count++ > MAX_SOFT_RETRY) ||
2298                     le32_to_cpu(slot_ctx->tt_info) & TT_SLOT)
2299                         break;
2300                 *status = 0;
2301                 xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
2302                                         ep_ring->stream_id, td, EP_SOFT_RESET);
2303                 return 0;
2304         default:
2305                 /* do nothing */
2306                 break;
2307         }
2308 
2309         if (ep_trb == td->last_trb)
2310                 td->urb->actual_length = requested - remaining;
2311         else
2312                 td->urb->actual_length =
2313                         sum_trb_lengths(xhci, ep_ring, ep_trb) +
2314                         ep_trb_len - remaining;
2315 finish_td:
2316         if (remaining > requested) {
2317                 xhci_warn(xhci, "bad transfer trb length %d in event trb\n",
2318                           remaining);
2319                 td->urb->actual_length = 0;
2320         }
2321         return finish_td(xhci, td, event, ep, status);
2322 }
2323 
2324 /*
2325  * If this function returns an error condition, it means it got a Transfer
2326  * event with a corrupted Slot ID, Endpoint ID, or TRB DMA address.
2327  * At this point, the host controller is probably hosed and should be reset.
2328  */
2329 static int handle_tx_event(struct xhci_hcd *xhci,
2330                 struct xhci_transfer_event *event)
2331 {
2332         struct xhci_virt_device *xdev;
2333         struct xhci_virt_ep *ep;
2334         struct xhci_ring *ep_ring;
2335         unsigned int slot_id;
2336         int ep_index;
2337         struct xhci_td *td = NULL;
2338         dma_addr_t ep_trb_dma;
2339         struct xhci_segment *ep_seg;
2340         union xhci_trb *ep_trb;
2341         int status = -EINPROGRESS;
2342         struct xhci_ep_ctx *ep_ctx;
2343         struct list_head *tmp;
2344         u32 trb_comp_code;
2345         int td_num = 0;
2346         bool handling_skipped_tds = false;
2347 
2348         slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2349         ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2350         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2351         ep_trb_dma = le64_to_cpu(event->buffer);
2352 
2353         xdev = xhci->devs[slot_id];
2354         if (!xdev) {
2355                 xhci_err(xhci, "ERROR Transfer event pointed to bad slot %u\n",
2356                          slot_id);
2357                 goto err_out;
2358         }
2359 
2360         ep = &xdev->eps[ep_index];
2361         ep_ring = xhci_dma_to_transfer_ring(ep, ep_trb_dma);
2362         ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2363 
2364         if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) {
2365                 xhci_err(xhci,
2366                          "ERROR Transfer event for disabled endpoint slot %u ep %u\n",
2367                           slot_id, ep_index);
2368                 goto err_out;
2369         }
2370 
2371         /* Some transfer events don't always point to a trb, see xhci 4.17.4 */
2372         if (!ep_ring) {
2373                 switch (trb_comp_code) {
2374                 case COMP_STALL_ERROR:
2375                 case COMP_USB_TRANSACTION_ERROR:
2376                 case COMP_INVALID_STREAM_TYPE_ERROR:
2377                 case COMP_INVALID_STREAM_ID_ERROR:
2378                         xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index, 0,
2379                                                      NULL, EP_SOFT_RESET);
2380                         goto cleanup;
2381                 case COMP_RING_UNDERRUN:
2382                 case COMP_RING_OVERRUN:
2383                 case COMP_STOPPED_LENGTH_INVALID:
2384                         goto cleanup;
2385                 default:
2386                         xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n",
2387                                  slot_id, ep_index);
2388                         goto err_out;
2389                 }
2390         }
2391 
2392         /* Count current td numbers if ep->skip is set */
2393         if (ep->skip) {
2394                 list_for_each(tmp, &ep_ring->td_list)
2395                         td_num++;
2396         }
2397 
2398         /* Look for common error cases */
2399         switch (trb_comp_code) {
2400         /* Skip codes that require special handling depending on
2401          * transfer type
2402          */
2403         case COMP_SUCCESS:
2404                 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0)
2405                         break;
2406                 if (xhci->quirks & XHCI_TRUST_TX_LENGTH ||
2407                     ep_ring->last_td_was_short)
2408                         trb_comp_code = COMP_SHORT_PACKET;
2409                 else
2410                         xhci_warn_ratelimited(xhci,
2411                                               "WARN Successful completion on short TX for slot %u ep %u: needs XHCI_TRUST_TX_LENGTH quirk?\n",
2412                                               slot_id, ep_index);
2413         case COMP_SHORT_PACKET:
2414                 break;
2415         /* Completion codes for endpoint stopped state */
2416         case COMP_STOPPED:
2417                 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n",
2418                          slot_id, ep_index);
2419                 break;
2420         case COMP_STOPPED_LENGTH_INVALID:
2421                 xhci_dbg(xhci,
2422                          "Stopped on No-op or Link TRB for slot %u ep %u\n",
2423                          slot_id, ep_index);
2424                 break;
2425         case COMP_STOPPED_SHORT_PACKET:
2426                 xhci_dbg(xhci,
2427                          "Stopped with short packet transfer detected for slot %u ep %u\n",
2428                          slot_id, ep_index);
2429                 break;
2430         /* Completion codes for endpoint halted state */
2431         case COMP_STALL_ERROR:
2432                 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id,
2433                          ep_index);
2434                 ep->ep_state |= EP_HALTED;
2435                 status = -EPIPE;
2436                 break;
2437         case COMP_SPLIT_TRANSACTION_ERROR:
2438         case COMP_USB_TRANSACTION_ERROR:
2439                 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n",
2440                          slot_id, ep_index);
2441                 status = -EPROTO;
2442                 break;
2443         case COMP_BABBLE_DETECTED_ERROR:
2444                 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n",
2445                          slot_id, ep_index);
2446                 status = -EOVERFLOW;
2447                 break;
2448         /* Completion codes for endpoint error state */
2449         case COMP_TRB_ERROR:
2450                 xhci_warn(xhci,
2451                           "WARN: TRB error for slot %u ep %u on endpoint\n",
2452                           slot_id, ep_index);
2453                 status = -EILSEQ;
2454                 break;
2455         /* completion codes not indicating endpoint state change */
2456         case COMP_DATA_BUFFER_ERROR:
2457                 xhci_warn(xhci,
2458                           "WARN: HC couldn't access mem fast enough for slot %u ep %u\n",
2459                           slot_id, ep_index);
2460                 status = -ENOSR;
2461                 break;
2462         case COMP_BANDWIDTH_OVERRUN_ERROR:
2463                 xhci_warn(xhci,
2464                           "WARN: bandwidth overrun event for slot %u ep %u on endpoint\n",
2465                           slot_id, ep_index);
2466                 break;
2467         case COMP_ISOCH_BUFFER_OVERRUN:
2468                 xhci_warn(xhci,
2469                           "WARN: buffer overrun event for slot %u ep %u on endpoint",
2470                           slot_id, ep_index);
2471                 break;
2472         case COMP_RING_UNDERRUN:
2473                 /*
2474                  * When the Isoch ring is empty, the xHC will generate
2475                  * a Ring Overrun Event for IN Isoch endpoint or Ring
2476                  * Underrun Event for OUT Isoch endpoint.
2477                  */
2478                 xhci_dbg(xhci, "underrun event on endpoint\n");
2479                 if (!list_empty(&ep_ring->td_list))
2480                         xhci_dbg(xhci, "Underrun Event for slot %d ep %d "
2481                                         "still with TDs queued?\n",
2482                                  TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2483                                  ep_index);
2484                 goto cleanup;
2485         case COMP_RING_OVERRUN:
2486                 xhci_dbg(xhci, "overrun event on endpoint\n");
2487                 if (!list_empty(&ep_ring->td_list))
2488                         xhci_dbg(xhci, "Overrun Event for slot %d ep %d "
2489                                         "still with TDs queued?\n",
2490                                  TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2491                                  ep_index);
2492                 goto cleanup;
2493         case COMP_MISSED_SERVICE_ERROR:
2494                 /*
2495                  * When encounter missed service error, one or more isoc tds
2496                  * may be missed by xHC.
2497                  * Set skip flag of the ep_ring; Complete the missed tds as
2498                  * short transfer when process the ep_ring next time.
2499                  */
2500                 ep->skip = true;
2501                 xhci_dbg(xhci,
2502                          "Miss service interval error for slot %u ep %u, set skip flag\n",
2503                          slot_id, ep_index);
2504                 goto cleanup;
2505         case COMP_NO_PING_RESPONSE_ERROR:
2506                 ep->skip = true;
2507                 xhci_dbg(xhci,
2508                          "No Ping response error for slot %u ep %u, Skip one Isoc TD\n",
2509                          slot_id, ep_index);
2510                 goto cleanup;
2511 
2512         case COMP_INCOMPATIBLE_DEVICE_ERROR:
2513                 /* needs disable slot command to recover */
2514                 xhci_warn(xhci,
2515                           "WARN: detect an incompatible device for slot %u ep %u",
2516                           slot_id, ep_index);
2517                 status = -EPROTO;
2518                 break;
2519         default:
2520                 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) {
2521                         status = 0;
2522                         break;
2523                 }
2524                 xhci_warn(xhci,
2525                           "ERROR Unknown event condition %u for slot %u ep %u , HC probably busted\n",
2526                           trb_comp_code, slot_id, ep_index);
2527                 goto cleanup;
2528         }
2529 
2530         do {
2531                 /* This TRB should be in the TD at the head of this ring's
2532                  * TD list.
2533                  */
2534                 if (list_empty(&ep_ring->td_list)) {
2535                         /*
2536                          * Don't print wanings if it's due to a stopped endpoint
2537                          * generating an extra completion event if the device
2538                          * was suspended. Or, a event for the last TRB of a
2539                          * short TD we already got a short event for.
2540                          * The short TD is already removed from the TD list.
2541                          */
2542 
2543                         if (!(trb_comp_code == COMP_STOPPED ||
2544                               trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
2545                               ep_ring->last_td_was_short)) {
2546                                 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n",
2547                                                 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2548                                                 ep_index);
2549                         }
2550                         if (ep->skip) {
2551                                 ep->skip = false;
2552                                 xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n",
2553                                          slot_id, ep_index);
2554                         }
2555                         if (trb_comp_code == COMP_STALL_ERROR ||
2556                             xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
2557                                                               trb_comp_code)) {
2558                                 xhci_cleanup_halted_endpoint(xhci, slot_id,
2559                                                              ep_index,
2560                                                              ep_ring->stream_id,
2561                                                              NULL,
2562                                                              EP_HARD_RESET);
2563                         }
2564                         goto cleanup;
2565                 }
2566 
2567                 /* We've skipped all the TDs on the ep ring when ep->skip set */
2568                 if (ep->skip && td_num == 0) {
2569                         ep->skip = false;
2570                         xhci_dbg(xhci, "All tds on the ep_ring skipped. Clear skip flag for slot %u ep %u.\n",
2571                                  slot_id, ep_index);
2572                         goto cleanup;
2573                 }
2574 
2575                 td = list_first_entry(&ep_ring->td_list, struct xhci_td,
2576                                       td_list);
2577                 if (ep->skip)
2578                         td_num--;
2579 
2580                 /* Is this a TRB in the currently executing TD? */
2581                 ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,
2582                                 td->last_trb, ep_trb_dma, false);
2583 
2584                 /*
2585                  * Skip the Force Stopped Event. The event_trb(event_dma) of FSE
2586                  * is not in the current TD pointed by ep_ring->dequeue because
2587                  * that the hardware dequeue pointer still at the previous TRB
2588                  * of the current TD. The previous TRB maybe a Link TD or the
2589                  * last TRB of the previous TD. The command completion handle
2590                  * will take care the rest.
2591                  */
2592                 if (!ep_seg && (trb_comp_code == COMP_STOPPED ||
2593                            trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) {
2594                         goto cleanup;
2595                 }
2596 
2597                 if (!ep_seg) {
2598                         if (!ep->skip ||
2599                             !usb_endpoint_xfer_isoc(&td->urb->ep->desc)) {
2600                                 /* Some host controllers give a spurious
2601                                  * successful event after a short transfer.
2602                                  * Ignore it.
2603                                  */
2604                                 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) &&
2605                                                 ep_ring->last_td_was_short) {
2606                                         ep_ring->last_td_was_short = false;
2607                                         goto cleanup;
2608                                 }
2609                                 /* HC is busted, give up! */
2610                                 xhci_err(xhci,
2611                                         "ERROR Transfer event TRB DMA ptr not "
2612                                         "part of current TD ep_index %d "
2613                                         "comp_code %u\n", ep_index,
2614                                         trb_comp_code);
2615                                 trb_in_td(xhci, ep_ring->deq_seg,
2616                                           ep_ring->dequeue, td->last_trb,
2617                                           ep_trb_dma, true);
2618                                 return -ESHUTDOWN;
2619                         }
2620 
2621                         skip_isoc_td(xhci, td, event, ep, &status);
2622                         goto cleanup;
2623                 }
2624                 if (trb_comp_code == COMP_SHORT_PACKET)
2625                         ep_ring->last_td_was_short = true;
2626                 else
2627                         ep_ring->last_td_was_short = false;
2628 
2629                 if (ep->skip) {
2630                         xhci_dbg(xhci,
2631                                  "Found td. Clear skip flag for slot %u ep %u.\n",
2632                                  slot_id, ep_index);
2633                         ep->skip = false;
2634                 }
2635 
2636                 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) /
2637                                                 sizeof(*ep_trb)];
2638 
2639                 trace_xhci_handle_transfer(ep_ring,
2640                                 (struct xhci_generic_trb *) ep_trb);
2641 
2642                 /*
2643                  * No-op TRB could trigger interrupts in a case where
2644                  * a URB was killed and a STALL_ERROR happens right
2645                  * after the endpoint ring stopped. Reset the halted
2646                  * endpoint. Otherwise, the endpoint remains stalled
2647                  * indefinitely.
2648                  */
2649                 if (trb_is_noop(ep_trb)) {
2650                         if (trb_comp_code == COMP_STALL_ERROR ||
2651                             xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
2652                                                               trb_comp_code))
2653                                 xhci_cleanup_halted_endpoint(xhci, slot_id,
2654                                                              ep_index,
2655                                                              ep_ring->stream_id,
2656                                                              td, EP_HARD_RESET);
2657                         goto cleanup;
2658                 }
2659 
2660                 /* update the urb's actual_length and give back to the core */
2661                 if (usb_endpoint_xfer_control(&td->urb->ep->desc))
2662                         process_ctrl_td(xhci, td, ep_trb, event, ep, &status);
2663                 else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc))
2664                         process_isoc_td(xhci, td, ep_trb, event, ep, &status);
2665                 else
2666                         process_bulk_intr_td(xhci, td, ep_trb, event, ep,
2667                                              &status);
2668 cleanup:
2669                 handling_skipped_tds = ep->skip &&
2670                         trb_comp_code != COMP_MISSED_SERVICE_ERROR &&
2671                         trb_comp_code != COMP_NO_PING_RESPONSE_ERROR;
2672 
2673                 /*
2674                  * Do not update event ring dequeue pointer if we're in a loop
2675                  * processing missed tds.
2676                  */
2677                 if (!handling_skipped_tds)
2678                         inc_deq(xhci, xhci->event_ring);
2679 
2680         /*
2681          * If ep->skip is set, it means there are missed tds on the
2682          * endpoint ring need to take care of.
2683          * Process them as short transfer until reach the td pointed by
2684          * the event.
2685          */
2686         } while (handling_skipped_tds);
2687 
2688         return 0;
2689 
2690 err_out:
2691         xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
2692                  (unsigned long long) xhci_trb_virt_to_dma(
2693                          xhci->event_ring->deq_seg,
2694                          xhci->event_ring->dequeue),
2695                  lower_32_bits(le64_to_cpu(event->buffer)),
2696                  upper_32_bits(le64_to_cpu(event->buffer)),
2697                  le32_to_cpu(event->transfer_len),
2698                  le32_to_cpu(event->flags));
2699         return -ENODEV;
2700 }
2701 
2702 /*
2703  * This function handles all OS-owned events on the event ring.  It may drop
2704  * xhci->lock between event processing (e.g. to pass up port status changes).
2705  * Returns >0 for "possibly more events to process" (caller should call again),
2706  * otherwise 0 if done.  In future, <0 returns should indicate error code.
2707  */
2708 static int xhci_handle_event(struct xhci_hcd *xhci)
2709 {
2710         union xhci_trb *event;
2711         int update_ptrs = 1;
2712         int ret;
2713 
2714         /* Event ring hasn't been allocated yet. */
2715         if (!xhci->event_ring || !xhci->event_ring->dequeue) {
2716                 xhci_err(xhci, "ERROR event ring not ready\n");
2717                 return -ENOMEM;
2718         }
2719 
2720         event = xhci->event_ring->dequeue;
2721         /* Does the HC or OS own the TRB? */
2722         if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
2723             xhci->event_ring->cycle_state)
2724                 return 0;
2725 
2726         trace_xhci_handle_event(xhci->event_ring, &event->generic);
2727 
2728         /*
2729          * Barrier between reading the TRB_CYCLE (valid) flag above and any
2730          * speculative reads of the event's flags/data below.
2731          */
2732         rmb();
2733         /* FIXME: Handle more event types. */
2734         switch (le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) {
2735         case TRB_TYPE(TRB_COMPLETION):
2736                 handle_cmd_completion(xhci, &event->event_cmd);
2737                 break;
2738         case TRB_TYPE(TRB_PORT_STATUS):
2739                 handle_port_status(xhci, event);
2740                 update_ptrs = 0;
2741                 break;
2742         case TRB_TYPE(TRB_TRANSFER):
2743                 ret = handle_tx_event(xhci, &event->trans_event);
2744                 if (ret >= 0)
2745                         update_ptrs = 0;
2746                 break;
2747         case TRB_TYPE(TRB_DEV_NOTE):
2748                 handle_device_notification(xhci, event);
2749                 break;
2750         default:
2751                 if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >=
2752                     TRB_TYPE(48))
2753                         handle_vendor_event(xhci, event);
2754                 else
2755                         xhci_warn(xhci, "ERROR unknown event type %d\n",
2756                                   TRB_FIELD_TO_TYPE(
2757                                   le32_to_cpu(event->event_cmd.flags)));
2758         }
2759         /* Any of the above functions may drop and re-acquire the lock, so check
2760          * to make sure a watchdog timer didn't mark the host as non-responsive.
2761          */
2762         if (xhci->xhc_state & XHCI_STATE_DYING) {
2763                 xhci_dbg(xhci, "xHCI host dying, returning from "
2764                                 "event handler.\n");
2765                 return 0;
2766         }
2767 
2768         if (update_ptrs)
2769                 /* Update SW event ring dequeue pointer */
2770                 inc_deq(xhci, xhci->event_ring);
2771 
2772         /* Are there more items on the event ring?  Caller will call us again to
2773          * check.
2774          */
2775         return 1;
2776 }
2777 
2778 /*
2779  * Update Event Ring Dequeue Pointer:
2780  * - When all events have finished
2781  * - To avoid "Event Ring Full Error" condition
2782  */
2783 static void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
2784                 union xhci_trb *event_ring_deq)
2785 {
2786         u64 temp_64;
2787         dma_addr_t deq;
2788 
2789         temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2790         /* If necessary, update the HW's version of the event ring deq ptr. */
2791         if (event_ring_deq != xhci->event_ring->dequeue) {
2792                 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
2793                                 xhci->event_ring->dequeue);
2794                 if (deq == 0)
2795                         xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n");
2796                 /*
2797                  * Per 4.9.4, Software writes to the ERDP register shall
2798                  * always advance the Event Ring Dequeue Pointer value.
2799                  */
2800                 if ((temp_64 & (u64) ~ERST_PTR_MASK) ==
2801                                 ((u64) deq & (u64) ~ERST_PTR_MASK))
2802                         return;
2803 
2804                 /* Update HC event ring dequeue pointer */
2805                 temp_64 &= ERST_PTR_MASK;
2806                 temp_64 |= ((u64) deq & (u64) ~ERST_PTR_MASK);
2807         }
2808 
2809         /* Clear the event handler busy flag (RW1C) */
2810         temp_64 |= ERST_EHB;
2811         xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue);
2812 }
2813 
2814 /*
2815  * xHCI spec says we can get an interrupt, and if the HC has an error condition,
2816  * we might get bad data out of the event ring.  Section 4.10.2.7 has a list of
2817  * indicators of an event TRB error, but we check the status *first* to be safe.
2818  */
2819 irqreturn_t xhci_irq(struct usb_hcd *hcd)
2820 {
2821         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2822         union xhci_trb *event_ring_deq;
2823         irqreturn_t ret = IRQ_NONE;
2824         unsigned long flags;
2825         u64 temp_64;
2826         u32 status;
2827         int event_loop = 0;
2828 
2829         spin_lock_irqsave(&xhci->lock, flags);
2830         /* Check if the xHC generated the interrupt, or the irq is shared */
2831         status = readl(&xhci->op_regs->status);
2832         if (status == ~(u32)0) {
2833                 xhci_hc_died(xhci);
2834                 ret = IRQ_HANDLED;
2835                 goto out;
2836         }
2837 
2838         if (!(status & STS_EINT))
2839                 goto out;
2840 
2841         if (status & STS_FATAL) {
2842                 xhci_warn(xhci, "WARNING: Host System Error\n");
2843                 xhci_halt(xhci);
2844                 ret = IRQ_HANDLED;
2845                 goto out;
2846         }
2847 
2848         /*
2849          * Clear the op reg interrupt status first,
2850          * so we can receive interrupts from other MSI-X interrupters.
2851          * Write 1 to clear the interrupt status.
2852          */
2853         status |= STS_EINT;
2854         writel(status, &xhci->op_regs->status);
2855 
2856         if (!hcd->msi_enabled) {
2857                 u32 irq_pending;
2858                 irq_pending = readl(&xhci->ir_set->irq_pending);
2859                 irq_pending |= IMAN_IP;
2860                 writel(irq_pending, &xhci->ir_set->irq_pending);
2861         }
2862 
2863         if (xhci->xhc_state & XHCI_STATE_DYING ||
2864             xhci->xhc_state & XHCI_STATE_HALTED) {
2865                 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. "
2866                                 "Shouldn't IRQs be disabled?\n");
2867                 /* Clear the event handler busy flag (RW1C);
2868                  * the event ring should be empty.
2869                  */
2870                 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2871                 xhci_write_64(xhci, temp_64 | ERST_EHB,
2872                                 &xhci->ir_set->erst_dequeue);
2873                 ret = IRQ_HANDLED;
2874                 goto out;
2875         }
2876 
2877         event_ring_deq = xhci->event_ring->dequeue;
2878         /* FIXME this should be a delayed service routine
2879          * that clears the EHB.
2880          */
2881         while (xhci_handle_event(xhci) > 0) {
2882                 if (event_loop++ < TRBS_PER_SEGMENT / 2)
2883                         continue;
2884                 xhci_update_erst_dequeue(xhci, event_ring_deq);
2885                 event_loop = 0;
2886         }
2887 
2888         xhci_update_erst_dequeue(xhci, event_ring_deq);
2889         ret = IRQ_HANDLED;
2890 
2891 out:
2892         spin_unlock_irqrestore(&xhci->lock, flags);
2893 
2894         return ret;
2895 }
2896 
2897 irqreturn_t xhci_msi_irq(int irq, void *hcd)
2898 {
2899         return xhci_irq(hcd);
2900 }
2901 
2902 /****           Endpoint Ring Operations        ****/
2903 
2904 /*
2905  * Generic function for queueing a TRB on a ring.
2906  * The caller must have checked to make sure there's room on the ring.
2907  *
2908  * @more_trbs_coming:   Will you enqueue more TRBs before calling
2909  *                      prepare_transfer()?
2910  */
2911 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
2912                 bool more_trbs_coming,
2913                 u32 field1, u32 field2, u32 field3, u32 field4)
2914 {
2915         struct xhci_generic_trb *trb;
2916 
2917         trb = &ring->enqueue->generic;
2918         trb->field[0] = cpu_to_le32(field1);
2919         trb->field[1] = cpu_to_le32(field2);
2920         trb->field[2] = cpu_to_le32(field3);
2921         trb->field[3] = cpu_to_le32(field4);
2922 
2923         trace_xhci_queue_trb(ring, trb);
2924 
2925         inc_enq(xhci, ring, more_trbs_coming);
2926 }
2927 
2928 /*
2929  * Does various checks on the endpoint ring, and makes it ready to queue num_trbs.
2930  * FIXME allocate segments if the ring is full.
2931  */
2932 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
2933                 u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
2934 {
2935         unsigned int num_trbs_needed;
2936 
2937         /* Make sure the endpoint has been added to xHC schedule */
2938         switch (ep_state) {
2939         case EP_STATE_DISABLED:
2940                 /*
2941                  * USB core changed config/interfaces without notifying us,
2942                  * or hardware is reporting the wrong state.
2943                  */
2944                 xhci_warn(xhci, "WARN urb submitted to disabled ep\n");
2945                 return -ENOENT;
2946         case EP_STATE_ERROR:
2947                 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n");
2948                 /* FIXME event handling code for error needs to clear it */
2949                 /* XXX not sure if this should be -ENOENT or not */
2950                 return -EINVAL;
2951         case EP_STATE_HALTED:
2952                 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n");
2953         case EP_STATE_STOPPED:
2954         case EP_STATE_RUNNING:
2955                 break;
2956         default:
2957                 xhci_err(xhci, "ERROR unknown endpoint state for ep\n");
2958                 /*
2959                  * FIXME issue Configure Endpoint command to try to get the HC
2960                  * back into a known state.
2961                  */
2962                 return -EINVAL;
2963         }
2964 
2965         while (1) {
2966                 if (room_on_ring(xhci, ep_ring, num_trbs))
2967                         break;
2968 
2969                 if (ep_ring == xhci->cmd_ring) {
2970                         xhci_err(xhci, "Do not support expand command ring\n");
2971                         return -ENOMEM;
2972                 }
2973 
2974                 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion,
2975                                 "ERROR no room on ep ring, try ring expansion");
2976                 num_trbs_needed = num_trbs - ep_ring->num_trbs_free;
2977                 if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed,
2978                                         mem_flags)) {
2979                         xhci_err(xhci, "Ring expansion failed\n");
2980                         return -ENOMEM;
2981                 }
2982         }
2983 
2984         while (trb_is_link(ep_ring->enqueue)) {
2985                 /* If we're not dealing with 0.95 hardware or isoc rings
2986                  * on AMD 0.96 host, clear the chain bit.
2987                  */
2988                 if (!xhci_link_trb_quirk(xhci) &&
2989                     !(ep_ring->type == TYPE_ISOC &&
2990                       (xhci->quirks & XHCI_AMD_0x96_HOST)))
2991                         ep_ring->enqueue->link.control &=
2992                                 cpu_to_le32(~TRB_CHAIN);
2993                 else
2994                         ep_ring->enqueue->link.control |=
2995                                 cpu_to_le32(TRB_CHAIN);
2996 
2997                 wmb();
2998                 ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);
2999 
3000                 /* Toggle the cycle bit after the last ring segment. */
3001                 if (link_trb_toggles_cycle(ep_ring->enqueue))
3002                         ep_ring->cycle_state ^= 1;
3003 
3004                 ep_ring->enq_seg = ep_ring->enq_seg->next;
3005                 ep_ring->enqueue = ep_ring->enq_seg->trbs;
3006         }
3007         return 0;
3008 }
3009 
3010 static int prepare_transfer(struct xhci_hcd *xhci,
3011                 struct xhci_virt_device *xdev,
3012                 unsigned int ep_index,
3013                 unsigned int stream_id,
3014                 unsigned int num_trbs,
3015                 struct urb *urb,
3016                 unsigned int td_index,
3017                 gfp_t mem_flags)
3018 {
3019         int ret;
3020         struct urb_priv *urb_priv;
3021         struct xhci_td  *td;
3022         struct xhci_ring *ep_ring;
3023         struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
3024 
3025         ep_ring = xhci_stream_id_to_ring(xdev, ep_index, stream_id);
3026         if (!ep_ring) {
3027                 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n",
3028                                 stream_id);
3029                 return -EINVAL;
3030         }
3031 
3032         ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
3033                            num_trbs, mem_flags);
3034         if (ret)
3035                 return ret;
3036 
3037         urb_priv = urb->hcpriv;
3038         td = &urb_priv->td[td_index];
3039 
3040         INIT_LIST_HEAD(&td->td_list);
3041         INIT_LIST_HEAD(&td->cancelled_td_list);
3042 
3043         if (td_index == 0) {
3044                 ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb);
3045                 if (unlikely(ret))
3046                         return ret;
3047         }
3048 
3049         td->urb = urb;
3050         /* Add this TD to the tail of the endpoint ring's TD list */
3051         list_add_tail(&td->td_list, &ep_ring->td_list);
3052         td->start_seg = ep_ring->enq_seg;
3053         td->first_trb = ep_ring->enqueue;
3054 
3055         return 0;
3056 }
3057 
3058 unsigned int count_trbs(u64 addr, u64 len)
3059 {
3060         unsigned int num_trbs;
3061 
3062         num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)),
3063                         TRB_MAX_BUFF_SIZE);
3064         if (num_trbs == 0)
3065                 num_trbs++;
3066 
3067         return num_trbs;
3068 }
3069 
3070 static inline unsigned int count_trbs_needed(struct urb *urb)
3071 {
3072         return count_trbs(urb->transfer_dma, urb->transfer_buffer_length);
3073 }
3074 
3075 static unsigned int count_sg_trbs_needed(struct urb *urb)
3076 {
3077         struct scatterlist *sg;
3078         unsigned int i, len, full_len, num_trbs = 0;
3079 
3080         full_len = urb->transfer_buffer_length;
3081 
3082         for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) {
3083                 len = sg_dma_len(sg);
3084                 num_trbs += count_trbs(sg_dma_address(sg), len);
3085                 len = min_t(unsigned int, len, full_len);
3086                 full_len -= len;
3087                 if (full_len == 0)
3088                         break;
3089         }
3090 
3091         return num_trbs;
3092 }
3093 
3094 static unsigned int count_isoc_trbs_needed(struct urb *urb, int i)
3095 {
3096         u64 addr, len;
3097 
3098         addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset);
3099         len = urb->iso_frame_desc[i].length;
3100 
3101         return count_trbs(addr, len);
3102 }
3103 
3104 static void check_trb_math(struct urb *urb, int running_total)
3105 {
3106         if (unlikely(running_total != urb->transfer_buffer_length))
3107                 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, "
3108                                 "queued %#x (%d), asked for %#x (%d)\n",
3109                                 __func__,
3110                                 urb->ep->desc.bEndpointAddress,
3111                                 running_total, running_total,
3112                                 urb->transfer_buffer_length,
3113                                 urb->transfer_buffer_length);
3114 }
3115 
3116 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id,
3117                 unsigned int ep_index, unsigned int stream_id, int start_cycle,
3118                 struct xhci_generic_trb *start_trb)
3119 {
3120         /*
3121          * Pass all the TRBs to the hardware at once and make sure this write
3122          * isn't reordered.
3123          */
3124         wmb();
3125         if (start_cycle)
3126                 start_trb->field[3] |= cpu_to_le32(start_cycle);
3127         else
3128                 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
3129         xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id);
3130 }
3131 
3132 static void check_interval(struct xhci_hcd *xhci, struct urb *urb,
3133                                                 struct xhci_ep_ctx *ep_ctx)
3134 {
3135         int xhci_interval;
3136         int ep_interval;
3137 
3138         xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info));
3139         ep_interval = urb->interval;
3140 
3141         /* Convert to microframes */
3142         if (urb->dev->speed == USB_SPEED_LOW ||
3143                         urb->dev->speed == USB_SPEED_FULL)
3144                 ep_interval *= 8;
3145 
3146         /* FIXME change this to a warning and a suggestion to use the new API
3147          * to set the polling interval (once the API is added).
3148          */
3149         if (xhci_interval != ep_interval) {
3150                 dev_dbg_ratelimited(&urb->dev->dev,
3151                                 "Driver uses different interval (%d microframe%s) than xHCI (%d microframe%s)\n",
3152                                 ep_interval, ep_interval == 1 ? "" : "s",
3153                                 xhci_interval, xhci_interval == 1 ? "" : "s");
3154                 urb->interval = xhci_interval;
3155                 /* Convert back to frames for LS/FS devices */
3156                 if (urb->dev->speed == USB_SPEED_LOW ||
3157                                 urb->dev->speed == USB_SPEED_FULL)
3158                         urb->interval /= 8;
3159         }
3160 }
3161 
3162 /*
3163  * xHCI uses normal TRBs for both bulk and interrupt.  When the interrupt
3164  * endpoint is to be serviced, the xHC will consume (at most) one TD.  A TD
3165  * (comprised of sg list entries) can take several service intervals to
3166  * transmit.
3167  */
3168 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3169                 struct urb *urb, int slot_id, unsigned int ep_index)
3170 {
3171         struct xhci_ep_ctx *ep_ctx;
3172 
3173         ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index);
3174         check_interval(xhci, urb, ep_ctx);
3175 
3176         return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index);
3177 }
3178 
3179 /*
3180  * For xHCI 1.0 host controllers, TD size is the number of max packet sized
3181  * packets remaining in the TD (*not* including this TRB).
3182  *
3183  * Total TD packet count = total_packet_count =
3184  *     DIV_ROUND_UP(TD size in bytes / wMaxPacketSize)
3185  *
3186  * Packets transferred up to and including this TRB = packets_transferred =
3187  *     rounddown(total bytes transferred including this TRB / wMaxPacketSize)
3188  *
3189  * TD size = total_packet_count - packets_transferred
3190  *
3191  * For xHCI 0.96 and older, TD size field should be the remaining bytes
3192  * including this TRB, right shifted by 10
3193  *
3194  * For all hosts it must fit in bits 21:17, so it can't be bigger than 31.
3195  * This is taken care of in the TRB_TD_SIZE() macro
3196  *
3197  * The last TRB in a TD must have the TD size set to zero.
3198  */
3199 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred,
3200                               int trb_buff_len, unsigned int td_total_len,
3201                               struct urb *urb, bool more_trbs_coming)
3202 {
3203         u32 maxp, total_packet_count;
3204 
3205         /* MTK xHCI 0.96 contains some features from 1.0 */
3206         if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST))
3207                 return ((td_total_len - transferred) >> 10);
3208 
3209         /* One TRB with a zero-length data packet. */
3210         if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
3211             trb_buff_len == td_total_len)
3212                 return 0;
3213 
3214         /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */
3215         if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100))
3216                 trb_buff_len = 0;
3217 
3218         maxp = usb_endpoint_maxp(&urb->ep->desc);
3219         total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
3220 
3221         /* Queueing functions don't count the current TRB into transferred */
3222         return (total_packet_count - ((transferred + trb_buff_len) / maxp));
3223 }
3224 
3225 
3226 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
3227                          u32 *trb_buff_len, struct xhci_segment *seg)
3228 {
3229         struct device *dev = xhci_to_hcd(xhci)->self.controller;
3230         unsigned int unalign;
3231         unsigned int max_pkt;
3232         u32 new_buff_len;
3233         size_t len;
3234 
3235         max_pkt = usb_endpoint_maxp(&urb->ep->desc);
3236         unalign = (enqd_len + *trb_buff_len) % max_pkt;
3237 
3238         /* we got lucky, last normal TRB data on segment is packet aligned */
3239         if (unalign == 0)
3240                 return 0;
3241 
3242         xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n",
3243                  unalign, *trb_buff_len);
3244 
3245         /* is the last nornal TRB alignable by splitting it */
3246         if (*trb_buff_len > unalign) {
3247                 *trb_buff_len -= unalign;
3248                 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len);
3249                 return 0;
3250         }
3251 
3252         /*
3253          * We want enqd_len + trb_buff_len to sum up to a number aligned to
3254          * number which is divisible by the endpoint's wMaxPacketSize. IOW:
3255          * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0.
3256          */
3257         new_buff_len = max_pkt - (enqd_len % max_pkt);
3258 
3259         if (new_buff_len > (urb->transfer_buffer_length - enqd_len))
3260                 new_buff_len = (urb->transfer_buffer_length - enqd_len);
3261 
3262         /* create a max max_pkt sized bounce buffer pointed to by last trb */
3263         if (usb_urb_dir_out(urb)) {
3264                 len = sg_pcopy_to_buffer(urb->sg, urb->num_sgs,
3265                                    seg->bounce_buf, new_buff_len, enqd_len);
3266                 if (len != new_buff_len)
3267                         xhci_warn(xhci,
3268                                 "WARN Wrong bounce buffer write length: %zu != %d\n",
3269                                 len, new_buff_len);
3270                 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
3271                                                  max_pkt, DMA_TO_DEVICE);
3272         } else {
3273                 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
3274                                                  max_pkt, DMA_FROM_DEVICE);
3275         }
3276 
3277         if (dma_mapping_error(dev, seg->bounce_dma)) {
3278                 /* try without aligning. Some host controllers survive */
3279                 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n");
3280                 return 0;
3281         }
3282         *trb_buff_len = new_buff_len;
3283         seg->bounce_len = new_buff_len;
3284         seg->bounce_offs = enqd_len;
3285 
3286         xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len);
3287 
3288         return 1;
3289 }
3290 
3291 /* This is very similar to what ehci-q.c qtd_fill() does */
3292 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3293                 struct urb *urb, int slot_id, unsigned int ep_index)
3294 {
3295         struct xhci_ring *ring;
3296         struct urb_priv *urb_priv;
3297         struct xhci_td *td;
3298         struct xhci_generic_trb *start_trb;
3299         struct scatterlist *sg = NULL;
3300         bool more_trbs_coming = true;
3301         bool need_zero_pkt = false;
3302         bool first_trb = true;
3303         unsigned int num_trbs;
3304         unsigned int start_cycle, num_sgs = 0;
3305         unsigned int enqd_len, block_len, trb_buff_len, full_len;
3306         int sent_len, ret;
3307         u32 field, length_field, remainder;
3308         u64 addr, send_addr;
3309 
3310         ring = xhci_urb_to_transfer_ring(xhci, urb);
3311         if (!ring)
3312                 return -EINVAL;
3313 
3314         full_len = urb->transfer_buffer_length;
3315         /* If we have scatter/gather list, we use it. */
3316         if (urb->num_sgs) {
3317                 num_sgs = urb->num_mapped_sgs;
3318                 sg = urb->sg;
3319                 addr = (u64) sg_dma_address(sg);
3320                 block_len = sg_dma_len(sg);
3321                 num_trbs = count_sg_trbs_needed(urb);
3322         } else {
3323                 num_trbs = count_trbs_needed(urb);
3324                 addr = (u64) urb->transfer_dma;
3325                 block_len = full_len;
3326         }
3327         ret = prepare_transfer(xhci, xhci->devs[slot_id],
3328                         ep_index, urb->stream_id,
3329                         num_trbs, urb, 0, mem_flags);
3330         if (unlikely(ret < 0))
3331                 return ret;
3332 
3333         urb_priv = urb->hcpriv;
3334 
3335         /* Deal with URB_ZERO_PACKET - need one more td/trb */
3336         if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1)
3337                 need_zero_pkt = true;
3338 
3339         td = &urb_priv->td[0];
3340 
3341         /*
3342          * Don't give the first TRB to the hardware (by toggling the cycle bit)
3343          * until we've finished creating all the other TRBs.  The ring's cycle
3344          * state may change as we enqueue the other TRBs, so save it too.
3345          */
3346         start_trb = &ring->enqueue->generic;
3347         start_cycle = ring->cycle_state;
3348         send_addr = addr;
3349 
3350         /* Queue the TRBs, even if they are zero-length */
3351         for (enqd_len = 0; first_trb || enqd_len < full_len;
3352                         enqd_len += trb_buff_len) {
3353                 field = TRB_TYPE(TRB_NORMAL);
3354 
3355                 /* TRB buffer should not cross 64KB boundaries */
3356                 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
3357                 trb_buff_len = min_t(unsigned int, trb_buff_len, block_len);
3358 
3359                 if (enqd_len + trb_buff_len > full_len)
3360                         trb_buff_len = full_len - enqd_len;
3361 
3362                 /* Don't change the cycle bit of the first TRB until later */
3363                 if (first_trb) {
3364                         first_trb = false;
3365                         if (start_cycle == 0)
3366                                 field |= TRB_CYCLE;
3367                 } else
3368                         field |= ring->cycle_state;
3369 
3370                 /* Chain all the TRBs together; clear the chain bit in the last
3371                  * TRB to indicate it's the last TRB in the chain.
3372                  */
3373                 if (enqd_len + trb_buff_len < full_len) {
3374                         field |= TRB_CHAIN;
3375                         if (trb_is_link(ring->enqueue + 1)) {
3376                                 if (xhci_align_td(xhci, urb, enqd_len,
3377                                                   &trb_buff_len,
3378                                                   ring->enq_seg)) {
3379                                         send_addr = ring->enq_seg->bounce_dma;
3380                                         /* assuming TD won't span 2 segs */
3381                                         td->bounce_seg = ring->enq_seg;
3382                                 }
3383                         }
3384                 }
3385                 if (enqd_len + trb_buff_len >= full_len) {
3386                         field &= ~TRB_CHAIN;
3387                         field |= TRB_IOC;
3388                         more_trbs_coming = false;
3389                         td->last_trb = ring->enqueue;
3390 
3391                         if (xhci_urb_suitable_for_idt(urb)) {
3392                                 memcpy(&send_addr, urb->transfer_buffer,
3393                                        trb_buff_len);
3394                                 le64_to_cpus(&send_addr);
3395                                 field |= TRB_IDT;
3396                         }
3397                 }
3398 
3399                 /* Only set interrupt on short packet for IN endpoints */
3400                 if (usb_urb_dir_in(urb))
3401                         field |= TRB_ISP;
3402 
3403                 /* Set the TRB length, TD size, and interrupter fields. */
3404                 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len,
3405                                               full_len, urb, more_trbs_coming);
3406 
3407                 length_field = TRB_LEN(trb_buff_len) |
3408                         TRB_TD_SIZE(remainder) |
3409                         TRB_INTR_TARGET(0);
3410 
3411                 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt,
3412                                 lower_32_bits(send_addr),
3413                                 upper_32_bits(send_addr),
3414                                 length_field,
3415                                 field);
3416 
3417                 addr += trb_buff_len;
3418                 sent_len = trb_buff_len;
3419 
3420                 while (sg && sent_len >= block_len) {
3421                         /* New sg entry */
3422                         --num_sgs;
3423                         sent_len -= block_len;
3424                         sg = sg_next(sg);
3425                         if (num_sgs != 0 && sg) {
3426                                 block_len = sg_dma_len(sg);
3427                                 addr = (u64) sg_dma_address(sg);
3428                                 addr += sent_len;
3429                         }
3430                 }
3431                 block_len -= sent_len;
3432                 send_addr = addr;
3433         }
3434 
3435         if (need_zero_pkt) {
3436                 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3437                                        ep_index, urb->stream_id,
3438                                        1, urb, 1, mem_flags);
3439                 urb_priv->td[1].last_trb = ring->enqueue;
3440                 field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC;
3441                 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field);
3442         }
3443 
3444         check_trb_math(urb, enqd_len);
3445         giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
3446                         start_cycle, start_trb);
3447         return 0;
3448 }
3449 
3450 /* Caller must have locked xhci->lock */
3451 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3452                 struct urb *urb, int slot_id, unsigned int ep_index)
3453 {
3454         struct xhci_ring *ep_ring;
3455         int num_trbs;
3456         int ret;
3457         struct usb_ctrlrequest *setup;
3458         struct xhci_generic_trb *start_trb;
3459         int start_cycle;
3460         u32 field;
3461         struct urb_priv *urb_priv;
3462         struct xhci_td *td;
3463 
3464         ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
3465         if (!ep_ring)
3466                 return -EINVAL;
3467 
3468         /*
3469          * Need to copy setup packet into setup TRB, so we can't use the setup
3470          * DMA address.
3471          */
3472         if (!urb->setup_packet)
3473                 return -EINVAL;
3474 
3475         /* 1 TRB for setup, 1 for status */
3476         num_trbs = 2;
3477         /*
3478          * Don't need to check if we need additional event data and normal TRBs,
3479          * since data in control transfers will never get bigger than 16MB
3480          * XXX: can we get a buffer that crosses 64KB boundaries?
3481          */
3482         if (urb->transfer_buffer_length > 0)
3483                 num_trbs++;
3484         ret = prepare_transfer(xhci, xhci->devs[slot_id],
3485                         ep_index, urb->stream_id,
3486                         num_trbs, urb, 0, mem_flags);
3487         if (ret < 0)
3488                 return ret;
3489 
3490         urb_priv = urb->hcpriv;
3491         td = &urb_priv->td[0];
3492 
3493         /*
3494          * Don't give the first TRB to the hardware (by toggling the cycle bit)
3495          * until we've finished creating all the other TRBs.  The ring's cycle
3496          * state may change as we enqueue the other TRBs, so save it too.
3497          */
3498         start_trb = &ep_ring->enqueue->generic;
3499         start_cycle = ep_ring->cycle_state;
3500 
3501         /* Queue setup TRB - see section 6.4.1.2.1 */
3502         /* FIXME better way to translate setup_packet into two u32 fields? */
3503         setup = (struct usb_ctrlrequest *) urb->setup_packet;
3504         field = 0;
3505         field |= TRB_IDT | TRB_TYPE(TRB_SETUP);
3506         if (start_cycle == 0)
3507                 field |= 0x1;
3508 
3509         /* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */
3510         if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) {
3511                 if (urb->transfer_buffer_length > 0) {
3512                         if (setup->bRequestType & USB_DIR_IN)
3513                                 field |= TRB_TX_TYPE(TRB_DATA_IN);
3514                         else
3515                                 field |= TRB_TX_TYPE(TRB_DATA_OUT);
3516                 }
3517         }
3518 
3519         queue_trb(xhci, ep_ring, true,
3520                   setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16,
3521                   le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16,
3522                   TRB_LEN(8) | TRB_INTR_TARGET(0),
3523                   /* Immediate data in pointer */
3524                   field);
3525 
3526         /* If there's data, queue data TRBs */
3527         /* Only set interrupt on short packet for IN endpoints */
3528         if (usb_urb_dir_in(urb))
3529                 field = TRB_ISP | TRB_TYPE(TRB_DATA);
3530         else
3531                 field = TRB_TYPE(TRB_DATA);
3532 
3533         if (urb->transfer_buffer_length > 0) {
3534                 u32 length_field, remainder;
3535                 u64 addr;
3536 
3537                 if (xhci_urb_suitable_for_idt(urb)) {
3538                         memcpy(&addr, urb->transfer_buffer,
3539                                urb->transfer_buffer_length);
3540                         le64_to_cpus(&addr);
3541                         field |= TRB_IDT;
3542                 } else {
3543                         addr = (u64) urb->transfer_dma;
3544                 }
3545 
3546                 remainder = xhci_td_remainder(xhci, 0,
3547                                 urb->transfer_buffer_length,
3548                                 urb->transfer_buffer_length,
3549                                 urb, 1);
3550                 length_field = TRB_LEN(urb->transfer_buffer_length) |
3551                                 TRB_TD_SIZE(remainder) |
3552                                 TRB_INTR_TARGET(0);
3553                 if (setup->bRequestType & USB_DIR_IN)
3554                         field |= TRB_DIR_IN;
3555                 queue_trb(xhci, ep_ring, true,
3556                                 lower_32_bits(addr),
3557                                 upper_32_bits(addr),
3558                                 length_field,
3559                                 field | ep_ring->cycle_state);
3560         }
3561 
3562         /* Save the DMA address of the last TRB in the TD */
3563         td->last_trb = ep_ring->enqueue;
3564 
3565         /* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */
3566         /* If the device sent data, the status stage is an OUT transfer */
3567         if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN)
3568                 field = 0;
3569         else
3570                 field = TRB_DIR_IN;
3571         queue_trb(xhci, ep_ring, false,
3572                         0,
3573                         0,
3574                         TRB_INTR_TARGET(0),
3575                         /* Event on completion */
3576                         field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state);
3577 
3578         giveback_first_trb(xhci, slot_id, ep_index, 0,
3579                         start_cycle, start_trb);
3580         return 0;
3581 }
3582 
3583 /*
3584  * The transfer burst count field of the isochronous TRB defines the number of
3585  * bursts that are required to move all packets in this TD.  Only SuperSpeed
3586  * devices can burst up to bMaxBurst number of packets per service interval.
3587  * This field is zero based, meaning a value of zero in the field means one
3588  * burst.  Basically, for everything but SuperSpeed devices, this field will be
3589  * zero.  Only xHCI 1.0 host controllers support this field.
3590  */
3591 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci,
3592                 struct urb *urb, unsigned int total_packet_count)
3593 {
3594         unsigned int max_burst;
3595 
3596         if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER)
3597                 return 0;
3598 
3599         max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3600         return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1;
3601 }
3602 
3603 /*
3604  * Returns the number of packets in the last "burst" of packets.  This field is
3605  * valid for all speeds of devices.  USB 2.0 devices can only do one "burst", so
3606  * the last burst packet count is equal to the total number of packets in the
3607  * TD.  SuperSpeed endpoints can have up to 3 bursts.  All but the last burst
3608  * must contain (bMaxBurst + 1) number of packets, but the last burst can
3609  * contain 1 to (bMaxBurst + 1) packets.
3610  */
3611 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci,
3612                 struct urb *urb, unsigned int total_packet_count)
3613 {
3614         unsigned int max_burst;
3615         unsigned int residue;
3616 
3617         if (xhci->hci_version < 0x100)
3618                 return 0;
3619 
3620         if (urb->dev->speed >= USB_SPEED_SUPER) {
3621                 /* bMaxBurst is zero based: 0 means 1 packet per burst */
3622                 max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3623                 residue = total_packet_count % (max_burst + 1);
3624                 /* If residue is zero, the last burst contains (max_burst + 1)
3625                  * number of packets, but the TLBPC field is zero-based.
3626                  */
3627                 if (residue == 0)
3628                         return max_burst;
3629                 return residue - 1;
3630         }
3631         if (total_packet_count == 0)
3632                 return 0;
3633         return total_packet_count - 1;
3634 }
3635 
3636 /*
3637  * Calculates Frame ID field of the isochronous TRB identifies the
3638  * target frame that the Interval associated with this Isochronous
3639  * Transfer Descriptor will start on. Refer to 4.11.2.5 in 1.1 spec.
3640  *
3641  * Returns actual frame id on success, negative value on error.
3642  */
3643 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci,
3644                 struct urb *urb, int index)
3645 {
3646         int start_frame, ist, ret = 0;
3647         int start_frame_id, end_frame_id, current_frame_id;
3648 
3649         if (urb->dev->speed == USB_SPEED_LOW ||
3650                         urb->dev->speed == USB_SPEED_FULL)
3651                 start_frame = urb->start_frame + index * urb->interval;
3652         else
3653                 start_frame = (urb->start_frame + index * urb->interval) >> 3;
3654 
3655         /* Isochronous Scheduling Threshold (IST, bits 0~3 in HCSPARAMS2):
3656          *
3657          * If bit [3] of IST is cleared to '0', software can add a TRB no
3658          * later than IST[2:0] Microframes before that TRB is scheduled to
3659          * be executed.
3660          * If bit [3] of IST is set to '1', software can add a TRB no later
3661          * than IST[2:0] Frames before that TRB is scheduled to be executed.
3662          */
3663         ist = HCS_IST(xhci->hcs_params2) & 0x7;
3664         if (HCS_IST(xhci->hcs_params2) & (1 << 3))
3665                 ist <<= 3;
3666 
3667         /* Software shall not schedule an Isoch TD with a Frame ID value that
3668          * is less than the Start Frame ID or greater than the End Frame ID,
3669          * where:
3670          *
3671          * End Frame ID = (Current MFINDEX register value + 895 ms.) MOD 2048
3672          * Start Frame ID = (Current MFINDEX register value + IST + 1) MOD 2048
3673          *
3674          * Both the End Frame ID and Start Frame ID values are calculated
3675          * in microframes. When software determines the valid Frame ID value;
3676          * The End Frame ID value should be rounded down to the nearest Frame
3677          * boundary, and the Start Frame ID value should be rounded up to the
3678          * nearest Frame boundary.
3679          */
3680         current_frame_id = readl(&xhci->run_regs->microframe_index);
3681         start_frame_id = roundup(current_frame_id + ist + 1, 8);
3682         end_frame_id = rounddown(current_frame_id + 895 * 8, 8);
3683 
3684         start_frame &= 0x7ff;
3685         start_frame_id = (start_frame_id >> 3) & 0x7ff;
3686         end_frame_id = (end_frame_id >> 3) & 0x7ff;
3687 
3688         xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n",
3689                  __func__, index, readl(&xhci->run_regs->microframe_index),
3690                  start_frame_id, end_frame_id, start_frame);
3691 
3692         if (start_frame_id < end_frame_id) {
3693                 if (start_frame > end_frame_id ||
3694                                 start_frame < start_frame_id)
3695                         ret = -EINVAL;
3696         } else if (start_frame_id > end_frame_id) {
3697                 if ((start_frame > end_frame_id &&
3698                                 start_frame < start_frame_id))
3699                         ret = -EINVAL;
3700         } else {
3701                         ret = -EINVAL;
3702         }
3703 
3704         if (index == 0) {
3705                 if (ret == -EINVAL || start_frame == start_frame_id) {
3706                         start_frame = start_frame_id + 1;
3707                         if (urb->dev->speed == USB_SPEED_LOW ||
3708                                         urb->dev->speed == USB_SPEED_FULL)
3709                                 urb->start_frame = start_frame;
3710                         else
3711                                 urb->start_frame = start_frame << 3;
3712                         ret = 0;
3713                 }
3714         }
3715 
3716         if (ret) {
3717                 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n",
3718                                 start_frame, current_frame_id, index,
3719                                 start_frame_id, end_frame_id);
3720                 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n");
3721                 return ret;
3722         }
3723 
3724         return start_frame;
3725 }
3726 
3727 /* This is for isoc transfer */
3728 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3729                 struct urb *urb, int slot_id, unsigned int ep_index)
3730 {
3731         struct xhci_ring *ep_ring;
3732         struct urb_priv *urb_priv;
3733         struct xhci_td *td;
3734         int num_tds, trbs_per_td;
3735         struct xhci_generic_trb *start_trb;
3736         bool first_trb;
3737         int start_cycle;
3738         u32 field, length_field;
3739         int running_total, trb_buff_len, td_len, td_remain_len, ret;
3740         u64 start_addr, addr;
3741         int i, j;
3742         bool more_trbs_coming;
3743         struct xhci_virt_ep *xep;
3744         int frame_id;
3745 
3746         xep = &xhci->devs[slot_id]->eps[ep_index];
3747         ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;
3748 
3749         num_tds = urb->number_of_packets;
3750         if (num_tds < 1) {
3751                 xhci_dbg(xhci, "Isoc URB with zero packets?\n");
3752                 return -EINVAL;
3753         }
3754         start_addr = (u64) urb->transfer_dma;
3755         start_trb = &ep_ring->enqueue->generic;
3756         start_cycle = ep_ring->cycle_state;
3757 
3758         urb_priv = urb->hcpriv;
3759         /* Queue the TRBs for each TD, even if they are zero-length */
3760         for (i = 0; i < num_tds; i++) {
3761                 unsigned int total_pkt_count, max_pkt;
3762                 unsigned int burst_count, last_burst_pkt_count;
3763                 u32 sia_frame_id;
3764 
3765                 first_trb = true;
3766                 running_total = 0;
3767                 addr = start_addr + urb->iso_frame_desc[i].offset;
3768                 td_len = urb->iso_frame_desc[i].length;
3769                 td_remain_len = td_len;
3770                 max_pkt = usb_endpoint_maxp(&urb->ep->desc);
3771                 total_pkt_count = DIV_ROUND_UP(td_len, max_pkt);
3772 
3773                 /* A zero-length transfer still involves at least one packet. */
3774                 if (total_pkt_count == 0)
3775                         total_pkt_count++;
3776                 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count);
3777                 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci,
3778                                                         urb, total_pkt_count);
3779 
3780                 trbs_per_td = count_isoc_trbs_needed(urb, i);
3781 
3782                 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index,
3783                                 urb->stream_id, trbs_per_td, urb, i, mem_flags);
3784                 if (ret < 0) {
3785                         if (i == 0)
3786                                 return ret;
3787                         goto cleanup;
3788                 }
3789                 td = &urb_priv->td[i];
3790 
3791                 /* use SIA as default, if frame id is used overwrite it */
3792                 sia_frame_id = TRB_SIA;
3793                 if (!(urb->transfer_flags & URB_ISO_ASAP) &&
3794                     HCC_CFC(xhci->hcc_params)) {
3795                         frame_id = xhci_get_isoc_frame_id(xhci, urb, i);
3796                         if (frame_id >= 0)
3797                                 sia_frame_id = TRB_FRAME_ID(frame_id);
3798                 }
3799                 /*
3800                  * Set isoc specific data for the first TRB in a TD.
3801                  * Prevent HW from getting the TRBs by keeping the cycle state
3802                  * inverted in the first TDs isoc TRB.
3803                  */
3804                 field = TRB_TYPE(TRB_ISOC) |
3805                         TRB_TLBPC(last_burst_pkt_count) |
3806                         sia_frame_id |
3807                         (i ? ep_ring->cycle_state : !start_cycle);
3808 
3809                 /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */
3810                 if (!xep->use_extended_tbc)
3811                         field |= TRB_TBC(burst_count);
3812 
3813                 /* fill the rest of the TRB fields, and remaining normal TRBs */
3814                 for (j = 0; j < trbs_per_td; j++) {
3815                         u32 remainder = 0;
3816 
3817                         /* only first TRB is isoc, overwrite otherwise */
3818                         if (!first_trb)
3819                                 field = TRB_TYPE(TRB_NORMAL) |
3820                                         ep_ring->cycle_state;
3821 
3822                         /* Only set interrupt on short packet for IN EPs */
3823                         if (usb_urb_dir_in(urb))
3824                                 field |= TRB_ISP;
3825 
3826                         /* Set the chain bit for all except the last TRB  */
3827                         if (j < trbs_per_td - 1) {
3828                                 more_trbs_coming = true;
3829                                 field |= TRB_CHAIN;
3830                         } else {
3831                                 more_trbs_coming = false;
3832                                 td->last_trb = ep_ring->enqueue;
3833                                 field |= TRB_IOC;
3834                                 /* set BEI, except for the last TD */
3835                                 if (xhci->hci_version >= 0x100 &&
3836                                     !(xhci->quirks & XHCI_AVOID_BEI) &&
3837                                     i < num_tds - 1)
3838                                         field |= TRB_BEI;
3839                         }
3840                         /* Calculate TRB length */
3841                         trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
3842                         if (trb_buff_len > td_remain_len)
3843                                 trb_buff_len = td_remain_len;
3844 
3845                         /* Set the TRB length, TD size, & interrupter fields. */
3846                         remainder = xhci_td_remainder(xhci, running_total,
3847                                                    trb_buff_len, td_len,
3848                                                    urb, more_trbs_coming);
3849 
3850                         length_field = TRB_LEN(trb_buff_len) |
3851                                 TRB_INTR_TARGET(0);
3852 
3853                         /* xhci 1.1 with ETE uses TD Size field for TBC */
3854                         if (first_trb && xep->use_extended_tbc)
3855                                 length_field |= TRB_TD_SIZE_TBC(burst_count);
3856                         else
3857                                 length_field |= TRB_TD_SIZE(remainder);
3858                         first_trb = false;
3859 
3860                         queue_trb(xhci, ep_ring, more_trbs_coming,
3861                                 lower_32_bits(addr),
3862                                 upper_32_bits(addr),
3863                                 length_field,
3864                                 field);
3865                         running_total += trb_buff_len;
3866 
3867                         addr += trb_buff_len;
3868                         td_remain_len -= trb_buff_len;
3869                 }
3870 
3871                 /* Check TD length */
3872                 if (running_total != td_len) {
3873                         xhci_err(xhci, "ISOC TD length unmatch\n");
3874                         ret = -EINVAL;
3875                         goto cleanup;
3876                 }
3877         }
3878 
3879         /* store the next frame id */
3880         if (HCC_CFC(xhci->hcc_params))
3881                 xep->next_frame_id = urb->start_frame + num_tds * urb->interval;
3882 
3883         if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
3884                 if (xhci->quirks & XHCI_AMD_PLL_FIX)
3885                         usb_amd_quirk_pll_disable();
3886         }
3887         xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++;
3888 
3889         giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
3890                         start_cycle, start_trb);
3891         return 0;
3892 cleanup:
3893         /* Clean up a partially enqueued isoc transfer. */
3894 
3895         for (i--; i >= 0; i--)
3896                 list_del_init(&urb_priv->td[i].td_list);
3897 
3898         /* Use the first TD as a temporary variable to turn the TDs we've queued
3899          * into No-ops with a software-owned cycle bit. That way the hardware
3900          * won't accidentally start executing bogus TDs when we partially
3901          * overwrite them.  td->first_trb and td->start_seg are already set.
3902          */
3903         urb_priv->td[0].last_trb = ep_ring->enqueue;
3904         /* Every TRB except the first & last will have its cycle bit flipped. */
3905         td_to_noop(xhci, ep_ring, &urb_priv->td[0], true);
3906 
3907         /* Reset the ring enqueue back to the first TRB and its cycle bit. */
3908         ep_ring->enqueue = urb_priv->td[0].first_trb;
3909         ep_ring->enq_seg = urb_priv->td[0].start_seg;
3910         ep_ring->cycle_state = start_cycle;
3911         ep_ring->num_trbs_free = ep_ring->num_trbs_free_temp;
3912         usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);
3913         return ret;
3914 }
3915 
3916 /*
3917  * Check transfer ring to guarantee there is enough room for the urb.
3918  * Update ISO URB start_frame and interval.
3919  * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to
3920  * update urb->start_frame if URB_ISO_ASAP is set in transfer_flags or
3921  * Contiguous Frame ID is not supported by HC.
3922  */
3923 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
3924                 struct urb *urb, int slot_id, unsigned int ep_index)
3925 {
3926         struct xhci_virt_device *xdev;
3927         struct xhci_ring *ep_ring;
3928         struct xhci_ep_ctx *ep_ctx;
3929         int start_frame;
3930         int num_tds, num_trbs, i;
3931         int ret;
3932         struct xhci_virt_ep *xep;
3933         int ist;
3934 
3935         xdev = xhci->devs[slot_id];
3936         xep = &xhci->devs[slot_id]->eps[ep_index];
3937         ep_ring = xdev->eps[ep_index].ring;
3938         ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
3939 
3940         num_trbs = 0;
3941         num_tds = urb->number_of_packets;
3942         for (i = 0; i < num_tds; i++)
3943                 num_trbs += count_isoc_trbs_needed(urb, i);
3944 
3945         /* Check the ring to guarantee there is enough room for the whole urb.
3946          * Do not insert any td of the urb to the ring if the check failed.
3947          */
3948         ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
3949                            num_trbs, mem_flags);
3950         if (ret)
3951                 return ret;
3952 
3953         /*
3954          * Check interval value. This should be done before we start to
3955          * calculate the start frame value.
3956          */
3957         check_interval(xhci, urb, ep_ctx);
3958 
3959         /* Calculate the start frame and put it in urb->start_frame. */
3960         if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {
3961                 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_RUNNING) {
3962                         urb->start_frame = xep->next_frame_id;
3963                         goto skip_start_over;
3964                 }
3965         }
3966 
3967         start_frame = readl(&xhci->run_regs->microframe_index);
3968         start_frame &= 0x3fff;
3969         /*
3970          * Round up to the next frame and consider the time before trb really
3971          * gets scheduled by hardare.
3972          */
3973         ist = HCS_IST(xhci->hcs_params2) & 0x7;
3974         if (HCS_IST(xhci->hcs_params2) & (1 << 3))
3975                 ist <<= 3;
3976         start_frame += ist + XHCI_CFC_DELAY;
3977         start_frame = roundup(start_frame, 8);
3978 
3979         /*
3980          * Round up to the next ESIT (Endpoint Service Interval Time) if ESIT
3981          * is greate than 8 microframes.
3982          */
3983         if (urb->dev->speed == USB_SPEED_LOW ||
3984                         urb->dev->speed == USB_SPEED_FULL) {
3985                 start_frame = roundup(start_frame, urb->interval << 3);
3986                 urb->start_frame = start_frame >> 3;
3987         } else {
3988                 start_frame = roundup(start_frame, urb->interval);
3989                 urb->start_frame = start_frame;
3990         }
3991 
3992 skip_start_over:
3993         ep_ring->num_trbs_free_temp = ep_ring->num_trbs_free;
3994 
3995         return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index);
3996 }
3997 
3998 /****           Command Ring Operations         ****/
3999 
4000 /* Generic function for queueing a command TRB on the command ring.
4001  * Check to make sure there's room on the command ring for one command TRB.
4002  * Also check that there's room reserved for commands that must not fail.
4003  * If this is a command that must not fail, meaning command_must_succeed = TRUE,
4004  * then only check for the number of reserved spots.
4005  * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
4006  * because the command event handler may want to resubmit a failed command.
4007  */
4008 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
4009                          u32 field1, u32 field2,
4010                          u32 field3, u32 field4, bool command_must_succeed)
4011 {
4012         int reserved_trbs = xhci->cmd_ring_reserved_trbs;
4013         int ret;
4014 
4015         if ((xhci->xhc_state & XHCI_STATE_DYING) ||
4016                 (xhci->xhc_state & XHCI_STATE_HALTED)) {
4017                 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n");
4018                 return -ESHUTDOWN;
4019         }
4020 
4021         if (!command_must_succeed)
4022                 reserved_trbs++;
4023 
4024         ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING,
4025                         reserved_trbs, GFP_ATOMIC);
4026         if (ret < 0) {
4027                 xhci_err(xhci, "ERR: No room for command on command ring\n");
4028                 if (command_must_succeed)
4029                         xhci_err(xhci, "ERR: Reserved TRB counting for "
4030                                         "unfailable commands failed.\n");
4031                 return ret;
4032         }
4033 
4034         cmd->command_trb = xhci->cmd_ring->enqueue;
4035 
4036         /* if there are no other commands queued we start the timeout timer */
4037         if (list_empty(&xhci->cmd_list)) {
4038                 xhci->current_cmd = cmd;
4039                 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
4040         }
4041 
4042         list_add_tail(&cmd->cmd_list, &xhci->cmd_list);
4043 
4044         queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3,
4045                         field4 | xhci->cmd_ring->cycle_state);
4046         return 0;
4047 }
4048 
4049 /* Queue a slot enable or disable request on the command ring */
4050 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd,
4051                 u32 trb_type, u32 slot_id)
4052 {
4053         return queue_command(xhci, cmd, 0, 0, 0,
4054                         TRB_TYPE(trb_type) | SLOT_ID_FOR_TRB(slot_id), false);
4055 }
4056 
4057 /* Queue an address device command TRB */
4058 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
4059                 dma_addr_t in_ctx_ptr, u32 slot_id, enum xhci_setup_dev setup)
4060 {
4061         return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4062                         upper_32_bits(in_ctx_ptr), 0,
4063                         TRB_TYPE(TRB_ADDR_DEV) | SLOT_ID_FOR_TRB(slot_id)
4064                         | (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0), false);
4065 }
4066 
4067 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
4068                 u32 field1, u32 field2, u32 field3, u32 field4)
4069 {
4070         return queue_command(xhci, cmd, field1, field2, field3, field4, false);
4071 }
4072 
4073 /* Queue a reset device command TRB */
4074 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
4075                 u32 slot_id)
4076 {
4077         return queue_command(xhci, cmd, 0, 0, 0,
4078                         TRB_TYPE(TRB_RESET_DEV) | SLOT_ID_FOR_TRB(slot_id),
4079                         false);
4080 }
4081 
4082 /* Queue a configure endpoint command TRB */
4083 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
4084                 struct xhci_command *cmd, dma_addr_t in_ctx_ptr,
4085                 u32 slot_id, bool command_must_succeed)
4086 {
4087         return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4088                         upper_32_bits(in_ctx_ptr), 0,
4089                         TRB_TYPE(TRB_CONFIG_EP) | SLOT_ID_FOR_TRB(slot_id),
4090                         command_must_succeed);
4091 }
4092 
4093 /* Queue an evaluate context command TRB */
4094 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd,
4095                 dma_addr_t in_ctx_ptr, u32 slot_id, bool command_must_succeed)
4096 {
4097         return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4098                         upper_32_bits(in_ctx_ptr), 0,
4099                         TRB_TYPE(TRB_EVAL_CONTEXT) | SLOT_ID_FOR_TRB(slot_id),
4100                         command_must_succeed);
4101 }
4102 
4103 /*
4104  * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop
4105  * activity on an endpoint that is about to be suspended.
4106  */
4107 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd,
4108                              int slot_id, unsigned int ep_index, int suspend)
4109 {
4110         u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
4111         u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
4112         u32 type = TRB_TYPE(TRB_STOP_RING);
4113         u32 trb_suspend = SUSPEND_PORT_FOR_TRB(suspend);
4114 
4115         return queue_command(xhci, cmd, 0, 0, 0,
4116                         trb_slot_id | trb_ep_index | type | trb_suspend, false);
4117 }
4118 
4119 /* Set Transfer Ring Dequeue Pointer command */
4120 void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
4121                 unsigned int slot_id, unsigned int ep_index,
4122                 struct xhci_dequeue_state *deq_state)
4123 {
4124         dma_addr_t addr;
4125         u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
4126         u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
4127         u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id);
4128         u32 trb_sct = 0;
4129         u32 type = TRB_TYPE(TRB_SET_DEQ);
4130         struct xhci_virt_ep *ep;
4131         struct xhci_command *cmd;
4132         int ret;
4133 
4134         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
4135                 "Set TR Deq Ptr cmd, new deq seg = %p (0x%llx dma), new deq ptr = %p (0x%llx dma), new cycle = %u",
4136                 deq_state->new_deq_seg,
4137                 (unsigned long long)deq_state->new_deq_seg->dma,
4138                 deq_state->new_deq_ptr,
4139                 (unsigned long long)xhci_trb_virt_to_dma(
4140                         deq_state->new_deq_seg, deq_state->new_deq_ptr),
4141                 deq_state->new_cycle_state);
4142 
4143         addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
4144                                     deq_state->new_deq_ptr);
4145         if (addr == 0) {
4146                 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4147                 xhci_warn(xhci, "WARN deq seg = %p, deq pt = %p\n",
4148                           deq_state->new_deq_seg, deq_state->new_deq_ptr);
4149                 return;
4150         }
4151         ep = &xhci->devs[slot_id]->eps[ep_index];
4152         if ((ep->ep_state & SET_DEQ_PENDING)) {
4153                 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4154                 xhci_warn(xhci, "A Set TR Deq Ptr command is pending.\n");
4155                 return;
4156         }
4157 
4158         /* This function gets called from contexts where it cannot sleep */
4159         cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC);
4160         if (!cmd)
4161                 return;
4162 
4163         ep->queued_deq_seg = deq_state->new_deq_seg;
4164         ep->queued_deq_ptr = deq_state->new_deq_ptr;
4165         if (deq_state->stream_id)
4166                 trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
4167         ret = queue_command(xhci, cmd,
4168                 lower_32_bits(addr) | trb_sct | deq_state->new_cycle_state,
4169                 upper_32_bits(addr), trb_stream_id,
4170                 trb_slot_id | trb_ep_index | type, false);
4171         if (ret < 0) {
4172                 xhci_free_command(xhci, cmd);
4173                 return;
4174         }
4175 
4176         /* Stop the TD queueing code from ringing the doorbell until
4177          * this command completes.  The HC won't set the dequeue pointer
4178          * if the ring is running, and ringing the doorbell starts the
4179          * ring running.
4180          */
4181         ep->ep_state |= SET_DEQ_PENDING;
4182 }
4183 
4184 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd,
4185                         int slot_id, unsigned int ep_index,
4186                         enum xhci_ep_reset_type reset_type)
4187 {
4188         u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
4189         u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
4190         u32 type = TRB_TYPE(TRB_RESET_EP);
4191 
4192         if (reset_type == EP_SOFT_RESET)
4193                 type |= TRB_TSP;
4194 
4195         return queue_command(xhci, cmd, 0, 0, 0,
4196                         trb_slot_id | trb_ep_index | type, false);
4197 }