drivers/usb/cdns3/gadget.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
cdns3_set_register_bit
cdns3_ep_addr_to_index
cdns3_get_dma_pos
cdns3_next_request
cdns3_next_align_buf
cdns3_next_priv_request
cdns3_select_ep
cdns3_trb_virt_to_dma
cdns3_ring_size
cdns3_allocate_trb_pool
cdns3_free_trb_pool
cdns3_ep_stall_flush
cdns3_hw_reset_eps_config
cdns3_ep_inc_trb
cdns3_ep_inc_enq
cdns3_ep_inc_deq
cdns3_move_deq_to_next_trb
cdns3_allow_enable_l1
cdns3_get_speed
cdns3_start_all_request
cdns3_wa2_descmiss_copy_data
cdns3_wa2_gadget_giveback
cdns3_wa2_gadget_ep_queue
cdns3_wa2_remove_old_request
cdns3_wa2_descmissing_packet
cdns3_gadget_giveback
cdns3_wa1_restore_cycle_bit
cdns3_free_aligned_request_buf
cdns3_prepare_aligned_request_buf
cdns3_wa1_update_guard
cdns3_wa1_tray_restore_cycle_bit
cdns3_ep_run_transfer
cdns3_set_hw_configuration
cdns3_request_handled
cdns3_transfer_completed
cdns3_rearm_transfer
cdns3_check_ep_interrupt_proceed
cdns3_disconnect_gadget
cdns3_check_usb_interrupt_proceed
cdns3_device_irq_handler
cdns3_device_thread_irq_handler
cdns3_ep_onchip_buffer_reserve
cdns3_configure_dmult
cdns3_ep_config
cdns3_ep_dir_is_correct
cdns3_find_available_ep
cdns3_gadget_match_ep
cdns3_gadget_ep_alloc_request
cdns3_gadget_ep_free_request
cdns3_gadget_ep_enable
cdns3_gadget_ep_disable
__cdns3_gadget_ep_queue
cdns3_gadget_ep_queue
cdns3_gadget_ep_dequeue
__cdns3_gadget_ep_set_halt
__cdns3_gadget_ep_clear_halt
cdns3_gadget_ep_set_halt
cdns3_gadget_get_frame
__cdns3_gadget_wakeup
cdns3_gadget_wakeup
cdns3_gadget_set_selfpowered
cdns3_gadget_pullup
cdns3_gadget_config
cdns3_gadget_udc_start
cdns3_gadget_udc_stop
cdns3_free_all_eps
cdns3_init_eps
cdns3_gadget_exit
cdns3_gadget_start
__cdns3_gadget_init
cdns3_gadget_suspend
cdns3_gadget_resume
cdns3_gadget_init
   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Cadence USBSS DRD Driver - gadget side.
   4  *
   5  * Copyright (C) 2018-2019 Cadence Design Systems.
   6  * Copyright (C) 2017-2018 NXP
   7  *
   8  * Authors: Pawel Jez <pjez@cadence.com>,
   9  *          Pawel Laszczak <pawell@cadence.com>
  10  *          Peter Chen <peter.chen@nxp.com>
  11  */
  12 
  13 /*
  14  * Work around 1:
  15  * At some situations, the controller may get stale data address in TRB
  16  * at below sequences:
  17  * 1. Controller read TRB includes data address
  18  * 2. Software updates TRBs includes data address and Cycle bit
  19  * 3. Controller read TRB which includes Cycle bit
  20  * 4. DMA run with stale data address
  21  *
  22  * To fix this problem, driver needs to make the first TRB in TD as invalid.
  23  * After preparing all TRBs driver needs to check the position of DMA and
  24  * if the DMA point to the first just added TRB and doorbell is 1,
  25  * then driver must defer making this TRB as valid. This TRB will be make
  26  * as valid during adding next TRB only if DMA is stopped or at TRBERR
  27  * interrupt.
  28  *
  29  * Issue has been fixed in DEV_VER_V3 version of controller.
  30  *
  31  * Work around 2:
  32  * Controller for OUT endpoints has shared on-chip buffers for all incoming
  33  * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
  34  * in correct order. If the first packet in the buffer will not be handled,
  35  * then the following packets directed for other endpoints and  functions
  36  * will be blocked.
  37  * Additionally the packets directed to one endpoint can block entire on-chip
  38  * buffers. In this case transfer to other endpoints also will blocked.
  39  *
  40  * To resolve this issue after raising the descriptor missing interrupt
  41  * driver prepares internal usb_request object and use it to arm DMA transfer.
  42  *
  43  * The problematic situation was observed in case when endpoint has been enabled
  44  * but no usb_request were queued. Driver try detects such endpoints and will
  45  * use this workaround only for these endpoint.
  46  *
  47  * Driver use limited number of buffer. This number can be set by macro
  48  * CDNS3_WA2_NUM_BUFFERS.
  49  *
  50  * Such blocking situation was observed on ACM gadget. For this function
  51  * host send OUT data packet but ACM function is not prepared for this packet.
  52  * It's cause that buffer placed in on chip memory block transfer to other
  53  * endpoints.
  54  *
  55  * Issue has been fixed in DEV_VER_V2 version of controller.
  56  *
  57  */
  58 
  59 #include <linux/dma-mapping.h>
  60 #include <linux/usb/gadget.h>
  61 #include <linux/module.h>
  62 #include <linux/iopoll.h>
  63 
  64 #include "core.h"
  65 #include "gadget-export.h"
  66 #include "gadget.h"
  67 #include "trace.h"
  68 #include "drd.h"
  69 
  70 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  71                                    struct usb_request *request,
  72                                    gfp_t gfp_flags);
  73 
  74 /**
  75  * cdns3_set_register_bit - set bit in given register.
  76  * @ptr: address of device controller register to be read and changed
  77  * @mask: bits requested to set
  78  */
  79 void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  80 {
  81         mask = readl(ptr) | mask;
  82         writel(mask, ptr);
  83 }
  84 
  85 /**
  86  * cdns3_ep_addr_to_index - Macro converts endpoint address to
  87  * index of endpoint object in cdns3_device.eps[] container
  88  * @ep_addr: endpoint address for which endpoint object is required
  89  *
  90  */
  91 u8 cdns3_ep_addr_to_index(u8 ep_addr)
  92 {
  93         return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
  94 }
  95 
  96 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
  97                              struct cdns3_endpoint *priv_ep)
  98 {
  99         int dma_index;
 100 
 101         dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
 102 
 103         return dma_index / TRB_SIZE;
 104 }
 105 
 106 /**
 107  * cdns3_next_request - returns next request from list
 108  * @list: list containing requests
 109  *
 110  * Returns request or NULL if no requests in list
 111  */
 112 struct usb_request *cdns3_next_request(struct list_head *list)
 113 {
 114         return list_first_entry_or_null(list, struct usb_request, list);
 115 }
 116 
 117 /**
 118  * cdns3_next_align_buf - returns next buffer from list
 119  * @list: list containing buffers
 120  *
 121  * Returns buffer or NULL if no buffers in list
 122  */
 123 struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
 124 {
 125         return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
 126 }
 127 
 128 /**
 129  * cdns3_next_priv_request - returns next request from list
 130  * @list: list containing requests
 131  *
 132  * Returns request or NULL if no requests in list
 133  */
 134 struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
 135 {
 136         return list_first_entry_or_null(list, struct cdns3_request, list);
 137 }
 138 
 139 /**
 140  * select_ep - selects endpoint
 141  * @priv_dev:  extended gadget object
 142  * @ep: endpoint address
 143  */
 144 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
 145 {
 146         if (priv_dev->selected_ep == ep)
 147                 return;
 148 
 149         priv_dev->selected_ep = ep;
 150         writel(ep, &priv_dev->regs->ep_sel);
 151 }
 152 
 153 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
 154                                  struct cdns3_trb *trb)
 155 {
 156         u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
 157 
 158         return priv_ep->trb_pool_dma + offset;
 159 }
 160 
 161 int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
 162 {
 163         switch (priv_ep->type) {
 164         case USB_ENDPOINT_XFER_ISOC:
 165                 return TRB_ISO_RING_SIZE;
 166         case USB_ENDPOINT_XFER_CONTROL:
 167                 return TRB_CTRL_RING_SIZE;
 168         default:
 169                 return TRB_RING_SIZE;
 170         }
 171 }
 172 
 173 /**
 174  * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
 175  * @priv_ep:  endpoint object
 176  *
 177  * Function will return 0 on success or -ENOMEM on allocation error
 178  */
 179 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
 180 {
 181         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 182         int ring_size = cdns3_ring_size(priv_ep);
 183         struct cdns3_trb *link_trb;
 184 
 185         if (!priv_ep->trb_pool) {
 186                 priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
 187                                                        ring_size,
 188                                                        &priv_ep->trb_pool_dma,
 189                                                        GFP_DMA32 | GFP_ATOMIC);
 190                 if (!priv_ep->trb_pool)
 191                         return -ENOMEM;
 192         } else {
 193                 memset(priv_ep->trb_pool, 0, ring_size);
 194         }
 195 
 196         if (!priv_ep->num)
 197                 return 0;
 198 
 199         priv_ep->num_trbs = ring_size / TRB_SIZE;
 200         /* Initialize the last TRB as Link TRB. */
 201         link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
 202         link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
 203         link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
 204 
 205         return 0;
 206 }
 207 
 208 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
 209 {
 210         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 211 
 212         if (priv_ep->trb_pool) {
 213                 dma_free_coherent(priv_dev->sysdev,
 214                                   cdns3_ring_size(priv_ep),
 215                                   priv_ep->trb_pool, priv_ep->trb_pool_dma);
 216                 priv_ep->trb_pool = NULL;
 217         }
 218 }
 219 
 220 /**
 221  * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
 222  * @priv_ep: endpoint object
 223  *
 224  * Endpoint must be selected before call to this function
 225  */
 226 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
 227 {
 228         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 229         int val;
 230 
 231         trace_cdns3_halt(priv_ep, 1, 1);
 232 
 233         writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
 234                &priv_dev->regs->ep_cmd);
 235 
 236         /* wait for DFLUSH cleared */
 237         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
 238                                   !(val & EP_CMD_DFLUSH), 1, 1000);
 239         priv_ep->flags |= EP_STALLED;
 240         priv_ep->flags &= ~EP_STALL_PENDING;
 241 }
 242 
 243 /**
 244  * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
 245  * @priv_dev: extended gadget object
 246  */
 247 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
 248 {
 249         writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
 250 
 251         cdns3_allow_enable_l1(priv_dev, 0);
 252         priv_dev->hw_configured_flag = 0;
 253         priv_dev->onchip_used_size = 0;
 254         priv_dev->out_mem_is_allocated = 0;
 255         priv_dev->wait_for_setup = 0;
 256 }
 257 
 258 /**
 259  * cdns3_ep_inc_trb - increment a trb index.
 260  * @index: Pointer to the TRB index to increment.
 261  * @cs: Cycle state
 262  * @trb_in_seg: number of TRBs in segment
 263  *
 264  * The index should never point to the link TRB. After incrementing,
 265  * if it is point to the link TRB, wrap around to the beginning and revert
 266  * cycle state bit The
 267  * link TRB is always at the last TRB entry.
 268  */
 269 static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
 270 {
 271         (*index)++;
 272         if (*index == (trb_in_seg - 1)) {
 273                 *index = 0;
 274                 *cs ^=  1;
 275         }
 276 }
 277 
 278 /**
 279  * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
 280  * @priv_ep: The endpoint whose enqueue pointer we're incrementing
 281  */
 282 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
 283 {
 284         priv_ep->free_trbs--;
 285         cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
 286 }
 287 
 288 /**
 289  * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
 290  * @priv_ep: The endpoint whose dequeue pointer we're incrementing
 291  */
 292 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
 293 {
 294         priv_ep->free_trbs++;
 295         cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
 296 }
 297 
 298 void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
 299 {
 300         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 301         int current_trb = priv_req->start_trb;
 302 
 303         while (current_trb != priv_req->end_trb) {
 304                 cdns3_ep_inc_deq(priv_ep);
 305                 current_trb = priv_ep->dequeue;
 306         }
 307 
 308         cdns3_ep_inc_deq(priv_ep);
 309 }
 310 
 311 /**
 312  * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
 313  * @priv_dev: Extended gadget object
 314  * @enable: Enable/disable permit to transition to L1.
 315  *
 316  * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
 317  * then controller answer with ACK handshake.
 318  * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
 319  * then controller answer with NYET handshake.
 320  */
 321 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
 322 {
 323         if (enable)
 324                 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
 325         else
 326                 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
 327 }
 328 
 329 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
 330 {
 331         u32 reg;
 332 
 333         reg = readl(&priv_dev->regs->usb_sts);
 334 
 335         if (DEV_SUPERSPEED(reg))
 336                 return USB_SPEED_SUPER;
 337         else if (DEV_HIGHSPEED(reg))
 338                 return USB_SPEED_HIGH;
 339         else if (DEV_FULLSPEED(reg))
 340                 return USB_SPEED_FULL;
 341         else if (DEV_LOWSPEED(reg))
 342                 return USB_SPEED_LOW;
 343         return USB_SPEED_UNKNOWN;
 344 }
 345 
 346 /**
 347  * cdns3_start_all_request - add to ring all request not started
 348  * @priv_dev: Extended gadget object
 349  * @priv_ep: The endpoint for whom request will be started.
 350  *
 351  * Returns return ENOMEM if transfer ring i not enough TRBs to start
 352  *         all requests.
 353  */
 354 static int cdns3_start_all_request(struct cdns3_device *priv_dev,
 355                                    struct cdns3_endpoint *priv_ep)
 356 {
 357         struct usb_request *request;
 358         int ret = 0;
 359 
 360         while (!list_empty(&priv_ep->deferred_req_list)) {
 361                 request = cdns3_next_request(&priv_ep->deferred_req_list);
 362 
 363                 ret = cdns3_ep_run_transfer(priv_ep, request);
 364                 if (ret)
 365                         return ret;
 366 
 367                 list_del(&request->list);
 368                 list_add_tail(&request->list,
 369                               &priv_ep->pending_req_list);
 370         }
 371 
 372         priv_ep->flags &= ~EP_RING_FULL;
 373         return ret;
 374 }
 375 
 376 /*
 377  * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
 378  * driver try to detect whether endpoint need additional internal
 379  * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
 380  * if before first DESCMISS interrupt the DMA will be armed.
 381  */
 382 #define cdns3_wa2_enable_detection(priv_dev, ep_priv, reg) do { \
 383         if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
 384                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
 385                 (reg) |= EP_STS_EN_DESCMISEN; \
 386         } } while (0)
 387 
 388 /**
 389  * cdns3_wa2_descmiss_copy_data copy data from internal requests to
 390  * request queued by class driver.
 391  * @priv_ep: extended endpoint object
 392  * @request: request object
 393  */
 394 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
 395                                          struct usb_request *request)
 396 {
 397         struct usb_request *descmiss_req;
 398         struct cdns3_request *descmiss_priv_req;
 399 
 400         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 401                 int chunk_end;
 402                 int length;
 403 
 404                 descmiss_priv_req =
 405                         cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 406                 descmiss_req = &descmiss_priv_req->request;
 407 
 408                 /* driver can't touch pending request */
 409                 if (descmiss_priv_req->flags & REQUEST_PENDING)
 410                         break;
 411 
 412                 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
 413                 length = request->actual + descmiss_req->actual;
 414 
 415                 request->status = descmiss_req->status;
 416 
 417                 if (length <= request->length) {
 418                         memcpy(&((u8 *)request->buf)[request->actual],
 419                                descmiss_req->buf,
 420                                descmiss_req->actual);
 421                         request->actual = length;
 422                 } else {
 423                         /* It should never occures */
 424                         request->status = -ENOMEM;
 425                 }
 426 
 427                 list_del_init(&descmiss_priv_req->list);
 428 
 429                 kfree(descmiss_req->buf);
 430                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
 431                 --priv_ep->wa2_counter;
 432 
 433                 if (!chunk_end)
 434                         break;
 435         }
 436 }
 437 
 438 struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
 439                                               struct cdns3_endpoint *priv_ep,
 440                                               struct cdns3_request *priv_req)
 441 {
 442         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
 443             priv_req->flags & REQUEST_INTERNAL) {
 444                 struct usb_request *req;
 445 
 446                 req = cdns3_next_request(&priv_ep->deferred_req_list);
 447 
 448                 priv_ep->descmis_req = NULL;
 449 
 450                 if (!req)
 451                         return NULL;
 452 
 453                 cdns3_wa2_descmiss_copy_data(priv_ep, req);
 454                 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
 455                     req->length != req->actual) {
 456                         /* wait for next part of transfer */
 457                         return NULL;
 458                 }
 459 
 460                 if (req->status == -EINPROGRESS)
 461                         req->status = 0;
 462 
 463                 list_del_init(&req->list);
 464                 cdns3_start_all_request(priv_dev, priv_ep);
 465                 return req;
 466         }
 467 
 468         return &priv_req->request;
 469 }
 470 
 471 int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
 472                               struct cdns3_endpoint *priv_ep,
 473                               struct cdns3_request *priv_req)
 474 {
 475         int deferred = 0;
 476 
 477         /*
 478          * If transfer was queued before DESCMISS appear than we
 479          * can disable handling of DESCMISS interrupt. Driver assumes that it
 480          * can disable special treatment for this endpoint.
 481          */
 482         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 483                 u32 reg;
 484 
 485                 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
 486                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 487                 reg = readl(&priv_dev->regs->ep_sts_en);
 488                 reg &= ~EP_STS_EN_DESCMISEN;
 489                 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
 490                 writel(reg, &priv_dev->regs->ep_sts_en);
 491         }
 492 
 493         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
 494                 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 495                 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
 496 
 497                 /*
 498                  *  DESCMISS transfer has been finished, so data will be
 499                  *  directly copied from internal allocated usb_request
 500                  *  objects.
 501                  */
 502                 if (pending_empty && !descmiss_empty &&
 503                     !(priv_req->flags & REQUEST_INTERNAL)) {
 504                         cdns3_wa2_descmiss_copy_data(priv_ep,
 505                                                      &priv_req->request);
 506 
 507                         trace_cdns3_wa2(priv_ep, "get internal stored data");
 508 
 509                         list_add_tail(&priv_req->request.list,
 510                                       &priv_ep->pending_req_list);
 511                         cdns3_gadget_giveback(priv_ep, priv_req,
 512                                               priv_req->request.status);
 513 
 514                         /*
 515                          * Intentionally driver returns positive value as
 516                          * correct value. It informs that transfer has
 517                          * been finished.
 518                          */
 519                         return EINPROGRESS;
 520                 }
 521 
 522                 /*
 523                  * Driver will wait for completion DESCMISS transfer,
 524                  * before starts new, not DESCMISS transfer.
 525                  */
 526                 if (!pending_empty && !descmiss_empty) {
 527                         trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
 528                         deferred = 1;
 529                 }
 530 
 531                 if (priv_req->flags & REQUEST_INTERNAL)
 532                         list_add_tail(&priv_req->list,
 533                                       &priv_ep->wa2_descmiss_req_list);
 534         }
 535 
 536         return deferred;
 537 }
 538 
 539 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
 540 {
 541         struct cdns3_request *priv_req;
 542 
 543         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 544                 u8 chain;
 545 
 546                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 547                 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
 548 
 549                 trace_cdns3_wa2(priv_ep, "removes eldest request");
 550 
 551                 kfree(priv_req->request.buf);
 552                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
 553                                              &priv_req->request);
 554                 list_del_init(&priv_req->list);
 555                 --priv_ep->wa2_counter;
 556 
 557                 if (!chain)
 558                         break;
 559         }
 560 }
 561 
 562 /**
 563  * cdns3_wa2_descmissing_packet - handles descriptor missing event.
 564  * @priv_dev: extended gadget object
 565  *
 566  * This function is used only for WA2. For more information see Work around 2
 567  * description.
 568  */
 569 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
 570 {
 571         struct cdns3_request *priv_req;
 572         struct usb_request *request;
 573 
 574         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 575                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 576                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
 577         }
 578 
 579         trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
 580 
 581         if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS)
 582                 cdns3_wa2_remove_old_request(priv_ep);
 583 
 584         request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
 585                                                 GFP_ATOMIC);
 586         if (!request)
 587                 goto err;
 588 
 589         priv_req = to_cdns3_request(request);
 590         priv_req->flags |= REQUEST_INTERNAL;
 591 
 592         /* if this field is still assigned it indicate that transfer related
 593          * with this request has not been finished yet. Driver in this
 594          * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
 595          * flag to previous one. It will indicate that current request is
 596          * part of the previous one.
 597          */
 598         if (priv_ep->descmis_req)
 599                 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
 600 
 601         priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
 602                                         GFP_ATOMIC);
 603         priv_ep->wa2_counter++;
 604 
 605         if (!priv_req->request.buf) {
 606                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 607                 goto err;
 608         }
 609 
 610         priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
 611         priv_ep->descmis_req = priv_req;
 612 
 613         __cdns3_gadget_ep_queue(&priv_ep->endpoint,
 614                                 &priv_ep->descmis_req->request,
 615                                 GFP_ATOMIC);
 616 
 617         return;
 618 
 619 err:
 620         dev_err(priv_ep->cdns3_dev->dev,
 621                 "Failed: No sufficient memory for DESCMIS\n");
 622 }
 623 
 624 /**
 625  * cdns3_gadget_giveback - call struct usb_request's ->complete callback
 626  * @priv_ep: The endpoint to whom the request belongs to
 627  * @priv_req: The request we're giving back
 628  * @status: completion code for the request
 629  *
 630  * Must be called with controller's lock held and interrupts disabled. This
 631  * function will unmap @req and call its ->complete() callback to notify upper
 632  * layers that it has completed.
 633  */
 634 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
 635                            struct cdns3_request *priv_req,
 636                            int status)
 637 {
 638         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 639         struct usb_request *request = &priv_req->request;
 640 
 641         list_del_init(&request->list);
 642 
 643         if (request->status == -EINPROGRESS)
 644                 request->status = status;
 645 
 646         usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
 647                                         priv_ep->dir);
 648 
 649         if ((priv_req->flags & REQUEST_UNALIGNED) &&
 650             priv_ep->dir == USB_DIR_OUT && !request->status)
 651                 memcpy(request->buf, priv_req->aligned_buf->buf,
 652                        request->length);
 653 
 654         priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
 655         trace_cdns3_gadget_giveback(priv_req);
 656 
 657         if (priv_dev->dev_ver < DEV_VER_V2) {
 658                 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
 659                                                     priv_req);
 660                 if (!request)
 661                         return;
 662         }
 663 
 664         if (request->complete) {
 665                 spin_unlock(&priv_dev->lock);
 666                 usb_gadget_giveback_request(&priv_ep->endpoint,
 667                                             request);
 668                 spin_lock(&priv_dev->lock);
 669         }
 670 
 671         if (request->buf == priv_dev->zlp_buf)
 672                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 673 }
 674 
 675 void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
 676 {
 677         /* Work around for stale data address in TRB*/
 678         if (priv_ep->wa1_set) {
 679                 trace_cdns3_wa1(priv_ep, "restore cycle bit");
 680 
 681                 priv_ep->wa1_set = 0;
 682                 priv_ep->wa1_trb_index = 0xFFFF;
 683                 if (priv_ep->wa1_cycle_bit) {
 684                         priv_ep->wa1_trb->control =
 685                                 priv_ep->wa1_trb->control | 0x1;
 686                 } else {
 687                         priv_ep->wa1_trb->control =
 688                                 priv_ep->wa1_trb->control & ~0x1;
 689                 }
 690         }
 691 }
 692 
 693 static void cdns3_free_aligned_request_buf(struct work_struct *work)
 694 {
 695         struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
 696                                         aligned_buf_wq);
 697         struct cdns3_aligned_buf *buf, *tmp;
 698         unsigned long flags;
 699 
 700         spin_lock_irqsave(&priv_dev->lock, flags);
 701 
 702         list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
 703                 if (!buf->in_use) {
 704                         list_del(&buf->list);
 705 
 706                         /*
 707                          * Re-enable interrupts to free DMA capable memory.
 708                          * Driver can't free this memory with disabled
 709                          * interrupts.
 710                          */
 711                         spin_unlock_irqrestore(&priv_dev->lock, flags);
 712                         dma_free_coherent(priv_dev->sysdev, buf->size,
 713                                           buf->buf, buf->dma);
 714                         kfree(buf);
 715                         spin_lock_irqsave(&priv_dev->lock, flags);
 716                 }
 717         }
 718 
 719         spin_unlock_irqrestore(&priv_dev->lock, flags);
 720 }
 721 
 722 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
 723 {
 724         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 725         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 726         struct cdns3_aligned_buf *buf;
 727 
 728         /* check if buffer is aligned to 8. */
 729         if (!((uintptr_t)priv_req->request.buf & 0x7))
 730                 return 0;
 731 
 732         buf = priv_req->aligned_buf;
 733 
 734         if (!buf || priv_req->request.length > buf->size) {
 735                 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
 736                 if (!buf)
 737                         return -ENOMEM;
 738 
 739                 buf->size = priv_req->request.length;
 740 
 741                 buf->buf = dma_alloc_coherent(priv_dev->sysdev,
 742                                               buf->size,
 743                                               &buf->dma,
 744                                               GFP_ATOMIC);
 745                 if (!buf->buf) {
 746                         kfree(buf);
 747                         return -ENOMEM;
 748                 }
 749 
 750                 if (priv_req->aligned_buf) {
 751                         trace_cdns3_free_aligned_request(priv_req);
 752                         priv_req->aligned_buf->in_use = 0;
 753                         queue_work(system_freezable_wq,
 754                                    &priv_dev->aligned_buf_wq);
 755                 }
 756 
 757                 buf->in_use = 1;
 758                 priv_req->aligned_buf = buf;
 759 
 760                 list_add_tail(&buf->list,
 761                               &priv_dev->aligned_buf_list);
 762         }
 763 
 764         if (priv_ep->dir == USB_DIR_IN) {
 765                 memcpy(buf->buf, priv_req->request.buf,
 766                        priv_req->request.length);
 767         }
 768 
 769         priv_req->flags |= REQUEST_UNALIGNED;
 770         trace_cdns3_prepare_aligned_request(priv_req);
 771 
 772         return 0;
 773 }
 774 
 775 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
 776                                   struct cdns3_trb *trb)
 777 {
 778         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 779 
 780         if (!priv_ep->wa1_set) {
 781                 u32 doorbell;
 782 
 783                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 784 
 785                 if (doorbell) {
 786                         priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
 787                         priv_ep->wa1_set = 1;
 788                         priv_ep->wa1_trb = trb;
 789                         priv_ep->wa1_trb_index = priv_ep->enqueue;
 790                         trace_cdns3_wa1(priv_ep, "set guard");
 791                         return 0;
 792                 }
 793         }
 794         return 1;
 795 }
 796 
 797 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
 798                                              struct cdns3_endpoint *priv_ep)
 799 {
 800         int dma_index;
 801         u32 doorbell;
 802 
 803         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 804         dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 805 
 806         if (!doorbell || dma_index != priv_ep->wa1_trb_index)
 807                 cdns3_wa1_restore_cycle_bit(priv_ep);
 808 }
 809 
 810 /**
 811  * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
 812  * @priv_ep: endpoint object
 813  *
 814  * Returns zero on success or negative value on failure
 815  */
 816 int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
 817                           struct usb_request *request)
 818 {
 819         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 820         struct cdns3_request *priv_req;
 821         struct cdns3_trb *trb;
 822         dma_addr_t trb_dma;
 823         u32 togle_pcs = 1;
 824         int sg_iter = 0;
 825         int num_trb;
 826         int address;
 827         u32 control;
 828         int pcs;
 829 
 830         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
 831                 num_trb = priv_ep->interval;
 832         else
 833                 num_trb = request->num_sgs ? request->num_sgs : 1;
 834 
 835         if (num_trb > priv_ep->free_trbs) {
 836                 priv_ep->flags |= EP_RING_FULL;
 837                 return -ENOBUFS;
 838         }
 839 
 840         priv_req = to_cdns3_request(request);
 841         address = priv_ep->endpoint.desc->bEndpointAddress;
 842 
 843         priv_ep->flags |= EP_PENDING_REQUEST;
 844 
 845         /* must allocate buffer aligned to 8 */
 846         if (priv_req->flags & REQUEST_UNALIGNED)
 847                 trb_dma = priv_req->aligned_buf->dma;
 848         else
 849                 trb_dma = request->dma;
 850 
 851         trb = priv_ep->trb_pool + priv_ep->enqueue;
 852         priv_req->start_trb = priv_ep->enqueue;
 853         priv_req->trb = trb;
 854 
 855         cdns3_select_ep(priv_ep->cdns3_dev, address);
 856 
 857         /* prepare ring */
 858         if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
 859                 struct cdns3_trb *link_trb;
 860                 int doorbell, dma_index;
 861                 u32 ch_bit = 0;
 862 
 863                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 864                 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 865 
 866                 /* Driver can't update LINK TRB if it is current processed. */
 867                 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
 868                         priv_ep->flags |= EP_DEFERRED_DRDY;
 869                         return -ENOBUFS;
 870                 }
 871 
 872                 /*updating C bt in  Link TRB before starting DMA*/
 873                 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
 874                 /*
 875                  * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
 876                  * that DMA stuck at the LINK TRB.
 877                  * On the other hand, removing TRB_CHAIN for longer TRs for
 878                  * epXout cause that DMA stuck after handling LINK TRB.
 879                  * To eliminate this strange behavioral driver set TRB_CHAIN
 880                  * bit only for TR size > 2.
 881                  */
 882                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
 883                     TRBS_PER_SEGMENT > 2)
 884                         ch_bit = TRB_CHAIN;
 885 
 886                 link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
 887                                     TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
 888         }
 889 
 890         if (priv_dev->dev_ver <= DEV_VER_V2)
 891                 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
 892 
 893         /* set incorrect Cycle Bit for first trb*/
 894         control = priv_ep->pcs ? 0 : TRB_CYCLE;
 895 
 896         do {
 897                 u32 length;
 898                 u16 td_size = 0;
 899 
 900                 /* fill TRB */
 901                 control |= TRB_TYPE(TRB_NORMAL);
 902                 trb->buffer = TRB_BUFFER(request->num_sgs == 0
 903                                 ? trb_dma : request->sg[sg_iter].dma_address);
 904 
 905                 if (likely(!request->num_sgs))
 906                         length = request->length;
 907                 else
 908                         length = request->sg[sg_iter].length;
 909 
 910                 if (likely(priv_dev->dev_ver >= DEV_VER_V2))
 911                         td_size = DIV_ROUND_UP(length,
 912                                                priv_ep->endpoint.maxpacket);
 913 
 914                 trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
 915                                         TRB_LEN(length);
 916                 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
 917                         trb->length |= TRB_TDL_SS_SIZE(td_size);
 918                 else
 919                         control |= TRB_TDL_HS_SIZE(td_size);
 920 
 921                 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
 922 
 923                 /*
 924                  * first trb should be prepared as last to avoid processing
 925                  *  transfer to early
 926                  */
 927                 if (sg_iter != 0)
 928                         control |= pcs;
 929 
 930                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
 931                         control |= TRB_IOC | TRB_ISP;
 932                 } else {
 933                         /* for last element in TD or in SG list */
 934                         if (sg_iter == (num_trb - 1) && sg_iter != 0)
 935                                 control |= pcs | TRB_IOC | TRB_ISP;
 936                 }
 937 
 938                 if (sg_iter)
 939                         trb->control = control;
 940                 else
 941                         priv_req->trb->control = control;
 942 
 943                 control = 0;
 944                 ++sg_iter;
 945                 priv_req->end_trb = priv_ep->enqueue;
 946                 cdns3_ep_inc_enq(priv_ep);
 947                 trb = priv_ep->trb_pool + priv_ep->enqueue;
 948         } while (sg_iter < num_trb);
 949 
 950         trb = priv_req->trb;
 951 
 952         priv_req->flags |= REQUEST_PENDING;
 953 
 954         if (sg_iter == 1)
 955                 trb->control |= TRB_IOC | TRB_ISP;
 956 
 957         /*
 958          * Memory barrier - cycle bit must be set before other filds in trb.
 959          */
 960         wmb();
 961 
 962         /* give the TD to the consumer*/
 963         if (togle_pcs)
 964                 trb->control =  trb->control ^ 1;
 965 
 966         if (priv_dev->dev_ver <= DEV_VER_V2)
 967                 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
 968 
 969         trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
 970 
 971         /*
 972          * Memory barrier - Cycle Bit must be set before trb->length  and
 973          * trb->buffer fields.
 974          */
 975         wmb();
 976 
 977         /*
 978          * For DMULT mode we can set address to transfer ring only once after
 979          * enabling endpoint.
 980          */
 981         if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
 982                 /*
 983                  * Until SW is not ready to handle the OUT transfer the ISO OUT
 984                  * Endpoint should be disabled (EP_CFG.ENABLE = 0).
 985                  * EP_CFG_ENABLE must be set before updating ep_traddr.
 986                  */
 987                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
 988                     !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
 989                         priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
 990                         cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
 991                                                EP_CFG_ENABLE);
 992                 }
 993 
 994                 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
 995                                         priv_req->start_trb * TRB_SIZE),
 996                                         &priv_dev->regs->ep_traddr);
 997 
 998                 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
 999         }
1000 
1001         if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1002                 trace_cdns3_ring(priv_ep);
1003                 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1004                 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1005                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1006                 trace_cdns3_doorbell_epx(priv_ep->name,
1007                                          readl(&priv_dev->regs->ep_traddr));
1008         }
1009 
1010         /* WORKAROUND for transition to L0 */
1011         __cdns3_gadget_wakeup(priv_dev);
1012 
1013         return 0;
1014 }
1015 
1016 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1017 {
1018         struct cdns3_endpoint *priv_ep;
1019         struct usb_ep *ep;
1020         int val;
1021 
1022         if (priv_dev->hw_configured_flag)
1023                 return;
1024 
1025         writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1026         writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
1027 
1028         cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1029                                USB_CONF_U1EN | USB_CONF_U2EN);
1030 
1031         /* wait until configuration set */
1032         readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
1033                                   val & USB_STS_CFGSTS_MASK, 1, 100);
1034 
1035         priv_dev->hw_configured_flag = 1;
1036 
1037         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1038                 if (ep->enabled) {
1039                         priv_ep = ep_to_cdns3_ep(ep);
1040                         cdns3_start_all_request(priv_dev, priv_ep);
1041                 }
1042         }
1043 }
1044 
1045 /**
1046  * cdns3_request_handled - check whether request has been handled by DMA
1047  *
1048  * @priv_ep: extended endpoint object.
1049  * @priv_req: request object for checking
1050  *
1051  * Endpoint must be selected before invoking this function.
1052  *
1053  * Returns false if request has not been handled by DMA, else returns true.
1054  *
1055  * SR - start ring
1056  * ER -  end ring
1057  * DQ = priv_ep->dequeue - dequeue position
1058  * EQ = priv_ep->enqueue -  enqueue position
1059  * ST = priv_req->start_trb - index of first TRB in transfer ring
1060  * ET = priv_req->end_trb - index of last TRB in transfer ring
1061  * CI = current_index - index of processed TRB by DMA.
1062  *
1063  * As first step, function checks if cycle bit for priv_req->start_trb is
1064  * correct.
1065  *
1066  * some rules:
1067  * 1. priv_ep->dequeue never exceed current_index.
1068  * 2  priv_ep->enqueue never exceed priv_ep->dequeue
1069  * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1070  *    and priv_ep->free_trbs is zero.
1071  *    This case indicate that TR is full.
1072  *
1073  * Then We can split recognition into two parts:
1074  * Case 1 - priv_ep->dequeue < current_index
1075  *      SR ... EQ ... DQ ... CI ... ER
1076  *      SR ... DQ ... CI ... EQ ... ER
1077  *
1078  *      Request has been handled by DMA if ST and ET is between DQ and CI.
1079  *
1080  * Case 2 - priv_ep->dequeue > current_index
1081  * This situation take place when CI go through the LINK TRB at the end of
1082  * transfer ring.
1083  *      SR ... CI ... EQ ... DQ ... ER
1084  *
1085  *      Request has been handled by DMA if ET is less then CI or
1086  *      ET is greater or equal DQ.
1087  */
1088 static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
1089                                   struct cdns3_request *priv_req)
1090 {
1091         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1092         struct cdns3_trb *trb = priv_req->trb;
1093         int current_index = 0;
1094         int handled = 0;
1095         int doorbell;
1096 
1097         current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1098         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1099 
1100         trb = &priv_ep->trb_pool[priv_req->start_trb];
1101 
1102         if ((trb->control  & TRB_CYCLE) != priv_ep->ccs)
1103                 goto finish;
1104 
1105         if (doorbell == 1 && current_index == priv_ep->dequeue)
1106                 goto finish;
1107 
1108         /* The corner case for TRBS_PER_SEGMENT equal 2). */
1109         if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1110                 handled = 1;
1111                 goto finish;
1112         }
1113 
1114         if (priv_ep->enqueue == priv_ep->dequeue &&
1115             priv_ep->free_trbs == 0) {
1116                 handled = 1;
1117         } else if (priv_ep->dequeue < current_index) {
1118                 if ((current_index == (priv_ep->num_trbs - 1)) &&
1119                     !priv_ep->dequeue)
1120                         goto finish;
1121 
1122                 if (priv_req->end_trb >= priv_ep->dequeue &&
1123                     priv_req->end_trb < current_index)
1124                         handled = 1;
1125         } else if (priv_ep->dequeue  > current_index) {
1126                 if (priv_req->end_trb  < current_index ||
1127                     priv_req->end_trb >= priv_ep->dequeue)
1128                         handled = 1;
1129         }
1130 
1131 finish:
1132         trace_cdns3_request_handled(priv_req, current_index, handled);
1133 
1134         return handled;
1135 }
1136 
1137 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1138                                      struct cdns3_endpoint *priv_ep)
1139 {
1140         struct cdns3_request *priv_req;
1141         struct usb_request *request;
1142         struct cdns3_trb *trb;
1143 
1144         while (!list_empty(&priv_ep->pending_req_list)) {
1145                 request = cdns3_next_request(&priv_ep->pending_req_list);
1146                 priv_req = to_cdns3_request(request);
1147 
1148                 trb = priv_ep->trb_pool + priv_ep->dequeue;
1149 
1150                 /* Request was dequeued and TRB was changed to TRB_LINK. */
1151                 if (TRB_FIELD_TO_TYPE(trb->control) == TRB_LINK) {
1152                         trace_cdns3_complete_trb(priv_ep, trb);
1153                         cdns3_move_deq_to_next_trb(priv_req);
1154                 }
1155 
1156                 /* Re-select endpoint. It could be changed by other CPU during
1157                  * handling usb_gadget_giveback_request.
1158                  */
1159                 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1160 
1161                 if (!cdns3_request_handled(priv_ep, priv_req))
1162                         goto prepare_next_td;
1163 
1164                 trb = priv_ep->trb_pool + priv_ep->dequeue;
1165                 trace_cdns3_complete_trb(priv_ep, trb);
1166 
1167                 if (trb != priv_req->trb)
1168                         dev_warn(priv_dev->dev,
1169                                  "request_trb=0x%p, queue_trb=0x%p\n",
1170                                  priv_req->trb, trb);
1171 
1172                 request->actual = TRB_LEN(le32_to_cpu(trb->length));
1173                 cdns3_move_deq_to_next_trb(priv_req);
1174                 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1175 
1176                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1177                     TRBS_PER_SEGMENT == 2)
1178                         break;
1179         }
1180         priv_ep->flags &= ~EP_PENDING_REQUEST;
1181 
1182 prepare_next_td:
1183         if (!(priv_ep->flags & EP_STALLED) &&
1184             !(priv_ep->flags & EP_STALL_PENDING))
1185                 cdns3_start_all_request(priv_dev, priv_ep);
1186 }
1187 
1188 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1189 {
1190         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1191 
1192         cdns3_wa1_restore_cycle_bit(priv_ep);
1193 
1194         if (rearm) {
1195                 trace_cdns3_ring(priv_ep);
1196 
1197                 /* Cycle Bit must be updated before arming DMA. */
1198                 wmb();
1199                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1200 
1201                 __cdns3_gadget_wakeup(priv_dev);
1202 
1203                 trace_cdns3_doorbell_epx(priv_ep->name,
1204                                          readl(&priv_dev->regs->ep_traddr));
1205         }
1206 }
1207 
1208 /**
1209  * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1210  * @priv_ep: endpoint object
1211  *
1212  * Returns 0
1213  */
1214 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1215 {
1216         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1217         u32 ep_sts_reg;
1218 
1219         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1220 
1221         trace_cdns3_epx_irq(priv_dev, priv_ep);
1222 
1223         ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1224         writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1225 
1226         if (ep_sts_reg & EP_STS_TRBERR) {
1227                 if (priv_ep->flags & EP_STALL_PENDING &&
1228                     !(ep_sts_reg & EP_STS_DESCMIS &&
1229                     priv_dev->dev_ver < DEV_VER_V2)) {
1230                         cdns3_ep_stall_flush(priv_ep);
1231                 }
1232 
1233                 /*
1234                  * For isochronous transfer driver completes request on
1235                  * IOC or on TRBERR. IOC appears only when device receive
1236                  * OUT data packet. If host disable stream or lost some packet
1237                  * then the only way to finish all queued transfer is to do it
1238                  * on TRBERR event.
1239                  */
1240                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1241                     !priv_ep->wa1_set) {
1242                         if (!priv_ep->dir) {
1243                                 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1244 
1245                                 ep_cfg &= ~EP_CFG_ENABLE;
1246                                 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1247                                 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1248                         }
1249                         cdns3_transfer_completed(priv_dev, priv_ep);
1250                 } else if (!(priv_ep->flags & EP_STALLED) &&
1251                           !(priv_ep->flags & EP_STALL_PENDING)) {
1252                         if (priv_ep->flags & EP_DEFERRED_DRDY) {
1253                                 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1254                                 cdns3_start_all_request(priv_dev, priv_ep);
1255                         } else {
1256                                 cdns3_rearm_transfer(priv_ep,
1257                                                      priv_ep->wa1_set);
1258                         }
1259                 }
1260         }
1261 
1262         if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
1263                 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1264                         if (ep_sts_reg & EP_STS_ISP)
1265                                 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1266                         else
1267                                 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1268                 }
1269 
1270                 cdns3_transfer_completed(priv_dev, priv_ep);
1271         }
1272 
1273         /*
1274          * WA2: this condition should only be meet when
1275          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1276          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1277          * In other cases this interrupt will be disabled/
1278          */
1279         if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1280             !(priv_ep->flags & EP_STALLED))
1281                 cdns3_wa2_descmissing_packet(priv_ep);
1282 
1283         return 0;
1284 }
1285 
1286 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1287 {
1288         if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
1289                 spin_unlock(&priv_dev->lock);
1290                 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1291                 spin_lock(&priv_dev->lock);
1292         }
1293 }
1294 
1295 /**
1296  * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1297  * @priv_dev: extended gadget object
1298  * @usb_ists: bitmap representation of device's reported interrupts
1299  * (usb_ists register value)
1300  */
1301 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1302                                               u32 usb_ists)
1303 {
1304         int speed = 0;
1305 
1306         trace_cdns3_usb_irq(priv_dev, usb_ists);
1307         if (usb_ists & USB_ISTS_L1ENTI) {
1308                 /*
1309                  * WORKAROUND: CDNS3 controller has issue with hardware resuming
1310                  * from L1. To fix it, if any DMA transfer is pending driver
1311                  * must starts driving resume signal immediately.
1312                  */
1313                 if (readl(&priv_dev->regs->drbl))
1314                         __cdns3_gadget_wakeup(priv_dev);
1315         }
1316 
1317         /* Connection detected */
1318         if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1319                 speed = cdns3_get_speed(priv_dev);
1320                 priv_dev->gadget.speed = speed;
1321                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1322                 cdns3_ep0_config(priv_dev);
1323         }
1324 
1325         /* Disconnection detected */
1326         if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1327                 cdns3_disconnect_gadget(priv_dev);
1328                 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1329                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1330                 cdns3_hw_reset_eps_config(priv_dev);
1331         }
1332 
1333         if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1334                 if (priv_dev->gadget_driver &&
1335                     priv_dev->gadget_driver->suspend) {
1336                         spin_unlock(&priv_dev->lock);
1337                         priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1338                         spin_lock(&priv_dev->lock);
1339                 }
1340         }
1341 
1342         if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1343                 if (priv_dev->gadget_driver &&
1344                     priv_dev->gadget_driver->resume) {
1345                         spin_unlock(&priv_dev->lock);
1346                         priv_dev->gadget_driver->resume(&priv_dev->gadget);
1347                         spin_lock(&priv_dev->lock);
1348                 }
1349         }
1350 
1351         /* reset*/
1352         if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1353                 if (priv_dev->gadget_driver) {
1354                         spin_unlock(&priv_dev->lock);
1355                         usb_gadget_udc_reset(&priv_dev->gadget,
1356                                              priv_dev->gadget_driver);
1357                         spin_lock(&priv_dev->lock);
1358 
1359                         /*read again to check the actual speed*/
1360                         speed = cdns3_get_speed(priv_dev);
1361                         priv_dev->gadget.speed = speed;
1362                         cdns3_hw_reset_eps_config(priv_dev);
1363                         cdns3_ep0_config(priv_dev);
1364                 }
1365         }
1366 }
1367 
1368 /**
1369  * cdns3_device_irq_handler- interrupt handler for device part of controller
1370  *
1371  * @irq: irq number for cdns3 core device
1372  * @data: structure of cdns3
1373  *
1374  * Returns IRQ_HANDLED or IRQ_NONE
1375  */
1376 static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1377 {
1378         struct cdns3_device *priv_dev = data;
1379         irqreturn_t ret = IRQ_NONE;
1380         u32 reg;
1381 
1382         /* check USB device interrupt */
1383         reg = readl(&priv_dev->regs->usb_ists);
1384         if (reg) {
1385                 /* After masking interrupts the new interrupts won't be
1386                  * reported in usb_ists/ep_ists. In order to not lose some
1387                  * of them driver disables only detected interrupts.
1388                  * They will be enabled ASAP after clearing source of
1389                  * interrupt. This an unusual behavior only applies to
1390                  * usb_ists register.
1391                  */
1392                 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1393                 /* mask deferred interrupt. */
1394                 writel(reg, &priv_dev->regs->usb_ien);
1395                 ret = IRQ_WAKE_THREAD;
1396         }
1397 
1398         /* check endpoint interrupt */
1399         reg = readl(&priv_dev->regs->ep_ists);
1400         if (reg) {
1401                 writel(0, &priv_dev->regs->ep_ien);
1402                 ret = IRQ_WAKE_THREAD;
1403         }
1404 
1405         return ret;
1406 }
1407 
1408 /**
1409  * cdns3_device_thread_irq_handler- interrupt handler for device part
1410  * of controller
1411  *
1412  * @irq: irq number for cdns3 core device
1413  * @data: structure of cdns3
1414  *
1415  * Returns IRQ_HANDLED or IRQ_NONE
1416  */
1417 static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1418 {
1419         struct cdns3_device *priv_dev = data;
1420         irqreturn_t ret = IRQ_NONE;
1421         unsigned long flags;
1422         int bit;
1423         u32 reg;
1424 
1425         spin_lock_irqsave(&priv_dev->lock, flags);
1426 
1427         reg = readl(&priv_dev->regs->usb_ists);
1428         if (reg) {
1429                 writel(reg, &priv_dev->regs->usb_ists);
1430                 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1431                 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1432                 ret = IRQ_HANDLED;
1433         }
1434 
1435         reg = readl(&priv_dev->regs->ep_ists);
1436 
1437         /* handle default endpoint OUT */
1438         if (reg & EP_ISTS_EP_OUT0) {
1439                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1440                 ret = IRQ_HANDLED;
1441         }
1442 
1443         /* handle default endpoint IN */
1444         if (reg & EP_ISTS_EP_IN0) {
1445                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1446                 ret = IRQ_HANDLED;
1447         }
1448 
1449         /* check if interrupt from non default endpoint, if no exit */
1450         reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1451         if (!reg)
1452                 goto irqend;
1453 
1454         for_each_set_bit(bit, (unsigned long *)&reg,
1455                          sizeof(u32) * BITS_PER_BYTE) {
1456                 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1457                 ret = IRQ_HANDLED;
1458         }
1459 
1460 irqend:
1461         writel(~0, &priv_dev->regs->ep_ien);
1462         spin_unlock_irqrestore(&priv_dev->lock, flags);
1463 
1464         return ret;
1465 }
1466 
1467 /**
1468  * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1469  *
1470  * The real reservation will occur during write to EP_CFG register,
1471  * this function is used to check if the 'size' reservation is allowed.
1472  *
1473  * @priv_dev: extended gadget object
1474  * @size: the size (KB) for EP would like to allocate
1475  * @is_in: endpoint direction
1476  *
1477  * Return 0 if the required size can met or negative value on failure
1478  */
1479 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1480                                           int size, int is_in)
1481 {
1482         int remained;
1483 
1484         /* 2KB are reserved for EP0*/
1485         remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1486 
1487         if (is_in) {
1488                 if (remained < size)
1489                         return -EPERM;
1490 
1491                 priv_dev->onchip_used_size += size;
1492         } else {
1493                 int required;
1494 
1495                 /**
1496                  *  ALL OUT EPs are shared the same chunk onchip memory, so
1497                  * driver checks if it already has assigned enough buffers
1498                  */
1499                 if (priv_dev->out_mem_is_allocated >= size)
1500                         return 0;
1501 
1502                 required = size - priv_dev->out_mem_is_allocated;
1503 
1504                 if (required > remained)
1505                         return -EPERM;
1506 
1507                 priv_dev->out_mem_is_allocated += required;
1508                 priv_dev->onchip_used_size += required;
1509         }
1510 
1511         return 0;
1512 }
1513 
1514 void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1515                            struct cdns3_endpoint *priv_ep)
1516 {
1517         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1518 
1519         /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1520         if (priv_dev->dev_ver <= DEV_VER_V2)
1521                 writel(USB_CONF_DMULT, &regs->usb_conf);
1522 
1523         if (priv_dev->dev_ver == DEV_VER_V2)
1524                 writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
1525 
1526         if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
1527                 u32 mask;
1528 
1529                 if (priv_ep->dir)
1530                         mask = BIT(priv_ep->num + 16);
1531                 else
1532                         mask = BIT(priv_ep->num);
1533 
1534                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1535                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1536                         cdns3_set_register_bit(&regs->tdl_beh, mask);
1537                         cdns3_set_register_bit(&regs->tdl_beh2, mask);
1538                         cdns3_set_register_bit(&regs->dma_adv_td, mask);
1539                 }
1540 
1541                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1542                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1543 
1544                 cdns3_set_register_bit(&regs->dtrans, mask);
1545         }
1546 }
1547 
1548 /**
1549  * cdns3_ep_config Configure hardware endpoint
1550  * @priv_ep: extended endpoint object
1551  */
1552 void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
1553 {
1554         bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
1555         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1556         u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
1557         u32 max_packet_size = 0;
1558         u8 maxburst = 0;
1559         u32 ep_cfg = 0;
1560         u8 buffering;
1561         u8 mult = 0;
1562         int ret;
1563 
1564         buffering = CDNS3_EP_BUF_SIZE - 1;
1565 
1566         cdns3_configure_dmult(priv_dev, priv_ep);
1567 
1568         switch (priv_ep->type) {
1569         case USB_ENDPOINT_XFER_INT:
1570                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
1571 
1572                 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1573                     priv_dev->dev_ver > DEV_VER_V2)
1574                         ep_cfg |= EP_CFG_TDL_CHK;
1575                 break;
1576         case USB_ENDPOINT_XFER_BULK:
1577                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
1578 
1579                 if ((priv_dev->dev_ver == DEV_VER_V2  && !priv_ep->dir) ||
1580                     priv_dev->dev_ver > DEV_VER_V2)
1581                         ep_cfg |= EP_CFG_TDL_CHK;
1582                 break;
1583         default:
1584                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
1585                 mult = CDNS3_EP_ISO_HS_MULT - 1;
1586                 buffering = mult + 1;
1587         }
1588 
1589         switch (priv_dev->gadget.speed) {
1590         case USB_SPEED_FULL:
1591                 max_packet_size = is_iso_ep ? 1023 : 64;
1592                 break;
1593         case USB_SPEED_HIGH:
1594                 max_packet_size = is_iso_ep ? 1024 : 512;
1595                 break;
1596         case USB_SPEED_SUPER:
1597                 /* It's limitation that driver assumes in driver. */
1598                 mult = 0;
1599                 max_packet_size = 1024;
1600                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1601                         maxburst = CDNS3_EP_ISO_SS_BURST - 1;
1602                         buffering = (mult + 1) *
1603                                     (maxburst + 1);
1604 
1605                         if (priv_ep->interval > 1)
1606                                 buffering++;
1607                 } else {
1608                         maxburst = CDNS3_EP_BUF_SIZE - 1;
1609                 }
1610                 break;
1611         default:
1612                 /* all other speed are not supported */
1613                 return;
1614         }
1615 
1616         if (max_packet_size == 1024)
1617                 priv_ep->trb_burst_size = 128;
1618         else if (max_packet_size >= 512)
1619                 priv_ep->trb_burst_size = 64;
1620         else
1621                 priv_ep->trb_burst_size = 16;
1622 
1623         ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
1624                                              !!priv_ep->dir);
1625         if (ret) {
1626                 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
1627                 return;
1628         }
1629 
1630         ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
1631                   EP_CFG_MULT(mult) |
1632                   EP_CFG_BUFFERING(buffering) |
1633                   EP_CFG_MAXBURST(maxburst);
1634 
1635         cdns3_select_ep(priv_dev, bEndpointAddress);
1636         writel(ep_cfg, &priv_dev->regs->ep_cfg);
1637 
1638         dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
1639                 priv_ep->name, ep_cfg);
1640 }
1641 
1642 /* Find correct direction for HW endpoint according to description */
1643 static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
1644                                    struct cdns3_endpoint *priv_ep)
1645 {
1646         return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
1647                (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
1648 }
1649 
1650 static struct
1651 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
1652                                         struct usb_endpoint_descriptor *desc)
1653 {
1654         struct usb_ep *ep;
1655         struct cdns3_endpoint *priv_ep;
1656 
1657         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1658                 unsigned long num;
1659                 int ret;
1660                 /* ep name pattern likes epXin or epXout */
1661                 char c[2] = {ep->name[2], '\0'};
1662 
1663                 ret = kstrtoul(c, 10, &num);
1664                 if (ret)
1665                         return ERR_PTR(ret);
1666 
1667                 priv_ep = ep_to_cdns3_ep(ep);
1668                 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
1669                         if (!(priv_ep->flags & EP_CLAIMED)) {
1670                                 priv_ep->num  = num;
1671                                 return priv_ep;
1672                         }
1673                 }
1674         }
1675 
1676         return ERR_PTR(-ENOENT);
1677 }
1678 
1679 /*
1680  *  Cadence IP has one limitation that all endpoints must be configured
1681  * (Type & MaxPacketSize) before setting configuration through hardware
1682  * register, it means we can't change endpoints configuration after
1683  * set_configuration.
1684  *
1685  * This function set EP_CLAIMED flag which is added when the gadget driver
1686  * uses usb_ep_autoconfig to configure specific endpoint;
1687  * When the udc driver receives set_configurion request,
1688  * it goes through all claimed endpoints, and configure all endpoints
1689  * accordingly.
1690  *
1691  * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
1692  * ep_cfg register which can be changed after set_configuration, and do
1693  * some software operation accordingly.
1694  */
1695 static struct
1696 usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
1697                               struct usb_endpoint_descriptor *desc,
1698                               struct usb_ss_ep_comp_descriptor *comp_desc)
1699 {
1700         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1701         struct cdns3_endpoint *priv_ep;
1702         unsigned long flags;
1703 
1704         priv_ep = cdns3_find_available_ep(priv_dev, desc);
1705         if (IS_ERR(priv_ep)) {
1706                 dev_err(priv_dev->dev, "no available ep\n");
1707                 return NULL;
1708         }
1709 
1710         dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
1711 
1712         spin_lock_irqsave(&priv_dev->lock, flags);
1713         priv_ep->endpoint.desc = desc;
1714         priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
1715         priv_ep->type = usb_endpoint_type(desc);
1716         priv_ep->flags |= EP_CLAIMED;
1717         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1718 
1719         spin_unlock_irqrestore(&priv_dev->lock, flags);
1720         return &priv_ep->endpoint;
1721 }
1722 
1723 /**
1724  * cdns3_gadget_ep_alloc_request Allocates request
1725  * @ep: endpoint object associated with request
1726  * @gfp_flags: gfp flags
1727  *
1728  * Returns allocated request address, NULL on allocation error
1729  */
1730 struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
1731                                                   gfp_t gfp_flags)
1732 {
1733         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1734         struct cdns3_request *priv_req;
1735 
1736         priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
1737         if (!priv_req)
1738                 return NULL;
1739 
1740         priv_req->priv_ep = priv_ep;
1741 
1742         trace_cdns3_alloc_request(priv_req);
1743         return &priv_req->request;
1744 }
1745 
1746 /**
1747  * cdns3_gadget_ep_free_request Free memory occupied by request
1748  * @ep: endpoint object associated with request
1749  * @request: request to free memory
1750  */
1751 void cdns3_gadget_ep_free_request(struct usb_ep *ep,
1752                                   struct usb_request *request)
1753 {
1754         struct cdns3_request *priv_req = to_cdns3_request(request);
1755 
1756         if (priv_req->aligned_buf)
1757                 priv_req->aligned_buf->in_use = 0;
1758 
1759         trace_cdns3_free_request(priv_req);
1760         kfree(priv_req);
1761 }
1762 
1763 /**
1764  * cdns3_gadget_ep_enable Enable endpoint
1765  * @ep: endpoint object
1766  * @desc: endpoint descriptor
1767  *
1768  * Returns 0 on success, error code elsewhere
1769  */
1770 static int cdns3_gadget_ep_enable(struct usb_ep *ep,
1771                                   const struct usb_endpoint_descriptor *desc)
1772 {
1773         struct cdns3_endpoint *priv_ep;
1774         struct cdns3_device *priv_dev;
1775         u32 reg = EP_STS_EN_TRBERREN;
1776         u32 bEndpointAddress;
1777         unsigned long flags;
1778         int enable = 1;
1779         int ret;
1780         int val;
1781 
1782         priv_ep = ep_to_cdns3_ep(ep);
1783         priv_dev = priv_ep->cdns3_dev;
1784 
1785         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1786                 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
1787                 return -EINVAL;
1788         }
1789 
1790         if (!desc->wMaxPacketSize) {
1791                 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
1792                 return -EINVAL;
1793         }
1794 
1795         if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
1796                           "%s is already enabled\n", priv_ep->name))
1797                 return 0;
1798 
1799         spin_lock_irqsave(&priv_dev->lock, flags);
1800 
1801         priv_ep->endpoint.desc = desc;
1802         priv_ep->type = usb_endpoint_type(desc);
1803         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1804 
1805         if (priv_ep->interval > ISO_MAX_INTERVAL &&
1806             priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1807                 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
1808                         ISO_MAX_INTERVAL);
1809 
1810                 ret =  -EINVAL;
1811                 goto exit;
1812         }
1813 
1814         ret = cdns3_allocate_trb_pool(priv_ep);
1815 
1816         if (ret)
1817                 goto exit;
1818 
1819         bEndpointAddress = priv_ep->num | priv_ep->dir;
1820         cdns3_select_ep(priv_dev, bEndpointAddress);
1821 
1822         trace_cdns3_gadget_ep_enable(priv_ep);
1823 
1824         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1825 
1826         ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1827                                         !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1828                                         1, 1000);
1829 
1830         if (unlikely(ret)) {
1831                 cdns3_free_trb_pool(priv_ep);
1832                 ret =  -EINVAL;
1833                 goto exit;
1834         }
1835 
1836         /* enable interrupt for selected endpoint */
1837         cdns3_set_register_bit(&priv_dev->regs->ep_ien,
1838                                BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
1839 
1840         if (priv_dev->dev_ver < DEV_VER_V2)
1841                 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
1842 
1843         writel(reg, &priv_dev->regs->ep_sts_en);
1844 
1845         /*
1846          * For some versions of controller at some point during ISO OUT traffic
1847          * DMA reads Transfer Ring for the EP which has never got doorbell.
1848          * This issue was detected only on simulation, but to avoid this issue
1849          * driver add protection against it. To fix it driver enable ISO OUT
1850          * endpoint before setting DRBL. This special treatment of ISO OUT
1851          * endpoints are recommended by controller specification.
1852          */
1853         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
1854                 enable = 0;
1855 
1856         if (enable)
1857                 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
1858 
1859         ep->desc = desc;
1860         priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
1861                             EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
1862         priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
1863         priv_ep->wa1_set = 0;
1864         priv_ep->enqueue = 0;
1865         priv_ep->dequeue = 0;
1866         reg = readl(&priv_dev->regs->ep_sts);
1867         priv_ep->pcs = !!EP_STS_CCS(reg);
1868         priv_ep->ccs = !!EP_STS_CCS(reg);
1869         /* one TRB is reserved for link TRB used in DMULT mode*/
1870         priv_ep->free_trbs = priv_ep->num_trbs - 1;
1871 exit:
1872         spin_unlock_irqrestore(&priv_dev->lock, flags);
1873 
1874         return ret;
1875 }
1876 
1877 /**
1878  * cdns3_gadget_ep_disable Disable endpoint
1879  * @ep: endpoint object
1880  *
1881  * Returns 0 on success, error code elsewhere
1882  */
1883 static int cdns3_gadget_ep_disable(struct usb_ep *ep)
1884 {
1885         struct cdns3_endpoint *priv_ep;
1886         struct cdns3_request *priv_req;
1887         struct cdns3_device *priv_dev;
1888         struct usb_request *request;
1889         unsigned long flags;
1890         int ret = 0;
1891         u32 ep_cfg;
1892         int val;
1893 
1894         if (!ep) {
1895                 pr_err("usbss: invalid parameters\n");
1896                 return -EINVAL;
1897         }
1898 
1899         priv_ep = ep_to_cdns3_ep(ep);
1900         priv_dev = priv_ep->cdns3_dev;
1901 
1902         if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
1903                           "%s is already disabled\n", priv_ep->name))
1904                 return 0;
1905 
1906         spin_lock_irqsave(&priv_dev->lock, flags);
1907 
1908         trace_cdns3_gadget_ep_disable(priv_ep);
1909 
1910         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
1911 
1912         ep_cfg = readl(&priv_dev->regs->ep_cfg);
1913         ep_cfg &= ~EP_CFG_ENABLE;
1914         writel(ep_cfg, &priv_dev->regs->ep_cfg);
1915 
1916         /**
1917          * Driver needs some time before resetting endpoint.
1918          * It need waits for clearing DBUSY bit or for timeout expired.
1919          * 10us is enough time for controller to stop transfer.
1920          */
1921         readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
1922                                   !(val & EP_STS_DBUSY), 1, 10);
1923         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1924 
1925         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1926                                   !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1927                                   1, 1000);
1928         if (unlikely(ret))
1929                 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
1930                         priv_ep->name);
1931 
1932         while (!list_empty(&priv_ep->pending_req_list)) {
1933                 request = cdns3_next_request(&priv_ep->pending_req_list);
1934 
1935                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1936                                       -ESHUTDOWN);
1937         }
1938 
1939         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
1940                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
1941 
1942                 kfree(priv_req->request.buf);
1943                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
1944                                              &priv_req->request);
1945                 list_del_init(&priv_req->list);
1946                 --priv_ep->wa2_counter;
1947         }
1948 
1949         while (!list_empty(&priv_ep->deferred_req_list)) {
1950                 request = cdns3_next_request(&priv_ep->deferred_req_list);
1951 
1952                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1953                                       -ESHUTDOWN);
1954         }
1955 
1956         priv_ep->descmis_req = NULL;
1957 
1958         ep->desc = NULL;
1959         priv_ep->flags &= ~EP_ENABLED;
1960 
1961         spin_unlock_irqrestore(&priv_dev->lock, flags);
1962 
1963         return ret;
1964 }
1965 
1966 /**
1967  * cdns3_gadget_ep_queue Transfer data on endpoint
1968  * @ep: endpoint object
1969  * @request: request object
1970  * @gfp_flags: gfp flags
1971  *
1972  * Returns 0 on success, error code elsewhere
1973  */
1974 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
1975                                    struct usb_request *request,
1976                                    gfp_t gfp_flags)
1977 {
1978         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1979         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1980         struct cdns3_request *priv_req;
1981         int ret = 0;
1982 
1983         request->actual = 0;
1984         request->status = -EINPROGRESS;
1985         priv_req = to_cdns3_request(request);
1986         trace_cdns3_ep_queue(priv_req);
1987 
1988         if (priv_dev->dev_ver < DEV_VER_V2) {
1989                 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
1990                                                 priv_req);
1991 
1992                 if (ret == EINPROGRESS)
1993                         return 0;
1994         }
1995 
1996         ret = cdns3_prepare_aligned_request_buf(priv_req);
1997         if (ret < 0)
1998                 return ret;
1999 
2000         ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2001                                             usb_endpoint_dir_in(ep->desc));
2002         if (ret)
2003                 return ret;
2004 
2005         list_add_tail(&request->list, &priv_ep->deferred_req_list);
2006 
2007         /*
2008          * If hardware endpoint configuration has not been set yet then
2009          * just queue request in deferred list. Transfer will be started in
2010          * cdns3_set_hw_configuration.
2011          */
2012         if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) &&
2013             !(priv_ep->flags & EP_STALL_PENDING))
2014                 cdns3_start_all_request(priv_dev, priv_ep);
2015 
2016         return 0;
2017 }
2018 
2019 static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2020                                  gfp_t gfp_flags)
2021 {
2022         struct usb_request *zlp_request;
2023         struct cdns3_endpoint *priv_ep;
2024         struct cdns3_device *priv_dev;
2025         unsigned long flags;
2026         int ret;
2027 
2028         if (!request || !ep)
2029                 return -EINVAL;
2030 
2031         priv_ep = ep_to_cdns3_ep(ep);
2032         priv_dev = priv_ep->cdns3_dev;
2033 
2034         spin_lock_irqsave(&priv_dev->lock, flags);
2035 
2036         ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2037 
2038         if (ret == 0 && request->zero && request->length &&
2039             (request->length % ep->maxpacket == 0)) {
2040                 struct cdns3_request *priv_req;
2041 
2042                 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2043                 zlp_request->buf = priv_dev->zlp_buf;
2044                 zlp_request->length = 0;
2045 
2046                 priv_req = to_cdns3_request(zlp_request);
2047                 priv_req->flags |= REQUEST_ZLP;
2048 
2049                 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2050                         priv_ep->name);
2051                 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2052         }
2053 
2054         spin_unlock_irqrestore(&priv_dev->lock, flags);
2055         return ret;
2056 }
2057 
2058 /**
2059  * cdns3_gadget_ep_dequeue Remove request from transfer queue
2060  * @ep: endpoint object associated with request
2061  * @request: request object
2062  *
2063  * Returns 0 on success, error code elsewhere
2064  */
2065 int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2066                             struct usb_request *request)
2067 {
2068         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2069         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2070         struct usb_request *req, *req_temp;
2071         struct cdns3_request *priv_req;
2072         struct cdns3_trb *link_trb;
2073         u8 req_on_hw_ring = 0;
2074         unsigned long flags;
2075         int ret = 0;
2076 
2077         if (!ep || !request || !ep->desc)
2078                 return -EINVAL;
2079 
2080         spin_lock_irqsave(&priv_dev->lock, flags);
2081 
2082         priv_req = to_cdns3_request(request);
2083 
2084         trace_cdns3_ep_dequeue(priv_req);
2085 
2086         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2087 
2088         list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2089                                  list) {
2090                 if (request == req) {
2091                         req_on_hw_ring = 1;
2092                         goto found;
2093                 }
2094         }
2095 
2096         list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2097                                  list) {
2098                 if (request == req)
2099                         goto found;
2100         }
2101 
2102         goto not_found;
2103 
2104 found:
2105         link_trb = priv_req->trb;
2106 
2107         /* Update ring only if removed request is on pending_req_list list */
2108         if (req_on_hw_ring && link_trb) {
2109                 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
2110                         ((priv_req->end_trb + 1) * TRB_SIZE));
2111                 link_trb->control = (link_trb->control & TRB_CYCLE) |
2112                                     TRB_TYPE(TRB_LINK) | TRB_CHAIN;
2113 
2114                 if (priv_ep->wa1_trb == priv_req->trb)
2115                         cdns3_wa1_restore_cycle_bit(priv_ep);
2116         }
2117 
2118         cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2119 
2120 not_found:
2121         spin_unlock_irqrestore(&priv_dev->lock, flags);
2122         return ret;
2123 }
2124 
2125 /**
2126  * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
2127  * Should be called after acquiring spin_lock and selecting ep
2128  * @ep: endpoint object to set stall on.
2129  */
2130 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2131 {
2132         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2133 
2134         trace_cdns3_halt(priv_ep, 1, 0);
2135 
2136         if (!(priv_ep->flags & EP_STALLED)) {
2137                 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2138 
2139                 if (!(ep_sts_reg & EP_STS_DBUSY))
2140                         cdns3_ep_stall_flush(priv_ep);
2141                 else
2142                         priv_ep->flags |= EP_STALL_PENDING;
2143         }
2144 }
2145 
2146 /**
2147  * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
2148  * Should be called after acquiring spin_lock and selecting ep
2149  * @ep: endpoint object to clear stall on
2150  */
2151 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2152 {
2153         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2154         struct usb_request *request;
2155         struct cdns3_request *priv_req;
2156         struct cdns3_trb *trb = NULL;
2157         int ret;
2158         int val;
2159 
2160         trace_cdns3_halt(priv_ep, 0, 0);
2161 
2162         request = cdns3_next_request(&priv_ep->pending_req_list);
2163         if (request) {
2164                 priv_req = to_cdns3_request(request);
2165                 trb = priv_req->trb;
2166                 if (trb)
2167                         trb->control = trb->control ^ TRB_CYCLE;
2168         }
2169 
2170         writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2171 
2172         /* wait for EPRST cleared */
2173         ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2174                                         !(val & EP_CMD_EPRST), 1, 100);
2175         if (ret)
2176                 return -EINVAL;
2177 
2178         priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2179 
2180         if (request) {
2181                 if (trb)
2182                         trb->control = trb->control ^ TRB_CYCLE;
2183                 cdns3_rearm_transfer(priv_ep, 1);
2184         }
2185 
2186         cdns3_start_all_request(priv_dev, priv_ep);
2187         return ret;
2188 }
2189 
2190 /**
2191  * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
2192  * @ep: endpoint object to set/clear stall on
2193  * @value: 1 for set stall, 0 for clear stall
2194  *
2195  * Returns 0 on success, error code elsewhere
2196  */
2197 int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2198 {
2199         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2200         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2201         unsigned long flags;
2202         int ret = 0;
2203 
2204         if (!(priv_ep->flags & EP_ENABLED))
2205                 return -EPERM;
2206 
2207         spin_lock_irqsave(&priv_dev->lock, flags);
2208 
2209         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2210 
2211         if (!value) {
2212                 priv_ep->flags &= ~EP_WEDGE;
2213                 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2214         } else {
2215                 __cdns3_gadget_ep_set_halt(priv_ep);
2216         }
2217 
2218         spin_unlock_irqrestore(&priv_dev->lock, flags);
2219 
2220         return ret;
2221 }
2222 
2223 extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2224 
2225 static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2226         .enable = cdns3_gadget_ep_enable,
2227         .disable = cdns3_gadget_ep_disable,
2228         .alloc_request = cdns3_gadget_ep_alloc_request,
2229         .free_request = cdns3_gadget_ep_free_request,
2230         .queue = cdns3_gadget_ep_queue,
2231         .dequeue = cdns3_gadget_ep_dequeue,
2232         .set_halt = cdns3_gadget_ep_set_halt,
2233         .set_wedge = cdns3_gadget_ep_set_wedge,
2234 };
2235 
2236 /**
2237  * cdns3_gadget_get_frame Returns number of actual ITP frame
2238  * @gadget: gadget object
2239  *
2240  * Returns number of actual ITP frame
2241  */
2242 static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2243 {
2244         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2245 
2246         return readl(&priv_dev->regs->usb_itpn);
2247 }
2248 
2249 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2250 {
2251         enum usb_device_speed speed;
2252 
2253         speed = cdns3_get_speed(priv_dev);
2254 
2255         if (speed >= USB_SPEED_SUPER)
2256                 return 0;
2257 
2258         /* Start driving resume signaling to indicate remote wakeup. */
2259         writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2260 
2261         return 0;
2262 }
2263 
2264 static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2265 {
2266         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2267         unsigned long flags;
2268         int ret = 0;
2269 
2270         spin_lock_irqsave(&priv_dev->lock, flags);
2271         ret = __cdns3_gadget_wakeup(priv_dev);
2272         spin_unlock_irqrestore(&priv_dev->lock, flags);
2273         return ret;
2274 }
2275 
2276 static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2277                                         int is_selfpowered)
2278 {
2279         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2280         unsigned long flags;
2281 
2282         spin_lock_irqsave(&priv_dev->lock, flags);
2283         priv_dev->is_selfpowered = !!is_selfpowered;
2284         spin_unlock_irqrestore(&priv_dev->lock, flags);
2285         return 0;
2286 }
2287 
2288 static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2289 {
2290         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2291 
2292         if (is_on)
2293                 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2294         else
2295                 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2296 
2297         return 0;
2298 }
2299 
2300 static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2301 {
2302         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2303         u32 reg;
2304 
2305         cdns3_ep0_config(priv_dev);
2306 
2307         /* enable interrupts for endpoint 0 (in and out) */
2308         writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
2309 
2310         /*
2311          * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2312          * revision of controller.
2313          */
2314         if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2315                 reg = readl(&regs->dbg_link1);
2316 
2317                 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2318                 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2319                        DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2320                 writel(reg, &regs->dbg_link1);
2321         }
2322 
2323         /*
2324          * By default some platforms has set protected access to memory.
2325          * This cause problem with cache, so driver restore non-secure
2326          * access to memory.
2327          */
2328         reg = readl(&regs->dma_axi_ctrl);
2329         reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2330                DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2331         writel(reg, &regs->dma_axi_ctrl);
2332 
2333         /* enable generic interrupt*/
2334         writel(USB_IEN_INIT, &regs->usb_ien);
2335         writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
2336 
2337         cdns3_configure_dmult(priv_dev, NULL);
2338 }
2339 
2340 /**
2341  * cdns3_gadget_udc_start Gadget start
2342  * @gadget: gadget object
2343  * @driver: driver which operates on this gadget
2344  *
2345  * Returns 0 on success, error code elsewhere
2346  */
2347 static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2348                                   struct usb_gadget_driver *driver)
2349 {
2350         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2351         unsigned long flags;
2352         enum usb_device_speed max_speed = driver->max_speed;
2353 
2354         spin_lock_irqsave(&priv_dev->lock, flags);
2355         priv_dev->gadget_driver = driver;
2356 
2357         /* limit speed if necessary */
2358         max_speed = min(driver->max_speed, gadget->max_speed);
2359 
2360         switch (max_speed) {
2361         case USB_SPEED_FULL:
2362                 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2363                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2364                 break;
2365         case USB_SPEED_HIGH:
2366                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2367                 break;
2368         case USB_SPEED_SUPER:
2369                 break;
2370         default:
2371                 dev_err(priv_dev->dev,
2372                         "invalid maximum_speed parameter %d\n",
2373                         max_speed);
2374                 /* fall through */
2375         case USB_SPEED_UNKNOWN:
2376                 /* default to superspeed */
2377                 max_speed = USB_SPEED_SUPER;
2378                 break;
2379         }
2380 
2381         cdns3_gadget_config(priv_dev);
2382         spin_unlock_irqrestore(&priv_dev->lock, flags);
2383         return 0;
2384 }
2385 
2386 /**
2387  * cdns3_gadget_udc_stop Stops gadget
2388  * @gadget: gadget object
2389  *
2390  * Returns 0
2391  */
2392 static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2393 {
2394         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2395         struct cdns3_endpoint *priv_ep;
2396         u32 bEndpointAddress;
2397         struct usb_ep *ep;
2398         int ret = 0;
2399         int val;
2400 
2401         priv_dev->gadget_driver = NULL;
2402 
2403         priv_dev->onchip_used_size = 0;
2404         priv_dev->out_mem_is_allocated = 0;
2405         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2406 
2407         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2408                 priv_ep = ep_to_cdns3_ep(ep);
2409                 bEndpointAddress = priv_ep->num | priv_ep->dir;
2410                 cdns3_select_ep(priv_dev, bEndpointAddress);
2411                 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2412                 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2413                                           !(val & EP_CMD_EPRST), 1, 100);
2414 
2415                 priv_ep->flags &= ~EP_CLAIMED;
2416         }
2417 
2418         /* disable interrupt for device */
2419         writel(0, &priv_dev->regs->usb_ien);
2420         writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2421 
2422         return ret;
2423 }
2424 
2425 static const struct usb_gadget_ops cdns3_gadget_ops = {
2426         .get_frame = cdns3_gadget_get_frame,
2427         .wakeup = cdns3_gadget_wakeup,
2428         .set_selfpowered = cdns3_gadget_set_selfpowered,
2429         .pullup = cdns3_gadget_pullup,
2430         .udc_start = cdns3_gadget_udc_start,
2431         .udc_stop = cdns3_gadget_udc_stop,
2432         .match_ep = cdns3_gadget_match_ep,
2433 };
2434 
2435 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2436 {
2437         int i;
2438 
2439         /* ep0 OUT point to ep0 IN. */
2440         priv_dev->eps[16] = NULL;
2441 
2442         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2443                 if (priv_dev->eps[i]) {
2444                         cdns3_free_trb_pool(priv_dev->eps[i]);
2445                         devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2446                 }
2447 }
2448 
2449 /**
2450  * cdns3_init_eps Initializes software endpoints of gadget
2451  * @cdns3: extended gadget object
2452  *
2453  * Returns 0 on success, error code elsewhere
2454  */
2455 static int cdns3_init_eps(struct cdns3_device *priv_dev)
2456 {
2457         u32 ep_enabled_reg, iso_ep_reg;
2458         struct cdns3_endpoint *priv_ep;
2459         int ep_dir, ep_number;
2460         u32 ep_mask;
2461         int ret = 0;
2462         int i;
2463 
2464         /* Read it from USB_CAP3 to USB_CAP5 */
2465         ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
2466         iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
2467 
2468         dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
2469 
2470         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
2471                 ep_dir = i >> 4;        /* i div 16 */
2472                 ep_number = i & 0xF;    /* i % 16 */
2473                 ep_mask = BIT(i);
2474 
2475                 if (!(ep_enabled_reg & ep_mask))
2476                         continue;
2477 
2478                 if (ep_dir && !ep_number) {
2479                         priv_dev->eps[i] = priv_dev->eps[0];
2480                         continue;
2481                 }
2482 
2483                 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
2484                                        GFP_KERNEL);
2485                 if (!priv_ep)
2486                         goto err;
2487 
2488                 /* set parent of endpoint object */
2489                 priv_ep->cdns3_dev = priv_dev;
2490                 priv_dev->eps[i] = priv_ep;
2491                 priv_ep->num = ep_number;
2492                 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
2493 
2494                 if (!ep_number) {
2495                         ret = cdns3_init_ep0(priv_dev, priv_ep);
2496                         if (ret) {
2497                                 dev_err(priv_dev->dev, "Failed to init ep0\n");
2498                                 goto err;
2499                         }
2500                 } else {
2501                         snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
2502                                  ep_number, !!ep_dir ? "in" : "out");
2503                         priv_ep->endpoint.name = priv_ep->name;
2504 
2505                         usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
2506                                                    CDNS3_EP_MAX_PACKET_LIMIT);
2507                         priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
2508                         priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
2509                         if (ep_dir)
2510                                 priv_ep->endpoint.caps.dir_in = 1;
2511                         else
2512                                 priv_ep->endpoint.caps.dir_out = 1;
2513 
2514                         if (iso_ep_reg & ep_mask)
2515                                 priv_ep->endpoint.caps.type_iso = 1;
2516 
2517                         priv_ep->endpoint.caps.type_bulk = 1;
2518                         priv_ep->endpoint.caps.type_int = 1;
2519 
2520                         list_add_tail(&priv_ep->endpoint.ep_list,
2521                                       &priv_dev->gadget.ep_list);
2522                 }
2523 
2524                 priv_ep->flags = 0;
2525 
2526                 dev_info(priv_dev->dev, "Initialized  %s support: %s %s\n",
2527                          priv_ep->name,
2528                          priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
2529                          priv_ep->endpoint.caps.type_iso ? "ISO" : "");
2530 
2531                 INIT_LIST_HEAD(&priv_ep->pending_req_list);
2532                 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
2533                 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
2534         }
2535 
2536         return 0;
2537 err:
2538         cdns3_free_all_eps(priv_dev);
2539         return -ENOMEM;
2540 }
2541 
2542 void cdns3_gadget_exit(struct cdns3 *cdns)
2543 {
2544         struct cdns3_device *priv_dev;
2545 
2546         priv_dev = cdns->gadget_dev;
2547 
2548         devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
2549 
2550         pm_runtime_mark_last_busy(cdns->dev);
2551         pm_runtime_put_autosuspend(cdns->dev);
2552 
2553         usb_del_gadget_udc(&priv_dev->gadget);
2554 
2555         cdns3_free_all_eps(priv_dev);
2556 
2557         while (!list_empty(&priv_dev->aligned_buf_list)) {
2558                 struct cdns3_aligned_buf *buf;
2559 
2560                 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
2561                 dma_free_coherent(priv_dev->sysdev, buf->size,
2562                                   buf->buf,
2563                                   buf->dma);
2564 
2565                 list_del(&buf->list);
2566                 kfree(buf);
2567         }
2568 
2569         dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
2570                           priv_dev->setup_dma);
2571 
2572         kfree(priv_dev->zlp_buf);
2573         kfree(priv_dev);
2574         cdns->gadget_dev = NULL;
2575         cdns3_drd_switch_gadget(cdns, 0);
2576 }
2577 
2578 static int cdns3_gadget_start(struct cdns3 *cdns)
2579 {
2580         struct cdns3_device *priv_dev;
2581         u32 max_speed;
2582         int ret;
2583 
2584         priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
2585         if (!priv_dev)
2586                 return -ENOMEM;
2587 
2588         cdns->gadget_dev = priv_dev;
2589         priv_dev->sysdev = cdns->dev;
2590         priv_dev->dev = cdns->dev;
2591         priv_dev->regs = cdns->dev_regs;
2592 
2593         device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
2594                                  &priv_dev->onchip_buffers);
2595 
2596         if (priv_dev->onchip_buffers <=  0) {
2597                 u32 reg = readl(&priv_dev->regs->usb_cap2);
2598 
2599                 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
2600         }
2601 
2602         if (!priv_dev->onchip_buffers)
2603                 priv_dev->onchip_buffers = 256;
2604 
2605         max_speed = usb_get_maximum_speed(cdns->dev);
2606 
2607         /* Check the maximum_speed parameter */
2608         switch (max_speed) {
2609         case USB_SPEED_FULL:
2610         case USB_SPEED_HIGH:
2611         case USB_SPEED_SUPER:
2612                 break;
2613         default:
2614                 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
2615                         max_speed);
2616                 /* fall through */
2617         case USB_SPEED_UNKNOWN:
2618                 /* default to superspeed */
2619                 max_speed = USB_SPEED_SUPER;
2620                 break;
2621         }
2622 
2623         /* fill gadget fields */
2624         priv_dev->gadget.max_speed = max_speed;
2625         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2626         priv_dev->gadget.ops = &cdns3_gadget_ops;
2627         priv_dev->gadget.name = "usb-ss-gadget";
2628         priv_dev->gadget.sg_supported = 1;
2629         priv_dev->gadget.quirk_avoids_skb_reserve = 1;
2630 
2631         spin_lock_init(&priv_dev->lock);
2632         INIT_WORK(&priv_dev->pending_status_wq,
2633                   cdns3_pending_setup_status_handler);
2634 
2635         INIT_WORK(&priv_dev->aligned_buf_wq,
2636                   cdns3_free_aligned_request_buf);
2637 
2638         /* initialize endpoint container */
2639         INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
2640         INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
2641 
2642         ret = cdns3_init_eps(priv_dev);
2643         if (ret) {
2644                 dev_err(priv_dev->dev, "Failed to create endpoints\n");
2645                 goto err1;
2646         }
2647 
2648         /* allocate memory for setup packet buffer */
2649         priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
2650                                                  &priv_dev->setup_dma, GFP_DMA);
2651         if (!priv_dev->setup_buf) {
2652                 ret = -ENOMEM;
2653                 goto err2;
2654         }
2655 
2656         priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
2657 
2658         dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
2659                 readl(&priv_dev->regs->usb_cap6));
2660         dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
2661                 readl(&priv_dev->regs->usb_cap1));
2662         dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
2663                 readl(&priv_dev->regs->usb_cap2));
2664 
2665         priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
2666 
2667         priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2668         if (!priv_dev->zlp_buf) {
2669                 ret = -ENOMEM;
2670                 goto err3;
2671         }
2672 
2673         /* add USB gadget device */
2674         ret = usb_add_gadget_udc(priv_dev->dev, &priv_dev->gadget);
2675         if (ret < 0) {
2676                 dev_err(priv_dev->dev,
2677                         "Failed to register USB device controller\n");
2678                 goto err4;
2679         }
2680 
2681         return 0;
2682 err4:
2683         kfree(priv_dev->zlp_buf);
2684 err3:
2685         dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
2686                           priv_dev->setup_dma);
2687 err2:
2688         cdns3_free_all_eps(priv_dev);
2689 err1:
2690         cdns->gadget_dev = NULL;
2691         return ret;
2692 }
2693 
2694 static int __cdns3_gadget_init(struct cdns3 *cdns)
2695 {
2696         int ret = 0;
2697 
2698         /* Ensure 32-bit DMA Mask in case we switched back from Host mode */
2699         ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
2700         if (ret) {
2701                 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
2702                 return ret;
2703         }
2704 
2705         cdns3_drd_switch_gadget(cdns, 1);
2706         pm_runtime_get_sync(cdns->dev);
2707 
2708         ret = cdns3_gadget_start(cdns);
2709         if (ret)
2710                 return ret;
2711 
2712         /*
2713          * Because interrupt line can be shared with other components in
2714          * driver it can't use IRQF_ONESHOT flag here.
2715          */
2716         ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
2717                                         cdns3_device_irq_handler,
2718                                         cdns3_device_thread_irq_handler,
2719                                         IRQF_SHARED, dev_name(cdns->dev),
2720                                         cdns->gadget_dev);
2721 
2722         if (ret)
2723                 goto err0;
2724 
2725         return 0;
2726 err0:
2727         cdns3_gadget_exit(cdns);
2728         return ret;
2729 }
2730 
2731 static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
2732 {
2733         struct cdns3_device *priv_dev = cdns->gadget_dev;
2734 
2735         cdns3_disconnect_gadget(priv_dev);
2736 
2737         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2738         usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
2739         cdns3_hw_reset_eps_config(priv_dev);
2740 
2741         /* disable interrupt for device */
2742         writel(0, &priv_dev->regs->usb_ien);
2743 
2744         return 0;
2745 }
2746 
2747 static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
2748 {
2749         struct cdns3_device *priv_dev = cdns->gadget_dev;
2750 
2751         if (!priv_dev->gadget_driver)
2752                 return 0;
2753 
2754         cdns3_gadget_config(priv_dev);
2755 
2756         return 0;
2757 }
2758 
2759 /**
2760  * cdns3_gadget_init - initialize device structure
2761  *
2762  * cdns: cdns3 instance
2763  *
2764  * This function initializes the gadget.
2765  */
2766 int cdns3_gadget_init(struct cdns3 *cdns)
2767 {
2768         struct cdns3_role_driver *rdrv;
2769 
2770         rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2771         if (!rdrv)
2772                 return -ENOMEM;
2773 
2774         rdrv->start     = __cdns3_gadget_init;
2775         rdrv->stop      = cdns3_gadget_exit;
2776         rdrv->suspend   = cdns3_gadget_suspend;
2777         rdrv->resume    = cdns3_gadget_resume;
2778         rdrv->state     = CDNS3_ROLE_STATE_INACTIVE;
2779         rdrv->name      = "gadget";
2780         cdns->roles[USB_ROLE_DEVICE] = rdrv;
2781 
2782         return 0;
2783 }
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/usb/cdns3/gadget.c

DEFINITIONS
