root/drivers/net/ethernet/chelsio/cxgb3/sge.c

DEFINITIONS

1. fl_to_qset
2. rspq_to_qset
3. txq_to_qset
4. refill_rspq
5. need_skb_unmap
6. unmap_skb
7. free_tx_desc
8. reclaim_completed_tx
9. should_restart_tx
10. clear_rx_desc
11. free_rx_bufs
12. add_one_rx_buf
13. add_one_rx_chunk
14. alloc_pg_chunk
15. ring_fl_db
16. refill_fl
17. __refill_fl
18. recycle_rx_buf
19. alloc_ring
20. t3_reset_qset
21. t3_free_qset
22. init_qset_cntxt
23. sgl_len
24. flits_to_desc
25. get_packet
26. get_packet_pg
27. get_imm_packet
28. calc_tx_descs
29. map_skb
30. write_sgl
31. check_ring_tx_db
32. wr_gen2
33. write_wr_hdr_sgl
34. write_tx_pkt_wr
35. t3_stop_tx_queue
36. t3_eth_xmit
37. write_imm
38. check_desc_avail
39. reclaim_completed_tx_imm
40. immediate
41. ctrl_xmit
42. restart_ctrlq
43. t3_mgmt_tx
44. deferred_unmap_destructor
45. setup_deferred_unmapping
46. write_ofld_wr
47. calc_tx_descs_ofld
48. ofld_xmit
49. restart_offloadq
50. queue_set
51. is_ctrl_pkt
52. t3_offload_tx
53. offload_enqueue
54. deliver_partial_bundle
55. ofld_poll
56. rx_offload
57. restart_tx
58. cxgb3_arp_process
59. is_arp
60. cxgb3_process_iscsi_prov_pack
61. rx_eth
62. is_eth_tcp
63. lro_add_page
64. handle_rsp_cntrl_info
65. check_ring_db
66. is_new_response
67. clear_rspq_bufstate
68. process_responses
69. is_pure_response
70. napi_rx_handler
71. napi_is_scheduled
72. process_pure_responses
73. handle_responses
74. t3_sge_intr_msix
75. t3_sge_intr_msix_napi
76. t3_intr_msi
77. rspq_check_napi
78. t3_intr_msi_napi
79. process_responses_gts
80. t3_intr
81. t3b_intr
82. t3b_intr_napi
83. t3_intr_handler
84. t3_sge_err_intr_handler
85. sge_timer_tx
86. sge_timer_rx
87. t3_update_qset_coalesce
88. t3_sge_alloc_qset
89. t3_start_sge_timers
90. t3_stop_sge_timers
91. t3_free_sge_resources
92. t3_sge_start
93. t3_sge_stop
94. t3_sge_init
95. t3_sge_prep

   1 /*
   2  * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/skbuff.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/etherdevice.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/ip.h>
  37 #include <linux/tcp.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/slab.h>
  40 #include <linux/prefetch.h>
  41 #include <net/arp.h>
  42 #include "common.h"
  43 #include "regs.h"
  44 #include "sge_defs.h"
  45 #include "t3_cpl.h"
  46 #include "firmware_exports.h"
  47 #include "cxgb3_offload.h"
  48 
  49 #define USE_GTS 0
  50 
  51 #define SGE_RX_SM_BUF_SIZE 1536
  52 
  53 #define SGE_RX_COPY_THRES  256
  54 #define SGE_RX_PULL_LEN    128
  55 
  56 #define SGE_PG_RSVD SMP_CACHE_BYTES
  57 /*
  58  * Page chunk size for FL0 buffers if FL0 is to be populated with page chunks.
  59  * It must be a divisor of PAGE_SIZE.  If set to 0 FL0 will use sk_buffs
  60  * directly.
  61  */
  62 #define FL0_PG_CHUNK_SIZE  2048
  63 #define FL0_PG_ORDER 0
  64 #define FL0_PG_ALLOC_SIZE (PAGE_SIZE << FL0_PG_ORDER)
  65 #define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192)
  66 #define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1)
  67 #define FL1_PG_ALLOC_SIZE (PAGE_SIZE << FL1_PG_ORDER)
  68 
  69 #define SGE_RX_DROP_THRES 16
  70 #define RX_RECLAIM_PERIOD (HZ/4)
  71 
  72 /*
  73  * Max number of Rx buffers we replenish at a time.
  74  */
  75 #define MAX_RX_REFILL 16U
  76 /*
  77  * Period of the Tx buffer reclaim timer.  This timer does not need to run
  78  * frequently as Tx buffers are usually reclaimed by new Tx packets.
  79  */
  80 #define TX_RECLAIM_PERIOD (HZ / 4)
  81 #define TX_RECLAIM_TIMER_CHUNK 64U
  82 #define TX_RECLAIM_CHUNK 16U
  83 
  84 /* WR size in bytes */
  85 #define WR_LEN (WR_FLITS * 8)
  86 
  87 /*
  88  * Types of Tx queues in each queue set.  Order here matters, do not change.
  89  */
  90 enum { TXQ_ETH, TXQ_OFLD, TXQ_CTRL };
  91 
  92 /* Values for sge_txq.flags */
  93 enum {
  94         TXQ_RUNNING = 1 << 0,   /* fetch engine is running */
  95         TXQ_LAST_PKT_DB = 1 << 1,       /* last packet rang the doorbell */
  96 };
  97 
  98 struct tx_desc {
  99         __be64 flit[TX_DESC_FLITS];
 100 };
 101 
 102 struct rx_desc {
 103         __be32 addr_lo;
 104         __be32 len_gen;
 105         __be32 gen2;
 106         __be32 addr_hi;
 107 };
 108 
 109 struct tx_sw_desc {             /* SW state per Tx descriptor */
 110         struct sk_buff *skb;
 111         u8 eop;       /* set if last descriptor for packet */
 112         u8 addr_idx;  /* buffer index of first SGL entry in descriptor */
 113         u8 fragidx;   /* first page fragment associated with descriptor */
 114         s8 sflit;     /* start flit of first SGL entry in descriptor */
 115 };
 116 
 117 struct rx_sw_desc {                /* SW state per Rx descriptor */
 118         union {
 119                 struct sk_buff *skb;
 120                 struct fl_pg_chunk pg_chunk;
 121         };
 122         DEFINE_DMA_UNMAP_ADDR(dma_addr);
 123 };
 124 
 125 struct rsp_desc {               /* response queue descriptor */
 126         struct rss_header rss_hdr;
 127         __be32 flags;
 128         __be32 len_cq;
 129         u8 imm_data[47];
 130         u8 intr_gen;
 131 };
 132 
 133 /*
 134  * Holds unmapping information for Tx packets that need deferred unmapping.
 135  * This structure lives at skb->head and must be allocated by callers.
 136  */
 137 struct deferred_unmap_info {
 138         struct pci_dev *pdev;
 139         dma_addr_t addr[MAX_SKB_FRAGS + 1];
 140 };
 141 
 142 /*
 143  * Maps a number of flits to the number of Tx descriptors that can hold them.
 144  * The formula is
 145  *
 146  * desc = 1 + (flits - 2) / (WR_FLITS - 1).
 147  *
 148  * HW allows up to 4 descriptors to be combined into a WR.
 149  */
 150 static u8 flit_desc_map[] = {
 151         0,
 152 #if SGE_NUM_GENBITS == 1
 153         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
 154         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 155         3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
 156         4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
 157 #elif SGE_NUM_GENBITS == 2
 158         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
 159         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 160         3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
 161         4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
 162 #else
 163 # error "SGE_NUM_GENBITS must be 1 or 2"
 164 #endif
 165 };
 166 
 167 static inline struct sge_qset *fl_to_qset(const struct sge_fl *q, int qidx)
 168 {
 169         return container_of(q, struct sge_qset, fl[qidx]);
 170 }
 171 
 172 static inline struct sge_qset *rspq_to_qset(const struct sge_rspq *q)
 173 {
 174         return container_of(q, struct sge_qset, rspq);
 175 }
 176 
 177 static inline struct sge_qset *txq_to_qset(const struct sge_txq *q, int qidx)
 178 {
 179         return container_of(q, struct sge_qset, txq[qidx]);
 180 }
 181 
 182 /**
 183  *      refill_rspq - replenish an SGE response queue
 184  *      @adapter: the adapter
 185  *      @q: the response queue to replenish
 186  *      @credits: how many new responses to make available
 187  *
 188  *      Replenishes a response queue by making the supplied number of responses
 189  *      available to HW.
 190  */
 191 static inline void refill_rspq(struct adapter *adapter,
 192                                const struct sge_rspq *q, unsigned int credits)
 193 {
 194         rmb();
 195         t3_write_reg(adapter, A_SG_RSPQ_CREDIT_RETURN,
 196                      V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
 197 }
 198 
 199 /**
 200  *      need_skb_unmap - does the platform need unmapping of sk_buffs?
 201  *
 202  *      Returns true if the platform needs sk_buff unmapping.  The compiler
 203  *      optimizes away unnecessary code if this returns true.
 204  */
 205 static inline int need_skb_unmap(void)
 206 {
 207 #ifdef CONFIG_NEED_DMA_MAP_STATE
 208         return 1;
 209 #else
 210         return 0;
 211 #endif
 212 }
 213 
 214 /**
 215  *      unmap_skb - unmap a packet main body and its page fragments
 216  *      @skb: the packet
 217  *      @q: the Tx queue containing Tx descriptors for the packet
 218  *      @cidx: index of Tx descriptor
 219  *      @pdev: the PCI device
 220  *
 221  *      Unmap the main body of an sk_buff and its page fragments, if any.
 222  *      Because of the fairly complicated structure of our SGLs and the desire
 223  *      to conserve space for metadata, the information necessary to unmap an
 224  *      sk_buff is spread across the sk_buff itself (buffer lengths), the HW Tx
 225  *      descriptors (the physical addresses of the various data buffers), and
 226  *      the SW descriptor state (assorted indices).  The send functions
 227  *      initialize the indices for the first packet descriptor so we can unmap
 228  *      the buffers held in the first Tx descriptor here, and we have enough
 229  *      information at this point to set the state for the next Tx descriptor.
 230  *
 231  *      Note that it is possible to clean up the first descriptor of a packet
 232  *      before the send routines have written the next descriptors, but this
 233  *      race does not cause any problem.  We just end up writing the unmapping
 234  *      info for the descriptor first.
 235  */
 236 static inline void unmap_skb(struct sk_buff *skb, struct sge_txq *q,
 237                              unsigned int cidx, struct pci_dev *pdev)
 238 {
 239         const struct sg_ent *sgp;
 240         struct tx_sw_desc *d = &q->sdesc[cidx];
 241         int nfrags, frag_idx, curflit, j = d->addr_idx;
 242 
 243         sgp = (struct sg_ent *)&q->desc[cidx].flit[d->sflit];
 244         frag_idx = d->fragidx;
 245 
 246         if (frag_idx == 0 && skb_headlen(skb)) {
 247                 pci_unmap_single(pdev, be64_to_cpu(sgp->addr[0]),
 248                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 249                 j = 1;
 250         }
 251 
 252         curflit = d->sflit + 1 + j;
 253         nfrags = skb_shinfo(skb)->nr_frags;
 254 
 255         while (frag_idx < nfrags && curflit < WR_FLITS) {
 256                 pci_unmap_page(pdev, be64_to_cpu(sgp->addr[j]),
 257                                skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]),
 258                                PCI_DMA_TODEVICE);
 259                 j ^= 1;
 260                 if (j == 0) {
 261                         sgp++;
 262                         curflit++;
 263                 }
 264                 curflit++;
 265                 frag_idx++;
 266         }
 267 
 268         if (frag_idx < nfrags) {   /* SGL continues into next Tx descriptor */
 269                 d = cidx + 1 == q->size ? q->sdesc : d + 1;
 270                 d->fragidx = frag_idx;
 271                 d->addr_idx = j;
 272                 d->sflit = curflit - WR_FLITS - j; /* sflit can be -1 */
 273         }
 274 }
 275 
 276 /**
 277  *      free_tx_desc - reclaims Tx descriptors and their buffers
 278  *      @adapter: the adapter
 279  *      @q: the Tx queue to reclaim descriptors from
 280  *      @n: the number of descriptors to reclaim
 281  *
 282  *      Reclaims Tx descriptors from an SGE Tx queue and frees the associated
 283  *      Tx buffers.  Called with the Tx queue lock held.
 284  */
 285 static void free_tx_desc(struct adapter *adapter, struct sge_txq *q,
 286                          unsigned int n)
 287 {
 288         struct tx_sw_desc *d;
 289         struct pci_dev *pdev = adapter->pdev;
 290         unsigned int cidx = q->cidx;
 291 
 292         const int need_unmap = need_skb_unmap() &&
 293                                q->cntxt_id >= FW_TUNNEL_SGEEC_START;
 294 
 295         d = &q->sdesc[cidx];
 296         while (n--) {
 297                 if (d->skb) {   /* an SGL is present */
 298                         if (need_unmap)
 299                                 unmap_skb(d->skb, q, cidx, pdev);
 300                         if (d->eop) {
 301                                 dev_consume_skb_any(d->skb);
 302                                 d->skb = NULL;
 303                         }
 304                 }
 305                 ++d;
 306                 if (++cidx == q->size) {
 307                         cidx = 0;
 308                         d = q->sdesc;
 309                 }
 310         }
 311         q->cidx = cidx;
 312 }
 313 
 314 /**
 315  *      reclaim_completed_tx - reclaims completed Tx descriptors
 316  *      @adapter: the adapter
 317  *      @q: the Tx queue to reclaim completed descriptors from
 318  *      @chunk: maximum number of descriptors to reclaim
 319  *
 320  *      Reclaims Tx descriptors that the SGE has indicated it has processed,
 321  *      and frees the associated buffers if possible.  Called with the Tx
 322  *      queue's lock held.
 323  */
 324 static inline unsigned int reclaim_completed_tx(struct adapter *adapter,
 325                                                 struct sge_txq *q,
 326                                                 unsigned int chunk)
 327 {
 328         unsigned int reclaim = q->processed - q->cleaned;
 329 
 330         reclaim = min(chunk, reclaim);
 331         if (reclaim) {
 332                 free_tx_desc(adapter, q, reclaim);
 333                 q->cleaned += reclaim;
 334                 q->in_use -= reclaim;
 335         }
 336         return q->processed - q->cleaned;
 337 }
 338 
 339 /**
 340  *      should_restart_tx - are there enough resources to restart a Tx queue?
 341  *      @q: the Tx queue
 342  *
 343  *      Checks if there are enough descriptors to restart a suspended Tx queue.
 344  */
 345 static inline int should_restart_tx(const struct sge_txq *q)
 346 {
 347         unsigned int r = q->processed - q->cleaned;
 348 
 349         return q->in_use - r < (q->size >> 1);
 350 }
 351 
 352 static void clear_rx_desc(struct pci_dev *pdev, const struct sge_fl *q,
 353                           struct rx_sw_desc *d)
 354 {
 355         if (q->use_pages && d->pg_chunk.page) {
 356                 (*d->pg_chunk.p_cnt)--;
 357                 if (!*d->pg_chunk.p_cnt)
 358                         pci_unmap_page(pdev,
 359                                        d->pg_chunk.mapping,
 360                                        q->alloc_size, PCI_DMA_FROMDEVICE);
 361 
 362                 put_page(d->pg_chunk.page);
 363                 d->pg_chunk.page = NULL;
 364         } else {
 365                 pci_unmap_single(pdev, dma_unmap_addr(d, dma_addr),
 366                                  q->buf_size, PCI_DMA_FROMDEVICE);
 367                 kfree_skb(d->skb);
 368                 d->skb = NULL;
 369         }
 370 }
 371 
 372 /**
 373  *      free_rx_bufs - free the Rx buffers on an SGE free list
 374  *      @pdev: the PCI device associated with the adapter
 375  *      @rxq: the SGE free list to clean up
 376  *
 377  *      Release the buffers on an SGE free-buffer Rx queue.  HW fetching from
 378  *      this queue should be stopped before calling this function.
 379  */
 380 static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q)
 381 {
 382         unsigned int cidx = q->cidx;
 383 
 384         while (q->credits--) {
 385                 struct rx_sw_desc *d = &q->sdesc[cidx];
 386 
 387 
 388                 clear_rx_desc(pdev, q, d);
 389                 if (++cidx == q->size)
 390                         cidx = 0;
 391         }
 392 
 393         if (q->pg_chunk.page) {
 394                 __free_pages(q->pg_chunk.page, q->order);
 395                 q->pg_chunk.page = NULL;
 396         }
 397 }
 398 
 399 /**
 400  *      add_one_rx_buf - add a packet buffer to a free-buffer list
 401  *      @va:  buffer start VA
 402  *      @len: the buffer length
 403  *      @d: the HW Rx descriptor to write
 404  *      @sd: the SW Rx descriptor to write
 405  *      @gen: the generation bit value
 406  *      @pdev: the PCI device associated with the adapter
 407  *
 408  *      Add a buffer of the given length to the supplied HW and SW Rx
 409  *      descriptors.
 410  */
 411 static inline int add_one_rx_buf(void *va, unsigned int len,
 412                                  struct rx_desc *d, struct rx_sw_desc *sd,
 413                                  unsigned int gen, struct pci_dev *pdev)
 414 {
 415         dma_addr_t mapping;
 416 
 417         mapping = pci_map_single(pdev, va, len, PCI_DMA_FROMDEVICE);
 418         if (unlikely(pci_dma_mapping_error(pdev, mapping)))
 419                 return -ENOMEM;
 420 
 421         dma_unmap_addr_set(sd, dma_addr, mapping);
 422 
 423         d->addr_lo = cpu_to_be32(mapping);
 424         d->addr_hi = cpu_to_be32((u64) mapping >> 32);
 425         dma_wmb();
 426         d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
 427         d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
 428         return 0;
 429 }
 430 
 431 static inline int add_one_rx_chunk(dma_addr_t mapping, struct rx_desc *d,
 432                                    unsigned int gen)
 433 {
 434         d->addr_lo = cpu_to_be32(mapping);
 435         d->addr_hi = cpu_to_be32((u64) mapping >> 32);
 436         dma_wmb();
 437         d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
 438         d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
 439         return 0;
 440 }
 441 
 442 static int alloc_pg_chunk(struct adapter *adapter, struct sge_fl *q,
 443                           struct rx_sw_desc *sd, gfp_t gfp,
 444                           unsigned int order)
 445 {
 446         if (!q->pg_chunk.page) {
 447                 dma_addr_t mapping;
 448 
 449                 q->pg_chunk.page = alloc_pages(gfp, order);
 450                 if (unlikely(!q->pg_chunk.page))
 451                         return -ENOMEM;
 452                 q->pg_chunk.va = page_address(q->pg_chunk.page);
 453                 q->pg_chunk.p_cnt = q->pg_chunk.va + (PAGE_SIZE << order) -
 454                                     SGE_PG_RSVD;
 455                 q->pg_chunk.offset = 0;
 456                 mapping = pci_map_page(adapter->pdev, q->pg_chunk.page,
 457                                        0, q->alloc_size, PCI_DMA_FROMDEVICE);
 458                 if (unlikely(pci_dma_mapping_error(adapter->pdev, mapping))) {
 459                         __free_pages(q->pg_chunk.page, order);
 460                         q->pg_chunk.page = NULL;
 461                         return -EIO;
 462                 }
 463                 q->pg_chunk.mapping = mapping;
 464         }
 465         sd->pg_chunk = q->pg_chunk;
 466 
 467         prefetch(sd->pg_chunk.p_cnt);
 468 
 469         q->pg_chunk.offset += q->buf_size;
 470         if (q->pg_chunk.offset == (PAGE_SIZE << order))
 471                 q->pg_chunk.page = NULL;
 472         else {
 473                 q->pg_chunk.va += q->buf_size;
 474                 get_page(q->pg_chunk.page);
 475         }
 476 
 477         if (sd->pg_chunk.offset == 0)
 478                 *sd->pg_chunk.p_cnt = 1;
 479         else
 480                 *sd->pg_chunk.p_cnt += 1;
 481 
 482         return 0;
 483 }
 484 
 485 static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
 486 {
 487         if (q->pend_cred >= q->credits / 4) {
 488                 q->pend_cred = 0;
 489                 wmb();
 490                 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
 491         }
 492 }
 493 
 494 /**
 495  *      refill_fl - refill an SGE free-buffer list
 496  *      @adapter: the adapter
 497  *      @q: the free-list to refill
 498  *      @n: the number of new buffers to allocate
 499  *      @gfp: the gfp flags for allocating new buffers
 500  *
 501  *      (Re)populate an SGE free-buffer list with up to @n new packet buffers,
 502  *      allocated with the supplied gfp flags.  The caller must assure that
 503  *      @n does not exceed the queue's capacity.
 504  */
 505 static int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp)
 506 {
 507         struct rx_sw_desc *sd = &q->sdesc[q->pidx];
 508         struct rx_desc *d = &q->desc[q->pidx];
 509         unsigned int count = 0;
 510 
 511         while (n--) {
 512                 dma_addr_t mapping;
 513                 int err;
 514 
 515                 if (q->use_pages) {
 516                         if (unlikely(alloc_pg_chunk(adap, q, sd, gfp,
 517                                                     q->order))) {
 518 nomem:                          q->alloc_failed++;
 519                                 break;
 520                         }
 521                         mapping = sd->pg_chunk.mapping + sd->pg_chunk.offset;
 522                         dma_unmap_addr_set(sd, dma_addr, mapping);
 523 
 524                         add_one_rx_chunk(mapping, d, q->gen);
 525                         pci_dma_sync_single_for_device(adap->pdev, mapping,
 526                                                 q->buf_size - SGE_PG_RSVD,
 527                                                 PCI_DMA_FROMDEVICE);
 528                 } else {
 529                         void *buf_start;
 530 
 531                         struct sk_buff *skb = alloc_skb(q->buf_size, gfp);
 532                         if (!skb)
 533                                 goto nomem;
 534 
 535                         sd->skb = skb;
 536                         buf_start = skb->data;
 537                         err = add_one_rx_buf(buf_start, q->buf_size, d, sd,
 538                                              q->gen, adap->pdev);
 539                         if (unlikely(err)) {
 540                                 clear_rx_desc(adap->pdev, q, sd);
 541                                 break;
 542                         }
 543                 }
 544 
 545                 d++;
 546                 sd++;
 547                 if (++q->pidx == q->size) {
 548                         q->pidx = 0;
 549                         q->gen ^= 1;
 550                         sd = q->sdesc;
 551                         d = q->desc;
 552                 }
 553                 count++;
 554         }
 555 
 556         q->credits += count;
 557         q->pend_cred += count;
 558         ring_fl_db(adap, q);
 559 
 560         return count;
 561 }
 562 
 563 static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
 564 {
 565         refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits),
 566                   GFP_ATOMIC | __GFP_COMP);
 567 }
 568 
 569 /**
 570  *      recycle_rx_buf - recycle a receive buffer
 571  *      @adapter: the adapter
 572  *      @q: the SGE free list
 573  *      @idx: index of buffer to recycle
 574  *
 575  *      Recycles the specified buffer on the given free list by adding it at
 576  *      the next available slot on the list.
 577  */
 578 static void recycle_rx_buf(struct adapter *adap, struct sge_fl *q,
 579                            unsigned int idx)
 580 {
 581         struct rx_desc *from = &q->desc[idx];
 582         struct rx_desc *to = &q->desc[q->pidx];
 583 
 584         q->sdesc[q->pidx] = q->sdesc[idx];
 585         to->addr_lo = from->addr_lo;    /* already big endian */
 586         to->addr_hi = from->addr_hi;    /* likewise */
 587         dma_wmb();
 588         to->len_gen = cpu_to_be32(V_FLD_GEN1(q->gen));
 589         to->gen2 = cpu_to_be32(V_FLD_GEN2(q->gen));
 590 
 591         if (++q->pidx == q->size) {
 592                 q->pidx = 0;
 593                 q->gen ^= 1;
 594         }
 595 
 596         q->credits++;
 597         q->pend_cred++;
 598         ring_fl_db(adap, q);
 599 }
 600 
 601 /**
 602  *      alloc_ring - allocate resources for an SGE descriptor ring
 603  *      @pdev: the PCI device
 604  *      @nelem: the number of descriptors
 605  *      @elem_size: the size of each descriptor
 606  *      @sw_size: the size of the SW state associated with each ring element
 607  *      @phys: the physical address of the allocated ring
 608  *      @metadata: address of the array holding the SW state for the ring
 609  *
 610  *      Allocates resources for an SGE descriptor ring, such as Tx queues,
 611  *      free buffer lists, or response queues.  Each SGE ring requires
 612  *      space for its HW descriptors plus, optionally, space for the SW state
 613  *      associated with each HW entry (the metadata).  The function returns
 614  *      three values: the virtual address for the HW ring (the return value
 615  *      of the function), the physical address of the HW ring, and the address
 616  *      of the SW ring.
 617  */
 618 static void *alloc_ring(struct pci_dev *pdev, size_t nelem, size_t elem_size,
 619                         size_t sw_size, dma_addr_t * phys, void *metadata)
 620 {
 621         size_t len = nelem * elem_size;
 622         void *s = NULL;
 623         void *p = dma_alloc_coherent(&pdev->dev, len, phys, GFP_KERNEL);
 624 
 625         if (!p)
 626                 return NULL;
 627         if (sw_size && metadata) {
 628                 s = kcalloc(nelem, sw_size, GFP_KERNEL);
 629 
 630                 if (!s) {
 631                         dma_free_coherent(&pdev->dev, len, p, *phys);
 632                         return NULL;
 633                 }
 634                 *(void **)metadata = s;
 635         }
 636         return p;
 637 }
 638 
 639 /**
 640  *      t3_reset_qset - reset a sge qset
 641  *      @q: the queue set
 642  *
 643  *      Reset the qset structure.
 644  *      the NAPI structure is preserved in the event of
 645  *      the qset's reincarnation, for example during EEH recovery.
 646  */
 647 static void t3_reset_qset(struct sge_qset *q)
 648 {
 649         if (q->adap &&
 650             !(q->adap->flags & NAPI_INIT)) {
 651                 memset(q, 0, sizeof(*q));
 652                 return;
 653         }
 654 
 655         q->adap = NULL;
 656         memset(&q->rspq, 0, sizeof(q->rspq));
 657         memset(q->fl, 0, sizeof(struct sge_fl) * SGE_RXQ_PER_SET);
 658         memset(q->txq, 0, sizeof(struct sge_txq) * SGE_TXQ_PER_SET);
 659         q->txq_stopped = 0;
 660         q->tx_reclaim_timer.function = NULL; /* for t3_stop_sge_timers() */
 661         q->rx_reclaim_timer.function = NULL;
 662         q->nomem = 0;
 663         napi_free_frags(&q->napi);
 664 }
 665 
 666 
 667 /**
 668  *      free_qset - free the resources of an SGE queue set
 669  *      @adapter: the adapter owning the queue set
 670  *      @q: the queue set
 671  *
 672  *      Release the HW and SW resources associated with an SGE queue set, such
 673  *      as HW contexts, packet buffers, and descriptor rings.  Traffic to the
 674  *      queue set must be quiesced prior to calling this.
 675  */
 676 static void t3_free_qset(struct adapter *adapter, struct sge_qset *q)
 677 {
 678         int i;
 679         struct pci_dev *pdev = adapter->pdev;
 680 
 681         for (i = 0; i < SGE_RXQ_PER_SET; ++i)
 682                 if (q->fl[i].desc) {
 683                         spin_lock_irq(&adapter->sge.reg_lock);
 684                         t3_sge_disable_fl(adapter, q->fl[i].cntxt_id);
 685                         spin_unlock_irq(&adapter->sge.reg_lock);
 686                         free_rx_bufs(pdev, &q->fl[i]);
 687                         kfree(q->fl[i].sdesc);
 688                         dma_free_coherent(&pdev->dev,
 689                                           q->fl[i].size *
 690                                           sizeof(struct rx_desc), q->fl[i].desc,
 691                                           q->fl[i].phys_addr);
 692                 }
 693 
 694         for (i = 0; i < SGE_TXQ_PER_SET; ++i)
 695                 if (q->txq[i].desc) {
 696                         spin_lock_irq(&adapter->sge.reg_lock);
 697                         t3_sge_enable_ecntxt(adapter, q->txq[i].cntxt_id, 0);
 698                         spin_unlock_irq(&adapter->sge.reg_lock);
 699                         if (q->txq[i].sdesc) {
 700                                 free_tx_desc(adapter, &q->txq[i],
 701                                              q->txq[i].in_use);
 702                                 kfree(q->txq[i].sdesc);
 703                         }
 704                         dma_free_coherent(&pdev->dev,
 705                                           q->txq[i].size *
 706                                           sizeof(struct tx_desc),
 707                                           q->txq[i].desc, q->txq[i].phys_addr);
 708                         __skb_queue_purge(&q->txq[i].sendq);
 709                 }
 710 
 711         if (q->rspq.desc) {
 712                 spin_lock_irq(&adapter->sge.reg_lock);
 713                 t3_sge_disable_rspcntxt(adapter, q->rspq.cntxt_id);
 714                 spin_unlock_irq(&adapter->sge.reg_lock);
 715                 dma_free_coherent(&pdev->dev,
 716                                   q->rspq.size * sizeof(struct rsp_desc),
 717                                   q->rspq.desc, q->rspq.phys_addr);
 718         }
 719 
 720         t3_reset_qset(q);
 721 }
 722 
 723 /**
 724  *      init_qset_cntxt - initialize an SGE queue set context info
 725  *      @qs: the queue set
 726  *      @id: the queue set id
 727  *
 728  *      Initializes the TIDs and context ids for the queues of a queue set.
 729  */
 730 static void init_qset_cntxt(struct sge_qset *qs, unsigned int id)
 731 {
 732         qs->rspq.cntxt_id = id;
 733         qs->fl[0].cntxt_id = 2 * id;
 734         qs->fl[1].cntxt_id = 2 * id + 1;
 735         qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
 736         qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
 737         qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
 738         qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
 739         qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
 740 }
 741 
 742 /**
 743  *      sgl_len - calculates the size of an SGL of the given capacity
 744  *      @n: the number of SGL entries
 745  *
 746  *      Calculates the number of flits needed for a scatter/gather list that
 747  *      can hold the given number of entries.
 748  */
 749 static inline unsigned int sgl_len(unsigned int n)
 750 {
 751         /* alternatively: 3 * (n / 2) + 2 * (n & 1) */
 752         return (3 * n) / 2 + (n & 1);
 753 }
 754 
 755 /**
 756  *      flits_to_desc - returns the num of Tx descriptors for the given flits
 757  *      @n: the number of flits
 758  *
 759  *      Calculates the number of Tx descriptors needed for the supplied number
 760  *      of flits.
 761  */
 762 static inline unsigned int flits_to_desc(unsigned int n)
 763 {
 764         BUG_ON(n >= ARRAY_SIZE(flit_desc_map));
 765         return flit_desc_map[n];
 766 }
 767 
 768 /**
 769  *      get_packet - return the next ingress packet buffer from a free list
 770  *      @adap: the adapter that received the packet
 771  *      @fl: the SGE free list holding the packet
 772  *      @len: the packet length including any SGE padding
 773  *      @drop_thres: # of remaining buffers before we start dropping packets
 774  *
 775  *      Get the next packet from a free list and complete setup of the
 776  *      sk_buff.  If the packet is small we make a copy and recycle the
 777  *      original buffer, otherwise we use the original buffer itself.  If a
 778  *      positive drop threshold is supplied packets are dropped and their
 779  *      buffers recycled if (a) the number of remaining buffers is under the
 780  *      threshold and the packet is too big to copy, or (b) the packet should
 781  *      be copied but there is no memory for the copy.
 782  */
 783 static struct sk_buff *get_packet(struct adapter *adap, struct sge_fl *fl,
 784                                   unsigned int len, unsigned int drop_thres)
 785 {
 786         struct sk_buff *skb = NULL;
 787         struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
 788 
 789         prefetch(sd->skb->data);
 790         fl->credits--;
 791 
 792         if (len <= SGE_RX_COPY_THRES) {
 793                 skb = alloc_skb(len, GFP_ATOMIC);
 794                 if (likely(skb != NULL)) {
 795                         __skb_put(skb, len);
 796                         pci_dma_sync_single_for_cpu(adap->pdev,
 797                                             dma_unmap_addr(sd, dma_addr), len,
 798                                             PCI_DMA_FROMDEVICE);
 799                         memcpy(skb->data, sd->skb->data, len);
 800                         pci_dma_sync_single_for_device(adap->pdev,
 801                                             dma_unmap_addr(sd, dma_addr), len,
 802                                             PCI_DMA_FROMDEVICE);
 803                 } else if (!drop_thres)
 804                         goto use_orig_buf;
 805 recycle:
 806                 recycle_rx_buf(adap, fl, fl->cidx);
 807                 return skb;
 808         }
 809 
 810         if (unlikely(fl->credits < drop_thres) &&
 811             refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits - 1),
 812                       GFP_ATOMIC | __GFP_COMP) == 0)
 813                 goto recycle;
 814 
 815 use_orig_buf:
 816         pci_unmap_single(adap->pdev, dma_unmap_addr(sd, dma_addr),
 817                          fl->buf_size, PCI_DMA_FROMDEVICE);
 818         skb = sd->skb;
 819         skb_put(skb, len);
 820         __refill_fl(adap, fl);
 821         return skb;
 822 }
 823 
 824 /**
 825  *      get_packet_pg - return the next ingress packet buffer from a free list
 826  *      @adap: the adapter that received the packet
 827  *      @fl: the SGE free list holding the packet
 828  *      @len: the packet length including any SGE padding
 829  *      @drop_thres: # of remaining buffers before we start dropping packets
 830  *
 831  *      Get the next packet from a free list populated with page chunks.
 832  *      If the packet is small we make a copy and recycle the original buffer,
 833  *      otherwise we attach the original buffer as a page fragment to a fresh
 834  *      sk_buff.  If a positive drop threshold is supplied packets are dropped
 835  *      and their buffers recycled if (a) the number of remaining buffers is
 836  *      under the threshold and the packet is too big to copy, or (b) there's
 837  *      no system memory.
 838  *
 839  *      Note: this function is similar to @get_packet but deals with Rx buffers
 840  *      that are page chunks rather than sk_buffs.
 841  */
 842 static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl,
 843                                      struct sge_rspq *q, unsigned int len,
 844                                      unsigned int drop_thres)
 845 {
 846         struct sk_buff *newskb, *skb;
 847         struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
 848 
 849         dma_addr_t dma_addr = dma_unmap_addr(sd, dma_addr);
 850 
 851         newskb = skb = q->pg_skb;
 852         if (!skb && (len <= SGE_RX_COPY_THRES)) {
 853                 newskb = alloc_skb(len, GFP_ATOMIC);
 854                 if (likely(newskb != NULL)) {
 855                         __skb_put(newskb, len);
 856                         pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
 857                                             PCI_DMA_FROMDEVICE);
 858                         memcpy(newskb->data, sd->pg_chunk.va, len);
 859                         pci_dma_sync_single_for_device(adap->pdev, dma_addr,
 860                                                        len,
 861                                                        PCI_DMA_FROMDEVICE);
 862                 } else if (!drop_thres)
 863                         return NULL;
 864 recycle:
 865                 fl->credits--;
 866                 recycle_rx_buf(adap, fl, fl->cidx);
 867                 q->rx_recycle_buf++;
 868                 return newskb;
 869         }
 870 
 871         if (unlikely(q->rx_recycle_buf || (!skb && fl->credits <= drop_thres)))
 872                 goto recycle;
 873 
 874         prefetch(sd->pg_chunk.p_cnt);
 875 
 876         if (!skb)
 877                 newskb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC);
 878 
 879         if (unlikely(!newskb)) {
 880                 if (!drop_thres)
 881                         return NULL;
 882                 goto recycle;
 883         }
 884 
 885         pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
 886                                     PCI_DMA_FROMDEVICE);
 887         (*sd->pg_chunk.p_cnt)--;
 888         if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
 889                 pci_unmap_page(adap->pdev,
 890                                sd->pg_chunk.mapping,
 891                                fl->alloc_size,
 892                                PCI_DMA_FROMDEVICE);
 893         if (!skb) {
 894                 __skb_put(newskb, SGE_RX_PULL_LEN);
 895                 memcpy(newskb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN);
 896                 skb_fill_page_desc(newskb, 0, sd->pg_chunk.page,
 897                                    sd->pg_chunk.offset + SGE_RX_PULL_LEN,
 898                                    len - SGE_RX_PULL_LEN);
 899                 newskb->len = len;
 900                 newskb->data_len = len - SGE_RX_PULL_LEN;
 901                 newskb->truesize += newskb->data_len;
 902         } else {
 903                 skb_fill_page_desc(newskb, skb_shinfo(newskb)->nr_frags,
 904                                    sd->pg_chunk.page,
 905                                    sd->pg_chunk.offset, len);
 906                 newskb->len += len;
 907                 newskb->data_len += len;
 908                 newskb->truesize += len;
 909         }
 910 
 911         fl->credits--;
 912         /*
 913          * We do not refill FLs here, we let the caller do it to overlap a
 914          * prefetch.
 915          */
 916         return newskb;
 917 }
 918 
 919 /**
 920  *      get_imm_packet - return the next ingress packet buffer from a response
 921  *      @resp: the response descriptor containing the packet data
 922  *
 923  *      Return a packet containing the immediate data of the given response.
 924  */
 925 static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp)
 926 {
 927         struct sk_buff *skb = alloc_skb(IMMED_PKT_SIZE, GFP_ATOMIC);
 928 
 929         if (skb) {
 930                 __skb_put(skb, IMMED_PKT_SIZE);
 931                 skb_copy_to_linear_data(skb, resp->imm_data, IMMED_PKT_SIZE);
 932         }
 933         return skb;
 934 }
 935 
 936 /**
 937  *      calc_tx_descs - calculate the number of Tx descriptors for a packet
 938  *      @skb: the packet
 939  *
 940  *      Returns the number of Tx descriptors needed for the given Ethernet
 941  *      packet.  Ethernet packets require addition of WR and CPL headers.
 942  */
 943 static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
 944 {
 945         unsigned int flits;
 946 
 947         if (skb->len <= WR_LEN - sizeof(struct cpl_tx_pkt))
 948                 return 1;
 949 
 950         flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 2;
 951         if (skb_shinfo(skb)->gso_size)
 952                 flits++;
 953         return flits_to_desc(flits);
 954 }
 955 
 956 /*      map_skb - map a packet main body and its page fragments
 957  *      @pdev: the PCI device
 958  *      @skb: the packet
 959  *      @addr: placeholder to save the mapped addresses
 960  *
 961  *      map the main body of an sk_buff and its page fragments, if any.
 962  */
 963 static int map_skb(struct pci_dev *pdev, const struct sk_buff *skb,
 964                    dma_addr_t *addr)
 965 {
 966         const skb_frag_t *fp, *end;
 967         const struct skb_shared_info *si;
 968 
 969         if (skb_headlen(skb)) {
 970                 *addr = pci_map_single(pdev, skb->data, skb_headlen(skb),
 971                                        PCI_DMA_TODEVICE);
 972                 if (pci_dma_mapping_error(pdev, *addr))
 973                         goto out_err;
 974                 addr++;
 975         }
 976 
 977         si = skb_shinfo(skb);
 978         end = &si->frags[si->nr_frags];
 979 
 980         for (fp = si->frags; fp < end; fp++) {
 981                 *addr = skb_frag_dma_map(&pdev->dev, fp, 0, skb_frag_size(fp),
 982                                          DMA_TO_DEVICE);
 983                 if (pci_dma_mapping_error(pdev, *addr))
 984                         goto unwind;
 985                 addr++;
 986         }
 987         return 0;
 988 
 989 unwind:
 990         while (fp-- > si->frags)
 991                 dma_unmap_page(&pdev->dev, *--addr, skb_frag_size(fp),
 992                                DMA_TO_DEVICE);
 993 
 994         pci_unmap_single(pdev, addr[-1], skb_headlen(skb), PCI_DMA_TODEVICE);
 995 out_err:
 996         return -ENOMEM;
 997 }
 998 
 999 /**
1000  *      write_sgl - populate a scatter/gather list for a packet
1001  *      @skb: the packet
1002  *      @sgp: the SGL to populate
1003  *      @start: start address of skb main body data to include in the SGL
1004  *      @len: length of skb main body data to include in the SGL
1005  *      @addr: the list of the mapped addresses
1006  *
1007  *      Copies the scatter/gather list for the buffers that make up a packet
1008  *      and returns the SGL size in 8-byte words.  The caller must size the SGL
1009  *      appropriately.
1010  */
1011 static inline unsigned int write_sgl(const struct sk_buff *skb,
1012                                      struct sg_ent *sgp, unsigned char *start,
1013                                      unsigned int len, const dma_addr_t *addr)
1014 {
1015         unsigned int i, j = 0, k = 0, nfrags;
1016 
1017         if (len) {
1018                 sgp->len[0] = cpu_to_be32(len);
1019                 sgp->addr[j++] = cpu_to_be64(addr[k++]);
1020         }
1021 
1022         nfrags = skb_shinfo(skb)->nr_frags;
1023         for (i = 0; i < nfrags; i++) {
1024                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1025 
1026                 sgp->len[j] = cpu_to_be32(skb_frag_size(frag));
1027                 sgp->addr[j] = cpu_to_be64(addr[k++]);
1028                 j ^= 1;
1029                 if (j == 0)
1030                         ++sgp;
1031         }
1032         if (j)
1033                 sgp->len[j] = 0;
1034         return ((nfrags + (len != 0)) * 3) / 2 + j;
1035 }
1036 
1037 /**
1038  *      check_ring_tx_db - check and potentially ring a Tx queue's doorbell
1039  *      @adap: the adapter
1040  *      @q: the Tx queue
1041  *
1042  *      Ring the doorbel if a Tx queue is asleep.  There is a natural race,
1043  *      where the HW is going to sleep just after we checked, however,
1044  *      then the interrupt handler will detect the outstanding TX packet
1045  *      and ring the doorbell for us.
1046  *
1047  *      When GTS is disabled we unconditionally ring the doorbell.
1048  */
1049 static inline void check_ring_tx_db(struct adapter *adap, struct sge_txq *q)
1050 {
1051 #if USE_GTS
1052         clear_bit(TXQ_LAST_PKT_DB, &q->flags);
1053         if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
1054                 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1055                 t3_write_reg(adap, A_SG_KDOORBELL,
1056                              F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1057         }
1058 #else
1059         wmb();                  /* write descriptors before telling HW */
1060         t3_write_reg(adap, A_SG_KDOORBELL,
1061                      F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1062 #endif
1063 }
1064 
1065 static inline void wr_gen2(struct tx_desc *d, unsigned int gen)
1066 {
1067 #if SGE_NUM_GENBITS == 2
1068         d->flit[TX_DESC_FLITS - 1] = cpu_to_be64(gen);
1069 #endif
1070 }
1071 
1072 /**
1073  *      write_wr_hdr_sgl - write a WR header and, optionally, SGL
1074  *      @ndesc: number of Tx descriptors spanned by the SGL
1075  *      @skb: the packet corresponding to the WR
1076  *      @d: first Tx descriptor to be written
1077  *      @pidx: index of above descriptors
1078  *      @q: the SGE Tx queue
1079  *      @sgl: the SGL
1080  *      @flits: number of flits to the start of the SGL in the first descriptor
1081  *      @sgl_flits: the SGL size in flits
1082  *      @gen: the Tx descriptor generation
1083  *      @wr_hi: top 32 bits of WR header based on WR type (big endian)
1084  *      @wr_lo: low 32 bits of WR header based on WR type (big endian)
1085  *
1086  *      Write a work request header and an associated SGL.  If the SGL is
1087  *      small enough to fit into one Tx descriptor it has already been written
1088  *      and we just need to write the WR header.  Otherwise we distribute the
1089  *      SGL across the number of descriptors it spans.
1090  */
1091 static void write_wr_hdr_sgl(unsigned int ndesc, struct sk_buff *skb,
1092                              struct tx_desc *d, unsigned int pidx,
1093                              const struct sge_txq *q,
1094                              const struct sg_ent *sgl,
1095                              unsigned int flits, unsigned int sgl_flits,
1096                              unsigned int gen, __be32 wr_hi,
1097                              __be32 wr_lo)
1098 {
1099         struct work_request_hdr *wrp = (struct work_request_hdr *)d;
1100         struct tx_sw_desc *sd = &q->sdesc[pidx];
1101 
1102         sd->skb = skb;
1103         if (need_skb_unmap()) {
1104                 sd->fragidx = 0;
1105                 sd->addr_idx = 0;
1106                 sd->sflit = flits;
1107         }
1108 
1109         if (likely(ndesc == 1)) {
1110                 sd->eop = 1;
1111                 wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
1112                                    V_WR_SGLSFLT(flits)) | wr_hi;
1113                 dma_wmb();
1114                 wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) |
1115                                    V_WR_GEN(gen)) | wr_lo;
1116                 wr_gen2(d, gen);
1117         } else {
1118                 unsigned int ogen = gen;
1119                 const u64 *fp = (const u64 *)sgl;
1120                 struct work_request_hdr *wp = wrp;
1121 
1122                 wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
1123                                    V_WR_SGLSFLT(flits)) | wr_hi;
1124 
1125                 while (sgl_flits) {
1126                         unsigned int avail = WR_FLITS - flits;
1127 
1128                         if (avail > sgl_flits)
1129                                 avail = sgl_flits;
1130                         memcpy(&d->flit[flits], fp, avail * sizeof(*fp));
1131                         sgl_flits -= avail;
1132                         ndesc--;
1133                         if (!sgl_flits)
1134                                 break;
1135 
1136                         fp += avail;
1137                         d++;
1138                         sd->eop = 0;
1139                         sd++;
1140                         if (++pidx == q->size) {
1141                                 pidx = 0;
1142                                 gen ^= 1;
1143                                 d = q->desc;
1144                                 sd = q->sdesc;
1145                         }
1146 
1147                         sd->skb = skb;
1148                         wrp = (struct work_request_hdr *)d;
1149                         wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
1150                                            V_WR_SGLSFLT(1)) | wr_hi;
1151                         wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS,
1152                                                         sgl_flits + 1)) |
1153                                            V_WR_GEN(gen)) | wr_lo;
1154                         wr_gen2(d, gen);
1155                         flits = 1;
1156                 }
1157                 sd->eop = 1;
1158                 wrp->wr_hi |= htonl(F_WR_EOP);
1159                 dma_wmb();
1160                 wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
1161                 wr_gen2((struct tx_desc *)wp, ogen);
1162                 WARN_ON(ndesc != 0);
1163         }
1164 }
1165 
1166 /**
1167  *      write_tx_pkt_wr - write a TX_PKT work request
1168  *      @adap: the adapter
1169  *      @skb: the packet to send
1170  *      @pi: the egress interface
1171  *      @pidx: index of the first Tx descriptor to write
1172  *      @gen: the generation value to use
1173  *      @q: the Tx queue
1174  *      @ndesc: number of descriptors the packet will occupy
1175  *      @compl: the value of the COMPL bit to use
1176  *
1177  *      Generate a TX_PKT work request to send the supplied packet.
1178  */
1179 static void write_tx_pkt_wr(struct adapter *adap, struct sk_buff *skb,
1180                             const struct port_info *pi,
1181                             unsigned int pidx, unsigned int gen,
1182                             struct sge_txq *q, unsigned int ndesc,
1183                             unsigned int compl, const dma_addr_t *addr)
1184 {
1185         unsigned int flits, sgl_flits, cntrl, tso_info;
1186         struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1187         struct tx_desc *d = &q->desc[pidx];
1188         struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)d;
1189 
1190         cpl->len = htonl(skb->len);
1191         cntrl = V_TXPKT_INTF(pi->port_id);
1192 
1193         if (skb_vlan_tag_present(skb))
1194                 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(skb_vlan_tag_get(skb));
1195 
1196         tso_info = V_LSO_MSS(skb_shinfo(skb)->gso_size);
1197         if (tso_info) {
1198                 int eth_type;
1199                 struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)cpl;
1200 
1201                 d->flit[2] = 0;
1202                 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
1203                 hdr->cntrl = htonl(cntrl);
1204                 eth_type = skb_network_offset(skb) == ETH_HLEN ?
1205                     CPL_ETH_II : CPL_ETH_II_VLAN;
1206                 tso_info |= V_LSO_ETH_TYPE(eth_type) |
1207                     V_LSO_IPHDR_WORDS(ip_hdr(skb)->ihl) |
1208                     V_LSO_TCPHDR_WORDS(tcp_hdr(skb)->doff);
1209                 hdr->lso_info = htonl(tso_info);
1210                 flits = 3;
1211         } else {
1212                 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
1213                 cntrl |= F_TXPKT_IPCSUM_DIS;    /* SW calculates IP csum */
1214                 cntrl |= V_TXPKT_L4CSUM_DIS(skb->ip_summed != CHECKSUM_PARTIAL);
1215                 cpl->cntrl = htonl(cntrl);
1216 
1217                 if (skb->len <= WR_LEN - sizeof(*cpl)) {
1218                         q->sdesc[pidx].skb = NULL;
1219                         if (!skb->data_len)
1220                                 skb_copy_from_linear_data(skb, &d->flit[2],
1221                                                           skb->len);
1222                         else
1223                                 skb_copy_bits(skb, 0, &d->flit[2], skb->len);
1224 
1225                         flits = (skb->len + 7) / 8 + 2;
1226                         cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(skb->len & 7) |
1227                                               V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT)
1228                                               | F_WR_SOP | F_WR_EOP | compl);
1229                         dma_wmb();
1230                         cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(gen) |
1231                                               V_WR_TID(q->token));
1232                         wr_gen2(d, gen);
1233                         dev_consume_skb_any(skb);
1234                         return;
1235                 }
1236 
1237                 flits = 2;
1238         }
1239 
1240         sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1241         sgl_flits = write_sgl(skb, sgp, skb->data, skb_headlen(skb), addr);
1242 
1243         write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits, gen,
1244                          htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | compl),
1245                          htonl(V_WR_TID(q->token)));
1246 }
1247 
1248 static inline void t3_stop_tx_queue(struct netdev_queue *txq,
1249                                     struct sge_qset *qs, struct sge_txq *q)
1250 {
1251         netif_tx_stop_queue(txq);
1252         set_bit(TXQ_ETH, &qs->txq_stopped);
1253         q->stops++;
1254 }
1255 
1256 /**
1257  *      eth_xmit - add a packet to the Ethernet Tx queue
1258  *      @skb: the packet
1259  *      @dev: the egress net device
1260  *
1261  *      Add a packet to an SGE Tx queue.  Runs with softirqs disabled.
1262  */
1263 netdev_tx_t t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1264 {
1265         int qidx;
1266         unsigned int ndesc, pidx, credits, gen, compl;
1267         const struct port_info *pi = netdev_priv(dev);
1268         struct adapter *adap = pi->adapter;
1269         struct netdev_queue *txq;
1270         struct sge_qset *qs;
1271         struct sge_txq *q;
1272         dma_addr_t addr[MAX_SKB_FRAGS + 1];
1273 
1274         /*
1275          * The chip min packet length is 9 octets but play safe and reject
1276          * anything shorter than an Ethernet header.
1277          */
1278         if (unlikely(skb->len < ETH_HLEN)) {
1279                 dev_kfree_skb_any(skb);
1280                 return NETDEV_TX_OK;
1281         }
1282 
1283         qidx = skb_get_queue_mapping(skb);
1284         qs = &pi->qs[qidx];
1285         q = &qs->txq[TXQ_ETH];
1286         txq = netdev_get_tx_queue(dev, qidx);
1287 
1288         reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1289 
1290         credits = q->size - q->in_use;
1291         ndesc = calc_tx_descs(skb);
1292 
1293         if (unlikely(credits < ndesc)) {
1294                 t3_stop_tx_queue(txq, qs, q);
1295                 dev_err(&adap->pdev->dev,
1296                         "%s: Tx ring %u full while queue awake!\n",
1297                         dev->name, q->cntxt_id & 7);
1298                 return NETDEV_TX_BUSY;
1299         }
1300 
1301         /* Check if ethernet packet can't be sent as immediate data */
1302         if (skb->len > (WR_LEN - sizeof(struct cpl_tx_pkt))) {
1303                 if (unlikely(map_skb(adap->pdev, skb, addr) < 0)) {
1304                         dev_kfree_skb(skb);
1305                         return NETDEV_TX_OK;
1306                 }
1307         }
1308 
1309         q->in_use += ndesc;
1310         if (unlikely(credits - ndesc < q->stop_thres)) {
1311                 t3_stop_tx_queue(txq, qs, q);
1312 
1313                 if (should_restart_tx(q) &&
1314                     test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1315                         q->restarts++;
1316                         netif_tx_start_queue(txq);
1317                 }
1318         }
1319 
1320         gen = q->gen;
1321         q->unacked += ndesc;
1322         compl = (q->unacked & 8) << (S_WR_COMPL - 3);
1323         q->unacked &= 7;
1324         pidx = q->pidx;
1325         q->pidx += ndesc;
1326         if (q->pidx >= q->size) {
1327                 q->pidx -= q->size;
1328                 q->gen ^= 1;
1329         }
1330 
1331         /* update port statistics */
1332         if (skb->ip_summed == CHECKSUM_PARTIAL)
1333                 qs->port_stats[SGE_PSTAT_TX_CSUM]++;
1334         if (skb_shinfo(skb)->gso_size)
1335                 qs->port_stats[SGE_PSTAT_TSO]++;
1336         if (skb_vlan_tag_present(skb))
1337                 qs->port_stats[SGE_PSTAT_VLANINS]++;
1338 
1339         /*
1340          * We do not use Tx completion interrupts to free DMAd Tx packets.
1341          * This is good for performance but means that we rely on new Tx
1342          * packets arriving to run the destructors of completed packets,
1343          * which open up space in their sockets' send queues.  Sometimes
1344          * we do not get such new packets causing Tx to stall.  A single
1345          * UDP transmitter is a good example of this situation.  We have
1346          * a clean up timer that periodically reclaims completed packets
1347          * but it doesn't run often enough (nor do we want it to) to prevent
1348          * lengthy stalls.  A solution to this problem is to run the
1349          * destructor early, after the packet is queued but before it's DMAd.
1350          * A cons is that we lie to socket memory accounting, but the amount
1351          * of extra memory is reasonable (limited by the number of Tx
1352          * descriptors), the packets do actually get freed quickly by new
1353          * packets almost always, and for protocols like TCP that wait for
1354          * acks to really free up the data the extra memory is even less.
1355          * On the positive side we run the destructors on the sending CPU
1356          * rather than on a potentially different completing CPU, usually a
1357          * good thing.  We also run them without holding our Tx queue lock,
1358          * unlike what reclaim_completed_tx() would otherwise do.
1359          *
1360          * Run the destructor before telling the DMA engine about the packet
1361          * to make sure it doesn't complete and get freed prematurely.
1362          */
1363         if (likely(!skb_shared(skb)))
1364                 skb_orphan(skb);
1365 
1366         write_tx_pkt_wr(adap, skb, pi, pidx, gen, q, ndesc, compl, addr);
1367         check_ring_tx_db(adap, q);
1368         return NETDEV_TX_OK;
1369 }
1370 
1371 /**
1372  *      write_imm - write a packet into a Tx descriptor as immediate data
1373  *      @d: the Tx descriptor to write
1374  *      @skb: the packet
1375  *      @len: the length of packet data to write as immediate data
1376  *      @gen: the generation bit value to write
1377  *
1378  *      Writes a packet as immediate data into a Tx descriptor.  The packet
1379  *      contains a work request at its beginning.  We must write the packet
1380  *      carefully so the SGE doesn't read it accidentally before it's written
1381  *      in its entirety.
1382  */
1383 static inline void write_imm(struct tx_desc *d, struct sk_buff *skb,
1384                              unsigned int len, unsigned int gen)
1385 {
1386         struct work_request_hdr *from = (struct work_request_hdr *)skb->data;
1387         struct work_request_hdr *to = (struct work_request_hdr *)d;
1388 
1389         if (likely(!skb->data_len))
1390                 memcpy(&to[1], &from[1], len - sizeof(*from));
1391         else
1392                 skb_copy_bits(skb, sizeof(*from), &to[1], len - sizeof(*from));
1393 
1394         to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP |
1395                                         V_WR_BCNTLFLT(len & 7));
1396         dma_wmb();
1397         to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) |
1398                                         V_WR_LEN((len + 7) / 8));
1399         wr_gen2(d, gen);
1400         kfree_skb(skb);
1401 }
1402 
1403 /**
1404  *      check_desc_avail - check descriptor availability on a send queue
1405  *      @adap: the adapter
1406  *      @q: the send queue
1407  *      @skb: the packet needing the descriptors
1408  *      @ndesc: the number of Tx descriptors needed
1409  *      @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
1410  *
1411  *      Checks if the requested number of Tx descriptors is available on an
1412  *      SGE send queue.  If the queue is already suspended or not enough
1413  *      descriptors are available the packet is queued for later transmission.
1414  *      Must be called with the Tx queue locked.
1415  *
1416  *      Returns 0 if enough descriptors are available, 1 if there aren't
1417  *      enough descriptors and the packet has been queued, and 2 if the caller
1418  *      needs to retry because there weren't enough descriptors at the
1419  *      beginning of the call but some freed up in the mean time.
1420  */
1421 static inline int check_desc_avail(struct adapter *adap, struct sge_txq *q,
1422                                    struct sk_buff *skb, unsigned int ndesc,
1423                                    unsigned int qid)
1424 {
1425         if (unlikely(!skb_queue_empty(&q->sendq))) {
1426               addq_exit:__skb_queue_tail(&q->sendq, skb);
1427                 return 1;
1428         }
1429         if (unlikely(q->size - q->in_use < ndesc)) {
1430                 struct sge_qset *qs = txq_to_qset(q, qid);
1431 
1432                 set_bit(qid, &qs->txq_stopped);
1433                 smp_mb__after_atomic();
1434 
1435                 if (should_restart_tx(q) &&
1436                     test_and_clear_bit(qid, &qs->txq_stopped))
1437                         return 2;
1438 
1439                 q->stops++;
1440                 goto addq_exit;
1441         }
1442         return 0;
1443 }
1444 
1445 /**
1446  *      reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1447  *      @q: the SGE control Tx queue
1448  *
1449  *      This is a variant of reclaim_completed_tx() that is used for Tx queues
1450  *      that send only immediate data (presently just the control queues) and
1451  *      thus do not have any sk_buffs to release.
1452  */
1453 static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1454 {
1455         unsigned int reclaim = q->processed - q->cleaned;
1456 
1457         q->in_use -= reclaim;
1458         q->cleaned += reclaim;
1459 }
1460 
1461 static inline int immediate(const struct sk_buff *skb)
1462 {
1463         return skb->len <= WR_LEN;
1464 }
1465 
1466 /**
1467  *      ctrl_xmit - send a packet through an SGE control Tx queue
1468  *      @adap: the adapter
1469  *      @q: the control queue
1470  *      @skb: the packet
1471  *
1472  *      Send a packet through an SGE control Tx queue.  Packets sent through
1473  *      a control queue must fit entirely as immediate data in a single Tx
1474  *      descriptor and have no page fragments.
1475  */
1476 static int ctrl_xmit(struct adapter *adap, struct sge_txq *q,
1477                      struct sk_buff *skb)
1478 {
1479         int ret;
1480         struct work_request_hdr *wrp = (struct work_request_hdr *)skb->data;
1481 
1482         if (unlikely(!immediate(skb))) {
1483                 WARN_ON(1);
1484                 dev_kfree_skb(skb);
1485                 return NET_XMIT_SUCCESS;
1486         }
1487 
1488         wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP);
1489         wrp->wr_lo = htonl(V_WR_TID(q->token));
1490 
1491         spin_lock(&q->lock);
1492       again:reclaim_completed_tx_imm(q);
1493 
1494         ret = check_desc_avail(adap, q, skb, 1, TXQ_CTRL);
1495         if (unlikely(ret)) {
1496                 if (ret == 1) {
1497                         spin_unlock(&q->lock);
1498                         return NET_XMIT_CN;
1499                 }
1500                 goto again;
1501         }
1502 
1503         write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1504 
1505         q->in_use++;
1506         if (++q->pidx >= q->size) {
1507                 q->pidx = 0;
1508                 q->gen ^= 1;
1509         }
1510         spin_unlock(&q->lock);
1511         wmb();
1512         t3_write_reg(adap, A_SG_KDOORBELL,
1513                      F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1514         return NET_XMIT_SUCCESS;
1515 }
1516 
1517 /**
1518  *      restart_ctrlq - restart a suspended control queue
1519  *      @qs: the queue set cotaining the control queue
1520  *
1521  *      Resumes transmission on a suspended Tx control queue.
1522  */
1523 static void restart_ctrlq(unsigned long data)
1524 {
1525         struct sk_buff *skb;
1526         struct sge_qset *qs = (struct sge_qset *)data;
1527         struct sge_txq *q = &qs->txq[TXQ_CTRL];
1528 
1529         spin_lock(&q->lock);
1530       again:reclaim_completed_tx_imm(q);
1531 
1532         while (q->in_use < q->size &&
1533                (skb = __skb_dequeue(&q->sendq)) != NULL) {
1534 
1535                 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1536 
1537                 if (++q->pidx >= q->size) {
1538                         q->pidx = 0;
1539                         q->gen ^= 1;
1540                 }
1541                 q->in_use++;
1542         }
1543 
1544         if (!skb_queue_empty(&q->sendq)) {
1545                 set_bit(TXQ_CTRL, &qs->txq_stopped);
1546                 smp_mb__after_atomic();
1547 
1548                 if (should_restart_tx(q) &&
1549                     test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
1550                         goto again;
1551                 q->stops++;
1552         }
1553 
1554         spin_unlock(&q->lock);
1555         wmb();
1556         t3_write_reg(qs->adap, A_SG_KDOORBELL,
1557                      F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1558 }
1559 
1560 /*
1561  * Send a management message through control queue 0
1562  */
1563 int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
1564 {
1565         int ret;
1566         local_bh_disable();
1567         ret = ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], skb);
1568         local_bh_enable();
1569 
1570         return ret;
1571 }
1572 
1573 /**
1574  *      deferred_unmap_destructor - unmap a packet when it is freed
1575  *      @skb: the packet
1576  *
1577  *      This is the packet destructor used for Tx packets that need to remain
1578  *      mapped until they are freed rather than until their Tx descriptors are
1579  *      freed.
1580  */
1581 static void deferred_unmap_destructor(struct sk_buff *skb)
1582 {
1583         int i;
1584         const dma_addr_t *p;
1585         const struct skb_shared_info *si;
1586         const struct deferred_unmap_info *dui;
1587 
1588         dui = (struct deferred_unmap_info *)skb->head;
1589         p = dui->addr;
1590 
1591         if (skb_tail_pointer(skb) - skb_transport_header(skb))
1592                 pci_unmap_single(dui->pdev, *p++, skb_tail_pointer(skb) -
1593                                  skb_transport_header(skb), PCI_DMA_TODEVICE);
1594 
1595         si = skb_shinfo(skb);
1596         for (i = 0; i < si->nr_frags; i++)
1597                 pci_unmap_page(dui->pdev, *p++, skb_frag_size(&si->frags[i]),
1598                                PCI_DMA_TODEVICE);
1599 }
1600 
1601 static void setup_deferred_unmapping(struct sk_buff *skb, struct pci_dev *pdev,
1602                                      const struct sg_ent *sgl, int sgl_flits)
1603 {
1604         dma_addr_t *p;
1605         struct deferred_unmap_info *dui;
1606 
1607         dui = (struct deferred_unmap_info *)skb->head;
1608         dui->pdev = pdev;
1609         for (p = dui->addr; sgl_flits >= 3; sgl++, sgl_flits -= 3) {
1610                 *p++ = be64_to_cpu(sgl->addr[0]);
1611                 *p++ = be64_to_cpu(sgl->addr[1]);
1612         }
1613         if (sgl_flits)
1614                 *p = be64_to_cpu(sgl->addr[0]);
1615 }
1616 
1617 /**
1618  *      write_ofld_wr - write an offload work request
1619  *      @adap: the adapter
1620  *      @skb: the packet to send
1621  *      @q: the Tx queue
1622  *      @pidx: index of the first Tx descriptor to write
1623  *      @gen: the generation value to use
1624  *      @ndesc: number of descriptors the packet will occupy
1625  *
1626  *      Write an offload work request to send the supplied packet.  The packet
1627  *      data already carry the work request with most fields populated.
1628  */
1629 static void write_ofld_wr(struct adapter *adap, struct sk_buff *skb,
1630                           struct sge_txq *q, unsigned int pidx,
1631                           unsigned int gen, unsigned int ndesc,
1632                           const dma_addr_t *addr)
1633 {
1634         unsigned int sgl_flits, flits;
1635         struct work_request_hdr *from;
1636         struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1637         struct tx_desc *d = &q->desc[pidx];
1638 
1639         if (immediate(skb)) {
1640                 q->sdesc[pidx].skb = NULL;
1641                 write_imm(d, skb, skb->len, gen);
1642                 return;
1643         }
1644 
1645         /* Only TX_DATA builds SGLs */
1646 
1647         from = (struct work_request_hdr *)skb->data;
1648         memcpy(&d->flit[1], &from[1],
1649                skb_transport_offset(skb) - sizeof(*from));
1650 
1651         flits = skb_transport_offset(skb) / 8;
1652         sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1653         sgl_flits = write_sgl(skb, sgp, skb_transport_header(skb),
1654                               skb_tail_pointer(skb) - skb_transport_header(skb),
1655                               addr);
1656         if (need_skb_unmap()) {
1657                 setup_deferred_unmapping(skb, adap->pdev, sgp, sgl_flits);
1658                 skb->destructor = deferred_unmap_destructor;
1659         }
1660 
1661         write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits,
1662                          gen, from->wr_hi, from->wr_lo);
1663 }
1664 
1665 /**
1666  *      calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet
1667  *      @skb: the packet
1668  *
1669  *      Returns the number of Tx descriptors needed for the given offload
1670  *      packet.  These packets are already fully constructed.
1671  */
1672 static inline unsigned int calc_tx_descs_ofld(const struct sk_buff *skb)
1673 {
1674         unsigned int flits, cnt;
1675 
1676         if (skb->len <= WR_LEN)
1677                 return 1;       /* packet fits as immediate data */
1678 
1679         flits = skb_transport_offset(skb) / 8;  /* headers */
1680         cnt = skb_shinfo(skb)->nr_frags;
1681         if (skb_tail_pointer(skb) != skb_transport_header(skb))
1682                 cnt++;
1683         return flits_to_desc(flits + sgl_len(cnt));
1684 }
1685 
1686 /**
1687  *      ofld_xmit - send a packet through an offload queue
1688  *      @adap: the adapter
1689  *      @q: the Tx offload queue
1690  *      @skb: the packet
1691  *
1692  *      Send an offload packet through an SGE offload queue.
1693  */
1694 static int ofld_xmit(struct adapter *adap, struct sge_txq *q,
1695                      struct sk_buff *skb)
1696 {
1697         int ret;
1698         unsigned int ndesc = calc_tx_descs_ofld(skb), pidx, gen;
1699 
1700         spin_lock(&q->lock);
1701 again:  reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1702 
1703         ret = check_desc_avail(adap, q, skb, ndesc, TXQ_OFLD);
1704         if (unlikely(ret)) {
1705                 if (ret == 1) {
1706                         skb->priority = ndesc;  /* save for restart */
1707                         spin_unlock(&q->lock);
1708                         return NET_XMIT_CN;
1709                 }
1710                 goto again;
1711         }
1712 
1713         if (!immediate(skb) &&
1714             map_skb(adap->pdev, skb, (dma_addr_t *)skb->head)) {
1715                 spin_unlock(&q->lock);
1716                 return NET_XMIT_SUCCESS;
1717         }
1718 
1719         gen = q->gen;
1720         q->in_use += ndesc;
1721         pidx = q->pidx;
1722         q->pidx += ndesc;
1723         if (q->pidx >= q->size) {
1724                 q->pidx -= q->size;
1725                 q->gen ^= 1;
1726         }
1727         spin_unlock(&q->lock);
1728 
1729         write_ofld_wr(adap, skb, q, pidx, gen, ndesc, (dma_addr_t *)skb->head);
1730         check_ring_tx_db(adap, q);
1731         return NET_XMIT_SUCCESS;
1732 }
1733 
1734 /**
1735  *      restart_offloadq - restart a suspended offload queue
1736  *      @qs: the queue set cotaining the offload queue
1737  *
1738  *      Resumes transmission on a suspended Tx offload queue.
1739  */
1740 static void restart_offloadq(unsigned long data)
1741 {
1742         struct sk_buff *skb;
1743         struct sge_qset *qs = (struct sge_qset *)data;
1744         struct sge_txq *q = &qs->txq[TXQ_OFLD];
1745         const struct port_info *pi = netdev_priv(qs->netdev);
1746         struct adapter *adap = pi->adapter;
1747         unsigned int written = 0;
1748 
1749         spin_lock(&q->lock);
1750 again:  reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1751 
1752         while ((skb = skb_peek(&q->sendq)) != NULL) {
1753                 unsigned int gen, pidx;
1754                 unsigned int ndesc = skb->priority;
1755 
1756                 if (unlikely(q->size - q->in_use < ndesc)) {
1757                         set_bit(TXQ_OFLD, &qs->txq_stopped);
1758                         smp_mb__after_atomic();
1759 
1760                         if (should_restart_tx(q) &&
1761                             test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
1762                                 goto again;
1763                         q->stops++;
1764                         break;
1765                 }
1766 
1767                 if (!immediate(skb) &&
1768                     map_skb(adap->pdev, skb, (dma_addr_t *)skb->head))
1769                         break;
1770 
1771                 gen = q->gen;
1772                 q->in_use += ndesc;
1773                 pidx = q->pidx;
1774                 q->pidx += ndesc;
1775                 written += ndesc;
1776                 if (q->pidx >= q->size) {
1777                         q->pidx -= q->size;
1778                         q->gen ^= 1;
1779                 }
1780                 __skb_unlink(skb, &q->sendq);
1781                 spin_unlock(&q->lock);
1782 
1783                 write_ofld_wr(adap, skb, q, pidx, gen, ndesc,
1784                               (dma_addr_t *)skb->head);
1785                 spin_lock(&q->lock);
1786         }
1787         spin_unlock(&q->lock);
1788 
1789 #if USE_GTS
1790         set_bit(TXQ_RUNNING, &q->flags);
1791         set_bit(TXQ_LAST_PKT_DB, &q->flags);
1792 #endif
1793         wmb();
1794         if (likely(written))
1795                 t3_write_reg(adap, A_SG_KDOORBELL,
1796                              F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1797 }
1798 
1799 /**
1800  *      queue_set - return the queue set a packet should use
1801  *      @skb: the packet
1802  *
1803  *      Maps a packet to the SGE queue set it should use.  The desired queue
1804  *      set is carried in bits 1-3 in the packet's priority.
1805  */
1806 static inline int queue_set(const struct sk_buff *skb)
1807 {
1808         return skb->priority >> 1;
1809 }
1810 
1811 /**
1812  *      is_ctrl_pkt - return whether an offload packet is a control packet
1813  *      @skb: the packet
1814  *
1815  *      Determines whether an offload packet should use an OFLD or a CTRL
1816  *      Tx queue.  This is indicated by bit 0 in the packet's priority.
1817  */
1818 static inline int is_ctrl_pkt(const struct sk_buff *skb)
1819 {
1820         return skb->priority & 1;
1821 }
1822 
1823 /**
1824  *      t3_offload_tx - send an offload packet
1825  *      @tdev: the offload device to send to
1826  *      @skb: the packet
1827  *
1828  *      Sends an offload packet.  We use the packet priority to select the
1829  *      appropriate Tx queue as follows: bit 0 indicates whether the packet
1830  *      should be sent as regular or control, bits 1-3 select the queue set.
1831  */
1832 int t3_offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
1833 {
1834         struct adapter *adap = tdev2adap(tdev);
1835         struct sge_qset *qs = &adap->sge.qs[queue_set(skb)];
1836 
1837         if (unlikely(is_ctrl_pkt(skb)))
1838                 return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], skb);
1839 
1840         return ofld_xmit(adap, &qs->txq[TXQ_OFLD], skb);
1841 }
1842 
1843 /**
1844  *      offload_enqueue - add an offload packet to an SGE offload receive queue
1845  *      @q: the SGE response queue
1846  *      @skb: the packet
1847  *
1848  *      Add a new offload packet to an SGE response queue's offload packet
1849  *      queue.  If the packet is the first on the queue it schedules the RX
1850  *      softirq to process the queue.
1851  */
1852 static inline void offload_enqueue(struct sge_rspq *q, struct sk_buff *skb)
1853 {
1854         int was_empty = skb_queue_empty(&q->rx_queue);
1855 
1856         __skb_queue_tail(&q->rx_queue, skb);
1857 
1858         if (was_empty) {
1859                 struct sge_qset *qs = rspq_to_qset(q);
1860 
1861                 napi_schedule(&qs->napi);
1862         }
1863 }
1864 
1865 /**
1866  *      deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts
1867  *      @tdev: the offload device that will be receiving the packets
1868  *      @q: the SGE response queue that assembled the bundle
1869  *      @skbs: the partial bundle
1870  *      @n: the number of packets in the bundle
1871  *
1872  *      Delivers a (partial) bundle of Rx offload packets to an offload device.
1873  */
1874 static inline void deliver_partial_bundle(struct t3cdev *tdev,
1875                                           struct sge_rspq *q,
1876                                           struct sk_buff *skbs[], int n)
1877 {
1878         if (n) {
1879                 q->offload_bundles++;
1880                 tdev->recv(tdev, skbs, n);
1881         }
1882 }
1883 
1884 /**
1885  *      ofld_poll - NAPI handler for offload packets in interrupt mode
1886  *      @dev: the network device doing the polling
1887  *      @budget: polling budget
1888  *
1889  *      The NAPI handler for offload packets when a response queue is serviced
1890  *      by the hard interrupt handler, i.e., when it's operating in non-polling
1891  *      mode.  Creates small packet batches and sends them through the offload
1892  *      receive handler.  Batches need to be of modest size as we do prefetches
1893  *      on the packets in each.
1894  */
1895 static int ofld_poll(struct napi_struct *napi, int budget)
1896 {
1897         struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
1898         struct sge_rspq *q = &qs->rspq;
1899         struct adapter *adapter = qs->adap;
1900         int work_done = 0;
1901 
1902         while (work_done < budget) {
1903                 struct sk_buff *skb, *tmp, *skbs[RX_BUNDLE_SIZE];
1904                 struct sk_buff_head queue;
1905                 int ngathered;
1906 
1907                 spin_lock_irq(&q->lock);
1908                 __skb_queue_head_init(&queue);
1909                 skb_queue_splice_init(&q->rx_queue, &queue);
1910                 if (skb_queue_empty(&queue)) {
1911                         napi_complete_done(napi, work_done);
1912                         spin_unlock_irq(&q->lock);
1913                         return work_done;
1914                 }
1915                 spin_unlock_irq(&q->lock);
1916 
1917                 ngathered = 0;
1918                 skb_queue_walk_safe(&queue, skb, tmp) {
1919                         if (work_done >= budget)
1920                                 break;
1921                         work_done++;
1922 
1923                         __skb_unlink(skb, &queue);
1924                         prefetch(skb->data);
1925                         skbs[ngathered] = skb;
1926                         if (++ngathered == RX_BUNDLE_SIZE) {
1927                                 q->offload_bundles++;
1928                                 adapter->tdev.recv(&adapter->tdev, skbs,
1929                                                    ngathered);
1930                                 ngathered = 0;
1931                         }
1932                 }
1933                 if (!skb_queue_empty(&queue)) {
1934                         /* splice remaining packets back onto Rx queue */
1935                         spin_lock_irq(&q->lock);
1936                         skb_queue_splice(&queue, &q->rx_queue);
1937                         spin_unlock_irq(&q->lock);
1938                 }
1939                 deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered);
1940         }
1941 
1942         return work_done;
1943 }
1944 
1945 /**
1946  *      rx_offload - process a received offload packet
1947  *      @tdev: the offload device receiving the packet
1948  *      @rq: the response queue that received the packet
1949  *      @skb: the packet
1950  *      @rx_gather: a gather list of packets if we are building a bundle
1951  *      @gather_idx: index of the next available slot in the bundle
1952  *
1953  *      Process an ingress offload pakcet and add it to the offload ingress
1954  *      queue.  Returns the index of the next available slot in the bundle.
1955  */
1956 static inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq,
1957                              struct sk_buff *skb, struct sk_buff *rx_gather[],
1958                              unsigned int gather_idx)
1959 {
1960         skb_reset_mac_header(skb);
1961         skb_reset_network_header(skb);
1962         skb_reset_transport_header(skb);
1963 
1964         if (rq->polling) {
1965                 rx_gather[gather_idx++] = skb;
1966                 if (gather_idx == RX_BUNDLE_SIZE) {
1967                         tdev->recv(tdev, rx_gather, RX_BUNDLE_SIZE);
1968                         gather_idx = 0;
1969                         rq->offload_bundles++;
1970                 }
1971         } else
1972                 offload_enqueue(rq, skb);
1973 
1974         return gather_idx;
1975 }
1976 
1977 /**
1978  *      restart_tx - check whether to restart suspended Tx queues
1979  *      @qs: the queue set to resume
1980  *
1981  *      Restarts suspended Tx queues of an SGE queue set if they have enough
1982  *      free resources to resume operation.
1983  */
1984 static void restart_tx(struct sge_qset *qs)
1985 {
1986         if (test_bit(TXQ_ETH, &qs->txq_stopped) &&
1987             should_restart_tx(&qs->txq[TXQ_ETH]) &&
1988             test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1989                 qs->txq[TXQ_ETH].restarts++;
1990                 if (netif_running(qs->netdev))
1991                         netif_tx_wake_queue(qs->tx_q);
1992         }
1993 
1994         if (test_bit(TXQ_OFLD, &qs->txq_stopped) &&
1995             should_restart_tx(&qs->txq[TXQ_OFLD]) &&
1996             test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
1997                 qs->txq[TXQ_OFLD].restarts++;
1998                 tasklet_schedule(&qs->txq[TXQ_OFLD].qresume_tsk);
1999         }
2000         if (test_bit(TXQ_CTRL, &qs->txq_stopped) &&
2001             should_restart_tx(&qs->txq[TXQ_CTRL]) &&
2002             test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
2003                 qs->txq[TXQ_CTRL].restarts++;
2004                 tasklet_schedule(&qs->txq[TXQ_CTRL].qresume_tsk);
2005         }
2006 }
2007 
2008 /**
2009  *      cxgb3_arp_process - process an ARP request probing a private IP address
2010  *      @adapter: the adapter
2011  *      @skb: the skbuff containing the ARP request
2012  *
2013  *      Check if the ARP request is probing the private IP address
2014  *      dedicated to iSCSI, generate an ARP reply if so.
2015  */
2016 static void cxgb3_arp_process(struct port_info *pi, struct sk_buff *skb)
2017 {
2018         struct net_device *dev = skb->dev;
2019         struct arphdr *arp;
2020         unsigned char *arp_ptr;
2021         unsigned char *sha;
2022         __be32 sip, tip;
2023 
2024         if (!dev)
2025                 return;
2026 
2027         skb_reset_network_header(skb);
2028         arp = arp_hdr(skb);
2029 
2030         if (arp->ar_op != htons(ARPOP_REQUEST))
2031                 return;
2032 
2033         arp_ptr = (unsigned char *)(arp + 1);
2034         sha = arp_ptr;
2035         arp_ptr += dev->addr_len;
2036         memcpy(&sip, arp_ptr, sizeof(sip));
2037         arp_ptr += sizeof(sip);
2038         arp_ptr += dev->addr_len;
2039         memcpy(&tip, arp_ptr, sizeof(tip));
2040 
2041         if (tip != pi->iscsi_ipv4addr)
2042                 return;
2043 
2044         arp_send(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha,
2045                  pi->iscsic.mac_addr, sha);
2046 
2047 }
2048 
2049 static inline int is_arp(struct sk_buff *skb)
2050 {
2051         return skb->protocol == htons(ETH_P_ARP);
2052 }
2053 
2054 static void cxgb3_process_iscsi_prov_pack(struct port_info *pi,
2055                                         struct sk_buff *skb)
2056 {
2057         if (is_arp(skb)) {
2058                 cxgb3_arp_process(pi, skb);
2059                 return;
2060         }
2061 
2062         if (pi->iscsic.recv)
2063                 pi->iscsic.recv(pi, skb);
2064 
2065 }
2066 
2067 /**
2068  *      rx_eth - process an ingress ethernet packet
2069  *      @adap: the adapter
2070  *      @rq: the response queue that received the packet
2071  *      @skb: the packet
2072  *      @pad: amount of padding at the start of the buffer
2073  *
2074  *      Process an ingress ethernet pakcet and deliver it to the stack.
2075  *      The padding is 2 if the packet was delivered in an Rx buffer and 0
2076  *      if it was immediate data in a response.
2077  */
2078 static void rx_eth(struct adapter *adap, struct sge_rspq *rq,
2079                    struct sk_buff *skb, int pad, int lro)
2080 {
2081         struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)(skb->data + pad);
2082         struct sge_qset *qs = rspq_to_qset(rq);
2083         struct port_info *pi;
2084 
2085         skb_pull(skb, sizeof(*p) + pad);
2086         skb->protocol = eth_type_trans(skb, adap->port[p->iff]);
2087         pi = netdev_priv(skb->dev);
2088         if ((skb->dev->features & NETIF_F_RXCSUM) && p->csum_valid &&
2089             p->csum == htons(0xffff) && !p->fragment) {
2090                 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2091                 skb->ip_summed = CHECKSUM_UNNECESSARY;
2092         } else
2093                 skb_checksum_none_assert(skb);
2094         skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);
2095 
2096         if (p->vlan_valid) {
2097                 qs->port_stats[SGE_PSTAT_VLANEX]++;
2098                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(p->vlan));
2099         }
2100         if (rq->polling) {
2101                 if (lro)
2102                         napi_gro_receive(&qs->napi, skb);
2103                 else {
2104                         if (unlikely(pi->iscsic.flags))
2105                                 cxgb3_process_iscsi_prov_pack(pi, skb);
2106                         netif_receive_skb(skb);
2107                 }
2108         } else
2109                 netif_rx(skb);
2110 }
2111 
2112 static inline int is_eth_tcp(u32 rss)
2113 {
2114         return G_HASHTYPE(ntohl(rss)) == RSS_HASH_4_TUPLE;
2115 }
2116 
2117 /**
2118  *      lro_add_page - add a page chunk to an LRO session
2119  *      @adap: the adapter
2120  *      @qs: the associated queue set
2121  *      @fl: the free list containing the page chunk to add
2122  *      @len: packet length
2123  *      @complete: Indicates the last fragment of a frame
2124  *
2125  *      Add a received packet contained in a page chunk to an existing LRO
2126  *      session.
2127  */
2128 static void lro_add_page(struct adapter *adap, struct sge_qset *qs,
2129                          struct sge_fl *fl, int len, int complete)
2130 {
2131         struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
2132         struct port_info *pi = netdev_priv(qs->netdev);
2133         struct sk_buff *skb = NULL;
2134         struct cpl_rx_pkt *cpl;
2135         skb_frag_t *rx_frag;
2136         int nr_frags;
2137         int offset = 0;
2138 
2139         if (!qs->nomem) {
2140                 skb = napi_get_frags(&qs->napi);
2141                 qs->nomem = !skb;
2142         }
2143 
2144         fl->credits--;
2145 
2146         pci_dma_sync_single_for_cpu(adap->pdev,
2147                                     dma_unmap_addr(sd, dma_addr),
2148                                     fl->buf_size - SGE_PG_RSVD,
2149                                     PCI_DMA_FROMDEVICE);
2150 
2151         (*sd->pg_chunk.p_cnt)--;
2152         if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
2153                 pci_unmap_page(adap->pdev,
2154                                sd->pg_chunk.mapping,
2155                                fl->alloc_size,
2156                                PCI_DMA_FROMDEVICE);
2157 
2158         if (!skb) {
2159                 put_page(sd->pg_chunk.page);
2160                 if (complete)
2161                         qs->nomem = 0;
2162                 return;
2163         }
2164 
2165         rx_frag = skb_shinfo(skb)->frags;
2166         nr_frags = skb_shinfo(skb)->nr_frags;
2167 
2168         if (!nr_frags) {
2169                 offset = 2 + sizeof(struct cpl_rx_pkt);
2170                 cpl = qs->lro_va = sd->pg_chunk.va + 2;
2171 
2172                 if ((qs->netdev->features & NETIF_F_RXCSUM) &&
2173                      cpl->csum_valid && cpl->csum == htons(0xffff)) {
2174                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2175                         qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2176                 } else
2177                         skb->ip_summed = CHECKSUM_NONE;
2178         } else
2179                 cpl = qs->lro_va;
2180 
2181         len -= offset;
2182 
2183         rx_frag += nr_frags;
2184         __skb_frag_set_page(rx_frag, sd->pg_chunk.page);
2185         skb_frag_off_set(rx_frag, sd->pg_chunk.offset + offset);
2186         skb_frag_size_set(rx_frag, len);
2187 
2188         skb->len += len;
2189         skb->data_len += len;
2190         skb->truesize += len;
2191         skb_shinfo(skb)->nr_frags++;
2192 
2193         if (!complete)
2194                 return;
2195 
2196         skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);
2197 
2198         if (cpl->vlan_valid) {
2199                 qs->port_stats[SGE_PSTAT_VLANEX]++;
2200                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(cpl->vlan));
2201         }
2202         napi_gro_frags(&qs->napi);
2203 }
2204 
2205 /**
2206  *      handle_rsp_cntrl_info - handles control information in a response
2207  *      @qs: the queue set corresponding to the response
2208  *      @flags: the response control flags
2209  *
2210  *      Handles the control information of an SGE response, such as GTS
2211  *      indications and completion credits for the queue set's Tx queues.
2212  *      HW coalesces credits, we don't do any extra SW coalescing.
2213  */
2214 static inline void handle_rsp_cntrl_info(struct sge_qset *qs, u32 flags)
2215 {
2216         unsigned int credits;
2217 
2218 #if USE_GTS
2219         if (flags & F_RSPD_TXQ0_GTS)
2220                 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
2221 #endif
2222 
2223         credits = G_RSPD_TXQ0_CR(flags);
2224         if (credits)
2225                 qs->txq[TXQ_ETH].processed += credits;
2226 
2227         credits = G_RSPD_TXQ2_CR(flags);
2228         if (credits)
2229                 qs->txq[TXQ_CTRL].processed += credits;
2230 
2231 # if USE_GTS
2232         if (flags & F_RSPD_TXQ1_GTS)
2233                 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);
2234 # endif
2235         credits = G_RSPD_TXQ1_CR(flags);
2236         if (credits)
2237                 qs->txq[TXQ_OFLD].processed += credits;
2238 }
2239 
2240 /**
2241  *      check_ring_db - check if we need to ring any doorbells
2242  *      @adapter: the adapter
2243  *      @qs: the queue set whose Tx queues are to be examined
2244  *      @sleeping: indicates which Tx queue sent GTS
2245  *
2246  *      Checks if some of a queue set's Tx queues need to ring their doorbells
2247  *      to resume transmission after idling while they still have unprocessed
2248  *      descriptors.
2249  */
2250 static void check_ring_db(struct adapter *adap, struct sge_qset *qs,
2251                           unsigned int sleeping)
2252 {
2253         if (sleeping & F_RSPD_TXQ0_GTS) {
2254                 struct sge_txq *txq = &qs->txq[TXQ_ETH];
2255 
2256                 if (txq->cleaned + txq->in_use != txq->processed &&
2257                     !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2258                         set_bit(TXQ_RUNNING, &txq->flags);
2259                         t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2260                                      V_EGRCNTX(txq->cntxt_id));
2261                 }
2262         }
2263 
2264         if (sleeping & F_RSPD_TXQ1_GTS) {
2265                 struct sge_txq *txq = &qs->txq[TXQ_OFLD];
2266 
2267                 if (txq->cleaned + txq->in_use != txq->processed &&
2268                     !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2269                         set_bit(TXQ_RUNNING, &txq->flags);
2270                         t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2271                                      V_EGRCNTX(txq->cntxt_id));
2272                 }
2273         }
2274 }
2275 
2276 /**
2277  *      is_new_response - check if a response is newly written
2278  *      @r: the response descriptor
2279  *      @q: the response queue
2280  *
2281  *      Returns true if a response descriptor contains a yet unprocessed
2282  *      response.
2283  */
2284 static inline int is_new_response(const struct rsp_desc *r,
2285                                   const struct sge_rspq *q)
2286 {
2287         return (r->intr_gen & F_RSPD_GEN2) == q->gen;
2288 }
2289 
2290 static inline void clear_rspq_bufstate(struct sge_rspq * const q)
2291 {
2292         q->pg_skb = NULL;
2293         q->rx_recycle_buf = 0;
2294 }
2295 
2296 #define RSPD_GTS_MASK  (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
2297 #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
2298                         V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
2299                         V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
2300                         V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
2301 
2302 /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */
2303 #define NOMEM_INTR_DELAY 2500
2304 
2305 /**
2306  *      process_responses - process responses from an SGE response queue
2307  *      @adap: the adapter
2308  *      @qs: the queue set to which the response queue belongs
2309  *      @budget: how many responses can be processed in this round
2310  *
2311  *      Process responses from an SGE response queue up to the supplied budget.
2312  *      Responses include received packets as well as credits and other events
2313  *      for the queues that belong to the response queue's queue set.
2314  *      A negative budget is effectively unlimited.
2315  *
2316  *      Additionally choose the interrupt holdoff time for the next interrupt
2317  *      on this queue.  If the system is under memory shortage use a fairly
2318  *      long delay to help recovery.
2319  */
2320 static int process_responses(struct adapter *adap, struct sge_qset *qs,
2321                              int budget)
2322 {
2323         struct sge_rspq *q = &qs->rspq;
2324         struct rsp_desc *r = &q->desc[q->cidx];
2325         int budget_left = budget;
2326         unsigned int sleeping = 0;
2327         struct sk_buff *offload_skbs[RX_BUNDLE_SIZE];
2328         int ngathered = 0;
2329 
2330         q->next_holdoff = q->holdoff_tmr;
2331 
2332         while (likely(budget_left && is_new_response(r, q))) {
2333                 int packet_complete, eth, ethpad = 2;
2334                 int lro = !!(qs->netdev->features & NETIF_F_GRO);
2335                 struct sk_buff *skb = NULL;
2336                 u32 len, flags;
2337                 __be32 rss_hi, rss_lo;
2338 
2339                 dma_rmb();
2340                 eth = r->rss_hdr.opcode == CPL_RX_PKT;
2341                 rss_hi = *(const __be32 *)r;
2342                 rss_lo = r->rss_hdr.rss_hash_val;
2343                 flags = ntohl(r->flags);
2344 
2345                 if (unlikely(flags & F_RSPD_ASYNC_NOTIF)) {
2346                         skb = alloc_skb(AN_PKT_SIZE, GFP_ATOMIC);
2347                         if (!skb)
2348                                 goto no_mem;
2349 
2350                         __skb_put_data(skb, r, AN_PKT_SIZE);
2351                         skb->data[0] = CPL_ASYNC_NOTIF;
2352                         rss_hi = htonl(CPL_ASYNC_NOTIF << 24);
2353                         q->async_notif++;
2354                 } else if (flags & F_RSPD_IMM_DATA_VALID) {
2355                         skb = get_imm_packet(r);
2356                         if (unlikely(!skb)) {
2357 no_mem:
2358                                 q->next_holdoff = NOMEM_INTR_DELAY;
2359                                 q->nomem++;
2360                                 /* consume one credit since we tried */
2361                                 budget_left--;
2362                                 break;
2363                         }
2364                         q->imm_data++;
2365                         ethpad = 0;
2366                 } else if ((len = ntohl(r->len_cq)) != 0) {
2367                         struct sge_fl *fl;
2368 
2369                         lro &= eth && is_eth_tcp(rss_hi);
2370 
2371                         fl = (len & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];
2372                         if (fl->use_pages) {
2373                                 void *addr = fl->sdesc[fl->cidx].pg_chunk.va;
2374 
2375                                 prefetch(addr);
2376 #if L1_CACHE_BYTES < 128
2377                                 prefetch(addr + L1_CACHE_BYTES);
2378 #endif
2379                                 __refill_fl(adap, fl);
2380                                 if (lro > 0) {
2381                                         lro_add_page(adap, qs, fl,
2382                                                      G_RSPD_LEN(len),
2383                                                      flags & F_RSPD_EOP);
2384                                         goto next_fl;
2385                                 }
2386 
2387                                 skb = get_packet_pg(adap, fl, q,
2388                                                     G_RSPD_LEN(len),
2389                                                     eth ?
2390                                                     SGE_RX_DROP_THRES : 0);
2391                                 q->pg_skb = skb;
2392                         } else
2393                                 skb = get_packet(adap, fl, G_RSPD_LEN(len),
2394                                                  eth ? SGE_RX_DROP_THRES : 0);
2395                         if (unlikely(!skb)) {
2396                                 if (!eth)
2397                                         goto no_mem;
2398                                 q->rx_drops++;
2399                         } else if (unlikely(r->rss_hdr.opcode == CPL_TRACE_PKT))
2400                                 __skb_pull(skb, 2);
2401 next_fl:
2402                         if (++fl->cidx == fl->size)
2403                                 fl->cidx = 0;
2404                 } else
2405                         q->pure_rsps++;
2406 
2407                 if (flags & RSPD_CTRL_MASK) {
2408                         sleeping |= flags & RSPD_GTS_MASK;
2409                         handle_rsp_cntrl_info(qs, flags);
2410                 }
2411 
2412                 r++;
2413                 if (unlikely(++q->cidx == q->size)) {
2414                         q->cidx = 0;
2415                         q->gen ^= 1;
2416                         r = q->desc;
2417                 }
2418                 prefetch(r);
2419 
2420                 if (++q->credits >= (q->size / 4)) {
2421                         refill_rspq(adap, q, q->credits);
2422                         q->credits = 0;
2423                 }
2424 
2425                 packet_complete = flags &
2426                                   (F_RSPD_EOP | F_RSPD_IMM_DATA_VALID |
2427                                    F_RSPD_ASYNC_NOTIF);
2428 
2429                 if (skb != NULL && packet_complete) {
2430                         if (eth)
2431                                 rx_eth(adap, q, skb, ethpad, lro);
2432                         else {
2433                                 q->offload_pkts++;
2434                                 /* Preserve the RSS info in csum & priority */
2435                                 skb->csum = rss_hi;
2436                                 skb->priority = rss_lo;
2437                                 ngathered = rx_offload(&adap->tdev, q, skb,
2438                                                        offload_skbs,
2439                                                        ngathered);
2440                         }
2441 
2442                         if (flags & F_RSPD_EOP)
2443                                 clear_rspq_bufstate(q);
2444                 }
2445                 --budget_left;
2446         }
2447 
2448         deliver_partial_bundle(&adap->tdev, q, offload_skbs, ngathered);
2449 
2450         if (sleeping)
2451                 check_ring_db(adap, qs, sleeping);
2452 
2453         smp_mb();               /* commit Tx queue .processed updates */
2454         if (unlikely(qs->txq_stopped != 0))
2455                 restart_tx(qs);
2456 
2457         budget -= budget_left;
2458         return budget;
2459 }
2460 
2461 static inline int is_pure_response(const struct rsp_desc *r)
2462 {
2463         __be32 n = r->flags & htonl(F_RSPD_ASYNC_NOTIF | F_RSPD_IMM_DATA_VALID);
2464 
2465         return (n | r->len_cq) == 0;
2466 }
2467 
2468 /**
2469  *      napi_rx_handler - the NAPI handler for Rx processing
2470  *      @napi: the napi instance
2471  *      @budget: how many packets we can process in this round
2472  *
2473  *      Handler for new data events when using NAPI.
2474  */
2475 static int napi_rx_handler(struct napi_struct *napi, int budget)
2476 {
2477         struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
2478         struct adapter *adap = qs->adap;
2479         int work_done = process_responses(adap, qs, budget);
2480 
2481         if (likely(work_done < budget)) {
2482                 napi_complete_done(napi, work_done);
2483 
2484                 /*
2485                  * Because we don't atomically flush the following
2486                  * write it is possible that in very rare cases it can
2487                  * reach the device in a way that races with a new
2488                  * response being written plus an error interrupt
2489                  * causing the NAPI interrupt handler below to return
2490                  * unhandled status to the OS.  To protect against
2491                  * this would require flushing the write and doing
2492                  * both the write and the flush with interrupts off.
2493                  * Way too expensive and unjustifiable given the
2494                  * rarity of the race.
2495                  *
2496                  * The race cannot happen at all with MSI-X.
2497                  */
2498                 t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |
2499                              V_NEWTIMER(qs->rspq.next_holdoff) |
2500                              V_NEWINDEX(qs->rspq.cidx));
2501         }
2502         return work_done;
2503 }
2504 
2505 /*
2506  * Returns true if the device is already scheduled for polling.
2507  */
2508 static inline int napi_is_scheduled(struct napi_struct *napi)
2509 {
2510         return test_bit(NAPI_STATE_SCHED, &napi->state);
2511 }
2512 
2513 /**
2514  *      process_pure_responses - process pure responses from a response queue
2515  *      @adap: the adapter
2516  *      @qs: the queue set owning the response queue
2517  *      @r: the first pure response to process
2518  *
2519  *      A simpler version of process_responses() that handles only pure (i.e.,
2520  *      non data-carrying) responses.  Such respones are too light-weight to
2521  *      justify calling a softirq under NAPI, so we handle them specially in
2522  *      the interrupt handler.  The function is called with a pointer to a
2523  *      response, which the caller must ensure is a valid pure response.
2524  *
2525  *      Returns 1 if it encounters a valid data-carrying response, 0 otherwise.
2526  */
2527 static int process_pure_responses(struct adapter *adap, struct sge_qset *qs,
2528                                   struct rsp_desc *r)
2529 {
2530         struct sge_rspq *q = &qs->rspq;
2531         unsigned int sleeping = 0;
2532 
2533         do {
2534                 u32 flags = ntohl(r->flags);
2535 
2536                 r++;
2537                 if (unlikely(++q->cidx == q->size)) {
2538                         q->cidx = 0;
2539                         q->gen ^= 1;
2540                         r = q->desc;
2541                 }
2542                 prefetch(r);
2543 
2544                 if (flags & RSPD_CTRL_MASK) {
2545                         sleeping |= flags & RSPD_GTS_MASK;
2546                         handle_rsp_cntrl_info(qs, flags);
2547                 }
2548 
2549                 q->pure_rsps++;
2550                 if (++q->credits >= (q->size / 4)) {
2551                         refill_rspq(adap, q, q->credits);
2552                         q->credits = 0;
2553                 }
2554                 if (!is_new_response(r, q))
2555                         break;
2556                 dma_rmb();
2557         } while (is_pure_response(r));
2558 
2559         if (sleeping)
2560                 check_ring_db(adap, qs, sleeping);
2561 
2562         smp_mb();               /* commit Tx queue .processed updates */
2563         if (unlikely(qs->txq_stopped != 0))
2564                 restart_tx(qs);
2565 
2566         return is_new_response(r, q);
2567 }
2568 
2569 /**
2570  *      handle_responses - decide what to do with new responses in NAPI mode
2571  *      @adap: the adapter
2572  *      @q: the response queue
2573  *
2574  *      This is used by the NAPI interrupt handlers to decide what to do with
2575  *      new SGE responses.  If there are no new responses it returns -1.  If
2576  *      there are new responses and they are pure (i.e., non-data carrying)
2577  *      it handles them straight in hard interrupt context as they are very
2578  *      cheap and don't deliver any packets.  Finally, if there are any data
2579  *      signaling responses it schedules the NAPI handler.  Returns 1 if it
2580  *      schedules NAPI, 0 if all new responses were pure.
2581  *
2582  *      The caller must ascertain NAPI is not already running.
2583  */
2584 static inline int handle_responses(struct adapter *adap, struct sge_rspq *q)
2585 {
2586         struct sge_qset *qs = rspq_to_qset(q);
2587         struct rsp_desc *r = &q->desc[q->cidx];
2588 
2589         if (!is_new_response(r, q))
2590                 return -1;
2591         dma_rmb();
2592         if (is_pure_response(r) && process_pure_responses(adap, qs, r) == 0) {
2593                 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2594                              V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx));
2595                 return 0;
2596         }
2597         napi_schedule(&qs->napi);
2598         return 1;
2599 }
2600 
2601 /*
2602  * The MSI-X interrupt handler for an SGE response queue for the non-NAPI case
2603  * (i.e., response queue serviced in hard interrupt).
2604  */
2605 static irqreturn_t t3_sge_intr_msix(int irq, void *cookie)
2606 {
2607         struct sge_qset *qs = cookie;
2608         struct adapter *adap = qs->adap;
2609         struct sge_rspq *q = &qs->rspq;
2610 
2611         spin_lock(&q->lock);
2612         if (process_responses(adap, qs, -1) == 0)
2613                 q->unhandled_irqs++;
2614         t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2615                      V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2616         spin_unlock(&q->lock);
2617         return IRQ_HANDLED;
2618 }
2619 
2620 /*
2621  * The MSI-X interrupt handler for an SGE response queue for the NAPI case
2622  * (i.e., response queue serviced by NAPI polling).
2623  */
2624 static irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie)
2625 {
2626         struct sge_qset *qs = cookie;
2627         struct sge_rspq *q = &qs->rspq;
2628 
2629         spin_lock(&q->lock);
2630 
2631         if (handle_responses(qs->adap, q) < 0)
2632                 q->unhandled_irqs++;
2633         spin_unlock(&q->lock);
2634         return IRQ_HANDLED;
2635 }
2636 
2637 /*
2638  * The non-NAPI MSI interrupt handler.  This needs to handle data events from
2639  * SGE response queues as well as error and other async events as they all use
2640  * the same MSI vector.  We use one SGE response queue per port in this mode
2641  * and protect all response queues with queue 0's lock.
2642  */
2643 static irqreturn_t t3_intr_msi(int irq, void *cookie)
2644 {
2645         int new_packets = 0;
2646         struct adapter *adap = cookie;
2647         struct sge_rspq *q = &adap->sge.qs[0].rspq;
2648 
2649         spin_lock(&q->lock);
2650 
2651         if (process_responses(adap, &adap->sge.qs[0], -1)) {
2652                 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2653                              V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2654                 new_packets = 1;
2655         }
2656 
2657         if (adap->params.nports == 2 &&
2658             process_responses(adap, &adap->sge.qs[1], -1)) {
2659                 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2660 
2661                 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q1->cntxt_id) |
2662                              V_NEWTIMER(q1->next_holdoff) |
2663                              V_NEWINDEX(q1->cidx));
2664                 new_packets = 1;
2665         }
2666 
2667         if (!new_packets && t3_slow_intr_handler(adap) == 0)
2668                 q->unhandled_irqs++;
2669 
2670         spin_unlock(&q->lock);
2671         return IRQ_HANDLED;
2672 }
2673 
2674 static int rspq_check_napi(struct sge_qset *qs)
2675 {
2676         struct sge_rspq *q = &qs->rspq;
2677 
2678         if (!napi_is_scheduled(&qs->napi) &&
2679             is_new_response(&q->desc[q->cidx], q)) {
2680                 napi_schedule(&qs->napi);
2681                 return 1;
2682         }
2683         return 0;
2684 }
2685 
2686 /*
2687  * The MSI interrupt handler for the NAPI case (i.e., response queues serviced
2688  * by NAPI polling).  Handles data events from SGE response queues as well as
2689  * error and other async events as they all use the same MSI vector.  We use
2690  * one SGE response queue per port in this mode and protect all response
2691  * queues with queue 0's lock.
2692  */
2693 static irqreturn_t t3_intr_msi_napi(int irq, void *cookie)
2694 {
2695         int new_packets;
2696         struct adapter *adap = cookie;
2697         struct sge_rspq *q = &adap->sge.qs[0].rspq;
2698 
2699         spin_lock(&q->lock);
2700 
2701         new_packets = rspq_check_napi(&adap->sge.qs[0]);
2702         if (adap->params.nports == 2)
2703                 new_packets += rspq_check_napi(&adap->sge.qs[1]);
2704         if (!new_packets && t3_slow_intr_handler(adap) == 0)
2705                 q->unhandled_irqs++;
2706 
2707         spin_unlock(&q->lock);
2708         return IRQ_HANDLED;
2709 }
2710 
2711 /*
2712  * A helper function that processes responses and issues GTS.
2713  */
2714 static inline int process_responses_gts(struct adapter *adap,
2715                                         struct sge_rspq *rq)
2716 {
2717         int work;
2718 
2719         work = process_responses(adap, rspq_to_qset(rq), -1);
2720         t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) |
2721                      V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx));
2722         return work;
2723 }
2724 
2725 /*
2726  * The legacy INTx interrupt handler.  This needs to handle data events from
2727  * SGE response queues as well as error and other async events as they all use
2728  * the same interrupt pin.  We use one SGE response queue per port in this mode
2729  * and protect all response queues with queue 0's lock.
2730  */
2731 static irqreturn_t t3_intr(int irq, void *cookie)
2732 {
2733         int work_done, w0, w1;
2734         struct adapter *adap = cookie;
2735         struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2736         struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2737 
2738         spin_lock(&q0->lock);
2739 
2740         w0 = is_new_response(&q0->desc[q0->cidx], q0);
2741         w1 = adap->params.nports == 2 &&
2742             is_new_response(&q1->desc[q1->cidx], q1);
2743 
2744         if (likely(w0 | w1)) {
2745                 t3_write_reg(adap, A_PL_CLI, 0);
2746                 t3_read_reg(adap, A_PL_CLI);    /* flush */
2747 
2748                 if (likely(w0))
2749                         process_responses_gts(adap, q0);
2750 
2751                 if (w1)
2752                         process_responses_gts(adap, q1);
2753 
2754                 work_done = w0 | w1;
2755         } else
2756                 work_done = t3_slow_intr_handler(adap);
2757 
2758         spin_unlock(&q0->lock);
2759         return IRQ_RETVAL(work_done != 0);
2760 }
2761 
2762 /*
2763  * Interrupt handler for legacy INTx interrupts for T3B-based cards.
2764  * Handles data events from SGE response queues as well as error and other
2765  * async events as they all use the same interrupt pin.  We use one SGE
2766  * response queue per port in this mode and protect all response queues with
2767  * queue 0's lock.
2768  */
2769 static irqreturn_t t3b_intr(int irq, void *cookie)
2770 {
2771         u32 map;
2772         struct adapter *adap = cookie;
2773         struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2774 
2775         t3_write_reg(adap, A_PL_CLI, 0);
2776         map = t3_read_reg(adap, A_SG_DATA_INTR);
2777 
2778         if (unlikely(!map))     /* shared interrupt, most likely */
2779                 return IRQ_NONE;
2780 
2781         spin_lock(&q0->lock);
2782 
2783         if (unlikely(map & F_ERRINTR))
2784                 t3_slow_intr_handler(adap);
2785 
2786         if (likely(map & 1))
2787                 process_responses_gts(adap, q0);
2788 
2789         if (map & 2)
2790                 process_responses_gts(adap, &adap->sge.qs[1].rspq);
2791 
2792         spin_unlock(&q0->lock);
2793         return IRQ_HANDLED;
2794 }
2795 
2796 /*
2797  * NAPI interrupt handler for legacy INTx interrupts for T3B-based cards.
2798  * Handles data events from SGE response queues as well as error and other
2799  * async events as they all use the same interrupt pin.  We use one SGE
2800  * response queue per port in this mode and protect all response queues with
2801  * queue 0's lock.
2802  */
2803 static irqreturn_t t3b_intr_napi(int irq, void *cookie)
2804 {
2805         u32 map;
2806         struct adapter *adap = cookie;
2807         struct sge_qset *qs0 = &adap->sge.qs[0];
2808         struct sge_rspq *q0 = &qs0->rspq;
2809 
2810         t3_write_reg(adap, A_PL_CLI, 0);
2811         map = t3_read_reg(adap, A_SG_DATA_INTR);
2812 
2813         if (unlikely(!map))     /* shared interrupt, most likely */
2814                 return IRQ_NONE;
2815 
2816         spin_lock(&q0->lock);
2817 
2818         if (unlikely(map & F_ERRINTR))
2819                 t3_slow_intr_handler(adap);
2820 
2821         if (likely(map & 1))
2822                 napi_schedule(&qs0->napi);
2823 
2824         if (map & 2)
2825                 napi_schedule(&adap->sge.qs[1].napi);
2826 
2827         spin_unlock(&q0->lock);
2828         return IRQ_HANDLED;
2829 }
2830 
2831 /**
2832  *      t3_intr_handler - select the top-level interrupt handler
2833  *      @adap: the adapter
2834  *      @polling: whether using NAPI to service response queues
2835  *
2836  *      Selects the top-level interrupt handler based on the type of interrupts
2837  *      (MSI-X, MSI, or legacy) and whether NAPI will be used to service the
2838  *      response queues.
2839  */
2840 irq_handler_t t3_intr_handler(struct adapter *adap, int polling)
2841 {
2842         if (adap->flags & USING_MSIX)
2843                 return polling ? t3_sge_intr_msix_napi : t3_sge_intr_msix;
2844         if (adap->flags & USING_MSI)
2845                 return polling ? t3_intr_msi_napi : t3_intr_msi;
2846         if (adap->params.rev > 0)
2847                 return polling ? t3b_intr_napi : t3b_intr;
2848         return t3_intr;
2849 }
2850 
2851 #define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
2852                     F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
2853                     V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
2854                     F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
2855                     F_HIRCQPARITYERROR)
2856 #define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
2857 #define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
2858                       F_RSPQDISABLED)
2859 
2860 /**
2861  *      t3_sge_err_intr_handler - SGE async event interrupt handler
2862  *      @adapter: the adapter
2863  *
2864  *      Interrupt handler for SGE asynchronous (non-data) events.
2865  */
2866 void t3_sge_err_intr_handler(struct adapter *adapter)
2867 {
2868         unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE) &
2869                                  ~F_FLEMPTY;
2870 
2871         if (status & SGE_PARERR)
2872                 CH_ALERT(adapter, "SGE parity error (0x%x)\n",
2873                          status & SGE_PARERR);
2874         if (status & SGE_FRAMINGERR)
2875                 CH_ALERT(adapter, "SGE framing error (0x%x)\n",
2876                          status & SGE_FRAMINGERR);
2877 
2878         if (status & F_RSPQCREDITOVERFOW)
2879                 CH_ALERT(adapter, "SGE response queue credit overflow\n");
2880 
2881         if (status & F_RSPQDISABLED) {
2882                 v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS);
2883 
2884                 CH_ALERT(adapter,
2885                          "packet delivered to disabled response queue "
2886                          "(0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff);
2887         }
2888 
2889         if (status & (F_HIPIODRBDROPERR | F_LOPIODRBDROPERR))
2890                 queue_work(cxgb3_wq, &adapter->db_drop_task);
2891 
2892         if (status & (F_HIPRIORITYDBFULL | F_LOPRIORITYDBFULL))
2893                 queue_work(cxgb3_wq, &adapter->db_full_task);
2894 
2895         if (status & (F_HIPRIORITYDBEMPTY | F_LOPRIORITYDBEMPTY))
2896                 queue_work(cxgb3_wq, &adapter->db_empty_task);
2897 
2898         t3_write_reg(adapter, A_SG_INT_CAUSE, status);
2899         if (status &  SGE_FATALERR)
2900                 t3_fatal_err(adapter);
2901 }
2902 
2903 /**
2904  *      sge_timer_tx - perform periodic maintenance of an SGE qset
2905  *      @data: the SGE queue set to maintain
2906  *
2907  *      Runs periodically from a timer to perform maintenance of an SGE queue
2908  *      set.  It performs two tasks:
2909  *
2910  *      Cleans up any completed Tx descriptors that may still be pending.
2911  *      Normal descriptor cleanup happens when new packets are added to a Tx
2912  *      queue so this timer is relatively infrequent and does any cleanup only
2913  *      if the Tx queue has not seen any new packets in a while.  We make a
2914  *      best effort attempt to reclaim descriptors, in that we don't wait
2915  *      around if we cannot get a queue's lock (which most likely is because
2916  *      someone else is queueing new packets and so will also handle the clean
2917  *      up).  Since control queues use immediate data exclusively we don't
2918  *      bother cleaning them up here.
2919  *
2920  */
2921 static void sge_timer_tx(struct timer_list *t)
2922 {
2923         struct sge_qset *qs = from_timer(qs, t, tx_reclaim_timer);
2924         struct port_info *pi = netdev_priv(qs->netdev);
2925         struct adapter *adap = pi->adapter;
2926         unsigned int tbd[SGE_TXQ_PER_SET] = {0, 0};
2927         unsigned long next_period;
2928 
2929         if (__netif_tx_trylock(qs->tx_q)) {
2930                 tbd[TXQ_ETH] = reclaim_completed_tx(adap, &qs->txq[TXQ_ETH],
2931                                                      TX_RECLAIM_TIMER_CHUNK);
2932                 __netif_tx_unlock(qs->tx_q);
2933         }
2934 
2935         if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {
2936                 tbd[TXQ_OFLD] = reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD],
2937                                                      TX_RECLAIM_TIMER_CHUNK);
2938                 spin_unlock(&qs->txq[TXQ_OFLD].lock);
2939         }
2940 
2941         next_period = TX_RECLAIM_PERIOD >>
2942                       (max(tbd[TXQ_ETH], tbd[TXQ_OFLD]) /
2943                       TX_RECLAIM_TIMER_CHUNK);
2944         mod_timer(&qs->tx_reclaim_timer, jiffies + next_period);
2945 }
2946 
2947 /**
2948  *      sge_timer_rx - perform periodic maintenance of an SGE qset
2949  *      @data: the SGE queue set to maintain
2950  *
2951  *      a) Replenishes Rx queues that have run out due to memory shortage.
2952  *      Normally new Rx buffers are added when existing ones are consumed but
2953  *      when out of memory a queue can become empty.  We try to add only a few
2954  *      buffers here, the queue will be replenished fully as these new buffers
2955  *      are used up if memory shortage has subsided.
2956  *
2957  *      b) Return coalesced response queue credits in case a response queue is
2958  *      starved.
2959  *
2960  */
2961 static void sge_timer_rx(struct timer_list *t)
2962 {
2963         spinlock_t *lock;
2964         struct sge_qset *qs = from_timer(qs, t, rx_reclaim_timer);
2965         struct port_info *pi = netdev_priv(qs->netdev);
2966         struct adapter *adap = pi->adapter;
2967         u32 status;
2968 
2969         lock = adap->params.rev > 0 ?
2970                &qs->rspq.lock : &adap->sge.qs[0].rspq.lock;
2971 
2972         if (!spin_trylock_irq(lock))
2973                 goto out;
2974 
2975         if (napi_is_scheduled(&qs->napi))
2976                 goto unlock;
2977 
2978         if (adap->params.rev < 4) {
2979                 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS);
2980 
2981                 if (status & (1 << qs->rspq.cntxt_id)) {
2982                         qs->rspq.starved++;
2983                         if (qs->rspq.credits) {
2984                                 qs->rspq.credits--;
2985                                 refill_rspq(adap, &qs->rspq, 1);
2986                                 qs->rspq.restarted++;
2987                                 t3_write_reg(adap, A_SG_RSPQ_FL_STATUS,
2988                                              1 << qs->rspq.cntxt_id);
2989                         }
2990                 }
2991         }
2992 
2993         if (qs->fl[0].credits < qs->fl[0].size)
2994                 __refill_fl(adap, &qs->fl[0]);
2995         if (qs->fl[1].credits < qs->fl[1].size)
2996                 __refill_fl(adap, &qs->fl[1]);
2997 
2998 unlock:
2999         spin_unlock_irq(lock);
3000 out:
3001         mod_timer(&qs->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
3002 }
3003 
3004 /**
3005  *      t3_update_qset_coalesce - update coalescing settings for a queue set
3006  *      @qs: the SGE queue set
3007  *      @p: new queue set parameters
3008  *
3009  *      Update the coalescing settings for an SGE queue set.  Nothing is done
3010  *      if the queue set is not initialized yet.
3011  */
3012 void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)
3013 {
3014         qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);/* can't be 0 */
3015         qs->rspq.polling = p->polling;
3016         qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll;
3017 }
3018 
3019 /**
3020  *      t3_sge_alloc_qset - initialize an SGE queue set
3021  *      @adapter: the adapter
3022  *      @id: the queue set id
3023  *      @nports: how many Ethernet ports will be using this queue set
3024  *      @irq_vec_idx: the IRQ vector index for response queue interrupts
3025  *      @p: configuration parameters for this queue set
3026  *      @ntxq: number of Tx queues for the queue set
3027  *      @netdev: net device associated with this queue set
3028  *      @netdevq: net device TX queue associated with this queue set
3029  *
3030  *      Allocate resources and initialize an SGE queue set.  A queue set
3031  *      comprises a response queue, two Rx free-buffer queues, and up to 3
3032  *      Tx queues.  The Tx queues are assigned roles in the order Ethernet
3033  *      queue, offload queue, and control queue.
3034  */
3035 int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
3036                       int irq_vec_idx, const struct qset_params *p,
3037                       int ntxq, struct net_device *dev,
3038                       struct netdev_queue *netdevq)
3039 {
3040         int i, avail, ret = -ENOMEM;
3041         struct sge_qset *q = &adapter->sge.qs[id];
3042 
3043         init_qset_cntxt(q, id);
3044         timer_setup(&q->tx_reclaim_timer, sge_timer_tx, 0);
3045         timer_setup(&q->rx_reclaim_timer, sge_timer_rx, 0);
3046 
3047         q->fl[0].desc = alloc_ring(adapter->pdev, p->fl_size,
3048                                    sizeof(struct rx_desc),
3049                                    sizeof(struct rx_sw_desc),
3050                                    &q->fl[0].phys_addr, &q->fl[0].sdesc);
3051         if (!q->fl[0].desc)
3052                 goto err;
3053 
3054         q->fl[1].desc = alloc_ring(adapter->pdev, p->jumbo_size,
3055                                    sizeof(struct rx_desc),
3056                                    sizeof(struct rx_sw_desc),
3057                                    &q->fl[1].phys_addr, &q->fl[1].sdesc);
3058         if (!q->fl[1].desc)
3059                 goto err;
3060 
3061         q->rspq.desc = alloc_ring(adapter->pdev, p->rspq_size,
3062                                   sizeof(struct rsp_desc), 0,
3063                                   &q->rspq.phys_addr, NULL);
3064         if (!q->rspq.desc)
3065                 goto err;
3066 
3067         for (i = 0; i < ntxq; ++i) {
3068                 /*
3069                  * The control queue always uses immediate data so does not
3070                  * need to keep track of any sk_buffs.
3071                  */
3072                 size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
3073 
3074                 q->txq[i].desc = alloc_ring(adapter->pdev, p->txq_size[i],
3075                                             sizeof(struct tx_desc), sz,
3076                                             &q->txq[i].phys_addr,
3077                                             &q->txq[i].sdesc);
3078                 if (!q->txq[i].desc)
3079                         goto err;
3080 
3081                 q->txq[i].gen = 1;
3082                 q->txq[i].size = p->txq_size[i];
3083                 spin_lock_init(&q->txq[i].lock);
3084                 skb_queue_head_init(&q->txq[i].sendq);
3085         }
3086 
3087         tasklet_init(&q->txq[TXQ_OFLD].qresume_tsk, restart_offloadq,
3088                      (unsigned long)q);
3089         tasklet_init(&q->txq[TXQ_CTRL].qresume_tsk, restart_ctrlq,
3090                      (unsigned long)q);
3091 
3092         q->fl[0].gen = q->fl[1].gen = 1;
3093         q->fl[0].size = p->fl_size;
3094         q->fl[1].size = p->jumbo_size;
3095 
3096         q->rspq.gen = 1;
3097         q->rspq.size = p->rspq_size;
3098         spin_lock_init(&q->rspq.lock);
3099         skb_queue_head_init(&q->rspq.rx_queue);
3100 
3101         q->txq[TXQ_ETH].stop_thres = nports *
3102             flits_to_desc(sgl_len(MAX_SKB_FRAGS + 1) + 3);
3103 
3104 #if FL0_PG_CHUNK_SIZE > 0
3105         q->fl[0].buf_size = FL0_PG_CHUNK_SIZE;
3106 #else
3107         q->fl[0].buf_size = SGE_RX_SM_BUF_SIZE + sizeof(struct cpl_rx_data);
3108 #endif
3109 #if FL1_PG_CHUNK_SIZE > 0
3110         q->fl[1].buf_size = FL1_PG_CHUNK_SIZE;
3111 #else
3112         q->fl[1].buf_size = is_offload(adapter) ?
3113                 (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
3114                 MAX_FRAME_SIZE + 2 + sizeof(struct cpl_rx_pkt);
3115 #endif
3116 
3117         q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0;
3118         q->fl[1].use_pages = FL1_PG_CHUNK_SIZE > 0;
3119         q->fl[0].order = FL0_PG_ORDER;
3120         q->fl[1].order = FL1_PG_ORDER;
3121         q->fl[0].alloc_size = FL0_PG_ALLOC_SIZE;
3122         q->fl[1].alloc_size = FL1_PG_ALLOC_SIZE;
3123 
3124         spin_lock_irq(&adapter->sge.reg_lock);
3125 
3126         /* FL threshold comparison uses < */
3127         ret = t3_sge_init_rspcntxt(adapter, q->rspq.cntxt_id, irq_vec_idx,
3128                                    q->rspq.phys_addr, q->rspq.size,
3129                                    q->fl[0].buf_size - SGE_PG_RSVD, 1, 0);
3130         if (ret)
3131                 goto err_unlock;
3132 
3133         for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
3134                 ret = t3_sge_init_flcntxt(adapter, q->fl[i].cntxt_id, 0,
3135                                           q->fl[i].phys_addr, q->fl[i].size,
3136                                           q->fl[i].buf_size - SGE_PG_RSVD,
3137                                           p->cong_thres, 1, 0);
3138                 if (ret)
3139                         goto err_unlock;
3140         }
3141 
3142         ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
3143                                  SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
3144                                  q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
3145                                  1, 0);
3146         if (ret)
3147                 goto err_unlock;
3148 
3149         if (ntxq > 1) {
3150                 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_OFLD].cntxt_id,
3151                                          USE_GTS, SGE_CNTXT_OFLD, id,
3152                                          q->txq[TXQ_OFLD].phys_addr,
3153                                          q->txq[TXQ_OFLD].size, 0, 1, 0);
3154                 if (ret)
3155                         goto err_unlock;
3156         }
3157 
3158         if (ntxq > 2) {
3159                 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_CTRL].cntxt_id, 0,
3160                                          SGE_CNTXT_CTRL, id,
3161                                          q->txq[TXQ_CTRL].phys_addr,
3162                                          q->txq[TXQ_CTRL].size,
3163                                          q->txq[TXQ_CTRL].token, 1, 0);
3164                 if (ret)
3165                         goto err_unlock;
3166         }
3167 
3168         spin_unlock_irq(&adapter->sge.reg_lock);
3169 
3170         q->adap = adapter;
3171         q->netdev = dev;
3172         q->tx_q = netdevq;
3173         t3_update_qset_coalesce(q, p);
3174 
3175         avail = refill_fl(adapter, &q->fl[0], q->fl[0].size,
3176                           GFP_KERNEL | __GFP_COMP);
3177         if (!avail) {
3178                 CH_ALERT(adapter, "free list queue 0 initialization failed\n");
3179                 goto err;
3180         }
3181         if (avail < q->fl[0].size)
3182                 CH_WARN(adapter, "free list queue 0 enabled with %d credits\n",
3183                         avail);
3184 
3185         avail = refill_fl(adapter, &q->fl[1], q->fl[1].size,
3186                           GFP_KERNEL | __GFP_COMP);
3187         if (avail < q->fl[1].size)
3188                 CH_WARN(adapter, "free list queue 1 enabled with %d credits\n",
3189                         avail);
3190         refill_rspq(adapter, &q->rspq, q->rspq.size - 1);
3191 
3192         t3_write_reg(adapter, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
3193                      V_NEWTIMER(q->rspq.holdoff_tmr));
3194 
3195         return 0;
3196 
3197 err_unlock:
3198         spin_unlock_irq(&adapter->sge.reg_lock);
3199 err:
3200         t3_free_qset(adapter, q);
3201         return ret;
3202 }
3203 
3204 /**
3205  *      t3_start_sge_timers - start SGE timer call backs
3206  *      @adap: the adapter
3207  *
3208  *      Starts each SGE queue set's timer call back
3209  */
3210 void t3_start_sge_timers(struct adapter *adap)
3211 {
3212         int i;
3213 
3214         for (i = 0; i < SGE_QSETS; ++i) {
3215                 struct sge_qset *q = &adap->sge.qs[i];
3216 
3217                 if (q->tx_reclaim_timer.function)
3218                         mod_timer(&q->tx_reclaim_timer,
3219                                   jiffies + TX_RECLAIM_PERIOD);
3220 
3221                 if (q->rx_reclaim_timer.function)
3222                         mod_timer(&q->rx_reclaim_timer,
3223                                   jiffies + RX_RECLAIM_PERIOD);
3224         }
3225 }
3226 
3227 /**
3228  *      t3_stop_sge_timers - stop SGE timer call backs
3229  *      @adap: the adapter
3230  *
3231  *      Stops each SGE queue set's timer call back
3232  */
3233 void t3_stop_sge_timers(struct adapter *adap)
3234 {
3235         int i;
3236 
3237         for (i = 0; i < SGE_QSETS; ++i) {
3238                 struct sge_qset *q = &adap->sge.qs[i];
3239 
3240                 if (q->tx_reclaim_timer.function)
3241                         del_timer_sync(&q->tx_reclaim_timer);
3242                 if (q->rx_reclaim_timer.function)
3243                         del_timer_sync(&q->rx_reclaim_timer);
3244         }
3245 }
3246 
3247 /**
3248  *      t3_free_sge_resources - free SGE resources
3249  *      @adap: the adapter
3250  *
3251  *      Frees resources used by the SGE queue sets.
3252  */
3253 void t3_free_sge_resources(struct adapter *adap)
3254 {
3255         int i;
3256 
3257         for (i = 0; i < SGE_QSETS; ++i)
3258                 t3_free_qset(adap, &adap->sge.qs[i]);
3259 }
3260 
3261 /**
3262  *      t3_sge_start - enable SGE
3263  *      @adap: the adapter
3264  *
3265  *      Enables the SGE for DMAs.  This is the last step in starting packet
3266  *      transfers.
3267  */
3268 void t3_sge_start(struct adapter *adap)
3269 {
3270         t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
3271 }
3272 
3273 /**
3274  *      t3_sge_stop - disable SGE operation
3275  *      @adap: the adapter
3276  *
3277  *      Disables the DMA engine.  This can be called in emeregencies (e.g.,
3278  *      from error interrupts) or from normal process context.  In the latter
3279  *      case it also disables any pending queue restart tasklets.  Note that
3280  *      if it is called in interrupt context it cannot disable the restart
3281  *      tasklets as it cannot wait, however the tasklets will have no effect
3282  *      since the doorbells are disabled and the driver will call this again
3283  *      later from process context, at which time the tasklets will be stopped
3284  *      if they are still running.
3285  */
3286 void t3_sge_stop(struct adapter *adap)
3287 {
3288         t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, 0);
3289         if (!in_interrupt()) {
3290                 int i;
3291 
3292                 for (i = 0; i < SGE_QSETS; ++i) {
3293                         struct sge_qset *qs = &adap->sge.qs[i];
3294 
3295                         tasklet_kill(&qs->txq[TXQ_OFLD].qresume_tsk);
3296                         tasklet_kill(&qs->txq[TXQ_CTRL].qresume_tsk);
3297                 }
3298         }
3299 }
3300 
3301 /**
3302  *      t3_sge_init - initialize SGE
3303  *      @adap: the adapter
3304  *      @p: the SGE parameters
3305  *
3306  *      Performs SGE initialization needed every time after a chip reset.
3307  *      We do not initialize any of the queue sets here, instead the driver
3308  *      top-level must request those individually.  We also do not enable DMA
3309  *      here, that should be done after the queues have been set up.
3310  */
3311 void t3_sge_init(struct adapter *adap, struct sge_params *p)
3312 {
3313         unsigned int ctrl, ups = ffs(pci_resource_len(adap->pdev, 2) >> 12);
3314 
3315         ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
3316             F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
3317             V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
3318             V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
3319 #if SGE_NUM_GENBITS == 1
3320         ctrl |= F_EGRGENCTRL;
3321 #endif
3322         if (adap->params.rev > 0) {
3323                 if (!(adap->flags & (USING_MSIX | USING_MSI)))
3324                         ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
3325         }
3326         t3_write_reg(adap, A_SG_CONTROL, ctrl);
3327         t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
3328                      V_LORCQDRBTHRSH(512));
3329         t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
3330         t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
3331                      V_TIMEOUT(200 * core_ticks_per_usec(adap)));
3332         t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
3333                      adap->params.rev < T3_REV_C ? 1000 : 500);
3334         t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
3335         t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
3336         t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
3337         t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff));
3338         t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024);
3339 }
3340 
3341 /**
3342  *      t3_sge_prep - one-time SGE initialization
3343  *      @adap: the associated adapter
3344  *      @p: SGE parameters
3345  *
3346  *      Performs one-time initialization of SGE SW state.  Includes determining
3347  *      defaults for the assorted SGE parameters, which admins can change until
3348  *      they are used to initialize the SGE.
3349  */
3350 void t3_sge_prep(struct adapter *adap, struct sge_params *p)
3351 {
3352         int i;
3353 
3354         p->max_pkt_size = (16 * 1024) - sizeof(struct cpl_rx_data) -
3355             SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3356 
3357         for (i = 0; i < SGE_QSETS; ++i) {
3358                 struct qset_params *q = p->qset + i;
3359 
3360                 q->polling = adap->params.rev > 0;
3361                 q->coalesce_usecs = 5;
3362                 q->rspq_size = 1024;
3363                 q->fl_size = 1024;
3364                 q->jumbo_size = 512;
3365                 q->txq_size[TXQ_ETH] = 1024;
3366                 q->txq_size[TXQ_OFLD] = 1024;
3367                 q->txq_size[TXQ_CTRL] = 256;
3368                 q->cong_thres = 0;
3369         }
3370 
3371         spin_lock_init(&adap->sge.reg_lock);
3372 }