1 /*
2 * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet
3 * driver for Linux.
4 *
5 * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35
36 #include <linux/skbuff.h>
37 #include <linux/netdevice.h>
38 #include <linux/etherdevice.h>
39 #include <linux/if_vlan.h>
40 #include <linux/ip.h>
41 #include <net/ipv6.h>
42 #include <net/tcp.h>
43 #include <linux/dma-mapping.h>
44 #include <linux/prefetch.h>
45
46 #include "t4vf_common.h"
47 #include "t4vf_defs.h"
48
49 #include "../cxgb4/t4_regs.h"
50 #include "../cxgb4/t4_values.h"
51 #include "../cxgb4/t4fw_api.h"
52 #include "../cxgb4/t4_msg.h"
53
54 /*
55 * Constants ...
56 */
57 enum {
58 /*
59 * Egress Queue sizes, producer and consumer indices are all in units
60 * of Egress Context Units bytes. Note that as far as the hardware is
61 * concerned, the free list is an Egress Queue (the host produces free
62 * buffers which the hardware consumes) and free list entries are
63 * 64-bit PCI DMA addresses.
64 */
65 EQ_UNIT = SGE_EQ_IDXSIZE,
66 FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
67 TXD_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
68
69 /*
70 * Max number of TX descriptors we clean up at a time. Should be
71 * modest as freeing skbs isn't cheap and it happens while holding
72 * locks. We just need to free packets faster than they arrive, we
73 * eventually catch up and keep the amortized cost reasonable.
74 */
75 MAX_TX_RECLAIM = 16,
76
77 /*
78 * Max number of Rx buffers we replenish at a time. Again keep this
79 * modest, allocating buffers isn't cheap either.
80 */
81 MAX_RX_REFILL = 16,
82
83 /*
84 * Period of the Rx queue check timer. This timer is infrequent as it
85 * has something to do only when the system experiences severe memory
86 * shortage.
87 */
88 RX_QCHECK_PERIOD = (HZ / 2),
89
90 /*
91 * Period of the TX queue check timer and the maximum number of TX
92 * descriptors to be reclaimed by the TX timer.
93 */
94 TX_QCHECK_PERIOD = (HZ / 2),
95 MAX_TIMER_TX_RECLAIM = 100,
96
97 /*
98 * Suspend an Ethernet TX queue with fewer available descriptors than
99 * this. We always want to have room for a maximum sized packet:
100 * inline immediate data + MAX_SKB_FRAGS. This is the same as
101 * calc_tx_flits() for a TSO packet with nr_frags == MAX_SKB_FRAGS
102 * (see that function and its helpers for a description of the
103 * calculation).
104 */
105 ETHTXQ_MAX_FRAGS = MAX_SKB_FRAGS + 1,
106 ETHTXQ_MAX_SGL_LEN = ((3 * (ETHTXQ_MAX_FRAGS-1))/2 +
107 ((ETHTXQ_MAX_FRAGS-1) & 1) +
108 2),
109 ETHTXQ_MAX_HDR = (sizeof(struct fw_eth_tx_pkt_vm_wr) +
110 sizeof(struct cpl_tx_pkt_lso_core) +
111 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64),
112 ETHTXQ_MAX_FLITS = ETHTXQ_MAX_SGL_LEN + ETHTXQ_MAX_HDR,
113
114 ETHTXQ_STOP_THRES = 1 + DIV_ROUND_UP(ETHTXQ_MAX_FLITS, TXD_PER_EQ_UNIT),
115
116 /*
117 * Max TX descriptor space we allow for an Ethernet packet to be
118 * inlined into a WR. This is limited by the maximum value which
119 * we can specify for immediate data in the firmware Ethernet TX
120 * Work Request.
121 */
122 MAX_IMM_TX_PKT_LEN = FW_WR_IMMDLEN_M,
123
124 /*
125 * Max size of a WR sent through a control TX queue.
126 */
127 MAX_CTRL_WR_LEN = 256,
128
129 /*
130 * Maximum amount of data which we'll ever need to inline into a
131 * TX ring: max(MAX_IMM_TX_PKT_LEN, MAX_CTRL_WR_LEN).
132 */
133 MAX_IMM_TX_LEN = (MAX_IMM_TX_PKT_LEN > MAX_CTRL_WR_LEN
134 ? MAX_IMM_TX_PKT_LEN
135 : MAX_CTRL_WR_LEN),
136
137 /*
138 * For incoming packets less than RX_COPY_THRES, we copy the data into
139 * an skb rather than referencing the data. We allocate enough
140 * in-line room in skb's to accommodate pulling in RX_PULL_LEN bytes
141 * of the data (header).
142 */
143 RX_COPY_THRES = 256,
144 RX_PULL_LEN = 128,
145
146 /*
147 * Main body length for sk_buffs used for RX Ethernet packets with
148 * fragments. Should be >= RX_PULL_LEN but possibly bigger to give
149 * pskb_may_pull() some room.
150 */
151 RX_SKB_LEN = 512,
152 };
153
154 /*
155 * Software state per TX descriptor.
156 */
157 struct tx_sw_desc {
158 struct sk_buff *skb; /* socket buffer of TX data source */
159 struct ulptx_sgl *sgl; /* scatter/gather list in TX Queue */
160 };
161
162 /*
163 * Software state per RX Free List descriptor. We keep track of the allocated
164 * FL page, its size, and its PCI DMA address (if the page is mapped). The FL
165 * page size and its PCI DMA mapped state are stored in the low bits of the
166 * PCI DMA address as per below.
167 */
168 struct rx_sw_desc {
169 struct page *page; /* Free List page buffer */
170 dma_addr_t dma_addr; /* PCI DMA address (if mapped) */
171 /* and flags (see below) */
172 };
173
174 /*
175 * The low bits of rx_sw_desc.dma_addr have special meaning. Note that the
176 * SGE also uses the low 4 bits to determine the size of the buffer. It uses
177 * those bits to index into the SGE_FL_BUFFER_SIZE[index] register array.
178 * Since we only use SGE_FL_BUFFER_SIZE0 and SGE_FL_BUFFER_SIZE1, these low 4
179 * bits can only contain a 0 or a 1 to indicate which size buffer we're giving
180 * to the SGE. Thus, our software state of "is the buffer mapped for DMA" is
181 * maintained in an inverse sense so the hardware never sees that bit high.
182 */
183 enum {
184 RX_LARGE_BUF = 1 << 0, /* buffer is SGE_FL_BUFFER_SIZE[1] */
185 RX_UNMAPPED_BUF = 1 << 1, /* buffer is not mapped */
186 };
187
188 /**
189 * get_buf_addr - return DMA buffer address of software descriptor
190 * @sdesc: pointer to the software buffer descriptor
191 *
192 * Return the DMA buffer address of a software descriptor (stripping out
193 * our low-order flag bits).
194 */
195 static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *sdesc)
196 {
197 return sdesc->dma_addr & ~(dma_addr_t)(RX_LARGE_BUF | RX_UNMAPPED_BUF);
198 }
199
200 /**
201 * is_buf_mapped - is buffer mapped for DMA?
202 * @sdesc: pointer to the software buffer descriptor
203 *
204 * Determine whether the buffer associated with a software descriptor in
205 * mapped for DMA or not.
206 */
207 static inline bool is_buf_mapped(const struct rx_sw_desc *sdesc)
208 {
209 return !(sdesc->dma_addr & RX_UNMAPPED_BUF);
210 }
211
212 /**
213 * need_skb_unmap - does the platform need unmapping of sk_buffs?
214 *
215 * Returns true if the platform needs sk_buff unmapping. The compiler
216 * optimizes away unnecessary code if this returns true.
217 */
218 static inline int need_skb_unmap(void)
219 {
220 #ifdef CONFIG_NEED_DMA_MAP_STATE
221 return 1;
222 #else
223 return 0;
224 #endif
225 }
226
227 /**
228 * txq_avail - return the number of available slots in a TX queue
229 * @tq: the TX queue
230 *
231 * Returns the number of available descriptors in a TX queue.
232 */
233 static inline unsigned int txq_avail(const struct sge_txq *tq)
234 {
235 return tq->size - 1 - tq->in_use;
236 }
237
238 /**
239 * fl_cap - return the capacity of a Free List
240 * @fl: the Free List
241 *
242 * Returns the capacity of a Free List. The capacity is less than the
243 * size because an Egress Queue Index Unit worth of descriptors needs to
244 * be left unpopulated, otherwise the Producer and Consumer indices PIDX
245 * and CIDX will match and the hardware will think the FL is empty.
246 */
247 static inline unsigned int fl_cap(const struct sge_fl *fl)
248 {
249 return fl->size - FL_PER_EQ_UNIT;
250 }
251
252 /**
253 * fl_starving - return whether a Free List is starving.
254 * @adapter: pointer to the adapter
255 * @fl: the Free List
256 *
257 * Tests specified Free List to see whether the number of buffers
258 * available to the hardware has falled below our "starvation"
259 * threshold.
260 */
261 static inline bool fl_starving(const struct adapter *adapter,
262 const struct sge_fl *fl)
263 {
264 const struct sge *s = &adapter->sge;
265
266 return fl->avail - fl->pend_cred <= s->fl_starve_thres;
267 }
268
269 /**
270 * map_skb - map an skb for DMA to the device
271 * @dev: the egress net device
272 * @skb: the packet to map
273 * @addr: a pointer to the base of the DMA mapping array
274 *
275 * Map an skb for DMA to the device and return an array of DMA addresses.
276 */
277 static int map_skb(struct device *dev, const struct sk_buff *skb,
278 dma_addr_t *addr)
279 {
280 const skb_frag_t *fp, *end;
281 const struct skb_shared_info *si;
282
283 *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
284 if (dma_mapping_error(dev, *addr))
285 goto out_err;
286
287 si = skb_shinfo(skb);
288 end = &si->frags[si->nr_frags];
289 for (fp = si->frags; fp < end; fp++) {
290 *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp),
291 DMA_TO_DEVICE);
292 if (dma_mapping_error(dev, *addr))
293 goto unwind;
294 }
295 return 0;
296
297 unwind:
298 while (fp-- > si->frags)
299 dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE);
300 dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE);
301
302 out_err:
303 return -ENOMEM;
304 }
305
306 static void unmap_sgl(struct device *dev, const struct sk_buff *skb,
307 const struct ulptx_sgl *sgl, const struct sge_txq *tq)
308 {
309 const struct ulptx_sge_pair *p;
310 unsigned int nfrags = skb_shinfo(skb)->nr_frags;
311
312 if (likely(skb_headlen(skb)))
313 dma_unmap_single(dev, be64_to_cpu(sgl->addr0),
314 be32_to_cpu(sgl->len0), DMA_TO_DEVICE);
315 else {
316 dma_unmap_page(dev, be64_to_cpu(sgl->addr0),
317 be32_to_cpu(sgl->len0), DMA_TO_DEVICE);
318 nfrags--;
319 }
320
321 /*
322 * the complexity below is because of the possibility of a wrap-around
323 * in the middle of an SGL
324 */
325 for (p = sgl->sge; nfrags >= 2; nfrags -= 2) {
326 if (likely((u8 *)(p + 1) <= (u8 *)tq->stat)) {
327 unmap:
328 dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
329 be32_to_cpu(p->len[0]), DMA_TO_DEVICE);
330 dma_unmap_page(dev, be64_to_cpu(p->addr[1]),
331 be32_to_cpu(p->len[1]), DMA_TO_DEVICE);
332 p++;
333 } else if ((u8 *)p == (u8 *)tq->stat) {
334 p = (const struct ulptx_sge_pair *)tq->desc;
335 goto unmap;
336 } else if ((u8 *)p + 8 == (u8 *)tq->stat) {
337 const __be64 *addr = (const __be64 *)tq->desc;
338
339 dma_unmap_page(dev, be64_to_cpu(addr[0]),
340 be32_to_cpu(p->len[0]), DMA_TO_DEVICE);
341 dma_unmap_page(dev, be64_to_cpu(addr[1]),
342 be32_to_cpu(p->len[1]), DMA_TO_DEVICE);
343 p = (const struct ulptx_sge_pair *)&addr[2];
344 } else {
345 const __be64 *addr = (const __be64 *)tq->desc;
346
347 dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
348 be32_to_cpu(p->len[0]), DMA_TO_DEVICE);
349 dma_unmap_page(dev, be64_to_cpu(addr[0]),
350 be32_to_cpu(p->len[1]), DMA_TO_DEVICE);
351 p = (const struct ulptx_sge_pair *)&addr[1];
352 }
353 }
354 if (nfrags) {
355 __be64 addr;
356
357 if ((u8 *)p == (u8 *)tq->stat)
358 p = (const struct ulptx_sge_pair *)tq->desc;
359 addr = ((u8 *)p + 16 <= (u8 *)tq->stat
360 ? p->addr[0]
361 : *(const __be64 *)tq->desc);
362 dma_unmap_page(dev, be64_to_cpu(addr), be32_to_cpu(p->len[0]),
363 DMA_TO_DEVICE);
364 }
365 }
366
367 /**
368 * free_tx_desc - reclaims TX descriptors and their buffers
369 * @adapter: the adapter
370 * @tq: the TX queue to reclaim descriptors from
371 * @n: the number of descriptors to reclaim
372 * @unmap: whether the buffers should be unmapped for DMA
373 *
374 * Reclaims TX descriptors from an SGE TX queue and frees the associated
375 * TX buffers. Called with the TX queue lock held.
376 */
377 static void free_tx_desc(struct adapter *adapter, struct sge_txq *tq,
378 unsigned int n, bool unmap)
379 {
380 struct tx_sw_desc *sdesc;
381 unsigned int cidx = tq->cidx;
382 struct device *dev = adapter->pdev_dev;
383
384 const int need_unmap = need_skb_unmap() && unmap;
385
386 sdesc = &tq->sdesc[cidx];
387 while (n--) {
388 /*
389 * If we kept a reference to the original TX skb, we need to
390 * unmap it from PCI DMA space (if required) and free it.
391 */
392 if (sdesc->skb) {
393 if (need_unmap)
394 unmap_sgl(dev, sdesc->skb, sdesc->sgl, tq);
395 dev_consume_skb_any(sdesc->skb);
396 sdesc->skb = NULL;
397 }
398
399 sdesc++;
400 if (++cidx == tq->size) {
401 cidx = 0;
402 sdesc = tq->sdesc;
403 }
404 }
405 tq->cidx = cidx;
406 }
407
408 /*
409 * Return the number of reclaimable descriptors in a TX queue.
410 */
411 static inline int reclaimable(const struct sge_txq *tq)
412 {
413 int hw_cidx = be16_to_cpu(tq->stat->cidx);
414 int reclaimable = hw_cidx - tq->cidx;
415 if (reclaimable < 0)
416 reclaimable += tq->size;
417 return reclaimable;
418 }
419
420 /**
421 * reclaim_completed_tx - reclaims completed TX descriptors
422 * @adapter: the adapter
423 * @tq: the TX queue to reclaim completed descriptors from
424 * @unmap: whether the buffers should be unmapped for DMA
425 *
426 * Reclaims TX descriptors that the SGE has indicated it has processed,
427 * and frees the associated buffers if possible. Called with the TX
428 * queue locked.
429 */
430 static inline void reclaim_completed_tx(struct adapter *adapter,
431 struct sge_txq *tq,
432 bool unmap)
433 {
434 int avail = reclaimable(tq);
435
436 if (avail) {
437 /*
438 * Limit the amount of clean up work we do at a time to keep
439 * the TX lock hold time O(1).
440 */
441 if (avail > MAX_TX_RECLAIM)
442 avail = MAX_TX_RECLAIM;
443
444 free_tx_desc(adapter, tq, avail, unmap);
445 tq->in_use -= avail;
446 }
447 }
448
449 /**
450 * get_buf_size - return the size of an RX Free List buffer.
451 * @adapter: pointer to the associated adapter
452 * @sdesc: pointer to the software buffer descriptor
453 */
454 static inline int get_buf_size(const struct adapter *adapter,
455 const struct rx_sw_desc *sdesc)
456 {
457 const struct sge *s = &adapter->sge;
458
459 return (s->fl_pg_order > 0 && (sdesc->dma_addr & RX_LARGE_BUF)
460 ? (PAGE_SIZE << s->fl_pg_order) : PAGE_SIZE);
461 }
462
463 /**
464 * free_rx_bufs - free RX buffers on an SGE Free List
465 * @adapter: the adapter
466 * @fl: the SGE Free List to free buffers from
467 * @n: how many buffers to free
468 *
469 * Release the next @n buffers on an SGE Free List RX queue. The
470 * buffers must be made inaccessible to hardware before calling this
471 * function.
472 */
473 static void free_rx_bufs(struct adapter *adapter, struct sge_fl *fl, int n)
474 {
475 while (n--) {
476 struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx];
477
478 if (is_buf_mapped(sdesc))
479 dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),
480 get_buf_size(adapter, sdesc),
481 PCI_DMA_FROMDEVICE);
482 put_page(sdesc->page);
483 sdesc->page = NULL;
484 if (++fl->cidx == fl->size)
485 fl->cidx = 0;
486 fl->avail--;
487 }
488 }
489
490 /**
491 * unmap_rx_buf - unmap the current RX buffer on an SGE Free List
492 * @adapter: the adapter
493 * @fl: the SGE Free List
494 *
495 * Unmap the current buffer on an SGE Free List RX queue. The
496 * buffer must be made inaccessible to HW before calling this function.
497 *
498 * This is similar to @free_rx_bufs above but does not free the buffer.
499 * Do note that the FL still loses any further access to the buffer.
500 * This is used predominantly to "transfer ownership" of an FL buffer
501 * to another entity (typically an skb's fragment list).
502 */
503 static void unmap_rx_buf(struct adapter *adapter, struct sge_fl *fl)
504 {
505 struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx];
506
507 if (is_buf_mapped(sdesc))
508 dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),
509 get_buf_size(adapter, sdesc),
510 PCI_DMA_FROMDEVICE);
511 sdesc->page = NULL;
512 if (++fl->cidx == fl->size)
513 fl->cidx = 0;
514 fl->avail--;
515 }
516
517 /**
518 * ring_fl_db - righ doorbell on free list
519 * @adapter: the adapter
520 * @fl: the Free List whose doorbell should be rung ...
521 *
522 * Tell the Scatter Gather Engine that there are new free list entries
523 * available.
524 */
525 static inline void ring_fl_db(struct adapter *adapter, struct sge_fl *fl)
526 {
527 u32 val = adapter->params.arch.sge_fl_db;
528
529 /* The SGE keeps track of its Producer and Consumer Indices in terms
530 * of Egress Queue Units so we can only tell it about integral numbers
531 * of multiples of Free List Entries per Egress Queue Units ...
532 */
533 if (fl->pend_cred >= FL_PER_EQ_UNIT) {
534 if (is_t4(adapter->params.chip))
535 val |= PIDX_V(fl->pend_cred / FL_PER_EQ_UNIT);
536 else
537 val |= PIDX_T5_V(fl->pend_cred / FL_PER_EQ_UNIT);
538
539 /* Make sure all memory writes to the Free List queue are
540 * committed before we tell the hardware about them.
541 */
542 wmb();
543
544 /* If we don't have access to the new User Doorbell (T5+), use
545 * the old doorbell mechanism; otherwise use the new BAR2
546 * mechanism.
547 */
548 if (unlikely(fl->bar2_addr == NULL)) {
549 t4_write_reg(adapter,
550 T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL,
551 QID_V(fl->cntxt_id) | val);
552 } else {
553 writel(val | QID_V(fl->bar2_qid),
554 fl->bar2_addr + SGE_UDB_KDOORBELL);
555
556 /* This Write memory Barrier will force the write to
557 * the User Doorbell area to be flushed.
558 */
559 wmb();
560 }
561 fl->pend_cred %= FL_PER_EQ_UNIT;
562 }
563 }
564
565 /**
566 * set_rx_sw_desc - initialize software RX buffer descriptor
567 * @sdesc: pointer to the softwore RX buffer descriptor
568 * @page: pointer to the page data structure backing the RX buffer
569 * @dma_addr: PCI DMA address (possibly with low-bit flags)
570 */
571 static inline void set_rx_sw_desc(struct rx_sw_desc *sdesc, struct page *page,
572 dma_addr_t dma_addr)
573 {
574 sdesc->page = page;
575 sdesc->dma_addr = dma_addr;
576 }
577
578 /*
579 * Support for poisoning RX buffers ...
580 */
581 #define POISON_BUF_VAL -1
582
583 static inline void poison_buf(struct page *page, size_t sz)
584 {
585 #if POISON_BUF_VAL >= 0
586 memset(page_address(page), POISON_BUF_VAL, sz);
587 #endif
588 }
589
590 /**
591 * refill_fl - refill an SGE RX buffer ring
592 * @adapter: the adapter
593 * @fl: the Free List ring to refill
594 * @n: the number of new buffers to allocate
595 * @gfp: the gfp flags for the allocations
596 *
597 * (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
598 * allocated with the supplied gfp flags. The caller must assure that
599 * @n does not exceed the queue's capacity -- i.e. (cidx == pidx) _IN
600 * EGRESS QUEUE UNITS_ indicates an empty Free List! Returns the number
601 * of buffers allocated. If afterwards the queue is found critically low,
602 * mark it as starving in the bitmap of starving FLs.
603 */
604 static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl,
605 int n, gfp_t gfp)
606 {
607 struct sge *s = &adapter->sge;
608 struct page *page;
609 dma_addr_t dma_addr;
610 unsigned int cred = fl->avail;
611 __be64 *d = &fl->desc[fl->pidx];
612 struct rx_sw_desc *sdesc = &fl->sdesc[fl->pidx];
613
614 /*
615 * Sanity: ensure that the result of adding n Free List buffers
616 * won't result in wrapping the SGE's Producer Index around to
617 * it's Consumer Index thereby indicating an empty Free List ...
618 */
619 BUG_ON(fl->avail + n > fl->size - FL_PER_EQ_UNIT);
620
621 gfp |= __GFP_NOWARN;
622
623 /*
624 * If we support large pages, prefer large buffers and fail over to
625 * small pages if we can't allocate large pages to satisfy the refill.
626 * If we don't support large pages, drop directly into the small page
627 * allocation code.
628 */
629 if (s->fl_pg_order == 0)
630 goto alloc_small_pages;
631
632 while (n) {
633 page = __dev_alloc_pages(gfp, s->fl_pg_order);
634 if (unlikely(!page)) {
635 /*
636 * We've failed inour attempt to allocate a "large
637 * page". Fail over to the "small page" allocation
638 * below.
639 */
640 fl->large_alloc_failed++;
641 break;
642 }
643 poison_buf(page, PAGE_SIZE << s->fl_pg_order);
644
645 dma_addr = dma_map_page(adapter->pdev_dev, page, 0,
646 PAGE_SIZE << s->fl_pg_order,
647 PCI_DMA_FROMDEVICE);
648 if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {
649 /*
650 * We've run out of DMA mapping space. Free up the
651 * buffer and return with what we've managed to put
652 * into the free list. We don't want to fail over to
653 * the small page allocation below in this case
654 * because DMA mapping resources are typically
655 * critical resources once they become scarse.
656 */
657 __free_pages(page, s->fl_pg_order);
658 goto out;
659 }
660 dma_addr |= RX_LARGE_BUF;
661 *d++ = cpu_to_be64(dma_addr);
662
663 set_rx_sw_desc(sdesc, page, dma_addr);
664 sdesc++;
665
666 fl->avail++;
667 if (++fl->pidx == fl->size) {
668 fl->pidx = 0;
669 sdesc = fl->sdesc;
670 d = fl->desc;
671 }
672 n--;
673 }
674
675 alloc_small_pages:
676 while (n--) {
677 page = __dev_alloc_page(gfp);
678 if (unlikely(!page)) {
679 fl->alloc_failed++;
680 break;
681 }
682 poison_buf(page, PAGE_SIZE);
683
684 dma_addr = dma_map_page(adapter->pdev_dev, page, 0, PAGE_SIZE,
685 PCI_DMA_FROMDEVICE);
686 if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {
687 put_page(page);
688 break;
689 }
690 *d++ = cpu_to_be64(dma_addr);
691
692 set_rx_sw_desc(sdesc, page, dma_addr);
693 sdesc++;
694
695 fl->avail++;
696 if (++fl->pidx == fl->size) {
697 fl->pidx = 0;
698 sdesc = fl->sdesc;
699 d = fl->desc;
700 }
701 }
702
703 out:
704 /*
705 * Update our accounting state to incorporate the new Free List
706 * buffers, tell the hardware about them and return the number of
707 * buffers which we were able to allocate.
708 */
709 cred = fl->avail - cred;
710 fl->pend_cred += cred;
711 ring_fl_db(adapter, fl);
712
713 if (unlikely(fl_starving(adapter, fl))) {
714 smp_wmb();
715 set_bit(fl->cntxt_id, adapter->sge.starving_fl);
716 }
717
718 return cred;
719 }
720
721 /*
722 * Refill a Free List to its capacity or the Maximum Refill Increment,
723 * whichever is smaller ...
724 */
725 static inline void __refill_fl(struct adapter *adapter, struct sge_fl *fl)
726 {
727 refill_fl(adapter, fl,
728 min((unsigned int)MAX_RX_REFILL, fl_cap(fl) - fl->avail),
729 GFP_ATOMIC);
730 }
731
732 /**
733 * alloc_ring - allocate resources for an SGE descriptor ring
734 * @dev: the PCI device's core device
735 * @nelem: the number of descriptors
736 * @hwsize: the size of each hardware descriptor
737 * @swsize: the size of each software descriptor
738 * @busaddrp: the physical PCI bus address of the allocated ring
739 * @swringp: return address pointer for software ring
740 * @stat_size: extra space in hardware ring for status information
741 *
742 * Allocates resources for an SGE descriptor ring, such as TX queues,
743 * free buffer lists, response queues, etc. Each SGE ring requires
744 * space for its hardware descriptors plus, optionally, space for software
745 * state associated with each hardware entry (the metadata). The function
746 * returns three values: the virtual address for the hardware ring (the
747 * return value of the function), the PCI bus address of the hardware
748 * ring (in *busaddrp), and the address of the software ring (in swringp).
749 * Both the hardware and software rings are returned zeroed out.
750 */
751 static void *alloc_ring(struct device *dev, size_t nelem, size_t hwsize,
752 size_t swsize, dma_addr_t *busaddrp, void *swringp,
753 size_t stat_size)
754 {
755 /*
756 * Allocate the hardware ring and PCI DMA bus address space for said.
757 */
758 size_t hwlen = nelem * hwsize + stat_size;
759 void *hwring = dma_alloc_coherent(dev, hwlen, busaddrp, GFP_KERNEL);
760
761 if (!hwring)
762 return NULL;
763
764 /*
765 * If the caller wants a software ring, allocate it and return a
766 * pointer to it in *swringp.
767 */
768 BUG_ON((swsize != 0) != (swringp != NULL));
769 if (swsize) {
770 void *swring = kcalloc(nelem, swsize, GFP_KERNEL);
771
772 if (!swring) {
773 dma_free_coherent(dev, hwlen, hwring, *busaddrp);
774 return NULL;
775 }
776 *(void **)swringp = swring;
777 }
778
779 return hwring;
780 }
781
782 /**
783 * sgl_len - calculates the size of an SGL of the given capacity
784 * @n: the number of SGL entries
785 *
786 * Calculates the number of flits (8-byte units) needed for a Direct
787 * Scatter/Gather List that can hold the given number of entries.
788 */
789 static inline unsigned int sgl_len(unsigned int n)
790 {
791 /*
792 * A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA
793 * addresses. The DSGL Work Request starts off with a 32-bit DSGL
794 * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N,
795 * repeated sequences of { Length[i], Length[i+1], Address[i],
796 * Address[i+1] } (this ensures that all addresses are on 64-bit
797 * boundaries). If N is even, then Length[N+1] should be set to 0 and
798 * Address[N+1] is omitted.
799 *
800 * The following calculation incorporates all of the above. It's
801 * somewhat hard to follow but, briefly: the "+2" accounts for the
802 * first two flits which include the DSGL header, Length0 and
803 * Address0; the "(3*(n-1))/2" covers the main body of list entries (3
804 * flits for every pair of the remaining N) +1 if (n-1) is odd; and
805 * finally the "+((n-1)&1)" adds the one remaining flit needed if
806 * (n-1) is odd ...
807 */
808 n--;
809 return (3 * n) / 2 + (n & 1) + 2;
810 }
811
812 /**
813 * flits_to_desc - returns the num of TX descriptors for the given flits
814 * @flits: the number of flits
815 *
816 * Returns the number of TX descriptors needed for the supplied number
817 * of flits.
818 */
819 static inline unsigned int flits_to_desc(unsigned int flits)
820 {
821 BUG_ON(flits > SGE_MAX_WR_LEN / sizeof(__be64));
822 return DIV_ROUND_UP(flits, TXD_PER_EQ_UNIT);
823 }
824
825 /**
826 * is_eth_imm - can an Ethernet packet be sent as immediate data?
827 * @skb: the packet
828 *
829 * Returns whether an Ethernet packet is small enough to fit completely as
830 * immediate data.
831 */
832 static inline int is_eth_imm(const struct sk_buff *skb)
833 {
834 /*
835 * The VF Driver uses the FW_ETH_TX_PKT_VM_WR firmware Work Request
836 * which does not accommodate immediate data. We could dike out all
837 * of the support code for immediate data but that would tie our hands
838 * too much if we ever want to enhace the firmware. It would also
839 * create more differences between the PF and VF Drivers.
840 */
841 return false;
842 }
843
844 /**
845 * calc_tx_flits - calculate the number of flits for a packet TX WR
846 * @skb: the packet
847 *
848 * Returns the number of flits needed for a TX Work Request for the
849 * given Ethernet packet, including the needed WR and CPL headers.
850 */
851 static inline unsigned int calc_tx_flits(const struct sk_buff *skb)
852 {
853 unsigned int flits;
854
855 /*
856 * If the skb is small enough, we can pump it out as a work request
857 * with only immediate data. In that case we just have to have the
858 * TX Packet header plus the skb data in the Work Request.
859 */
860 if (is_eth_imm(skb))
861 return DIV_ROUND_UP(skb->len + sizeof(struct cpl_tx_pkt),
862 sizeof(__be64));
863
864 /*
865 * Otherwise, we're going to have to construct a Scatter gather list
866 * of the skb body and fragments. We also include the flits necessary
867 * for the TX Packet Work Request and CPL. We always have a firmware
868 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
869 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
870 * message or, if we're doing a Large Send Offload, an LSO CPL message
871 * with an embedded TX Packet Write CPL message.
872 */
873 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
874 if (skb_shinfo(skb)->gso_size)
875 flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
876 sizeof(struct cpl_tx_pkt_lso_core) +
877 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
878 else
879 flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
880 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
881 return flits;
882 }
883
884 /**
885 * write_sgl - populate a Scatter/Gather List for a packet
886 * @skb: the packet
887 * @tq: the TX queue we are writing into
888 * @sgl: starting location for writing the SGL
889 * @end: points right after the end of the SGL
890 * @start: start offset into skb main-body data to include in the SGL
891 * @addr: the list of DMA bus addresses for the SGL elements
892 *
893 * Generates a Scatter/Gather List for the buffers that make up a packet.
894 * The caller must provide adequate space for the SGL that will be written.
895 * The SGL includes all of the packet's page fragments and the data in its
896 * main body except for the first @start bytes. @pos must be 16-byte
897 * aligned and within a TX descriptor with available space. @end points
898 * write after the end of the SGL but does not account for any potential
899 * wrap around, i.e., @end > @tq->stat.
900 */
901 static void write_sgl(const struct sk_buff *skb, struct sge_txq *tq,
902 struct ulptx_sgl *sgl, u64 *end, unsigned int start,
903 const dma_addr_t *addr)
904 {
905 unsigned int i, len;
906 struct ulptx_sge_pair *to;
907 const struct skb_shared_info *si = skb_shinfo(skb);
908 unsigned int nfrags = si->nr_frags;
909 struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1];
910
911 len = skb_headlen(skb) - start;
912 if (likely(len)) {
913 sgl->len0 = htonl(len);
914 sgl->addr0 = cpu_to_be64(addr[0] + start);
915 nfrags++;
916 } else {
917 sgl->len0 = htonl(skb_frag_size(&si->frags[0]));
918 sgl->addr0 = cpu_to_be64(addr[1]);
919 }
920
921 sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
922 ULPTX_NSGE_V(nfrags));
923 if (likely(--nfrags == 0))
924 return;
925 /*
926 * Most of the complexity below deals with the possibility we hit the
927 * end of the queue in the middle of writing the SGL. For this case
928 * only we create the SGL in a temporary buffer and then copy it.
929 */
930 to = (u8 *)end > (u8 *)tq->stat ? buf : sgl->sge;
931
932 for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) {
933 to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
934 to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i]));
935 to->addr[0] = cpu_to_be64(addr[i]);
936 to->addr[1] = cpu_to_be64(addr[++i]);
937 }
938 if (nfrags) {
939 to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
940 to->len[1] = cpu_to_be32(0);
941 to->addr[0] = cpu_to_be64(addr[i + 1]);
942 }
943 if (unlikely((u8 *)end > (u8 *)tq->stat)) {
944 unsigned int part0 = (u8 *)tq->stat - (u8 *)sgl->sge, part1;
945
946 if (likely(part0))
947 memcpy(sgl->sge, buf, part0);
948 part1 = (u8 *)end - (u8 *)tq->stat;
949 memcpy(tq->desc, (u8 *)buf + part0, part1);
950 end = (void *)tq->desc + part1;
951 }
952 if ((uintptr_t)end & 8) /* 0-pad to multiple of 16 */
953 *end = 0;
954 }
955
956 /**
957 * check_ring_tx_db - check and potentially ring a TX queue's doorbell
958 * @adapter: the adapter
959 * @tq: the TX queue
960 * @n: number of new descriptors to give to HW
961 *
962 * Ring the doorbel for a TX queue.
963 */
964 static inline void ring_tx_db(struct adapter *adapter, struct sge_txq *tq,
965 int n)
966 {
967 /* Make sure that all writes to the TX Descriptors are committed
968 * before we tell the hardware about them.
969 */
970 wmb();
971
972 /* If we don't have access to the new User Doorbell (T5+), use the old
973 * doorbell mechanism; otherwise use the new BAR2 mechanism.
974 */
975 if (unlikely(tq->bar2_addr == NULL)) {
976 u32 val = PIDX_V(n);
977
978 t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL,
979 QID_V(tq->cntxt_id) | val);
980 } else {
981 u32 val = PIDX_T5_V(n);
982
983 /* T4 and later chips share the same PIDX field offset within
984 * the doorbell, but T5 and later shrank the field in order to
985 * gain a bit for Doorbell Priority. The field was absurdly
986 * large in the first place (14 bits) so we just use the T5
987 * and later limits and warn if a Queue ID is too large.
988 */
989 WARN_ON(val & DBPRIO_F);
990
991 /* If we're only writing a single Egress Unit and the BAR2
992 * Queue ID is 0, we can use the Write Combining Doorbell
993 * Gather Buffer; otherwise we use the simple doorbell.
994 */
995 if (n == 1 && tq->bar2_qid == 0) {
996 unsigned int index = (tq->pidx
997 ? (tq->pidx - 1)
998 : (tq->size - 1));
999 __be64 *src = (__be64 *)&tq->desc[index];
1000 __be64 __iomem *dst = (__be64 __iomem *)(tq->bar2_addr +
1001 SGE_UDB_WCDOORBELL);
1002 unsigned int count = EQ_UNIT / sizeof(__be64);
1003
1004 /* Copy the TX Descriptor in a tight loop in order to
1005 * try to get it to the adapter in a single Write
1006 * Combined transfer on the PCI-E Bus. If the Write
1007 * Combine fails (say because of an interrupt, etc.)
1008 * the hardware will simply take the last write as a
1009 * simple doorbell write with a PIDX Increment of 1
1010 * and will fetch the TX Descriptor from memory via
1011 * DMA.
1012 */
1013 while (count) {
1014 /* the (__force u64) is because the compiler
1015 * doesn't understand the endian swizzling
1016 * going on
1017 */
1018 writeq((__force u64)*src, dst);
1019 src++;
1020 dst++;
1021 count--;
1022 }
1023 } else
1024 writel(val | QID_V(tq->bar2_qid),
1025 tq->bar2_addr + SGE_UDB_KDOORBELL);
1026
1027 /* This Write Memory Barrier will force the write to the User
1028 * Doorbell area to be flushed. This is needed to prevent
1029 * writes on different CPUs for the same queue from hitting
1030 * the adapter out of order. This is required when some Work
1031 * Requests take the Write Combine Gather Buffer path (user
1032 * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some
1033 * take the traditional path where we simply increment the
1034 * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the
1035 * hardware DMA read the actual Work Request.
1036 */
1037 wmb();
1038 }
1039 }
1040
1041 /**
1042 * inline_tx_skb - inline a packet's data into TX descriptors
1043 * @skb: the packet
1044 * @tq: the TX queue where the packet will be inlined
1045 * @pos: starting position in the TX queue to inline the packet
1046 *
1047 * Inline a packet's contents directly into TX descriptors, starting at
1048 * the given position within the TX DMA ring.
1049 * Most of the complexity of this operation is dealing with wrap arounds
1050 * in the middle of the packet we want to inline.
1051 */
1052 static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *tq,
1053 void *pos)
1054 {
1055 u64 *p;
1056 int left = (void *)tq->stat - pos;
1057
1058 if (likely(skb->len <= left)) {
1059 if (likely(!skb->data_len))
1060 skb_copy_from_linear_data(skb, pos, skb->len);
1061 else
1062 skb_copy_bits(skb, 0, pos, skb->len);
1063 pos += skb->len;
1064 } else {
1065 skb_copy_bits(skb, 0, pos, left);
1066 skb_copy_bits(skb, left, tq->desc, skb->len - left);
1067 pos = (void *)tq->desc + (skb->len - left);
1068 }
1069
1070 /* 0-pad to multiple of 16 */
1071 p = PTR_ALIGN(pos, 8);
1072 if ((uintptr_t)p & 8)
1073 *p = 0;
1074 }
1075
1076 /*
1077 * Figure out what HW csum a packet wants and return the appropriate control
1078 * bits.
1079 */
1080 static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb)
1081 {
1082 int csum_type;
1083 const struct iphdr *iph = ip_hdr(skb);
1084
1085 if (iph->version == 4) {
1086 if (iph->protocol == IPPROTO_TCP)
1087 csum_type = TX_CSUM_TCPIP;
1088 else if (iph->protocol == IPPROTO_UDP)
1089 csum_type = TX_CSUM_UDPIP;
1090 else {
1091 nocsum:
1092 /*
1093 * unknown protocol, disable HW csum
1094 * and hope a bad packet is detected
1095 */
1096 return TXPKT_L4CSUM_DIS_F;
1097 }
1098 } else {
1099 /*
1100 * this doesn't work with extension headers
1101 */
1102 const struct ipv6hdr *ip6h = (const struct ipv6hdr *)iph;
1103
1104 if (ip6h->nexthdr == IPPROTO_TCP)
1105 csum_type = TX_CSUM_TCPIP6;
1106 else if (ip6h->nexthdr == IPPROTO_UDP)
1107 csum_type = TX_CSUM_UDPIP6;
1108 else
1109 goto nocsum;
1110 }
1111
1112 if (likely(csum_type >= TX_CSUM_TCPIP)) {
1113 u64 hdr_len = TXPKT_IPHDR_LEN_V(skb_network_header_len(skb));
1114 int eth_hdr_len = skb_network_offset(skb) - ETH_HLEN;
1115
1116 if (chip <= CHELSIO_T5)
1117 hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1118 else
1119 hdr_len |= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1120 return TXPKT_CSUM_TYPE_V(csum_type) | hdr_len;
1121 } else {
1122 int start = skb_transport_offset(skb);
1123
1124 return TXPKT_CSUM_TYPE_V(csum_type) |
1125 TXPKT_CSUM_START_V(start) |
1126 TXPKT_CSUM_LOC_V(start + skb->csum_offset);
1127 }
1128 }
1129
1130 /*
1131 * Stop an Ethernet TX queue and record that state change.
1132 */
1133 static void txq_stop(struct sge_eth_txq *txq)
1134 {
1135 netif_tx_stop_queue(txq->txq);
1136 txq->q.stops++;
1137 }
1138
1139 /*
1140 * Advance our software state for a TX queue by adding n in use descriptors.
1141 */
1142 static inline void txq_advance(struct sge_txq *tq, unsigned int n)
1143 {
1144 tq->in_use += n;
1145 tq->pidx += n;
1146 if (tq->pidx >= tq->size)
1147 tq->pidx -= tq->size;
1148 }
1149
1150 /**
1151 * t4vf_eth_xmit - add a packet to an Ethernet TX queue
1152 * @skb: the packet
1153 * @dev: the egress net device
1154 *
1155 * Add a packet to an SGE Ethernet TX queue. Runs with softirqs disabled.
1156 */
1157 int t4vf_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1158 {
1159 u32 wr_mid;
1160 u64 cntrl, *end;
1161 int qidx, credits, max_pkt_len;
1162 unsigned int flits, ndesc;
1163 struct adapter *adapter;
1164 struct sge_eth_txq *txq;
1165 const struct port_info *pi;
1166 struct fw_eth_tx_pkt_vm_wr *wr;
1167 struct cpl_tx_pkt_core *cpl;
1168 const struct skb_shared_info *ssi;
1169 dma_addr_t addr[MAX_SKB_FRAGS + 1];
1170 const size_t fw_hdr_copy_len = (sizeof(wr->ethmacdst) +
1171 sizeof(wr->ethmacsrc) +
1172 sizeof(wr->ethtype) +
1173 sizeof(wr->vlantci));
1174
1175 /*
1176 * The chip minimum packet length is 10 octets but the firmware
1177 * command that we are using requires that we copy the Ethernet header
1178 * (including the VLAN tag) into the header so we reject anything
1179 * smaller than that ...
1180 */
1181 if (unlikely(skb->len < fw_hdr_copy_len))
1182 goto out_free;
1183
1184 /* Discard the packet if the length is greater than mtu */
1185 max_pkt_len = ETH_HLEN + dev->mtu;
1186 if (skb_vlan_tagged(skb))
1187 max_pkt_len += VLAN_HLEN;
1188 if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
1189 goto out_free;
1190
1191 /*
1192 * Figure out which TX Queue we're going to use.
1193 */
1194 pi = netdev_priv(dev);
1195 adapter = pi->adapter;
1196 qidx = skb_get_queue_mapping(skb);
1197 BUG_ON(qidx >= pi->nqsets);
1198 txq = &adapter->sge.ethtxq[pi->first_qset + qidx];
1199
1200 if (pi->vlan_id && !skb_vlan_tag_present(skb))
1201 __vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q),
1202 pi->vlan_id);
1203
1204 /*
1205 * Take this opportunity to reclaim any TX Descriptors whose DMA
1206 * transfers have completed.
1207 */
1208 reclaim_completed_tx(adapter, &txq->q, true);
1209
1210 /*
1211 * Calculate the number of flits and TX Descriptors we're going to
1212 * need along with how many TX Descriptors will be left over after
1213 * we inject our Work Request.
1214 */
1215 flits = calc_tx_flits(skb);
1216 ndesc = flits_to_desc(flits);
1217 credits = txq_avail(&txq->q) - ndesc;
1218
1219 if (unlikely(credits < 0)) {
1220 /*
1221 * Not enough room for this packet's Work Request. Stop the
1222 * TX Queue and return a "busy" condition. The queue will get
1223 * started later on when the firmware informs us that space
1224 * has opened up.
1225 */
1226 txq_stop(txq);
1227 dev_err(adapter->pdev_dev,
1228 "%s: TX ring %u full while queue awake!\n",
1229 dev->name, qidx);
1230 return NETDEV_TX_BUSY;
1231 }
1232
1233 if (!is_eth_imm(skb) &&
1234 unlikely(map_skb(adapter->pdev_dev, skb, addr) < 0)) {
1235 /*
1236 * We need to map the skb into PCI DMA space (because it can't
1237 * be in-lined directly into the Work Request) and the mapping
1238 * operation failed. Record the error and drop the packet.
1239 */
1240 txq->mapping_err++;
1241 goto out_free;
1242 }
1243
1244 wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1245 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1246 /*
1247 * After we're done injecting the Work Request for this
1248 * packet, we'll be below our "stop threshold" so stop the TX
1249 * Queue now and schedule a request for an SGE Egress Queue
1250 * Update message. The queue will get started later on when
1251 * the firmware processes this Work Request and sends us an
1252 * Egress Queue Status Update message indicating that space
1253 * has opened up.
1254 */
1255 txq_stop(txq);
1256 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1257 }
1258
1259 /*
1260 * Start filling in our Work Request. Note that we do _not_ handle
1261 * the WR Header wrapping around the TX Descriptor Ring. If our
1262 * maximum header size ever exceeds one TX Descriptor, we'll need to
1263 * do something else here.
1264 */
1265 BUG_ON(DIV_ROUND_UP(ETHTXQ_MAX_HDR, TXD_PER_EQ_UNIT) > 1);
1266 wr = (void *)&txq->q.desc[txq->q.pidx];
1267 wr->equiq_to_len16 = cpu_to_be32(wr_mid);
1268 wr->r3[0] = cpu_to_be32(0);
1269 wr->r3[1] = cpu_to_be32(0);
1270 skb_copy_from_linear_data(skb, (void *)wr->ethmacdst, fw_hdr_copy_len);
1271 end = (u64 *)wr + flits;
1272
1273 /*
1274 * If this is a Large Send Offload packet we'll put in an LSO CPL
1275 * message with an encapsulated TX Packet CPL message. Otherwise we
1276 * just use a TX Packet CPL message.
1277 */
1278 ssi = skb_shinfo(skb);
1279 if (ssi->gso_size) {
1280 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
1281 bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1282 int l3hdr_len = skb_network_header_len(skb);
1283 int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1284
1285 wr->op_immdlen =
1286 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1287 FW_WR_IMMDLEN_V(sizeof(*lso) +
1288 sizeof(*cpl)));
1289 /*
1290 * Fill in the LSO CPL message.
1291 */
1292 lso->lso_ctrl =
1293 cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
1294 LSO_FIRST_SLICE_F |
1295 LSO_LAST_SLICE_F |
1296 LSO_IPV6_V(v6) |
1297 LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
1298 LSO_IPHDR_LEN_V(l3hdr_len / 4) |
1299 LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1300 lso->ipid_ofst = cpu_to_be16(0);
1301 lso->mss = cpu_to_be16(ssi->gso_size);
1302 lso->seqno_offset = cpu_to_be32(0);
1303 if (is_t4(adapter->params.chip))
1304 lso->len = cpu_to_be32(skb->len);
1305 else
1306 lso->len = cpu_to_be32(LSO_T5_XFER_SIZE_V(skb->len));
1307
1308 /*
1309 * Set up TX Packet CPL pointer, control word and perform
1310 * accounting.
1311 */
1312 cpl = (void *)(lso + 1);
1313
1314 if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
1315 cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1316 else
1317 cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1318
1319 cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
1320 TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
1321 TXPKT_IPHDR_LEN_V(l3hdr_len);
1322 txq->tso++;
1323 txq->tx_cso += ssi->gso_segs;
1324 } else {
1325 int len;
1326
1327 len = is_eth_imm(skb) ? skb->len + sizeof(*cpl) : sizeof(*cpl);
1328 wr->op_immdlen =
1329 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1330 FW_WR_IMMDLEN_V(len));
1331
1332 /*
1333 * Set up TX Packet CPL pointer, control word and perform
1334 * accounting.
1335 */
1336 cpl = (void *)(wr + 1);
1337 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1338 cntrl = hwcsum(adapter->params.chip, skb) |
1339 TXPKT_IPCSUM_DIS_F;
1340 txq->tx_cso++;
1341 } else
1342 cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
1343 }
1344
1345 /*
1346 * If there's a VLAN tag present, add that to the list of things to
1347 * do in this Work Request.
1348 */
1349 if (skb_vlan_tag_present(skb)) {
1350 txq->vlan_ins++;
1351 cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
1352 }
1353
1354 /*
1355 * Fill in the TX Packet CPL message header.
1356 */
1357 cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT) |
1358 TXPKT_INTF_V(pi->port_id) |
1359 TXPKT_PF_V(0));
1360 cpl->pack = cpu_to_be16(0);
1361 cpl->len = cpu_to_be16(skb->len);
1362 cpl->ctrl1 = cpu_to_be64(cntrl);
1363
1364 #ifdef T4_TRACE
1365 T4_TRACE5(adapter->tb[txq->q.cntxt_id & 7],
1366 "eth_xmit: ndesc %u, credits %u, pidx %u, len %u, frags %u",
1367 ndesc, credits, txq->q.pidx, skb->len, ssi->nr_frags);
1368 #endif
1369
1370 /*
1371 * Fill in the body of the TX Packet CPL message with either in-lined
1372 * data or a Scatter/Gather List.
1373 */
1374 if (is_eth_imm(skb)) {
1375 /*
1376 * In-line the packet's data and free the skb since we don't
1377 * need it any longer.
1378 */
1379 inline_tx_skb(skb, &txq->q, cpl + 1);
1380 dev_consume_skb_any(skb);
1381 } else {
1382 /*
1383 * Write the skb's Scatter/Gather list into the TX Packet CPL
1384 * message and retain a pointer to the skb so we can free it
1385 * later when its DMA completes. (We store the skb pointer
1386 * in the Software Descriptor corresponding to the last TX
1387 * Descriptor used by the Work Request.)
1388 *
1389 * The retained skb will be freed when the corresponding TX
1390 * Descriptors are reclaimed after their DMAs complete.
1391 * However, this could take quite a while since, in general,
1392 * the hardware is set up to be lazy about sending DMA
1393 * completion notifications to us and we mostly perform TX
1394 * reclaims in the transmit routine.
1395 *
1396 * This is good for performamce but means that we rely on new
1397 * TX packets arriving to run the destructors of completed
1398 * packets, which open up space in their sockets' send queues.
1399 * Sometimes we do not get such new packets causing TX to
1400 * stall. A single UDP transmitter is a good example of this
1401 * situation. We have a clean up timer that periodically
1402 * reclaims completed packets but it doesn't run often enough
1403 * (nor do we want it to) to prevent lengthy stalls. A
1404 * solution to this problem is to run the destructor early,
1405 * after the packet is queued but before it's DMAd. A con is
1406 * that we lie to socket memory accounting, but the amount of
1407 * extra memory is reasonable (limited by the number of TX
1408 * descriptors), the packets do actually get freed quickly by
1409 * new packets almost always, and for protocols like TCP that
1410 * wait for acks to really free up the data the extra memory
1411 * is even less. On the positive side we run the destructors
1412 * on the sending CPU rather than on a potentially different
1413 * completing CPU, usually a good thing.
1414 *
1415 * Run the destructor before telling the DMA engine about the
1416 * packet to make sure it doesn't complete and get freed
1417 * prematurely.
1418 */
1419 struct ulptx_sgl *sgl = (struct ulptx_sgl *)(cpl + 1);
1420 struct sge_txq *tq = &txq->q;
1421 int last_desc;
1422
1423 /*
1424 * If the Work Request header was an exact multiple of our TX
1425 * Descriptor length, then it's possible that the starting SGL
1426 * pointer lines up exactly with the end of our TX Descriptor
1427 * ring. If that's the case, wrap around to the beginning
1428 * here ...
1429 */
1430 if (unlikely((void *)sgl == (void *)tq->stat)) {
1431 sgl = (void *)tq->desc;
1432 end = ((void *)tq->desc + ((void *)end - (void *)tq->stat));
1433 }
1434
1435 write_sgl(skb, tq, sgl, end, 0, addr);
1436 skb_orphan(skb);
1437
1438 last_desc = tq->pidx + ndesc - 1;
1439 if (last_desc >= tq->size)
1440 last_desc -= tq->size;
1441 tq->sdesc[last_desc].skb = skb;
1442 tq->sdesc[last_desc].sgl = sgl;
1443 }
1444
1445 /*
1446 * Advance our internal TX Queue state, tell the hardware about
1447 * the new TX descriptors and return success.
1448 */
1449 txq_advance(&txq->q, ndesc);
1450 netif_trans_update(dev);
1451 ring_tx_db(adapter, &txq->q, ndesc);
1452 return NETDEV_TX_OK;
1453
1454 out_free:
1455 /*
1456 * An error of some sort happened. Free the TX skb and tell the
1457 * OS that we've "dealt" with the packet ...
1458 */
1459 dev_kfree_skb_any(skb);
1460 return NETDEV_TX_OK;
1461 }
1462
1463 /**
1464 * copy_frags - copy fragments from gather list into skb_shared_info
1465 * @skb: destination skb
1466 * @gl: source internal packet gather list
1467 * @offset: packet start offset in first page
1468 *
1469 * Copy an internal packet gather list into a Linux skb_shared_info
1470 * structure.
1471 */
1472 static inline void copy_frags(struct sk_buff *skb,
1473 const struct pkt_gl *gl,
1474 unsigned int offset)
1475 {
1476 int i;
1477
1478 /* usually there's just one frag */
1479 __skb_fill_page_desc(skb, 0, gl->frags[0].page,
1480 gl->frags[0].offset + offset,
1481 gl->frags[0].size - offset);
1482 skb_shinfo(skb)->nr_frags = gl->nfrags;
1483 for (i = 1; i < gl->nfrags; i++)
1484 __skb_fill_page_desc(skb, i, gl->frags[i].page,
1485 gl->frags[i].offset,
1486 gl->frags[i].size);
1487
1488 /* get a reference to the last page, we don't own it */
1489 get_page(gl->frags[gl->nfrags - 1].page);
1490 }
1491
1492 /**
1493 * t4vf_pktgl_to_skb - build an sk_buff from a packet gather list
1494 * @gl: the gather list
1495 * @skb_len: size of sk_buff main body if it carries fragments
1496 * @pull_len: amount of data to move to the sk_buff's main body
1497 *
1498 * Builds an sk_buff from the given packet gather list. Returns the
1499 * sk_buff or %NULL if sk_buff allocation failed.
1500 */
1501 static struct sk_buff *t4vf_pktgl_to_skb(const struct pkt_gl *gl,
1502 unsigned int skb_len,
1503 unsigned int pull_len)
1504 {
1505 struct sk_buff *skb;
1506
1507 /*
1508 * If the ingress packet is small enough, allocate an skb large enough
1509 * for all of the data and copy it inline. Otherwise, allocate an skb
1510 * with enough room to pull in the header and reference the rest of
1511 * the data via the skb fragment list.
1512 *
1513 * Below we rely on RX_COPY_THRES being less than the smallest Rx
1514 * buff! size, which is expected since buffers are at least
1515 * PAGE_SIZEd. In this case packets up to RX_COPY_THRES have only one
1516 * fragment.
1517 */
1518 if (gl->tot_len <= RX_COPY_THRES) {
1519 /* small packets have only one fragment */
1520 skb = alloc_skb(gl->tot_len, GFP_ATOMIC);
1521 if (unlikely(!skb))
1522 goto out;
1523 __skb_put(skb, gl->tot_len);
1524 skb_copy_to_linear_data(skb, gl->va, gl->tot_len);
1525 } else {
1526 skb = alloc_skb(skb_len, GFP_ATOMIC);
1527 if (unlikely(!skb))
1528 goto out;
1529 __skb_put(skb, pull_len);
1530 skb_copy_to_linear_data(skb, gl->va, pull_len);
1531
1532 copy_frags(skb, gl, pull_len);
1533 skb->len = gl->tot_len;
1534 skb->data_len = skb->len - pull_len;
1535 skb->truesize += skb->data_len;
1536 }
1537
1538 out:
1539 return skb;
1540 }
1541
1542 /**
1543 * t4vf_pktgl_free - free a packet gather list
1544 * @gl: the gather list
1545 *
1546 * Releases the pages of a packet gather list. We do not own the last
1547 * page on the list and do not free it.
1548 */
1549 static void t4vf_pktgl_free(const struct pkt_gl *gl)
1550 {
1551 int frag;
1552
1553 frag = gl->nfrags - 1;
1554 while (frag--)
1555 put_page(gl->frags[frag].page);
1556 }
1557
1558 /**
1559 * do_gro - perform Generic Receive Offload ingress packet processing
1560 * @rxq: ingress RX Ethernet Queue
1561 * @gl: gather list for ingress packet
1562 * @pkt: CPL header for last packet fragment
1563 *
1564 * Perform Generic Receive Offload (GRO) ingress packet processing.
1565 * We use the standard Linux GRO interfaces for this.
1566 */
1567 static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
1568 const struct cpl_rx_pkt *pkt)
1569 {
1570 struct adapter *adapter = rxq->rspq.adapter;
1571 struct sge *s = &adapter->sge;
1572 struct port_info *pi;
1573 int ret;
1574 struct sk_buff *skb;
1575
1576 skb = napi_get_frags(&rxq->rspq.napi);
1577 if (unlikely(!skb)) {
1578 t4vf_pktgl_free(gl);
1579 rxq->stats.rx_drops++;
1580 return;
1581 }
1582
1583 copy_frags(skb, gl, s->pktshift);
1584 skb->len = gl->tot_len - s->pktshift;
1585 skb->data_len = skb->len;
1586 skb->truesize += skb->data_len;
1587 skb->ip_summed = CHECKSUM_UNNECESSARY;
1588 skb_record_rx_queue(skb, rxq->rspq.idx);
1589 pi = netdev_priv(skb->dev);
1590
1591 if (pkt->vlan_ex && !pi->vlan_id) {
1592 __vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q),
1593 be16_to_cpu(pkt->vlan));
1594 rxq->stats.vlan_ex++;
1595 }
1596 ret = napi_gro_frags(&rxq->rspq.napi);
1597
1598 if (ret == GRO_HELD)
1599 rxq->stats.lro_pkts++;
1600 else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE)
1601 rxq->stats.lro_merged++;
1602 rxq->stats.pkts++;
1603 rxq->stats.rx_cso++;
1604 }
1605
1606 /**
1607 * t4vf_ethrx_handler - process an ingress ethernet packet
1608 * @rspq: the response queue that received the packet
1609 * @rsp: the response queue descriptor holding the RX_PKT message
1610 * @gl: the gather list of packet fragments
1611 *
1612 * Process an ingress ethernet packet and deliver it to the stack.
1613 */
1614 int t4vf_ethrx_handler(struct sge_rspq *rspq, const __be64 *rsp,
1615 const struct pkt_gl *gl)
1616 {
1617 struct sk_buff *skb;
1618 const struct cpl_rx_pkt *pkt = (void *)rsp;
1619 bool csum_ok = pkt->csum_calc && !pkt->err_vec &&
1620 (rspq->netdev->features & NETIF_F_RXCSUM);
1621 struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq);
1622 struct adapter *adapter = rspq->adapter;
1623 struct sge *s = &adapter->sge;
1624 struct port_info *pi;
1625
1626 /*
1627 * If this is a good TCP packet and we have Generic Receive Offload
1628 * enabled, handle the packet in the GRO path.
1629 */
1630 if ((pkt->l2info & cpu_to_be32(RXF_TCP_F)) &&
1631 (rspq->netdev->features & NETIF_F_GRO) && csum_ok &&
1632 !pkt->ip_frag) {
1633 do_gro(rxq, gl, pkt);
1634 return 0;
1635 }
1636
1637 /*
1638 * Convert the Packet Gather List into an skb.
1639 */
1640 skb = t4vf_pktgl_to_skb(gl, RX_SKB_LEN, RX_PULL_LEN);
1641 if (unlikely(!skb)) {
1642 t4vf_pktgl_free(gl);
1643 rxq->stats.rx_drops++;
1644 return 0;
1645 }
1646 __skb_pull(skb, s->pktshift);
1647 skb->protocol = eth_type_trans(skb, rspq->netdev);
1648 skb_record_rx_queue(skb, rspq->idx);
1649 pi = netdev_priv(skb->dev);
1650 rxq->stats.pkts++;
1651
1652 if (csum_ok && !pkt->err_vec &&
1653 (be32_to_cpu(pkt->l2info) & (RXF_UDP_F | RXF_TCP_F))) {
1654 if (!pkt->ip_frag) {
1655 skb->ip_summed = CHECKSUM_UNNECESSARY;
1656 rxq->stats.rx_cso++;
1657 } else if (pkt->l2info & htonl(RXF_IP_F)) {
1658 __sum16 c = (__force __sum16)pkt->csum;
1659 skb->csum = csum_unfold(c);
1660 skb->ip_summed = CHECKSUM_COMPLETE;
1661 rxq->stats.rx_cso++;
1662 }
1663 } else
1664 skb_checksum_none_assert(skb);
1665
1666 if (pkt->vlan_ex && !pi->vlan_id) {
1667 rxq->stats.vlan_ex++;
1668 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1669 be16_to_cpu(pkt->vlan));
1670 }
1671
1672 netif_receive_skb(skb);
1673
1674 return 0;
1675 }
1676
1677 /**
1678 * is_new_response - check if a response is newly written
1679 * @rc: the response control descriptor
1680 * @rspq: the response queue
1681 *
1682 * Returns true if a response descriptor contains a yet unprocessed
1683 * response.
1684 */
1685 static inline bool is_new_response(const struct rsp_ctrl *rc,
1686 const struct sge_rspq *rspq)
1687 {
1688 return ((rc->type_gen >> RSPD_GEN_S) & 0x1) == rspq->gen;
1689 }
1690
1691 /**
1692 * restore_rx_bufs - put back a packet's RX buffers
1693 * @gl: the packet gather list
1694 * @fl: the SGE Free List
1695 * @nfrags: how many fragments in @si
1696 *
1697 * Called when we find out that the current packet, @si, can't be
1698 * processed right away for some reason. This is a very rare event and
1699 * there's no effort to make this suspension/resumption process
1700 * particularly efficient.
1701 *
1702 * We implement the suspension by putting all of the RX buffers associated
1703 * with the current packet back on the original Free List. The buffers
1704 * have already been unmapped and are left unmapped, we mark them as
1705 * unmapped in order to prevent further unmapping attempts. (Effectively
1706 * this function undoes the series of @unmap_rx_buf calls which were done
1707 * to create the current packet's gather list.) This leaves us ready to
1708 * restart processing of the packet the next time we start processing the
1709 * RX Queue ...
1710 */
1711 static void restore_rx_bufs(const struct pkt_gl *gl, struct sge_fl *fl,
1712 int frags)
1713 {
1714 struct rx_sw_desc *sdesc;
1715
1716 while (frags--) {
1717 if (fl->cidx == 0)
1718 fl->cidx = fl->size - 1;
1719 else
1720 fl->cidx--;
1721 sdesc = &fl->sdesc[fl->cidx];
1722 sdesc->page = gl->frags[frags].page;
1723 sdesc->dma_addr |= RX_UNMAPPED_BUF;
1724 fl->avail++;
1725 }
1726 }
1727
1728 /**
1729 * rspq_next - advance to the next entry in a response queue
1730 * @rspq: the queue
1731 *
1732 * Updates the state of a response queue to advance it to the next entry.
1733 */
1734 static inline void rspq_next(struct sge_rspq *rspq)
1735 {
1736 rspq->cur_desc = (void *)rspq->cur_desc + rspq->iqe_len;
1737 if (unlikely(++rspq->cidx == rspq->size)) {
1738 rspq->cidx = 0;
1739 rspq->gen ^= 1;
1740 rspq->cur_desc = rspq->desc;
1741 }
1742 }
1743
1744 /**
1745 * process_responses - process responses from an SGE response queue
1746 * @rspq: the ingress response queue to process
1747 * @budget: how many responses can be processed in this round
1748 *
1749 * Process responses from a Scatter Gather Engine response queue up to
1750 * the supplied budget. Responses include received packets as well as
1751 * control messages from firmware or hardware.
1752 *
1753 * Additionally choose the interrupt holdoff time for the next interrupt
1754 * on this queue. If the system is under memory shortage use a fairly
1755 * long delay to help recovery.
1756 */
1757 static int process_responses(struct sge_rspq *rspq, int budget)
1758 {
1759 struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq);
1760 struct adapter *adapter = rspq->adapter;
1761 struct sge *s = &adapter->sge;
1762 int budget_left = budget;
1763
1764 while (likely(budget_left)) {
1765 int ret, rsp_type;
1766 const struct rsp_ctrl *rc;
1767
1768 rc = (void *)rspq->cur_desc + (rspq->iqe_len - sizeof(*rc));
1769 if (!is_new_response(rc, rspq))
1770 break;
1771
1772 /*
1773 * Figure out what kind of response we've received from the
1774 * SGE.
1775 */
1776 dma_rmb();
1777 rsp_type = RSPD_TYPE_G(rc->type_gen);
1778 if (likely(rsp_type == RSPD_TYPE_FLBUF_X)) {
1779 struct page_frag *fp;
1780 struct pkt_gl gl;
1781 const struct rx_sw_desc *sdesc;
1782 u32 bufsz, frag;
1783 u32 len = be32_to_cpu(rc->pldbuflen_qid);
1784
1785 /*
1786 * If we get a "new buffer" message from the SGE we
1787 * need to move on to the next Free List buffer.
1788 */
1789 if (len & RSPD_NEWBUF_F) {
1790 /*
1791 * We get one "new buffer" message when we
1792 * first start up a queue so we need to ignore
1793 * it when our offset into the buffer is 0.
1794 */
1795 if (likely(rspq->offset > 0)) {
1796 free_rx_bufs(rspq->adapter, &rxq->fl,
1797 1);
1798 rspq->offset = 0;
1799 }
1800 len = RSPD_LEN_G(len);
1801 }
1802 gl.tot_len = len;
1803
1804 /*
1805 * Gather packet fragments.
1806 */
1807 for (frag = 0, fp = gl.frags; /**/; frag++, fp++) {
1808 BUG_ON(frag >= MAX_SKB_FRAGS);
1809 BUG_ON(rxq->fl.avail == 0);
1810 sdesc = &rxq->fl.sdesc[rxq->fl.cidx];
1811 bufsz = get_buf_size(adapter, sdesc);
1812 fp->page = sdesc->page;
1813 fp->offset = rspq->offset;
1814 fp->size = min(bufsz, len);
1815 len -= fp->size;
1816 if (!len)
1817 break;
1818 unmap_rx_buf(rspq->adapter, &rxq->fl);
1819 }
1820 gl.nfrags = frag+1;
1821
1822 /*
1823 * Last buffer remains mapped so explicitly make it
1824 * coherent for CPU access and start preloading first
1825 * cache line ...
1826 */
1827 dma_sync_single_for_cpu(rspq->adapter->pdev_dev,
1828 get_buf_addr(sdesc),
1829 fp->size, DMA_FROM_DEVICE);
1830 gl.va = (page_address(gl.frags[0].page) +
1831 gl.frags[0].offset);
1832 prefetch(gl.va);
1833
1834 /*
1835 * Hand the new ingress packet to the handler for
1836 * this Response Queue.
1837 */
1838 ret = rspq->handler(rspq, rspq->cur_desc, &gl);
1839 if (likely(ret == 0))
1840 rspq->offset += ALIGN(fp->size, s->fl_align);
1841 else
1842 restore_rx_bufs(&gl, &rxq->fl, frag);
1843 } else if (likely(rsp_type == RSPD_TYPE_CPL_X)) {
1844 ret = rspq->handler(rspq, rspq->cur_desc, NULL);
1845 } else {
1846 WARN_ON(rsp_type > RSPD_TYPE_CPL_X);
1847 ret = 0;
1848 }
1849
1850 if (unlikely(ret)) {
1851 /*
1852 * Couldn't process descriptor, back off for recovery.
1853 * We use the SGE's last timer which has the longest
1854 * interrupt coalescing value ...
1855 */
1856 const int NOMEM_TIMER_IDX = SGE_NTIMERS-1;
1857 rspq->next_intr_params =
1858 QINTR_TIMER_IDX_V(NOMEM_TIMER_IDX);
1859 break;
1860 }
1861
1862 rspq_next(rspq);
1863 budget_left--;
1864 }
1865
1866 /*
1867 * If this is a Response Queue with an associated Free List and
1868 * at least two Egress Queue units available in the Free List
1869 * for new buffer pointers, refill the Free List.
1870 */
1871 if (rspq->offset >= 0 &&
1872 fl_cap(&rxq->fl) - rxq->fl.avail >= 2*FL_PER_EQ_UNIT)
1873 __refill_fl(rspq->adapter, &rxq->fl);
1874 return budget - budget_left;
1875 }
1876
1877 /**
1878 * napi_rx_handler - the NAPI handler for RX processing
1879 * @napi: the napi instance
1880 * @budget: how many packets we can process in this round
1881 *
1882 * Handler for new data events when using NAPI. This does not need any
1883 * locking or protection from interrupts as data interrupts are off at
1884 * this point and other adapter interrupts do not interfere (the latter
1885 * in not a concern at all with MSI-X as non-data interrupts then have
1886 * a separate handler).
1887 */
1888 static int napi_rx_handler(struct napi_struct *napi, int budget)
1889 {
1890 unsigned int intr_params;
1891 struct sge_rspq *rspq = container_of(napi, struct sge_rspq, napi);
1892 int work_done = process_responses(rspq, budget);
1893 u32 val;
1894
1895 if (likely(work_done < budget)) {
1896 napi_complete_done(napi, work_done);
1897 intr_params = rspq->next_intr_params;
1898 rspq->next_intr_params = rspq->intr_params;
1899 } else
1900 intr_params = QINTR_TIMER_IDX_V(SGE_TIMER_UPD_CIDX);
1901
1902 if (unlikely(work_done == 0))
1903 rspq->unhandled_irqs++;
1904
1905 val = CIDXINC_V(work_done) | SEINTARM_V(intr_params);
1906 /* If we don't have access to the new User GTS (T5+), use the old
1907 * doorbell mechanism; otherwise use the new BAR2 mechanism.
1908 */
1909 if (unlikely(!rspq->bar2_addr)) {
1910 t4_write_reg(rspq->adapter,
1911 T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
1912 val | INGRESSQID_V((u32)rspq->cntxt_id));
1913 } else {
1914 writel(val | INGRESSQID_V(rspq->bar2_qid),
1915 rspq->bar2_addr + SGE_UDB_GTS);
1916 wmb();
1917 }
1918 return work_done;
1919 }
1920
1921 /*
1922 * The MSI-X interrupt handler for an SGE response queue for the NAPI case
1923 * (i.e., response queue serviced by NAPI polling).
1924 */
1925 irqreturn_t t4vf_sge_intr_msix(int irq, void *cookie)
1926 {
1927 struct sge_rspq *rspq = cookie;
1928
1929 napi_schedule(&rspq->napi);
1930 return IRQ_HANDLED;
1931 }
1932
1933 /*
1934 * Process the indirect interrupt entries in the interrupt queue and kick off
1935 * NAPI for each queue that has generated an entry.
1936 */
1937 static unsigned int process_intrq(struct adapter *adapter)
1938 {
1939 struct sge *s = &adapter->sge;
1940 struct sge_rspq *intrq = &s->intrq;
1941 unsigned int work_done;
1942 u32 val;
1943
1944 spin_lock(&adapter->sge.intrq_lock);
1945 for (work_done = 0; ; work_done++) {
1946 const struct rsp_ctrl *rc;
1947 unsigned int qid, iq_idx;
1948 struct sge_rspq *rspq;
1949
1950 /*
1951 * Grab the next response from the interrupt queue and bail
1952 * out if it's not a new response.
1953 */
1954 rc = (void *)intrq->cur_desc + (intrq->iqe_len - sizeof(*rc));
1955 if (!is_new_response(rc, intrq))
1956 break;
1957
1958 /*
1959 * If the response isn't a forwarded interrupt message issue a
1960 * error and go on to the next response message. This should
1961 * never happen ...
1962 */
1963 dma_rmb();
1964 if (unlikely(RSPD_TYPE_G(rc->type_gen) != RSPD_TYPE_INTR_X)) {
1965 dev_err(adapter->pdev_dev,
1966 "Unexpected INTRQ response type %d\n",
1967 RSPD_TYPE_G(rc->type_gen));
1968 continue;
1969 }
1970
1971 /*
1972 * Extract the Queue ID from the interrupt message and perform
1973 * sanity checking to make sure it really refers to one of our
1974 * Ingress Queues which is active and matches the queue's ID.
1975 * None of these error conditions should ever happen so we may
1976 * want to either make them fatal and/or conditionalized under
1977 * DEBUG.
1978 */
1979 qid = RSPD_QID_G(be32_to_cpu(rc->pldbuflen_qid));
1980 iq_idx = IQ_IDX(s, qid);
1981 if (unlikely(iq_idx >= MAX_INGQ)) {
1982 dev_err(adapter->pdev_dev,
1983 "Ingress QID %d out of range\n", qid);
1984 continue;
1985 }
1986 rspq = s->ingr_map[iq_idx];
1987 if (unlikely(rspq == NULL)) {
1988 dev_err(adapter->pdev_dev,
1989 "Ingress QID %d RSPQ=NULL\n", qid);
1990 continue;
1991 }
1992 if (unlikely(rspq->abs_id != qid)) {
1993 dev_err(adapter->pdev_dev,
1994 "Ingress QID %d refers to RSPQ %d\n",
1995 qid, rspq->abs_id);
1996 continue;
1997 }
1998
1999 /*
2000 * Schedule NAPI processing on the indicated Response Queue
2001 * and move on to the next entry in the Forwarded Interrupt
2002 * Queue.
2003 */
2004 napi_schedule(&rspq->napi);
2005 rspq_next(intrq);
2006 }
2007
2008 val = CIDXINC_V(work_done) | SEINTARM_V(intrq->intr_params);
2009 /* If we don't have access to the new User GTS (T5+), use the old
2010 * doorbell mechanism; otherwise use the new BAR2 mechanism.
2011 */
2012 if (unlikely(!intrq->bar2_addr)) {
2013 t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
2014 val | INGRESSQID_V(intrq->cntxt_id));
2015 } else {
2016 writel(val | INGRESSQID_V(intrq->bar2_qid),
2017 intrq->bar2_addr + SGE_UDB_GTS);
2018 wmb();
2019 }
2020
2021 spin_unlock(&adapter->sge.intrq_lock);
2022
2023 return work_done;
2024 }
2025
2026 /*
2027 * The MSI interrupt handler handles data events from SGE response queues as
2028 * well as error and other async events as they all use the same MSI vector.
2029 */
2030 static irqreturn_t t4vf_intr_msi(int irq, void *cookie)
2031 {
2032 struct adapter *adapter = cookie;
2033
2034 process_intrq(adapter);
2035 return IRQ_HANDLED;
2036 }
2037
2038 /**
2039 * t4vf_intr_handler - select the top-level interrupt handler
2040 * @adapter: the adapter
2041 *
2042 * Selects the top-level interrupt handler based on the type of interrupts
2043 * (MSI-X or MSI).
2044 */
2045 irq_handler_t t4vf_intr_handler(struct adapter *adapter)
2046 {
2047 BUG_ON((adapter->flags &
2048 (CXGB4VF_USING_MSIX | CXGB4VF_USING_MSI)) == 0);
2049 if (adapter->flags & CXGB4VF_USING_MSIX)
2050 return t4vf_sge_intr_msix;
2051 else
2052 return t4vf_intr_msi;
2053 }
2054
2055 /**
2056 * sge_rx_timer_cb - perform periodic maintenance of SGE RX queues
2057 * @data: the adapter
2058 *
2059 * Runs periodically from a timer to perform maintenance of SGE RX queues.
2060 *
2061 * a) Replenishes RX queues that have run out due to memory shortage.
2062 * Normally new RX buffers are added when existing ones are consumed but
2063 * when out of memory a queue can become empty. We schedule NAPI to do
2064 * the actual refill.
2065 */
2066 static void sge_rx_timer_cb(struct timer_list *t)
2067 {
2068 struct adapter *adapter = from_timer(adapter, t, sge.rx_timer);
2069 struct sge *s = &adapter->sge;
2070 unsigned int i;
2071
2072 /*
2073 * Scan the "Starving Free Lists" flag array looking for any Free
2074 * Lists in need of more free buffers. If we find one and it's not
2075 * being actively polled, then bump its "starving" counter and attempt
2076 * to refill it. If we're successful in adding enough buffers to push
2077 * the Free List over the starving threshold, then we can clear its
2078 * "starving" status.
2079 */
2080 for (i = 0; i < ARRAY_SIZE(s->starving_fl); i++) {
2081 unsigned long m;
2082
2083 for (m = s->starving_fl[i]; m; m &= m - 1) {
2084 unsigned int id = __ffs(m) + i * BITS_PER_LONG;
2085 struct sge_fl *fl = s->egr_map[id];
2086
2087 clear_bit(id, s->starving_fl);
2088 smp_mb__after_atomic();
2089
2090 /*
2091 * Since we are accessing fl without a lock there's a
2092 * small probability of a false positive where we
2093 * schedule napi but the FL is no longer starving.
2094 * No biggie.
2095 */
2096 if (fl_starving(adapter, fl)) {
2097 struct sge_eth_rxq *rxq;
2098
2099 rxq = container_of(fl, struct sge_eth_rxq, fl);
2100 if (napi_reschedule(&rxq->rspq.napi))
2101 fl->starving++;
2102 else
2103 set_bit(id, s->starving_fl);
2104 }
2105 }
2106 }
2107
2108 /*
2109 * Reschedule the next scan for starving Free Lists ...
2110 */
2111 mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD);
2112 }
2113
2114 /**
2115 * sge_tx_timer_cb - perform periodic maintenance of SGE Tx queues
2116 * @data: the adapter
2117 *
2118 * Runs periodically from a timer to perform maintenance of SGE TX queues.
2119 *
2120 * b) Reclaims completed Tx packets for the Ethernet queues. Normally
2121 * packets are cleaned up by new Tx packets, this timer cleans up packets
2122 * when no new packets are being submitted. This is essential for pktgen,
2123 * at least.
2124 */
2125 static void sge_tx_timer_cb(struct timer_list *t)
2126 {
2127 struct adapter *adapter = from_timer(adapter, t, sge.tx_timer);
2128 struct sge *s = &adapter->sge;
2129 unsigned int i, budget;
2130
2131 budget = MAX_TIMER_TX_RECLAIM;
2132 i = s->ethtxq_rover;
2133 do {
2134 struct sge_eth_txq *txq = &s->ethtxq[i];
2135
2136 if (reclaimable(&txq->q) && __netif_tx_trylock(txq->txq)) {
2137 int avail = reclaimable(&txq->q);
2138
2139 if (avail > budget)
2140 avail = budget;
2141
2142 free_tx_desc(adapter, &txq->q, avail, true);
2143 txq->q.in_use -= avail;
2144 __netif_tx_unlock(txq->txq);
2145
2146 budget -= avail;
2147 if (!budget)
2148 break;
2149 }
2150
2151 i++;
2152 if (i >= s->ethqsets)
2153 i = 0;
2154 } while (i != s->ethtxq_rover);
2155 s->ethtxq_rover = i;
2156
2157 /*
2158 * If we found too many reclaimable packets schedule a timer in the
2159 * near future to continue where we left off. Otherwise the next timer
2160 * will be at its normal interval.
2161 */
2162 mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2));
2163 }
2164
2165 /**
2166 * bar2_address - return the BAR2 address for an SGE Queue's Registers
2167 * @adapter: the adapter
2168 * @qid: the SGE Queue ID
2169 * @qtype: the SGE Queue Type (Egress or Ingress)
2170 * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
2171 *
2172 * Returns the BAR2 address for the SGE Queue Registers associated with
2173 * @qid. If BAR2 SGE Registers aren't available, returns NULL. Also
2174 * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE
2175 * Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID"
2176 * Registers are supported (e.g. the Write Combining Doorbell Buffer).
2177 */
2178 static void __iomem *bar2_address(struct adapter *adapter,
2179 unsigned int qid,
2180 enum t4_bar2_qtype qtype,
2181 unsigned int *pbar2_qid)
2182 {
2183 u64 bar2_qoffset;
2184 int ret;
2185
2186 ret = t4vf_bar2_sge_qregs(adapter, qid, qtype,
2187 &bar2_qoffset, pbar2_qid);
2188 if (ret)
2189 return NULL;
2190
2191 return adapter->bar2 + bar2_qoffset;
2192 }
2193
2194 /**
2195 * t4vf_sge_alloc_rxq - allocate an SGE RX Queue
2196 * @adapter: the adapter
2197 * @rspq: pointer to to the new rxq's Response Queue to be filled in
2198 * @iqasynch: if 0, a normal rspq; if 1, an asynchronous event queue
2199 * @dev: the network device associated with the new rspq
2200 * @intr_dest: MSI-X vector index (overriden in MSI mode)
2201 * @fl: pointer to the new rxq's Free List to be filled in
2202 * @hnd: the interrupt handler to invoke for the rspq
2203 */
2204 int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq,
2205 bool iqasynch, struct net_device *dev,
2206 int intr_dest,
2207 struct sge_fl *fl, rspq_handler_t hnd)
2208 {
2209 struct sge *s = &adapter->sge;
2210 struct port_info *pi = netdev_priv(dev);
2211 struct fw_iq_cmd cmd, rpl;
2212 int ret, iqandst, flsz = 0;
2213 int relaxed = !(adapter->flags & CXGB4VF_ROOT_NO_RELAXED_ORDERING);
2214
2215 /*
2216 * If we're using MSI interrupts and we're not initializing the
2217 * Forwarded Interrupt Queue itself, then set up this queue for
2218 * indirect interrupts to the Forwarded Interrupt Queue. Obviously
2219 * the Forwarded Interrupt Queue must be set up before any other
2220 * ingress queue ...
2221 */
2222 if ((adapter->flags & CXGB4VF_USING_MSI) &&
2223 rspq != &adapter->sge.intrq) {
2224 iqandst = SGE_INTRDST_IQ;
2225 intr_dest = adapter->sge.intrq.abs_id;
2226 } else
2227 iqandst = SGE_INTRDST_PCI;
2228
2229 /*
2230 * Allocate the hardware ring for the Response Queue. The size needs
2231 * to be a multiple of 16 which includes the mandatory status entry
2232 * (regardless of whether the Status Page capabilities are enabled or
2233 * not).
2234 */
2235 rspq->size = roundup(rspq->size, 16);
2236 rspq->desc = alloc_ring(adapter->pdev_dev, rspq->size, rspq->iqe_len,
2237 0, &rspq->phys_addr, NULL, 0);
2238 if (!rspq->desc)
2239 return -ENOMEM;
2240
2241 /*
2242 * Fill in the Ingress Queue Command. Note: Ideally this code would
2243 * be in t4vf_hw.c but there are so many parameters and dependencies
2244 * on our Linux SGE state that we would end up having to pass tons of
2245 * parameters. We'll have to think about how this might be migrated
2246 * into OS-independent common code ...
2247 */
2248 memset(&cmd, 0, sizeof(cmd));
2249 cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) |
2250 FW_CMD_REQUEST_F |
2251 FW_CMD_WRITE_F |
2252 FW_CMD_EXEC_F);
2253 cmd.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_ALLOC_F |
2254 FW_IQ_CMD_IQSTART_F |
2255 FW_LEN16(cmd));
2256 cmd.type_to_iqandstindex =
2257 cpu_to_be32(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP) |
2258 FW_IQ_CMD_IQASYNCH_V(iqasynch) |
2259 FW_IQ_CMD_VIID_V(pi->viid) |
2260 FW_IQ_CMD_IQANDST_V(iqandst) |
2261 FW_IQ_CMD_IQANUS_V(1) |
2262 FW_IQ_CMD_IQANUD_V(SGE_UPDATEDEL_INTR) |
2263 FW_IQ_CMD_IQANDSTINDEX_V(intr_dest));
2264 cmd.iqdroprss_to_iqesize =
2265 cpu_to_be16(FW_IQ_CMD_IQPCIECH_V(pi->port_id) |
2266 FW_IQ_CMD_IQGTSMODE_F |
2267 FW_IQ_CMD_IQINTCNTTHRESH_V(rspq->pktcnt_idx) |
2268 FW_IQ_CMD_IQESIZE_V(ilog2(rspq->iqe_len) - 4));
2269 cmd.iqsize = cpu_to_be16(rspq->size);
2270 cmd.iqaddr = cpu_to_be64(rspq->phys_addr);
2271
2272 if (fl) {
2273 unsigned int chip_ver =
2274 CHELSIO_CHIP_VERSION(adapter->params.chip);
2275 /*
2276 * Allocate the ring for the hardware free list (with space
2277 * for its status page) along with the associated software
2278 * descriptor ring. The free list size needs to be a multiple
2279 * of the Egress Queue Unit and at least 2 Egress Units larger
2280 * than the SGE's Egress Congrestion Threshold
2281 * (fl_starve_thres - 1).
2282 */
2283 if (fl->size < s->fl_starve_thres - 1 + 2 * FL_PER_EQ_UNIT)
2284 fl->size = s->fl_starve_thres - 1 + 2 * FL_PER_EQ_UNIT;
2285 fl->size = roundup(fl->size, FL_PER_EQ_UNIT);
2286 fl->desc = alloc_ring(adapter->pdev_dev, fl->size,
2287 sizeof(__be64), sizeof(struct rx_sw_desc),
2288 &fl->addr, &fl->sdesc, s->stat_len);
2289 if (!fl->desc) {
2290 ret = -ENOMEM;
2291 goto err;
2292 }
2293
2294 /*
2295 * Calculate the size of the hardware free list ring plus
2296 * Status Page (which the SGE will place after the end of the
2297 * free list ring) in Egress Queue Units.
2298 */
2299 flsz = (fl->size / FL_PER_EQ_UNIT +
2300 s->stat_len / EQ_UNIT);
2301
2302 /*
2303 * Fill in all the relevant firmware Ingress Queue Command
2304 * fields for the free list.
2305 */
2306 cmd.iqns_to_fl0congen =
2307 cpu_to_be32(
2308 FW_IQ_CMD_FL0HOSTFCMODE_V(SGE_HOSTFCMODE_NONE) |
2309 FW_IQ_CMD_FL0PACKEN_F |
2310 FW_IQ_CMD_FL0FETCHRO_V(relaxed) |
2311 FW_IQ_CMD_FL0DATARO_V(relaxed) |
2312 FW_IQ_CMD_FL0PADEN_F);
2313
2314 /* In T6, for egress queue type FL there is internal overhead
2315 * of 16B for header going into FLM module. Hence the maximum
2316 * allowed burst size is 448 bytes. For T4/T5, the hardware
2317 * doesn't coalesce fetch requests if more than 64 bytes of
2318 * Free List pointers are provided, so we use a 128-byte Fetch
2319 * Burst Minimum there (T6 implements coalescing so we can use
2320 * the smaller 64-byte value there).
2321 */
2322 cmd.fl0dcaen_to_fl0cidxfthresh =
2323 cpu_to_be16(
2324 FW_IQ_CMD_FL0FBMIN_V(chip_ver <= CHELSIO_T5
2325 ? FETCHBURSTMIN_128B_X
2326 : FETCHBURSTMIN_64B_T6_X) |
2327 FW_IQ_CMD_FL0FBMAX_V((chip_ver <= CHELSIO_T5) ?
2328 FETCHBURSTMAX_512B_X :
2329 FETCHBURSTMAX_256B_X));
2330 cmd.fl0size = cpu_to_be16(flsz);
2331 cmd.fl0addr = cpu_to_be64(fl->addr);
2332 }
2333
2334 /*
2335 * Issue the firmware Ingress Queue Command and extract the results if
2336 * it completes successfully.
2337 */
2338 ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);
2339 if (ret)
2340 goto err;
2341
2342 netif_napi_add(dev, &rspq->napi, napi_rx_handler, 64);
2343 rspq->cur_desc = rspq->desc;
2344 rspq->cidx = 0;
2345 rspq->gen = 1;
2346 rspq->next_intr_params = rspq->intr_params;
2347 rspq->cntxt_id = be16_to_cpu(rpl.iqid);
2348 rspq->bar2_addr = bar2_address(adapter,
2349 rspq->cntxt_id,
2350 T4_BAR2_QTYPE_INGRESS,
2351 &rspq->bar2_qid);
2352 rspq->abs_id = be16_to_cpu(rpl.physiqid);
2353 rspq->size--; /* subtract status entry */
2354 rspq->adapter = adapter;
2355 rspq->netdev = dev;
2356 rspq->handler = hnd;
2357
2358 /* set offset to -1 to distinguish ingress queues without FL */
2359 rspq->offset = fl ? 0 : -1;
2360
2361 if (fl) {
2362 fl->cntxt_id = be16_to_cpu(rpl.fl0id);
2363 fl->avail = 0;
2364 fl->pend_cred = 0;
2365 fl->pidx = 0;
2366 fl->cidx = 0;
2367 fl->alloc_failed = 0;
2368 fl->large_alloc_failed = 0;
2369 fl->starving = 0;
2370
2371 /* Note, we must initialize the BAR2 Free List User Doorbell
2372 * information before refilling the Free List!
2373 */
2374 fl->bar2_addr = bar2_address(adapter,
2375 fl->cntxt_id,
2376 T4_BAR2_QTYPE_EGRESS,
2377 &fl->bar2_qid);
2378
2379 refill_fl(adapter, fl, fl_cap(fl), GFP_KERNEL);
2380 }
2381
2382 return 0;
2383
2384 err:
2385 /*
2386 * An error occurred. Clean up our partial allocation state and
2387 * return the error.
2388 */
2389 if (rspq->desc) {
2390 dma_free_coherent(adapter->pdev_dev, rspq->size * rspq->iqe_len,
2391 rspq->desc, rspq->phys_addr);
2392 rspq->desc = NULL;
2393 }
2394 if (fl && fl->desc) {
2395 kfree(fl->sdesc);
2396 fl->sdesc = NULL;
2397 dma_free_coherent(adapter->pdev_dev, flsz * EQ_UNIT,
2398 fl->desc, fl->addr);
2399 fl->desc = NULL;
2400 }
2401 return ret;
2402 }
2403
2404 /**
2405 * t4vf_sge_alloc_eth_txq - allocate an SGE Ethernet TX Queue
2406 * @adapter: the adapter
2407 * @txq: pointer to the new txq to be filled in
2408 * @devq: the network TX queue associated with the new txq
2409 * @iqid: the relative ingress queue ID to which events relating to
2410 * the new txq should be directed
2411 */
2412 int t4vf_sge_alloc_eth_txq(struct adapter *adapter, struct sge_eth_txq *txq,
2413 struct net_device *dev, struct netdev_queue *devq,
2414 unsigned int iqid)
2415 {
2416 unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
2417 struct port_info *pi = netdev_priv(dev);
2418 struct fw_eq_eth_cmd cmd, rpl;
2419 struct sge *s = &adapter->sge;
2420 int ret, nentries;
2421
2422 /*
2423 * Calculate the size of the hardware TX Queue (including the Status
2424 * Page on the end of the TX Queue) in units of TX Descriptors.
2425 */
2426 nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
2427
2428 /*
2429 * Allocate the hardware ring for the TX ring (with space for its
2430 * status page) along with the associated software descriptor ring.
2431 */
2432 txq->q.desc = alloc_ring(adapter->pdev_dev, txq->q.size,
2433 sizeof(struct tx_desc),
2434 sizeof(struct tx_sw_desc),
2435 &txq->q.phys_addr, &txq->q.sdesc, s->stat_len);
2436 if (!txq->q.desc)
2437 return -ENOMEM;
2438
2439 /*
2440 * Fill in the Egress Queue Command. Note: As with the direct use of
2441 * the firmware Ingress Queue COmmand above in our RXQ allocation
2442 * routine, ideally, this code would be in t4vf_hw.c. Again, we'll
2443 * have to see if there's some reasonable way to parameterize it
2444 * into the common code ...
2445 */
2446 memset(&cmd, 0, sizeof(cmd));
2447 cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_ETH_CMD) |
2448 FW_CMD_REQUEST_F |
2449 FW_CMD_WRITE_F |
2450 FW_CMD_EXEC_F);
2451 cmd.alloc_to_len16 = cpu_to_be32(FW_EQ_ETH_CMD_ALLOC_F |
2452 FW_EQ_ETH_CMD_EQSTART_F |
2453 FW_LEN16(cmd));
2454 cmd.autoequiqe_to_viid = cpu_to_be32(FW_EQ_ETH_CMD_AUTOEQUEQE_F |
2455 FW_EQ_ETH_CMD_VIID_V(pi->viid));
2456 cmd.fetchszm_to_iqid =
2457 cpu_to_be32(FW_EQ_ETH_CMD_HOSTFCMODE_V(SGE_HOSTFCMODE_STPG) |
2458 FW_EQ_ETH_CMD_PCIECHN_V(pi->port_id) |
2459 FW_EQ_ETH_CMD_IQID_V(iqid));
2460 cmd.dcaen_to_eqsize =
2461 cpu_to_be32(FW_EQ_ETH_CMD_FBMIN_V(chip_ver <= CHELSIO_T5
2462 ? FETCHBURSTMIN_64B_X
2463 : FETCHBURSTMIN_64B_T6_X) |
2464 FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
2465 FW_EQ_ETH_CMD_CIDXFTHRESH_V(
2466 CIDXFLUSHTHRESH_32_X) |
2467 FW_EQ_ETH_CMD_EQSIZE_V(nentries));
2468 cmd.eqaddr = cpu_to_be64(txq->q.phys_addr);
2469
2470 /*
2471 * Issue the firmware Egress Queue Command and extract the results if
2472 * it completes successfully.
2473 */
2474 ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);
2475 if (ret) {
2476 /*
2477 * The girmware Ingress Queue Command failed for some reason.
2478 * Free up our partial allocation state and return the error.
2479 */
2480 kfree(txq->q.sdesc);
2481 txq->q.sdesc = NULL;
2482 dma_free_coherent(adapter->pdev_dev,
2483 nentries * sizeof(struct tx_desc),
2484 txq->q.desc, txq->q.phys_addr);
2485 txq->q.desc = NULL;
2486 return ret;
2487 }
2488
2489 txq->q.in_use = 0;
2490 txq->q.cidx = 0;
2491 txq->q.pidx = 0;
2492 txq->q.stat = (void *)&txq->q.desc[txq->q.size];
2493 txq->q.cntxt_id = FW_EQ_ETH_CMD_EQID_G(be32_to_cpu(rpl.eqid_pkd));
2494 txq->q.bar2_addr = bar2_address(adapter,
2495 txq->q.cntxt_id,
2496 T4_BAR2_QTYPE_EGRESS,
2497 &txq->q.bar2_qid);
2498 txq->q.abs_id =
2499 FW_EQ_ETH_CMD_PHYSEQID_G(be32_to_cpu(rpl.physeqid_pkd));
2500 txq->txq = devq;
2501 txq->tso = 0;
2502 txq->tx_cso = 0;
2503 txq->vlan_ins = 0;
2504 txq->q.stops = 0;
2505 txq->q.restarts = 0;
2506 txq->mapping_err = 0;
2507 return 0;
2508 }
2509
2510 /*
2511 * Free the DMA map resources associated with a TX queue.
2512 */
2513 static void free_txq(struct adapter *adapter, struct sge_txq *tq)
2514 {
2515 struct sge *s = &adapter->sge;
2516
2517 dma_free_coherent(adapter->pdev_dev,
2518 tq->size * sizeof(*tq->desc) + s->stat_len,
2519 tq->desc, tq->phys_addr);
2520 tq->cntxt_id = 0;
2521 tq->sdesc = NULL;
2522 tq->desc = NULL;
2523 }
2524
2525 /*
2526 * Free the resources associated with a response queue (possibly including a
2527 * free list).
2528 */
2529 static void free_rspq_fl(struct adapter *adapter, struct sge_rspq *rspq,
2530 struct sge_fl *fl)
2531 {
2532 struct sge *s = &adapter->sge;
2533 unsigned int flid = fl ? fl->cntxt_id : 0xffff;
2534
2535 t4vf_iq_free(adapter, FW_IQ_TYPE_FL_INT_CAP,
2536 rspq->cntxt_id, flid, 0xffff);
2537 dma_free_coherent(adapter->pdev_dev, (rspq->size + 1) * rspq->iqe_len,
2538 rspq->desc, rspq->phys_addr);
2539 netif_napi_del(&rspq->napi);
2540 rspq->netdev = NULL;
2541 rspq->cntxt_id = 0;
2542 rspq->abs_id = 0;
2543 rspq->desc = NULL;
2544
2545 if (fl) {
2546 free_rx_bufs(adapter, fl, fl->avail);
2547 dma_free_coherent(adapter->pdev_dev,
2548 fl->size * sizeof(*fl->desc) + s->stat_len,
2549 fl->desc, fl->addr);
2550 kfree(fl->sdesc);
2551 fl->sdesc = NULL;
2552 fl->cntxt_id = 0;
2553 fl->desc = NULL;
2554 }
2555 }
2556
2557 /**
2558 * t4vf_free_sge_resources - free SGE resources
2559 * @adapter: the adapter
2560 *
2561 * Frees resources used by the SGE queue sets.
2562 */
2563 void t4vf_free_sge_resources(struct adapter *adapter)
2564 {
2565 struct sge *s = &adapter->sge;
2566 struct sge_eth_rxq *rxq = s->ethrxq;
2567 struct sge_eth_txq *txq = s->ethtxq;
2568 struct sge_rspq *evtq = &s->fw_evtq;
2569 struct sge_rspq *intrq = &s->intrq;
2570 int qs;
2571
2572 for (qs = 0; qs < adapter->sge.ethqsets; qs++, rxq++, txq++) {
2573 if (rxq->rspq.desc)
2574 free_rspq_fl(adapter, &rxq->rspq, &rxq->fl);
2575 if (txq->q.desc) {
2576 t4vf_eth_eq_free(adapter, txq->q.cntxt_id);
2577 free_tx_desc(adapter, &txq->q, txq->q.in_use, true);
2578 kfree(txq->q.sdesc);
2579 free_txq(adapter, &txq->q);
2580 }
2581 }
2582 if (evtq->desc)
2583 free_rspq_fl(adapter, evtq, NULL);
2584 if (intrq->desc)
2585 free_rspq_fl(adapter, intrq, NULL);
2586 }
2587
2588 /**
2589 * t4vf_sge_start - enable SGE operation
2590 * @adapter: the adapter
2591 *
2592 * Start tasklets and timers associated with the DMA engine.
2593 */
2594 void t4vf_sge_start(struct adapter *adapter)
2595 {
2596 adapter->sge.ethtxq_rover = 0;
2597 mod_timer(&adapter->sge.rx_timer, jiffies + RX_QCHECK_PERIOD);
2598 mod_timer(&adapter->sge.tx_timer, jiffies + TX_QCHECK_PERIOD);
2599 }
2600
2601 /**
2602 * t4vf_sge_stop - disable SGE operation
2603 * @adapter: the adapter
2604 *
2605 * Stop tasklets and timers associated with the DMA engine. Note that
2606 * this is effective only if measures have been taken to disable any HW
2607 * events that may restart them.
2608 */
2609 void t4vf_sge_stop(struct adapter *adapter)
2610 {
2611 struct sge *s = &adapter->sge;
2612
2613 if (s->rx_timer.function)
2614 del_timer_sync(&s->rx_timer);
2615 if (s->tx_timer.function)
2616 del_timer_sync(&s->tx_timer);
2617 }
2618
2619 /**
2620 * t4vf_sge_init - initialize SGE
2621 * @adapter: the adapter
2622 *
2623 * Performs SGE initialization needed every time after a chip reset.
2624 * We do not initialize any of the queue sets here, instead the driver
2625 * top-level must request those individually. We also do not enable DMA
2626 * here, that should be done after the queues have been set up.
2627 */
2628 int t4vf_sge_init(struct adapter *adapter)
2629 {
2630 struct sge_params *sge_params = &adapter->params.sge;
2631 u32 fl_small_pg = sge_params->sge_fl_buffer_size[0];
2632 u32 fl_large_pg = sge_params->sge_fl_buffer_size[1];
2633 struct sge *s = &adapter->sge;
2634
2635 /*
2636 * Start by vetting the basic SGE parameters which have been set up by
2637 * the Physical Function Driver. Ideally we should be able to deal
2638 * with _any_ configuration. Practice is different ...
2639 */
2640
2641 /* We only bother using the Large Page logic if the Large Page Buffer
2642 * is larger than our Page Size Buffer.
2643 */
2644 if (fl_large_pg <= fl_small_pg)
2645 fl_large_pg = 0;
2646
2647 /* The Page Size Buffer must be exactly equal to our Page Size and the
2648 * Large Page Size Buffer should be 0 (per above) or a power of 2.
2649 */
2650 if (fl_small_pg != PAGE_SIZE ||
2651 (fl_large_pg & (fl_large_pg - 1)) != 0) {
2652 dev_err(adapter->pdev_dev, "bad SGE FL buffer sizes [%d, %d]\n",
2653 fl_small_pg, fl_large_pg);
2654 return -EINVAL;
2655 }
2656 if ((sge_params->sge_control & RXPKTCPLMODE_F) !=
2657 RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X)) {
2658 dev_err(adapter->pdev_dev, "bad SGE CPL MODE\n");
2659 return -EINVAL;
2660 }
2661
2662 /*
2663 * Now translate the adapter parameters into our internal forms.
2664 */
2665 if (fl_large_pg)
2666 s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT;
2667 s->stat_len = ((sge_params->sge_control & EGRSTATUSPAGESIZE_F)
2668 ? 128 : 64);
2669 s->pktshift = PKTSHIFT_G(sge_params->sge_control);
2670 s->fl_align = t4vf_fl_pkt_align(adapter);
2671
2672 /* A FL with <= fl_starve_thres buffers is starving and a periodic
2673 * timer will attempt to refill it. This needs to be larger than the
2674 * SGE's Egress Congestion Threshold. If it isn't, then we can get
2675 * stuck waiting for new packets while the SGE is waiting for us to
2676 * give it more Free List entries. (Note that the SGE's Egress
2677 * Congestion Threshold is in units of 2 Free List pointers.)
2678 */
2679 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
2680 case CHELSIO_T4:
2681 s->fl_starve_thres =
2682 EGRTHRESHOLD_G(sge_params->sge_congestion_control);
2683 break;
2684 case CHELSIO_T5:
2685 s->fl_starve_thres =
2686 EGRTHRESHOLDPACKING_G(sge_params->sge_congestion_control);
2687 break;
2688 case CHELSIO_T6:
2689 default:
2690 s->fl_starve_thres =
2691 T6_EGRTHRESHOLDPACKING_G(sge_params->sge_congestion_control);
2692 break;
2693 }
2694 s->fl_starve_thres = s->fl_starve_thres * 2 + 1;
2695
2696 /*
2697 * Set up tasklet timers.
2698 */
2699 timer_setup(&s->rx_timer, sge_rx_timer_cb, 0);
2700 timer_setup(&s->tx_timer, sge_tx_timer_cb, 0);
2701
2702 /*
2703 * Initialize Forwarded Interrupt Queue lock.
2704 */
2705 spin_lock_init(&s->intrq_lock);
2706
2707 return 0;
2708 }