This source file includes following definitions.
- fl_mtu_bufsize
- get_buf_addr
- is_buf_mapped
- txq_avail
- fl_cap
- fl_starving
- cxgb4_map_skb
- unmap_skb
- deferred_unmap_destructor
- unmap_sgl
- free_tx_desc
- reclaimable
- reclaim_completed_tx
- cxgb4_reclaim_completed_tx
- get_buf_size
- free_rx_bufs
- unmap_rx_buf
- ring_fl_db
- set_rx_sw_desc
- refill_fl
- __refill_fl
- alloc_ring
- sgl_len
- flits_to_desc
- is_eth_imm
- calc_tx_flits
- calc_tx_descs
- cxgb4_write_sgl
- cxgb_pio_copy
- cxgb4_ring_tx_db
- cxgb4_inline_tx_skb
- inline_tx_skb_header
- hwcsum
- eth_txq_stop
- txq_advance
- cxgb_fcoe_offload
- cxgb_encap_offload_supported
- t6_fill_tnl_lso
- t4_sge_eth_txq_egress_update
- cxgb4_eth_xmit
- t4vf_is_eth_imm
- t4vf_calc_tx_flits
- cxgb4_vf_eth_xmit
- t4_start_xmit
- reclaim_completed_tx_imm
- is_imm
- ctrlq_check_stop
- ctrl_xmit
- restart_ctrlq
- t4_mgmt_tx
- is_ofld_imm
- calc_tx_flits_ofld
- txq_stop_maperr
- ofldtxq_stop
- service_ofldq
- ofld_xmit
- restart_ofldq
- skb_txq
- is_ctrl_pkt
- uld_send
- t4_ofld_send
- cxgb4_ofld_send
- inline_tx_header
- ofld_xmit_direct
- cxgb4_immdata_send
- t4_crypto_send
- cxgb4_crypto_send
- copy_frags
- cxgb4_pktgl_to_skb
- t4_pktgl_free
- handle_trace_pkt
- cxgb4_sgetim_to_hwtstamp
- do_gro
- t4_systim_to_hwstamp
- t4_rx_hststamp
- t4_tx_hststamp
- t4_tx_completion_handler
- t4_ethrx_handler
- restore_rx_bufs
- is_new_response
- rspq_next
- process_responses
- napi_rx_handler
- t4_sge_intr_msix
- process_intrq
- t4_intr_msi
- t4_intr_intx
- t4_intr_handler
- sge_rx_timer_cb
- sge_tx_timer_cb
- bar2_address
- t4_sge_alloc_rxq
- init_txq
- t4_sge_alloc_eth_txq
- t4_sge_alloc_ctrl_txq
- t4_sge_mod_ctrl_txq
- t4_sge_alloc_uld_txq
- free_txq
- free_rspq_fl
- t4_free_ofld_rxqs
- t4_free_sge_resources
- t4_sge_start
- t4_sge_stop
- t4_sge_init_soft
- t4_sge_init
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35 #include <linux/skbuff.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/if_vlan.h>
39 #include <linux/ip.h>
40 #include <linux/dma-mapping.h>
41 #include <linux/jiffies.h>
42 #include <linux/prefetch.h>
43 #include <linux/export.h>
44 #include <net/xfrm.h>
45 #include <net/ipv6.h>
46 #include <net/tcp.h>
47 #include <net/busy_poll.h>
48 #ifdef CONFIG_CHELSIO_T4_FCOE
49 #include <scsi/fc/fc_fcoe.h>
50 #endif
51 #include "cxgb4.h"
52 #include "t4_regs.h"
53 #include "t4_values.h"
54 #include "t4_msg.h"
55 #include "t4fw_api.h"
56 #include "cxgb4_ptp.h"
57 #include "cxgb4_uld.h"
58
59
60
61
62
63 #if PAGE_SHIFT >= 16
64 # define FL_PG_ORDER 0
65 #else
66 # define FL_PG_ORDER (16 - PAGE_SHIFT)
67 #endif
68
69
70 #define RX_COPY_THRES 256
71 #define RX_PULL_LEN 128
72
73
74
75
76
77 #define RX_PKT_SKB_LEN 512
78
79
80
81
82
83
84
85
86 #define MAX_TX_RECLAIM 32
87
88
89
90
91
92 #define MAX_RX_REFILL 16U
93
94
95
96
97
98 #define RX_QCHECK_PERIOD (HZ / 2)
99
100
101
102
103 #define TX_QCHECK_PERIOD (HZ / 2)
104
105
106
107
108 #define MAX_TIMER_TX_RECLAIM 100
109
110
111
112
113 #define NOMEM_TMR_IDX (SGE_NTIMERS - 1)
114
115
116
117
118
119 #define TXQ_STOP_THRES (SGE_MAX_WR_LEN / sizeof(struct tx_desc))
120
121
122
123
124
125 #define MAX_IMM_TX_PKT_LEN 256
126
127
128
129
130 #define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN
131
132 struct rx_sw_desc {
133 struct page *page;
134 dma_addr_t dma_addr;
135 };
136
137
138
139
140
141
142
143 #define FL_MTU_SMALL 1500
144 #define FL_MTU_LARGE 9000
145
146 static inline unsigned int fl_mtu_bufsize(struct adapter *adapter,
147 unsigned int mtu)
148 {
149 struct sge *s = &adapter->sge;
150
151 return ALIGN(s->pktshift + ETH_HLEN + VLAN_HLEN + mtu, s->fl_align);
152 }
153
154 #define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL)
155 #define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE)
156
157
158
159
160
161
162
163
164
165
166
167 enum {
168 RX_BUF_FLAGS = 0x1f,
169 RX_BUF_SIZE = 0x0f,
170 RX_UNMAPPED_BUF = 0x10,
171
172
173
174
175
176
177
178
179 RX_SMALL_PG_BUF = 0x0,
180 RX_LARGE_PG_BUF = 0x1,
181
182 RX_SMALL_MTU_BUF = 0x2,
183 RX_LARGE_MTU_BUF = 0x3,
184 };
185
186 static int timer_pkt_quota[] = {1, 1, 2, 3, 4, 5};
187 #define MIN_NAPI_WORK 1
188
189 static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *d)
190 {
191 return d->dma_addr & ~(dma_addr_t)RX_BUF_FLAGS;
192 }
193
194 static inline bool is_buf_mapped(const struct rx_sw_desc *d)
195 {
196 return !(d->dma_addr & RX_UNMAPPED_BUF);
197 }
198
199
200
201
202
203
204
205
206 static inline unsigned int txq_avail(const struct sge_txq *q)
207 {
208 return q->size - 1 - q->in_use;
209 }
210
211
212
213
214
215
216
217
218
219 static inline unsigned int fl_cap(const struct sge_fl *fl)
220 {
221 return fl->size - 8;
222 }
223
224
225
226
227
228
229
230
231
232
233 static inline bool fl_starving(const struct adapter *adapter,
234 const struct sge_fl *fl)
235 {
236 const struct sge *s = &adapter->sge;
237
238 return fl->avail - fl->pend_cred <= s->fl_starve_thres;
239 }
240
241 int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb,
242 dma_addr_t *addr)
243 {
244 const skb_frag_t *fp, *end;
245 const struct skb_shared_info *si;
246
247 *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
248 if (dma_mapping_error(dev, *addr))
249 goto out_err;
250
251 si = skb_shinfo(skb);
252 end = &si->frags[si->nr_frags];
253
254 for (fp = si->frags; fp < end; fp++) {
255 *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp),
256 DMA_TO_DEVICE);
257 if (dma_mapping_error(dev, *addr))
258 goto unwind;
259 }
260 return 0;
261
262 unwind:
263 while (fp-- > si->frags)
264 dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE);
265
266 dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE);
267 out_err:
268 return -ENOMEM;
269 }
270 EXPORT_SYMBOL(cxgb4_map_skb);
271
272 #ifdef CONFIG_NEED_DMA_MAP_STATE
273 static void unmap_skb(struct device *dev, const struct sk_buff *skb,
274 const dma_addr_t *addr)
275 {
276 const skb_frag_t *fp, *end;
277 const struct skb_shared_info *si;
278
279 dma_unmap_single(dev, *addr++, skb_headlen(skb), DMA_TO_DEVICE);
280
281 si = skb_shinfo(skb);
282 end = &si->frags[si->nr_frags];
283 for (fp = si->frags; fp < end; fp++)
284 dma_unmap_page(dev, *addr++, skb_frag_size(fp), DMA_TO_DEVICE);
285 }
286
287
288
289
290
291
292
293
294
295 static void deferred_unmap_destructor(struct sk_buff *skb)
296 {
297 unmap_skb(skb->dev->dev.parent, skb, (dma_addr_t *)skb->head);
298 }
299 #endif
300
301 static void unmap_sgl(struct device *dev, const struct sk_buff *skb,
302 const struct ulptx_sgl *sgl, const struct sge_txq *q)
303 {
304 const struct ulptx_sge_pair *p;
305 unsigned int nfrags = skb_shinfo(skb)->nr_frags;
306
307 if (likely(skb_headlen(skb)))
308 dma_unmap_single(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
309 DMA_TO_DEVICE);
310 else {
311 dma_unmap_page(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
312 DMA_TO_DEVICE);
313 nfrags--;
314 }
315
316
317
318
319
320 for (p = sgl->sge; nfrags >= 2; nfrags -= 2) {
321 if (likely((u8 *)(p + 1) <= (u8 *)q->stat)) {
322 unmap: dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
323 ntohl(p->len[0]), DMA_TO_DEVICE);
324 dma_unmap_page(dev, be64_to_cpu(p->addr[1]),
325 ntohl(p->len[1]), DMA_TO_DEVICE);
326 p++;
327 } else if ((u8 *)p == (u8 *)q->stat) {
328 p = (const struct ulptx_sge_pair *)q->desc;
329 goto unmap;
330 } else if ((u8 *)p + 8 == (u8 *)q->stat) {
331 const __be64 *addr = (const __be64 *)q->desc;
332
333 dma_unmap_page(dev, be64_to_cpu(addr[0]),
334 ntohl(p->len[0]), DMA_TO_DEVICE);
335 dma_unmap_page(dev, be64_to_cpu(addr[1]),
336 ntohl(p->len[1]), DMA_TO_DEVICE);
337 p = (const struct ulptx_sge_pair *)&addr[2];
338 } else {
339 const __be64 *addr = (const __be64 *)q->desc;
340
341 dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
342 ntohl(p->len[0]), DMA_TO_DEVICE);
343 dma_unmap_page(dev, be64_to_cpu(addr[0]),
344 ntohl(p->len[1]), DMA_TO_DEVICE);
345 p = (const struct ulptx_sge_pair *)&addr[1];
346 }
347 }
348 if (nfrags) {
349 __be64 addr;
350
351 if ((u8 *)p == (u8 *)q->stat)
352 p = (const struct ulptx_sge_pair *)q->desc;
353 addr = (u8 *)p + 16 <= (u8 *)q->stat ? p->addr[0] :
354 *(const __be64 *)q->desc;
355 dma_unmap_page(dev, be64_to_cpu(addr), ntohl(p->len[0]),
356 DMA_TO_DEVICE);
357 }
358 }
359
360
361
362
363
364
365
366
367
368
369
370 void free_tx_desc(struct adapter *adap, struct sge_txq *q,
371 unsigned int n, bool unmap)
372 {
373 struct tx_sw_desc *d;
374 unsigned int cidx = q->cidx;
375 struct device *dev = adap->pdev_dev;
376
377 d = &q->sdesc[cidx];
378 while (n--) {
379 if (d->skb) {
380 if (unmap)
381 unmap_sgl(dev, d->skb, d->sgl, q);
382 dev_consume_skb_any(d->skb);
383 d->skb = NULL;
384 }
385 ++d;
386 if (++cidx == q->size) {
387 cidx = 0;
388 d = q->sdesc;
389 }
390 }
391 q->cidx = cidx;
392 }
393
394
395
396
397 static inline int reclaimable(const struct sge_txq *q)
398 {
399 int hw_cidx = ntohs(READ_ONCE(q->stat->cidx));
400 hw_cidx -= q->cidx;
401 return hw_cidx < 0 ? hw_cidx + q->size : hw_cidx;
402 }
403
404
405
406
407
408
409
410
411
412
413
414
415 static inline int reclaim_completed_tx(struct adapter *adap, struct sge_txq *q,
416 int maxreclaim, bool unmap)
417 {
418 int reclaim = reclaimable(q);
419
420 if (reclaim) {
421
422
423
424
425 if (maxreclaim < 0)
426 maxreclaim = MAX_TX_RECLAIM;
427 if (reclaim > maxreclaim)
428 reclaim = maxreclaim;
429
430 free_tx_desc(adap, q, reclaim, unmap);
431 q->in_use -= reclaim;
432 }
433
434 return reclaim;
435 }
436
437
438
439
440
441
442
443
444
445
446
447 void cxgb4_reclaim_completed_tx(struct adapter *adap, struct sge_txq *q,
448 bool unmap)
449 {
450 (void)reclaim_completed_tx(adap, q, -1, unmap);
451 }
452 EXPORT_SYMBOL(cxgb4_reclaim_completed_tx);
453
454 static inline int get_buf_size(struct adapter *adapter,
455 const struct rx_sw_desc *d)
456 {
457 struct sge *s = &adapter->sge;
458 unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE;
459 int buf_size;
460
461 switch (rx_buf_size_idx) {
462 case RX_SMALL_PG_BUF:
463 buf_size = PAGE_SIZE;
464 break;
465
466 case RX_LARGE_PG_BUF:
467 buf_size = PAGE_SIZE << s->fl_pg_order;
468 break;
469
470 case RX_SMALL_MTU_BUF:
471 buf_size = FL_MTU_SMALL_BUFSIZE(adapter);
472 break;
473
474 case RX_LARGE_MTU_BUF:
475 buf_size = FL_MTU_LARGE_BUFSIZE(adapter);
476 break;
477
478 default:
479 BUG();
480 }
481
482 return buf_size;
483 }
484
485
486
487
488
489
490
491
492
493
494 static void free_rx_bufs(struct adapter *adap, struct sge_fl *q, int n)
495 {
496 while (n--) {
497 struct rx_sw_desc *d = &q->sdesc[q->cidx];
498
499 if (is_buf_mapped(d))
500 dma_unmap_page(adap->pdev_dev, get_buf_addr(d),
501 get_buf_size(adap, d),
502 PCI_DMA_FROMDEVICE);
503 put_page(d->page);
504 d->page = NULL;
505 if (++q->cidx == q->size)
506 q->cidx = 0;
507 q->avail--;
508 }
509 }
510
511
512
513
514
515
516
517
518
519
520
521
522 static void unmap_rx_buf(struct adapter *adap, struct sge_fl *q)
523 {
524 struct rx_sw_desc *d = &q->sdesc[q->cidx];
525
526 if (is_buf_mapped(d))
527 dma_unmap_page(adap->pdev_dev, get_buf_addr(d),
528 get_buf_size(adap, d), PCI_DMA_FROMDEVICE);
529 d->page = NULL;
530 if (++q->cidx == q->size)
531 q->cidx = 0;
532 q->avail--;
533 }
534
535 static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
536 {
537 if (q->pend_cred >= 8) {
538 u32 val = adap->params.arch.sge_fl_db;
539
540 if (is_t4(adap->params.chip))
541 val |= PIDX_V(q->pend_cred / 8);
542 else
543 val |= PIDX_T5_V(q->pend_cred / 8);
544
545
546
547
548 wmb();
549
550
551
552
553
554 if (unlikely(q->bar2_addr == NULL)) {
555 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
556 val | QID_V(q->cntxt_id));
557 } else {
558 writel(val | QID_V(q->bar2_qid),
559 q->bar2_addr + SGE_UDB_KDOORBELL);
560
561
562
563
564 wmb();
565 }
566 q->pend_cred &= 7;
567 }
568 }
569
570 static inline void set_rx_sw_desc(struct rx_sw_desc *sd, struct page *pg,
571 dma_addr_t mapping)
572 {
573 sd->page = pg;
574 sd->dma_addr = mapping;
575 }
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591 static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n,
592 gfp_t gfp)
593 {
594 struct sge *s = &adap->sge;
595 struct page *pg;
596 dma_addr_t mapping;
597 unsigned int cred = q->avail;
598 __be64 *d = &q->desc[q->pidx];
599 struct rx_sw_desc *sd = &q->sdesc[q->pidx];
600 int node;
601
602 #ifdef CONFIG_DEBUG_FS
603 if (test_bit(q->cntxt_id - adap->sge.egr_start, adap->sge.blocked_fl))
604 goto out;
605 #endif
606
607 gfp |= __GFP_NOWARN;
608 node = dev_to_node(adap->pdev_dev);
609
610 if (s->fl_pg_order == 0)
611 goto alloc_small_pages;
612
613
614
615
616 while (n) {
617 pg = alloc_pages_node(node, gfp | __GFP_COMP, s->fl_pg_order);
618 if (unlikely(!pg)) {
619 q->large_alloc_failed++;
620 break;
621 }
622
623 mapping = dma_map_page(adap->pdev_dev, pg, 0,
624 PAGE_SIZE << s->fl_pg_order,
625 PCI_DMA_FROMDEVICE);
626 if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
627 __free_pages(pg, s->fl_pg_order);
628 q->mapping_err++;
629 goto out;
630 }
631 mapping |= RX_LARGE_PG_BUF;
632 *d++ = cpu_to_be64(mapping);
633
634 set_rx_sw_desc(sd, pg, mapping);
635 sd++;
636
637 q->avail++;
638 if (++q->pidx == q->size) {
639 q->pidx = 0;
640 sd = q->sdesc;
641 d = q->desc;
642 }
643 n--;
644 }
645
646 alloc_small_pages:
647 while (n--) {
648 pg = alloc_pages_node(node, gfp, 0);
649 if (unlikely(!pg)) {
650 q->alloc_failed++;
651 break;
652 }
653
654 mapping = dma_map_page(adap->pdev_dev, pg, 0, PAGE_SIZE,
655 PCI_DMA_FROMDEVICE);
656 if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
657 put_page(pg);
658 q->mapping_err++;
659 goto out;
660 }
661 *d++ = cpu_to_be64(mapping);
662
663 set_rx_sw_desc(sd, pg, mapping);
664 sd++;
665
666 q->avail++;
667 if (++q->pidx == q->size) {
668 q->pidx = 0;
669 sd = q->sdesc;
670 d = q->desc;
671 }
672 }
673
674 out: cred = q->avail - cred;
675 q->pend_cred += cred;
676 ring_fl_db(adap, q);
677
678 if (unlikely(fl_starving(adap, q))) {
679 smp_wmb();
680 q->low++;
681 set_bit(q->cntxt_id - adap->sge.egr_start,
682 adap->sge.starving_fl);
683 }
684
685 return cred;
686 }
687
688 static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
689 {
690 refill_fl(adap, fl, min(MAX_RX_REFILL, fl_cap(fl) - fl->avail),
691 GFP_ATOMIC);
692 }
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713 static void *alloc_ring(struct device *dev, size_t nelem, size_t elem_size,
714 size_t sw_size, dma_addr_t *phys, void *metadata,
715 size_t stat_size, int node)
716 {
717 size_t len = nelem * elem_size + stat_size;
718 void *s = NULL;
719 void *p = dma_alloc_coherent(dev, len, phys, GFP_KERNEL);
720
721 if (!p)
722 return NULL;
723 if (sw_size) {
724 s = kcalloc_node(sw_size, nelem, GFP_KERNEL, node);
725
726 if (!s) {
727 dma_free_coherent(dev, len, p, *phys);
728 return NULL;
729 }
730 }
731 if (metadata)
732 *(void **)metadata = s;
733 return p;
734 }
735
736
737
738
739
740
741
742
743 static inline unsigned int sgl_len(unsigned int n)
744 {
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761 n--;
762 return (3 * n) / 2 + (n & 1) + 2;
763 }
764
765
766
767
768
769
770
771
772 static inline unsigned int flits_to_desc(unsigned int n)
773 {
774 BUG_ON(n > SGE_MAX_WR_LEN / 8);
775 return DIV_ROUND_UP(n, 8);
776 }
777
778
779
780
781
782
783
784
785 static inline int is_eth_imm(const struct sk_buff *skb, unsigned int chip_ver)
786 {
787 int hdrlen = 0;
788
789 if (skb->encapsulation && skb_shinfo(skb)->gso_size &&
790 chip_ver > CHELSIO_T5) {
791 hdrlen = sizeof(struct cpl_tx_tnl_lso);
792 hdrlen += sizeof(struct cpl_tx_pkt_core);
793 } else {
794 hdrlen = skb_shinfo(skb)->gso_size ?
795 sizeof(struct cpl_tx_pkt_lso_core) : 0;
796 hdrlen += sizeof(struct cpl_tx_pkt);
797 }
798 if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
799 return hdrlen;
800 return 0;
801 }
802
803
804
805
806
807
808
809
810 static inline unsigned int calc_tx_flits(const struct sk_buff *skb,
811 unsigned int chip_ver)
812 {
813 unsigned int flits;
814 int hdrlen = is_eth_imm(skb, chip_ver);
815
816
817
818
819
820
821 if (hdrlen)
822 return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
823
824
825
826
827
828
829
830
831
832 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
833 if (skb_shinfo(skb)->gso_size) {
834 if (skb->encapsulation && chip_ver > CHELSIO_T5)
835 hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
836 sizeof(struct cpl_tx_tnl_lso);
837 else
838 hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
839 sizeof(struct cpl_tx_pkt_lso_core);
840
841 hdrlen += sizeof(struct cpl_tx_pkt_core);
842 flits += (hdrlen / sizeof(__be64));
843 } else {
844 flits += (sizeof(struct fw_eth_tx_pkt_wr) +
845 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
846 }
847 return flits;
848 }
849
850
851
852
853
854
855
856
857 static inline unsigned int calc_tx_descs(const struct sk_buff *skb,
858 unsigned int chip_ver)
859 {
860 return flits_to_desc(calc_tx_flits(skb, chip_ver));
861 }
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880 void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q,
881 struct ulptx_sgl *sgl, u64 *end, unsigned int start,
882 const dma_addr_t *addr)
883 {
884 unsigned int i, len;
885 struct ulptx_sge_pair *to;
886 const struct skb_shared_info *si = skb_shinfo(skb);
887 unsigned int nfrags = si->nr_frags;
888 struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1];
889
890 len = skb_headlen(skb) - start;
891 if (likely(len)) {
892 sgl->len0 = htonl(len);
893 sgl->addr0 = cpu_to_be64(addr[0] + start);
894 nfrags++;
895 } else {
896 sgl->len0 = htonl(skb_frag_size(&si->frags[0]));
897 sgl->addr0 = cpu_to_be64(addr[1]);
898 }
899
900 sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
901 ULPTX_NSGE_V(nfrags));
902 if (likely(--nfrags == 0))
903 return;
904
905
906
907
908
909 to = (u8 *)end > (u8 *)q->stat ? buf : sgl->sge;
910
911 for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) {
912 to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
913 to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i]));
914 to->addr[0] = cpu_to_be64(addr[i]);
915 to->addr[1] = cpu_to_be64(addr[++i]);
916 }
917 if (nfrags) {
918 to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
919 to->len[1] = cpu_to_be32(0);
920 to->addr[0] = cpu_to_be64(addr[i + 1]);
921 }
922 if (unlikely((u8 *)end > (u8 *)q->stat)) {
923 unsigned int part0 = (u8 *)q->stat - (u8 *)sgl->sge, part1;
924
925 if (likely(part0))
926 memcpy(sgl->sge, buf, part0);
927 part1 = (u8 *)end - (u8 *)q->stat;
928 memcpy(q->desc, (u8 *)buf + part0, part1);
929 end = (void *)q->desc + part1;
930 }
931 if ((uintptr_t)end & 8)
932 *end = 0;
933 }
934 EXPORT_SYMBOL(cxgb4_write_sgl);
935
936
937
938
939
940 static void cxgb_pio_copy(u64 __iomem *dst, u64 *src)
941 {
942 int count = 8;
943
944 while (count) {
945 writeq(*src, dst);
946 src++;
947 dst++;
948 count--;
949 }
950 }
951
952
953
954
955
956
957
958
959
960 inline void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n)
961 {
962
963
964
965 wmb();
966
967
968
969
970 if (unlikely(q->bar2_addr == NULL)) {
971 u32 val = PIDX_V(n);
972 unsigned long flags;
973
974
975
976
977 spin_lock_irqsave(&q->db_lock, flags);
978 if (!q->db_disabled)
979 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
980 QID_V(q->cntxt_id) | val);
981 else
982 q->db_pidx_inc += n;
983 q->db_pidx = q->pidx;
984 spin_unlock_irqrestore(&q->db_lock, flags);
985 } else {
986 u32 val = PIDX_T5_V(n);
987
988
989
990
991
992
993
994 WARN_ON(val & DBPRIO_F);
995
996
997
998
999
1000 if (n == 1 && q->bar2_qid == 0) {
1001 int index = (q->pidx
1002 ? (q->pidx - 1)
1003 : (q->size - 1));
1004 u64 *wr = (u64 *)&q->desc[index];
1005
1006 cxgb_pio_copy((u64 __iomem *)
1007 (q->bar2_addr + SGE_UDB_WCDOORBELL),
1008 wr);
1009 } else {
1010 writel(val | QID_V(q->bar2_qid),
1011 q->bar2_addr + SGE_UDB_KDOORBELL);
1012 }
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024 wmb();
1025 }
1026 }
1027 EXPORT_SYMBOL(cxgb4_ring_tx_db);
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040 void cxgb4_inline_tx_skb(const struct sk_buff *skb,
1041 const struct sge_txq *q, void *pos)
1042 {
1043 int left = (void *)q->stat - pos;
1044 u64 *p;
1045
1046 if (likely(skb->len <= left)) {
1047 if (likely(!skb->data_len))
1048 skb_copy_from_linear_data(skb, pos, skb->len);
1049 else
1050 skb_copy_bits(skb, 0, pos, skb->len);
1051 pos += skb->len;
1052 } else {
1053 skb_copy_bits(skb, 0, pos, left);
1054 skb_copy_bits(skb, left, q->desc, skb->len - left);
1055 pos = (void *)q->desc + (skb->len - left);
1056 }
1057
1058
1059 p = PTR_ALIGN(pos, 8);
1060 if ((uintptr_t)p & 8)
1061 *p = 0;
1062 }
1063 EXPORT_SYMBOL(cxgb4_inline_tx_skb);
1064
1065 static void *inline_tx_skb_header(const struct sk_buff *skb,
1066 const struct sge_txq *q, void *pos,
1067 int length)
1068 {
1069 u64 *p;
1070 int left = (void *)q->stat - pos;
1071
1072 if (likely(length <= left)) {
1073 memcpy(pos, skb->data, length);
1074 pos += length;
1075 } else {
1076 memcpy(pos, skb->data, left);
1077 memcpy(q->desc, skb->data + left, length - left);
1078 pos = (void *)q->desc + (length - left);
1079 }
1080
1081 p = PTR_ALIGN(pos, 8);
1082 if ((uintptr_t)p & 8) {
1083 *p = 0;
1084 return p + 1;
1085 }
1086 return p;
1087 }
1088
1089
1090
1091
1092
1093 static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb)
1094 {
1095 int csum_type;
1096 bool inner_hdr_csum = false;
1097 u16 proto, ver;
1098
1099 if (skb->encapsulation &&
1100 (CHELSIO_CHIP_VERSION(chip) > CHELSIO_T5))
1101 inner_hdr_csum = true;
1102
1103 if (inner_hdr_csum) {
1104 ver = inner_ip_hdr(skb)->version;
1105 proto = (ver == 4) ? inner_ip_hdr(skb)->protocol :
1106 inner_ipv6_hdr(skb)->nexthdr;
1107 } else {
1108 ver = ip_hdr(skb)->version;
1109 proto = (ver == 4) ? ip_hdr(skb)->protocol :
1110 ipv6_hdr(skb)->nexthdr;
1111 }
1112
1113 if (ver == 4) {
1114 if (proto == IPPROTO_TCP)
1115 csum_type = TX_CSUM_TCPIP;
1116 else if (proto == IPPROTO_UDP)
1117 csum_type = TX_CSUM_UDPIP;
1118 else {
1119 nocsum:
1120
1121
1122
1123 return TXPKT_L4CSUM_DIS_F;
1124 }
1125 } else {
1126
1127
1128
1129 if (proto == IPPROTO_TCP)
1130 csum_type = TX_CSUM_TCPIP6;
1131 else if (proto == IPPROTO_UDP)
1132 csum_type = TX_CSUM_UDPIP6;
1133 else
1134 goto nocsum;
1135 }
1136
1137 if (likely(csum_type >= TX_CSUM_TCPIP)) {
1138 int eth_hdr_len, l4_len;
1139 u64 hdr_len;
1140
1141 if (inner_hdr_csum) {
1142
1143
1144
1145 l4_len = skb_inner_network_header_len(skb);
1146 eth_hdr_len = skb_inner_network_offset(skb) - ETH_HLEN;
1147 } else {
1148 l4_len = skb_network_header_len(skb);
1149 eth_hdr_len = skb_network_offset(skb) - ETH_HLEN;
1150 }
1151 hdr_len = TXPKT_IPHDR_LEN_V(l4_len);
1152
1153 if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5)
1154 hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1155 else
1156 hdr_len |= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1157 return TXPKT_CSUM_TYPE_V(csum_type) | hdr_len;
1158 } else {
1159 int start = skb_transport_offset(skb);
1160
1161 return TXPKT_CSUM_TYPE_V(csum_type) |
1162 TXPKT_CSUM_START_V(start) |
1163 TXPKT_CSUM_LOC_V(start + skb->csum_offset);
1164 }
1165 }
1166
1167 static void eth_txq_stop(struct sge_eth_txq *q)
1168 {
1169 netif_tx_stop_queue(q->txq);
1170 q->q.stops++;
1171 }
1172
1173 static inline void txq_advance(struct sge_txq *q, unsigned int n)
1174 {
1175 q->in_use += n;
1176 q->pidx += n;
1177 if (q->pidx >= q->size)
1178 q->pidx -= q->size;
1179 }
1180
1181 #ifdef CONFIG_CHELSIO_T4_FCOE
1182 static inline int
1183 cxgb_fcoe_offload(struct sk_buff *skb, struct adapter *adap,
1184 const struct port_info *pi, u64 *cntrl)
1185 {
1186 const struct cxgb_fcoe *fcoe = &pi->fcoe;
1187
1188 if (!(fcoe->flags & CXGB_FCOE_ENABLED))
1189 return 0;
1190
1191 if (skb->protocol != htons(ETH_P_FCOE))
1192 return 0;
1193
1194 skb_reset_mac_header(skb);
1195 skb->mac_len = sizeof(struct ethhdr);
1196
1197 skb_set_network_header(skb, skb->mac_len);
1198 skb_set_transport_header(skb, skb->mac_len + sizeof(struct fcoe_hdr));
1199
1200 if (!cxgb_fcoe_sof_eof_supported(adap, skb))
1201 return -ENOTSUPP;
1202
1203
1204 *cntrl = TXPKT_CSUM_TYPE_V(TX_CSUM_FCOE) |
1205 TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F |
1206 TXPKT_CSUM_START_V(CXGB_FCOE_TXPKT_CSUM_START) |
1207 TXPKT_CSUM_END_V(CXGB_FCOE_TXPKT_CSUM_END) |
1208 TXPKT_CSUM_LOC_V(CXGB_FCOE_TXPKT_CSUM_END);
1209 return 0;
1210 }
1211 #endif
1212
1213
1214
1215
1216 enum cpl_tx_tnl_lso_type cxgb_encap_offload_supported(struct sk_buff *skb)
1217 {
1218 u8 l4_hdr = 0;
1219 enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
1220 struct port_info *pi = netdev_priv(skb->dev);
1221 struct adapter *adapter = pi->adapter;
1222
1223 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1224 skb->inner_protocol != htons(ETH_P_TEB))
1225 return tnl_type;
1226
1227 switch (vlan_get_protocol(skb)) {
1228 case htons(ETH_P_IP):
1229 l4_hdr = ip_hdr(skb)->protocol;
1230 break;
1231 case htons(ETH_P_IPV6):
1232 l4_hdr = ipv6_hdr(skb)->nexthdr;
1233 break;
1234 default:
1235 return tnl_type;
1236 }
1237
1238 switch (l4_hdr) {
1239 case IPPROTO_UDP:
1240 if (adapter->vxlan_port == udp_hdr(skb)->dest)
1241 tnl_type = TX_TNL_TYPE_VXLAN;
1242 else if (adapter->geneve_port == udp_hdr(skb)->dest)
1243 tnl_type = TX_TNL_TYPE_GENEVE;
1244 break;
1245 default:
1246 return tnl_type;
1247 }
1248
1249 return tnl_type;
1250 }
1251
1252 static inline void t6_fill_tnl_lso(struct sk_buff *skb,
1253 struct cpl_tx_tnl_lso *tnl_lso,
1254 enum cpl_tx_tnl_lso_type tnl_type)
1255 {
1256 u32 val;
1257 int in_eth_xtra_len;
1258 int l3hdr_len = skb_network_header_len(skb);
1259 int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1260 const struct skb_shared_info *ssi = skb_shinfo(skb);
1261 bool v6 = (ip_hdr(skb)->version == 6);
1262
1263 val = CPL_TX_TNL_LSO_OPCODE_V(CPL_TX_TNL_LSO) |
1264 CPL_TX_TNL_LSO_FIRST_F |
1265 CPL_TX_TNL_LSO_LAST_F |
1266 (v6 ? CPL_TX_TNL_LSO_IPV6OUT_F : 0) |
1267 CPL_TX_TNL_LSO_ETHHDRLENOUT_V(eth_xtra_len / 4) |
1268 CPL_TX_TNL_LSO_IPHDRLENOUT_V(l3hdr_len / 4) |
1269 (v6 ? 0 : CPL_TX_TNL_LSO_IPHDRCHKOUT_F) |
1270 CPL_TX_TNL_LSO_IPLENSETOUT_F |
1271 (v6 ? 0 : CPL_TX_TNL_LSO_IPIDINCOUT_F);
1272 tnl_lso->op_to_IpIdSplitOut = htonl(val);
1273
1274 tnl_lso->IpIdOffsetOut = 0;
1275
1276
1277 val = skb_inner_mac_header(skb) - skb_mac_header(skb);
1278 in_eth_xtra_len = skb_inner_network_header(skb) -
1279 skb_inner_mac_header(skb) - ETH_HLEN;
1280
1281 switch (tnl_type) {
1282 case TX_TNL_TYPE_VXLAN:
1283 case TX_TNL_TYPE_GENEVE:
1284 tnl_lso->UdpLenSetOut_to_TnlHdrLen =
1285 htons(CPL_TX_TNL_LSO_UDPCHKCLROUT_F |
1286 CPL_TX_TNL_LSO_UDPLENSETOUT_F);
1287 break;
1288 default:
1289 tnl_lso->UdpLenSetOut_to_TnlHdrLen = 0;
1290 break;
1291 }
1292
1293 tnl_lso->UdpLenSetOut_to_TnlHdrLen |=
1294 htons(CPL_TX_TNL_LSO_TNLHDRLEN_V(val) |
1295 CPL_TX_TNL_LSO_TNLTYPE_V(tnl_type));
1296
1297 tnl_lso->r1 = 0;
1298
1299 val = CPL_TX_TNL_LSO_ETHHDRLEN_V(in_eth_xtra_len / 4) |
1300 CPL_TX_TNL_LSO_IPV6_V(inner_ip_hdr(skb)->version == 6) |
1301 CPL_TX_TNL_LSO_IPHDRLEN_V(skb_inner_network_header_len(skb) / 4) |
1302 CPL_TX_TNL_LSO_TCPHDRLEN_V(inner_tcp_hdrlen(skb) / 4);
1303 tnl_lso->Flow_to_TcpHdrLen = htonl(val);
1304
1305 tnl_lso->IpIdOffset = htons(0);
1306
1307 tnl_lso->IpIdSplit_to_Mss = htons(CPL_TX_TNL_LSO_MSS_V(ssi->gso_size));
1308 tnl_lso->TCPSeqOffset = htonl(0);
1309 tnl_lso->EthLenOffset_Size = htonl(CPL_TX_TNL_LSO_SIZE_V(skb->len));
1310 }
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324 int t4_sge_eth_txq_egress_update(struct adapter *adap, struct sge_eth_txq *eq,
1325 int maxreclaim)
1326 {
1327 unsigned int reclaimed, hw_cidx;
1328 struct sge_txq *q = &eq->q;
1329 int hw_in_use;
1330
1331 if (!q->in_use || !__netif_tx_trylock(eq->txq))
1332 return 0;
1333
1334
1335 reclaimed = reclaim_completed_tx(adap, &eq->q, maxreclaim, true);
1336
1337 hw_cidx = ntohs(READ_ONCE(q->stat->cidx));
1338 hw_in_use = q->pidx - hw_cidx;
1339 if (hw_in_use < 0)
1340 hw_in_use += q->size;
1341
1342
1343
1344
1345
1346
1347 if (netif_tx_queue_stopped(eq->txq) && hw_in_use < (q->size / 2)) {
1348 netif_tx_wake_queue(eq->txq);
1349 eq->q.restarts++;
1350 }
1351
1352 __netif_tx_unlock(eq->txq);
1353 return reclaimed;
1354 }
1355
1356
1357
1358
1359
1360
1361
1362
1363 static netdev_tx_t cxgb4_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1364 {
1365 u32 wr_mid, ctrl0, op;
1366 u64 cntrl, *end, *sgl;
1367 int qidx, credits;
1368 unsigned int flits, ndesc;
1369 struct adapter *adap;
1370 struct sge_eth_txq *q;
1371 const struct port_info *pi;
1372 struct fw_eth_tx_pkt_wr *wr;
1373 struct cpl_tx_pkt_core *cpl;
1374 const struct skb_shared_info *ssi;
1375 dma_addr_t addr[MAX_SKB_FRAGS + 1];
1376 bool immediate = false;
1377 int len, max_pkt_len;
1378 bool ptp_enabled = is_ptp_enabled(skb, dev);
1379 unsigned int chip_ver;
1380 enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
1381
1382 #ifdef CONFIG_CHELSIO_T4_FCOE
1383 int err;
1384 #endif
1385
1386
1387
1388
1389
1390 if (unlikely(skb->len < ETH_HLEN)) {
1391 out_free: dev_kfree_skb_any(skb);
1392 return NETDEV_TX_OK;
1393 }
1394
1395
1396 max_pkt_len = ETH_HLEN + dev->mtu;
1397 if (skb_vlan_tagged(skb))
1398 max_pkt_len += VLAN_HLEN;
1399 if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
1400 goto out_free;
1401
1402 pi = netdev_priv(dev);
1403 adap = pi->adapter;
1404 ssi = skb_shinfo(skb);
1405 #ifdef CONFIG_CHELSIO_IPSEC_INLINE
1406 if (xfrm_offload(skb) && !ssi->gso_size)
1407 return adap->uld[CXGB4_ULD_CRYPTO].tx_handler(skb, dev);
1408 #endif
1409
1410 qidx = skb_get_queue_mapping(skb);
1411 if (ptp_enabled) {
1412 spin_lock(&adap->ptp_lock);
1413 if (!(adap->ptp_tx_skb)) {
1414 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1415 adap->ptp_tx_skb = skb_get(skb);
1416 } else {
1417 spin_unlock(&adap->ptp_lock);
1418 goto out_free;
1419 }
1420 q = &adap->sge.ptptxq;
1421 } else {
1422 q = &adap->sge.ethtxq[qidx + pi->first_qset];
1423 }
1424 skb_tx_timestamp(skb);
1425
1426 reclaim_completed_tx(adap, &q->q, -1, true);
1427 cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
1428
1429 #ifdef CONFIG_CHELSIO_T4_FCOE
1430 err = cxgb_fcoe_offload(skb, adap, pi, &cntrl);
1431 if (unlikely(err == -ENOTSUPP)) {
1432 if (ptp_enabled)
1433 spin_unlock(&adap->ptp_lock);
1434 goto out_free;
1435 }
1436 #endif
1437
1438 chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
1439 flits = calc_tx_flits(skb, chip_ver);
1440 ndesc = flits_to_desc(flits);
1441 credits = txq_avail(&q->q) - ndesc;
1442
1443 if (unlikely(credits < 0)) {
1444 eth_txq_stop(q);
1445 dev_err(adap->pdev_dev,
1446 "%s: Tx ring %u full while queue awake!\n",
1447 dev->name, qidx);
1448 if (ptp_enabled)
1449 spin_unlock(&adap->ptp_lock);
1450 return NETDEV_TX_BUSY;
1451 }
1452
1453 if (is_eth_imm(skb, chip_ver))
1454 immediate = true;
1455
1456 if (skb->encapsulation && chip_ver > CHELSIO_T5)
1457 tnl_type = cxgb_encap_offload_supported(skb);
1458
1459 if (!immediate &&
1460 unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) {
1461 q->mapping_err++;
1462 if (ptp_enabled)
1463 spin_unlock(&adap->ptp_lock);
1464 goto out_free;
1465 }
1466
1467 wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1468 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1469
1470
1471
1472
1473
1474
1475
1476
1477 eth_txq_stop(q);
1478 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1479 }
1480
1481 wr = (void *)&q->q.desc[q->q.pidx];
1482 wr->equiq_to_len16 = htonl(wr_mid);
1483 wr->r3 = cpu_to_be64(0);
1484 end = (u64 *)wr + flits;
1485
1486 len = immediate ? skb->len : 0;
1487 len += sizeof(*cpl);
1488 if (ssi->gso_size) {
1489 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
1490 bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1491 int l3hdr_len = skb_network_header_len(skb);
1492 int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1493 struct cpl_tx_tnl_lso *tnl_lso = (void *)(wr + 1);
1494
1495 if (tnl_type)
1496 len += sizeof(*tnl_lso);
1497 else
1498 len += sizeof(*lso);
1499
1500 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1501 FW_WR_IMMDLEN_V(len));
1502 if (tnl_type) {
1503 struct iphdr *iph = ip_hdr(skb);
1504
1505 t6_fill_tnl_lso(skb, tnl_lso, tnl_type);
1506 cpl = (void *)(tnl_lso + 1);
1507
1508
1509
1510 if (iph->version == 4) {
1511 iph->check = 0;
1512 iph->tot_len = 0;
1513 iph->check = (u16)(~ip_fast_csum((u8 *)iph,
1514 iph->ihl));
1515 }
1516 if (skb->ip_summed == CHECKSUM_PARTIAL)
1517 cntrl = hwcsum(adap->params.chip, skb);
1518 } else {
1519 lso->lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
1520 LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F |
1521 LSO_IPV6_V(v6) |
1522 LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
1523 LSO_IPHDR_LEN_V(l3hdr_len / 4) |
1524 LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1525 lso->ipid_ofst = htons(0);
1526 lso->mss = htons(ssi->gso_size);
1527 lso->seqno_offset = htonl(0);
1528 if (is_t4(adap->params.chip))
1529 lso->len = htonl(skb->len);
1530 else
1531 lso->len = htonl(LSO_T5_XFER_SIZE_V(skb->len));
1532 cpl = (void *)(lso + 1);
1533
1534 if (CHELSIO_CHIP_VERSION(adap->params.chip)
1535 <= CHELSIO_T5)
1536 cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1537 else
1538 cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1539
1540 cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
1541 TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
1542 TXPKT_IPHDR_LEN_V(l3hdr_len);
1543 }
1544 sgl = (u64 *)(cpl + 1);
1545 if (unlikely((u8 *)sgl >= (u8 *)q->q.stat)) {
1546
1547
1548
1549
1550 if (sgl == (u64 *)q->q.stat) {
1551 int left = (u8 *)end - (u8 *)q->q.stat;
1552
1553 end = (void *)q->q.desc + left;
1554 sgl = (void *)q->q.desc;
1555 }
1556 }
1557 q->tso++;
1558 q->tx_cso += ssi->gso_segs;
1559 } else {
1560 if (ptp_enabled)
1561 op = FW_PTP_TX_PKT_WR;
1562 else
1563 op = FW_ETH_TX_PKT_WR;
1564 wr->op_immdlen = htonl(FW_WR_OP_V(op) |
1565 FW_WR_IMMDLEN_V(len));
1566 cpl = (void *)(wr + 1);
1567 sgl = (u64 *)(cpl + 1);
1568 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1569 cntrl = hwcsum(adap->params.chip, skb) |
1570 TXPKT_IPCSUM_DIS_F;
1571 q->tx_cso++;
1572 }
1573 }
1574
1575 if (skb_vlan_tag_present(skb)) {
1576 q->vlan_ins++;
1577 cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
1578 #ifdef CONFIG_CHELSIO_T4_FCOE
1579 if (skb->protocol == htons(ETH_P_FCOE))
1580 cntrl |= TXPKT_VLAN_V(
1581 ((skb->priority & 0x7) << VLAN_PRIO_SHIFT));
1582 #endif
1583 }
1584
1585 ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
1586 TXPKT_PF_V(adap->pf);
1587 if (ptp_enabled)
1588 ctrl0 |= TXPKT_TSTAMP_F;
1589 #ifdef CONFIG_CHELSIO_T4_DCB
1590 if (is_t4(adap->params.chip))
1591 ctrl0 |= TXPKT_OVLAN_IDX_V(q->dcb_prio);
1592 else
1593 ctrl0 |= TXPKT_T5_OVLAN_IDX_V(q->dcb_prio);
1594 #endif
1595 cpl->ctrl0 = htonl(ctrl0);
1596 cpl->pack = htons(0);
1597 cpl->len = htons(skb->len);
1598 cpl->ctrl1 = cpu_to_be64(cntrl);
1599
1600 if (immediate) {
1601 cxgb4_inline_tx_skb(skb, &q->q, sgl);
1602 dev_consume_skb_any(skb);
1603 } else {
1604 int last_desc;
1605
1606 cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, 0, addr);
1607 skb_orphan(skb);
1608
1609 last_desc = q->q.pidx + ndesc - 1;
1610 if (last_desc >= q->q.size)
1611 last_desc -= q->q.size;
1612 q->q.sdesc[last_desc].skb = skb;
1613 q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl;
1614 }
1615
1616 txq_advance(&q->q, ndesc);
1617
1618 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1619 if (ptp_enabled)
1620 spin_unlock(&adap->ptp_lock);
1621 return NETDEV_TX_OK;
1622 }
1623
1624
1625 enum {
1626
1627
1628
1629
1630
1631
1632 EQ_UNIT = SGE_EQ_IDXSIZE,
1633 FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
1634 TXD_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
1635
1636 T4VF_ETHTXQ_MAX_HDR = (sizeof(struct fw_eth_tx_pkt_vm_wr) +
1637 sizeof(struct cpl_tx_pkt_lso_core) +
1638 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64),
1639 };
1640
1641
1642
1643
1644
1645
1646
1647
1648 static inline int t4vf_is_eth_imm(const struct sk_buff *skb)
1649 {
1650
1651
1652
1653
1654
1655
1656 return false;
1657 }
1658
1659
1660
1661
1662
1663
1664
1665
1666 static inline unsigned int t4vf_calc_tx_flits(const struct sk_buff *skb)
1667 {
1668 unsigned int flits;
1669
1670
1671
1672
1673
1674 if (t4vf_is_eth_imm(skb))
1675 return DIV_ROUND_UP(skb->len + sizeof(struct cpl_tx_pkt),
1676 sizeof(__be64));
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
1687 if (skb_shinfo(skb)->gso_size)
1688 flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
1689 sizeof(struct cpl_tx_pkt_lso_core) +
1690 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
1691 else
1692 flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
1693 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
1694 return flits;
1695 }
1696
1697
1698
1699
1700
1701
1702
1703
1704 static netdev_tx_t cxgb4_vf_eth_xmit(struct sk_buff *skb,
1705 struct net_device *dev)
1706 {
1707 dma_addr_t addr[MAX_SKB_FRAGS + 1];
1708 const struct skb_shared_info *ssi;
1709 struct fw_eth_tx_pkt_vm_wr *wr;
1710 int qidx, credits, max_pkt_len;
1711 struct cpl_tx_pkt_core *cpl;
1712 const struct port_info *pi;
1713 unsigned int flits, ndesc;
1714 struct sge_eth_txq *txq;
1715 struct adapter *adapter;
1716 u64 cntrl, *end;
1717 u32 wr_mid;
1718 const size_t fw_hdr_copy_len = sizeof(wr->ethmacdst) +
1719 sizeof(wr->ethmacsrc) +
1720 sizeof(wr->ethtype) +
1721 sizeof(wr->vlantci);
1722
1723
1724
1725
1726
1727
1728 if (unlikely(skb->len < fw_hdr_copy_len))
1729 goto out_free;
1730
1731
1732 max_pkt_len = ETH_HLEN + dev->mtu;
1733 if (skb_vlan_tag_present(skb))
1734 max_pkt_len += VLAN_HLEN;
1735 if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
1736 goto out_free;
1737
1738
1739 pi = netdev_priv(dev);
1740 adapter = pi->adapter;
1741 qidx = skb_get_queue_mapping(skb);
1742 WARN_ON(qidx >= pi->nqsets);
1743 txq = &adapter->sge.ethtxq[pi->first_qset + qidx];
1744
1745
1746
1747
1748 reclaim_completed_tx(adapter, &txq->q, -1, true);
1749
1750
1751
1752
1753
1754 flits = t4vf_calc_tx_flits(skb);
1755 ndesc = flits_to_desc(flits);
1756 credits = txq_avail(&txq->q) - ndesc;
1757
1758 if (unlikely(credits < 0)) {
1759
1760
1761
1762
1763
1764 eth_txq_stop(txq);
1765 dev_err(adapter->pdev_dev,
1766 "%s: TX ring %u full while queue awake!\n",
1767 dev->name, qidx);
1768 return NETDEV_TX_BUSY;
1769 }
1770
1771 if (!t4vf_is_eth_imm(skb) &&
1772 unlikely(cxgb4_map_skb(adapter->pdev_dev, skb, addr) < 0)) {
1773
1774
1775
1776
1777 txq->mapping_err++;
1778 goto out_free;
1779 }
1780
1781 wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1782 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1783
1784
1785
1786
1787
1788
1789
1790
1791 eth_txq_stop(txq);
1792 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1793 }
1794
1795
1796
1797
1798
1799
1800 WARN_ON(DIV_ROUND_UP(T4VF_ETHTXQ_MAX_HDR, TXD_PER_EQ_UNIT) > 1);
1801 wr = (void *)&txq->q.desc[txq->q.pidx];
1802 wr->equiq_to_len16 = cpu_to_be32(wr_mid);
1803 wr->r3[0] = cpu_to_be32(0);
1804 wr->r3[1] = cpu_to_be32(0);
1805 skb_copy_from_linear_data(skb, (void *)wr->ethmacdst, fw_hdr_copy_len);
1806 end = (u64 *)wr + flits;
1807
1808
1809
1810
1811
1812 ssi = skb_shinfo(skb);
1813 if (ssi->gso_size) {
1814 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
1815 bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1816 int l3hdr_len = skb_network_header_len(skb);
1817 int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1818
1819 wr->op_immdlen =
1820 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1821 FW_WR_IMMDLEN_V(sizeof(*lso) +
1822 sizeof(*cpl)));
1823
1824 lso->lso_ctrl =
1825 cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
1826 LSO_FIRST_SLICE_F |
1827 LSO_LAST_SLICE_F |
1828 LSO_IPV6_V(v6) |
1829 LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
1830 LSO_IPHDR_LEN_V(l3hdr_len / 4) |
1831 LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1832 lso->ipid_ofst = cpu_to_be16(0);
1833 lso->mss = cpu_to_be16(ssi->gso_size);
1834 lso->seqno_offset = cpu_to_be32(0);
1835 if (is_t4(adapter->params.chip))
1836 lso->len = cpu_to_be32(skb->len);
1837 else
1838 lso->len = cpu_to_be32(LSO_T5_XFER_SIZE_V(skb->len));
1839
1840
1841
1842
1843 cpl = (void *)(lso + 1);
1844
1845 if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
1846 cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1847 else
1848 cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1849
1850 cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
1851 TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
1852 TXPKT_IPHDR_LEN_V(l3hdr_len);
1853 txq->tso++;
1854 txq->tx_cso += ssi->gso_segs;
1855 } else {
1856 int len;
1857
1858 len = (t4vf_is_eth_imm(skb)
1859 ? skb->len + sizeof(*cpl)
1860 : sizeof(*cpl));
1861 wr->op_immdlen =
1862 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1863 FW_WR_IMMDLEN_V(len));
1864
1865
1866
1867
1868 cpl = (void *)(wr + 1);
1869 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1870 cntrl = hwcsum(adapter->params.chip, skb) |
1871 TXPKT_IPCSUM_DIS_F;
1872 txq->tx_cso++;
1873 } else {
1874 cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
1875 }
1876 }
1877
1878
1879
1880
1881 if (skb_vlan_tag_present(skb)) {
1882 txq->vlan_ins++;
1883 cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
1884 }
1885
1886
1887 cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT) |
1888 TXPKT_INTF_V(pi->port_id) |
1889 TXPKT_PF_V(0));
1890 cpl->pack = cpu_to_be16(0);
1891 cpl->len = cpu_to_be16(skb->len);
1892 cpl->ctrl1 = cpu_to_be64(cntrl);
1893
1894
1895
1896
1897 if (t4vf_is_eth_imm(skb)) {
1898
1899
1900
1901 cxgb4_inline_tx_skb(skb, &txq->q, cpl + 1);
1902 dev_consume_skb_any(skb);
1903 } else {
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940 struct ulptx_sgl *sgl = (struct ulptx_sgl *)(cpl + 1);
1941 struct sge_txq *tq = &txq->q;
1942 int last_desc;
1943
1944
1945
1946
1947
1948
1949
1950 if (unlikely((void *)sgl == (void *)tq->stat)) {
1951 sgl = (void *)tq->desc;
1952 end = (void *)((void *)tq->desc +
1953 ((void *)end - (void *)tq->stat));
1954 }
1955
1956 cxgb4_write_sgl(skb, tq, sgl, end, 0, addr);
1957 skb_orphan(skb);
1958
1959 last_desc = tq->pidx + ndesc - 1;
1960 if (last_desc >= tq->size)
1961 last_desc -= tq->size;
1962 tq->sdesc[last_desc].skb = skb;
1963 tq->sdesc[last_desc].sgl = sgl;
1964 }
1965
1966
1967
1968
1969 txq_advance(&txq->q, ndesc);
1970
1971 cxgb4_ring_tx_db(adapter, &txq->q, ndesc);
1972 return NETDEV_TX_OK;
1973
1974 out_free:
1975
1976
1977
1978 dev_kfree_skb_any(skb);
1979 return NETDEV_TX_OK;
1980 }
1981
1982 netdev_tx_t t4_start_xmit(struct sk_buff *skb, struct net_device *dev)
1983 {
1984 struct port_info *pi = netdev_priv(dev);
1985
1986 if (unlikely(pi->eth_flags & PRIV_FLAG_PORT_TX_VM))
1987 return cxgb4_vf_eth_xmit(skb, dev);
1988
1989 return cxgb4_eth_xmit(skb, dev);
1990 }
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000 static inline void reclaim_completed_tx_imm(struct sge_txq *q)
2001 {
2002 int hw_cidx = ntohs(READ_ONCE(q->stat->cidx));
2003 int reclaim = hw_cidx - q->cidx;
2004
2005 if (reclaim < 0)
2006 reclaim += q->size;
2007
2008 q->in_use -= reclaim;
2009 q->cidx = hw_cidx;
2010 }
2011
2012
2013
2014
2015
2016
2017
2018 static inline int is_imm(const struct sk_buff *skb)
2019 {
2020 return skb->len <= MAX_CTRL_WR_LEN;
2021 }
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033 static void ctrlq_check_stop(struct sge_ctrl_txq *q, struct fw_wr_hdr *wr)
2034 {
2035 reclaim_completed_tx_imm(&q->q);
2036 if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) {
2037 wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F);
2038 q->q.stops++;
2039 q->full = 1;
2040 }
2041 }
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051 static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb)
2052 {
2053 unsigned int ndesc;
2054 struct fw_wr_hdr *wr;
2055
2056 if (unlikely(!is_imm(skb))) {
2057 WARN_ON(1);
2058 dev_kfree_skb(skb);
2059 return NET_XMIT_DROP;
2060 }
2061
2062 ndesc = DIV_ROUND_UP(skb->len, sizeof(struct tx_desc));
2063 spin_lock(&q->sendq.lock);
2064
2065 if (unlikely(q->full)) {
2066 skb->priority = ndesc;
2067 __skb_queue_tail(&q->sendq, skb);
2068 spin_unlock(&q->sendq.lock);
2069 return NET_XMIT_CN;
2070 }
2071
2072 wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
2073 cxgb4_inline_tx_skb(skb, &q->q, wr);
2074
2075 txq_advance(&q->q, ndesc);
2076 if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES))
2077 ctrlq_check_stop(q, wr);
2078
2079 cxgb4_ring_tx_db(q->adap, &q->q, ndesc);
2080 spin_unlock(&q->sendq.lock);
2081
2082 kfree_skb(skb);
2083 return NET_XMIT_SUCCESS;
2084 }
2085
2086
2087
2088
2089
2090
2091
2092 static void restart_ctrlq(unsigned long data)
2093 {
2094 struct sk_buff *skb;
2095 unsigned int written = 0;
2096 struct sge_ctrl_txq *q = (struct sge_ctrl_txq *)data;
2097
2098 spin_lock(&q->sendq.lock);
2099 reclaim_completed_tx_imm(&q->q);
2100 BUG_ON(txq_avail(&q->q) < TXQ_STOP_THRES);
2101
2102 while ((skb = __skb_dequeue(&q->sendq)) != NULL) {
2103 struct fw_wr_hdr *wr;
2104 unsigned int ndesc = skb->priority;
2105
2106 written += ndesc;
2107
2108
2109
2110 wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
2111 txq_advance(&q->q, ndesc);
2112 spin_unlock(&q->sendq.lock);
2113
2114 cxgb4_inline_tx_skb(skb, &q->q, wr);
2115 kfree_skb(skb);
2116
2117 if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) {
2118 unsigned long old = q->q.stops;
2119
2120 ctrlq_check_stop(q, wr);
2121 if (q->q.stops != old) {
2122 spin_lock(&q->sendq.lock);
2123 goto ringdb;
2124 }
2125 }
2126 if (written > 16) {
2127 cxgb4_ring_tx_db(q->adap, &q->q, written);
2128 written = 0;
2129 }
2130 spin_lock(&q->sendq.lock);
2131 }
2132 q->full = 0;
2133 ringdb:
2134 if (written)
2135 cxgb4_ring_tx_db(q->adap, &q->q, written);
2136 spin_unlock(&q->sendq.lock);
2137 }
2138
2139
2140
2141
2142
2143
2144
2145
2146 int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
2147 {
2148 int ret;
2149
2150 local_bh_disable();
2151 ret = ctrl_xmit(&adap->sge.ctrlq[0], skb);
2152 local_bh_enable();
2153 return ret;
2154 }
2155
2156
2157
2158
2159
2160
2161
2162
2163 static inline int is_ofld_imm(const struct sk_buff *skb)
2164 {
2165 struct work_request_hdr *req = (struct work_request_hdr *)skb->data;
2166 unsigned long opcode = FW_WR_OP_G(ntohl(req->wr_hi));
2167
2168 if (opcode == FW_CRYPTO_LOOKASIDE_WR)
2169 return skb->len <= SGE_MAX_WR_LEN;
2170 else
2171 return skb->len <= MAX_IMM_TX_PKT_LEN;
2172 }
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
2183 {
2184 unsigned int flits, cnt;
2185
2186 if (is_ofld_imm(skb))
2187 return DIV_ROUND_UP(skb->len, 8);
2188
2189 flits = skb_transport_offset(skb) / 8U;
2190 cnt = skb_shinfo(skb)->nr_frags;
2191 if (skb_tail_pointer(skb) != skb_transport_header(skb))
2192 cnt++;
2193 return flits + sgl_len(cnt);
2194 }
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205 static void txq_stop_maperr(struct sge_uld_txq *q)
2206 {
2207 q->mapping_err++;
2208 q->q.stops++;
2209 set_bit(q->q.cntxt_id - q->adap->sge.egr_start,
2210 q->adap->sge.txq_maperr);
2211 }
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221 static void ofldtxq_stop(struct sge_uld_txq *q, struct fw_wr_hdr *wr)
2222 {
2223 wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F);
2224 q->q.stops++;
2225 q->full = 1;
2226 }
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246 static void service_ofldq(struct sge_uld_txq *q)
2247 {
2248 u64 *pos, *before, *end;
2249 int credits;
2250 struct sk_buff *skb;
2251 struct sge_txq *txq;
2252 unsigned int left;
2253 unsigned int written = 0;
2254 unsigned int flits, ndesc;
2255
2256
2257
2258
2259
2260
2261
2262 if (q->service_ofldq_running)
2263 return;
2264 q->service_ofldq_running = true;
2265
2266 while ((skb = skb_peek(&q->sendq)) != NULL && !q->full) {
2267
2268
2269
2270
2271
2272
2273
2274 spin_unlock(&q->sendq.lock);
2275
2276 cxgb4_reclaim_completed_tx(q->adap, &q->q, false);
2277
2278 flits = skb->priority;
2279 ndesc = flits_to_desc(flits);
2280 credits = txq_avail(&q->q) - ndesc;
2281 BUG_ON(credits < 0);
2282 if (unlikely(credits < TXQ_STOP_THRES))
2283 ofldtxq_stop(q, (struct fw_wr_hdr *)skb->data);
2284
2285 pos = (u64 *)&q->q.desc[q->q.pidx];
2286 if (is_ofld_imm(skb))
2287 cxgb4_inline_tx_skb(skb, &q->q, pos);
2288 else if (cxgb4_map_skb(q->adap->pdev_dev, skb,
2289 (dma_addr_t *)skb->head)) {
2290 txq_stop_maperr(q);
2291 spin_lock(&q->sendq.lock);
2292 break;
2293 } else {
2294 int last_desc, hdr_len = skb_transport_offset(skb);
2295
2296
2297
2298
2299 before = (u64 *)pos;
2300 end = (u64 *)pos + flits;
2301 txq = &q->q;
2302 pos = (void *)inline_tx_skb_header(skb, &q->q,
2303 (void *)pos,
2304 hdr_len);
2305 if (before > (u64 *)pos) {
2306 left = (u8 *)end - (u8 *)txq->stat;
2307 end = (void *)txq->desc + left;
2308 }
2309
2310
2311
2312
2313
2314 if (pos == (u64 *)txq->stat) {
2315 left = (u8 *)end - (u8 *)txq->stat;
2316 end = (void *)txq->desc + left;
2317 pos = (void *)txq->desc;
2318 }
2319
2320 cxgb4_write_sgl(skb, &q->q, (void *)pos,
2321 end, hdr_len,
2322 (dma_addr_t *)skb->head);
2323 #ifdef CONFIG_NEED_DMA_MAP_STATE
2324 skb->dev = q->adap->port[0];
2325 skb->destructor = deferred_unmap_destructor;
2326 #endif
2327 last_desc = q->q.pidx + ndesc - 1;
2328 if (last_desc >= q->q.size)
2329 last_desc -= q->q.size;
2330 q->q.sdesc[last_desc].skb = skb;
2331 }
2332
2333 txq_advance(&q->q, ndesc);
2334 written += ndesc;
2335 if (unlikely(written > 32)) {
2336 cxgb4_ring_tx_db(q->adap, &q->q, written);
2337 written = 0;
2338 }
2339
2340
2341
2342
2343
2344
2345 spin_lock(&q->sendq.lock);
2346 __skb_unlink(skb, &q->sendq);
2347 if (is_ofld_imm(skb))
2348 kfree_skb(skb);
2349 }
2350 if (likely(written))
2351 cxgb4_ring_tx_db(q->adap, &q->q, written);
2352
2353
2354
2355
2356 q->service_ofldq_running = false;
2357 }
2358
2359
2360
2361
2362
2363
2364
2365
2366 static int ofld_xmit(struct sge_uld_txq *q, struct sk_buff *skb)
2367 {
2368 skb->priority = calc_tx_flits_ofld(skb);
2369 spin_lock(&q->sendq.lock);
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379 __skb_queue_tail(&q->sendq, skb);
2380 if (q->sendq.qlen == 1)
2381 service_ofldq(q);
2382
2383 spin_unlock(&q->sendq.lock);
2384 return NET_XMIT_SUCCESS;
2385 }
2386
2387
2388
2389
2390
2391
2392
2393 static void restart_ofldq(unsigned long data)
2394 {
2395 struct sge_uld_txq *q = (struct sge_uld_txq *)data;
2396
2397 spin_lock(&q->sendq.lock);
2398 q->full = 0;
2399 service_ofldq(q);
2400 spin_unlock(&q->sendq.lock);
2401 }
2402
2403
2404
2405
2406
2407
2408
2409
2410 static inline unsigned int skb_txq(const struct sk_buff *skb)
2411 {
2412 return skb->queue_mapping >> 1;
2413 }
2414
2415
2416
2417
2418
2419
2420
2421
2422 static inline unsigned int is_ctrl_pkt(const struct sk_buff *skb)
2423 {
2424 return skb->queue_mapping & 1;
2425 }
2426
2427 static inline int uld_send(struct adapter *adap, struct sk_buff *skb,
2428 unsigned int tx_uld_type)
2429 {
2430 struct sge_uld_txq_info *txq_info;
2431 struct sge_uld_txq *txq;
2432 unsigned int idx = skb_txq(skb);
2433
2434 if (unlikely(is_ctrl_pkt(skb))) {
2435
2436 if (adap->tids.nsftids)
2437 idx = 0;
2438 return ctrl_xmit(&adap->sge.ctrlq[idx], skb);
2439 }
2440
2441 txq_info = adap->sge.uld_txq_info[tx_uld_type];
2442 if (unlikely(!txq_info)) {
2443 WARN_ON(true);
2444 return NET_XMIT_DROP;
2445 }
2446
2447 txq = &txq_info->uldtxq[idx];
2448 return ofld_xmit(txq, skb);
2449 }
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460 int t4_ofld_send(struct adapter *adap, struct sk_buff *skb)
2461 {
2462 int ret;
2463
2464 local_bh_disable();
2465 ret = uld_send(adap, skb, CXGB4_TX_OFLD);
2466 local_bh_enable();
2467 return ret;
2468 }
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478 int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb)
2479 {
2480 return t4_ofld_send(netdev2adap(dev), skb);
2481 }
2482 EXPORT_SYMBOL(cxgb4_ofld_send);
2483
2484 static void *inline_tx_header(const void *src,
2485 const struct sge_txq *q,
2486 void *pos, int length)
2487 {
2488 int left = (void *)q->stat - pos;
2489 u64 *p;
2490
2491 if (likely(length <= left)) {
2492 memcpy(pos, src, length);
2493 pos += length;
2494 } else {
2495 memcpy(pos, src, left);
2496 memcpy(q->desc, src + left, length - left);
2497 pos = (void *)q->desc + (length - left);
2498 }
2499
2500 p = PTR_ALIGN(pos, 8);
2501 if ((uintptr_t)p & 8) {
2502 *p = 0;
2503 return p + 1;
2504 }
2505 return p;
2506 }
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516 static int ofld_xmit_direct(struct sge_uld_txq *q, const void *src,
2517 unsigned int len)
2518 {
2519 unsigned int ndesc;
2520 int credits;
2521 u64 *pos;
2522
2523
2524 if (len > MAX_IMM_OFLD_TX_DATA_WR_LEN) {
2525 WARN_ON(1);
2526 return NET_XMIT_DROP;
2527 }
2528
2529
2530
2531
2532
2533 if (!spin_trylock(&q->sendq.lock))
2534 return NET_XMIT_DROP;
2535
2536 if (q->full || !skb_queue_empty(&q->sendq) ||
2537 q->service_ofldq_running) {
2538 spin_unlock(&q->sendq.lock);
2539 return NET_XMIT_DROP;
2540 }
2541 ndesc = flits_to_desc(DIV_ROUND_UP(len, 8));
2542 credits = txq_avail(&q->q) - ndesc;
2543 pos = (u64 *)&q->q.desc[q->q.pidx];
2544
2545
2546 inline_tx_header(src, &q->q, pos, len);
2547 if (unlikely(credits < TXQ_STOP_THRES))
2548 ofldtxq_stop(q, (struct fw_wr_hdr *)pos);
2549 txq_advance(&q->q, ndesc);
2550 cxgb4_ring_tx_db(q->adap, &q->q, ndesc);
2551
2552 spin_unlock(&q->sendq.lock);
2553 return NET_XMIT_SUCCESS;
2554 }
2555
2556 int cxgb4_immdata_send(struct net_device *dev, unsigned int idx,
2557 const void *src, unsigned int len)
2558 {
2559 struct sge_uld_txq_info *txq_info;
2560 struct sge_uld_txq *txq;
2561 struct adapter *adap;
2562 int ret;
2563
2564 adap = netdev2adap(dev);
2565
2566 local_bh_disable();
2567 txq_info = adap->sge.uld_txq_info[CXGB4_TX_OFLD];
2568 if (unlikely(!txq_info)) {
2569 WARN_ON(true);
2570 local_bh_enable();
2571 return NET_XMIT_DROP;
2572 }
2573 txq = &txq_info->uldtxq[idx];
2574
2575 ret = ofld_xmit_direct(txq, src, len);
2576 local_bh_enable();
2577 return net_xmit_eval(ret);
2578 }
2579 EXPORT_SYMBOL(cxgb4_immdata_send);
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590 static int t4_crypto_send(struct adapter *adap, struct sk_buff *skb)
2591 {
2592 int ret;
2593
2594 local_bh_disable();
2595 ret = uld_send(adap, skb, CXGB4_TX_CRYPTO);
2596 local_bh_enable();
2597 return ret;
2598 }
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608 int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb)
2609 {
2610 return t4_crypto_send(netdev2adap(dev), skb);
2611 }
2612 EXPORT_SYMBOL(cxgb4_crypto_send);
2613
2614 static inline void copy_frags(struct sk_buff *skb,
2615 const struct pkt_gl *gl, unsigned int offset)
2616 {
2617 int i;
2618
2619
2620 __skb_fill_page_desc(skb, 0, gl->frags[0].page,
2621 gl->frags[0].offset + offset,
2622 gl->frags[0].size - offset);
2623 skb_shinfo(skb)->nr_frags = gl->nfrags;
2624 for (i = 1; i < gl->nfrags; i++)
2625 __skb_fill_page_desc(skb, i, gl->frags[i].page,
2626 gl->frags[i].offset,
2627 gl->frags[i].size);
2628
2629
2630 get_page(gl->frags[gl->nfrags - 1].page);
2631 }
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642 struct sk_buff *cxgb4_pktgl_to_skb(const struct pkt_gl *gl,
2643 unsigned int skb_len, unsigned int pull_len)
2644 {
2645 struct sk_buff *skb;
2646
2647
2648
2649
2650
2651
2652 if (gl->tot_len <= RX_COPY_THRES) {
2653 skb = dev_alloc_skb(gl->tot_len);
2654 if (unlikely(!skb))
2655 goto out;
2656 __skb_put(skb, gl->tot_len);
2657 skb_copy_to_linear_data(skb, gl->va, gl->tot_len);
2658 } else {
2659 skb = dev_alloc_skb(skb_len);
2660 if (unlikely(!skb))
2661 goto out;
2662 __skb_put(skb, pull_len);
2663 skb_copy_to_linear_data(skb, gl->va, pull_len);
2664
2665 copy_frags(skb, gl, pull_len);
2666 skb->len = gl->tot_len;
2667 skb->data_len = skb->len - pull_len;
2668 skb->truesize += skb->data_len;
2669 }
2670 out: return skb;
2671 }
2672 EXPORT_SYMBOL(cxgb4_pktgl_to_skb);
2673
2674
2675
2676
2677
2678
2679
2680
2681 static void t4_pktgl_free(const struct pkt_gl *gl)
2682 {
2683 int n;
2684 const struct page_frag *p;
2685
2686 for (p = gl->frags, n = gl->nfrags - 1; n--; p++)
2687 put_page(p->page);
2688 }
2689
2690
2691
2692
2693
2694 static noinline int handle_trace_pkt(struct adapter *adap,
2695 const struct pkt_gl *gl)
2696 {
2697 struct sk_buff *skb;
2698
2699 skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
2700 if (unlikely(!skb)) {
2701 t4_pktgl_free(gl);
2702 return 0;
2703 }
2704
2705 if (is_t4(adap->params.chip))
2706 __skb_pull(skb, sizeof(struct cpl_trace_pkt));
2707 else
2708 __skb_pull(skb, sizeof(struct cpl_t5_trace_pkt));
2709
2710 skb_reset_mac_header(skb);
2711 skb->protocol = htons(0xffff);
2712 skb->dev = adap->port[0];
2713 netif_receive_skb(skb);
2714 return 0;
2715 }
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726 static void cxgb4_sgetim_to_hwtstamp(struct adapter *adap,
2727 struct skb_shared_hwtstamps *hwtstamps,
2728 u64 sgetstamp)
2729 {
2730 u64 ns;
2731 u64 tmp = (sgetstamp * 1000 * 1000 + adap->params.vpd.cclk / 2);
2732
2733 ns = div_u64(tmp, adap->params.vpd.cclk);
2734
2735 memset(hwtstamps, 0, sizeof(*hwtstamps));
2736 hwtstamps->hwtstamp = ns_to_ktime(ns);
2737 }
2738
2739 static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
2740 const struct cpl_rx_pkt *pkt, unsigned long tnl_hdr_len)
2741 {
2742 struct adapter *adapter = rxq->rspq.adap;
2743 struct sge *s = &adapter->sge;
2744 struct port_info *pi;
2745 int ret;
2746 struct sk_buff *skb;
2747
2748 skb = napi_get_frags(&rxq->rspq.napi);
2749 if (unlikely(!skb)) {
2750 t4_pktgl_free(gl);
2751 rxq->stats.rx_drops++;
2752 return;
2753 }
2754
2755 copy_frags(skb, gl, s->pktshift);
2756 if (tnl_hdr_len)
2757 skb->csum_level = 1;
2758 skb->len = gl->tot_len - s->pktshift;
2759 skb->data_len = skb->len;
2760 skb->truesize += skb->data_len;
2761 skb->ip_summed = CHECKSUM_UNNECESSARY;
2762 skb_record_rx_queue(skb, rxq->rspq.idx);
2763 pi = netdev_priv(skb->dev);
2764 if (pi->rxtstamp)
2765 cxgb4_sgetim_to_hwtstamp(adapter, skb_hwtstamps(skb),
2766 gl->sgetstamp);
2767 if (rxq->rspq.netdev->features & NETIF_F_RXHASH)
2768 skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val,
2769 PKT_HASH_TYPE_L3);
2770
2771 if (unlikely(pkt->vlan_ex)) {
2772 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan));
2773 rxq->stats.vlan_ex++;
2774 }
2775 ret = napi_gro_frags(&rxq->rspq.napi);
2776 if (ret == GRO_HELD)
2777 rxq->stats.lro_pkts++;
2778 else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE)
2779 rxq->stats.lro_merged++;
2780 rxq->stats.pkts++;
2781 rxq->stats.rx_cso++;
2782 }
2783
2784 enum {
2785 RX_NON_PTP_PKT = 0,
2786 RX_PTP_PKT_SUC = 1,
2787 RX_PTP_PKT_ERR = 2
2788 };
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798 static noinline int t4_systim_to_hwstamp(struct adapter *adapter,
2799 struct sk_buff *skb)
2800 {
2801 struct skb_shared_hwtstamps *hwtstamps;
2802 struct cpl_rx_mps_pkt *cpl = NULL;
2803 unsigned char *data;
2804 int offset;
2805
2806 cpl = (struct cpl_rx_mps_pkt *)skb->data;
2807 if (!(CPL_RX_MPS_PKT_TYPE_G(ntohl(cpl->op_to_r1_hi)) &
2808 X_CPL_RX_MPS_PKT_TYPE_PTP))
2809 return RX_PTP_PKT_ERR;
2810
2811 data = skb->data + sizeof(*cpl);
2812 skb_pull(skb, 2 * sizeof(u64) + sizeof(struct cpl_rx_mps_pkt));
2813 offset = ETH_HLEN + IPV4_HLEN(skb->data) + UDP_HLEN;
2814 if (skb->len < offset + OFF_PTP_SEQUENCE_ID + sizeof(short))
2815 return RX_PTP_PKT_ERR;
2816
2817 hwtstamps = skb_hwtstamps(skb);
2818 memset(hwtstamps, 0, sizeof(*hwtstamps));
2819 hwtstamps->hwtstamp = ns_to_ktime(be64_to_cpu(*((u64 *)data)));
2820
2821 return RX_PTP_PKT_SUC;
2822 }
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832 static int t4_rx_hststamp(struct adapter *adapter, const __be64 *rsp,
2833 struct sge_eth_rxq *rxq, struct sk_buff *skb)
2834 {
2835 int ret;
2836
2837 if (unlikely((*(u8 *)rsp == CPL_RX_MPS_PKT) &&
2838 !is_t4(adapter->params.chip))) {
2839 ret = t4_systim_to_hwstamp(adapter, skb);
2840 if (ret == RX_PTP_PKT_ERR) {
2841 kfree_skb(skb);
2842 rxq->stats.rx_drops++;
2843 }
2844 return ret;
2845 }
2846 return RX_NON_PTP_PKT;
2847 }
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857 static int t4_tx_hststamp(struct adapter *adapter, struct sk_buff *skb,
2858 struct net_device *dev)
2859 {
2860 struct port_info *pi = netdev_priv(dev);
2861
2862 if (!is_t4(adapter->params.chip) && adapter->ptp_tx_skb) {
2863 cxgb4_ptp_read_hwstamp(adapter, pi);
2864 kfree_skb(skb);
2865 return 0;
2866 }
2867 return 1;
2868 }
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885 static void t4_tx_completion_handler(struct sge_rspq *rspq,
2886 const __be64 *rsp,
2887 const struct pkt_gl *gl)
2888 {
2889 u8 opcode = ((const struct rss_header *)rsp)->opcode;
2890 struct port_info *pi = netdev_priv(rspq->netdev);
2891 struct adapter *adapter = rspq->adap;
2892 struct sge *s = &adapter->sge;
2893 struct sge_eth_txq *txq;
2894
2895
2896 rsp++;
2897
2898
2899
2900 if (unlikely(opcode == CPL_FW4_MSG &&
2901 ((const struct cpl_fw4_msg *)rsp)->type ==
2902 FW_TYPE_RSSCPL)) {
2903 rsp++;
2904 opcode = ((const struct rss_header *)rsp)->opcode;
2905 rsp++;
2906 }
2907
2908 if (unlikely(opcode != CPL_SGE_EGR_UPDATE)) {
2909 pr_info("%s: unexpected FW4/CPL %#x on Rx queue\n",
2910 __func__, opcode);
2911 return;
2912 }
2913
2914 txq = &s->ethtxq[pi->first_qset + rspq->idx];
2915 t4_sge_eth_txq_egress_update(adapter, txq, -1);
2916 }
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926 int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
2927 const struct pkt_gl *si)
2928 {
2929 bool csum_ok;
2930 struct sk_buff *skb;
2931 const struct cpl_rx_pkt *pkt;
2932 struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
2933 struct adapter *adapter = q->adap;
2934 struct sge *s = &q->adap->sge;
2935 int cpl_trace_pkt = is_t4(q->adap->params.chip) ?
2936 CPL_TRACE_PKT : CPL_TRACE_PKT_T5;
2937 u16 err_vec, tnl_hdr_len = 0;
2938 struct port_info *pi;
2939 int ret = 0;
2940
2941
2942
2943
2944 if (unlikely((*(u8 *)rsp == CPL_FW4_MSG) ||
2945 (*(u8 *)rsp == CPL_SGE_EGR_UPDATE))) {
2946 t4_tx_completion_handler(q, rsp, si);
2947 return 0;
2948 }
2949
2950 if (unlikely(*(u8 *)rsp == cpl_trace_pkt))
2951 return handle_trace_pkt(q->adap, si);
2952
2953 pkt = (const struct cpl_rx_pkt *)rsp;
2954
2955 if (q->adap->params.tp.rx_pkt_encap) {
2956 err_vec = T6_COMPR_RXERR_VEC_G(be16_to_cpu(pkt->err_vec));
2957 tnl_hdr_len = T6_RX_TNLHDR_LEN_G(ntohs(pkt->err_vec));
2958 } else {
2959 err_vec = be16_to_cpu(pkt->err_vec);
2960 }
2961
2962 csum_ok = pkt->csum_calc && !err_vec &&
2963 (q->netdev->features & NETIF_F_RXCSUM);
2964
2965 if (err_vec)
2966 rxq->stats.bad_rx_pkts++;
2967
2968 if (((pkt->l2info & htonl(RXF_TCP_F)) ||
2969 tnl_hdr_len) &&
2970 (q->netdev->features & NETIF_F_GRO) && csum_ok && !pkt->ip_frag) {
2971 do_gro(rxq, si, pkt, tnl_hdr_len);
2972 return 0;
2973 }
2974
2975 skb = cxgb4_pktgl_to_skb(si, RX_PKT_SKB_LEN, RX_PULL_LEN);
2976 if (unlikely(!skb)) {
2977 t4_pktgl_free(si);
2978 rxq->stats.rx_drops++;
2979 return 0;
2980 }
2981 pi = netdev_priv(q->netdev);
2982
2983
2984 if (unlikely(pi->ptp_enable)) {
2985 ret = t4_rx_hststamp(adapter, rsp, rxq, skb);
2986 if (ret == RX_PTP_PKT_ERR)
2987 return 0;
2988 }
2989 if (likely(!ret))
2990 __skb_pull(skb, s->pktshift);
2991
2992
2993 if (unlikely(pi->ptp_enable && !ret &&
2994 (pkt->l2info & htonl(RXF_UDP_F)) &&
2995 cxgb4_ptp_is_ptp_rx(skb))) {
2996 if (!t4_tx_hststamp(adapter, skb, q->netdev))
2997 return 0;
2998 }
2999
3000 skb->protocol = eth_type_trans(skb, q->netdev);
3001 skb_record_rx_queue(skb, q->idx);
3002 if (skb->dev->features & NETIF_F_RXHASH)
3003 skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val,
3004 PKT_HASH_TYPE_L3);
3005
3006 rxq->stats.pkts++;
3007
3008 if (pi->rxtstamp)
3009 cxgb4_sgetim_to_hwtstamp(q->adap, skb_hwtstamps(skb),
3010 si->sgetstamp);
3011 if (csum_ok && (pkt->l2info & htonl(RXF_UDP_F | RXF_TCP_F))) {
3012 if (!pkt->ip_frag) {
3013 skb->ip_summed = CHECKSUM_UNNECESSARY;
3014 rxq->stats.rx_cso++;
3015 } else if (pkt->l2info & htonl(RXF_IP_F)) {
3016 __sum16 c = (__force __sum16)pkt->csum;
3017 skb->csum = csum_unfold(c);
3018
3019 if (tnl_hdr_len) {
3020 skb->ip_summed = CHECKSUM_UNNECESSARY;
3021 skb->csum_level = 1;
3022 } else {
3023 skb->ip_summed = CHECKSUM_COMPLETE;
3024 }
3025 rxq->stats.rx_cso++;
3026 }
3027 } else {
3028 skb_checksum_none_assert(skb);
3029 #ifdef CONFIG_CHELSIO_T4_FCOE
3030 #define CPL_RX_PKT_FLAGS (RXF_PSH_F | RXF_SYN_F | RXF_UDP_F | \
3031 RXF_TCP_F | RXF_IP_F | RXF_IP6_F | RXF_LRO_F)
3032
3033 if (!(pkt->l2info & cpu_to_be32(CPL_RX_PKT_FLAGS))) {
3034 if ((pkt->l2info & cpu_to_be32(RXF_FCOE_F)) &&
3035 (pi->fcoe.flags & CXGB_FCOE_ENABLED)) {
3036 if (q->adap->params.tp.rx_pkt_encap)
3037 csum_ok = err_vec &
3038 T6_COMPR_RXERR_SUM_F;
3039 else
3040 csum_ok = err_vec & RXERR_CSUM_F;
3041 if (!csum_ok)
3042 skb->ip_summed = CHECKSUM_UNNECESSARY;
3043 }
3044 }
3045
3046 #undef CPL_RX_PKT_FLAGS
3047 #endif
3048 }
3049
3050 if (unlikely(pkt->vlan_ex)) {
3051 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan));
3052 rxq->stats.vlan_ex++;
3053 }
3054 skb_mark_napi_id(skb, &q->napi);
3055 netif_receive_skb(skb);
3056 return 0;
3057 }
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074 static void restore_rx_bufs(const struct pkt_gl *si, struct sge_fl *q,
3075 int frags)
3076 {
3077 struct rx_sw_desc *d;
3078
3079 while (frags--) {
3080 if (q->cidx == 0)
3081 q->cidx = q->size - 1;
3082 else
3083 q->cidx--;
3084 d = &q->sdesc[q->cidx];
3085 d->page = si->frags[frags].page;
3086 d->dma_addr |= RX_UNMAPPED_BUF;
3087 q->avail++;
3088 }
3089 }
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099 static inline bool is_new_response(const struct rsp_ctrl *r,
3100 const struct sge_rspq *q)
3101 {
3102 return (r->type_gen >> RSPD_GEN_S) == q->gen;
3103 }
3104
3105
3106
3107
3108
3109
3110
3111 static inline void rspq_next(struct sge_rspq *q)
3112 {
3113 q->cur_desc = (void *)q->cur_desc + q->iqe_len;
3114 if (unlikely(++q->cidx == q->size)) {
3115 q->cidx = 0;
3116 q->gen ^= 1;
3117 q->cur_desc = q->desc;
3118 }
3119 }
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134 static int process_responses(struct sge_rspq *q, int budget)
3135 {
3136 int ret, rsp_type;
3137 int budget_left = budget;
3138 const struct rsp_ctrl *rc;
3139 struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
3140 struct adapter *adapter = q->adap;
3141 struct sge *s = &adapter->sge;
3142
3143 while (likely(budget_left)) {
3144 rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc));
3145 if (!is_new_response(rc, q)) {
3146 if (q->flush_handler)
3147 q->flush_handler(q);
3148 break;
3149 }
3150
3151 dma_rmb();
3152 rsp_type = RSPD_TYPE_G(rc->type_gen);
3153 if (likely(rsp_type == RSPD_TYPE_FLBUF_X)) {
3154 struct page_frag *fp;
3155 struct pkt_gl si;
3156 const struct rx_sw_desc *rsd;
3157 u32 len = ntohl(rc->pldbuflen_qid), bufsz, frags;
3158
3159 if (len & RSPD_NEWBUF_F) {
3160 if (likely(q->offset > 0)) {
3161 free_rx_bufs(q->adap, &rxq->fl, 1);
3162 q->offset = 0;
3163 }
3164 len = RSPD_LEN_G(len);
3165 }
3166 si.tot_len = len;
3167
3168
3169 for (frags = 0, fp = si.frags; ; frags++, fp++) {
3170 rsd = &rxq->fl.sdesc[rxq->fl.cidx];
3171 bufsz = get_buf_size(adapter, rsd);
3172 fp->page = rsd->page;
3173 fp->offset = q->offset;
3174 fp->size = min(bufsz, len);
3175 len -= fp->size;
3176 if (!len)
3177 break;
3178 unmap_rx_buf(q->adap, &rxq->fl);
3179 }
3180
3181 si.sgetstamp = SGE_TIMESTAMP_G(
3182 be64_to_cpu(rc->last_flit));
3183
3184
3185
3186
3187 dma_sync_single_for_cpu(q->adap->pdev_dev,
3188 get_buf_addr(rsd),
3189 fp->size, DMA_FROM_DEVICE);
3190
3191 si.va = page_address(si.frags[0].page) +
3192 si.frags[0].offset;
3193 prefetch(si.va);
3194
3195 si.nfrags = frags + 1;
3196 ret = q->handler(q, q->cur_desc, &si);
3197 if (likely(ret == 0))
3198 q->offset += ALIGN(fp->size, s->fl_align);
3199 else
3200 restore_rx_bufs(&si, &rxq->fl, frags);
3201 } else if (likely(rsp_type == RSPD_TYPE_CPL_X)) {
3202 ret = q->handler(q, q->cur_desc, NULL);
3203 } else {
3204 ret = q->handler(q, (const __be64 *)rc, CXGB4_MSG_AN);
3205 }
3206
3207 if (unlikely(ret)) {
3208
3209 q->next_intr_params = QINTR_TIMER_IDX_V(NOMEM_TMR_IDX);
3210 break;
3211 }
3212
3213 rspq_next(q);
3214 budget_left--;
3215 }
3216
3217 if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 16)
3218 __refill_fl(q->adap, &rxq->fl);
3219 return budget - budget_left;
3220 }
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233 static int napi_rx_handler(struct napi_struct *napi, int budget)
3234 {
3235 unsigned int params;
3236 struct sge_rspq *q = container_of(napi, struct sge_rspq, napi);
3237 int work_done;
3238 u32 val;
3239
3240 work_done = process_responses(q, budget);
3241 if (likely(work_done < budget)) {
3242 int timer_index;
3243
3244 napi_complete_done(napi, work_done);
3245 timer_index = QINTR_TIMER_IDX_G(q->next_intr_params);
3246
3247 if (q->adaptive_rx) {
3248 if (work_done > max(timer_pkt_quota[timer_index],
3249 MIN_NAPI_WORK))
3250 timer_index = (timer_index + 1);
3251 else
3252 timer_index = timer_index - 1;
3253
3254 timer_index = clamp(timer_index, 0, SGE_TIMERREGS - 1);
3255 q->next_intr_params =
3256 QINTR_TIMER_IDX_V(timer_index) |
3257 QINTR_CNT_EN_V(0);
3258 params = q->next_intr_params;
3259 } else {
3260 params = q->next_intr_params;
3261 q->next_intr_params = q->intr_params;
3262 }
3263 } else
3264 params = QINTR_TIMER_IDX_V(7);
3265
3266 val = CIDXINC_V(work_done) | SEINTARM_V(params);
3267
3268
3269
3270
3271 if (unlikely(q->bar2_addr == NULL)) {
3272 t4_write_reg(q->adap, MYPF_REG(SGE_PF_GTS_A),
3273 val | INGRESSQID_V((u32)q->cntxt_id));
3274 } else {
3275 writel(val | INGRESSQID_V(q->bar2_qid),
3276 q->bar2_addr + SGE_UDB_GTS);
3277 wmb();
3278 }
3279 return work_done;
3280 }
3281
3282
3283
3284
3285 irqreturn_t t4_sge_intr_msix(int irq, void *cookie)
3286 {
3287 struct sge_rspq *q = cookie;
3288
3289 napi_schedule(&q->napi);
3290 return IRQ_HANDLED;
3291 }
3292
3293
3294
3295
3296
3297 static unsigned int process_intrq(struct adapter *adap)
3298 {
3299 unsigned int credits;
3300 const struct rsp_ctrl *rc;
3301 struct sge_rspq *q = &adap->sge.intrq;
3302 u32 val;
3303
3304 spin_lock(&adap->sge.intrq_lock);
3305 for (credits = 0; ; credits++) {
3306 rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc));
3307 if (!is_new_response(rc, q))
3308 break;
3309
3310 dma_rmb();
3311 if (RSPD_TYPE_G(rc->type_gen) == RSPD_TYPE_INTR_X) {
3312 unsigned int qid = ntohl(rc->pldbuflen_qid);
3313
3314 qid -= adap->sge.ingr_start;
3315 napi_schedule(&adap->sge.ingr_map[qid]->napi);
3316 }
3317
3318 rspq_next(q);
3319 }
3320
3321 val = CIDXINC_V(credits) | SEINTARM_V(q->intr_params);
3322
3323
3324
3325
3326 if (unlikely(q->bar2_addr == NULL)) {
3327 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
3328 val | INGRESSQID_V(q->cntxt_id));
3329 } else {
3330 writel(val | INGRESSQID_V(q->bar2_qid),
3331 q->bar2_addr + SGE_UDB_GTS);
3332 wmb();
3333 }
3334 spin_unlock(&adap->sge.intrq_lock);
3335 return credits;
3336 }
3337
3338
3339
3340
3341
3342 static irqreturn_t t4_intr_msi(int irq, void *cookie)
3343 {
3344 struct adapter *adap = cookie;
3345
3346 if (adap->flags & CXGB4_MASTER_PF)
3347 t4_slow_intr_handler(adap);
3348 process_intrq(adap);
3349 return IRQ_HANDLED;
3350 }
3351
3352
3353
3354
3355
3356
3357 static irqreturn_t t4_intr_intx(int irq, void *cookie)
3358 {
3359 struct adapter *adap = cookie;
3360
3361 t4_write_reg(adap, MYPF_REG(PCIE_PF_CLI_A), 0);
3362 if (((adap->flags & CXGB4_MASTER_PF) && t4_slow_intr_handler(adap)) |
3363 process_intrq(adap))
3364 return IRQ_HANDLED;
3365 return IRQ_NONE;
3366 }
3367
3368
3369
3370
3371
3372
3373
3374
3375 irq_handler_t t4_intr_handler(struct adapter *adap)
3376 {
3377 if (adap->flags & CXGB4_USING_MSIX)
3378 return t4_sge_intr_msix;
3379 if (adap->flags & CXGB4_USING_MSI)
3380 return t4_intr_msi;
3381 return t4_intr_intx;
3382 }
3383
3384 static void sge_rx_timer_cb(struct timer_list *t)
3385 {
3386 unsigned long m;
3387 unsigned int i;
3388 struct adapter *adap = from_timer(adap, t, sge.rx_timer);
3389 struct sge *s = &adap->sge;
3390
3391 for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++)
3392 for (m = s->starving_fl[i]; m; m &= m - 1) {
3393 struct sge_eth_rxq *rxq;
3394 unsigned int id = __ffs(m) + i * BITS_PER_LONG;
3395 struct sge_fl *fl = s->egr_map[id];
3396
3397 clear_bit(id, s->starving_fl);
3398 smp_mb__after_atomic();
3399
3400 if (fl_starving(adap, fl)) {
3401 rxq = container_of(fl, struct sge_eth_rxq, fl);
3402 if (napi_reschedule(&rxq->rspq.napi))
3403 fl->starving++;
3404 else
3405 set_bit(id, s->starving_fl);
3406 }
3407 }
3408
3409
3410
3411
3412 if (!(adap->flags & CXGB4_MASTER_PF))
3413 goto done;
3414
3415 t4_idma_monitor(adap, &s->idma_monitor, HZ, RX_QCHECK_PERIOD);
3416
3417 done:
3418 mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD);
3419 }
3420
3421 static void sge_tx_timer_cb(struct timer_list *t)
3422 {
3423 struct adapter *adap = from_timer(adap, t, sge.tx_timer);
3424 struct sge *s = &adap->sge;
3425 unsigned long m, period;
3426 unsigned int i, budget;
3427
3428 for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++)
3429 for (m = s->txq_maperr[i]; m; m &= m - 1) {
3430 unsigned long id = __ffs(m) + i * BITS_PER_LONG;
3431 struct sge_uld_txq *txq = s->egr_map[id];
3432
3433 clear_bit(id, s->txq_maperr);
3434 tasklet_schedule(&txq->qresume_tsk);
3435 }
3436
3437 if (!is_t4(adap->params.chip)) {
3438 struct sge_eth_txq *q = &s->ptptxq;
3439 int avail;
3440
3441 spin_lock(&adap->ptp_lock);
3442 avail = reclaimable(&q->q);
3443
3444 if (avail) {
3445 free_tx_desc(adap, &q->q, avail, false);
3446 q->q.in_use -= avail;
3447 }
3448 spin_unlock(&adap->ptp_lock);
3449 }
3450
3451 budget = MAX_TIMER_TX_RECLAIM;
3452 i = s->ethtxq_rover;
3453 do {
3454 budget -= t4_sge_eth_txq_egress_update(adap, &s->ethtxq[i],
3455 budget);
3456 if (!budget)
3457 break;
3458
3459 if (++i >= s->ethqsets)
3460 i = 0;
3461 } while (i != s->ethtxq_rover);
3462 s->ethtxq_rover = i;
3463
3464 if (budget == 0) {
3465
3466
3467
3468 period = 2;
3469 } else {
3470
3471
3472
3473 period = TX_QCHECK_PERIOD;
3474 }
3475
3476 mod_timer(&s->tx_timer, jiffies + period);
3477 }
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492 static void __iomem *bar2_address(struct adapter *adapter,
3493 unsigned int qid,
3494 enum t4_bar2_qtype qtype,
3495 unsigned int *pbar2_qid)
3496 {
3497 u64 bar2_qoffset;
3498 int ret;
3499
3500 ret = t4_bar2_sge_qregs(adapter, qid, qtype, 0,
3501 &bar2_qoffset, pbar2_qid);
3502 if (ret)
3503 return NULL;
3504
3505 return adapter->bar2 + bar2_qoffset;
3506 }
3507
3508
3509
3510
3511 int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
3512 struct net_device *dev, int intr_idx,
3513 struct sge_fl *fl, rspq_handler_t hnd,
3514 rspq_flush_handler_t flush_hnd, int cong)
3515 {
3516 int ret, flsz = 0;
3517 struct fw_iq_cmd c;
3518 struct sge *s = &adap->sge;
3519 struct port_info *pi = netdev_priv(dev);
3520 int relaxed = !(adap->flags & CXGB4_ROOT_NO_RELAXED_ORDERING);
3521
3522
3523 iq->size = roundup(iq->size, 16);
3524
3525 iq->desc = alloc_ring(adap->pdev_dev, iq->size, iq->iqe_len, 0,
3526 &iq->phys_addr, NULL, 0,
3527 dev_to_node(adap->pdev_dev));
3528 if (!iq->desc)
3529 return -ENOMEM;
3530
3531 memset(&c, 0, sizeof(c));
3532 c.op_to_vfn = htonl(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F |
3533 FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3534 FW_IQ_CMD_PFN_V(adap->pf) | FW_IQ_CMD_VFN_V(0));
3535 c.alloc_to_len16 = htonl(FW_IQ_CMD_ALLOC_F | FW_IQ_CMD_IQSTART_F |
3536 FW_LEN16(c));
3537 c.type_to_iqandstindex = htonl(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP) |
3538 FW_IQ_CMD_IQASYNCH_V(fwevtq) | FW_IQ_CMD_VIID_V(pi->viid) |
3539 FW_IQ_CMD_IQANDST_V(intr_idx < 0) |
3540 FW_IQ_CMD_IQANUD_V(UPDATEDELIVERY_INTERRUPT_X) |
3541 FW_IQ_CMD_IQANDSTINDEX_V(intr_idx >= 0 ? intr_idx :
3542 -intr_idx - 1));
3543 c.iqdroprss_to_iqesize = htons(FW_IQ_CMD_IQPCIECH_V(pi->tx_chan) |
3544 FW_IQ_CMD_IQGTSMODE_F |
3545 FW_IQ_CMD_IQINTCNTTHRESH_V(iq->pktcnt_idx) |
3546 FW_IQ_CMD_IQESIZE_V(ilog2(iq->iqe_len) - 4));
3547 c.iqsize = htons(iq->size);
3548 c.iqaddr = cpu_to_be64(iq->phys_addr);
3549 if (cong >= 0)
3550 c.iqns_to_fl0congen = htonl(FW_IQ_CMD_IQFLINTCONGEN_F |
3551 FW_IQ_CMD_IQTYPE_V(cong ? FW_IQ_IQTYPE_NIC
3552 : FW_IQ_IQTYPE_OFLD));
3553
3554 if (fl) {
3555 unsigned int chip_ver =
3556 CHELSIO_CHIP_VERSION(adap->params.chip);
3557
3558
3559
3560
3561
3562
3563
3564
3565 if (fl->size < s->fl_starve_thres - 1 + 2 * 8)
3566 fl->size = s->fl_starve_thres - 1 + 2 * 8;
3567 fl->size = roundup(fl->size, 8);
3568 fl->desc = alloc_ring(adap->pdev_dev, fl->size, sizeof(__be64),
3569 sizeof(struct rx_sw_desc), &fl->addr,
3570 &fl->sdesc, s->stat_len,
3571 dev_to_node(adap->pdev_dev));
3572 if (!fl->desc)
3573 goto fl_nomem;
3574
3575 flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc);
3576 c.iqns_to_fl0congen |= htonl(FW_IQ_CMD_FL0PACKEN_F |
3577 FW_IQ_CMD_FL0FETCHRO_V(relaxed) |
3578 FW_IQ_CMD_FL0DATARO_V(relaxed) |
3579 FW_IQ_CMD_FL0PADEN_F);
3580 if (cong >= 0)
3581 c.iqns_to_fl0congen |=
3582 htonl(FW_IQ_CMD_FL0CNGCHMAP_V(cong) |
3583 FW_IQ_CMD_FL0CONGCIF_F |
3584 FW_IQ_CMD_FL0CONGEN_F);
3585
3586
3587
3588
3589
3590
3591
3592
3593 c.fl0dcaen_to_fl0cidxfthresh =
3594 htons(FW_IQ_CMD_FL0FBMIN_V(chip_ver <= CHELSIO_T5 ?
3595 FETCHBURSTMIN_128B_X :
3596 FETCHBURSTMIN_64B_T6_X) |
3597 FW_IQ_CMD_FL0FBMAX_V((chip_ver <= CHELSIO_T5) ?
3598 FETCHBURSTMAX_512B_X :
3599 FETCHBURSTMAX_256B_X));
3600 c.fl0size = htons(flsz);
3601 c.fl0addr = cpu_to_be64(fl->addr);
3602 }
3603
3604 ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3605 if (ret)
3606 goto err;
3607
3608 netif_napi_add(dev, &iq->napi, napi_rx_handler, 64);
3609 iq->cur_desc = iq->desc;
3610 iq->cidx = 0;
3611 iq->gen = 1;
3612 iq->next_intr_params = iq->intr_params;
3613 iq->cntxt_id = ntohs(c.iqid);
3614 iq->abs_id = ntohs(c.physiqid);
3615 iq->bar2_addr = bar2_address(adap,
3616 iq->cntxt_id,
3617 T4_BAR2_QTYPE_INGRESS,
3618 &iq->bar2_qid);
3619 iq->size--;
3620 iq->netdev = dev;
3621 iq->handler = hnd;
3622 iq->flush_handler = flush_hnd;
3623
3624 memset(&iq->lro_mgr, 0, sizeof(struct t4_lro_mgr));
3625 skb_queue_head_init(&iq->lro_mgr.lroq);
3626
3627
3628 iq->offset = fl ? 0 : -1;
3629
3630 adap->sge.ingr_map[iq->cntxt_id - adap->sge.ingr_start] = iq;
3631
3632 if (fl) {
3633 fl->cntxt_id = ntohs(c.fl0id);
3634 fl->avail = fl->pend_cred = 0;
3635 fl->pidx = fl->cidx = 0;
3636 fl->alloc_failed = fl->large_alloc_failed = fl->starving = 0;
3637 adap->sge.egr_map[fl->cntxt_id - adap->sge.egr_start] = fl;
3638
3639
3640
3641
3642 fl->bar2_addr = bar2_address(adap,
3643 fl->cntxt_id,
3644 T4_BAR2_QTYPE_EGRESS,
3645 &fl->bar2_qid);
3646 refill_fl(adap, fl, fl_cap(fl), GFP_KERNEL);
3647 }
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657 if (!is_t4(adap->params.chip) && cong >= 0) {
3658 u32 param, val, ch_map = 0;
3659 int i;
3660 u16 cng_ch_bits_log = adap->params.arch.cng_ch_bits_log;
3661
3662 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
3663 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
3664 FW_PARAMS_PARAM_YZ_V(iq->cntxt_id));
3665 if (cong == 0) {
3666 val = CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_QUEUE_X);
3667 } else {
3668 val =
3669 CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_CHANNEL_X);
3670 for (i = 0; i < 4; i++) {
3671 if (cong & (1 << i))
3672 ch_map |= 1 << (i << cng_ch_bits_log);
3673 }
3674 val |= CONMCTXT_CNGCHMAP_V(ch_map);
3675 }
3676 ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
3677 ¶m, &val);
3678 if (ret)
3679 dev_warn(adap->pdev_dev, "Failed to set Congestion"
3680 " Manager Context for Ingress Queue %d: %d\n",
3681 iq->cntxt_id, -ret);
3682 }
3683
3684 return 0;
3685
3686 fl_nomem:
3687 ret = -ENOMEM;
3688 err:
3689 if (iq->desc) {
3690 dma_free_coherent(adap->pdev_dev, iq->size * iq->iqe_len,
3691 iq->desc, iq->phys_addr);
3692 iq->desc = NULL;
3693 }
3694 if (fl && fl->desc) {
3695 kfree(fl->sdesc);
3696 fl->sdesc = NULL;
3697 dma_free_coherent(adap->pdev_dev, flsz * sizeof(struct tx_desc),
3698 fl->desc, fl->addr);
3699 fl->desc = NULL;
3700 }
3701 return ret;
3702 }
3703
3704 static void init_txq(struct adapter *adap, struct sge_txq *q, unsigned int id)
3705 {
3706 q->cntxt_id = id;
3707 q->bar2_addr = bar2_address(adap,
3708 q->cntxt_id,
3709 T4_BAR2_QTYPE_EGRESS,
3710 &q->bar2_qid);
3711 q->in_use = 0;
3712 q->cidx = q->pidx = 0;
3713 q->stops = q->restarts = 0;
3714 q->stat = (void *)&q->desc[q->size];
3715 spin_lock_init(&q->db_lock);
3716 adap->sge.egr_map[id - adap->sge.egr_start] = q;
3717 }
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728 int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
3729 struct net_device *dev, struct netdev_queue *netdevq,
3730 unsigned int iqid, u8 dbqt)
3731 {
3732 unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
3733 struct port_info *pi = netdev_priv(dev);
3734 struct sge *s = &adap->sge;
3735 struct fw_eq_eth_cmd c;
3736 int ret, nentries;
3737
3738
3739 nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3740
3741 txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size,
3742 sizeof(struct tx_desc), sizeof(struct tx_sw_desc),
3743 &txq->q.phys_addr, &txq->q.sdesc, s->stat_len,
3744 netdev_queue_numa_node_read(netdevq));
3745 if (!txq->q.desc)
3746 return -ENOMEM;
3747
3748 memset(&c, 0, sizeof(c));
3749 c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_ETH_CMD) | FW_CMD_REQUEST_F |
3750 FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3751 FW_EQ_ETH_CMD_PFN_V(adap->pf) |
3752 FW_EQ_ETH_CMD_VFN_V(0));
3753 c.alloc_to_len16 = htonl(FW_EQ_ETH_CMD_ALLOC_F |
3754 FW_EQ_ETH_CMD_EQSTART_F | FW_LEN16(c));
3755
3756
3757
3758
3759
3760
3761
3762 c.autoequiqe_to_viid = htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F |
3763 FW_EQ_ETH_CMD_VIID_V(pi->viid));
3764
3765 c.fetchszm_to_iqid =
3766 htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
3767 FW_EQ_ETH_CMD_PCIECHN_V(pi->tx_chan) |
3768 FW_EQ_ETH_CMD_FETCHRO_F | FW_EQ_ETH_CMD_IQID_V(iqid));
3769
3770
3771 c.dcaen_to_eqsize =
3772 htonl(FW_EQ_ETH_CMD_FBMIN_V(chip_ver <= CHELSIO_T5
3773 ? FETCHBURSTMIN_64B_X
3774 : FETCHBURSTMIN_64B_T6_X) |
3775 FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
3776 FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
3777 FW_EQ_ETH_CMD_EQSIZE_V(nentries));
3778
3779 c.eqaddr = cpu_to_be64(txq->q.phys_addr);
3780
3781
3782
3783
3784
3785
3786
3787 if (dbqt)
3788 c.timeren_timerix =
3789 cpu_to_be32(FW_EQ_ETH_CMD_TIMEREN_F |
3790 FW_EQ_ETH_CMD_TIMERIX_V(txq->dbqtimerix));
3791
3792 ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3793 if (ret) {
3794 kfree(txq->q.sdesc);
3795 txq->q.sdesc = NULL;
3796 dma_free_coherent(adap->pdev_dev,
3797 nentries * sizeof(struct tx_desc),
3798 txq->q.desc, txq->q.phys_addr);
3799 txq->q.desc = NULL;
3800 return ret;
3801 }
3802
3803 txq->q.q_type = CXGB4_TXQ_ETH;
3804 init_txq(adap, &txq->q, FW_EQ_ETH_CMD_EQID_G(ntohl(c.eqid_pkd)));
3805 txq->txq = netdevq;
3806 txq->tso = txq->tx_cso = txq->vlan_ins = 0;
3807 txq->mapping_err = 0;
3808 txq->dbqt = dbqt;
3809
3810 return 0;
3811 }
3812
3813 int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
3814 struct net_device *dev, unsigned int iqid,
3815 unsigned int cmplqid)
3816 {
3817 unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
3818 struct port_info *pi = netdev_priv(dev);
3819 struct sge *s = &adap->sge;
3820 struct fw_eq_ctrl_cmd c;
3821 int ret, nentries;
3822
3823
3824 nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3825
3826 txq->q.desc = alloc_ring(adap->pdev_dev, nentries,
3827 sizeof(struct tx_desc), 0, &txq->q.phys_addr,
3828 NULL, 0, dev_to_node(adap->pdev_dev));
3829 if (!txq->q.desc)
3830 return -ENOMEM;
3831
3832 c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_CTRL_CMD) | FW_CMD_REQUEST_F |
3833 FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3834 FW_EQ_CTRL_CMD_PFN_V(adap->pf) |
3835 FW_EQ_CTRL_CMD_VFN_V(0));
3836 c.alloc_to_len16 = htonl(FW_EQ_CTRL_CMD_ALLOC_F |
3837 FW_EQ_CTRL_CMD_EQSTART_F | FW_LEN16(c));
3838 c.cmpliqid_eqid = htonl(FW_EQ_CTRL_CMD_CMPLIQID_V(cmplqid));
3839 c.physeqid_pkd = htonl(0);
3840 c.fetchszm_to_iqid =
3841 htonl(FW_EQ_CTRL_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
3842 FW_EQ_CTRL_CMD_PCIECHN_V(pi->tx_chan) |
3843 FW_EQ_CTRL_CMD_FETCHRO_F | FW_EQ_CTRL_CMD_IQID_V(iqid));
3844 c.dcaen_to_eqsize =
3845 htonl(FW_EQ_CTRL_CMD_FBMIN_V(chip_ver <= CHELSIO_T5
3846 ? FETCHBURSTMIN_64B_X
3847 : FETCHBURSTMIN_64B_T6_X) |
3848 FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
3849 FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
3850 FW_EQ_CTRL_CMD_EQSIZE_V(nentries));
3851 c.eqaddr = cpu_to_be64(txq->q.phys_addr);
3852
3853 ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3854 if (ret) {
3855 dma_free_coherent(adap->pdev_dev,
3856 nentries * sizeof(struct tx_desc),
3857 txq->q.desc, txq->q.phys_addr);
3858 txq->q.desc = NULL;
3859 return ret;
3860 }
3861
3862 txq->q.q_type = CXGB4_TXQ_CTRL;
3863 init_txq(adap, &txq->q, FW_EQ_CTRL_CMD_EQID_G(ntohl(c.cmpliqid_eqid)));
3864 txq->adap = adap;
3865 skb_queue_head_init(&txq->sendq);
3866 tasklet_init(&txq->qresume_tsk, restart_ctrlq, (unsigned long)txq);
3867 txq->full = 0;
3868 return 0;
3869 }
3870
3871 int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid,
3872 unsigned int cmplqid)
3873 {
3874 u32 param, val;
3875
3876 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
3877 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL) |
3878 FW_PARAMS_PARAM_YZ_V(eqid));
3879 val = cmplqid;
3880 return t4_set_params(adap, adap->mbox, adap->pf, 0, 1, ¶m, &val);
3881 }
3882
3883 int t4_sge_alloc_uld_txq(struct adapter *adap, struct sge_uld_txq *txq,
3884 struct net_device *dev, unsigned int iqid,
3885 unsigned int uld_type)
3886 {
3887 unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
3888 int ret, nentries;
3889 struct fw_eq_ofld_cmd c;
3890 struct sge *s = &adap->sge;
3891 struct port_info *pi = netdev_priv(dev);
3892 int cmd = FW_EQ_OFLD_CMD;
3893
3894
3895 nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3896
3897 txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size,
3898 sizeof(struct tx_desc), sizeof(struct tx_sw_desc),
3899 &txq->q.phys_addr, &txq->q.sdesc, s->stat_len,
3900 NUMA_NO_NODE);
3901 if (!txq->q.desc)
3902 return -ENOMEM;
3903
3904 memset(&c, 0, sizeof(c));
3905 if (unlikely(uld_type == CXGB4_TX_CRYPTO))
3906 cmd = FW_EQ_CTRL_CMD;
3907 c.op_to_vfn = htonl(FW_CMD_OP_V(cmd) | FW_CMD_REQUEST_F |
3908 FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3909 FW_EQ_OFLD_CMD_PFN_V(adap->pf) |
3910 FW_EQ_OFLD_CMD_VFN_V(0));
3911 c.alloc_to_len16 = htonl(FW_EQ_OFLD_CMD_ALLOC_F |
3912 FW_EQ_OFLD_CMD_EQSTART_F | FW_LEN16(c));
3913 c.fetchszm_to_iqid =
3914 htonl(FW_EQ_OFLD_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
3915 FW_EQ_OFLD_CMD_PCIECHN_V(pi->tx_chan) |
3916 FW_EQ_OFLD_CMD_FETCHRO_F | FW_EQ_OFLD_CMD_IQID_V(iqid));
3917 c.dcaen_to_eqsize =
3918 htonl(FW_EQ_OFLD_CMD_FBMIN_V(chip_ver <= CHELSIO_T5
3919 ? FETCHBURSTMIN_64B_X
3920 : FETCHBURSTMIN_64B_T6_X) |
3921 FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
3922 FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
3923 FW_EQ_OFLD_CMD_EQSIZE_V(nentries));
3924 c.eqaddr = cpu_to_be64(txq->q.phys_addr);
3925
3926 ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3927 if (ret) {
3928 kfree(txq->q.sdesc);
3929 txq->q.sdesc = NULL;
3930 dma_free_coherent(adap->pdev_dev,
3931 nentries * sizeof(struct tx_desc),
3932 txq->q.desc, txq->q.phys_addr);
3933 txq->q.desc = NULL;
3934 return ret;
3935 }
3936
3937 txq->q.q_type = CXGB4_TXQ_ULD;
3938 init_txq(adap, &txq->q, FW_EQ_OFLD_CMD_EQID_G(ntohl(c.eqid_pkd)));
3939 txq->adap = adap;
3940 skb_queue_head_init(&txq->sendq);
3941 tasklet_init(&txq->qresume_tsk, restart_ofldq, (unsigned long)txq);
3942 txq->full = 0;
3943 txq->mapping_err = 0;
3944 return 0;
3945 }
3946
3947 void free_txq(struct adapter *adap, struct sge_txq *q)
3948 {
3949 struct sge *s = &adap->sge;
3950
3951 dma_free_coherent(adap->pdev_dev,
3952 q->size * sizeof(struct tx_desc) + s->stat_len,
3953 q->desc, q->phys_addr);
3954 q->cntxt_id = 0;
3955 q->sdesc = NULL;
3956 q->desc = NULL;
3957 }
3958
3959 void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq,
3960 struct sge_fl *fl)
3961 {
3962 struct sge *s = &adap->sge;
3963 unsigned int fl_id = fl ? fl->cntxt_id : 0xffff;
3964
3965 adap->sge.ingr_map[rq->cntxt_id - adap->sge.ingr_start] = NULL;
3966 t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
3967 rq->cntxt_id, fl_id, 0xffff);
3968 dma_free_coherent(adap->pdev_dev, (rq->size + 1) * rq->iqe_len,
3969 rq->desc, rq->phys_addr);
3970 netif_napi_del(&rq->napi);
3971 rq->netdev = NULL;
3972 rq->cntxt_id = rq->abs_id = 0;
3973 rq->desc = NULL;
3974
3975 if (fl) {
3976 free_rx_bufs(adap, fl, fl->avail);
3977 dma_free_coherent(adap->pdev_dev, fl->size * 8 + s->stat_len,
3978 fl->desc, fl->addr);
3979 kfree(fl->sdesc);
3980 fl->sdesc = NULL;
3981 fl->cntxt_id = 0;
3982 fl->desc = NULL;
3983 }
3984 }
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994 void t4_free_ofld_rxqs(struct adapter *adap, int n, struct sge_ofld_rxq *q)
3995 {
3996 for ( ; n; n--, q++)
3997 if (q->rspq.desc)
3998 free_rspq_fl(adap, &q->rspq,
3999 q->fl.size ? &q->fl : NULL);
4000 }
4001
4002
4003
4004
4005
4006
4007
4008 void t4_free_sge_resources(struct adapter *adap)
4009 {
4010 int i;
4011 struct sge_eth_rxq *eq;
4012 struct sge_eth_txq *etq;
4013
4014
4015 for (i = 0; i < adap->sge.ethqsets; i++) {
4016 eq = &adap->sge.ethrxq[i];
4017 if (eq->rspq.desc)
4018 t4_iq_stop(adap, adap->mbox, adap->pf, 0,
4019 FW_IQ_TYPE_FL_INT_CAP,
4020 eq->rspq.cntxt_id,
4021 eq->fl.size ? eq->fl.cntxt_id : 0xffff,
4022 0xffff);
4023 }
4024
4025
4026 for (i = 0; i < adap->sge.ethqsets; i++) {
4027 eq = &adap->sge.ethrxq[i];
4028 if (eq->rspq.desc)
4029 free_rspq_fl(adap, &eq->rspq,
4030 eq->fl.size ? &eq->fl : NULL);
4031
4032 etq = &adap->sge.ethtxq[i];
4033 if (etq->q.desc) {
4034 t4_eth_eq_free(adap, adap->mbox, adap->pf, 0,
4035 etq->q.cntxt_id);
4036 __netif_tx_lock_bh(etq->txq);
4037 free_tx_desc(adap, &etq->q, etq->q.in_use, true);
4038 __netif_tx_unlock_bh(etq->txq);
4039 kfree(etq->q.sdesc);
4040 free_txq(adap, &etq->q);
4041 }
4042 }
4043
4044
4045 for (i = 0; i < ARRAY_SIZE(adap->sge.ctrlq); i++) {
4046 struct sge_ctrl_txq *cq = &adap->sge.ctrlq[i];
4047
4048 if (cq->q.desc) {
4049 tasklet_kill(&cq->qresume_tsk);
4050 t4_ctrl_eq_free(adap, adap->mbox, adap->pf, 0,
4051 cq->q.cntxt_id);
4052 __skb_queue_purge(&cq->sendq);
4053 free_txq(adap, &cq->q);
4054 }
4055 }
4056
4057 if (adap->sge.fw_evtq.desc)
4058 free_rspq_fl(adap, &adap->sge.fw_evtq, NULL);
4059
4060 if (adap->sge.intrq.desc)
4061 free_rspq_fl(adap, &adap->sge.intrq, NULL);
4062
4063 if (!is_t4(adap->params.chip)) {
4064 etq = &adap->sge.ptptxq;
4065 if (etq->q.desc) {
4066 t4_eth_eq_free(adap, adap->mbox, adap->pf, 0,
4067 etq->q.cntxt_id);
4068 spin_lock_bh(&adap->ptp_lock);
4069 free_tx_desc(adap, &etq->q, etq->q.in_use, true);
4070 spin_unlock_bh(&adap->ptp_lock);
4071 kfree(etq->q.sdesc);
4072 free_txq(adap, &etq->q);
4073 }
4074 }
4075
4076
4077 memset(adap->sge.egr_map, 0,
4078 adap->sge.egr_sz * sizeof(*adap->sge.egr_map));
4079 }
4080
4081 void t4_sge_start(struct adapter *adap)
4082 {
4083 adap->sge.ethtxq_rover = 0;
4084 mod_timer(&adap->sge.rx_timer, jiffies + RX_QCHECK_PERIOD);
4085 mod_timer(&adap->sge.tx_timer, jiffies + TX_QCHECK_PERIOD);
4086 }
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096 void t4_sge_stop(struct adapter *adap)
4097 {
4098 int i;
4099 struct sge *s = &adap->sge;
4100
4101 if (in_interrupt())
4102 return;
4103
4104 if (s->rx_timer.function)
4105 del_timer_sync(&s->rx_timer);
4106 if (s->tx_timer.function)
4107 del_timer_sync(&s->tx_timer);
4108
4109 if (is_offload(adap)) {
4110 struct sge_uld_txq_info *txq_info;
4111
4112 txq_info = adap->sge.uld_txq_info[CXGB4_TX_OFLD];
4113 if (txq_info) {
4114 struct sge_uld_txq *txq = txq_info->uldtxq;
4115
4116 for_each_ofldtxq(&adap->sge, i) {
4117 if (txq->q.desc)
4118 tasklet_kill(&txq->qresume_tsk);
4119 }
4120 }
4121 }
4122
4123 if (is_pci_uld(adap)) {
4124 struct sge_uld_txq_info *txq_info;
4125
4126 txq_info = adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
4127 if (txq_info) {
4128 struct sge_uld_txq *txq = txq_info->uldtxq;
4129
4130 for_each_ofldtxq(&adap->sge, i) {
4131 if (txq->q.desc)
4132 tasklet_kill(&txq->qresume_tsk);
4133 }
4134 }
4135 }
4136
4137 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) {
4138 struct sge_ctrl_txq *cq = &s->ctrlq[i];
4139
4140 if (cq->q.desc)
4141 tasklet_kill(&cq->qresume_tsk);
4142 }
4143 }
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153 static int t4_sge_init_soft(struct adapter *adap)
4154 {
4155 struct sge *s = &adap->sge;
4156 u32 fl_small_pg, fl_large_pg, fl_small_mtu, fl_large_mtu;
4157 u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
4158 u32 ingress_rx_threshold;
4159
4160
4161
4162
4163
4164
4165 if ((t4_read_reg(adap, SGE_CONTROL_A) & RXPKTCPLMODE_F) !=
4166 RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X)) {
4167 dev_err(adap->pdev_dev, "bad SGE CPL MODE\n");
4168 return -EINVAL;
4169 }
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179 #define READ_FL_BUF(x) \
4180 t4_read_reg(adap, SGE_FL_BUFFER_SIZE0_A+(x)*sizeof(u32))
4181
4182 fl_small_pg = READ_FL_BUF(RX_SMALL_PG_BUF);
4183 fl_large_pg = READ_FL_BUF(RX_LARGE_PG_BUF);
4184 fl_small_mtu = READ_FL_BUF(RX_SMALL_MTU_BUF);
4185 fl_large_mtu = READ_FL_BUF(RX_LARGE_MTU_BUF);
4186
4187
4188
4189
4190 if (fl_large_pg <= fl_small_pg)
4191 fl_large_pg = 0;
4192
4193 #undef READ_FL_BUF
4194
4195
4196
4197
4198 if (fl_small_pg != PAGE_SIZE ||
4199 (fl_large_pg & (fl_large_pg-1)) != 0) {
4200 dev_err(adap->pdev_dev, "bad SGE FL page buffer sizes [%d, %d]\n",
4201 fl_small_pg, fl_large_pg);
4202 return -EINVAL;
4203 }
4204 if (fl_large_pg)
4205 s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT;
4206
4207 if (fl_small_mtu < FL_MTU_SMALL_BUFSIZE(adap) ||
4208 fl_large_mtu < FL_MTU_LARGE_BUFSIZE(adap)) {
4209 dev_err(adap->pdev_dev, "bad SGE FL MTU sizes [%d, %d]\n",
4210 fl_small_mtu, fl_large_mtu);
4211 return -EINVAL;
4212 }
4213
4214
4215
4216
4217
4218 timer_value_0_and_1 = t4_read_reg(adap, SGE_TIMER_VALUE_0_AND_1_A);
4219 timer_value_2_and_3 = t4_read_reg(adap, SGE_TIMER_VALUE_2_AND_3_A);
4220 timer_value_4_and_5 = t4_read_reg(adap, SGE_TIMER_VALUE_4_AND_5_A);
4221 s->timer_val[0] = core_ticks_to_us(adap,
4222 TIMERVALUE0_G(timer_value_0_and_1));
4223 s->timer_val[1] = core_ticks_to_us(adap,
4224 TIMERVALUE1_G(timer_value_0_and_1));
4225 s->timer_val[2] = core_ticks_to_us(adap,
4226 TIMERVALUE2_G(timer_value_2_and_3));
4227 s->timer_val[3] = core_ticks_to_us(adap,
4228 TIMERVALUE3_G(timer_value_2_and_3));
4229 s->timer_val[4] = core_ticks_to_us(adap,
4230 TIMERVALUE4_G(timer_value_4_and_5));
4231 s->timer_val[5] = core_ticks_to_us(adap,
4232 TIMERVALUE5_G(timer_value_4_and_5));
4233
4234 ingress_rx_threshold = t4_read_reg(adap, SGE_INGRESS_RX_THRESHOLD_A);
4235 s->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);
4236 s->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);
4237 s->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);
4238 s->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);
4239
4240 return 0;
4241 }
4242
4243
4244
4245
4246
4247
4248
4249
4250 int t4_sge_init(struct adapter *adap)
4251 {
4252 struct sge *s = &adap->sge;
4253 u32 sge_control, sge_conm_ctrl;
4254 int ret, egress_threshold;
4255
4256
4257
4258
4259
4260 sge_control = t4_read_reg(adap, SGE_CONTROL_A);
4261 s->pktshift = PKTSHIFT_G(sge_control);
4262 s->stat_len = (sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64;
4263
4264 s->fl_align = t4_fl_pkt_align(adap);
4265 ret = t4_sge_init_soft(adap);
4266 if (ret < 0)
4267 return ret;
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281 sge_conm_ctrl = t4_read_reg(adap, SGE_CONM_CTRL_A);
4282 switch (CHELSIO_CHIP_VERSION(adap->params.chip)) {
4283 case CHELSIO_T4:
4284 egress_threshold = EGRTHRESHOLD_G(sge_conm_ctrl);
4285 break;
4286 case CHELSIO_T5:
4287 egress_threshold = EGRTHRESHOLDPACKING_G(sge_conm_ctrl);
4288 break;
4289 case CHELSIO_T6:
4290 egress_threshold = T6_EGRTHRESHOLDPACKING_G(sge_conm_ctrl);
4291 break;
4292 default:
4293 dev_err(adap->pdev_dev, "Unsupported Chip version %d\n",
4294 CHELSIO_CHIP_VERSION(adap->params.chip));
4295 return -EINVAL;
4296 }
4297 s->fl_starve_thres = 2*egress_threshold + 1;
4298
4299 t4_idma_monitor_init(adap, &s->idma_monitor);
4300
4301
4302
4303
4304 timer_setup(&s->rx_timer, sge_rx_timer_cb, 0);
4305 timer_setup(&s->tx_timer, sge_tx_timer_cb, 0);
4306
4307 spin_lock_init(&s->intrq_lock);
4308
4309 return 0;
4310 }