This source file includes following definitions.
- dpaa2_iova_to_virt
- validate_rx_csum
- free_rx_fd
- build_linear_skb
- build_frag_skb
- free_bufs
- xdp_release_buf
- xdp_enqueue
- run_xdp
- dpaa2_eth_rx
- consume_frames
- enable_tx_tstamp
- build_sg_fd
- build_single_fd
- free_tx_fd
- dpaa2_eth_tx
- dpaa2_eth_tx_conf
- set_rx_csum
- set_tx_csum
- add_bufs
- seed_pool
- drain_bufs
- drain_pool
- refill_pool
- pull_channel
- dpaa2_eth_poll
- enable_ch_napi
- disable_ch_napi
- dpaa2_eth_set_rx_taildrop
- link_state_update
- dpaa2_eth_open
- ingress_fq_count
- wait_for_ingress_fq_empty
- wait_for_egress_fq_empty
- dpaa2_eth_stop
- dpaa2_eth_set_addr
- dpaa2_eth_get_stats
- add_uc_hw_addr
- add_mc_hw_addr
- dpaa2_eth_set_rx_mode
- dpaa2_eth_set_features
- dpaa2_eth_ts_ioctl
- dpaa2_eth_ioctl
- xdp_mtu_valid
- set_rx_mfl
- dpaa2_eth_change_mtu
- update_rx_buffer_headroom
- setup_xdp
- dpaa2_eth_xdp
- dpaa2_eth_xdp_xmit_frame
- dpaa2_eth_xdp_xmit
- update_xps
- dpaa2_eth_setup_tc
- cdan_cb
- setup_dpcon
- free_dpcon
- alloc_channel
- free_channel
- setup_dpio
- free_dpio
- get_affine_channel
- set_fq_affinity
- setup_fqs
- setup_dpbp
- free_dpbp
- set_buffer_layout
- dpaa2_eth_enqueue_qd
- dpaa2_eth_enqueue_fq
- set_enqueue_mode
- set_pause
- update_tx_fqids
- setup_dpni
- free_dpni
- setup_rx_flow
- setup_tx_flow
- config_legacy_hash_key
- config_hash_key
- config_cls_key
- dpaa2_eth_cls_key_size
- dpaa2_eth_cls_fld_off
- dpaa2_eth_cls_trim_rule
- dpaa2_eth_set_dist_key
- dpaa2_eth_set_hash
- dpaa2_eth_set_cls
- dpaa2_eth_set_default_cls
- bind_dpni
- alloc_rings
- free_rings
- set_mac_addr
- netdev_init
- poll_link_state
- dpni_irq0_handler_thread
- setup_irqs
- add_ch_napi
- del_ch_napi
- dpaa2_eth_probe
- dpaa2_eth_remove
- dpaa2_eth_driver_init
- dpaa2_eth_driver_exit
1
2
3
4
5 #include <linux/init.h>
6 #include <linux/module.h>
7 #include <linux/platform_device.h>
8 #include <linux/etherdevice.h>
9 #include <linux/of_net.h>
10 #include <linux/interrupt.h>
11 #include <linux/msi.h>
12 #include <linux/kthread.h>
13 #include <linux/iommu.h>
14 #include <linux/net_tstamp.h>
15 #include <linux/fsl/mc.h>
16 #include <linux/bpf.h>
17 #include <linux/bpf_trace.h>
18 #include <net/sock.h>
19
20 #include "dpaa2-eth.h"
21
22
23
24
25 #define CREATE_TRACE_POINTS
26 #include "dpaa2-eth-trace.h"
27
28 MODULE_LICENSE("Dual BSD/GPL");
29 MODULE_AUTHOR("Freescale Semiconductor, Inc");
30 MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver");
31
32 static void *dpaa2_iova_to_virt(struct iommu_domain *domain,
33 dma_addr_t iova_addr)
34 {
35 phys_addr_t phys_addr;
36
37 phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
38
39 return phys_to_virt(phys_addr);
40 }
41
42 static void validate_rx_csum(struct dpaa2_eth_priv *priv,
43 u32 fd_status,
44 struct sk_buff *skb)
45 {
46 skb_checksum_none_assert(skb);
47
48
49 if (!(priv->net_dev->features & NETIF_F_RXCSUM))
50 return;
51
52
53 if (!((fd_status & DPAA2_FAS_L3CV) &&
54 (fd_status & DPAA2_FAS_L4CV)))
55 return;
56
57
58 skb->ip_summed = CHECKSUM_UNNECESSARY;
59 }
60
61
62
63
64 static void free_rx_fd(struct dpaa2_eth_priv *priv,
65 const struct dpaa2_fd *fd,
66 void *vaddr)
67 {
68 struct device *dev = priv->net_dev->dev.parent;
69 dma_addr_t addr = dpaa2_fd_get_addr(fd);
70 u8 fd_format = dpaa2_fd_get_format(fd);
71 struct dpaa2_sg_entry *sgt;
72 void *sg_vaddr;
73 int i;
74
75
76 if (fd_format == dpaa2_fd_single)
77 goto free_buf;
78 else if (fd_format != dpaa2_fd_sg)
79
80 return;
81
82
83
84
85 sgt = vaddr + dpaa2_fd_get_offset(fd);
86 for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) {
87 addr = dpaa2_sg_get_addr(&sgt[i]);
88 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
89 dma_unmap_page(dev, addr, priv->rx_buf_size,
90 DMA_BIDIRECTIONAL);
91
92 free_pages((unsigned long)sg_vaddr, 0);
93 if (dpaa2_sg_is_final(&sgt[i]))
94 break;
95 }
96
97 free_buf:
98 free_pages((unsigned long)vaddr, 0);
99 }
100
101
102 static struct sk_buff *build_linear_skb(struct dpaa2_eth_channel *ch,
103 const struct dpaa2_fd *fd,
104 void *fd_vaddr)
105 {
106 struct sk_buff *skb = NULL;
107 u16 fd_offset = dpaa2_fd_get_offset(fd);
108 u32 fd_length = dpaa2_fd_get_len(fd);
109
110 ch->buf_count--;
111
112 skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE);
113 if (unlikely(!skb))
114 return NULL;
115
116 skb_reserve(skb, fd_offset);
117 skb_put(skb, fd_length);
118
119 return skb;
120 }
121
122
123 static struct sk_buff *build_frag_skb(struct dpaa2_eth_priv *priv,
124 struct dpaa2_eth_channel *ch,
125 struct dpaa2_sg_entry *sgt)
126 {
127 struct sk_buff *skb = NULL;
128 struct device *dev = priv->net_dev->dev.parent;
129 void *sg_vaddr;
130 dma_addr_t sg_addr;
131 u16 sg_offset;
132 u32 sg_length;
133 struct page *page, *head_page;
134 int page_offset;
135 int i;
136
137 for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) {
138 struct dpaa2_sg_entry *sge = &sgt[i];
139
140
141
142
143
144
145 sg_addr = dpaa2_sg_get_addr(sge);
146 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr);
147 dma_unmap_page(dev, sg_addr, priv->rx_buf_size,
148 DMA_BIDIRECTIONAL);
149
150 sg_length = dpaa2_sg_get_len(sge);
151
152 if (i == 0) {
153
154 skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE);
155 if (unlikely(!skb)) {
156
157
158
159 free_pages((unsigned long)sg_vaddr, 0);
160
161
162
163
164 while (!dpaa2_sg_is_final(&sgt[i]) &&
165 i < DPAA2_ETH_MAX_SG_ENTRIES)
166 i++;
167 break;
168 }
169
170 sg_offset = dpaa2_sg_get_offset(sge);
171 skb_reserve(skb, sg_offset);
172 skb_put(skb, sg_length);
173 } else {
174
175 page = virt_to_page(sg_vaddr);
176 head_page = virt_to_head_page(sg_vaddr);
177
178
179
180
181
182
183 page_offset = ((unsigned long)sg_vaddr &
184 (PAGE_SIZE - 1)) +
185 (page_address(page) - page_address(head_page));
186
187 skb_add_rx_frag(skb, i - 1, head_page, page_offset,
188 sg_length, priv->rx_buf_size);
189 }
190
191 if (dpaa2_sg_is_final(sge))
192 break;
193 }
194
195 WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT");
196
197
198 ch->buf_count -= i + 2;
199
200 return skb;
201 }
202
203
204
205
206 static void free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, int count)
207 {
208 struct device *dev = priv->net_dev->dev.parent;
209 void *vaddr;
210 int i;
211
212 for (i = 0; i < count; i++) {
213 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]);
214 dma_unmap_page(dev, buf_array[i], priv->rx_buf_size,
215 DMA_BIDIRECTIONAL);
216 free_pages((unsigned long)vaddr, 0);
217 }
218 }
219
220 static void xdp_release_buf(struct dpaa2_eth_priv *priv,
221 struct dpaa2_eth_channel *ch,
222 dma_addr_t addr)
223 {
224 int err;
225
226 ch->xdp.drop_bufs[ch->xdp.drop_cnt++] = addr;
227 if (ch->xdp.drop_cnt < DPAA2_ETH_BUFS_PER_CMD)
228 return;
229
230 while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid,
231 ch->xdp.drop_bufs,
232 ch->xdp.drop_cnt)) == -EBUSY)
233 cpu_relax();
234
235 if (err) {
236 free_bufs(priv, ch->xdp.drop_bufs, ch->xdp.drop_cnt);
237 ch->buf_count -= ch->xdp.drop_cnt;
238 }
239
240 ch->xdp.drop_cnt = 0;
241 }
242
243 static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd,
244 void *buf_start, u16 queue_id)
245 {
246 struct dpaa2_eth_fq *fq;
247 struct dpaa2_faead *faead;
248 u32 ctrl, frc;
249 int i, err;
250
251
252 frc = dpaa2_fd_get_frc(fd);
253 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV);
254 dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL);
255
256
257
258
259
260 ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV;
261 faead = dpaa2_get_faead(buf_start, false);
262 faead->ctrl = cpu_to_le32(ctrl);
263 faead->conf_fqid = 0;
264
265 fq = &priv->fq[queue_id];
266 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
267 err = priv->enqueue(priv, fq, fd, 0);
268 if (err != -EBUSY)
269 break;
270 }
271
272 return err;
273 }
274
275 static u32 run_xdp(struct dpaa2_eth_priv *priv,
276 struct dpaa2_eth_channel *ch,
277 struct dpaa2_eth_fq *rx_fq,
278 struct dpaa2_fd *fd, void *vaddr)
279 {
280 dma_addr_t addr = dpaa2_fd_get_addr(fd);
281 struct rtnl_link_stats64 *percpu_stats;
282 struct bpf_prog *xdp_prog;
283 struct xdp_buff xdp;
284 u32 xdp_act = XDP_PASS;
285 int err;
286
287 percpu_stats = this_cpu_ptr(priv->percpu_stats);
288
289 rcu_read_lock();
290
291 xdp_prog = READ_ONCE(ch->xdp.prog);
292 if (!xdp_prog)
293 goto out;
294
295 xdp.data = vaddr + dpaa2_fd_get_offset(fd);
296 xdp.data_end = xdp.data + dpaa2_fd_get_len(fd);
297 xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
298 xdp_set_data_meta_invalid(&xdp);
299 xdp.rxq = &ch->xdp_rxq;
300
301 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp);
302
303
304 dpaa2_fd_set_offset(fd, xdp.data - vaddr);
305 dpaa2_fd_set_len(fd, xdp.data_end - xdp.data);
306
307 switch (xdp_act) {
308 case XDP_PASS:
309 break;
310 case XDP_TX:
311 err = xdp_enqueue(priv, fd, vaddr, rx_fq->flowid);
312 if (err) {
313 xdp_release_buf(priv, ch, addr);
314 percpu_stats->tx_errors++;
315 ch->stats.xdp_tx_err++;
316 } else {
317 percpu_stats->tx_packets++;
318 percpu_stats->tx_bytes += dpaa2_fd_get_len(fd);
319 ch->stats.xdp_tx++;
320 }
321 break;
322 default:
323 bpf_warn_invalid_xdp_action(xdp_act);
324
325 case XDP_ABORTED:
326 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
327
328 case XDP_DROP:
329 xdp_release_buf(priv, ch, addr);
330 ch->stats.xdp_drop++;
331 break;
332 case XDP_REDIRECT:
333 dma_unmap_page(priv->net_dev->dev.parent, addr,
334 priv->rx_buf_size, DMA_BIDIRECTIONAL);
335 ch->buf_count--;
336 xdp.data_hard_start = vaddr;
337 err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog);
338 if (unlikely(err))
339 ch->stats.xdp_drop++;
340 else
341 ch->stats.xdp_redirect++;
342 break;
343 }
344
345 ch->xdp.res |= xdp_act;
346 out:
347 rcu_read_unlock();
348 return xdp_act;
349 }
350
351
352 static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv,
353 struct dpaa2_eth_channel *ch,
354 const struct dpaa2_fd *fd,
355 struct dpaa2_eth_fq *fq)
356 {
357 dma_addr_t addr = dpaa2_fd_get_addr(fd);
358 u8 fd_format = dpaa2_fd_get_format(fd);
359 void *vaddr;
360 struct sk_buff *skb;
361 struct rtnl_link_stats64 *percpu_stats;
362 struct dpaa2_eth_drv_stats *percpu_extras;
363 struct device *dev = priv->net_dev->dev.parent;
364 struct dpaa2_fas *fas;
365 void *buf_data;
366 u32 status = 0;
367 u32 xdp_act;
368
369
370 trace_dpaa2_rx_fd(priv->net_dev, fd);
371
372 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
373 dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size,
374 DMA_BIDIRECTIONAL);
375
376 fas = dpaa2_get_fas(vaddr, false);
377 prefetch(fas);
378 buf_data = vaddr + dpaa2_fd_get_offset(fd);
379 prefetch(buf_data);
380
381 percpu_stats = this_cpu_ptr(priv->percpu_stats);
382 percpu_extras = this_cpu_ptr(priv->percpu_extras);
383
384 if (fd_format == dpaa2_fd_single) {
385 xdp_act = run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr);
386 if (xdp_act != XDP_PASS) {
387 percpu_stats->rx_packets++;
388 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
389 return;
390 }
391
392 dma_unmap_page(dev, addr, priv->rx_buf_size,
393 DMA_BIDIRECTIONAL);
394 skb = build_linear_skb(ch, fd, vaddr);
395 } else if (fd_format == dpaa2_fd_sg) {
396 WARN_ON(priv->xdp_prog);
397
398 dma_unmap_page(dev, addr, priv->rx_buf_size,
399 DMA_BIDIRECTIONAL);
400 skb = build_frag_skb(priv, ch, buf_data);
401 free_pages((unsigned long)vaddr, 0);
402 percpu_extras->rx_sg_frames++;
403 percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd);
404 } else {
405
406 goto err_frame_format;
407 }
408
409 if (unlikely(!skb))
410 goto err_build_skb;
411
412 prefetch(skb->data);
413
414
415 if (priv->rx_tstamp) {
416 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
417 __le64 *ts = dpaa2_get_ts(vaddr, false);
418 u64 ns;
419
420 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
421
422 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
423 shhwtstamps->hwtstamp = ns_to_ktime(ns);
424 }
425
426
427 if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) {
428 status = le32_to_cpu(fas->status);
429 validate_rx_csum(priv, status, skb);
430 }
431
432 skb->protocol = eth_type_trans(skb, priv->net_dev);
433 skb_record_rx_queue(skb, fq->flowid);
434
435 percpu_stats->rx_packets++;
436 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
437
438 list_add_tail(&skb->list, ch->rx_list);
439
440 return;
441
442 err_build_skb:
443 free_rx_fd(priv, fd, vaddr);
444 err_frame_format:
445 percpu_stats->rx_dropped++;
446 }
447
448
449
450
451
452
453
454 static int consume_frames(struct dpaa2_eth_channel *ch,
455 struct dpaa2_eth_fq **src)
456 {
457 struct dpaa2_eth_priv *priv = ch->priv;
458 struct dpaa2_eth_fq *fq = NULL;
459 struct dpaa2_dq *dq;
460 const struct dpaa2_fd *fd;
461 int cleaned = 0;
462 int is_last;
463
464 do {
465 dq = dpaa2_io_store_next(ch->store, &is_last);
466 if (unlikely(!dq)) {
467
468
469
470
471
472 continue;
473 }
474
475 fd = dpaa2_dq_fd(dq);
476 fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq);
477
478 fq->consume(priv, ch, fd, fq);
479 cleaned++;
480 } while (!is_last);
481
482 if (!cleaned)
483 return 0;
484
485 fq->stats.frames += cleaned;
486
487
488
489
490 if (src)
491 *src = fq;
492
493 return cleaned;
494 }
495
496
497 static void enable_tx_tstamp(struct dpaa2_fd *fd, void *buf_start)
498 {
499 struct dpaa2_faead *faead;
500 u32 ctrl, frc;
501
502
503 frc = dpaa2_fd_get_frc(fd);
504 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV);
505
506
507 ctrl = dpaa2_fd_get_ctrl(fd);
508 dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL);
509
510
511
512
513 ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD;
514 faead = dpaa2_get_faead(buf_start, true);
515 faead->ctrl = cpu_to_le32(ctrl);
516 }
517
518
519 static int build_sg_fd(struct dpaa2_eth_priv *priv,
520 struct sk_buff *skb,
521 struct dpaa2_fd *fd)
522 {
523 struct device *dev = priv->net_dev->dev.parent;
524 void *sgt_buf = NULL;
525 dma_addr_t addr;
526 int nr_frags = skb_shinfo(skb)->nr_frags;
527 struct dpaa2_sg_entry *sgt;
528 int i, err;
529 int sgt_buf_size;
530 struct scatterlist *scl, *crt_scl;
531 int num_sg;
532 int num_dma_bufs;
533 struct dpaa2_eth_swa *swa;
534
535
536
537
538
539
540 if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1))
541 return -EINVAL;
542
543 scl = kcalloc(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC);
544 if (unlikely(!scl))
545 return -ENOMEM;
546
547 sg_init_table(scl, nr_frags + 1);
548 num_sg = skb_to_sgvec(skb, scl, 0, skb->len);
549 num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL);
550 if (unlikely(!num_dma_bufs)) {
551 err = -ENOMEM;
552 goto dma_map_sg_failed;
553 }
554
555
556 sgt_buf_size = priv->tx_data_offset +
557 sizeof(struct dpaa2_sg_entry) * num_dma_bufs;
558 sgt_buf = napi_alloc_frag(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN);
559 if (unlikely(!sgt_buf)) {
560 err = -ENOMEM;
561 goto sgt_buf_alloc_failed;
562 }
563 sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN);
564 memset(sgt_buf, 0, sgt_buf_size);
565
566 sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset);
567
568
569
570
571
572
573
574
575 for_each_sg(scl, crt_scl, num_dma_bufs, i) {
576 dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl));
577 dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl));
578 }
579 dpaa2_sg_set_final(&sgt[i - 1], true);
580
581
582
583
584
585
586 swa = (struct dpaa2_eth_swa *)sgt_buf;
587 swa->type = DPAA2_ETH_SWA_SG;
588 swa->sg.skb = skb;
589 swa->sg.scl = scl;
590 swa->sg.num_sg = num_sg;
591 swa->sg.sgt_size = sgt_buf_size;
592
593
594 addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL);
595 if (unlikely(dma_mapping_error(dev, addr))) {
596 err = -ENOMEM;
597 goto dma_map_single_failed;
598 }
599 dpaa2_fd_set_offset(fd, priv->tx_data_offset);
600 dpaa2_fd_set_format(fd, dpaa2_fd_sg);
601 dpaa2_fd_set_addr(fd, addr);
602 dpaa2_fd_set_len(fd, skb->len);
603 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
604
605 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
606 enable_tx_tstamp(fd, sgt_buf);
607
608 return 0;
609
610 dma_map_single_failed:
611 skb_free_frag(sgt_buf);
612 sgt_buf_alloc_failed:
613 dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL);
614 dma_map_sg_failed:
615 kfree(scl);
616 return err;
617 }
618
619
620 static int build_single_fd(struct dpaa2_eth_priv *priv,
621 struct sk_buff *skb,
622 struct dpaa2_fd *fd)
623 {
624 struct device *dev = priv->net_dev->dev.parent;
625 u8 *buffer_start, *aligned_start;
626 struct dpaa2_eth_swa *swa;
627 dma_addr_t addr;
628
629 buffer_start = skb->data - dpaa2_eth_needed_headroom(priv, skb);
630
631
632
633
634 aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN,
635 DPAA2_ETH_TX_BUF_ALIGN);
636 if (aligned_start >= skb->head)
637 buffer_start = aligned_start;
638
639
640
641
642
643 swa = (struct dpaa2_eth_swa *)buffer_start;
644 swa->type = DPAA2_ETH_SWA_SINGLE;
645 swa->single.skb = skb;
646
647 addr = dma_map_single(dev, buffer_start,
648 skb_tail_pointer(skb) - buffer_start,
649 DMA_BIDIRECTIONAL);
650 if (unlikely(dma_mapping_error(dev, addr)))
651 return -ENOMEM;
652
653 dpaa2_fd_set_addr(fd, addr);
654 dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start));
655 dpaa2_fd_set_len(fd, skb->len);
656 dpaa2_fd_set_format(fd, dpaa2_fd_single);
657 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
658
659 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
660 enable_tx_tstamp(fd, buffer_start);
661
662 return 0;
663 }
664
665
666
667
668
669
670
671
672 static void free_tx_fd(const struct dpaa2_eth_priv *priv,
673 struct dpaa2_eth_fq *fq,
674 const struct dpaa2_fd *fd, bool in_napi)
675 {
676 struct device *dev = priv->net_dev->dev.parent;
677 dma_addr_t fd_addr;
678 struct sk_buff *skb = NULL;
679 unsigned char *buffer_start;
680 struct dpaa2_eth_swa *swa;
681 u8 fd_format = dpaa2_fd_get_format(fd);
682 u32 fd_len = dpaa2_fd_get_len(fd);
683
684 fd_addr = dpaa2_fd_get_addr(fd);
685 buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr);
686 swa = (struct dpaa2_eth_swa *)buffer_start;
687
688 if (fd_format == dpaa2_fd_single) {
689 if (swa->type == DPAA2_ETH_SWA_SINGLE) {
690 skb = swa->single.skb;
691
692
693
694 dma_unmap_single(dev, fd_addr,
695 skb_tail_pointer(skb) - buffer_start,
696 DMA_BIDIRECTIONAL);
697 } else {
698 WARN_ONCE(swa->type != DPAA2_ETH_SWA_XDP, "Wrong SWA type");
699 dma_unmap_single(dev, fd_addr, swa->xdp.dma_size,
700 DMA_BIDIRECTIONAL);
701 }
702 } else if (fd_format == dpaa2_fd_sg) {
703 skb = swa->sg.skb;
704
705
706 dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg,
707 DMA_BIDIRECTIONAL);
708 kfree(swa->sg.scl);
709
710
711 dma_unmap_single(dev, fd_addr, swa->sg.sgt_size,
712 DMA_BIDIRECTIONAL);
713 } else {
714 netdev_dbg(priv->net_dev, "Invalid FD format\n");
715 return;
716 }
717
718 if (swa->type != DPAA2_ETH_SWA_XDP && in_napi) {
719 fq->dq_frames++;
720 fq->dq_bytes += fd_len;
721 }
722
723 if (swa->type == DPAA2_ETH_SWA_XDP) {
724 xdp_return_frame(swa->xdp.xdpf);
725 return;
726 }
727
728
729 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
730 struct skb_shared_hwtstamps shhwtstamps;
731 __le64 *ts = dpaa2_get_ts(buffer_start, true);
732 u64 ns;
733
734 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
735
736 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
737 shhwtstamps.hwtstamp = ns_to_ktime(ns);
738 skb_tstamp_tx(skb, &shhwtstamps);
739 }
740
741
742 if (fd_format != dpaa2_fd_single)
743 skb_free_frag(buffer_start);
744
745
746 napi_consume_skb(skb, in_napi);
747 }
748
749 static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev)
750 {
751 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
752 struct dpaa2_fd fd;
753 struct rtnl_link_stats64 *percpu_stats;
754 struct dpaa2_eth_drv_stats *percpu_extras;
755 struct dpaa2_eth_fq *fq;
756 struct netdev_queue *nq;
757 u16 queue_mapping;
758 unsigned int needed_headroom;
759 u32 fd_len;
760 u8 prio = 0;
761 int err, i;
762
763 percpu_stats = this_cpu_ptr(priv->percpu_stats);
764 percpu_extras = this_cpu_ptr(priv->percpu_extras);
765
766 needed_headroom = dpaa2_eth_needed_headroom(priv, skb);
767 if (skb_headroom(skb) < needed_headroom) {
768 struct sk_buff *ns;
769
770 ns = skb_realloc_headroom(skb, needed_headroom);
771 if (unlikely(!ns)) {
772 percpu_stats->tx_dropped++;
773 goto err_alloc_headroom;
774 }
775 percpu_extras->tx_reallocs++;
776
777 if (skb->sk)
778 skb_set_owner_w(ns, skb->sk);
779
780 dev_kfree_skb(skb);
781 skb = ns;
782 }
783
784
785
786
787 skb = skb_unshare(skb, GFP_ATOMIC);
788 if (unlikely(!skb)) {
789
790 percpu_stats->tx_dropped++;
791 return NETDEV_TX_OK;
792 }
793
794
795 memset(&fd, 0, sizeof(fd));
796
797 if (skb_is_nonlinear(skb)) {
798 err = build_sg_fd(priv, skb, &fd);
799 percpu_extras->tx_sg_frames++;
800 percpu_extras->tx_sg_bytes += skb->len;
801 } else {
802 err = build_single_fd(priv, skb, &fd);
803 }
804
805 if (unlikely(err)) {
806 percpu_stats->tx_dropped++;
807 goto err_build_fd;
808 }
809
810
811 trace_dpaa2_tx_fd(net_dev, &fd);
812
813
814
815
816
817 queue_mapping = skb_get_queue_mapping(skb);
818
819 if (net_dev->num_tc) {
820 prio = netdev_txq_to_tc(net_dev, queue_mapping);
821
822
823
824 prio = net_dev->num_tc - prio - 1;
825
826
827
828 queue_mapping %= dpaa2_eth_queue_count(priv);
829 }
830 fq = &priv->fq[queue_mapping];
831
832 fd_len = dpaa2_fd_get_len(&fd);
833 nq = netdev_get_tx_queue(net_dev, queue_mapping);
834 netdev_tx_sent_queue(nq, fd_len);
835
836
837
838
839 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
840 err = priv->enqueue(priv, fq, &fd, prio);
841 if (err != -EBUSY)
842 break;
843 }
844 percpu_extras->tx_portal_busy += i;
845 if (unlikely(err < 0)) {
846 percpu_stats->tx_errors++;
847
848 free_tx_fd(priv, fq, &fd, false);
849 netdev_tx_completed_queue(nq, 1, fd_len);
850 } else {
851 percpu_stats->tx_packets++;
852 percpu_stats->tx_bytes += fd_len;
853 }
854
855 return NETDEV_TX_OK;
856
857 err_build_fd:
858 err_alloc_headroom:
859 dev_kfree_skb(skb);
860
861 return NETDEV_TX_OK;
862 }
863
864
865 static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv,
866 struct dpaa2_eth_channel *ch __always_unused,
867 const struct dpaa2_fd *fd,
868 struct dpaa2_eth_fq *fq)
869 {
870 struct rtnl_link_stats64 *percpu_stats;
871 struct dpaa2_eth_drv_stats *percpu_extras;
872 u32 fd_len = dpaa2_fd_get_len(fd);
873 u32 fd_errors;
874
875
876 trace_dpaa2_tx_conf_fd(priv->net_dev, fd);
877
878 percpu_extras = this_cpu_ptr(priv->percpu_extras);
879 percpu_extras->tx_conf_frames++;
880 percpu_extras->tx_conf_bytes += fd_len;
881
882
883 fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK;
884 free_tx_fd(priv, fq, fd, true);
885
886 if (likely(!fd_errors))
887 return;
888
889 if (net_ratelimit())
890 netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n",
891 fd_errors);
892
893 percpu_stats = this_cpu_ptr(priv->percpu_stats);
894
895 percpu_stats->tx_errors++;
896 }
897
898 static int set_rx_csum(struct dpaa2_eth_priv *priv, bool enable)
899 {
900 int err;
901
902 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
903 DPNI_OFF_RX_L3_CSUM, enable);
904 if (err) {
905 netdev_err(priv->net_dev,
906 "dpni_set_offload(RX_L3_CSUM) failed\n");
907 return err;
908 }
909
910 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
911 DPNI_OFF_RX_L4_CSUM, enable);
912 if (err) {
913 netdev_err(priv->net_dev,
914 "dpni_set_offload(RX_L4_CSUM) failed\n");
915 return err;
916 }
917
918 return 0;
919 }
920
921 static int set_tx_csum(struct dpaa2_eth_priv *priv, bool enable)
922 {
923 int err;
924
925 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
926 DPNI_OFF_TX_L3_CSUM, enable);
927 if (err) {
928 netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n");
929 return err;
930 }
931
932 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
933 DPNI_OFF_TX_L4_CSUM, enable);
934 if (err) {
935 netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n");
936 return err;
937 }
938
939 return 0;
940 }
941
942
943
944
945 static int add_bufs(struct dpaa2_eth_priv *priv,
946 struct dpaa2_eth_channel *ch, u16 bpid)
947 {
948 struct device *dev = priv->net_dev->dev.parent;
949 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
950 struct page *page;
951 dma_addr_t addr;
952 int i, err;
953
954 for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) {
955
956
957
958
959
960
961
962 page = dev_alloc_pages(0);
963 if (!page)
964 goto err_alloc;
965
966 addr = dma_map_page(dev, page, 0, priv->rx_buf_size,
967 DMA_BIDIRECTIONAL);
968 if (unlikely(dma_mapping_error(dev, addr)))
969 goto err_map;
970
971 buf_array[i] = addr;
972
973
974 trace_dpaa2_eth_buf_seed(priv->net_dev,
975 page, DPAA2_ETH_RX_BUF_RAW_SIZE,
976 addr, priv->rx_buf_size,
977 bpid);
978 }
979
980 release_bufs:
981
982 while ((err = dpaa2_io_service_release(ch->dpio, bpid,
983 buf_array, i)) == -EBUSY)
984 cpu_relax();
985
986
987
988
989 if (err) {
990 free_bufs(priv, buf_array, i);
991 return 0;
992 }
993
994 return i;
995
996 err_map:
997 __free_pages(page, 0);
998 err_alloc:
999
1000
1001
1002 if (i)
1003 goto release_bufs;
1004
1005 return 0;
1006 }
1007
1008 static int seed_pool(struct dpaa2_eth_priv *priv, u16 bpid)
1009 {
1010 int i, j;
1011 int new_count;
1012
1013 for (j = 0; j < priv->num_channels; j++) {
1014 for (i = 0; i < DPAA2_ETH_NUM_BUFS;
1015 i += DPAA2_ETH_BUFS_PER_CMD) {
1016 new_count = add_bufs(priv, priv->channel[j], bpid);
1017 priv->channel[j]->buf_count += new_count;
1018
1019 if (new_count < DPAA2_ETH_BUFS_PER_CMD) {
1020 return -ENOMEM;
1021 }
1022 }
1023 }
1024
1025 return 0;
1026 }
1027
1028
1029
1030
1031
1032 static void drain_bufs(struct dpaa2_eth_priv *priv, int count)
1033 {
1034 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
1035 int ret;
1036
1037 do {
1038 ret = dpaa2_io_service_acquire(NULL, priv->bpid,
1039 buf_array, count);
1040 if (ret < 0) {
1041 netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n");
1042 return;
1043 }
1044 free_bufs(priv, buf_array, ret);
1045 } while (ret);
1046 }
1047
1048 static void drain_pool(struct dpaa2_eth_priv *priv)
1049 {
1050 int i;
1051
1052 drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD);
1053 drain_bufs(priv, 1);
1054
1055 for (i = 0; i < priv->num_channels; i++)
1056 priv->channel[i]->buf_count = 0;
1057 }
1058
1059
1060
1061
1062 static int refill_pool(struct dpaa2_eth_priv *priv,
1063 struct dpaa2_eth_channel *ch,
1064 u16 bpid)
1065 {
1066 int new_count;
1067
1068 if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH))
1069 return 0;
1070
1071 do {
1072 new_count = add_bufs(priv, ch, bpid);
1073 if (unlikely(!new_count)) {
1074
1075 break;
1076 }
1077 ch->buf_count += new_count;
1078 } while (ch->buf_count < DPAA2_ETH_NUM_BUFS);
1079
1080 if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS))
1081 return -ENOMEM;
1082
1083 return 0;
1084 }
1085
1086 static int pull_channel(struct dpaa2_eth_channel *ch)
1087 {
1088 int err;
1089 int dequeues = -1;
1090
1091
1092 do {
1093 err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id,
1094 ch->store);
1095 dequeues++;
1096 cpu_relax();
1097 } while (err == -EBUSY);
1098
1099 ch->stats.dequeue_portal_busy += dequeues;
1100 if (unlikely(err))
1101 ch->stats.pull_err++;
1102
1103 return err;
1104 }
1105
1106
1107
1108
1109
1110
1111
1112 static int dpaa2_eth_poll(struct napi_struct *napi, int budget)
1113 {
1114 struct dpaa2_eth_channel *ch;
1115 struct dpaa2_eth_priv *priv;
1116 int rx_cleaned = 0, txconf_cleaned = 0;
1117 struct dpaa2_eth_fq *fq, *txc_fq = NULL;
1118 struct netdev_queue *nq;
1119 int store_cleaned, work_done;
1120 struct list_head rx_list;
1121 int err;
1122
1123 ch = container_of(napi, struct dpaa2_eth_channel, napi);
1124 ch->xdp.res = 0;
1125 priv = ch->priv;
1126
1127 INIT_LIST_HEAD(&rx_list);
1128 ch->rx_list = &rx_list;
1129
1130 do {
1131 err = pull_channel(ch);
1132 if (unlikely(err))
1133 break;
1134
1135
1136 refill_pool(priv, ch, priv->bpid);
1137
1138 store_cleaned = consume_frames(ch, &fq);
1139 if (!store_cleaned)
1140 break;
1141 if (fq->type == DPAA2_RX_FQ) {
1142 rx_cleaned += store_cleaned;
1143 } else {
1144 txconf_cleaned += store_cleaned;
1145
1146 txc_fq = fq;
1147 }
1148
1149
1150
1151
1152 if (rx_cleaned >= budget ||
1153 txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) {
1154 work_done = budget;
1155 goto out;
1156 }
1157 } while (store_cleaned);
1158
1159
1160
1161
1162 napi_complete_done(napi, rx_cleaned);
1163 do {
1164 err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx);
1165 cpu_relax();
1166 } while (err == -EBUSY);
1167 WARN_ONCE(err, "CDAN notifications rearm failed on core %d",
1168 ch->nctx.desired_cpu);
1169
1170 work_done = max(rx_cleaned, 1);
1171
1172 out:
1173 netif_receive_skb_list(ch->rx_list);
1174
1175 if (txc_fq && txc_fq->dq_frames) {
1176 nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid);
1177 netdev_tx_completed_queue(nq, txc_fq->dq_frames,
1178 txc_fq->dq_bytes);
1179 txc_fq->dq_frames = 0;
1180 txc_fq->dq_bytes = 0;
1181 }
1182
1183 if (ch->xdp.res & XDP_REDIRECT)
1184 xdp_do_flush_map();
1185
1186 return work_done;
1187 }
1188
1189 static void enable_ch_napi(struct dpaa2_eth_priv *priv)
1190 {
1191 struct dpaa2_eth_channel *ch;
1192 int i;
1193
1194 for (i = 0; i < priv->num_channels; i++) {
1195 ch = priv->channel[i];
1196 napi_enable(&ch->napi);
1197 }
1198 }
1199
1200 static void disable_ch_napi(struct dpaa2_eth_priv *priv)
1201 {
1202 struct dpaa2_eth_channel *ch;
1203 int i;
1204
1205 for (i = 0; i < priv->num_channels; i++) {
1206 ch = priv->channel[i];
1207 napi_disable(&ch->napi);
1208 }
1209 }
1210
1211 static void dpaa2_eth_set_rx_taildrop(struct dpaa2_eth_priv *priv, bool enable)
1212 {
1213 struct dpni_taildrop td = {0};
1214 int i, err;
1215
1216 if (priv->rx_td_enabled == enable)
1217 return;
1218
1219 td.enable = enable;
1220 td.threshold = DPAA2_ETH_TAILDROP_THRESH;
1221
1222 for (i = 0; i < priv->num_fqs; i++) {
1223 if (priv->fq[i].type != DPAA2_RX_FQ)
1224 continue;
1225 err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token,
1226 DPNI_CP_QUEUE, DPNI_QUEUE_RX, 0,
1227 priv->fq[i].flowid, &td);
1228 if (err) {
1229 netdev_err(priv->net_dev,
1230 "dpni_set_taildrop() failed\n");
1231 break;
1232 }
1233 }
1234
1235 priv->rx_td_enabled = enable;
1236 }
1237
1238 static void update_tx_fqids(struct dpaa2_eth_priv *priv);
1239
1240 static int link_state_update(struct dpaa2_eth_priv *priv)
1241 {
1242 struct dpni_link_state state = {0};
1243 bool tx_pause;
1244 int err;
1245
1246 err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state);
1247 if (unlikely(err)) {
1248 netdev_err(priv->net_dev,
1249 "dpni_get_link_state() failed\n");
1250 return err;
1251 }
1252
1253
1254
1255
1256
1257 tx_pause = !!(state.options & DPNI_LINK_OPT_PAUSE) ^
1258 !!(state.options & DPNI_LINK_OPT_ASYM_PAUSE);
1259 dpaa2_eth_set_rx_taildrop(priv, !tx_pause);
1260
1261
1262 if (priv->link_state.up == state.up)
1263 goto out;
1264
1265 if (state.up) {
1266 update_tx_fqids(priv);
1267 netif_carrier_on(priv->net_dev);
1268 netif_tx_start_all_queues(priv->net_dev);
1269 } else {
1270 netif_tx_stop_all_queues(priv->net_dev);
1271 netif_carrier_off(priv->net_dev);
1272 }
1273
1274 netdev_info(priv->net_dev, "Link Event: state %s\n",
1275 state.up ? "up" : "down");
1276
1277 out:
1278 priv->link_state = state;
1279
1280 return 0;
1281 }
1282
1283 static int dpaa2_eth_open(struct net_device *net_dev)
1284 {
1285 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1286 int err;
1287
1288 err = seed_pool(priv, priv->bpid);
1289 if (err) {
1290
1291
1292
1293
1294 netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n",
1295 priv->dpbp_dev->obj_desc.id, priv->bpid);
1296 }
1297
1298
1299
1300
1301
1302 netif_tx_stop_all_queues(net_dev);
1303 enable_ch_napi(priv);
1304
1305
1306
1307
1308 netif_carrier_off(net_dev);
1309
1310 err = dpni_enable(priv->mc_io, 0, priv->mc_token);
1311 if (err < 0) {
1312 netdev_err(net_dev, "dpni_enable() failed\n");
1313 goto enable_err;
1314 }
1315
1316
1317
1318
1319 err = link_state_update(priv);
1320 if (err < 0) {
1321 netdev_err(net_dev, "Can't update link state\n");
1322 goto link_state_err;
1323 }
1324
1325 return 0;
1326
1327 link_state_err:
1328 enable_err:
1329 disable_ch_napi(priv);
1330 drain_pool(priv);
1331 return err;
1332 }
1333
1334
1335 static u32 ingress_fq_count(struct dpaa2_eth_priv *priv)
1336 {
1337 struct dpaa2_eth_fq *fq;
1338 u32 fcnt = 0, bcnt = 0, total = 0;
1339 int i, err;
1340
1341 for (i = 0; i < priv->num_fqs; i++) {
1342 fq = &priv->fq[i];
1343 err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt);
1344 if (err) {
1345 netdev_warn(priv->net_dev, "query_fq_count failed");
1346 break;
1347 }
1348 total += fcnt;
1349 }
1350
1351 return total;
1352 }
1353
1354 static void wait_for_ingress_fq_empty(struct dpaa2_eth_priv *priv)
1355 {
1356 int retries = 10;
1357 u32 pending;
1358
1359 do {
1360 pending = ingress_fq_count(priv);
1361 if (pending)
1362 msleep(100);
1363 } while (pending && --retries);
1364 }
1365
1366 #define DPNI_TX_PENDING_VER_MAJOR 7
1367 #define DPNI_TX_PENDING_VER_MINOR 13
1368 static void wait_for_egress_fq_empty(struct dpaa2_eth_priv *priv)
1369 {
1370 union dpni_statistics stats;
1371 int retries = 10;
1372 int err;
1373
1374 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_TX_PENDING_VER_MAJOR,
1375 DPNI_TX_PENDING_VER_MINOR) < 0)
1376 goto out;
1377
1378 do {
1379 err = dpni_get_statistics(priv->mc_io, 0, priv->mc_token, 6,
1380 &stats);
1381 if (err)
1382 goto out;
1383 if (stats.page_6.tx_pending_frames == 0)
1384 return;
1385 } while (--retries);
1386
1387 out:
1388 msleep(500);
1389 }
1390
1391 static int dpaa2_eth_stop(struct net_device *net_dev)
1392 {
1393 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1394 int dpni_enabled = 0;
1395 int retries = 10;
1396
1397 netif_tx_stop_all_queues(net_dev);
1398 netif_carrier_off(net_dev);
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410 wait_for_egress_fq_empty(priv);
1411
1412 do {
1413 dpni_disable(priv->mc_io, 0, priv->mc_token);
1414 dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled);
1415 if (dpni_enabled)
1416
1417 msleep(100);
1418 } while (dpni_enabled && --retries);
1419 if (!retries) {
1420 netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n");
1421
1422
1423
1424 }
1425
1426 wait_for_ingress_fq_empty(priv);
1427 disable_ch_napi(priv);
1428
1429
1430 drain_pool(priv);
1431
1432 return 0;
1433 }
1434
1435 static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr)
1436 {
1437 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1438 struct device *dev = net_dev->dev.parent;
1439 int err;
1440
1441 err = eth_mac_addr(net_dev, addr);
1442 if (err < 0) {
1443 dev_err(dev, "eth_mac_addr() failed (%d)\n", err);
1444 return err;
1445 }
1446
1447 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token,
1448 net_dev->dev_addr);
1449 if (err) {
1450 dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err);
1451 return err;
1452 }
1453
1454 return 0;
1455 }
1456
1457
1458
1459
1460 static void dpaa2_eth_get_stats(struct net_device *net_dev,
1461 struct rtnl_link_stats64 *stats)
1462 {
1463 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1464 struct rtnl_link_stats64 *percpu_stats;
1465 u64 *cpustats;
1466 u64 *netstats = (u64 *)stats;
1467 int i, j;
1468 int num = sizeof(struct rtnl_link_stats64) / sizeof(u64);
1469
1470 for_each_possible_cpu(i) {
1471 percpu_stats = per_cpu_ptr(priv->percpu_stats, i);
1472 cpustats = (u64 *)percpu_stats;
1473 for (j = 0; j < num; j++)
1474 netstats[j] += cpustats[j];
1475 }
1476 }
1477
1478
1479
1480
1481 static void add_uc_hw_addr(const struct net_device *net_dev,
1482 struct dpaa2_eth_priv *priv)
1483 {
1484 struct netdev_hw_addr *ha;
1485 int err;
1486
1487 netdev_for_each_uc_addr(ha, net_dev) {
1488 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token,
1489 ha->addr);
1490 if (err)
1491 netdev_warn(priv->net_dev,
1492 "Could not add ucast MAC %pM to the filtering table (err %d)\n",
1493 ha->addr, err);
1494 }
1495 }
1496
1497
1498
1499
1500 static void add_mc_hw_addr(const struct net_device *net_dev,
1501 struct dpaa2_eth_priv *priv)
1502 {
1503 struct netdev_hw_addr *ha;
1504 int err;
1505
1506 netdev_for_each_mc_addr(ha, net_dev) {
1507 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token,
1508 ha->addr);
1509 if (err)
1510 netdev_warn(priv->net_dev,
1511 "Could not add mcast MAC %pM to the filtering table (err %d)\n",
1512 ha->addr, err);
1513 }
1514 }
1515
1516 static void dpaa2_eth_set_rx_mode(struct net_device *net_dev)
1517 {
1518 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1519 int uc_count = netdev_uc_count(net_dev);
1520 int mc_count = netdev_mc_count(net_dev);
1521 u8 max_mac = priv->dpni_attrs.mac_filter_entries;
1522 u32 options = priv->dpni_attrs.options;
1523 u16 mc_token = priv->mc_token;
1524 struct fsl_mc_io *mc_io = priv->mc_io;
1525 int err;
1526
1527
1528 if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0)
1529 netdev_info(net_dev,
1530 "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n",
1531 max_mac);
1532
1533
1534 if (uc_count > max_mac) {
1535 netdev_info(net_dev,
1536 "Unicast addr count reached %d, max allowed is %d; forcing promisc\n",
1537 uc_count, max_mac);
1538 goto force_promisc;
1539 }
1540 if (mc_count + uc_count > max_mac) {
1541 netdev_info(net_dev,
1542 "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n",
1543 uc_count + mc_count, max_mac);
1544 goto force_mc_promisc;
1545 }
1546
1547
1548 if (net_dev->flags & IFF_PROMISC)
1549 goto force_promisc;
1550 if (net_dev->flags & IFF_ALLMULTI) {
1551
1552
1553
1554
1555
1556
1557
1558 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1);
1559 if (err)
1560 netdev_warn(net_dev, "Can't set uc promisc\n");
1561
1562
1563 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0);
1564 if (err)
1565 netdev_warn(net_dev, "Can't clear uc filters\n");
1566 add_uc_hw_addr(net_dev, priv);
1567
1568
1569 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0);
1570 if (err)
1571 netdev_warn(net_dev, "Can't clear uc promisc\n");
1572 goto force_mc_promisc;
1573 }
1574
1575
1576
1577
1578 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1);
1579 if (err)
1580 netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err);
1581 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1);
1582 if (err)
1583 netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err);
1584
1585
1586 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1);
1587 if (err)
1588 netdev_warn(net_dev, "Can't clear mac filters\n");
1589 add_mc_hw_addr(net_dev, priv);
1590 add_uc_hw_addr(net_dev, priv);
1591
1592
1593
1594
1595 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0);
1596 if (err)
1597 netdev_warn(net_dev, "Can't clear ucast promisc\n");
1598 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0);
1599 if (err)
1600 netdev_warn(net_dev, "Can't clear mcast promisc\n");
1601
1602 return;
1603
1604 force_promisc:
1605 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1);
1606 if (err)
1607 netdev_warn(net_dev, "Can't set ucast promisc\n");
1608 force_mc_promisc:
1609 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1);
1610 if (err)
1611 netdev_warn(net_dev, "Can't set mcast promisc\n");
1612 }
1613
1614 static int dpaa2_eth_set_features(struct net_device *net_dev,
1615 netdev_features_t features)
1616 {
1617 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1618 netdev_features_t changed = features ^ net_dev->features;
1619 bool enable;
1620 int err;
1621
1622 if (changed & NETIF_F_RXCSUM) {
1623 enable = !!(features & NETIF_F_RXCSUM);
1624 err = set_rx_csum(priv, enable);
1625 if (err)
1626 return err;
1627 }
1628
1629 if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
1630 enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM));
1631 err = set_tx_csum(priv, enable);
1632 if (err)
1633 return err;
1634 }
1635
1636 return 0;
1637 }
1638
1639 static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1640 {
1641 struct dpaa2_eth_priv *priv = netdev_priv(dev);
1642 struct hwtstamp_config config;
1643
1644 if (copy_from_user(&config, rq->ifr_data, sizeof(config)))
1645 return -EFAULT;
1646
1647 switch (config.tx_type) {
1648 case HWTSTAMP_TX_OFF:
1649 priv->tx_tstamp = false;
1650 break;
1651 case HWTSTAMP_TX_ON:
1652 priv->tx_tstamp = true;
1653 break;
1654 default:
1655 return -ERANGE;
1656 }
1657
1658 if (config.rx_filter == HWTSTAMP_FILTER_NONE) {
1659 priv->rx_tstamp = false;
1660 } else {
1661 priv->rx_tstamp = true;
1662
1663 config.rx_filter = HWTSTAMP_FILTER_ALL;
1664 }
1665
1666 return copy_to_user(rq->ifr_data, &config, sizeof(config)) ?
1667 -EFAULT : 0;
1668 }
1669
1670 static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1671 {
1672 if (cmd == SIOCSHWTSTAMP)
1673 return dpaa2_eth_ts_ioctl(dev, rq, cmd);
1674
1675 return -EINVAL;
1676 }
1677
1678 static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu)
1679 {
1680 int mfl, linear_mfl;
1681
1682 mfl = DPAA2_ETH_L2_MAX_FRM(mtu);
1683 linear_mfl = priv->rx_buf_size - DPAA2_ETH_RX_HWA_SIZE -
1684 dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM;
1685
1686 if (mfl > linear_mfl) {
1687 netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n",
1688 linear_mfl - VLAN_ETH_HLEN);
1689 return false;
1690 }
1691
1692 return true;
1693 }
1694
1695 static int set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp)
1696 {
1697 int mfl, err;
1698
1699
1700
1701
1702
1703
1704 if (has_xdp)
1705 mfl = DPAA2_ETH_L2_MAX_FRM(mtu);
1706 else
1707 mfl = DPAA2_ETH_MFL;
1708
1709 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl);
1710 if (err) {
1711 netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n");
1712 return err;
1713 }
1714
1715 return 0;
1716 }
1717
1718 static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu)
1719 {
1720 struct dpaa2_eth_priv *priv = netdev_priv(dev);
1721 int err;
1722
1723 if (!priv->xdp_prog)
1724 goto out;
1725
1726 if (!xdp_mtu_valid(priv, new_mtu))
1727 return -EINVAL;
1728
1729 err = set_rx_mfl(priv, new_mtu, true);
1730 if (err)
1731 return err;
1732
1733 out:
1734 dev->mtu = new_mtu;
1735 return 0;
1736 }
1737
1738 static int update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp)
1739 {
1740 struct dpni_buffer_layout buf_layout = {0};
1741 int err;
1742
1743 err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token,
1744 DPNI_QUEUE_RX, &buf_layout);
1745 if (err) {
1746 netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n");
1747 return err;
1748 }
1749
1750
1751 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) +
1752 (has_xdp ? XDP_PACKET_HEADROOM : 0);
1753 buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM;
1754 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
1755 DPNI_QUEUE_RX, &buf_layout);
1756 if (err) {
1757 netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n");
1758 return err;
1759 }
1760
1761 return 0;
1762 }
1763
1764 static int setup_xdp(struct net_device *dev, struct bpf_prog *prog)
1765 {
1766 struct dpaa2_eth_priv *priv = netdev_priv(dev);
1767 struct dpaa2_eth_channel *ch;
1768 struct bpf_prog *old;
1769 bool up, need_update;
1770 int i, err;
1771
1772 if (prog && !xdp_mtu_valid(priv, dev->mtu))
1773 return -EINVAL;
1774
1775 if (prog) {
1776 prog = bpf_prog_add(prog, priv->num_channels);
1777 if (IS_ERR(prog))
1778 return PTR_ERR(prog);
1779 }
1780
1781 up = netif_running(dev);
1782 need_update = (!!priv->xdp_prog != !!prog);
1783
1784 if (up)
1785 dpaa2_eth_stop(dev);
1786
1787
1788
1789
1790
1791
1792 if (need_update) {
1793 err = set_rx_mfl(priv, dev->mtu, !!prog);
1794 if (err)
1795 goto out_err;
1796 err = update_rx_buffer_headroom(priv, !!prog);
1797 if (err)
1798 goto out_err;
1799 }
1800
1801 old = xchg(&priv->xdp_prog, prog);
1802 if (old)
1803 bpf_prog_put(old);
1804
1805 for (i = 0; i < priv->num_channels; i++) {
1806 ch = priv->channel[i];
1807 old = xchg(&ch->xdp.prog, prog);
1808 if (old)
1809 bpf_prog_put(old);
1810 }
1811
1812 if (up) {
1813 err = dpaa2_eth_open(dev);
1814 if (err)
1815 return err;
1816 }
1817
1818 return 0;
1819
1820 out_err:
1821 if (prog)
1822 bpf_prog_sub(prog, priv->num_channels);
1823 if (up)
1824 dpaa2_eth_open(dev);
1825
1826 return err;
1827 }
1828
1829 static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1830 {
1831 struct dpaa2_eth_priv *priv = netdev_priv(dev);
1832
1833 switch (xdp->command) {
1834 case XDP_SETUP_PROG:
1835 return setup_xdp(dev, xdp->prog);
1836 case XDP_QUERY_PROG:
1837 xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
1838 break;
1839 default:
1840 return -EINVAL;
1841 }
1842
1843 return 0;
1844 }
1845
1846 static int dpaa2_eth_xdp_xmit_frame(struct net_device *net_dev,
1847 struct xdp_frame *xdpf)
1848 {
1849 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1850 struct device *dev = net_dev->dev.parent;
1851 struct rtnl_link_stats64 *percpu_stats;
1852 struct dpaa2_eth_drv_stats *percpu_extras;
1853 unsigned int needed_headroom;
1854 struct dpaa2_eth_swa *swa;
1855 struct dpaa2_eth_fq *fq;
1856 struct dpaa2_fd fd;
1857 void *buffer_start, *aligned_start;
1858 dma_addr_t addr;
1859 int err, i;
1860
1861
1862
1863
1864 needed_headroom = dpaa2_eth_needed_headroom(priv, NULL);
1865 if (xdpf->headroom < needed_headroom)
1866 return -EINVAL;
1867
1868 percpu_stats = this_cpu_ptr(priv->percpu_stats);
1869 percpu_extras = this_cpu_ptr(priv->percpu_extras);
1870
1871
1872 memset(&fd, 0, sizeof(fd));
1873
1874
1875 buffer_start = xdpf->data - needed_headroom;
1876 aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN,
1877 DPAA2_ETH_TX_BUF_ALIGN);
1878 if (aligned_start >= xdpf->data - xdpf->headroom)
1879 buffer_start = aligned_start;
1880
1881 swa = (struct dpaa2_eth_swa *)buffer_start;
1882
1883 swa->type = DPAA2_ETH_SWA_XDP;
1884 swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start;
1885 swa->xdp.xdpf = xdpf;
1886
1887 addr = dma_map_single(dev, buffer_start,
1888 swa->xdp.dma_size,
1889 DMA_BIDIRECTIONAL);
1890 if (unlikely(dma_mapping_error(dev, addr))) {
1891 percpu_stats->tx_dropped++;
1892 return -ENOMEM;
1893 }
1894
1895 dpaa2_fd_set_addr(&fd, addr);
1896 dpaa2_fd_set_offset(&fd, xdpf->data - buffer_start);
1897 dpaa2_fd_set_len(&fd, xdpf->len);
1898 dpaa2_fd_set_format(&fd, dpaa2_fd_single);
1899 dpaa2_fd_set_ctrl(&fd, FD_CTRL_PTA);
1900
1901 fq = &priv->fq[smp_processor_id() % dpaa2_eth_queue_count(priv)];
1902 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
1903 err = priv->enqueue(priv, fq, &fd, 0);
1904 if (err != -EBUSY)
1905 break;
1906 }
1907 percpu_extras->tx_portal_busy += i;
1908 if (unlikely(err < 0)) {
1909 percpu_stats->tx_errors++;
1910
1911 return err;
1912 }
1913
1914 percpu_stats->tx_packets++;
1915 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fd);
1916
1917 return 0;
1918 }
1919
1920 static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n,
1921 struct xdp_frame **frames, u32 flags)
1922 {
1923 int drops = 0;
1924 int i, err;
1925
1926 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1927 return -EINVAL;
1928
1929 if (!netif_running(net_dev))
1930 return -ENETDOWN;
1931
1932 for (i = 0; i < n; i++) {
1933 struct xdp_frame *xdpf = frames[i];
1934
1935 err = dpaa2_eth_xdp_xmit_frame(net_dev, xdpf);
1936 if (err) {
1937 xdp_return_frame_rx_napi(xdpf);
1938 drops++;
1939 }
1940 }
1941
1942 return n - drops;
1943 }
1944
1945 static int update_xps(struct dpaa2_eth_priv *priv)
1946 {
1947 struct net_device *net_dev = priv->net_dev;
1948 struct cpumask xps_mask;
1949 struct dpaa2_eth_fq *fq;
1950 int i, num_queues, netdev_queues;
1951 int err = 0;
1952
1953 num_queues = dpaa2_eth_queue_count(priv);
1954 netdev_queues = (net_dev->num_tc ? : 1) * num_queues;
1955
1956
1957
1958
1959 for (i = 0; i < netdev_queues; i++) {
1960 fq = &priv->fq[i % num_queues];
1961
1962 cpumask_clear(&xps_mask);
1963 cpumask_set_cpu(fq->target_cpu, &xps_mask);
1964
1965 err = netif_set_xps_queue(net_dev, &xps_mask, i);
1966 if (err) {
1967 netdev_warn_once(net_dev, "Error setting XPS queue\n");
1968 break;
1969 }
1970 }
1971
1972 return err;
1973 }
1974
1975 static int dpaa2_eth_setup_tc(struct net_device *net_dev,
1976 enum tc_setup_type type, void *type_data)
1977 {
1978 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1979 struct tc_mqprio_qopt *mqprio = type_data;
1980 u8 num_tc, num_queues;
1981 int i;
1982
1983 if (type != TC_SETUP_QDISC_MQPRIO)
1984 return -EINVAL;
1985
1986 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1987 num_queues = dpaa2_eth_queue_count(priv);
1988 num_tc = mqprio->num_tc;
1989
1990 if (num_tc == net_dev->num_tc)
1991 return 0;
1992
1993 if (num_tc > dpaa2_eth_tc_count(priv)) {
1994 netdev_err(net_dev, "Max %d traffic classes supported\n",
1995 dpaa2_eth_tc_count(priv));
1996 return -EINVAL;
1997 }
1998
1999 if (!num_tc) {
2000 netdev_reset_tc(net_dev);
2001 netif_set_real_num_tx_queues(net_dev, num_queues);
2002 goto out;
2003 }
2004
2005 netdev_set_num_tc(net_dev, num_tc);
2006 netif_set_real_num_tx_queues(net_dev, num_tc * num_queues);
2007
2008 for (i = 0; i < num_tc; i++)
2009 netdev_set_tc_queue(net_dev, i, num_queues, i * num_queues);
2010
2011 out:
2012 update_xps(priv);
2013
2014 return 0;
2015 }
2016
2017 static const struct net_device_ops dpaa2_eth_ops = {
2018 .ndo_open = dpaa2_eth_open,
2019 .ndo_start_xmit = dpaa2_eth_tx,
2020 .ndo_stop = dpaa2_eth_stop,
2021 .ndo_set_mac_address = dpaa2_eth_set_addr,
2022 .ndo_get_stats64 = dpaa2_eth_get_stats,
2023 .ndo_set_rx_mode = dpaa2_eth_set_rx_mode,
2024 .ndo_set_features = dpaa2_eth_set_features,
2025 .ndo_do_ioctl = dpaa2_eth_ioctl,
2026 .ndo_change_mtu = dpaa2_eth_change_mtu,
2027 .ndo_bpf = dpaa2_eth_xdp,
2028 .ndo_xdp_xmit = dpaa2_eth_xdp_xmit,
2029 .ndo_setup_tc = dpaa2_eth_setup_tc,
2030 };
2031
2032 static void cdan_cb(struct dpaa2_io_notification_ctx *ctx)
2033 {
2034 struct dpaa2_eth_channel *ch;
2035
2036 ch = container_of(ctx, struct dpaa2_eth_channel, nctx);
2037
2038
2039 ch->stats.cdan++;
2040
2041 napi_schedule_irqoff(&ch->napi);
2042 }
2043
2044
2045 static struct fsl_mc_device *setup_dpcon(struct dpaa2_eth_priv *priv)
2046 {
2047 struct fsl_mc_device *dpcon;
2048 struct device *dev = priv->net_dev->dev.parent;
2049 struct dpcon_attr attrs;
2050 int err;
2051
2052 err = fsl_mc_object_allocate(to_fsl_mc_device(dev),
2053 FSL_MC_POOL_DPCON, &dpcon);
2054 if (err) {
2055 if (err == -ENXIO)
2056 err = -EPROBE_DEFER;
2057 else
2058 dev_info(dev, "Not enough DPCONs, will go on as-is\n");
2059 return ERR_PTR(err);
2060 }
2061
2062 err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle);
2063 if (err) {
2064 dev_err(dev, "dpcon_open() failed\n");
2065 goto free;
2066 }
2067
2068 err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle);
2069 if (err) {
2070 dev_err(dev, "dpcon_reset() failed\n");
2071 goto close;
2072 }
2073
2074 err = dpcon_get_attributes(priv->mc_io, 0, dpcon->mc_handle, &attrs);
2075 if (err) {
2076 dev_err(dev, "dpcon_get_attributes() failed\n");
2077 goto close;
2078 }
2079
2080 err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle);
2081 if (err) {
2082 dev_err(dev, "dpcon_enable() failed\n");
2083 goto close;
2084 }
2085
2086 return dpcon;
2087
2088 close:
2089 dpcon_close(priv->mc_io, 0, dpcon->mc_handle);
2090 free:
2091 fsl_mc_object_free(dpcon);
2092
2093 return NULL;
2094 }
2095
2096 static void free_dpcon(struct dpaa2_eth_priv *priv,
2097 struct fsl_mc_device *dpcon)
2098 {
2099 dpcon_disable(priv->mc_io, 0, dpcon->mc_handle);
2100 dpcon_close(priv->mc_io, 0, dpcon->mc_handle);
2101 fsl_mc_object_free(dpcon);
2102 }
2103
2104 static struct dpaa2_eth_channel *
2105 alloc_channel(struct dpaa2_eth_priv *priv)
2106 {
2107 struct dpaa2_eth_channel *channel;
2108 struct dpcon_attr attr;
2109 struct device *dev = priv->net_dev->dev.parent;
2110 int err;
2111
2112 channel = kzalloc(sizeof(*channel), GFP_KERNEL);
2113 if (!channel)
2114 return NULL;
2115
2116 channel->dpcon = setup_dpcon(priv);
2117 if (IS_ERR_OR_NULL(channel->dpcon)) {
2118 err = PTR_ERR_OR_ZERO(channel->dpcon);
2119 goto err_setup;
2120 }
2121
2122 err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle,
2123 &attr);
2124 if (err) {
2125 dev_err(dev, "dpcon_get_attributes() failed\n");
2126 goto err_get_attr;
2127 }
2128
2129 channel->dpcon_id = attr.id;
2130 channel->ch_id = attr.qbman_ch_id;
2131 channel->priv = priv;
2132
2133 return channel;
2134
2135 err_get_attr:
2136 free_dpcon(priv, channel->dpcon);
2137 err_setup:
2138 kfree(channel);
2139 return ERR_PTR(err);
2140 }
2141
2142 static void free_channel(struct dpaa2_eth_priv *priv,
2143 struct dpaa2_eth_channel *channel)
2144 {
2145 free_dpcon(priv, channel->dpcon);
2146 kfree(channel);
2147 }
2148
2149
2150
2151
2152 static int setup_dpio(struct dpaa2_eth_priv *priv)
2153 {
2154 struct dpaa2_io_notification_ctx *nctx;
2155 struct dpaa2_eth_channel *channel;
2156 struct dpcon_notification_cfg dpcon_notif_cfg;
2157 struct device *dev = priv->net_dev->dev.parent;
2158 int i, err;
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169 cpumask_clear(&priv->dpio_cpumask);
2170 for_each_online_cpu(i) {
2171
2172 channel = alloc_channel(priv);
2173 if (IS_ERR_OR_NULL(channel)) {
2174 err = PTR_ERR_OR_ZERO(channel);
2175 if (err != -EPROBE_DEFER)
2176 dev_info(dev,
2177 "No affine channel for cpu %d and above\n", i);
2178 goto err_alloc_ch;
2179 }
2180
2181 priv->channel[priv->num_channels] = channel;
2182
2183 nctx = &channel->nctx;
2184 nctx->is_cdan = 1;
2185 nctx->cb = cdan_cb;
2186 nctx->id = channel->ch_id;
2187 nctx->desired_cpu = i;
2188
2189
2190 channel->dpio = dpaa2_io_service_select(i);
2191 err = dpaa2_io_service_register(channel->dpio, nctx, dev);
2192 if (err) {
2193 dev_dbg(dev, "No affine DPIO for cpu %d\n", i);
2194
2195
2196
2197
2198
2199 err = -EPROBE_DEFER;
2200 goto err_service_reg;
2201 }
2202
2203
2204 dpcon_notif_cfg.dpio_id = nctx->dpio_id;
2205 dpcon_notif_cfg.priority = 0;
2206 dpcon_notif_cfg.user_ctx = nctx->qman64;
2207 err = dpcon_set_notification(priv->mc_io, 0,
2208 channel->dpcon->mc_handle,
2209 &dpcon_notif_cfg);
2210 if (err) {
2211 dev_err(dev, "dpcon_set_notification failed()\n");
2212 goto err_set_cdan;
2213 }
2214
2215
2216
2217
2218 cpumask_set_cpu(i, &priv->dpio_cpumask);
2219 priv->num_channels++;
2220
2221
2222
2223
2224 if (priv->num_channels == priv->dpni_attrs.num_queues)
2225 break;
2226 }
2227
2228 return 0;
2229
2230 err_set_cdan:
2231 dpaa2_io_service_deregister(channel->dpio, nctx, dev);
2232 err_service_reg:
2233 free_channel(priv, channel);
2234 err_alloc_ch:
2235 if (err == -EPROBE_DEFER) {
2236 for (i = 0; i < priv->num_channels; i++) {
2237 channel = priv->channel[i];
2238 nctx = &channel->nctx;
2239 dpaa2_io_service_deregister(channel->dpio, nctx, dev);
2240 free_channel(priv, channel);
2241 }
2242 priv->num_channels = 0;
2243 return err;
2244 }
2245
2246 if (cpumask_empty(&priv->dpio_cpumask)) {
2247 dev_err(dev, "No cpu with an affine DPIO/DPCON\n");
2248 return -ENODEV;
2249 }
2250
2251 dev_info(dev, "Cores %*pbl available for processing ingress traffic\n",
2252 cpumask_pr_args(&priv->dpio_cpumask));
2253
2254 return 0;
2255 }
2256
2257 static void free_dpio(struct dpaa2_eth_priv *priv)
2258 {
2259 struct device *dev = priv->net_dev->dev.parent;
2260 struct dpaa2_eth_channel *ch;
2261 int i;
2262
2263
2264 for (i = 0; i < priv->num_channels; i++) {
2265 ch = priv->channel[i];
2266 dpaa2_io_service_deregister(ch->dpio, &ch->nctx, dev);
2267 free_channel(priv, ch);
2268 }
2269 }
2270
2271 static struct dpaa2_eth_channel *get_affine_channel(struct dpaa2_eth_priv *priv,
2272 int cpu)
2273 {
2274 struct device *dev = priv->net_dev->dev.parent;
2275 int i;
2276
2277 for (i = 0; i < priv->num_channels; i++)
2278 if (priv->channel[i]->nctx.desired_cpu == cpu)
2279 return priv->channel[i];
2280
2281
2282
2283
2284 dev_warn(dev, "No affine channel found for cpu %d\n", cpu);
2285
2286 return priv->channel[0];
2287 }
2288
2289 static void set_fq_affinity(struct dpaa2_eth_priv *priv)
2290 {
2291 struct device *dev = priv->net_dev->dev.parent;
2292 struct dpaa2_eth_fq *fq;
2293 int rx_cpu, txc_cpu;
2294 int i;
2295
2296
2297
2298
2299
2300 rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask);
2301
2302 for (i = 0; i < priv->num_fqs; i++) {
2303 fq = &priv->fq[i];
2304 switch (fq->type) {
2305 case DPAA2_RX_FQ:
2306 fq->target_cpu = rx_cpu;
2307 rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask);
2308 if (rx_cpu >= nr_cpu_ids)
2309 rx_cpu = cpumask_first(&priv->dpio_cpumask);
2310 break;
2311 case DPAA2_TX_CONF_FQ:
2312 fq->target_cpu = txc_cpu;
2313 txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask);
2314 if (txc_cpu >= nr_cpu_ids)
2315 txc_cpu = cpumask_first(&priv->dpio_cpumask);
2316 break;
2317 default:
2318 dev_err(dev, "Unknown FQ type: %d\n", fq->type);
2319 }
2320 fq->channel = get_affine_channel(priv, fq->target_cpu);
2321 }
2322
2323 update_xps(priv);
2324 }
2325
2326 static void setup_fqs(struct dpaa2_eth_priv *priv)
2327 {
2328 int i;
2329
2330
2331
2332
2333
2334 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) {
2335 priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ;
2336 priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf;
2337 priv->fq[priv->num_fqs++].flowid = (u16)i;
2338 }
2339
2340 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) {
2341 priv->fq[priv->num_fqs].type = DPAA2_RX_FQ;
2342 priv->fq[priv->num_fqs].consume = dpaa2_eth_rx;
2343 priv->fq[priv->num_fqs++].flowid = (u16)i;
2344 }
2345
2346
2347 set_fq_affinity(priv);
2348 }
2349
2350
2351 static int setup_dpbp(struct dpaa2_eth_priv *priv)
2352 {
2353 int err;
2354 struct fsl_mc_device *dpbp_dev;
2355 struct device *dev = priv->net_dev->dev.parent;
2356 struct dpbp_attr dpbp_attrs;
2357
2358 err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP,
2359 &dpbp_dev);
2360 if (err) {
2361 if (err == -ENXIO)
2362 err = -EPROBE_DEFER;
2363 else
2364 dev_err(dev, "DPBP device allocation failed\n");
2365 return err;
2366 }
2367
2368 priv->dpbp_dev = dpbp_dev;
2369
2370 err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id,
2371 &dpbp_dev->mc_handle);
2372 if (err) {
2373 dev_err(dev, "dpbp_open() failed\n");
2374 goto err_open;
2375 }
2376
2377 err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle);
2378 if (err) {
2379 dev_err(dev, "dpbp_reset() failed\n");
2380 goto err_reset;
2381 }
2382
2383 err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle);
2384 if (err) {
2385 dev_err(dev, "dpbp_enable() failed\n");
2386 goto err_enable;
2387 }
2388
2389 err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle,
2390 &dpbp_attrs);
2391 if (err) {
2392 dev_err(dev, "dpbp_get_attributes() failed\n");
2393 goto err_get_attr;
2394 }
2395 priv->bpid = dpbp_attrs.bpid;
2396
2397 return 0;
2398
2399 err_get_attr:
2400 dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle);
2401 err_enable:
2402 err_reset:
2403 dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle);
2404 err_open:
2405 fsl_mc_object_free(dpbp_dev);
2406
2407 return err;
2408 }
2409
2410 static void free_dpbp(struct dpaa2_eth_priv *priv)
2411 {
2412 drain_pool(priv);
2413 dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle);
2414 dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle);
2415 fsl_mc_object_free(priv->dpbp_dev);
2416 }
2417
2418 static int set_buffer_layout(struct dpaa2_eth_priv *priv)
2419 {
2420 struct device *dev = priv->net_dev->dev.parent;
2421 struct dpni_buffer_layout buf_layout = {0};
2422 u16 rx_buf_align;
2423 int err;
2424
2425
2426
2427
2428
2429 if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) ||
2430 priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0))
2431 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1;
2432 else
2433 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN;
2434
2435
2436
2437
2438 priv->rx_buf_size = ALIGN_DOWN(DPAA2_ETH_RX_BUF_SIZE, rx_buf_align);
2439
2440
2441 buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE;
2442 buf_layout.pass_timestamp = true;
2443 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE |
2444 DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
2445 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
2446 DPNI_QUEUE_TX, &buf_layout);
2447 if (err) {
2448 dev_err(dev, "dpni_set_buffer_layout(TX) failed\n");
2449 return err;
2450 }
2451
2452
2453 buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
2454 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
2455 DPNI_QUEUE_TX_CONFIRM, &buf_layout);
2456 if (err) {
2457 dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n");
2458 return err;
2459 }
2460
2461
2462
2463
2464 err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token,
2465 &priv->tx_data_offset);
2466 if (err) {
2467 dev_err(dev, "dpni_get_tx_data_offset() failed\n");
2468 return err;
2469 }
2470
2471 if ((priv->tx_data_offset % 64) != 0)
2472 dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n",
2473 priv->tx_data_offset);
2474
2475
2476 buf_layout.pass_frame_status = true;
2477 buf_layout.pass_parser_result = true;
2478 buf_layout.data_align = rx_buf_align;
2479 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv);
2480 buf_layout.private_data_size = 0;
2481 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT |
2482 DPNI_BUF_LAYOUT_OPT_FRAME_STATUS |
2483 DPNI_BUF_LAYOUT_OPT_DATA_ALIGN |
2484 DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM |
2485 DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
2486 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
2487 DPNI_QUEUE_RX, &buf_layout);
2488 if (err) {
2489 dev_err(dev, "dpni_set_buffer_layout(RX) failed\n");
2490 return err;
2491 }
2492
2493 return 0;
2494 }
2495
2496 #define DPNI_ENQUEUE_FQID_VER_MAJOR 7
2497 #define DPNI_ENQUEUE_FQID_VER_MINOR 9
2498
2499 static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv,
2500 struct dpaa2_eth_fq *fq,
2501 struct dpaa2_fd *fd, u8 prio)
2502 {
2503 return dpaa2_io_service_enqueue_qd(fq->channel->dpio,
2504 priv->tx_qdid, prio,
2505 fq->tx_qdbin, fd);
2506 }
2507
2508 static inline int dpaa2_eth_enqueue_fq(struct dpaa2_eth_priv *priv,
2509 struct dpaa2_eth_fq *fq,
2510 struct dpaa2_fd *fd, u8 prio)
2511 {
2512 return dpaa2_io_service_enqueue_fq(fq->channel->dpio,
2513 fq->tx_fqid[prio], fd);
2514 }
2515
2516 static void set_enqueue_mode(struct dpaa2_eth_priv *priv)
2517 {
2518 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR,
2519 DPNI_ENQUEUE_FQID_VER_MINOR) < 0)
2520 priv->enqueue = dpaa2_eth_enqueue_qd;
2521 else
2522 priv->enqueue = dpaa2_eth_enqueue_fq;
2523 }
2524
2525 static int set_pause(struct dpaa2_eth_priv *priv)
2526 {
2527 struct device *dev = priv->net_dev->dev.parent;
2528 struct dpni_link_cfg link_cfg = {0};
2529 int err;
2530
2531
2532 err = dpni_get_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg);
2533 if (err) {
2534 dev_err(dev, "dpni_get_link_cfg() failed\n");
2535 return err;
2536 }
2537
2538
2539 link_cfg.options |= DPNI_LINK_OPT_PAUSE;
2540 link_cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE;
2541 err = dpni_set_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg);
2542 if (err) {
2543 dev_err(dev, "dpni_set_link_cfg() failed\n");
2544 return err;
2545 }
2546
2547 priv->link_state.options = link_cfg.options;
2548
2549 return 0;
2550 }
2551
2552 static void update_tx_fqids(struct dpaa2_eth_priv *priv)
2553 {
2554 struct dpni_queue_id qid = {0};
2555 struct dpaa2_eth_fq *fq;
2556 struct dpni_queue queue;
2557 int i, j, err;
2558
2559
2560
2561
2562 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR,
2563 DPNI_ENQUEUE_FQID_VER_MINOR) < 0)
2564 return;
2565
2566 for (i = 0; i < priv->num_fqs; i++) {
2567 fq = &priv->fq[i];
2568 if (fq->type != DPAA2_TX_CONF_FQ)
2569 continue;
2570 for (j = 0; j < dpaa2_eth_tc_count(priv); j++) {
2571 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2572 DPNI_QUEUE_TX, j, fq->flowid,
2573 &queue, &qid);
2574 if (err)
2575 goto out_err;
2576
2577 fq->tx_fqid[j] = qid.fqid;
2578 if (fq->tx_fqid[j] == 0)
2579 goto out_err;
2580 }
2581 }
2582
2583 priv->enqueue = dpaa2_eth_enqueue_fq;
2584
2585 return;
2586
2587 out_err:
2588 netdev_info(priv->net_dev,
2589 "Error reading Tx FQID, fallback to QDID-based enqueue\n");
2590 priv->enqueue = dpaa2_eth_enqueue_qd;
2591 }
2592
2593
2594 static int setup_dpni(struct fsl_mc_device *ls_dev)
2595 {
2596 struct device *dev = &ls_dev->dev;
2597 struct dpaa2_eth_priv *priv;
2598 struct net_device *net_dev;
2599 int err;
2600
2601 net_dev = dev_get_drvdata(dev);
2602 priv = netdev_priv(net_dev);
2603
2604
2605 err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token);
2606 if (err) {
2607 dev_err(dev, "dpni_open() failed\n");
2608 return err;
2609 }
2610
2611
2612 err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major,
2613 &priv->dpni_ver_minor);
2614 if (err) {
2615 dev_err(dev, "dpni_get_api_version() failed\n");
2616 goto close;
2617 }
2618 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) {
2619 dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n",
2620 priv->dpni_ver_major, priv->dpni_ver_minor,
2621 DPNI_VER_MAJOR, DPNI_VER_MINOR);
2622 err = -ENOTSUPP;
2623 goto close;
2624 }
2625
2626 ls_dev->mc_io = priv->mc_io;
2627 ls_dev->mc_handle = priv->mc_token;
2628
2629 err = dpni_reset(priv->mc_io, 0, priv->mc_token);
2630 if (err) {
2631 dev_err(dev, "dpni_reset() failed\n");
2632 goto close;
2633 }
2634
2635 err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token,
2636 &priv->dpni_attrs);
2637 if (err) {
2638 dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err);
2639 goto close;
2640 }
2641
2642 err = set_buffer_layout(priv);
2643 if (err)
2644 goto close;
2645
2646 set_enqueue_mode(priv);
2647
2648
2649 if (dpaa2_eth_has_pause_support(priv)) {
2650 err = set_pause(priv);
2651 if (err)
2652 goto close;
2653 }
2654
2655 priv->cls_rules = devm_kzalloc(dev, sizeof(struct dpaa2_eth_cls_rule) *
2656 dpaa2_eth_fs_count(priv), GFP_KERNEL);
2657 if (!priv->cls_rules)
2658 goto close;
2659
2660 return 0;
2661
2662 close:
2663 dpni_close(priv->mc_io, 0, priv->mc_token);
2664
2665 return err;
2666 }
2667
2668 static void free_dpni(struct dpaa2_eth_priv *priv)
2669 {
2670 int err;
2671
2672 err = dpni_reset(priv->mc_io, 0, priv->mc_token);
2673 if (err)
2674 netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n",
2675 err);
2676
2677 dpni_close(priv->mc_io, 0, priv->mc_token);
2678 }
2679
2680 static int setup_rx_flow(struct dpaa2_eth_priv *priv,
2681 struct dpaa2_eth_fq *fq)
2682 {
2683 struct device *dev = priv->net_dev->dev.parent;
2684 struct dpni_queue queue;
2685 struct dpni_queue_id qid;
2686 int err;
2687
2688 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2689 DPNI_QUEUE_RX, 0, fq->flowid, &queue, &qid);
2690 if (err) {
2691 dev_err(dev, "dpni_get_queue(RX) failed\n");
2692 return err;
2693 }
2694
2695 fq->fqid = qid.fqid;
2696
2697 queue.destination.id = fq->channel->dpcon_id;
2698 queue.destination.type = DPNI_DEST_DPCON;
2699 queue.destination.priority = 1;
2700 queue.user_context = (u64)(uintptr_t)fq;
2701 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token,
2702 DPNI_QUEUE_RX, 0, fq->flowid,
2703 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST,
2704 &queue);
2705 if (err) {
2706 dev_err(dev, "dpni_set_queue(RX) failed\n");
2707 return err;
2708 }
2709
2710
2711 err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev,
2712 fq->flowid);
2713 if (err) {
2714 dev_err(dev, "xdp_rxq_info_reg failed\n");
2715 return err;
2716 }
2717
2718 err = xdp_rxq_info_reg_mem_model(&fq->channel->xdp_rxq,
2719 MEM_TYPE_PAGE_ORDER0, NULL);
2720 if (err) {
2721 dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n");
2722 return err;
2723 }
2724
2725 return 0;
2726 }
2727
2728 static int setup_tx_flow(struct dpaa2_eth_priv *priv,
2729 struct dpaa2_eth_fq *fq)
2730 {
2731 struct device *dev = priv->net_dev->dev.parent;
2732 struct dpni_queue queue;
2733 struct dpni_queue_id qid;
2734 int i, err;
2735
2736 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) {
2737 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2738 DPNI_QUEUE_TX, i, fq->flowid,
2739 &queue, &qid);
2740 if (err) {
2741 dev_err(dev, "dpni_get_queue(TX) failed\n");
2742 return err;
2743 }
2744 fq->tx_fqid[i] = qid.fqid;
2745 }
2746
2747
2748 fq->tx_qdbin = qid.qdbin;
2749
2750 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2751 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid,
2752 &queue, &qid);
2753 if (err) {
2754 dev_err(dev, "dpni_get_queue(TX_CONF) failed\n");
2755 return err;
2756 }
2757
2758 fq->fqid = qid.fqid;
2759
2760 queue.destination.id = fq->channel->dpcon_id;
2761 queue.destination.type = DPNI_DEST_DPCON;
2762 queue.destination.priority = 0;
2763 queue.user_context = (u64)(uintptr_t)fq;
2764 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token,
2765 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid,
2766 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST,
2767 &queue);
2768 if (err) {
2769 dev_err(dev, "dpni_set_queue(TX_CONF) failed\n");
2770 return err;
2771 }
2772
2773 return 0;
2774 }
2775
2776
2777 static const struct dpaa2_eth_dist_fields dist_fields[] = {
2778 {
2779
2780 .rxnfc_field = RXH_L2DA,
2781 .cls_prot = NET_PROT_ETH,
2782 .cls_field = NH_FLD_ETH_DA,
2783 .id = DPAA2_ETH_DIST_ETHDST,
2784 .size = 6,
2785 }, {
2786 .cls_prot = NET_PROT_ETH,
2787 .cls_field = NH_FLD_ETH_SA,
2788 .id = DPAA2_ETH_DIST_ETHSRC,
2789 .size = 6,
2790 }, {
2791
2792
2793
2794
2795 .cls_prot = NET_PROT_ETH,
2796 .cls_field = NH_FLD_ETH_TYPE,
2797 .id = DPAA2_ETH_DIST_ETHTYPE,
2798 .size = 2,
2799 }, {
2800
2801 .rxnfc_field = RXH_VLAN,
2802 .cls_prot = NET_PROT_VLAN,
2803 .cls_field = NH_FLD_VLAN_TCI,
2804 .id = DPAA2_ETH_DIST_VLAN,
2805 .size = 2,
2806 }, {
2807
2808 .rxnfc_field = RXH_IP_SRC,
2809 .cls_prot = NET_PROT_IP,
2810 .cls_field = NH_FLD_IP_SRC,
2811 .id = DPAA2_ETH_DIST_IPSRC,
2812 .size = 4,
2813 }, {
2814 .rxnfc_field = RXH_IP_DST,
2815 .cls_prot = NET_PROT_IP,
2816 .cls_field = NH_FLD_IP_DST,
2817 .id = DPAA2_ETH_DIST_IPDST,
2818 .size = 4,
2819 }, {
2820 .rxnfc_field = RXH_L3_PROTO,
2821 .cls_prot = NET_PROT_IP,
2822 .cls_field = NH_FLD_IP_PROTO,
2823 .id = DPAA2_ETH_DIST_IPPROTO,
2824 .size = 1,
2825 }, {
2826
2827
2828
2829 .rxnfc_field = RXH_L4_B_0_1,
2830 .cls_prot = NET_PROT_UDP,
2831 .cls_field = NH_FLD_UDP_PORT_SRC,
2832 .id = DPAA2_ETH_DIST_L4SRC,
2833 .size = 2,
2834 }, {
2835 .rxnfc_field = RXH_L4_B_2_3,
2836 .cls_prot = NET_PROT_UDP,
2837 .cls_field = NH_FLD_UDP_PORT_DST,
2838 .id = DPAA2_ETH_DIST_L4DST,
2839 .size = 2,
2840 },
2841 };
2842
2843
2844 static int config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key)
2845 {
2846 struct device *dev = priv->net_dev->dev.parent;
2847 struct dpni_rx_tc_dist_cfg dist_cfg;
2848 int err;
2849
2850 memset(&dist_cfg, 0, sizeof(dist_cfg));
2851
2852 dist_cfg.key_cfg_iova = key;
2853 dist_cfg.dist_size = dpaa2_eth_queue_count(priv);
2854 dist_cfg.dist_mode = DPNI_DIST_MODE_HASH;
2855
2856 err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 0, &dist_cfg);
2857 if (err)
2858 dev_err(dev, "dpni_set_rx_tc_dist failed\n");
2859
2860 return err;
2861 }
2862
2863
2864 static int config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key)
2865 {
2866 struct device *dev = priv->net_dev->dev.parent;
2867 struct dpni_rx_dist_cfg dist_cfg;
2868 int err;
2869
2870 memset(&dist_cfg, 0, sizeof(dist_cfg));
2871
2872 dist_cfg.key_cfg_iova = key;
2873 dist_cfg.dist_size = dpaa2_eth_queue_count(priv);
2874 dist_cfg.enable = 1;
2875
2876 err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg);
2877 if (err)
2878 dev_err(dev, "dpni_set_rx_hash_dist failed\n");
2879
2880 return err;
2881 }
2882
2883
2884 static int config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key)
2885 {
2886 struct device *dev = priv->net_dev->dev.parent;
2887 struct dpni_rx_dist_cfg dist_cfg;
2888 int err;
2889
2890 memset(&dist_cfg, 0, sizeof(dist_cfg));
2891
2892 dist_cfg.key_cfg_iova = key;
2893 dist_cfg.dist_size = dpaa2_eth_queue_count(priv);
2894 dist_cfg.enable = 1;
2895
2896 err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg);
2897 if (err)
2898 dev_err(dev, "dpni_set_rx_fs_dist failed\n");
2899
2900 return err;
2901 }
2902
2903
2904 int dpaa2_eth_cls_key_size(u64 fields)
2905 {
2906 int i, size = 0;
2907
2908 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) {
2909 if (!(fields & dist_fields[i].id))
2910 continue;
2911 size += dist_fields[i].size;
2912 }
2913
2914 return size;
2915 }
2916
2917
2918 int dpaa2_eth_cls_fld_off(int prot, int field)
2919 {
2920 int i, off = 0;
2921
2922 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) {
2923 if (dist_fields[i].cls_prot == prot &&
2924 dist_fields[i].cls_field == field)
2925 return off;
2926 off += dist_fields[i].size;
2927 }
2928
2929 WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n");
2930 return 0;
2931 }
2932
2933
2934
2935
2936 void dpaa2_eth_cls_trim_rule(void *key_mem, u64 fields)
2937 {
2938 int off = 0, new_off = 0;
2939 int i, size;
2940
2941 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) {
2942 size = dist_fields[i].size;
2943 if (dist_fields[i].id & fields) {
2944 memcpy(key_mem + new_off, key_mem + off, size);
2945 new_off += size;
2946 }
2947 off += size;
2948 }
2949 }
2950
2951
2952
2953
2954 static int dpaa2_eth_set_dist_key(struct net_device *net_dev,
2955 enum dpaa2_eth_rx_dist type, u64 flags)
2956 {
2957 struct device *dev = net_dev->dev.parent;
2958 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
2959 struct dpkg_profile_cfg cls_cfg;
2960 u32 rx_hash_fields = 0;
2961 dma_addr_t key_iova;
2962 u8 *dma_mem;
2963 int i;
2964 int err = 0;
2965
2966 memset(&cls_cfg, 0, sizeof(cls_cfg));
2967
2968 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) {
2969 struct dpkg_extract *key =
2970 &cls_cfg.extracts[cls_cfg.num_extracts];
2971
2972
2973
2974
2975 if (!(flags & dist_fields[i].id))
2976 continue;
2977 if (type == DPAA2_ETH_RX_DIST_HASH)
2978 rx_hash_fields |= dist_fields[i].rxnfc_field;
2979
2980 if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) {
2981 dev_err(dev, "error adding key extraction rule, too many rules?\n");
2982 return -E2BIG;
2983 }
2984
2985 key->type = DPKG_EXTRACT_FROM_HDR;
2986 key->extract.from_hdr.prot = dist_fields[i].cls_prot;
2987 key->extract.from_hdr.type = DPKG_FULL_FIELD;
2988 key->extract.from_hdr.field = dist_fields[i].cls_field;
2989 cls_cfg.num_extracts++;
2990 }
2991
2992 dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL);
2993 if (!dma_mem)
2994 return -ENOMEM;
2995
2996 err = dpni_prepare_key_cfg(&cls_cfg, dma_mem);
2997 if (err) {
2998 dev_err(dev, "dpni_prepare_key_cfg error %d\n", err);
2999 goto free_key;
3000 }
3001
3002
3003 key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE,
3004 DMA_TO_DEVICE);
3005 if (dma_mapping_error(dev, key_iova)) {
3006 dev_err(dev, "DMA mapping failed\n");
3007 err = -ENOMEM;
3008 goto free_key;
3009 }
3010
3011 if (type == DPAA2_ETH_RX_DIST_HASH) {
3012 if (dpaa2_eth_has_legacy_dist(priv))
3013 err = config_legacy_hash_key(priv, key_iova);
3014 else
3015 err = config_hash_key(priv, key_iova);
3016 } else {
3017 err = config_cls_key(priv, key_iova);
3018 }
3019
3020 dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE,
3021 DMA_TO_DEVICE);
3022 if (!err && type == DPAA2_ETH_RX_DIST_HASH)
3023 priv->rx_hash_fields = rx_hash_fields;
3024
3025 free_key:
3026 kfree(dma_mem);
3027 return err;
3028 }
3029
3030 int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags)
3031 {
3032 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
3033 u64 key = 0;
3034 int i;
3035
3036 if (!dpaa2_eth_hash_enabled(priv))
3037 return -EOPNOTSUPP;
3038
3039 for (i = 0; i < ARRAY_SIZE(dist_fields); i++)
3040 if (dist_fields[i].rxnfc_field & flags)
3041 key |= dist_fields[i].id;
3042
3043 return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, key);
3044 }
3045
3046 int dpaa2_eth_set_cls(struct net_device *net_dev, u64 flags)
3047 {
3048 return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_CLS, flags);
3049 }
3050
3051 static int dpaa2_eth_set_default_cls(struct dpaa2_eth_priv *priv)
3052 {
3053 struct device *dev = priv->net_dev->dev.parent;
3054 int err;
3055
3056
3057 if (dpaa2_eth_has_legacy_dist(priv)) {
3058 dev_dbg(dev, "Rx cls not supported by current MC version\n");
3059 return -EOPNOTSUPP;
3060 }
3061
3062 if (!dpaa2_eth_fs_enabled(priv)) {
3063 dev_dbg(dev, "Rx cls disabled in DPNI options\n");
3064 return -EOPNOTSUPP;
3065 }
3066
3067 if (!dpaa2_eth_hash_enabled(priv)) {
3068 dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n");
3069 return -EOPNOTSUPP;
3070 }
3071
3072
3073
3074
3075
3076 if (!dpaa2_eth_fs_mask_enabled(priv))
3077 goto out;
3078
3079 err = dpaa2_eth_set_cls(priv->net_dev, DPAA2_ETH_DIST_ALL);
3080 if (err)
3081 return err;
3082
3083 out:
3084 priv->rx_cls_enabled = 1;
3085
3086 return 0;
3087 }
3088
3089
3090
3091
3092 static int bind_dpni(struct dpaa2_eth_priv *priv)
3093 {
3094 struct net_device *net_dev = priv->net_dev;
3095 struct device *dev = net_dev->dev.parent;
3096 struct dpni_pools_cfg pools_params;
3097 struct dpni_error_cfg err_cfg;
3098 int err = 0;
3099 int i;
3100
3101 pools_params.num_dpbp = 1;
3102 pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id;
3103 pools_params.pools[0].backup_pool = 0;
3104 pools_params.pools[0].buffer_size = priv->rx_buf_size;
3105 err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);
3106 if (err) {
3107 dev_err(dev, "dpni_set_pools() failed\n");
3108 return err;
3109 }
3110
3111
3112
3113
3114 err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT);
3115 if (err && err != -EOPNOTSUPP)
3116 dev_err(dev, "Failed to configure hashing\n");
3117
3118
3119
3120
3121 err = dpaa2_eth_set_default_cls(priv);
3122 if (err && err != -EOPNOTSUPP)
3123 dev_err(dev, "Failed to configure Rx classification key\n");
3124
3125
3126 err_cfg.errors = DPAA2_FAS_RX_ERR_MASK;
3127 err_cfg.set_frame_annotation = 1;
3128 err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD;
3129 err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token,
3130 &err_cfg);
3131 if (err) {
3132 dev_err(dev, "dpni_set_errors_behavior failed\n");
3133 return err;
3134 }
3135
3136
3137 for (i = 0; i < priv->num_fqs; i++) {
3138 switch (priv->fq[i].type) {
3139 case DPAA2_RX_FQ:
3140 err = setup_rx_flow(priv, &priv->fq[i]);
3141 break;
3142 case DPAA2_TX_CONF_FQ:
3143 err = setup_tx_flow(priv, &priv->fq[i]);
3144 break;
3145 default:
3146 dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type);
3147 return -EINVAL;
3148 }
3149 if (err)
3150 return err;
3151 }
3152
3153 err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token,
3154 DPNI_QUEUE_TX, &priv->tx_qdid);
3155 if (err) {
3156 dev_err(dev, "dpni_get_qdid() failed\n");
3157 return err;
3158 }
3159
3160 return 0;
3161 }
3162
3163
3164 static int alloc_rings(struct dpaa2_eth_priv *priv)
3165 {
3166 struct net_device *net_dev = priv->net_dev;
3167 struct device *dev = net_dev->dev.parent;
3168 int i;
3169
3170 for (i = 0; i < priv->num_channels; i++) {
3171 priv->channel[i]->store =
3172 dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev);
3173 if (!priv->channel[i]->store) {
3174 netdev_err(net_dev, "dpaa2_io_store_create() failed\n");
3175 goto err_ring;
3176 }
3177 }
3178
3179 return 0;
3180
3181 err_ring:
3182 for (i = 0; i < priv->num_channels; i++) {
3183 if (!priv->channel[i]->store)
3184 break;
3185 dpaa2_io_store_destroy(priv->channel[i]->store);
3186 }
3187
3188 return -ENOMEM;
3189 }
3190
3191 static void free_rings(struct dpaa2_eth_priv *priv)
3192 {
3193 int i;
3194
3195 for (i = 0; i < priv->num_channels; i++)
3196 dpaa2_io_store_destroy(priv->channel[i]->store);
3197 }
3198
3199 static int set_mac_addr(struct dpaa2_eth_priv *priv)
3200 {
3201 struct net_device *net_dev = priv->net_dev;
3202 struct device *dev = net_dev->dev.parent;
3203 u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN];
3204 int err;
3205
3206
3207 err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr);
3208 if (err) {
3209 dev_err(dev, "dpni_get_port_mac_addr() failed\n");
3210 return err;
3211 }
3212
3213
3214 err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token,
3215 dpni_mac_addr);
3216 if (err) {
3217 dev_err(dev, "dpni_get_primary_mac_addr() failed\n");
3218 return err;
3219 }
3220
3221
3222 if (!is_zero_ether_addr(mac_addr)) {
3223
3224 if (!ether_addr_equal(mac_addr, dpni_mac_addr)) {
3225 err = dpni_set_primary_mac_addr(priv->mc_io, 0,
3226 priv->mc_token,
3227 mac_addr);
3228 if (err) {
3229 dev_err(dev, "dpni_set_primary_mac_addr() failed\n");
3230 return err;
3231 }
3232 }
3233 memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len);
3234 } else if (is_zero_ether_addr(dpni_mac_addr)) {
3235
3236
3237
3238 eth_hw_addr_random(net_dev);
3239 dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n");
3240
3241 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token,
3242 net_dev->dev_addr);
3243 if (err) {
3244 dev_err(dev, "dpni_set_primary_mac_addr() failed\n");
3245 return err;
3246 }
3247
3248
3249
3250
3251
3252
3253 net_dev->addr_assign_type = NET_ADDR_PERM;
3254 } else {
3255
3256
3257
3258 memcpy(net_dev->dev_addr, dpni_mac_addr, net_dev->addr_len);
3259 }
3260
3261 return 0;
3262 }
3263
3264 static int netdev_init(struct net_device *net_dev)
3265 {
3266 struct device *dev = net_dev->dev.parent;
3267 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
3268 u32 options = priv->dpni_attrs.options;
3269 u64 supported = 0, not_supported = 0;
3270 u8 bcast_addr[ETH_ALEN];
3271 u8 num_queues;
3272 int err;
3273
3274 net_dev->netdev_ops = &dpaa2_eth_ops;
3275 net_dev->ethtool_ops = &dpaa2_ethtool_ops;
3276
3277 err = set_mac_addr(priv);
3278 if (err)
3279 return err;
3280
3281
3282 eth_broadcast_addr(bcast_addr);
3283 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr);
3284 if (err) {
3285 dev_err(dev, "dpni_add_mac_addr() failed\n");
3286 return err;
3287 }
3288
3289
3290 net_dev->max_mtu = DPAA2_ETH_MAX_MTU;
3291 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token,
3292 DPAA2_ETH_MFL);
3293 if (err) {
3294 dev_err(dev, "dpni_set_max_frame_length() failed\n");
3295 return err;
3296 }
3297
3298
3299 num_queues = dpaa2_eth_queue_count(priv);
3300 err = netif_set_real_num_tx_queues(net_dev, num_queues);
3301 if (err) {
3302 dev_err(dev, "netif_set_real_num_tx_queues() failed\n");
3303 return err;
3304 }
3305 err = netif_set_real_num_rx_queues(net_dev, num_queues);
3306 if (err) {
3307 dev_err(dev, "netif_set_real_num_rx_queues() failed\n");
3308 return err;
3309 }
3310
3311
3312 supported |= IFF_LIVE_ADDR_CHANGE;
3313
3314 if (options & DPNI_OPT_NO_MAC_FILTER)
3315 not_supported |= IFF_UNICAST_FLT;
3316 else
3317 supported |= IFF_UNICAST_FLT;
3318
3319 net_dev->priv_flags |= supported;
3320 net_dev->priv_flags &= ~not_supported;
3321
3322
3323 net_dev->features = NETIF_F_RXCSUM |
3324 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3325 NETIF_F_SG | NETIF_F_HIGHDMA |
3326 NETIF_F_LLTX;
3327 net_dev->hw_features = net_dev->features;
3328
3329 return 0;
3330 }
3331
3332 static int poll_link_state(void *arg)
3333 {
3334 struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg;
3335 int err;
3336
3337 while (!kthread_should_stop()) {
3338 err = link_state_update(priv);
3339 if (unlikely(err))
3340 return err;
3341
3342 msleep(DPAA2_ETH_LINK_STATE_REFRESH);
3343 }
3344
3345 return 0;
3346 }
3347
3348 static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg)
3349 {
3350 u32 status = ~0;
3351 struct device *dev = (struct device *)arg;
3352 struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev);
3353 struct net_device *net_dev = dev_get_drvdata(dev);
3354 int err;
3355
3356 err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle,
3357 DPNI_IRQ_INDEX, &status);
3358 if (unlikely(err)) {
3359 netdev_err(net_dev, "Can't get irq status (err %d)\n", err);
3360 return IRQ_HANDLED;
3361 }
3362
3363 if (status & DPNI_IRQ_EVENT_LINK_CHANGED)
3364 link_state_update(netdev_priv(net_dev));
3365
3366 if (status & DPNI_IRQ_EVENT_ENDPOINT_CHANGED)
3367 set_mac_addr(netdev_priv(net_dev));
3368
3369 return IRQ_HANDLED;
3370 }
3371
3372 static int setup_irqs(struct fsl_mc_device *ls_dev)
3373 {
3374 int err = 0;
3375 struct fsl_mc_device_irq *irq;
3376
3377 err = fsl_mc_allocate_irqs(ls_dev);
3378 if (err) {
3379 dev_err(&ls_dev->dev, "MC irqs allocation failed\n");
3380 return err;
3381 }
3382
3383 irq = ls_dev->irqs[0];
3384 err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq,
3385 NULL, dpni_irq0_handler_thread,
3386 IRQF_NO_SUSPEND | IRQF_ONESHOT,
3387 dev_name(&ls_dev->dev), &ls_dev->dev);
3388 if (err < 0) {
3389 dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err);
3390 goto free_mc_irq;
3391 }
3392
3393 err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle,
3394 DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED |
3395 DPNI_IRQ_EVENT_ENDPOINT_CHANGED);
3396 if (err < 0) {
3397 dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err);
3398 goto free_irq;
3399 }
3400
3401 err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle,
3402 DPNI_IRQ_INDEX, 1);
3403 if (err < 0) {
3404 dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err);
3405 goto free_irq;
3406 }
3407
3408 return 0;
3409
3410 free_irq:
3411 devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev);
3412 free_mc_irq:
3413 fsl_mc_free_irqs(ls_dev);
3414
3415 return err;
3416 }
3417
3418 static void add_ch_napi(struct dpaa2_eth_priv *priv)
3419 {
3420 int i;
3421 struct dpaa2_eth_channel *ch;
3422
3423 for (i = 0; i < priv->num_channels; i++) {
3424 ch = priv->channel[i];
3425
3426 netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll,
3427 NAPI_POLL_WEIGHT);
3428 }
3429 }
3430
3431 static void del_ch_napi(struct dpaa2_eth_priv *priv)
3432 {
3433 int i;
3434 struct dpaa2_eth_channel *ch;
3435
3436 for (i = 0; i < priv->num_channels; i++) {
3437 ch = priv->channel[i];
3438 netif_napi_del(&ch->napi);
3439 }
3440 }
3441
3442 static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev)
3443 {
3444 struct device *dev;
3445 struct net_device *net_dev = NULL;
3446 struct dpaa2_eth_priv *priv = NULL;
3447 int err = 0;
3448
3449 dev = &dpni_dev->dev;
3450
3451
3452 net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_NETDEV_QUEUES);
3453 if (!net_dev) {
3454 dev_err(dev, "alloc_etherdev_mq() failed\n");
3455 return -ENOMEM;
3456 }
3457
3458 SET_NETDEV_DEV(net_dev, dev);
3459 dev_set_drvdata(dev, net_dev);
3460
3461 priv = netdev_priv(net_dev);
3462 priv->net_dev = net_dev;
3463
3464 priv->iommu_domain = iommu_get_domain_for_dev(dev);
3465
3466
3467 err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL,
3468 &priv->mc_io);
3469 if (err) {
3470 if (err == -ENXIO)
3471 err = -EPROBE_DEFER;
3472 else
3473 dev_err(dev, "MC portal allocation failed\n");
3474 goto err_portal_alloc;
3475 }
3476
3477
3478 err = setup_dpni(dpni_dev);
3479 if (err)
3480 goto err_dpni_setup;
3481
3482 err = setup_dpio(priv);
3483 if (err)
3484 goto err_dpio_setup;
3485
3486 setup_fqs(priv);
3487
3488 err = setup_dpbp(priv);
3489 if (err)
3490 goto err_dpbp_setup;
3491
3492 err = bind_dpni(priv);
3493 if (err)
3494 goto err_bind;
3495
3496
3497 add_ch_napi(priv);
3498
3499
3500 priv->percpu_stats = alloc_percpu(*priv->percpu_stats);
3501 if (!priv->percpu_stats) {
3502 dev_err(dev, "alloc_percpu(percpu_stats) failed\n");
3503 err = -ENOMEM;
3504 goto err_alloc_percpu_stats;
3505 }
3506 priv->percpu_extras = alloc_percpu(*priv->percpu_extras);
3507 if (!priv->percpu_extras) {
3508 dev_err(dev, "alloc_percpu(percpu_extras) failed\n");
3509 err = -ENOMEM;
3510 goto err_alloc_percpu_extras;
3511 }
3512
3513 err = netdev_init(net_dev);
3514 if (err)
3515 goto err_netdev_init;
3516
3517
3518 err = set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM));
3519 if (err)
3520 goto err_csum;
3521
3522 err = set_tx_csum(priv, !!(net_dev->features &
3523 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)));
3524 if (err)
3525 goto err_csum;
3526
3527 err = alloc_rings(priv);
3528 if (err)
3529 goto err_alloc_rings;
3530
3531 err = setup_irqs(dpni_dev);
3532 if (err) {
3533 netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n");
3534 priv->poll_thread = kthread_run(poll_link_state, priv,
3535 "%s_poll_link", net_dev->name);
3536 if (IS_ERR(priv->poll_thread)) {
3537 dev_err(dev, "Error starting polling thread\n");
3538 goto err_poll_thread;
3539 }
3540 priv->do_link_poll = true;
3541 }
3542
3543 err = register_netdev(net_dev);
3544 if (err < 0) {
3545 dev_err(dev, "register_netdev() failed\n");
3546 goto err_netdev_reg;
3547 }
3548
3549 #ifdef CONFIG_DEBUG_FS
3550 dpaa2_dbg_add(priv);
3551 #endif
3552
3553 dev_info(dev, "Probed interface %s\n", net_dev->name);
3554 return 0;
3555
3556 err_netdev_reg:
3557 if (priv->do_link_poll)
3558 kthread_stop(priv->poll_thread);
3559 else
3560 fsl_mc_free_irqs(dpni_dev);
3561 err_poll_thread:
3562 free_rings(priv);
3563 err_alloc_rings:
3564 err_csum:
3565 err_netdev_init:
3566 free_percpu(priv->percpu_extras);
3567 err_alloc_percpu_extras:
3568 free_percpu(priv->percpu_stats);
3569 err_alloc_percpu_stats:
3570 del_ch_napi(priv);
3571 err_bind:
3572 free_dpbp(priv);
3573 err_dpbp_setup:
3574 free_dpio(priv);
3575 err_dpio_setup:
3576 free_dpni(priv);
3577 err_dpni_setup:
3578 fsl_mc_portal_free(priv->mc_io);
3579 err_portal_alloc:
3580 dev_set_drvdata(dev, NULL);
3581 free_netdev(net_dev);
3582
3583 return err;
3584 }
3585
3586 static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev)
3587 {
3588 struct device *dev;
3589 struct net_device *net_dev;
3590 struct dpaa2_eth_priv *priv;
3591
3592 dev = &ls_dev->dev;
3593 net_dev = dev_get_drvdata(dev);
3594 priv = netdev_priv(net_dev);
3595
3596 #ifdef CONFIG_DEBUG_FS
3597 dpaa2_dbg_remove(priv);
3598 #endif
3599 unregister_netdev(net_dev);
3600
3601 if (priv->do_link_poll)
3602 kthread_stop(priv->poll_thread);
3603 else
3604 fsl_mc_free_irqs(ls_dev);
3605
3606 free_rings(priv);
3607 free_percpu(priv->percpu_stats);
3608 free_percpu(priv->percpu_extras);
3609
3610 del_ch_napi(priv);
3611 free_dpbp(priv);
3612 free_dpio(priv);
3613 free_dpni(priv);
3614
3615 fsl_mc_portal_free(priv->mc_io);
3616
3617 free_netdev(net_dev);
3618
3619 dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name);
3620
3621 return 0;
3622 }
3623
3624 static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = {
3625 {
3626 .vendor = FSL_MC_VENDOR_FREESCALE,
3627 .obj_type = "dpni",
3628 },
3629 { .vendor = 0x0 }
3630 };
3631 MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table);
3632
3633 static struct fsl_mc_driver dpaa2_eth_driver = {
3634 .driver = {
3635 .name = KBUILD_MODNAME,
3636 .owner = THIS_MODULE,
3637 },
3638 .probe = dpaa2_eth_probe,
3639 .remove = dpaa2_eth_remove,
3640 .match_id_table = dpaa2_eth_match_id_table
3641 };
3642
3643 static int __init dpaa2_eth_driver_init(void)
3644 {
3645 int err;
3646
3647 dpaa2_eth_dbg_init();
3648 err = fsl_mc_driver_register(&dpaa2_eth_driver);
3649 if (err) {
3650 dpaa2_eth_dbg_exit();
3651 return err;
3652 }
3653
3654 return 0;
3655 }
3656
3657 static void __exit dpaa2_eth_driver_exit(void)
3658 {
3659 dpaa2_eth_dbg_exit();
3660 fsl_mc_driver_unregister(&dpaa2_eth_driver);
3661 }
3662
3663 module_init(dpaa2_eth_driver_init);
3664 module_exit(dpaa2_eth_driver_exit);