This source file includes following definitions.
- rvu_poll_reg
- rvu_alloc_rsrc
- rvu_alloc_rsrc_contig
- rvu_free_rsrc_contig
- rvu_rsrc_check_contig
- rvu_free_rsrc
- rvu_rsrc_free_count
- rvu_alloc_bitmap
- rvu_get_lf
- rvu_get_blkaddr
- rvu_update_rsrc_map
- rvu_get_pf
- rvu_get_pf_numvfs
- rvu_get_hwvf
- rvu_get_pfvf
- is_pf_func_valid
- is_block_implemented
- rvu_check_block_implemented
- rvu_lf_reset
- rvu_block_reset
- rvu_reset_all_blocks
- rvu_scan_block
- rvu_check_min_msix_vec
- rvu_setup_msix_resources
- rvu_free_hw_resources
- rvu_setup_hw_resources
- rvu_aq_free
- rvu_aq_alloc
- rvu_mbox_handler_ready
- rvu_get_rsrc_mapcount
- is_pffunc_map_valid
- rvu_lookup_rsrc
- rvu_detach_block
- rvu_detach_rsrcs
- rvu_mbox_handler_detach_resources
- rvu_attach_block
- rvu_check_rsrc_availability
- rvu_mbox_handler_attach_resources
- rvu_get_msix_offset
- rvu_set_msix_offset
- rvu_clear_msix_offset
- rvu_mbox_handler_msix_offset
- rvu_mbox_handler_vf_flr
- rvu_process_mbox_msg
- __rvu_mbox_handler
- rvu_afpf_mbox_handler
- rvu_afvf_mbox_handler
- __rvu_mbox_up_handler
- rvu_afpf_mbox_up_handler
- rvu_afvf_mbox_up_handler
- rvu_mbox_init
- rvu_mbox_destroy
- rvu_queue_work
- rvu_mbox_intr_handler
- rvu_enable_mbox_intr
- rvu_blklf_teardown
- __rvu_flr_handler
- rvu_afvf_flr_handler
- rvu_flr_handler
- rvu_afvf_queue_flr_work
- rvu_flr_intr_handler
- rvu_me_handle_vfset
- rvu_me_vf_intr_handler
- rvu_me_pf_intr_handler
- rvu_unregister_interrupts
- rvu_afvf_msix_vectors_num_ok
- rvu_register_interrupts
- rvu_flr_wq_destroy
- rvu_flr_init
- rvu_disable_afvf_intr
- rvu_enable_afvf_intr
- lbk_get_num_chans
- rvu_enable_sriov
- rvu_disable_sriov
- rvu_update_module_params
- rvu_probe
- rvu_remove
- rvu_init_module
- rvu_cleanup_module
1
2
3
4
5
6
7
8
9
10
11 #include <linux/module.h>
12 #include <linux/interrupt.h>
13 #include <linux/delay.h>
14 #include <linux/irq.h>
15 #include <linux/pci.h>
16 #include <linux/sysfs.h>
17
18 #include "cgx.h"
19 #include "rvu.h"
20 #include "rvu_reg.h"
21
22 #define DRV_NAME "octeontx2-af"
23 #define DRV_STRING "Marvell OcteonTX2 RVU Admin Function Driver"
24 #define DRV_VERSION "1.0"
25
26 static int rvu_get_hwvf(struct rvu *rvu, int pcifunc);
27
28 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
29 struct rvu_block *block, int lf);
30 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
31 struct rvu_block *block, int lf);
32 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc);
33
34 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
35 int type, int num,
36 void (mbox_handler)(struct work_struct *),
37 void (mbox_up_handler)(struct work_struct *));
38 enum {
39 TYPE_AFVF,
40 TYPE_AFPF,
41 };
42
43
44 static const struct pci_device_id rvu_id_table[] = {
45 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_AF) },
46 { 0, }
47 };
48
49 MODULE_AUTHOR("Marvell International Ltd.");
50 MODULE_DESCRIPTION(DRV_STRING);
51 MODULE_LICENSE("GPL v2");
52 MODULE_VERSION(DRV_VERSION);
53 MODULE_DEVICE_TABLE(pci, rvu_id_table);
54
55 static char *mkex_profile;
56 module_param(mkex_profile, charp, 0000);
57 MODULE_PARM_DESC(mkex_profile, "MKEX profile name string");
58
59
60
61
62 int rvu_poll_reg(struct rvu *rvu, u64 block, u64 offset, u64 mask, bool zero)
63 {
64 unsigned long timeout = jiffies + usecs_to_jiffies(100);
65 void __iomem *reg;
66 u64 reg_val;
67
68 reg = rvu->afreg_base + ((block << 28) | offset);
69 while (time_before(jiffies, timeout)) {
70 reg_val = readq(reg);
71 if (zero && !(reg_val & mask))
72 return 0;
73 if (!zero && (reg_val & mask))
74 return 0;
75 usleep_range(1, 5);
76 timeout--;
77 }
78 return -EBUSY;
79 }
80
81 int rvu_alloc_rsrc(struct rsrc_bmap *rsrc)
82 {
83 int id;
84
85 if (!rsrc->bmap)
86 return -EINVAL;
87
88 id = find_first_zero_bit(rsrc->bmap, rsrc->max);
89 if (id >= rsrc->max)
90 return -ENOSPC;
91
92 __set_bit(id, rsrc->bmap);
93
94 return id;
95 }
96
97 int rvu_alloc_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc)
98 {
99 int start;
100
101 if (!rsrc->bmap)
102 return -EINVAL;
103
104 start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
105 if (start >= rsrc->max)
106 return -ENOSPC;
107
108 bitmap_set(rsrc->bmap, start, nrsrc);
109 return start;
110 }
111
112 static void rvu_free_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc, int start)
113 {
114 if (!rsrc->bmap)
115 return;
116 if (start >= rsrc->max)
117 return;
118
119 bitmap_clear(rsrc->bmap, start, nrsrc);
120 }
121
122 bool rvu_rsrc_check_contig(struct rsrc_bmap *rsrc, int nrsrc)
123 {
124 int start;
125
126 if (!rsrc->bmap)
127 return false;
128
129 start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
130 if (start >= rsrc->max)
131 return false;
132
133 return true;
134 }
135
136 void rvu_free_rsrc(struct rsrc_bmap *rsrc, int id)
137 {
138 if (!rsrc->bmap)
139 return;
140
141 __clear_bit(id, rsrc->bmap);
142 }
143
144 int rvu_rsrc_free_count(struct rsrc_bmap *rsrc)
145 {
146 int used;
147
148 if (!rsrc->bmap)
149 return 0;
150
151 used = bitmap_weight(rsrc->bmap, rsrc->max);
152 return (rsrc->max - used);
153 }
154
155 int rvu_alloc_bitmap(struct rsrc_bmap *rsrc)
156 {
157 rsrc->bmap = kcalloc(BITS_TO_LONGS(rsrc->max),
158 sizeof(long), GFP_KERNEL);
159 if (!rsrc->bmap)
160 return -ENOMEM;
161 return 0;
162 }
163
164
165 int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot)
166 {
167 u16 match = 0;
168 int lf;
169
170 mutex_lock(&rvu->rsrc_lock);
171 for (lf = 0; lf < block->lf.max; lf++) {
172 if (block->fn_map[lf] == pcifunc) {
173 if (slot == match) {
174 mutex_unlock(&rvu->rsrc_lock);
175 return lf;
176 }
177 match++;
178 }
179 }
180 mutex_unlock(&rvu->rsrc_lock);
181 return -ENODEV;
182 }
183
184
185
186
187
188
189
190 int rvu_get_blkaddr(struct rvu *rvu, int blktype, u16 pcifunc)
191 {
192 int devnum, blkaddr = -ENODEV;
193 u64 cfg, reg;
194 bool is_pf;
195
196 switch (blktype) {
197 case BLKTYPE_NPC:
198 blkaddr = BLKADDR_NPC;
199 goto exit;
200 case BLKTYPE_NPA:
201 blkaddr = BLKADDR_NPA;
202 goto exit;
203 case BLKTYPE_NIX:
204
205 if (!pcifunc) {
206 blkaddr = BLKADDR_NIX0;
207 goto exit;
208 }
209 break;
210 case BLKTYPE_SSO:
211 blkaddr = BLKADDR_SSO;
212 goto exit;
213 case BLKTYPE_SSOW:
214 blkaddr = BLKADDR_SSOW;
215 goto exit;
216 case BLKTYPE_TIM:
217 blkaddr = BLKADDR_TIM;
218 goto exit;
219 case BLKTYPE_CPT:
220
221 if (!pcifunc) {
222 blkaddr = BLKADDR_CPT0;
223 goto exit;
224 }
225 break;
226 }
227
228
229 if (pcifunc & RVU_PFVF_FUNC_MASK) {
230 is_pf = false;
231 devnum = rvu_get_hwvf(rvu, pcifunc);
232 } else {
233 is_pf = true;
234 devnum = rvu_get_pf(pcifunc);
235 }
236
237
238 if (blktype == BLKTYPE_NIX) {
239 reg = is_pf ? RVU_PRIV_PFX_NIX0_CFG : RVU_PRIV_HWVFX_NIX0_CFG;
240 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
241 if (cfg)
242 blkaddr = BLKADDR_NIX0;
243 }
244
245
246 if (blktype == BLKTYPE_CPT) {
247 reg = is_pf ? RVU_PRIV_PFX_CPT0_CFG : RVU_PRIV_HWVFX_CPT0_CFG;
248 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
249 if (cfg)
250 blkaddr = BLKADDR_CPT0;
251 }
252
253 exit:
254 if (is_block_implemented(rvu->hw, blkaddr))
255 return blkaddr;
256 return -ENODEV;
257 }
258
259 static void rvu_update_rsrc_map(struct rvu *rvu, struct rvu_pfvf *pfvf,
260 struct rvu_block *block, u16 pcifunc,
261 u16 lf, bool attach)
262 {
263 int devnum, num_lfs = 0;
264 bool is_pf;
265 u64 reg;
266
267 if (lf >= block->lf.max) {
268 dev_err(&rvu->pdev->dev,
269 "%s: FATAL: LF %d is >= %s's max lfs i.e %d\n",
270 __func__, lf, block->name, block->lf.max);
271 return;
272 }
273
274
275 if (pcifunc & RVU_PFVF_FUNC_MASK) {
276 is_pf = false;
277 devnum = rvu_get_hwvf(rvu, pcifunc);
278 } else {
279 is_pf = true;
280 devnum = rvu_get_pf(pcifunc);
281 }
282
283 block->fn_map[lf] = attach ? pcifunc : 0;
284
285 switch (block->type) {
286 case BLKTYPE_NPA:
287 pfvf->npalf = attach ? true : false;
288 num_lfs = pfvf->npalf;
289 break;
290 case BLKTYPE_NIX:
291 pfvf->nixlf = attach ? true : false;
292 num_lfs = pfvf->nixlf;
293 break;
294 case BLKTYPE_SSO:
295 attach ? pfvf->sso++ : pfvf->sso--;
296 num_lfs = pfvf->sso;
297 break;
298 case BLKTYPE_SSOW:
299 attach ? pfvf->ssow++ : pfvf->ssow--;
300 num_lfs = pfvf->ssow;
301 break;
302 case BLKTYPE_TIM:
303 attach ? pfvf->timlfs++ : pfvf->timlfs--;
304 num_lfs = pfvf->timlfs;
305 break;
306 case BLKTYPE_CPT:
307 attach ? pfvf->cptlfs++ : pfvf->cptlfs--;
308 num_lfs = pfvf->cptlfs;
309 break;
310 }
311
312 reg = is_pf ? block->pf_lfcnt_reg : block->vf_lfcnt_reg;
313 rvu_write64(rvu, BLKADDR_RVUM, reg | (devnum << 16), num_lfs);
314 }
315
316 inline int rvu_get_pf(u16 pcifunc)
317 {
318 return (pcifunc >> RVU_PFVF_PF_SHIFT) & RVU_PFVF_PF_MASK;
319 }
320
321 void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf)
322 {
323 u64 cfg;
324
325
326 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
327 *numvfs = (cfg >> 12) & 0xFF;
328 *hwvf = cfg & 0xFFF;
329 }
330
331 static int rvu_get_hwvf(struct rvu *rvu, int pcifunc)
332 {
333 int pf, func;
334 u64 cfg;
335
336 pf = rvu_get_pf(pcifunc);
337 func = pcifunc & RVU_PFVF_FUNC_MASK;
338
339
340 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
341
342 return ((cfg & 0xFFF) + func - 1);
343 }
344
345 struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc)
346 {
347
348 if (pcifunc & RVU_PFVF_FUNC_MASK)
349 return &rvu->hwvf[rvu_get_hwvf(rvu, pcifunc)];
350 else
351 return &rvu->pf[rvu_get_pf(pcifunc)];
352 }
353
354 static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)
355 {
356 int pf, vf, nvfs;
357 u64 cfg;
358
359 pf = rvu_get_pf(pcifunc);
360 if (pf >= rvu->hw->total_pfs)
361 return false;
362
363 if (!(pcifunc & RVU_PFVF_FUNC_MASK))
364 return true;
365
366
367 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
368 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
369 nvfs = (cfg >> 12) & 0xFF;
370 if (vf >= nvfs)
371 return false;
372
373 return true;
374 }
375
376 bool is_block_implemented(struct rvu_hwinfo *hw, int blkaddr)
377 {
378 struct rvu_block *block;
379
380 if (blkaddr < BLKADDR_RVUM || blkaddr >= BLK_COUNT)
381 return false;
382
383 block = &hw->block[blkaddr];
384 return block->implemented;
385 }
386
387 static void rvu_check_block_implemented(struct rvu *rvu)
388 {
389 struct rvu_hwinfo *hw = rvu->hw;
390 struct rvu_block *block;
391 int blkid;
392 u64 cfg;
393
394
395 for (blkid = 0; blkid < BLK_COUNT; blkid++) {
396 block = &hw->block[blkid];
397 cfg = rvupf_read64(rvu, RVU_PF_BLOCK_ADDRX_DISC(blkid));
398 if (cfg & BIT_ULL(11))
399 block->implemented = true;
400 }
401 }
402
403 int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf)
404 {
405 int err;
406
407 if (!block->implemented)
408 return 0;
409
410 rvu_write64(rvu, block->addr, block->lfreset_reg, lf | BIT_ULL(12));
411 err = rvu_poll_reg(rvu, block->addr, block->lfreset_reg, BIT_ULL(12),
412 true);
413 return err;
414 }
415
416 static void rvu_block_reset(struct rvu *rvu, int blkaddr, u64 rst_reg)
417 {
418 struct rvu_block *block = &rvu->hw->block[blkaddr];
419
420 if (!block->implemented)
421 return;
422
423 rvu_write64(rvu, blkaddr, rst_reg, BIT_ULL(0));
424 rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true);
425 }
426
427 static void rvu_reset_all_blocks(struct rvu *rvu)
428 {
429
430 rvu_block_reset(rvu, BLKADDR_NPA, NPA_AF_BLK_RST);
431 rvu_block_reset(rvu, BLKADDR_NIX0, NIX_AF_BLK_RST);
432 rvu_block_reset(rvu, BLKADDR_NPC, NPC_AF_BLK_RST);
433 rvu_block_reset(rvu, BLKADDR_SSO, SSO_AF_BLK_RST);
434 rvu_block_reset(rvu, BLKADDR_TIM, TIM_AF_BLK_RST);
435 rvu_block_reset(rvu, BLKADDR_CPT0, CPT_AF_BLK_RST);
436 rvu_block_reset(rvu, BLKADDR_NDC0, NDC_AF_BLK_RST);
437 rvu_block_reset(rvu, BLKADDR_NDC1, NDC_AF_BLK_RST);
438 rvu_block_reset(rvu, BLKADDR_NDC2, NDC_AF_BLK_RST);
439 }
440
441 static void rvu_scan_block(struct rvu *rvu, struct rvu_block *block)
442 {
443 struct rvu_pfvf *pfvf;
444 u64 cfg;
445 int lf;
446
447 for (lf = 0; lf < block->lf.max; lf++) {
448 cfg = rvu_read64(rvu, block->addr,
449 block->lfcfg_reg | (lf << block->lfshift));
450 if (!(cfg & BIT_ULL(63)))
451 continue;
452
453
454 __set_bit(lf, block->lf.bmap);
455
456
457 pfvf = rvu_get_pfvf(rvu, (cfg >> 8) & 0xFFFF);
458 rvu_update_rsrc_map(rvu, pfvf, block,
459 (cfg >> 8) & 0xFFFF, lf, true);
460
461
462 rvu_set_msix_offset(rvu, pfvf, block, lf);
463 }
464 }
465
466 static void rvu_check_min_msix_vec(struct rvu *rvu, int nvecs, int pf, int vf)
467 {
468 int min_vecs;
469
470 if (!vf)
471 goto check_pf;
472
473 if (!nvecs) {
474 dev_warn(rvu->dev,
475 "PF%d:VF%d is configured with zero msix vectors, %d\n",
476 pf, vf - 1, nvecs);
477 }
478 return;
479
480 check_pf:
481 if (pf == 0)
482 min_vecs = RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT;
483 else
484 min_vecs = RVU_PF_INT_VEC_CNT;
485
486 if (!(nvecs < min_vecs))
487 return;
488 dev_warn(rvu->dev,
489 "PF%d is configured with too few vectors, %d, min is %d\n",
490 pf, nvecs, min_vecs);
491 }
492
493 static int rvu_setup_msix_resources(struct rvu *rvu)
494 {
495 struct rvu_hwinfo *hw = rvu->hw;
496 int pf, vf, numvfs, hwvf, err;
497 int nvecs, offset, max_msix;
498 struct rvu_pfvf *pfvf;
499 u64 cfg, phy_addr;
500 dma_addr_t iova;
501
502 for (pf = 0; pf < hw->total_pfs; pf++) {
503 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
504
505 if (!((cfg >> 20) & 0x01))
506 continue;
507
508 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
509
510 pfvf = &rvu->pf[pf];
511
512 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_MSIX_CFG(pf));
513 pfvf->msix.max = ((cfg >> 32) & 0xFFF) + 1;
514 rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, 0);
515
516
517 err = rvu_alloc_bitmap(&pfvf->msix);
518 if (err)
519 return err;
520
521
522 pfvf->msix_lfmap = devm_kcalloc(rvu->dev, pfvf->msix.max,
523 sizeof(u16), GFP_KERNEL);
524 if (!pfvf->msix_lfmap)
525 return -ENOMEM;
526
527
528
529
530 if (!pf)
531 goto setup_vfmsix;
532
533
534
535
536
537 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(pf));
538 nvecs = (cfg >> 12) & 0xFF;
539 cfg &= ~0x7FFULL;
540 offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
541 rvu_write64(rvu, BLKADDR_RVUM,
542 RVU_PRIV_PFX_INT_CFG(pf), cfg | offset);
543 setup_vfmsix:
544
545 for (vf = 0; vf < numvfs; vf++) {
546 pfvf = &rvu->hwvf[hwvf + vf];
547
548 cfg = rvu_read64(rvu, BLKADDR_RVUM,
549 RVU_PRIV_PFX_MSIX_CFG(pf));
550 pfvf->msix.max = (cfg & 0xFFF) + 1;
551 rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, vf + 1);
552
553
554 err = rvu_alloc_bitmap(&pfvf->msix);
555 if (err)
556 return err;
557
558 pfvf->msix_lfmap =
559 devm_kcalloc(rvu->dev, pfvf->msix.max,
560 sizeof(u16), GFP_KERNEL);
561 if (!pfvf->msix_lfmap)
562 return -ENOMEM;
563
564
565
566
567
568 cfg = rvu_read64(rvu, BLKADDR_RVUM,
569 RVU_PRIV_HWVFX_INT_CFG(hwvf + vf));
570 nvecs = (cfg >> 12) & 0xFF;
571 cfg &= ~0x7FFULL;
572 offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
573 rvu_write64(rvu, BLKADDR_RVUM,
574 RVU_PRIV_HWVFX_INT_CFG(hwvf + vf),
575 cfg | offset);
576 }
577 }
578
579
580
581
582
583 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
584 max_msix = cfg & 0xFFFFF;
585 phy_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE);
586 iova = dma_map_resource(rvu->dev, phy_addr,
587 max_msix * PCI_MSIX_ENTRY_SIZE,
588 DMA_BIDIRECTIONAL, 0);
589
590 if (dma_mapping_error(rvu->dev, iova))
591 return -ENOMEM;
592
593 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE, (u64)iova);
594 rvu->msix_base_iova = iova;
595
596 return 0;
597 }
598
599 static void rvu_free_hw_resources(struct rvu *rvu)
600 {
601 struct rvu_hwinfo *hw = rvu->hw;
602 struct rvu_block *block;
603 struct rvu_pfvf *pfvf;
604 int id, max_msix;
605 u64 cfg;
606
607 rvu_npa_freemem(rvu);
608 rvu_npc_freemem(rvu);
609 rvu_nix_freemem(rvu);
610
611
612 for (id = 0; id < BLK_COUNT; id++) {
613 block = &hw->block[id];
614 kfree(block->lf.bmap);
615 }
616
617
618 for (id = 0; id < hw->total_pfs; id++) {
619 pfvf = &rvu->pf[id];
620 kfree(pfvf->msix.bmap);
621 }
622
623 for (id = 0; id < hw->total_vfs; id++) {
624 pfvf = &rvu->hwvf[id];
625 kfree(pfvf->msix.bmap);
626 }
627
628
629 if (!rvu->msix_base_iova)
630 return;
631 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
632 max_msix = cfg & 0xFFFFF;
633 dma_unmap_resource(rvu->dev, rvu->msix_base_iova,
634 max_msix * PCI_MSIX_ENTRY_SIZE,
635 DMA_BIDIRECTIONAL, 0);
636
637 mutex_destroy(&rvu->rsrc_lock);
638 }
639
640 static int rvu_setup_hw_resources(struct rvu *rvu)
641 {
642 struct rvu_hwinfo *hw = rvu->hw;
643 struct rvu_block *block;
644 int blkid, err;
645 u64 cfg;
646
647
648 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
649 hw->total_pfs = (cfg >> 32) & 0xFF;
650 hw->total_vfs = (cfg >> 20) & 0xFFF;
651 hw->max_vfs_per_pf = (cfg >> 40) & 0xFF;
652
653
654 block = &hw->block[BLKADDR_NPA];
655 if (!block->implemented)
656 goto nix;
657 cfg = rvu_read64(rvu, BLKADDR_NPA, NPA_AF_CONST);
658 block->lf.max = (cfg >> 16) & 0xFFF;
659 block->addr = BLKADDR_NPA;
660 block->type = BLKTYPE_NPA;
661 block->lfshift = 8;
662 block->lookup_reg = NPA_AF_RVU_LF_CFG_DEBUG;
663 block->pf_lfcnt_reg = RVU_PRIV_PFX_NPA_CFG;
664 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NPA_CFG;
665 block->lfcfg_reg = NPA_PRIV_LFX_CFG;
666 block->msixcfg_reg = NPA_PRIV_LFX_INT_CFG;
667 block->lfreset_reg = NPA_AF_LF_RST;
668 sprintf(block->name, "NPA");
669 err = rvu_alloc_bitmap(&block->lf);
670 if (err)
671 return err;
672
673 nix:
674
675 block = &hw->block[BLKADDR_NIX0];
676 if (!block->implemented)
677 goto sso;
678 cfg = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_CONST2);
679 block->lf.max = cfg & 0xFFF;
680 block->addr = BLKADDR_NIX0;
681 block->type = BLKTYPE_NIX;
682 block->lfshift = 8;
683 block->lookup_reg = NIX_AF_RVU_LF_CFG_DEBUG;
684 block->pf_lfcnt_reg = RVU_PRIV_PFX_NIX0_CFG;
685 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NIX0_CFG;
686 block->lfcfg_reg = NIX_PRIV_LFX_CFG;
687 block->msixcfg_reg = NIX_PRIV_LFX_INT_CFG;
688 block->lfreset_reg = NIX_AF_LF_RST;
689 sprintf(block->name, "NIX");
690 err = rvu_alloc_bitmap(&block->lf);
691 if (err)
692 return err;
693
694 sso:
695
696 block = &hw->block[BLKADDR_SSO];
697 if (!block->implemented)
698 goto ssow;
699 cfg = rvu_read64(rvu, BLKADDR_SSO, SSO_AF_CONST);
700 block->lf.max = cfg & 0xFFFF;
701 block->addr = BLKADDR_SSO;
702 block->type = BLKTYPE_SSO;
703 block->multislot = true;
704 block->lfshift = 3;
705 block->lookup_reg = SSO_AF_RVU_LF_CFG_DEBUG;
706 block->pf_lfcnt_reg = RVU_PRIV_PFX_SSO_CFG;
707 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSO_CFG;
708 block->lfcfg_reg = SSO_PRIV_LFX_HWGRP_CFG;
709 block->msixcfg_reg = SSO_PRIV_LFX_HWGRP_INT_CFG;
710 block->lfreset_reg = SSO_AF_LF_HWGRP_RST;
711 sprintf(block->name, "SSO GROUP");
712 err = rvu_alloc_bitmap(&block->lf);
713 if (err)
714 return err;
715
716 ssow:
717
718 block = &hw->block[BLKADDR_SSOW];
719 if (!block->implemented)
720 goto tim;
721 block->lf.max = (cfg >> 56) & 0xFF;
722 block->addr = BLKADDR_SSOW;
723 block->type = BLKTYPE_SSOW;
724 block->multislot = true;
725 block->lfshift = 3;
726 block->lookup_reg = SSOW_AF_RVU_LF_HWS_CFG_DEBUG;
727 block->pf_lfcnt_reg = RVU_PRIV_PFX_SSOW_CFG;
728 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSOW_CFG;
729 block->lfcfg_reg = SSOW_PRIV_LFX_HWS_CFG;
730 block->msixcfg_reg = SSOW_PRIV_LFX_HWS_INT_CFG;
731 block->lfreset_reg = SSOW_AF_LF_HWS_RST;
732 sprintf(block->name, "SSOWS");
733 err = rvu_alloc_bitmap(&block->lf);
734 if (err)
735 return err;
736
737 tim:
738
739 block = &hw->block[BLKADDR_TIM];
740 if (!block->implemented)
741 goto cpt;
742 cfg = rvu_read64(rvu, BLKADDR_TIM, TIM_AF_CONST);
743 block->lf.max = cfg & 0xFFFF;
744 block->addr = BLKADDR_TIM;
745 block->type = BLKTYPE_TIM;
746 block->multislot = true;
747 block->lfshift = 3;
748 block->lookup_reg = TIM_AF_RVU_LF_CFG_DEBUG;
749 block->pf_lfcnt_reg = RVU_PRIV_PFX_TIM_CFG;
750 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_TIM_CFG;
751 block->lfcfg_reg = TIM_PRIV_LFX_CFG;
752 block->msixcfg_reg = TIM_PRIV_LFX_INT_CFG;
753 block->lfreset_reg = TIM_AF_LF_RST;
754 sprintf(block->name, "TIM");
755 err = rvu_alloc_bitmap(&block->lf);
756 if (err)
757 return err;
758
759 cpt:
760
761 block = &hw->block[BLKADDR_CPT0];
762 if (!block->implemented)
763 goto init;
764 cfg = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_CONSTANTS0);
765 block->lf.max = cfg & 0xFF;
766 block->addr = BLKADDR_CPT0;
767 block->type = BLKTYPE_CPT;
768 block->multislot = true;
769 block->lfshift = 3;
770 block->lookup_reg = CPT_AF_RVU_LF_CFG_DEBUG;
771 block->pf_lfcnt_reg = RVU_PRIV_PFX_CPT0_CFG;
772 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_CPT0_CFG;
773 block->lfcfg_reg = CPT_PRIV_LFX_CFG;
774 block->msixcfg_reg = CPT_PRIV_LFX_INT_CFG;
775 block->lfreset_reg = CPT_AF_LF_RST;
776 sprintf(block->name, "CPT");
777 err = rvu_alloc_bitmap(&block->lf);
778 if (err)
779 return err;
780
781 init:
782
783 rvu->pf = devm_kcalloc(rvu->dev, hw->total_pfs,
784 sizeof(struct rvu_pfvf), GFP_KERNEL);
785 if (!rvu->pf)
786 return -ENOMEM;
787
788 rvu->hwvf = devm_kcalloc(rvu->dev, hw->total_vfs,
789 sizeof(struct rvu_pfvf), GFP_KERNEL);
790 if (!rvu->hwvf)
791 return -ENOMEM;
792
793 mutex_init(&rvu->rsrc_lock);
794
795 err = rvu_setup_msix_resources(rvu);
796 if (err)
797 return err;
798
799 for (blkid = 0; blkid < BLK_COUNT; blkid++) {
800 block = &hw->block[blkid];
801 if (!block->lf.bmap)
802 continue;
803
804
805 block->fn_map = devm_kcalloc(rvu->dev, block->lf.max,
806 sizeof(u16), GFP_KERNEL);
807 if (!block->fn_map)
808 return -ENOMEM;
809
810
811
812
813 rvu_scan_block(rvu, block);
814 }
815
816 err = rvu_npc_init(rvu);
817 if (err)
818 goto exit;
819
820 err = rvu_cgx_init(rvu);
821 if (err)
822 goto exit;
823
824 err = rvu_npa_init(rvu);
825 if (err)
826 goto cgx_err;
827
828 err = rvu_nix_init(rvu);
829 if (err)
830 goto cgx_err;
831
832 return 0;
833
834 cgx_err:
835 rvu_cgx_exit(rvu);
836 exit:
837 return err;
838 }
839
840
841 void rvu_aq_free(struct rvu *rvu, struct admin_queue *aq)
842 {
843 if (!aq)
844 return;
845
846 qmem_free(rvu->dev, aq->inst);
847 qmem_free(rvu->dev, aq->res);
848 devm_kfree(rvu->dev, aq);
849 }
850
851 int rvu_aq_alloc(struct rvu *rvu, struct admin_queue **ad_queue,
852 int qsize, int inst_size, int res_size)
853 {
854 struct admin_queue *aq;
855 int err;
856
857 *ad_queue = devm_kzalloc(rvu->dev, sizeof(*aq), GFP_KERNEL);
858 if (!*ad_queue)
859 return -ENOMEM;
860 aq = *ad_queue;
861
862
863 err = qmem_alloc(rvu->dev, &aq->inst, qsize, inst_size);
864 if (err) {
865 devm_kfree(rvu->dev, aq);
866 return err;
867 }
868
869
870 err = qmem_alloc(rvu->dev, &aq->res, qsize, res_size);
871 if (err) {
872 rvu_aq_free(rvu, aq);
873 return err;
874 }
875
876 spin_lock_init(&aq->lock);
877 return 0;
878 }
879
880 static int rvu_mbox_handler_ready(struct rvu *rvu, struct msg_req *req,
881 struct ready_msg_rsp *rsp)
882 {
883 return 0;
884 }
885
886
887
888
889 static u16 rvu_get_rsrc_mapcount(struct rvu_pfvf *pfvf, int blktype)
890 {
891 switch (blktype) {
892 case BLKTYPE_NPA:
893 return pfvf->npalf ? 1 : 0;
894 case BLKTYPE_NIX:
895 return pfvf->nixlf ? 1 : 0;
896 case BLKTYPE_SSO:
897 return pfvf->sso;
898 case BLKTYPE_SSOW:
899 return pfvf->ssow;
900 case BLKTYPE_TIM:
901 return pfvf->timlfs;
902 case BLKTYPE_CPT:
903 return pfvf->cptlfs;
904 }
905 return 0;
906 }
907
908 bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype)
909 {
910 struct rvu_pfvf *pfvf;
911
912 if (!is_pf_func_valid(rvu, pcifunc))
913 return false;
914
915 pfvf = rvu_get_pfvf(rvu, pcifunc);
916
917
918 if (!rvu_get_rsrc_mapcount(pfvf, blktype))
919 return false;
920
921 return true;
922 }
923
924 static int rvu_lookup_rsrc(struct rvu *rvu, struct rvu_block *block,
925 int pcifunc, int slot)
926 {
927 u64 val;
928
929 val = ((u64)pcifunc << 24) | (slot << 16) | (1ULL << 13);
930 rvu_write64(rvu, block->addr, block->lookup_reg, val);
931
932
933 while (rvu_read64(rvu, block->addr, block->lookup_reg) & (1ULL << 13))
934 ;
935
936 val = rvu_read64(rvu, block->addr, block->lookup_reg);
937
938
939 if (!(val & (1ULL << 12)))
940 return -1;
941
942 return (val & 0xFFF);
943 }
944
945 static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype)
946 {
947 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
948 struct rvu_hwinfo *hw = rvu->hw;
949 struct rvu_block *block;
950 int slot, lf, num_lfs;
951 int blkaddr;
952
953 blkaddr = rvu_get_blkaddr(rvu, blktype, pcifunc);
954 if (blkaddr < 0)
955 return;
956
957 block = &hw->block[blkaddr];
958
959 num_lfs = rvu_get_rsrc_mapcount(pfvf, block->type);
960 if (!num_lfs)
961 return;
962
963 for (slot = 0; slot < num_lfs; slot++) {
964 lf = rvu_lookup_rsrc(rvu, block, pcifunc, slot);
965 if (lf < 0)
966 continue;
967
968
969 rvu_write64(rvu, blkaddr, block->lfcfg_reg |
970 (lf << block->lfshift), 0x00ULL);
971
972
973 rvu_update_rsrc_map(rvu, pfvf, block,
974 pcifunc, lf, false);
975
976
977 rvu_free_rsrc(&block->lf, lf);
978
979
980 rvu_clear_msix_offset(rvu, pfvf, block, lf);
981 }
982 }
983
984 static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach,
985 u16 pcifunc)
986 {
987 struct rvu_hwinfo *hw = rvu->hw;
988 bool detach_all = true;
989 struct rvu_block *block;
990 int blkid;
991
992 mutex_lock(&rvu->rsrc_lock);
993
994
995 if (detach && detach->partial)
996 detach_all = false;
997
998
999
1000
1001 for (blkid = 0; blkid < BLK_COUNT; blkid++) {
1002 block = &hw->block[blkid];
1003 if (!block->lf.bmap)
1004 continue;
1005 if (!detach_all && detach) {
1006 if (blkid == BLKADDR_NPA && !detach->npalf)
1007 continue;
1008 else if ((blkid == BLKADDR_NIX0) && !detach->nixlf)
1009 continue;
1010 else if ((blkid == BLKADDR_SSO) && !detach->sso)
1011 continue;
1012 else if ((blkid == BLKADDR_SSOW) && !detach->ssow)
1013 continue;
1014 else if ((blkid == BLKADDR_TIM) && !detach->timlfs)
1015 continue;
1016 else if ((blkid == BLKADDR_CPT0) && !detach->cptlfs)
1017 continue;
1018 }
1019 rvu_detach_block(rvu, pcifunc, block->type);
1020 }
1021
1022 mutex_unlock(&rvu->rsrc_lock);
1023 return 0;
1024 }
1025
1026 static int rvu_mbox_handler_detach_resources(struct rvu *rvu,
1027 struct rsrc_detach *detach,
1028 struct msg_rsp *rsp)
1029 {
1030 return rvu_detach_rsrcs(rvu, detach, detach->hdr.pcifunc);
1031 }
1032
1033 static void rvu_attach_block(struct rvu *rvu, int pcifunc,
1034 int blktype, int num_lfs)
1035 {
1036 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1037 struct rvu_hwinfo *hw = rvu->hw;
1038 struct rvu_block *block;
1039 int slot, lf;
1040 int blkaddr;
1041 u64 cfg;
1042
1043 if (!num_lfs)
1044 return;
1045
1046 blkaddr = rvu_get_blkaddr(rvu, blktype, 0);
1047 if (blkaddr < 0)
1048 return;
1049
1050 block = &hw->block[blkaddr];
1051 if (!block->lf.bmap)
1052 return;
1053
1054 for (slot = 0; slot < num_lfs; slot++) {
1055
1056 lf = rvu_alloc_rsrc(&block->lf);
1057 if (lf < 0)
1058 return;
1059
1060 cfg = (1ULL << 63) | (pcifunc << 8) | slot;
1061 rvu_write64(rvu, blkaddr, block->lfcfg_reg |
1062 (lf << block->lfshift), cfg);
1063 rvu_update_rsrc_map(rvu, pfvf, block,
1064 pcifunc, lf, true);
1065
1066
1067 rvu_set_msix_offset(rvu, pfvf, block, lf);
1068 }
1069 }
1070
1071 static int rvu_check_rsrc_availability(struct rvu *rvu,
1072 struct rsrc_attach *req, u16 pcifunc)
1073 {
1074 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1075 struct rvu_hwinfo *hw = rvu->hw;
1076 struct rvu_block *block;
1077 int free_lfs, mappedlfs;
1078
1079
1080 if (req->npalf && !rvu_get_rsrc_mapcount(pfvf, BLKTYPE_NPA)) {
1081 block = &hw->block[BLKADDR_NPA];
1082 free_lfs = rvu_rsrc_free_count(&block->lf);
1083 if (!free_lfs)
1084 goto fail;
1085 } else if (req->npalf) {
1086 dev_err(&rvu->pdev->dev,
1087 "Func 0x%x: Invalid req, already has NPA\n",
1088 pcifunc);
1089 return -EINVAL;
1090 }
1091
1092
1093 if (req->nixlf && !rvu_get_rsrc_mapcount(pfvf, BLKTYPE_NIX)) {
1094 block = &hw->block[BLKADDR_NIX0];
1095 free_lfs = rvu_rsrc_free_count(&block->lf);
1096 if (!free_lfs)
1097 goto fail;
1098 } else if (req->nixlf) {
1099 dev_err(&rvu->pdev->dev,
1100 "Func 0x%x: Invalid req, already has NIX\n",
1101 pcifunc);
1102 return -EINVAL;
1103 }
1104
1105 if (req->sso) {
1106 block = &hw->block[BLKADDR_SSO];
1107
1108 if (req->sso > block->lf.max) {
1109 dev_err(&rvu->pdev->dev,
1110 "Func 0x%x: Invalid SSO req, %d > max %d\n",
1111 pcifunc, req->sso, block->lf.max);
1112 return -EINVAL;
1113 }
1114 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1115 free_lfs = rvu_rsrc_free_count(&block->lf);
1116
1117 if (req->sso > mappedlfs &&
1118 ((req->sso - mappedlfs) > free_lfs))
1119 goto fail;
1120 }
1121
1122 if (req->ssow) {
1123 block = &hw->block[BLKADDR_SSOW];
1124 if (req->ssow > block->lf.max) {
1125 dev_err(&rvu->pdev->dev,
1126 "Func 0x%x: Invalid SSOW req, %d > max %d\n",
1127 pcifunc, req->sso, block->lf.max);
1128 return -EINVAL;
1129 }
1130 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1131 free_lfs = rvu_rsrc_free_count(&block->lf);
1132 if (req->ssow > mappedlfs &&
1133 ((req->ssow - mappedlfs) > free_lfs))
1134 goto fail;
1135 }
1136
1137 if (req->timlfs) {
1138 block = &hw->block[BLKADDR_TIM];
1139 if (req->timlfs > block->lf.max) {
1140 dev_err(&rvu->pdev->dev,
1141 "Func 0x%x: Invalid TIMLF req, %d > max %d\n",
1142 pcifunc, req->timlfs, block->lf.max);
1143 return -EINVAL;
1144 }
1145 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1146 free_lfs = rvu_rsrc_free_count(&block->lf);
1147 if (req->timlfs > mappedlfs &&
1148 ((req->timlfs - mappedlfs) > free_lfs))
1149 goto fail;
1150 }
1151
1152 if (req->cptlfs) {
1153 block = &hw->block[BLKADDR_CPT0];
1154 if (req->cptlfs > block->lf.max) {
1155 dev_err(&rvu->pdev->dev,
1156 "Func 0x%x: Invalid CPTLF req, %d > max %d\n",
1157 pcifunc, req->cptlfs, block->lf.max);
1158 return -EINVAL;
1159 }
1160 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1161 free_lfs = rvu_rsrc_free_count(&block->lf);
1162 if (req->cptlfs > mappedlfs &&
1163 ((req->cptlfs - mappedlfs) > free_lfs))
1164 goto fail;
1165 }
1166
1167 return 0;
1168
1169 fail:
1170 dev_info(rvu->dev, "Request for %s failed\n", block->name);
1171 return -ENOSPC;
1172 }
1173
1174 static int rvu_mbox_handler_attach_resources(struct rvu *rvu,
1175 struct rsrc_attach *attach,
1176 struct msg_rsp *rsp)
1177 {
1178 u16 pcifunc = attach->hdr.pcifunc;
1179 int err;
1180
1181
1182 if (!attach->modify)
1183 rvu_detach_rsrcs(rvu, NULL, pcifunc);
1184
1185 mutex_lock(&rvu->rsrc_lock);
1186
1187
1188 err = rvu_check_rsrc_availability(rvu, attach, pcifunc);
1189 if (err)
1190 goto exit;
1191
1192
1193 if (attach->npalf)
1194 rvu_attach_block(rvu, pcifunc, BLKTYPE_NPA, 1);
1195
1196 if (attach->nixlf)
1197 rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1);
1198
1199 if (attach->sso) {
1200
1201
1202
1203
1204
1205 if (attach->modify)
1206 rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO);
1207 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO, attach->sso);
1208 }
1209
1210 if (attach->ssow) {
1211 if (attach->modify)
1212 rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW);
1213 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW, attach->ssow);
1214 }
1215
1216 if (attach->timlfs) {
1217 if (attach->modify)
1218 rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM);
1219 rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM, attach->timlfs);
1220 }
1221
1222 if (attach->cptlfs) {
1223 if (attach->modify)
1224 rvu_detach_block(rvu, pcifunc, BLKTYPE_CPT);
1225 rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT, attach->cptlfs);
1226 }
1227
1228 exit:
1229 mutex_unlock(&rvu->rsrc_lock);
1230 return err;
1231 }
1232
1233 static u16 rvu_get_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1234 int blkaddr, int lf)
1235 {
1236 u16 vec;
1237
1238 if (lf < 0)
1239 return MSIX_VECTOR_INVALID;
1240
1241 for (vec = 0; vec < pfvf->msix.max; vec++) {
1242 if (pfvf->msix_lfmap[vec] == MSIX_BLKLF(blkaddr, lf))
1243 return vec;
1244 }
1245 return MSIX_VECTOR_INVALID;
1246 }
1247
1248 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1249 struct rvu_block *block, int lf)
1250 {
1251 u16 nvecs, vec, offset;
1252 u64 cfg;
1253
1254 cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1255 (lf << block->lfshift));
1256 nvecs = (cfg >> 12) & 0xFF;
1257
1258
1259 if (!rvu_rsrc_check_contig(&pfvf->msix, nvecs))
1260 return;
1261
1262 offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
1263
1264
1265 rvu_write64(rvu, block->addr, block->msixcfg_reg |
1266 (lf << block->lfshift), (cfg & ~0x7FFULL) | offset);
1267
1268
1269 for (vec = 0; vec < nvecs; vec++)
1270 pfvf->msix_lfmap[offset + vec] = MSIX_BLKLF(block->addr, lf);
1271 }
1272
1273 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1274 struct rvu_block *block, int lf)
1275 {
1276 u16 nvecs, vec, offset;
1277 u64 cfg;
1278
1279 cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1280 (lf << block->lfshift));
1281 nvecs = (cfg >> 12) & 0xFF;
1282
1283
1284 rvu_write64(rvu, block->addr, block->msixcfg_reg |
1285 (lf << block->lfshift), cfg & ~0x7FFULL);
1286
1287 offset = rvu_get_msix_offset(rvu, pfvf, block->addr, lf);
1288
1289
1290 for (vec = 0; vec < nvecs; vec++)
1291 pfvf->msix_lfmap[offset + vec] = 0;
1292
1293
1294 rvu_free_rsrc_contig(&pfvf->msix, nvecs, offset);
1295 }
1296
1297 static int rvu_mbox_handler_msix_offset(struct rvu *rvu, struct msg_req *req,
1298 struct msix_offset_rsp *rsp)
1299 {
1300 struct rvu_hwinfo *hw = rvu->hw;
1301 u16 pcifunc = req->hdr.pcifunc;
1302 struct rvu_pfvf *pfvf;
1303 int lf, slot;
1304
1305 pfvf = rvu_get_pfvf(rvu, pcifunc);
1306 if (!pfvf->msix.bmap)
1307 return 0;
1308
1309
1310 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NPA], pcifunc, 0);
1311 rsp->npa_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NPA, lf);
1312
1313 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NIX0], pcifunc, 0);
1314 rsp->nix_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NIX0, lf);
1315
1316 rsp->sso = pfvf->sso;
1317 for (slot = 0; slot < rsp->sso; slot++) {
1318 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSO], pcifunc, slot);
1319 rsp->sso_msixoff[slot] =
1320 rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSO, lf);
1321 }
1322
1323 rsp->ssow = pfvf->ssow;
1324 for (slot = 0; slot < rsp->ssow; slot++) {
1325 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSOW], pcifunc, slot);
1326 rsp->ssow_msixoff[slot] =
1327 rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSOW, lf);
1328 }
1329
1330 rsp->timlfs = pfvf->timlfs;
1331 for (slot = 0; slot < rsp->timlfs; slot++) {
1332 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_TIM], pcifunc, slot);
1333 rsp->timlf_msixoff[slot] =
1334 rvu_get_msix_offset(rvu, pfvf, BLKADDR_TIM, lf);
1335 }
1336
1337 rsp->cptlfs = pfvf->cptlfs;
1338 for (slot = 0; slot < rsp->cptlfs; slot++) {
1339 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT0], pcifunc, slot);
1340 rsp->cptlf_msixoff[slot] =
1341 rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT0, lf);
1342 }
1343 return 0;
1344 }
1345
1346 static int rvu_mbox_handler_vf_flr(struct rvu *rvu, struct msg_req *req,
1347 struct msg_rsp *rsp)
1348 {
1349 u16 pcifunc = req->hdr.pcifunc;
1350 u16 vf, numvfs;
1351 u64 cfg;
1352
1353 vf = pcifunc & RVU_PFVF_FUNC_MASK;
1354 cfg = rvu_read64(rvu, BLKADDR_RVUM,
1355 RVU_PRIV_PFX_CFG(rvu_get_pf(pcifunc)));
1356 numvfs = (cfg >> 12) & 0xFF;
1357
1358 if (vf && vf <= numvfs)
1359 __rvu_flr_handler(rvu, pcifunc);
1360 else
1361 return RVU_INVALID_VF_ID;
1362
1363 return 0;
1364 }
1365
1366 static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid,
1367 struct mbox_msghdr *req)
1368 {
1369 struct rvu *rvu = pci_get_drvdata(mbox->pdev);
1370
1371
1372 if (req->sig != OTX2_MBOX_REQ_SIG)
1373 goto bad_message;
1374
1375 switch (req->id) {
1376 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \
1377 case _id: { \
1378 struct _rsp_type *rsp; \
1379 int err; \
1380 \
1381 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \
1382 mbox, devid, \
1383 sizeof(struct _rsp_type)); \
1384 \
1385 \
1386 if (!rsp && \
1387 _id != MBOX_MSG_DETACH_RESOURCES && \
1388 _id != MBOX_MSG_NIX_TXSCH_FREE && \
1389 _id != MBOX_MSG_VF_FLR) \
1390 return -ENOMEM; \
1391 if (rsp) { \
1392 rsp->hdr.id = _id; \
1393 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \
1394 rsp->hdr.pcifunc = req->pcifunc; \
1395 rsp->hdr.rc = 0; \
1396 } \
1397 \
1398 err = rvu_mbox_handler_ ## _fn_name(rvu, \
1399 (struct _req_type *)req, \
1400 rsp); \
1401 if (rsp && err) \
1402 rsp->hdr.rc = err; \
1403 \
1404 return rsp ? err : -ENOMEM; \
1405 }
1406 MBOX_MESSAGES
1407 #undef M
1408
1409 bad_message:
1410 default:
1411 otx2_reply_invalid_msg(mbox, devid, req->pcifunc, req->id);
1412 return -ENODEV;
1413 }
1414 }
1415
1416 static void __rvu_mbox_handler(struct rvu_work *mwork, int type)
1417 {
1418 struct rvu *rvu = mwork->rvu;
1419 int offset, err, id, devid;
1420 struct otx2_mbox_dev *mdev;
1421 struct mbox_hdr *req_hdr;
1422 struct mbox_msghdr *msg;
1423 struct mbox_wq_info *mw;
1424 struct otx2_mbox *mbox;
1425
1426 switch (type) {
1427 case TYPE_AFPF:
1428 mw = &rvu->afpf_wq_info;
1429 break;
1430 case TYPE_AFVF:
1431 mw = &rvu->afvf_wq_info;
1432 break;
1433 default:
1434 return;
1435 }
1436
1437 devid = mwork - mw->mbox_wrk;
1438 mbox = &mw->mbox;
1439 mdev = &mbox->dev[devid];
1440
1441
1442 req_hdr = mdev->mbase + mbox->rx_start;
1443 if (req_hdr->num_msgs == 0)
1444 return;
1445
1446 offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
1447
1448 for (id = 0; id < req_hdr->num_msgs; id++) {
1449 msg = mdev->mbase + offset;
1450
1451
1452 switch (type) {
1453 case TYPE_AFPF:
1454 msg->pcifunc &=
1455 ~(RVU_PFVF_PF_MASK << RVU_PFVF_PF_SHIFT);
1456 msg->pcifunc |= (devid << RVU_PFVF_PF_SHIFT);
1457 break;
1458 case TYPE_AFVF:
1459 msg->pcifunc &=
1460 ~(RVU_PFVF_FUNC_MASK << RVU_PFVF_FUNC_SHIFT);
1461 msg->pcifunc |= (devid << RVU_PFVF_FUNC_SHIFT) + 1;
1462 break;
1463 }
1464
1465 err = rvu_process_mbox_msg(mbox, devid, msg);
1466 if (!err) {
1467 offset = mbox->rx_start + msg->next_msgoff;
1468 continue;
1469 }
1470
1471 if (msg->pcifunc & RVU_PFVF_FUNC_MASK)
1472 dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d:VF%d\n",
1473 err, otx2_mbox_id2name(msg->id),
1474 msg->id, devid,
1475 (msg->pcifunc & RVU_PFVF_FUNC_MASK) - 1);
1476 else
1477 dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d\n",
1478 err, otx2_mbox_id2name(msg->id),
1479 msg->id, devid);
1480 }
1481
1482
1483 otx2_mbox_msg_send(mbox, devid);
1484 }
1485
1486 static inline void rvu_afpf_mbox_handler(struct work_struct *work)
1487 {
1488 struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1489
1490 __rvu_mbox_handler(mwork, TYPE_AFPF);
1491 }
1492
1493 static inline void rvu_afvf_mbox_handler(struct work_struct *work)
1494 {
1495 struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1496
1497 __rvu_mbox_handler(mwork, TYPE_AFVF);
1498 }
1499
1500 static void __rvu_mbox_up_handler(struct rvu_work *mwork, int type)
1501 {
1502 struct rvu *rvu = mwork->rvu;
1503 struct otx2_mbox_dev *mdev;
1504 struct mbox_hdr *rsp_hdr;
1505 struct mbox_msghdr *msg;
1506 struct mbox_wq_info *mw;
1507 struct otx2_mbox *mbox;
1508 int offset, id, devid;
1509
1510 switch (type) {
1511 case TYPE_AFPF:
1512 mw = &rvu->afpf_wq_info;
1513 break;
1514 case TYPE_AFVF:
1515 mw = &rvu->afvf_wq_info;
1516 break;
1517 default:
1518 return;
1519 }
1520
1521 devid = mwork - mw->mbox_wrk_up;
1522 mbox = &mw->mbox_up;
1523 mdev = &mbox->dev[devid];
1524
1525 rsp_hdr = mdev->mbase + mbox->rx_start;
1526 if (rsp_hdr->num_msgs == 0) {
1527 dev_warn(rvu->dev, "mbox up handler: num_msgs = 0\n");
1528 return;
1529 }
1530
1531 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
1532
1533 for (id = 0; id < rsp_hdr->num_msgs; id++) {
1534 msg = mdev->mbase + offset;
1535
1536 if (msg->id >= MBOX_MSG_MAX) {
1537 dev_err(rvu->dev,
1538 "Mbox msg with unknown ID 0x%x\n", msg->id);
1539 goto end;
1540 }
1541
1542 if (msg->sig != OTX2_MBOX_RSP_SIG) {
1543 dev_err(rvu->dev,
1544 "Mbox msg with wrong signature %x, ID 0x%x\n",
1545 msg->sig, msg->id);
1546 goto end;
1547 }
1548
1549 switch (msg->id) {
1550 case MBOX_MSG_CGX_LINK_EVENT:
1551 break;
1552 default:
1553 if (msg->rc)
1554 dev_err(rvu->dev,
1555 "Mbox msg response has err %d, ID 0x%x\n",
1556 msg->rc, msg->id);
1557 break;
1558 }
1559 end:
1560 offset = mbox->rx_start + msg->next_msgoff;
1561 mdev->msgs_acked++;
1562 }
1563
1564 otx2_mbox_reset(mbox, devid);
1565 }
1566
1567 static inline void rvu_afpf_mbox_up_handler(struct work_struct *work)
1568 {
1569 struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1570
1571 __rvu_mbox_up_handler(mwork, TYPE_AFPF);
1572 }
1573
1574 static inline void rvu_afvf_mbox_up_handler(struct work_struct *work)
1575 {
1576 struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1577
1578 __rvu_mbox_up_handler(mwork, TYPE_AFVF);
1579 }
1580
1581 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
1582 int type, int num,
1583 void (mbox_handler)(struct work_struct *),
1584 void (mbox_up_handler)(struct work_struct *))
1585 {
1586 void __iomem *hwbase = NULL, *reg_base;
1587 int err, i, dir, dir_up;
1588 struct rvu_work *mwork;
1589 const char *name;
1590 u64 bar4_addr;
1591
1592 switch (type) {
1593 case TYPE_AFPF:
1594 name = "rvu_afpf_mailbox";
1595 bar4_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PF_BAR4_ADDR);
1596 dir = MBOX_DIR_AFPF;
1597 dir_up = MBOX_DIR_AFPF_UP;
1598 reg_base = rvu->afreg_base;
1599 break;
1600 case TYPE_AFVF:
1601 name = "rvu_afvf_mailbox";
1602 bar4_addr = rvupf_read64(rvu, RVU_PF_VF_BAR4_ADDR);
1603 dir = MBOX_DIR_PFVF;
1604 dir_up = MBOX_DIR_PFVF_UP;
1605 reg_base = rvu->pfreg_base;
1606 break;
1607 default:
1608 return -EINVAL;
1609 }
1610
1611 mw->mbox_wq = alloc_workqueue(name,
1612 WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,
1613 num);
1614 if (!mw->mbox_wq)
1615 return -ENOMEM;
1616
1617 mw->mbox_wrk = devm_kcalloc(rvu->dev, num,
1618 sizeof(struct rvu_work), GFP_KERNEL);
1619 if (!mw->mbox_wrk) {
1620 err = -ENOMEM;
1621 goto exit;
1622 }
1623
1624 mw->mbox_wrk_up = devm_kcalloc(rvu->dev, num,
1625 sizeof(struct rvu_work), GFP_KERNEL);
1626 if (!mw->mbox_wrk_up) {
1627 err = -ENOMEM;
1628 goto exit;
1629 }
1630
1631
1632
1633
1634
1635 hwbase = ioremap_wc(bar4_addr, MBOX_SIZE * num);
1636 if (!hwbase) {
1637 dev_err(rvu->dev, "Unable to map mailbox region\n");
1638 err = -ENOMEM;
1639 goto exit;
1640 }
1641
1642 err = otx2_mbox_init(&mw->mbox, hwbase, rvu->pdev, reg_base, dir, num);
1643 if (err)
1644 goto exit;
1645
1646 err = otx2_mbox_init(&mw->mbox_up, hwbase, rvu->pdev,
1647 reg_base, dir_up, num);
1648 if (err)
1649 goto exit;
1650
1651 for (i = 0; i < num; i++) {
1652 mwork = &mw->mbox_wrk[i];
1653 mwork->rvu = rvu;
1654 INIT_WORK(&mwork->work, mbox_handler);
1655
1656 mwork = &mw->mbox_wrk_up[i];
1657 mwork->rvu = rvu;
1658 INIT_WORK(&mwork->work, mbox_up_handler);
1659 }
1660
1661 return 0;
1662 exit:
1663 if (hwbase)
1664 iounmap((void __iomem *)hwbase);
1665 destroy_workqueue(mw->mbox_wq);
1666 return err;
1667 }
1668
1669 static void rvu_mbox_destroy(struct mbox_wq_info *mw)
1670 {
1671 if (mw->mbox_wq) {
1672 flush_workqueue(mw->mbox_wq);
1673 destroy_workqueue(mw->mbox_wq);
1674 mw->mbox_wq = NULL;
1675 }
1676
1677 if (mw->mbox.hwbase)
1678 iounmap((void __iomem *)mw->mbox.hwbase);
1679
1680 otx2_mbox_destroy(&mw->mbox);
1681 otx2_mbox_destroy(&mw->mbox_up);
1682 }
1683
1684 static void rvu_queue_work(struct mbox_wq_info *mw, int first,
1685 int mdevs, u64 intr)
1686 {
1687 struct otx2_mbox_dev *mdev;
1688 struct otx2_mbox *mbox;
1689 struct mbox_hdr *hdr;
1690 int i;
1691
1692 for (i = first; i < mdevs; i++) {
1693
1694 if (!(intr & BIT_ULL(i - first)))
1695 continue;
1696
1697 mbox = &mw->mbox;
1698 mdev = &mbox->dev[i];
1699 hdr = mdev->mbase + mbox->rx_start;
1700 if (hdr->num_msgs)
1701 queue_work(mw->mbox_wq, &mw->mbox_wrk[i].work);
1702
1703 mbox = &mw->mbox_up;
1704 mdev = &mbox->dev[i];
1705 hdr = mdev->mbase + mbox->rx_start;
1706 if (hdr->num_msgs)
1707 queue_work(mw->mbox_wq, &mw->mbox_wrk_up[i].work);
1708 }
1709 }
1710
1711 static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
1712 {
1713 struct rvu *rvu = (struct rvu *)rvu_irq;
1714 int vfs = rvu->vfs;
1715 u64 intr;
1716
1717 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT);
1718
1719 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT, intr);
1720
1721
1722 rmb();
1723
1724 rvu_queue_work(&rvu->afpf_wq_info, 0, rvu->hw->total_pfs, intr);
1725
1726
1727 if (vfs > 64) {
1728 intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(1));
1729 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), intr);
1730
1731 rvu_queue_work(&rvu->afvf_wq_info, 64, vfs, intr);
1732 vfs -= 64;
1733 }
1734
1735 intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(0));
1736 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), intr);
1737
1738 rvu_queue_work(&rvu->afvf_wq_info, 0, vfs, intr);
1739
1740 return IRQ_HANDLED;
1741 }
1742
1743 static void rvu_enable_mbox_intr(struct rvu *rvu)
1744 {
1745 struct rvu_hwinfo *hw = rvu->hw;
1746
1747
1748 rvu_write64(rvu, BLKADDR_RVUM,
1749 RVU_AF_PFAF_MBOX_INT, INTR_MASK(hw->total_pfs));
1750
1751
1752 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1S,
1753 INTR_MASK(hw->total_pfs) & ~1ULL);
1754 }
1755
1756 static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr)
1757 {
1758 struct rvu_block *block;
1759 int slot, lf, num_lfs;
1760 int err;
1761
1762 block = &rvu->hw->block[blkaddr];
1763 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
1764 block->type);
1765 if (!num_lfs)
1766 return;
1767 for (slot = 0; slot < num_lfs; slot++) {
1768 lf = rvu_get_lf(rvu, block, pcifunc, slot);
1769 if (lf < 0)
1770 continue;
1771
1772
1773 if (block->addr == BLKADDR_NIX0)
1774 rvu_nix_lf_teardown(rvu, pcifunc, block->addr, lf);
1775 else if (block->addr == BLKADDR_NPA)
1776 rvu_npa_lf_teardown(rvu, pcifunc, lf);
1777
1778 err = rvu_lf_reset(rvu, block, lf);
1779 if (err) {
1780 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
1781 block->addr, lf);
1782 }
1783 }
1784 }
1785
1786 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc)
1787 {
1788 mutex_lock(&rvu->flr_lock);
1789
1790
1791
1792
1793
1794 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX0);
1795 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT0);
1796 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_TIM);
1797 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSOW);
1798 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSO);
1799 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NPA);
1800 rvu_detach_rsrcs(rvu, NULL, pcifunc);
1801 mutex_unlock(&rvu->flr_lock);
1802 }
1803
1804 static void rvu_afvf_flr_handler(struct rvu *rvu, int vf)
1805 {
1806 int reg = 0;
1807
1808
1809 __rvu_flr_handler(rvu, vf + 1);
1810
1811 if (vf >= 64) {
1812 reg = 1;
1813 vf = vf - 64;
1814 }
1815
1816
1817 rvupf_write64(rvu, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
1818 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
1819 }
1820
1821 static void rvu_flr_handler(struct work_struct *work)
1822 {
1823 struct rvu_work *flrwork = container_of(work, struct rvu_work, work);
1824 struct rvu *rvu = flrwork->rvu;
1825 u16 pcifunc, numvfs, vf;
1826 u64 cfg;
1827 int pf;
1828
1829 pf = flrwork - rvu->flr_wrk;
1830 if (pf >= rvu->hw->total_pfs) {
1831 rvu_afvf_flr_handler(rvu, pf - rvu->hw->total_pfs);
1832 return;
1833 }
1834
1835 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
1836 numvfs = (cfg >> 12) & 0xFF;
1837 pcifunc = pf << RVU_PFVF_PF_SHIFT;
1838
1839 for (vf = 0; vf < numvfs; vf++)
1840 __rvu_flr_handler(rvu, (pcifunc | (vf + 1)));
1841
1842 __rvu_flr_handler(rvu, pcifunc);
1843
1844
1845 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND, BIT_ULL(pf));
1846
1847
1848 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S, BIT_ULL(pf));
1849 }
1850
1851 static void rvu_afvf_queue_flr_work(struct rvu *rvu, int start_vf, int numvfs)
1852 {
1853 int dev, vf, reg = 0;
1854 u64 intr;
1855
1856 if (start_vf >= 64)
1857 reg = 1;
1858
1859 intr = rvupf_read64(rvu, RVU_PF_VFFLR_INTX(reg));
1860 if (!intr)
1861 return;
1862
1863 for (vf = 0; vf < numvfs; vf++) {
1864 if (!(intr & BIT_ULL(vf)))
1865 continue;
1866 dev = vf + start_vf + rvu->hw->total_pfs;
1867 queue_work(rvu->flr_wq, &rvu->flr_wrk[dev].work);
1868
1869 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
1870 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(reg), BIT_ULL(vf));
1871 }
1872 }
1873
1874 static irqreturn_t rvu_flr_intr_handler(int irq, void *rvu_irq)
1875 {
1876 struct rvu *rvu = (struct rvu *)rvu_irq;
1877 u64 intr;
1878 u8 pf;
1879
1880 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT);
1881 if (!intr)
1882 goto afvf_flr;
1883
1884 for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
1885 if (intr & (1ULL << pf)) {
1886
1887 queue_work(rvu->flr_wq, &rvu->flr_wrk[pf].work);
1888
1889 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT,
1890 BIT_ULL(pf));
1891
1892 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
1893 BIT_ULL(pf));
1894 }
1895 }
1896
1897 afvf_flr:
1898 rvu_afvf_queue_flr_work(rvu, 0, 64);
1899 if (rvu->vfs > 64)
1900 rvu_afvf_queue_flr_work(rvu, 64, rvu->vfs - 64);
1901
1902 return IRQ_HANDLED;
1903 }
1904
1905 static void rvu_me_handle_vfset(struct rvu *rvu, int idx, u64 intr)
1906 {
1907 int vf;
1908
1909
1910
1911
1912 for (vf = 0; vf < 64; vf++) {
1913 if (intr & (1ULL << vf)) {
1914
1915 rvupf_write64(rvu, RVU_PF_VFTRPENDX(idx), BIT_ULL(vf));
1916
1917 rvupf_write64(rvu, RVU_PF_VFME_INTX(idx), BIT_ULL(vf));
1918 }
1919 }
1920 }
1921
1922
1923 static irqreturn_t rvu_me_vf_intr_handler(int irq, void *rvu_irq)
1924 {
1925 struct rvu *rvu = (struct rvu *)rvu_irq;
1926 int vfset;
1927 u64 intr;
1928
1929 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
1930
1931 for (vfset = 0; vfset <= 1; vfset++) {
1932 intr = rvupf_read64(rvu, RVU_PF_VFME_INTX(vfset));
1933 if (intr)
1934 rvu_me_handle_vfset(rvu, vfset, intr);
1935 }
1936
1937 return IRQ_HANDLED;
1938 }
1939
1940
1941 static irqreturn_t rvu_me_pf_intr_handler(int irq, void *rvu_irq)
1942 {
1943 struct rvu *rvu = (struct rvu *)rvu_irq;
1944 u64 intr;
1945 u8 pf;
1946
1947 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
1948
1949
1950
1951
1952 for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
1953 if (intr & (1ULL << pf)) {
1954
1955 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND,
1956 BIT_ULL(pf));
1957
1958 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT,
1959 BIT_ULL(pf));
1960 }
1961 }
1962
1963 return IRQ_HANDLED;
1964 }
1965
1966 static void rvu_unregister_interrupts(struct rvu *rvu)
1967 {
1968 int irq;
1969
1970
1971 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1C,
1972 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
1973
1974
1975 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
1976 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
1977
1978
1979 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1C,
1980 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
1981
1982 for (irq = 0; irq < rvu->num_vec; irq++) {
1983 if (rvu->irq_allocated[irq])
1984 free_irq(pci_irq_vector(rvu->pdev, irq), rvu);
1985 }
1986
1987 pci_free_irq_vectors(rvu->pdev);
1988 rvu->num_vec = 0;
1989 }
1990
1991 static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu)
1992 {
1993 struct rvu_pfvf *pfvf = &rvu->pf[0];
1994 int offset;
1995
1996 pfvf = &rvu->pf[0];
1997 offset = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
1998
1999
2000
2001
2002
2003 return (pfvf->msix.max >= RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT) &&
2004 offset;
2005 }
2006
2007 static int rvu_register_interrupts(struct rvu *rvu)
2008 {
2009 int ret, offset, pf_vec_start;
2010
2011 rvu->num_vec = pci_msix_vec_count(rvu->pdev);
2012
2013 rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec,
2014 NAME_SIZE, GFP_KERNEL);
2015 if (!rvu->irq_name)
2016 return -ENOMEM;
2017
2018 rvu->irq_allocated = devm_kcalloc(rvu->dev, rvu->num_vec,
2019 sizeof(bool), GFP_KERNEL);
2020 if (!rvu->irq_allocated)
2021 return -ENOMEM;
2022
2023
2024 ret = pci_alloc_irq_vectors(rvu->pdev, rvu->num_vec,
2025 rvu->num_vec, PCI_IRQ_MSIX);
2026 if (ret < 0) {
2027 dev_err(rvu->dev,
2028 "RVUAF: Request for %d msix vectors failed, ret %d\n",
2029 rvu->num_vec, ret);
2030 return ret;
2031 }
2032
2033
2034 sprintf(&rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], "RVUAF Mbox");
2035 ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_MBOX),
2036 rvu_mbox_intr_handler, 0,
2037 &rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], rvu);
2038 if (ret) {
2039 dev_err(rvu->dev,
2040 "RVUAF: IRQ registration failed for mbox irq\n");
2041 goto fail;
2042 }
2043
2044 rvu->irq_allocated[RVU_AF_INT_VEC_MBOX] = true;
2045
2046
2047 rvu_enable_mbox_intr(rvu);
2048
2049
2050 sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
2051 "RVUAF FLR");
2052 ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFFLR),
2053 rvu_flr_intr_handler, 0,
2054 &rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
2055 rvu);
2056 if (ret) {
2057 dev_err(rvu->dev,
2058 "RVUAF: IRQ registration failed for FLR\n");
2059 goto fail;
2060 }
2061 rvu->irq_allocated[RVU_AF_INT_VEC_PFFLR] = true;
2062
2063
2064 rvu_write64(rvu, BLKADDR_RVUM,
2065 RVU_AF_PFFLR_INT, INTR_MASK(rvu->hw->total_pfs));
2066
2067 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,
2068 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2069
2070
2071 sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
2072 "RVUAF ME");
2073 ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFME),
2074 rvu_me_pf_intr_handler, 0,
2075 &rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
2076 rvu);
2077 if (ret) {
2078 dev_err(rvu->dev,
2079 "RVUAF: IRQ registration failed for ME\n");
2080 }
2081 rvu->irq_allocated[RVU_AF_INT_VEC_PFME] = true;
2082
2083
2084 rvu_write64(rvu, BLKADDR_RVUM,
2085 RVU_AF_PFME_INT, INTR_MASK(rvu->hw->total_pfs));
2086
2087 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1S,
2088 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2089
2090 if (!rvu_afvf_msix_vectors_num_ok(rvu))
2091 return 0;
2092
2093
2094 pf_vec_start = rvu_read64(rvu, BLKADDR_RVUM,
2095 RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
2096
2097
2098 offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX0;
2099 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox0");
2100 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2101 rvu_mbox_intr_handler, 0,
2102 &rvu->irq_name[offset * NAME_SIZE],
2103 rvu);
2104 if (ret)
2105 dev_err(rvu->dev,
2106 "RVUAF: IRQ registration failed for Mbox0\n");
2107
2108 rvu->irq_allocated[offset] = true;
2109
2110
2111
2112
2113 offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX1;
2114 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox1");
2115 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2116 rvu_mbox_intr_handler, 0,
2117 &rvu->irq_name[offset * NAME_SIZE],
2118 rvu);
2119 if (ret)
2120 dev_err(rvu->dev,
2121 "RVUAF: IRQ registration failed for Mbox1\n");
2122
2123 rvu->irq_allocated[offset] = true;
2124
2125
2126 offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR0;
2127 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR0");
2128 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2129 rvu_flr_intr_handler, 0,
2130 &rvu->irq_name[offset * NAME_SIZE], rvu);
2131 if (ret) {
2132 dev_err(rvu->dev,
2133 "RVUAF: IRQ registration failed for RVUAFVF FLR0\n");
2134 goto fail;
2135 }
2136 rvu->irq_allocated[offset] = true;
2137
2138 offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR1;
2139 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR1");
2140 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2141 rvu_flr_intr_handler, 0,
2142 &rvu->irq_name[offset * NAME_SIZE], rvu);
2143 if (ret) {
2144 dev_err(rvu->dev,
2145 "RVUAF: IRQ registration failed for RVUAFVF FLR1\n");
2146 goto fail;
2147 }
2148 rvu->irq_allocated[offset] = true;
2149
2150
2151 offset = pf_vec_start + RVU_PF_INT_VEC_VFME0;
2152 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME0");
2153 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2154 rvu_me_vf_intr_handler, 0,
2155 &rvu->irq_name[offset * NAME_SIZE], rvu);
2156 if (ret) {
2157 dev_err(rvu->dev,
2158 "RVUAF: IRQ registration failed for RVUAFVF ME0\n");
2159 goto fail;
2160 }
2161 rvu->irq_allocated[offset] = true;
2162
2163 offset = pf_vec_start + RVU_PF_INT_VEC_VFME1;
2164 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME1");
2165 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2166 rvu_me_vf_intr_handler, 0,
2167 &rvu->irq_name[offset * NAME_SIZE], rvu);
2168 if (ret) {
2169 dev_err(rvu->dev,
2170 "RVUAF: IRQ registration failed for RVUAFVF ME1\n");
2171 goto fail;
2172 }
2173 rvu->irq_allocated[offset] = true;
2174 return 0;
2175
2176 fail:
2177 rvu_unregister_interrupts(rvu);
2178 return ret;
2179 }
2180
2181 static void rvu_flr_wq_destroy(struct rvu *rvu)
2182 {
2183 if (rvu->flr_wq) {
2184 flush_workqueue(rvu->flr_wq);
2185 destroy_workqueue(rvu->flr_wq);
2186 rvu->flr_wq = NULL;
2187 }
2188 }
2189
2190 static int rvu_flr_init(struct rvu *rvu)
2191 {
2192 int dev, num_devs;
2193 u64 cfg;
2194 int pf;
2195
2196
2197 for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
2198 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
2199 rvu_write64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf),
2200 cfg | BIT_ULL(22));
2201 }
2202
2203 rvu->flr_wq = alloc_workqueue("rvu_afpf_flr",
2204 WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,
2205 1);
2206 if (!rvu->flr_wq)
2207 return -ENOMEM;
2208
2209 num_devs = rvu->hw->total_pfs + pci_sriov_get_totalvfs(rvu->pdev);
2210 rvu->flr_wrk = devm_kcalloc(rvu->dev, num_devs,
2211 sizeof(struct rvu_work), GFP_KERNEL);
2212 if (!rvu->flr_wrk) {
2213 destroy_workqueue(rvu->flr_wq);
2214 return -ENOMEM;
2215 }
2216
2217 for (dev = 0; dev < num_devs; dev++) {
2218 rvu->flr_wrk[dev].rvu = rvu;
2219 INIT_WORK(&rvu->flr_wrk[dev].work, rvu_flr_handler);
2220 }
2221
2222 mutex_init(&rvu->flr_lock);
2223
2224 return 0;
2225 }
2226
2227 static void rvu_disable_afvf_intr(struct rvu *rvu)
2228 {
2229 int vfs = rvu->vfs;
2230
2231 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), INTR_MASK(vfs));
2232 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
2233 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
2234 if (vfs <= 64)
2235 return;
2236
2237 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1),
2238 INTR_MASK(vfs - 64));
2239 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
2240 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
2241 }
2242
2243 static void rvu_enable_afvf_intr(struct rvu *rvu)
2244 {
2245 int vfs = rvu->vfs;
2246
2247
2248
2249
2250
2251 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), INTR_MASK(vfs));
2252 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(vfs));
2253
2254
2255 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(0), INTR_MASK(vfs));
2256 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(vfs));
2257 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(vfs));
2258
2259
2260 if (vfs <= 64)
2261 return;
2262
2263 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), INTR_MASK(vfs - 64));
2264 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
2265 INTR_MASK(vfs - 64));
2266
2267 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(1), INTR_MASK(vfs - 64));
2268 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
2269 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
2270 }
2271
2272 #define PCI_DEVID_OCTEONTX2_LBK 0xA061
2273
2274 static int lbk_get_num_chans(void)
2275 {
2276 struct pci_dev *pdev;
2277 void __iomem *base;
2278 int ret = -EIO;
2279
2280 pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_LBK,
2281 NULL);
2282 if (!pdev)
2283 goto err;
2284
2285 base = pci_ioremap_bar(pdev, 0);
2286 if (!base)
2287 goto err_put;
2288
2289
2290 ret = (readq(base + 0x10) >> 32) & 0xffff;
2291 iounmap(base);
2292 err_put:
2293 pci_dev_put(pdev);
2294 err:
2295 return ret;
2296 }
2297
2298 static int rvu_enable_sriov(struct rvu *rvu)
2299 {
2300 struct pci_dev *pdev = rvu->pdev;
2301 int err, chans, vfs;
2302
2303 if (!rvu_afvf_msix_vectors_num_ok(rvu)) {
2304 dev_warn(&pdev->dev,
2305 "Skipping SRIOV enablement since not enough IRQs are available\n");
2306 return 0;
2307 }
2308
2309 chans = lbk_get_num_chans();
2310 if (chans < 0)
2311 return chans;
2312
2313 vfs = pci_sriov_get_totalvfs(pdev);
2314
2315
2316 if (vfs > chans)
2317 vfs = chans;
2318
2319
2320
2321
2322
2323
2324 if (vfs & 0x1) {
2325 dev_warn(&pdev->dev,
2326 "Number of VFs should be even. Enabling %d out of %d.\n",
2327 vfs - 1, vfs);
2328 vfs--;
2329 }
2330
2331 if (!vfs)
2332 return 0;
2333
2334
2335
2336
2337
2338 rvu->vfs = vfs;
2339
2340 err = rvu_mbox_init(rvu, &rvu->afvf_wq_info, TYPE_AFVF, vfs,
2341 rvu_afvf_mbox_handler, rvu_afvf_mbox_up_handler);
2342 if (err)
2343 return err;
2344
2345 rvu_enable_afvf_intr(rvu);
2346
2347 mb();
2348
2349 err = pci_enable_sriov(pdev, vfs);
2350 if (err) {
2351 rvu_disable_afvf_intr(rvu);
2352 rvu_mbox_destroy(&rvu->afvf_wq_info);
2353 return err;
2354 }
2355
2356 return 0;
2357 }
2358
2359 static void rvu_disable_sriov(struct rvu *rvu)
2360 {
2361 rvu_disable_afvf_intr(rvu);
2362 rvu_mbox_destroy(&rvu->afvf_wq_info);
2363 pci_disable_sriov(rvu->pdev);
2364 }
2365
2366 static void rvu_update_module_params(struct rvu *rvu)
2367 {
2368 const char *default_pfl_name = "default";
2369
2370 strscpy(rvu->mkex_pfl_name,
2371 mkex_profile ? mkex_profile : default_pfl_name, MKEX_NAME_LEN);
2372 }
2373
2374 static int rvu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2375 {
2376 struct device *dev = &pdev->dev;
2377 struct rvu *rvu;
2378 int err;
2379
2380 rvu = devm_kzalloc(dev, sizeof(*rvu), GFP_KERNEL);
2381 if (!rvu)
2382 return -ENOMEM;
2383
2384 rvu->hw = devm_kzalloc(dev, sizeof(struct rvu_hwinfo), GFP_KERNEL);
2385 if (!rvu->hw) {
2386 devm_kfree(dev, rvu);
2387 return -ENOMEM;
2388 }
2389
2390 pci_set_drvdata(pdev, rvu);
2391 rvu->pdev = pdev;
2392 rvu->dev = &pdev->dev;
2393
2394 err = pci_enable_device(pdev);
2395 if (err) {
2396 dev_err(dev, "Failed to enable PCI device\n");
2397 goto err_freemem;
2398 }
2399
2400 err = pci_request_regions(pdev, DRV_NAME);
2401 if (err) {
2402 dev_err(dev, "PCI request regions failed 0x%x\n", err);
2403 goto err_disable_device;
2404 }
2405
2406 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
2407 if (err) {
2408 dev_err(dev, "Unable to set DMA mask\n");
2409 goto err_release_regions;
2410 }
2411
2412 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
2413 if (err) {
2414 dev_err(dev, "Unable to set consistent DMA mask\n");
2415 goto err_release_regions;
2416 }
2417
2418
2419 rvu->afreg_base = pcim_iomap(pdev, PCI_AF_REG_BAR_NUM, 0);
2420 rvu->pfreg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0);
2421 if (!rvu->afreg_base || !rvu->pfreg_base) {
2422 dev_err(dev, "Unable to map admin function CSRs, aborting\n");
2423 err = -ENOMEM;
2424 goto err_release_regions;
2425 }
2426
2427
2428 rvu_update_module_params(rvu);
2429
2430
2431 rvu_check_block_implemented(rvu);
2432
2433 rvu_reset_all_blocks(rvu);
2434
2435 err = rvu_setup_hw_resources(rvu);
2436 if (err)
2437 goto err_release_regions;
2438
2439
2440 err = rvu_mbox_init(rvu, &rvu->afpf_wq_info, TYPE_AFPF,
2441 rvu->hw->total_pfs, rvu_afpf_mbox_handler,
2442 rvu_afpf_mbox_up_handler);
2443 if (err)
2444 goto err_hwsetup;
2445
2446 err = rvu_flr_init(rvu);
2447 if (err)
2448 goto err_mbox;
2449
2450 err = rvu_register_interrupts(rvu);
2451 if (err)
2452 goto err_flr;
2453
2454
2455 err = rvu_enable_sriov(rvu);
2456 if (err)
2457 goto err_irq;
2458
2459 return 0;
2460 err_irq:
2461 rvu_unregister_interrupts(rvu);
2462 err_flr:
2463 rvu_flr_wq_destroy(rvu);
2464 err_mbox:
2465 rvu_mbox_destroy(&rvu->afpf_wq_info);
2466 err_hwsetup:
2467 rvu_cgx_exit(rvu);
2468 rvu_reset_all_blocks(rvu);
2469 rvu_free_hw_resources(rvu);
2470 err_release_regions:
2471 pci_release_regions(pdev);
2472 err_disable_device:
2473 pci_disable_device(pdev);
2474 err_freemem:
2475 pci_set_drvdata(pdev, NULL);
2476 devm_kfree(&pdev->dev, rvu->hw);
2477 devm_kfree(dev, rvu);
2478 return err;
2479 }
2480
2481 static void rvu_remove(struct pci_dev *pdev)
2482 {
2483 struct rvu *rvu = pci_get_drvdata(pdev);
2484
2485 rvu_unregister_interrupts(rvu);
2486 rvu_flr_wq_destroy(rvu);
2487 rvu_cgx_exit(rvu);
2488 rvu_mbox_destroy(&rvu->afpf_wq_info);
2489 rvu_disable_sriov(rvu);
2490 rvu_reset_all_blocks(rvu);
2491 rvu_free_hw_resources(rvu);
2492
2493 pci_release_regions(pdev);
2494 pci_disable_device(pdev);
2495 pci_set_drvdata(pdev, NULL);
2496
2497 devm_kfree(&pdev->dev, rvu->hw);
2498 devm_kfree(&pdev->dev, rvu);
2499 }
2500
2501 static struct pci_driver rvu_driver = {
2502 .name = DRV_NAME,
2503 .id_table = rvu_id_table,
2504 .probe = rvu_probe,
2505 .remove = rvu_remove,
2506 };
2507
2508 static int __init rvu_init_module(void)
2509 {
2510 int err;
2511
2512 pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
2513
2514 err = pci_register_driver(&cgx_driver);
2515 if (err < 0)
2516 return err;
2517
2518 err = pci_register_driver(&rvu_driver);
2519 if (err < 0)
2520 pci_unregister_driver(&cgx_driver);
2521
2522 return err;
2523 }
2524
2525 static void __exit rvu_cleanup_module(void)
2526 {
2527 pci_unregister_driver(&rvu_driver);
2528 pci_unregister_driver(&cgx_driver);
2529 }
2530
2531 module_init(rvu_init_module);
2532 module_exit(rvu_cleanup_module);