This source file includes following definitions.
- nfp_map_ptr_record
- nfp_map_ptrs_forget
- nfp_map_ptrs_record
- nfp_prog_prepare
- nfp_prog_free
- nfp_bpf_verifier_prep
- nfp_bpf_translate
- nfp_bpf_destroy
- nfp_map_bpf_byte_swap
- nfp_map_bpf_byte_swap_record
- nfp_bpf_map_lookup_entry
- nfp_bpf_map_update_entry
- nfp_bpf_map_get_next_key
- nfp_bpf_map_delete_elem
- nfp_bpf_map_alloc
- nfp_bpf_map_free
- nfp_ndo_bpf
- nfp_bpf_perf_event_copy
- nfp_bpf_event_output
- nfp_net_bpf_load
- nfp_net_bpf_start
- nfp_net_bpf_stop
- nfp_net_bpf_offload
1
2
3
4
5
6
7
8
9 #define pr_fmt(fmt) "NFP net bpf: " fmt
10
11 #include <linux/bpf.h>
12 #include <linux/kernel.h>
13 #include <linux/netdevice.h>
14 #include <linux/pci.h>
15 #include <linux/jiffies.h>
16 #include <linux/timer.h>
17 #include <linux/list.h>
18 #include <linux/mm.h>
19
20 #include <net/pkt_cls.h>
21 #include <net/tc_act/tc_gact.h>
22 #include <net/tc_act/tc_mirred.h>
23
24 #include "main.h"
25 #include "../ccm.h"
26 #include "../nfp_app.h"
27 #include "../nfp_net_ctrl.h"
28 #include "../nfp_net.h"
29
30 static int
31 nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
32 struct bpf_map *map)
33 {
34 struct nfp_bpf_neutral_map *record;
35 int err;
36
37
38 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map->id,
39 nfp_bpf_maps_neutral_params);
40 if (record) {
41 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
42 record->count++;
43 return 0;
44 }
45
46
47
48
49 map = bpf_map_inc(map, false);
50 if (IS_ERR(map))
51 return PTR_ERR(map);
52
53 record = kmalloc(sizeof(*record), GFP_KERNEL);
54 if (!record) {
55 err = -ENOMEM;
56 goto err_map_put;
57 }
58
59 record->ptr = map;
60 record->map_id = map->id;
61 record->count = 1;
62
63 err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l,
64 nfp_bpf_maps_neutral_params);
65 if (err)
66 goto err_free_rec;
67
68 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
69
70 return 0;
71
72 err_free_rec:
73 kfree(record);
74 err_map_put:
75 bpf_map_put(map);
76 return err;
77 }
78
79 static void
80 nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog)
81 {
82 bool freed = false;
83 int i;
84
85 for (i = 0; i < nfp_prog->map_records_cnt; i++) {
86 if (--nfp_prog->map_records[i]->count) {
87 nfp_prog->map_records[i] = NULL;
88 continue;
89 }
90
91 WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral,
92 &nfp_prog->map_records[i]->l,
93 nfp_bpf_maps_neutral_params));
94 freed = true;
95 }
96
97 if (freed) {
98 synchronize_rcu();
99
100 for (i = 0; i < nfp_prog->map_records_cnt; i++)
101 if (nfp_prog->map_records[i]) {
102 bpf_map_put(nfp_prog->map_records[i]->ptr);
103 kfree(nfp_prog->map_records[i]);
104 }
105 }
106
107 kfree(nfp_prog->map_records);
108 nfp_prog->map_records = NULL;
109 nfp_prog->map_records_cnt = 0;
110 }
111
112 static int
113 nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
114 struct bpf_prog *prog)
115 {
116 int i, cnt, err;
117
118
119 cnt = 0;
120 for (i = 0; i < prog->aux->used_map_cnt; i++)
121 if (bpf_map_offload_neutral(prog->aux->used_maps[i]))
122 cnt++;
123 if (!cnt)
124 return 0;
125
126 nfp_prog->map_records = kmalloc_array(cnt,
127 sizeof(nfp_prog->map_records[0]),
128 GFP_KERNEL);
129 if (!nfp_prog->map_records)
130 return -ENOMEM;
131
132 for (i = 0; i < prog->aux->used_map_cnt; i++)
133 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) {
134 err = nfp_map_ptr_record(bpf, nfp_prog,
135 prog->aux->used_maps[i]);
136 if (err) {
137 nfp_map_ptrs_forget(bpf, nfp_prog);
138 return err;
139 }
140 }
141 WARN_ON(cnt != nfp_prog->map_records_cnt);
142
143 return 0;
144 }
145
146 static int
147 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
148 unsigned int cnt)
149 {
150 struct nfp_insn_meta *meta;
151 unsigned int i;
152
153 for (i = 0; i < cnt; i++) {
154 meta = kzalloc(sizeof(*meta), GFP_KERNEL);
155 if (!meta)
156 return -ENOMEM;
157
158 meta->insn = prog[i];
159 meta->n = i;
160 if (is_mbpf_alu(meta)) {
161 meta->umin_src = U64_MAX;
162 meta->umin_dst = U64_MAX;
163 }
164
165 list_add_tail(&meta->l, &nfp_prog->insns);
166 }
167 nfp_prog->n_insns = cnt;
168
169 nfp_bpf_jit_prepare(nfp_prog);
170
171 return 0;
172 }
173
174 static void nfp_prog_free(struct nfp_prog *nfp_prog)
175 {
176 struct nfp_insn_meta *meta, *tmp;
177
178 kfree(nfp_prog->subprog);
179
180 list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
181 list_del(&meta->l);
182 kfree(meta);
183 }
184 kfree(nfp_prog);
185 }
186
187 static int nfp_bpf_verifier_prep(struct bpf_prog *prog)
188 {
189 struct nfp_prog *nfp_prog;
190 int ret;
191
192 nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
193 if (!nfp_prog)
194 return -ENOMEM;
195 prog->aux->offload->dev_priv = nfp_prog;
196
197 INIT_LIST_HEAD(&nfp_prog->insns);
198 nfp_prog->type = prog->type;
199 nfp_prog->bpf = bpf_offload_dev_priv(prog->aux->offload->offdev);
200
201 ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
202 if (ret)
203 goto err_free;
204
205 nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
206
207 return 0;
208
209 err_free:
210 nfp_prog_free(nfp_prog);
211
212 return ret;
213 }
214
215 static int nfp_bpf_translate(struct bpf_prog *prog)
216 {
217 struct nfp_net *nn = netdev_priv(prog->aux->offload->netdev);
218 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
219 unsigned int max_instr;
220 int err;
221
222
223 if (prog->aux->offload->opt_failed)
224 return -EINVAL;
225
226 max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
227 nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
228
229 nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
230 if (!nfp_prog->prog)
231 return -ENOMEM;
232
233 err = nfp_bpf_jit(nfp_prog);
234 if (err)
235 return err;
236
237 prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
238 prog->aux->offload->jited_image = nfp_prog->prog;
239
240 return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog);
241 }
242
243 static void nfp_bpf_destroy(struct bpf_prog *prog)
244 {
245 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
246
247 kvfree(nfp_prog->prog);
248 nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog);
249 nfp_prog_free(nfp_prog);
250 }
251
252
253
254
255 static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value)
256 {
257 u32 *word = value;
258 unsigned int i;
259
260 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
261 if (nfp_map->use_map[i].type == NFP_MAP_USE_ATOMIC_CNT)
262 word[i] = (__force u32)cpu_to_be32(word[i]);
263 }
264
265
266
267
268 static void
269 nfp_map_bpf_byte_swap_record(struct nfp_bpf_map *nfp_map, void *value)
270 {
271 u32 *word = value;
272 unsigned int i;
273
274 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
275 if (nfp_map->use_map[i].type == NFP_MAP_UNUSED &&
276 word[i] != (__force u32)cpu_to_be32(word[i]))
277 nfp_map->use_map[i].non_zero_update = 1;
278 }
279
280 static int
281 nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap,
282 void *key, void *value)
283 {
284 int err;
285
286 err = nfp_bpf_ctrl_lookup_entry(offmap, key, value);
287 if (err)
288 return err;
289
290 nfp_map_bpf_byte_swap(offmap->dev_priv, value);
291 return 0;
292 }
293
294 static int
295 nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap,
296 void *key, void *value, u64 flags)
297 {
298 nfp_map_bpf_byte_swap(offmap->dev_priv, value);
299 nfp_map_bpf_byte_swap_record(offmap->dev_priv, value);
300 return nfp_bpf_ctrl_update_entry(offmap, key, value, flags);
301 }
302
303 static int
304 nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
305 void *key, void *next_key)
306 {
307 if (!key)
308 return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
309 return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
310 }
311
312 static int
313 nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
314 {
315 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
316 return -EINVAL;
317 return nfp_bpf_ctrl_del_entry(offmap, key);
318 }
319
320 static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
321 .map_get_next_key = nfp_bpf_map_get_next_key,
322 .map_lookup_elem = nfp_bpf_map_lookup_entry,
323 .map_update_elem = nfp_bpf_map_update_entry,
324 .map_delete_elem = nfp_bpf_map_delete_elem,
325 };
326
327 static int
328 nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
329 {
330 struct nfp_bpf_map *nfp_map;
331 unsigned int use_map_size;
332 long long int res;
333
334 if (!bpf->maps.types)
335 return -EOPNOTSUPP;
336
337 if (offmap->map.map_flags ||
338 offmap->map.numa_node != NUMA_NO_NODE) {
339 pr_info("map flags are not supported\n");
340 return -EINVAL;
341 }
342
343 if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
344 pr_info("map type not supported\n");
345 return -EOPNOTSUPP;
346 }
347 if (bpf->maps.max_maps == bpf->maps_in_use) {
348 pr_info("too many maps for a device\n");
349 return -ENOMEM;
350 }
351 if (bpf->maps.max_elems - bpf->map_elems_in_use <
352 offmap->map.max_entries) {
353 pr_info("map with too many elements: %u, left: %u\n",
354 offmap->map.max_entries,
355 bpf->maps.max_elems - bpf->map_elems_in_use);
356 return -ENOMEM;
357 }
358
359 if (round_up(offmap->map.key_size, 8) +
360 round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
361 pr_info("map elements too large: %u, FW max element size (key+value): %u\n",
362 round_up(offmap->map.key_size, 8) +
363 round_up(offmap->map.value_size, 8),
364 bpf->maps.max_elem_sz);
365 return -ENOMEM;
366 }
367 if (offmap->map.key_size > bpf->maps.max_key_sz) {
368 pr_info("map key size %u, FW max is %u\n",
369 offmap->map.key_size, bpf->maps.max_key_sz);
370 return -ENOMEM;
371 }
372 if (offmap->map.value_size > bpf->maps.max_val_sz) {
373 pr_info("map value size %u, FW max is %u\n",
374 offmap->map.value_size, bpf->maps.max_val_sz);
375 return -ENOMEM;
376 }
377
378 use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) *
379 FIELD_SIZEOF(struct nfp_bpf_map, use_map[0]);
380
381 nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER);
382 if (!nfp_map)
383 return -ENOMEM;
384
385 offmap->dev_priv = nfp_map;
386 nfp_map->offmap = offmap;
387 nfp_map->bpf = bpf;
388 spin_lock_init(&nfp_map->cache_lock);
389
390 res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
391 if (res < 0) {
392 kfree(nfp_map);
393 return res;
394 }
395
396 nfp_map->tid = res;
397 offmap->dev_ops = &nfp_bpf_map_ops;
398 bpf->maps_in_use++;
399 bpf->map_elems_in_use += offmap->map.max_entries;
400 list_add_tail(&nfp_map->l, &bpf->map_list);
401
402 return 0;
403 }
404
405 static int
406 nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
407 {
408 struct nfp_bpf_map *nfp_map = offmap->dev_priv;
409
410 nfp_bpf_ctrl_free_map(bpf, nfp_map);
411 dev_consume_skb_any(nfp_map->cache);
412 WARN_ON_ONCE(nfp_map->cache_blockers);
413 list_del_init(&nfp_map->l);
414 bpf->map_elems_in_use -= offmap->map.max_entries;
415 bpf->maps_in_use--;
416 kfree(nfp_map);
417
418 return 0;
419 }
420
421 int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
422 {
423 switch (bpf->command) {
424 case BPF_OFFLOAD_MAP_ALLOC:
425 return nfp_bpf_map_alloc(app->priv, bpf->offmap);
426 case BPF_OFFLOAD_MAP_FREE:
427 return nfp_bpf_map_free(app->priv, bpf->offmap);
428 default:
429 return -EINVAL;
430 }
431 }
432
433 static unsigned long
434 nfp_bpf_perf_event_copy(void *dst, const void *src,
435 unsigned long off, unsigned long len)
436 {
437 memcpy(dst, src + off, len);
438 return 0;
439 }
440
441 int nfp_bpf_event_output(struct nfp_app_bpf *bpf, const void *data,
442 unsigned int len)
443 {
444 struct cmsg_bpf_event *cbe = (void *)data;
445 struct nfp_bpf_neutral_map *record;
446 u32 pkt_size, data_size, map_id;
447 u64 map_id_full;
448
449 if (len < sizeof(struct cmsg_bpf_event))
450 return -EINVAL;
451
452 pkt_size = be32_to_cpu(cbe->pkt_size);
453 data_size = be32_to_cpu(cbe->data_size);
454 map_id_full = be64_to_cpu(cbe->map_ptr);
455 map_id = map_id_full;
456
457 if (len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size)
458 return -EINVAL;
459 if (cbe->hdr.ver != NFP_CCM_ABI_VERSION)
460 return -EINVAL;
461
462 rcu_read_lock();
463 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map_id,
464 nfp_bpf_maps_neutral_params);
465 if (!record || map_id_full > U32_MAX) {
466 rcu_read_unlock();
467 cmsg_warn(bpf, "perf event: map id %lld (0x%llx) not recognized, dropping event\n",
468 map_id_full, map_id_full);
469 return -EINVAL;
470 }
471
472 bpf_event_output(record->ptr, be32_to_cpu(cbe->cpu_id),
473 &cbe->data[round_up(pkt_size, 4)], data_size,
474 cbe->data, pkt_size, nfp_bpf_perf_event_copy);
475 rcu_read_unlock();
476
477 return 0;
478 }
479
480 static int
481 nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog,
482 struct netlink_ext_ack *extack)
483 {
484 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
485 unsigned int fw_mtu, pkt_off, max_stack, max_prog_len;
486 dma_addr_t dma_addr;
487 void *img;
488 int err;
489
490 fw_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
491 pkt_off = min(prog->aux->max_pkt_offset, nn->dp.netdev->mtu);
492 if (fw_mtu < pkt_off) {
493 NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with potential packet access beyond HW packet split boundary");
494 return -EOPNOTSUPP;
495 }
496
497 max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
498 if (nfp_prog->stack_size > max_stack) {
499 NL_SET_ERR_MSG_MOD(extack, "stack too large");
500 return -EOPNOTSUPP;
501 }
502
503 max_prog_len = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
504 if (nfp_prog->prog_len > max_prog_len) {
505 NL_SET_ERR_MSG_MOD(extack, "program too long");
506 return -EOPNOTSUPP;
507 }
508
509 img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
510 if (IS_ERR(img))
511 return PTR_ERR(img);
512
513 dma_addr = dma_map_single(nn->dp.dev, img,
514 nfp_prog->prog_len * sizeof(u64),
515 DMA_TO_DEVICE);
516 if (dma_mapping_error(nn->dp.dev, dma_addr)) {
517 kfree(img);
518 return -ENOMEM;
519 }
520
521 nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
522 nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
523
524
525 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
526 if (err)
527 NL_SET_ERR_MSG_MOD(extack,
528 "FW command error while loading BPF");
529
530 dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
531 DMA_TO_DEVICE);
532 kfree(img);
533
534 return err;
535 }
536
537 static void
538 nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack)
539 {
540 int err;
541
542
543 nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
544 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
545 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
546 if (err)
547 NL_SET_ERR_MSG_MOD(extack,
548 "FW command error while enabling BPF");
549 }
550
551 static int nfp_net_bpf_stop(struct nfp_net *nn)
552 {
553 if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
554 return 0;
555
556 nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
557 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
558
559 return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
560 }
561
562 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
563 bool old_prog, struct netlink_ext_ack *extack)
564 {
565 int err;
566
567 if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev))
568 return -EINVAL;
569
570 if (prog && old_prog) {
571 u8 cap;
572
573 cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
574 if (!(cap & NFP_NET_BPF_CAP_RELO)) {
575 NL_SET_ERR_MSG_MOD(extack,
576 "FW does not support live reload");
577 return -EBUSY;
578 }
579 }
580
581
582 if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
583 return -EBUSY;
584
585 if (old_prog && !prog)
586 return nfp_net_bpf_stop(nn);
587
588 err = nfp_net_bpf_load(nn, prog, extack);
589 if (err)
590 return err;
591
592 if (!old_prog)
593 nfp_net_bpf_start(nn, extack);
594
595 return 0;
596 }
597
598 const struct bpf_prog_offload_ops nfp_bpf_dev_ops = {
599 .insn_hook = nfp_verify_insn,
600 .finalize = nfp_bpf_finalize,
601 .replace_insn = nfp_bpf_opt_replace_insn,
602 .remove_insns = nfp_bpf_opt_remove_insns,
603 .prepare = nfp_bpf_verifier_prep,
604 .translate = nfp_bpf_translate,
605 .destroy = nfp_bpf_destroy,
606 };