This source file includes following definitions.
- nfp_net_ebpf_capable
- nfp_bpf_xdp_offload
- nfp_bpf_extra_cap
- nfp_bpf_vnic_alloc
- nfp_bpf_vnic_free
- nfp_bpf_setup_tc_block_cb
- nfp_bpf_setup_tc
- nfp_bpf_check_mtu
- nfp_bpf_parse_cap_adjust_head
- nfp_bpf_parse_cap_func
- nfp_bpf_parse_cap_maps
- nfp_bpf_parse_cap_random
- nfp_bpf_parse_cap_qsel
- nfp_bpf_parse_cap_adjust_tail
- nfp_bpf_parse_cap_cmsg_multi_ent
- nfp_bpf_parse_cap_abi_version
- nfp_bpf_parse_capabilities
- nfp_bpf_init_capabilities
- nfp_bpf_ndo_init
- nfp_bpf_ndo_uninit
- nfp_bpf_start
- nfp_bpf_init
- nfp_bpf_clean
1
2
3
4 #include <net/pkt_cls.h>
5
6 #include "../nfpcore/nfp_cpp.h"
7 #include "../nfpcore/nfp_nffw.h"
8 #include "../nfpcore/nfp_nsp.h"
9 #include "../nfp_app.h"
10 #include "../nfp_main.h"
11 #include "../nfp_net.h"
12 #include "../nfp_port.h"
13 #include "fw.h"
14 #include "main.h"
15
16 const struct rhashtable_params nfp_bpf_maps_neutral_params = {
17 .nelem_hint = 4,
18 .key_len = FIELD_SIZEOF(struct bpf_map, id),
19 .key_offset = offsetof(struct nfp_bpf_neutral_map, map_id),
20 .head_offset = offsetof(struct nfp_bpf_neutral_map, l),
21 .automatic_shrinking = true,
22 };
23
24 static bool nfp_net_ebpf_capable(struct nfp_net *nn)
25 {
26 #ifdef __LITTLE_ENDIAN
27 struct nfp_app_bpf *bpf = nn->app->priv;
28
29 return nn->cap & NFP_NET_CFG_CTRL_BPF &&
30 bpf->abi_version &&
31 nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version;
32 #else
33 return false;
34 #endif
35 }
36
37 static int
38 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
39 struct bpf_prog *prog, struct netlink_ext_ack *extack)
40 {
41 bool running, xdp_running;
42
43 if (!nfp_net_ebpf_capable(nn))
44 return -EINVAL;
45
46 running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
47 xdp_running = running && nn->xdp_hw.prog;
48
49 if (!prog && !xdp_running)
50 return 0;
51 if (prog && running && !xdp_running)
52 return -EBUSY;
53
54 return nfp_net_bpf_offload(nn, prog, running, extack);
55 }
56
57 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
58 {
59 return nfp_net_ebpf_capable(nn) ? "BPF" : "";
60 }
61
62 static int
63 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
64 {
65 struct nfp_pf *pf = app->pf;
66 struct nfp_bpf_vnic *bv;
67 int err;
68
69 if (!pf->eth_tbl) {
70 nfp_err(pf->cpp, "No ETH table\n");
71 return -EINVAL;
72 }
73 if (pf->max_data_vnics != pf->eth_tbl->count) {
74 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
75 pf->max_data_vnics, pf->eth_tbl->count);
76 return -EINVAL;
77 }
78
79 bv = kzalloc(sizeof(*bv), GFP_KERNEL);
80 if (!bv)
81 return -ENOMEM;
82 nn->app_priv = bv;
83
84 err = nfp_app_nic_vnic_alloc(app, nn, id);
85 if (err)
86 goto err_free_priv;
87
88 bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
89 bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
90
91 return 0;
92 err_free_priv:
93 kfree(nn->app_priv);
94 return err;
95 }
96
97 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
98 {
99 struct nfp_bpf_vnic *bv = nn->app_priv;
100
101 WARN_ON(bv->tc_prog);
102 kfree(bv);
103 }
104
105 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
106 void *type_data, void *cb_priv)
107 {
108 struct tc_cls_bpf_offload *cls_bpf = type_data;
109 struct nfp_net *nn = cb_priv;
110 struct bpf_prog *oldprog;
111 struct nfp_bpf_vnic *bv;
112 int err;
113
114 if (type != TC_SETUP_CLSBPF) {
115 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
116 "only offload of BPF classifiers supported");
117 return -EOPNOTSUPP;
118 }
119 if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common))
120 return -EOPNOTSUPP;
121 if (!nfp_net_ebpf_capable(nn)) {
122 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
123 "NFP firmware does not support eBPF offload");
124 return -EOPNOTSUPP;
125 }
126 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
127 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
128 "only ETH_P_ALL supported as filter protocol");
129 return -EOPNOTSUPP;
130 }
131
132
133 if (!cls_bpf->exts_integrated ||
134 tcf_exts_has_actions(cls_bpf->exts)) {
135 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
136 "only direct action with no legacy actions supported");
137 return -EOPNOTSUPP;
138 }
139
140 if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
141 return -EOPNOTSUPP;
142
143 bv = nn->app_priv;
144 oldprog = cls_bpf->oldprog;
145
146
147 if (bv->tc_prog != oldprog) {
148 oldprog = NULL;
149 if (!cls_bpf->prog)
150 return 0;
151 }
152
153 err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog,
154 cls_bpf->common.extack);
155 if (err)
156 return err;
157
158 bv->tc_prog = cls_bpf->prog;
159 nn->port->tc_offload_cnt = !!bv->tc_prog;
160 return 0;
161 }
162
163 static LIST_HEAD(nfp_bpf_block_cb_list);
164
165 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
166 enum tc_setup_type type, void *type_data)
167 {
168 struct nfp_net *nn = netdev_priv(netdev);
169
170 switch (type) {
171 case TC_SETUP_BLOCK:
172 return flow_block_cb_setup_simple(type_data,
173 &nfp_bpf_block_cb_list,
174 nfp_bpf_setup_tc_block_cb,
175 nn, nn, true);
176 default:
177 return -EOPNOTSUPP;
178 }
179 }
180
181 static int
182 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
183 {
184 struct nfp_net *nn = netdev_priv(netdev);
185 unsigned int max_mtu;
186
187 if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
188 return 0;
189
190 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
191 if (new_mtu > max_mtu) {
192 nn_info(nn, "BPF offload active, MTU over %u not supported\n",
193 max_mtu);
194 return -EBUSY;
195 }
196 return 0;
197 }
198
199 static int
200 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value,
201 u32 length)
202 {
203 struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
204 struct nfp_cpp *cpp = bpf->app->pf->cpp;
205
206 if (length < sizeof(*cap)) {
207 nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
208 return -EINVAL;
209 }
210
211 bpf->adjust_head.flags = readl(&cap->flags);
212 bpf->adjust_head.off_min = readl(&cap->off_min);
213 bpf->adjust_head.off_max = readl(&cap->off_max);
214 bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
215 bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);
216
217 if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
218 nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
219 return -EINVAL;
220 }
221 if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) ||
222 !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
223 nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
224 memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
225 return 0;
226 }
227
228 return 0;
229 }
230
231 static int
232 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
233 {
234 struct nfp_bpf_cap_tlv_func __iomem *cap = value;
235
236 if (length < sizeof(*cap)) {
237 nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length);
238 return -EINVAL;
239 }
240
241 switch (readl(&cap->func_id)) {
242 case BPF_FUNC_map_lookup_elem:
243 bpf->helpers.map_lookup = readl(&cap->func_addr);
244 break;
245 case BPF_FUNC_map_update_elem:
246 bpf->helpers.map_update = readl(&cap->func_addr);
247 break;
248 case BPF_FUNC_map_delete_elem:
249 bpf->helpers.map_delete = readl(&cap->func_addr);
250 break;
251 case BPF_FUNC_perf_event_output:
252 bpf->helpers.perf_event_output = readl(&cap->func_addr);
253 break;
254 }
255
256 return 0;
257 }
258
259 static int
260 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
261 {
262 struct nfp_bpf_cap_tlv_maps __iomem *cap = value;
263
264 if (length < sizeof(*cap)) {
265 nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length);
266 return -EINVAL;
267 }
268
269 bpf->maps.types = readl(&cap->types);
270 bpf->maps.max_maps = readl(&cap->max_maps);
271 bpf->maps.max_elems = readl(&cap->max_elems);
272 bpf->maps.max_key_sz = readl(&cap->max_key_sz);
273 bpf->maps.max_val_sz = readl(&cap->max_val_sz);
274 bpf->maps.max_elem_sz = readl(&cap->max_elem_sz);
275
276 return 0;
277 }
278
279 static int
280 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
281 u32 length)
282 {
283 bpf->pseudo_random = true;
284 return 0;
285 }
286
287 static int
288 nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
289 {
290 bpf->queue_select = true;
291 return 0;
292 }
293
294 static int
295 nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value,
296 u32 length)
297 {
298 bpf->adjust_tail = true;
299 return 0;
300 }
301
302 static int
303 nfp_bpf_parse_cap_cmsg_multi_ent(struct nfp_app_bpf *bpf, void __iomem *value,
304 u32 length)
305 {
306 bpf->cmsg_multi_ent = true;
307 return 0;
308 }
309
310 static int
311 nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value,
312 u32 length)
313 {
314 if (length < 4) {
315 nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n",
316 length);
317 return -EINVAL;
318 }
319
320 bpf->abi_version = readl(value);
321 if (bpf->abi_version < 2 || bpf->abi_version > 3) {
322 nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n",
323 bpf->abi_version);
324 bpf->abi_version = 0;
325 }
326
327 return 0;
328 }
329
330 static int nfp_bpf_parse_capabilities(struct nfp_app *app)
331 {
332 struct nfp_cpp *cpp = app->pf->cpp;
333 struct nfp_cpp_area *area;
334 u8 __iomem *mem, *start;
335
336 mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
337 8, &area);
338 if (IS_ERR(mem))
339 return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);
340
341 start = mem;
342 while (mem - start + 8 <= nfp_cpp_area_size(area)) {
343 u8 __iomem *value;
344 u32 type, length;
345
346 type = readl(mem);
347 length = readl(mem + 4);
348 value = mem + 8;
349
350 mem += 8 + length;
351 if (mem - start > nfp_cpp_area_size(area))
352 goto err_release_free;
353
354 switch (type) {
355 case NFP_BPF_CAP_TYPE_FUNC:
356 if (nfp_bpf_parse_cap_func(app->priv, value, length))
357 goto err_release_free;
358 break;
359 case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
360 if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
361 length))
362 goto err_release_free;
363 break;
364 case NFP_BPF_CAP_TYPE_MAPS:
365 if (nfp_bpf_parse_cap_maps(app->priv, value, length))
366 goto err_release_free;
367 break;
368 case NFP_BPF_CAP_TYPE_RANDOM:
369 if (nfp_bpf_parse_cap_random(app->priv, value, length))
370 goto err_release_free;
371 break;
372 case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
373 if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
374 goto err_release_free;
375 break;
376 case NFP_BPF_CAP_TYPE_ADJUST_TAIL:
377 if (nfp_bpf_parse_cap_adjust_tail(app->priv, value,
378 length))
379 goto err_release_free;
380 break;
381 case NFP_BPF_CAP_TYPE_ABI_VERSION:
382 if (nfp_bpf_parse_cap_abi_version(app->priv, value,
383 length))
384 goto err_release_free;
385 break;
386 case NFP_BPF_CAP_TYPE_CMSG_MULTI_ENT:
387 if (nfp_bpf_parse_cap_cmsg_multi_ent(app->priv, value,
388 length))
389 goto err_release_free;
390 break;
391 default:
392 nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
393 break;
394 }
395 }
396 if (mem - start != nfp_cpp_area_size(area)) {
397 nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n",
398 mem - start, nfp_cpp_area_size(area));
399 goto err_release_free;
400 }
401
402 nfp_cpp_area_release_free(area);
403
404 return 0;
405
406 err_release_free:
407 nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start);
408 nfp_cpp_area_release_free(area);
409 return -EINVAL;
410 }
411
412 static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf)
413 {
414 bpf->abi_version = 2;
415 }
416
417 static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev)
418 {
419 struct nfp_app_bpf *bpf = app->priv;
420
421 return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev);
422 }
423
424 static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev)
425 {
426 struct nfp_app_bpf *bpf = app->priv;
427
428 bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev);
429 }
430
431 static int nfp_bpf_start(struct nfp_app *app)
432 {
433 struct nfp_app_bpf *bpf = app->priv;
434
435 if (app->ctrl->dp.mtu < nfp_bpf_ctrl_cmsg_min_mtu(bpf)) {
436 nfp_err(bpf->app->cpp,
437 "ctrl channel MTU below min required %u < %u\n",
438 app->ctrl->dp.mtu, nfp_bpf_ctrl_cmsg_min_mtu(bpf));
439 return -EINVAL;
440 }
441
442 if (bpf->cmsg_multi_ent)
443 bpf->cmsg_cache_cnt = nfp_bpf_ctrl_cmsg_cache_cnt(bpf);
444 else
445 bpf->cmsg_cache_cnt = 1;
446
447 return 0;
448 }
449
450 static int nfp_bpf_init(struct nfp_app *app)
451 {
452 struct nfp_app_bpf *bpf;
453 int err;
454
455 bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
456 if (!bpf)
457 return -ENOMEM;
458 bpf->app = app;
459 app->priv = bpf;
460
461 INIT_LIST_HEAD(&bpf->map_list);
462
463 err = nfp_ccm_init(&bpf->ccm, app);
464 if (err)
465 goto err_free_bpf;
466
467 err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params);
468 if (err)
469 goto err_clean_ccm;
470
471 nfp_bpf_init_capabilities(bpf);
472
473 err = nfp_bpf_parse_capabilities(app);
474 if (err)
475 goto err_free_neutral_maps;
476
477 if (bpf->abi_version < 3) {
478 bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4;
479 bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4;
480 } else {
481 bpf->cmsg_key_sz = bpf->maps.max_key_sz;
482 bpf->cmsg_val_sz = bpf->maps.max_val_sz;
483 app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf);
484 }
485
486 bpf->bpf_dev = bpf_offload_dev_create(&nfp_bpf_dev_ops, bpf);
487 err = PTR_ERR_OR_ZERO(bpf->bpf_dev);
488 if (err)
489 goto err_free_neutral_maps;
490
491 return 0;
492
493 err_free_neutral_maps:
494 rhashtable_destroy(&bpf->maps_neutral);
495 err_clean_ccm:
496 nfp_ccm_clean(&bpf->ccm);
497 err_free_bpf:
498 kfree(bpf);
499 return err;
500 }
501
502 static void nfp_bpf_clean(struct nfp_app *app)
503 {
504 struct nfp_app_bpf *bpf = app->priv;
505
506 bpf_offload_dev_destroy(bpf->bpf_dev);
507 nfp_ccm_clean(&bpf->ccm);
508 WARN_ON(!list_empty(&bpf->map_list));
509 WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
510 rhashtable_free_and_destroy(&bpf->maps_neutral,
511 nfp_check_rhashtable_empty, NULL);
512 kfree(bpf);
513 }
514
515 const struct nfp_app_type app_bpf = {
516 .id = NFP_APP_BPF_NIC,
517 .name = "ebpf",
518
519 .ctrl_cap_mask = 0,
520
521 .init = nfp_bpf_init,
522 .clean = nfp_bpf_clean,
523 .start = nfp_bpf_start,
524
525 .check_mtu = nfp_bpf_check_mtu,
526
527 .extra_cap = nfp_bpf_extra_cap,
528
529 .ndo_init = nfp_bpf_ndo_init,
530 .ndo_uninit = nfp_bpf_ndo_uninit,
531
532 .vnic_alloc = nfp_bpf_vnic_alloc,
533 .vnic_free = nfp_bpf_vnic_free,
534
535 .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx,
536 .ctrl_msg_rx_raw = nfp_bpf_ctrl_msg_rx_raw,
537
538 .setup_tc = nfp_bpf_setup_tc,
539 .bpf = nfp_ndo_bpf,
540 .xdp_offload = nfp_bpf_xdp_offload,
541 };