This source file includes following definitions.
- nfp_bpf_cmsg_alloc
- nfp_bpf_cmsg_map_req_size
- nfp_bpf_cmsg_map_req_alloc
- nfp_bpf_cmsg_map_reply_size
- nfp_bpf_ctrl_rc_to_errno
- nfp_bpf_ctrl_alloc_map
- nfp_bpf_ctrl_free_map
- nfp_bpf_ctrl_req_key
- nfp_bpf_ctrl_req_val
- nfp_bpf_ctrl_reply_key
- nfp_bpf_ctrl_reply_val
- nfp_bpf_ctrl_op_cache_invalidate
- nfp_bpf_ctrl_op_cache_capable
- nfp_bpf_ctrl_op_cache_fill
- nfp_bpf_ctrl_op_cache_get
- nfp_bpf_ctrl_op_cache_put
- nfp_bpf_ctrl_entry_op
- nfp_bpf_ctrl_update_entry
- nfp_bpf_ctrl_del_entry
- nfp_bpf_ctrl_lookup_entry
- nfp_bpf_ctrl_getfirst_entry
- nfp_bpf_ctrl_getnext_entry
- nfp_bpf_ctrl_cmsg_min_mtu
- nfp_bpf_ctrl_cmsg_mtu
- nfp_bpf_ctrl_cmsg_cache_cnt
- nfp_bpf_ctrl_msg_rx
- nfp_bpf_ctrl_msg_rx_raw
1
2
3
4 #include <linux/bpf.h>
5 #include <linux/bitops.h>
6 #include <linux/bug.h>
7 #include <linux/jiffies.h>
8 #include <linux/skbuff.h>
9 #include <linux/timekeeping.h>
10
11 #include "../ccm.h"
12 #include "../nfp_app.h"
13 #include "../nfp_net.h"
14 #include "fw.h"
15 #include "main.h"
16
17 static struct sk_buff *
18 nfp_bpf_cmsg_alloc(struct nfp_app_bpf *bpf, unsigned int size)
19 {
20 struct sk_buff *skb;
21
22 skb = nfp_app_ctrl_msg_alloc(bpf->app, size, GFP_KERNEL);
23 skb_put(skb, size);
24
25 return skb;
26 }
27
28 static unsigned int
29 nfp_bpf_cmsg_map_req_size(struct nfp_app_bpf *bpf, unsigned int n)
30 {
31 unsigned int size;
32
33 size = sizeof(struct cmsg_req_map_op);
34 size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n;
35
36 return size;
37 }
38
39 static struct sk_buff *
40 nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n)
41 {
42 return nfp_bpf_cmsg_alloc(bpf, nfp_bpf_cmsg_map_req_size(bpf, n));
43 }
44
45 static unsigned int
46 nfp_bpf_cmsg_map_reply_size(struct nfp_app_bpf *bpf, unsigned int n)
47 {
48 unsigned int size;
49
50 size = sizeof(struct cmsg_reply_map_op);
51 size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n;
52
53 return size;
54 }
55
56 static int
57 nfp_bpf_ctrl_rc_to_errno(struct nfp_app_bpf *bpf,
58 struct cmsg_reply_map_simple *reply)
59 {
60 static const int res_table[] = {
61 [CMSG_RC_SUCCESS] = 0,
62 [CMSG_RC_ERR_MAP_FD] = -EBADFD,
63 [CMSG_RC_ERR_MAP_NOENT] = -ENOENT,
64 [CMSG_RC_ERR_MAP_ERR] = -EINVAL,
65 [CMSG_RC_ERR_MAP_PARSE] = -EIO,
66 [CMSG_RC_ERR_MAP_EXIST] = -EEXIST,
67 [CMSG_RC_ERR_MAP_NOMEM] = -ENOMEM,
68 [CMSG_RC_ERR_MAP_E2BIG] = -E2BIG,
69 };
70 u32 rc;
71
72 rc = be32_to_cpu(reply->rc);
73 if (rc >= ARRAY_SIZE(res_table)) {
74 cmsg_warn(bpf, "FW responded with invalid status: %u\n", rc);
75 return -EIO;
76 }
77
78 return res_table[rc];
79 }
80
81 long long int
82 nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map)
83 {
84 struct cmsg_reply_map_alloc_tbl *reply;
85 struct cmsg_req_map_alloc_tbl *req;
86 struct sk_buff *skb;
87 u32 tid;
88 int err;
89
90 skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
91 if (!skb)
92 return -ENOMEM;
93
94 req = (void *)skb->data;
95 req->key_size = cpu_to_be32(map->key_size);
96 req->value_size = cpu_to_be32(map->value_size);
97 req->max_entries = cpu_to_be32(map->max_entries);
98 req->map_type = cpu_to_be32(map->map_type);
99 req->map_flags = 0;
100
101 skb = nfp_ccm_communicate(&bpf->ccm, skb, NFP_CCM_TYPE_BPF_MAP_ALLOC,
102 sizeof(*reply));
103 if (IS_ERR(skb))
104 return PTR_ERR(skb);
105
106 reply = (void *)skb->data;
107 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
108 if (err)
109 goto err_free;
110
111 tid = be32_to_cpu(reply->tid);
112 dev_consume_skb_any(skb);
113
114 return tid;
115 err_free:
116 dev_kfree_skb_any(skb);
117 return err;
118 }
119
120 void nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map)
121 {
122 struct cmsg_reply_map_free_tbl *reply;
123 struct cmsg_req_map_free_tbl *req;
124 struct sk_buff *skb;
125 int err;
126
127 skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
128 if (!skb) {
129 cmsg_warn(bpf, "leaking map - failed to allocate msg\n");
130 return;
131 }
132
133 req = (void *)skb->data;
134 req->tid = cpu_to_be32(nfp_map->tid);
135
136 skb = nfp_ccm_communicate(&bpf->ccm, skb, NFP_CCM_TYPE_BPF_MAP_FREE,
137 sizeof(*reply));
138 if (IS_ERR(skb)) {
139 cmsg_warn(bpf, "leaking map - I/O error\n");
140 return;
141 }
142
143 reply = (void *)skb->data;
144 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
145 if (err)
146 cmsg_warn(bpf, "leaking map - FW responded with: %d\n", err);
147
148 dev_consume_skb_any(skb);
149 }
150
151 static void *
152 nfp_bpf_ctrl_req_key(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req,
153 unsigned int n)
154 {
155 return &req->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n];
156 }
157
158 static void *
159 nfp_bpf_ctrl_req_val(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req,
160 unsigned int n)
161 {
162 return &req->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n];
163 }
164
165 static void *
166 nfp_bpf_ctrl_reply_key(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply,
167 unsigned int n)
168 {
169 return &reply->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n];
170 }
171
172 static void *
173 nfp_bpf_ctrl_reply_val(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply,
174 unsigned int n)
175 {
176 return &reply->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n];
177 }
178
179 static bool nfp_bpf_ctrl_op_cache_invalidate(enum nfp_ccm_type op)
180 {
181 return op == NFP_CCM_TYPE_BPF_MAP_UPDATE ||
182 op == NFP_CCM_TYPE_BPF_MAP_DELETE;
183 }
184
185 static bool nfp_bpf_ctrl_op_cache_capable(enum nfp_ccm_type op)
186 {
187 return op == NFP_CCM_TYPE_BPF_MAP_LOOKUP ||
188 op == NFP_CCM_TYPE_BPF_MAP_GETNEXT;
189 }
190
191 static bool nfp_bpf_ctrl_op_cache_fill(enum nfp_ccm_type op)
192 {
193 return op == NFP_CCM_TYPE_BPF_MAP_GETFIRST ||
194 op == NFP_CCM_TYPE_BPF_MAP_GETNEXT;
195 }
196
197 static unsigned int
198 nfp_bpf_ctrl_op_cache_get(struct nfp_bpf_map *nfp_map, enum nfp_ccm_type op,
199 const u8 *key, u8 *out_key, u8 *out_value,
200 u32 *cache_gen)
201 {
202 struct bpf_map *map = &nfp_map->offmap->map;
203 struct nfp_app_bpf *bpf = nfp_map->bpf;
204 unsigned int i, count, n_entries;
205 struct cmsg_reply_map_op *reply;
206
207 n_entries = nfp_bpf_ctrl_op_cache_fill(op) ? bpf->cmsg_cache_cnt : 1;
208
209 spin_lock(&nfp_map->cache_lock);
210 *cache_gen = nfp_map->cache_gen;
211 if (nfp_map->cache_blockers)
212 n_entries = 1;
213
214 if (nfp_bpf_ctrl_op_cache_invalidate(op))
215 goto exit_block;
216 if (!nfp_bpf_ctrl_op_cache_capable(op))
217 goto exit_unlock;
218
219 if (!nfp_map->cache)
220 goto exit_unlock;
221 if (nfp_map->cache_to < ktime_get_ns())
222 goto exit_invalidate;
223
224 reply = (void *)nfp_map->cache->data;
225 count = be32_to_cpu(reply->count);
226
227 for (i = 0; i < count; i++) {
228 void *cached_key;
229
230 cached_key = nfp_bpf_ctrl_reply_key(bpf, reply, i);
231 if (memcmp(cached_key, key, map->key_size))
232 continue;
233
234 if (op == NFP_CCM_TYPE_BPF_MAP_LOOKUP)
235 memcpy(out_value, nfp_bpf_ctrl_reply_val(bpf, reply, i),
236 map->value_size);
237 if (op == NFP_CCM_TYPE_BPF_MAP_GETNEXT) {
238 if (i + 1 == count)
239 break;
240
241 memcpy(out_key,
242 nfp_bpf_ctrl_reply_key(bpf, reply, i + 1),
243 map->key_size);
244 }
245
246 n_entries = 0;
247 goto exit_unlock;
248 }
249 goto exit_unlock;
250
251 exit_block:
252 nfp_map->cache_blockers++;
253 exit_invalidate:
254 dev_consume_skb_any(nfp_map->cache);
255 nfp_map->cache = NULL;
256 exit_unlock:
257 spin_unlock(&nfp_map->cache_lock);
258 return n_entries;
259 }
260
261 static void
262 nfp_bpf_ctrl_op_cache_put(struct nfp_bpf_map *nfp_map, enum nfp_ccm_type op,
263 struct sk_buff *skb, u32 cache_gen)
264 {
265 bool blocker, filler;
266
267 blocker = nfp_bpf_ctrl_op_cache_invalidate(op);
268 filler = nfp_bpf_ctrl_op_cache_fill(op);
269 if (blocker || filler) {
270 u64 to = 0;
271
272 if (filler)
273 to = ktime_get_ns() + NFP_BPF_MAP_CACHE_TIME_NS;
274
275 spin_lock(&nfp_map->cache_lock);
276 if (blocker) {
277 nfp_map->cache_blockers--;
278 nfp_map->cache_gen++;
279 }
280 if (filler && !nfp_map->cache_blockers &&
281 nfp_map->cache_gen == cache_gen) {
282 nfp_map->cache_to = to;
283 swap(nfp_map->cache, skb);
284 }
285 spin_unlock(&nfp_map->cache_lock);
286 }
287
288 dev_consume_skb_any(skb);
289 }
290
291 static int
292 nfp_bpf_ctrl_entry_op(struct bpf_offloaded_map *offmap, enum nfp_ccm_type op,
293 u8 *key, u8 *value, u64 flags, u8 *out_key, u8 *out_value)
294 {
295 struct nfp_bpf_map *nfp_map = offmap->dev_priv;
296 unsigned int n_entries, reply_entries, count;
297 struct nfp_app_bpf *bpf = nfp_map->bpf;
298 struct bpf_map *map = &offmap->map;
299 struct cmsg_reply_map_op *reply;
300 struct cmsg_req_map_op *req;
301 struct sk_buff *skb;
302 u32 cache_gen;
303 int err;
304
305
306 if (flags >> 32)
307 return -EOPNOTSUPP;
308
309
310 n_entries = nfp_bpf_ctrl_op_cache_get(nfp_map, op, key, out_key,
311 out_value, &cache_gen);
312 if (!n_entries)
313 return 0;
314
315 skb = nfp_bpf_cmsg_map_req_alloc(bpf, 1);
316 if (!skb) {
317 err = -ENOMEM;
318 goto err_cache_put;
319 }
320
321 req = (void *)skb->data;
322 req->tid = cpu_to_be32(nfp_map->tid);
323 req->count = cpu_to_be32(n_entries);
324 req->flags = cpu_to_be32(flags);
325
326
327 if (key)
328 memcpy(nfp_bpf_ctrl_req_key(bpf, req, 0), key, map->key_size);
329 if (value)
330 memcpy(nfp_bpf_ctrl_req_val(bpf, req, 0), value,
331 map->value_size);
332
333 skb = nfp_ccm_communicate(&bpf->ccm, skb, op, 0);
334 if (IS_ERR(skb)) {
335 err = PTR_ERR(skb);
336 goto err_cache_put;
337 }
338
339 if (skb->len < sizeof(*reply)) {
340 cmsg_warn(bpf, "cmsg drop - type 0x%02x too short %d!\n",
341 op, skb->len);
342 err = -EIO;
343 goto err_free;
344 }
345
346 reply = (void *)skb->data;
347 count = be32_to_cpu(reply->count);
348 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
349
350
351
352 reply_entries = count + !!err;
353 if (n_entries > 1 && count)
354 err = 0;
355 if (err)
356 goto err_free;
357
358 if (skb->len != nfp_bpf_cmsg_map_reply_size(bpf, reply_entries)) {
359 cmsg_warn(bpf, "cmsg drop - type 0x%02x too short %d for %d entries!\n",
360 op, skb->len, reply_entries);
361 err = -EIO;
362 goto err_free;
363 }
364
365
366 if (out_key)
367 memcpy(out_key, nfp_bpf_ctrl_reply_key(bpf, reply, 0),
368 map->key_size);
369 if (out_value)
370 memcpy(out_value, nfp_bpf_ctrl_reply_val(bpf, reply, 0),
371 map->value_size);
372
373 nfp_bpf_ctrl_op_cache_put(nfp_map, op, skb, cache_gen);
374
375 return 0;
376 err_free:
377 dev_kfree_skb_any(skb);
378 err_cache_put:
379 nfp_bpf_ctrl_op_cache_put(nfp_map, op, NULL, cache_gen);
380 return err;
381 }
382
383 int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap,
384 void *key, void *value, u64 flags)
385 {
386 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_UPDATE,
387 key, value, flags, NULL, NULL);
388 }
389
390 int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key)
391 {
392 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_DELETE,
393 key, NULL, 0, NULL, NULL);
394 }
395
396 int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap,
397 void *key, void *value)
398 {
399 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_LOOKUP,
400 key, NULL, 0, NULL, value);
401 }
402
403 int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap,
404 void *next_key)
405 {
406 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_GETFIRST,
407 NULL, NULL, 0, next_key, NULL);
408 }
409
410 int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap,
411 void *key, void *next_key)
412 {
413 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_GETNEXT,
414 key, NULL, 0, next_key, NULL);
415 }
416
417 unsigned int nfp_bpf_ctrl_cmsg_min_mtu(struct nfp_app_bpf *bpf)
418 {
419 return max(nfp_bpf_cmsg_map_req_size(bpf, 1),
420 nfp_bpf_cmsg_map_reply_size(bpf, 1));
421 }
422
423 unsigned int nfp_bpf_ctrl_cmsg_mtu(struct nfp_app_bpf *bpf)
424 {
425 return max3(NFP_NET_DEFAULT_MTU,
426 nfp_bpf_cmsg_map_req_size(bpf, NFP_BPF_MAP_CACHE_CNT),
427 nfp_bpf_cmsg_map_reply_size(bpf, NFP_BPF_MAP_CACHE_CNT));
428 }
429
430 unsigned int nfp_bpf_ctrl_cmsg_cache_cnt(struct nfp_app_bpf *bpf)
431 {
432 unsigned int mtu, req_max, reply_max, entry_sz;
433
434 mtu = bpf->app->ctrl->dp.mtu;
435 entry_sz = bpf->cmsg_key_sz + bpf->cmsg_val_sz;
436 req_max = (mtu - sizeof(struct cmsg_req_map_op)) / entry_sz;
437 reply_max = (mtu - sizeof(struct cmsg_reply_map_op)) / entry_sz;
438
439 return min3(req_max, reply_max, NFP_BPF_MAP_CACHE_CNT);
440 }
441
442 void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb)
443 {
444 struct nfp_app_bpf *bpf = app->priv;
445
446 if (unlikely(skb->len < sizeof(struct cmsg_reply_map_simple))) {
447 cmsg_warn(bpf, "cmsg drop - too short %d!\n", skb->len);
448 dev_kfree_skb_any(skb);
449 return;
450 }
451
452 if (nfp_ccm_get_type(skb) == NFP_CCM_TYPE_BPF_BPF_EVENT) {
453 if (!nfp_bpf_event_output(bpf, skb->data, skb->len))
454 dev_consume_skb_any(skb);
455 else
456 dev_kfree_skb_any(skb);
457 }
458
459 nfp_ccm_rx(&bpf->ccm, skb);
460 }
461
462 void
463 nfp_bpf_ctrl_msg_rx_raw(struct nfp_app *app, const void *data, unsigned int len)
464 {
465 const struct nfp_ccm_hdr *hdr = data;
466 struct nfp_app_bpf *bpf = app->priv;
467
468 if (unlikely(len < sizeof(struct cmsg_reply_map_simple))) {
469 cmsg_warn(bpf, "cmsg drop - too short %d!\n", len);
470 return;
471 }
472
473 if (hdr->type == NFP_CCM_TYPE_BPF_BPF_EVENT)
474 nfp_bpf_event_output(bpf, data, len);
475 else
476 cmsg_warn(bpf, "cmsg drop - msg type %d with raw buffer!\n",
477 hdr->type);
478 }