This source file includes following definitions.
- bpf_test_run
- bpf_test_finish
- bpf_test_init
- bpf_ctx_init
- bpf_ctx_finish
- range_is_zero
- convert___skb_to_skb
- convert_skb_to___skb
- bpf_prog_test_run_skb
- bpf_prog_test_run_xdp
- verify_user_bpf_flow_keys
- bpf_prog_test_run_flow_dissector
1
2
3
4 #include <linux/bpf.h>
5 #include <linux/slab.h>
6 #include <linux/vmalloc.h>
7 #include <linux/etherdevice.h>
8 #include <linux/filter.h>
9 #include <linux/sched/signal.h>
10 #include <net/bpf_sk_storage.h>
11 #include <net/sock.h>
12 #include <net/tcp.h>
13
14 #define CREATE_TRACE_POINTS
15 #include <trace/events/bpf_test_run.h>
16
17 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
18 u32 *retval, u32 *time)
19 {
20 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL };
21 enum bpf_cgroup_storage_type stype;
22 u64 time_start, time_spent = 0;
23 int ret = 0;
24 u32 i;
25
26 for_each_cgroup_storage_type(stype) {
27 storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
28 if (IS_ERR(storage[stype])) {
29 storage[stype] = NULL;
30 for_each_cgroup_storage_type(stype)
31 bpf_cgroup_storage_free(storage[stype]);
32 return -ENOMEM;
33 }
34 }
35
36 if (!repeat)
37 repeat = 1;
38
39 rcu_read_lock();
40 preempt_disable();
41 time_start = ktime_get_ns();
42 for (i = 0; i < repeat; i++) {
43 bpf_cgroup_storage_set(storage);
44 *retval = BPF_PROG_RUN(prog, ctx);
45
46 if (signal_pending(current)) {
47 ret = -EINTR;
48 break;
49 }
50
51 if (need_resched()) {
52 time_spent += ktime_get_ns() - time_start;
53 preempt_enable();
54 rcu_read_unlock();
55
56 cond_resched();
57
58 rcu_read_lock();
59 preempt_disable();
60 time_start = ktime_get_ns();
61 }
62 }
63 time_spent += ktime_get_ns() - time_start;
64 preempt_enable();
65 rcu_read_unlock();
66
67 do_div(time_spent, repeat);
68 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
69
70 for_each_cgroup_storage_type(stype)
71 bpf_cgroup_storage_free(storage[stype]);
72
73 return ret;
74 }
75
76 static int bpf_test_finish(const union bpf_attr *kattr,
77 union bpf_attr __user *uattr, const void *data,
78 u32 size, u32 retval, u32 duration)
79 {
80 void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
81 int err = -EFAULT;
82 u32 copy_size = size;
83
84
85
86
87 if (kattr->test.data_size_out &&
88 copy_size > kattr->test.data_size_out) {
89 copy_size = kattr->test.data_size_out;
90 err = -ENOSPC;
91 }
92
93 if (data_out && copy_to_user(data_out, data, copy_size))
94 goto out;
95 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
96 goto out;
97 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
98 goto out;
99 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
100 goto out;
101 if (err != -ENOSPC)
102 err = 0;
103 out:
104 trace_bpf_test_finish(&err);
105 return err;
106 }
107
108 static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
109 u32 headroom, u32 tailroom)
110 {
111 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
112 void *data;
113
114 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
115 return ERR_PTR(-EINVAL);
116
117 data = kzalloc(size + headroom + tailroom, GFP_USER);
118 if (!data)
119 return ERR_PTR(-ENOMEM);
120
121 if (copy_from_user(data + headroom, data_in, size)) {
122 kfree(data);
123 return ERR_PTR(-EFAULT);
124 }
125 return data;
126 }
127
128 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
129 {
130 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
131 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
132 u32 size = kattr->test.ctx_size_in;
133 void *data;
134 int err;
135
136 if (!data_in && !data_out)
137 return NULL;
138
139 data = kzalloc(max_size, GFP_USER);
140 if (!data)
141 return ERR_PTR(-ENOMEM);
142
143 if (data_in) {
144 err = bpf_check_uarg_tail_zero(data_in, max_size, size);
145 if (err) {
146 kfree(data);
147 return ERR_PTR(err);
148 }
149
150 size = min_t(u32, max_size, size);
151 if (copy_from_user(data, data_in, size)) {
152 kfree(data);
153 return ERR_PTR(-EFAULT);
154 }
155 }
156 return data;
157 }
158
159 static int bpf_ctx_finish(const union bpf_attr *kattr,
160 union bpf_attr __user *uattr, const void *data,
161 u32 size)
162 {
163 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
164 int err = -EFAULT;
165 u32 copy_size = size;
166
167 if (!data || !data_out)
168 return 0;
169
170 if (copy_size > kattr->test.ctx_size_out) {
171 copy_size = kattr->test.ctx_size_out;
172 err = -ENOSPC;
173 }
174
175 if (copy_to_user(data_out, data, copy_size))
176 goto out;
177 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
178 goto out;
179 if (err != -ENOSPC)
180 err = 0;
181 out:
182 return err;
183 }
184
185
186
187
188
189
190
191
192
193
194 static inline bool range_is_zero(void *buf, size_t from, size_t to)
195 {
196 return !memchr_inv((u8 *)buf + from, 0, to - from);
197 }
198
199 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
200 {
201 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
202
203 if (!__skb)
204 return 0;
205
206
207 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, priority)))
208 return -EINVAL;
209
210
211
212 if (!range_is_zero(__skb, offsetof(struct __sk_buff, priority) +
213 FIELD_SIZEOF(struct __sk_buff, priority),
214 offsetof(struct __sk_buff, cb)))
215 return -EINVAL;
216
217
218
219 if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) +
220 FIELD_SIZEOF(struct __sk_buff, cb),
221 sizeof(struct __sk_buff)))
222 return -EINVAL;
223
224 skb->priority = __skb->priority;
225 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
226
227 return 0;
228 }
229
230 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
231 {
232 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
233
234 if (!__skb)
235 return;
236
237 __skb->priority = skb->priority;
238 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
239 }
240
241 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
242 union bpf_attr __user *uattr)
243 {
244 bool is_l2 = false, is_direct_pkt_access = false;
245 u32 size = kattr->test.data_size_in;
246 u32 repeat = kattr->test.repeat;
247 struct __sk_buff *ctx = NULL;
248 u32 retval, duration;
249 int hh_len = ETH_HLEN;
250 struct sk_buff *skb;
251 struct sock *sk;
252 void *data;
253 int ret;
254
255 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
256 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
257 if (IS_ERR(data))
258 return PTR_ERR(data);
259
260 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
261 if (IS_ERR(ctx)) {
262 kfree(data);
263 return PTR_ERR(ctx);
264 }
265
266 switch (prog->type) {
267 case BPF_PROG_TYPE_SCHED_CLS:
268 case BPF_PROG_TYPE_SCHED_ACT:
269 is_l2 = true;
270
271 case BPF_PROG_TYPE_LWT_IN:
272 case BPF_PROG_TYPE_LWT_OUT:
273 case BPF_PROG_TYPE_LWT_XMIT:
274 is_direct_pkt_access = true;
275 break;
276 default:
277 break;
278 }
279
280 sk = kzalloc(sizeof(struct sock), GFP_USER);
281 if (!sk) {
282 kfree(data);
283 kfree(ctx);
284 return -ENOMEM;
285 }
286 sock_net_set(sk, current->nsproxy->net_ns);
287 sock_init_data(NULL, sk);
288
289 skb = build_skb(data, 0);
290 if (!skb) {
291 kfree(data);
292 kfree(ctx);
293 kfree(sk);
294 return -ENOMEM;
295 }
296 skb->sk = sk;
297
298 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
299 __skb_put(skb, size);
300 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev);
301 skb_reset_network_header(skb);
302
303 if (is_l2)
304 __skb_push(skb, hh_len);
305 if (is_direct_pkt_access)
306 bpf_compute_data_pointers(skb);
307 ret = convert___skb_to_skb(skb, ctx);
308 if (ret)
309 goto out;
310 ret = bpf_test_run(prog, skb, repeat, &retval, &duration);
311 if (ret)
312 goto out;
313 if (!is_l2) {
314 if (skb_headroom(skb) < hh_len) {
315 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
316
317 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
318 ret = -ENOMEM;
319 goto out;
320 }
321 }
322 memset(__skb_push(skb, hh_len), 0, hh_len);
323 }
324 convert_skb_to___skb(skb, ctx);
325
326 size = skb->len;
327
328 if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
329 size = skb_headlen(skb);
330 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
331 if (!ret)
332 ret = bpf_ctx_finish(kattr, uattr, ctx,
333 sizeof(struct __sk_buff));
334 out:
335 kfree_skb(skb);
336 bpf_sk_storage_free(sk);
337 kfree(sk);
338 kfree(ctx);
339 return ret;
340 }
341
342 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
343 union bpf_attr __user *uattr)
344 {
345 u32 size = kattr->test.data_size_in;
346 u32 repeat = kattr->test.repeat;
347 struct netdev_rx_queue *rxqueue;
348 struct xdp_buff xdp = {};
349 u32 retval, duration;
350 void *data;
351 int ret;
352
353 if (kattr->test.ctx_in || kattr->test.ctx_out)
354 return -EINVAL;
355
356 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0);
357 if (IS_ERR(data))
358 return PTR_ERR(data);
359
360 xdp.data_hard_start = data;
361 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN;
362 xdp.data_meta = xdp.data;
363 xdp.data_end = xdp.data + size;
364
365 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
366 xdp.rxq = &rxqueue->xdp_rxq;
367
368 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration);
369 if (ret)
370 goto out;
371 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN ||
372 xdp.data_end != xdp.data + size)
373 size = xdp.data_end - xdp.data;
374 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
375 out:
376 kfree(data);
377 return ret;
378 }
379
380 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
381 {
382
383 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
384 return -EINVAL;
385
386
387
388 if (!range_is_zero(ctx, offsetof(struct bpf_flow_keys, flags) +
389 FIELD_SIZEOF(struct bpf_flow_keys, flags),
390 sizeof(struct bpf_flow_keys)))
391 return -EINVAL;
392
393 return 0;
394 }
395
396 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
397 const union bpf_attr *kattr,
398 union bpf_attr __user *uattr)
399 {
400 u32 size = kattr->test.data_size_in;
401 struct bpf_flow_dissector ctx = {};
402 u32 repeat = kattr->test.repeat;
403 struct bpf_flow_keys *user_ctx;
404 struct bpf_flow_keys flow_keys;
405 u64 time_start, time_spent = 0;
406 const struct ethhdr *eth;
407 unsigned int flags = 0;
408 u32 retval, duration;
409 void *data;
410 int ret;
411 u32 i;
412
413 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
414 return -EINVAL;
415
416 if (size < ETH_HLEN)
417 return -EINVAL;
418
419 data = bpf_test_init(kattr, size, 0, 0);
420 if (IS_ERR(data))
421 return PTR_ERR(data);
422
423 eth = (struct ethhdr *)data;
424
425 if (!repeat)
426 repeat = 1;
427
428 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
429 if (IS_ERR(user_ctx)) {
430 kfree(data);
431 return PTR_ERR(user_ctx);
432 }
433 if (user_ctx) {
434 ret = verify_user_bpf_flow_keys(user_ctx);
435 if (ret)
436 goto out;
437 flags = user_ctx->flags;
438 }
439
440 ctx.flow_keys = &flow_keys;
441 ctx.data = data;
442 ctx.data_end = (__u8 *)data + size;
443
444 rcu_read_lock();
445 preempt_disable();
446 time_start = ktime_get_ns();
447 for (i = 0; i < repeat; i++) {
448 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
449 size, flags);
450
451 if (signal_pending(current)) {
452 preempt_enable();
453 rcu_read_unlock();
454
455 ret = -EINTR;
456 goto out;
457 }
458
459 if (need_resched()) {
460 time_spent += ktime_get_ns() - time_start;
461 preempt_enable();
462 rcu_read_unlock();
463
464 cond_resched();
465
466 rcu_read_lock();
467 preempt_disable();
468 time_start = ktime_get_ns();
469 }
470 }
471 time_spent += ktime_get_ns() - time_start;
472 preempt_enable();
473 rcu_read_unlock();
474
475 do_div(time_spent, repeat);
476 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
477
478 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
479 retval, duration);
480 if (!ret)
481 ret = bpf_ctx_finish(kattr, uattr, user_ctx,
482 sizeof(struct bpf_flow_keys));
483
484 out:
485 kfree(user_ctx);
486 kfree(data);
487 return ret;
488 }