1/* bpf_jit.S : BPF JIT helper functions
2 *
3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; version 2
8 * of the License.
9 */
10#include <linux/linkage.h>
11#include <asm/dwarf2.h>
12
13/*
14 * Calling convention :
15 * rbx : skb pointer (callee saved)
16 * esi : offset of byte(s) to fetch in skb (can be scratched)
17 * r10 : copy of skb->data
18 * r9d : hlen = skb->len - skb->data_len
19 */
20#define SKBDATA	%r10
21#define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
22#define MAX_BPF_STACK (512 /* from filter.h */ + \
23	32 /* space for rbx,r13,r14,r15 */ + \
24	8 /* space for skb_copy_bits */)
25
26sk_load_word:
27	.globl	sk_load_word
28
29	test	%esi,%esi
30	js	bpf_slow_path_word_neg
31
32sk_load_word_positive_offset:
33	.globl	sk_load_word_positive_offset
34
35	mov	%r9d,%eax		# hlen
36	sub	%esi,%eax		# hlen - offset
37	cmp	$3,%eax
38	jle	bpf_slow_path_word
39	mov     (SKBDATA,%rsi),%eax
40	bswap   %eax  			/* ntohl() */
41	ret
42
43sk_load_half:
44	.globl	sk_load_half
45
46	test	%esi,%esi
47	js	bpf_slow_path_half_neg
48
49sk_load_half_positive_offset:
50	.globl	sk_load_half_positive_offset
51
52	mov	%r9d,%eax
53	sub	%esi,%eax		#	hlen - offset
54	cmp	$1,%eax
55	jle	bpf_slow_path_half
56	movzwl	(SKBDATA,%rsi),%eax
57	rol	$8,%ax			# ntohs()
58	ret
59
60sk_load_byte:
61	.globl	sk_load_byte
62
63	test	%esi,%esi
64	js	bpf_slow_path_byte_neg
65
66sk_load_byte_positive_offset:
67	.globl	sk_load_byte_positive_offset
68
69	cmp	%esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
70	jle	bpf_slow_path_byte
71	movzbl	(SKBDATA,%rsi),%eax
72	ret
73
74/* rsi contains offset and can be scratched */
75#define bpf_slow_path_common(LEN)		\
76	mov	%rbx, %rdi; /* arg1 == skb */	\
77	push	%r9;				\
78	push	SKBDATA;			\
79/* rsi already has offset */			\
80	mov	$LEN,%ecx;	/* len */	\
81	lea	- MAX_BPF_STACK + 32(%rbp),%rdx;			\
82	call	skb_copy_bits;			\
83	test    %eax,%eax;			\
84	pop	SKBDATA;			\
85	pop	%r9;
86
87
88bpf_slow_path_word:
89	bpf_slow_path_common(4)
90	js	bpf_error
91	mov	- MAX_BPF_STACK + 32(%rbp),%eax
92	bswap	%eax
93	ret
94
95bpf_slow_path_half:
96	bpf_slow_path_common(2)
97	js	bpf_error
98	mov	- MAX_BPF_STACK + 32(%rbp),%ax
99	rol	$8,%ax
100	movzwl	%ax,%eax
101	ret
102
103bpf_slow_path_byte:
104	bpf_slow_path_common(1)
105	js	bpf_error
106	movzbl	- MAX_BPF_STACK + 32(%rbp),%eax
107	ret
108
109#define sk_negative_common(SIZE)				\
110	mov	%rbx, %rdi; /* arg1 == skb */			\
111	push	%r9;						\
112	push	SKBDATA;					\
113/* rsi already has offset */					\
114	mov	$SIZE,%edx;	/* size */			\
115	call	bpf_internal_load_pointer_neg_helper;		\
116	test	%rax,%rax;					\
117	pop	SKBDATA;					\
118	pop	%r9;						\
119	jz	bpf_error
120
121bpf_slow_path_word_neg:
122	cmp	SKF_MAX_NEG_OFF, %esi	/* test range */
123	jl	bpf_error	/* offset lower -> error  */
124sk_load_word_negative_offset:
125	.globl	sk_load_word_negative_offset
126	sk_negative_common(4)
127	mov	(%rax), %eax
128	bswap	%eax
129	ret
130
131bpf_slow_path_half_neg:
132	cmp	SKF_MAX_NEG_OFF, %esi
133	jl	bpf_error
134sk_load_half_negative_offset:
135	.globl	sk_load_half_negative_offset
136	sk_negative_common(2)
137	mov	(%rax),%ax
138	rol	$8,%ax
139	movzwl	%ax,%eax
140	ret
141
142bpf_slow_path_byte_neg:
143	cmp	SKF_MAX_NEG_OFF, %esi
144	jl	bpf_error
145sk_load_byte_negative_offset:
146	.globl	sk_load_byte_negative_offset
147	sk_negative_common(1)
148	movzbl	(%rax), %eax
149	ret
150
151bpf_error:
152# force a return 0 from jit handler
153	xor	%eax,%eax
154	mov	- MAX_BPF_STACK(%rbp),%rbx
155	mov	- MAX_BPF_STACK + 8(%rbp),%r13
156	mov	- MAX_BPF_STACK + 16(%rbp),%r14
157	mov	- MAX_BPF_STACK + 24(%rbp),%r15
158	leaveq
159	ret
160