1/* bpf_jit.S : BPF JIT helper functions
2 *
3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; version 2
8 * of the License.
9 */
10#include <linux/linkage.h>
11
12/*
13 * Calling convention :
14 * rbx : skb pointer (callee saved)
15 * esi : offset of byte(s) to fetch in skb (can be scratched)
16 * r10 : copy of skb->data
17 * r9d : hlen = skb->len - skb->data_len
18 */
19#define SKBDATA	%r10
20#define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
21#define MAX_BPF_STACK (512 /* from filter.h */ + \
22	32 /* space for rbx,r13,r14,r15 */ + \
23	8 /* space for skb_copy_bits */)
24
25sk_load_word:
26	.globl	sk_load_word
27
28	test	%esi,%esi
29	js	bpf_slow_path_word_neg
30
31sk_load_word_positive_offset:
32	.globl	sk_load_word_positive_offset
33
34	mov	%r9d,%eax		# hlen
35	sub	%esi,%eax		# hlen - offset
36	cmp	$3,%eax
37	jle	bpf_slow_path_word
38	mov     (SKBDATA,%rsi),%eax
39	bswap   %eax  			/* ntohl() */
40	ret
41
42sk_load_half:
43	.globl	sk_load_half
44
45	test	%esi,%esi
46	js	bpf_slow_path_half_neg
47
48sk_load_half_positive_offset:
49	.globl	sk_load_half_positive_offset
50
51	mov	%r9d,%eax
52	sub	%esi,%eax		#	hlen - offset
53	cmp	$1,%eax
54	jle	bpf_slow_path_half
55	movzwl	(SKBDATA,%rsi),%eax
56	rol	$8,%ax			# ntohs()
57	ret
58
59sk_load_byte:
60	.globl	sk_load_byte
61
62	test	%esi,%esi
63	js	bpf_slow_path_byte_neg
64
65sk_load_byte_positive_offset:
66	.globl	sk_load_byte_positive_offset
67
68	cmp	%esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
69	jle	bpf_slow_path_byte
70	movzbl	(SKBDATA,%rsi),%eax
71	ret
72
73/* rsi contains offset and can be scratched */
74#define bpf_slow_path_common(LEN)		\
75	mov	%rbx, %rdi; /* arg1 == skb */	\
76	push	%r9;				\
77	push	SKBDATA;			\
78/* rsi already has offset */			\
79	mov	$LEN,%ecx;	/* len */	\
80	lea	- MAX_BPF_STACK + 32(%rbp),%rdx;			\
81	call	skb_copy_bits;			\
82	test    %eax,%eax;			\
83	pop	SKBDATA;			\
84	pop	%r9;
85
86
87bpf_slow_path_word:
88	bpf_slow_path_common(4)
89	js	bpf_error
90	mov	- MAX_BPF_STACK + 32(%rbp),%eax
91	bswap	%eax
92	ret
93
94bpf_slow_path_half:
95	bpf_slow_path_common(2)
96	js	bpf_error
97	mov	- MAX_BPF_STACK + 32(%rbp),%ax
98	rol	$8,%ax
99	movzwl	%ax,%eax
100	ret
101
102bpf_slow_path_byte:
103	bpf_slow_path_common(1)
104	js	bpf_error
105	movzbl	- MAX_BPF_STACK + 32(%rbp),%eax
106	ret
107
108#define sk_negative_common(SIZE)				\
109	mov	%rbx, %rdi; /* arg1 == skb */			\
110	push	%r9;						\
111	push	SKBDATA;					\
112/* rsi already has offset */					\
113	mov	$SIZE,%edx;	/* size */			\
114	call	bpf_internal_load_pointer_neg_helper;		\
115	test	%rax,%rax;					\
116	pop	SKBDATA;					\
117	pop	%r9;						\
118	jz	bpf_error
119
120bpf_slow_path_word_neg:
121	cmp	SKF_MAX_NEG_OFF, %esi	/* test range */
122	jl	bpf_error	/* offset lower -> error  */
123sk_load_word_negative_offset:
124	.globl	sk_load_word_negative_offset
125	sk_negative_common(4)
126	mov	(%rax), %eax
127	bswap	%eax
128	ret
129
130bpf_slow_path_half_neg:
131	cmp	SKF_MAX_NEG_OFF, %esi
132	jl	bpf_error
133sk_load_half_negative_offset:
134	.globl	sk_load_half_negative_offset
135	sk_negative_common(2)
136	mov	(%rax),%ax
137	rol	$8,%ax
138	movzwl	%ax,%eax
139	ret
140
141bpf_slow_path_byte_neg:
142	cmp	SKF_MAX_NEG_OFF, %esi
143	jl	bpf_error
144sk_load_byte_negative_offset:
145	.globl	sk_load_byte_negative_offset
146	sk_negative_common(1)
147	movzbl	(%rax), %eax
148	ret
149
150bpf_error:
151# force a return 0 from jit handler
152	xor	%eax,%eax
153	mov	- MAX_BPF_STACK(%rbp),%rbx
154	mov	- MAX_BPF_STACK + 8(%rbp),%r13
155	mov	- MAX_BPF_STACK + 16(%rbp),%r14
156	mov	- MAX_BPF_STACK + 24(%rbp),%r15
157	leaveq
158	ret
159