1/* bpf_jit.S : BPF JIT helper functions 2 * 3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com) 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License 7 * as published by the Free Software Foundation; version 2 8 * of the License. 9 */ 10#include <linux/linkage.h> 11 12/* 13 * Calling convention : 14 * rbx : skb pointer (callee saved) 15 * esi : offset of byte(s) to fetch in skb (can be scratched) 16 * r10 : copy of skb->data 17 * r9d : hlen = skb->len - skb->data_len 18 */ 19#define SKBDATA %r10 20#define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */ 21#define MAX_BPF_STACK (512 /* from filter.h */ + \ 22 32 /* space for rbx,r13,r14,r15 */ + \ 23 8 /* space for skb_copy_bits */) 24 25sk_load_word: 26 .globl sk_load_word 27 28 test %esi,%esi 29 js bpf_slow_path_word_neg 30 31sk_load_word_positive_offset: 32 .globl sk_load_word_positive_offset 33 34 mov %r9d,%eax # hlen 35 sub %esi,%eax # hlen - offset 36 cmp $3,%eax 37 jle bpf_slow_path_word 38 mov (SKBDATA,%rsi),%eax 39 bswap %eax /* ntohl() */ 40 ret 41 42sk_load_half: 43 .globl sk_load_half 44 45 test %esi,%esi 46 js bpf_slow_path_half_neg 47 48sk_load_half_positive_offset: 49 .globl sk_load_half_positive_offset 50 51 mov %r9d,%eax 52 sub %esi,%eax # hlen - offset 53 cmp $1,%eax 54 jle bpf_slow_path_half 55 movzwl (SKBDATA,%rsi),%eax 56 rol $8,%ax # ntohs() 57 ret 58 59sk_load_byte: 60 .globl sk_load_byte 61 62 test %esi,%esi 63 js bpf_slow_path_byte_neg 64 65sk_load_byte_positive_offset: 66 .globl sk_load_byte_positive_offset 67 68 cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */ 69 jle bpf_slow_path_byte 70 movzbl (SKBDATA,%rsi),%eax 71 ret 72 73/* rsi contains offset and can be scratched */ 74#define bpf_slow_path_common(LEN) \ 75 mov %rbx, %rdi; /* arg1 == skb */ \ 76 push %r9; \ 77 push SKBDATA; \ 78/* rsi already has offset */ \ 79 mov $LEN,%ecx; /* len */ \ 80 lea - MAX_BPF_STACK + 32(%rbp),%rdx; \ 81 call skb_copy_bits; \ 82 test %eax,%eax; \ 83 pop SKBDATA; \ 84 pop %r9; 85 86 87bpf_slow_path_word: 88 bpf_slow_path_common(4) 89 js bpf_error 90 mov - MAX_BPF_STACK + 32(%rbp),%eax 91 bswap %eax 92 ret 93 94bpf_slow_path_half: 95 bpf_slow_path_common(2) 96 js bpf_error 97 mov - MAX_BPF_STACK + 32(%rbp),%ax 98 rol $8,%ax 99 movzwl %ax,%eax 100 ret 101 102bpf_slow_path_byte: 103 bpf_slow_path_common(1) 104 js bpf_error 105 movzbl - MAX_BPF_STACK + 32(%rbp),%eax 106 ret 107 108#define sk_negative_common(SIZE) \ 109 mov %rbx, %rdi; /* arg1 == skb */ \ 110 push %r9; \ 111 push SKBDATA; \ 112/* rsi already has offset */ \ 113 mov $SIZE,%edx; /* size */ \ 114 call bpf_internal_load_pointer_neg_helper; \ 115 test %rax,%rax; \ 116 pop SKBDATA; \ 117 pop %r9; \ 118 jz bpf_error 119 120bpf_slow_path_word_neg: 121 cmp SKF_MAX_NEG_OFF, %esi /* test range */ 122 jl bpf_error /* offset lower -> error */ 123sk_load_word_negative_offset: 124 .globl sk_load_word_negative_offset 125 sk_negative_common(4) 126 mov (%rax), %eax 127 bswap %eax 128 ret 129 130bpf_slow_path_half_neg: 131 cmp SKF_MAX_NEG_OFF, %esi 132 jl bpf_error 133sk_load_half_negative_offset: 134 .globl sk_load_half_negative_offset 135 sk_negative_common(2) 136 mov (%rax),%ax 137 rol $8,%ax 138 movzwl %ax,%eax 139 ret 140 141bpf_slow_path_byte_neg: 142 cmp SKF_MAX_NEG_OFF, %esi 143 jl bpf_error 144sk_load_byte_negative_offset: 145 .globl sk_load_byte_negative_offset 146 sk_negative_common(1) 147 movzbl (%rax), %eax 148 ret 149 150bpf_error: 151# force a return 0 from jit handler 152 xor %eax,%eax 153 mov - MAX_BPF_STACK(%rbp),%rbx 154 mov - MAX_BPF_STACK + 8(%rbp),%r13 155 mov - MAX_BPF_STACK + 16(%rbp),%r14 156 mov - MAX_BPF_STACK + 24(%rbp),%r15 157 leaveq 158 ret 159