This source file includes following definitions.
- __nft_trace_packet
- nft_trace_packet
- nft_cmp_fast_eval
- nft_payload_fast_eval
- nft_update_chain_stats
- expr_call_ops_eval
- nft_do_chain
- nf_tables_core_module_init
- nf_tables_core_module_exit
1
2
3
4
5
6
7
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/init.h>
11 #include <linux/list.h>
12 #include <linux/rculist.h>
13 #include <linux/skbuff.h>
14 #include <linux/netlink.h>
15 #include <linux/netfilter.h>
16 #include <linux/static_key.h>
17 #include <linux/netfilter/nfnetlink.h>
18 #include <linux/netfilter/nf_tables.h>
19 #include <net/netfilter/nf_tables_core.h>
20 #include <net/netfilter/nf_tables.h>
21 #include <net/netfilter/nf_log.h>
22 #include <net/netfilter/nft_meta.h>
23
24 static noinline void __nft_trace_packet(struct nft_traceinfo *info,
25 const struct nft_chain *chain,
26 enum nft_trace_types type)
27 {
28 const struct nft_pktinfo *pkt = info->pkt;
29
30 if (!info->trace || !pkt->skb->nf_trace)
31 return;
32
33 info->chain = chain;
34 info->type = type;
35
36 nft_trace_notify(info);
37 }
38
39 static inline void nft_trace_packet(struct nft_traceinfo *info,
40 const struct nft_chain *chain,
41 const struct nft_rule *rule,
42 enum nft_trace_types type)
43 {
44 if (static_branch_unlikely(&nft_trace_enabled)) {
45 info->rule = rule;
46 __nft_trace_packet(info, chain, type);
47 }
48 }
49
50 static void nft_cmp_fast_eval(const struct nft_expr *expr,
51 struct nft_regs *regs)
52 {
53 const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
54 u32 mask = nft_cmp_fast_mask(priv->len);
55
56 if ((regs->data[priv->sreg] & mask) == priv->data)
57 return;
58 regs->verdict.code = NFT_BREAK;
59 }
60
61 static bool nft_payload_fast_eval(const struct nft_expr *expr,
62 struct nft_regs *regs,
63 const struct nft_pktinfo *pkt)
64 {
65 const struct nft_payload *priv = nft_expr_priv(expr);
66 const struct sk_buff *skb = pkt->skb;
67 u32 *dest = ®s->data[priv->dreg];
68 unsigned char *ptr;
69
70 if (priv->base == NFT_PAYLOAD_NETWORK_HEADER)
71 ptr = skb_network_header(skb);
72 else {
73 if (!pkt->tprot_set)
74 return false;
75 ptr = skb_network_header(skb) + pkt->xt.thoff;
76 }
77
78 ptr += priv->offset;
79
80 if (unlikely(ptr + priv->len > skb_tail_pointer(skb)))
81 return false;
82
83 *dest = 0;
84 if (priv->len == 2)
85 *(u16 *)dest = *(u16 *)ptr;
86 else if (priv->len == 4)
87 *(u32 *)dest = *(u32 *)ptr;
88 else
89 *(u8 *)dest = *(u8 *)ptr;
90 return true;
91 }
92
93 DEFINE_STATIC_KEY_FALSE(nft_counters_enabled);
94
95 static noinline void nft_update_chain_stats(const struct nft_chain *chain,
96 const struct nft_pktinfo *pkt)
97 {
98 struct nft_base_chain *base_chain;
99 struct nft_stats __percpu *pstats;
100 struct nft_stats *stats;
101
102 base_chain = nft_base_chain(chain);
103
104 rcu_read_lock();
105 pstats = READ_ONCE(base_chain->stats);
106 if (pstats) {
107 local_bh_disable();
108 stats = this_cpu_ptr(pstats);
109 u64_stats_update_begin(&stats->syncp);
110 stats->pkts++;
111 stats->bytes += pkt->skb->len;
112 u64_stats_update_end(&stats->syncp);
113 local_bh_enable();
114 }
115 rcu_read_unlock();
116 }
117
118 struct nft_jumpstack {
119 const struct nft_chain *chain;
120 struct nft_rule *const *rules;
121 };
122
123 static void expr_call_ops_eval(const struct nft_expr *expr,
124 struct nft_regs *regs,
125 struct nft_pktinfo *pkt)
126 {
127 #ifdef CONFIG_RETPOLINE
128 unsigned long e = (unsigned long)expr->ops->eval;
129 #define X(e, fun) \
130 do { if ((e) == (unsigned long)(fun)) \
131 return fun(expr, regs, pkt); } while (0)
132
133 X(e, nft_payload_eval);
134 X(e, nft_cmp_eval);
135 X(e, nft_meta_get_eval);
136 X(e, nft_lookup_eval);
137 X(e, nft_range_eval);
138 X(e, nft_immediate_eval);
139 X(e, nft_byteorder_eval);
140 X(e, nft_dynset_eval);
141 X(e, nft_rt_get_eval);
142 X(e, nft_bitwise_eval);
143 #undef X
144 #endif
145 expr->ops->eval(expr, regs, pkt);
146 }
147
148 unsigned int
149 nft_do_chain(struct nft_pktinfo *pkt, void *priv)
150 {
151 const struct nft_chain *chain = priv, *basechain = chain;
152 const struct net *net = nft_net(pkt);
153 struct nft_rule *const *rules;
154 const struct nft_rule *rule;
155 const struct nft_expr *expr, *last;
156 struct nft_regs regs;
157 unsigned int stackptr = 0;
158 struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
159 bool genbit = READ_ONCE(net->nft.gencursor);
160 struct nft_traceinfo info;
161
162 info.trace = false;
163 if (static_branch_unlikely(&nft_trace_enabled))
164 nft_trace_init(&info, pkt, ®s.verdict, basechain);
165 do_chain:
166 if (genbit)
167 rules = rcu_dereference(chain->rules_gen_1);
168 else
169 rules = rcu_dereference(chain->rules_gen_0);
170
171 next_rule:
172 rule = *rules;
173 regs.verdict.code = NFT_CONTINUE;
174 for (; *rules ; rules++) {
175 rule = *rules;
176 nft_rule_for_each_expr(expr, last, rule) {
177 if (expr->ops == &nft_cmp_fast_ops)
178 nft_cmp_fast_eval(expr, ®s);
179 else if (expr->ops != &nft_payload_fast_ops ||
180 !nft_payload_fast_eval(expr, ®s, pkt))
181 expr_call_ops_eval(expr, ®s, pkt);
182
183 if (regs.verdict.code != NFT_CONTINUE)
184 break;
185 }
186
187 switch (regs.verdict.code) {
188 case NFT_BREAK:
189 regs.verdict.code = NFT_CONTINUE;
190 continue;
191 case NFT_CONTINUE:
192 nft_trace_packet(&info, chain, rule,
193 NFT_TRACETYPE_RULE);
194 continue;
195 }
196 break;
197 }
198
199 switch (regs.verdict.code & NF_VERDICT_MASK) {
200 case NF_ACCEPT:
201 case NF_DROP:
202 case NF_QUEUE:
203 case NF_STOLEN:
204 nft_trace_packet(&info, chain, rule,
205 NFT_TRACETYPE_RULE);
206 return regs.verdict.code;
207 }
208
209 switch (regs.verdict.code) {
210 case NFT_JUMP:
211 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE))
212 return NF_DROP;
213 jumpstack[stackptr].chain = chain;
214 jumpstack[stackptr].rules = rules + 1;
215 stackptr++;
216
217 case NFT_GOTO:
218 nft_trace_packet(&info, chain, rule,
219 NFT_TRACETYPE_RULE);
220
221 chain = regs.verdict.chain;
222 goto do_chain;
223 case NFT_CONTINUE:
224 case NFT_RETURN:
225 nft_trace_packet(&info, chain, rule,
226 NFT_TRACETYPE_RETURN);
227 break;
228 default:
229 WARN_ON(1);
230 }
231
232 if (stackptr > 0) {
233 stackptr--;
234 chain = jumpstack[stackptr].chain;
235 rules = jumpstack[stackptr].rules;
236 goto next_rule;
237 }
238
239 nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY);
240
241 if (static_branch_unlikely(&nft_counters_enabled))
242 nft_update_chain_stats(basechain, pkt);
243
244 return nft_base_chain(basechain)->policy;
245 }
246 EXPORT_SYMBOL_GPL(nft_do_chain);
247
248 static struct nft_expr_type *nft_basic_types[] = {
249 &nft_imm_type,
250 &nft_cmp_type,
251 &nft_lookup_type,
252 &nft_bitwise_type,
253 &nft_byteorder_type,
254 &nft_payload_type,
255 &nft_dynset_type,
256 &nft_range_type,
257 &nft_meta_type,
258 &nft_rt_type,
259 &nft_exthdr_type,
260 };
261
262 static struct nft_object_type *nft_basic_objects[] = {
263 #ifdef CONFIG_NETWORK_SECMARK
264 &nft_secmark_obj_type,
265 #endif
266 };
267
268 int __init nf_tables_core_module_init(void)
269 {
270 int err, i, j = 0;
271
272 for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) {
273 err = nft_register_obj(nft_basic_objects[i]);
274 if (err)
275 goto err;
276 }
277
278 for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) {
279 err = nft_register_expr(nft_basic_types[j]);
280 if (err)
281 goto err;
282 }
283
284 return 0;
285
286 err:
287 while (j-- > 0)
288 nft_unregister_expr(nft_basic_types[j]);
289
290 while (i-- > 0)
291 nft_unregister_obj(nft_basic_objects[i]);
292
293 return err;
294 }
295
296 void nf_tables_core_module_exit(void)
297 {
298 int i;
299
300 i = ARRAY_SIZE(nft_basic_types);
301 while (i-- > 0)
302 nft_unregister_expr(nft_basic_types[i]);
303
304 i = ARRAY_SIZE(nft_basic_objects);
305 while (i-- > 0)
306 nft_unregister_obj(nft_basic_objects[i]);
307 }