This source file includes following definitions.
- jump_entry_code
- jump_entry_target
- jump_entry_key
- jump_entry_code
- jump_entry_target
- jump_entry_key
- jump_entry_is_branch
- jump_entry_is_init
- jump_entry_set_init
- static_key_false
- static_key_true
- static_key_count
- jump_label_init
- static_key_false
- static_key_true
- static_key_slow_inc
- static_key_slow_dec
- jump_label_text_reserved
- jump_label_lock
- jump_label_unlock
- jump_label_apply_nops
- static_key_enable
- static_key_disable
1
2 #ifndef _LINUX_JUMP_LABEL_H
3 #define _LINUX_JUMP_LABEL_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74 #ifndef __ASSEMBLY__
75
76 #include <linux/types.h>
77 #include <linux/compiler.h>
78
79 extern bool static_key_initialized;
80
81 #define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \
82 "%s(): static key '%pS' used before call to jump_label_init()", \
83 __func__, (key))
84
85 #ifdef CONFIG_JUMP_LABEL
86
87 struct static_key {
88 atomic_t enabled;
89
90
91
92
93
94
95
96
97
98
99
100
101
102 union {
103 unsigned long type;
104 struct jump_entry *entries;
105 struct static_key_mod *next;
106 };
107 };
108
109 #else
110 struct static_key {
111 atomic_t enabled;
112 };
113 #endif
114 #endif
115
116 #ifdef CONFIG_JUMP_LABEL
117 #include <asm/jump_label.h>
118
119 #ifndef __ASSEMBLY__
120 #ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE
121
122 struct jump_entry {
123 s32 code;
124 s32 target;
125 long key;
126 };
127
128 static inline unsigned long jump_entry_code(const struct jump_entry *entry)
129 {
130 return (unsigned long)&entry->code + entry->code;
131 }
132
133 static inline unsigned long jump_entry_target(const struct jump_entry *entry)
134 {
135 return (unsigned long)&entry->target + entry->target;
136 }
137
138 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
139 {
140 long offset = entry->key & ~3L;
141
142 return (struct static_key *)((unsigned long)&entry->key + offset);
143 }
144
145 #else
146
147 static inline unsigned long jump_entry_code(const struct jump_entry *entry)
148 {
149 return entry->code;
150 }
151
152 static inline unsigned long jump_entry_target(const struct jump_entry *entry)
153 {
154 return entry->target;
155 }
156
157 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
158 {
159 return (struct static_key *)((unsigned long)entry->key & ~3UL);
160 }
161
162 #endif
163
164 static inline bool jump_entry_is_branch(const struct jump_entry *entry)
165 {
166 return (unsigned long)entry->key & 1UL;
167 }
168
169 static inline bool jump_entry_is_init(const struct jump_entry *entry)
170 {
171 return (unsigned long)entry->key & 2UL;
172 }
173
174 static inline void jump_entry_set_init(struct jump_entry *entry)
175 {
176 entry->key |= 2;
177 }
178
179 #endif
180 #endif
181
182 #ifndef __ASSEMBLY__
183
184 enum jump_label_type {
185 JUMP_LABEL_NOP = 0,
186 JUMP_LABEL_JMP,
187 };
188
189 struct module;
190
191 #ifdef CONFIG_JUMP_LABEL
192
193 #define JUMP_TYPE_FALSE 0UL
194 #define JUMP_TYPE_TRUE 1UL
195 #define JUMP_TYPE_LINKED 2UL
196 #define JUMP_TYPE_MASK 3UL
197
198 static __always_inline bool static_key_false(struct static_key *key)
199 {
200 return arch_static_branch(key, false);
201 }
202
203 static __always_inline bool static_key_true(struct static_key *key)
204 {
205 return !arch_static_branch(key, true);
206 }
207
208 extern struct jump_entry __start___jump_table[];
209 extern struct jump_entry __stop___jump_table[];
210
211 extern void jump_label_init(void);
212 extern void jump_label_lock(void);
213 extern void jump_label_unlock(void);
214 extern void arch_jump_label_transform(struct jump_entry *entry,
215 enum jump_label_type type);
216 extern void arch_jump_label_transform_static(struct jump_entry *entry,
217 enum jump_label_type type);
218 extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
219 enum jump_label_type type);
220 extern void arch_jump_label_transform_apply(void);
221 extern int jump_label_text_reserved(void *start, void *end);
222 extern void static_key_slow_inc(struct static_key *key);
223 extern void static_key_slow_dec(struct static_key *key);
224 extern void static_key_slow_inc_cpuslocked(struct static_key *key);
225 extern void static_key_slow_dec_cpuslocked(struct static_key *key);
226 extern void jump_label_apply_nops(struct module *mod);
227 extern int static_key_count(struct static_key *key);
228 extern void static_key_enable(struct static_key *key);
229 extern void static_key_disable(struct static_key *key);
230 extern void static_key_enable_cpuslocked(struct static_key *key);
231 extern void static_key_disable_cpuslocked(struct static_key *key);
232
233
234
235
236
237
238
239
240 #define STATIC_KEY_INIT_TRUE \
241 { .enabled = { 1 }, \
242 { .entries = (void *)JUMP_TYPE_TRUE } }
243 #define STATIC_KEY_INIT_FALSE \
244 { .enabled = { 0 }, \
245 { .entries = (void *)JUMP_TYPE_FALSE } }
246
247 #else
248
249 #include <linux/atomic.h>
250 #include <linux/bug.h>
251
252 static inline int static_key_count(struct static_key *key)
253 {
254 return atomic_read(&key->enabled);
255 }
256
257 static __always_inline void jump_label_init(void)
258 {
259 static_key_initialized = true;
260 }
261
262 static __always_inline bool static_key_false(struct static_key *key)
263 {
264 if (unlikely(static_key_count(key) > 0))
265 return true;
266 return false;
267 }
268
269 static __always_inline bool static_key_true(struct static_key *key)
270 {
271 if (likely(static_key_count(key) > 0))
272 return true;
273 return false;
274 }
275
276 static inline void static_key_slow_inc(struct static_key *key)
277 {
278 STATIC_KEY_CHECK_USE(key);
279 atomic_inc(&key->enabled);
280 }
281
282 static inline void static_key_slow_dec(struct static_key *key)
283 {
284 STATIC_KEY_CHECK_USE(key);
285 atomic_dec(&key->enabled);
286 }
287
288 #define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)
289 #define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)
290
291 static inline int jump_label_text_reserved(void *start, void *end)
292 {
293 return 0;
294 }
295
296 static inline void jump_label_lock(void) {}
297 static inline void jump_label_unlock(void) {}
298
299 static inline int jump_label_apply_nops(struct module *mod)
300 {
301 return 0;
302 }
303
304 static inline void static_key_enable(struct static_key *key)
305 {
306 STATIC_KEY_CHECK_USE(key);
307
308 if (atomic_read(&key->enabled) != 0) {
309 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
310 return;
311 }
312 atomic_set(&key->enabled, 1);
313 }
314
315 static inline void static_key_disable(struct static_key *key)
316 {
317 STATIC_KEY_CHECK_USE(key);
318
319 if (atomic_read(&key->enabled) != 1) {
320 WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
321 return;
322 }
323 atomic_set(&key->enabled, 0);
324 }
325
326 #define static_key_enable_cpuslocked(k) static_key_enable((k))
327 #define static_key_disable_cpuslocked(k) static_key_disable((k))
328
329 #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
330 #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
331
332 #endif
333
334 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
335 #define jump_label_enabled static_key_enabled
336
337
338
339
340
341
342
343
344
345
346 struct static_key_true {
347 struct static_key key;
348 };
349
350 struct static_key_false {
351 struct static_key key;
352 };
353
354 #define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
355 #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
356
357 #define DEFINE_STATIC_KEY_TRUE(name) \
358 struct static_key_true name = STATIC_KEY_TRUE_INIT
359
360 #define DEFINE_STATIC_KEY_TRUE_RO(name) \
361 struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
362
363 #define DECLARE_STATIC_KEY_TRUE(name) \
364 extern struct static_key_true name
365
366 #define DEFINE_STATIC_KEY_FALSE(name) \
367 struct static_key_false name = STATIC_KEY_FALSE_INIT
368
369 #define DEFINE_STATIC_KEY_FALSE_RO(name) \
370 struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
371
372 #define DECLARE_STATIC_KEY_FALSE(name) \
373 extern struct static_key_false name
374
375 #define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
376 struct static_key_true name[count] = { \
377 [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
378 }
379
380 #define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
381 struct static_key_false name[count] = { \
382 [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
383 }
384
385 extern bool ____wrong_branch_error(void);
386
387 #define static_key_enabled(x) \
388 ({ \
389 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
390 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
391 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
392 ____wrong_branch_error(); \
393 static_key_count((struct static_key *)x) > 0; \
394 })
395
396 #ifdef CONFIG_JUMP_LABEL
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454 #define static_branch_likely(x) \
455 ({ \
456 bool branch; \
457 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
458 branch = !arch_static_branch(&(x)->key, true); \
459 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
460 branch = !arch_static_branch_jump(&(x)->key, true); \
461 else \
462 branch = ____wrong_branch_error(); \
463 likely(branch); \
464 })
465
466 #define static_branch_unlikely(x) \
467 ({ \
468 bool branch; \
469 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
470 branch = arch_static_branch_jump(&(x)->key, false); \
471 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
472 branch = arch_static_branch(&(x)->key, false); \
473 else \
474 branch = ____wrong_branch_error(); \
475 unlikely(branch); \
476 })
477
478 #else
479
480 #define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
481 #define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
482
483 #endif
484
485
486
487
488
489 #define static_branch_inc(x) static_key_slow_inc(&(x)->key)
490 #define static_branch_dec(x) static_key_slow_dec(&(x)->key)
491 #define static_branch_inc_cpuslocked(x) static_key_slow_inc_cpuslocked(&(x)->key)
492 #define static_branch_dec_cpuslocked(x) static_key_slow_dec_cpuslocked(&(x)->key)
493
494
495
496
497
498 #define static_branch_enable(x) static_key_enable(&(x)->key)
499 #define static_branch_disable(x) static_key_disable(&(x)->key)
500 #define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key)
501 #define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key)
502
503 #endif
504
505 #endif