38 	if (jea->key < jeb->key)  in jump_label_cmp()
41 	if (jea->key > jeb->key)  in jump_label_cmp()
57 static void jump_label_update(struct static_key *key);
59 void static_key_slow_inc(struct static_key *key)  in static_key_slow_inc()  argument
62 	if (atomic_inc_not_zero(&key->enabled))  in static_key_slow_inc()
66 	if (atomic_inc_return(&key->enabled) == 1)  in static_key_slow_inc()
67 		jump_label_update(key);  in static_key_slow_inc()
72 static void __static_key_slow_dec(struct static_key *key,  in __static_key_slow_dec()  argument
75 	if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {  in __static_key_slow_dec()
76 		WARN(atomic_read(&key->enabled) < 0,  in __static_key_slow_dec()
82 		atomic_inc(&key->enabled);  in __static_key_slow_dec()
85 		jump_label_update(key);  in __static_key_slow_dec()
92 	struct static_key_deferred *key =  in jump_label_update_timeout()  local
94 	__static_key_slow_dec(&key->key, 0, NULL);  in jump_label_update_timeout()
97 void static_key_slow_dec(struct static_key *key)  in static_key_slow_dec()  argument
100 	__static_key_slow_dec(key, 0, NULL);  in static_key_slow_dec()
104 void static_key_slow_dec_deferred(struct static_key_deferred *key)  in static_key_slow_dec_deferred()  argument
107 	__static_key_slow_dec(&key->key, key->timeout, &key->work);  in static_key_slow_dec_deferred()
111 void jump_label_rate_limit(struct static_key_deferred *key,  in jump_label_rate_limit()  argument
115 	key->timeout = rl;  in jump_label_rate_limit()
116 	INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);  in jump_label_rate_limit()
156 static inline struct jump_entry *static_key_entries(struct static_key *key)  in static_key_entries()  argument
158 	return (struct jump_entry *)((unsigned long)key->entries & ~JUMP_TYPE_MASK);  in static_key_entries()
161 static inline bool static_key_type(struct static_key *key)  in static_key_type()  argument
163 	return (unsigned long)key->entries & JUMP_TYPE_MASK;  in static_key_type()
168 	return (struct static_key *)((unsigned long)entry->key & ~1UL);  in jump_entry_key()
173 	return (unsigned long)entry->key & 1UL;  in jump_entry_branch()
178 	struct static_key *key = jump_entry_key(entry);  in jump_label_type()  local
179 	bool enabled = static_key_enabled(key);  in jump_label_type()
186 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
190 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
205 	struct static_key *key = NULL;  in jump_label_init()  local
219 		if (iterk == key)  in jump_label_init()
222 		key = iterk;  in jump_label_init()
226 		*((unsigned long *)&key->entries) += (unsigned long)iter;  in jump_label_init()
228 		key->next = NULL;  in jump_label_init()
239 	struct static_key *key = jump_entry_key(entry);  in jump_label_init_type()  local
240 	bool type = static_key_type(key);  in jump_label_init_type()
268 static void __jump_label_mod_update(struct static_key *key)  in __jump_label_mod_update()  argument
272 	for (mod = key->next; mod; mod = mod->next) {  in __jump_label_mod_update()
275 		__jump_label_update(key, mod->entries,  in __jump_label_mod_update()
310 	struct static_key *key = NULL;  in jump_label_add_module()  local
323 		if (iterk == key)  in jump_label_add_module()
326 		key = iterk;  in jump_label_add_module()
327 		if (within_module(iter->key, mod)) {  in jump_label_add_module()
331 			*((unsigned long *)&key->entries) += (unsigned long)iter;  in jump_label_add_module()
332 			key->next = NULL;  in jump_label_add_module()
340 		jlm->next = key->next;  in jump_label_add_module()
341 		key->next = jlm;  in jump_label_add_module()
345 			__jump_label_update(key, iter, iter_stop);  in jump_label_add_module()
356 	struct static_key *key = NULL;  in jump_label_del_module()  local
360 		if (jump_entry_key(iter) == key)  in jump_label_del_module()
363 		key = jump_entry_key(iter);  in jump_label_del_module()
365 		if (within_module(iter->key, mod))  in jump_label_del_module()
368 		prev = &key->next;  in jump_label_del_module()
369 		jlm = key->next;  in jump_label_del_module()
465 static void jump_label_update(struct static_key *key)  in jump_label_update()  argument
468 	struct jump_entry *entry = static_key_entries(key);  in jump_label_update()
472 	__jump_label_mod_update(key);  in jump_label_update()
475 	mod = __module_address((unsigned long)key);  in jump_label_update()
482 		__jump_label_update(key, entry, stop);  in jump_label_update()
494 		WARN_ON(static_key_enabled(&sk_true.key) != true);  in jump_label_test()
495 		WARN_ON(static_key_enabled(&sk_false.key) != false);  in jump_label_test()
505 		WARN_ON(static_key_enabled(&sk_true.key) == true);  in jump_label_test()
506 		WARN_ON(static_key_enabled(&sk_false.key) == false);  in jump_label_test()