519 	struct hlist_head		*hash;  member
696 	memset(stat->hash, 0,  in ftrace_profile_reset()
763 	if (stat->hash) {  in ftrace_profile_init_cpu()
775 	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);  in ftrace_profile_init_cpu()
777 	if (!stat->hash)  in ftrace_profile_init_cpu()
782 		kfree(stat->hash);  in ftrace_profile_init_cpu()
783 		stat->hash = NULL;  in ftrace_profile_init_cpu()
813 	hhd = &stat->hash[key];  in ftrace_find_profiled_func()
832 	hlist_add_head_rcu(&rec->node, &stat->hash[key]);  in ftrace_add_profile()
885 	if (!stat->hash || !ftrace_profile_enabled)  in function_profile_call()
916 	if (!stat->hash || !ftrace_profile_enabled)  in profile_graph_return()
1220 static bool __always_inline ftrace_hash_empty(struct ftrace_hash *hash)  in ftrace_hash_empty()  argument
1222 	return !hash || !hash->count;  in ftrace_hash_empty()
1226 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)  in ftrace_lookup_ip()  argument
1232 	if (ftrace_hash_empty(hash))  in ftrace_lookup_ip()
1235 	if (hash->size_bits > 0)  in ftrace_lookup_ip()
1236 		key = hash_long(ip, hash->size_bits);  in ftrace_lookup_ip()
1240 	hhd = &hash->buckets[key];  in ftrace_lookup_ip()
1249 static void __add_hash_entry(struct ftrace_hash *hash,  in __add_hash_entry()  argument
1255 	if (hash->size_bits)  in __add_hash_entry()
1256 		key = hash_long(entry->ip, hash->size_bits);  in __add_hash_entry()
1260 	hhd = &hash->buckets[key];  in __add_hash_entry()
1262 	hash->count++;  in __add_hash_entry()
1265 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)  in add_hash_entry()  argument
1274 	__add_hash_entry(hash, entry);  in add_hash_entry()
1280 free_hash_entry(struct ftrace_hash *hash,  in free_hash_entry()  argument
1285 	hash->count--;  in free_hash_entry()
1289 remove_hash_entry(struct ftrace_hash *hash,  in remove_hash_entry()  argument
1293 	hash->count--;  in remove_hash_entry()
1296 static void ftrace_hash_clear(struct ftrace_hash *hash)  in ftrace_hash_clear()  argument
1301 	int size = 1 << hash->size_bits;  in ftrace_hash_clear()
1304 	if (!hash->count)  in ftrace_hash_clear()
1308 		hhd = &hash->buckets[i];  in ftrace_hash_clear()
1310 			free_hash_entry(hash, entry);  in ftrace_hash_clear()
1312 	FTRACE_WARN_ON(hash->count);  in ftrace_hash_clear()
1315 static void free_ftrace_hash(struct ftrace_hash *hash)  in free_ftrace_hash()  argument
1317 	if (!hash || hash == EMPTY_HASH)  in free_ftrace_hash()
1319 	ftrace_hash_clear(hash);  in free_ftrace_hash()
1320 	kfree(hash->buckets);  in free_ftrace_hash()
1321 	kfree(hash);  in free_ftrace_hash()
1326 	struct ftrace_hash *hash;  in __free_ftrace_hash_rcu()  local
1328 	hash = container_of(rcu, struct ftrace_hash, rcu);  in __free_ftrace_hash_rcu()
1329 	free_ftrace_hash(hash);  in __free_ftrace_hash_rcu()
1332 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)  in free_ftrace_hash_rcu()  argument
1334 	if (!hash || hash == EMPTY_HASH)  in free_ftrace_hash_rcu()
1336 	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);  in free_ftrace_hash_rcu()
1348 	struct ftrace_hash *hash;  in alloc_ftrace_hash()  local
1351 	hash = kzalloc(sizeof(*hash), GFP_KERNEL);  in alloc_ftrace_hash()
1352 	if (!hash)  in alloc_ftrace_hash()
1356 	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);  in alloc_ftrace_hash()
1358 	if (!hash->buckets) {  in alloc_ftrace_hash()
1359 		kfree(hash);  in alloc_ftrace_hash()
1363 	hash->size_bits = size_bits;  in alloc_ftrace_hash()
1365 	return hash;  in alloc_ftrace_hash()
1369 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)  in alloc_and_copy_ftrace_hash()  argument
1382 	if (ftrace_hash_empty(hash))  in alloc_and_copy_ftrace_hash()
1385 	size = 1 << hash->size_bits;  in alloc_and_copy_ftrace_hash()
1387 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {  in alloc_and_copy_ftrace_hash()
1394 	FTRACE_WARN_ON(new_hash->count != hash->count);  in alloc_and_copy_ftrace_hash()
1485 			     struct ftrace_ops_hash *hash)  in hash_contains_ip()  argument
1493 	return (ftrace_hash_empty(hash->filter_hash) ||  in hash_contains_ip()
1494 		ftrace_lookup_ip(hash->filter_hash, ip)) &&  in hash_contains_ip()
1495 		(ftrace_hash_empty(hash->notrace_hash) ||  in hash_contains_ip()
1496 		 !ftrace_lookup_ip(hash->notrace_hash, ip));  in hash_contains_ip()
1514 	struct ftrace_ops_hash hash;  in ftrace_ops_test()  local
1527 	hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);  in ftrace_ops_test()
1528 	hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);  in ftrace_ops_test()
1530 	if (hash_contains_ip(ip, &hash))  in ftrace_ops_test()
1646 	struct ftrace_hash *hash;  in __ftrace_hash_rec_update()  local
1669 		hash = ops->func_hash->filter_hash;  in __ftrace_hash_rec_update()
1671 		if (ftrace_hash_empty(hash))  in __ftrace_hash_rec_update()
1675 		hash = ops->func_hash->notrace_hash;  in __ftrace_hash_rec_update()
1681 		if (ftrace_hash_empty(hash))  in __ftrace_hash_rec_update()
1698 			in_hash = !!ftrace_lookup_ip(hash, rec->ip);  in __ftrace_hash_rec_update()
1783 		if (!all && count == hash->count)  in __ftrace_hash_rec_update()
1910 	struct ftrace_hash *hash = ops->func_hash->filter_hash;  in ftrace_hash_ipmodify_enable()  local
1912 	if (ftrace_hash_empty(hash))  in ftrace_hash_ipmodify_enable()
1913 		hash = NULL;  in ftrace_hash_ipmodify_enable()
1915 	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);  in ftrace_hash_ipmodify_enable()
1921 	struct ftrace_hash *hash = ops->func_hash->filter_hash;  in ftrace_hash_ipmodify_disable()  local
1923 	if (ftrace_hash_empty(hash))  in ftrace_hash_ipmodify_disable()
1924 		hash = NULL;  in ftrace_hash_ipmodify_disable()
1926 	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);  in ftrace_hash_ipmodify_disable()
3006 	struct ftrace_hash		*hash;  member
3344 	struct ftrace_hash *hash;  in ftrace_regex_open()  local
3367 		hash = ops->func_hash->notrace_hash;  in ftrace_regex_open()
3369 		hash = ops->func_hash->filter_hash;  in ftrace_regex_open()
3375 			iter->hash = alloc_ftrace_hash(size_bits);  in ftrace_regex_open()
3377 			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);  in ftrace_regex_open()
3379 		if (!iter->hash) {  in ftrace_regex_open()
3396 			free_ftrace_hash(iter->hash);  in ftrace_regex_open()
3465 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)  in enter_record()  argument
3470 	entry = ftrace_lookup_ip(hash, rec->ip);  in enter_record()
3476 		free_hash_entry(hash, entry);  in enter_record()
3482 		ret = add_hash_entry(hash, rec->ip);  in enter_record()
3528 match_records(struct ftrace_hash *hash, char *func, int len, char *mod)  in match_records()  argument
3559 			ret = enter_record(hash, rec, clear_filter);  in match_records()
3574 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)  in ftrace_match_records()  argument
3576 	return match_records(hash, buff, len, NULL);  in ftrace_match_records()
3586 ftrace_mod_callback(struct ftrace_hash *hash,  in ftrace_mod_callback()  argument
3598 	ret = match_records(hash, func, strlen(func), module);  in ftrace_mod_callback()
3716 	struct ftrace_hash *hash;  in register_ftrace_function_probe()  local
3738 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);  in register_ftrace_function_probe()
3739 	if (!hash) {  in register_ftrace_function_probe()
3781 		ret = enter_record(hash, rec, 0);  in register_ftrace_function_probe()
3796 	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);  in register_ftrace_function_probe()
3809 	free_ftrace_hash(hash);  in register_ftrace_function_probe()
3830 	struct ftrace_hash *hash;  in __unregister_ftrace_function_probe()  local
3852 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);  in __unregister_ftrace_function_probe()
3853 	if (!hash)  in __unregister_ftrace_function_probe()
3879 			rec_entry = ftrace_lookup_ip(hash, entry->ip);  in __unregister_ftrace_function_probe()
3882 				free_hash_entry(hash, rec_entry);  in __unregister_ftrace_function_probe()
3894 	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);  in __unregister_ftrace_function_probe()
3907 	free_ftrace_hash(hash);  in __unregister_ftrace_function_probe()
3978 static int ftrace_process_regex(struct ftrace_hash *hash,  in ftrace_process_regex()  argument
3988 		ret = ftrace_match_records(hash, func, len);  in ftrace_process_regex()
4003 			ret = p->func(hash, func, command, next, enable);  in ftrace_process_regex()
4040 		ret = ftrace_process_regex(iter->hash, parser->buffer,  in ftrace_regex_write()
4067 ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)  in ftrace_match_addr()  argument
4075 		entry = ftrace_lookup_ip(hash, ip);  in ftrace_match_addr()
4078 		free_hash_entry(hash, entry);  in ftrace_match_addr()
4082 	return add_hash_entry(hash, ip);  in ftrace_match_addr()
4123 	struct ftrace_hash *hash;  in ftrace_set_hash()  local
4137 		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);  in ftrace_set_hash()
4139 		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);  in ftrace_set_hash()
4141 	if (!hash) {  in ftrace_set_hash()
4146 	if (buf && !ftrace_match_records(hash, buf, len)) {  in ftrace_set_hash()
4151 		ret = ftrace_match_addr(hash, ip, remove);  in ftrace_set_hash()
4160 	ret = ftrace_hash_move(ops, enable, orig_hash, hash);  in ftrace_set_hash()
4170 	free_ftrace_hash(hash);  in ftrace_set_hash()
4391 		ftrace_match_records(iter->hash, parser->buffer, parser->idx);  in ftrace_regex_release()
4411 				       orig_hash, iter->hash);  in ftrace_regex_release()
4420 	free_ftrace_hash(iter->hash);  in ftrace_regex_release()