Lines Matching refs:rec
539 struct ftrace_profile *rec = v; in function_stat_next() local
542 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); in function_stat_next()
546 rec++; in function_stat_next()
548 if ((void *)rec >= (void *)&pg->records[pg->index]) { in function_stat_next()
552 rec = &pg->records[0]; in function_stat_next()
553 if (!rec->counter) in function_stat_next()
557 return rec; in function_stat_next()
617 struct ftrace_profile *rec = v; in function_stat_show() local
628 if (unlikely(rec->counter == 0)) { in function_stat_show()
633 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); in function_stat_show()
634 seq_printf(m, " %-30.30s %10lu", str, rec->counter); in function_stat_show()
638 avg = rec->time; in function_stat_show()
639 do_div(avg, rec->counter); in function_stat_show()
642 if (rec->counter <= 1) in function_stat_show()
649 stddev = rec->counter * rec->time_squared - in function_stat_show()
650 rec->time * rec->time; in function_stat_show()
656 do_div(stddev, rec->counter * (rec->counter - 1) * 1000); in function_stat_show()
660 trace_print_graph_duration(rec->time, &s); in function_stat_show()
798 struct ftrace_profile *rec; in ftrace_find_profiled_func() local
808 hlist_for_each_entry_rcu_notrace(rec, hhd, node) { in ftrace_find_profiled_func()
809 if (rec->ip == ip) in ftrace_find_profiled_func()
810 return rec; in ftrace_find_profiled_func()
817 struct ftrace_profile *rec) in ftrace_add_profile() argument
821 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); in ftrace_add_profile()
822 hlist_add_head_rcu(&rec->node, &stat->hash[key]); in ftrace_add_profile()
831 struct ftrace_profile *rec = NULL; in ftrace_profile_alloc() local
841 rec = ftrace_find_profiled_func(stat, ip); in ftrace_profile_alloc()
842 if (rec) in ftrace_profile_alloc()
851 rec = &stat->pages->records[stat->pages->index++]; in ftrace_profile_alloc()
852 rec->ip = ip; in ftrace_profile_alloc()
853 ftrace_add_profile(stat, rec); in ftrace_profile_alloc()
858 return rec; in ftrace_profile_alloc()
866 struct ftrace_profile *rec; in function_profile_call() local
878 rec = ftrace_find_profiled_func(stat, ip); in function_profile_call()
879 if (!rec) { in function_profile_call()
880 rec = ftrace_profile_alloc(stat, ip); in function_profile_call()
881 if (!rec) in function_profile_call()
885 rec->counter++; in function_profile_call()
901 struct ftrace_profile *rec; in profile_graph_return() local
930 rec = ftrace_find_profiled_func(stat, trace->func); in profile_graph_return()
931 if (rec) { in profile_graph_return()
932 rec->time += calltime; in profile_graph_return()
933 rec->time_squared += calltime * calltime; in profile_graph_return()
1532 #define do_for_each_ftrace_rec(pg, rec) \ argument
1536 rec = &pg->records[_____i];
1546 const struct dyn_ftrace *rec = b; in ftrace_cmp_recs() local
1548 if (key->flags < rec->ip) in ftrace_cmp_recs()
1550 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) in ftrace_cmp_recs()
1558 struct dyn_ftrace *rec; in ftrace_location_range() local
1568 rec = bsearch(&key, pg->records, pg->index, in ftrace_location_range()
1571 if (rec) in ftrace_location_range()
1572 return rec->ip; in ftrace_location_range()
1613 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) in test_rec_ops_needs_regs() argument
1621 if (ftrace_ops_test(ops, rec->ip, rec)) { in test_rec_ops_needs_regs()
1639 struct dyn_ftrace *rec; in __ftrace_hash_rec_update() local
1675 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_rec_update()
1685 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) in __ftrace_hash_rec_update()
1688 in_hash = !!ftrace_lookup_ip(hash, rec->ip); in __ftrace_hash_rec_update()
1689 in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip); in __ftrace_hash_rec_update()
1711 rec->flags++; in __ftrace_hash_rec_update()
1712 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX)) in __ftrace_hash_rec_update()
1720 if (ftrace_rec_count(rec) == 1 && ops->trampoline) in __ftrace_hash_rec_update()
1721 rec->flags |= FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1729 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1736 rec->flags |= FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1738 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0)) in __ftrace_hash_rec_update()
1740 rec->flags--; in __ftrace_hash_rec_update()
1748 if (ftrace_rec_count(rec) > 0 && in __ftrace_hash_rec_update()
1749 rec->flags & FTRACE_FL_REGS && in __ftrace_hash_rec_update()
1751 if (!test_rec_ops_needs_regs(rec)) in __ftrace_hash_rec_update()
1752 rec->flags &= ~FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1764 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1839 struct dyn_ftrace *rec, *end = NULL; in __ftrace_hash_update_ipmodify() local
1857 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_update_ipmodify()
1859 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1860 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1866 if (rec->flags & FTRACE_FL_IPMODIFY) in __ftrace_hash_update_ipmodify()
1868 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1870 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1876 end = rec; in __ftrace_hash_update_ipmodify()
1879 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_update_ipmodify()
1880 if (rec == end) in __ftrace_hash_update_ipmodify()
1883 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1884 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1889 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1891 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1944 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1958 void ftrace_bug(int failed, struct dyn_ftrace *rec) in ftrace_bug() argument
1960 unsigned long ip = rec ? rec->ip : 0; in ftrace_bug()
1985 if (rec) { in ftrace_bug()
1988 pr_info("ftrace record flags: %lx\n", rec->flags); in ftrace_bug()
1989 pr_cont(" (%ld)%s", ftrace_rec_count(rec), in ftrace_bug()
1990 rec->flags & FTRACE_FL_REGS ? " R" : " "); in ftrace_bug()
1991 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_bug()
1992 ops = ftrace_find_tramp_ops_any(rec); in ftrace_bug()
2000 ip = ftrace_get_addr_curr(rec); in ftrace_bug()
2005 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) in ftrace_check_record() argument
2020 if (enable && ftrace_rec_count(rec)) in ftrace_check_record()
2029 if (!(rec->flags & FTRACE_FL_REGS) != in ftrace_check_record()
2030 !(rec->flags & FTRACE_FL_REGS_EN)) in ftrace_check_record()
2033 if (!(rec->flags & FTRACE_FL_TRAMP) != in ftrace_check_record()
2034 !(rec->flags & FTRACE_FL_TRAMP_EN)) in ftrace_check_record()
2039 if ((rec->flags & FTRACE_FL_ENABLED) == flag) in ftrace_check_record()
2044 flag ^= rec->flags & FTRACE_FL_ENABLED; in ftrace_check_record()
2047 rec->flags |= FTRACE_FL_ENABLED; in ftrace_check_record()
2049 if (rec->flags & FTRACE_FL_REGS) in ftrace_check_record()
2050 rec->flags |= FTRACE_FL_REGS_EN; in ftrace_check_record()
2052 rec->flags &= ~FTRACE_FL_REGS_EN; in ftrace_check_record()
2055 if (rec->flags & FTRACE_FL_TRAMP) in ftrace_check_record()
2056 rec->flags |= FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2058 rec->flags &= ~FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2078 if (!ftrace_rec_count(rec)) in ftrace_check_record()
2079 rec->flags = 0; in ftrace_check_record()
2085 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | in ftrace_check_record()
2100 int ftrace_update_record(struct dyn_ftrace *rec, int enable) in ftrace_update_record() argument
2102 return ftrace_check_record(rec, enable, 1); in ftrace_update_record()
2114 int ftrace_test_record(struct dyn_ftrace *rec, int enable) in ftrace_test_record() argument
2116 return ftrace_check_record(rec, enable, 0); in ftrace_test_record()
2120 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec) in ftrace_find_tramp_ops_any() argument
2123 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_any()
2138 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec) in ftrace_find_tramp_ops_curr() argument
2141 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_curr()
2208 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec) in ftrace_find_tramp_ops_new() argument
2211 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_new()
2232 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec) in ftrace_get_addr_new() argument
2237 if (rec->flags & FTRACE_FL_TRAMP) { in ftrace_get_addr_new()
2238 ops = ftrace_find_tramp_ops_new(rec); in ftrace_get_addr_new()
2241 (void *)rec->ip, (void *)rec->ip, rec->flags); in ftrace_get_addr_new()
2248 if (rec->flags & FTRACE_FL_REGS) in ftrace_get_addr_new()
2264 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec) in ftrace_get_addr_curr() argument
2269 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_get_addr_curr()
2270 ops = ftrace_find_tramp_ops_curr(rec); in ftrace_get_addr_curr()
2273 (void *)rec->ip, (void *)rec->ip); in ftrace_get_addr_curr()
2280 if (rec->flags & FTRACE_FL_REGS_EN) in ftrace_get_addr_curr()
2287 __ftrace_replace_code(struct dyn_ftrace *rec, int enable) in __ftrace_replace_code() argument
2293 ftrace_addr = ftrace_get_addr_new(rec); in __ftrace_replace_code()
2296 ftrace_old_addr = ftrace_get_addr_curr(rec); in __ftrace_replace_code()
2298 ret = ftrace_update_record(rec, enable); in __ftrace_replace_code()
2305 return ftrace_make_call(rec, ftrace_addr); in __ftrace_replace_code()
2308 return ftrace_make_nop(NULL, rec, ftrace_old_addr); in __ftrace_replace_code()
2311 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr); in __ftrace_replace_code()
2319 struct dyn_ftrace *rec; in ftrace_replace_code() local
2326 do_for_each_ftrace_rec(pg, rec) { in ftrace_replace_code()
2327 failed = __ftrace_replace_code(rec, enable); in ftrace_replace_code()
2329 ftrace_bug(failed, rec); in ftrace_replace_code()
2409 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) in ftrace_code_disable() argument
2416 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); in ftrace_code_disable()
2418 ftrace_bug(ret, rec); in ftrace_code_disable()
2689 struct dyn_ftrace *rec; in ftrace_shutdown() local
2691 do_for_each_ftrace_rec(pg, rec) { in ftrace_shutdown()
2692 if (FTRACE_WARN_ON_ONCE(rec->flags)) in ftrace_shutdown()
2694 (void *)rec->ip, rec->flags); in ftrace_shutdown()
2785 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) in ops_references_rec() argument
2797 !ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip)) in ops_references_rec()
2801 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) in ops_references_rec()
2807 static int referenced_filters(struct dyn_ftrace *rec) in referenced_filters() argument
2813 if (ops_references_rec(ops, rec)) in referenced_filters()
3075 struct ftrace_func_probe *rec; in t_hash_show() local
3077 rec = iter->probe; in t_hash_show()
3078 if (WARN_ON_ONCE(!rec)) in t_hash_show()
3081 if (rec->ops->print) in t_hash_show()
3082 return rec->ops->print(m, rec->ip, rec->ops, rec->data); in t_hash_show()
3084 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func); in t_hash_show()
3086 if (rec->data) in t_hash_show()
3087 seq_printf(m, ":%p", rec->data); in t_hash_show()
3098 struct dyn_ftrace *rec = NULL; in t_next() local
3120 rec = &iter->pg->records[iter->idx++]; in t_next()
3122 !(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) || in t_next()
3125 !ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) || in t_next()
3128 !(rec->flags & FTRACE_FL_ENABLED))) { in t_next()
3130 rec = NULL; in t_next()
3135 if (!rec) in t_next()
3138 iter->func = rec; in t_next()
3213 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) in arch_ftrace_trampoline_func() argument
3219 struct dyn_ftrace *rec) in add_trampoline_func() argument
3223 ptr = arch_ftrace_trampoline_func(ops, rec); in add_trampoline_func()
3231 struct dyn_ftrace *rec; in t_show() local
3244 rec = iter->func; in t_show()
3246 if (!rec) in t_show()
3249 seq_printf(m, "%ps", (void *)rec->ip); in t_show()
3254 ftrace_rec_count(rec), in t_show()
3255 rec->flags & FTRACE_FL_REGS ? " R" : " ", in t_show()
3256 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " "); in t_show()
3257 if (rec->flags & FTRACE_FL_TRAMP_EN) { in t_show()
3258 ops = ftrace_find_tramp_ops_any(rec); in t_show()
3266 add_trampoline_func(m, ops, rec); in t_show()
3447 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not) in enter_record() argument
3452 entry = ftrace_lookup_ip(hash, rec->ip); in enter_record()
3464 ret = add_hash_entry(hash, rec->ip); in enter_record()
3470 ftrace_match_record(struct dyn_ftrace *rec, char *mod, in ftrace_match_record() argument
3476 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); in ftrace_match_record()
3497 struct dyn_ftrace *rec; in match_records() local
3513 do_for_each_ftrace_rec(pg, rec) { in match_records()
3514 if (ftrace_match_record(rec, mod, search, search_len, type)) { in match_records()
3515 ret = enter_record(hash, rec, not); in match_records()
3701 struct dyn_ftrace *rec; in register_ftrace_function_probe() local
3734 do_for_each_ftrace_rec(pg, rec) { in register_ftrace_function_probe()
3736 if (!ftrace_match_record(rec, NULL, search, len, type)) in register_ftrace_function_probe()
3757 if (ops->init(ops, rec->ip, &entry->data) < 0) { in register_ftrace_function_probe()
3764 ret = enter_record(hash, rec, 0); in register_ftrace_function_probe()
3772 entry->ip = rec->ip; in register_ftrace_function_probe()
4603 struct dyn_ftrace *rec; in ftrace_set_func() local
4626 do_for_each_ftrace_rec(pg, rec) { in ftrace_set_func()
4628 if (ftrace_match_record(rec, NULL, search, search_len, type)) { in ftrace_set_func()
4632 if (array[i] == rec->ip) { in ftrace_set_func()
4641 array[(*idx)++] = rec->ip; in ftrace_set_func()
4798 struct dyn_ftrace *rec; in ftrace_process_locs() local
4861 rec = &pg->records[pg->index++]; in ftrace_process_locs()
4862 rec->ip = addr; in ftrace_process_locs()
4897 struct dyn_ftrace *rec; in ftrace_release_mod() local
4913 rec = &pg->records[0]; in ftrace_release_mod()
4914 if (within_module_core(rec->ip, mod)) { in ftrace_release_mod()