Lines Matching refs:p
307 struct kprobe *p; in get_kprobe() local
310 hlist_for_each_entry_rcu(p, head, hlist) { in get_kprobe()
311 if (p->addr == addr) in get_kprobe()
312 return p; in get_kprobe()
319 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
322 static inline int kprobe_aggrprobe(struct kprobe *p) in kprobe_aggrprobe() argument
324 return p->pre_handler == aggr_pre_handler; in kprobe_aggrprobe()
328 static inline int kprobe_unused(struct kprobe *p) in kprobe_unused() argument
330 return kprobe_aggrprobe(p) && kprobe_disabled(p) && in kprobe_unused()
331 list_empty(&p->list); in kprobe_unused()
337 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p) in copy_kprobe() argument
339 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t)); in copy_kprobe()
340 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn)); in copy_kprobe()
351 void opt_pre_handler(struct kprobe *p, struct pt_regs *regs) in opt_pre_handler() argument
355 list_for_each_entry_rcu(kp, &p->list, list) { in opt_pre_handler()
366 static void free_aggr_kprobe(struct kprobe *p) in free_aggr_kprobe() argument
370 op = container_of(p, struct optimized_kprobe, kp); in free_aggr_kprobe()
372 arch_remove_kprobe(p); in free_aggr_kprobe()
377 static inline int kprobe_optready(struct kprobe *p) in kprobe_optready() argument
381 if (kprobe_aggrprobe(p)) { in kprobe_optready()
382 op = container_of(p, struct optimized_kprobe, kp); in kprobe_optready()
390 static inline int kprobe_disarmed(struct kprobe *p) in kprobe_disarmed() argument
395 if (!kprobe_aggrprobe(p)) in kprobe_disarmed()
396 return kprobe_disabled(p); in kprobe_disarmed()
398 op = container_of(p, struct optimized_kprobe, kp); in kprobe_disarmed()
400 return kprobe_disabled(p) && list_empty(&op->list); in kprobe_disarmed()
404 static int kprobe_queued(struct kprobe *p) in kprobe_queued() argument
408 if (kprobe_aggrprobe(p)) { in kprobe_queued()
409 op = container_of(p, struct optimized_kprobe, kp); in kprobe_queued()
423 struct kprobe *p = NULL; in get_optimized_kprobe() local
427 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++) in get_optimized_kprobe()
428 p = get_kprobe((void *)(addr - i)); in get_optimized_kprobe()
430 if (p && kprobe_optready(p)) { in get_optimized_kprobe()
431 op = container_of(p, struct optimized_kprobe, kp); in get_optimized_kprobe()
433 return p; in get_optimized_kprobe()
585 static void optimize_kprobe(struct kprobe *p) in optimize_kprobe() argument
590 if (!kprobe_optready(p) || !kprobes_allow_optimization || in optimize_kprobe()
591 (kprobe_disabled(p) || kprobes_all_disarmed)) in optimize_kprobe()
595 if (p->break_handler || p->post_handler) in optimize_kprobe()
598 op = container_of(p, struct optimized_kprobe, kp); in optimize_kprobe()
629 static void unoptimize_kprobe(struct kprobe *p, bool force) in unoptimize_kprobe() argument
633 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p)) in unoptimize_kprobe()
636 op = container_of(p, struct optimized_kprobe, kp); in unoptimize_kprobe()
637 if (!kprobe_optimized(p)) { in unoptimize_kprobe()
689 static void kill_optimized_kprobe(struct kprobe *p) in kill_optimized_kprobe() argument
693 op = container_of(p, struct optimized_kprobe, kp); in kill_optimized_kprobe()
699 if (kprobe_unused(p)) { in kill_optimized_kprobe()
715 static void prepare_optimized_kprobe(struct kprobe *p) in prepare_optimized_kprobe() argument
719 op = container_of(p, struct optimized_kprobe, kp); in prepare_optimized_kprobe()
720 arch_prepare_optimized_kprobe(op, p); in prepare_optimized_kprobe()
724 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) in alloc_aggr_kprobe() argument
733 op->kp.addr = p->addr; in alloc_aggr_kprobe()
734 arch_prepare_optimized_kprobe(op, p); in alloc_aggr_kprobe()
739 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
745 static void try_to_optimize_kprobe(struct kprobe *p) in try_to_optimize_kprobe() argument
751 if (kprobe_ftrace(p)) in try_to_optimize_kprobe()
758 ap = alloc_aggr_kprobe(p); in try_to_optimize_kprobe()
770 init_aggr_kprobe(ap, p); in try_to_optimize_kprobe()
782 struct kprobe *p; in optimize_all_kprobes() local
793 hlist_for_each_entry_rcu(p, head, hlist) in optimize_all_kprobes()
794 if (!kprobe_disabled(p)) in optimize_all_kprobes()
795 optimize_kprobe(p); in optimize_all_kprobes()
805 struct kprobe *p; in unoptimize_all_kprobes() local
818 hlist_for_each_entry_rcu(p, head, hlist) { in unoptimize_all_kprobes()
819 if (!kprobe_disabled(p)) in unoptimize_all_kprobes()
820 unoptimize_kprobe(p, false); in unoptimize_all_kprobes()
853 static void __arm_kprobe(struct kprobe *p) in __arm_kprobe() argument
858 _p = get_optimized_kprobe((unsigned long)p->addr); in __arm_kprobe()
863 arch_arm_kprobe(p); in __arm_kprobe()
864 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ in __arm_kprobe()
868 static void __disarm_kprobe(struct kprobe *p, bool reopt) in __disarm_kprobe() argument
873 unoptimize_kprobe(p, kprobes_all_disarmed); in __disarm_kprobe()
875 if (!kprobe_queued(p)) { in __disarm_kprobe()
876 arch_disarm_kprobe(p); in __disarm_kprobe()
878 _p = get_optimized_kprobe((unsigned long)p->addr); in __disarm_kprobe()
887 #define optimize_kprobe(p) do {} while (0) argument
888 #define unoptimize_kprobe(p, f) do {} while (0) argument
889 #define kill_optimized_kprobe(p) do {} while (0) argument
890 #define prepare_optimized_kprobe(p) do {} while (0) argument
891 #define try_to_optimize_kprobe(p) do {} while (0) argument
892 #define __arm_kprobe(p) arch_arm_kprobe(p) argument
893 #define __disarm_kprobe(p, o) arch_disarm_kprobe(p) argument
894 #define kprobe_disarmed(p) kprobe_disabled(p) argument
904 static void free_aggr_kprobe(struct kprobe *p) in free_aggr_kprobe() argument
906 arch_remove_kprobe(p); in free_aggr_kprobe()
907 kfree(p); in free_aggr_kprobe()
910 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) in alloc_aggr_kprobe() argument
924 static int prepare_kprobe(struct kprobe *p) in prepare_kprobe() argument
926 if (!kprobe_ftrace(p)) in prepare_kprobe()
927 return arch_prepare_kprobe(p); in prepare_kprobe()
929 return arch_prepare_kprobe_ftrace(p); in prepare_kprobe()
933 static void arm_kprobe_ftrace(struct kprobe *p) in arm_kprobe_ftrace() argument
938 (unsigned long)p->addr, 0, 0); in arm_kprobe_ftrace()
939 WARN(ret < 0, "Failed to arm kprobe-ftrace at %p (%d)\n", p->addr, ret); in arm_kprobe_ftrace()
948 static void disarm_kprobe_ftrace(struct kprobe *p) in disarm_kprobe_ftrace() argument
958 (unsigned long)p->addr, 1, 0); in disarm_kprobe_ftrace()
959 WARN(ret < 0, "Failed to disarm kprobe-ftrace at %p (%d)\n", p->addr, ret); in disarm_kprobe_ftrace()
962 #define prepare_kprobe(p) arch_prepare_kprobe(p) argument
963 #define arm_kprobe_ftrace(p) do {} while (0) argument
964 #define disarm_kprobe_ftrace(p) do {} while (0) argument
1001 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) in aggr_pre_handler() argument
1005 list_for_each_entry_rcu(kp, &p->list, list) { in aggr_pre_handler()
1017 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, in aggr_post_handler() argument
1022 list_for_each_entry_rcu(kp, &p->list, list) { in aggr_post_handler()
1032 static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, in aggr_fault_handler() argument
1049 static int aggr_break_handler(struct kprobe *p, struct pt_regs *regs) in aggr_break_handler() argument
1064 void kprobes_inc_nmissed_count(struct kprobe *p) in kprobes_inc_nmissed_count() argument
1067 if (!kprobe_aggrprobe(p)) { in kprobes_inc_nmissed_count()
1068 p->nmissed++; in kprobes_inc_nmissed_count()
1070 list_for_each_entry_rcu(kp, &p->list, list) in kprobes_inc_nmissed_count()
1207 static int add_new_kprobe(struct kprobe *ap, struct kprobe *p) in add_new_kprobe() argument
1209 BUG_ON(kprobe_gone(ap) || kprobe_gone(p)); in add_new_kprobe()
1211 if (p->break_handler || p->post_handler) in add_new_kprobe()
1214 if (p->break_handler) { in add_new_kprobe()
1217 list_add_tail_rcu(&p->list, &ap->list); in add_new_kprobe()
1220 list_add_rcu(&p->list, &ap->list); in add_new_kprobe()
1221 if (p->post_handler && !ap->post_handler) in add_new_kprobe()
1231 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p) in init_aggr_kprobe() argument
1234 copy_kprobe(p, ap); in init_aggr_kprobe()
1236 ap->addr = p->addr; in init_aggr_kprobe()
1237 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED; in init_aggr_kprobe()
1241 if (p->post_handler && !kprobe_gone(p)) in init_aggr_kprobe()
1243 if (p->break_handler && !kprobe_gone(p)) in init_aggr_kprobe()
1249 list_add_rcu(&p->list, &ap->list); in init_aggr_kprobe()
1250 hlist_replace_rcu(&p->hlist, &ap->hlist); in init_aggr_kprobe()
1257 static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p) in register_aggr_kprobe() argument
1311 copy_kprobe(ap, p); in register_aggr_kprobe()
1312 ret = add_new_kprobe(ap, p); in register_aggr_kprobe()
1319 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) { in register_aggr_kprobe()
1359 static kprobe_opcode_t *kprobe_addr(struct kprobe *p) in kprobe_addr() argument
1361 kprobe_opcode_t *addr = p->addr; in kprobe_addr()
1363 if ((p->symbol_name && p->addr) || in kprobe_addr()
1364 (!p->symbol_name && !p->addr)) in kprobe_addr()
1367 if (p->symbol_name) { in kprobe_addr()
1368 kprobe_lookup_name(p->symbol_name, addr); in kprobe_addr()
1373 addr = (kprobe_opcode_t *)(((char *)addr) + p->offset); in kprobe_addr()
1382 static struct kprobe *__get_valid_kprobe(struct kprobe *p) in __get_valid_kprobe() argument
1386 ap = get_kprobe(p->addr); in __get_valid_kprobe()
1390 if (p != ap) { in __get_valid_kprobe()
1392 if (list_p == p) in __get_valid_kprobe()
1402 static inline int check_kprobe_rereg(struct kprobe *p) in check_kprobe_rereg() argument
1407 if (__get_valid_kprobe(p)) in check_kprobe_rereg()
1414 int __weak arch_check_ftrace_location(struct kprobe *p) in arch_check_ftrace_location() argument
1418 ftrace_addr = ftrace_location((unsigned long)p->addr); in arch_check_ftrace_location()
1422 if ((unsigned long)p->addr != ftrace_addr) in arch_check_ftrace_location()
1424 p->flags |= KPROBE_FLAG_FTRACE; in arch_check_ftrace_location()
1432 static int check_kprobe_address_safe(struct kprobe *p, in check_kprobe_address_safe() argument
1437 ret = arch_check_ftrace_location(p); in check_kprobe_address_safe()
1444 if (!kernel_text_address((unsigned long) p->addr) || in check_kprobe_address_safe()
1445 within_kprobe_blacklist((unsigned long) p->addr) || in check_kprobe_address_safe()
1446 jump_label_text_reserved(p->addr, p->addr)) { in check_kprobe_address_safe()
1452 *probed_mod = __module_text_address((unsigned long) p->addr); in check_kprobe_address_safe()
1467 if (within_module_init((unsigned long)p->addr, *probed_mod) && in check_kprobe_address_safe()
1481 int register_kprobe(struct kprobe *p) in register_kprobe() argument
1489 addr = kprobe_addr(p); in register_kprobe()
1492 p->addr = addr; in register_kprobe()
1494 ret = check_kprobe_rereg(p); in register_kprobe()
1499 p->flags &= KPROBE_FLAG_DISABLED; in register_kprobe()
1500 p->nmissed = 0; in register_kprobe()
1501 INIT_LIST_HEAD(&p->list); in register_kprobe()
1503 ret = check_kprobe_address_safe(p, &probed_mod); in register_kprobe()
1509 old_p = get_kprobe(p->addr); in register_kprobe()
1512 ret = register_aggr_kprobe(old_p, p); in register_kprobe()
1517 ret = prepare_kprobe(p); in register_kprobe()
1522 INIT_HLIST_NODE(&p->hlist); in register_kprobe()
1523 hlist_add_head_rcu(&p->hlist, in register_kprobe()
1524 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); in register_kprobe()
1526 if (!kprobes_all_disarmed && !kprobe_disabled(p)) in register_kprobe()
1527 arm_kprobe(p); in register_kprobe()
1530 try_to_optimize_kprobe(p); in register_kprobe()
1559 static struct kprobe *__disable_kprobe(struct kprobe *p) in __disable_kprobe() argument
1564 orig_p = __get_valid_kprobe(p); in __disable_kprobe()
1568 if (!kprobe_disabled(p)) { in __disable_kprobe()
1570 if (p != orig_p) in __disable_kprobe()
1571 p->flags |= KPROBE_FLAG_DISABLED; in __disable_kprobe()
1574 if (p == orig_p || aggr_kprobe_disabled(orig_p)) { in __disable_kprobe()
1592 static int __unregister_kprobe_top(struct kprobe *p) in __unregister_kprobe_top() argument
1597 ap = __disable_kprobe(p); in __unregister_kprobe_top()
1601 if (ap == p) in __unregister_kprobe_top()
1619 if (p->break_handler && !kprobe_gone(p)) in __unregister_kprobe_top()
1621 if (p->post_handler && !kprobe_gone(p)) { in __unregister_kprobe_top()
1623 if ((list_p != p) && (list_p->post_handler)) in __unregister_kprobe_top()
1633 list_del_rcu(&p->list); in __unregister_kprobe_top()
1649 static void __unregister_kprobe_bottom(struct kprobe *p) in __unregister_kprobe_bottom() argument
1653 if (list_empty(&p->list)) in __unregister_kprobe_bottom()
1655 arch_remove_kprobe(p); in __unregister_kprobe_bottom()
1656 else if (list_is_singular(&p->list)) { in __unregister_kprobe_bottom()
1658 ap = list_entry(p->list.next, struct kprobe, list); in __unregister_kprobe_bottom()
1659 list_del(&p->list); in __unregister_kprobe_bottom()
1683 void unregister_kprobe(struct kprobe *p) in unregister_kprobe() argument
1685 unregister_kprobes(&p, 1); in unregister_kprobe()
1786 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) in pre_handler_kretprobe() argument
1788 struct kretprobe *rp = container_of(p, struct kretprobe, kp); in pre_handler_kretprobe()
1959 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) in pre_handler_kretprobe() argument
1968 static void kill_kprobe(struct kprobe *p) in kill_kprobe() argument
1972 p->flags |= KPROBE_FLAG_GONE; in kill_kprobe()
1973 if (kprobe_aggrprobe(p)) { in kill_kprobe()
1978 list_for_each_entry_rcu(kp, &p->list, list) in kill_kprobe()
1980 p->post_handler = NULL; in kill_kprobe()
1981 p->break_handler = NULL; in kill_kprobe()
1982 kill_optimized_kprobe(p); in kill_kprobe()
1988 arch_remove_kprobe(p); in kill_kprobe()
2011 struct kprobe *p; in enable_kprobe() local
2016 p = __get_valid_kprobe(kp); in enable_kprobe()
2017 if (unlikely(p == NULL)) { in enable_kprobe()
2028 if (p != kp) in enable_kprobe()
2031 if (!kprobes_all_disarmed && kprobe_disabled(p)) { in enable_kprobe()
2032 p->flags &= ~KPROBE_FLAG_DISABLED; in enable_kprobe()
2033 arm_kprobe(p); in enable_kprobe()
2091 struct kprobe *p; in kprobes_module_callback() local
2107 hlist_for_each_entry_rcu(p, head, hlist) in kprobes_module_callback()
2108 if (within_module_init((unsigned long)p->addr, mod) || in kprobes_module_callback()
2110 within_module_core((unsigned long)p->addr, mod))) { in kprobes_module_callback()
2116 kill_kprobe(p); in kprobes_module_callback()
2188 static void report_probe(struct seq_file *pi, struct kprobe *p, in report_probe() argument
2193 if (p->pre_handler == pre_handler_kretprobe) in report_probe()
2195 else if (p->pre_handler == setjmp_pre_handler) in report_probe()
2202 p->addr, kprobe_type, sym, offset, in report_probe()
2206 p->addr, kprobe_type, p->addr); in report_probe()
2209 pp = p; in report_probe()
2211 (kprobe_gone(p) ? "[GONE]" : ""), in report_probe()
2212 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""), in report_probe()
2238 struct kprobe *p, *kp; in show_kprobe_addr() local
2246 hlist_for_each_entry_rcu(p, head, hlist) { in show_kprobe_addr()
2247 sym = kallsyms_lookup((unsigned long)p->addr, NULL, in show_kprobe_addr()
2249 if (kprobe_aggrprobe(p)) { in show_kprobe_addr()
2250 list_for_each_entry_rcu(kp, &p->list, list) in show_kprobe_addr()
2251 report_probe(pi, kp, sym, offset, modname, p); in show_kprobe_addr()
2253 report_probe(pi, p, sym, offset, modname, NULL); in show_kprobe_addr()
2321 struct kprobe *p; in arm_all_kprobes() local
2339 hlist_for_each_entry_rcu(p, head, hlist) in arm_all_kprobes()
2340 if (!kprobe_disabled(p)) in arm_all_kprobes()
2341 arm_kprobe(p); in arm_all_kprobes()
2354 struct kprobe *p; in disarm_all_kprobes() local
2370 hlist_for_each_entry_rcu(p, head, hlist) { in disarm_all_kprobes()
2371 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) in disarm_all_kprobes()
2372 disarm_kprobe(p, false); in disarm_all_kprobes()