337 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)  in copy_kprobe()  argument
339 	memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));  in copy_kprobe()
340 	memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));  in copy_kprobe()
668 static void reuse_unused_kprobe(struct kprobe *ap)  in reuse_unused_kprobe()  argument
672 	BUG_ON(!kprobe_unused(ap));  in reuse_unused_kprobe()
677 	op = container_of(ap, struct optimized_kprobe, kp);  in reuse_unused_kprobe()
680 			"aggrprobe@%p\n", ap->addr);  in reuse_unused_kprobe()
682 	ap->flags &= ~KPROBE_FLAG_DISABLED;  in reuse_unused_kprobe()
684 	BUG_ON(!kprobe_optready(ap));  in reuse_unused_kprobe()
685 	optimize_kprobe(ap);  in reuse_unused_kprobe()
739 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
747 	struct kprobe *ap;  in try_to_optimize_kprobe()  local
758 	ap = alloc_aggr_kprobe(p);  in try_to_optimize_kprobe()
759 	if (!ap)  in try_to_optimize_kprobe()
762 	op = container_of(ap, struct optimized_kprobe, kp);  in try_to_optimize_kprobe()
770 	init_aggr_kprobe(ap, p);  in try_to_optimize_kprobe()
771 	optimize_kprobe(ap);	/* This just kicks optimizer thread */  in try_to_optimize_kprobe()
898 static void reuse_unused_kprobe(struct kprobe *ap)  in reuse_unused_kprobe()  argument
901 	BUG_ON(kprobe_unused(ap));  in reuse_unused_kprobe()
1207 static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)  in add_new_kprobe()  argument
1209 	BUG_ON(kprobe_gone(ap) || kprobe_gone(p));  in add_new_kprobe()
1212 		unoptimize_kprobe(ap, true);	/* Fall back to normal kprobe */  in add_new_kprobe()
1215 		if (ap->break_handler)  in add_new_kprobe()
1217 		list_add_tail_rcu(&p->list, &ap->list);  in add_new_kprobe()
1218 		ap->break_handler = aggr_break_handler;  in add_new_kprobe()
1220 		list_add_rcu(&p->list, &ap->list);  in add_new_kprobe()
1221 	if (p->post_handler && !ap->post_handler)  in add_new_kprobe()
1222 		ap->post_handler = aggr_post_handler;  in add_new_kprobe()
1231 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)  in init_aggr_kprobe()  argument
1234 	copy_kprobe(p, ap);  in init_aggr_kprobe()
1235 	flush_insn_slot(ap);  in init_aggr_kprobe()
1236 	ap->addr = p->addr;  in init_aggr_kprobe()
1237 	ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;  in init_aggr_kprobe()
1238 	ap->pre_handler = aggr_pre_handler;  in init_aggr_kprobe()
1239 	ap->fault_handler = aggr_fault_handler;  in init_aggr_kprobe()
1242 		ap->post_handler = aggr_post_handler;  in init_aggr_kprobe()
1244 		ap->break_handler = aggr_break_handler;  in init_aggr_kprobe()
1246 	INIT_LIST_HEAD(&ap->list);  in init_aggr_kprobe()
1247 	INIT_HLIST_NODE(&ap->hlist);  in init_aggr_kprobe()
1249 	list_add_rcu(&p->list, &ap->list);  in init_aggr_kprobe()
1250 	hlist_replace_rcu(&p->hlist, &ap->hlist);  in init_aggr_kprobe()
1260 	struct kprobe *ap = orig_p;  in register_aggr_kprobe()  local
1273 		ap = alloc_aggr_kprobe(orig_p);  in register_aggr_kprobe()
1274 		if (!ap) {  in register_aggr_kprobe()
1278 		init_aggr_kprobe(ap, orig_p);  in register_aggr_kprobe()
1279 	} else if (kprobe_unused(ap))  in register_aggr_kprobe()
1281 		reuse_unused_kprobe(ap);  in register_aggr_kprobe()
1283 	if (kprobe_gone(ap)) {  in register_aggr_kprobe()
1290 		ret = arch_prepare_kprobe(ap);  in register_aggr_kprobe()
1300 		prepare_optimized_kprobe(ap);  in register_aggr_kprobe()
1306 		ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)  in register_aggr_kprobe()
1311 	copy_kprobe(ap, p);  in register_aggr_kprobe()
1312 	ret = add_new_kprobe(ap, p);  in register_aggr_kprobe()
1319 	if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {  in register_aggr_kprobe()
1320 		ap->flags &= ~KPROBE_FLAG_DISABLED;  in register_aggr_kprobe()
1323 			arm_kprobe(ap);  in register_aggr_kprobe()
1384 	struct kprobe *ap, *list_p;  in __get_valid_kprobe()  local
1386 	ap = get_kprobe(p->addr);  in __get_valid_kprobe()
1387 	if (unlikely(!ap))  in __get_valid_kprobe()
1390 	if (p != ap) {  in __get_valid_kprobe()
1391 		list_for_each_entry_rcu(list_p, &ap->list, list)  in __get_valid_kprobe()
1398 	return ap;  in __get_valid_kprobe()
1543 static int aggr_kprobe_disabled(struct kprobe *ap)  in aggr_kprobe_disabled()  argument
1547 	list_for_each_entry_rcu(kp, &ap->list, list)  in aggr_kprobe_disabled()
1594 	struct kprobe *ap, *list_p;  in __unregister_kprobe_top()  local
1597 	ap = __disable_kprobe(p);  in __unregister_kprobe_top()
1598 	if (ap == NULL)  in __unregister_kprobe_top()
1601 	if (ap == p)  in __unregister_kprobe_top()
1609 	WARN_ON(!kprobe_aggrprobe(ap));  in __unregister_kprobe_top()
1611 	if (list_is_singular(&ap->list) && kprobe_disarmed(ap))  in __unregister_kprobe_top()
1620 			ap->break_handler = NULL;  in __unregister_kprobe_top()
1622 			list_for_each_entry_rcu(list_p, &ap->list, list) {  in __unregister_kprobe_top()
1626 			ap->post_handler = NULL;  in __unregister_kprobe_top()
1634 		if (!kprobe_disabled(ap) && !kprobes_all_disarmed)  in __unregister_kprobe_top()
1639 			optimize_kprobe(ap);  in __unregister_kprobe_top()
1644 	BUG_ON(!kprobe_disarmed(ap));  in __unregister_kprobe_top()
1645 	hlist_del_rcu(&ap->hlist);  in __unregister_kprobe_top()
1651 	struct kprobe *ap;  in __unregister_kprobe_bottom()  local
1658 		ap = list_entry(p->list.next, struct kprobe, list);  in __unregister_kprobe_bottom()
1660 		free_aggr_kprobe(ap);  in __unregister_kprobe_bottom()