286 static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)  in uprobe_init_insn()  argument
290 	insn_init(insn, auprobe->insn, sizeof(auprobe->insn), x86_64);  in uprobe_init_insn()
340 static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)  in riprel_analyze()  argument
355 		cursor = auprobe->insn + insn_offset_rex_prefix(insn);  in riprel_analyze()
372 		cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1;  in riprel_analyze()
438 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_SI;  in riprel_analyze()
441 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_DI;  in riprel_analyze()
445 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_BX;  in riprel_analyze()
452 	cursor = auprobe->insn + insn_offset_modrm(insn);  in riprel_analyze()
462 scratch_reg(struct arch_uprobe *auprobe, struct pt_regs *regs)  in scratch_reg()  argument
464 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_SI)  in scratch_reg()
466 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_DI)  in scratch_reg()
475 static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_pre_xol()  argument
477 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {  in riprel_pre_xol()
479 		unsigned long *sr = scratch_reg(auprobe, regs);  in riprel_pre_xol()
482 		*sr = utask->vaddr + auprobe->defparam.ilen;  in riprel_pre_xol()
486 static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_post_xol()  argument
488 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {  in riprel_post_xol()
490 		unsigned long *sr = scratch_reg(auprobe, regs);  in riprel_post_xol()
499 static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)  in riprel_analyze()  argument
502 static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_pre_xol()  argument
505 static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_post_xol()  argument
522 static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in default_pre_xol_op()  argument
524 	riprel_pre_xol(auprobe, regs);  in default_pre_xol_op()
556 static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in default_post_xol_op()  argument
560 	riprel_post_xol(auprobe, regs);  in default_post_xol_op()
561 	if (auprobe->defparam.fixups & UPROBE_FIX_IP) {  in default_post_xol_op()
564 	} else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) {  in default_post_xol_op()
566 		if (push_ret_address(regs, utask->vaddr + auprobe->defparam.ilen))  in default_post_xol_op()
570 	if (auprobe->defparam.fixups & UPROBE_FIX_SETF)  in default_post_xol_op()
576 static void default_abort_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in default_abort_op()  argument
578 	riprel_post_xol(auprobe, regs);  in default_abort_op()
587 static bool branch_is_call(struct arch_uprobe *auprobe)  in branch_is_call()  argument
589 	return auprobe->branch.opc1 == 0xe8;  in branch_is_call()
621 static bool check_jmp_cond(struct arch_uprobe *auprobe, struct pt_regs *regs)  in check_jmp_cond()  argument
625 	switch (auprobe->branch.opc1) {  in check_jmp_cond()
640 static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in branch_emulate_op()  argument
642 	unsigned long new_ip = regs->ip += auprobe->branch.ilen;  in branch_emulate_op()
643 	unsigned long offs = (long)auprobe->branch.offs;  in branch_emulate_op()
645 	if (branch_is_call(auprobe)) {  in branch_emulate_op()
657 	} else if (!check_jmp_cond(auprobe, regs)) {  in branch_emulate_op()
665 static int branch_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in branch_post_xol_op()  argument
667 	BUG_ON(!branch_is_call(auprobe));  in branch_post_xol_op()
678 static void branch_clear_offset(struct arch_uprobe *auprobe, struct insn *insn)  in branch_clear_offset()  argument
694 	memset(auprobe->insn + insn_offset_immediate(insn),  in branch_clear_offset()
704 static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)  in branch_setup_xol_ops()  argument
716 		branch_clear_offset(auprobe, insn);  in branch_setup_xol_ops()
742 	auprobe->branch.opc1 = opc1;  in branch_setup_xol_ops()
743 	auprobe->branch.ilen = insn->length;  in branch_setup_xol_ops()
744 	auprobe->branch.offs = insn->immediate.value;  in branch_setup_xol_ops()
746 	auprobe->ops = &branch_xol_ops;  in branch_setup_xol_ops()
757 int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr)  in arch_uprobe_analyze_insn()  argument
763 	ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));  in arch_uprobe_analyze_insn()
767 	ret = branch_setup_xol_ops(auprobe, &insn);  in arch_uprobe_analyze_insn()
777 		auprobe->defparam.fixups |= UPROBE_FIX_SETF;  in arch_uprobe_analyze_insn()
800 		riprel_analyze(auprobe, &insn);  in arch_uprobe_analyze_insn()
803 	auprobe->defparam.ilen = insn.length;  in arch_uprobe_analyze_insn()
804 	auprobe->defparam.fixups |= fix_ip_or_call;  in arch_uprobe_analyze_insn()
806 	auprobe->ops = &default_xol_ops;  in arch_uprobe_analyze_insn()
815 int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_pre_xol()  argument
819 	if (auprobe->ops->pre_xol) {  in arch_uprobe_pre_xol()
820 		int err = auprobe->ops->pre_xol(auprobe, regs);  in arch_uprobe_pre_xol()
862 int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_post_xol()  argument
871 	if (auprobe->ops->post_xol) {  in arch_uprobe_post_xol()
872 		err = auprobe->ops->post_xol(auprobe, regs);  in arch_uprobe_post_xol()
933 void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_abort_xol()  argument
937 	if (auprobe->ops->abort)  in arch_uprobe_abort_xol()
938 		auprobe->ops->abort(auprobe, regs);  in arch_uprobe_abort_xol()
947 static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)  in __skip_sstep()  argument
949 	if (auprobe->ops->emulate)  in __skip_sstep()
950 		return auprobe->ops->emulate(auprobe, regs);  in __skip_sstep()
954 bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_skip_sstep()  argument
956 	bool ret = __skip_sstep(auprobe, regs);  in arch_uprobe_skip_sstep()