285 static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)  in uprobe_init_insn()  argument
289 	insn_init(insn, auprobe->insn, sizeof(auprobe->insn), x86_64);  in uprobe_init_insn()
344 static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)  in riprel_analyze()  argument
359 		cursor = auprobe->insn + insn_offset_rex_prefix(insn);  in riprel_analyze()
376 		cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1;  in riprel_analyze()
442 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_SI;  in riprel_analyze()
445 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_DI;  in riprel_analyze()
449 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_BX;  in riprel_analyze()
456 	cursor = auprobe->insn + insn_offset_modrm(insn);  in riprel_analyze()
466 scratch_reg(struct arch_uprobe *auprobe, struct pt_regs *regs)  in scratch_reg()  argument
468 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_SI)  in scratch_reg()
470 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_DI)  in scratch_reg()
479 static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_pre_xol()  argument
481 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {  in riprel_pre_xol()
483 		unsigned long *sr = scratch_reg(auprobe, regs);  in riprel_pre_xol()
486 		*sr = utask->vaddr + auprobe->defparam.ilen;  in riprel_pre_xol()
490 static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_post_xol()  argument
492 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {  in riprel_post_xol()
494 		unsigned long *sr = scratch_reg(auprobe, regs);  in riprel_post_xol()
507 static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)  in riprel_analyze()  argument
510 static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_pre_xol()  argument
513 static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in riprel_post_xol()  argument
530 static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in default_pre_xol_op()  argument
532 	riprel_pre_xol(auprobe, regs);  in default_pre_xol_op()
564 static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in default_post_xol_op()  argument
568 	riprel_post_xol(auprobe, regs);  in default_post_xol_op()
569 	if (auprobe->defparam.fixups & UPROBE_FIX_IP) {  in default_post_xol_op()
572 	} else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) {  in default_post_xol_op()
574 		if (push_ret_address(regs, utask->vaddr + auprobe->defparam.ilen))  in default_post_xol_op()
578 	if (auprobe->defparam.fixups & UPROBE_FIX_SETF)  in default_post_xol_op()
584 static void default_abort_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in default_abort_op()  argument
586 	riprel_post_xol(auprobe, regs);  in default_abort_op()
595 static bool branch_is_call(struct arch_uprobe *auprobe)  in branch_is_call()  argument
597 	return auprobe->branch.opc1 == 0xe8;  in branch_is_call()
629 static bool check_jmp_cond(struct arch_uprobe *auprobe, struct pt_regs *regs)  in check_jmp_cond()  argument
633 	switch (auprobe->branch.opc1) {  in check_jmp_cond()
648 static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in branch_emulate_op()  argument
650 	unsigned long new_ip = regs->ip += auprobe->branch.ilen;  in branch_emulate_op()
651 	unsigned long offs = (long)auprobe->branch.offs;  in branch_emulate_op()
653 	if (branch_is_call(auprobe)) {  in branch_emulate_op()
665 	} else if (!check_jmp_cond(auprobe, regs)) {  in branch_emulate_op()
673 static int branch_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)  in branch_post_xol_op()  argument
675 	BUG_ON(!branch_is_call(auprobe));  in branch_post_xol_op()
686 static void branch_clear_offset(struct arch_uprobe *auprobe, struct insn *insn)  in branch_clear_offset()  argument
702 	memset(auprobe->insn + insn_offset_immediate(insn),  in branch_clear_offset()
712 static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)  in branch_setup_xol_ops()  argument
724 		branch_clear_offset(auprobe, insn);  in branch_setup_xol_ops()
750 	auprobe->branch.opc1 = opc1;  in branch_setup_xol_ops()
751 	auprobe->branch.ilen = insn->length;  in branch_setup_xol_ops()
752 	auprobe->branch.offs = insn->immediate.value;  in branch_setup_xol_ops()
754 	auprobe->ops = &branch_xol_ops;  in branch_setup_xol_ops()
765 int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr)  in arch_uprobe_analyze_insn()  argument
771 	ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));  in arch_uprobe_analyze_insn()
775 	ret = branch_setup_xol_ops(auprobe, &insn);  in arch_uprobe_analyze_insn()
785 		auprobe->defparam.fixups |= UPROBE_FIX_SETF;  in arch_uprobe_analyze_insn()
808 		riprel_analyze(auprobe, &insn);  in arch_uprobe_analyze_insn()
811 	auprobe->defparam.ilen = insn.length;  in arch_uprobe_analyze_insn()
812 	auprobe->defparam.fixups |= fix_ip_or_call;  in arch_uprobe_analyze_insn()
814 	auprobe->ops = &default_xol_ops;  in arch_uprobe_analyze_insn()
823 int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_pre_xol()  argument
827 	if (auprobe->ops->pre_xol) {  in arch_uprobe_pre_xol()
828 		int err = auprobe->ops->pre_xol(auprobe, regs);  in arch_uprobe_pre_xol()
870 int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_post_xol()  argument
879 	if (auprobe->ops->post_xol) {  in arch_uprobe_post_xol()
880 		err = auprobe->ops->post_xol(auprobe, regs);  in arch_uprobe_post_xol()
941 void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_abort_xol()  argument
945 	if (auprobe->ops->abort)  in arch_uprobe_abort_xol()
946 		auprobe->ops->abort(auprobe, regs);  in arch_uprobe_abort_xol()
955 static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)  in __skip_sstep()  argument
957 	if (auprobe->ops->emulate)  in __skip_sstep()
958 		return auprobe->ops->emulate(auprobe, regs);  in __skip_sstep()
962 bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)  in arch_uprobe_skip_sstep()  argument
964 	bool ret = __skip_sstep(auprobe, regs);  in arch_uprobe_skip_sstep()