1/* 2 * Kernel Probes (KProbes) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * Copyright (C) IBM Corporation, 2002, 2004 19 * 20 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 21 * Probes initial implementation ( includes contributions from 22 * Rusty Russell). 23 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 24 * interface to access function arguments. 25 * 2004-Oct Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 26 * <prasanna@in.ibm.com> adapted for x86_64 from i386. 27 * 2005-Mar Roland McGrath <roland@redhat.com> 28 * Fixed to handle %rip-relative addressing mode correctly. 29 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 30 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 31 * <prasanna@in.ibm.com> added function-return probes. 32 * 2005-May Rusty Lynch <rusty.lynch@intel.com> 33 * Added function return probes functionality 34 * 2006-Feb Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added 35 * kprobe-booster and kretprobe-booster for i386. 36 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster 37 * and kretprobe-booster for x86-64 38 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven 39 * <arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com> 40 * unified x86 kprobes code. 41 */ 42#include <linux/kprobes.h> 43#include <linux/ptrace.h> 44#include <linux/string.h> 45#include <linux/slab.h> 46#include <linux/hardirq.h> 47#include <linux/preempt.h> 48#include <linux/module.h> 49#include <linux/kdebug.h> 50#include <linux/kallsyms.h> 51#include <linux/ftrace.h> 52 53#include <asm/cacheflush.h> 54#include <asm/desc.h> 55#include <asm/pgtable.h> 56#include <asm/uaccess.h> 57#include <asm/alternative.h> 58#include <asm/insn.h> 59#include <asm/debugreg.h> 60 61#include "common.h" 62 63void jprobe_return_end(void); 64 65DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 66DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 67 68#define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs)) 69 70#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\ 71 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \ 72 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \ 73 (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \ 74 (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \ 75 << (row % 32)) 76 /* 77 * Undefined/reserved opcodes, conditional jump, Opcode Extension 78 * Groups, and some special opcodes can not boost. 79 * This is non-const and volatile to keep gcc from statically 80 * optimizing it out, as variable_test_bit makes gcc think only 81 * *(unsigned long*) is used. 82 */ 83static volatile u32 twobyte_is_boostable[256 / 32] = { 84 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 85 /* ---------------------------------------------- */ 86 W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */ 87 W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1) , /* 10 */ 88 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */ 89 W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */ 90 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */ 91 W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */ 92 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */ 93 W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */ 94 W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */ 95 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */ 96 W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */ 97 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */ 98 W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */ 99 W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */ 100 W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */ 101 W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0) /* f0 */ 102 /* ----------------------------------------------- */ 103 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 104}; 105#undef W 106 107struct kretprobe_blackpoint kretprobe_blacklist[] = { 108 {"__switch_to", }, /* This function switches only current task, but 109 doesn't switch kernel stack.*/ 110 {NULL, NULL} /* Terminator */ 111}; 112 113const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist); 114 115static nokprobe_inline void 116__synthesize_relative_insn(void *from, void *to, u8 op) 117{ 118 struct __arch_relative_insn { 119 u8 op; 120 s32 raddr; 121 } __packed *insn; 122 123 insn = (struct __arch_relative_insn *)from; 124 insn->raddr = (s32)((long)(to) - ((long)(from) + 5)); 125 insn->op = op; 126} 127 128/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/ 129void synthesize_reljump(void *from, void *to) 130{ 131 __synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE); 132} 133NOKPROBE_SYMBOL(synthesize_reljump); 134 135/* Insert a call instruction at address 'from', which calls address 'to'.*/ 136void synthesize_relcall(void *from, void *to) 137{ 138 __synthesize_relative_insn(from, to, RELATIVECALL_OPCODE); 139} 140NOKPROBE_SYMBOL(synthesize_relcall); 141 142/* 143 * Skip the prefixes of the instruction. 144 */ 145static kprobe_opcode_t *skip_prefixes(kprobe_opcode_t *insn) 146{ 147 insn_attr_t attr; 148 149 attr = inat_get_opcode_attribute((insn_byte_t)*insn); 150 while (inat_is_legacy_prefix(attr)) { 151 insn++; 152 attr = inat_get_opcode_attribute((insn_byte_t)*insn); 153 } 154#ifdef CONFIG_X86_64 155 if (inat_is_rex_prefix(attr)) 156 insn++; 157#endif 158 return insn; 159} 160NOKPROBE_SYMBOL(skip_prefixes); 161 162/* 163 * Returns non-zero if opcode is boostable. 164 * RIP relative instructions are adjusted at copying time in 64 bits mode 165 */ 166int can_boost(kprobe_opcode_t *opcodes) 167{ 168 kprobe_opcode_t opcode; 169 kprobe_opcode_t *orig_opcodes = opcodes; 170 171 if (search_exception_tables((unsigned long)opcodes)) 172 return 0; /* Page fault may occur on this address. */ 173 174retry: 175 if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1) 176 return 0; 177 opcode = *(opcodes++); 178 179 /* 2nd-byte opcode */ 180 if (opcode == 0x0f) { 181 if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1) 182 return 0; 183 return test_bit(*opcodes, 184 (unsigned long *)twobyte_is_boostable); 185 } 186 187 switch (opcode & 0xf0) { 188#ifdef CONFIG_X86_64 189 case 0x40: 190 goto retry; /* REX prefix is boostable */ 191#endif 192 case 0x60: 193 if (0x63 < opcode && opcode < 0x67) 194 goto retry; /* prefixes */ 195 /* can't boost Address-size override and bound */ 196 return (opcode != 0x62 && opcode != 0x67); 197 case 0x70: 198 return 0; /* can't boost conditional jump */ 199 case 0xc0: 200 /* can't boost software-interruptions */ 201 return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf; 202 case 0xd0: 203 /* can boost AA* and XLAT */ 204 return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7); 205 case 0xe0: 206 /* can boost in/out and absolute jmps */ 207 return ((opcode & 0x04) || opcode == 0xea); 208 case 0xf0: 209 if ((opcode & 0x0c) == 0 && opcode != 0xf1) 210 goto retry; /* lock/rep(ne) prefix */ 211 /* clear and set flags are boostable */ 212 return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe)); 213 default: 214 /* segment override prefixes are boostable */ 215 if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e) 216 goto retry; /* prefixes */ 217 /* CS override prefix and call are not boostable */ 218 return (opcode != 0x2e && opcode != 0x9a); 219 } 220} 221 222static unsigned long 223__recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr) 224{ 225 struct kprobe *kp; 226 unsigned long faddr; 227 228 kp = get_kprobe((void *)addr); 229 faddr = ftrace_location(addr); 230 /* 231 * Addresses inside the ftrace location are refused by 232 * arch_check_ftrace_location(). Something went terribly wrong 233 * if such an address is checked here. 234 */ 235 if (WARN_ON(faddr && faddr != addr)) 236 return 0UL; 237 /* 238 * Use the current code if it is not modified by Kprobe 239 * and it cannot be modified by ftrace. 240 */ 241 if (!kp && !faddr) 242 return addr; 243 244 /* 245 * Basically, kp->ainsn.insn has an original instruction. 246 * However, RIP-relative instruction can not do single-stepping 247 * at different place, __copy_instruction() tweaks the displacement of 248 * that instruction. In that case, we can't recover the instruction 249 * from the kp->ainsn.insn. 250 * 251 * On the other hand, in case on normal Kprobe, kp->opcode has a copy 252 * of the first byte of the probed instruction, which is overwritten 253 * by int3. And the instruction at kp->addr is not modified by kprobes 254 * except for the first byte, we can recover the original instruction 255 * from it and kp->opcode. 256 * 257 * In case of Kprobes using ftrace, we do not have a copy of 258 * the original instruction. In fact, the ftrace location might 259 * be modified at anytime and even could be in an inconsistent state. 260 * Fortunately, we know that the original code is the ideal 5-byte 261 * long NOP. 262 */ 263 memcpy(buf, (void *)addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); 264 if (faddr) 265 memcpy(buf, ideal_nops[NOP_ATOMIC5], 5); 266 else 267 buf[0] = kp->opcode; 268 return (unsigned long)buf; 269} 270 271/* 272 * Recover the probed instruction at addr for further analysis. 273 * Caller must lock kprobes by kprobe_mutex, or disable preemption 274 * for preventing to release referencing kprobes. 275 * Returns zero if the instruction can not get recovered. 276 */ 277unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr) 278{ 279 unsigned long __addr; 280 281 __addr = __recover_optprobed_insn(buf, addr); 282 if (__addr != addr) 283 return __addr; 284 285 return __recover_probed_insn(buf, addr); 286} 287 288/* Check if paddr is at an instruction boundary */ 289static int can_probe(unsigned long paddr) 290{ 291 unsigned long addr, __addr, offset = 0; 292 struct insn insn; 293 kprobe_opcode_t buf[MAX_INSN_SIZE]; 294 295 if (!kallsyms_lookup_size_offset(paddr, NULL, &offset)) 296 return 0; 297 298 /* Decode instructions */ 299 addr = paddr - offset; 300 while (addr < paddr) { 301 /* 302 * Check if the instruction has been modified by another 303 * kprobe, in which case we replace the breakpoint by the 304 * original instruction in our buffer. 305 * Also, jump optimization will change the breakpoint to 306 * relative-jump. Since the relative-jump itself is 307 * normally used, we just go through if there is no kprobe. 308 */ 309 __addr = recover_probed_instruction(buf, addr); 310 if (!__addr) 311 return 0; 312 kernel_insn_init(&insn, (void *)__addr, MAX_INSN_SIZE); 313 insn_get_length(&insn); 314 315 /* 316 * Another debugging subsystem might insert this breakpoint. 317 * In that case, we can't recover it. 318 */ 319 if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) 320 return 0; 321 addr += insn.length; 322 } 323 324 return (addr == paddr); 325} 326 327/* 328 * Returns non-zero if opcode modifies the interrupt flag. 329 */ 330static int is_IF_modifier(kprobe_opcode_t *insn) 331{ 332 /* Skip prefixes */ 333 insn = skip_prefixes(insn); 334 335 switch (*insn) { 336 case 0xfa: /* cli */ 337 case 0xfb: /* sti */ 338 case 0xcf: /* iret/iretd */ 339 case 0x9d: /* popf/popfd */ 340 return 1; 341 } 342 343 return 0; 344} 345 346/* 347 * Copy an instruction and adjust the displacement if the instruction 348 * uses the %rip-relative addressing mode. 349 * If it does, Return the address of the 32-bit displacement word. 350 * If not, return null. 351 * Only applicable to 64-bit x86. 352 */ 353int __copy_instruction(u8 *dest, u8 *src) 354{ 355 struct insn insn; 356 kprobe_opcode_t buf[MAX_INSN_SIZE]; 357 int length; 358 unsigned long recovered_insn = 359 recover_probed_instruction(buf, (unsigned long)src); 360 361 if (!recovered_insn) 362 return 0; 363 kernel_insn_init(&insn, (void *)recovered_insn, MAX_INSN_SIZE); 364 insn_get_length(&insn); 365 length = insn.length; 366 367 /* Another subsystem puts a breakpoint, failed to recover */ 368 if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) 369 return 0; 370 memcpy(dest, insn.kaddr, length); 371 372#ifdef CONFIG_X86_64 373 if (insn_rip_relative(&insn)) { 374 s64 newdisp; 375 u8 *disp; 376 kernel_insn_init(&insn, dest, length); 377 insn_get_displacement(&insn); 378 /* 379 * The copied instruction uses the %rip-relative addressing 380 * mode. Adjust the displacement for the difference between 381 * the original location of this instruction and the location 382 * of the copy that will actually be run. The tricky bit here 383 * is making sure that the sign extension happens correctly in 384 * this calculation, since we need a signed 32-bit result to 385 * be sign-extended to 64 bits when it's added to the %rip 386 * value and yield the same 64-bit result that the sign- 387 * extension of the original signed 32-bit displacement would 388 * have given. 389 */ 390 newdisp = (u8 *) src + (s64) insn.displacement.value - (u8 *) dest; 391 if ((s64) (s32) newdisp != newdisp) { 392 pr_err("Kprobes error: new displacement does not fit into s32 (%llx)\n", newdisp); 393 pr_err("\tSrc: %p, Dest: %p, old disp: %x\n", src, dest, insn.displacement.value); 394 return 0; 395 } 396 disp = (u8 *) dest + insn_offset_displacement(&insn); 397 *(s32 *) disp = (s32) newdisp; 398 } 399#endif 400 return length; 401} 402 403static int arch_copy_kprobe(struct kprobe *p) 404{ 405 int ret; 406 407 /* Copy an instruction with recovering if other optprobe modifies it.*/ 408 ret = __copy_instruction(p->ainsn.insn, p->addr); 409 if (!ret) 410 return -EINVAL; 411 412 /* 413 * __copy_instruction can modify the displacement of the instruction, 414 * but it doesn't affect boostable check. 415 */ 416 if (can_boost(p->ainsn.insn)) 417 p->ainsn.boostable = 0; 418 else 419 p->ainsn.boostable = -1; 420 421 /* Check whether the instruction modifies Interrupt Flag or not */ 422 p->ainsn.if_modifier = is_IF_modifier(p->ainsn.insn); 423 424 /* Also, displacement change doesn't affect the first byte */ 425 p->opcode = p->ainsn.insn[0]; 426 427 return 0; 428} 429 430int arch_prepare_kprobe(struct kprobe *p) 431{ 432 if (alternatives_text_reserved(p->addr, p->addr)) 433 return -EINVAL; 434 435 if (!can_probe((unsigned long)p->addr)) 436 return -EILSEQ; 437 /* insn: must be on special executable page on x86. */ 438 p->ainsn.insn = get_insn_slot(); 439 if (!p->ainsn.insn) 440 return -ENOMEM; 441 442 return arch_copy_kprobe(p); 443} 444 445void arch_arm_kprobe(struct kprobe *p) 446{ 447 text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1); 448} 449 450void arch_disarm_kprobe(struct kprobe *p) 451{ 452 text_poke(p->addr, &p->opcode, 1); 453} 454 455void arch_remove_kprobe(struct kprobe *p) 456{ 457 if (p->ainsn.insn) { 458 free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1)); 459 p->ainsn.insn = NULL; 460 } 461} 462 463static nokprobe_inline void 464save_previous_kprobe(struct kprobe_ctlblk *kcb) 465{ 466 kcb->prev_kprobe.kp = kprobe_running(); 467 kcb->prev_kprobe.status = kcb->kprobe_status; 468 kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags; 469 kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags; 470} 471 472static nokprobe_inline void 473restore_previous_kprobe(struct kprobe_ctlblk *kcb) 474{ 475 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 476 kcb->kprobe_status = kcb->prev_kprobe.status; 477 kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags; 478 kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags; 479} 480 481static nokprobe_inline void 482set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 483 struct kprobe_ctlblk *kcb) 484{ 485 __this_cpu_write(current_kprobe, p); 486 kcb->kprobe_saved_flags = kcb->kprobe_old_flags 487 = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF)); 488 if (p->ainsn.if_modifier) 489 kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF; 490} 491 492static nokprobe_inline void clear_btf(void) 493{ 494 if (test_thread_flag(TIF_BLOCKSTEP)) { 495 unsigned long debugctl = get_debugctlmsr(); 496 497 debugctl &= ~DEBUGCTLMSR_BTF; 498 update_debugctlmsr(debugctl); 499 } 500} 501 502static nokprobe_inline void restore_btf(void) 503{ 504 if (test_thread_flag(TIF_BLOCKSTEP)) { 505 unsigned long debugctl = get_debugctlmsr(); 506 507 debugctl |= DEBUGCTLMSR_BTF; 508 update_debugctlmsr(debugctl); 509 } 510} 511 512void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) 513{ 514 unsigned long *sara = stack_addr(regs); 515 516 ri->ret_addr = (kprobe_opcode_t *) *sara; 517 518 /* Replace the return addr with trampoline addr */ 519 *sara = (unsigned long) &kretprobe_trampoline; 520} 521NOKPROBE_SYMBOL(arch_prepare_kretprobe); 522 523static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, 524 struct kprobe_ctlblk *kcb, int reenter) 525{ 526 if (setup_detour_execution(p, regs, reenter)) 527 return; 528 529#if !defined(CONFIG_PREEMPT) 530 if (p->ainsn.boostable == 1 && !p->post_handler) { 531 /* Boost up -- we can execute copied instructions directly */ 532 if (!reenter) 533 reset_current_kprobe(); 534 /* 535 * Reentering boosted probe doesn't reset current_kprobe, 536 * nor set current_kprobe, because it doesn't use single 537 * stepping. 538 */ 539 regs->ip = (unsigned long)p->ainsn.insn; 540 preempt_enable_no_resched(); 541 return; 542 } 543#endif 544 if (reenter) { 545 save_previous_kprobe(kcb); 546 set_current_kprobe(p, regs, kcb); 547 kcb->kprobe_status = KPROBE_REENTER; 548 } else 549 kcb->kprobe_status = KPROBE_HIT_SS; 550 /* Prepare real single stepping */ 551 clear_btf(); 552 regs->flags |= X86_EFLAGS_TF; 553 regs->flags &= ~X86_EFLAGS_IF; 554 /* single step inline if the instruction is an int3 */ 555 if (p->opcode == BREAKPOINT_INSTRUCTION) 556 regs->ip = (unsigned long)p->addr; 557 else 558 regs->ip = (unsigned long)p->ainsn.insn; 559} 560NOKPROBE_SYMBOL(setup_singlestep); 561 562/* 563 * We have reentered the kprobe_handler(), since another probe was hit while 564 * within the handler. We save the original kprobes variables and just single 565 * step on the instruction of the new probe without calling any user handlers. 566 */ 567static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs, 568 struct kprobe_ctlblk *kcb) 569{ 570 switch (kcb->kprobe_status) { 571 case KPROBE_HIT_SSDONE: 572 case KPROBE_HIT_ACTIVE: 573 case KPROBE_HIT_SS: 574 kprobes_inc_nmissed_count(p); 575 setup_singlestep(p, regs, kcb, 1); 576 break; 577 case KPROBE_REENTER: 578 /* A probe has been hit in the codepath leading up to, or just 579 * after, single-stepping of a probed instruction. This entire 580 * codepath should strictly reside in .kprobes.text section. 581 * Raise a BUG or we'll continue in an endless reentering loop 582 * and eventually a stack overflow. 583 */ 584 printk(KERN_WARNING "Unrecoverable kprobe detected at %p.\n", 585 p->addr); 586 dump_kprobe(p); 587 BUG(); 588 default: 589 /* impossible cases */ 590 WARN_ON(1); 591 return 0; 592 } 593 594 return 1; 595} 596NOKPROBE_SYMBOL(reenter_kprobe); 597 598/* 599 * Interrupts are disabled on entry as trap3 is an interrupt gate and they 600 * remain disabled throughout this function. 601 */ 602int kprobe_int3_handler(struct pt_regs *regs) 603{ 604 kprobe_opcode_t *addr; 605 struct kprobe *p; 606 struct kprobe_ctlblk *kcb; 607 608 if (user_mode(regs)) 609 return 0; 610 611 addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t)); 612 /* 613 * We don't want to be preempted for the entire 614 * duration of kprobe processing. We conditionally 615 * re-enable preemption at the end of this function, 616 * and also in reenter_kprobe() and setup_singlestep(). 617 */ 618 preempt_disable(); 619 620 kcb = get_kprobe_ctlblk(); 621 p = get_kprobe(addr); 622 623 if (p) { 624 if (kprobe_running()) { 625 if (reenter_kprobe(p, regs, kcb)) 626 return 1; 627 } else { 628 set_current_kprobe(p, regs, kcb); 629 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 630 631 /* 632 * If we have no pre-handler or it returned 0, we 633 * continue with normal processing. If we have a 634 * pre-handler and it returned non-zero, it prepped 635 * for calling the break_handler below on re-entry 636 * for jprobe processing, so get out doing nothing 637 * more here. 638 */ 639 if (!p->pre_handler || !p->pre_handler(p, regs)) 640 setup_singlestep(p, regs, kcb, 0); 641 return 1; 642 } 643 } else if (*addr != BREAKPOINT_INSTRUCTION) { 644 /* 645 * The breakpoint instruction was removed right 646 * after we hit it. Another cpu has removed 647 * either a probepoint or a debugger breakpoint 648 * at this address. In either case, no further 649 * handling of this interrupt is appropriate. 650 * Back up over the (now missing) int3 and run 651 * the original instruction. 652 */ 653 regs->ip = (unsigned long)addr; 654 preempt_enable_no_resched(); 655 return 1; 656 } else if (kprobe_running()) { 657 p = __this_cpu_read(current_kprobe); 658 if (p->break_handler && p->break_handler(p, regs)) { 659 if (!skip_singlestep(p, regs, kcb)) 660 setup_singlestep(p, regs, kcb, 0); 661 return 1; 662 } 663 } /* else: not a kprobe fault; let the kernel handle it */ 664 665 preempt_enable_no_resched(); 666 return 0; 667} 668NOKPROBE_SYMBOL(kprobe_int3_handler); 669 670/* 671 * When a retprobed function returns, this code saves registers and 672 * calls trampoline_handler() runs, which calls the kretprobe's handler. 673 */ 674static void __used kretprobe_trampoline_holder(void) 675{ 676 asm volatile ( 677 ".global kretprobe_trampoline\n" 678 "kretprobe_trampoline: \n" 679#ifdef CONFIG_X86_64 680 /* We don't bother saving the ss register */ 681 " pushq %rsp\n" 682 " pushfq\n" 683 SAVE_REGS_STRING 684 " movq %rsp, %rdi\n" 685 " call trampoline_handler\n" 686 /* Replace saved sp with true return address. */ 687 " movq %rax, 152(%rsp)\n" 688 RESTORE_REGS_STRING 689 " popfq\n" 690#else 691 " pushf\n" 692 SAVE_REGS_STRING 693 " movl %esp, %eax\n" 694 " call trampoline_handler\n" 695 /* Move flags to cs */ 696 " movl 56(%esp), %edx\n" 697 " movl %edx, 52(%esp)\n" 698 /* Replace saved flags with true return address. */ 699 " movl %eax, 56(%esp)\n" 700 RESTORE_REGS_STRING 701 " popf\n" 702#endif 703 " ret\n"); 704} 705NOKPROBE_SYMBOL(kretprobe_trampoline_holder); 706NOKPROBE_SYMBOL(kretprobe_trampoline); 707 708/* 709 * Called from kretprobe_trampoline 710 */ 711__visible __used void *trampoline_handler(struct pt_regs *regs) 712{ 713 struct kretprobe_instance *ri = NULL; 714 struct hlist_head *head, empty_rp; 715 struct hlist_node *tmp; 716 unsigned long flags, orig_ret_address = 0; 717 unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline; 718 kprobe_opcode_t *correct_ret_addr = NULL; 719 720 INIT_HLIST_HEAD(&empty_rp); 721 kretprobe_hash_lock(current, &head, &flags); 722 /* fixup registers */ 723#ifdef CONFIG_X86_64 724 regs->cs = __KERNEL_CS; 725#else 726 regs->cs = __KERNEL_CS | get_kernel_rpl(); 727 regs->gs = 0; 728#endif 729 regs->ip = trampoline_address; 730 regs->orig_ax = ~0UL; 731 732 /* 733 * It is possible to have multiple instances associated with a given 734 * task either because multiple functions in the call path have 735 * return probes installed on them, and/or more than one 736 * return probe was registered for a target function. 737 * 738 * We can handle this because: 739 * - instances are always pushed into the head of the list 740 * - when multiple return probes are registered for the same 741 * function, the (chronologically) first instance's ret_addr 742 * will be the real return address, and all the rest will 743 * point to kretprobe_trampoline. 744 */ 745 hlist_for_each_entry_safe(ri, tmp, head, hlist) { 746 if (ri->task != current) 747 /* another task is sharing our hash bucket */ 748 continue; 749 750 orig_ret_address = (unsigned long)ri->ret_addr; 751 752 if (orig_ret_address != trampoline_address) 753 /* 754 * This is the real return address. Any other 755 * instances associated with this task are for 756 * other calls deeper on the call stack 757 */ 758 break; 759 } 760 761 kretprobe_assert(ri, orig_ret_address, trampoline_address); 762 763 correct_ret_addr = ri->ret_addr; 764 hlist_for_each_entry_safe(ri, tmp, head, hlist) { 765 if (ri->task != current) 766 /* another task is sharing our hash bucket */ 767 continue; 768 769 orig_ret_address = (unsigned long)ri->ret_addr; 770 if (ri->rp && ri->rp->handler) { 771 __this_cpu_write(current_kprobe, &ri->rp->kp); 772 get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE; 773 ri->ret_addr = correct_ret_addr; 774 ri->rp->handler(ri, regs); 775 __this_cpu_write(current_kprobe, NULL); 776 } 777 778 recycle_rp_inst(ri, &empty_rp); 779 780 if (orig_ret_address != trampoline_address) 781 /* 782 * This is the real return address. Any other 783 * instances associated with this task are for 784 * other calls deeper on the call stack 785 */ 786 break; 787 } 788 789 kretprobe_hash_unlock(current, &flags); 790 791 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { 792 hlist_del(&ri->hlist); 793 kfree(ri); 794 } 795 return (void *)orig_ret_address; 796} 797NOKPROBE_SYMBOL(trampoline_handler); 798 799/* 800 * Called after single-stepping. p->addr is the address of the 801 * instruction whose first byte has been replaced by the "int 3" 802 * instruction. To avoid the SMP problems that can occur when we 803 * temporarily put back the original opcode to single-step, we 804 * single-stepped a copy of the instruction. The address of this 805 * copy is p->ainsn.insn. 806 * 807 * This function prepares to return from the post-single-step 808 * interrupt. We have to fix up the stack as follows: 809 * 810 * 0) Except in the case of absolute or indirect jump or call instructions, 811 * the new ip is relative to the copied instruction. We need to make 812 * it relative to the original instruction. 813 * 814 * 1) If the single-stepped instruction was pushfl, then the TF and IF 815 * flags are set in the just-pushed flags, and may need to be cleared. 816 * 817 * 2) If the single-stepped instruction was a call, the return address 818 * that is atop the stack is the address following the copied instruction. 819 * We need to make it the address following the original instruction. 820 * 821 * If this is the first time we've single-stepped the instruction at 822 * this probepoint, and the instruction is boostable, boost it: add a 823 * jump instruction after the copied instruction, that jumps to the next 824 * instruction after the probepoint. 825 */ 826static void resume_execution(struct kprobe *p, struct pt_regs *regs, 827 struct kprobe_ctlblk *kcb) 828{ 829 unsigned long *tos = stack_addr(regs); 830 unsigned long copy_ip = (unsigned long)p->ainsn.insn; 831 unsigned long orig_ip = (unsigned long)p->addr; 832 kprobe_opcode_t *insn = p->ainsn.insn; 833 834 /* Skip prefixes */ 835 insn = skip_prefixes(insn); 836 837 regs->flags &= ~X86_EFLAGS_TF; 838 switch (*insn) { 839 case 0x9c: /* pushfl */ 840 *tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF); 841 *tos |= kcb->kprobe_old_flags; 842 break; 843 case 0xc2: /* iret/ret/lret */ 844 case 0xc3: 845 case 0xca: 846 case 0xcb: 847 case 0xcf: 848 case 0xea: /* jmp absolute -- ip is correct */ 849 /* ip is already adjusted, no more changes required */ 850 p->ainsn.boostable = 1; 851 goto no_change; 852 case 0xe8: /* call relative - Fix return addr */ 853 *tos = orig_ip + (*tos - copy_ip); 854 break; 855#ifdef CONFIG_X86_32 856 case 0x9a: /* call absolute -- same as call absolute, indirect */ 857 *tos = orig_ip + (*tos - copy_ip); 858 goto no_change; 859#endif 860 case 0xff: 861 if ((insn[1] & 0x30) == 0x10) { 862 /* 863 * call absolute, indirect 864 * Fix return addr; ip is correct. 865 * But this is not boostable 866 */ 867 *tos = orig_ip + (*tos - copy_ip); 868 goto no_change; 869 } else if (((insn[1] & 0x31) == 0x20) || 870 ((insn[1] & 0x31) == 0x21)) { 871 /* 872 * jmp near and far, absolute indirect 873 * ip is correct. And this is boostable 874 */ 875 p->ainsn.boostable = 1; 876 goto no_change; 877 } 878 default: 879 break; 880 } 881 882 if (p->ainsn.boostable == 0) { 883 if ((regs->ip > copy_ip) && 884 (regs->ip - copy_ip) + 5 < MAX_INSN_SIZE) { 885 /* 886 * These instructions can be executed directly if it 887 * jumps back to correct address. 888 */ 889 synthesize_reljump((void *)regs->ip, 890 (void *)orig_ip + (regs->ip - copy_ip)); 891 p->ainsn.boostable = 1; 892 } else { 893 p->ainsn.boostable = -1; 894 } 895 } 896 897 regs->ip += orig_ip - copy_ip; 898 899no_change: 900 restore_btf(); 901} 902NOKPROBE_SYMBOL(resume_execution); 903 904/* 905 * Interrupts are disabled on entry as trap1 is an interrupt gate and they 906 * remain disabled throughout this function. 907 */ 908int kprobe_debug_handler(struct pt_regs *regs) 909{ 910 struct kprobe *cur = kprobe_running(); 911 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 912 913 if (!cur) 914 return 0; 915 916 resume_execution(cur, regs, kcb); 917 regs->flags |= kcb->kprobe_saved_flags; 918 919 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { 920 kcb->kprobe_status = KPROBE_HIT_SSDONE; 921 cur->post_handler(cur, regs, 0); 922 } 923 924 /* Restore back the original saved kprobes variables and continue. */ 925 if (kcb->kprobe_status == KPROBE_REENTER) { 926 restore_previous_kprobe(kcb); 927 goto out; 928 } 929 reset_current_kprobe(); 930out: 931 preempt_enable_no_resched(); 932 933 /* 934 * if somebody else is singlestepping across a probe point, flags 935 * will have TF set, in which case, continue the remaining processing 936 * of do_debug, as if this is not a probe hit. 937 */ 938 if (regs->flags & X86_EFLAGS_TF) 939 return 0; 940 941 return 1; 942} 943NOKPROBE_SYMBOL(kprobe_debug_handler); 944 945int kprobe_fault_handler(struct pt_regs *regs, int trapnr) 946{ 947 struct kprobe *cur = kprobe_running(); 948 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 949 950 if (unlikely(regs->ip == (unsigned long)cur->ainsn.insn)) { 951 /* This must happen on single-stepping */ 952 WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS && 953 kcb->kprobe_status != KPROBE_REENTER); 954 /* 955 * We are here because the instruction being single 956 * stepped caused a page fault. We reset the current 957 * kprobe and the ip points back to the probe address 958 * and allow the page fault handler to continue as a 959 * normal page fault. 960 */ 961 regs->ip = (unsigned long)cur->addr; 962 regs->flags |= kcb->kprobe_old_flags; 963 if (kcb->kprobe_status == KPROBE_REENTER) 964 restore_previous_kprobe(kcb); 965 else 966 reset_current_kprobe(); 967 preempt_enable_no_resched(); 968 } else if (kcb->kprobe_status == KPROBE_HIT_ACTIVE || 969 kcb->kprobe_status == KPROBE_HIT_SSDONE) { 970 /* 971 * We increment the nmissed count for accounting, 972 * we can also use npre/npostfault count for accounting 973 * these specific fault cases. 974 */ 975 kprobes_inc_nmissed_count(cur); 976 977 /* 978 * We come here because instructions in the pre/post 979 * handler caused the page_fault, this could happen 980 * if handler tries to access user space by 981 * copy_from_user(), get_user() etc. Let the 982 * user-specified handler try to fix it first. 983 */ 984 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) 985 return 1; 986 987 /* 988 * In case the user-specified fault handler returned 989 * zero, try to fix up. 990 */ 991 if (fixup_exception(regs)) 992 return 1; 993 994 /* 995 * fixup routine could not handle it, 996 * Let do_page_fault() fix it. 997 */ 998 } 999 1000 return 0; 1001} 1002NOKPROBE_SYMBOL(kprobe_fault_handler); 1003 1004/* 1005 * Wrapper routine for handling exceptions. 1006 */ 1007int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, 1008 void *data) 1009{ 1010 struct die_args *args = data; 1011 int ret = NOTIFY_DONE; 1012 1013 if (args->regs && user_mode(args->regs)) 1014 return ret; 1015 1016 if (val == DIE_GPF) { 1017 /* 1018 * To be potentially processing a kprobe fault and to 1019 * trust the result from kprobe_running(), we have 1020 * be non-preemptible. 1021 */ 1022 if (!preemptible() && kprobe_running() && 1023 kprobe_fault_handler(args->regs, args->trapnr)) 1024 ret = NOTIFY_STOP; 1025 } 1026 return ret; 1027} 1028NOKPROBE_SYMBOL(kprobe_exceptions_notify); 1029 1030int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) 1031{ 1032 struct jprobe *jp = container_of(p, struct jprobe, kp); 1033 unsigned long addr; 1034 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 1035 1036 kcb->jprobe_saved_regs = *regs; 1037 kcb->jprobe_saved_sp = stack_addr(regs); 1038 addr = (unsigned long)(kcb->jprobe_saved_sp); 1039 1040 /* 1041 * As Linus pointed out, gcc assumes that the callee 1042 * owns the argument space and could overwrite it, e.g. 1043 * tailcall optimization. So, to be absolutely safe 1044 * we also save and restore enough stack bytes to cover 1045 * the argument area. 1046 */ 1047 memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr, 1048 MIN_STACK_SIZE(addr)); 1049 regs->flags &= ~X86_EFLAGS_IF; 1050 trace_hardirqs_off(); 1051 regs->ip = (unsigned long)(jp->entry); 1052 1053 /* 1054 * jprobes use jprobe_return() which skips the normal return 1055 * path of the function, and this messes up the accounting of the 1056 * function graph tracer to get messed up. 1057 * 1058 * Pause function graph tracing while performing the jprobe function. 1059 */ 1060 pause_graph_tracing(); 1061 return 1; 1062} 1063NOKPROBE_SYMBOL(setjmp_pre_handler); 1064 1065void jprobe_return(void) 1066{ 1067 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 1068 1069 asm volatile ( 1070#ifdef CONFIG_X86_64 1071 " xchg %%rbx,%%rsp \n" 1072#else 1073 " xchgl %%ebx,%%esp \n" 1074#endif 1075 " int3 \n" 1076 " .globl jprobe_return_end\n" 1077 " jprobe_return_end: \n" 1078 " nop \n"::"b" 1079 (kcb->jprobe_saved_sp):"memory"); 1080} 1081NOKPROBE_SYMBOL(jprobe_return); 1082NOKPROBE_SYMBOL(jprobe_return_end); 1083 1084int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) 1085{ 1086 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 1087 u8 *addr = (u8 *) (regs->ip - 1); 1088 struct jprobe *jp = container_of(p, struct jprobe, kp); 1089 void *saved_sp = kcb->jprobe_saved_sp; 1090 1091 if ((addr > (u8 *) jprobe_return) && 1092 (addr < (u8 *) jprobe_return_end)) { 1093 if (stack_addr(regs) != saved_sp) { 1094 struct pt_regs *saved_regs = &kcb->jprobe_saved_regs; 1095 printk(KERN_ERR 1096 "current sp %p does not match saved sp %p\n", 1097 stack_addr(regs), saved_sp); 1098 printk(KERN_ERR "Saved registers for jprobe %p\n", jp); 1099 show_regs(saved_regs); 1100 printk(KERN_ERR "Current registers\n"); 1101 show_regs(regs); 1102 BUG(); 1103 } 1104 /* It's OK to start function graph tracing again */ 1105 unpause_graph_tracing(); 1106 *regs = kcb->jprobe_saved_regs; 1107 memcpy(saved_sp, kcb->jprobes_stack, MIN_STACK_SIZE(saved_sp)); 1108 preempt_enable_no_resched(); 1109 return 1; 1110 } 1111 return 0; 1112} 1113NOKPROBE_SYMBOL(longjmp_break_handler); 1114 1115bool arch_within_kprobe_blacklist(unsigned long addr) 1116{ 1117 return (addr >= (unsigned long)__kprobes_text_start && 1118 addr < (unsigned long)__kprobes_text_end) || 1119 (addr >= (unsigned long)__entry_text_start && 1120 addr < (unsigned long)__entry_text_end); 1121} 1122 1123int __init arch_init_kprobes(void) 1124{ 1125 return 0; 1126} 1127 1128int arch_trampoline_kprobe(struct kprobe *p) 1129{ 1130 return 0; 1131} 1132