1/* 2 * arch/arm/kernel/kprobes.c 3 * 4 * Kprobes on ARM 5 * 6 * Abhishek Sagar <sagar.abhishek@gmail.com> 7 * Copyright (C) 2006, 2007 Motorola Inc. 8 * 9 * Nicolas Pitre <nico@marvell.com> 10 * Copyright (C) 2007 Marvell Ltd. 11 * 12 * This program is free software; you can redistribute it and/or modify 13 * it under the terms of the GNU General Public License version 2 as 14 * published by the Free Software Foundation. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 19 * General Public License for more details. 20 */ 21 22#include <linux/kernel.h> 23#include <linux/kprobes.h> 24#include <linux/module.h> 25#include <linux/slab.h> 26#include <linux/stop_machine.h> 27#include <linux/stringify.h> 28#include <asm/traps.h> 29#include <asm/opcodes.h> 30#include <asm/cacheflush.h> 31#include <linux/percpu.h> 32#include <linux/bug.h> 33#include <asm/patch.h> 34 35#include "../decode-arm.h" 36#include "../decode-thumb.h" 37#include "core.h" 38 39#define MIN_STACK_SIZE(addr) \ 40 min((unsigned long)MAX_STACK_SIZE, \ 41 (unsigned long)current_thread_info() + THREAD_START_SP - (addr)) 42 43#define flush_insns(addr, size) \ 44 flush_icache_range((unsigned long)(addr), \ 45 (unsigned long)(addr) + \ 46 (size)) 47 48/* Used as a marker in ARM_pc to note when we're in a jprobe. */ 49#define JPROBE_MAGIC_ADDR 0xffffffff 50 51DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 52DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 53 54 55int __kprobes arch_prepare_kprobe(struct kprobe *p) 56{ 57 kprobe_opcode_t insn; 58 kprobe_opcode_t tmp_insn[MAX_INSN_SIZE]; 59 unsigned long addr = (unsigned long)p->addr; 60 bool thumb; 61 kprobe_decode_insn_t *decode_insn; 62 const union decode_action *actions; 63 int is; 64 const struct decode_checker **checkers; 65 66 if (in_exception_text(addr)) 67 return -EINVAL; 68 69#ifdef CONFIG_THUMB2_KERNEL 70 thumb = true; 71 addr &= ~1; /* Bit 0 would normally be set to indicate Thumb code */ 72 insn = __mem_to_opcode_thumb16(((u16 *)addr)[0]); 73 if (is_wide_instruction(insn)) { 74 u16 inst2 = __mem_to_opcode_thumb16(((u16 *)addr)[1]); 75 insn = __opcode_thumb32_compose(insn, inst2); 76 decode_insn = thumb32_probes_decode_insn; 77 actions = kprobes_t32_actions; 78 checkers = kprobes_t32_checkers; 79 } else { 80 decode_insn = thumb16_probes_decode_insn; 81 actions = kprobes_t16_actions; 82 checkers = kprobes_t16_checkers; 83 } 84#else /* !CONFIG_THUMB2_KERNEL */ 85 thumb = false; 86 if (addr & 0x3) 87 return -EINVAL; 88 insn = __mem_to_opcode_arm(*p->addr); 89 decode_insn = arm_probes_decode_insn; 90 actions = kprobes_arm_actions; 91 checkers = kprobes_arm_checkers; 92#endif 93 94 p->opcode = insn; 95 p->ainsn.insn = tmp_insn; 96 97 switch ((*decode_insn)(insn, &p->ainsn, true, actions, checkers)) { 98 case INSN_REJECTED: /* not supported */ 99 return -EINVAL; 100 101 case INSN_GOOD: /* instruction uses slot */ 102 p->ainsn.insn = get_insn_slot(); 103 if (!p->ainsn.insn) 104 return -ENOMEM; 105 for (is = 0; is < MAX_INSN_SIZE; ++is) 106 p->ainsn.insn[is] = tmp_insn[is]; 107 flush_insns(p->ainsn.insn, 108 sizeof(p->ainsn.insn[0]) * MAX_INSN_SIZE); 109 p->ainsn.insn_fn = (probes_insn_fn_t *) 110 ((uintptr_t)p->ainsn.insn | thumb); 111 break; 112 113 case INSN_GOOD_NO_SLOT: /* instruction doesn't need insn slot */ 114 p->ainsn.insn = NULL; 115 break; 116 } 117 118 /* 119 * Never instrument insn like 'str r0, [sp, +/-r1]'. Also, insn likes 120 * 'str r0, [sp, #-68]' should also be prohibited. 121 * See __und_svc. 122 */ 123 if ((p->ainsn.stack_space < 0) || 124 (p->ainsn.stack_space > MAX_STACK_SIZE)) 125 return -EINVAL; 126 127 return 0; 128} 129 130void __kprobes arch_arm_kprobe(struct kprobe *p) 131{ 132 unsigned int brkp; 133 void *addr; 134 135 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { 136 /* Remove any Thumb flag */ 137 addr = (void *)((uintptr_t)p->addr & ~1); 138 139 if (is_wide_instruction(p->opcode)) 140 brkp = KPROBE_THUMB32_BREAKPOINT_INSTRUCTION; 141 else 142 brkp = KPROBE_THUMB16_BREAKPOINT_INSTRUCTION; 143 } else { 144 kprobe_opcode_t insn = p->opcode; 145 146 addr = p->addr; 147 brkp = KPROBE_ARM_BREAKPOINT_INSTRUCTION; 148 149 if (insn >= 0xe0000000) 150 brkp |= 0xe0000000; /* Unconditional instruction */ 151 else 152 brkp |= insn & 0xf0000000; /* Copy condition from insn */ 153 } 154 155 patch_text(addr, brkp); 156} 157 158/* 159 * The actual disarming is done here on each CPU and synchronized using 160 * stop_machine. This synchronization is necessary on SMP to avoid removing 161 * a probe between the moment the 'Undefined Instruction' exception is raised 162 * and the moment the exception handler reads the faulting instruction from 163 * memory. It is also needed to atomically set the two half-words of a 32-bit 164 * Thumb breakpoint. 165 */ 166struct patch { 167 void *addr; 168 unsigned int insn; 169}; 170 171static int __kprobes_remove_breakpoint(void *data) 172{ 173 struct patch *p = data; 174 __patch_text(p->addr, p->insn); 175 return 0; 176} 177 178void __kprobes kprobes_remove_breakpoint(void *addr, unsigned int insn) 179{ 180 struct patch p = { 181 .addr = addr, 182 .insn = insn, 183 }; 184 stop_machine(__kprobes_remove_breakpoint, &p, cpu_online_mask); 185} 186 187void __kprobes arch_disarm_kprobe(struct kprobe *p) 188{ 189 kprobes_remove_breakpoint((void *)((uintptr_t)p->addr & ~1), 190 p->opcode); 191} 192 193void __kprobes arch_remove_kprobe(struct kprobe *p) 194{ 195 if (p->ainsn.insn) { 196 free_insn_slot(p->ainsn.insn, 0); 197 p->ainsn.insn = NULL; 198 } 199} 200 201static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 202{ 203 kcb->prev_kprobe.kp = kprobe_running(); 204 kcb->prev_kprobe.status = kcb->kprobe_status; 205} 206 207static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 208{ 209 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 210 kcb->kprobe_status = kcb->prev_kprobe.status; 211} 212 213static void __kprobes set_current_kprobe(struct kprobe *p) 214{ 215 __this_cpu_write(current_kprobe, p); 216} 217 218static void __kprobes 219singlestep_skip(struct kprobe *p, struct pt_regs *regs) 220{ 221#ifdef CONFIG_THUMB2_KERNEL 222 regs->ARM_cpsr = it_advance(regs->ARM_cpsr); 223 if (is_wide_instruction(p->opcode)) 224 regs->ARM_pc += 4; 225 else 226 regs->ARM_pc += 2; 227#else 228 regs->ARM_pc += 4; 229#endif 230} 231 232static inline void __kprobes 233singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) 234{ 235 p->ainsn.insn_singlestep(p->opcode, &p->ainsn, regs); 236} 237 238/* 239 * Called with IRQs disabled. IRQs must remain disabled from that point 240 * all the way until processing this kprobe is complete. The current 241 * kprobes implementation cannot process more than one nested level of 242 * kprobe, and that level is reserved for user kprobe handlers, so we can't 243 * risk encountering a new kprobe in an interrupt handler. 244 */ 245void __kprobes kprobe_handler(struct pt_regs *regs) 246{ 247 struct kprobe *p, *cur; 248 struct kprobe_ctlblk *kcb; 249 250 kcb = get_kprobe_ctlblk(); 251 cur = kprobe_running(); 252 253#ifdef CONFIG_THUMB2_KERNEL 254 /* 255 * First look for a probe which was registered using an address with 256 * bit 0 set, this is the usual situation for pointers to Thumb code. 257 * If not found, fallback to looking for one with bit 0 clear. 258 */ 259 p = get_kprobe((kprobe_opcode_t *)(regs->ARM_pc | 1)); 260 if (!p) 261 p = get_kprobe((kprobe_opcode_t *)regs->ARM_pc); 262 263#else /* ! CONFIG_THUMB2_KERNEL */ 264 p = get_kprobe((kprobe_opcode_t *)regs->ARM_pc); 265#endif 266 267 if (p) { 268 if (cur) { 269 /* Kprobe is pending, so we're recursing. */ 270 switch (kcb->kprobe_status) { 271 case KPROBE_HIT_ACTIVE: 272 case KPROBE_HIT_SSDONE: 273 /* A pre- or post-handler probe got us here. */ 274 kprobes_inc_nmissed_count(p); 275 save_previous_kprobe(kcb); 276 set_current_kprobe(p); 277 kcb->kprobe_status = KPROBE_REENTER; 278 singlestep(p, regs, kcb); 279 restore_previous_kprobe(kcb); 280 break; 281 default: 282 /* impossible cases */ 283 BUG(); 284 } 285 } else if (p->ainsn.insn_check_cc(regs->ARM_cpsr)) { 286 /* Probe hit and conditional execution check ok. */ 287 set_current_kprobe(p); 288 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 289 290 /* 291 * If we have no pre-handler or it returned 0, we 292 * continue with normal processing. If we have a 293 * pre-handler and it returned non-zero, it prepped 294 * for calling the break_handler below on re-entry, 295 * so get out doing nothing more here. 296 */ 297 if (!p->pre_handler || !p->pre_handler(p, regs)) { 298 kcb->kprobe_status = KPROBE_HIT_SS; 299 singlestep(p, regs, kcb); 300 if (p->post_handler) { 301 kcb->kprobe_status = KPROBE_HIT_SSDONE; 302 p->post_handler(p, regs, 0); 303 } 304 reset_current_kprobe(); 305 } 306 } else { 307 /* 308 * Probe hit but conditional execution check failed, 309 * so just skip the instruction and continue as if 310 * nothing had happened. 311 */ 312 singlestep_skip(p, regs); 313 } 314 } else if (cur) { 315 /* We probably hit a jprobe. Call its break handler. */ 316 if (cur->break_handler && cur->break_handler(cur, regs)) { 317 kcb->kprobe_status = KPROBE_HIT_SS; 318 singlestep(cur, regs, kcb); 319 if (cur->post_handler) { 320 kcb->kprobe_status = KPROBE_HIT_SSDONE; 321 cur->post_handler(cur, regs, 0); 322 } 323 } 324 reset_current_kprobe(); 325 } else { 326 /* 327 * The probe was removed and a race is in progress. 328 * There is nothing we can do about it. Let's restart 329 * the instruction. By the time we can restart, the 330 * real instruction will be there. 331 */ 332 } 333} 334 335static int __kprobes kprobe_trap_handler(struct pt_regs *regs, unsigned int instr) 336{ 337 unsigned long flags; 338 local_irq_save(flags); 339 kprobe_handler(regs); 340 local_irq_restore(flags); 341 return 0; 342} 343 344int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) 345{ 346 struct kprobe *cur = kprobe_running(); 347 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 348 349 switch (kcb->kprobe_status) { 350 case KPROBE_HIT_SS: 351 case KPROBE_REENTER: 352 /* 353 * We are here because the instruction being single 354 * stepped caused a page fault. We reset the current 355 * kprobe and the PC to point back to the probe address 356 * and allow the page fault handler to continue as a 357 * normal page fault. 358 */ 359 regs->ARM_pc = (long)cur->addr; 360 if (kcb->kprobe_status == KPROBE_REENTER) { 361 restore_previous_kprobe(kcb); 362 } else { 363 reset_current_kprobe(); 364 } 365 break; 366 367 case KPROBE_HIT_ACTIVE: 368 case KPROBE_HIT_SSDONE: 369 /* 370 * We increment the nmissed count for accounting, 371 * we can also use npre/npostfault count for accounting 372 * these specific fault cases. 373 */ 374 kprobes_inc_nmissed_count(cur); 375 376 /* 377 * We come here because instructions in the pre/post 378 * handler caused the page_fault, this could happen 379 * if handler tries to access user space by 380 * copy_from_user(), get_user() etc. Let the 381 * user-specified handler try to fix it. 382 */ 383 if (cur->fault_handler && cur->fault_handler(cur, regs, fsr)) 384 return 1; 385 break; 386 387 default: 388 break; 389 } 390 391 return 0; 392} 393 394int __kprobes kprobe_exceptions_notify(struct notifier_block *self, 395 unsigned long val, void *data) 396{ 397 /* 398 * notify_die() is currently never called on ARM, 399 * so this callback is currently empty. 400 */ 401 return NOTIFY_DONE; 402} 403 404/* 405 * When a retprobed function returns, trampoline_handler() is called, 406 * calling the kretprobe's handler. We construct a struct pt_regs to 407 * give a view of registers r0-r11 to the user return-handler. This is 408 * not a complete pt_regs structure, but that should be plenty sufficient 409 * for kretprobe handlers which should normally be interested in r0 only 410 * anyway. 411 */ 412void __naked __kprobes kretprobe_trampoline(void) 413{ 414 __asm__ __volatile__ ( 415 "stmdb sp!, {r0 - r11} \n\t" 416 "mov r0, sp \n\t" 417 "bl trampoline_handler \n\t" 418 "mov lr, r0 \n\t" 419 "ldmia sp!, {r0 - r11} \n\t" 420#ifdef CONFIG_THUMB2_KERNEL 421 "bx lr \n\t" 422#else 423 "mov pc, lr \n\t" 424#endif 425 : : : "memory"); 426} 427 428/* Called from kretprobe_trampoline */ 429static __used __kprobes void *trampoline_handler(struct pt_regs *regs) 430{ 431 struct kretprobe_instance *ri = NULL; 432 struct hlist_head *head, empty_rp; 433 struct hlist_node *tmp; 434 unsigned long flags, orig_ret_address = 0; 435 unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline; 436 437 INIT_HLIST_HEAD(&empty_rp); 438 kretprobe_hash_lock(current, &head, &flags); 439 440 /* 441 * It is possible to have multiple instances associated with a given 442 * task either because multiple functions in the call path have 443 * a return probe installed on them, and/or more than one return 444 * probe was registered for a target function. 445 * 446 * We can handle this because: 447 * - instances are always inserted at the head of the list 448 * - when multiple return probes are registered for the same 449 * function, the first instance's ret_addr will point to the 450 * real return address, and all the rest will point to 451 * kretprobe_trampoline 452 */ 453 hlist_for_each_entry_safe(ri, tmp, head, hlist) { 454 if (ri->task != current) 455 /* another task is sharing our hash bucket */ 456 continue; 457 458 if (ri->rp && ri->rp->handler) { 459 __this_cpu_write(current_kprobe, &ri->rp->kp); 460 get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE; 461 ri->rp->handler(ri, regs); 462 __this_cpu_write(current_kprobe, NULL); 463 } 464 465 orig_ret_address = (unsigned long)ri->ret_addr; 466 recycle_rp_inst(ri, &empty_rp); 467 468 if (orig_ret_address != trampoline_address) 469 /* 470 * This is the real return address. Any other 471 * instances associated with this task are for 472 * other calls deeper on the call stack 473 */ 474 break; 475 } 476 477 kretprobe_assert(ri, orig_ret_address, trampoline_address); 478 kretprobe_hash_unlock(current, &flags); 479 480 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { 481 hlist_del(&ri->hlist); 482 kfree(ri); 483 } 484 485 return (void *)orig_ret_address; 486} 487 488void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 489 struct pt_regs *regs) 490{ 491 ri->ret_addr = (kprobe_opcode_t *)regs->ARM_lr; 492 493 /* Replace the return addr with trampoline addr. */ 494 regs->ARM_lr = (unsigned long)&kretprobe_trampoline; 495} 496 497int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) 498{ 499 struct jprobe *jp = container_of(p, struct jprobe, kp); 500 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 501 long sp_addr = regs->ARM_sp; 502 long cpsr; 503 504 kcb->jprobe_saved_regs = *regs; 505 memcpy(kcb->jprobes_stack, (void *)sp_addr, MIN_STACK_SIZE(sp_addr)); 506 regs->ARM_pc = (long)jp->entry; 507 508 cpsr = regs->ARM_cpsr | PSR_I_BIT; 509#ifdef CONFIG_THUMB2_KERNEL 510 /* Set correct Thumb state in cpsr */ 511 if (regs->ARM_pc & 1) 512 cpsr |= PSR_T_BIT; 513 else 514 cpsr &= ~PSR_T_BIT; 515#endif 516 regs->ARM_cpsr = cpsr; 517 518 preempt_disable(); 519 return 1; 520} 521 522void __kprobes jprobe_return(void) 523{ 524 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 525 526 __asm__ __volatile__ ( 527 /* 528 * Setup an empty pt_regs. Fill SP and PC fields as 529 * they're needed by longjmp_break_handler. 530 * 531 * We allocate some slack between the original SP and start of 532 * our fabricated regs. To be precise we want to have worst case 533 * covered which is STMFD with all 16 regs so we allocate 2 * 534 * sizeof(struct_pt_regs)). 535 * 536 * This is to prevent any simulated instruction from writing 537 * over the regs when they are accessing the stack. 538 */ 539#ifdef CONFIG_THUMB2_KERNEL 540 "sub r0, %0, %1 \n\t" 541 "mov sp, r0 \n\t" 542#else 543 "sub sp, %0, %1 \n\t" 544#endif 545 "ldr r0, ="__stringify(JPROBE_MAGIC_ADDR)"\n\t" 546 "str %0, [sp, %2] \n\t" 547 "str r0, [sp, %3] \n\t" 548 "mov r0, sp \n\t" 549 "bl kprobe_handler \n\t" 550 551 /* 552 * Return to the context saved by setjmp_pre_handler 553 * and restored by longjmp_break_handler. 554 */ 555#ifdef CONFIG_THUMB2_KERNEL 556 "ldr lr, [sp, %2] \n\t" /* lr = saved sp */ 557 "ldrd r0, r1, [sp, %5] \n\t" /* r0,r1 = saved lr,pc */ 558 "ldr r2, [sp, %4] \n\t" /* r2 = saved psr */ 559 "stmdb lr!, {r0, r1, r2} \n\t" /* push saved lr and */ 560 /* rfe context */ 561 "ldmia sp, {r0 - r12} \n\t" 562 "mov sp, lr \n\t" 563 "ldr lr, [sp], #4 \n\t" 564 "rfeia sp! \n\t" 565#else 566 "ldr r0, [sp, %4] \n\t" 567 "msr cpsr_cxsf, r0 \n\t" 568 "ldmia sp, {r0 - pc} \n\t" 569#endif 570 : 571 : "r" (kcb->jprobe_saved_regs.ARM_sp), 572 "I" (sizeof(struct pt_regs) * 2), 573 "J" (offsetof(struct pt_regs, ARM_sp)), 574 "J" (offsetof(struct pt_regs, ARM_pc)), 575 "J" (offsetof(struct pt_regs, ARM_cpsr)), 576 "J" (offsetof(struct pt_regs, ARM_lr)) 577 : "memory", "cc"); 578} 579 580int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) 581{ 582 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 583 long stack_addr = kcb->jprobe_saved_regs.ARM_sp; 584 long orig_sp = regs->ARM_sp; 585 struct jprobe *jp = container_of(p, struct jprobe, kp); 586 587 if (regs->ARM_pc == JPROBE_MAGIC_ADDR) { 588 if (orig_sp != stack_addr) { 589 struct pt_regs *saved_regs = 590 (struct pt_regs *)kcb->jprobe_saved_regs.ARM_sp; 591 printk("current sp %lx does not match saved sp %lx\n", 592 orig_sp, stack_addr); 593 printk("Saved registers for jprobe %p\n", jp); 594 show_regs(saved_regs); 595 printk("Current registers\n"); 596 show_regs(regs); 597 BUG(); 598 } 599 *regs = kcb->jprobe_saved_regs; 600 memcpy((void *)stack_addr, kcb->jprobes_stack, 601 MIN_STACK_SIZE(stack_addr)); 602 preempt_enable_no_resched(); 603 return 1; 604 } 605 return 0; 606} 607 608int __kprobes arch_trampoline_kprobe(struct kprobe *p) 609{ 610 return 0; 611} 612 613#ifdef CONFIG_THUMB2_KERNEL 614 615static struct undef_hook kprobes_thumb16_break_hook = { 616 .instr_mask = 0xffff, 617 .instr_val = KPROBE_THUMB16_BREAKPOINT_INSTRUCTION, 618 .cpsr_mask = MODE_MASK, 619 .cpsr_val = SVC_MODE, 620 .fn = kprobe_trap_handler, 621}; 622 623static struct undef_hook kprobes_thumb32_break_hook = { 624 .instr_mask = 0xffffffff, 625 .instr_val = KPROBE_THUMB32_BREAKPOINT_INSTRUCTION, 626 .cpsr_mask = MODE_MASK, 627 .cpsr_val = SVC_MODE, 628 .fn = kprobe_trap_handler, 629}; 630 631#else /* !CONFIG_THUMB2_KERNEL */ 632 633static struct undef_hook kprobes_arm_break_hook = { 634 .instr_mask = 0x0fffffff, 635 .instr_val = KPROBE_ARM_BREAKPOINT_INSTRUCTION, 636 .cpsr_mask = MODE_MASK, 637 .cpsr_val = SVC_MODE, 638 .fn = kprobe_trap_handler, 639}; 640 641#endif /* !CONFIG_THUMB2_KERNEL */ 642 643int __init arch_init_kprobes() 644{ 645 arm_probes_decode_init(); 646#ifdef CONFIG_THUMB2_KERNEL 647 register_undef_hook(&kprobes_thumb16_break_hook); 648 register_undef_hook(&kprobes_thumb32_break_hook); 649#else 650 register_undef_hook(&kprobes_arm_break_hook); 651#endif 652 return 0; 653} 654