1/* 2 * Copyright 2010 Tilera Corporation. All Rights Reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation, version 2. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 11 * NON INFRINGEMENT. See the GNU General Public License for 12 * more details. 13 * 14 * From i386 code copyright (C) 1995 Linus Torvalds 15 */ 16 17#include <linux/signal.h> 18#include <linux/sched.h> 19#include <linux/kernel.h> 20#include <linux/errno.h> 21#include <linux/string.h> 22#include <linux/types.h> 23#include <linux/ptrace.h> 24#include <linux/mman.h> 25#include <linux/mm.h> 26#include <linux/smp.h> 27#include <linux/interrupt.h> 28#include <linux/init.h> 29#include <linux/tty.h> 30#include <linux/vt_kern.h> /* For unblank_screen() */ 31#include <linux/highmem.h> 32#include <linux/module.h> 33#include <linux/kprobes.h> 34#include <linux/hugetlb.h> 35#include <linux/syscalls.h> 36#include <linux/uaccess.h> 37#include <linux/kdebug.h> 38#include <linux/context_tracking.h> 39 40#include <asm/pgalloc.h> 41#include <asm/sections.h> 42#include <asm/traps.h> 43#include <asm/syscalls.h> 44 45#include <arch/interrupts.h> 46 47static noinline void force_sig_info_fault(const char *type, int si_signo, 48 int si_code, unsigned long address, 49 int fault_num, 50 struct task_struct *tsk, 51 struct pt_regs *regs) 52{ 53 siginfo_t info; 54 55 if (unlikely(tsk->pid < 2)) { 56 panic("Signal %d (code %d) at %#lx sent to %s!", 57 si_signo, si_code & 0xffff, address, 58 is_idle_task(tsk) ? "the idle task" : "init"); 59 } 60 61 info.si_signo = si_signo; 62 info.si_errno = 0; 63 info.si_code = si_code; 64 info.si_addr = (void __user *)address; 65 info.si_trapno = fault_num; 66 trace_unhandled_signal(type, regs, address, si_signo); 67 force_sig_info(si_signo, &info, tsk); 68} 69 70#ifndef __tilegx__ 71/* 72 * Synthesize the fault a PL0 process would get by doing a word-load of 73 * an unaligned address or a high kernel address. 74 */ 75SYSCALL_DEFINE1(cmpxchg_badaddr, unsigned long, address) 76{ 77 struct pt_regs *regs = current_pt_regs(); 78 79 if (address >= PAGE_OFFSET) 80 force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR, 81 address, INT_DTLB_MISS, current, regs); 82 else 83 force_sig_info_fault("atomic alignment fault", SIGBUS, 84 BUS_ADRALN, address, 85 INT_UNALIGN_DATA, current, regs); 86 87 /* 88 * Adjust pc to point at the actual instruction, which is unusual 89 * for syscalls normally, but is appropriate when we are claiming 90 * that a syscall swint1 caused a page fault or bus error. 91 */ 92 regs->pc -= 8; 93 94 /* 95 * Mark this as a caller-save interrupt, like a normal page fault, 96 * so that when we go through the signal handler path we will 97 * properly restore r0, r1, and r2 for the signal handler arguments. 98 */ 99 regs->flags |= PT_FLAGS_CALLER_SAVES; 100 101 return 0; 102} 103#endif 104 105static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) 106{ 107 unsigned index = pgd_index(address); 108 pgd_t *pgd_k; 109 pud_t *pud, *pud_k; 110 pmd_t *pmd, *pmd_k; 111 112 pgd += index; 113 pgd_k = init_mm.pgd + index; 114 115 if (!pgd_present(*pgd_k)) 116 return NULL; 117 118 pud = pud_offset(pgd, address); 119 pud_k = pud_offset(pgd_k, address); 120 if (!pud_present(*pud_k)) 121 return NULL; 122 123 pmd = pmd_offset(pud, address); 124 pmd_k = pmd_offset(pud_k, address); 125 if (!pmd_present(*pmd_k)) 126 return NULL; 127 if (!pmd_present(*pmd)) 128 set_pmd(pmd, *pmd_k); 129 else 130 BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k)); 131 return pmd_k; 132} 133 134/* 135 * Handle a fault on the vmalloc area. 136 */ 137static inline int vmalloc_fault(pgd_t *pgd, unsigned long address) 138{ 139 pmd_t *pmd_k; 140 pte_t *pte_k; 141 142 /* Make sure we are in vmalloc area */ 143 if (!(address >= VMALLOC_START && address < VMALLOC_END)) 144 return -1; 145 146 /* 147 * Synchronize this task's top level page-table 148 * with the 'reference' page table. 149 */ 150 pmd_k = vmalloc_sync_one(pgd, address); 151 if (!pmd_k) 152 return -1; 153 pte_k = pte_offset_kernel(pmd_k, address); 154 if (!pte_present(*pte_k)) 155 return -1; 156 return 0; 157} 158 159/* Wait until this PTE has completed migration. */ 160static void wait_for_migration(pte_t *pte) 161{ 162 if (pte_migrating(*pte)) { 163 /* 164 * Wait until the migrater fixes up this pte. 165 * We scale the loop count by the clock rate so we'll wait for 166 * a few seconds here. 167 */ 168 int retries = 0; 169 int bound = get_clock_rate(); 170 while (pte_migrating(*pte)) { 171 barrier(); 172 if (++retries > bound) 173 panic("Hit migrating PTE (%#llx) and page PFN %#lx still migrating", 174 pte->val, pte_pfn(*pte)); 175 } 176 } 177} 178 179/* 180 * It's not generally safe to use "current" to get the page table pointer, 181 * since we might be running an oprofile interrupt in the middle of a 182 * task switch. 183 */ 184static pgd_t *get_current_pgd(void) 185{ 186 HV_Context ctx = hv_inquire_context(); 187 unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT; 188 struct page *pgd_page = pfn_to_page(pgd_pfn); 189 BUG_ON(PageHighMem(pgd_page)); 190 return (pgd_t *) __va(ctx.page_table); 191} 192 193/* 194 * We can receive a page fault from a migrating PTE at any time. 195 * Handle it by just waiting until the fault resolves. 196 * 197 * It's also possible to get a migrating kernel PTE that resolves 198 * itself during the downcall from hypervisor to Linux. We just check 199 * here to see if the PTE seems valid, and if so we retry it. 200 * 201 * NOTE! We MUST NOT take any locks for this case. We may be in an 202 * interrupt or a critical region, and must do as little as possible. 203 * Similarly, we can't use atomic ops here, since we may be handling a 204 * fault caused by an atomic op access. 205 * 206 * If we find a migrating PTE while we're in an NMI context, and we're 207 * at a PC that has a registered exception handler, we don't wait, 208 * since this thread may (e.g.) have been interrupted while migrating 209 * its own stack, which would then cause us to self-deadlock. 210 */ 211static int handle_migrating_pte(pgd_t *pgd, int fault_num, 212 unsigned long address, unsigned long pc, 213 int is_kernel_mode, int write) 214{ 215 pud_t *pud; 216 pmd_t *pmd; 217 pte_t *pte; 218 pte_t pteval; 219 220 if (pgd_addr_invalid(address)) 221 return 0; 222 223 pgd += pgd_index(address); 224 pud = pud_offset(pgd, address); 225 if (!pud || !pud_present(*pud)) 226 return 0; 227 pmd = pmd_offset(pud, address); 228 if (!pmd || !pmd_present(*pmd)) 229 return 0; 230 pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) : 231 pte_offset_kernel(pmd, address); 232 pteval = *pte; 233 if (pte_migrating(pteval)) { 234 if (in_nmi() && search_exception_tables(pc)) 235 return 0; 236 wait_for_migration(pte); 237 return 1; 238 } 239 240 if (!is_kernel_mode || !pte_present(pteval)) 241 return 0; 242 if (fault_num == INT_ITLB_MISS) { 243 if (pte_exec(pteval)) 244 return 1; 245 } else if (write) { 246 if (pte_write(pteval)) 247 return 1; 248 } else { 249 if (pte_read(pteval)) 250 return 1; 251 } 252 253 return 0; 254} 255 256/* 257 * This routine is responsible for faulting in user pages. 258 * It passes the work off to one of the appropriate routines. 259 * It returns true if the fault was successfully handled. 260 */ 261static int handle_page_fault(struct pt_regs *regs, 262 int fault_num, 263 int is_page_fault, 264 unsigned long address, 265 int write) 266{ 267 struct task_struct *tsk; 268 struct mm_struct *mm; 269 struct vm_area_struct *vma; 270 unsigned long stack_offset; 271 int fault; 272 int si_code; 273 int is_kernel_mode; 274 pgd_t *pgd; 275 unsigned int flags; 276 277 /* on TILE, protection faults are always writes */ 278 if (!is_page_fault) 279 write = 1; 280 281 flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 282 283 is_kernel_mode = !user_mode(regs); 284 285 tsk = validate_current(); 286 287 /* 288 * Check to see if we might be overwriting the stack, and bail 289 * out if so. The page fault code is a relatively likely 290 * place to get trapped in an infinite regress, and once we 291 * overwrite the whole stack, it becomes very hard to recover. 292 */ 293 stack_offset = stack_pointer & (THREAD_SIZE-1); 294 if (stack_offset < THREAD_SIZE / 8) { 295 pr_alert("Potential stack overrun: sp %#lx\n", stack_pointer); 296 show_regs(regs); 297 pr_alert("Killing current process %d/%s\n", 298 tsk->pid, tsk->comm); 299 do_group_exit(SIGKILL); 300 } 301 302 /* 303 * Early on, we need to check for migrating PTE entries; 304 * see homecache.c. If we find a migrating PTE, we wait until 305 * the backing page claims to be done migrating, then we proceed. 306 * For kernel PTEs, we rewrite the PTE and return and retry. 307 * Otherwise, we treat the fault like a normal "no PTE" fault, 308 * rather than trying to patch up the existing PTE. 309 */ 310 pgd = get_current_pgd(); 311 if (handle_migrating_pte(pgd, fault_num, address, regs->pc, 312 is_kernel_mode, write)) 313 return 1; 314 315 si_code = SEGV_MAPERR; 316 317 /* 318 * We fault-in kernel-space virtual memory on-demand. The 319 * 'reference' page table is init_mm.pgd. 320 * 321 * NOTE! We MUST NOT take any locks for this case. We may 322 * be in an interrupt or a critical region, and should 323 * only copy the information from the master page table, 324 * nothing more. 325 * 326 * This verifies that the fault happens in kernel space 327 * and that the fault was not a protection fault. 328 */ 329 if (unlikely(address >= TASK_SIZE && 330 !is_arch_mappable_range(address, 0))) { 331 if (is_kernel_mode && is_page_fault && 332 vmalloc_fault(pgd, address) >= 0) 333 return 1; 334 /* 335 * Don't take the mm semaphore here. If we fixup a prefetch 336 * fault we could otherwise deadlock. 337 */ 338 mm = NULL; /* happy compiler */ 339 vma = NULL; 340 goto bad_area_nosemaphore; 341 } 342 343 /* 344 * If we're trying to touch user-space addresses, we must 345 * be either at PL0, or else with interrupts enabled in the 346 * kernel, so either way we can re-enable interrupts here 347 * unless we are doing atomic access to user space with 348 * interrupts disabled. 349 */ 350 if (!(regs->flags & PT_FLAGS_DISABLE_IRQ)) 351 local_irq_enable(); 352 353 mm = tsk->mm; 354 355 /* 356 * If we're in an interrupt, have no user context or are running in an 357 * region with pagefaults disabled then we must not take the fault. 358 */ 359 if (pagefault_disabled() || !mm) { 360 vma = NULL; /* happy compiler */ 361 goto bad_area_nosemaphore; 362 } 363 364 if (!is_kernel_mode) 365 flags |= FAULT_FLAG_USER; 366 367 /* 368 * When running in the kernel we expect faults to occur only to 369 * addresses in user space. All other faults represent errors in the 370 * kernel and should generate an OOPS. Unfortunately, in the case of an 371 * erroneous fault occurring in a code path which already holds mmap_sem 372 * we will deadlock attempting to validate the fault against the 373 * address space. Luckily the kernel only validly references user 374 * space from well defined areas of code, which are listed in the 375 * exceptions table. 376 * 377 * As the vast majority of faults will be valid we will only perform 378 * the source reference check when there is a possibility of a deadlock. 379 * Attempt to lock the address space, if we cannot we then validate the 380 * source. If this is invalid we can skip the address space check, 381 * thus avoiding the deadlock. 382 */ 383 if (!down_read_trylock(&mm->mmap_sem)) { 384 if (is_kernel_mode && 385 !search_exception_tables(regs->pc)) { 386 vma = NULL; /* happy compiler */ 387 goto bad_area_nosemaphore; 388 } 389 390retry: 391 down_read(&mm->mmap_sem); 392 } 393 394 vma = find_vma(mm, address); 395 if (!vma) 396 goto bad_area; 397 if (vma->vm_start <= address) 398 goto good_area; 399 if (!(vma->vm_flags & VM_GROWSDOWN)) 400 goto bad_area; 401 if (regs->sp < PAGE_OFFSET) { 402 /* 403 * accessing the stack below sp is always a bug. 404 */ 405 if (address < regs->sp) 406 goto bad_area; 407 } 408 if (expand_stack(vma, address)) 409 goto bad_area; 410 411/* 412 * Ok, we have a good vm_area for this memory access, so 413 * we can handle it.. 414 */ 415good_area: 416 si_code = SEGV_ACCERR; 417 if (fault_num == INT_ITLB_MISS) { 418 if (!(vma->vm_flags & VM_EXEC)) 419 goto bad_area; 420 } else if (write) { 421#ifdef TEST_VERIFY_AREA 422 if (!is_page_fault && regs->cs == KERNEL_CS) 423 pr_err("WP fault at " REGFMT "\n", regs->eip); 424#endif 425 if (!(vma->vm_flags & VM_WRITE)) 426 goto bad_area; 427 flags |= FAULT_FLAG_WRITE; 428 } else { 429 if (!is_page_fault || !(vma->vm_flags & VM_READ)) 430 goto bad_area; 431 } 432 433 /* 434 * If for any reason at all we couldn't handle the fault, 435 * make sure we exit gracefully rather than endlessly redo 436 * the fault. 437 */ 438 fault = handle_mm_fault(mm, vma, address, flags); 439 440 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) 441 return 0; 442 443 if (unlikely(fault & VM_FAULT_ERROR)) { 444 if (fault & VM_FAULT_OOM) 445 goto out_of_memory; 446 else if (fault & VM_FAULT_SIGSEGV) 447 goto bad_area; 448 else if (fault & VM_FAULT_SIGBUS) 449 goto do_sigbus; 450 BUG(); 451 } 452 if (flags & FAULT_FLAG_ALLOW_RETRY) { 453 if (fault & VM_FAULT_MAJOR) 454 tsk->maj_flt++; 455 else 456 tsk->min_flt++; 457 if (fault & VM_FAULT_RETRY) { 458 flags &= ~FAULT_FLAG_ALLOW_RETRY; 459 flags |= FAULT_FLAG_TRIED; 460 461 /* 462 * No need to up_read(&mm->mmap_sem) as we would 463 * have already released it in __lock_page_or_retry 464 * in mm/filemap.c. 465 */ 466 goto retry; 467 } 468 } 469 470#if CHIP_HAS_TILE_DMA() 471 /* If this was a DMA TLB fault, restart the DMA engine. */ 472 switch (fault_num) { 473 case INT_DMATLB_MISS: 474 case INT_DMATLB_MISS_DWNCL: 475 case INT_DMATLB_ACCESS: 476 case INT_DMATLB_ACCESS_DWNCL: 477 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK); 478 break; 479 } 480#endif 481 482 up_read(&mm->mmap_sem); 483 return 1; 484 485/* 486 * Something tried to access memory that isn't in our memory map.. 487 * Fix it, but check if it's kernel or user first.. 488 */ 489bad_area: 490 up_read(&mm->mmap_sem); 491 492bad_area_nosemaphore: 493 /* User mode accesses just cause a SIGSEGV */ 494 if (!is_kernel_mode) { 495 /* 496 * It's possible to have interrupts off here. 497 */ 498 local_irq_enable(); 499 500 force_sig_info_fault("segfault", SIGSEGV, si_code, address, 501 fault_num, tsk, regs); 502 return 0; 503 } 504 505no_context: 506 /* Are we prepared to handle this kernel fault? */ 507 if (fixup_exception(regs)) 508 return 0; 509 510/* 511 * Oops. The kernel tried to access some bad page. We'll have to 512 * terminate things with extreme prejudice. 513 */ 514 515 bust_spinlocks(1); 516 517 /* FIXME: no lookup_address() yet */ 518#ifdef SUPPORT_LOOKUP_ADDRESS 519 if (fault_num == INT_ITLB_MISS) { 520 pte_t *pte = lookup_address(address); 521 522 if (pte && pte_present(*pte) && !pte_exec_kernel(*pte)) 523 pr_crit("kernel tried to execute non-executable page - exploit attempt? (uid: %d)\n", 524 current->uid); 525 } 526#endif 527 if (address < PAGE_SIZE) 528 pr_alert("Unable to handle kernel NULL pointer dereference\n"); 529 else 530 pr_alert("Unable to handle kernel paging request\n"); 531 pr_alert(" at virtual address " REGFMT ", pc " REGFMT "\n", 532 address, regs->pc); 533 534 show_regs(regs); 535 536 if (unlikely(tsk->pid < 2)) { 537 panic("Kernel page fault running %s!", 538 is_idle_task(tsk) ? "the idle task" : "init"); 539 } 540 541 /* 542 * More FIXME: we should probably copy the i386 here and 543 * implement a generic die() routine. Not today. 544 */ 545#ifdef SUPPORT_DIE 546 die("Oops", regs); 547#endif 548 bust_spinlocks(1); 549 550 do_group_exit(SIGKILL); 551 552/* 553 * We ran out of memory, or some other thing happened to us that made 554 * us unable to handle the page fault gracefully. 555 */ 556out_of_memory: 557 up_read(&mm->mmap_sem); 558 if (is_kernel_mode) 559 goto no_context; 560 pagefault_out_of_memory(); 561 return 0; 562 563do_sigbus: 564 up_read(&mm->mmap_sem); 565 566 /* Kernel mode? Handle exceptions or die */ 567 if (is_kernel_mode) 568 goto no_context; 569 570 force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address, 571 fault_num, tsk, regs); 572 return 0; 573} 574 575#ifndef __tilegx__ 576 577/* We must release ICS before panicking or we won't get anywhere. */ 578#define ics_panic(fmt, ...) \ 579do { \ 580 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \ 581 panic(fmt, ##__VA_ARGS__); \ 582} while (0) 583 584/* 585 * When we take an ITLB or DTLB fault or access violation in the 586 * supervisor while the critical section bit is set, the hypervisor is 587 * reluctant to write new values into the EX_CONTEXT_K_x registers, 588 * since that might indicate we have not yet squirreled the SPR 589 * contents away and can thus safely take a recursive interrupt. 590 * Accordingly, the hypervisor passes us the PC via SYSTEM_SAVE_K_2. 591 * 592 * Note that this routine is called before homecache_tlb_defer_enter(), 593 * which means that we can properly unlock any atomics that might 594 * be used there (good), but also means we must be very sensitive 595 * to not touch any data structures that might be located in memory 596 * that could migrate, as we could be entering the kernel on a dataplane 597 * cpu that has been deferring kernel TLB updates. This means, for 598 * example, that we can't migrate init_mm or its pgd. 599 */ 600struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num, 601 unsigned long address, 602 unsigned long info) 603{ 604 unsigned long pc = info & ~1; 605 int write = info & 1; 606 pgd_t *pgd = get_current_pgd(); 607 608 /* Retval is 1 at first since we will handle the fault fully. */ 609 struct intvec_state state = { 610 do_page_fault, fault_num, address, write, 1 611 }; 612 613 /* Validate that we are plausibly in the right routine. */ 614 if ((pc & 0x7) != 0 || pc < PAGE_OFFSET || 615 (fault_num != INT_DTLB_MISS && 616 fault_num != INT_DTLB_ACCESS)) { 617 unsigned long old_pc = regs->pc; 618 regs->pc = pc; 619 ics_panic("Bad ICS page fault args: old PC %#lx, fault %d/%d at %#lx", 620 old_pc, fault_num, write, address); 621 } 622 623 /* We might be faulting on a vmalloc page, so check that first. */ 624 if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0) 625 return state; 626 627 /* 628 * If we faulted with ICS set in sys_cmpxchg, we are providing 629 * a user syscall service that should generate a signal on 630 * fault. We didn't set up a kernel stack on initial entry to 631 * sys_cmpxchg, but instead had one set up by the fault, which 632 * (because sys_cmpxchg never releases ICS) came to us via the 633 * SYSTEM_SAVE_K_2 mechanism, and thus EX_CONTEXT_K_[01] are 634 * still referencing the original user code. We release the 635 * atomic lock and rewrite pt_regs so that it appears that we 636 * came from user-space directly, and after we finish the 637 * fault we'll go back to user space and re-issue the swint. 638 * This way the backtrace information is correct if we need to 639 * emit a stack dump at any point while handling this. 640 * 641 * Must match register use in sys_cmpxchg(). 642 */ 643 if (pc >= (unsigned long) sys_cmpxchg && 644 pc < (unsigned long) __sys_cmpxchg_end) { 645#ifdef CONFIG_SMP 646 /* Don't unlock before we could have locked. */ 647 if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) { 648 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]); 649 __atomic_fault_unlock(lock_ptr); 650 } 651#endif 652 regs->sp = regs->regs[27]; 653 } 654 655 /* 656 * We can also fault in the atomic assembly, in which 657 * case we use the exception table to do the first-level fixup. 658 * We may re-fixup again in the real fault handler if it 659 * turns out the faulting address is just bad, and not, 660 * for example, migrating. 661 */ 662 else if (pc >= (unsigned long) __start_atomic_asm_code && 663 pc < (unsigned long) __end_atomic_asm_code) { 664 const struct exception_table_entry *fixup; 665#ifdef CONFIG_SMP 666 /* Unlock the atomic lock. */ 667 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]); 668 __atomic_fault_unlock(lock_ptr); 669#endif 670 fixup = search_exception_tables(pc); 671 if (!fixup) 672 ics_panic("ICS atomic fault not in table: PC %#lx, fault %d", 673 pc, fault_num); 674 regs->pc = fixup->fixup; 675 regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0); 676 } 677 678 /* 679 * Now that we have released the atomic lock (if necessary), 680 * it's safe to spin if the PTE that caused the fault was migrating. 681 */ 682 if (fault_num == INT_DTLB_ACCESS) 683 write = 1; 684 if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write)) 685 return state; 686 687 /* Return zero so that we continue on with normal fault handling. */ 688 state.retval = 0; 689 return state; 690} 691 692#endif /* !__tilegx__ */ 693 694/* 695 * This routine handles page faults. It determines the address, and the 696 * problem, and then passes it handle_page_fault() for normal DTLB and 697 * ITLB issues, and for DMA or SN processor faults when we are in user 698 * space. For the latter, if we're in kernel mode, we just save the 699 * interrupt away appropriately and return immediately. We can't do 700 * page faults for user code while in kernel mode. 701 */ 702static inline void __do_page_fault(struct pt_regs *regs, int fault_num, 703 unsigned long address, unsigned long write) 704{ 705 int is_page_fault; 706 707#ifdef CONFIG_KPROBES 708 /* 709 * This is to notify the fault handler of the kprobes. The 710 * exception code is redundant as it is also carried in REGS, 711 * but we pass it anyhow. 712 */ 713 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1, 714 regs->faultnum, SIGSEGV) == NOTIFY_STOP) 715 return; 716#endif 717 718#ifdef __tilegx__ 719 /* 720 * We don't need early do_page_fault_ics() support, since unlike 721 * Pro we don't need to worry about unlocking the atomic locks. 722 * There is only one current case in GX where we touch any memory 723 * under ICS other than our own kernel stack, and we handle that 724 * here. (If we crash due to trying to touch our own stack, 725 * we're in too much trouble for C code to help out anyway.) 726 */ 727 if (write & ~1) { 728 unsigned long pc = write & ~1; 729 if (pc >= (unsigned long) __start_unalign_asm_code && 730 pc < (unsigned long) __end_unalign_asm_code) { 731 struct thread_info *ti = current_thread_info(); 732 /* 733 * Our EX_CONTEXT is still what it was from the 734 * initial unalign exception, but now we've faulted 735 * on the JIT page. We would like to complete the 736 * page fault however is appropriate, and then retry 737 * the instruction that caused the unalign exception. 738 * Our state has been "corrupted" by setting the low 739 * bit in "sp", and stashing r0..r3 in the 740 * thread_info area, so we revert all of that, then 741 * continue as if this were a normal page fault. 742 */ 743 regs->sp &= ~1UL; 744 regs->regs[0] = ti->unalign_jit_tmp[0]; 745 regs->regs[1] = ti->unalign_jit_tmp[1]; 746 regs->regs[2] = ti->unalign_jit_tmp[2]; 747 regs->regs[3] = ti->unalign_jit_tmp[3]; 748 write &= 1; 749 } else { 750 pr_alert("%s/%d: ICS set at page fault at %#lx: %#lx\n", 751 current->comm, current->pid, pc, address); 752 show_regs(regs); 753 do_group_exit(SIGKILL); 754 } 755 } 756#else 757 /* This case should have been handled by do_page_fault_ics(). */ 758 BUG_ON(write & ~1); 759#endif 760 761#if CHIP_HAS_TILE_DMA() 762 /* 763 * If it's a DMA fault, suspend the transfer while we're 764 * handling the miss; we'll restart after it's handled. If we 765 * don't suspend, it's possible that this process could swap 766 * out and back in, and restart the engine since the DMA is 767 * still 'running'. 768 */ 769 if (fault_num == INT_DMATLB_MISS || 770 fault_num == INT_DMATLB_ACCESS || 771 fault_num == INT_DMATLB_MISS_DWNCL || 772 fault_num == INT_DMATLB_ACCESS_DWNCL) { 773 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK); 774 while (__insn_mfspr(SPR_DMA_USER_STATUS) & 775 SPR_DMA_STATUS__BUSY_MASK) 776 ; 777 } 778#endif 779 780 /* Validate fault num and decide if this is a first-time page fault. */ 781 switch (fault_num) { 782 case INT_ITLB_MISS: 783 case INT_DTLB_MISS: 784#if CHIP_HAS_TILE_DMA() 785 case INT_DMATLB_MISS: 786 case INT_DMATLB_MISS_DWNCL: 787#endif 788 is_page_fault = 1; 789 break; 790 791 case INT_DTLB_ACCESS: 792#if CHIP_HAS_TILE_DMA() 793 case INT_DMATLB_ACCESS: 794 case INT_DMATLB_ACCESS_DWNCL: 795#endif 796 is_page_fault = 0; 797 break; 798 799 default: 800 panic("Bad fault number %d in do_page_fault", fault_num); 801 } 802 803#if CHIP_HAS_TILE_DMA() 804 if (!user_mode(regs)) { 805 struct async_tlb *async; 806 switch (fault_num) { 807#if CHIP_HAS_TILE_DMA() 808 case INT_DMATLB_MISS: 809 case INT_DMATLB_ACCESS: 810 case INT_DMATLB_MISS_DWNCL: 811 case INT_DMATLB_ACCESS_DWNCL: 812 async = ¤t->thread.dma_async_tlb; 813 break; 814#endif 815 default: 816 async = NULL; 817 } 818 if (async) { 819 820 /* 821 * No vmalloc check required, so we can allow 822 * interrupts immediately at this point. 823 */ 824 local_irq_enable(); 825 826 set_thread_flag(TIF_ASYNC_TLB); 827 if (async->fault_num != 0) { 828 panic("Second async fault %d; old fault was %d (%#lx/%ld)", 829 fault_num, async->fault_num, 830 address, write); 831 } 832 BUG_ON(fault_num == 0); 833 async->fault_num = fault_num; 834 async->is_fault = is_page_fault; 835 async->is_write = write; 836 async->address = address; 837 return; 838 } 839 } 840#endif 841 842 handle_page_fault(regs, fault_num, is_page_fault, address, write); 843} 844 845void do_page_fault(struct pt_regs *regs, int fault_num, 846 unsigned long address, unsigned long write) 847{ 848 enum ctx_state prev_state = exception_enter(); 849 __do_page_fault(regs, fault_num, address, write); 850 exception_exit(prev_state); 851} 852 853#if CHIP_HAS_TILE_DMA() 854/* 855 * This routine effectively re-issues asynchronous page faults 856 * when we are returning to user space. 857 */ 858void do_async_page_fault(struct pt_regs *regs) 859{ 860 struct async_tlb *async = ¤t->thread.dma_async_tlb; 861 862 /* 863 * Clear thread flag early. If we re-interrupt while processing 864 * code here, we will reset it and recall this routine before 865 * returning to user space. 866 */ 867 clear_thread_flag(TIF_ASYNC_TLB); 868 869 if (async->fault_num) { 870 /* 871 * Clear async->fault_num before calling the page-fault 872 * handler so that if we re-interrupt before returning 873 * from the function we have somewhere to put the 874 * information from the new interrupt. 875 */ 876 int fault_num = async->fault_num; 877 async->fault_num = 0; 878 handle_page_fault(regs, fault_num, async->is_fault, 879 async->address, async->is_write); 880 } 881} 882#endif /* CHIP_HAS_TILE_DMA() */ 883 884 885void vmalloc_sync_all(void) 886{ 887#ifdef __tilegx__ 888 /* Currently all L1 kernel pmd's are static and shared. */ 889 BUILD_BUG_ON(pgd_index(VMALLOC_END - PAGE_SIZE) != 890 pgd_index(VMALLOC_START)); 891#else 892 /* 893 * Note that races in the updates of insync and start aren't 894 * problematic: insync can only get set bits added, and updates to 895 * start are only improving performance (without affecting correctness 896 * if undone). 897 */ 898 static DECLARE_BITMAP(insync, PTRS_PER_PGD); 899 static unsigned long start = PAGE_OFFSET; 900 unsigned long address; 901 902 BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK); 903 for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) { 904 if (!test_bit(pgd_index(address), insync)) { 905 unsigned long flags; 906 struct list_head *pos; 907 908 spin_lock_irqsave(&pgd_lock, flags); 909 list_for_each(pos, &pgd_list) 910 if (!vmalloc_sync_one(list_to_pgd(pos), 911 address)) { 912 /* Must be at first entry in list. */ 913 BUG_ON(pos != pgd_list.next); 914 break; 915 } 916 spin_unlock_irqrestore(&pgd_lock, flags); 917 if (pos != pgd_list.next) 918 set_bit(pgd_index(address), insync); 919 } 920 if (address == start && test_bit(pgd_index(address), insync)) 921 start = address + PGDIR_SIZE; 922 } 923#endif 924} 925