1/* 2 * 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 */ 5 6/* 7 * entry.S contains the system-call and fault low-level handling routines. 8 * This also contains the timer-interrupt handler, as well as all interrupts 9 * and faults that can result in a task-switch. 10 * 11 * NOTE: This code handles signal-recognition, which happens every time 12 * after a timer-interrupt and after each system call. 13 * 14 * I changed all the .align's to 4 (16 byte alignment), as that's faster 15 * on a 486. 16 * 17 * Stack layout in 'syscall_exit': 18 * ptrace needs to have all regs on the stack. 19 * if the order here is changed, it needs to be 20 * updated in fork.c:copy_process, signal.c:do_signal, 21 * ptrace.c and ptrace.h 22 * 23 * 0(%esp) - %ebx 24 * 4(%esp) - %ecx 25 * 8(%esp) - %edx 26 * C(%esp) - %esi 27 * 10(%esp) - %edi 28 * 14(%esp) - %ebp 29 * 18(%esp) - %eax 30 * 1C(%esp) - %ds 31 * 20(%esp) - %es 32 * 24(%esp) - %fs 33 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 34 * 2C(%esp) - orig_eax 35 * 30(%esp) - %eip 36 * 34(%esp) - %cs 37 * 38(%esp) - %eflags 38 * 3C(%esp) - %oldesp 39 * 40(%esp) - %oldss 40 * 41 * "current" is in register %ebx during any slow entries. 42 */ 43 44#include <linux/linkage.h> 45#include <linux/err.h> 46#include <asm/thread_info.h> 47#include <asm/irqflags.h> 48#include <asm/errno.h> 49#include <asm/segment.h> 50#include <asm/smp.h> 51#include <asm/page_types.h> 52#include <asm/percpu.h> 53#include <asm/dwarf2.h> 54#include <asm/processor-flags.h> 55#include <asm/ftrace.h> 56#include <asm/irq_vectors.h> 57#include <asm/cpufeature.h> 58#include <asm/alternative-asm.h> 59#include <asm/asm.h> 60#include <asm/smap.h> 61 62/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 63#include <linux/elf-em.h> 64#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) 65#define __AUDIT_ARCH_LE 0x40000000 66 67#ifndef CONFIG_AUDITSYSCALL 68#define sysenter_audit syscall_trace_entry 69#define sysexit_audit syscall_exit_work 70#endif 71 72 .section .entry.text, "ax" 73 74/* 75 * We use macros for low-level operations which need to be overridden 76 * for paravirtualization. The following will never clobber any registers: 77 * INTERRUPT_RETURN (aka. "iret") 78 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 79 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 80 * 81 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 82 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 83 * Allowing a register to be clobbered can shrink the paravirt replacement 84 * enough to patch inline, increasing performance. 85 */ 86 87#ifdef CONFIG_PREEMPT 88#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 89#else 90#define preempt_stop(clobbers) 91#define resume_kernel restore_all 92#endif 93 94.macro TRACE_IRQS_IRET 95#ifdef CONFIG_TRACE_IRQFLAGS 96 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off? 97 jz 1f 98 TRACE_IRQS_ON 991: 100#endif 101.endm 102 103/* 104 * User gs save/restore 105 * 106 * %gs is used for userland TLS and kernel only uses it for stack 107 * canary which is required to be at %gs:20 by gcc. Read the comment 108 * at the top of stackprotector.h for more info. 109 * 110 * Local labels 98 and 99 are used. 111 */ 112#ifdef CONFIG_X86_32_LAZY_GS 113 114 /* unfortunately push/pop can't be no-op */ 115.macro PUSH_GS 116 pushl_cfi $0 117.endm 118.macro POP_GS pop=0 119 addl $(4 + \pop), %esp 120 CFI_ADJUST_CFA_OFFSET -(4 + \pop) 121.endm 122.macro POP_GS_EX 123.endm 124 125 /* all the rest are no-op */ 126.macro PTGS_TO_GS 127.endm 128.macro PTGS_TO_GS_EX 129.endm 130.macro GS_TO_REG reg 131.endm 132.macro REG_TO_PTGS reg 133.endm 134.macro SET_KERNEL_GS reg 135.endm 136 137#else /* CONFIG_X86_32_LAZY_GS */ 138 139.macro PUSH_GS 140 pushl_cfi %gs 141 /*CFI_REL_OFFSET gs, 0*/ 142.endm 143 144.macro POP_GS pop=0 14598: popl_cfi %gs 146 /*CFI_RESTORE gs*/ 147 .if \pop <> 0 148 add $\pop, %esp 149 CFI_ADJUST_CFA_OFFSET -\pop 150 .endif 151.endm 152.macro POP_GS_EX 153.pushsection .fixup, "ax" 15499: movl $0, (%esp) 155 jmp 98b 156.popsection 157 _ASM_EXTABLE(98b,99b) 158.endm 159 160.macro PTGS_TO_GS 16198: mov PT_GS(%esp), %gs 162.endm 163.macro PTGS_TO_GS_EX 164.pushsection .fixup, "ax" 16599: movl $0, PT_GS(%esp) 166 jmp 98b 167.popsection 168 _ASM_EXTABLE(98b,99b) 169.endm 170 171.macro GS_TO_REG reg 172 movl %gs, \reg 173 /*CFI_REGISTER gs, \reg*/ 174.endm 175.macro REG_TO_PTGS reg 176 movl \reg, PT_GS(%esp) 177 /*CFI_REL_OFFSET gs, PT_GS*/ 178.endm 179.macro SET_KERNEL_GS reg 180 movl $(__KERNEL_STACK_CANARY), \reg 181 movl \reg, %gs 182.endm 183 184#endif /* CONFIG_X86_32_LAZY_GS */ 185 186.macro SAVE_ALL 187 cld 188 PUSH_GS 189 pushl_cfi %fs 190 /*CFI_REL_OFFSET fs, 0;*/ 191 pushl_cfi %es 192 /*CFI_REL_OFFSET es, 0;*/ 193 pushl_cfi %ds 194 /*CFI_REL_OFFSET ds, 0;*/ 195 pushl_cfi %eax 196 CFI_REL_OFFSET eax, 0 197 pushl_cfi %ebp 198 CFI_REL_OFFSET ebp, 0 199 pushl_cfi %edi 200 CFI_REL_OFFSET edi, 0 201 pushl_cfi %esi 202 CFI_REL_OFFSET esi, 0 203 pushl_cfi %edx 204 CFI_REL_OFFSET edx, 0 205 pushl_cfi %ecx 206 CFI_REL_OFFSET ecx, 0 207 pushl_cfi %ebx 208 CFI_REL_OFFSET ebx, 0 209 movl $(__USER_DS), %edx 210 movl %edx, %ds 211 movl %edx, %es 212 movl $(__KERNEL_PERCPU), %edx 213 movl %edx, %fs 214 SET_KERNEL_GS %edx 215.endm 216 217.macro RESTORE_INT_REGS 218 popl_cfi %ebx 219 CFI_RESTORE ebx 220 popl_cfi %ecx 221 CFI_RESTORE ecx 222 popl_cfi %edx 223 CFI_RESTORE edx 224 popl_cfi %esi 225 CFI_RESTORE esi 226 popl_cfi %edi 227 CFI_RESTORE edi 228 popl_cfi %ebp 229 CFI_RESTORE ebp 230 popl_cfi %eax 231 CFI_RESTORE eax 232.endm 233 234.macro RESTORE_REGS pop=0 235 RESTORE_INT_REGS 2361: popl_cfi %ds 237 /*CFI_RESTORE ds;*/ 2382: popl_cfi %es 239 /*CFI_RESTORE es;*/ 2403: popl_cfi %fs 241 /*CFI_RESTORE fs;*/ 242 POP_GS \pop 243.pushsection .fixup, "ax" 2444: movl $0, (%esp) 245 jmp 1b 2465: movl $0, (%esp) 247 jmp 2b 2486: movl $0, (%esp) 249 jmp 3b 250.popsection 251 _ASM_EXTABLE(1b,4b) 252 _ASM_EXTABLE(2b,5b) 253 _ASM_EXTABLE(3b,6b) 254 POP_GS_EX 255.endm 256 257.macro RING0_INT_FRAME 258 CFI_STARTPROC simple 259 CFI_SIGNAL_FRAME 260 CFI_DEF_CFA esp, 3*4 261 /*CFI_OFFSET cs, -2*4;*/ 262 CFI_OFFSET eip, -3*4 263.endm 264 265.macro RING0_EC_FRAME 266 CFI_STARTPROC simple 267 CFI_SIGNAL_FRAME 268 CFI_DEF_CFA esp, 4*4 269 /*CFI_OFFSET cs, -2*4;*/ 270 CFI_OFFSET eip, -3*4 271.endm 272 273.macro RING0_PTREGS_FRAME 274 CFI_STARTPROC simple 275 CFI_SIGNAL_FRAME 276 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX 277 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/ 278 CFI_OFFSET eip, PT_EIP-PT_OLDESP 279 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/ 280 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/ 281 CFI_OFFSET eax, PT_EAX-PT_OLDESP 282 CFI_OFFSET ebp, PT_EBP-PT_OLDESP 283 CFI_OFFSET edi, PT_EDI-PT_OLDESP 284 CFI_OFFSET esi, PT_ESI-PT_OLDESP 285 CFI_OFFSET edx, PT_EDX-PT_OLDESP 286 CFI_OFFSET ecx, PT_ECX-PT_OLDESP 287 CFI_OFFSET ebx, PT_EBX-PT_OLDESP 288.endm 289 290ENTRY(ret_from_fork) 291 CFI_STARTPROC 292 pushl_cfi %eax 293 call schedule_tail 294 GET_THREAD_INFO(%ebp) 295 popl_cfi %eax 296 pushl_cfi $0x0202 # Reset kernel eflags 297 popfl_cfi 298 jmp syscall_exit 299 CFI_ENDPROC 300END(ret_from_fork) 301 302ENTRY(ret_from_kernel_thread) 303 CFI_STARTPROC 304 pushl_cfi %eax 305 call schedule_tail 306 GET_THREAD_INFO(%ebp) 307 popl_cfi %eax 308 pushl_cfi $0x0202 # Reset kernel eflags 309 popfl_cfi 310 movl PT_EBP(%esp),%eax 311 call *PT_EBX(%esp) 312 movl $0,PT_EAX(%esp) 313 jmp syscall_exit 314 CFI_ENDPROC 315ENDPROC(ret_from_kernel_thread) 316 317/* 318 * Return to user mode is not as complex as all this looks, 319 * but we want the default path for a system call return to 320 * go as quickly as possible which is why some of this is 321 * less clear than it otherwise should be. 322 */ 323 324 # userspace resumption stub bypassing syscall exit tracing 325 ALIGN 326 RING0_PTREGS_FRAME 327ret_from_exception: 328 preempt_stop(CLBR_ANY) 329ret_from_intr: 330 GET_THREAD_INFO(%ebp) 331#ifdef CONFIG_VM86 332 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 333 movb PT_CS(%esp), %al 334 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 335#else 336 /* 337 * We can be coming here from child spawned by kernel_thread(). 338 */ 339 movl PT_CS(%esp), %eax 340 andl $SEGMENT_RPL_MASK, %eax 341#endif 342 cmpl $USER_RPL, %eax 343 jb resume_kernel # not returning to v8086 or userspace 344 345ENTRY(resume_userspace) 346 LOCKDEP_SYS_EXIT 347 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 348 # setting need_resched or sigpending 349 # between sampling and the iret 350 TRACE_IRQS_OFF 351 movl TI_flags(%ebp), %ecx 352 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 353 # int/exception return? 354 jne work_pending 355 jmp restore_all 356END(ret_from_exception) 357 358#ifdef CONFIG_PREEMPT 359ENTRY(resume_kernel) 360 DISABLE_INTERRUPTS(CLBR_ANY) 361need_resched: 362 cmpl $0,PER_CPU_VAR(__preempt_count) 363 jnz restore_all 364 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? 365 jz restore_all 366 call preempt_schedule_irq 367 jmp need_resched 368END(resume_kernel) 369#endif 370 CFI_ENDPROC 371 372/* SYSENTER_RETURN points to after the "sysenter" instruction in 373 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 374 375 # sysenter call handler stub 376ENTRY(ia32_sysenter_target) 377 CFI_STARTPROC simple 378 CFI_SIGNAL_FRAME 379 CFI_DEF_CFA esp, 0 380 CFI_REGISTER esp, ebp 381 movl TSS_sysenter_sp0(%esp),%esp 382sysenter_past_esp: 383 /* 384 * Interrupts are disabled here, but we can't trace it until 385 * enough kernel state to call TRACE_IRQS_OFF can be called - but 386 * we immediately enable interrupts at that point anyway. 387 */ 388 pushl_cfi $__USER_DS 389 /*CFI_REL_OFFSET ss, 0*/ 390 pushl_cfi %ebp 391 CFI_REL_OFFSET esp, 0 392 pushfl_cfi 393 orl $X86_EFLAGS_IF, (%esp) 394 pushl_cfi $__USER_CS 395 /*CFI_REL_OFFSET cs, 0*/ 396 /* 397 * Push current_thread_info()->sysenter_return to the stack. 398 * A tiny bit of offset fixup is necessary: TI_sysenter_return 399 * is relative to thread_info, which is at the bottom of the 400 * kernel stack page. 4*4 means the 4 words pushed above; 401 * TOP_OF_KERNEL_STACK_PADDING takes us to the top of the stack; 402 * and THREAD_SIZE takes us to the bottom. 403 */ 404 pushl_cfi ((TI_sysenter_return) - THREAD_SIZE + TOP_OF_KERNEL_STACK_PADDING + 4*4)(%esp) 405 CFI_REL_OFFSET eip, 0 406 407 pushl_cfi %eax 408 SAVE_ALL 409 ENABLE_INTERRUPTS(CLBR_NONE) 410 411/* 412 * Load the potential sixth argument from user stack. 413 * Careful about security. 414 */ 415 cmpl $__PAGE_OFFSET-3,%ebp 416 jae syscall_fault 417 ASM_STAC 4181: movl (%ebp),%ebp 419 ASM_CLAC 420 movl %ebp,PT_EBP(%esp) 421 _ASM_EXTABLE(1b,syscall_fault) 422 423 GET_THREAD_INFO(%ebp) 424 425 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 426 jnz sysenter_audit 427sysenter_do_call: 428 cmpl $(NR_syscalls), %eax 429 jae sysenter_badsys 430 call *sys_call_table(,%eax,4) 431sysenter_after_call: 432 movl %eax,PT_EAX(%esp) 433 LOCKDEP_SYS_EXIT 434 DISABLE_INTERRUPTS(CLBR_ANY) 435 TRACE_IRQS_OFF 436 movl TI_flags(%ebp), %ecx 437 testl $_TIF_ALLWORK_MASK, %ecx 438 jnz sysexit_audit 439sysenter_exit: 440/* if something modifies registers it must also disable sysexit */ 441 movl PT_EIP(%esp), %edx 442 movl PT_OLDESP(%esp), %ecx 443 xorl %ebp,%ebp 444 TRACE_IRQS_ON 4451: mov PT_FS(%esp), %fs 446 PTGS_TO_GS 447 ENABLE_INTERRUPTS_SYSEXIT 448 449#ifdef CONFIG_AUDITSYSCALL 450sysenter_audit: 451 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 452 jnz syscall_trace_entry 453 /* movl PT_EAX(%esp), %eax already set, syscall number: 1st arg to audit */ 454 movl PT_EBX(%esp), %edx /* ebx/a0: 2nd arg to audit */ 455 /* movl PT_ECX(%esp), %ecx already set, a1: 3nd arg to audit */ 456 pushl_cfi PT_ESI(%esp) /* a3: 5th arg */ 457 pushl_cfi PT_EDX+4(%esp) /* a2: 4th arg */ 458 call __audit_syscall_entry 459 popl_cfi %ecx /* get that remapped edx off the stack */ 460 popl_cfi %ecx /* get that remapped esi off the stack */ 461 movl PT_EAX(%esp),%eax /* reload syscall number */ 462 jmp sysenter_do_call 463 464sysexit_audit: 465 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 466 jnz syscall_exit_work 467 TRACE_IRQS_ON 468 ENABLE_INTERRUPTS(CLBR_ANY) 469 movl %eax,%edx /* second arg, syscall return value */ 470 cmpl $-MAX_ERRNO,%eax /* is it an error ? */ 471 setbe %al /* 1 if so, 0 if not */ 472 movzbl %al,%eax /* zero-extend that */ 473 call __audit_syscall_exit 474 DISABLE_INTERRUPTS(CLBR_ANY) 475 TRACE_IRQS_OFF 476 movl TI_flags(%ebp), %ecx 477 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 478 jnz syscall_exit_work 479 movl PT_EAX(%esp),%eax /* reload syscall return value */ 480 jmp sysenter_exit 481#endif 482 483 CFI_ENDPROC 484.pushsection .fixup,"ax" 4852: movl $0,PT_FS(%esp) 486 jmp 1b 487.popsection 488 _ASM_EXTABLE(1b,2b) 489 PTGS_TO_GS_EX 490ENDPROC(ia32_sysenter_target) 491 492 # system call handler stub 493ENTRY(system_call) 494 RING0_INT_FRAME # can't unwind into user space anyway 495 ASM_CLAC 496 pushl_cfi %eax # save orig_eax 497 SAVE_ALL 498 GET_THREAD_INFO(%ebp) 499 # system call tracing in operation / emulation 500 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 501 jnz syscall_trace_entry 502 cmpl $(NR_syscalls), %eax 503 jae syscall_badsys 504syscall_call: 505 call *sys_call_table(,%eax,4) 506syscall_after_call: 507 movl %eax,PT_EAX(%esp) # store the return value 508syscall_exit: 509 LOCKDEP_SYS_EXIT 510 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 511 # setting need_resched or sigpending 512 # between sampling and the iret 513 TRACE_IRQS_OFF 514 movl TI_flags(%ebp), %ecx 515 testl $_TIF_ALLWORK_MASK, %ecx # current->work 516 jnz syscall_exit_work 517 518restore_all: 519 TRACE_IRQS_IRET 520restore_all_notrace: 521#ifdef CONFIG_X86_ESPFIX32 522 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 523 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 524 # are returning to the kernel. 525 # See comments in process.c:copy_thread() for details. 526 movb PT_OLDSS(%esp), %ah 527 movb PT_CS(%esp), %al 528 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 529 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 530 CFI_REMEMBER_STATE 531 je ldt_ss # returning to user-space with LDT SS 532#endif 533restore_nocheck: 534 RESTORE_REGS 4 # skip orig_eax/error_code 535irq_return: 536 INTERRUPT_RETURN 537.section .fixup,"ax" 538ENTRY(iret_exc) 539 pushl $0 # no error code 540 pushl $do_iret_error 541 jmp error_code 542.previous 543 _ASM_EXTABLE(irq_return,iret_exc) 544 545#ifdef CONFIG_X86_ESPFIX32 546 CFI_RESTORE_STATE 547ldt_ss: 548#ifdef CONFIG_PARAVIRT 549 /* 550 * The kernel can't run on a non-flat stack if paravirt mode 551 * is active. Rather than try to fixup the high bits of 552 * ESP, bypass this code entirely. This may break DOSemu 553 * and/or Wine support in a paravirt VM, although the option 554 * is still available to implement the setting of the high 555 * 16-bits in the INTERRUPT_RETURN paravirt-op. 556 */ 557 cmpl $0, pv_info+PARAVIRT_enabled 558 jne restore_nocheck 559#endif 560 561/* 562 * Setup and switch to ESPFIX stack 563 * 564 * We're returning to userspace with a 16 bit stack. The CPU will not 565 * restore the high word of ESP for us on executing iret... This is an 566 * "official" bug of all the x86-compatible CPUs, which we can work 567 * around to make dosemu and wine happy. We do this by preloading the 568 * high word of ESP with the high word of the userspace ESP while 569 * compensating for the offset by changing to the ESPFIX segment with 570 * a base address that matches for the difference. 571 */ 572#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 573 mov %esp, %edx /* load kernel esp */ 574 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 575 mov %dx, %ax /* eax: new kernel esp */ 576 sub %eax, %edx /* offset (low word is 0) */ 577 shr $16, %edx 578 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 579 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 580 pushl_cfi $__ESPFIX_SS 581 pushl_cfi %eax /* new kernel esp */ 582 /* Disable interrupts, but do not irqtrace this section: we 583 * will soon execute iret and the tracer was already set to 584 * the irqstate after the iret */ 585 DISABLE_INTERRUPTS(CLBR_EAX) 586 lss (%esp), %esp /* switch to espfix segment */ 587 CFI_ADJUST_CFA_OFFSET -8 588 jmp restore_nocheck 589#endif 590 CFI_ENDPROC 591ENDPROC(system_call) 592 593 # perform work that needs to be done immediately before resumption 594 ALIGN 595 RING0_PTREGS_FRAME # can't unwind into user space anyway 596work_pending: 597 testb $_TIF_NEED_RESCHED, %cl 598 jz work_notifysig 599work_resched: 600 call schedule 601 LOCKDEP_SYS_EXIT 602 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 603 # setting need_resched or sigpending 604 # between sampling and the iret 605 TRACE_IRQS_OFF 606 movl TI_flags(%ebp), %ecx 607 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 608 # than syscall tracing? 609 jz restore_all 610 testb $_TIF_NEED_RESCHED, %cl 611 jnz work_resched 612 613work_notifysig: # deal with pending signals and 614 # notify-resume requests 615#ifdef CONFIG_VM86 616 testl $X86_EFLAGS_VM, PT_EFLAGS(%esp) 617 movl %esp, %eax 618 jnz work_notifysig_v86 # returning to kernel-space or 619 # vm86-space 6201: 621#else 622 movl %esp, %eax 623#endif 624 TRACE_IRQS_ON 625 ENABLE_INTERRUPTS(CLBR_NONE) 626 movb PT_CS(%esp), %bl 627 andb $SEGMENT_RPL_MASK, %bl 628 cmpb $USER_RPL, %bl 629 jb resume_kernel 630 xorl %edx, %edx 631 call do_notify_resume 632 jmp resume_userspace 633 634#ifdef CONFIG_VM86 635 ALIGN 636work_notifysig_v86: 637 pushl_cfi %ecx # save ti_flags for do_notify_resume 638 call save_v86_state # %eax contains pt_regs pointer 639 popl_cfi %ecx 640 movl %eax, %esp 641 jmp 1b 642#endif 643END(work_pending) 644 645 # perform syscall exit tracing 646 ALIGN 647syscall_trace_entry: 648 movl $-ENOSYS,PT_EAX(%esp) 649 movl %esp, %eax 650 call syscall_trace_enter 651 /* What it returned is what we'll actually use. */ 652 cmpl $(NR_syscalls), %eax 653 jnae syscall_call 654 jmp syscall_exit 655END(syscall_trace_entry) 656 657 # perform syscall exit tracing 658 ALIGN 659syscall_exit_work: 660 testl $_TIF_WORK_SYSCALL_EXIT, %ecx 661 jz work_pending 662 TRACE_IRQS_ON 663 ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call 664 # schedule() instead 665 movl %esp, %eax 666 call syscall_trace_leave 667 jmp resume_userspace 668END(syscall_exit_work) 669 CFI_ENDPROC 670 671 RING0_INT_FRAME # can't unwind into user space anyway 672syscall_fault: 673 ASM_CLAC 674 GET_THREAD_INFO(%ebp) 675 movl $-EFAULT,PT_EAX(%esp) 676 jmp resume_userspace 677END(syscall_fault) 678 679syscall_badsys: 680 movl $-ENOSYS,%eax 681 jmp syscall_after_call 682END(syscall_badsys) 683 684sysenter_badsys: 685 movl $-ENOSYS,%eax 686 jmp sysenter_after_call 687END(sysenter_badsys) 688 CFI_ENDPROC 689 690.macro FIXUP_ESPFIX_STACK 691/* 692 * Switch back for ESPFIX stack to the normal zerobased stack 693 * 694 * We can't call C functions using the ESPFIX stack. This code reads 695 * the high word of the segment base from the GDT and swiches to the 696 * normal stack and adjusts ESP with the matching offset. 697 */ 698#ifdef CONFIG_X86_ESPFIX32 699 /* fixup the stack */ 700 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 701 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 702 shl $16, %eax 703 addl %esp, %eax /* the adjusted stack pointer */ 704 pushl_cfi $__KERNEL_DS 705 pushl_cfi %eax 706 lss (%esp), %esp /* switch to the normal stack segment */ 707 CFI_ADJUST_CFA_OFFSET -8 708#endif 709.endm 710.macro UNWIND_ESPFIX_STACK 711#ifdef CONFIG_X86_ESPFIX32 712 movl %ss, %eax 713 /* see if on espfix stack */ 714 cmpw $__ESPFIX_SS, %ax 715 jne 27f 716 movl $__KERNEL_DS, %eax 717 movl %eax, %ds 718 movl %eax, %es 719 /* switch to normal stack */ 720 FIXUP_ESPFIX_STACK 72127: 722#endif 723.endm 724 725/* 726 * Build the entry stubs with some assembler magic. 727 * We pack 1 stub into every 8-byte block. 728 */ 729 .align 8 730ENTRY(irq_entries_start) 731 RING0_INT_FRAME 732 vector=FIRST_EXTERNAL_VECTOR 733 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 734 pushl_cfi $(~vector+0x80) /* Note: always in signed byte range */ 735 vector=vector+1 736 jmp common_interrupt 737 CFI_ADJUST_CFA_OFFSET -4 738 .align 8 739 .endr 740END(irq_entries_start) 741 742/* 743 * the CPU automatically disables interrupts when executing an IRQ vector, 744 * so IRQ-flags tracing has to follow that: 745 */ 746 .p2align CONFIG_X86_L1_CACHE_SHIFT 747common_interrupt: 748 ASM_CLAC 749 addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */ 750 SAVE_ALL 751 TRACE_IRQS_OFF 752 movl %esp,%eax 753 call do_IRQ 754 jmp ret_from_intr 755ENDPROC(common_interrupt) 756 CFI_ENDPROC 757 758#define BUILD_INTERRUPT3(name, nr, fn) \ 759ENTRY(name) \ 760 RING0_INT_FRAME; \ 761 ASM_CLAC; \ 762 pushl_cfi $~(nr); \ 763 SAVE_ALL; \ 764 TRACE_IRQS_OFF \ 765 movl %esp,%eax; \ 766 call fn; \ 767 jmp ret_from_intr; \ 768 CFI_ENDPROC; \ 769ENDPROC(name) 770 771 772#ifdef CONFIG_TRACING 773#define TRACE_BUILD_INTERRUPT(name, nr) \ 774 BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name) 775#else 776#define TRACE_BUILD_INTERRUPT(name, nr) 777#endif 778 779#define BUILD_INTERRUPT(name, nr) \ 780 BUILD_INTERRUPT3(name, nr, smp_##name); \ 781 TRACE_BUILD_INTERRUPT(name, nr) 782 783/* The include is where all of the SMP etc. interrupts come from */ 784#include <asm/entry_arch.h> 785 786ENTRY(coprocessor_error) 787 RING0_INT_FRAME 788 ASM_CLAC 789 pushl_cfi $0 790 pushl_cfi $do_coprocessor_error 791 jmp error_code 792 CFI_ENDPROC 793END(coprocessor_error) 794 795ENTRY(simd_coprocessor_error) 796 RING0_INT_FRAME 797 ASM_CLAC 798 pushl_cfi $0 799#ifdef CONFIG_X86_INVD_BUG 800 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 801 ALTERNATIVE "pushl_cfi $do_general_protection", \ 802 "pushl $do_simd_coprocessor_error", \ 803 X86_FEATURE_XMM 804#else 805 pushl_cfi $do_simd_coprocessor_error 806#endif 807 jmp error_code 808 CFI_ENDPROC 809END(simd_coprocessor_error) 810 811ENTRY(device_not_available) 812 RING0_INT_FRAME 813 ASM_CLAC 814 pushl_cfi $-1 # mark this as an int 815 pushl_cfi $do_device_not_available 816 jmp error_code 817 CFI_ENDPROC 818END(device_not_available) 819 820#ifdef CONFIG_PARAVIRT 821ENTRY(native_iret) 822 iret 823 _ASM_EXTABLE(native_iret, iret_exc) 824END(native_iret) 825 826ENTRY(native_irq_enable_sysexit) 827 sti 828 sysexit 829END(native_irq_enable_sysexit) 830#endif 831 832ENTRY(overflow) 833 RING0_INT_FRAME 834 ASM_CLAC 835 pushl_cfi $0 836 pushl_cfi $do_overflow 837 jmp error_code 838 CFI_ENDPROC 839END(overflow) 840 841ENTRY(bounds) 842 RING0_INT_FRAME 843 ASM_CLAC 844 pushl_cfi $0 845 pushl_cfi $do_bounds 846 jmp error_code 847 CFI_ENDPROC 848END(bounds) 849 850ENTRY(invalid_op) 851 RING0_INT_FRAME 852 ASM_CLAC 853 pushl_cfi $0 854 pushl_cfi $do_invalid_op 855 jmp error_code 856 CFI_ENDPROC 857END(invalid_op) 858 859ENTRY(coprocessor_segment_overrun) 860 RING0_INT_FRAME 861 ASM_CLAC 862 pushl_cfi $0 863 pushl_cfi $do_coprocessor_segment_overrun 864 jmp error_code 865 CFI_ENDPROC 866END(coprocessor_segment_overrun) 867 868ENTRY(invalid_TSS) 869 RING0_EC_FRAME 870 ASM_CLAC 871 pushl_cfi $do_invalid_TSS 872 jmp error_code 873 CFI_ENDPROC 874END(invalid_TSS) 875 876ENTRY(segment_not_present) 877 RING0_EC_FRAME 878 ASM_CLAC 879 pushl_cfi $do_segment_not_present 880 jmp error_code 881 CFI_ENDPROC 882END(segment_not_present) 883 884ENTRY(stack_segment) 885 RING0_EC_FRAME 886 ASM_CLAC 887 pushl_cfi $do_stack_segment 888 jmp error_code 889 CFI_ENDPROC 890END(stack_segment) 891 892ENTRY(alignment_check) 893 RING0_EC_FRAME 894 ASM_CLAC 895 pushl_cfi $do_alignment_check 896 jmp error_code 897 CFI_ENDPROC 898END(alignment_check) 899 900ENTRY(divide_error) 901 RING0_INT_FRAME 902 ASM_CLAC 903 pushl_cfi $0 # no error code 904 pushl_cfi $do_divide_error 905 jmp error_code 906 CFI_ENDPROC 907END(divide_error) 908 909#ifdef CONFIG_X86_MCE 910ENTRY(machine_check) 911 RING0_INT_FRAME 912 ASM_CLAC 913 pushl_cfi $0 914 pushl_cfi machine_check_vector 915 jmp error_code 916 CFI_ENDPROC 917END(machine_check) 918#endif 919 920ENTRY(spurious_interrupt_bug) 921 RING0_INT_FRAME 922 ASM_CLAC 923 pushl_cfi $0 924 pushl_cfi $do_spurious_interrupt_bug 925 jmp error_code 926 CFI_ENDPROC 927END(spurious_interrupt_bug) 928 929#ifdef CONFIG_XEN 930/* Xen doesn't set %esp to be precisely what the normal sysenter 931 entrypoint expects, so fix it up before using the normal path. */ 932ENTRY(xen_sysenter_target) 933 RING0_INT_FRAME 934 addl $5*4, %esp /* remove xen-provided frame */ 935 CFI_ADJUST_CFA_OFFSET -5*4 936 jmp sysenter_past_esp 937 CFI_ENDPROC 938 939ENTRY(xen_hypervisor_callback) 940 CFI_STARTPROC 941 pushl_cfi $-1 /* orig_ax = -1 => not a system call */ 942 SAVE_ALL 943 TRACE_IRQS_OFF 944 945 /* Check to see if we got the event in the critical 946 region in xen_iret_direct, after we've reenabled 947 events and checked for pending events. This simulates 948 iret instruction's behaviour where it delivers a 949 pending interrupt when enabling interrupts. */ 950 movl PT_EIP(%esp),%eax 951 cmpl $xen_iret_start_crit,%eax 952 jb 1f 953 cmpl $xen_iret_end_crit,%eax 954 jae 1f 955 956 jmp xen_iret_crit_fixup 957 958ENTRY(xen_do_upcall) 9591: mov %esp, %eax 960 call xen_evtchn_do_upcall 961#ifndef CONFIG_PREEMPT 962 call xen_maybe_preempt_hcall 963#endif 964 jmp ret_from_intr 965 CFI_ENDPROC 966ENDPROC(xen_hypervisor_callback) 967 968# Hypervisor uses this for application faults while it executes. 969# We get here for two reasons: 970# 1. Fault while reloading DS, ES, FS or GS 971# 2. Fault while executing IRET 972# Category 1 we fix up by reattempting the load, and zeroing the segment 973# register if the load fails. 974# Category 2 we fix up by jumping to do_iret_error. We cannot use the 975# normal Linux return path in this case because if we use the IRET hypercall 976# to pop the stack frame we end up in an infinite loop of failsafe callbacks. 977# We distinguish between categories by maintaining a status value in EAX. 978ENTRY(xen_failsafe_callback) 979 CFI_STARTPROC 980 pushl_cfi %eax 981 movl $1,%eax 9821: mov 4(%esp),%ds 9832: mov 8(%esp),%es 9843: mov 12(%esp),%fs 9854: mov 16(%esp),%gs 986 /* EAX == 0 => Category 1 (Bad segment) 987 EAX != 0 => Category 2 (Bad IRET) */ 988 testl %eax,%eax 989 popl_cfi %eax 990 lea 16(%esp),%esp 991 CFI_ADJUST_CFA_OFFSET -16 992 jz 5f 993 jmp iret_exc 9945: pushl_cfi $-1 /* orig_ax = -1 => not a system call */ 995 SAVE_ALL 996 jmp ret_from_exception 997 CFI_ENDPROC 998 999.section .fixup,"ax" 10006: xorl %eax,%eax 1001 movl %eax,4(%esp) 1002 jmp 1b 10037: xorl %eax,%eax 1004 movl %eax,8(%esp) 1005 jmp 2b 10068: xorl %eax,%eax 1007 movl %eax,12(%esp) 1008 jmp 3b 10099: xorl %eax,%eax 1010 movl %eax,16(%esp) 1011 jmp 4b 1012.previous 1013 _ASM_EXTABLE(1b,6b) 1014 _ASM_EXTABLE(2b,7b) 1015 _ASM_EXTABLE(3b,8b) 1016 _ASM_EXTABLE(4b,9b) 1017ENDPROC(xen_failsafe_callback) 1018 1019BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 1020 xen_evtchn_do_upcall) 1021 1022#endif /* CONFIG_XEN */ 1023 1024#if IS_ENABLED(CONFIG_HYPERV) 1025 1026BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 1027 hyperv_vector_handler) 1028 1029#endif /* CONFIG_HYPERV */ 1030 1031#ifdef CONFIG_FUNCTION_TRACER 1032#ifdef CONFIG_DYNAMIC_FTRACE 1033 1034ENTRY(mcount) 1035 ret 1036END(mcount) 1037 1038ENTRY(ftrace_caller) 1039 pushl %eax 1040 pushl %ecx 1041 pushl %edx 1042 pushl $0 /* Pass NULL as regs pointer */ 1043 movl 4*4(%esp), %eax 1044 movl 0x4(%ebp), %edx 1045 movl function_trace_op, %ecx 1046 subl $MCOUNT_INSN_SIZE, %eax 1047 1048.globl ftrace_call 1049ftrace_call: 1050 call ftrace_stub 1051 1052 addl $4,%esp /* skip NULL pointer */ 1053 popl %edx 1054 popl %ecx 1055 popl %eax 1056ftrace_ret: 1057#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1058.globl ftrace_graph_call 1059ftrace_graph_call: 1060 jmp ftrace_stub 1061#endif 1062 1063.globl ftrace_stub 1064ftrace_stub: 1065 ret 1066END(ftrace_caller) 1067 1068ENTRY(ftrace_regs_caller) 1069 pushf /* push flags before compare (in cs location) */ 1070 1071 /* 1072 * i386 does not save SS and ESP when coming from kernel. 1073 * Instead, to get sp, ®s->sp is used (see ptrace.h). 1074 * Unfortunately, that means eflags must be at the same location 1075 * as the current return ip is. We move the return ip into the 1076 * ip location, and move flags into the return ip location. 1077 */ 1078 pushl 4(%esp) /* save return ip into ip slot */ 1079 1080 pushl $0 /* Load 0 into orig_ax */ 1081 pushl %gs 1082 pushl %fs 1083 pushl %es 1084 pushl %ds 1085 pushl %eax 1086 pushl %ebp 1087 pushl %edi 1088 pushl %esi 1089 pushl %edx 1090 pushl %ecx 1091 pushl %ebx 1092 1093 movl 13*4(%esp), %eax /* Get the saved flags */ 1094 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ 1095 /* clobbering return ip */ 1096 movl $__KERNEL_CS,13*4(%esp) 1097 1098 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 1099 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 1100 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 1101 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 1102 pushl %esp /* Save pt_regs as 4th parameter */ 1103 1104GLOBAL(ftrace_regs_call) 1105 call ftrace_stub 1106 1107 addl $4, %esp /* Skip pt_regs */ 1108 movl 14*4(%esp), %eax /* Move flags back into cs */ 1109 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ 1110 movl 12*4(%esp), %eax /* Get return ip from regs->ip */ 1111 movl %eax, 14*4(%esp) /* Put return ip back for ret */ 1112 1113 popl %ebx 1114 popl %ecx 1115 popl %edx 1116 popl %esi 1117 popl %edi 1118 popl %ebp 1119 popl %eax 1120 popl %ds 1121 popl %es 1122 popl %fs 1123 popl %gs 1124 addl $8, %esp /* Skip orig_ax and ip */ 1125 popf /* Pop flags at end (no addl to corrupt flags) */ 1126 jmp ftrace_ret 1127 1128 popf 1129 jmp ftrace_stub 1130#else /* ! CONFIG_DYNAMIC_FTRACE */ 1131 1132ENTRY(mcount) 1133 cmpl $__PAGE_OFFSET, %esp 1134 jb ftrace_stub /* Paging not enabled yet? */ 1135 1136 cmpl $ftrace_stub, ftrace_trace_function 1137 jnz trace 1138#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1139 cmpl $ftrace_stub, ftrace_graph_return 1140 jnz ftrace_graph_caller 1141 1142 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 1143 jnz ftrace_graph_caller 1144#endif 1145.globl ftrace_stub 1146ftrace_stub: 1147 ret 1148 1149 /* taken from glibc */ 1150trace: 1151 pushl %eax 1152 pushl %ecx 1153 pushl %edx 1154 movl 0xc(%esp), %eax 1155 movl 0x4(%ebp), %edx 1156 subl $MCOUNT_INSN_SIZE, %eax 1157 1158 call *ftrace_trace_function 1159 1160 popl %edx 1161 popl %ecx 1162 popl %eax 1163 jmp ftrace_stub 1164END(mcount) 1165#endif /* CONFIG_DYNAMIC_FTRACE */ 1166#endif /* CONFIG_FUNCTION_TRACER */ 1167 1168#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1169ENTRY(ftrace_graph_caller) 1170 pushl %eax 1171 pushl %ecx 1172 pushl %edx 1173 movl 0xc(%esp), %eax 1174 lea 0x4(%ebp), %edx 1175 movl (%ebp), %ecx 1176 subl $MCOUNT_INSN_SIZE, %eax 1177 call prepare_ftrace_return 1178 popl %edx 1179 popl %ecx 1180 popl %eax 1181 ret 1182END(ftrace_graph_caller) 1183 1184.globl return_to_handler 1185return_to_handler: 1186 pushl %eax 1187 pushl %edx 1188 movl %ebp, %eax 1189 call ftrace_return_to_handler 1190 movl %eax, %ecx 1191 popl %edx 1192 popl %eax 1193 jmp *%ecx 1194#endif 1195 1196#ifdef CONFIG_TRACING 1197ENTRY(trace_page_fault) 1198 RING0_EC_FRAME 1199 ASM_CLAC 1200 pushl_cfi $trace_do_page_fault 1201 jmp error_code 1202 CFI_ENDPROC 1203END(trace_page_fault) 1204#endif 1205 1206ENTRY(page_fault) 1207 RING0_EC_FRAME 1208 ASM_CLAC 1209 pushl_cfi $do_page_fault 1210 ALIGN 1211error_code: 1212 /* the function address is in %gs's slot on the stack */ 1213 pushl_cfi %fs 1214 /*CFI_REL_OFFSET fs, 0*/ 1215 pushl_cfi %es 1216 /*CFI_REL_OFFSET es, 0*/ 1217 pushl_cfi %ds 1218 /*CFI_REL_OFFSET ds, 0*/ 1219 pushl_cfi_reg eax 1220 pushl_cfi_reg ebp 1221 pushl_cfi_reg edi 1222 pushl_cfi_reg esi 1223 pushl_cfi_reg edx 1224 pushl_cfi_reg ecx 1225 pushl_cfi_reg ebx 1226 cld 1227 movl $(__KERNEL_PERCPU), %ecx 1228 movl %ecx, %fs 1229 UNWIND_ESPFIX_STACK 1230 GS_TO_REG %ecx 1231 movl PT_GS(%esp), %edi # get the function address 1232 movl PT_ORIG_EAX(%esp), %edx # get the error code 1233 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 1234 REG_TO_PTGS %ecx 1235 SET_KERNEL_GS %ecx 1236 movl $(__USER_DS), %ecx 1237 movl %ecx, %ds 1238 movl %ecx, %es 1239 TRACE_IRQS_OFF 1240 movl %esp,%eax # pt_regs pointer 1241 call *%edi 1242 jmp ret_from_exception 1243 CFI_ENDPROC 1244END(page_fault) 1245 1246/* 1247 * Debug traps and NMI can happen at the one SYSENTER instruction 1248 * that sets up the real kernel stack. Check here, since we can't 1249 * allow the wrong stack to be used. 1250 * 1251 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 1252 * already pushed 3 words if it hits on the sysenter instruction: 1253 * eflags, cs and eip. 1254 * 1255 * We just load the right stack, and push the three (known) values 1256 * by hand onto the new stack - while updating the return eip past 1257 * the instruction that would have done it for sysenter. 1258 */ 1259.macro FIX_STACK offset ok label 1260 cmpw $__KERNEL_CS, 4(%esp) 1261 jne \ok 1262\label: 1263 movl TSS_sysenter_sp0 + \offset(%esp), %esp 1264 CFI_DEF_CFA esp, 0 1265 CFI_UNDEFINED eip 1266 pushfl_cfi 1267 pushl_cfi $__KERNEL_CS 1268 pushl_cfi $sysenter_past_esp 1269 CFI_REL_OFFSET eip, 0 1270.endm 1271 1272ENTRY(debug) 1273 RING0_INT_FRAME 1274 ASM_CLAC 1275 cmpl $ia32_sysenter_target,(%esp) 1276 jne debug_stack_correct 1277 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 1278debug_stack_correct: 1279 pushl_cfi $-1 # mark this as an int 1280 SAVE_ALL 1281 TRACE_IRQS_OFF 1282 xorl %edx,%edx # error code 0 1283 movl %esp,%eax # pt_regs pointer 1284 call do_debug 1285 jmp ret_from_exception 1286 CFI_ENDPROC 1287END(debug) 1288 1289/* 1290 * NMI is doubly nasty. It can happen _while_ we're handling 1291 * a debug fault, and the debug fault hasn't yet been able to 1292 * clear up the stack. So we first check whether we got an 1293 * NMI on the sysenter entry path, but after that we need to 1294 * check whether we got an NMI on the debug path where the debug 1295 * fault happened on the sysenter path. 1296 */ 1297ENTRY(nmi) 1298 RING0_INT_FRAME 1299 ASM_CLAC 1300#ifdef CONFIG_X86_ESPFIX32 1301 pushl_cfi %eax 1302 movl %ss, %eax 1303 cmpw $__ESPFIX_SS, %ax 1304 popl_cfi %eax 1305 je nmi_espfix_stack 1306#endif 1307 cmpl $ia32_sysenter_target,(%esp) 1308 je nmi_stack_fixup 1309 pushl_cfi %eax 1310 movl %esp,%eax 1311 /* Do not access memory above the end of our stack page, 1312 * it might not exist. 1313 */ 1314 andl $(THREAD_SIZE-1),%eax 1315 cmpl $(THREAD_SIZE-20),%eax 1316 popl_cfi %eax 1317 jae nmi_stack_correct 1318 cmpl $ia32_sysenter_target,12(%esp) 1319 je nmi_debug_stack_check 1320nmi_stack_correct: 1321 /* We have a RING0_INT_FRAME here */ 1322 pushl_cfi %eax 1323 SAVE_ALL 1324 xorl %edx,%edx # zero error code 1325 movl %esp,%eax # pt_regs pointer 1326 call do_nmi 1327 jmp restore_all_notrace 1328 CFI_ENDPROC 1329 1330nmi_stack_fixup: 1331 RING0_INT_FRAME 1332 FIX_STACK 12, nmi_stack_correct, 1 1333 jmp nmi_stack_correct 1334 1335nmi_debug_stack_check: 1336 /* We have a RING0_INT_FRAME here */ 1337 cmpw $__KERNEL_CS,16(%esp) 1338 jne nmi_stack_correct 1339 cmpl $debug,(%esp) 1340 jb nmi_stack_correct 1341 cmpl $debug_esp_fix_insn,(%esp) 1342 ja nmi_stack_correct 1343 FIX_STACK 24, nmi_stack_correct, 1 1344 jmp nmi_stack_correct 1345 1346#ifdef CONFIG_X86_ESPFIX32 1347nmi_espfix_stack: 1348 /* We have a RING0_INT_FRAME here. 1349 * 1350 * create the pointer to lss back 1351 */ 1352 pushl_cfi %ss 1353 pushl_cfi %esp 1354 addl $4, (%esp) 1355 /* copy the iret frame of 12 bytes */ 1356 .rept 3 1357 pushl_cfi 16(%esp) 1358 .endr 1359 pushl_cfi %eax 1360 SAVE_ALL 1361 FIXUP_ESPFIX_STACK # %eax == %esp 1362 xorl %edx,%edx # zero error code 1363 call do_nmi 1364 RESTORE_REGS 1365 lss 12+4(%esp), %esp # back to espfix stack 1366 CFI_ADJUST_CFA_OFFSET -24 1367 jmp irq_return 1368#endif 1369 CFI_ENDPROC 1370END(nmi) 1371 1372ENTRY(int3) 1373 RING0_INT_FRAME 1374 ASM_CLAC 1375 pushl_cfi $-1 # mark this as an int 1376 SAVE_ALL 1377 TRACE_IRQS_OFF 1378 xorl %edx,%edx # zero error code 1379 movl %esp,%eax # pt_regs pointer 1380 call do_int3 1381 jmp ret_from_exception 1382 CFI_ENDPROC 1383END(int3) 1384 1385ENTRY(general_protection) 1386 RING0_EC_FRAME 1387 pushl_cfi $do_general_protection 1388 jmp error_code 1389 CFI_ENDPROC 1390END(general_protection) 1391 1392#ifdef CONFIG_KVM_GUEST 1393ENTRY(async_page_fault) 1394 RING0_EC_FRAME 1395 ASM_CLAC 1396 pushl_cfi $do_async_page_fault 1397 jmp error_code 1398 CFI_ENDPROC 1399END(async_page_fault) 1400#endif 1401 1402