root/arch/x86/entry/entry_32.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  *  Copyright (C) 1991,1992  Linus Torvalds
   4  *
   5  * entry_32.S contains the system-call and low-level fault and trap handling routines.
   6  *
   7  * Stack layout while running C code:
   8  *      ptrace needs to have all registers on the stack.
   9  *      If the order here is changed, it needs to be
  10  *      updated in fork.c:copy_process(), signal.c:do_signal(),
  11  *      ptrace.c and ptrace.h
  12  *
  13  *       0(%esp) - %ebx
  14  *       4(%esp) - %ecx
  15  *       8(%esp) - %edx
  16  *       C(%esp) - %esi
  17  *      10(%esp) - %edi
  18  *      14(%esp) - %ebp
  19  *      18(%esp) - %eax
  20  *      1C(%esp) - %ds
  21  *      20(%esp) - %es
  22  *      24(%esp) - %fs
  23  *      28(%esp) - %gs          saved iff !CONFIG_X86_32_LAZY_GS
  24  *      2C(%esp) - orig_eax
  25  *      30(%esp) - %eip
  26  *      34(%esp) - %cs
  27  *      38(%esp) - %eflags
  28  *      3C(%esp) - %oldesp
  29  *      40(%esp) - %oldss
  30  */
  31 
  32 #include <linux/linkage.h>
  33 #include <linux/err.h>
  34 #include <asm/thread_info.h>
  35 #include <asm/irqflags.h>
  36 #include <asm/errno.h>
  37 #include <asm/segment.h>
  38 #include <asm/smp.h>
  39 #include <asm/percpu.h>
  40 #include <asm/processor-flags.h>
  41 #include <asm/irq_vectors.h>
  42 #include <asm/cpufeatures.h>
  43 #include <asm/alternative-asm.h>
  44 #include <asm/asm.h>
  45 #include <asm/smap.h>
  46 #include <asm/frame.h>
  47 #include <asm/nospec-branch.h>
  48 
  49 #include "calling.h"
  50 
  51         .section .entry.text, "ax"
  52 
  53 /*
  54  * We use macros for low-level operations which need to be overridden
  55  * for paravirtualization.  The following will never clobber any registers:
  56  *   INTERRUPT_RETURN (aka. "iret")
  57  *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
  58  *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
  59  *
  60  * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
  61  * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
  62  * Allowing a register to be clobbered can shrink the paravirt replacement
  63  * enough to patch inline, increasing performance.
  64  */
  65 
  66 #ifdef CONFIG_PREEMPTION
  67 # define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
  68 #else
  69 # define preempt_stop(clobbers)
  70 #endif
  71 
  72 .macro TRACE_IRQS_IRET
  73 #ifdef CONFIG_TRACE_IRQFLAGS
  74         testl   $X86_EFLAGS_IF, PT_EFLAGS(%esp)     # interrupts off?
  75         jz      1f
  76         TRACE_IRQS_ON
  77 1:
  78 #endif
  79 .endm
  80 
  81 #define PTI_SWITCH_MASK         (1 << PAGE_SHIFT)
  82 
  83 /*
  84  * User gs save/restore
  85  *
  86  * %gs is used for userland TLS and kernel only uses it for stack
  87  * canary which is required to be at %gs:20 by gcc.  Read the comment
  88  * at the top of stackprotector.h for more info.
  89  *
  90  * Local labels 98 and 99 are used.
  91  */
  92 #ifdef CONFIG_X86_32_LAZY_GS
  93 
  94  /* unfortunately push/pop can't be no-op */
  95 .macro PUSH_GS
  96         pushl   $0
  97 .endm
  98 .macro POP_GS pop=0
  99         addl    $(4 + \pop), %esp
 100 .endm
 101 .macro POP_GS_EX
 102 .endm
 103 
 104  /* all the rest are no-op */
 105 .macro PTGS_TO_GS
 106 .endm
 107 .macro PTGS_TO_GS_EX
 108 .endm
 109 .macro GS_TO_REG reg
 110 .endm
 111 .macro REG_TO_PTGS reg
 112 .endm
 113 .macro SET_KERNEL_GS reg
 114 .endm
 115 
 116 #else   /* CONFIG_X86_32_LAZY_GS */
 117 
 118 .macro PUSH_GS
 119         pushl   %gs
 120 .endm
 121 
 122 .macro POP_GS pop=0
 123 98:     popl    %gs
 124   .if \pop <> 0
 125         add     $\pop, %esp
 126   .endif
 127 .endm
 128 .macro POP_GS_EX
 129 .pushsection .fixup, "ax"
 130 99:     movl    $0, (%esp)
 131         jmp     98b
 132 .popsection
 133         _ASM_EXTABLE(98b, 99b)
 134 .endm
 135 
 136 .macro PTGS_TO_GS
 137 98:     mov     PT_GS(%esp), %gs
 138 .endm
 139 .macro PTGS_TO_GS_EX
 140 .pushsection .fixup, "ax"
 141 99:     movl    $0, PT_GS(%esp)
 142         jmp     98b
 143 .popsection
 144         _ASM_EXTABLE(98b, 99b)
 145 .endm
 146 
 147 .macro GS_TO_REG reg
 148         movl    %gs, \reg
 149 .endm
 150 .macro REG_TO_PTGS reg
 151         movl    \reg, PT_GS(%esp)
 152 .endm
 153 .macro SET_KERNEL_GS reg
 154         movl    $(__KERNEL_STACK_CANARY), \reg
 155         movl    \reg, %gs
 156 .endm
 157 
 158 #endif /* CONFIG_X86_32_LAZY_GS */
 159 
 160 /* Unconditionally switch to user cr3 */
 161 .macro SWITCH_TO_USER_CR3 scratch_reg:req
 162         ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
 163 
 164         movl    %cr3, \scratch_reg
 165         orl     $PTI_SWITCH_MASK, \scratch_reg
 166         movl    \scratch_reg, %cr3
 167 .Lend_\@:
 168 .endm
 169 
 170 .macro BUG_IF_WRONG_CR3 no_user_check=0
 171 #ifdef CONFIG_DEBUG_ENTRY
 172         ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
 173         .if \no_user_check == 0
 174         /* coming from usermode? */
 175         testl   $USER_SEGMENT_RPL_MASK, PT_CS(%esp)
 176         jz      .Lend_\@
 177         .endif
 178         /* On user-cr3? */
 179         movl    %cr3, %eax
 180         testl   $PTI_SWITCH_MASK, %eax
 181         jnz     .Lend_\@
 182         /* From userspace with kernel cr3 - BUG */
 183         ud2
 184 .Lend_\@:
 185 #endif
 186 .endm
 187 
 188 /*
 189  * Switch to kernel cr3 if not already loaded and return current cr3 in
 190  * \scratch_reg
 191  */
 192 .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
 193         ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
 194         movl    %cr3, \scratch_reg
 195         /* Test if we are already on kernel CR3 */
 196         testl   $PTI_SWITCH_MASK, \scratch_reg
 197         jz      .Lend_\@
 198         andl    $(~PTI_SWITCH_MASK), \scratch_reg
 199         movl    \scratch_reg, %cr3
 200         /* Return original CR3 in \scratch_reg */
 201         orl     $PTI_SWITCH_MASK, \scratch_reg
 202 .Lend_\@:
 203 .endm
 204 
 205 #define CS_FROM_ENTRY_STACK     (1 << 31)
 206 #define CS_FROM_USER_CR3        (1 << 30)
 207 #define CS_FROM_KERNEL          (1 << 29)
 208 #define CS_FROM_ESPFIX          (1 << 28)
 209 
 210 .macro FIXUP_FRAME
 211         /*
 212          * The high bits of the CS dword (__csh) are used for CS_FROM_*.
 213          * Clear them in case hardware didn't do this for us.
 214          */
 215         andl    $0x0000ffff, 4*4(%esp)
 216 
 217 #ifdef CONFIG_VM86
 218         testl   $X86_EFLAGS_VM, 5*4(%esp)
 219         jnz     .Lfrom_usermode_no_fixup_\@
 220 #endif
 221         testl   $USER_SEGMENT_RPL_MASK, 4*4(%esp)
 222         jnz     .Lfrom_usermode_no_fixup_\@
 223 
 224         orl     $CS_FROM_KERNEL, 4*4(%esp)
 225 
 226         /*
 227          * When we're here from kernel mode; the (exception) stack looks like:
 228          *
 229          *  6*4(%esp) - <previous context>
 230          *  5*4(%esp) - flags
 231          *  4*4(%esp) - cs
 232          *  3*4(%esp) - ip
 233          *  2*4(%esp) - orig_eax
 234          *  1*4(%esp) - gs / function
 235          *  0*4(%esp) - fs
 236          *
 237          * Lets build a 5 entry IRET frame after that, such that struct pt_regs
 238          * is complete and in particular regs->sp is correct. This gives us
 239          * the original 6 enties as gap:
 240          *
 241          * 14*4(%esp) - <previous context>
 242          * 13*4(%esp) - gap / flags
 243          * 12*4(%esp) - gap / cs
 244          * 11*4(%esp) - gap / ip
 245          * 10*4(%esp) - gap / orig_eax
 246          *  9*4(%esp) - gap / gs / function
 247          *  8*4(%esp) - gap / fs
 248          *  7*4(%esp) - ss
 249          *  6*4(%esp) - sp
 250          *  5*4(%esp) - flags
 251          *  4*4(%esp) - cs
 252          *  3*4(%esp) - ip
 253          *  2*4(%esp) - orig_eax
 254          *  1*4(%esp) - gs / function
 255          *  0*4(%esp) - fs
 256          */
 257 
 258         pushl   %ss             # ss
 259         pushl   %esp            # sp (points at ss)
 260         addl    $7*4, (%esp)    # point sp back at the previous context
 261         pushl   7*4(%esp)       # flags
 262         pushl   7*4(%esp)       # cs
 263         pushl   7*4(%esp)       # ip
 264         pushl   7*4(%esp)       # orig_eax
 265         pushl   7*4(%esp)       # gs / function
 266         pushl   7*4(%esp)       # fs
 267 .Lfrom_usermode_no_fixup_\@:
 268 .endm
 269 
 270 .macro IRET_FRAME
 271         /*
 272          * We're called with %ds, %es, %fs, and %gs from the interrupted
 273          * frame, so we shouldn't use them.  Also, we may be in ESPFIX
 274          * mode and therefore have a nonzero SS base and an offset ESP,
 275          * so any attempt to access the stack needs to use SS.  (except for
 276          * accesses through %esp, which automatically use SS.)
 277          */
 278         testl $CS_FROM_KERNEL, 1*4(%esp)
 279         jz .Lfinished_frame_\@
 280 
 281         /*
 282          * Reconstruct the 3 entry IRET frame right after the (modified)
 283          * regs->sp without lowering %esp in between, such that an NMI in the
 284          * middle doesn't scribble our stack.
 285          */
 286         pushl   %eax
 287         pushl   %ecx
 288         movl    5*4(%esp), %eax         # (modified) regs->sp
 289 
 290         movl    4*4(%esp), %ecx         # flags
 291         movl    %ecx, %ss:-1*4(%eax)
 292 
 293         movl    3*4(%esp), %ecx         # cs
 294         andl    $0x0000ffff, %ecx
 295         movl    %ecx, %ss:-2*4(%eax)
 296 
 297         movl    2*4(%esp), %ecx         # ip
 298         movl    %ecx, %ss:-3*4(%eax)
 299 
 300         movl    1*4(%esp), %ecx         # eax
 301         movl    %ecx, %ss:-4*4(%eax)
 302 
 303         popl    %ecx
 304         lea     -4*4(%eax), %esp
 305         popl    %eax
 306 .Lfinished_frame_\@:
 307 .endm
 308 
 309 .macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 skip_gs=0 unwind_espfix=0
 310         cld
 311 .if \skip_gs == 0
 312         PUSH_GS
 313 .endif
 314         pushl   %fs
 315 
 316         pushl   %eax
 317         movl    $(__KERNEL_PERCPU), %eax
 318         movl    %eax, %fs
 319 .if \unwind_espfix > 0
 320         UNWIND_ESPFIX_STACK
 321 .endif
 322         popl    %eax
 323 
 324         FIXUP_FRAME
 325         pushl   %es
 326         pushl   %ds
 327         pushl   \pt_regs_ax
 328         pushl   %ebp
 329         pushl   %edi
 330         pushl   %esi
 331         pushl   %edx
 332         pushl   %ecx
 333         pushl   %ebx
 334         movl    $(__USER_DS), %edx
 335         movl    %edx, %ds
 336         movl    %edx, %es
 337 .if \skip_gs == 0
 338         SET_KERNEL_GS %edx
 339 .endif
 340         /* Switch to kernel stack if necessary */
 341 .if \switch_stacks > 0
 342         SWITCH_TO_KERNEL_STACK
 343 .endif
 344 .endm
 345 
 346 .macro SAVE_ALL_NMI cr3_reg:req unwind_espfix=0
 347         SAVE_ALL unwind_espfix=\unwind_espfix
 348 
 349         BUG_IF_WRONG_CR3
 350 
 351         /*
 352          * Now switch the CR3 when PTI is enabled.
 353          *
 354          * We can enter with either user or kernel cr3, the code will
 355          * store the old cr3 in \cr3_reg and switches to the kernel cr3
 356          * if necessary.
 357          */
 358         SWITCH_TO_KERNEL_CR3 scratch_reg=\cr3_reg
 359 
 360 .Lend_\@:
 361 .endm
 362 
 363 .macro RESTORE_INT_REGS
 364         popl    %ebx
 365         popl    %ecx
 366         popl    %edx
 367         popl    %esi
 368         popl    %edi
 369         popl    %ebp
 370         popl    %eax
 371 .endm
 372 
 373 .macro RESTORE_REGS pop=0
 374         RESTORE_INT_REGS
 375 1:      popl    %ds
 376 2:      popl    %es
 377 3:      popl    %fs
 378         POP_GS \pop
 379         IRET_FRAME
 380 .pushsection .fixup, "ax"
 381 4:      movl    $0, (%esp)
 382         jmp     1b
 383 5:      movl    $0, (%esp)
 384         jmp     2b
 385 6:      movl    $0, (%esp)
 386         jmp     3b
 387 .popsection
 388         _ASM_EXTABLE(1b, 4b)
 389         _ASM_EXTABLE(2b, 5b)
 390         _ASM_EXTABLE(3b, 6b)
 391         POP_GS_EX
 392 .endm
 393 
 394 .macro RESTORE_ALL_NMI cr3_reg:req pop=0
 395         /*
 396          * Now switch the CR3 when PTI is enabled.
 397          *
 398          * We enter with kernel cr3 and switch the cr3 to the value
 399          * stored on \cr3_reg, which is either a user or a kernel cr3.
 400          */
 401         ALTERNATIVE "jmp .Lswitched_\@", "", X86_FEATURE_PTI
 402 
 403         testl   $PTI_SWITCH_MASK, \cr3_reg
 404         jz      .Lswitched_\@
 405 
 406         /* User cr3 in \cr3_reg - write it to hardware cr3 */
 407         movl    \cr3_reg, %cr3
 408 
 409 .Lswitched_\@:
 410 
 411         BUG_IF_WRONG_CR3
 412 
 413         RESTORE_REGS pop=\pop
 414 .endm
 415 
 416 .macro CHECK_AND_APPLY_ESPFIX
 417 #ifdef CONFIG_X86_ESPFIX32
 418 #define GDT_ESPFIX_OFFSET (GDT_ENTRY_ESPFIX_SS * 8)
 419 #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + GDT_ESPFIX_OFFSET
 420 
 421         ALTERNATIVE     "jmp .Lend_\@", "", X86_BUG_ESPFIX
 422 
 423         movl    PT_EFLAGS(%esp), %eax           # mix EFLAGS, SS and CS
 424         /*
 425          * Warning: PT_OLDSS(%esp) contains the wrong/random values if we
 426          * are returning to the kernel.
 427          * See comments in process.c:copy_thread() for details.
 428          */
 429         movb    PT_OLDSS(%esp), %ah
 430         movb    PT_CS(%esp), %al
 431         andl    $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
 432         cmpl    $((SEGMENT_LDT << 8) | USER_RPL), %eax
 433         jne     .Lend_\@        # returning to user-space with LDT SS
 434 
 435         /*
 436          * Setup and switch to ESPFIX stack
 437          *
 438          * We're returning to userspace with a 16 bit stack. The CPU will not
 439          * restore the high word of ESP for us on executing iret... This is an
 440          * "official" bug of all the x86-compatible CPUs, which we can work
 441          * around to make dosemu and wine happy. We do this by preloading the
 442          * high word of ESP with the high word of the userspace ESP while
 443          * compensating for the offset by changing to the ESPFIX segment with
 444          * a base address that matches for the difference.
 445          */
 446         mov     %esp, %edx                      /* load kernel esp */
 447         mov     PT_OLDESP(%esp), %eax           /* load userspace esp */
 448         mov     %dx, %ax                        /* eax: new kernel esp */
 449         sub     %eax, %edx                      /* offset (low word is 0) */
 450         shr     $16, %edx
 451         mov     %dl, GDT_ESPFIX_SS + 4          /* bits 16..23 */
 452         mov     %dh, GDT_ESPFIX_SS + 7          /* bits 24..31 */
 453         pushl   $__ESPFIX_SS
 454         pushl   %eax                            /* new kernel esp */
 455         /*
 456          * Disable interrupts, but do not irqtrace this section: we
 457          * will soon execute iret and the tracer was already set to
 458          * the irqstate after the IRET:
 459          */
 460         DISABLE_INTERRUPTS(CLBR_ANY)
 461         lss     (%esp), %esp                    /* switch to espfix segment */
 462 .Lend_\@:
 463 #endif /* CONFIG_X86_ESPFIX32 */
 464 .endm
 465 
 466 /*
 467  * Called with pt_regs fully populated and kernel segments loaded,
 468  * so we can access PER_CPU and use the integer registers.
 469  *
 470  * We need to be very careful here with the %esp switch, because an NMI
 471  * can happen everywhere. If the NMI handler finds itself on the
 472  * entry-stack, it will overwrite the task-stack and everything we
 473  * copied there. So allocate the stack-frame on the task-stack and
 474  * switch to it before we do any copying.
 475  */
 476 
 477 .macro SWITCH_TO_KERNEL_STACK
 478 
 479         ALTERNATIVE     "", "jmp .Lend_\@", X86_FEATURE_XENPV
 480 
 481         BUG_IF_WRONG_CR3
 482 
 483         SWITCH_TO_KERNEL_CR3 scratch_reg=%eax
 484 
 485         /*
 486          * %eax now contains the entry cr3 and we carry it forward in
 487          * that register for the time this macro runs
 488          */
 489 
 490         /* Are we on the entry stack? Bail out if not! */
 491         movl    PER_CPU_VAR(cpu_entry_area), %ecx
 492         addl    $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
 493         subl    %esp, %ecx      /* ecx = (end of entry_stack) - esp */
 494         cmpl    $SIZEOF_entry_stack, %ecx
 495         jae     .Lend_\@
 496 
 497         /* Load stack pointer into %esi and %edi */
 498         movl    %esp, %esi
 499         movl    %esi, %edi
 500 
 501         /* Move %edi to the top of the entry stack */
 502         andl    $(MASK_entry_stack), %edi
 503         addl    $(SIZEOF_entry_stack), %edi
 504 
 505         /* Load top of task-stack into %edi */
 506         movl    TSS_entry2task_stack(%edi), %edi
 507 
 508         /* Special case - entry from kernel mode via entry stack */
 509 #ifdef CONFIG_VM86
 510         movl    PT_EFLAGS(%esp), %ecx           # mix EFLAGS and CS
 511         movb    PT_CS(%esp), %cl
 512         andl    $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %ecx
 513 #else
 514         movl    PT_CS(%esp), %ecx
 515         andl    $SEGMENT_RPL_MASK, %ecx
 516 #endif
 517         cmpl    $USER_RPL, %ecx
 518         jb      .Lentry_from_kernel_\@
 519 
 520         /* Bytes to copy */
 521         movl    $PTREGS_SIZE, %ecx
 522 
 523 #ifdef CONFIG_VM86
 524         testl   $X86_EFLAGS_VM, PT_EFLAGS(%esi)
 525         jz      .Lcopy_pt_regs_\@
 526 
 527         /*
 528          * Stack-frame contains 4 additional segment registers when
 529          * coming from VM86 mode
 530          */
 531         addl    $(4 * 4), %ecx
 532 
 533 #endif
 534 .Lcopy_pt_regs_\@:
 535 
 536         /* Allocate frame on task-stack */
 537         subl    %ecx, %edi
 538 
 539         /* Switch to task-stack */
 540         movl    %edi, %esp
 541 
 542         /*
 543          * We are now on the task-stack and can safely copy over the
 544          * stack-frame
 545          */
 546         shrl    $2, %ecx
 547         cld
 548         rep movsl
 549 
 550         jmp .Lend_\@
 551 
 552 .Lentry_from_kernel_\@:
 553 
 554         /*
 555          * This handles the case when we enter the kernel from
 556          * kernel-mode and %esp points to the entry-stack. When this
 557          * happens we need to switch to the task-stack to run C code,
 558          * but switch back to the entry-stack again when we approach
 559          * iret and return to the interrupted code-path. This usually
 560          * happens when we hit an exception while restoring user-space
 561          * segment registers on the way back to user-space or when the
 562          * sysenter handler runs with eflags.tf set.
 563          *
 564          * When we switch to the task-stack here, we can't trust the
 565          * contents of the entry-stack anymore, as the exception handler
 566          * might be scheduled out or moved to another CPU. Therefore we
 567          * copy the complete entry-stack to the task-stack and set a
 568          * marker in the iret-frame (bit 31 of the CS dword) to detect
 569          * what we've done on the iret path.
 570          *
 571          * On the iret path we copy everything back and switch to the
 572          * entry-stack, so that the interrupted kernel code-path
 573          * continues on the same stack it was interrupted with.
 574          *
 575          * Be aware that an NMI can happen anytime in this code.
 576          *
 577          * %esi: Entry-Stack pointer (same as %esp)
 578          * %edi: Top of the task stack
 579          * %eax: CR3 on kernel entry
 580          */
 581 
 582         /* Calculate number of bytes on the entry stack in %ecx */
 583         movl    %esi, %ecx
 584 
 585         /* %ecx to the top of entry-stack */
 586         andl    $(MASK_entry_stack), %ecx
 587         addl    $(SIZEOF_entry_stack), %ecx
 588 
 589         /* Number of bytes on the entry stack to %ecx */
 590         sub     %esi, %ecx
 591 
 592         /* Mark stackframe as coming from entry stack */
 593         orl     $CS_FROM_ENTRY_STACK, PT_CS(%esp)
 594 
 595         /*
 596          * Test the cr3 used to enter the kernel and add a marker
 597          * so that we can switch back to it before iret.
 598          */
 599         testl   $PTI_SWITCH_MASK, %eax
 600         jz      .Lcopy_pt_regs_\@
 601         orl     $CS_FROM_USER_CR3, PT_CS(%esp)
 602 
 603         /*
 604          * %esi and %edi are unchanged, %ecx contains the number of
 605          * bytes to copy. The code at .Lcopy_pt_regs_\@ will allocate
 606          * the stack-frame on task-stack and copy everything over
 607          */
 608         jmp .Lcopy_pt_regs_\@
 609 
 610 .Lend_\@:
 611 .endm
 612 
 613 /*
 614  * Switch back from the kernel stack to the entry stack.
 615  *
 616  * The %esp register must point to pt_regs on the task stack. It will
 617  * first calculate the size of the stack-frame to copy, depending on
 618  * whether we return to VM86 mode or not. With that it uses 'rep movsl'
 619  * to copy the contents of the stack over to the entry stack.
 620  *
 621  * We must be very careful here, as we can't trust the contents of the
 622  * task-stack once we switched to the entry-stack. When an NMI happens
 623  * while on the entry-stack, the NMI handler will switch back to the top
 624  * of the task stack, overwriting our stack-frame we are about to copy.
 625  * Therefore we switch the stack only after everything is copied over.
 626  */
 627 .macro SWITCH_TO_ENTRY_STACK
 628 
 629         ALTERNATIVE     "", "jmp .Lend_\@", X86_FEATURE_XENPV
 630 
 631         /* Bytes to copy */
 632         movl    $PTREGS_SIZE, %ecx
 633 
 634 #ifdef CONFIG_VM86
 635         testl   $(X86_EFLAGS_VM), PT_EFLAGS(%esp)
 636         jz      .Lcopy_pt_regs_\@
 637 
 638         /* Additional 4 registers to copy when returning to VM86 mode */
 639         addl    $(4 * 4), %ecx
 640 
 641 .Lcopy_pt_regs_\@:
 642 #endif
 643 
 644         /* Initialize source and destination for movsl */
 645         movl    PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi
 646         subl    %ecx, %edi
 647         movl    %esp, %esi
 648 
 649         /* Save future stack pointer in %ebx */
 650         movl    %edi, %ebx
 651 
 652         /* Copy over the stack-frame */
 653         shrl    $2, %ecx
 654         cld
 655         rep movsl
 656 
 657         /*
 658          * Switch to entry-stack - needs to happen after everything is
 659          * copied because the NMI handler will overwrite the task-stack
 660          * when on entry-stack
 661          */
 662         movl    %ebx, %esp
 663 
 664 .Lend_\@:
 665 .endm
 666 
 667 /*
 668  * This macro handles the case when we return to kernel-mode on the iret
 669  * path and have to switch back to the entry stack and/or user-cr3
 670  *
 671  * See the comments below the .Lentry_from_kernel_\@ label in the
 672  * SWITCH_TO_KERNEL_STACK macro for more details.
 673  */
 674 .macro PARANOID_EXIT_TO_KERNEL_MODE
 675 
 676         /*
 677          * Test if we entered the kernel with the entry-stack. Most
 678          * likely we did not, because this code only runs on the
 679          * return-to-kernel path.
 680          */
 681         testl   $CS_FROM_ENTRY_STACK, PT_CS(%esp)
 682         jz      .Lend_\@
 683 
 684         /* Unlikely slow-path */
 685 
 686         /* Clear marker from stack-frame */
 687         andl    $(~CS_FROM_ENTRY_STACK), PT_CS(%esp)
 688 
 689         /* Copy the remaining task-stack contents to entry-stack */
 690         movl    %esp, %esi
 691         movl    PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi
 692 
 693         /* Bytes on the task-stack to ecx */
 694         movl    PER_CPU_VAR(cpu_tss_rw + TSS_sp1), %ecx
 695         subl    %esi, %ecx
 696 
 697         /* Allocate stack-frame on entry-stack */
 698         subl    %ecx, %edi
 699 
 700         /*
 701          * Save future stack-pointer, we must not switch until the
 702          * copy is done, otherwise the NMI handler could destroy the
 703          * contents of the task-stack we are about to copy.
 704          */
 705         movl    %edi, %ebx
 706 
 707         /* Do the copy */
 708         shrl    $2, %ecx
 709         cld
 710         rep movsl
 711 
 712         /* Safe to switch to entry-stack now */
 713         movl    %ebx, %esp
 714 
 715         /*
 716          * We came from entry-stack and need to check if we also need to
 717          * switch back to user cr3.
 718          */
 719         testl   $CS_FROM_USER_CR3, PT_CS(%esp)
 720         jz      .Lend_\@
 721 
 722         /* Clear marker from stack-frame */
 723         andl    $(~CS_FROM_USER_CR3), PT_CS(%esp)
 724 
 725         SWITCH_TO_USER_CR3 scratch_reg=%eax
 726 
 727 .Lend_\@:
 728 .endm
 729 /*
 730  * %eax: prev task
 731  * %edx: next task
 732  */
 733 ENTRY(__switch_to_asm)
 734         /*
 735          * Save callee-saved registers
 736          * This must match the order in struct inactive_task_frame
 737          */
 738         pushl   %ebp
 739         pushl   %ebx
 740         pushl   %edi
 741         pushl   %esi
 742         pushfl
 743 
 744         /* switch stack */
 745         movl    %esp, TASK_threadsp(%eax)
 746         movl    TASK_threadsp(%edx), %esp
 747 
 748 #ifdef CONFIG_STACKPROTECTOR
 749         movl    TASK_stack_canary(%edx), %ebx
 750         movl    %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
 751 #endif
 752 
 753 #ifdef CONFIG_RETPOLINE
 754         /*
 755          * When switching from a shallower to a deeper call stack
 756          * the RSB may either underflow or use entries populated
 757          * with userspace addresses. On CPUs where those concerns
 758          * exist, overwrite the RSB with entries which capture
 759          * speculative execution to prevent attack.
 760          */
 761         FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
 762 #endif
 763 
 764         /* restore callee-saved registers */
 765         popfl
 766         popl    %esi
 767         popl    %edi
 768         popl    %ebx
 769         popl    %ebp
 770 
 771         jmp     __switch_to
 772 END(__switch_to_asm)
 773 
 774 /*
 775  * The unwinder expects the last frame on the stack to always be at the same
 776  * offset from the end of the page, which allows it to validate the stack.
 777  * Calling schedule_tail() directly would break that convention because its an
 778  * asmlinkage function so its argument has to be pushed on the stack.  This
 779  * wrapper creates a proper "end of stack" frame header before the call.
 780  */
 781 ENTRY(schedule_tail_wrapper)
 782         FRAME_BEGIN
 783 
 784         pushl   %eax
 785         call    schedule_tail
 786         popl    %eax
 787 
 788         FRAME_END
 789         ret
 790 ENDPROC(schedule_tail_wrapper)
 791 /*
 792  * A newly forked process directly context switches into this address.
 793  *
 794  * eax: prev task we switched from
 795  * ebx: kernel thread func (NULL for user thread)
 796  * edi: kernel thread arg
 797  */
 798 ENTRY(ret_from_fork)
 799         call    schedule_tail_wrapper
 800 
 801         testl   %ebx, %ebx
 802         jnz     1f              /* kernel threads are uncommon */
 803 
 804 2:
 805         /* When we fork, we trace the syscall return in the child, too. */
 806         movl    %esp, %eax
 807         call    syscall_return_slowpath
 808         STACKLEAK_ERASE
 809         jmp     restore_all
 810 
 811         /* kernel thread */
 812 1:      movl    %edi, %eax
 813         CALL_NOSPEC %ebx
 814         /*
 815          * A kernel thread is allowed to return here after successfully
 816          * calling do_execve().  Exit to userspace to complete the execve()
 817          * syscall.
 818          */
 819         movl    $0, PT_EAX(%esp)
 820         jmp     2b
 821 END(ret_from_fork)
 822 
 823 /*
 824  * Return to user mode is not as complex as all this looks,
 825  * but we want the default path for a system call return to
 826  * go as quickly as possible which is why some of this is
 827  * less clear than it otherwise should be.
 828  */
 829 
 830         # userspace resumption stub bypassing syscall exit tracing
 831         ALIGN
 832 ret_from_exception:
 833         preempt_stop(CLBR_ANY)
 834 ret_from_intr:
 835 #ifdef CONFIG_VM86
 836         movl    PT_EFLAGS(%esp), %eax           # mix EFLAGS and CS
 837         movb    PT_CS(%esp), %al
 838         andl    $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
 839 #else
 840         /*
 841          * We can be coming here from child spawned by kernel_thread().
 842          */
 843         movl    PT_CS(%esp), %eax
 844         andl    $SEGMENT_RPL_MASK, %eax
 845 #endif
 846         cmpl    $USER_RPL, %eax
 847         jb      restore_all_kernel              # not returning to v8086 or userspace
 848 
 849 ENTRY(resume_userspace)
 850         DISABLE_INTERRUPTS(CLBR_ANY)
 851         TRACE_IRQS_OFF
 852         movl    %esp, %eax
 853         call    prepare_exit_to_usermode
 854         jmp     restore_all
 855 END(ret_from_exception)
 856 
 857 GLOBAL(__begin_SYSENTER_singlestep_region)
 858 /*
 859  * All code from here through __end_SYSENTER_singlestep_region is subject
 860  * to being single-stepped if a user program sets TF and executes SYSENTER.
 861  * There is absolutely nothing that we can do to prevent this from happening
 862  * (thanks Intel!).  To keep our handling of this situation as simple as
 863  * possible, we handle TF just like AC and NT, except that our #DB handler
 864  * will ignore all of the single-step traps generated in this range.
 865  */
 866 
 867 #ifdef CONFIG_XEN_PV
 868 /*
 869  * Xen doesn't set %esp to be precisely what the normal SYSENTER
 870  * entry point expects, so fix it up before using the normal path.
 871  */
 872 ENTRY(xen_sysenter_target)
 873         addl    $5*4, %esp                      /* remove xen-provided frame */
 874         jmp     .Lsysenter_past_esp
 875 #endif
 876 
 877 /*
 878  * 32-bit SYSENTER entry.
 879  *
 880  * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
 881  * if X86_FEATURE_SEP is available.  This is the preferred system call
 882  * entry on 32-bit systems.
 883  *
 884  * The SYSENTER instruction, in principle, should *only* occur in the
 885  * vDSO.  In practice, a small number of Android devices were shipped
 886  * with a copy of Bionic that inlined a SYSENTER instruction.  This
 887  * never happened in any of Google's Bionic versions -- it only happened
 888  * in a narrow range of Intel-provided versions.
 889  *
 890  * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs.
 891  * IF and VM in RFLAGS are cleared (IOW: interrupts are off).
 892  * SYSENTER does not save anything on the stack,
 893  * and does not save old EIP (!!!), ESP, or EFLAGS.
 894  *
 895  * To avoid losing track of EFLAGS.VM (and thus potentially corrupting
 896  * user and/or vm86 state), we explicitly disable the SYSENTER
 897  * instruction in vm86 mode by reprogramming the MSRs.
 898  *
 899  * Arguments:
 900  * eax  system call number
 901  * ebx  arg1
 902  * ecx  arg2
 903  * edx  arg3
 904  * esi  arg4
 905  * edi  arg5
 906  * ebp  user stack
 907  * 0(%ebp) arg6
 908  */
 909 ENTRY(entry_SYSENTER_32)
 910         /*
 911          * On entry-stack with all userspace-regs live - save and
 912          * restore eflags and %eax to use it as scratch-reg for the cr3
 913          * switch.
 914          */
 915         pushfl
 916         pushl   %eax
 917         BUG_IF_WRONG_CR3 no_user_check=1
 918         SWITCH_TO_KERNEL_CR3 scratch_reg=%eax
 919         popl    %eax
 920         popfl
 921 
 922         /* Stack empty again, switch to task stack */
 923         movl    TSS_entry2task_stack(%esp), %esp
 924 
 925 .Lsysenter_past_esp:
 926         pushl   $__USER_DS              /* pt_regs->ss */
 927         pushl   %ebp                    /* pt_regs->sp (stashed in bp) */
 928         pushfl                          /* pt_regs->flags (except IF = 0) */
 929         orl     $X86_EFLAGS_IF, (%esp)  /* Fix IF */
 930         pushl   $__USER_CS              /* pt_regs->cs */
 931         pushl   $0                      /* pt_regs->ip = 0 (placeholder) */
 932         pushl   %eax                    /* pt_regs->orig_ax */
 933         SAVE_ALL pt_regs_ax=$-ENOSYS    /* save rest, stack already switched */
 934 
 935         /*
 936          * SYSENTER doesn't filter flags, so we need to clear NT, AC
 937          * and TF ourselves.  To save a few cycles, we can check whether
 938          * either was set instead of doing an unconditional popfq.
 939          * This needs to happen before enabling interrupts so that
 940          * we don't get preempted with NT set.
 941          *
 942          * If TF is set, we will single-step all the way to here -- do_debug
 943          * will ignore all the traps.  (Yes, this is slow, but so is
 944          * single-stepping in general.  This allows us to avoid having
 945          * a more complicated code to handle the case where a user program
 946          * forces us to single-step through the SYSENTER entry code.)
 947          *
 948          * NB.: .Lsysenter_fix_flags is a label with the code under it moved
 949          * out-of-line as an optimization: NT is unlikely to be set in the
 950          * majority of the cases and instead of polluting the I$ unnecessarily,
 951          * we're keeping that code behind a branch which will predict as
 952          * not-taken and therefore its instructions won't be fetched.
 953          */
 954         testl   $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp)
 955         jnz     .Lsysenter_fix_flags
 956 .Lsysenter_flags_fixed:
 957 
 958         /*
 959          * User mode is traced as though IRQs are on, and SYSENTER
 960          * turned them off.
 961          */
 962         TRACE_IRQS_OFF
 963 
 964         movl    %esp, %eax
 965         call    do_fast_syscall_32
 966         /* XEN PV guests always use IRET path */
 967         ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \
 968                     "jmp .Lsyscall_32_done", X86_FEATURE_XENPV
 969 
 970         STACKLEAK_ERASE
 971 
 972 /* Opportunistic SYSEXIT */
 973         TRACE_IRQS_ON                   /* User mode traces as IRQs on. */
 974 
 975         /*
 976          * Setup entry stack - we keep the pointer in %eax and do the
 977          * switch after almost all user-state is restored.
 978          */
 979 
 980         /* Load entry stack pointer and allocate frame for eflags/eax */
 981         movl    PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %eax
 982         subl    $(2*4), %eax
 983 
 984         /* Copy eflags and eax to entry stack */
 985         movl    PT_EFLAGS(%esp), %edi
 986         movl    PT_EAX(%esp), %esi
 987         movl    %edi, (%eax)
 988         movl    %esi, 4(%eax)
 989 
 990         /* Restore user registers and segments */
 991         movl    PT_EIP(%esp), %edx      /* pt_regs->ip */
 992         movl    PT_OLDESP(%esp), %ecx   /* pt_regs->sp */
 993 1:      mov     PT_FS(%esp), %fs
 994         PTGS_TO_GS
 995 
 996         popl    %ebx                    /* pt_regs->bx */
 997         addl    $2*4, %esp              /* skip pt_regs->cx and pt_regs->dx */
 998         popl    %esi                    /* pt_regs->si */
 999         popl    %edi                    /* pt_regs->di */
1000         popl    %ebp                    /* pt_regs->bp */
1001 
1002         /* Switch to entry stack */
1003         movl    %eax, %esp
1004 
1005         /* Now ready to switch the cr3 */
1006         SWITCH_TO_USER_CR3 scratch_reg=%eax
1007 
1008         /*
1009          * Restore all flags except IF. (We restore IF separately because
1010          * STI gives a one-instruction window in which we won't be interrupted,
1011          * whereas POPF does not.)
1012          */
1013         btrl    $X86_EFLAGS_IF_BIT, (%esp)
1014         BUG_IF_WRONG_CR3 no_user_check=1
1015         popfl
1016         popl    %eax
1017 
1018         /*
1019          * Return back to the vDSO, which will pop ecx and edx.
1020          * Don't bother with DS and ES (they already contain __USER_DS).
1021          */
1022         sti
1023         sysexit
1024 
1025 .pushsection .fixup, "ax"
1026 2:      movl    $0, PT_FS(%esp)
1027         jmp     1b
1028 .popsection
1029         _ASM_EXTABLE(1b, 2b)
1030         PTGS_TO_GS_EX
1031 
1032 .Lsysenter_fix_flags:
1033         pushl   $X86_EFLAGS_FIXED
1034         popfl
1035         jmp     .Lsysenter_flags_fixed
1036 GLOBAL(__end_SYSENTER_singlestep_region)
1037 ENDPROC(entry_SYSENTER_32)
1038 
1039 /*
1040  * 32-bit legacy system call entry.
1041  *
1042  * 32-bit x86 Linux system calls traditionally used the INT $0x80
1043  * instruction.  INT $0x80 lands here.
1044  *
1045  * This entry point can be used by any 32-bit perform system calls.
1046  * Instances of INT $0x80 can be found inline in various programs and
1047  * libraries.  It is also used by the vDSO's __kernel_vsyscall
1048  * fallback for hardware that doesn't support a faster entry method.
1049  * Restarted 32-bit system calls also fall back to INT $0x80
1050  * regardless of what instruction was originally used to do the system
1051  * call.  (64-bit programs can use INT $0x80 as well, but they can
1052  * only run on 64-bit kernels and therefore land in
1053  * entry_INT80_compat.)
1054  *
1055  * This is considered a slow path.  It is not used by most libc
1056  * implementations on modern hardware except during process startup.
1057  *
1058  * Arguments:
1059  * eax  system call number
1060  * ebx  arg1
1061  * ecx  arg2
1062  * edx  arg3
1063  * esi  arg4
1064  * edi  arg5
1065  * ebp  arg6
1066  */
1067 ENTRY(entry_INT80_32)
1068         ASM_CLAC
1069         pushl   %eax                    /* pt_regs->orig_ax */
1070 
1071         SAVE_ALL pt_regs_ax=$-ENOSYS switch_stacks=1    /* save rest */
1072 
1073         /*
1074          * User mode is traced as though IRQs are on, and the interrupt gate
1075          * turned them off.
1076          */
1077         TRACE_IRQS_OFF
1078 
1079         movl    %esp, %eax
1080         call    do_int80_syscall_32
1081 .Lsyscall_32_done:
1082 
1083         STACKLEAK_ERASE
1084 
1085 restore_all:
1086         TRACE_IRQS_IRET
1087         SWITCH_TO_ENTRY_STACK
1088 .Lrestore_all_notrace:
1089         CHECK_AND_APPLY_ESPFIX
1090 .Lrestore_nocheck:
1091         /* Switch back to user CR3 */
1092         SWITCH_TO_USER_CR3 scratch_reg=%eax
1093 
1094         BUG_IF_WRONG_CR3
1095 
1096         /* Restore user state */
1097         RESTORE_REGS pop=4                      # skip orig_eax/error_code
1098 .Lirq_return:
1099         /*
1100          * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
1101          * when returning from IPI handler and when returning from
1102          * scheduler to user-space.
1103          */
1104         INTERRUPT_RETURN
1105 
1106 restore_all_kernel:
1107 #ifdef CONFIG_PREEMPTION
1108         DISABLE_INTERRUPTS(CLBR_ANY)
1109         cmpl    $0, PER_CPU_VAR(__preempt_count)
1110         jnz     .Lno_preempt
1111         testl   $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ?
1112         jz      .Lno_preempt
1113         call    preempt_schedule_irq
1114 .Lno_preempt:
1115 #endif
1116         TRACE_IRQS_IRET
1117         PARANOID_EXIT_TO_KERNEL_MODE
1118         BUG_IF_WRONG_CR3
1119         RESTORE_REGS 4
1120         jmp     .Lirq_return
1121 
1122 .section .fixup, "ax"
1123 ENTRY(iret_exc  )
1124         pushl   $0                              # no error code
1125         pushl   $do_iret_error
1126 
1127 #ifdef CONFIG_DEBUG_ENTRY
1128         /*
1129          * The stack-frame here is the one that iret faulted on, so its a
1130          * return-to-user frame. We are on kernel-cr3 because we come here from
1131          * the fixup code. This confuses the CR3 checker, so switch to user-cr3
1132          * as the checker expects it.
1133          */
1134         pushl   %eax
1135         SWITCH_TO_USER_CR3 scratch_reg=%eax
1136         popl    %eax
1137 #endif
1138 
1139         jmp     common_exception
1140 .previous
1141         _ASM_EXTABLE(.Lirq_return, iret_exc)
1142 ENDPROC(entry_INT80_32)
1143 
1144 .macro FIXUP_ESPFIX_STACK
1145 /*
1146  * Switch back for ESPFIX stack to the normal zerobased stack
1147  *
1148  * We can't call C functions using the ESPFIX stack. This code reads
1149  * the high word of the segment base from the GDT and swiches to the
1150  * normal stack and adjusts ESP with the matching offset.
1151  *
1152  * We might be on user CR3 here, so percpu data is not mapped and we can't
1153  * access the GDT through the percpu segment.  Instead, use SGDT to find
1154  * the cpu_entry_area alias of the GDT.
1155  */
1156 #ifdef CONFIG_X86_ESPFIX32
1157         /* fixup the stack */
1158         pushl   %ecx
1159         subl    $2*4, %esp
1160         sgdt    (%esp)
1161         movl    2(%esp), %ecx                           /* GDT address */
1162         /*
1163          * Careful: ECX is a linear pointer, so we need to force base
1164          * zero.  %cs is the only known-linear segment we have right now.
1165          */
1166         mov     %cs:GDT_ESPFIX_OFFSET + 4(%ecx), %al    /* bits 16..23 */
1167         mov     %cs:GDT_ESPFIX_OFFSET + 7(%ecx), %ah    /* bits 24..31 */
1168         shl     $16, %eax
1169         addl    $2*4, %esp
1170         popl    %ecx
1171         addl    %esp, %eax                      /* the adjusted stack pointer */
1172         pushl   $__KERNEL_DS
1173         pushl   %eax
1174         lss     (%esp), %esp                    /* switch to the normal stack segment */
1175 #endif
1176 .endm
1177 
1178 .macro UNWIND_ESPFIX_STACK
1179         /* It's safe to clobber %eax, all other regs need to be preserved */
1180 #ifdef CONFIG_X86_ESPFIX32
1181         movl    %ss, %eax
1182         /* see if on espfix stack */
1183         cmpw    $__ESPFIX_SS, %ax
1184         jne     .Lno_fixup_\@
1185         /* switch to normal stack */
1186         FIXUP_ESPFIX_STACK
1187 .Lno_fixup_\@:
1188 #endif
1189 .endm
1190 
1191 /*
1192  * Build the entry stubs with some assembler magic.
1193  * We pack 1 stub into every 8-byte block.
1194  */
1195         .align 8
1196 ENTRY(irq_entries_start)
1197     vector=FIRST_EXTERNAL_VECTOR
1198     .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
1199         pushl   $(~vector+0x80)                 /* Note: always in signed byte range */
1200     vector=vector+1
1201         jmp     common_interrupt
1202         .align  8
1203     .endr
1204 END(irq_entries_start)
1205 
1206 #ifdef CONFIG_X86_LOCAL_APIC
1207         .align 8
1208 ENTRY(spurious_entries_start)
1209     vector=FIRST_SYSTEM_VECTOR
1210     .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR)
1211         pushl   $(~vector+0x80)                 /* Note: always in signed byte range */
1212     vector=vector+1
1213         jmp     common_spurious
1214         .align  8
1215     .endr
1216 END(spurious_entries_start)
1217 
1218 common_spurious:
1219         ASM_CLAC
1220         addl    $-0x80, (%esp)                  /* Adjust vector into the [-256, -1] range */
1221         SAVE_ALL switch_stacks=1
1222         ENCODE_FRAME_POINTER
1223         TRACE_IRQS_OFF
1224         movl    %esp, %eax
1225         call    smp_spurious_interrupt
1226         jmp     ret_from_intr
1227 ENDPROC(common_spurious)
1228 #endif
1229 
1230 /*
1231  * the CPU automatically disables interrupts when executing an IRQ vector,
1232  * so IRQ-flags tracing has to follow that:
1233  */
1234         .p2align CONFIG_X86_L1_CACHE_SHIFT
1235 common_interrupt:
1236         ASM_CLAC
1237         addl    $-0x80, (%esp)                  /* Adjust vector into the [-256, -1] range */
1238 
1239         SAVE_ALL switch_stacks=1
1240         ENCODE_FRAME_POINTER
1241         TRACE_IRQS_OFF
1242         movl    %esp, %eax
1243         call    do_IRQ
1244         jmp     ret_from_intr
1245 ENDPROC(common_interrupt)
1246 
1247 #define BUILD_INTERRUPT3(name, nr, fn)                  \
1248 ENTRY(name)                                             \
1249         ASM_CLAC;                                       \
1250         pushl   $~(nr);                                 \
1251         SAVE_ALL switch_stacks=1;                       \
1252         ENCODE_FRAME_POINTER;                           \
1253         TRACE_IRQS_OFF                                  \
1254         movl    %esp, %eax;                             \
1255         call    fn;                                     \
1256         jmp     ret_from_intr;                          \
1257 ENDPROC(name)
1258 
1259 #define BUILD_INTERRUPT(name, nr)               \
1260         BUILD_INTERRUPT3(name, nr, smp_##name); \
1261 
1262 /* The include is where all of the SMP etc. interrupts come from */
1263 #include <asm/entry_arch.h>
1264 
1265 ENTRY(coprocessor_error)
1266         ASM_CLAC
1267         pushl   $0
1268         pushl   $do_coprocessor_error
1269         jmp     common_exception
1270 END(coprocessor_error)
1271 
1272 ENTRY(simd_coprocessor_error)
1273         ASM_CLAC
1274         pushl   $0
1275 #ifdef CONFIG_X86_INVD_BUG
1276         /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
1277         ALTERNATIVE "pushl      $do_general_protection",        \
1278                     "pushl      $do_simd_coprocessor_error",    \
1279                     X86_FEATURE_XMM
1280 #else
1281         pushl   $do_simd_coprocessor_error
1282 #endif
1283         jmp     common_exception
1284 END(simd_coprocessor_error)
1285 
1286 ENTRY(device_not_available)
1287         ASM_CLAC
1288         pushl   $-1                             # mark this as an int
1289         pushl   $do_device_not_available
1290         jmp     common_exception
1291 END(device_not_available)
1292 
1293 #ifdef CONFIG_PARAVIRT
1294 ENTRY(native_iret)
1295         iret
1296         _ASM_EXTABLE(native_iret, iret_exc)
1297 END(native_iret)
1298 #endif
1299 
1300 ENTRY(overflow)
1301         ASM_CLAC
1302         pushl   $0
1303         pushl   $do_overflow
1304         jmp     common_exception
1305 END(overflow)
1306 
1307 ENTRY(bounds)
1308         ASM_CLAC
1309         pushl   $0
1310         pushl   $do_bounds
1311         jmp     common_exception
1312 END(bounds)
1313 
1314 ENTRY(invalid_op)
1315         ASM_CLAC
1316         pushl   $0
1317         pushl   $do_invalid_op
1318         jmp     common_exception
1319 END(invalid_op)
1320 
1321 ENTRY(coprocessor_segment_overrun)
1322         ASM_CLAC
1323         pushl   $0
1324         pushl   $do_coprocessor_segment_overrun
1325         jmp     common_exception
1326 END(coprocessor_segment_overrun)
1327 
1328 ENTRY(invalid_TSS)
1329         ASM_CLAC
1330         pushl   $do_invalid_TSS
1331         jmp     common_exception
1332 END(invalid_TSS)
1333 
1334 ENTRY(segment_not_present)
1335         ASM_CLAC
1336         pushl   $do_segment_not_present
1337         jmp     common_exception
1338 END(segment_not_present)
1339 
1340 ENTRY(stack_segment)
1341         ASM_CLAC
1342         pushl   $do_stack_segment
1343         jmp     common_exception
1344 END(stack_segment)
1345 
1346 ENTRY(alignment_check)
1347         ASM_CLAC
1348         pushl   $do_alignment_check
1349         jmp     common_exception
1350 END(alignment_check)
1351 
1352 ENTRY(divide_error)
1353         ASM_CLAC
1354         pushl   $0                              # no error code
1355         pushl   $do_divide_error
1356         jmp     common_exception
1357 END(divide_error)
1358 
1359 #ifdef CONFIG_X86_MCE
1360 ENTRY(machine_check)
1361         ASM_CLAC
1362         pushl   $0
1363         pushl   machine_check_vector
1364         jmp     common_exception
1365 END(machine_check)
1366 #endif
1367 
1368 ENTRY(spurious_interrupt_bug)
1369         ASM_CLAC
1370         pushl   $0
1371         pushl   $do_spurious_interrupt_bug
1372         jmp     common_exception
1373 END(spurious_interrupt_bug)
1374 
1375 #ifdef CONFIG_XEN_PV
1376 ENTRY(xen_hypervisor_callback)
1377         /*
1378          * Check to see if we got the event in the critical
1379          * region in xen_iret_direct, after we've reenabled
1380          * events and checked for pending events.  This simulates
1381          * iret instruction's behaviour where it delivers a
1382          * pending interrupt when enabling interrupts:
1383          */
1384         cmpl    $xen_iret_start_crit, (%esp)
1385         jb      1f
1386         cmpl    $xen_iret_end_crit, (%esp)
1387         jae     1f
1388         call    xen_iret_crit_fixup
1389 1:
1390         pushl   $-1                             /* orig_ax = -1 => not a system call */
1391         SAVE_ALL
1392         ENCODE_FRAME_POINTER
1393         TRACE_IRQS_OFF
1394         mov     %esp, %eax
1395         call    xen_evtchn_do_upcall
1396 #ifndef CONFIG_PREEMPTION
1397         call    xen_maybe_preempt_hcall
1398 #endif
1399         jmp     ret_from_intr
1400 ENDPROC(xen_hypervisor_callback)
1401 
1402 /*
1403  * Hypervisor uses this for application faults while it executes.
1404  * We get here for two reasons:
1405  *  1. Fault while reloading DS, ES, FS or GS
1406  *  2. Fault while executing IRET
1407  * Category 1 we fix up by reattempting the load, and zeroing the segment
1408  * register if the load fails.
1409  * Category 2 we fix up by jumping to do_iret_error. We cannot use the
1410  * normal Linux return path in this case because if we use the IRET hypercall
1411  * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1412  * We distinguish between categories by maintaining a status value in EAX.
1413  */
1414 ENTRY(xen_failsafe_callback)
1415         pushl   %eax
1416         movl    $1, %eax
1417 1:      mov     4(%esp), %ds
1418 2:      mov     8(%esp), %es
1419 3:      mov     12(%esp), %fs
1420 4:      mov     16(%esp), %gs
1421         /* EAX == 0 => Category 1 (Bad segment)
1422            EAX != 0 => Category 2 (Bad IRET) */
1423         testl   %eax, %eax
1424         popl    %eax
1425         lea     16(%esp), %esp
1426         jz      5f
1427         jmp     iret_exc
1428 5:      pushl   $-1                             /* orig_ax = -1 => not a system call */
1429         SAVE_ALL
1430         ENCODE_FRAME_POINTER
1431         jmp     ret_from_exception
1432 
1433 .section .fixup, "ax"
1434 6:      xorl    %eax, %eax
1435         movl    %eax, 4(%esp)
1436         jmp     1b
1437 7:      xorl    %eax, %eax
1438         movl    %eax, 8(%esp)
1439         jmp     2b
1440 8:      xorl    %eax, %eax
1441         movl    %eax, 12(%esp)
1442         jmp     3b
1443 9:      xorl    %eax, %eax
1444         movl    %eax, 16(%esp)
1445         jmp     4b
1446 .previous
1447         _ASM_EXTABLE(1b, 6b)
1448         _ASM_EXTABLE(2b, 7b)
1449         _ASM_EXTABLE(3b, 8b)
1450         _ASM_EXTABLE(4b, 9b)
1451 ENDPROC(xen_failsafe_callback)
1452 #endif /* CONFIG_XEN_PV */
1453 
1454 #ifdef CONFIG_XEN_PVHVM
1455 BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
1456                  xen_evtchn_do_upcall)
1457 #endif
1458 
1459 
1460 #if IS_ENABLED(CONFIG_HYPERV)
1461 
1462 BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
1463                  hyperv_vector_handler)
1464 
1465 BUILD_INTERRUPT3(hyperv_reenlightenment_vector, HYPERV_REENLIGHTENMENT_VECTOR,
1466                  hyperv_reenlightenment_intr)
1467 
1468 BUILD_INTERRUPT3(hv_stimer0_callback_vector, HYPERV_STIMER0_VECTOR,
1469                  hv_stimer0_vector_handler)
1470 
1471 #endif /* CONFIG_HYPERV */
1472 
1473 ENTRY(page_fault)
1474         ASM_CLAC
1475         pushl   $do_page_fault
1476         jmp     common_exception_read_cr2
1477 END(page_fault)
1478 
1479 common_exception_read_cr2:
1480         /* the function address is in %gs's slot on the stack */
1481         SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1
1482 
1483         ENCODE_FRAME_POINTER
1484 
1485         /* fixup %gs */
1486         GS_TO_REG %ecx
1487         movl    PT_GS(%esp), %edi
1488         REG_TO_PTGS %ecx
1489         SET_KERNEL_GS %ecx
1490 
1491         GET_CR2_INTO(%ecx)                      # might clobber %eax
1492 
1493         /* fixup orig %eax */
1494         movl    PT_ORIG_EAX(%esp), %edx         # get the error code
1495         movl    $-1, PT_ORIG_EAX(%esp)          # no syscall to restart
1496 
1497         TRACE_IRQS_OFF
1498         movl    %esp, %eax                      # pt_regs pointer
1499         CALL_NOSPEC %edi
1500         jmp     ret_from_exception
1501 END(common_exception_read_cr2)
1502 
1503 common_exception:
1504         /* the function address is in %gs's slot on the stack */
1505         SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1
1506         ENCODE_FRAME_POINTER
1507 
1508         /* fixup %gs */
1509         GS_TO_REG %ecx
1510         movl    PT_GS(%esp), %edi               # get the function address
1511         REG_TO_PTGS %ecx
1512         SET_KERNEL_GS %ecx
1513 
1514         /* fixup orig %eax */
1515         movl    PT_ORIG_EAX(%esp), %edx         # get the error code
1516         movl    $-1, PT_ORIG_EAX(%esp)          # no syscall to restart
1517 
1518         TRACE_IRQS_OFF
1519         movl    %esp, %eax                      # pt_regs pointer
1520         CALL_NOSPEC %edi
1521         jmp     ret_from_exception
1522 END(common_exception)
1523 
1524 ENTRY(debug)
1525         /*
1526          * Entry from sysenter is now handled in common_exception
1527          */
1528         ASM_CLAC
1529         pushl   $-1                             # mark this as an int
1530         pushl   $do_debug
1531         jmp     common_exception
1532 END(debug)
1533 
1534 /*
1535  * NMI is doubly nasty.  It can happen on the first instruction of
1536  * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning
1537  * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32
1538  * switched stacks.  We handle both conditions by simply checking whether we
1539  * interrupted kernel code running on the SYSENTER stack.
1540  */
1541 ENTRY(nmi)
1542         ASM_CLAC
1543 
1544 #ifdef CONFIG_X86_ESPFIX32
1545         /*
1546          * ESPFIX_SS is only ever set on the return to user path
1547          * after we've switched to the entry stack.
1548          */
1549         pushl   %eax
1550         movl    %ss, %eax
1551         cmpw    $__ESPFIX_SS, %ax
1552         popl    %eax
1553         je      .Lnmi_espfix_stack
1554 #endif
1555 
1556         pushl   %eax                            # pt_regs->orig_ax
1557         SAVE_ALL_NMI cr3_reg=%edi
1558         ENCODE_FRAME_POINTER
1559         xorl    %edx, %edx                      # zero error code
1560         movl    %esp, %eax                      # pt_regs pointer
1561 
1562         /* Are we currently on the SYSENTER stack? */
1563         movl    PER_CPU_VAR(cpu_entry_area), %ecx
1564         addl    $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
1565         subl    %eax, %ecx      /* ecx = (end of entry_stack) - esp */
1566         cmpl    $SIZEOF_entry_stack, %ecx
1567         jb      .Lnmi_from_sysenter_stack
1568 
1569         /* Not on SYSENTER stack. */
1570         call    do_nmi
1571         jmp     .Lnmi_return
1572 
1573 .Lnmi_from_sysenter_stack:
1574         /*
1575          * We're on the SYSENTER stack.  Switch off.  No one (not even debug)
1576          * is using the thread stack right now, so it's safe for us to use it.
1577          */
1578         movl    %esp, %ebx
1579         movl    PER_CPU_VAR(cpu_current_top_of_stack), %esp
1580         call    do_nmi
1581         movl    %ebx, %esp
1582 
1583 .Lnmi_return:
1584 #ifdef CONFIG_X86_ESPFIX32
1585         testl   $CS_FROM_ESPFIX, PT_CS(%esp)
1586         jnz     .Lnmi_from_espfix
1587 #endif
1588 
1589         CHECK_AND_APPLY_ESPFIX
1590         RESTORE_ALL_NMI cr3_reg=%edi pop=4
1591         jmp     .Lirq_return
1592 
1593 #ifdef CONFIG_X86_ESPFIX32
1594 .Lnmi_espfix_stack:
1595         /*
1596          * Create the pointer to LSS back
1597          */
1598         pushl   %ss
1599         pushl   %esp
1600         addl    $4, (%esp)
1601 
1602         /* Copy the (short) IRET frame */
1603         pushl   4*4(%esp)       # flags
1604         pushl   4*4(%esp)       # cs
1605         pushl   4*4(%esp)       # ip
1606 
1607         pushl   %eax            # orig_ax
1608 
1609         SAVE_ALL_NMI cr3_reg=%edi unwind_espfix=1
1610         ENCODE_FRAME_POINTER
1611 
1612         /* clear CS_FROM_KERNEL, set CS_FROM_ESPFIX */
1613         xorl    $(CS_FROM_ESPFIX | CS_FROM_KERNEL), PT_CS(%esp)
1614 
1615         xorl    %edx, %edx                      # zero error code
1616         movl    %esp, %eax                      # pt_regs pointer
1617         jmp     .Lnmi_from_sysenter_stack
1618 
1619 .Lnmi_from_espfix:
1620         RESTORE_ALL_NMI cr3_reg=%edi
1621         /*
1622          * Because we cleared CS_FROM_KERNEL, IRET_FRAME 'forgot' to
1623          * fix up the gap and long frame:
1624          *
1625          *  3 - original frame  (exception)
1626          *  2 - ESPFIX block    (above)
1627          *  6 - gap             (FIXUP_FRAME)
1628          *  5 - long frame      (FIXUP_FRAME)
1629          *  1 - orig_ax
1630          */
1631         lss     (1+5+6)*4(%esp), %esp                   # back to espfix stack
1632         jmp     .Lirq_return
1633 #endif
1634 END(nmi)
1635 
1636 ENTRY(int3)
1637         ASM_CLAC
1638         pushl   $-1                             # mark this as an int
1639 
1640         SAVE_ALL switch_stacks=1
1641         ENCODE_FRAME_POINTER
1642         TRACE_IRQS_OFF
1643         xorl    %edx, %edx                      # zero error code
1644         movl    %esp, %eax                      # pt_regs pointer
1645         call    do_int3
1646         jmp     ret_from_exception
1647 END(int3)
1648 
1649 ENTRY(general_protection)
1650         ASM_CLAC
1651         pushl   $do_general_protection
1652         jmp     common_exception
1653 END(general_protection)
1654 
1655 #ifdef CONFIG_KVM_GUEST
1656 ENTRY(async_page_fault)
1657         ASM_CLAC
1658         pushl   $do_async_page_fault
1659         jmp     common_exception_read_cr2
1660 END(async_page_fault)
1661 #endif
1662 
1663 ENTRY(rewind_stack_do_exit)
1664         /* Prevent any naive code from trying to unwind to our caller. */
1665         xorl    %ebp, %ebp
1666 
1667         movl    PER_CPU_VAR(cpu_current_top_of_stack), %esi
1668         leal    -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp
1669 
1670         call    do_exit
1671 1:      jmp 1b
1672 END(rewind_stack_do_exit)