arch/x86/mm/fault.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
kmmio_fault
check_prefetch_opcode
is_prefetch
vmalloc_sync_one
vmalloc_sync
vmalloc_sync_mappings
vmalloc_sync_unmappings
vmalloc_fault
check_v8086_mode
low_pfn
dump_pagetable
vmalloc_sync_mappings
vmalloc_sync_unmappings
vmalloc_fault
check_v8086_mode
bad_address
dump_pagetable
is_errata93
is_errata100
is_f00f_bug
show_ldttss
show_fault_oops
pgtable_bad
set_signal_archinfo
no_context
show_signal_msg
is_vsyscall_vaddr
__bad_area_nosemaphore
bad_area_nosemaphore
__bad_area
bad_area
bad_area_access_from_pkeys
bad_area_access_error
do_sigbus
mm_fault_error
spurious_kernel_fault_check
spurious_kernel_fault
access_error
fault_in_kernel_space
do_kern_addr_fault
do_user_addr_fault
__do_page_fault
trace_page_fault_entries
do_page_fault
   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1995  Linus Torvalds
   4  *  Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs.
   5  *  Copyright (C) 2008-2009, Red Hat Inc., Ingo Molnar
   6  */
   7 #include <linux/sched.h>                /* test_thread_flag(), ...      */
   8 #include <linux/sched/task_stack.h>     /* task_stack_*(), ...          */
   9 #include <linux/kdebug.h>               /* oops_begin/end, ...          */
  10 #include <linux/extable.h>              /* search_exception_tables      */
  11 #include <linux/memblock.h>             /* max_low_pfn                  */
  12 #include <linux/kprobes.h>              /* NOKPROBE_SYMBOL, ...         */
  13 #include <linux/mmiotrace.h>            /* kmmio_handler, ...           */
  14 #include <linux/perf_event.h>           /* perf_sw_event                */
  15 #include <linux/hugetlb.h>              /* hstate_index_to_shift        */
  16 #include <linux/prefetch.h>             /* prefetchw                    */
  17 #include <linux/context_tracking.h>     /* exception_enter(), ...       */
  18 #include <linux/uaccess.h>              /* faulthandler_disabled()      */
  19 #include <linux/efi.h>                  /* efi_recover_from_page_fault()*/
  20 #include <linux/mm_types.h>
  21 
  22 #include <asm/cpufeature.h>             /* boot_cpu_has, ...            */
  23 #include <asm/traps.h>                  /* dotraplinkage, ...           */
  24 #include <asm/pgalloc.h>                /* pgd_*(), ...                 */
  25 #include <asm/fixmap.h>                 /* VSYSCALL_ADDR                */
  26 #include <asm/vsyscall.h>               /* emulate_vsyscall             */
  27 #include <asm/vm86.h>                   /* struct vm86                  */
  28 #include <asm/mmu_context.h>            /* vma_pkey()                   */
  29 #include <asm/efi.h>                    /* efi_recover_from_page_fault()*/
  30 #include <asm/desc.h>                   /* store_idt(), ...             */
  31 #include <asm/cpu_entry_area.h>         /* exception stack              */
  32 
  33 #define CREATE_TRACE_POINTS
  34 #include <asm/trace/exceptions.h>
  35 
  36 /*
  37  * Returns 0 if mmiotrace is disabled, or if the fault is not
  38  * handled by mmiotrace:
  39  */
  40 static nokprobe_inline int
  41 kmmio_fault(struct pt_regs *regs, unsigned long addr)
  42 {
  43         if (unlikely(is_kmmio_active()))
  44                 if (kmmio_handler(regs, addr) == 1)
  45                         return -1;
  46         return 0;
  47 }
  48 
  49 /*
  50  * Prefetch quirks:
  51  *
  52  * 32-bit mode:
  53  *
  54  *   Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
  55  *   Check that here and ignore it.
  56  *
  57  * 64-bit mode:
  58  *
  59  *   Sometimes the CPU reports invalid exceptions on prefetch.
  60  *   Check that here and ignore it.
  61  *
  62  * Opcode checker based on code by Richard Brunner.
  63  */
  64 static inline int
  65 check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
  66                       unsigned char opcode, int *prefetch)
  67 {
  68         unsigned char instr_hi = opcode & 0xf0;
  69         unsigned char instr_lo = opcode & 0x0f;
  70 
  71         switch (instr_hi) {
  72         case 0x20:
  73         case 0x30:
  74                 /*
  75                  * Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes.
  76                  * In X86_64 long mode, the CPU will signal invalid
  77                  * opcode if some of these prefixes are present so
  78                  * X86_64 will never get here anyway
  79                  */
  80                 return ((instr_lo & 7) == 0x6);
  81 #ifdef CONFIG_X86_64
  82         case 0x40:
  83                 /*
  84                  * In AMD64 long mode 0x40..0x4F are valid REX prefixes
  85                  * Need to figure out under what instruction mode the
  86                  * instruction was issued. Could check the LDT for lm,
  87                  * but for now it's good enough to assume that long
  88                  * mode only uses well known segments or kernel.
  89                  */
  90                 return (!user_mode(regs) || user_64bit_mode(regs));
  91 #endif
  92         case 0x60:
  93                 /* 0x64 thru 0x67 are valid prefixes in all modes. */
  94                 return (instr_lo & 0xC) == 0x4;
  95         case 0xF0:
  96                 /* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */
  97                 return !instr_lo || (instr_lo>>1) == 1;
  98         case 0x00:
  99                 /* Prefetch instruction is 0x0F0D or 0x0F18 */
 100                 if (probe_kernel_address(instr, opcode))
 101                         return 0;
 102 
 103                 *prefetch = (instr_lo == 0xF) &&
 104                         (opcode == 0x0D || opcode == 0x18);
 105                 return 0;
 106         default:
 107                 return 0;
 108         }
 109 }
 110 
 111 static int
 112 is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
 113 {
 114         unsigned char *max_instr;
 115         unsigned char *instr;
 116         int prefetch = 0;
 117 
 118         /*
 119          * If it was a exec (instruction fetch) fault on NX page, then
 120          * do not ignore the fault:
 121          */
 122         if (error_code & X86_PF_INSTR)
 123                 return 0;
 124 
 125         instr = (void *)convert_ip_to_linear(current, regs);
 126         max_instr = instr + 15;
 127 
 128         if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE_MAX)
 129                 return 0;
 130 
 131         while (instr < max_instr) {
 132                 unsigned char opcode;
 133 
 134                 if (probe_kernel_address(instr, opcode))
 135                         break;
 136 
 137                 instr++;
 138 
 139                 if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
 140                         break;
 141         }
 142         return prefetch;
 143 }
 144 
 145 DEFINE_SPINLOCK(pgd_lock);
 146 LIST_HEAD(pgd_list);
 147 
 148 #ifdef CONFIG_X86_32
 149 static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
 150 {
 151         unsigned index = pgd_index(address);
 152         pgd_t *pgd_k;
 153         p4d_t *p4d, *p4d_k;
 154         pud_t *pud, *pud_k;
 155         pmd_t *pmd, *pmd_k;
 156 
 157         pgd += index;
 158         pgd_k = init_mm.pgd + index;
 159 
 160         if (!pgd_present(*pgd_k))
 161                 return NULL;
 162 
 163         /*
 164          * set_pgd(pgd, *pgd_k); here would be useless on PAE
 165          * and redundant with the set_pmd() on non-PAE. As would
 166          * set_p4d/set_pud.
 167          */
 168         p4d = p4d_offset(pgd, address);
 169         p4d_k = p4d_offset(pgd_k, address);
 170         if (!p4d_present(*p4d_k))
 171                 return NULL;
 172 
 173         pud = pud_offset(p4d, address);
 174         pud_k = pud_offset(p4d_k, address);
 175         if (!pud_present(*pud_k))
 176                 return NULL;
 177 
 178         pmd = pmd_offset(pud, address);
 179         pmd_k = pmd_offset(pud_k, address);
 180 
 181         if (pmd_present(*pmd) != pmd_present(*pmd_k))
 182                 set_pmd(pmd, *pmd_k);
 183 
 184         if (!pmd_present(*pmd_k))
 185                 return NULL;
 186         else
 187                 BUG_ON(pmd_pfn(*pmd) != pmd_pfn(*pmd_k));
 188 
 189         return pmd_k;
 190 }
 191 
 192 static void vmalloc_sync(void)
 193 {
 194         unsigned long address;
 195 
 196         if (SHARED_KERNEL_PMD)
 197                 return;
 198 
 199         for (address = VMALLOC_START & PMD_MASK;
 200              address >= TASK_SIZE_MAX && address < VMALLOC_END;
 201              address += PMD_SIZE) {
 202                 struct page *page;
 203 
 204                 spin_lock(&pgd_lock);
 205                 list_for_each_entry(page, &pgd_list, lru) {
 206                         spinlock_t *pgt_lock;
 207 
 208                         /* the pgt_lock only for Xen */
 209                         pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
 210 
 211                         spin_lock(pgt_lock);
 212                         vmalloc_sync_one(page_address(page), address);
 213                         spin_unlock(pgt_lock);
 214                 }
 215                 spin_unlock(&pgd_lock);
 216         }
 217 }
 218 
 219 void vmalloc_sync_mappings(void)
 220 {
 221         vmalloc_sync();
 222 }
 223 
 224 void vmalloc_sync_unmappings(void)
 225 {
 226         vmalloc_sync();
 227 }
 228 
 229 /*
 230  * 32-bit:
 231  *
 232  *   Handle a fault on the vmalloc or module mapping area
 233  */
 234 static noinline int vmalloc_fault(unsigned long address)
 235 {
 236         unsigned long pgd_paddr;
 237         pmd_t *pmd_k;
 238         pte_t *pte_k;
 239 
 240         /* Make sure we are in vmalloc area: */
 241         if (!(address >= VMALLOC_START && address < VMALLOC_END))
 242                 return -1;
 243 
 244         /*
 245          * Synchronize this task's top level page-table
 246          * with the 'reference' page table.
 247          *
 248          * Do _not_ use "current" here. We might be inside
 249          * an interrupt in the middle of a task switch..
 250          */
 251         pgd_paddr = read_cr3_pa();
 252         pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
 253         if (!pmd_k)
 254                 return -1;
 255 
 256         if (pmd_large(*pmd_k))
 257                 return 0;
 258 
 259         pte_k = pte_offset_kernel(pmd_k, address);
 260         if (!pte_present(*pte_k))
 261                 return -1;
 262 
 263         return 0;
 264 }
 265 NOKPROBE_SYMBOL(vmalloc_fault);
 266 
 267 /*
 268  * Did it hit the DOS screen memory VA from vm86 mode?
 269  */
 270 static inline void
 271 check_v8086_mode(struct pt_regs *regs, unsigned long address,
 272                  struct task_struct *tsk)
 273 {
 274 #ifdef CONFIG_VM86
 275         unsigned long bit;
 276 
 277         if (!v8086_mode(regs) || !tsk->thread.vm86)
 278                 return;
 279 
 280         bit = (address - 0xA0000) >> PAGE_SHIFT;
 281         if (bit < 32)
 282                 tsk->thread.vm86->screen_bitmap |= 1 << bit;
 283 #endif
 284 }
 285 
 286 static bool low_pfn(unsigned long pfn)
 287 {
 288         return pfn < max_low_pfn;
 289 }
 290 
 291 static void dump_pagetable(unsigned long address)
 292 {
 293         pgd_t *base = __va(read_cr3_pa());
 294         pgd_t *pgd = &base[pgd_index(address)];
 295         p4d_t *p4d;
 296         pud_t *pud;
 297         pmd_t *pmd;
 298         pte_t *pte;
 299 
 300 #ifdef CONFIG_X86_PAE
 301         pr_info("*pdpt = %016Lx ", pgd_val(*pgd));
 302         if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
 303                 goto out;
 304 #define pr_pde pr_cont
 305 #else
 306 #define pr_pde pr_info
 307 #endif
 308         p4d = p4d_offset(pgd, address);
 309         pud = pud_offset(p4d, address);
 310         pmd = pmd_offset(pud, address);
 311         pr_pde("*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
 312 #undef pr_pde
 313 
 314         /*
 315          * We must not directly access the pte in the highpte
 316          * case if the page table is located in highmem.
 317          * And let's rather not kmap-atomic the pte, just in case
 318          * it's allocated already:
 319          */
 320         if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_large(*pmd))
 321                 goto out;
 322 
 323         pte = pte_offset_kernel(pmd, address);
 324         pr_cont("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
 325 out:
 326         pr_cont("\n");
 327 }
 328 
 329 #else /* CONFIG_X86_64: */
 330 
 331 void vmalloc_sync_mappings(void)
 332 {
 333         /*
 334          * 64-bit mappings might allocate new p4d/pud pages
 335          * that need to be propagated to all tasks' PGDs.
 336          */
 337         sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END);
 338 }
 339 
 340 void vmalloc_sync_unmappings(void)
 341 {
 342         /*
 343          * Unmappings never allocate or free p4d/pud pages.
 344          * No work is required here.
 345          */
 346 }
 347 
 348 /*
 349  * 64-bit:
 350  *
 351  *   Handle a fault on the vmalloc area
 352  */
 353 static noinline int vmalloc_fault(unsigned long address)
 354 {
 355         pgd_t *pgd, *pgd_k;
 356         p4d_t *p4d, *p4d_k;
 357         pud_t *pud;
 358         pmd_t *pmd;
 359         pte_t *pte;
 360 
 361         /* Make sure we are in vmalloc area: */
 362         if (!(address >= VMALLOC_START && address < VMALLOC_END))
 363                 return -1;
 364 
 365         /*
 366          * Copy kernel mappings over when needed. This can also
 367          * happen within a race in page table update. In the later
 368          * case just flush:
 369          */
 370         pgd = (pgd_t *)__va(read_cr3_pa()) + pgd_index(address);
 371         pgd_k = pgd_offset_k(address);
 372         if (pgd_none(*pgd_k))
 373                 return -1;
 374 
 375         if (pgtable_l5_enabled()) {
 376                 if (pgd_none(*pgd)) {
 377                         set_pgd(pgd, *pgd_k);
 378                         arch_flush_lazy_mmu_mode();
 379                 } else {
 380                         BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_k));
 381                 }
 382         }
 383 
 384         /* With 4-level paging, copying happens on the p4d level. */
 385         p4d = p4d_offset(pgd, address);
 386         p4d_k = p4d_offset(pgd_k, address);
 387         if (p4d_none(*p4d_k))
 388                 return -1;
 389 
 390         if (p4d_none(*p4d) && !pgtable_l5_enabled()) {
 391                 set_p4d(p4d, *p4d_k);
 392                 arch_flush_lazy_mmu_mode();
 393         } else {
 394                 BUG_ON(p4d_pfn(*p4d) != p4d_pfn(*p4d_k));
 395         }
 396 
 397         BUILD_BUG_ON(CONFIG_PGTABLE_LEVELS < 4);
 398 
 399         pud = pud_offset(p4d, address);
 400         if (pud_none(*pud))
 401                 return -1;
 402 
 403         if (pud_large(*pud))
 404                 return 0;
 405 
 406         pmd = pmd_offset(pud, address);
 407         if (pmd_none(*pmd))
 408                 return -1;
 409 
 410         if (pmd_large(*pmd))
 411                 return 0;
 412 
 413         pte = pte_offset_kernel(pmd, address);
 414         if (!pte_present(*pte))
 415                 return -1;
 416 
 417         return 0;
 418 }
 419 NOKPROBE_SYMBOL(vmalloc_fault);
 420 
 421 #ifdef CONFIG_CPU_SUP_AMD
 422 static const char errata93_warning[] =
 423 KERN_ERR 
 424 "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
 425 "******* Working around it, but it may cause SEGVs or burn power.\n"
 426 "******* Please consider a BIOS update.\n"
 427 "******* Disabling USB legacy in the BIOS may also help.\n";
 428 #endif
 429 
 430 /*
 431  * No vm86 mode in 64-bit mode:
 432  */
 433 static inline void
 434 check_v8086_mode(struct pt_regs *regs, unsigned long address,
 435                  struct task_struct *tsk)
 436 {
 437 }
 438 
 439 static int bad_address(void *p)
 440 {
 441         unsigned long dummy;
 442 
 443         return probe_kernel_address((unsigned long *)p, dummy);
 444 }
 445 
 446 static void dump_pagetable(unsigned long address)
 447 {
 448         pgd_t *base = __va(read_cr3_pa());
 449         pgd_t *pgd = base + pgd_index(address);
 450         p4d_t *p4d;
 451         pud_t *pud;
 452         pmd_t *pmd;
 453         pte_t *pte;
 454 
 455         if (bad_address(pgd))
 456                 goto bad;
 457 
 458         pr_info("PGD %lx ", pgd_val(*pgd));
 459 
 460         if (!pgd_present(*pgd))
 461                 goto out;
 462 
 463         p4d = p4d_offset(pgd, address);
 464         if (bad_address(p4d))
 465                 goto bad;
 466 
 467         pr_cont("P4D %lx ", p4d_val(*p4d));
 468         if (!p4d_present(*p4d) || p4d_large(*p4d))
 469                 goto out;
 470 
 471         pud = pud_offset(p4d, address);
 472         if (bad_address(pud))
 473                 goto bad;
 474 
 475         pr_cont("PUD %lx ", pud_val(*pud));
 476         if (!pud_present(*pud) || pud_large(*pud))
 477                 goto out;
 478 
 479         pmd = pmd_offset(pud, address);
 480         if (bad_address(pmd))
 481                 goto bad;
 482 
 483         pr_cont("PMD %lx ", pmd_val(*pmd));
 484         if (!pmd_present(*pmd) || pmd_large(*pmd))
 485                 goto out;
 486 
 487         pte = pte_offset_kernel(pmd, address);
 488         if (bad_address(pte))
 489                 goto bad;
 490 
 491         pr_cont("PTE %lx", pte_val(*pte));
 492 out:
 493         pr_cont("\n");
 494         return;
 495 bad:
 496         pr_info("BAD\n");
 497 }
 498 
 499 #endif /* CONFIG_X86_64 */
 500 
 501 /*
 502  * Workaround for K8 erratum #93 & buggy BIOS.
 503  *
 504  * BIOS SMM functions are required to use a specific workaround
 505  * to avoid corruption of the 64bit RIP register on C stepping K8.
 506  *
 507  * A lot of BIOS that didn't get tested properly miss this.
 508  *
 509  * The OS sees this as a page fault with the upper 32bits of RIP cleared.
 510  * Try to work around it here.
 511  *
 512  * Note we only handle faults in kernel here.
 513  * Does nothing on 32-bit.
 514  */
 515 static int is_errata93(struct pt_regs *regs, unsigned long address)
 516 {
 517 #if defined(CONFIG_X86_64) && defined(CONFIG_CPU_SUP_AMD)
 518         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD
 519             || boot_cpu_data.x86 != 0xf)
 520                 return 0;
 521 
 522         if (address != regs->ip)
 523                 return 0;
 524 
 525         if ((address >> 32) != 0)
 526                 return 0;
 527 
 528         address |= 0xffffffffUL << 32;
 529         if ((address >= (u64)_stext && address <= (u64)_etext) ||
 530             (address >= MODULES_VADDR && address <= MODULES_END)) {
 531                 printk_once(errata93_warning);
 532                 regs->ip = address;
 533                 return 1;
 534         }
 535 #endif
 536         return 0;
 537 }
 538 
 539 /*
 540  * Work around K8 erratum #100 K8 in compat mode occasionally jumps
 541  * to illegal addresses >4GB.
 542  *
 543  * We catch this in the page fault handler because these addresses
 544  * are not reachable. Just detect this case and return.  Any code
 545  * segment in LDT is compatibility mode.
 546  */
 547 static int is_errata100(struct pt_regs *regs, unsigned long address)
 548 {
 549 #ifdef CONFIG_X86_64
 550         if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
 551                 return 1;
 552 #endif
 553         return 0;
 554 }
 555 
 556 static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
 557 {
 558 #ifdef CONFIG_X86_F00F_BUG
 559         unsigned long nr;
 560 
 561         /*
 562          * Pentium F0 0F C7 C8 bug workaround:
 563          */
 564         if (boot_cpu_has_bug(X86_BUG_F00F)) {
 565                 nr = (address - idt_descr.address) >> 3;
 566 
 567                 if (nr == 6) {
 568                         do_invalid_op(regs, 0);
 569                         return 1;
 570                 }
 571         }
 572 #endif
 573         return 0;
 574 }
 575 
 576 static void show_ldttss(const struct desc_ptr *gdt, const char *name, u16 index)
 577 {
 578         u32 offset = (index >> 3) * sizeof(struct desc_struct);
 579         unsigned long addr;
 580         struct ldttss_desc desc;
 581 
 582         if (index == 0) {
 583                 pr_alert("%s: NULL\n", name);
 584                 return;
 585         }
 586 
 587         if (offset + sizeof(struct ldttss_desc) >= gdt->size) {
 588                 pr_alert("%s: 0x%hx -- out of bounds\n", name, index);
 589                 return;
 590         }
 591 
 592         if (probe_kernel_read(&desc, (void *)(gdt->address + offset),
 593                               sizeof(struct ldttss_desc))) {
 594                 pr_alert("%s: 0x%hx -- GDT entry is not readable\n",
 595                          name, index);
 596                 return;
 597         }
 598 
 599         addr = desc.base0 | (desc.base1 << 16) | ((unsigned long)desc.base2 << 24);
 600 #ifdef CONFIG_X86_64
 601         addr |= ((u64)desc.base3 << 32);
 602 #endif
 603         pr_alert("%s: 0x%hx -- base=0x%lx limit=0x%x\n",
 604                  name, index, addr, (desc.limit0 | (desc.limit1 << 16)));
 605 }
 606 
 607 static void
 608 show_fault_oops(struct pt_regs *regs, unsigned long error_code, unsigned long address)
 609 {
 610         if (!oops_may_print())
 611                 return;
 612 
 613         if (error_code & X86_PF_INSTR) {
 614                 unsigned int level;
 615                 pgd_t *pgd;
 616                 pte_t *pte;
 617 
 618                 pgd = __va(read_cr3_pa());
 619                 pgd += pgd_index(address);
 620 
 621                 pte = lookup_address_in_pgd(pgd, address, &level);
 622 
 623                 if (pte && pte_present(*pte) && !pte_exec(*pte))
 624                         pr_crit("kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n",
 625                                 from_kuid(&init_user_ns, current_uid()));
 626                 if (pte && pte_present(*pte) && pte_exec(*pte) &&
 627                                 (pgd_flags(*pgd) & _PAGE_USER) &&
 628                                 (__read_cr4() & X86_CR4_SMEP))
 629                         pr_crit("unable to execute userspace code (SMEP?) (uid: %d)\n",
 630                                 from_kuid(&init_user_ns, current_uid()));
 631         }
 632 
 633         if (address < PAGE_SIZE && !user_mode(regs))
 634                 pr_alert("BUG: kernel NULL pointer dereference, address: %px\n",
 635                         (void *)address);
 636         else
 637                 pr_alert("BUG: unable to handle page fault for address: %px\n",
 638                         (void *)address);
 639 
 640         pr_alert("#PF: %s %s in %s mode\n",
 641                  (error_code & X86_PF_USER)  ? "user" : "supervisor",
 642                  (error_code & X86_PF_INSTR) ? "instruction fetch" :
 643                  (error_code & X86_PF_WRITE) ? "write access" :
 644                                                "read access",
 645                              user_mode(regs) ? "user" : "kernel");
 646         pr_alert("#PF: error_code(0x%04lx) - %s\n", error_code,
 647                  !(error_code & X86_PF_PROT) ? "not-present page" :
 648                  (error_code & X86_PF_RSVD)  ? "reserved bit violation" :
 649                  (error_code & X86_PF_PK)    ? "protection keys violation" :
 650                                                "permissions violation");
 651 
 652         if (!(error_code & X86_PF_USER) && user_mode(regs)) {
 653                 struct desc_ptr idt, gdt;
 654                 u16 ldtr, tr;
 655 
 656                 /*
 657                  * This can happen for quite a few reasons.  The more obvious
 658                  * ones are faults accessing the GDT, or LDT.  Perhaps
 659                  * surprisingly, if the CPU tries to deliver a benign or
 660                  * contributory exception from user code and gets a page fault
 661                  * during delivery, the page fault can be delivered as though
 662                  * it originated directly from user code.  This could happen
 663                  * due to wrong permissions on the IDT, GDT, LDT, TSS, or
 664                  * kernel or IST stack.
 665                  */
 666                 store_idt(&idt);
 667 
 668                 /* Usable even on Xen PV -- it's just slow. */
 669                 native_store_gdt(&gdt);
 670 
 671                 pr_alert("IDT: 0x%lx (limit=0x%hx) GDT: 0x%lx (limit=0x%hx)\n",
 672                          idt.address, idt.size, gdt.address, gdt.size);
 673 
 674                 store_ldt(ldtr);
 675                 show_ldttss(&gdt, "LDTR", ldtr);
 676 
 677                 store_tr(tr);
 678                 show_ldttss(&gdt, "TR", tr);
 679         }
 680 
 681         dump_pagetable(address);
 682 }
 683 
 684 static noinline void
 685 pgtable_bad(struct pt_regs *regs, unsigned long error_code,
 686             unsigned long address)
 687 {
 688         struct task_struct *tsk;
 689         unsigned long flags;
 690         int sig;
 691 
 692         flags = oops_begin();
 693         tsk = current;
 694         sig = SIGKILL;
 695 
 696         printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
 697                tsk->comm, address);
 698         dump_pagetable(address);
 699 
 700         if (__die("Bad pagetable", regs, error_code))
 701                 sig = 0;
 702 
 703         oops_end(flags, regs, sig);
 704 }
 705 
 706 static void set_signal_archinfo(unsigned long address,
 707                                 unsigned long error_code)
 708 {
 709         struct task_struct *tsk = current;
 710 
 711         /*
 712          * To avoid leaking information about the kernel page
 713          * table layout, pretend that user-mode accesses to
 714          * kernel addresses are always protection faults.
 715          *
 716          * NB: This means that failed vsyscalls with vsyscall=none
 717          * will have the PROT bit.  This doesn't leak any
 718          * information and does not appear to cause any problems.
 719          */
 720         if (address >= TASK_SIZE_MAX)
 721                 error_code |= X86_PF_PROT;
 722 
 723         tsk->thread.trap_nr = X86_TRAP_PF;
 724         tsk->thread.error_code = error_code | X86_PF_USER;
 725         tsk->thread.cr2 = address;
 726 }
 727 
 728 static noinline void
 729 no_context(struct pt_regs *regs, unsigned long error_code,
 730            unsigned long address, int signal, int si_code)
 731 {
 732         struct task_struct *tsk = current;
 733         unsigned long flags;
 734         int sig;
 735 
 736         if (user_mode(regs)) {
 737                 /*
 738                  * This is an implicit supervisor-mode access from user
 739                  * mode.  Bypass all the kernel-mode recovery code and just
 740                  * OOPS.
 741                  */
 742                 goto oops;
 743         }
 744 
 745         /* Are we prepared to handle this kernel fault? */
 746         if (fixup_exception(regs, X86_TRAP_PF, error_code, address)) {
 747                 /*
 748                  * Any interrupt that takes a fault gets the fixup. This makes
 749                  * the below recursive fault logic only apply to a faults from
 750                  * task context.
 751                  */
 752                 if (in_interrupt())
 753                         return;
 754 
 755                 /*
 756                  * Per the above we're !in_interrupt(), aka. task context.
 757                  *
 758                  * In this case we need to make sure we're not recursively
 759                  * faulting through the emulate_vsyscall() logic.
 760                  */
 761                 if (current->thread.sig_on_uaccess_err && signal) {
 762                         set_signal_archinfo(address, error_code);
 763 
 764                         /* XXX: hwpoison faults will set the wrong code. */
 765                         force_sig_fault(signal, si_code, (void __user *)address);
 766                 }
 767 
 768                 /*
 769                  * Barring that, we can do the fixup and be happy.
 770                  */
 771                 return;
 772         }
 773 
 774 #ifdef CONFIG_VMAP_STACK
 775         /*
 776          * Stack overflow?  During boot, we can fault near the initial
 777          * stack in the direct map, but that's not an overflow -- check
 778          * that we're in vmalloc space to avoid this.
 779          */
 780         if (is_vmalloc_addr((void *)address) &&
 781             (((unsigned long)tsk->stack - 1 - address < PAGE_SIZE) ||
 782              address - ((unsigned long)tsk->stack + THREAD_SIZE) < PAGE_SIZE)) {
 783                 unsigned long stack = __this_cpu_ist_top_va(DF) - sizeof(void *);
 784                 /*
 785                  * We're likely to be running with very little stack space
 786                  * left.  It's plausible that we'd hit this condition but
 787                  * double-fault even before we get this far, in which case
 788                  * we're fine: the double-fault handler will deal with it.
 789                  *
 790                  * We don't want to make it all the way into the oops code
 791                  * and then double-fault, though, because we're likely to
 792                  * break the console driver and lose most of the stack dump.
 793                  */
 794                 asm volatile ("movq %[stack], %%rsp\n\t"
 795                               "call handle_stack_overflow\n\t"
 796                               "1: jmp 1b"
 797                               : ASM_CALL_CONSTRAINT
 798                               : "D" ("kernel stack overflow (page fault)"),
 799                                 "S" (regs), "d" (address),
 800                                 [stack] "rm" (stack));
 801                 unreachable();
 802         }
 803 #endif
 804 
 805         /*
 806          * 32-bit:
 807          *
 808          *   Valid to do another page fault here, because if this fault
 809          *   had been triggered by is_prefetch fixup_exception would have
 810          *   handled it.
 811          *
 812          * 64-bit:
 813          *
 814          *   Hall of shame of CPU/BIOS bugs.
 815          */
 816         if (is_prefetch(regs, error_code, address))
 817                 return;
 818 
 819         if (is_errata93(regs, address))
 820                 return;
 821 
 822         /*
 823          * Buggy firmware could access regions which might page fault, try to
 824          * recover from such faults.
 825          */
 826         if (IS_ENABLED(CONFIG_EFI))
 827                 efi_recover_from_page_fault(address);
 828 
 829 oops:
 830         /*
 831          * Oops. The kernel tried to access some bad page. We'll have to
 832          * terminate things with extreme prejudice:
 833          */
 834         flags = oops_begin();
 835 
 836         show_fault_oops(regs, error_code, address);
 837 
 838         if (task_stack_end_corrupted(tsk))
 839                 printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
 840 
 841         sig = SIGKILL;
 842         if (__die("Oops", regs, error_code))
 843                 sig = 0;
 844 
 845         /* Executive summary in case the body of the oops scrolled away */
 846         printk(KERN_DEFAULT "CR2: %016lx\n", address);
 847 
 848         oops_end(flags, regs, sig);
 849 }
 850 
 851 /*
 852  * Print out info about fatal segfaults, if the show_unhandled_signals
 853  * sysctl is set:
 854  */
 855 static inline void
 856 show_signal_msg(struct pt_regs *regs, unsigned long error_code,
 857                 unsigned long address, struct task_struct *tsk)
 858 {
 859         const char *loglvl = task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG;
 860 
 861         if (!unhandled_signal(tsk, SIGSEGV))
 862                 return;
 863 
 864         if (!printk_ratelimit())
 865                 return;
 866 
 867         printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx",
 868                 loglvl, tsk->comm, task_pid_nr(tsk), address,
 869                 (void *)regs->ip, (void *)regs->sp, error_code);
 870 
 871         print_vma_addr(KERN_CONT " in ", regs->ip);
 872 
 873         printk(KERN_CONT "\n");
 874 
 875         show_opcodes(regs, loglvl);
 876 }
 877 
 878 /*
 879  * The (legacy) vsyscall page is the long page in the kernel portion
 880  * of the address space that has user-accessible permissions.
 881  */
 882 static bool is_vsyscall_vaddr(unsigned long vaddr)
 883 {
 884         return unlikely((vaddr & PAGE_MASK) == VSYSCALL_ADDR);
 885 }
 886 
 887 static void
 888 __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 889                        unsigned long address, u32 pkey, int si_code)
 890 {
 891         struct task_struct *tsk = current;
 892 
 893         /* User mode accesses just cause a SIGSEGV */
 894         if (user_mode(regs) && (error_code & X86_PF_USER)) {
 895                 /*
 896                  * It's possible to have interrupts off here:
 897                  */
 898                 local_irq_enable();
 899 
 900                 /*
 901                  * Valid to do another page fault here because this one came
 902                  * from user space:
 903                  */
 904                 if (is_prefetch(regs, error_code, address))
 905                         return;
 906 
 907                 if (is_errata100(regs, address))
 908                         return;
 909 
 910                 /*
 911                  * To avoid leaking information about the kernel page table
 912                  * layout, pretend that user-mode accesses to kernel addresses
 913                  * are always protection faults.
 914                  */
 915                 if (address >= TASK_SIZE_MAX)
 916                         error_code |= X86_PF_PROT;
 917 
 918                 if (likely(show_unhandled_signals))
 919                         show_signal_msg(regs, error_code, address, tsk);
 920 
 921                 set_signal_archinfo(address, error_code);
 922 
 923                 if (si_code == SEGV_PKUERR)
 924                         force_sig_pkuerr((void __user *)address, pkey);
 925 
 926                 force_sig_fault(SIGSEGV, si_code, (void __user *)address);
 927 
 928                 return;
 929         }
 930 
 931         if (is_f00f_bug(regs, address))
 932                 return;
 933 
 934         no_context(regs, error_code, address, SIGSEGV, si_code);
 935 }
 936 
 937 static noinline void
 938 bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 939                      unsigned long address)
 940 {
 941         __bad_area_nosemaphore(regs, error_code, address, 0, SEGV_MAPERR);
 942 }
 943 
 944 static void
 945 __bad_area(struct pt_regs *regs, unsigned long error_code,
 946            unsigned long address, u32 pkey, int si_code)
 947 {
 948         struct mm_struct *mm = current->mm;
 949         /*
 950          * Something tried to access memory that isn't in our memory map..
 951          * Fix it, but check if it's kernel or user first..
 952          */
 953         up_read(&mm->mmap_sem);
 954 
 955         __bad_area_nosemaphore(regs, error_code, address, pkey, si_code);
 956 }
 957 
 958 static noinline void
 959 bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
 960 {
 961         __bad_area(regs, error_code, address, 0, SEGV_MAPERR);
 962 }
 963 
 964 static inline bool bad_area_access_from_pkeys(unsigned long error_code,
 965                 struct vm_area_struct *vma)
 966 {
 967         /* This code is always called on the current mm */
 968         bool foreign = false;
 969 
 970         if (!boot_cpu_has(X86_FEATURE_OSPKE))
 971                 return false;
 972         if (error_code & X86_PF_PK)
 973                 return true;
 974         /* this checks permission keys on the VMA: */
 975         if (!arch_vma_access_permitted(vma, (error_code & X86_PF_WRITE),
 976                                        (error_code & X86_PF_INSTR), foreign))
 977                 return true;
 978         return false;
 979 }
 980 
 981 static noinline void
 982 bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
 983                       unsigned long address, struct vm_area_struct *vma)
 984 {
 985         /*
 986          * This OSPKE check is not strictly necessary at runtime.
 987          * But, doing it this way allows compiler optimizations
 988          * if pkeys are compiled out.
 989          */
 990         if (bad_area_access_from_pkeys(error_code, vma)) {
 991                 /*
 992                  * A protection key fault means that the PKRU value did not allow
 993                  * access to some PTE.  Userspace can figure out what PKRU was
 994                  * from the XSAVE state.  This function captures the pkey from
 995                  * the vma and passes it to userspace so userspace can discover
 996                  * which protection key was set on the PTE.
 997                  *
 998                  * If we get here, we know that the hardware signaled a X86_PF_PK
 999                  * fault and that there was a VMA once we got in the fault
1000                  * handler.  It does *not* guarantee that the VMA we find here
1001                  * was the one that we faulted on.
1002                  *
1003                  * 1. T1   : mprotect_key(foo, PAGE_SIZE, pkey=4);
1004                  * 2. T1   : set PKRU to deny access to pkey=4, touches page
1005                  * 3. T1   : faults...
1006                  * 4.    T2: mprotect_key(foo, PAGE_SIZE, pkey=5);
1007                  * 5. T1   : enters fault handler, takes mmap_sem, etc...
1008                  * 6. T1   : reaches here, sees vma_pkey(vma)=5, when we really
1009                  *           faulted on a pte with its pkey=4.
1010                  */
1011                 u32 pkey = vma_pkey(vma);
1012 
1013                 __bad_area(regs, error_code, address, pkey, SEGV_PKUERR);
1014         } else {
1015                 __bad_area(regs, error_code, address, 0, SEGV_ACCERR);
1016         }
1017 }
1018 
1019 static void
1020 do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
1021           vm_fault_t fault)
1022 {
1023         /* Kernel mode? Handle exceptions or die: */
1024         if (!(error_code & X86_PF_USER)) {
1025                 no_context(regs, error_code, address, SIGBUS, BUS_ADRERR);
1026                 return;
1027         }
1028 
1029         /* User-space => ok to do another page fault: */
1030         if (is_prefetch(regs, error_code, address))
1031                 return;
1032 
1033         set_signal_archinfo(address, error_code);
1034 
1035 #ifdef CONFIG_MEMORY_FAILURE
1036         if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
1037                 struct task_struct *tsk = current;
1038                 unsigned lsb = 0;
1039 
1040                 pr_err(
1041         "MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
1042                         tsk->comm, tsk->pid, address);
1043                 if (fault & VM_FAULT_HWPOISON_LARGE)
1044                         lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
1045                 if (fault & VM_FAULT_HWPOISON)
1046                         lsb = PAGE_SHIFT;
1047                 force_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb);
1048                 return;
1049         }
1050 #endif
1051         force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
1052 }
1053 
1054 static noinline void
1055 mm_fault_error(struct pt_regs *regs, unsigned long error_code,
1056                unsigned long address, vm_fault_t fault)
1057 {
1058         if (fatal_signal_pending(current) && !(error_code & X86_PF_USER)) {
1059                 no_context(regs, error_code, address, 0, 0);
1060                 return;
1061         }
1062 
1063         if (fault & VM_FAULT_OOM) {
1064                 /* Kernel mode? Handle exceptions or die: */
1065                 if (!(error_code & X86_PF_USER)) {
1066                         no_context(regs, error_code, address,
1067                                    SIGSEGV, SEGV_MAPERR);
1068                         return;
1069                 }
1070 
1071                 /*
1072                  * We ran out of memory, call the OOM killer, and return the
1073                  * userspace (which will retry the fault, or kill us if we got
1074                  * oom-killed):
1075                  */
1076                 pagefault_out_of_memory();
1077         } else {
1078                 if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
1079                              VM_FAULT_HWPOISON_LARGE))
1080                         do_sigbus(regs, error_code, address, fault);
1081                 else if (fault & VM_FAULT_SIGSEGV)
1082                         bad_area_nosemaphore(regs, error_code, address);
1083                 else
1084                         BUG();
1085         }
1086 }
1087 
1088 static int spurious_kernel_fault_check(unsigned long error_code, pte_t *pte)
1089 {
1090         if ((error_code & X86_PF_WRITE) && !pte_write(*pte))
1091                 return 0;
1092 
1093         if ((error_code & X86_PF_INSTR) && !pte_exec(*pte))
1094                 return 0;
1095 
1096         return 1;
1097 }
1098 
1099 /*
1100  * Handle a spurious fault caused by a stale TLB entry.
1101  *
1102  * This allows us to lazily refresh the TLB when increasing the
1103  * permissions of a kernel page (RO -> RW or NX -> X).  Doing it
1104  * eagerly is very expensive since that implies doing a full
1105  * cross-processor TLB flush, even if no stale TLB entries exist
1106  * on other processors.
1107  *
1108  * Spurious faults may only occur if the TLB contains an entry with
1109  * fewer permission than the page table entry.  Non-present (P = 0)
1110  * and reserved bit (R = 1) faults are never spurious.
1111  *
1112  * There are no security implications to leaving a stale TLB when
1113  * increasing the permissions on a page.
1114  *
1115  * Returns non-zero if a spurious fault was handled, zero otherwise.
1116  *
1117  * See Intel Developer's Manual Vol 3 Section 4.10.4.3, bullet 3
1118  * (Optional Invalidation).
1119  */
1120 static noinline int
1121 spurious_kernel_fault(unsigned long error_code, unsigned long address)
1122 {
1123         pgd_t *pgd;
1124         p4d_t *p4d;
1125         pud_t *pud;
1126         pmd_t *pmd;
1127         pte_t *pte;
1128         int ret;
1129 
1130         /*
1131          * Only writes to RO or instruction fetches from NX may cause
1132          * spurious faults.
1133          *
1134          * These could be from user or supervisor accesses but the TLB
1135          * is only lazily flushed after a kernel mapping protection
1136          * change, so user accesses are not expected to cause spurious
1137          * faults.
1138          */
1139         if (error_code != (X86_PF_WRITE | X86_PF_PROT) &&
1140             error_code != (X86_PF_INSTR | X86_PF_PROT))
1141                 return 0;
1142 
1143         pgd = init_mm.pgd + pgd_index(address);
1144         if (!pgd_present(*pgd))
1145                 return 0;
1146 
1147         p4d = p4d_offset(pgd, address);
1148         if (!p4d_present(*p4d))
1149                 return 0;
1150 
1151         if (p4d_large(*p4d))
1152                 return spurious_kernel_fault_check(error_code, (pte_t *) p4d);
1153 
1154         pud = pud_offset(p4d, address);
1155         if (!pud_present(*pud))
1156                 return 0;
1157 
1158         if (pud_large(*pud))
1159                 return spurious_kernel_fault_check(error_code, (pte_t *) pud);
1160 
1161         pmd = pmd_offset(pud, address);
1162         if (!pmd_present(*pmd))
1163                 return 0;
1164 
1165         if (pmd_large(*pmd))
1166                 return spurious_kernel_fault_check(error_code, (pte_t *) pmd);
1167 
1168         pte = pte_offset_kernel(pmd, address);
1169         if (!pte_present(*pte))
1170                 return 0;
1171 
1172         ret = spurious_kernel_fault_check(error_code, pte);
1173         if (!ret)
1174                 return 0;
1175 
1176         /*
1177          * Make sure we have permissions in PMD.
1178          * If not, then there's a bug in the page tables:
1179          */
1180         ret = spurious_kernel_fault_check(error_code, (pte_t *) pmd);
1181         WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
1182 
1183         return ret;
1184 }
1185 NOKPROBE_SYMBOL(spurious_kernel_fault);
1186 
1187 int show_unhandled_signals = 1;
1188 
1189 static inline int
1190 access_error(unsigned long error_code, struct vm_area_struct *vma)
1191 {
1192         /* This is only called for the current mm, so: */
1193         bool foreign = false;
1194 
1195         /*
1196          * Read or write was blocked by protection keys.  This is
1197          * always an unconditional error and can never result in
1198          * a follow-up action to resolve the fault, like a COW.
1199          */
1200         if (error_code & X86_PF_PK)
1201                 return 1;
1202 
1203         /*
1204          * Make sure to check the VMA so that we do not perform
1205          * faults just to hit a X86_PF_PK as soon as we fill in a
1206          * page.
1207          */
1208         if (!arch_vma_access_permitted(vma, (error_code & X86_PF_WRITE),
1209                                        (error_code & X86_PF_INSTR), foreign))
1210                 return 1;
1211 
1212         if (error_code & X86_PF_WRITE) {
1213                 /* write, present and write, not present: */
1214                 if (unlikely(!(vma->vm_flags & VM_WRITE)))
1215                         return 1;
1216                 return 0;
1217         }
1218 
1219         /* read, present: */
1220         if (unlikely(error_code & X86_PF_PROT))
1221                 return 1;
1222 
1223         /* read, not present: */
1224         if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
1225                 return 1;
1226 
1227         return 0;
1228 }
1229 
1230 static int fault_in_kernel_space(unsigned long address)
1231 {
1232         /*
1233          * On 64-bit systems, the vsyscall page is at an address above
1234          * TASK_SIZE_MAX, but is not considered part of the kernel
1235          * address space.
1236          */
1237         if (IS_ENABLED(CONFIG_X86_64) && is_vsyscall_vaddr(address))
1238                 return false;
1239 
1240         return address >= TASK_SIZE_MAX;
1241 }
1242 
1243 /*
1244  * Called for all faults where 'address' is part of the kernel address
1245  * space.  Might get called for faults that originate from *code* that
1246  * ran in userspace or the kernel.
1247  */
1248 static void
1249 do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code,
1250                    unsigned long address)
1251 {
1252         /*
1253          * Protection keys exceptions only happen on user pages.  We
1254          * have no user pages in the kernel portion of the address
1255          * space, so do not expect them here.
1256          */
1257         WARN_ON_ONCE(hw_error_code & X86_PF_PK);
1258 
1259         /*
1260          * We can fault-in kernel-space virtual memory on-demand. The
1261          * 'reference' page table is init_mm.pgd.
1262          *
1263          * NOTE! We MUST NOT take any locks for this case. We may
1264          * be in an interrupt or a critical region, and should
1265          * only copy the information from the master page table,
1266          * nothing more.
1267          *
1268          * Before doing this on-demand faulting, ensure that the
1269          * fault is not any of the following:
1270          * 1. A fault on a PTE with a reserved bit set.
1271          * 2. A fault caused by a user-mode access.  (Do not demand-
1272          *    fault kernel memory due to user-mode accesses).
1273          * 3. A fault caused by a page-level protection violation.
1274          *    (A demand fault would be on a non-present page which
1275          *     would have X86_PF_PROT==0).
1276          */
1277         if (!(hw_error_code & (X86_PF_RSVD | X86_PF_USER | X86_PF_PROT))) {
1278                 if (vmalloc_fault(address) >= 0)
1279                         return;
1280         }
1281 
1282         /* Was the fault spurious, caused by lazy TLB invalidation? */
1283         if (spurious_kernel_fault(hw_error_code, address))
1284                 return;
1285 
1286         /* kprobes don't want to hook the spurious faults: */
1287         if (kprobe_page_fault(regs, X86_TRAP_PF))
1288                 return;
1289 
1290         /*
1291          * Note, despite being a "bad area", there are quite a few
1292          * acceptable reasons to get here, such as erratum fixups
1293          * and handling kernel code that can fault, like get_user().
1294          *
1295          * Don't take the mm semaphore here. If we fixup a prefetch
1296          * fault we could otherwise deadlock:
1297          */
1298         bad_area_nosemaphore(regs, hw_error_code, address);
1299 }
1300 NOKPROBE_SYMBOL(do_kern_addr_fault);
1301 
1302 /* Handle faults in the user portion of the address space */
1303 static inline
1304 void do_user_addr_fault(struct pt_regs *regs,
1305                         unsigned long hw_error_code,
1306                         unsigned long address)
1307 {
1308         struct vm_area_struct *vma;
1309         struct task_struct *tsk;
1310         struct mm_struct *mm;
1311         vm_fault_t fault, major = 0;
1312         unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
1313 
1314         tsk = current;
1315         mm = tsk->mm;
1316 
1317         /* kprobes don't want to hook the spurious faults: */
1318         if (unlikely(kprobe_page_fault(regs, X86_TRAP_PF)))
1319                 return;
1320 
1321         /*
1322          * Reserved bits are never expected to be set on
1323          * entries in the user portion of the page tables.
1324          */
1325         if (unlikely(hw_error_code & X86_PF_RSVD))
1326                 pgtable_bad(regs, hw_error_code, address);
1327 
1328         /*
1329          * If SMAP is on, check for invalid kernel (supervisor) access to user
1330          * pages in the user address space.  The odd case here is WRUSS,
1331          * which, according to the preliminary documentation, does not respect
1332          * SMAP and will have the USER bit set so, in all cases, SMAP
1333          * enforcement appears to be consistent with the USER bit.
1334          */
1335         if (unlikely(cpu_feature_enabled(X86_FEATURE_SMAP) &&
1336                      !(hw_error_code & X86_PF_USER) &&
1337                      !(regs->flags & X86_EFLAGS_AC)))
1338         {
1339                 bad_area_nosemaphore(regs, hw_error_code, address);
1340                 return;
1341         }
1342 
1343         /*
1344          * If we're in an interrupt, have no user context or are running
1345          * in a region with pagefaults disabled then we must not take the fault
1346          */
1347         if (unlikely(faulthandler_disabled() || !mm)) {
1348                 bad_area_nosemaphore(regs, hw_error_code, address);
1349                 return;
1350         }
1351 
1352         /*
1353          * It's safe to allow irq's after cr2 has been saved and the
1354          * vmalloc fault has been handled.
1355          *
1356          * User-mode registers count as a user access even for any
1357          * potential system fault or CPU buglet:
1358          */
1359         if (user_mode(regs)) {
1360                 local_irq_enable();
1361                 flags |= FAULT_FLAG_USER;
1362         } else {
1363                 if (regs->flags & X86_EFLAGS_IF)
1364                         local_irq_enable();
1365         }
1366 
1367         perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
1368 
1369         if (hw_error_code & X86_PF_WRITE)
1370                 flags |= FAULT_FLAG_WRITE;
1371         if (hw_error_code & X86_PF_INSTR)
1372                 flags |= FAULT_FLAG_INSTRUCTION;
1373 
1374 #ifdef CONFIG_X86_64
1375         /*
1376          * Faults in the vsyscall page might need emulation.  The
1377          * vsyscall page is at a high address (>PAGE_OFFSET), but is
1378          * considered to be part of the user address space.
1379          *
1380          * The vsyscall page does not have a "real" VMA, so do this
1381          * emulation before we go searching for VMAs.
1382          *
1383          * PKRU never rejects instruction fetches, so we don't need
1384          * to consider the PF_PK bit.
1385          */
1386         if (is_vsyscall_vaddr(address)) {
1387                 if (emulate_vsyscall(hw_error_code, regs, address))
1388                         return;
1389         }
1390 #endif
1391 
1392         /*
1393          * Kernel-mode access to the user address space should only occur
1394          * on well-defined single instructions listed in the exception
1395          * tables.  But, an erroneous kernel fault occurring outside one of
1396          * those areas which also holds mmap_sem might deadlock attempting
1397          * to validate the fault against the address space.
1398          *
1399          * Only do the expensive exception table search when we might be at
1400          * risk of a deadlock.  This happens if we
1401          * 1. Failed to acquire mmap_sem, and
1402          * 2. The access did not originate in userspace.
1403          */
1404         if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
1405                 if (!user_mode(regs) && !search_exception_tables(regs->ip)) {
1406                         /*
1407                          * Fault from code in kernel from
1408                          * which we do not expect faults.
1409                          */
1410                         bad_area_nosemaphore(regs, hw_error_code, address);
1411                         return;
1412                 }
1413 retry:
1414                 down_read(&mm->mmap_sem);
1415         } else {
1416                 /*
1417                  * The above down_read_trylock() might have succeeded in
1418                  * which case we'll have missed the might_sleep() from
1419                  * down_read():
1420                  */
1421                 might_sleep();
1422         }
1423 
1424         vma = find_vma(mm, address);
1425         if (unlikely(!vma)) {
1426                 bad_area(regs, hw_error_code, address);
1427                 return;
1428         }
1429         if (likely(vma->vm_start <= address))
1430                 goto good_area;
1431         if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
1432                 bad_area(regs, hw_error_code, address);
1433                 return;
1434         }
1435         if (unlikely(expand_stack(vma, address))) {
1436                 bad_area(regs, hw_error_code, address);
1437                 return;
1438         }
1439 
1440         /*
1441          * Ok, we have a good vm_area for this memory access, so
1442          * we can handle it..
1443          */
1444 good_area:
1445         if (unlikely(access_error(hw_error_code, vma))) {
1446                 bad_area_access_error(regs, hw_error_code, address, vma);
1447                 return;
1448         }
1449 
1450         /*
1451          * If for any reason at all we couldn't handle the fault,
1452          * make sure we exit gracefully rather than endlessly redo
1453          * the fault.  Since we never set FAULT_FLAG_RETRY_NOWAIT, if
1454          * we get VM_FAULT_RETRY back, the mmap_sem has been unlocked.
1455          *
1456          * Note that handle_userfault() may also release and reacquire mmap_sem
1457          * (and not return with VM_FAULT_RETRY), when returning to userland to
1458          * repeat the page fault later with a VM_FAULT_NOPAGE retval
1459          * (potentially after handling any pending signal during the return to
1460          * userland). The return to userland is identified whenever
1461          * FAULT_FLAG_USER|FAULT_FLAG_KILLABLE are both set in flags.
1462          */
1463         fault = handle_mm_fault(vma, address, flags);
1464         major |= fault & VM_FAULT_MAJOR;
1465 
1466         /*
1467          * If we need to retry the mmap_sem has already been released,
1468          * and if there is a fatal signal pending there is no guarantee
1469          * that we made any progress. Handle this case first.
1470          */
1471         if (unlikely(fault & VM_FAULT_RETRY)) {
1472                 /* Retry at most once */
1473                 if (flags & FAULT_FLAG_ALLOW_RETRY) {
1474                         flags &= ~FAULT_FLAG_ALLOW_RETRY;
1475                         flags |= FAULT_FLAG_TRIED;
1476                         if (!fatal_signal_pending(tsk))
1477                                 goto retry;
1478                 }
1479 
1480                 /* User mode? Just return to handle the fatal exception */
1481                 if (flags & FAULT_FLAG_USER)
1482                         return;
1483 
1484                 /* Not returning to user mode? Handle exceptions or die: */
1485                 no_context(regs, hw_error_code, address, SIGBUS, BUS_ADRERR);
1486                 return;
1487         }
1488 
1489         up_read(&mm->mmap_sem);
1490         if (unlikely(fault & VM_FAULT_ERROR)) {
1491                 mm_fault_error(regs, hw_error_code, address, fault);
1492                 return;
1493         }
1494 
1495         /*
1496          * Major/minor page fault accounting. If any of the events
1497          * returned VM_FAULT_MAJOR, we account it as a major fault.
1498          */
1499         if (major) {
1500                 tsk->maj_flt++;
1501                 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
1502         } else {
1503                 tsk->min_flt++;
1504                 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
1505         }
1506 
1507         check_v8086_mode(regs, address, tsk);
1508 }
1509 NOKPROBE_SYMBOL(do_user_addr_fault);
1510 
1511 /*
1512  * Explicitly marked noinline such that the function tracer sees this as the
1513  * page_fault entry point.
1514  */
1515 static noinline void
1516 __do_page_fault(struct pt_regs *regs, unsigned long hw_error_code,
1517                 unsigned long address)
1518 {
1519         prefetchw(&current->mm->mmap_sem);
1520 
1521         if (unlikely(kmmio_fault(regs, address)))
1522                 return;
1523 
1524         /* Was the fault on kernel-controlled part of the address space? */
1525         if (unlikely(fault_in_kernel_space(address)))
1526                 do_kern_addr_fault(regs, hw_error_code, address);
1527         else
1528                 do_user_addr_fault(regs, hw_error_code, address);
1529 }
1530 NOKPROBE_SYMBOL(__do_page_fault);
1531 
1532 static __always_inline void
1533 trace_page_fault_entries(struct pt_regs *regs, unsigned long error_code,
1534                          unsigned long address)
1535 {
1536         if (!trace_pagefault_enabled())
1537                 return;
1538 
1539         if (user_mode(regs))
1540                 trace_page_fault_user(address, regs, error_code);
1541         else
1542                 trace_page_fault_kernel(address, regs, error_code);
1543 }
1544 
1545 dotraplinkage void
1546 do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
1547 {
1548         enum ctx_state prev_state;
1549 
1550         prev_state = exception_enter();
1551         trace_page_fault_entries(regs, error_code, address);
1552         __do_page_fault(regs, error_code, address);
1553         exception_exit(prev_state);
1554 }
1555 NOKPROBE_SYMBOL(do_page_fault);
/* [<][>][^][v][top][bottom][index][help] */
root/arch/x86/mm/fault.c

DEFINITIONS
