1/* 2 * Copyright (C) 2008, 2009 Intel Corporation 3 * Authors: Andi Kleen, Fengguang Wu 4 * 5 * This software may be redistributed and/or modified under the terms of 6 * the GNU General Public License ("GPL") version 2 only as published by the 7 * Free Software Foundation. 8 * 9 * High level machine check handler. Handles pages reported by the 10 * hardware as being corrupted usually due to a multi-bit ECC memory or cache 11 * failure. 12 * 13 * In addition there is a "soft offline" entry point that allows stop using 14 * not-yet-corrupted-by-suspicious pages without killing anything. 15 * 16 * Handles page cache pages in various states. The tricky part 17 * here is that we can access any page asynchronously in respect to 18 * other VM users, because memory failures could happen anytime and 19 * anywhere. This could violate some of their assumptions. This is why 20 * this code has to be extremely careful. Generally it tries to use 21 * normal locking rules, as in get the standard locks, even if that means 22 * the error handling takes potentially a long time. 23 * 24 * There are several operations here with exponential complexity because 25 * of unsuitable VM data structures. For example the operation to map back 26 * from RMAP chains to processes has to walk the complete process list and 27 * has non linear complexity with the number. But since memory corruptions 28 * are rare we hope to get away with this. This avoids impacting the core 29 * VM. 30 */ 31 32/* 33 * Notebook: 34 * - hugetlb needs more code 35 * - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages 36 * - pass bad pages to kdump next kernel 37 */ 38#include <linux/kernel.h> 39#include <linux/mm.h> 40#include <linux/page-flags.h> 41#include <linux/kernel-page-flags.h> 42#include <linux/sched.h> 43#include <linux/ksm.h> 44#include <linux/rmap.h> 45#include <linux/export.h> 46#include <linux/pagemap.h> 47#include <linux/swap.h> 48#include <linux/backing-dev.h> 49#include <linux/migrate.h> 50#include <linux/page-isolation.h> 51#include <linux/suspend.h> 52#include <linux/slab.h> 53#include <linux/swapops.h> 54#include <linux/hugetlb.h> 55#include <linux/memory_hotplug.h> 56#include <linux/mm_inline.h> 57#include <linux/kfifo.h> 58#include "internal.h" 59 60int sysctl_memory_failure_early_kill __read_mostly = 0; 61 62int sysctl_memory_failure_recovery __read_mostly = 1; 63 64atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); 65 66#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE) 67 68u32 hwpoison_filter_enable = 0; 69u32 hwpoison_filter_dev_major = ~0U; 70u32 hwpoison_filter_dev_minor = ~0U; 71u64 hwpoison_filter_flags_mask; 72u64 hwpoison_filter_flags_value; 73EXPORT_SYMBOL_GPL(hwpoison_filter_enable); 74EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major); 75EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor); 76EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask); 77EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value); 78 79static int hwpoison_filter_dev(struct page *p) 80{ 81 struct address_space *mapping; 82 dev_t dev; 83 84 if (hwpoison_filter_dev_major == ~0U && 85 hwpoison_filter_dev_minor == ~0U) 86 return 0; 87 88 /* 89 * page_mapping() does not accept slab pages. 90 */ 91 if (PageSlab(p)) 92 return -EINVAL; 93 94 mapping = page_mapping(p); 95 if (mapping == NULL || mapping->host == NULL) 96 return -EINVAL; 97 98 dev = mapping->host->i_sb->s_dev; 99 if (hwpoison_filter_dev_major != ~0U && 100 hwpoison_filter_dev_major != MAJOR(dev)) 101 return -EINVAL; 102 if (hwpoison_filter_dev_minor != ~0U && 103 hwpoison_filter_dev_minor != MINOR(dev)) 104 return -EINVAL; 105 106 return 0; 107} 108 109static int hwpoison_filter_flags(struct page *p) 110{ 111 if (!hwpoison_filter_flags_mask) 112 return 0; 113 114 if ((stable_page_flags(p) & hwpoison_filter_flags_mask) == 115 hwpoison_filter_flags_value) 116 return 0; 117 else 118 return -EINVAL; 119} 120 121/* 122 * This allows stress tests to limit test scope to a collection of tasks 123 * by putting them under some memcg. This prevents killing unrelated/important 124 * processes such as /sbin/init. Note that the target task may share clean 125 * pages with init (eg. libc text), which is harmless. If the target task 126 * share _dirty_ pages with another task B, the test scheme must make sure B 127 * is also included in the memcg. At last, due to race conditions this filter 128 * can only guarantee that the page either belongs to the memcg tasks, or is 129 * a freed page. 130 */ 131#ifdef CONFIG_MEMCG_SWAP 132u64 hwpoison_filter_memcg; 133EXPORT_SYMBOL_GPL(hwpoison_filter_memcg); 134static int hwpoison_filter_task(struct page *p) 135{ 136 struct mem_cgroup *mem; 137 struct cgroup_subsys_state *css; 138 unsigned long ino; 139 140 if (!hwpoison_filter_memcg) 141 return 0; 142 143 mem = try_get_mem_cgroup_from_page(p); 144 if (!mem) 145 return -EINVAL; 146 147 css = mem_cgroup_css(mem); 148 ino = cgroup_ino(css->cgroup); 149 css_put(css); 150 151 if (ino != hwpoison_filter_memcg) 152 return -EINVAL; 153 154 return 0; 155} 156#else 157static int hwpoison_filter_task(struct page *p) { return 0; } 158#endif 159 160int hwpoison_filter(struct page *p) 161{ 162 if (!hwpoison_filter_enable) 163 return 0; 164 165 if (hwpoison_filter_dev(p)) 166 return -EINVAL; 167 168 if (hwpoison_filter_flags(p)) 169 return -EINVAL; 170 171 if (hwpoison_filter_task(p)) 172 return -EINVAL; 173 174 return 0; 175} 176#else 177int hwpoison_filter(struct page *p) 178{ 179 return 0; 180} 181#endif 182 183EXPORT_SYMBOL_GPL(hwpoison_filter); 184 185/* 186 * Send all the processes who have the page mapped a signal. 187 * ``action optional'' if they are not immediately affected by the error 188 * ``action required'' if error happened in current execution context 189 */ 190static int kill_proc(struct task_struct *t, unsigned long addr, int trapno, 191 unsigned long pfn, struct page *page, int flags) 192{ 193 struct siginfo si; 194 int ret; 195 196 printk(KERN_ERR 197 "MCE %#lx: Killing %s:%d due to hardware memory corruption\n", 198 pfn, t->comm, t->pid); 199 si.si_signo = SIGBUS; 200 si.si_errno = 0; 201 si.si_addr = (void *)addr; 202#ifdef __ARCH_SI_TRAPNO 203 si.si_trapno = trapno; 204#endif 205 si.si_addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT; 206 207 if ((flags & MF_ACTION_REQUIRED) && t->mm == current->mm) { 208 si.si_code = BUS_MCEERR_AR; 209 ret = force_sig_info(SIGBUS, &si, current); 210 } else { 211 /* 212 * Don't use force here, it's convenient if the signal 213 * can be temporarily blocked. 214 * This could cause a loop when the user sets SIGBUS 215 * to SIG_IGN, but hopefully no one will do that? 216 */ 217 si.si_code = BUS_MCEERR_AO; 218 ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */ 219 } 220 if (ret < 0) 221 printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n", 222 t->comm, t->pid, ret); 223 return ret; 224} 225 226/* 227 * When a unknown page type is encountered drain as many buffers as possible 228 * in the hope to turn the page into a LRU or free page, which we can handle. 229 */ 230void shake_page(struct page *p, int access) 231{ 232 if (!PageSlab(p)) { 233 lru_add_drain_all(); 234 if (PageLRU(p)) 235 return; 236 drain_all_pages(page_zone(p)); 237 if (PageLRU(p) || is_free_buddy_page(p)) 238 return; 239 } 240 241 /* 242 * Only call shrink_node_slabs here (which would also shrink 243 * other caches) if access is not potentially fatal. 244 */ 245 if (access) 246 drop_slab_node(page_to_nid(p)); 247} 248EXPORT_SYMBOL_GPL(shake_page); 249 250/* 251 * Kill all processes that have a poisoned page mapped and then isolate 252 * the page. 253 * 254 * General strategy: 255 * Find all processes having the page mapped and kill them. 256 * But we keep a page reference around so that the page is not 257 * actually freed yet. 258 * Then stash the page away 259 * 260 * There's no convenient way to get back to mapped processes 261 * from the VMAs. So do a brute-force search over all 262 * running processes. 263 * 264 * Remember that machine checks are not common (or rather 265 * if they are common you have other problems), so this shouldn't 266 * be a performance issue. 267 * 268 * Also there are some races possible while we get from the 269 * error detection to actually handle it. 270 */ 271 272struct to_kill { 273 struct list_head nd; 274 struct task_struct *tsk; 275 unsigned long addr; 276 char addr_valid; 277}; 278 279/* 280 * Failure handling: if we can't find or can't kill a process there's 281 * not much we can do. We just print a message and ignore otherwise. 282 */ 283 284/* 285 * Schedule a process for later kill. 286 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. 287 * TBD would GFP_NOIO be enough? 288 */ 289static void add_to_kill(struct task_struct *tsk, struct page *p, 290 struct vm_area_struct *vma, 291 struct list_head *to_kill, 292 struct to_kill **tkc) 293{ 294 struct to_kill *tk; 295 296 if (*tkc) { 297 tk = *tkc; 298 *tkc = NULL; 299 } else { 300 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC); 301 if (!tk) { 302 printk(KERN_ERR 303 "MCE: Out of memory while machine check handling\n"); 304 return; 305 } 306 } 307 tk->addr = page_address_in_vma(p, vma); 308 tk->addr_valid = 1; 309 310 /* 311 * In theory we don't have to kill when the page was 312 * munmaped. But it could be also a mremap. Since that's 313 * likely very rare kill anyways just out of paranoia, but use 314 * a SIGKILL because the error is not contained anymore. 315 */ 316 if (tk->addr == -EFAULT) { 317 pr_info("MCE: Unable to find user space address %lx in %s\n", 318 page_to_pfn(p), tsk->comm); 319 tk->addr_valid = 0; 320 } 321 get_task_struct(tsk); 322 tk->tsk = tsk; 323 list_add_tail(&tk->nd, to_kill); 324} 325 326/* 327 * Kill the processes that have been collected earlier. 328 * 329 * Only do anything when DOIT is set, otherwise just free the list 330 * (this is used for clean pages which do not need killing) 331 * Also when FAIL is set do a force kill because something went 332 * wrong earlier. 333 */ 334static void kill_procs(struct list_head *to_kill, int forcekill, int trapno, 335 int fail, struct page *page, unsigned long pfn, 336 int flags) 337{ 338 struct to_kill *tk, *next; 339 340 list_for_each_entry_safe (tk, next, to_kill, nd) { 341 if (forcekill) { 342 /* 343 * In case something went wrong with munmapping 344 * make sure the process doesn't catch the 345 * signal and then access the memory. Just kill it. 346 */ 347 if (fail || tk->addr_valid == 0) { 348 printk(KERN_ERR 349 "MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", 350 pfn, tk->tsk->comm, tk->tsk->pid); 351 force_sig(SIGKILL, tk->tsk); 352 } 353 354 /* 355 * In theory the process could have mapped 356 * something else on the address in-between. We could 357 * check for that, but we need to tell the 358 * process anyways. 359 */ 360 else if (kill_proc(tk->tsk, tk->addr, trapno, 361 pfn, page, flags) < 0) 362 printk(KERN_ERR 363 "MCE %#lx: Cannot send advisory machine check signal to %s:%d\n", 364 pfn, tk->tsk->comm, tk->tsk->pid); 365 } 366 put_task_struct(tk->tsk); 367 kfree(tk); 368 } 369} 370 371/* 372 * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO) 373 * on behalf of the thread group. Return task_struct of the (first found) 374 * dedicated thread if found, and return NULL otherwise. 375 * 376 * We already hold read_lock(&tasklist_lock) in the caller, so we don't 377 * have to call rcu_read_lock/unlock() in this function. 378 */ 379static struct task_struct *find_early_kill_thread(struct task_struct *tsk) 380{ 381 struct task_struct *t; 382 383 for_each_thread(tsk, t) 384 if ((t->flags & PF_MCE_PROCESS) && (t->flags & PF_MCE_EARLY)) 385 return t; 386 return NULL; 387} 388 389/* 390 * Determine whether a given process is "early kill" process which expects 391 * to be signaled when some page under the process is hwpoisoned. 392 * Return task_struct of the dedicated thread (main thread unless explicitly 393 * specified) if the process is "early kill," and otherwise returns NULL. 394 */ 395static struct task_struct *task_early_kill(struct task_struct *tsk, 396 int force_early) 397{ 398 struct task_struct *t; 399 if (!tsk->mm) 400 return NULL; 401 if (force_early) 402 return tsk; 403 t = find_early_kill_thread(tsk); 404 if (t) 405 return t; 406 if (sysctl_memory_failure_early_kill) 407 return tsk; 408 return NULL; 409} 410 411/* 412 * Collect processes when the error hit an anonymous page. 413 */ 414static void collect_procs_anon(struct page *page, struct list_head *to_kill, 415 struct to_kill **tkc, int force_early) 416{ 417 struct vm_area_struct *vma; 418 struct task_struct *tsk; 419 struct anon_vma *av; 420 pgoff_t pgoff; 421 422 av = page_lock_anon_vma_read(page); 423 if (av == NULL) /* Not actually mapped anymore */ 424 return; 425 426 pgoff = page_to_pgoff(page); 427 read_lock(&tasklist_lock); 428 for_each_process (tsk) { 429 struct anon_vma_chain *vmac; 430 struct task_struct *t = task_early_kill(tsk, force_early); 431 432 if (!t) 433 continue; 434 anon_vma_interval_tree_foreach(vmac, &av->rb_root, 435 pgoff, pgoff) { 436 vma = vmac->vma; 437 if (!page_mapped_in_vma(page, vma)) 438 continue; 439 if (vma->vm_mm == t->mm) 440 add_to_kill(t, page, vma, to_kill, tkc); 441 } 442 } 443 read_unlock(&tasklist_lock); 444 page_unlock_anon_vma_read(av); 445} 446 447/* 448 * Collect processes when the error hit a file mapped page. 449 */ 450static void collect_procs_file(struct page *page, struct list_head *to_kill, 451 struct to_kill **tkc, int force_early) 452{ 453 struct vm_area_struct *vma; 454 struct task_struct *tsk; 455 struct address_space *mapping = page->mapping; 456 457 i_mmap_lock_read(mapping); 458 read_lock(&tasklist_lock); 459 for_each_process(tsk) { 460 pgoff_t pgoff = page_to_pgoff(page); 461 struct task_struct *t = task_early_kill(tsk, force_early); 462 463 if (!t) 464 continue; 465 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, 466 pgoff) { 467 /* 468 * Send early kill signal to tasks where a vma covers 469 * the page but the corrupted page is not necessarily 470 * mapped it in its pte. 471 * Assume applications who requested early kill want 472 * to be informed of all such data corruptions. 473 */ 474 if (vma->vm_mm == t->mm) 475 add_to_kill(t, page, vma, to_kill, tkc); 476 } 477 } 478 read_unlock(&tasklist_lock); 479 i_mmap_unlock_read(mapping); 480} 481 482/* 483 * Collect the processes who have the corrupted page mapped to kill. 484 * This is done in two steps for locking reasons. 485 * First preallocate one tokill structure outside the spin locks, 486 * so that we can kill at least one process reasonably reliable. 487 */ 488static void collect_procs(struct page *page, struct list_head *tokill, 489 int force_early) 490{ 491 struct to_kill *tk; 492 493 if (!page->mapping) 494 return; 495 496 tk = kmalloc(sizeof(struct to_kill), GFP_NOIO); 497 if (!tk) 498 return; 499 if (PageAnon(page)) 500 collect_procs_anon(page, tokill, &tk, force_early); 501 else 502 collect_procs_file(page, tokill, &tk, force_early); 503 kfree(tk); 504} 505 506/* 507 * Error handlers for various types of pages. 508 */ 509 510enum outcome { 511 IGNORED, /* Error: cannot be handled */ 512 FAILED, /* Error: handling failed */ 513 DELAYED, /* Will be handled later */ 514 RECOVERED, /* Successfully recovered */ 515}; 516 517static const char *action_name[] = { 518 [IGNORED] = "Ignored", 519 [FAILED] = "Failed", 520 [DELAYED] = "Delayed", 521 [RECOVERED] = "Recovered", 522}; 523 524enum action_page_type { 525 MSG_KERNEL, 526 MSG_KERNEL_HIGH_ORDER, 527 MSG_SLAB, 528 MSG_DIFFERENT_COMPOUND, 529 MSG_POISONED_HUGE, 530 MSG_HUGE, 531 MSG_FREE_HUGE, 532 MSG_UNMAP_FAILED, 533 MSG_DIRTY_SWAPCACHE, 534 MSG_CLEAN_SWAPCACHE, 535 MSG_DIRTY_MLOCKED_LRU, 536 MSG_CLEAN_MLOCKED_LRU, 537 MSG_DIRTY_UNEVICTABLE_LRU, 538 MSG_CLEAN_UNEVICTABLE_LRU, 539 MSG_DIRTY_LRU, 540 MSG_CLEAN_LRU, 541 MSG_TRUNCATED_LRU, 542 MSG_BUDDY, 543 MSG_BUDDY_2ND, 544 MSG_UNKNOWN, 545}; 546 547static const char * const action_page_types[] = { 548 [MSG_KERNEL] = "reserved kernel page", 549 [MSG_KERNEL_HIGH_ORDER] = "high-order kernel page", 550 [MSG_SLAB] = "kernel slab page", 551 [MSG_DIFFERENT_COMPOUND] = "different compound page after locking", 552 [MSG_POISONED_HUGE] = "huge page already hardware poisoned", 553 [MSG_HUGE] = "huge page", 554 [MSG_FREE_HUGE] = "free huge page", 555 [MSG_UNMAP_FAILED] = "unmapping failed page", 556 [MSG_DIRTY_SWAPCACHE] = "dirty swapcache page", 557 [MSG_CLEAN_SWAPCACHE] = "clean swapcache page", 558 [MSG_DIRTY_MLOCKED_LRU] = "dirty mlocked LRU page", 559 [MSG_CLEAN_MLOCKED_LRU] = "clean mlocked LRU page", 560 [MSG_DIRTY_UNEVICTABLE_LRU] = "dirty unevictable LRU page", 561 [MSG_CLEAN_UNEVICTABLE_LRU] = "clean unevictable LRU page", 562 [MSG_DIRTY_LRU] = "dirty LRU page", 563 [MSG_CLEAN_LRU] = "clean LRU page", 564 [MSG_TRUNCATED_LRU] = "already truncated LRU page", 565 [MSG_BUDDY] = "free buddy page", 566 [MSG_BUDDY_2ND] = "free buddy page (2nd try)", 567 [MSG_UNKNOWN] = "unknown page", 568}; 569 570/* 571 * XXX: It is possible that a page is isolated from LRU cache, 572 * and then kept in swap cache or failed to remove from page cache. 573 * The page count will stop it from being freed by unpoison. 574 * Stress tests should be aware of this memory leak problem. 575 */ 576static int delete_from_lru_cache(struct page *p) 577{ 578 if (!isolate_lru_page(p)) { 579 /* 580 * Clear sensible page flags, so that the buddy system won't 581 * complain when the page is unpoison-and-freed. 582 */ 583 ClearPageActive(p); 584 ClearPageUnevictable(p); 585 /* 586 * drop the page count elevated by isolate_lru_page() 587 */ 588 page_cache_release(p); 589 return 0; 590 } 591 return -EIO; 592} 593 594/* 595 * Error hit kernel page. 596 * Do nothing, try to be lucky and not touch this instead. For a few cases we 597 * could be more sophisticated. 598 */ 599static int me_kernel(struct page *p, unsigned long pfn) 600{ 601 return IGNORED; 602} 603 604/* 605 * Page in unknown state. Do nothing. 606 */ 607static int me_unknown(struct page *p, unsigned long pfn) 608{ 609 printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn); 610 return FAILED; 611} 612 613/* 614 * Clean (or cleaned) page cache page. 615 */ 616static int me_pagecache_clean(struct page *p, unsigned long pfn) 617{ 618 int err; 619 int ret = FAILED; 620 struct address_space *mapping; 621 622 delete_from_lru_cache(p); 623 624 /* 625 * For anonymous pages we're done the only reference left 626 * should be the one m_f() holds. 627 */ 628 if (PageAnon(p)) 629 return RECOVERED; 630 631 /* 632 * Now truncate the page in the page cache. This is really 633 * more like a "temporary hole punch" 634 * Don't do this for block devices when someone else 635 * has a reference, because it could be file system metadata 636 * and that's not safe to truncate. 637 */ 638 mapping = page_mapping(p); 639 if (!mapping) { 640 /* 641 * Page has been teared down in the meanwhile 642 */ 643 return FAILED; 644 } 645 646 /* 647 * Truncation is a bit tricky. Enable it per file system for now. 648 * 649 * Open: to take i_mutex or not for this? Right now we don't. 650 */ 651 if (mapping->a_ops->error_remove_page) { 652 err = mapping->a_ops->error_remove_page(mapping, p); 653 if (err != 0) { 654 printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n", 655 pfn, err); 656 } else if (page_has_private(p) && 657 !try_to_release_page(p, GFP_NOIO)) { 658 pr_info("MCE %#lx: failed to release buffers\n", pfn); 659 } else { 660 ret = RECOVERED; 661 } 662 } else { 663 /* 664 * If the file system doesn't support it just invalidate 665 * This fails on dirty or anything with private pages 666 */ 667 if (invalidate_inode_page(p)) 668 ret = RECOVERED; 669 else 670 printk(KERN_INFO "MCE %#lx: Failed to invalidate\n", 671 pfn); 672 } 673 return ret; 674} 675 676/* 677 * Dirty pagecache page 678 * Issues: when the error hit a hole page the error is not properly 679 * propagated. 680 */ 681static int me_pagecache_dirty(struct page *p, unsigned long pfn) 682{ 683 struct address_space *mapping = page_mapping(p); 684 685 SetPageError(p); 686 /* TBD: print more information about the file. */ 687 if (mapping) { 688 /* 689 * IO error will be reported by write(), fsync(), etc. 690 * who check the mapping. 691 * This way the application knows that something went 692 * wrong with its dirty file data. 693 * 694 * There's one open issue: 695 * 696 * The EIO will be only reported on the next IO 697 * operation and then cleared through the IO map. 698 * Normally Linux has two mechanisms to pass IO error 699 * first through the AS_EIO flag in the address space 700 * and then through the PageError flag in the page. 701 * Since we drop pages on memory failure handling the 702 * only mechanism open to use is through AS_AIO. 703 * 704 * This has the disadvantage that it gets cleared on 705 * the first operation that returns an error, while 706 * the PageError bit is more sticky and only cleared 707 * when the page is reread or dropped. If an 708 * application assumes it will always get error on 709 * fsync, but does other operations on the fd before 710 * and the page is dropped between then the error 711 * will not be properly reported. 712 * 713 * This can already happen even without hwpoisoned 714 * pages: first on metadata IO errors (which only 715 * report through AS_EIO) or when the page is dropped 716 * at the wrong time. 717 * 718 * So right now we assume that the application DTRT on 719 * the first EIO, but we're not worse than other parts 720 * of the kernel. 721 */ 722 mapping_set_error(mapping, EIO); 723 } 724 725 return me_pagecache_clean(p, pfn); 726} 727 728/* 729 * Clean and dirty swap cache. 730 * 731 * Dirty swap cache page is tricky to handle. The page could live both in page 732 * cache and swap cache(ie. page is freshly swapped in). So it could be 733 * referenced concurrently by 2 types of PTEs: 734 * normal PTEs and swap PTEs. We try to handle them consistently by calling 735 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs, 736 * and then 737 * - clear dirty bit to prevent IO 738 * - remove from LRU 739 * - but keep in the swap cache, so that when we return to it on 740 * a later page fault, we know the application is accessing 741 * corrupted data and shall be killed (we installed simple 742 * interception code in do_swap_page to catch it). 743 * 744 * Clean swap cache pages can be directly isolated. A later page fault will 745 * bring in the known good data from disk. 746 */ 747static int me_swapcache_dirty(struct page *p, unsigned long pfn) 748{ 749 ClearPageDirty(p); 750 /* Trigger EIO in shmem: */ 751 ClearPageUptodate(p); 752 753 if (!delete_from_lru_cache(p)) 754 return DELAYED; 755 else 756 return FAILED; 757} 758 759static int me_swapcache_clean(struct page *p, unsigned long pfn) 760{ 761 delete_from_swap_cache(p); 762 763 if (!delete_from_lru_cache(p)) 764 return RECOVERED; 765 else 766 return FAILED; 767} 768 769/* 770 * Huge pages. Needs work. 771 * Issues: 772 * - Error on hugepage is contained in hugepage unit (not in raw page unit.) 773 * To narrow down kill region to one page, we need to break up pmd. 774 */ 775static int me_huge_page(struct page *p, unsigned long pfn) 776{ 777 int res = 0; 778 struct page *hpage = compound_head(p); 779 /* 780 * We can safely recover from error on free or reserved (i.e. 781 * not in-use) hugepage by dequeuing it from freelist. 782 * To check whether a hugepage is in-use or not, we can't use 783 * page->lru because it can be used in other hugepage operations, 784 * such as __unmap_hugepage_range() and gather_surplus_pages(). 785 * So instead we use page_mapping() and PageAnon(). 786 * We assume that this function is called with page lock held, 787 * so there is no race between isolation and mapping/unmapping. 788 */ 789 if (!(page_mapping(hpage) || PageAnon(hpage))) { 790 res = dequeue_hwpoisoned_huge_page(hpage); 791 if (!res) 792 return RECOVERED; 793 } 794 return DELAYED; 795} 796 797/* 798 * Various page states we can handle. 799 * 800 * A page state is defined by its current page->flags bits. 801 * The table matches them in order and calls the right handler. 802 * 803 * This is quite tricky because we can access page at any time 804 * in its live cycle, so all accesses have to be extremely careful. 805 * 806 * This is not complete. More states could be added. 807 * For any missing state don't attempt recovery. 808 */ 809 810#define dirty (1UL << PG_dirty) 811#define sc (1UL << PG_swapcache) 812#define unevict (1UL << PG_unevictable) 813#define mlock (1UL << PG_mlocked) 814#define writeback (1UL << PG_writeback) 815#define lru (1UL << PG_lru) 816#define swapbacked (1UL << PG_swapbacked) 817#define head (1UL << PG_head) 818#define tail (1UL << PG_tail) 819#define compound (1UL << PG_compound) 820#define slab (1UL << PG_slab) 821#define reserved (1UL << PG_reserved) 822 823static struct page_state { 824 unsigned long mask; 825 unsigned long res; 826 enum action_page_type type; 827 int (*action)(struct page *p, unsigned long pfn); 828} error_states[] = { 829 { reserved, reserved, MSG_KERNEL, me_kernel }, 830 /* 831 * free pages are specially detected outside this table: 832 * PG_buddy pages only make a small fraction of all free pages. 833 */ 834 835 /* 836 * Could in theory check if slab page is free or if we can drop 837 * currently unused objects without touching them. But just 838 * treat it as standard kernel for now. 839 */ 840 { slab, slab, MSG_SLAB, me_kernel }, 841 842#ifdef CONFIG_PAGEFLAGS_EXTENDED 843 { head, head, MSG_HUGE, me_huge_page }, 844 { tail, tail, MSG_HUGE, me_huge_page }, 845#else 846 { compound, compound, MSG_HUGE, me_huge_page }, 847#endif 848 849 { sc|dirty, sc|dirty, MSG_DIRTY_SWAPCACHE, me_swapcache_dirty }, 850 { sc|dirty, sc, MSG_CLEAN_SWAPCACHE, me_swapcache_clean }, 851 852 { mlock|dirty, mlock|dirty, MSG_DIRTY_MLOCKED_LRU, me_pagecache_dirty }, 853 { mlock|dirty, mlock, MSG_CLEAN_MLOCKED_LRU, me_pagecache_clean }, 854 855 { unevict|dirty, unevict|dirty, MSG_DIRTY_UNEVICTABLE_LRU, me_pagecache_dirty }, 856 { unevict|dirty, unevict, MSG_CLEAN_UNEVICTABLE_LRU, me_pagecache_clean }, 857 858 { lru|dirty, lru|dirty, MSG_DIRTY_LRU, me_pagecache_dirty }, 859 { lru|dirty, lru, MSG_CLEAN_LRU, me_pagecache_clean }, 860 861 /* 862 * Catchall entry: must be at end. 863 */ 864 { 0, 0, MSG_UNKNOWN, me_unknown }, 865}; 866 867#undef dirty 868#undef sc 869#undef unevict 870#undef mlock 871#undef writeback 872#undef lru 873#undef swapbacked 874#undef head 875#undef tail 876#undef compound 877#undef slab 878#undef reserved 879 880/* 881 * "Dirty/Clean" indication is not 100% accurate due to the possibility of 882 * setting PG_dirty outside page lock. See also comment above set_page_dirty(). 883 */ 884static void action_result(unsigned long pfn, enum action_page_type type, int result) 885{ 886 pr_err("MCE %#lx: recovery action for %s: %s\n", 887 pfn, action_page_types[type], action_name[result]); 888} 889 890static int page_action(struct page_state *ps, struct page *p, 891 unsigned long pfn) 892{ 893 int result; 894 int count; 895 896 result = ps->action(p, pfn); 897 898 count = page_count(p) - 1; 899 if (ps->action == me_swapcache_dirty && result == DELAYED) 900 count--; 901 if (count != 0) { 902 printk(KERN_ERR 903 "MCE %#lx: %s still referenced by %d users\n", 904 pfn, action_page_types[ps->type], count); 905 result = FAILED; 906 } 907 action_result(pfn, ps->type, result); 908 909 /* Could do more checks here if page looks ok */ 910 /* 911 * Could adjust zone counters here to correct for the missing page. 912 */ 913 914 return (result == RECOVERED || result == DELAYED) ? 0 : -EBUSY; 915} 916 917/* 918 * Do all that is necessary to remove user space mappings. Unmap 919 * the pages and send SIGBUS to the processes if the data was dirty. 920 */ 921static int hwpoison_user_mappings(struct page *p, unsigned long pfn, 922 int trapno, int flags, struct page **hpagep) 923{ 924 enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS; 925 struct address_space *mapping; 926 LIST_HEAD(tokill); 927 int ret; 928 int kill = 1, forcekill; 929 struct page *hpage = *hpagep; 930 struct page *ppage; 931 932 /* 933 * Here we are interested only in user-mapped pages, so skip any 934 * other types of pages. 935 */ 936 if (PageReserved(p) || PageSlab(p)) 937 return SWAP_SUCCESS; 938 if (!(PageLRU(hpage) || PageHuge(p))) 939 return SWAP_SUCCESS; 940 941 /* 942 * This check implies we don't kill processes if their pages 943 * are in the swap cache early. Those are always late kills. 944 */ 945 if (!page_mapped(hpage)) 946 return SWAP_SUCCESS; 947 948 if (PageKsm(p)) { 949 pr_err("MCE %#lx: can't handle KSM pages.\n", pfn); 950 return SWAP_FAIL; 951 } 952 953 if (PageSwapCache(p)) { 954 printk(KERN_ERR 955 "MCE %#lx: keeping poisoned page in swap cache\n", pfn); 956 ttu |= TTU_IGNORE_HWPOISON; 957 } 958 959 /* 960 * Propagate the dirty bit from PTEs to struct page first, because we 961 * need this to decide if we should kill or just drop the page. 962 * XXX: the dirty test could be racy: set_page_dirty() may not always 963 * be called inside page lock (it's recommended but not enforced). 964 */ 965 mapping = page_mapping(hpage); 966 if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping && 967 mapping_cap_writeback_dirty(mapping)) { 968 if (page_mkclean(hpage)) { 969 SetPageDirty(hpage); 970 } else { 971 kill = 0; 972 ttu |= TTU_IGNORE_HWPOISON; 973 printk(KERN_INFO 974 "MCE %#lx: corrupted page was clean: dropped without side effects\n", 975 pfn); 976 } 977 } 978 979 /* 980 * ppage: poisoned page 981 * if p is regular page(4k page) 982 * ppage == real poisoned page; 983 * else p is hugetlb or THP, ppage == head page. 984 */ 985 ppage = hpage; 986 987 if (PageTransHuge(hpage)) { 988 /* 989 * Verify that this isn't a hugetlbfs head page, the check for 990 * PageAnon is just for avoid tripping a split_huge_page 991 * internal debug check, as split_huge_page refuses to deal with 992 * anything that isn't an anon page. PageAnon can't go away fro 993 * under us because we hold a refcount on the hpage, without a 994 * refcount on the hpage. split_huge_page can't be safely called 995 * in the first place, having a refcount on the tail isn't 996 * enough * to be safe. 997 */ 998 if (!PageHuge(hpage) && PageAnon(hpage)) { 999 if (unlikely(split_huge_page(hpage))) { 1000 /* 1001 * FIXME: if splitting THP is failed, it is 1002 * better to stop the following operation rather 1003 * than causing panic by unmapping. System might 1004 * survive if the page is freed later. 1005 */ 1006 printk(KERN_INFO 1007 "MCE %#lx: failed to split THP\n", pfn); 1008 1009 BUG_ON(!PageHWPoison(p)); 1010 return SWAP_FAIL; 1011 } 1012 /* 1013 * We pinned the head page for hwpoison handling, 1014 * now we split the thp and we are interested in 1015 * the hwpoisoned raw page, so move the refcount 1016 * to it. Similarly, page lock is shifted. 1017 */ 1018 if (hpage != p) { 1019 if (!(flags & MF_COUNT_INCREASED)) { 1020 put_page(hpage); 1021 get_page(p); 1022 } 1023 lock_page(p); 1024 unlock_page(hpage); 1025 *hpagep = p; 1026 } 1027 /* THP is split, so ppage should be the real poisoned page. */ 1028 ppage = p; 1029 } 1030 } 1031 1032 /* 1033 * First collect all the processes that have the page 1034 * mapped in dirty form. This has to be done before try_to_unmap, 1035 * because ttu takes the rmap data structures down. 1036 * 1037 * Error handling: We ignore errors here because 1038 * there's nothing that can be done. 1039 */ 1040 if (kill) 1041 collect_procs(ppage, &tokill, flags & MF_ACTION_REQUIRED); 1042 1043 ret = try_to_unmap(ppage, ttu); 1044 if (ret != SWAP_SUCCESS) 1045 printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n", 1046 pfn, page_mapcount(ppage)); 1047 1048 /* 1049 * Now that the dirty bit has been propagated to the 1050 * struct page and all unmaps done we can decide if 1051 * killing is needed or not. Only kill when the page 1052 * was dirty or the process is not restartable, 1053 * otherwise the tokill list is merely 1054 * freed. When there was a problem unmapping earlier 1055 * use a more force-full uncatchable kill to prevent 1056 * any accesses to the poisoned memory. 1057 */ 1058 forcekill = PageDirty(ppage) || (flags & MF_MUST_KILL); 1059 kill_procs(&tokill, forcekill, trapno, 1060 ret != SWAP_SUCCESS, p, pfn, flags); 1061 1062 return ret; 1063} 1064 1065static void set_page_hwpoison_huge_page(struct page *hpage) 1066{ 1067 int i; 1068 int nr_pages = 1 << compound_order(hpage); 1069 for (i = 0; i < nr_pages; i++) 1070 SetPageHWPoison(hpage + i); 1071} 1072 1073static void clear_page_hwpoison_huge_page(struct page *hpage) 1074{ 1075 int i; 1076 int nr_pages = 1 << compound_order(hpage); 1077 for (i = 0; i < nr_pages; i++) 1078 ClearPageHWPoison(hpage + i); 1079} 1080 1081/** 1082 * memory_failure - Handle memory failure of a page. 1083 * @pfn: Page Number of the corrupted page 1084 * @trapno: Trap number reported in the signal to user space. 1085 * @flags: fine tune action taken 1086 * 1087 * This function is called by the low level machine check code 1088 * of an architecture when it detects hardware memory corruption 1089 * of a page. It tries its best to recover, which includes 1090 * dropping pages, killing processes etc. 1091 * 1092 * The function is primarily of use for corruptions that 1093 * happen outside the current execution context (e.g. when 1094 * detected by a background scrubber) 1095 * 1096 * Must run in process context (e.g. a work queue) with interrupts 1097 * enabled and no spinlocks hold. 1098 */ 1099int memory_failure(unsigned long pfn, int trapno, int flags) 1100{ 1101 struct page_state *ps; 1102 struct page *p; 1103 struct page *hpage; 1104 int res; 1105 unsigned int nr_pages; 1106 unsigned long page_flags; 1107 1108 if (!sysctl_memory_failure_recovery) 1109 panic("Memory failure from trap %d on page %lx", trapno, pfn); 1110 1111 if (!pfn_valid(pfn)) { 1112 printk(KERN_ERR 1113 "MCE %#lx: memory outside kernel control\n", 1114 pfn); 1115 return -ENXIO; 1116 } 1117 1118 p = pfn_to_page(pfn); 1119 hpage = compound_head(p); 1120 if (TestSetPageHWPoison(p)) { 1121 printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn); 1122 return 0; 1123 } 1124 1125 /* 1126 * Currently errors on hugetlbfs pages are measured in hugepage units, 1127 * so nr_pages should be 1 << compound_order. OTOH when errors are on 1128 * transparent hugepages, they are supposed to be split and error 1129 * measurement is done in normal page units. So nr_pages should be one 1130 * in this case. 1131 */ 1132 if (PageHuge(p)) 1133 nr_pages = 1 << compound_order(hpage); 1134 else /* normal page or thp */ 1135 nr_pages = 1; 1136 atomic_long_add(nr_pages, &num_poisoned_pages); 1137 1138 /* 1139 * We need/can do nothing about count=0 pages. 1140 * 1) it's a free page, and therefore in safe hand: 1141 * prep_new_page() will be the gate keeper. 1142 * 2) it's a free hugepage, which is also safe: 1143 * an affected hugepage will be dequeued from hugepage freelist, 1144 * so there's no concern about reusing it ever after. 1145 * 3) it's part of a non-compound high order page. 1146 * Implies some kernel user: cannot stop them from 1147 * R/W the page; let's pray that the page has been 1148 * used and will be freed some time later. 1149 * In fact it's dangerous to directly bump up page count from 0, 1150 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch. 1151 */ 1152 if (!(flags & MF_COUNT_INCREASED) && 1153 !get_page_unless_zero(hpage)) { 1154 if (is_free_buddy_page(p)) { 1155 action_result(pfn, MSG_BUDDY, DELAYED); 1156 return 0; 1157 } else if (PageHuge(hpage)) { 1158 /* 1159 * Check "filter hit" and "race with other subpage." 1160 */ 1161 lock_page(hpage); 1162 if (PageHWPoison(hpage)) { 1163 if ((hwpoison_filter(p) && TestClearPageHWPoison(p)) 1164 || (p != hpage && TestSetPageHWPoison(hpage))) { 1165 atomic_long_sub(nr_pages, &num_poisoned_pages); 1166 unlock_page(hpage); 1167 return 0; 1168 } 1169 } 1170 set_page_hwpoison_huge_page(hpage); 1171 res = dequeue_hwpoisoned_huge_page(hpage); 1172 action_result(pfn, MSG_FREE_HUGE, 1173 res ? IGNORED : DELAYED); 1174 unlock_page(hpage); 1175 return res; 1176 } else { 1177 action_result(pfn, MSG_KERNEL_HIGH_ORDER, IGNORED); 1178 return -EBUSY; 1179 } 1180 } 1181 1182 /* 1183 * We ignore non-LRU pages for good reasons. 1184 * - PG_locked is only well defined for LRU pages and a few others 1185 * - to avoid races with __set_page_locked() 1186 * - to avoid races with __SetPageSlab*() (and more non-atomic ops) 1187 * The check (unnecessarily) ignores LRU pages being isolated and 1188 * walked by the page reclaim code, however that's not a big loss. 1189 */ 1190 if (!PageHuge(p)) { 1191 if (!PageLRU(hpage)) 1192 shake_page(hpage, 0); 1193 if (!PageLRU(hpage)) { 1194 /* 1195 * shake_page could have turned it free. 1196 */ 1197 if (is_free_buddy_page(p)) { 1198 if (flags & MF_COUNT_INCREASED) 1199 action_result(pfn, MSG_BUDDY, DELAYED); 1200 else 1201 action_result(pfn, MSG_BUDDY_2ND, 1202 DELAYED); 1203 return 0; 1204 } 1205 } 1206 } 1207 1208 lock_page(hpage); 1209 1210 /* 1211 * The page could have changed compound pages during the locking. 1212 * If this happens just bail out. 1213 */ 1214 if (compound_head(p) != hpage) { 1215 action_result(pfn, MSG_DIFFERENT_COMPOUND, IGNORED); 1216 res = -EBUSY; 1217 goto out; 1218 } 1219 1220 /* 1221 * We use page flags to determine what action should be taken, but 1222 * the flags can be modified by the error containment action. One 1223 * example is an mlocked page, where PG_mlocked is cleared by 1224 * page_remove_rmap() in try_to_unmap_one(). So to determine page status 1225 * correctly, we save a copy of the page flags at this time. 1226 */ 1227 page_flags = p->flags; 1228 1229 /* 1230 * unpoison always clear PG_hwpoison inside page lock 1231 */ 1232 if (!PageHWPoison(p)) { 1233 printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn); 1234 atomic_long_sub(nr_pages, &num_poisoned_pages); 1235 put_page(hpage); 1236 res = 0; 1237 goto out; 1238 } 1239 if (hwpoison_filter(p)) { 1240 if (TestClearPageHWPoison(p)) 1241 atomic_long_sub(nr_pages, &num_poisoned_pages); 1242 unlock_page(hpage); 1243 put_page(hpage); 1244 return 0; 1245 } 1246 1247 if (!PageHuge(p) && !PageTransTail(p) && !PageLRU(p)) 1248 goto identify_page_state; 1249 1250 /* 1251 * For error on the tail page, we should set PG_hwpoison 1252 * on the head page to show that the hugepage is hwpoisoned 1253 */ 1254 if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) { 1255 action_result(pfn, MSG_POISONED_HUGE, IGNORED); 1256 unlock_page(hpage); 1257 put_page(hpage); 1258 return 0; 1259 } 1260 /* 1261 * Set PG_hwpoison on all pages in an error hugepage, 1262 * because containment is done in hugepage unit for now. 1263 * Since we have done TestSetPageHWPoison() for the head page with 1264 * page lock held, we can safely set PG_hwpoison bits on tail pages. 1265 */ 1266 if (PageHuge(p)) 1267 set_page_hwpoison_huge_page(hpage); 1268 1269 /* 1270 * It's very difficult to mess with pages currently under IO 1271 * and in many cases impossible, so we just avoid it here. 1272 */ 1273 wait_on_page_writeback(p); 1274 1275 /* 1276 * Now take care of user space mappings. 1277 * Abort on fail: __delete_from_page_cache() assumes unmapped page. 1278 * 1279 * When the raw error page is thp tail page, hpage points to the raw 1280 * page after thp split. 1281 */ 1282 if (hwpoison_user_mappings(p, pfn, trapno, flags, &hpage) 1283 != SWAP_SUCCESS) { 1284 action_result(pfn, MSG_UNMAP_FAILED, IGNORED); 1285 res = -EBUSY; 1286 goto out; 1287 } 1288 1289 /* 1290 * Torn down by someone else? 1291 */ 1292 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) { 1293 action_result(pfn, MSG_TRUNCATED_LRU, IGNORED); 1294 res = -EBUSY; 1295 goto out; 1296 } 1297 1298identify_page_state: 1299 res = -EBUSY; 1300 /* 1301 * The first check uses the current page flags which may not have any 1302 * relevant information. The second check with the saved page flagss is 1303 * carried out only if the first check can't determine the page status. 1304 */ 1305 for (ps = error_states;; ps++) 1306 if ((p->flags & ps->mask) == ps->res) 1307 break; 1308 1309 page_flags |= (p->flags & (1UL << PG_dirty)); 1310 1311 if (!ps->mask) 1312 for (ps = error_states;; ps++) 1313 if ((page_flags & ps->mask) == ps->res) 1314 break; 1315 res = page_action(ps, p, pfn); 1316out: 1317 unlock_page(hpage); 1318 return res; 1319} 1320EXPORT_SYMBOL_GPL(memory_failure); 1321 1322#define MEMORY_FAILURE_FIFO_ORDER 4 1323#define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER) 1324 1325struct memory_failure_entry { 1326 unsigned long pfn; 1327 int trapno; 1328 int flags; 1329}; 1330 1331struct memory_failure_cpu { 1332 DECLARE_KFIFO(fifo, struct memory_failure_entry, 1333 MEMORY_FAILURE_FIFO_SIZE); 1334 spinlock_t lock; 1335 struct work_struct work; 1336}; 1337 1338static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu); 1339 1340/** 1341 * memory_failure_queue - Schedule handling memory failure of a page. 1342 * @pfn: Page Number of the corrupted page 1343 * @trapno: Trap number reported in the signal to user space. 1344 * @flags: Flags for memory failure handling 1345 * 1346 * This function is called by the low level hardware error handler 1347 * when it detects hardware memory corruption of a page. It schedules 1348 * the recovering of error page, including dropping pages, killing 1349 * processes etc. 1350 * 1351 * The function is primarily of use for corruptions that 1352 * happen outside the current execution context (e.g. when 1353 * detected by a background scrubber) 1354 * 1355 * Can run in IRQ context. 1356 */ 1357void memory_failure_queue(unsigned long pfn, int trapno, int flags) 1358{ 1359 struct memory_failure_cpu *mf_cpu; 1360 unsigned long proc_flags; 1361 struct memory_failure_entry entry = { 1362 .pfn = pfn, 1363 .trapno = trapno, 1364 .flags = flags, 1365 }; 1366 1367 mf_cpu = &get_cpu_var(memory_failure_cpu); 1368 spin_lock_irqsave(&mf_cpu->lock, proc_flags); 1369 if (kfifo_put(&mf_cpu->fifo, entry)) 1370 schedule_work_on(smp_processor_id(), &mf_cpu->work); 1371 else 1372 pr_err("Memory failure: buffer overflow when queuing memory failure at %#lx\n", 1373 pfn); 1374 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 1375 put_cpu_var(memory_failure_cpu); 1376} 1377EXPORT_SYMBOL_GPL(memory_failure_queue); 1378 1379static void memory_failure_work_func(struct work_struct *work) 1380{ 1381 struct memory_failure_cpu *mf_cpu; 1382 struct memory_failure_entry entry = { 0, }; 1383 unsigned long proc_flags; 1384 int gotten; 1385 1386 mf_cpu = this_cpu_ptr(&memory_failure_cpu); 1387 for (;;) { 1388 spin_lock_irqsave(&mf_cpu->lock, proc_flags); 1389 gotten = kfifo_get(&mf_cpu->fifo, &entry); 1390 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 1391 if (!gotten) 1392 break; 1393 if (entry.flags & MF_SOFT_OFFLINE) 1394 soft_offline_page(pfn_to_page(entry.pfn), entry.flags); 1395 else 1396 memory_failure(entry.pfn, entry.trapno, entry.flags); 1397 } 1398} 1399 1400static int __init memory_failure_init(void) 1401{ 1402 struct memory_failure_cpu *mf_cpu; 1403 int cpu; 1404 1405 for_each_possible_cpu(cpu) { 1406 mf_cpu = &per_cpu(memory_failure_cpu, cpu); 1407 spin_lock_init(&mf_cpu->lock); 1408 INIT_KFIFO(mf_cpu->fifo); 1409 INIT_WORK(&mf_cpu->work, memory_failure_work_func); 1410 } 1411 1412 return 0; 1413} 1414core_initcall(memory_failure_init); 1415 1416/** 1417 * unpoison_memory - Unpoison a previously poisoned page 1418 * @pfn: Page number of the to be unpoisoned page 1419 * 1420 * Software-unpoison a page that has been poisoned by 1421 * memory_failure() earlier. 1422 * 1423 * This is only done on the software-level, so it only works 1424 * for linux injected failures, not real hardware failures 1425 * 1426 * Returns 0 for success, otherwise -errno. 1427 */ 1428int unpoison_memory(unsigned long pfn) 1429{ 1430 struct page *page; 1431 struct page *p; 1432 int freeit = 0; 1433 unsigned int nr_pages; 1434 1435 if (!pfn_valid(pfn)) 1436 return -ENXIO; 1437 1438 p = pfn_to_page(pfn); 1439 page = compound_head(p); 1440 1441 if (!PageHWPoison(p)) { 1442 pr_info("MCE: Page was already unpoisoned %#lx\n", pfn); 1443 return 0; 1444 } 1445 1446 /* 1447 * unpoison_memory() can encounter thp only when the thp is being 1448 * worked by memory_failure() and the page lock is not held yet. 1449 * In such case, we yield to memory_failure() and make unpoison fail. 1450 */ 1451 if (!PageHuge(page) && PageTransHuge(page)) { 1452 pr_info("MCE: Memory failure is now running on %#lx\n", pfn); 1453 return 0; 1454 } 1455 1456 nr_pages = 1 << compound_order(page); 1457 1458 if (!get_page_unless_zero(page)) { 1459 /* 1460 * Since HWPoisoned hugepage should have non-zero refcount, 1461 * race between memory failure and unpoison seems to happen. 1462 * In such case unpoison fails and memory failure runs 1463 * to the end. 1464 */ 1465 if (PageHuge(page)) { 1466 pr_info("MCE: Memory failure is now running on free hugepage %#lx\n", pfn); 1467 return 0; 1468 } 1469 if (TestClearPageHWPoison(p)) 1470 atomic_long_dec(&num_poisoned_pages); 1471 pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn); 1472 return 0; 1473 } 1474 1475 lock_page(page); 1476 /* 1477 * This test is racy because PG_hwpoison is set outside of page lock. 1478 * That's acceptable because that won't trigger kernel panic. Instead, 1479 * the PG_hwpoison page will be caught and isolated on the entrance to 1480 * the free buddy page pool. 1481 */ 1482 if (TestClearPageHWPoison(page)) { 1483 pr_info("MCE: Software-unpoisoned page %#lx\n", pfn); 1484 atomic_long_sub(nr_pages, &num_poisoned_pages); 1485 freeit = 1; 1486 if (PageHuge(page)) 1487 clear_page_hwpoison_huge_page(page); 1488 } 1489 unlock_page(page); 1490 1491 put_page(page); 1492 if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1)) 1493 put_page(page); 1494 1495 return 0; 1496} 1497EXPORT_SYMBOL(unpoison_memory); 1498 1499static struct page *new_page(struct page *p, unsigned long private, int **x) 1500{ 1501 int nid = page_to_nid(p); 1502 if (PageHuge(p)) 1503 return alloc_huge_page_node(page_hstate(compound_head(p)), 1504 nid); 1505 else 1506 return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0); 1507} 1508 1509/* 1510 * Safely get reference count of an arbitrary page. 1511 * Returns 0 for a free page, -EIO for a zero refcount page 1512 * that is not free, and 1 for any other page type. 1513 * For 1 the page is returned with increased page count, otherwise not. 1514 */ 1515static int __get_any_page(struct page *p, unsigned long pfn, int flags) 1516{ 1517 int ret; 1518 1519 if (flags & MF_COUNT_INCREASED) 1520 return 1; 1521 1522 /* 1523 * When the target page is a free hugepage, just remove it 1524 * from free hugepage list. 1525 */ 1526 if (!get_page_unless_zero(compound_head(p))) { 1527 if (PageHuge(p)) { 1528 pr_info("%s: %#lx free huge page\n", __func__, pfn); 1529 ret = 0; 1530 } else if (is_free_buddy_page(p)) { 1531 pr_info("%s: %#lx free buddy page\n", __func__, pfn); 1532 ret = 0; 1533 } else { 1534 pr_info("%s: %#lx: unknown zero refcount page type %lx\n", 1535 __func__, pfn, p->flags); 1536 ret = -EIO; 1537 } 1538 } else { 1539 /* Not a free page */ 1540 ret = 1; 1541 } 1542 return ret; 1543} 1544 1545static int get_any_page(struct page *page, unsigned long pfn, int flags) 1546{ 1547 int ret = __get_any_page(page, pfn, flags); 1548 1549 if (ret == 1 && !PageHuge(page) && !PageLRU(page)) { 1550 /* 1551 * Try to free it. 1552 */ 1553 put_page(page); 1554 shake_page(page, 1); 1555 1556 /* 1557 * Did it turn free? 1558 */ 1559 ret = __get_any_page(page, pfn, 0); 1560 if (ret == 1 && !PageLRU(page)) { 1561 /* Drop page reference which is from __get_any_page() */ 1562 put_page(page); 1563 pr_info("soft_offline: %#lx: unknown non LRU page type %lx\n", 1564 pfn, page->flags); 1565 return -EIO; 1566 } 1567 } 1568 return ret; 1569} 1570 1571static int soft_offline_huge_page(struct page *page, int flags) 1572{ 1573 int ret; 1574 unsigned long pfn = page_to_pfn(page); 1575 struct page *hpage = compound_head(page); 1576 LIST_HEAD(pagelist); 1577 1578 /* 1579 * This double-check of PageHWPoison is to avoid the race with 1580 * memory_failure(). See also comment in __soft_offline_page(). 1581 */ 1582 lock_page(hpage); 1583 if (PageHWPoison(hpage)) { 1584 unlock_page(hpage); 1585 put_page(hpage); 1586 pr_info("soft offline: %#lx hugepage already poisoned\n", pfn); 1587 return -EBUSY; 1588 } 1589 unlock_page(hpage); 1590 1591 ret = isolate_huge_page(hpage, &pagelist); 1592 /* 1593 * get_any_page() and isolate_huge_page() takes a refcount each, 1594 * so need to drop one here. 1595 */ 1596 put_page(hpage); 1597 if (!ret) { 1598 pr_info("soft offline: %#lx hugepage failed to isolate\n", pfn); 1599 return -EBUSY; 1600 } 1601 1602 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL, 1603 MIGRATE_SYNC, MR_MEMORY_FAILURE); 1604 if (ret) { 1605 pr_info("soft offline: %#lx: migration failed %d, type %lx\n", 1606 pfn, ret, page->flags); 1607 /* 1608 * We know that soft_offline_huge_page() tries to migrate 1609 * only one hugepage pointed to by hpage, so we need not 1610 * run through the pagelist here. 1611 */ 1612 putback_active_hugepage(hpage); 1613 if (ret > 0) 1614 ret = -EIO; 1615 } else { 1616 /* overcommit hugetlb page will be freed to buddy */ 1617 if (PageHuge(page)) { 1618 set_page_hwpoison_huge_page(hpage); 1619 dequeue_hwpoisoned_huge_page(hpage); 1620 atomic_long_add(1 << compound_order(hpage), 1621 &num_poisoned_pages); 1622 } else { 1623 SetPageHWPoison(page); 1624 atomic_long_inc(&num_poisoned_pages); 1625 } 1626 } 1627 return ret; 1628} 1629 1630static int __soft_offline_page(struct page *page, int flags) 1631{ 1632 int ret; 1633 unsigned long pfn = page_to_pfn(page); 1634 1635 /* 1636 * Check PageHWPoison again inside page lock because PageHWPoison 1637 * is set by memory_failure() outside page lock. Note that 1638 * memory_failure() also double-checks PageHWPoison inside page lock, 1639 * so there's no race between soft_offline_page() and memory_failure(). 1640 */ 1641 lock_page(page); 1642 wait_on_page_writeback(page); 1643 if (PageHWPoison(page)) { 1644 unlock_page(page); 1645 put_page(page); 1646 pr_info("soft offline: %#lx page already poisoned\n", pfn); 1647 return -EBUSY; 1648 } 1649 /* 1650 * Try to invalidate first. This should work for 1651 * non dirty unmapped page cache pages. 1652 */ 1653 ret = invalidate_inode_page(page); 1654 unlock_page(page); 1655 /* 1656 * RED-PEN would be better to keep it isolated here, but we 1657 * would need to fix isolation locking first. 1658 */ 1659 if (ret == 1) { 1660 put_page(page); 1661 pr_info("soft_offline: %#lx: invalidated\n", pfn); 1662 SetPageHWPoison(page); 1663 atomic_long_inc(&num_poisoned_pages); 1664 return 0; 1665 } 1666 1667 /* 1668 * Simple invalidation didn't work. 1669 * Try to migrate to a new page instead. migrate.c 1670 * handles a large number of cases for us. 1671 */ 1672 ret = isolate_lru_page(page); 1673 /* 1674 * Drop page reference which is came from get_any_page() 1675 * successful isolate_lru_page() already took another one. 1676 */ 1677 put_page(page); 1678 if (!ret) { 1679 LIST_HEAD(pagelist); 1680 inc_zone_page_state(page, NR_ISOLATED_ANON + 1681 page_is_file_cache(page)); 1682 list_add(&page->lru, &pagelist); 1683 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL, 1684 MIGRATE_SYNC, MR_MEMORY_FAILURE); 1685 if (ret) { 1686 if (!list_empty(&pagelist)) { 1687 list_del(&page->lru); 1688 dec_zone_page_state(page, NR_ISOLATED_ANON + 1689 page_is_file_cache(page)); 1690 putback_lru_page(page); 1691 } 1692 1693 pr_info("soft offline: %#lx: migration failed %d, type %lx\n", 1694 pfn, ret, page->flags); 1695 if (ret > 0) 1696 ret = -EIO; 1697 } else { 1698 SetPageHWPoison(page); 1699 atomic_long_inc(&num_poisoned_pages); 1700 } 1701 } else { 1702 pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n", 1703 pfn, ret, page_count(page), page->flags); 1704 } 1705 return ret; 1706} 1707 1708/** 1709 * soft_offline_page - Soft offline a page. 1710 * @page: page to offline 1711 * @flags: flags. Same as memory_failure(). 1712 * 1713 * Returns 0 on success, otherwise negated errno. 1714 * 1715 * Soft offline a page, by migration or invalidation, 1716 * without killing anything. This is for the case when 1717 * a page is not corrupted yet (so it's still valid to access), 1718 * but has had a number of corrected errors and is better taken 1719 * out. 1720 * 1721 * The actual policy on when to do that is maintained by 1722 * user space. 1723 * 1724 * This should never impact any application or cause data loss, 1725 * however it might take some time. 1726 * 1727 * This is not a 100% solution for all memory, but tries to be 1728 * ``good enough'' for the majority of memory. 1729 */ 1730int soft_offline_page(struct page *page, int flags) 1731{ 1732 int ret; 1733 unsigned long pfn = page_to_pfn(page); 1734 struct page *hpage = compound_head(page); 1735 1736 if (PageHWPoison(page)) { 1737 pr_info("soft offline: %#lx page already poisoned\n", pfn); 1738 return -EBUSY; 1739 } 1740 if (!PageHuge(page) && PageTransHuge(hpage)) { 1741 if (PageAnon(hpage) && unlikely(split_huge_page(hpage))) { 1742 pr_info("soft offline: %#lx: failed to split THP\n", 1743 pfn); 1744 return -EBUSY; 1745 } 1746 } 1747 1748 get_online_mems(); 1749 1750 ret = get_any_page(page, pfn, flags); 1751 put_online_mems(); 1752 if (ret > 0) { /* for in-use pages */ 1753 if (PageHuge(page)) 1754 ret = soft_offline_huge_page(page, flags); 1755 else 1756 ret = __soft_offline_page(page, flags); 1757 } else if (ret == 0) { /* for free pages */ 1758 if (PageHuge(page)) { 1759 set_page_hwpoison_huge_page(hpage); 1760 if (!dequeue_hwpoisoned_huge_page(hpage)) 1761 atomic_long_add(1 << compound_order(hpage), 1762 &num_poisoned_pages); 1763 } else { 1764 if (!TestSetPageHWPoison(page)) 1765 atomic_long_inc(&num_poisoned_pages); 1766 } 1767 } 1768 return ret; 1769} 1770