1#ifndef _LINUX_MM_H 2#define _LINUX_MM_H 3 4#include <linux/errno.h> 5 6#ifdef __KERNEL__ 7 8#include <linux/mmdebug.h> 9#include <linux/gfp.h> 10#include <linux/bug.h> 11#include <linux/list.h> 12#include <linux/mmzone.h> 13#include <linux/rbtree.h> 14#include <linux/atomic.h> 15#include <linux/debug_locks.h> 16#include <linux/mm_types.h> 17#include <linux/range.h> 18#include <linux/pfn.h> 19#include <linux/bit_spinlock.h> 20#include <linux/shrinker.h> 21#include <linux/resource.h> 22#include <linux/page_ext.h> 23 24struct mempolicy; 25struct anon_vma; 26struct anon_vma_chain; 27struct file_ra_state; 28struct user_struct; 29struct writeback_control; 30 31#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */ 32extern unsigned long max_mapnr; 33 34static inline void set_max_mapnr(unsigned long limit) 35{ 36 max_mapnr = limit; 37} 38#else 39static inline void set_max_mapnr(unsigned long limit) { } 40#endif 41 42extern unsigned long totalram_pages; 43extern void * high_memory; 44extern int page_cluster; 45 46#ifdef CONFIG_SYSCTL 47extern int sysctl_legacy_va_layout; 48#else 49#define sysctl_legacy_va_layout 0 50#endif 51 52#include <asm/page.h> 53#include <asm/pgtable.h> 54#include <asm/processor.h> 55 56#ifndef __pa_symbol 57#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) 58#endif 59 60/* 61 * To prevent common memory management code establishing 62 * a zero page mapping on a read fault. 63 * This macro should be defined within <asm/pgtable.h>. 64 * s390 does this to prevent multiplexing of hardware bits 65 * related to the physical page in case of virtualization. 66 */ 67#ifndef mm_forbids_zeropage 68#define mm_forbids_zeropage(X) (0) 69#endif 70 71extern unsigned long sysctl_user_reserve_kbytes; 72extern unsigned long sysctl_admin_reserve_kbytes; 73 74extern int sysctl_overcommit_memory; 75extern int sysctl_overcommit_ratio; 76extern unsigned long sysctl_overcommit_kbytes; 77 78extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *, 79 size_t *, loff_t *); 80extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *, 81 size_t *, loff_t *); 82 83#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) 84 85/* to align the pointer to the (next) page boundary */ 86#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE) 87 88/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */ 89#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)addr, PAGE_SIZE) 90 91/* 92 * Linux kernel virtual memory manager primitives. 93 * The idea being to have a "virtual" mm in the same way 94 * we have a virtual fs - giving a cleaner interface to the 95 * mm details, and allowing different kinds of memory mappings 96 * (from shared memory to executable loading to arbitrary 97 * mmap() functions). 98 */ 99 100extern struct kmem_cache *vm_area_cachep; 101 102#ifndef CONFIG_MMU 103extern struct rb_root nommu_region_tree; 104extern struct rw_semaphore nommu_region_sem; 105 106extern unsigned int kobjsize(const void *objp); 107#endif 108 109/* 110 * vm_flags in vm_area_struct, see mm_types.h. 111 */ 112#define VM_NONE 0x00000000 113 114#define VM_READ 0x00000001 /* currently active flags */ 115#define VM_WRITE 0x00000002 116#define VM_EXEC 0x00000004 117#define VM_SHARED 0x00000008 118 119/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */ 120#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */ 121#define VM_MAYWRITE 0x00000020 122#define VM_MAYEXEC 0x00000040 123#define VM_MAYSHARE 0x00000080 124 125#define VM_GROWSDOWN 0x00000100 /* general info on the segment */ 126#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */ 127#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */ 128 129#define VM_LOCKED 0x00002000 130#define VM_IO 0x00004000 /* Memory mapped I/O or similar */ 131 132 /* Used by sys_madvise() */ 133#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */ 134#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */ 135 136#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */ 137#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */ 138#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */ 139#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */ 140#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */ 141#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */ 142#define VM_ARCH_2 0x02000000 143#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */ 144 145#ifdef CONFIG_MEM_SOFT_DIRTY 146# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */ 147#else 148# define VM_SOFTDIRTY 0 149#endif 150 151#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */ 152#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */ 153#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */ 154#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */ 155 156#if defined(CONFIG_X86) 157# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */ 158#elif defined(CONFIG_PPC) 159# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */ 160#elif defined(CONFIG_PARISC) 161# define VM_GROWSUP VM_ARCH_1 162#elif defined(CONFIG_METAG) 163# define VM_GROWSUP VM_ARCH_1 164#elif defined(CONFIG_IA64) 165# define VM_GROWSUP VM_ARCH_1 166#elif !defined(CONFIG_MMU) 167# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */ 168#endif 169 170#if defined(CONFIG_X86) 171/* MPX specific bounds table or bounds directory */ 172# define VM_MPX VM_ARCH_2 173#endif 174 175#ifndef VM_GROWSUP 176# define VM_GROWSUP VM_NONE 177#endif 178 179/* Bits set in the VMA until the stack is in its final location */ 180#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ) 181 182#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */ 183#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS 184#endif 185 186#ifdef CONFIG_STACK_GROWSUP 187#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT) 188#else 189#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT) 190#endif 191 192/* 193 * Special vmas that are non-mergable, non-mlock()able. 194 * Note: mm/huge_memory.c VM_NO_THP depends on this definition. 195 */ 196#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP) 197 198/* This mask defines which mm->def_flags a process can inherit its parent */ 199#define VM_INIT_DEF_MASK VM_NOHUGEPAGE 200 201/* 202 * mapping from the currently active vm_flags protection bits (the 203 * low four bits) to a page protection mask.. 204 */ 205extern pgprot_t protection_map[16]; 206 207#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */ 208#define FAULT_FLAG_MKWRITE 0x02 /* Fault was mkwrite of existing pte */ 209#define FAULT_FLAG_ALLOW_RETRY 0x04 /* Retry fault if blocking */ 210#define FAULT_FLAG_RETRY_NOWAIT 0x08 /* Don't drop mmap_sem and wait when retrying */ 211#define FAULT_FLAG_KILLABLE 0x10 /* The fault task is in SIGKILL killable region */ 212#define FAULT_FLAG_TRIED 0x20 /* Second try */ 213#define FAULT_FLAG_USER 0x40 /* The fault originated in userspace */ 214 215/* 216 * vm_fault is filled by the the pagefault handler and passed to the vma's 217 * ->fault function. The vma's ->fault is responsible for returning a bitmask 218 * of VM_FAULT_xxx flags that give details about how the fault was handled. 219 * 220 * pgoff should be used in favour of virtual_address, if possible. 221 */ 222struct vm_fault { 223 unsigned int flags; /* FAULT_FLAG_xxx flags */ 224 pgoff_t pgoff; /* Logical page offset based on vma */ 225 void __user *virtual_address; /* Faulting virtual address */ 226 227 struct page *cow_page; /* Handler may choose to COW */ 228 struct page *page; /* ->fault handlers should return a 229 * page here, unless VM_FAULT_NOPAGE 230 * is set (which is also implied by 231 * VM_FAULT_ERROR). 232 */ 233 /* for ->map_pages() only */ 234 pgoff_t max_pgoff; /* map pages for offset from pgoff till 235 * max_pgoff inclusive */ 236 pte_t *pte; /* pte entry associated with ->pgoff */ 237}; 238 239/* 240 * These are the virtual MM functions - opening of an area, closing and 241 * unmapping it (needed to keep files on disk up-to-date etc), pointer 242 * to the functions called when a no-page or a wp-page exception occurs. 243 */ 244struct vm_operations_struct { 245 void (*open)(struct vm_area_struct * area); 246 void (*close)(struct vm_area_struct * area); 247 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf); 248 void (*map_pages)(struct vm_area_struct *vma, struct vm_fault *vmf); 249 250 /* notification that a previously read-only page is about to become 251 * writable, if an error is returned it will cause a SIGBUS */ 252 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf); 253 254 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */ 255 int (*pfn_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf); 256 257 /* called by access_process_vm when get_user_pages() fails, typically 258 * for use by special VMAs that can switch between memory and hardware 259 */ 260 int (*access)(struct vm_area_struct *vma, unsigned long addr, 261 void *buf, int len, int write); 262 263 /* Called by the /proc/PID/maps code to ask the vma whether it 264 * has a special name. Returning non-NULL will also cause this 265 * vma to be dumped unconditionally. */ 266 const char *(*name)(struct vm_area_struct *vma); 267 268#ifdef CONFIG_NUMA 269 /* 270 * set_policy() op must add a reference to any non-NULL @new mempolicy 271 * to hold the policy upon return. Caller should pass NULL @new to 272 * remove a policy and fall back to surrounding context--i.e. do not 273 * install a MPOL_DEFAULT policy, nor the task or system default 274 * mempolicy. 275 */ 276 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new); 277 278 /* 279 * get_policy() op must add reference [mpol_get()] to any policy at 280 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure 281 * in mm/mempolicy.c will do this automatically. 282 * get_policy() must NOT add a ref if the policy at (vma,addr) is not 283 * marked as MPOL_SHARED. vma policies are protected by the mmap_sem. 284 * If no [shared/vma] mempolicy exists at the addr, get_policy() op 285 * must return NULL--i.e., do not "fallback" to task or system default 286 * policy. 287 */ 288 struct mempolicy *(*get_policy)(struct vm_area_struct *vma, 289 unsigned long addr); 290#endif 291 /* 292 * Called by vm_normal_page() for special PTEs to find the 293 * page for @addr. This is useful if the default behavior 294 * (using pte_page()) would not find the correct page. 295 */ 296 struct page *(*find_special_page)(struct vm_area_struct *vma, 297 unsigned long addr); 298}; 299 300struct mmu_gather; 301struct inode; 302 303#define page_private(page) ((page)->private) 304#define set_page_private(page, v) ((page)->private = (v)) 305 306/* It's valid only if the page is free path or free_list */ 307static inline void set_freepage_migratetype(struct page *page, int migratetype) 308{ 309 page->index = migratetype; 310} 311 312/* It's valid only if the page is free path or free_list */ 313static inline int get_freepage_migratetype(struct page *page) 314{ 315 return page->index; 316} 317 318/* 319 * FIXME: take this include out, include page-flags.h in 320 * files which need it (119 of them) 321 */ 322#include <linux/page-flags.h> 323#include <linux/huge_mm.h> 324 325/* 326 * Methods to modify the page usage count. 327 * 328 * What counts for a page usage: 329 * - cache mapping (page->mapping) 330 * - private data (page->private) 331 * - page mapped in a task's page tables, each mapping 332 * is counted separately 333 * 334 * Also, many kernel routines increase the page count before a critical 335 * routine so they can be sure the page doesn't go away from under them. 336 */ 337 338/* 339 * Drop a ref, return true if the refcount fell to zero (the page has no users) 340 */ 341static inline int put_page_testzero(struct page *page) 342{ 343 VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page); 344 return atomic_dec_and_test(&page->_count); 345} 346 347/* 348 * Try to grab a ref unless the page has a refcount of zero, return false if 349 * that is the case. 350 * This can be called when MMU is off so it must not access 351 * any of the virtual mappings. 352 */ 353static inline int get_page_unless_zero(struct page *page) 354{ 355 return atomic_inc_not_zero(&page->_count); 356} 357 358/* 359 * Try to drop a ref unless the page has a refcount of one, return false if 360 * that is the case. 361 * This is to make sure that the refcount won't become zero after this drop. 362 * This can be called when MMU is off so it must not access 363 * any of the virtual mappings. 364 */ 365static inline int put_page_unless_one(struct page *page) 366{ 367 return atomic_add_unless(&page->_count, -1, 1); 368} 369 370extern int page_is_ram(unsigned long pfn); 371extern int region_is_ram(resource_size_t phys_addr, unsigned long size); 372 373/* Support for virtually mapped pages */ 374struct page *vmalloc_to_page(const void *addr); 375unsigned long vmalloc_to_pfn(const void *addr); 376 377/* 378 * Determine if an address is within the vmalloc range 379 * 380 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there 381 * is no special casing required. 382 */ 383static inline int is_vmalloc_addr(const void *x) 384{ 385#ifdef CONFIG_MMU 386 unsigned long addr = (unsigned long)x; 387 388 return addr >= VMALLOC_START && addr < VMALLOC_END; 389#else 390 return 0; 391#endif 392} 393#ifdef CONFIG_MMU 394extern int is_vmalloc_or_module_addr(const void *x); 395#else 396static inline int is_vmalloc_or_module_addr(const void *x) 397{ 398 return 0; 399} 400#endif 401 402extern void kvfree(const void *addr); 403 404static inline void compound_lock(struct page *page) 405{ 406#ifdef CONFIG_TRANSPARENT_HUGEPAGE 407 VM_BUG_ON_PAGE(PageSlab(page), page); 408 bit_spin_lock(PG_compound_lock, &page->flags); 409#endif 410} 411 412static inline void compound_unlock(struct page *page) 413{ 414#ifdef CONFIG_TRANSPARENT_HUGEPAGE 415 VM_BUG_ON_PAGE(PageSlab(page), page); 416 bit_spin_unlock(PG_compound_lock, &page->flags); 417#endif 418} 419 420static inline unsigned long compound_lock_irqsave(struct page *page) 421{ 422 unsigned long uninitialized_var(flags); 423#ifdef CONFIG_TRANSPARENT_HUGEPAGE 424 local_irq_save(flags); 425 compound_lock(page); 426#endif 427 return flags; 428} 429 430static inline void compound_unlock_irqrestore(struct page *page, 431 unsigned long flags) 432{ 433#ifdef CONFIG_TRANSPARENT_HUGEPAGE 434 compound_unlock(page); 435 local_irq_restore(flags); 436#endif 437} 438 439static inline struct page *compound_head_by_tail(struct page *tail) 440{ 441 struct page *head = tail->first_page; 442 443 /* 444 * page->first_page may be a dangling pointer to an old 445 * compound page, so recheck that it is still a tail 446 * page before returning. 447 */ 448 smp_rmb(); 449 if (likely(PageTail(tail))) 450 return head; 451 return tail; 452} 453 454/* 455 * Since either compound page could be dismantled asynchronously in THP 456 * or we access asynchronously arbitrary positioned struct page, there 457 * would be tail flag race. To handle this race, we should call 458 * smp_rmb() before checking tail flag. compound_head_by_tail() did it. 459 */ 460static inline struct page *compound_head(struct page *page) 461{ 462 if (unlikely(PageTail(page))) 463 return compound_head_by_tail(page); 464 return page; 465} 466 467/* 468 * If we access compound page synchronously such as access to 469 * allocated page, there is no need to handle tail flag race, so we can 470 * check tail flag directly without any synchronization primitive. 471 */ 472static inline struct page *compound_head_fast(struct page *page) 473{ 474 if (unlikely(PageTail(page))) 475 return page->first_page; 476 return page; 477} 478 479/* 480 * The atomic page->_mapcount, starts from -1: so that transitions 481 * both from it and to it can be tracked, using atomic_inc_and_test 482 * and atomic_add_negative(-1). 483 */ 484static inline void page_mapcount_reset(struct page *page) 485{ 486 atomic_set(&(page)->_mapcount, -1); 487} 488 489static inline int page_mapcount(struct page *page) 490{ 491 VM_BUG_ON_PAGE(PageSlab(page), page); 492 return atomic_read(&page->_mapcount) + 1; 493} 494 495static inline int page_count(struct page *page) 496{ 497 return atomic_read(&compound_head(page)->_count); 498} 499 500static inline bool __compound_tail_refcounted(struct page *page) 501{ 502 return !PageSlab(page) && !PageHeadHuge(page); 503} 504 505/* 506 * This takes a head page as parameter and tells if the 507 * tail page reference counting can be skipped. 508 * 509 * For this to be safe, PageSlab and PageHeadHuge must remain true on 510 * any given page where they return true here, until all tail pins 511 * have been released. 512 */ 513static inline bool compound_tail_refcounted(struct page *page) 514{ 515 VM_BUG_ON_PAGE(!PageHead(page), page); 516 return __compound_tail_refcounted(page); 517} 518 519static inline void get_huge_page_tail(struct page *page) 520{ 521 /* 522 * __split_huge_page_refcount() cannot run from under us. 523 */ 524 VM_BUG_ON_PAGE(!PageTail(page), page); 525 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 526 VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page); 527 if (compound_tail_refcounted(page->first_page)) 528 atomic_inc(&page->_mapcount); 529} 530 531extern bool __get_page_tail(struct page *page); 532 533static inline void get_page(struct page *page) 534{ 535 if (unlikely(PageTail(page))) 536 if (likely(__get_page_tail(page))) 537 return; 538 /* 539 * Getting a normal page or the head of a compound page 540 * requires to already have an elevated page->_count. 541 */ 542 VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page); 543 atomic_inc(&page->_count); 544} 545 546static inline struct page *virt_to_head_page(const void *x) 547{ 548 struct page *page = virt_to_page(x); 549 550 /* 551 * We don't need to worry about synchronization of tail flag 552 * when we call virt_to_head_page() since it is only called for 553 * already allocated page and this page won't be freed until 554 * this virt_to_head_page() is finished. So use _fast variant. 555 */ 556 return compound_head_fast(page); 557} 558 559/* 560 * Setup the page count before being freed into the page allocator for 561 * the first time (boot or memory hotplug) 562 */ 563static inline void init_page_count(struct page *page) 564{ 565 atomic_set(&page->_count, 1); 566} 567 568void put_page(struct page *page); 569void put_pages_list(struct list_head *pages); 570 571void split_page(struct page *page, unsigned int order); 572int split_free_page(struct page *page); 573 574/* 575 * Compound pages have a destructor function. Provide a 576 * prototype for that function and accessor functions. 577 * These are _only_ valid on the head of a PG_compound page. 578 */ 579 580static inline void set_compound_page_dtor(struct page *page, 581 compound_page_dtor *dtor) 582{ 583 page[1].compound_dtor = dtor; 584} 585 586static inline compound_page_dtor *get_compound_page_dtor(struct page *page) 587{ 588 return page[1].compound_dtor; 589} 590 591static inline unsigned int compound_order(struct page *page) 592{ 593 if (!PageHead(page)) 594 return 0; 595 return page[1].compound_order; 596} 597 598static inline void set_compound_order(struct page *page, unsigned long order) 599{ 600 page[1].compound_order = order; 601} 602 603#ifdef CONFIG_MMU 604/* 605 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when 606 * servicing faults for write access. In the normal case, do always want 607 * pte_mkwrite. But get_user_pages can cause write faults for mappings 608 * that do not have writing enabled, when used by access_process_vm. 609 */ 610static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) 611{ 612 if (likely(vma->vm_flags & VM_WRITE)) 613 pte = pte_mkwrite(pte); 614 return pte; 615} 616 617void do_set_pte(struct vm_area_struct *vma, unsigned long address, 618 struct page *page, pte_t *pte, bool write, bool anon); 619#endif 620 621/* 622 * Multiple processes may "see" the same page. E.g. for untouched 623 * mappings of /dev/null, all processes see the same page full of 624 * zeroes, and text pages of executables and shared libraries have 625 * only one copy in memory, at most, normally. 626 * 627 * For the non-reserved pages, page_count(page) denotes a reference count. 628 * page_count() == 0 means the page is free. page->lru is then used for 629 * freelist management in the buddy allocator. 630 * page_count() > 0 means the page has been allocated. 631 * 632 * Pages are allocated by the slab allocator in order to provide memory 633 * to kmalloc and kmem_cache_alloc. In this case, the management of the 634 * page, and the fields in 'struct page' are the responsibility of mm/slab.c 635 * unless a particular usage is carefully commented. (the responsibility of 636 * freeing the kmalloc memory is the caller's, of course). 637 * 638 * A page may be used by anyone else who does a __get_free_page(). 639 * In this case, page_count still tracks the references, and should only 640 * be used through the normal accessor functions. The top bits of page->flags 641 * and page->virtual store page management information, but all other fields 642 * are unused and could be used privately, carefully. The management of this 643 * page is the responsibility of the one who allocated it, and those who have 644 * subsequently been given references to it. 645 * 646 * The other pages (we may call them "pagecache pages") are completely 647 * managed by the Linux memory manager: I/O, buffers, swapping etc. 648 * The following discussion applies only to them. 649 * 650 * A pagecache page contains an opaque `private' member, which belongs to the 651 * page's address_space. Usually, this is the address of a circular list of 652 * the page's disk buffers. PG_private must be set to tell the VM to call 653 * into the filesystem to release these pages. 654 * 655 * A page may belong to an inode's memory mapping. In this case, page->mapping 656 * is the pointer to the inode, and page->index is the file offset of the page, 657 * in units of PAGE_CACHE_SIZE. 658 * 659 * If pagecache pages are not associated with an inode, they are said to be 660 * anonymous pages. These may become associated with the swapcache, and in that 661 * case PG_swapcache is set, and page->private is an offset into the swapcache. 662 * 663 * In either case (swapcache or inode backed), the pagecache itself holds one 664 * reference to the page. Setting PG_private should also increment the 665 * refcount. The each user mapping also has a reference to the page. 666 * 667 * The pagecache pages are stored in a per-mapping radix tree, which is 668 * rooted at mapping->page_tree, and indexed by offset. 669 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space 670 * lists, we instead now tag pages as dirty/writeback in the radix tree. 671 * 672 * All pagecache pages may be subject to I/O: 673 * - inode pages may need to be read from disk, 674 * - inode pages which have been modified and are MAP_SHARED may need 675 * to be written back to the inode on disk, 676 * - anonymous pages (including MAP_PRIVATE file mappings) which have been 677 * modified may need to be swapped out to swap space and (later) to be read 678 * back into memory. 679 */ 680 681/* 682 * The zone field is never updated after free_area_init_core() 683 * sets it, so none of the operations on it need to be atomic. 684 */ 685 686/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */ 687#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) 688#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) 689#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) 690#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) 691 692/* 693 * Define the bit shifts to access each section. For non-existent 694 * sections we define the shift as 0; that plus a 0 mask ensures 695 * the compiler will optimise away reference to them. 696 */ 697#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) 698#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) 699#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) 700#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0)) 701 702/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ 703#ifdef NODE_NOT_IN_PAGE_FLAGS 704#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT) 705#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \ 706 SECTIONS_PGOFF : ZONES_PGOFF) 707#else 708#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT) 709#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \ 710 NODES_PGOFF : ZONES_PGOFF) 711#endif 712 713#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0)) 714 715#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS 716#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS 717#endif 718 719#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) 720#define NODES_MASK ((1UL << NODES_WIDTH) - 1) 721#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) 722#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1) 723#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) 724 725static inline enum zone_type page_zonenum(const struct page *page) 726{ 727 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; 728} 729 730#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 731#define SECTION_IN_PAGE_FLAGS 732#endif 733 734/* 735 * The identification function is mainly used by the buddy allocator for 736 * determining if two pages could be buddies. We are not really identifying 737 * the zone since we could be using the section number id if we do not have 738 * node id available in page flags. 739 * We only guarantee that it will return the same value for two combinable 740 * pages in a zone. 741 */ 742static inline int page_zone_id(struct page *page) 743{ 744 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK; 745} 746 747static inline int zone_to_nid(struct zone *zone) 748{ 749#ifdef CONFIG_NUMA 750 return zone->node; 751#else 752 return 0; 753#endif 754} 755 756#ifdef NODE_NOT_IN_PAGE_FLAGS 757extern int page_to_nid(const struct page *page); 758#else 759static inline int page_to_nid(const struct page *page) 760{ 761 return (page->flags >> NODES_PGSHIFT) & NODES_MASK; 762} 763#endif 764 765#ifdef CONFIG_NUMA_BALANCING 766static inline int cpu_pid_to_cpupid(int cpu, int pid) 767{ 768 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK); 769} 770 771static inline int cpupid_to_pid(int cpupid) 772{ 773 return cpupid & LAST__PID_MASK; 774} 775 776static inline int cpupid_to_cpu(int cpupid) 777{ 778 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK; 779} 780 781static inline int cpupid_to_nid(int cpupid) 782{ 783 return cpu_to_node(cpupid_to_cpu(cpupid)); 784} 785 786static inline bool cpupid_pid_unset(int cpupid) 787{ 788 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK); 789} 790 791static inline bool cpupid_cpu_unset(int cpupid) 792{ 793 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK); 794} 795 796static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid) 797{ 798 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid); 799} 800 801#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid) 802#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 803static inline int page_cpupid_xchg_last(struct page *page, int cpupid) 804{ 805 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK); 806} 807 808static inline int page_cpupid_last(struct page *page) 809{ 810 return page->_last_cpupid; 811} 812static inline void page_cpupid_reset_last(struct page *page) 813{ 814 page->_last_cpupid = -1 & LAST_CPUPID_MASK; 815} 816#else 817static inline int page_cpupid_last(struct page *page) 818{ 819 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; 820} 821 822extern int page_cpupid_xchg_last(struct page *page, int cpupid); 823 824static inline void page_cpupid_reset_last(struct page *page) 825{ 826 int cpupid = (1 << LAST_CPUPID_SHIFT) - 1; 827 828 page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT); 829 page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT; 830} 831#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */ 832#else /* !CONFIG_NUMA_BALANCING */ 833static inline int page_cpupid_xchg_last(struct page *page, int cpupid) 834{ 835 return page_to_nid(page); /* XXX */ 836} 837 838static inline int page_cpupid_last(struct page *page) 839{ 840 return page_to_nid(page); /* XXX */ 841} 842 843static inline int cpupid_to_nid(int cpupid) 844{ 845 return -1; 846} 847 848static inline int cpupid_to_pid(int cpupid) 849{ 850 return -1; 851} 852 853static inline int cpupid_to_cpu(int cpupid) 854{ 855 return -1; 856} 857 858static inline int cpu_pid_to_cpupid(int nid, int pid) 859{ 860 return -1; 861} 862 863static inline bool cpupid_pid_unset(int cpupid) 864{ 865 return 1; 866} 867 868static inline void page_cpupid_reset_last(struct page *page) 869{ 870} 871 872static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) 873{ 874 return false; 875} 876#endif /* CONFIG_NUMA_BALANCING */ 877 878static inline struct zone *page_zone(const struct page *page) 879{ 880 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; 881} 882 883#ifdef SECTION_IN_PAGE_FLAGS 884static inline void set_page_section(struct page *page, unsigned long section) 885{ 886 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); 887 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; 888} 889 890static inline unsigned long page_to_section(const struct page *page) 891{ 892 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; 893} 894#endif 895 896static inline void set_page_zone(struct page *page, enum zone_type zone) 897{ 898 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); 899 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT; 900} 901 902static inline void set_page_node(struct page *page, unsigned long node) 903{ 904 page->flags &= ~(NODES_MASK << NODES_PGSHIFT); 905 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT; 906} 907 908static inline void set_page_links(struct page *page, enum zone_type zone, 909 unsigned long node, unsigned long pfn) 910{ 911 set_page_zone(page, zone); 912 set_page_node(page, node); 913#ifdef SECTION_IN_PAGE_FLAGS 914 set_page_section(page, pfn_to_section_nr(pfn)); 915#endif 916} 917 918/* 919 * Some inline functions in vmstat.h depend on page_zone() 920 */ 921#include <linux/vmstat.h> 922 923static __always_inline void *lowmem_page_address(const struct page *page) 924{ 925 return __va(PFN_PHYS(page_to_pfn(page))); 926} 927 928#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) 929#define HASHED_PAGE_VIRTUAL 930#endif 931 932#if defined(WANT_PAGE_VIRTUAL) 933static inline void *page_address(const struct page *page) 934{ 935 return page->virtual; 936} 937static inline void set_page_address(struct page *page, void *address) 938{ 939 page->virtual = address; 940} 941#define page_address_init() do { } while(0) 942#endif 943 944#if defined(HASHED_PAGE_VIRTUAL) 945void *page_address(const struct page *page); 946void set_page_address(struct page *page, void *virtual); 947void page_address_init(void); 948#endif 949 950#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL) 951#define page_address(page) lowmem_page_address(page) 952#define set_page_address(page, address) do { } while(0) 953#define page_address_init() do { } while(0) 954#endif 955 956extern void *page_rmapping(struct page *page); 957extern struct anon_vma *page_anon_vma(struct page *page); 958extern struct address_space *page_mapping(struct page *page); 959 960extern struct address_space *__page_file_mapping(struct page *); 961 962static inline 963struct address_space *page_file_mapping(struct page *page) 964{ 965 if (unlikely(PageSwapCache(page))) 966 return __page_file_mapping(page); 967 968 return page->mapping; 969} 970 971/* 972 * Return the pagecache index of the passed page. Regular pagecache pages 973 * use ->index whereas swapcache pages use ->private 974 */ 975static inline pgoff_t page_index(struct page *page) 976{ 977 if (unlikely(PageSwapCache(page))) 978 return page_private(page); 979 return page->index; 980} 981 982extern pgoff_t __page_file_index(struct page *page); 983 984/* 985 * Return the file index of the page. Regular pagecache pages use ->index 986 * whereas swapcache pages use swp_offset(->private) 987 */ 988static inline pgoff_t page_file_index(struct page *page) 989{ 990 if (unlikely(PageSwapCache(page))) 991 return __page_file_index(page); 992 993 return page->index; 994} 995 996/* 997 * Return true if this page is mapped into pagetables. 998 */ 999static inline int page_mapped(struct page *page) 1000{ 1001 return atomic_read(&(page)->_mapcount) >= 0; 1002} 1003 1004/* 1005 * Return true only if the page has been allocated with 1006 * ALLOC_NO_WATERMARKS and the low watermark was not 1007 * met implying that the system is under some pressure. 1008 */ 1009static inline bool page_is_pfmemalloc(struct page *page) 1010{ 1011 /* 1012 * Page index cannot be this large so this must be 1013 * a pfmemalloc page. 1014 */ 1015 return page->index == -1UL; 1016} 1017 1018/* 1019 * Only to be called by the page allocator on a freshly allocated 1020 * page. 1021 */ 1022static inline void set_page_pfmemalloc(struct page *page) 1023{ 1024 page->index = -1UL; 1025} 1026 1027static inline void clear_page_pfmemalloc(struct page *page) 1028{ 1029 page->index = 0; 1030} 1031 1032/* 1033 * Different kinds of faults, as returned by handle_mm_fault(). 1034 * Used to decide whether a process gets delivered SIGBUS or 1035 * just gets major/minor fault counters bumped up. 1036 */ 1037 1038#define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */ 1039 1040#define VM_FAULT_OOM 0x0001 1041#define VM_FAULT_SIGBUS 0x0002 1042#define VM_FAULT_MAJOR 0x0004 1043#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */ 1044#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */ 1045#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */ 1046#define VM_FAULT_SIGSEGV 0x0040 1047 1048#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */ 1049#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */ 1050#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */ 1051#define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */ 1052 1053#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */ 1054 1055#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \ 1056 VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE | \ 1057 VM_FAULT_FALLBACK) 1058 1059/* Encode hstate index for a hwpoisoned large page */ 1060#define VM_FAULT_SET_HINDEX(x) ((x) << 12) 1061#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf) 1062 1063/* 1064 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM. 1065 */ 1066extern void pagefault_out_of_memory(void); 1067 1068#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) 1069 1070/* 1071 * Flags passed to show_mem() and show_free_areas() to suppress output in 1072 * various contexts. 1073 */ 1074#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */ 1075 1076extern void show_free_areas(unsigned int flags); 1077extern bool skip_free_areas_node(unsigned int flags, int nid); 1078 1079int shmem_zero_setup(struct vm_area_struct *); 1080#ifdef CONFIG_SHMEM 1081bool shmem_mapping(struct address_space *mapping); 1082#else 1083static inline bool shmem_mapping(struct address_space *mapping) 1084{ 1085 return false; 1086} 1087#endif 1088 1089extern int can_do_mlock(void); 1090extern int user_shm_lock(size_t, struct user_struct *); 1091extern void user_shm_unlock(size_t, struct user_struct *); 1092 1093/* 1094 * Parameter block passed down to zap_pte_range in exceptional cases. 1095 */ 1096struct zap_details { 1097 struct address_space *check_mapping; /* Check page->mapping if set */ 1098 pgoff_t first_index; /* Lowest page->index to unmap */ 1099 pgoff_t last_index; /* Highest page->index to unmap */ 1100}; 1101 1102struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 1103 pte_t pte); 1104 1105int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 1106 unsigned long size); 1107void zap_page_range(struct vm_area_struct *vma, unsigned long address, 1108 unsigned long size, struct zap_details *); 1109void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma, 1110 unsigned long start, unsigned long end); 1111 1112/** 1113 * mm_walk - callbacks for walk_page_range 1114 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry 1115 * this handler is required to be able to handle 1116 * pmd_trans_huge() pmds. They may simply choose to 1117 * split_huge_page() instead of handling it explicitly. 1118 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry 1119 * @pte_hole: if set, called for each hole at all levels 1120 * @hugetlb_entry: if set, called for each hugetlb entry 1121 * @test_walk: caller specific callback function to determine whether 1122 * we walk over the current vma or not. A positive returned 1123 * value means "do page table walk over the current vma," 1124 * and a negative one means "abort current page table walk 1125 * right now." 0 means "skip the current vma." 1126 * @mm: mm_struct representing the target process of page table walk 1127 * @vma: vma currently walked (NULL if walking outside vmas) 1128 * @private: private data for callbacks' usage 1129 * 1130 * (see the comment on walk_page_range() for more details) 1131 */ 1132struct mm_walk { 1133 int (*pmd_entry)(pmd_t *pmd, unsigned long addr, 1134 unsigned long next, struct mm_walk *walk); 1135 int (*pte_entry)(pte_t *pte, unsigned long addr, 1136 unsigned long next, struct mm_walk *walk); 1137 int (*pte_hole)(unsigned long addr, unsigned long next, 1138 struct mm_walk *walk); 1139 int (*hugetlb_entry)(pte_t *pte, unsigned long hmask, 1140 unsigned long addr, unsigned long next, 1141 struct mm_walk *walk); 1142 int (*test_walk)(unsigned long addr, unsigned long next, 1143 struct mm_walk *walk); 1144 struct mm_struct *mm; 1145 struct vm_area_struct *vma; 1146 void *private; 1147}; 1148 1149int walk_page_range(unsigned long addr, unsigned long end, 1150 struct mm_walk *walk); 1151int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk); 1152void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, 1153 unsigned long end, unsigned long floor, unsigned long ceiling); 1154int copy_page_range(struct mm_struct *dst, struct mm_struct *src, 1155 struct vm_area_struct *vma); 1156void unmap_mapping_range(struct address_space *mapping, 1157 loff_t const holebegin, loff_t const holelen, int even_cows); 1158int follow_pfn(struct vm_area_struct *vma, unsigned long address, 1159 unsigned long *pfn); 1160int follow_phys(struct vm_area_struct *vma, unsigned long address, 1161 unsigned int flags, unsigned long *prot, resource_size_t *phys); 1162int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 1163 void *buf, int len, int write); 1164 1165static inline void unmap_shared_mapping_range(struct address_space *mapping, 1166 loff_t const holebegin, loff_t const holelen) 1167{ 1168 unmap_mapping_range(mapping, holebegin, holelen, 0); 1169} 1170 1171extern void truncate_pagecache(struct inode *inode, loff_t new); 1172extern void truncate_setsize(struct inode *inode, loff_t newsize); 1173void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to); 1174void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end); 1175int truncate_inode_page(struct address_space *mapping, struct page *page); 1176int generic_error_remove_page(struct address_space *mapping, struct page *page); 1177int invalidate_inode_page(struct page *page); 1178 1179#ifdef CONFIG_MMU 1180extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, 1181 unsigned long address, unsigned int flags); 1182extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, 1183 unsigned long address, unsigned int fault_flags); 1184#else 1185static inline int handle_mm_fault(struct mm_struct *mm, 1186 struct vm_area_struct *vma, unsigned long address, 1187 unsigned int flags) 1188{ 1189 /* should never happen if there's no MMU */ 1190 BUG(); 1191 return VM_FAULT_SIGBUS; 1192} 1193static inline int fixup_user_fault(struct task_struct *tsk, 1194 struct mm_struct *mm, unsigned long address, 1195 unsigned int fault_flags) 1196{ 1197 /* should never happen if there's no MMU */ 1198 BUG(); 1199 return -EFAULT; 1200} 1201#endif 1202 1203extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write); 1204extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, 1205 void *buf, int len, int write); 1206 1207long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1208 unsigned long start, unsigned long nr_pages, 1209 unsigned int foll_flags, struct page **pages, 1210 struct vm_area_struct **vmas, int *nonblocking); 1211long get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1212 unsigned long start, unsigned long nr_pages, 1213 int write, int force, struct page **pages, 1214 struct vm_area_struct **vmas); 1215long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm, 1216 unsigned long start, unsigned long nr_pages, 1217 int write, int force, struct page **pages, 1218 int *locked); 1219long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm, 1220 unsigned long start, unsigned long nr_pages, 1221 int write, int force, struct page **pages, 1222 unsigned int gup_flags); 1223long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm, 1224 unsigned long start, unsigned long nr_pages, 1225 int write, int force, struct page **pages); 1226int get_user_pages_fast(unsigned long start, int nr_pages, int write, 1227 struct page **pages); 1228struct kvec; 1229int get_kernel_pages(const struct kvec *iov, int nr_pages, int write, 1230 struct page **pages); 1231int get_kernel_page(unsigned long start, int write, struct page **pages); 1232struct page *get_dump_page(unsigned long addr); 1233 1234extern int try_to_release_page(struct page * page, gfp_t gfp_mask); 1235extern void do_invalidatepage(struct page *page, unsigned int offset, 1236 unsigned int length); 1237 1238int __set_page_dirty_nobuffers(struct page *page); 1239int __set_page_dirty_no_writeback(struct page *page); 1240int redirty_page_for_writepage(struct writeback_control *wbc, 1241 struct page *page); 1242void account_page_dirtied(struct page *page, struct address_space *mapping); 1243void account_page_cleaned(struct page *page, struct address_space *mapping); 1244int set_page_dirty(struct page *page); 1245int set_page_dirty_lock(struct page *page); 1246int clear_page_dirty_for_io(struct page *page); 1247 1248int get_cmdline(struct task_struct *task, char *buffer, int buflen); 1249 1250/* Is the vma a continuation of the stack vma above it? */ 1251static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr) 1252{ 1253 return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN); 1254} 1255 1256static inline int stack_guard_page_start(struct vm_area_struct *vma, 1257 unsigned long addr) 1258{ 1259 return (vma->vm_flags & VM_GROWSDOWN) && 1260 (vma->vm_start == addr) && 1261 !vma_growsdown(vma->vm_prev, addr); 1262} 1263 1264/* Is the vma a continuation of the stack vma below it? */ 1265static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr) 1266{ 1267 return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP); 1268} 1269 1270static inline int stack_guard_page_end(struct vm_area_struct *vma, 1271 unsigned long addr) 1272{ 1273 return (vma->vm_flags & VM_GROWSUP) && 1274 (vma->vm_end == addr) && 1275 !vma_growsup(vma->vm_next, addr); 1276} 1277 1278extern struct task_struct *task_of_stack(struct task_struct *task, 1279 struct vm_area_struct *vma, bool in_group); 1280 1281extern unsigned long move_page_tables(struct vm_area_struct *vma, 1282 unsigned long old_addr, struct vm_area_struct *new_vma, 1283 unsigned long new_addr, unsigned long len, 1284 bool need_rmap_locks); 1285extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, 1286 unsigned long end, pgprot_t newprot, 1287 int dirty_accountable, int prot_numa); 1288extern int mprotect_fixup(struct vm_area_struct *vma, 1289 struct vm_area_struct **pprev, unsigned long start, 1290 unsigned long end, unsigned long newflags); 1291 1292/* 1293 * doesn't attempt to fault and will return short. 1294 */ 1295int __get_user_pages_fast(unsigned long start, int nr_pages, int write, 1296 struct page **pages); 1297/* 1298 * per-process(per-mm_struct) statistics. 1299 */ 1300static inline unsigned long get_mm_counter(struct mm_struct *mm, int member) 1301{ 1302 long val = atomic_long_read(&mm->rss_stat.count[member]); 1303 1304#ifdef SPLIT_RSS_COUNTING 1305 /* 1306 * counter is updated in asynchronous manner and may go to minus. 1307 * But it's never be expected number for users. 1308 */ 1309 if (val < 0) 1310 val = 0; 1311#endif 1312 return (unsigned long)val; 1313} 1314 1315static inline void add_mm_counter(struct mm_struct *mm, int member, long value) 1316{ 1317 atomic_long_add(value, &mm->rss_stat.count[member]); 1318} 1319 1320static inline void inc_mm_counter(struct mm_struct *mm, int member) 1321{ 1322 atomic_long_inc(&mm->rss_stat.count[member]); 1323} 1324 1325static inline void dec_mm_counter(struct mm_struct *mm, int member) 1326{ 1327 atomic_long_dec(&mm->rss_stat.count[member]); 1328} 1329 1330static inline unsigned long get_mm_rss(struct mm_struct *mm) 1331{ 1332 return get_mm_counter(mm, MM_FILEPAGES) + 1333 get_mm_counter(mm, MM_ANONPAGES); 1334} 1335 1336static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm) 1337{ 1338 return max(mm->hiwater_rss, get_mm_rss(mm)); 1339} 1340 1341static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm) 1342{ 1343 return max(mm->hiwater_vm, mm->total_vm); 1344} 1345 1346static inline void update_hiwater_rss(struct mm_struct *mm) 1347{ 1348 unsigned long _rss = get_mm_rss(mm); 1349 1350 if ((mm)->hiwater_rss < _rss) 1351 (mm)->hiwater_rss = _rss; 1352} 1353 1354static inline void update_hiwater_vm(struct mm_struct *mm) 1355{ 1356 if (mm->hiwater_vm < mm->total_vm) 1357 mm->hiwater_vm = mm->total_vm; 1358} 1359 1360static inline void reset_mm_hiwater_rss(struct mm_struct *mm) 1361{ 1362 mm->hiwater_rss = get_mm_rss(mm); 1363} 1364 1365static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, 1366 struct mm_struct *mm) 1367{ 1368 unsigned long hiwater_rss = get_mm_hiwater_rss(mm); 1369 1370 if (*maxrss < hiwater_rss) 1371 *maxrss = hiwater_rss; 1372} 1373 1374#if defined(SPLIT_RSS_COUNTING) 1375void sync_mm_rss(struct mm_struct *mm); 1376#else 1377static inline void sync_mm_rss(struct mm_struct *mm) 1378{ 1379} 1380#endif 1381 1382int vma_wants_writenotify(struct vm_area_struct *vma); 1383 1384extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, 1385 spinlock_t **ptl); 1386static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, 1387 spinlock_t **ptl) 1388{ 1389 pte_t *ptep; 1390 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl)); 1391 return ptep; 1392} 1393 1394#ifdef __PAGETABLE_PUD_FOLDED 1395static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, 1396 unsigned long address) 1397{ 1398 return 0; 1399} 1400#else 1401int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address); 1402#endif 1403 1404#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU) 1405static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud, 1406 unsigned long address) 1407{ 1408 return 0; 1409} 1410 1411static inline void mm_nr_pmds_init(struct mm_struct *mm) {} 1412 1413static inline unsigned long mm_nr_pmds(struct mm_struct *mm) 1414{ 1415 return 0; 1416} 1417 1418static inline void mm_inc_nr_pmds(struct mm_struct *mm) {} 1419static inline void mm_dec_nr_pmds(struct mm_struct *mm) {} 1420 1421#else 1422int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address); 1423 1424static inline void mm_nr_pmds_init(struct mm_struct *mm) 1425{ 1426 atomic_long_set(&mm->nr_pmds, 0); 1427} 1428 1429static inline unsigned long mm_nr_pmds(struct mm_struct *mm) 1430{ 1431 return atomic_long_read(&mm->nr_pmds); 1432} 1433 1434static inline void mm_inc_nr_pmds(struct mm_struct *mm) 1435{ 1436 atomic_long_inc(&mm->nr_pmds); 1437} 1438 1439static inline void mm_dec_nr_pmds(struct mm_struct *mm) 1440{ 1441 atomic_long_dec(&mm->nr_pmds); 1442} 1443#endif 1444 1445int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, 1446 pmd_t *pmd, unsigned long address); 1447int __pte_alloc_kernel(pmd_t *pmd, unsigned long address); 1448 1449/* 1450 * The following ifdef needed to get the 4level-fixup.h header to work. 1451 * Remove it when 4level-fixup.h has been removed. 1452 */ 1453#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK) 1454static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 1455{ 1456 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))? 1457 NULL: pud_offset(pgd, address); 1458} 1459 1460static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 1461{ 1462 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? 1463 NULL: pmd_offset(pud, address); 1464} 1465#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */ 1466 1467#if USE_SPLIT_PTE_PTLOCKS 1468#if ALLOC_SPLIT_PTLOCKS 1469void __init ptlock_cache_init(void); 1470extern bool ptlock_alloc(struct page *page); 1471extern void ptlock_free(struct page *page); 1472 1473static inline spinlock_t *ptlock_ptr(struct page *page) 1474{ 1475 return page->ptl; 1476} 1477#else /* ALLOC_SPLIT_PTLOCKS */ 1478static inline void ptlock_cache_init(void) 1479{ 1480} 1481 1482static inline bool ptlock_alloc(struct page *page) 1483{ 1484 return true; 1485} 1486 1487static inline void ptlock_free(struct page *page) 1488{ 1489} 1490 1491static inline spinlock_t *ptlock_ptr(struct page *page) 1492{ 1493 return &page->ptl; 1494} 1495#endif /* ALLOC_SPLIT_PTLOCKS */ 1496 1497static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 1498{ 1499 return ptlock_ptr(pmd_page(*pmd)); 1500} 1501 1502static inline bool ptlock_init(struct page *page) 1503{ 1504 /* 1505 * prep_new_page() initialize page->private (and therefore page->ptl) 1506 * with 0. Make sure nobody took it in use in between. 1507 * 1508 * It can happen if arch try to use slab for page table allocation: 1509 * slab code uses page->slab_cache and page->first_page (for tail 1510 * pages), which share storage with page->ptl. 1511 */ 1512 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page); 1513 if (!ptlock_alloc(page)) 1514 return false; 1515 spin_lock_init(ptlock_ptr(page)); 1516 return true; 1517} 1518 1519/* Reset page->mapping so free_pages_check won't complain. */ 1520static inline void pte_lock_deinit(struct page *page) 1521{ 1522 page->mapping = NULL; 1523 ptlock_free(page); 1524} 1525 1526#else /* !USE_SPLIT_PTE_PTLOCKS */ 1527/* 1528 * We use mm->page_table_lock to guard all pagetable pages of the mm. 1529 */ 1530static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) 1531{ 1532 return &mm->page_table_lock; 1533} 1534static inline void ptlock_cache_init(void) {} 1535static inline bool ptlock_init(struct page *page) { return true; } 1536static inline void pte_lock_deinit(struct page *page) {} 1537#endif /* USE_SPLIT_PTE_PTLOCKS */ 1538 1539static inline void pgtable_init(void) 1540{ 1541 ptlock_cache_init(); 1542 pgtable_cache_init(); 1543} 1544 1545static inline bool pgtable_page_ctor(struct page *page) 1546{ 1547 inc_zone_page_state(page, NR_PAGETABLE); 1548 return ptlock_init(page); 1549} 1550 1551static inline void pgtable_page_dtor(struct page *page) 1552{ 1553 pte_lock_deinit(page); 1554 dec_zone_page_state(page, NR_PAGETABLE); 1555} 1556 1557#define pte_offset_map_lock(mm, pmd, address, ptlp) \ 1558({ \ 1559 spinlock_t *__ptl = pte_lockptr(mm, pmd); \ 1560 pte_t *__pte = pte_offset_map(pmd, address); \ 1561 *(ptlp) = __ptl; \ 1562 spin_lock(__ptl); \ 1563 __pte; \ 1564}) 1565 1566#define pte_unmap_unlock(pte, ptl) do { \ 1567 spin_unlock(ptl); \ 1568 pte_unmap(pte); \ 1569} while (0) 1570 1571#define pte_alloc_map(mm, vma, pmd, address) \ 1572 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma, \ 1573 pmd, address))? \ 1574 NULL: pte_offset_map(pmd, address)) 1575 1576#define pte_alloc_map_lock(mm, pmd, address, ptlp) \ 1577 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL, \ 1578 pmd, address))? \ 1579 NULL: pte_offset_map_lock(mm, pmd, address, ptlp)) 1580 1581#define pte_alloc_kernel(pmd, address) \ 1582 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \ 1583 NULL: pte_offset_kernel(pmd, address)) 1584 1585#if USE_SPLIT_PMD_PTLOCKS 1586 1587static struct page *pmd_to_page(pmd_t *pmd) 1588{ 1589 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); 1590 return virt_to_page((void *)((unsigned long) pmd & mask)); 1591} 1592 1593static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 1594{ 1595 return ptlock_ptr(pmd_to_page(pmd)); 1596} 1597 1598static inline bool pgtable_pmd_page_ctor(struct page *page) 1599{ 1600#ifdef CONFIG_TRANSPARENT_HUGEPAGE 1601 page->pmd_huge_pte = NULL; 1602#endif 1603 return ptlock_init(page); 1604} 1605 1606static inline void pgtable_pmd_page_dtor(struct page *page) 1607{ 1608#ifdef CONFIG_TRANSPARENT_HUGEPAGE 1609 VM_BUG_ON_PAGE(page->pmd_huge_pte, page); 1610#endif 1611 ptlock_free(page); 1612} 1613 1614#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte) 1615 1616#else 1617 1618static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) 1619{ 1620 return &mm->page_table_lock; 1621} 1622 1623static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; } 1624static inline void pgtable_pmd_page_dtor(struct page *page) {} 1625 1626#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte) 1627 1628#endif 1629 1630static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) 1631{ 1632 spinlock_t *ptl = pmd_lockptr(mm, pmd); 1633 spin_lock(ptl); 1634 return ptl; 1635} 1636 1637extern void free_area_init(unsigned long * zones_size); 1638extern void free_area_init_node(int nid, unsigned long * zones_size, 1639 unsigned long zone_start_pfn, unsigned long *zholes_size); 1640extern void free_initmem(void); 1641 1642/* 1643 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK) 1644 * into the buddy system. The freed pages will be poisoned with pattern 1645 * "poison" if it's within range [0, UCHAR_MAX]. 1646 * Return pages freed into the buddy system. 1647 */ 1648extern unsigned long free_reserved_area(void *start, void *end, 1649 int poison, char *s); 1650 1651#ifdef CONFIG_HIGHMEM 1652/* 1653 * Free a highmem page into the buddy system, adjusting totalhigh_pages 1654 * and totalram_pages. 1655 */ 1656extern void free_highmem_page(struct page *page); 1657#endif 1658 1659extern void adjust_managed_page_count(struct page *page, long count); 1660extern void mem_init_print_info(const char *str); 1661 1662/* Free the reserved page into the buddy system, so it gets managed. */ 1663static inline void __free_reserved_page(struct page *page) 1664{ 1665 ClearPageReserved(page); 1666 init_page_count(page); 1667 __free_page(page); 1668} 1669 1670static inline void free_reserved_page(struct page *page) 1671{ 1672 __free_reserved_page(page); 1673 adjust_managed_page_count(page, 1); 1674} 1675 1676static inline void mark_page_reserved(struct page *page) 1677{ 1678 SetPageReserved(page); 1679 adjust_managed_page_count(page, -1); 1680} 1681 1682/* 1683 * Default method to free all the __init memory into the buddy system. 1684 * The freed pages will be poisoned with pattern "poison" if it's within 1685 * range [0, UCHAR_MAX]. 1686 * Return pages freed into the buddy system. 1687 */ 1688static inline unsigned long free_initmem_default(int poison) 1689{ 1690 extern char __init_begin[], __init_end[]; 1691 1692 return free_reserved_area(&__init_begin, &__init_end, 1693 poison, "unused kernel"); 1694} 1695 1696static inline unsigned long get_num_physpages(void) 1697{ 1698 int nid; 1699 unsigned long phys_pages = 0; 1700 1701 for_each_online_node(nid) 1702 phys_pages += node_present_pages(nid); 1703 1704 return phys_pages; 1705} 1706 1707#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 1708/* 1709 * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its 1710 * zones, allocate the backing mem_map and account for memory holes in a more 1711 * architecture independent manner. This is a substitute for creating the 1712 * zone_sizes[] and zholes_size[] arrays and passing them to 1713 * free_area_init_node() 1714 * 1715 * An architecture is expected to register range of page frames backed by 1716 * physical memory with memblock_add[_node]() before calling 1717 * free_area_init_nodes() passing in the PFN each zone ends at. At a basic 1718 * usage, an architecture is expected to do something like 1719 * 1720 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn, 1721 * max_highmem_pfn}; 1722 * for_each_valid_physical_page_range() 1723 * memblock_add_node(base, size, nid) 1724 * free_area_init_nodes(max_zone_pfns); 1725 * 1726 * free_bootmem_with_active_regions() calls free_bootmem_node() for each 1727 * registered physical page range. Similarly 1728 * sparse_memory_present_with_active_regions() calls memory_present() for 1729 * each range when SPARSEMEM is enabled. 1730 * 1731 * See mm/page_alloc.c for more information on each function exposed by 1732 * CONFIG_HAVE_MEMBLOCK_NODE_MAP. 1733 */ 1734extern void free_area_init_nodes(unsigned long *max_zone_pfn); 1735unsigned long node_map_pfn_alignment(void); 1736unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn, 1737 unsigned long end_pfn); 1738extern unsigned long absent_pages_in_range(unsigned long start_pfn, 1739 unsigned long end_pfn); 1740extern void get_pfn_range_for_nid(unsigned int nid, 1741 unsigned long *start_pfn, unsigned long *end_pfn); 1742extern unsigned long find_min_pfn_with_active_regions(void); 1743extern void free_bootmem_with_active_regions(int nid, 1744 unsigned long max_low_pfn); 1745extern void sparse_memory_present_with_active_regions(int nid); 1746 1747#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 1748 1749#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \ 1750 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) 1751static inline int __early_pfn_to_nid(unsigned long pfn) 1752{ 1753 return 0; 1754} 1755#else 1756/* please see mm/page_alloc.c */ 1757extern int __meminit early_pfn_to_nid(unsigned long pfn); 1758/* there is a per-arch backend function. */ 1759extern int __meminit __early_pfn_to_nid(unsigned long pfn); 1760#endif 1761 1762extern void set_dma_reserve(unsigned long new_dma_reserve); 1763extern void memmap_init_zone(unsigned long, int, unsigned long, 1764 unsigned long, enum memmap_context); 1765extern void setup_per_zone_wmarks(void); 1766extern int __meminit init_per_zone_wmark_min(void); 1767extern void mem_init(void); 1768extern void __init mmap_init(void); 1769extern void show_mem(unsigned int flags); 1770extern void si_meminfo(struct sysinfo * val); 1771extern void si_meminfo_node(struct sysinfo *val, int nid); 1772 1773extern __printf(3, 4) 1774void warn_alloc_failed(gfp_t gfp_mask, unsigned int order, 1775 const char *fmt, ...); 1776 1777extern void setup_per_cpu_pageset(void); 1778 1779extern void zone_pcp_update(struct zone *zone); 1780extern void zone_pcp_reset(struct zone *zone); 1781 1782/* page_alloc.c */ 1783extern int min_free_kbytes; 1784 1785/* nommu.c */ 1786extern atomic_long_t mmap_pages_allocated; 1787extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); 1788 1789/* interval_tree.c */ 1790void vma_interval_tree_insert(struct vm_area_struct *node, 1791 struct rb_root *root); 1792void vma_interval_tree_insert_after(struct vm_area_struct *node, 1793 struct vm_area_struct *prev, 1794 struct rb_root *root); 1795void vma_interval_tree_remove(struct vm_area_struct *node, 1796 struct rb_root *root); 1797struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root, 1798 unsigned long start, unsigned long last); 1799struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, 1800 unsigned long start, unsigned long last); 1801 1802#define vma_interval_tree_foreach(vma, root, start, last) \ 1803 for (vma = vma_interval_tree_iter_first(root, start, last); \ 1804 vma; vma = vma_interval_tree_iter_next(vma, start, last)) 1805 1806void anon_vma_interval_tree_insert(struct anon_vma_chain *node, 1807 struct rb_root *root); 1808void anon_vma_interval_tree_remove(struct anon_vma_chain *node, 1809 struct rb_root *root); 1810struct anon_vma_chain *anon_vma_interval_tree_iter_first( 1811 struct rb_root *root, unsigned long start, unsigned long last); 1812struct anon_vma_chain *anon_vma_interval_tree_iter_next( 1813 struct anon_vma_chain *node, unsigned long start, unsigned long last); 1814#ifdef CONFIG_DEBUG_VM_RB 1815void anon_vma_interval_tree_verify(struct anon_vma_chain *node); 1816#endif 1817 1818#define anon_vma_interval_tree_foreach(avc, root, start, last) \ 1819 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \ 1820 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last)) 1821 1822/* mmap.c */ 1823extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin); 1824extern int vma_adjust(struct vm_area_struct *vma, unsigned long start, 1825 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert); 1826extern struct vm_area_struct *vma_merge(struct mm_struct *, 1827 struct vm_area_struct *prev, unsigned long addr, unsigned long end, 1828 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t, 1829 struct mempolicy *); 1830extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); 1831extern int split_vma(struct mm_struct *, 1832 struct vm_area_struct *, unsigned long addr, int new_below); 1833extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); 1834extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *, 1835 struct rb_node **, struct rb_node *); 1836extern void unlink_file_vma(struct vm_area_struct *); 1837extern struct vm_area_struct *copy_vma(struct vm_area_struct **, 1838 unsigned long addr, unsigned long len, pgoff_t pgoff, 1839 bool *need_rmap_locks); 1840extern void exit_mmap(struct mm_struct *); 1841 1842static inline int check_data_rlimit(unsigned long rlim, 1843 unsigned long new, 1844 unsigned long start, 1845 unsigned long end_data, 1846 unsigned long start_data) 1847{ 1848 if (rlim < RLIM_INFINITY) { 1849 if (((new - start) + (end_data - start_data)) > rlim) 1850 return -ENOSPC; 1851 } 1852 1853 return 0; 1854} 1855 1856extern int mm_take_all_locks(struct mm_struct *mm); 1857extern void mm_drop_all_locks(struct mm_struct *mm); 1858 1859extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); 1860extern struct file *get_mm_exe_file(struct mm_struct *mm); 1861 1862extern int may_expand_vm(struct mm_struct *mm, unsigned long npages); 1863extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm, 1864 unsigned long addr, unsigned long len, 1865 unsigned long flags, 1866 const struct vm_special_mapping *spec); 1867/* This is an obsolete alternative to _install_special_mapping. */ 1868extern int install_special_mapping(struct mm_struct *mm, 1869 unsigned long addr, unsigned long len, 1870 unsigned long flags, struct page **pages); 1871 1872extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); 1873 1874extern unsigned long mmap_region(struct file *file, unsigned long addr, 1875 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff); 1876extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr, 1877 unsigned long len, unsigned long prot, unsigned long flags, 1878 unsigned long pgoff, unsigned long *populate); 1879extern int do_munmap(struct mm_struct *, unsigned long, size_t); 1880 1881#ifdef CONFIG_MMU 1882extern int __mm_populate(unsigned long addr, unsigned long len, 1883 int ignore_errors); 1884static inline void mm_populate(unsigned long addr, unsigned long len) 1885{ 1886 /* Ignore errors */ 1887 (void) __mm_populate(addr, len, 1); 1888} 1889#else 1890static inline void mm_populate(unsigned long addr, unsigned long len) {} 1891#endif 1892 1893/* These take the mm semaphore themselves */ 1894extern unsigned long vm_brk(unsigned long, unsigned long); 1895extern int vm_munmap(unsigned long, size_t); 1896extern unsigned long vm_mmap(struct file *, unsigned long, 1897 unsigned long, unsigned long, 1898 unsigned long, unsigned long); 1899 1900struct vm_unmapped_area_info { 1901#define VM_UNMAPPED_AREA_TOPDOWN 1 1902 unsigned long flags; 1903 unsigned long length; 1904 unsigned long low_limit; 1905 unsigned long high_limit; 1906 unsigned long align_mask; 1907 unsigned long align_offset; 1908}; 1909 1910extern unsigned long unmapped_area(struct vm_unmapped_area_info *info); 1911extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info); 1912 1913/* 1914 * Search for an unmapped address range. 1915 * 1916 * We are looking for a range that: 1917 * - does not intersect with any VMA; 1918 * - is contained within the [low_limit, high_limit) interval; 1919 * - is at least the desired size. 1920 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask) 1921 */ 1922static inline unsigned long 1923vm_unmapped_area(struct vm_unmapped_area_info *info) 1924{ 1925 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) 1926 return unmapped_area_topdown(info); 1927 else 1928 return unmapped_area(info); 1929} 1930 1931/* truncate.c */ 1932extern void truncate_inode_pages(struct address_space *, loff_t); 1933extern void truncate_inode_pages_range(struct address_space *, 1934 loff_t lstart, loff_t lend); 1935extern void truncate_inode_pages_final(struct address_space *); 1936 1937/* generic vm_area_ops exported for stackable file systems */ 1938extern int filemap_fault(struct vm_area_struct *, struct vm_fault *); 1939extern void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf); 1940extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf); 1941 1942/* mm/page-writeback.c */ 1943int write_one_page(struct page *page, int wait); 1944void task_dirty_inc(struct task_struct *tsk); 1945 1946/* readahead.c */ 1947#define VM_MAX_READAHEAD 128 /* kbytes */ 1948#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */ 1949 1950int force_page_cache_readahead(struct address_space *mapping, struct file *filp, 1951 pgoff_t offset, unsigned long nr_to_read); 1952 1953void page_cache_sync_readahead(struct address_space *mapping, 1954 struct file_ra_state *ra, 1955 struct file *filp, 1956 pgoff_t offset, 1957 unsigned long size); 1958 1959void page_cache_async_readahead(struct address_space *mapping, 1960 struct file_ra_state *ra, 1961 struct file *filp, 1962 struct page *pg, 1963 pgoff_t offset, 1964 unsigned long size); 1965 1966unsigned long max_sane_readahead(unsigned long nr); 1967 1968/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */ 1969extern int expand_stack(struct vm_area_struct *vma, unsigned long address); 1970 1971/* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */ 1972extern int expand_downwards(struct vm_area_struct *vma, 1973 unsigned long address); 1974#if VM_GROWSUP 1975extern int expand_upwards(struct vm_area_struct *vma, unsigned long address); 1976#else 1977 #define expand_upwards(vma, address) (0) 1978#endif 1979 1980/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ 1981extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); 1982extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, 1983 struct vm_area_struct **pprev); 1984 1985/* Look up the first VMA which intersects the interval start_addr..end_addr-1, 1986 NULL if none. Assume start_addr < end_addr. */ 1987static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr) 1988{ 1989 struct vm_area_struct * vma = find_vma(mm,start_addr); 1990 1991 if (vma && end_addr <= vma->vm_start) 1992 vma = NULL; 1993 return vma; 1994} 1995 1996static inline unsigned long vma_pages(struct vm_area_struct *vma) 1997{ 1998 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 1999} 2000 2001/* Look up the first VMA which exactly match the interval vm_start ... vm_end */ 2002static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, 2003 unsigned long vm_start, unsigned long vm_end) 2004{ 2005 struct vm_area_struct *vma = find_vma(mm, vm_start); 2006 2007 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) 2008 vma = NULL; 2009 2010 return vma; 2011} 2012 2013#ifdef CONFIG_MMU 2014pgprot_t vm_get_page_prot(unsigned long vm_flags); 2015void vma_set_page_prot(struct vm_area_struct *vma); 2016#else 2017static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) 2018{ 2019 return __pgprot(0); 2020} 2021static inline void vma_set_page_prot(struct vm_area_struct *vma) 2022{ 2023 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); 2024} 2025#endif 2026 2027#ifdef CONFIG_NUMA_BALANCING 2028unsigned long change_prot_numa(struct vm_area_struct *vma, 2029 unsigned long start, unsigned long end); 2030#endif 2031 2032struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr); 2033int remap_pfn_range(struct vm_area_struct *, unsigned long addr, 2034 unsigned long pfn, unsigned long size, pgprot_t); 2035int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); 2036int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 2037 unsigned long pfn); 2038int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 2039 unsigned long pfn); 2040int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len); 2041 2042 2043struct page *follow_page_mask(struct vm_area_struct *vma, 2044 unsigned long address, unsigned int foll_flags, 2045 unsigned int *page_mask); 2046 2047static inline struct page *follow_page(struct vm_area_struct *vma, 2048 unsigned long address, unsigned int foll_flags) 2049{ 2050 unsigned int unused_page_mask; 2051 return follow_page_mask(vma, address, foll_flags, &unused_page_mask); 2052} 2053 2054#define FOLL_WRITE 0x01 /* check pte is writable */ 2055#define FOLL_TOUCH 0x02 /* mark page accessed */ 2056#define FOLL_GET 0x04 /* do get_page on page */ 2057#define FOLL_DUMP 0x08 /* give error on hole if it would be zero */ 2058#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */ 2059#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO 2060 * and return without waiting upon it */ 2061#define FOLL_POPULATE 0x40 /* fault in page */ 2062#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */ 2063#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */ 2064#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */ 2065#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */ 2066#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */ 2067 2068typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, 2069 void *data); 2070extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, 2071 unsigned long size, pte_fn_t fn, void *data); 2072 2073#ifdef CONFIG_PROC_FS 2074void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long); 2075#else 2076static inline void vm_stat_account(struct mm_struct *mm, 2077 unsigned long flags, struct file *file, long pages) 2078{ 2079 mm->total_vm += pages; 2080} 2081#endif /* CONFIG_PROC_FS */ 2082 2083#ifdef CONFIG_DEBUG_PAGEALLOC 2084extern bool _debug_pagealloc_enabled; 2085extern void __kernel_map_pages(struct page *page, int numpages, int enable); 2086 2087static inline bool debug_pagealloc_enabled(void) 2088{ 2089 return _debug_pagealloc_enabled; 2090} 2091 2092static inline void 2093kernel_map_pages(struct page *page, int numpages, int enable) 2094{ 2095 if (!debug_pagealloc_enabled()) 2096 return; 2097 2098 __kernel_map_pages(page, numpages, enable); 2099} 2100#ifdef CONFIG_HIBERNATION 2101extern bool kernel_page_present(struct page *page); 2102#endif /* CONFIG_HIBERNATION */ 2103#else 2104static inline void 2105kernel_map_pages(struct page *page, int numpages, int enable) {} 2106#ifdef CONFIG_HIBERNATION 2107static inline bool kernel_page_present(struct page *page) { return true; } 2108#endif /* CONFIG_HIBERNATION */ 2109#endif 2110 2111#ifdef __HAVE_ARCH_GATE_AREA 2112extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm); 2113extern int in_gate_area_no_mm(unsigned long addr); 2114extern int in_gate_area(struct mm_struct *mm, unsigned long addr); 2115#else 2116static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm) 2117{ 2118 return NULL; 2119} 2120static inline int in_gate_area_no_mm(unsigned long addr) { return 0; } 2121static inline int in_gate_area(struct mm_struct *mm, unsigned long addr) 2122{ 2123 return 0; 2124} 2125#endif /* __HAVE_ARCH_GATE_AREA */ 2126 2127#ifdef CONFIG_SYSCTL 2128extern int sysctl_drop_caches; 2129int drop_caches_sysctl_handler(struct ctl_table *, int, 2130 void __user *, size_t *, loff_t *); 2131#endif 2132 2133void drop_slab(void); 2134void drop_slab_node(int nid); 2135 2136#ifndef CONFIG_MMU 2137#define randomize_va_space 0 2138#else 2139extern int randomize_va_space; 2140#endif 2141 2142const char * arch_vma_name(struct vm_area_struct *vma); 2143void print_vma_addr(char *prefix, unsigned long rip); 2144 2145void sparse_mem_maps_populate_node(struct page **map_map, 2146 unsigned long pnum_begin, 2147 unsigned long pnum_end, 2148 unsigned long map_count, 2149 int nodeid); 2150 2151struct page *sparse_mem_map_populate(unsigned long pnum, int nid); 2152pgd_t *vmemmap_pgd_populate(unsigned long addr, int node); 2153pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node); 2154pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node); 2155pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node); 2156void *vmemmap_alloc_block(unsigned long size, int node); 2157void *vmemmap_alloc_block_buf(unsigned long size, int node); 2158void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); 2159int vmemmap_populate_basepages(unsigned long start, unsigned long end, 2160 int node); 2161int vmemmap_populate(unsigned long start, unsigned long end, int node); 2162void vmemmap_populate_print_last(void); 2163#ifdef CONFIG_MEMORY_HOTPLUG 2164void vmemmap_free(unsigned long start, unsigned long end); 2165#endif 2166void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, 2167 unsigned long size); 2168 2169enum mf_flags { 2170 MF_COUNT_INCREASED = 1 << 0, 2171 MF_ACTION_REQUIRED = 1 << 1, 2172 MF_MUST_KILL = 1 << 2, 2173 MF_SOFT_OFFLINE = 1 << 3, 2174}; 2175extern int memory_failure(unsigned long pfn, int trapno, int flags); 2176extern void memory_failure_queue(unsigned long pfn, int trapno, int flags); 2177extern int unpoison_memory(unsigned long pfn); 2178extern int sysctl_memory_failure_early_kill; 2179extern int sysctl_memory_failure_recovery; 2180extern void shake_page(struct page *p, int access); 2181extern atomic_long_t num_poisoned_pages; 2182extern int soft_offline_page(struct page *page, int flags); 2183 2184#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 2185extern void clear_huge_page(struct page *page, 2186 unsigned long addr, 2187 unsigned int pages_per_huge_page); 2188extern void copy_user_huge_page(struct page *dst, struct page *src, 2189 unsigned long addr, struct vm_area_struct *vma, 2190 unsigned int pages_per_huge_page); 2191#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 2192 2193extern struct page_ext_operations debug_guardpage_ops; 2194extern struct page_ext_operations page_poisoning_ops; 2195 2196#ifdef CONFIG_DEBUG_PAGEALLOC 2197extern unsigned int _debug_guardpage_minorder; 2198extern bool _debug_guardpage_enabled; 2199 2200static inline unsigned int debug_guardpage_minorder(void) 2201{ 2202 return _debug_guardpage_minorder; 2203} 2204 2205static inline bool debug_guardpage_enabled(void) 2206{ 2207 return _debug_guardpage_enabled; 2208} 2209 2210static inline bool page_is_guard(struct page *page) 2211{ 2212 struct page_ext *page_ext; 2213 2214 if (!debug_guardpage_enabled()) 2215 return false; 2216 2217 page_ext = lookup_page_ext(page); 2218 return test_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 2219} 2220#else 2221static inline unsigned int debug_guardpage_minorder(void) { return 0; } 2222static inline bool debug_guardpage_enabled(void) { return false; } 2223static inline bool page_is_guard(struct page *page) { return false; } 2224#endif /* CONFIG_DEBUG_PAGEALLOC */ 2225 2226#if MAX_NUMNODES > 1 2227void __init setup_nr_node_ids(void); 2228#else 2229static inline void setup_nr_node_ids(void) {} 2230#endif 2231 2232#endif /* __KERNEL__ */ 2233#endif /* _LINUX_MM_H */ 2234