This source file includes following definitions.
- early_init_on_alloc
- early_init_on_free
- get_pcppage_migratetype
- set_pcppage_migratetype
- pm_restore_gfp_mask
- pm_restrict_gfp_mask
- pm_suspended_storage
- kasan_free_nondeferred_pages
- early_page_uninitialised
- defer_init
- early_page_uninitialised
- defer_init
- get_pageblock_bitmap
- pfn_to_bitidx
- __get_pfnblock_flags_mask
- get_pfnblock_flags_mask
- get_pfnblock_migratetype
- set_pfnblock_flags_mask
- set_pageblock_migratetype
- page_outside_zone_boundaries
- page_is_consistent
- bad_range
- bad_range
- bad_page
- free_compound_page
- prep_compound_page
- early_debug_pagealloc
- init_debug_pagealloc
- debug_guardpage_minorder_setup
- set_page_guard
- clear_page_guard
- set_page_guard
- clear_page_guard
- set_page_order
- page_is_buddy
- task_capc
- compaction_capture
- task_capc
- compaction_capture
- __free_one_page
- page_expected_state
- free_pages_check_bad
- free_pages_check
- free_tail_pages_check
- kernel_init_free_pages
- free_pages_prepare
- free_pcp_prepare
- bulkfree_pcp_prepare
- free_pcp_prepare
- bulkfree_pcp_prepare
- prefetch_buddy
- free_pcppages_bulk
- free_one_page
- __init_single_page
- init_reserved_page
- init_reserved_page
- reserve_bootmem_region
- __free_pages_ok
- __free_pages_core
- early_pfn_to_nid
- early_pfn_in_nid
- early_pfn_in_nid
- memblock_free_pages
- __pageblock_pfn_to_page
- set_zone_contiguous
- clear_zone_contiguous
- deferred_free_range
- pgdat_init_report_one_done
- deferred_pfn_valid
- deferred_free_pages
- deferred_init_pages
- deferred_init_mem_pfn_range_in_zone
- deferred_init_maxorder
- deferred_init_memmap
- deferred_grow_zone
- _deferred_grow_zone
- page_alloc_init_late
- init_cma_reserved_pageblock
- expand
- check_new_page_bad
- check_new_page
- free_pages_prezeroed
- check_pcp_refill
- check_new_pcp
- check_pcp_refill
- check_new_pcp
- check_new_pages
- post_alloc_hook
- prep_new_page
- __rmqueue_smallest
- __rmqueue_cma_fallback
- __rmqueue_cma_fallback
- move_freepages
- move_freepages_block
- change_pageblock_range
- can_steal_fallback
- boost_watermark
- steal_suitable_fallback
- find_suitable_fallback
- reserve_highatomic_pageblock
- unreserve_highatomic_pageblock
- __rmqueue_fallback
- __rmqueue
- rmqueue_bulk
- drain_zone_pages
- drain_pages_zone
- drain_pages
- drain_local_pages
- drain_local_pages_wq
- drain_all_pages
- mark_free_pages
- free_unref_page_prepare
- free_unref_page_commit
- free_unref_page
- free_unref_page_list
- split_page
- __isolate_free_page
- zone_statistics
- __rmqueue_pcplist
- rmqueue_pcplist
- rmqueue
- setup_fail_page_alloc
- __should_fail_alloc_page
- fail_page_alloc_debugfs
- __should_fail_alloc_page
- should_fail_alloc_page
- __zone_watermark_ok
- zone_watermark_ok
- zone_watermark_fast
- zone_watermark_ok_safe
- zone_allows_reclaim
- zone_allows_reclaim
- alloc_flags_nofragment
- get_page_from_freelist
- warn_alloc_show_mem
- warn_alloc
- __alloc_pages_cpuset_fallback
- __alloc_pages_may_oom
- __alloc_pages_direct_compact
- should_compact_retry
- __alloc_pages_direct_compact
- should_compact_retry
- __need_fs_reclaim
- __fs_reclaim_acquire
- __fs_reclaim_release
- fs_reclaim_acquire
- fs_reclaim_release
- __perform_reclaim
- __alloc_pages_direct_reclaim
- wake_all_kswapds
- gfp_to_alloc_flags
- oom_reserves_allowed
- __gfp_pfmemalloc_flags
- gfp_pfmemalloc_allowed
- should_reclaim_retry
- check_retry_cpuset
- __alloc_pages_slowpath
- prepare_alloc_pages
- finalise_ac
- __alloc_pages_nodemask
- __get_free_pages
- get_zeroed_page
- free_the_page
- __free_pages
- free_pages
- __page_frag_cache_refill
- __page_frag_cache_drain
- page_frag_alloc
- page_frag_free
- make_alloc_exact
- alloc_pages_exact
- alloc_pages_exact_nid
- free_pages_exact
- nr_free_zone_pages
- nr_free_buffer_pages
- nr_free_pagecache_pages
- show_node
- si_mem_available
- si_meminfo
- si_meminfo_node
- show_mem_node_skip
- show_migration_types
- show_free_areas
- zoneref_set_zone
- build_zonerefs_node
- __parse_numa_zonelist_order
- setup_numa_zonelist_order
- numa_zonelist_order_handler
- find_next_best_node
- build_zonelists_in_node_order
- build_thisnode_zonelists
- build_zonelists
- local_memory_node
- build_zonelists
- __build_all_zonelists
- build_all_zonelists_init
- build_all_zonelists
- overlap_memmap_init
- memmap_init_zone
- memmap_init_zone_device
- zone_init_free_lists
- memmap_init
- zone_batchsize
- pageset_update
- pageset_set_batch
- pageset_init
- setup_pageset
- pageset_set_high
- pageset_set_high_and_batch
- zone_pageset_init
- setup_zone_pageset
- setup_per_cpu_pageset
- zone_pcp_init
- init_currently_empty_zone
- __early_pfn_to_nid
- free_bootmem_with_active_regions
- sparse_memory_present_with_active_regions
- get_pfn_range_for_nid
- find_usable_zone_for_movable
- adjust_zone_range_for_zone_movable
- zone_spanned_pages_in_node
- __absent_pages_in_range
- absent_pages_in_range
- zone_absent_pages_in_node
- zone_spanned_pages_in_node
- zone_absent_pages_in_node
- calculate_node_totalpages
- usemap_size
- setup_usemap
- setup_usemap
- set_pageblock_order
- set_pageblock_order
- calc_memmap_size
- pgdat_init_split_queue
- pgdat_init_split_queue
- pgdat_init_kcompactd
- pgdat_init_kcompactd
- pgdat_init_internals
- zone_init_internals
- free_area_init_core_hotplug
- free_area_init_core
- alloc_node_mem_map
- alloc_node_mem_map
- pgdat_set_deferred_range
- pgdat_set_deferred_range
- free_area_init_node
- zero_pfn_range
- zero_resv_unavail
- setup_nr_node_ids
- node_map_pfn_alignment
- find_min_pfn_for_node
- find_min_pfn_with_active_regions
- early_calculate_totalpages
- find_zone_movable_pfns_for_nodes
- check_for_memory
- free_area_init_nodes
- cmdline_parse_core
- cmdline_parse_kernelcore
- cmdline_parse_movablecore
- adjust_managed_page_count
- free_reserved_area
- free_highmem_page
- mem_init_print_info
- set_dma_reserve
- free_area_init
- page_alloc_cpu_dead
- set_hashdist
- page_alloc_init
- calculate_totalreserve_pages
- setup_per_zone_lowmem_reserve
- __setup_per_zone_wmarks
- setup_per_zone_wmarks
- init_per_zone_wmark_min
- core_initcall
- watermark_boost_factor_sysctl_handler
- watermark_scale_factor_sysctl_handler
- setup_min_unmapped_ratio
- sysctl_min_unmapped_ratio_sysctl_handler
- setup_min_slab_ratio
- sysctl_min_slab_ratio_sysctl_handler
- lowmem_reserve_ratio_sysctl_handler
- percpu_pagelist_fraction_sysctl_handler
- arch_reserved_kernel_pages
- alloc_large_system_hash
- has_unmovable_pages
- pfn_max_align_down
- pfn_max_align_up
- __alloc_contig_migrate_range
- alloc_contig_range
- free_contig_range
- zone_pcp_update
- zone_pcp_reset
- __offline_isolated_pages
- is_free_buddy_page
- set_hwpoison_free_buddy_page
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 #include <linux/stddef.h>
19 #include <linux/mm.h>
20 #include <linux/highmem.h>
21 #include <linux/swap.h>
22 #include <linux/interrupt.h>
23 #include <linux/pagemap.h>
24 #include <linux/jiffies.h>
25 #include <linux/memblock.h>
26 #include <linux/compiler.h>
27 #include <linux/kernel.h>
28 #include <linux/kasan.h>
29 #include <linux/module.h>
30 #include <linux/suspend.h>
31 #include <linux/pagevec.h>
32 #include <linux/blkdev.h>
33 #include <linux/slab.h>
34 #include <linux/ratelimit.h>
35 #include <linux/oom.h>
36 #include <linux/topology.h>
37 #include <linux/sysctl.h>
38 #include <linux/cpu.h>
39 #include <linux/cpuset.h>
40 #include <linux/memory_hotplug.h>
41 #include <linux/nodemask.h>
42 #include <linux/vmalloc.h>
43 #include <linux/vmstat.h>
44 #include <linux/mempolicy.h>
45 #include <linux/memremap.h>
46 #include <linux/stop_machine.h>
47 #include <linux/random.h>
48 #include <linux/sort.h>
49 #include <linux/pfn.h>
50 #include <linux/backing-dev.h>
51 #include <linux/fault-inject.h>
52 #include <linux/page-isolation.h>
53 #include <linux/debugobjects.h>
54 #include <linux/kmemleak.h>
55 #include <linux/compaction.h>
56 #include <trace/events/kmem.h>
57 #include <trace/events/oom.h>
58 #include <linux/prefetch.h>
59 #include <linux/mm_inline.h>
60 #include <linux/migrate.h>
61 #include <linux/hugetlb.h>
62 #include <linux/sched/rt.h>
63 #include <linux/sched/mm.h>
64 #include <linux/page_owner.h>
65 #include <linux/kthread.h>
66 #include <linux/memcontrol.h>
67 #include <linux/ftrace.h>
68 #include <linux/lockdep.h>
69 #include <linux/nmi.h>
70 #include <linux/psi.h>
71
72 #include <asm/sections.h>
73 #include <asm/tlbflush.h>
74 #include <asm/div64.h>
75 #include "internal.h"
76 #include "shuffle.h"
77
78
79 static DEFINE_MUTEX(pcp_batch_high_lock);
80 #define MIN_PERCPU_PAGELIST_FRACTION (8)
81
82 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
83 DEFINE_PER_CPU(int, numa_node);
84 EXPORT_PER_CPU_SYMBOL(numa_node);
85 #endif
86
87 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
88
89 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
90
91
92
93
94
95
96 DEFINE_PER_CPU(int, _numa_mem_);
97 EXPORT_PER_CPU_SYMBOL(_numa_mem_);
98 int _node_numa_mem_[MAX_NUMNODES];
99 #endif
100
101
102 struct pcpu_drain {
103 struct zone *zone;
104 struct work_struct work;
105 };
106 DEFINE_MUTEX(pcpu_drain_mutex);
107 DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
108
109 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
110 volatile unsigned long latent_entropy __latent_entropy;
111 EXPORT_SYMBOL(latent_entropy);
112 #endif
113
114
115
116
117 nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
118 [N_POSSIBLE] = NODE_MASK_ALL,
119 [N_ONLINE] = { { [0] = 1UL } },
120 #ifndef CONFIG_NUMA
121 [N_NORMAL_MEMORY] = { { [0] = 1UL } },
122 #ifdef CONFIG_HIGHMEM
123 [N_HIGH_MEMORY] = { { [0] = 1UL } },
124 #endif
125 [N_MEMORY] = { { [0] = 1UL } },
126 [N_CPU] = { { [0] = 1UL } },
127 #endif
128 };
129 EXPORT_SYMBOL(node_states);
130
131 atomic_long_t _totalram_pages __read_mostly;
132 EXPORT_SYMBOL(_totalram_pages);
133 unsigned long totalreserve_pages __read_mostly;
134 unsigned long totalcma_pages __read_mostly;
135
136 int percpu_pagelist_fraction;
137 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
138 #ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON
139 DEFINE_STATIC_KEY_TRUE(init_on_alloc);
140 #else
141 DEFINE_STATIC_KEY_FALSE(init_on_alloc);
142 #endif
143 EXPORT_SYMBOL(init_on_alloc);
144
145 #ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON
146 DEFINE_STATIC_KEY_TRUE(init_on_free);
147 #else
148 DEFINE_STATIC_KEY_FALSE(init_on_free);
149 #endif
150 EXPORT_SYMBOL(init_on_free);
151
152 static int __init early_init_on_alloc(char *buf)
153 {
154 int ret;
155 bool bool_result;
156
157 if (!buf)
158 return -EINVAL;
159 ret = kstrtobool(buf, &bool_result);
160 if (bool_result && page_poisoning_enabled())
161 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n");
162 if (bool_result)
163 static_branch_enable(&init_on_alloc);
164 else
165 static_branch_disable(&init_on_alloc);
166 return ret;
167 }
168 early_param("init_on_alloc", early_init_on_alloc);
169
170 static int __init early_init_on_free(char *buf)
171 {
172 int ret;
173 bool bool_result;
174
175 if (!buf)
176 return -EINVAL;
177 ret = kstrtobool(buf, &bool_result);
178 if (bool_result && page_poisoning_enabled())
179 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n");
180 if (bool_result)
181 static_branch_enable(&init_on_free);
182 else
183 static_branch_disable(&init_on_free);
184 return ret;
185 }
186 early_param("init_on_free", early_init_on_free);
187
188
189
190
191
192
193
194
195
196 static inline int get_pcppage_migratetype(struct page *page)
197 {
198 return page->index;
199 }
200
201 static inline void set_pcppage_migratetype(struct page *page, int migratetype)
202 {
203 page->index = migratetype;
204 }
205
206 #ifdef CONFIG_PM_SLEEP
207
208
209
210
211
212
213
214
215
216
217 static gfp_t saved_gfp_mask;
218
219 void pm_restore_gfp_mask(void)
220 {
221 WARN_ON(!mutex_is_locked(&system_transition_mutex));
222 if (saved_gfp_mask) {
223 gfp_allowed_mask = saved_gfp_mask;
224 saved_gfp_mask = 0;
225 }
226 }
227
228 void pm_restrict_gfp_mask(void)
229 {
230 WARN_ON(!mutex_is_locked(&system_transition_mutex));
231 WARN_ON(saved_gfp_mask);
232 saved_gfp_mask = gfp_allowed_mask;
233 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
234 }
235
236 bool pm_suspended_storage(void)
237 {
238 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
239 return false;
240 return true;
241 }
242 #endif
243
244 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
245 unsigned int pageblock_order __read_mostly;
246 #endif
247
248 static void __free_pages_ok(struct page *page, unsigned int order);
249
250
251
252
253
254
255
256
257
258
259
260
261 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
262 #ifdef CONFIG_ZONE_DMA
263 [ZONE_DMA] = 256,
264 #endif
265 #ifdef CONFIG_ZONE_DMA32
266 [ZONE_DMA32] = 256,
267 #endif
268 [ZONE_NORMAL] = 32,
269 #ifdef CONFIG_HIGHMEM
270 [ZONE_HIGHMEM] = 0,
271 #endif
272 [ZONE_MOVABLE] = 0,
273 };
274
275 static char * const zone_names[MAX_NR_ZONES] = {
276 #ifdef CONFIG_ZONE_DMA
277 "DMA",
278 #endif
279 #ifdef CONFIG_ZONE_DMA32
280 "DMA32",
281 #endif
282 "Normal",
283 #ifdef CONFIG_HIGHMEM
284 "HighMem",
285 #endif
286 "Movable",
287 #ifdef CONFIG_ZONE_DEVICE
288 "Device",
289 #endif
290 };
291
292 const char * const migratetype_names[MIGRATE_TYPES] = {
293 "Unmovable",
294 "Movable",
295 "Reclaimable",
296 "HighAtomic",
297 #ifdef CONFIG_CMA
298 "CMA",
299 #endif
300 #ifdef CONFIG_MEMORY_ISOLATION
301 "Isolate",
302 #endif
303 };
304
305 compound_page_dtor * const compound_page_dtors[] = {
306 NULL,
307 free_compound_page,
308 #ifdef CONFIG_HUGETLB_PAGE
309 free_huge_page,
310 #endif
311 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
312 free_transhuge_page,
313 #endif
314 };
315
316 int min_free_kbytes = 1024;
317 int user_min_free_kbytes = -1;
318 #ifdef CONFIG_DISCONTIGMEM
319
320
321
322
323
324
325
326
327
328 int watermark_boost_factor __read_mostly;
329 #else
330 int watermark_boost_factor __read_mostly = 15000;
331 #endif
332 int watermark_scale_factor = 10;
333
334 static unsigned long nr_kernel_pages __initdata;
335 static unsigned long nr_all_pages __initdata;
336 static unsigned long dma_reserve __initdata;
337
338 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
339 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
340 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
341 static unsigned long required_kernelcore __initdata;
342 static unsigned long required_kernelcore_percent __initdata;
343 static unsigned long required_movablecore __initdata;
344 static unsigned long required_movablecore_percent __initdata;
345 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
346 static bool mirrored_kernelcore __meminitdata;
347
348
349 int movable_zone;
350 EXPORT_SYMBOL(movable_zone);
351 #endif
352
353 #if MAX_NUMNODES > 1
354 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
355 unsigned int nr_online_nodes __read_mostly = 1;
356 EXPORT_SYMBOL(nr_node_ids);
357 EXPORT_SYMBOL(nr_online_nodes);
358 #endif
359
360 int page_group_by_mobility_disabled __read_mostly;
361
362 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
363
364
365
366
367
368 static DEFINE_STATIC_KEY_TRUE(deferred_pages);
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383 static inline void kasan_free_nondeferred_pages(struct page *page, int order)
384 {
385 if (!static_branch_unlikely(&deferred_pages))
386 kasan_free_pages(page, order);
387 }
388
389
390 static inline bool __meminit early_page_uninitialised(unsigned long pfn)
391 {
392 int nid = early_pfn_to_nid(pfn);
393
394 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
395 return true;
396
397 return false;
398 }
399
400
401
402
403
404 static bool __meminit
405 defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
406 {
407 static unsigned long prev_end_pfn, nr_initialised;
408
409
410
411
412
413 if (prev_end_pfn != end_pfn) {
414 prev_end_pfn = end_pfn;
415 nr_initialised = 0;
416 }
417
418
419 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
420 return false;
421
422
423
424
425
426 nr_initialised++;
427 if ((nr_initialised > PAGES_PER_SECTION) &&
428 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
429 NODE_DATA(nid)->first_deferred_pfn = pfn;
430 return true;
431 }
432 return false;
433 }
434 #else
435 #define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o)
436
437 static inline bool early_page_uninitialised(unsigned long pfn)
438 {
439 return false;
440 }
441
442 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
443 {
444 return false;
445 }
446 #endif
447
448
449 static inline unsigned long *get_pageblock_bitmap(struct page *page,
450 unsigned long pfn)
451 {
452 #ifdef CONFIG_SPARSEMEM
453 return section_to_usemap(__pfn_to_section(pfn));
454 #else
455 return page_zone(page)->pageblock_flags;
456 #endif
457 }
458
459 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
460 {
461 #ifdef CONFIG_SPARSEMEM
462 pfn &= (PAGES_PER_SECTION-1);
463 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
464 #else
465 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
466 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
467 #endif
468 }
469
470
471
472
473
474
475
476
477
478
479 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
480 unsigned long pfn,
481 unsigned long end_bitidx,
482 unsigned long mask)
483 {
484 unsigned long *bitmap;
485 unsigned long bitidx, word_bitidx;
486 unsigned long word;
487
488 bitmap = get_pageblock_bitmap(page, pfn);
489 bitidx = pfn_to_bitidx(page, pfn);
490 word_bitidx = bitidx / BITS_PER_LONG;
491 bitidx &= (BITS_PER_LONG-1);
492
493 word = bitmap[word_bitidx];
494 bitidx += end_bitidx;
495 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
496 }
497
498 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
499 unsigned long end_bitidx,
500 unsigned long mask)
501 {
502 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
503 }
504
505 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
506 {
507 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
508 }
509
510
511
512
513
514
515
516
517
518 void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
519 unsigned long pfn,
520 unsigned long end_bitidx,
521 unsigned long mask)
522 {
523 unsigned long *bitmap;
524 unsigned long bitidx, word_bitidx;
525 unsigned long old_word, word;
526
527 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
528 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
529
530 bitmap = get_pageblock_bitmap(page, pfn);
531 bitidx = pfn_to_bitidx(page, pfn);
532 word_bitidx = bitidx / BITS_PER_LONG;
533 bitidx &= (BITS_PER_LONG-1);
534
535 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
536
537 bitidx += end_bitidx;
538 mask <<= (BITS_PER_LONG - bitidx - 1);
539 flags <<= (BITS_PER_LONG - bitidx - 1);
540
541 word = READ_ONCE(bitmap[word_bitidx]);
542 for (;;) {
543 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
544 if (word == old_word)
545 break;
546 word = old_word;
547 }
548 }
549
550 void set_pageblock_migratetype(struct page *page, int migratetype)
551 {
552 if (unlikely(page_group_by_mobility_disabled &&
553 migratetype < MIGRATE_PCPTYPES))
554 migratetype = MIGRATE_UNMOVABLE;
555
556 set_pageblock_flags_group(page, (unsigned long)migratetype,
557 PB_migrate, PB_migrate_end);
558 }
559
560 #ifdef CONFIG_DEBUG_VM
561 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
562 {
563 int ret = 0;
564 unsigned seq;
565 unsigned long pfn = page_to_pfn(page);
566 unsigned long sp, start_pfn;
567
568 do {
569 seq = zone_span_seqbegin(zone);
570 start_pfn = zone->zone_start_pfn;
571 sp = zone->spanned_pages;
572 if (!zone_spans_pfn(zone, pfn))
573 ret = 1;
574 } while (zone_span_seqretry(zone, seq));
575
576 if (ret)
577 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
578 pfn, zone_to_nid(zone), zone->name,
579 start_pfn, start_pfn + sp);
580
581 return ret;
582 }
583
584 static int page_is_consistent(struct zone *zone, struct page *page)
585 {
586 if (!pfn_valid_within(page_to_pfn(page)))
587 return 0;
588 if (zone != page_zone(page))
589 return 0;
590
591 return 1;
592 }
593
594
595
596 static int __maybe_unused bad_range(struct zone *zone, struct page *page)
597 {
598 if (page_outside_zone_boundaries(zone, page))
599 return 1;
600 if (!page_is_consistent(zone, page))
601 return 1;
602
603 return 0;
604 }
605 #else
606 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
607 {
608 return 0;
609 }
610 #endif
611
612 static void bad_page(struct page *page, const char *reason,
613 unsigned long bad_flags)
614 {
615 static unsigned long resume;
616 static unsigned long nr_shown;
617 static unsigned long nr_unshown;
618
619
620
621
622
623 if (nr_shown == 60) {
624 if (time_before(jiffies, resume)) {
625 nr_unshown++;
626 goto out;
627 }
628 if (nr_unshown) {
629 pr_alert(
630 "BUG: Bad page state: %lu messages suppressed\n",
631 nr_unshown);
632 nr_unshown = 0;
633 }
634 nr_shown = 0;
635 }
636 if (nr_shown++ == 0)
637 resume = jiffies + 60 * HZ;
638
639 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
640 current->comm, page_to_pfn(page));
641 __dump_page(page, reason);
642 bad_flags &= page->flags;
643 if (bad_flags)
644 pr_alert("bad because of flags: %#lx(%pGp)\n",
645 bad_flags, &bad_flags);
646 dump_page_owner(page);
647
648 print_modules();
649 dump_stack();
650 out:
651
652 page_mapcount_reset(page);
653 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
654 }
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671 void free_compound_page(struct page *page)
672 {
673 mem_cgroup_uncharge(page);
674 __free_pages_ok(page, compound_order(page));
675 }
676
677 void prep_compound_page(struct page *page, unsigned int order)
678 {
679 int i;
680 int nr_pages = 1 << order;
681
682 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
683 set_compound_order(page, order);
684 __SetPageHead(page);
685 for (i = 1; i < nr_pages; i++) {
686 struct page *p = page + i;
687 set_page_count(p, 0);
688 p->mapping = TAIL_MAPPING;
689 set_compound_head(p, page);
690 }
691 atomic_set(compound_mapcount_ptr(page), -1);
692 }
693
694 #ifdef CONFIG_DEBUG_PAGEALLOC
695 unsigned int _debug_guardpage_minorder;
696
697 bool _debug_pagealloc_enabled_early __read_mostly
698 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
699 EXPORT_SYMBOL(_debug_pagealloc_enabled_early);
700 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
701 EXPORT_SYMBOL(_debug_pagealloc_enabled);
702
703 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
704
705 static int __init early_debug_pagealloc(char *buf)
706 {
707 return kstrtobool(buf, &_debug_pagealloc_enabled_early);
708 }
709 early_param("debug_pagealloc", early_debug_pagealloc);
710
711 void init_debug_pagealloc(void)
712 {
713 if (!debug_pagealloc_enabled())
714 return;
715
716 static_branch_enable(&_debug_pagealloc_enabled);
717
718 if (!debug_guardpage_minorder())
719 return;
720
721 static_branch_enable(&_debug_guardpage_enabled);
722 }
723
724 static int __init debug_guardpage_minorder_setup(char *buf)
725 {
726 unsigned long res;
727
728 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
729 pr_err("Bad debug_guardpage_minorder value\n");
730 return 0;
731 }
732 _debug_guardpage_minorder = res;
733 pr_info("Setting debug_guardpage_minorder to %lu\n", res);
734 return 0;
735 }
736 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
737
738 static inline bool set_page_guard(struct zone *zone, struct page *page,
739 unsigned int order, int migratetype)
740 {
741 if (!debug_guardpage_enabled())
742 return false;
743
744 if (order >= debug_guardpage_minorder())
745 return false;
746
747 __SetPageGuard(page);
748 INIT_LIST_HEAD(&page->lru);
749 set_page_private(page, order);
750
751 __mod_zone_freepage_state(zone, -(1 << order), migratetype);
752
753 return true;
754 }
755
756 static inline void clear_page_guard(struct zone *zone, struct page *page,
757 unsigned int order, int migratetype)
758 {
759 if (!debug_guardpage_enabled())
760 return;
761
762 __ClearPageGuard(page);
763
764 set_page_private(page, 0);
765 if (!is_migrate_isolate(migratetype))
766 __mod_zone_freepage_state(zone, (1 << order), migratetype);
767 }
768 #else
769 static inline bool set_page_guard(struct zone *zone, struct page *page,
770 unsigned int order, int migratetype) { return false; }
771 static inline void clear_page_guard(struct zone *zone, struct page *page,
772 unsigned int order, int migratetype) {}
773 #endif
774
775 static inline void set_page_order(struct page *page, unsigned int order)
776 {
777 set_page_private(page, order);
778 __SetPageBuddy(page);
779 }
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794 static inline int page_is_buddy(struct page *page, struct page *buddy,
795 unsigned int order)
796 {
797 if (page_is_guard(buddy) && page_order(buddy) == order) {
798 if (page_zone_id(page) != page_zone_id(buddy))
799 return 0;
800
801 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
802
803 return 1;
804 }
805
806 if (PageBuddy(buddy) && page_order(buddy) == order) {
807
808
809
810
811
812 if (page_zone_id(page) != page_zone_id(buddy))
813 return 0;
814
815 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
816
817 return 1;
818 }
819 return 0;
820 }
821
822 #ifdef CONFIG_COMPACTION
823 static inline struct capture_control *task_capc(struct zone *zone)
824 {
825 struct capture_control *capc = current->capture_control;
826
827 return capc &&
828 !(current->flags & PF_KTHREAD) &&
829 !capc->page &&
830 capc->cc->zone == zone &&
831 capc->cc->direct_compaction ? capc : NULL;
832 }
833
834 static inline bool
835 compaction_capture(struct capture_control *capc, struct page *page,
836 int order, int migratetype)
837 {
838 if (!capc || order != capc->cc->order)
839 return false;
840
841
842 if (is_migrate_cma(migratetype) ||
843 is_migrate_isolate(migratetype))
844 return false;
845
846
847
848
849
850
851
852 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
853 return false;
854
855 capc->page = page;
856 return true;
857 }
858
859 #else
860 static inline struct capture_control *task_capc(struct zone *zone)
861 {
862 return NULL;
863 }
864
865 static inline bool
866 compaction_capture(struct capture_control *capc, struct page *page,
867 int order, int migratetype)
868 {
869 return false;
870 }
871 #endif
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897 static inline void __free_one_page(struct page *page,
898 unsigned long pfn,
899 struct zone *zone, unsigned int order,
900 int migratetype)
901 {
902 unsigned long combined_pfn;
903 unsigned long uninitialized_var(buddy_pfn);
904 struct page *buddy;
905 unsigned int max_order;
906 struct capture_control *capc = task_capc(zone);
907
908 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
909
910 VM_BUG_ON(!zone_is_initialized(zone));
911 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
912
913 VM_BUG_ON(migratetype == -1);
914 if (likely(!is_migrate_isolate(migratetype)))
915 __mod_zone_freepage_state(zone, 1 << order, migratetype);
916
917 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
918 VM_BUG_ON_PAGE(bad_range(zone, page), page);
919
920 continue_merging:
921 while (order < max_order - 1) {
922 if (compaction_capture(capc, page, order, migratetype)) {
923 __mod_zone_freepage_state(zone, -(1 << order),
924 migratetype);
925 return;
926 }
927 buddy_pfn = __find_buddy_pfn(pfn, order);
928 buddy = page + (buddy_pfn - pfn);
929
930 if (!pfn_valid_within(buddy_pfn))
931 goto done_merging;
932 if (!page_is_buddy(page, buddy, order))
933 goto done_merging;
934
935
936
937
938 if (page_is_guard(buddy))
939 clear_page_guard(zone, buddy, order, migratetype);
940 else
941 del_page_from_free_area(buddy, &zone->free_area[order]);
942 combined_pfn = buddy_pfn & pfn;
943 page = page + (combined_pfn - pfn);
944 pfn = combined_pfn;
945 order++;
946 }
947 if (max_order < MAX_ORDER) {
948
949
950
951
952
953
954
955
956 if (unlikely(has_isolate_pageblock(zone))) {
957 int buddy_mt;
958
959 buddy_pfn = __find_buddy_pfn(pfn, order);
960 buddy = page + (buddy_pfn - pfn);
961 buddy_mt = get_pageblock_migratetype(buddy);
962
963 if (migratetype != buddy_mt
964 && (is_migrate_isolate(migratetype) ||
965 is_migrate_isolate(buddy_mt)))
966 goto done_merging;
967 }
968 max_order++;
969 goto continue_merging;
970 }
971
972 done_merging:
973 set_page_order(page, order);
974
975
976
977
978
979
980
981
982
983 if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn)
984 && !is_shuffle_order(order)) {
985 struct page *higher_page, *higher_buddy;
986 combined_pfn = buddy_pfn & pfn;
987 higher_page = page + (combined_pfn - pfn);
988 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
989 higher_buddy = higher_page + (buddy_pfn - combined_pfn);
990 if (pfn_valid_within(buddy_pfn) &&
991 page_is_buddy(higher_page, higher_buddy, order + 1)) {
992 add_to_free_area_tail(page, &zone->free_area[order],
993 migratetype);
994 return;
995 }
996 }
997
998 if (is_shuffle_order(order))
999 add_to_free_area_random(page, &zone->free_area[order],
1000 migratetype);
1001 else
1002 add_to_free_area(page, &zone->free_area[order], migratetype);
1003
1004 }
1005
1006
1007
1008
1009
1010
1011 static inline bool page_expected_state(struct page *page,
1012 unsigned long check_flags)
1013 {
1014 if (unlikely(atomic_read(&page->_mapcount) != -1))
1015 return false;
1016
1017 if (unlikely((unsigned long)page->mapping |
1018 page_ref_count(page) |
1019 #ifdef CONFIG_MEMCG
1020 (unsigned long)page->mem_cgroup |
1021 #endif
1022 (page->flags & check_flags)))
1023 return false;
1024
1025 return true;
1026 }
1027
1028 static void free_pages_check_bad(struct page *page)
1029 {
1030 const char *bad_reason;
1031 unsigned long bad_flags;
1032
1033 bad_reason = NULL;
1034 bad_flags = 0;
1035
1036 if (unlikely(atomic_read(&page->_mapcount) != -1))
1037 bad_reason = "nonzero mapcount";
1038 if (unlikely(page->mapping != NULL))
1039 bad_reason = "non-NULL mapping";
1040 if (unlikely(page_ref_count(page) != 0))
1041 bad_reason = "nonzero _refcount";
1042 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
1043 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
1044 bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
1045 }
1046 #ifdef CONFIG_MEMCG
1047 if (unlikely(page->mem_cgroup))
1048 bad_reason = "page still charged to cgroup";
1049 #endif
1050 bad_page(page, bad_reason, bad_flags);
1051 }
1052
1053 static inline int free_pages_check(struct page *page)
1054 {
1055 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
1056 return 0;
1057
1058
1059 free_pages_check_bad(page);
1060 return 1;
1061 }
1062
1063 static int free_tail_pages_check(struct page *head_page, struct page *page)
1064 {
1065 int ret = 1;
1066
1067
1068
1069
1070
1071 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
1072
1073 if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
1074 ret = 0;
1075 goto out;
1076 }
1077 switch (page - head_page) {
1078 case 1:
1079
1080 if (unlikely(compound_mapcount(page))) {
1081 bad_page(page, "nonzero compound_mapcount", 0);
1082 goto out;
1083 }
1084 break;
1085 case 2:
1086
1087
1088
1089
1090 break;
1091 default:
1092 if (page->mapping != TAIL_MAPPING) {
1093 bad_page(page, "corrupted mapping in tail page", 0);
1094 goto out;
1095 }
1096 break;
1097 }
1098 if (unlikely(!PageTail(page))) {
1099 bad_page(page, "PageTail not set", 0);
1100 goto out;
1101 }
1102 if (unlikely(compound_head(page) != head_page)) {
1103 bad_page(page, "compound_head not consistent", 0);
1104 goto out;
1105 }
1106 ret = 0;
1107 out:
1108 page->mapping = NULL;
1109 clear_compound_head(page);
1110 return ret;
1111 }
1112
1113 static void kernel_init_free_pages(struct page *page, int numpages)
1114 {
1115 int i;
1116
1117 for (i = 0; i < numpages; i++)
1118 clear_highpage(page + i);
1119 }
1120
1121 static __always_inline bool free_pages_prepare(struct page *page,
1122 unsigned int order, bool check_free)
1123 {
1124 int bad = 0;
1125
1126 VM_BUG_ON_PAGE(PageTail(page), page);
1127
1128 trace_mm_page_free(page, order);
1129
1130
1131
1132
1133
1134 if (unlikely(order)) {
1135 bool compound = PageCompound(page);
1136 int i;
1137
1138 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1139
1140 if (compound)
1141 ClearPageDoubleMap(page);
1142 for (i = 1; i < (1 << order); i++) {
1143 if (compound)
1144 bad += free_tail_pages_check(page, page + i);
1145 if (unlikely(free_pages_check(page + i))) {
1146 bad++;
1147 continue;
1148 }
1149 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1150 }
1151 }
1152 if (PageMappingFlags(page))
1153 page->mapping = NULL;
1154 if (memcg_kmem_enabled() && PageKmemcg(page))
1155 __memcg_kmem_uncharge(page, order);
1156 if (check_free)
1157 bad += free_pages_check(page);
1158 if (bad)
1159 return false;
1160
1161 page_cpupid_reset_last(page);
1162 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1163 reset_page_owner(page, order);
1164
1165 if (!PageHighMem(page)) {
1166 debug_check_no_locks_freed(page_address(page),
1167 PAGE_SIZE << order);
1168 debug_check_no_obj_freed(page_address(page),
1169 PAGE_SIZE << order);
1170 }
1171 if (want_init_on_free())
1172 kernel_init_free_pages(page, 1 << order);
1173
1174 kernel_poison_pages(page, 1 << order, 0);
1175
1176
1177
1178
1179
1180 arch_free_page(page, order);
1181
1182 if (debug_pagealloc_enabled_static())
1183 kernel_map_pages(page, 1 << order, 0);
1184
1185 kasan_free_nondeferred_pages(page, order);
1186
1187 return true;
1188 }
1189
1190 #ifdef CONFIG_DEBUG_VM
1191
1192
1193
1194
1195
1196 static bool free_pcp_prepare(struct page *page)
1197 {
1198 return free_pages_prepare(page, 0, true);
1199 }
1200
1201 static bool bulkfree_pcp_prepare(struct page *page)
1202 {
1203 if (debug_pagealloc_enabled_static())
1204 return free_pages_check(page);
1205 else
1206 return false;
1207 }
1208 #else
1209
1210
1211
1212
1213
1214
1215 static bool free_pcp_prepare(struct page *page)
1216 {
1217 if (debug_pagealloc_enabled_static())
1218 return free_pages_prepare(page, 0, true);
1219 else
1220 return free_pages_prepare(page, 0, false);
1221 }
1222
1223 static bool bulkfree_pcp_prepare(struct page *page)
1224 {
1225 return free_pages_check(page);
1226 }
1227 #endif
1228
1229 static inline void prefetch_buddy(struct page *page)
1230 {
1231 unsigned long pfn = page_to_pfn(page);
1232 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
1233 struct page *buddy = page + (buddy_pfn - pfn);
1234
1235 prefetch(buddy);
1236 }
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249 static void free_pcppages_bulk(struct zone *zone, int count,
1250 struct per_cpu_pages *pcp)
1251 {
1252 int migratetype = 0;
1253 int batch_free = 0;
1254 int prefetch_nr = 0;
1255 bool isolated_pageblocks;
1256 struct page *page, *tmp;
1257 LIST_HEAD(head);
1258
1259 while (count) {
1260 struct list_head *list;
1261
1262
1263
1264
1265
1266
1267
1268
1269 do {
1270 batch_free++;
1271 if (++migratetype == MIGRATE_PCPTYPES)
1272 migratetype = 0;
1273 list = &pcp->lists[migratetype];
1274 } while (list_empty(list));
1275
1276
1277 if (batch_free == MIGRATE_PCPTYPES)
1278 batch_free = count;
1279
1280 do {
1281 page = list_last_entry(list, struct page, lru);
1282
1283 list_del(&page->lru);
1284 pcp->count--;
1285
1286 if (bulkfree_pcp_prepare(page))
1287 continue;
1288
1289 list_add_tail(&page->lru, &head);
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300 if (prefetch_nr++ < pcp->batch)
1301 prefetch_buddy(page);
1302 } while (--count && --batch_free && !list_empty(list));
1303 }
1304
1305 spin_lock(&zone->lock);
1306 isolated_pageblocks = has_isolate_pageblock(zone);
1307
1308
1309
1310
1311
1312 list_for_each_entry_safe(page, tmp, &head, lru) {
1313 int mt = get_pcppage_migratetype(page);
1314
1315 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1316
1317 if (unlikely(isolated_pageblocks))
1318 mt = get_pageblock_migratetype(page);
1319
1320 __free_one_page(page, page_to_pfn(page), zone, 0, mt);
1321 trace_mm_page_pcpu_drain(page, 0, mt);
1322 }
1323 spin_unlock(&zone->lock);
1324 }
1325
1326 static void free_one_page(struct zone *zone,
1327 struct page *page, unsigned long pfn,
1328 unsigned int order,
1329 int migratetype)
1330 {
1331 spin_lock(&zone->lock);
1332 if (unlikely(has_isolate_pageblock(zone) ||
1333 is_migrate_isolate(migratetype))) {
1334 migratetype = get_pfnblock_migratetype(page, pfn);
1335 }
1336 __free_one_page(page, pfn, zone, order, migratetype);
1337 spin_unlock(&zone->lock);
1338 }
1339
1340 static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1341 unsigned long zone, int nid)
1342 {
1343 mm_zero_struct_page(page);
1344 set_page_links(page, zone, nid, pfn);
1345 init_page_count(page);
1346 page_mapcount_reset(page);
1347 page_cpupid_reset_last(page);
1348 page_kasan_tag_reset(page);
1349
1350 INIT_LIST_HEAD(&page->lru);
1351 #ifdef WANT_PAGE_VIRTUAL
1352
1353 if (!is_highmem_idx(zone))
1354 set_page_address(page, __va(pfn << PAGE_SHIFT));
1355 #endif
1356 }
1357
1358 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1359 static void __meminit init_reserved_page(unsigned long pfn)
1360 {
1361 pg_data_t *pgdat;
1362 int nid, zid;
1363
1364 if (!early_page_uninitialised(pfn))
1365 return;
1366
1367 nid = early_pfn_to_nid(pfn);
1368 pgdat = NODE_DATA(nid);
1369
1370 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1371 struct zone *zone = &pgdat->node_zones[zid];
1372
1373 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1374 break;
1375 }
1376 __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
1377 }
1378 #else
1379 static inline void init_reserved_page(unsigned long pfn)
1380 {
1381 }
1382 #endif
1383
1384
1385
1386
1387
1388
1389
1390 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
1391 {
1392 unsigned long start_pfn = PFN_DOWN(start);
1393 unsigned long end_pfn = PFN_UP(end);
1394
1395 for (; start_pfn < end_pfn; start_pfn++) {
1396 if (pfn_valid(start_pfn)) {
1397 struct page *page = pfn_to_page(start_pfn);
1398
1399 init_reserved_page(start_pfn);
1400
1401
1402 INIT_LIST_HEAD(&page->lru);
1403
1404
1405
1406
1407
1408
1409 __SetPageReserved(page);
1410 }
1411 }
1412 }
1413
1414 static void __free_pages_ok(struct page *page, unsigned int order)
1415 {
1416 unsigned long flags;
1417 int migratetype;
1418 unsigned long pfn = page_to_pfn(page);
1419
1420 if (!free_pages_prepare(page, order, true))
1421 return;
1422
1423 migratetype = get_pfnblock_migratetype(page, pfn);
1424 local_irq_save(flags);
1425 __count_vm_events(PGFREE, 1 << order);
1426 free_one_page(page_zone(page), page, pfn, order, migratetype);
1427 local_irq_restore(flags);
1428 }
1429
1430 void __free_pages_core(struct page *page, unsigned int order)
1431 {
1432 unsigned int nr_pages = 1 << order;
1433 struct page *p = page;
1434 unsigned int loop;
1435
1436 prefetchw(p);
1437 for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1438 prefetchw(p + 1);
1439 __ClearPageReserved(p);
1440 set_page_count(p, 0);
1441 }
1442 __ClearPageReserved(p);
1443 set_page_count(p, 0);
1444
1445 atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
1446 set_page_refcounted(page);
1447 __free_pages(page, order);
1448 }
1449
1450 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
1451 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1452
1453 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1454
1455 int __meminit early_pfn_to_nid(unsigned long pfn)
1456 {
1457 static DEFINE_SPINLOCK(early_pfn_lock);
1458 int nid;
1459
1460 spin_lock(&early_pfn_lock);
1461 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1462 if (nid < 0)
1463 nid = first_online_node;
1464 spin_unlock(&early_pfn_lock);
1465
1466 return nid;
1467 }
1468 #endif
1469
1470 #ifdef CONFIG_NODES_SPAN_OTHER_NODES
1471
1472 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1473 {
1474 int nid;
1475
1476 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1477 if (nid >= 0 && nid != node)
1478 return false;
1479 return true;
1480 }
1481
1482 #else
1483 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1484 {
1485 return true;
1486 }
1487 #endif
1488
1489
1490 void __init memblock_free_pages(struct page *page, unsigned long pfn,
1491 unsigned int order)
1492 {
1493 if (early_page_uninitialised(pfn))
1494 return;
1495 __free_pages_core(page, order);
1496 }
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515 struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1516 unsigned long end_pfn, struct zone *zone)
1517 {
1518 struct page *start_page;
1519 struct page *end_page;
1520
1521
1522 end_pfn--;
1523
1524 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1525 return NULL;
1526
1527 start_page = pfn_to_online_page(start_pfn);
1528 if (!start_page)
1529 return NULL;
1530
1531 if (page_zone(start_page) != zone)
1532 return NULL;
1533
1534 end_page = pfn_to_page(end_pfn);
1535
1536
1537 if (page_zone_id(start_page) != page_zone_id(end_page))
1538 return NULL;
1539
1540 return start_page;
1541 }
1542
1543 void set_zone_contiguous(struct zone *zone)
1544 {
1545 unsigned long block_start_pfn = zone->zone_start_pfn;
1546 unsigned long block_end_pfn;
1547
1548 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1549 for (; block_start_pfn < zone_end_pfn(zone);
1550 block_start_pfn = block_end_pfn,
1551 block_end_pfn += pageblock_nr_pages) {
1552
1553 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1554
1555 if (!__pageblock_pfn_to_page(block_start_pfn,
1556 block_end_pfn, zone))
1557 return;
1558 cond_resched();
1559 }
1560
1561
1562 zone->contiguous = true;
1563 }
1564
1565 void clear_zone_contiguous(struct zone *zone)
1566 {
1567 zone->contiguous = false;
1568 }
1569
1570 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1571 static void __init deferred_free_range(unsigned long pfn,
1572 unsigned long nr_pages)
1573 {
1574 struct page *page;
1575 unsigned long i;
1576
1577 if (!nr_pages)
1578 return;
1579
1580 page = pfn_to_page(pfn);
1581
1582
1583 if (nr_pages == pageblock_nr_pages &&
1584 (pfn & (pageblock_nr_pages - 1)) == 0) {
1585 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1586 __free_pages_core(page, pageblock_order);
1587 return;
1588 }
1589
1590 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1591 if ((pfn & (pageblock_nr_pages - 1)) == 0)
1592 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1593 __free_pages_core(page, 0);
1594 }
1595 }
1596
1597
1598 static atomic_t pgdat_init_n_undone __initdata;
1599 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1600
1601 static inline void __init pgdat_init_report_one_done(void)
1602 {
1603 if (atomic_dec_and_test(&pgdat_init_n_undone))
1604 complete(&pgdat_init_all_done_comp);
1605 }
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617 static inline bool __init deferred_pfn_valid(unsigned long pfn)
1618 {
1619 if (!pfn_valid_within(pfn))
1620 return false;
1621 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
1622 return false;
1623 return true;
1624 }
1625
1626
1627
1628
1629
1630 static void __init deferred_free_pages(unsigned long pfn,
1631 unsigned long end_pfn)
1632 {
1633 unsigned long nr_pgmask = pageblock_nr_pages - 1;
1634 unsigned long nr_free = 0;
1635
1636 for (; pfn < end_pfn; pfn++) {
1637 if (!deferred_pfn_valid(pfn)) {
1638 deferred_free_range(pfn - nr_free, nr_free);
1639 nr_free = 0;
1640 } else if (!(pfn & nr_pgmask)) {
1641 deferred_free_range(pfn - nr_free, nr_free);
1642 nr_free = 1;
1643 touch_nmi_watchdog();
1644 } else {
1645 nr_free++;
1646 }
1647 }
1648
1649 deferred_free_range(pfn - nr_free, nr_free);
1650 }
1651
1652
1653
1654
1655
1656
1657 static unsigned long __init deferred_init_pages(struct zone *zone,
1658 unsigned long pfn,
1659 unsigned long end_pfn)
1660 {
1661 unsigned long nr_pgmask = pageblock_nr_pages - 1;
1662 int nid = zone_to_nid(zone);
1663 unsigned long nr_pages = 0;
1664 int zid = zone_idx(zone);
1665 struct page *page = NULL;
1666
1667 for (; pfn < end_pfn; pfn++) {
1668 if (!deferred_pfn_valid(pfn)) {
1669 page = NULL;
1670 continue;
1671 } else if (!page || !(pfn & nr_pgmask)) {
1672 page = pfn_to_page(pfn);
1673 touch_nmi_watchdog();
1674 } else {
1675 page++;
1676 }
1677 __init_single_page(page, pfn, zid, nid);
1678 nr_pages++;
1679 }
1680 return (nr_pages);
1681 }
1682
1683
1684
1685
1686
1687
1688
1689 static bool __init
1690 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
1691 unsigned long *spfn, unsigned long *epfn,
1692 unsigned long first_init_pfn)
1693 {
1694 u64 j;
1695
1696
1697
1698
1699
1700
1701 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
1702 if (*epfn <= first_init_pfn)
1703 continue;
1704 if (*spfn < first_init_pfn)
1705 *spfn = first_init_pfn;
1706 *i = j;
1707 return true;
1708 }
1709
1710 return false;
1711 }
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723 static unsigned long __init
1724 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
1725 unsigned long *end_pfn)
1726 {
1727 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
1728 unsigned long spfn = *start_pfn, epfn = *end_pfn;
1729 unsigned long nr_pages = 0;
1730 u64 j = *i;
1731
1732
1733 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
1734 unsigned long t;
1735
1736 if (mo_pfn <= *start_pfn)
1737 break;
1738
1739 t = min(mo_pfn, *end_pfn);
1740 nr_pages += deferred_init_pages(zone, *start_pfn, t);
1741
1742 if (mo_pfn < *end_pfn) {
1743 *start_pfn = mo_pfn;
1744 break;
1745 }
1746 }
1747
1748
1749 swap(j, *i);
1750
1751 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
1752 unsigned long t;
1753
1754 if (mo_pfn <= spfn)
1755 break;
1756
1757 t = min(mo_pfn, epfn);
1758 deferred_free_pages(spfn, t);
1759
1760 if (mo_pfn <= epfn)
1761 break;
1762 }
1763
1764 return nr_pages;
1765 }
1766
1767
1768 static int __init deferred_init_memmap(void *data)
1769 {
1770 pg_data_t *pgdat = data;
1771 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1772 unsigned long spfn = 0, epfn = 0, nr_pages = 0;
1773 unsigned long first_init_pfn, flags;
1774 unsigned long start = jiffies;
1775 struct zone *zone;
1776 int zid;
1777 u64 i;
1778
1779
1780 if (!cpumask_empty(cpumask))
1781 set_cpus_allowed_ptr(current, cpumask);
1782
1783 pgdat_resize_lock(pgdat, &flags);
1784 first_init_pfn = pgdat->first_deferred_pfn;
1785 if (first_init_pfn == ULONG_MAX) {
1786 pgdat_resize_unlock(pgdat, &flags);
1787 pgdat_init_report_one_done();
1788 return 0;
1789 }
1790
1791
1792 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1793 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1794 pgdat->first_deferred_pfn = ULONG_MAX;
1795
1796
1797 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1798 zone = pgdat->node_zones + zid;
1799 if (first_init_pfn < zone_end_pfn(zone))
1800 break;
1801 }
1802
1803
1804 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
1805 first_init_pfn))
1806 goto zone_empty;
1807
1808
1809
1810
1811
1812
1813 while (spfn < epfn)
1814 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
1815 zone_empty:
1816 pgdat_resize_unlock(pgdat, &flags);
1817
1818
1819 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
1820
1821 pr_info("node %d initialised, %lu pages in %ums\n",
1822 pgdat->node_id, nr_pages, jiffies_to_msecs(jiffies - start));
1823
1824 pgdat_init_report_one_done();
1825 return 0;
1826 }
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843 static noinline bool __init
1844 deferred_grow_zone(struct zone *zone, unsigned int order)
1845 {
1846 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
1847 pg_data_t *pgdat = zone->zone_pgdat;
1848 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
1849 unsigned long spfn, epfn, flags;
1850 unsigned long nr_pages = 0;
1851 u64 i;
1852
1853
1854 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
1855 return false;
1856
1857 pgdat_resize_lock(pgdat, &flags);
1858
1859
1860
1861
1862
1863
1864
1865 if (!static_branch_unlikely(&deferred_pages)) {
1866 pgdat_resize_unlock(pgdat, &flags);
1867 return true;
1868 }
1869
1870
1871
1872
1873
1874 if (first_deferred_pfn != pgdat->first_deferred_pfn) {
1875 pgdat_resize_unlock(pgdat, &flags);
1876 return true;
1877 }
1878
1879
1880 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
1881 first_deferred_pfn)) {
1882 pgdat->first_deferred_pfn = ULONG_MAX;
1883 pgdat_resize_unlock(pgdat, &flags);
1884
1885 return first_deferred_pfn != ULONG_MAX;
1886 }
1887
1888
1889
1890
1891
1892
1893 while (spfn < epfn) {
1894
1895 first_deferred_pfn = spfn;
1896
1897 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
1898
1899
1900 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
1901 continue;
1902
1903
1904 if (nr_pages >= nr_pages_needed)
1905 break;
1906 }
1907
1908 pgdat->first_deferred_pfn = spfn;
1909 pgdat_resize_unlock(pgdat, &flags);
1910
1911 return nr_pages > 0;
1912 }
1913
1914
1915
1916
1917
1918
1919
1920 static bool __ref
1921 _deferred_grow_zone(struct zone *zone, unsigned int order)
1922 {
1923 return deferred_grow_zone(zone, order);
1924 }
1925
1926 #endif
1927
1928 void __init page_alloc_init_late(void)
1929 {
1930 struct zone *zone;
1931 int nid;
1932
1933 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1934
1935
1936 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
1937 for_each_node_state(nid, N_MEMORY) {
1938 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
1939 }
1940
1941
1942 wait_for_completion(&pgdat_init_all_done_comp);
1943
1944
1945
1946
1947
1948
1949 for_each_populated_zone(zone)
1950 zone_pcp_update(zone);
1951
1952
1953
1954
1955
1956 static_branch_disable(&deferred_pages);
1957
1958
1959 files_maxfiles_init();
1960 #endif
1961
1962
1963 memblock_discard();
1964
1965 for_each_node_state(nid, N_MEMORY)
1966 shuffle_free_memory(NODE_DATA(nid));
1967
1968 for_each_populated_zone(zone)
1969 set_zone_contiguous(zone);
1970 }
1971
1972 #ifdef CONFIG_CMA
1973
1974 void __init init_cma_reserved_pageblock(struct page *page)
1975 {
1976 unsigned i = pageblock_nr_pages;
1977 struct page *p = page;
1978
1979 do {
1980 __ClearPageReserved(p);
1981 set_page_count(p, 0);
1982 } while (++p, --i);
1983
1984 set_pageblock_migratetype(page, MIGRATE_CMA);
1985
1986 if (pageblock_order >= MAX_ORDER) {
1987 i = pageblock_nr_pages;
1988 p = page;
1989 do {
1990 set_page_refcounted(p);
1991 __free_pages(p, MAX_ORDER - 1);
1992 p += MAX_ORDER_NR_PAGES;
1993 } while (i -= MAX_ORDER_NR_PAGES);
1994 } else {
1995 set_page_refcounted(page);
1996 __free_pages(page, pageblock_order);
1997 }
1998
1999 adjust_managed_page_count(page, pageblock_nr_pages);
2000 }
2001 #endif
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017 static inline void expand(struct zone *zone, struct page *page,
2018 int low, int high, struct free_area *area,
2019 int migratetype)
2020 {
2021 unsigned long size = 1 << high;
2022
2023 while (high > low) {
2024 area--;
2025 high--;
2026 size >>= 1;
2027 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
2028
2029
2030
2031
2032
2033
2034
2035 if (set_page_guard(zone, &page[size], high, migratetype))
2036 continue;
2037
2038 add_to_free_area(&page[size], area, migratetype);
2039 set_page_order(&page[size], high);
2040 }
2041 }
2042
2043 static void check_new_page_bad(struct page *page)
2044 {
2045 const char *bad_reason = NULL;
2046 unsigned long bad_flags = 0;
2047
2048 if (unlikely(atomic_read(&page->_mapcount) != -1))
2049 bad_reason = "nonzero mapcount";
2050 if (unlikely(page->mapping != NULL))
2051 bad_reason = "non-NULL mapping";
2052 if (unlikely(page_ref_count(page) != 0))
2053 bad_reason = "nonzero _refcount";
2054 if (unlikely(page->flags & __PG_HWPOISON)) {
2055 bad_reason = "HWPoisoned (hardware-corrupted)";
2056 bad_flags = __PG_HWPOISON;
2057
2058 page_mapcount_reset(page);
2059 return;
2060 }
2061 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
2062 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
2063 bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
2064 }
2065 #ifdef CONFIG_MEMCG
2066 if (unlikely(page->mem_cgroup))
2067 bad_reason = "page still charged to cgroup";
2068 #endif
2069 bad_page(page, bad_reason, bad_flags);
2070 }
2071
2072
2073
2074
2075 static inline int check_new_page(struct page *page)
2076 {
2077 if (likely(page_expected_state(page,
2078 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
2079 return 0;
2080
2081 check_new_page_bad(page);
2082 return 1;
2083 }
2084
2085 static inline bool free_pages_prezeroed(void)
2086 {
2087 return (IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
2088 page_poisoning_enabled()) || want_init_on_free();
2089 }
2090
2091 #ifdef CONFIG_DEBUG_VM
2092
2093
2094
2095
2096
2097 static inline bool check_pcp_refill(struct page *page)
2098 {
2099 if (debug_pagealloc_enabled_static())
2100 return check_new_page(page);
2101 else
2102 return false;
2103 }
2104
2105 static inline bool check_new_pcp(struct page *page)
2106 {
2107 return check_new_page(page);
2108 }
2109 #else
2110
2111
2112
2113
2114
2115 static inline bool check_pcp_refill(struct page *page)
2116 {
2117 return check_new_page(page);
2118 }
2119 static inline bool check_new_pcp(struct page *page)
2120 {
2121 if (debug_pagealloc_enabled_static())
2122 return check_new_page(page);
2123 else
2124 return false;
2125 }
2126 #endif
2127
2128 static bool check_new_pages(struct page *page, unsigned int order)
2129 {
2130 int i;
2131 for (i = 0; i < (1 << order); i++) {
2132 struct page *p = page + i;
2133
2134 if (unlikely(check_new_page(p)))
2135 return true;
2136 }
2137
2138 return false;
2139 }
2140
2141 inline void post_alloc_hook(struct page *page, unsigned int order,
2142 gfp_t gfp_flags)
2143 {
2144 set_page_private(page, 0);
2145 set_page_refcounted(page);
2146
2147 arch_alloc_page(page, order);
2148 if (debug_pagealloc_enabled_static())
2149 kernel_map_pages(page, 1 << order, 1);
2150 kasan_alloc_pages(page, order);
2151 kernel_poison_pages(page, 1 << order, 1);
2152 set_page_owner(page, order, gfp_flags);
2153 }
2154
2155 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
2156 unsigned int alloc_flags)
2157 {
2158 post_alloc_hook(page, order, gfp_flags);
2159
2160 if (!free_pages_prezeroed() && want_init_on_alloc(gfp_flags))
2161 kernel_init_free_pages(page, 1 << order);
2162
2163 if (order && (gfp_flags & __GFP_COMP))
2164 prep_compound_page(page, order);
2165
2166
2167
2168
2169
2170
2171
2172 if (alloc_flags & ALLOC_NO_WATERMARKS)
2173 set_page_pfmemalloc(page);
2174 else
2175 clear_page_pfmemalloc(page);
2176 }
2177
2178
2179
2180
2181
2182 static __always_inline
2183 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
2184 int migratetype)
2185 {
2186 unsigned int current_order;
2187 struct free_area *area;
2188 struct page *page;
2189
2190
2191 for (current_order = order; current_order < MAX_ORDER; ++current_order) {
2192 area = &(zone->free_area[current_order]);
2193 page = get_page_from_free_area(area, migratetype);
2194 if (!page)
2195 continue;
2196 del_page_from_free_area(page, area);
2197 expand(zone, page, order, current_order, area, migratetype);
2198 set_pcppage_migratetype(page, migratetype);
2199 return page;
2200 }
2201
2202 return NULL;
2203 }
2204
2205
2206
2207
2208
2209
2210 static int fallbacks[MIGRATE_TYPES][4] = {
2211 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
2212 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
2213 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
2214 #ifdef CONFIG_CMA
2215 [MIGRATE_CMA] = { MIGRATE_TYPES },
2216 #endif
2217 #ifdef CONFIG_MEMORY_ISOLATION
2218 [MIGRATE_ISOLATE] = { MIGRATE_TYPES },
2219 #endif
2220 };
2221
2222 #ifdef CONFIG_CMA
2223 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2224 unsigned int order)
2225 {
2226 return __rmqueue_smallest(zone, order, MIGRATE_CMA);
2227 }
2228 #else
2229 static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2230 unsigned int order) { return NULL; }
2231 #endif
2232
2233
2234
2235
2236
2237
2238 static int move_freepages(struct zone *zone,
2239 struct page *start_page, struct page *end_page,
2240 int migratetype, int *num_movable)
2241 {
2242 struct page *page;
2243 unsigned int order;
2244 int pages_moved = 0;
2245
2246 for (page = start_page; page <= end_page;) {
2247 if (!pfn_valid_within(page_to_pfn(page))) {
2248 page++;
2249 continue;
2250 }
2251
2252 if (!PageBuddy(page)) {
2253
2254
2255
2256
2257
2258 if (num_movable &&
2259 (PageLRU(page) || __PageMovable(page)))
2260 (*num_movable)++;
2261
2262 page++;
2263 continue;
2264 }
2265
2266
2267 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2268 VM_BUG_ON_PAGE(page_zone(page) != zone, page);
2269
2270 order = page_order(page);
2271 move_to_free_area(page, &zone->free_area[order], migratetype);
2272 page += 1 << order;
2273 pages_moved += 1 << order;
2274 }
2275
2276 return pages_moved;
2277 }
2278
2279 int move_freepages_block(struct zone *zone, struct page *page,
2280 int migratetype, int *num_movable)
2281 {
2282 unsigned long start_pfn, end_pfn;
2283 struct page *start_page, *end_page;
2284
2285 if (num_movable)
2286 *num_movable = 0;
2287
2288 start_pfn = page_to_pfn(page);
2289 start_pfn = start_pfn & ~(pageblock_nr_pages-1);
2290 start_page = pfn_to_page(start_pfn);
2291 end_page = start_page + pageblock_nr_pages - 1;
2292 end_pfn = start_pfn + pageblock_nr_pages - 1;
2293
2294
2295 if (!zone_spans_pfn(zone, start_pfn))
2296 start_page = page;
2297 if (!zone_spans_pfn(zone, end_pfn))
2298 return 0;
2299
2300 return move_freepages(zone, start_page, end_page, migratetype,
2301 num_movable);
2302 }
2303
2304 static void change_pageblock_range(struct page *pageblock_page,
2305 int start_order, int migratetype)
2306 {
2307 int nr_pageblocks = 1 << (start_order - pageblock_order);
2308
2309 while (nr_pageblocks--) {
2310 set_pageblock_migratetype(pageblock_page, migratetype);
2311 pageblock_page += pageblock_nr_pages;
2312 }
2313 }
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327 static bool can_steal_fallback(unsigned int order, int start_mt)
2328 {
2329
2330
2331
2332
2333
2334
2335
2336 if (order >= pageblock_order)
2337 return true;
2338
2339 if (order >= pageblock_order / 2 ||
2340 start_mt == MIGRATE_RECLAIMABLE ||
2341 start_mt == MIGRATE_UNMOVABLE ||
2342 page_group_by_mobility_disabled)
2343 return true;
2344
2345 return false;
2346 }
2347
2348 static inline void boost_watermark(struct zone *zone)
2349 {
2350 unsigned long max_boost;
2351
2352 if (!watermark_boost_factor)
2353 return;
2354
2355
2356
2357
2358
2359
2360 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
2361 return;
2362
2363 max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2364 watermark_boost_factor, 10000);
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374 if (!max_boost)
2375 return;
2376
2377 max_boost = max(pageblock_nr_pages, max_boost);
2378
2379 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2380 max_boost);
2381 }
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391 static void steal_suitable_fallback(struct zone *zone, struct page *page,
2392 unsigned int alloc_flags, int start_type, bool whole_block)
2393 {
2394 unsigned int current_order = page_order(page);
2395 struct free_area *area;
2396 int free_pages, movable_pages, alike_pages;
2397 int old_block_type;
2398
2399 old_block_type = get_pageblock_migratetype(page);
2400
2401
2402
2403
2404
2405 if (is_migrate_highatomic(old_block_type))
2406 goto single_page;
2407
2408
2409 if (current_order >= pageblock_order) {
2410 change_pageblock_range(page, current_order, start_type);
2411 goto single_page;
2412 }
2413
2414
2415
2416
2417
2418
2419 boost_watermark(zone);
2420 if (alloc_flags & ALLOC_KSWAPD)
2421 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
2422
2423
2424 if (!whole_block)
2425 goto single_page;
2426
2427 free_pages = move_freepages_block(zone, page, start_type,
2428 &movable_pages);
2429
2430
2431
2432
2433
2434 if (start_type == MIGRATE_MOVABLE) {
2435 alike_pages = movable_pages;
2436 } else {
2437
2438
2439
2440
2441
2442
2443
2444 if (old_block_type == MIGRATE_MOVABLE)
2445 alike_pages = pageblock_nr_pages
2446 - (free_pages + movable_pages);
2447 else
2448 alike_pages = 0;
2449 }
2450
2451
2452 if (!free_pages)
2453 goto single_page;
2454
2455
2456
2457
2458
2459 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
2460 page_group_by_mobility_disabled)
2461 set_pageblock_migratetype(page, start_type);
2462
2463 return;
2464
2465 single_page:
2466 area = &zone->free_area[current_order];
2467 move_to_free_area(page, area, start_type);
2468 }
2469
2470
2471
2472
2473
2474
2475
2476 int find_suitable_fallback(struct free_area *area, unsigned int order,
2477 int migratetype, bool only_stealable, bool *can_steal)
2478 {
2479 int i;
2480 int fallback_mt;
2481
2482 if (area->nr_free == 0)
2483 return -1;
2484
2485 *can_steal = false;
2486 for (i = 0;; i++) {
2487 fallback_mt = fallbacks[migratetype][i];
2488 if (fallback_mt == MIGRATE_TYPES)
2489 break;
2490
2491 if (free_area_empty(area, fallback_mt))
2492 continue;
2493
2494 if (can_steal_fallback(order, migratetype))
2495 *can_steal = true;
2496
2497 if (!only_stealable)
2498 return fallback_mt;
2499
2500 if (*can_steal)
2501 return fallback_mt;
2502 }
2503
2504 return -1;
2505 }
2506
2507
2508
2509
2510
2511 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2512 unsigned int alloc_order)
2513 {
2514 int mt;
2515 unsigned long max_managed, flags;
2516
2517
2518
2519
2520
2521 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
2522 if (zone->nr_reserved_highatomic >= max_managed)
2523 return;
2524
2525 spin_lock_irqsave(&zone->lock, flags);
2526
2527
2528 if (zone->nr_reserved_highatomic >= max_managed)
2529 goto out_unlock;
2530
2531
2532 mt = get_pageblock_migratetype(page);
2533 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2534 && !is_migrate_cma(mt)) {
2535 zone->nr_reserved_highatomic += pageblock_nr_pages;
2536 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2537 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2538 }
2539
2540 out_unlock:
2541 spin_unlock_irqrestore(&zone->lock, flags);
2542 }
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2554 bool force)
2555 {
2556 struct zonelist *zonelist = ac->zonelist;
2557 unsigned long flags;
2558 struct zoneref *z;
2559 struct zone *zone;
2560 struct page *page;
2561 int order;
2562 bool ret;
2563
2564 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
2565 ac->nodemask) {
2566
2567
2568
2569
2570 if (!force && zone->nr_reserved_highatomic <=
2571 pageblock_nr_pages)
2572 continue;
2573
2574 spin_lock_irqsave(&zone->lock, flags);
2575 for (order = 0; order < MAX_ORDER; order++) {
2576 struct free_area *area = &(zone->free_area[order]);
2577
2578 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
2579 if (!page)
2580 continue;
2581
2582
2583
2584
2585
2586
2587
2588
2589 if (is_migrate_highatomic_page(page)) {
2590
2591
2592
2593
2594
2595
2596
2597 zone->nr_reserved_highatomic -= min(
2598 pageblock_nr_pages,
2599 zone->nr_reserved_highatomic);
2600 }
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611 set_pageblock_migratetype(page, ac->migratetype);
2612 ret = move_freepages_block(zone, page, ac->migratetype,
2613 NULL);
2614 if (ret) {
2615 spin_unlock_irqrestore(&zone->lock, flags);
2616 return ret;
2617 }
2618 }
2619 spin_unlock_irqrestore(&zone->lock, flags);
2620 }
2621
2622 return false;
2623 }
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635 static __always_inline bool
2636 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
2637 unsigned int alloc_flags)
2638 {
2639 struct free_area *area;
2640 int current_order;
2641 int min_order = order;
2642 struct page *page;
2643 int fallback_mt;
2644 bool can_steal;
2645
2646
2647
2648
2649
2650
2651 if (alloc_flags & ALLOC_NOFRAGMENT)
2652 min_order = pageblock_order;
2653
2654
2655
2656
2657
2658
2659 for (current_order = MAX_ORDER - 1; current_order >= min_order;
2660 --current_order) {
2661 area = &(zone->free_area[current_order]);
2662 fallback_mt = find_suitable_fallback(area, current_order,
2663 start_migratetype, false, &can_steal);
2664 if (fallback_mt == -1)
2665 continue;
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675 if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2676 && current_order > order)
2677 goto find_smallest;
2678
2679 goto do_steal;
2680 }
2681
2682 return false;
2683
2684 find_smallest:
2685 for (current_order = order; current_order < MAX_ORDER;
2686 current_order++) {
2687 area = &(zone->free_area[current_order]);
2688 fallback_mt = find_suitable_fallback(area, current_order,
2689 start_migratetype, false, &can_steal);
2690 if (fallback_mt != -1)
2691 break;
2692 }
2693
2694
2695
2696
2697
2698 VM_BUG_ON(current_order == MAX_ORDER);
2699
2700 do_steal:
2701 page = get_page_from_free_area(area, fallback_mt);
2702
2703 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2704 can_steal);
2705
2706 trace_mm_page_alloc_extfrag(page, order, current_order,
2707 start_migratetype, fallback_mt);
2708
2709 return true;
2710
2711 }
2712
2713
2714
2715
2716
2717 static __always_inline struct page *
2718 __rmqueue(struct zone *zone, unsigned int order, int migratetype,
2719 unsigned int alloc_flags)
2720 {
2721 struct page *page;
2722
2723 retry:
2724 page = __rmqueue_smallest(zone, order, migratetype);
2725 if (unlikely(!page)) {
2726 if (migratetype == MIGRATE_MOVABLE)
2727 page = __rmqueue_cma_fallback(zone, order);
2728
2729 if (!page && __rmqueue_fallback(zone, order, migratetype,
2730 alloc_flags))
2731 goto retry;
2732 }
2733
2734 trace_mm_page_alloc_zone_locked(page, order, migratetype);
2735 return page;
2736 }
2737
2738
2739
2740
2741
2742
2743 static int rmqueue_bulk(struct zone *zone, unsigned int order,
2744 unsigned long count, struct list_head *list,
2745 int migratetype, unsigned int alloc_flags)
2746 {
2747 int i, alloced = 0;
2748
2749 spin_lock(&zone->lock);
2750 for (i = 0; i < count; ++i) {
2751 struct page *page = __rmqueue(zone, order, migratetype,
2752 alloc_flags);
2753 if (unlikely(page == NULL))
2754 break;
2755
2756 if (unlikely(check_pcp_refill(page)))
2757 continue;
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769 list_add_tail(&page->lru, list);
2770 alloced++;
2771 if (is_migrate_cma(get_pcppage_migratetype(page)))
2772 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
2773 -(1 << order));
2774 }
2775
2776
2777
2778
2779
2780
2781
2782 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
2783 spin_unlock(&zone->lock);
2784 return alloced;
2785 }
2786
2787 #ifdef CONFIG_NUMA
2788
2789
2790
2791
2792
2793
2794
2795
2796 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
2797 {
2798 unsigned long flags;
2799 int to_drain, batch;
2800
2801 local_irq_save(flags);
2802 batch = READ_ONCE(pcp->batch);
2803 to_drain = min(pcp->count, batch);
2804 if (to_drain > 0)
2805 free_pcppages_bulk(zone, to_drain, pcp);
2806 local_irq_restore(flags);
2807 }
2808 #endif
2809
2810
2811
2812
2813
2814
2815
2816
2817 static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2818 {
2819 unsigned long flags;
2820 struct per_cpu_pageset *pset;
2821 struct per_cpu_pages *pcp;
2822
2823 local_irq_save(flags);
2824 pset = per_cpu_ptr(zone->pageset, cpu);
2825
2826 pcp = &pset->pcp;
2827 if (pcp->count)
2828 free_pcppages_bulk(zone, pcp->count, pcp);
2829 local_irq_restore(flags);
2830 }
2831
2832
2833
2834
2835
2836
2837
2838
2839 static void drain_pages(unsigned int cpu)
2840 {
2841 struct zone *zone;
2842
2843 for_each_populated_zone(zone) {
2844 drain_pages_zone(cpu, zone);
2845 }
2846 }
2847
2848
2849
2850
2851
2852
2853
2854 void drain_local_pages(struct zone *zone)
2855 {
2856 int cpu = smp_processor_id();
2857
2858 if (zone)
2859 drain_pages_zone(cpu, zone);
2860 else
2861 drain_pages(cpu);
2862 }
2863
2864 static void drain_local_pages_wq(struct work_struct *work)
2865 {
2866 struct pcpu_drain *drain;
2867
2868 drain = container_of(work, struct pcpu_drain, work);
2869
2870
2871
2872
2873
2874
2875
2876
2877 preempt_disable();
2878 drain_local_pages(drain->zone);
2879 preempt_enable();
2880 }
2881
2882
2883
2884
2885
2886
2887
2888
2889 void drain_all_pages(struct zone *zone)
2890 {
2891 int cpu;
2892
2893
2894
2895
2896
2897 static cpumask_t cpus_with_pcps;
2898
2899
2900
2901
2902
2903 if (WARN_ON_ONCE(!mm_percpu_wq))
2904 return;
2905
2906
2907
2908
2909
2910
2911 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
2912 if (!zone)
2913 return;
2914 mutex_lock(&pcpu_drain_mutex);
2915 }
2916
2917
2918
2919
2920
2921
2922
2923 for_each_online_cpu(cpu) {
2924 struct per_cpu_pageset *pcp;
2925 struct zone *z;
2926 bool has_pcps = false;
2927
2928 if (zone) {
2929 pcp = per_cpu_ptr(zone->pageset, cpu);
2930 if (pcp->pcp.count)
2931 has_pcps = true;
2932 } else {
2933 for_each_populated_zone(z) {
2934 pcp = per_cpu_ptr(z->pageset, cpu);
2935 if (pcp->pcp.count) {
2936 has_pcps = true;
2937 break;
2938 }
2939 }
2940 }
2941
2942 if (has_pcps)
2943 cpumask_set_cpu(cpu, &cpus_with_pcps);
2944 else
2945 cpumask_clear_cpu(cpu, &cpus_with_pcps);
2946 }
2947
2948 for_each_cpu(cpu, &cpus_with_pcps) {
2949 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
2950
2951 drain->zone = zone;
2952 INIT_WORK(&drain->work, drain_local_pages_wq);
2953 queue_work_on(cpu, mm_percpu_wq, &drain->work);
2954 }
2955 for_each_cpu(cpu, &cpus_with_pcps)
2956 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
2957
2958 mutex_unlock(&pcpu_drain_mutex);
2959 }
2960
2961 #ifdef CONFIG_HIBERNATION
2962
2963
2964
2965
2966 #define WD_PAGE_COUNT (128*1024)
2967
2968 void mark_free_pages(struct zone *zone)
2969 {
2970 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
2971 unsigned long flags;
2972 unsigned int order, t;
2973 struct page *page;
2974
2975 if (zone_is_empty(zone))
2976 return;
2977
2978 spin_lock_irqsave(&zone->lock, flags);
2979
2980 max_zone_pfn = zone_end_pfn(zone);
2981 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
2982 if (pfn_valid(pfn)) {
2983 page = pfn_to_page(pfn);
2984
2985 if (!--page_count) {
2986 touch_nmi_watchdog();
2987 page_count = WD_PAGE_COUNT;
2988 }
2989
2990 if (page_zone(page) != zone)
2991 continue;
2992
2993 if (!swsusp_page_is_forbidden(page))
2994 swsusp_unset_page_free(page);
2995 }
2996
2997 for_each_migratetype_order(order, t) {
2998 list_for_each_entry(page,
2999 &zone->free_area[order].free_list[t], lru) {
3000 unsigned long i;
3001
3002 pfn = page_to_pfn(page);
3003 for (i = 0; i < (1UL << order); i++) {
3004 if (!--page_count) {
3005 touch_nmi_watchdog();
3006 page_count = WD_PAGE_COUNT;
3007 }
3008 swsusp_set_page_free(pfn_to_page(pfn + i));
3009 }
3010 }
3011 }
3012 spin_unlock_irqrestore(&zone->lock, flags);
3013 }
3014 #endif
3015
3016 static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
3017 {
3018 int migratetype;
3019
3020 if (!free_pcp_prepare(page))
3021 return false;
3022
3023 migratetype = get_pfnblock_migratetype(page, pfn);
3024 set_pcppage_migratetype(page, migratetype);
3025 return true;
3026 }
3027
3028 static void free_unref_page_commit(struct page *page, unsigned long pfn)
3029 {
3030 struct zone *zone = page_zone(page);
3031 struct per_cpu_pages *pcp;
3032 int migratetype;
3033
3034 migratetype = get_pcppage_migratetype(page);
3035 __count_vm_event(PGFREE);
3036
3037
3038
3039
3040
3041
3042
3043
3044 if (migratetype >= MIGRATE_PCPTYPES) {
3045 if (unlikely(is_migrate_isolate(migratetype))) {
3046 free_one_page(zone, page, pfn, 0, migratetype);
3047 return;
3048 }
3049 migratetype = MIGRATE_MOVABLE;
3050 }
3051
3052 pcp = &this_cpu_ptr(zone->pageset)->pcp;
3053 list_add(&page->lru, &pcp->lists[migratetype]);
3054 pcp->count++;
3055 if (pcp->count >= pcp->high) {
3056 unsigned long batch = READ_ONCE(pcp->batch);
3057 free_pcppages_bulk(zone, batch, pcp);
3058 }
3059 }
3060
3061
3062
3063
3064 void free_unref_page(struct page *page)
3065 {
3066 unsigned long flags;
3067 unsigned long pfn = page_to_pfn(page);
3068
3069 if (!free_unref_page_prepare(page, pfn))
3070 return;
3071
3072 local_irq_save(flags);
3073 free_unref_page_commit(page, pfn);
3074 local_irq_restore(flags);
3075 }
3076
3077
3078
3079
3080 void free_unref_page_list(struct list_head *list)
3081 {
3082 struct page *page, *next;
3083 unsigned long flags, pfn;
3084 int batch_count = 0;
3085
3086
3087 list_for_each_entry_safe(page, next, list, lru) {
3088 pfn = page_to_pfn(page);
3089 if (!free_unref_page_prepare(page, pfn))
3090 list_del(&page->lru);
3091 set_page_private(page, pfn);
3092 }
3093
3094 local_irq_save(flags);
3095 list_for_each_entry_safe(page, next, list, lru) {
3096 unsigned long pfn = page_private(page);
3097
3098 set_page_private(page, 0);
3099 trace_mm_page_free_batched(page);
3100 free_unref_page_commit(page, pfn);
3101
3102
3103
3104
3105
3106 if (++batch_count == SWAP_CLUSTER_MAX) {
3107 local_irq_restore(flags);
3108 batch_count = 0;
3109 local_irq_save(flags);
3110 }
3111 }
3112 local_irq_restore(flags);
3113 }
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123 void split_page(struct page *page, unsigned int order)
3124 {
3125 int i;
3126
3127 VM_BUG_ON_PAGE(PageCompound(page), page);
3128 VM_BUG_ON_PAGE(!page_count(page), page);
3129
3130 for (i = 1; i < (1 << order); i++)
3131 set_page_refcounted(page + i);
3132 split_page_owner(page, order);
3133 }
3134 EXPORT_SYMBOL_GPL(split_page);
3135
3136 int __isolate_free_page(struct page *page, unsigned int order)
3137 {
3138 struct free_area *area = &page_zone(page)->free_area[order];
3139 unsigned long watermark;
3140 struct zone *zone;
3141 int mt;
3142
3143 BUG_ON(!PageBuddy(page));
3144
3145 zone = page_zone(page);
3146 mt = get_pageblock_migratetype(page);
3147
3148 if (!is_migrate_isolate(mt)) {
3149
3150
3151
3152
3153
3154
3155 watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
3156 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
3157 return 0;
3158
3159 __mod_zone_freepage_state(zone, -(1UL << order), mt);
3160 }
3161
3162
3163
3164 del_page_from_free_area(page, area);
3165
3166
3167
3168
3169
3170 if (order >= pageblock_order - 1) {
3171 struct page *endpage = page + (1 << order) - 1;
3172 for (; page < endpage; page += pageblock_nr_pages) {
3173 int mt = get_pageblock_migratetype(page);
3174 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
3175 && !is_migrate_highatomic(mt))
3176 set_pageblock_migratetype(page,
3177 MIGRATE_MOVABLE);
3178 }
3179 }
3180
3181
3182 return 1UL << order;
3183 }
3184
3185
3186
3187
3188
3189
3190 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
3191 {
3192 #ifdef CONFIG_NUMA
3193 enum numa_stat_item local_stat = NUMA_LOCAL;
3194
3195
3196 if (!static_branch_likely(&vm_numa_stat_key))
3197 return;
3198
3199 if (zone_to_nid(z) != numa_node_id())
3200 local_stat = NUMA_OTHER;
3201
3202 if (zone_to_nid(z) == zone_to_nid(preferred_zone))
3203 __inc_numa_state(z, NUMA_HIT);
3204 else {
3205 __inc_numa_state(z, NUMA_MISS);
3206 __inc_numa_state(preferred_zone, NUMA_FOREIGN);
3207 }
3208 __inc_numa_state(z, local_stat);
3209 #endif
3210 }
3211
3212
3213 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
3214 unsigned int alloc_flags,
3215 struct per_cpu_pages *pcp,
3216 struct list_head *list)
3217 {
3218 struct page *page;
3219
3220 do {
3221 if (list_empty(list)) {
3222 pcp->count += rmqueue_bulk(zone, 0,
3223 pcp->batch, list,
3224 migratetype, alloc_flags);
3225 if (unlikely(list_empty(list)))
3226 return NULL;
3227 }
3228
3229 page = list_first_entry(list, struct page, lru);
3230 list_del(&page->lru);
3231 pcp->count--;
3232 } while (check_new_pcp(page));
3233
3234 return page;
3235 }
3236
3237
3238 static struct page *rmqueue_pcplist(struct zone *preferred_zone,
3239 struct zone *zone, gfp_t gfp_flags,
3240 int migratetype, unsigned int alloc_flags)
3241 {
3242 struct per_cpu_pages *pcp;
3243 struct list_head *list;
3244 struct page *page;
3245 unsigned long flags;
3246
3247 local_irq_save(flags);
3248 pcp = &this_cpu_ptr(zone->pageset)->pcp;
3249 list = &pcp->lists[migratetype];
3250 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list);
3251 if (page) {
3252 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
3253 zone_statistics(preferred_zone, zone);
3254 }
3255 local_irq_restore(flags);
3256 return page;
3257 }
3258
3259
3260
3261
3262 static inline
3263 struct page *rmqueue(struct zone *preferred_zone,
3264 struct zone *zone, unsigned int order,
3265 gfp_t gfp_flags, unsigned int alloc_flags,
3266 int migratetype)
3267 {
3268 unsigned long flags;
3269 struct page *page;
3270
3271 if (likely(order == 0)) {
3272 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
3273 migratetype, alloc_flags);
3274 goto out;
3275 }
3276
3277
3278
3279
3280
3281 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
3282 spin_lock_irqsave(&zone->lock, flags);
3283
3284 do {
3285 page = NULL;
3286 if (alloc_flags & ALLOC_HARDER) {
3287 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3288 if (page)
3289 trace_mm_page_alloc_zone_locked(page, order, migratetype);
3290 }
3291 if (!page)
3292 page = __rmqueue(zone, order, migratetype, alloc_flags);
3293 } while (page && check_new_pages(page, order));
3294 spin_unlock(&zone->lock);
3295 if (!page)
3296 goto failed;
3297 __mod_zone_freepage_state(zone, -(1 << order),
3298 get_pcppage_migratetype(page));
3299
3300 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
3301 zone_statistics(preferred_zone, zone);
3302 local_irq_restore(flags);
3303
3304 out:
3305
3306 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
3307 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3308 wakeup_kswapd(zone, 0, 0, zone_idx(zone));
3309 }
3310
3311 VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
3312 return page;
3313
3314 failed:
3315 local_irq_restore(flags);
3316 return NULL;
3317 }
3318
3319 #ifdef CONFIG_FAIL_PAGE_ALLOC
3320
3321 static struct {
3322 struct fault_attr attr;
3323
3324 bool ignore_gfp_highmem;
3325 bool ignore_gfp_reclaim;
3326 u32 min_order;
3327 } fail_page_alloc = {
3328 .attr = FAULT_ATTR_INITIALIZER,
3329 .ignore_gfp_reclaim = true,
3330 .ignore_gfp_highmem = true,
3331 .min_order = 1,
3332 };
3333
3334 static int __init setup_fail_page_alloc(char *str)
3335 {
3336 return setup_fault_attr(&fail_page_alloc.attr, str);
3337 }
3338 __setup("fail_page_alloc=", setup_fail_page_alloc);
3339
3340 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3341 {
3342 if (order < fail_page_alloc.min_order)
3343 return false;
3344 if (gfp_mask & __GFP_NOFAIL)
3345 return false;
3346 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
3347 return false;
3348 if (fail_page_alloc.ignore_gfp_reclaim &&
3349 (gfp_mask & __GFP_DIRECT_RECLAIM))
3350 return false;
3351
3352 return should_fail(&fail_page_alloc.attr, 1 << order);
3353 }
3354
3355 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3356
3357 static int __init fail_page_alloc_debugfs(void)
3358 {
3359 umode_t mode = S_IFREG | 0600;
3360 struct dentry *dir;
3361
3362 dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
3363 &fail_page_alloc.attr);
3364
3365 debugfs_create_bool("ignore-gfp-wait", mode, dir,
3366 &fail_page_alloc.ignore_gfp_reclaim);
3367 debugfs_create_bool("ignore-gfp-highmem", mode, dir,
3368 &fail_page_alloc.ignore_gfp_highmem);
3369 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
3370
3371 return 0;
3372 }
3373
3374 late_initcall(fail_page_alloc_debugfs);
3375
3376 #endif
3377
3378 #else
3379
3380 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3381 {
3382 return false;
3383 }
3384
3385 #endif
3386
3387 static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3388 {
3389 return __should_fail_alloc_page(gfp_mask, order);
3390 }
3391 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
3392
3393
3394
3395
3396
3397
3398
3399 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3400 int classzone_idx, unsigned int alloc_flags,
3401 long free_pages)
3402 {
3403 long min = mark;
3404 int o;
3405 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3406
3407
3408 free_pages -= (1 << order) - 1;
3409
3410 if (alloc_flags & ALLOC_HIGH)
3411 min -= min / 2;
3412
3413
3414
3415
3416
3417
3418 if (likely(!alloc_harder)) {
3419 free_pages -= z->nr_reserved_highatomic;
3420 } else {
3421
3422
3423
3424
3425
3426
3427 if (alloc_flags & ALLOC_OOM)
3428 min -= min / 2;
3429 else
3430 min -= min / 4;
3431 }
3432
3433
3434 #ifdef CONFIG_CMA
3435
3436 if (!(alloc_flags & ALLOC_CMA))
3437 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
3438 #endif
3439
3440
3441
3442
3443
3444
3445 if (free_pages <= min + z->lowmem_reserve[classzone_idx])
3446 return false;
3447
3448
3449 if (!order)
3450 return true;
3451
3452
3453 for (o = order; o < MAX_ORDER; o++) {
3454 struct free_area *area = &z->free_area[o];
3455 int mt;
3456
3457 if (!area->nr_free)
3458 continue;
3459
3460 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
3461 if (!free_area_empty(area, mt))
3462 return true;
3463 }
3464
3465 #ifdef CONFIG_CMA
3466 if ((alloc_flags & ALLOC_CMA) &&
3467 !free_area_empty(area, MIGRATE_CMA)) {
3468 return true;
3469 }
3470 #endif
3471 if (alloc_harder &&
3472 !list_empty(&area->free_list[MIGRATE_HIGHATOMIC]))
3473 return true;
3474 }
3475 return false;
3476 }
3477
3478 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3479 int classzone_idx, unsigned int alloc_flags)
3480 {
3481 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3482 zone_page_state(z, NR_FREE_PAGES));
3483 }
3484
3485 static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
3486 unsigned long mark, int classzone_idx, unsigned int alloc_flags)
3487 {
3488 long free_pages = zone_page_state(z, NR_FREE_PAGES);
3489 long cma_pages = 0;
3490
3491 #ifdef CONFIG_CMA
3492
3493 if (!(alloc_flags & ALLOC_CMA))
3494 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
3495 #endif
3496
3497
3498
3499
3500
3501
3502
3503
3504 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
3505 return true;
3506
3507 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3508 free_pages);
3509 }
3510
3511 bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
3512 unsigned long mark, int classzone_idx)
3513 {
3514 long free_pages = zone_page_state(z, NR_FREE_PAGES);
3515
3516 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3517 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3518
3519 return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
3520 free_pages);
3521 }
3522
3523 #ifdef CONFIG_NUMA
3524 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3525 {
3526 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3527 node_reclaim_distance;
3528 }
3529 #else
3530 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3531 {
3532 return true;
3533 }
3534 #endif
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544 static inline unsigned int
3545 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
3546 {
3547 unsigned int alloc_flags = 0;
3548
3549 if (gfp_mask & __GFP_KSWAPD_RECLAIM)
3550 alloc_flags |= ALLOC_KSWAPD;
3551
3552 #ifdef CONFIG_ZONE_DMA32
3553 if (!zone)
3554 return alloc_flags;
3555
3556 if (zone_idx(zone) != ZONE_NORMAL)
3557 return alloc_flags;
3558
3559
3560
3561
3562
3563
3564 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
3565 if (nr_online_nodes > 1 && !populated_zone(--zone))
3566 return alloc_flags;
3567
3568 alloc_flags |= ALLOC_NOFRAGMENT;
3569 #endif
3570 return alloc_flags;
3571 }
3572
3573
3574
3575
3576
3577 static struct page *
3578 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
3579 const struct alloc_context *ac)
3580 {
3581 struct zoneref *z;
3582 struct zone *zone;
3583 struct pglist_data *last_pgdat_dirty_limit = NULL;
3584 bool no_fallback;
3585
3586 retry:
3587
3588
3589
3590
3591 no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
3592 z = ac->preferred_zoneref;
3593 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3594 ac->nodemask) {
3595 struct page *page;
3596 unsigned long mark;
3597
3598 if (cpusets_enabled() &&
3599 (alloc_flags & ALLOC_CPUSET) &&
3600 !__cpuset_zone_allowed(zone, gfp_mask))
3601 continue;
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621 if (ac->spread_dirty_pages) {
3622 if (last_pgdat_dirty_limit == zone->zone_pgdat)
3623 continue;
3624
3625 if (!node_dirty_ok(zone->zone_pgdat)) {
3626 last_pgdat_dirty_limit = zone->zone_pgdat;
3627 continue;
3628 }
3629 }
3630
3631 if (no_fallback && nr_online_nodes > 1 &&
3632 zone != ac->preferred_zoneref->zone) {
3633 int local_nid;
3634
3635
3636
3637
3638
3639
3640 local_nid = zone_to_nid(ac->preferred_zoneref->zone);
3641 if (zone_to_nid(zone) != local_nid) {
3642 alloc_flags &= ~ALLOC_NOFRAGMENT;
3643 goto retry;
3644 }
3645 }
3646
3647 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
3648 if (!zone_watermark_fast(zone, order, mark,
3649 ac_classzone_idx(ac), alloc_flags)) {
3650 int ret;
3651
3652 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3653
3654
3655
3656
3657 if (static_branch_unlikely(&deferred_pages)) {
3658 if (_deferred_grow_zone(zone, order))
3659 goto try_this_zone;
3660 }
3661 #endif
3662
3663 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
3664 if (alloc_flags & ALLOC_NO_WATERMARKS)
3665 goto try_this_zone;
3666
3667 if (node_reclaim_mode == 0 ||
3668 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
3669 continue;
3670
3671 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
3672 switch (ret) {
3673 case NODE_RECLAIM_NOSCAN:
3674
3675 continue;
3676 case NODE_RECLAIM_FULL:
3677
3678 continue;
3679 default:
3680
3681 if (zone_watermark_ok(zone, order, mark,
3682 ac_classzone_idx(ac), alloc_flags))
3683 goto try_this_zone;
3684
3685 continue;
3686 }
3687 }
3688
3689 try_this_zone:
3690 page = rmqueue(ac->preferred_zoneref->zone, zone, order,
3691 gfp_mask, alloc_flags, ac->migratetype);
3692 if (page) {
3693 prep_new_page(page, order, gfp_mask, alloc_flags);
3694
3695
3696
3697
3698
3699 if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
3700 reserve_highatomic_pageblock(page, zone, order);
3701
3702 return page;
3703 } else {
3704 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3705
3706 if (static_branch_unlikely(&deferred_pages)) {
3707 if (_deferred_grow_zone(zone, order))
3708 goto try_this_zone;
3709 }
3710 #endif
3711 }
3712 }
3713
3714
3715
3716
3717
3718 if (no_fallback) {
3719 alloc_flags &= ~ALLOC_NOFRAGMENT;
3720 goto retry;
3721 }
3722
3723 return NULL;
3724 }
3725
3726 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
3727 {
3728 unsigned int filter = SHOW_MEM_FILTER_NODES;
3729
3730
3731
3732
3733
3734
3735 if (!(gfp_mask & __GFP_NOMEMALLOC))
3736 if (tsk_is_oom_victim(current) ||
3737 (current->flags & (PF_MEMALLOC | PF_EXITING)))
3738 filter &= ~SHOW_MEM_FILTER_NODES;
3739 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
3740 filter &= ~SHOW_MEM_FILTER_NODES;
3741
3742 show_mem(filter, nodemask);
3743 }
3744
3745 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
3746 {
3747 struct va_format vaf;
3748 va_list args;
3749 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
3750
3751 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
3752 return;
3753
3754 va_start(args, fmt);
3755 vaf.fmt = fmt;
3756 vaf.va = &args;
3757 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
3758 current->comm, &vaf, gfp_mask, &gfp_mask,
3759 nodemask_pr_args(nodemask));
3760 va_end(args);
3761
3762 cpuset_print_current_mems_allowed();
3763 pr_cont("\n");
3764 dump_stack();
3765 warn_alloc_show_mem(gfp_mask, nodemask);
3766 }
3767
3768 static inline struct page *
3769 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
3770 unsigned int alloc_flags,
3771 const struct alloc_context *ac)
3772 {
3773 struct page *page;
3774
3775 page = get_page_from_freelist(gfp_mask, order,
3776 alloc_flags|ALLOC_CPUSET, ac);
3777
3778
3779
3780
3781 if (!page)
3782 page = get_page_from_freelist(gfp_mask, order,
3783 alloc_flags, ac);
3784
3785 return page;
3786 }
3787
3788 static inline struct page *
3789 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
3790 const struct alloc_context *ac, unsigned long *did_some_progress)
3791 {
3792 struct oom_control oc = {
3793 .zonelist = ac->zonelist,
3794 .nodemask = ac->nodemask,
3795 .memcg = NULL,
3796 .gfp_mask = gfp_mask,
3797 .order = order,
3798 };
3799 struct page *page;
3800
3801 *did_some_progress = 0;
3802
3803
3804
3805
3806
3807 if (!mutex_trylock(&oom_lock)) {
3808 *did_some_progress = 1;
3809 schedule_timeout_uninterruptible(1);
3810 return NULL;
3811 }
3812
3813
3814
3815
3816
3817
3818
3819
3820 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
3821 ~__GFP_DIRECT_RECLAIM, order,
3822 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
3823 if (page)
3824 goto out;
3825
3826
3827 if (current->flags & PF_DUMPCORE)
3828 goto out;
3829
3830 if (order > PAGE_ALLOC_COSTLY_ORDER)
3831 goto out;
3832
3833
3834
3835
3836
3837
3838 if (gfp_mask & __GFP_RETRY_MAYFAIL)
3839 goto out;
3840
3841 if (ac->high_zoneidx < ZONE_NORMAL)
3842 goto out;
3843 if (pm_suspended_storage())
3844 goto out;
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856 if (gfp_mask & __GFP_THISNODE)
3857 goto out;
3858
3859
3860 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
3861 *did_some_progress = 1;
3862
3863
3864
3865
3866
3867 if (gfp_mask & __GFP_NOFAIL)
3868 page = __alloc_pages_cpuset_fallback(gfp_mask, order,
3869 ALLOC_NO_WATERMARKS, ac);
3870 }
3871 out:
3872 mutex_unlock(&oom_lock);
3873 return page;
3874 }
3875
3876
3877
3878
3879
3880 #define MAX_COMPACT_RETRIES 16
3881
3882 #ifdef CONFIG_COMPACTION
3883
3884 static struct page *
3885 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3886 unsigned int alloc_flags, const struct alloc_context *ac,
3887 enum compact_priority prio, enum compact_result *compact_result)
3888 {
3889 struct page *page = NULL;
3890 unsigned long pflags;
3891 unsigned int noreclaim_flag;
3892
3893 if (!order)
3894 return NULL;
3895
3896 psi_memstall_enter(&pflags);
3897 noreclaim_flag = memalloc_noreclaim_save();
3898
3899 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
3900 prio, &page);
3901
3902 memalloc_noreclaim_restore(noreclaim_flag);
3903 psi_memstall_leave(&pflags);
3904
3905
3906
3907
3908
3909 count_vm_event(COMPACTSTALL);
3910
3911
3912 if (page)
3913 prep_new_page(page, order, gfp_mask, alloc_flags);
3914
3915
3916 if (!page)
3917 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3918
3919 if (page) {
3920 struct zone *zone = page_zone(page);
3921
3922 zone->compact_blockskip_flush = false;
3923 compaction_defer_reset(zone, order, true);
3924 count_vm_event(COMPACTSUCCESS);
3925 return page;
3926 }
3927
3928
3929
3930
3931
3932 count_vm_event(COMPACTFAIL);
3933
3934 cond_resched();
3935
3936 return NULL;
3937 }
3938
3939 static inline bool
3940 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
3941 enum compact_result compact_result,
3942 enum compact_priority *compact_priority,
3943 int *compaction_retries)
3944 {
3945 int max_retries = MAX_COMPACT_RETRIES;
3946 int min_priority;
3947 bool ret = false;
3948 int retries = *compaction_retries;
3949 enum compact_priority priority = *compact_priority;
3950
3951 if (!order)
3952 return false;
3953
3954 if (compaction_made_progress(compact_result))
3955 (*compaction_retries)++;
3956
3957
3958
3959
3960
3961
3962 if (compaction_failed(compact_result))
3963 goto check_priority;
3964
3965
3966
3967
3968
3969 if (compaction_needs_reclaim(compact_result)) {
3970 ret = compaction_zonelist_suitable(ac, order, alloc_flags);
3971 goto out;
3972 }
3973
3974
3975
3976
3977
3978
3979
3980 if (compaction_withdrawn(compact_result)) {
3981 goto check_priority;
3982 }
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992 if (order > PAGE_ALLOC_COSTLY_ORDER)
3993 max_retries /= 4;
3994 if (*compaction_retries <= max_retries) {
3995 ret = true;
3996 goto out;
3997 }
3998
3999
4000
4001
4002
4003 check_priority:
4004 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
4005 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
4006
4007 if (*compact_priority > min_priority) {
4008 (*compact_priority)--;
4009 *compaction_retries = 0;
4010 ret = true;
4011 }
4012 out:
4013 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
4014 return ret;
4015 }
4016 #else
4017 static inline struct page *
4018 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
4019 unsigned int alloc_flags, const struct alloc_context *ac,
4020 enum compact_priority prio, enum compact_result *compact_result)
4021 {
4022 *compact_result = COMPACT_SKIPPED;
4023 return NULL;
4024 }
4025
4026 static inline bool
4027 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
4028 enum compact_result compact_result,
4029 enum compact_priority *compact_priority,
4030 int *compaction_retries)
4031 {
4032 struct zone *zone;
4033 struct zoneref *z;
4034
4035 if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
4036 return false;
4037
4038
4039
4040
4041
4042
4043
4044 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
4045 ac->nodemask) {
4046 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
4047 ac_classzone_idx(ac), alloc_flags))
4048 return true;
4049 }
4050 return false;
4051 }
4052 #endif
4053
4054 #ifdef CONFIG_LOCKDEP
4055 static struct lockdep_map __fs_reclaim_map =
4056 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
4057
4058 static bool __need_fs_reclaim(gfp_t gfp_mask)
4059 {
4060 gfp_mask = current_gfp_context(gfp_mask);
4061
4062
4063 if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
4064 return false;
4065
4066
4067 if (current->flags & PF_MEMALLOC)
4068 return false;
4069
4070
4071 if (!(gfp_mask & __GFP_FS))
4072 return false;
4073
4074 if (gfp_mask & __GFP_NOLOCKDEP)
4075 return false;
4076
4077 return true;
4078 }
4079
4080 void __fs_reclaim_acquire(void)
4081 {
4082 lock_map_acquire(&__fs_reclaim_map);
4083 }
4084
4085 void __fs_reclaim_release(void)
4086 {
4087 lock_map_release(&__fs_reclaim_map);
4088 }
4089
4090 void fs_reclaim_acquire(gfp_t gfp_mask)
4091 {
4092 if (__need_fs_reclaim(gfp_mask))
4093 __fs_reclaim_acquire();
4094 }
4095 EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
4096
4097 void fs_reclaim_release(gfp_t gfp_mask)
4098 {
4099 if (__need_fs_reclaim(gfp_mask))
4100 __fs_reclaim_release();
4101 }
4102 EXPORT_SYMBOL_GPL(fs_reclaim_release);
4103 #endif
4104
4105
4106 static int
4107 __perform_reclaim(gfp_t gfp_mask, unsigned int order,
4108 const struct alloc_context *ac)
4109 {
4110 int progress;
4111 unsigned int noreclaim_flag;
4112 unsigned long pflags;
4113
4114 cond_resched();
4115
4116
4117 cpuset_memory_pressure_bump();
4118 psi_memstall_enter(&pflags);
4119 fs_reclaim_acquire(gfp_mask);
4120 noreclaim_flag = memalloc_noreclaim_save();
4121
4122 progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
4123 ac->nodemask);
4124
4125 memalloc_noreclaim_restore(noreclaim_flag);
4126 fs_reclaim_release(gfp_mask);
4127 psi_memstall_leave(&pflags);
4128
4129 cond_resched();
4130
4131 return progress;
4132 }
4133
4134
4135 static inline struct page *
4136 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
4137 unsigned int alloc_flags, const struct alloc_context *ac,
4138 unsigned long *did_some_progress)
4139 {
4140 struct page *page = NULL;
4141 bool drained = false;
4142
4143 *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
4144 if (unlikely(!(*did_some_progress)))
4145 return NULL;
4146
4147 retry:
4148 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4149
4150
4151
4152
4153
4154
4155 if (!page && !drained) {
4156 unreserve_highatomic_pageblock(ac, false);
4157 drain_all_pages(NULL);
4158 drained = true;
4159 goto retry;
4160 }
4161
4162 return page;
4163 }
4164
4165 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4166 const struct alloc_context *ac)
4167 {
4168 struct zoneref *z;
4169 struct zone *zone;
4170 pg_data_t *last_pgdat = NULL;
4171 enum zone_type high_zoneidx = ac->high_zoneidx;
4172
4173 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx,
4174 ac->nodemask) {
4175 if (last_pgdat != zone->zone_pgdat)
4176 wakeup_kswapd(zone, gfp_mask, order, high_zoneidx);
4177 last_pgdat = zone->zone_pgdat;
4178 }
4179 }
4180
4181 static inline unsigned int
4182 gfp_to_alloc_flags(gfp_t gfp_mask)
4183 {
4184 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
4185
4186
4187 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
4188
4189
4190
4191
4192
4193
4194
4195 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
4196
4197 if (gfp_mask & __GFP_ATOMIC) {
4198
4199
4200
4201
4202 if (!(gfp_mask & __GFP_NOMEMALLOC))
4203 alloc_flags |= ALLOC_HARDER;
4204
4205
4206
4207
4208 alloc_flags &= ~ALLOC_CPUSET;
4209 } else if (unlikely(rt_task(current)) && !in_interrupt())
4210 alloc_flags |= ALLOC_HARDER;
4211
4212 if (gfp_mask & __GFP_KSWAPD_RECLAIM)
4213 alloc_flags |= ALLOC_KSWAPD;
4214
4215 #ifdef CONFIG_CMA
4216 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
4217 alloc_flags |= ALLOC_CMA;
4218 #endif
4219 return alloc_flags;
4220 }
4221
4222 static bool oom_reserves_allowed(struct task_struct *tsk)
4223 {
4224 if (!tsk_is_oom_victim(tsk))
4225 return false;
4226
4227
4228
4229
4230
4231 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4232 return false;
4233
4234 return true;
4235 }
4236
4237
4238
4239
4240
4241 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
4242 {
4243 if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
4244 return 0;
4245 if (gfp_mask & __GFP_MEMALLOC)
4246 return ALLOC_NO_WATERMARKS;
4247 if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4248 return ALLOC_NO_WATERMARKS;
4249 if (!in_interrupt()) {
4250 if (current->flags & PF_MEMALLOC)
4251 return ALLOC_NO_WATERMARKS;
4252 else if (oom_reserves_allowed(current))
4253 return ALLOC_OOM;
4254 }
4255
4256 return 0;
4257 }
4258
4259 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
4260 {
4261 return !!__gfp_pfmemalloc_flags(gfp_mask);
4262 }
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274 static inline bool
4275 should_reclaim_retry(gfp_t gfp_mask, unsigned order,
4276 struct alloc_context *ac, int alloc_flags,
4277 bool did_some_progress, int *no_progress_loops)
4278 {
4279 struct zone *zone;
4280 struct zoneref *z;
4281 bool ret = false;
4282
4283
4284
4285
4286
4287
4288 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
4289 *no_progress_loops = 0;
4290 else
4291 (*no_progress_loops)++;
4292
4293
4294
4295
4296
4297 if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
4298
4299 return unreserve_highatomic_pageblock(ac, true);
4300 }
4301
4302
4303
4304
4305
4306
4307
4308 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
4309 ac->nodemask) {
4310 unsigned long available;
4311 unsigned long reclaimable;
4312 unsigned long min_wmark = min_wmark_pages(zone);
4313 bool wmark;
4314
4315 available = reclaimable = zone_reclaimable_pages(zone);
4316 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
4317
4318
4319
4320
4321
4322 wmark = __zone_watermark_ok(zone, order, min_wmark,
4323 ac_classzone_idx(ac), alloc_flags, available);
4324 trace_reclaim_retry_zone(z, order, reclaimable,
4325 available, min_wmark, *no_progress_loops, wmark);
4326 if (wmark) {
4327
4328
4329
4330
4331
4332
4333 if (!did_some_progress) {
4334 unsigned long write_pending;
4335
4336 write_pending = zone_page_state_snapshot(zone,
4337 NR_ZONE_WRITE_PENDING);
4338
4339 if (2 * write_pending > reclaimable) {
4340 congestion_wait(BLK_RW_ASYNC, HZ/10);
4341 return true;
4342 }
4343 }
4344
4345 ret = true;
4346 goto out;
4347 }
4348 }
4349
4350 out:
4351
4352
4353
4354
4355
4356
4357
4358 if (current->flags & PF_WQ_WORKER)
4359 schedule_timeout_uninterruptible(1);
4360 else
4361 cond_resched();
4362 return ret;
4363 }
4364
4365 static inline bool
4366 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
4367 {
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379 if (cpusets_enabled() && ac->nodemask &&
4380 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
4381 ac->nodemask = NULL;
4382 return true;
4383 }
4384
4385
4386
4387
4388
4389
4390
4391
4392 if (read_mems_allowed_retry(cpuset_mems_cookie))
4393 return true;
4394
4395 return false;
4396 }
4397
4398 static inline struct page *
4399 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
4400 struct alloc_context *ac)
4401 {
4402 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
4403 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
4404 struct page *page = NULL;
4405 unsigned int alloc_flags;
4406 unsigned long did_some_progress;
4407 enum compact_priority compact_priority;
4408 enum compact_result compact_result;
4409 int compaction_retries;
4410 int no_progress_loops;
4411 unsigned int cpuset_mems_cookie;
4412 int reserve_flags;
4413
4414
4415
4416
4417
4418 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
4419 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
4420 gfp_mask &= ~__GFP_ATOMIC;
4421
4422 retry_cpuset:
4423 compaction_retries = 0;
4424 no_progress_loops = 0;
4425 compact_priority = DEF_COMPACT_PRIORITY;
4426 cpuset_mems_cookie = read_mems_allowed_begin();
4427
4428
4429
4430
4431
4432
4433 alloc_flags = gfp_to_alloc_flags(gfp_mask);
4434
4435
4436
4437
4438
4439
4440
4441 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4442 ac->high_zoneidx, ac->nodemask);
4443 if (!ac->preferred_zoneref->zone)
4444 goto nopage;
4445
4446 if (alloc_flags & ALLOC_KSWAPD)
4447 wake_all_kswapds(order, gfp_mask, ac);
4448
4449
4450
4451
4452
4453 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4454 if (page)
4455 goto got_pg;
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466 if (can_direct_reclaim &&
4467 (costly_order ||
4468 (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
4469 && !gfp_pfmemalloc_allowed(gfp_mask)) {
4470 page = __alloc_pages_direct_compact(gfp_mask, order,
4471 alloc_flags, ac,
4472 INIT_COMPACT_PRIORITY,
4473 &compact_result);
4474 if (page)
4475 goto got_pg;
4476
4477 if (order >= pageblock_order && (gfp_mask & __GFP_IO) &&
4478 !(gfp_mask & __GFP_RETRY_MAYFAIL)) {
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496 if (compact_result == COMPACT_SKIPPED ||
4497 compact_result == COMPACT_DEFERRED)
4498 goto nopage;
4499 }
4500
4501
4502
4503
4504
4505 if (costly_order && (gfp_mask & __GFP_NORETRY)) {
4506
4507
4508
4509
4510
4511
4512
4513
4514 if (compact_result == COMPACT_DEFERRED)
4515 goto nopage;
4516
4517
4518
4519
4520
4521
4522 compact_priority = INIT_COMPACT_PRIORITY;
4523 }
4524 }
4525
4526 retry:
4527
4528 if (alloc_flags & ALLOC_KSWAPD)
4529 wake_all_kswapds(order, gfp_mask, ac);
4530
4531 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
4532 if (reserve_flags)
4533 alloc_flags = reserve_flags;
4534
4535
4536
4537
4538
4539
4540 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
4541 ac->nodemask = NULL;
4542 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4543 ac->high_zoneidx, ac->nodemask);
4544 }
4545
4546
4547 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4548 if (page)
4549 goto got_pg;
4550
4551
4552 if (!can_direct_reclaim)
4553 goto nopage;
4554
4555
4556 if (current->flags & PF_MEMALLOC)
4557 goto nopage;
4558
4559
4560 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
4561 &did_some_progress);
4562 if (page)
4563 goto got_pg;
4564
4565
4566 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
4567 compact_priority, &compact_result);
4568 if (page)
4569 goto got_pg;
4570
4571
4572 if (gfp_mask & __GFP_NORETRY)
4573 goto nopage;
4574
4575
4576
4577
4578
4579 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
4580 goto nopage;
4581
4582 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
4583 did_some_progress > 0, &no_progress_loops))
4584 goto retry;
4585
4586
4587
4588
4589
4590
4591
4592 if (did_some_progress > 0 &&
4593 should_compact_retry(ac, order, alloc_flags,
4594 compact_result, &compact_priority,
4595 &compaction_retries))
4596 goto retry;
4597
4598
4599
4600 if (check_retry_cpuset(cpuset_mems_cookie, ac))
4601 goto retry_cpuset;
4602
4603
4604 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
4605 if (page)
4606 goto got_pg;
4607
4608
4609 if (tsk_is_oom_victim(current) &&
4610 (alloc_flags == ALLOC_OOM ||
4611 (gfp_mask & __GFP_NOMEMALLOC)))
4612 goto nopage;
4613
4614
4615 if (did_some_progress) {
4616 no_progress_loops = 0;
4617 goto retry;
4618 }
4619
4620 nopage:
4621
4622 if (check_retry_cpuset(cpuset_mems_cookie, ac))
4623 goto retry_cpuset;
4624
4625
4626
4627
4628
4629 if (gfp_mask & __GFP_NOFAIL) {
4630
4631
4632
4633
4634 if (WARN_ON_ONCE(!can_direct_reclaim))
4635 goto fail;
4636
4637
4638
4639
4640
4641
4642 WARN_ON_ONCE(current->flags & PF_MEMALLOC);
4643
4644
4645
4646
4647
4648
4649
4650 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);
4651
4652
4653
4654
4655
4656
4657
4658 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
4659 if (page)
4660 goto got_pg;
4661
4662 cond_resched();
4663 goto retry;
4664 }
4665 fail:
4666 warn_alloc(gfp_mask, ac->nodemask,
4667 "page allocation failure: order:%u", order);
4668 got_pg:
4669 return page;
4670 }
4671
4672 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
4673 int preferred_nid, nodemask_t *nodemask,
4674 struct alloc_context *ac, gfp_t *alloc_mask,
4675 unsigned int *alloc_flags)
4676 {
4677 ac->high_zoneidx = gfp_zone(gfp_mask);
4678 ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
4679 ac->nodemask = nodemask;
4680 ac->migratetype = gfpflags_to_migratetype(gfp_mask);
4681
4682 if (cpusets_enabled()) {
4683 *alloc_mask |= __GFP_HARDWALL;
4684 if (!ac->nodemask)
4685 ac->nodemask = &cpuset_current_mems_allowed;
4686 else
4687 *alloc_flags |= ALLOC_CPUSET;
4688 }
4689
4690 fs_reclaim_acquire(gfp_mask);
4691 fs_reclaim_release(gfp_mask);
4692
4693 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
4694
4695 if (should_fail_alloc_page(gfp_mask, order))
4696 return false;
4697
4698 if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE)
4699 *alloc_flags |= ALLOC_CMA;
4700
4701 return true;
4702 }
4703
4704
4705 static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac)
4706 {
4707
4708 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
4709
4710
4711
4712
4713
4714
4715 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4716 ac->high_zoneidx, ac->nodemask);
4717 }
4718
4719
4720
4721
4722 struct page *
4723 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
4724 nodemask_t *nodemask)
4725 {
4726 struct page *page;
4727 unsigned int alloc_flags = ALLOC_WMARK_LOW;
4728 gfp_t alloc_mask;
4729 struct alloc_context ac = { };
4730
4731
4732
4733
4734
4735 if (unlikely(order >= MAX_ORDER)) {
4736 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
4737 return NULL;
4738 }
4739
4740 gfp_mask &= gfp_allowed_mask;
4741 alloc_mask = gfp_mask;
4742 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
4743 return NULL;
4744
4745 finalise_ac(gfp_mask, &ac);
4746
4747
4748
4749
4750
4751 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask);
4752
4753
4754 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
4755 if (likely(page))
4756 goto out;
4757
4758
4759
4760
4761
4762
4763
4764 alloc_mask = current_gfp_context(gfp_mask);
4765 ac.spread_dirty_pages = false;
4766
4767
4768
4769
4770
4771 if (unlikely(ac.nodemask != nodemask))
4772 ac.nodemask = nodemask;
4773
4774 page = __alloc_pages_slowpath(alloc_mask, order, &ac);
4775
4776 out:
4777 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
4778 unlikely(__memcg_kmem_charge(page, gfp_mask, order) != 0)) {
4779 __free_pages(page, order);
4780 page = NULL;
4781 }
4782
4783 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
4784
4785 return page;
4786 }
4787 EXPORT_SYMBOL(__alloc_pages_nodemask);
4788
4789
4790
4791
4792
4793
4794 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
4795 {
4796 struct page *page;
4797
4798 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
4799 if (!page)
4800 return 0;
4801 return (unsigned long) page_address(page);
4802 }
4803 EXPORT_SYMBOL(__get_free_pages);
4804
4805 unsigned long get_zeroed_page(gfp_t gfp_mask)
4806 {
4807 return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
4808 }
4809 EXPORT_SYMBOL(get_zeroed_page);
4810
4811 static inline void free_the_page(struct page *page, unsigned int order)
4812 {
4813 if (order == 0)
4814 free_unref_page(page);
4815 else
4816 __free_pages_ok(page, order);
4817 }
4818
4819 void __free_pages(struct page *page, unsigned int order)
4820 {
4821 if (put_page_testzero(page))
4822 free_the_page(page, order);
4823 }
4824 EXPORT_SYMBOL(__free_pages);
4825
4826 void free_pages(unsigned long addr, unsigned int order)
4827 {
4828 if (addr != 0) {
4829 VM_BUG_ON(!virt_addr_valid((void *)addr));
4830 __free_pages(virt_to_page((void *)addr), order);
4831 }
4832 }
4833
4834 EXPORT_SYMBOL(free_pages);
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
4848 gfp_t gfp_mask)
4849 {
4850 struct page *page = NULL;
4851 gfp_t gfp = gfp_mask;
4852
4853 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4854 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
4855 __GFP_NOMEMALLOC;
4856 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
4857 PAGE_FRAG_CACHE_MAX_ORDER);
4858 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
4859 #endif
4860 if (unlikely(!page))
4861 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
4862
4863 nc->va = page ? page_address(page) : NULL;
4864
4865 return page;
4866 }
4867
4868 void __page_frag_cache_drain(struct page *page, unsigned int count)
4869 {
4870 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
4871
4872 if (page_ref_sub_and_test(page, count))
4873 free_the_page(page, compound_order(page));
4874 }
4875 EXPORT_SYMBOL(__page_frag_cache_drain);
4876
4877 void *page_frag_alloc(struct page_frag_cache *nc,
4878 unsigned int fragsz, gfp_t gfp_mask)
4879 {
4880 unsigned int size = PAGE_SIZE;
4881 struct page *page;
4882 int offset;
4883
4884 if (unlikely(!nc->va)) {
4885 refill:
4886 page = __page_frag_cache_refill(nc, gfp_mask);
4887 if (!page)
4888 return NULL;
4889
4890 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4891
4892 size = nc->size;
4893 #endif
4894
4895
4896
4897 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
4898
4899
4900 nc->pfmemalloc = page_is_pfmemalloc(page);
4901 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
4902 nc->offset = size;
4903 }
4904
4905 offset = nc->offset - fragsz;
4906 if (unlikely(offset < 0)) {
4907 page = virt_to_page(nc->va);
4908
4909 if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
4910 goto refill;
4911
4912 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4913
4914 size = nc->size;
4915 #endif
4916
4917 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
4918
4919
4920 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
4921 offset = size - fragsz;
4922 }
4923
4924 nc->pagecnt_bias--;
4925 nc->offset = offset;
4926
4927 return nc->va + offset;
4928 }
4929 EXPORT_SYMBOL(page_frag_alloc);
4930
4931
4932
4933
4934 void page_frag_free(void *addr)
4935 {
4936 struct page *page = virt_to_head_page(addr);
4937
4938 if (unlikely(put_page_testzero(page)))
4939 free_the_page(page, compound_order(page));
4940 }
4941 EXPORT_SYMBOL(page_frag_free);
4942
4943 static void *make_alloc_exact(unsigned long addr, unsigned int order,
4944 size_t size)
4945 {
4946 if (addr) {
4947 unsigned long alloc_end = addr + (PAGE_SIZE << order);
4948 unsigned long used = addr + PAGE_ALIGN(size);
4949
4950 split_page(virt_to_page((void *)addr), order);
4951 while (used < alloc_end) {
4952 free_page(used);
4953 used += PAGE_SIZE;
4954 }
4955 }
4956 return (void *)addr;
4957 }
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974 void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
4975 {
4976 unsigned int order = get_order(size);
4977 unsigned long addr;
4978
4979 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
4980 gfp_mask &= ~__GFP_COMP;
4981
4982 addr = __get_free_pages(gfp_mask, order);
4983 return make_alloc_exact(addr, order, size);
4984 }
4985 EXPORT_SYMBOL(alloc_pages_exact);
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
5000 {
5001 unsigned int order = get_order(size);
5002 struct page *p;
5003
5004 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
5005 gfp_mask &= ~__GFP_COMP;
5006
5007 p = alloc_pages_node(nid, gfp_mask, order);
5008 if (!p)
5009 return NULL;
5010 return make_alloc_exact((unsigned long)page_address(p), order, size);
5011 }
5012
5013
5014
5015
5016
5017
5018
5019
5020 void free_pages_exact(void *virt, size_t size)
5021 {
5022 unsigned long addr = (unsigned long)virt;
5023 unsigned long end = addr + PAGE_ALIGN(size);
5024
5025 while (addr < end) {
5026 free_page(addr);
5027 addr += PAGE_SIZE;
5028 }
5029 }
5030 EXPORT_SYMBOL(free_pages_exact);
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044 static unsigned long nr_free_zone_pages(int offset)
5045 {
5046 struct zoneref *z;
5047 struct zone *zone;
5048
5049
5050 unsigned long sum = 0;
5051
5052 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
5053
5054 for_each_zone_zonelist(zone, z, zonelist, offset) {
5055 unsigned long size = zone_managed_pages(zone);
5056 unsigned long high = high_wmark_pages(zone);
5057 if (size > high)
5058 sum += size - high;
5059 }
5060
5061 return sum;
5062 }
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073 unsigned long nr_free_buffer_pages(void)
5074 {
5075 return nr_free_zone_pages(gfp_zone(GFP_USER));
5076 }
5077 EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087 unsigned long nr_free_pagecache_pages(void)
5088 {
5089 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
5090 }
5091
5092 static inline void show_node(struct zone *zone)
5093 {
5094 if (IS_ENABLED(CONFIG_NUMA))
5095 printk("Node %d ", zone_to_nid(zone));
5096 }
5097
5098 long si_mem_available(void)
5099 {
5100 long available;
5101 unsigned long pagecache;
5102 unsigned long wmark_low = 0;
5103 unsigned long pages[NR_LRU_LISTS];
5104 unsigned long reclaimable;
5105 struct zone *zone;
5106 int lru;
5107
5108 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
5109 pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
5110
5111 for_each_zone(zone)
5112 wmark_low += low_wmark_pages(zone);
5113
5114
5115
5116
5117
5118 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
5119
5120
5121
5122
5123
5124
5125 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
5126 pagecache -= min(pagecache / 2, wmark_low);
5127 available += pagecache;
5128
5129
5130
5131
5132
5133
5134 reclaimable = global_node_page_state(NR_SLAB_RECLAIMABLE) +
5135 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE);
5136 available += reclaimable - min(reclaimable / 2, wmark_low);
5137
5138 if (available < 0)
5139 available = 0;
5140 return available;
5141 }
5142 EXPORT_SYMBOL_GPL(si_mem_available);
5143
5144 void si_meminfo(struct sysinfo *val)
5145 {
5146 val->totalram = totalram_pages();
5147 val->sharedram = global_node_page_state(NR_SHMEM);
5148 val->freeram = global_zone_page_state(NR_FREE_PAGES);
5149 val->bufferram = nr_blockdev_pages();
5150 val->totalhigh = totalhigh_pages();
5151 val->freehigh = nr_free_highpages();
5152 val->mem_unit = PAGE_SIZE;
5153 }
5154
5155 EXPORT_SYMBOL(si_meminfo);
5156
5157 #ifdef CONFIG_NUMA
5158 void si_meminfo_node(struct sysinfo *val, int nid)
5159 {
5160 int zone_type;
5161 unsigned long managed_pages = 0;
5162 unsigned long managed_highpages = 0;
5163 unsigned long free_highpages = 0;
5164 pg_data_t *pgdat = NODE_DATA(nid);
5165
5166 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
5167 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]);
5168 val->totalram = managed_pages;
5169 val->sharedram = node_page_state(pgdat, NR_SHMEM);
5170 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
5171 #ifdef CONFIG_HIGHMEM
5172 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
5173 struct zone *zone = &pgdat->node_zones[zone_type];
5174
5175 if (is_highmem(zone)) {
5176 managed_highpages += zone_managed_pages(zone);
5177 free_highpages += zone_page_state(zone, NR_FREE_PAGES);
5178 }
5179 }
5180 val->totalhigh = managed_highpages;
5181 val->freehigh = free_highpages;
5182 #else
5183 val->totalhigh = managed_highpages;
5184 val->freehigh = free_highpages;
5185 #endif
5186 val->mem_unit = PAGE_SIZE;
5187 }
5188 #endif
5189
5190
5191
5192
5193
5194 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
5195 {
5196 if (!(flags & SHOW_MEM_FILTER_NODES))
5197 return false;
5198
5199
5200
5201
5202
5203
5204 if (!nodemask)
5205 nodemask = &cpuset_current_mems_allowed;
5206
5207 return !node_isset(nid, *nodemask);
5208 }
5209
5210 #define K(x) ((x) << (PAGE_SHIFT-10))
5211
5212 static void show_migration_types(unsigned char type)
5213 {
5214 static const char types[MIGRATE_TYPES] = {
5215 [MIGRATE_UNMOVABLE] = 'U',
5216 [MIGRATE_MOVABLE] = 'M',
5217 [MIGRATE_RECLAIMABLE] = 'E',
5218 [MIGRATE_HIGHATOMIC] = 'H',
5219 #ifdef CONFIG_CMA
5220 [MIGRATE_CMA] = 'C',
5221 #endif
5222 #ifdef CONFIG_MEMORY_ISOLATION
5223 [MIGRATE_ISOLATE] = 'I',
5224 #endif
5225 };
5226 char tmp[MIGRATE_TYPES + 1];
5227 char *p = tmp;
5228 int i;
5229
5230 for (i = 0; i < MIGRATE_TYPES; i++) {
5231 if (type & (1 << i))
5232 *p++ = types[i];
5233 }
5234
5235 *p = '\0';
5236 printk(KERN_CONT "(%s) ", tmp);
5237 }
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248 void show_free_areas(unsigned int filter, nodemask_t *nodemask)
5249 {
5250 unsigned long free_pcp = 0;
5251 int cpu;
5252 struct zone *zone;
5253 pg_data_t *pgdat;
5254
5255 for_each_populated_zone(zone) {
5256 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5257 continue;
5258
5259 for_each_online_cpu(cpu)
5260 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
5261 }
5262
5263 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
5264 " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
5265 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
5266 " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
5267 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
5268 " free:%lu free_pcp:%lu free_cma:%lu\n",
5269 global_node_page_state(NR_ACTIVE_ANON),
5270 global_node_page_state(NR_INACTIVE_ANON),
5271 global_node_page_state(NR_ISOLATED_ANON),
5272 global_node_page_state(NR_ACTIVE_FILE),
5273 global_node_page_state(NR_INACTIVE_FILE),
5274 global_node_page_state(NR_ISOLATED_FILE),
5275 global_node_page_state(NR_UNEVICTABLE),
5276 global_node_page_state(NR_FILE_DIRTY),
5277 global_node_page_state(NR_WRITEBACK),
5278 global_node_page_state(NR_UNSTABLE_NFS),
5279 global_node_page_state(NR_SLAB_RECLAIMABLE),
5280 global_node_page_state(NR_SLAB_UNRECLAIMABLE),
5281 global_node_page_state(NR_FILE_MAPPED),
5282 global_node_page_state(NR_SHMEM),
5283 global_zone_page_state(NR_PAGETABLE),
5284 global_zone_page_state(NR_BOUNCE),
5285 global_zone_page_state(NR_FREE_PAGES),
5286 free_pcp,
5287 global_zone_page_state(NR_FREE_CMA_PAGES));
5288
5289 for_each_online_pgdat(pgdat) {
5290 if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
5291 continue;
5292
5293 printk("Node %d"
5294 " active_anon:%lukB"
5295 " inactive_anon:%lukB"
5296 " active_file:%lukB"
5297 " inactive_file:%lukB"
5298 " unevictable:%lukB"
5299 " isolated(anon):%lukB"
5300 " isolated(file):%lukB"
5301 " mapped:%lukB"
5302 " dirty:%lukB"
5303 " writeback:%lukB"
5304 " shmem:%lukB"
5305 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
5306 " shmem_thp: %lukB"
5307 " shmem_pmdmapped: %lukB"
5308 " anon_thp: %lukB"
5309 #endif
5310 " writeback_tmp:%lukB"
5311 " unstable:%lukB"
5312 " all_unreclaimable? %s"
5313 "\n",
5314 pgdat->node_id,
5315 K(node_page_state(pgdat, NR_ACTIVE_ANON)),
5316 K(node_page_state(pgdat, NR_INACTIVE_ANON)),
5317 K(node_page_state(pgdat, NR_ACTIVE_FILE)),
5318 K(node_page_state(pgdat, NR_INACTIVE_FILE)),
5319 K(node_page_state(pgdat, NR_UNEVICTABLE)),
5320 K(node_page_state(pgdat, NR_ISOLATED_ANON)),
5321 K(node_page_state(pgdat, NR_ISOLATED_FILE)),
5322 K(node_page_state(pgdat, NR_FILE_MAPPED)),
5323 K(node_page_state(pgdat, NR_FILE_DIRTY)),
5324 K(node_page_state(pgdat, NR_WRITEBACK)),
5325 K(node_page_state(pgdat, NR_SHMEM)),
5326 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
5327 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
5328 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
5329 * HPAGE_PMD_NR),
5330 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
5331 #endif
5332 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
5333 K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
5334 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
5335 "yes" : "no");
5336 }
5337
5338 for_each_populated_zone(zone) {
5339 int i;
5340
5341 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5342 continue;
5343
5344 free_pcp = 0;
5345 for_each_online_cpu(cpu)
5346 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
5347
5348 show_node(zone);
5349 printk(KERN_CONT
5350 "%s"
5351 " free:%lukB"
5352 " min:%lukB"
5353 " low:%lukB"
5354 " high:%lukB"
5355 " active_anon:%lukB"
5356 " inactive_anon:%lukB"
5357 " active_file:%lukB"
5358 " inactive_file:%lukB"
5359 " unevictable:%lukB"
5360 " writepending:%lukB"
5361 " present:%lukB"
5362 " managed:%lukB"
5363 " mlocked:%lukB"
5364 " kernel_stack:%lukB"
5365 " pagetables:%lukB"
5366 " bounce:%lukB"
5367 " free_pcp:%lukB"
5368 " local_pcp:%ukB"
5369 " free_cma:%lukB"
5370 "\n",
5371 zone->name,
5372 K(zone_page_state(zone, NR_FREE_PAGES)),
5373 K(min_wmark_pages(zone)),
5374 K(low_wmark_pages(zone)),
5375 K(high_wmark_pages(zone)),
5376 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
5377 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
5378 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
5379 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
5380 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
5381 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
5382 K(zone->present_pages),
5383 K(zone_managed_pages(zone)),
5384 K(zone_page_state(zone, NR_MLOCK)),
5385 zone_page_state(zone, NR_KERNEL_STACK_KB),
5386 K(zone_page_state(zone, NR_PAGETABLE)),
5387 K(zone_page_state(zone, NR_BOUNCE)),
5388 K(free_pcp),
5389 K(this_cpu_read(zone->pageset->pcp.count)),
5390 K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
5391 printk("lowmem_reserve[]:");
5392 for (i = 0; i < MAX_NR_ZONES; i++)
5393 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
5394 printk(KERN_CONT "\n");
5395 }
5396
5397 for_each_populated_zone(zone) {
5398 unsigned int order;
5399 unsigned long nr[MAX_ORDER], flags, total = 0;
5400 unsigned char types[MAX_ORDER];
5401
5402 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5403 continue;
5404 show_node(zone);
5405 printk(KERN_CONT "%s: ", zone->name);
5406
5407 spin_lock_irqsave(&zone->lock, flags);
5408 for (order = 0; order < MAX_ORDER; order++) {
5409 struct free_area *area = &zone->free_area[order];
5410 int type;
5411
5412 nr[order] = area->nr_free;
5413 total += nr[order] << order;
5414
5415 types[order] = 0;
5416 for (type = 0; type < MIGRATE_TYPES; type++) {
5417 if (!free_area_empty(area, type))
5418 types[order] |= 1 << type;
5419 }
5420 }
5421 spin_unlock_irqrestore(&zone->lock, flags);
5422 for (order = 0; order < MAX_ORDER; order++) {
5423 printk(KERN_CONT "%lu*%lukB ",
5424 nr[order], K(1UL) << order);
5425 if (nr[order])
5426 show_migration_types(types[order]);
5427 }
5428 printk(KERN_CONT "= %lukB\n", K(total));
5429 }
5430
5431 hugetlb_show_meminfo();
5432
5433 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
5434
5435 show_swap_cache_info();
5436 }
5437
5438 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
5439 {
5440 zoneref->zone = zone;
5441 zoneref->zone_idx = zone_idx(zone);
5442 }
5443
5444
5445
5446
5447
5448
5449 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
5450 {
5451 struct zone *zone;
5452 enum zone_type zone_type = MAX_NR_ZONES;
5453 int nr_zones = 0;
5454
5455 do {
5456 zone_type--;
5457 zone = pgdat->node_zones + zone_type;
5458 if (managed_zone(zone)) {
5459 zoneref_set_zone(zone, &zonerefs[nr_zones++]);
5460 check_highest_zone(zone_type);
5461 }
5462 } while (zone_type);
5463
5464 return nr_zones;
5465 }
5466
5467 #ifdef CONFIG_NUMA
5468
5469 static int __parse_numa_zonelist_order(char *s)
5470 {
5471
5472
5473
5474
5475
5476
5477 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
5478 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s);
5479 return -EINVAL;
5480 }
5481 return 0;
5482 }
5483
5484 static __init int setup_numa_zonelist_order(char *s)
5485 {
5486 if (!s)
5487 return 0;
5488
5489 return __parse_numa_zonelist_order(s);
5490 }
5491 early_param("numa_zonelist_order", setup_numa_zonelist_order);
5492
5493 char numa_zonelist_order[] = "Node";
5494
5495
5496
5497
5498 int numa_zonelist_order_handler(struct ctl_table *table, int write,
5499 void __user *buffer, size_t *length,
5500 loff_t *ppos)
5501 {
5502 char *str;
5503 int ret;
5504
5505 if (!write)
5506 return proc_dostring(table, write, buffer, length, ppos);
5507 str = memdup_user_nul(buffer, 16);
5508 if (IS_ERR(str))
5509 return PTR_ERR(str);
5510
5511 ret = __parse_numa_zonelist_order(str);
5512 kfree(str);
5513 return ret;
5514 }
5515
5516
5517 #define MAX_NODE_LOAD (nr_online_nodes)
5518 static int node_load[MAX_NUMNODES];
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535 static int find_next_best_node(int node, nodemask_t *used_node_mask)
5536 {
5537 int n, val;
5538 int min_val = INT_MAX;
5539 int best_node = NUMA_NO_NODE;
5540 const struct cpumask *tmp = cpumask_of_node(0);
5541
5542
5543 if (!node_isset(node, *used_node_mask)) {
5544 node_set(node, *used_node_mask);
5545 return node;
5546 }
5547
5548 for_each_node_state(n, N_MEMORY) {
5549
5550
5551 if (node_isset(n, *used_node_mask))
5552 continue;
5553
5554
5555 val = node_distance(node, n);
5556
5557
5558 val += (n < node);
5559
5560
5561 tmp = cpumask_of_node(n);
5562 if (!cpumask_empty(tmp))
5563 val += PENALTY_FOR_NODE_WITH_CPUS;
5564
5565
5566 val *= (MAX_NODE_LOAD*MAX_NUMNODES);
5567 val += node_load[n];
5568
5569 if (val < min_val) {
5570 min_val = val;
5571 best_node = n;
5572 }
5573 }
5574
5575 if (best_node >= 0)
5576 node_set(best_node, *used_node_mask);
5577
5578 return best_node;
5579 }
5580
5581
5582
5583
5584
5585
5586
5587 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
5588 unsigned nr_nodes)
5589 {
5590 struct zoneref *zonerefs;
5591 int i;
5592
5593 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5594
5595 for (i = 0; i < nr_nodes; i++) {
5596 int nr_zones;
5597
5598 pg_data_t *node = NODE_DATA(node_order[i]);
5599
5600 nr_zones = build_zonerefs_node(node, zonerefs);
5601 zonerefs += nr_zones;
5602 }
5603 zonerefs->zone = NULL;
5604 zonerefs->zone_idx = 0;
5605 }
5606
5607
5608
5609
5610 static void build_thisnode_zonelists(pg_data_t *pgdat)
5611 {
5612 struct zoneref *zonerefs;
5613 int nr_zones;
5614
5615 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
5616 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5617 zonerefs += nr_zones;
5618 zonerefs->zone = NULL;
5619 zonerefs->zone_idx = 0;
5620 }
5621
5622
5623
5624
5625
5626
5627
5628
5629 static void build_zonelists(pg_data_t *pgdat)
5630 {
5631 static int node_order[MAX_NUMNODES];
5632 int node, load, nr_nodes = 0;
5633 nodemask_t used_mask;
5634 int local_node, prev_node;
5635
5636
5637 local_node = pgdat->node_id;
5638 load = nr_online_nodes;
5639 prev_node = local_node;
5640 nodes_clear(used_mask);
5641
5642 memset(node_order, 0, sizeof(node_order));
5643 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
5644
5645
5646
5647
5648
5649 if (node_distance(local_node, node) !=
5650 node_distance(local_node, prev_node))
5651 node_load[node] = load;
5652
5653 node_order[nr_nodes++] = node;
5654 prev_node = node;
5655 load--;
5656 }
5657
5658 build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
5659 build_thisnode_zonelists(pgdat);
5660 }
5661
5662 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
5663
5664
5665
5666
5667
5668
5669 int local_memory_node(int node)
5670 {
5671 struct zoneref *z;
5672
5673 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
5674 gfp_zone(GFP_KERNEL),
5675 NULL);
5676 return zone_to_nid(z->zone);
5677 }
5678 #endif
5679
5680 static void setup_min_unmapped_ratio(void);
5681 static void setup_min_slab_ratio(void);
5682 #else
5683
5684 static void build_zonelists(pg_data_t *pgdat)
5685 {
5686 int node, local_node;
5687 struct zoneref *zonerefs;
5688 int nr_zones;
5689
5690 local_node = pgdat->node_id;
5691
5692 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5693 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5694 zonerefs += nr_zones;
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704 for (node = local_node + 1; node < MAX_NUMNODES; node++) {
5705 if (!node_online(node))
5706 continue;
5707 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5708 zonerefs += nr_zones;
5709 }
5710 for (node = 0; node < local_node; node++) {
5711 if (!node_online(node))
5712 continue;
5713 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5714 zonerefs += nr_zones;
5715 }
5716
5717 zonerefs->zone = NULL;
5718 zonerefs->zone_idx = 0;
5719 }
5720
5721 #endif
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
5739 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
5740 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
5741
5742 static void __build_all_zonelists(void *data)
5743 {
5744 int nid;
5745 int __maybe_unused cpu;
5746 pg_data_t *self = data;
5747 static DEFINE_SPINLOCK(lock);
5748
5749 spin_lock(&lock);
5750
5751 #ifdef CONFIG_NUMA
5752 memset(node_load, 0, sizeof(node_load));
5753 #endif
5754
5755
5756
5757
5758
5759 if (self && !node_online(self->node_id)) {
5760 build_zonelists(self);
5761 } else {
5762 for_each_online_node(nid) {
5763 pg_data_t *pgdat = NODE_DATA(nid);
5764
5765 build_zonelists(pgdat);
5766 }
5767
5768 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
5769
5770
5771
5772
5773
5774
5775
5776
5777 for_each_online_cpu(cpu)
5778 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
5779 #endif
5780 }
5781
5782 spin_unlock(&lock);
5783 }
5784
5785 static noinline void __init
5786 build_all_zonelists_init(void)
5787 {
5788 int cpu;
5789
5790 __build_all_zonelists(NULL);
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805 for_each_possible_cpu(cpu)
5806 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
5807
5808 mminit_verify_zonelist();
5809 cpuset_init_current_mems_allowed();
5810 }
5811
5812
5813
5814
5815
5816
5817
5818 void __ref build_all_zonelists(pg_data_t *pgdat)
5819 {
5820 if (system_state == SYSTEM_BOOTING) {
5821 build_all_zonelists_init();
5822 } else {
5823 __build_all_zonelists(pgdat);
5824
5825 }
5826 vm_total_pages = nr_free_pagecache_pages();
5827
5828
5829
5830
5831
5832
5833
5834 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
5835 page_group_by_mobility_disabled = 1;
5836 else
5837 page_group_by_mobility_disabled = 0;
5838
5839 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n",
5840 nr_online_nodes,
5841 page_group_by_mobility_disabled ? "off" : "on",
5842 vm_total_pages);
5843 #ifdef CONFIG_NUMA
5844 pr_info("Policy zone: %s\n", zone_names[policy_zone]);
5845 #endif
5846 }
5847
5848
5849 static bool __meminit
5850 overlap_memmap_init(unsigned long zone, unsigned long *pfn)
5851 {
5852 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5853 static struct memblock_region *r;
5854
5855 if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
5856 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
5857 for_each_memblock(memory, r) {
5858 if (*pfn < memblock_region_memory_end_pfn(r))
5859 break;
5860 }
5861 }
5862 if (*pfn >= memblock_region_memory_base_pfn(r) &&
5863 memblock_is_mirror(r)) {
5864 *pfn = memblock_region_memory_end_pfn(r);
5865 return true;
5866 }
5867 }
5868 #endif
5869 return false;
5870 }
5871
5872
5873
5874
5875
5876
5877 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
5878 unsigned long start_pfn, enum memmap_context context,
5879 struct vmem_altmap *altmap)
5880 {
5881 unsigned long pfn, end_pfn = start_pfn + size;
5882 struct page *page;
5883
5884 if (highest_memmap_pfn < end_pfn - 1)
5885 highest_memmap_pfn = end_pfn - 1;
5886
5887 #ifdef CONFIG_ZONE_DEVICE
5888
5889
5890
5891
5892
5893
5894
5895 if (zone == ZONE_DEVICE) {
5896 if (!altmap)
5897 return;
5898
5899 if (start_pfn == altmap->base_pfn)
5900 start_pfn += altmap->reserve;
5901 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
5902 }
5903 #endif
5904
5905 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
5906
5907
5908
5909
5910 if (context == MEMMAP_EARLY) {
5911 if (!early_pfn_valid(pfn))
5912 continue;
5913 if (!early_pfn_in_nid(pfn, nid))
5914 continue;
5915 if (overlap_memmap_init(zone, &pfn))
5916 continue;
5917 if (defer_init(nid, pfn, end_pfn))
5918 break;
5919 }
5920
5921 page = pfn_to_page(pfn);
5922 __init_single_page(page, pfn, zone, nid);
5923 if (context == MEMMAP_HOTPLUG)
5924 __SetPageReserved(page);
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938 if (!(pfn & (pageblock_nr_pages - 1))) {
5939 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
5940 cond_resched();
5941 }
5942 }
5943 }
5944
5945 #ifdef CONFIG_ZONE_DEVICE
5946 void __ref memmap_init_zone_device(struct zone *zone,
5947 unsigned long start_pfn,
5948 unsigned long size,
5949 struct dev_pagemap *pgmap)
5950 {
5951 unsigned long pfn, end_pfn = start_pfn + size;
5952 struct pglist_data *pgdat = zone->zone_pgdat;
5953 struct vmem_altmap *altmap = pgmap_altmap(pgmap);
5954 unsigned long zone_idx = zone_idx(zone);
5955 unsigned long start = jiffies;
5956 int nid = pgdat->node_id;
5957
5958 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE))
5959 return;
5960
5961
5962
5963
5964
5965
5966 if (altmap) {
5967 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
5968 size = end_pfn - start_pfn;
5969 }
5970
5971 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
5972 struct page *page = pfn_to_page(pfn);
5973
5974 __init_single_page(page, pfn, zone_idx, nid);
5975
5976
5977
5978
5979
5980
5981
5982
5983 __SetPageReserved(page);
5984
5985
5986
5987
5988
5989
5990 page->pgmap = pgmap;
5991 page->zone_device_data = NULL;
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008 if (!(pfn & (pageblock_nr_pages - 1))) {
6009 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
6010 cond_resched();
6011 }
6012 }
6013
6014 pr_info("%s initialised %lu pages in %ums\n", __func__,
6015 size, jiffies_to_msecs(jiffies - start));
6016 }
6017
6018 #endif
6019 static void __meminit zone_init_free_lists(struct zone *zone)
6020 {
6021 unsigned int order, t;
6022 for_each_migratetype_order(order, t) {
6023 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
6024 zone->free_area[order].nr_free = 0;
6025 }
6026 }
6027
6028 void __meminit __weak memmap_init(unsigned long size, int nid,
6029 unsigned long zone, unsigned long start_pfn)
6030 {
6031 memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY, NULL);
6032 }
6033
6034 static int zone_batchsize(struct zone *zone)
6035 {
6036 #ifdef CONFIG_MMU
6037 int batch;
6038
6039
6040
6041
6042
6043 batch = zone_managed_pages(zone) / 1024;
6044
6045 if (batch * PAGE_SIZE > 1024 * 1024)
6046 batch = (1024 * 1024) / PAGE_SIZE;
6047 batch /= 4;
6048 if (batch < 1)
6049 batch = 1;
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061 batch = rounddown_pow_of_two(batch + batch/2) - 1;
6062
6063 return batch;
6064
6065 #else
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079 return 0;
6080 #endif
6081 }
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
6097 unsigned long batch)
6098 {
6099
6100 pcp->batch = 1;
6101 smp_wmb();
6102
6103
6104 pcp->high = high;
6105 smp_wmb();
6106
6107 pcp->batch = batch;
6108 }
6109
6110
6111 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
6112 {
6113 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
6114 }
6115
6116 static void pageset_init(struct per_cpu_pageset *p)
6117 {
6118 struct per_cpu_pages *pcp;
6119 int migratetype;
6120
6121 memset(p, 0, sizeof(*p));
6122
6123 pcp = &p->pcp;
6124 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
6125 INIT_LIST_HEAD(&pcp->lists[migratetype]);
6126 }
6127
6128 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
6129 {
6130 pageset_init(p);
6131 pageset_set_batch(p, batch);
6132 }
6133
6134
6135
6136
6137
6138 static void pageset_set_high(struct per_cpu_pageset *p,
6139 unsigned long high)
6140 {
6141 unsigned long batch = max(1UL, high / 4);
6142 if ((high / 4) > (PAGE_SHIFT * 8))
6143 batch = PAGE_SHIFT * 8;
6144
6145 pageset_update(&p->pcp, high, batch);
6146 }
6147
6148 static void pageset_set_high_and_batch(struct zone *zone,
6149 struct per_cpu_pageset *pcp)
6150 {
6151 if (percpu_pagelist_fraction)
6152 pageset_set_high(pcp,
6153 (zone_managed_pages(zone) /
6154 percpu_pagelist_fraction));
6155 else
6156 pageset_set_batch(pcp, zone_batchsize(zone));
6157 }
6158
6159 static void __meminit zone_pageset_init(struct zone *zone, int cpu)
6160 {
6161 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
6162
6163 pageset_init(pcp);
6164 pageset_set_high_and_batch(zone, pcp);
6165 }
6166
6167 void __meminit setup_zone_pageset(struct zone *zone)
6168 {
6169 int cpu;
6170 zone->pageset = alloc_percpu(struct per_cpu_pageset);
6171 for_each_possible_cpu(cpu)
6172 zone_pageset_init(zone, cpu);
6173 }
6174
6175
6176
6177
6178
6179 void __init setup_per_cpu_pageset(void)
6180 {
6181 struct pglist_data *pgdat;
6182 struct zone *zone;
6183
6184 for_each_populated_zone(zone)
6185 setup_zone_pageset(zone);
6186
6187 for_each_online_pgdat(pgdat)
6188 pgdat->per_cpu_nodestats =
6189 alloc_percpu(struct per_cpu_nodestat);
6190 }
6191
6192 static __meminit void zone_pcp_init(struct zone *zone)
6193 {
6194
6195
6196
6197
6198
6199 zone->pageset = &boot_pageset;
6200
6201 if (populated_zone(zone))
6202 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
6203 zone->name, zone->present_pages,
6204 zone_batchsize(zone));
6205 }
6206
6207 void __meminit init_currently_empty_zone(struct zone *zone,
6208 unsigned long zone_start_pfn,
6209 unsigned long size)
6210 {
6211 struct pglist_data *pgdat = zone->zone_pgdat;
6212 int zone_idx = zone_idx(zone) + 1;
6213
6214 if (zone_idx > pgdat->nr_zones)
6215 pgdat->nr_zones = zone_idx;
6216
6217 zone->zone_start_pfn = zone_start_pfn;
6218
6219 mminit_dprintk(MMINIT_TRACE, "memmap_init",
6220 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
6221 pgdat->node_id,
6222 (unsigned long)zone_idx(zone),
6223 zone_start_pfn, (zone_start_pfn + size));
6224
6225 zone_init_free_lists(zone);
6226 zone->initialized = 1;
6227 }
6228
6229 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6230 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
6231
6232
6233
6234
6235 int __meminit __early_pfn_to_nid(unsigned long pfn,
6236 struct mminit_pfnnid_cache *state)
6237 {
6238 unsigned long start_pfn, end_pfn;
6239 int nid;
6240
6241 if (state->last_start <= pfn && pfn < state->last_end)
6242 return state->last_nid;
6243
6244 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
6245 if (nid != NUMA_NO_NODE) {
6246 state->last_start = start_pfn;
6247 state->last_end = end_pfn;
6248 state->last_nid = nid;
6249 }
6250
6251 return nid;
6252 }
6253 #endif
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
6265 {
6266 unsigned long start_pfn, end_pfn;
6267 int i, this_nid;
6268
6269 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
6270 start_pfn = min(start_pfn, max_low_pfn);
6271 end_pfn = min(end_pfn, max_low_pfn);
6272
6273 if (start_pfn < end_pfn)
6274 memblock_free_early_nid(PFN_PHYS(start_pfn),
6275 (end_pfn - start_pfn) << PAGE_SHIFT,
6276 this_nid);
6277 }
6278 }
6279
6280
6281
6282
6283
6284
6285
6286
6287 void __init sparse_memory_present_with_active_regions(int nid)
6288 {
6289 unsigned long start_pfn, end_pfn;
6290 int i, this_nid;
6291
6292 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
6293 memory_present(this_nid, start_pfn, end_pfn);
6294 }
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307 void __init get_pfn_range_for_nid(unsigned int nid,
6308 unsigned long *start_pfn, unsigned long *end_pfn)
6309 {
6310 unsigned long this_start_pfn, this_end_pfn;
6311 int i;
6312
6313 *start_pfn = -1UL;
6314 *end_pfn = 0;
6315
6316 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
6317 *start_pfn = min(*start_pfn, this_start_pfn);
6318 *end_pfn = max(*end_pfn, this_end_pfn);
6319 }
6320
6321 if (*start_pfn == -1UL)
6322 *start_pfn = 0;
6323 }
6324
6325
6326
6327
6328
6329
6330 static void __init find_usable_zone_for_movable(void)
6331 {
6332 int zone_index;
6333 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
6334 if (zone_index == ZONE_MOVABLE)
6335 continue;
6336
6337 if (arch_zone_highest_possible_pfn[zone_index] >
6338 arch_zone_lowest_possible_pfn[zone_index])
6339 break;
6340 }
6341
6342 VM_BUG_ON(zone_index == -1);
6343 movable_zone = zone_index;
6344 }
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356 static void __init adjust_zone_range_for_zone_movable(int nid,
6357 unsigned long zone_type,
6358 unsigned long node_start_pfn,
6359 unsigned long node_end_pfn,
6360 unsigned long *zone_start_pfn,
6361 unsigned long *zone_end_pfn)
6362 {
6363
6364 if (zone_movable_pfn[nid]) {
6365
6366 if (zone_type == ZONE_MOVABLE) {
6367 *zone_start_pfn = zone_movable_pfn[nid];
6368 *zone_end_pfn = min(node_end_pfn,
6369 arch_zone_highest_possible_pfn[movable_zone]);
6370
6371
6372 } else if (!mirrored_kernelcore &&
6373 *zone_start_pfn < zone_movable_pfn[nid] &&
6374 *zone_end_pfn > zone_movable_pfn[nid]) {
6375 *zone_end_pfn = zone_movable_pfn[nid];
6376
6377
6378 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
6379 *zone_start_pfn = *zone_end_pfn;
6380 }
6381 }
6382
6383
6384
6385
6386
6387 static unsigned long __init zone_spanned_pages_in_node(int nid,
6388 unsigned long zone_type,
6389 unsigned long node_start_pfn,
6390 unsigned long node_end_pfn,
6391 unsigned long *zone_start_pfn,
6392 unsigned long *zone_end_pfn,
6393 unsigned long *ignored)
6394 {
6395 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
6396 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
6397
6398 if (!node_start_pfn && !node_end_pfn)
6399 return 0;
6400
6401
6402 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
6403 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
6404 adjust_zone_range_for_zone_movable(nid, zone_type,
6405 node_start_pfn, node_end_pfn,
6406 zone_start_pfn, zone_end_pfn);
6407
6408
6409 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
6410 return 0;
6411
6412
6413 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
6414 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
6415
6416
6417 return *zone_end_pfn - *zone_start_pfn;
6418 }
6419
6420
6421
6422
6423
6424 unsigned long __init __absent_pages_in_range(int nid,
6425 unsigned long range_start_pfn,
6426 unsigned long range_end_pfn)
6427 {
6428 unsigned long nr_absent = range_end_pfn - range_start_pfn;
6429 unsigned long start_pfn, end_pfn;
6430 int i;
6431
6432 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
6433 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
6434 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
6435 nr_absent -= end_pfn - start_pfn;
6436 }
6437 return nr_absent;
6438 }
6439
6440
6441
6442
6443
6444
6445
6446
6447 unsigned long __init absent_pages_in_range(unsigned long start_pfn,
6448 unsigned long end_pfn)
6449 {
6450 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
6451 }
6452
6453
6454 static unsigned long __init zone_absent_pages_in_node(int nid,
6455 unsigned long zone_type,
6456 unsigned long node_start_pfn,
6457 unsigned long node_end_pfn,
6458 unsigned long *ignored)
6459 {
6460 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
6461 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
6462 unsigned long zone_start_pfn, zone_end_pfn;
6463 unsigned long nr_absent;
6464
6465
6466 if (!node_start_pfn && !node_end_pfn)
6467 return 0;
6468
6469 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
6470 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
6471
6472 adjust_zone_range_for_zone_movable(nid, zone_type,
6473 node_start_pfn, node_end_pfn,
6474 &zone_start_pfn, &zone_end_pfn);
6475 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
6476
6477
6478
6479
6480
6481
6482 if (mirrored_kernelcore && zone_movable_pfn[nid]) {
6483 unsigned long start_pfn, end_pfn;
6484 struct memblock_region *r;
6485
6486 for_each_memblock(memory, r) {
6487 start_pfn = clamp(memblock_region_memory_base_pfn(r),
6488 zone_start_pfn, zone_end_pfn);
6489 end_pfn = clamp(memblock_region_memory_end_pfn(r),
6490 zone_start_pfn, zone_end_pfn);
6491
6492 if (zone_type == ZONE_MOVABLE &&
6493 memblock_is_mirror(r))
6494 nr_absent += end_pfn - start_pfn;
6495
6496 if (zone_type == ZONE_NORMAL &&
6497 !memblock_is_mirror(r))
6498 nr_absent += end_pfn - start_pfn;
6499 }
6500 }
6501
6502 return nr_absent;
6503 }
6504
6505 #else
6506 static inline unsigned long __init zone_spanned_pages_in_node(int nid,
6507 unsigned long zone_type,
6508 unsigned long node_start_pfn,
6509 unsigned long node_end_pfn,
6510 unsigned long *zone_start_pfn,
6511 unsigned long *zone_end_pfn,
6512 unsigned long *zones_size)
6513 {
6514 unsigned int zone;
6515
6516 *zone_start_pfn = node_start_pfn;
6517 for (zone = 0; zone < zone_type; zone++)
6518 *zone_start_pfn += zones_size[zone];
6519
6520 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type];
6521
6522 return zones_size[zone_type];
6523 }
6524
6525 static inline unsigned long __init zone_absent_pages_in_node(int nid,
6526 unsigned long zone_type,
6527 unsigned long node_start_pfn,
6528 unsigned long node_end_pfn,
6529 unsigned long *zholes_size)
6530 {
6531 if (!zholes_size)
6532 return 0;
6533
6534 return zholes_size[zone_type];
6535 }
6536
6537 #endif
6538
6539 static void __init calculate_node_totalpages(struct pglist_data *pgdat,
6540 unsigned long node_start_pfn,
6541 unsigned long node_end_pfn,
6542 unsigned long *zones_size,
6543 unsigned long *zholes_size)
6544 {
6545 unsigned long realtotalpages = 0, totalpages = 0;
6546 enum zone_type i;
6547
6548 for (i = 0; i < MAX_NR_ZONES; i++) {
6549 struct zone *zone = pgdat->node_zones + i;
6550 unsigned long zone_start_pfn, zone_end_pfn;
6551 unsigned long size, real_size;
6552
6553 size = zone_spanned_pages_in_node(pgdat->node_id, i,
6554 node_start_pfn,
6555 node_end_pfn,
6556 &zone_start_pfn,
6557 &zone_end_pfn,
6558 zones_size);
6559 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i,
6560 node_start_pfn, node_end_pfn,
6561 zholes_size);
6562 if (size)
6563 zone->zone_start_pfn = zone_start_pfn;
6564 else
6565 zone->zone_start_pfn = 0;
6566 zone->spanned_pages = size;
6567 zone->present_pages = real_size;
6568
6569 totalpages += size;
6570 realtotalpages += real_size;
6571 }
6572
6573 pgdat->node_spanned_pages = totalpages;
6574 pgdat->node_present_pages = realtotalpages;
6575 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
6576 realtotalpages);
6577 }
6578
6579 #ifndef CONFIG_SPARSEMEM
6580
6581
6582
6583
6584
6585
6586
6587 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
6588 {
6589 unsigned long usemapsize;
6590
6591 zonesize += zone_start_pfn & (pageblock_nr_pages-1);
6592 usemapsize = roundup(zonesize, pageblock_nr_pages);
6593 usemapsize = usemapsize >> pageblock_order;
6594 usemapsize *= NR_PAGEBLOCK_BITS;
6595 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
6596
6597 return usemapsize / 8;
6598 }
6599
6600 static void __ref setup_usemap(struct pglist_data *pgdat,
6601 struct zone *zone,
6602 unsigned long zone_start_pfn,
6603 unsigned long zonesize)
6604 {
6605 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
6606 zone->pageblock_flags = NULL;
6607 if (usemapsize) {
6608 zone->pageblock_flags =
6609 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
6610 pgdat->node_id);
6611 if (!zone->pageblock_flags)
6612 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
6613 usemapsize, zone->name, pgdat->node_id);
6614 }
6615 }
6616 #else
6617 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
6618 unsigned long zone_start_pfn, unsigned long zonesize) {}
6619 #endif
6620
6621 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
6622
6623
6624 void __init set_pageblock_order(void)
6625 {
6626 unsigned int order;
6627
6628
6629 if (pageblock_order)
6630 return;
6631
6632 if (HPAGE_SHIFT > PAGE_SHIFT)
6633 order = HUGETLB_PAGE_ORDER;
6634 else
6635 order = MAX_ORDER - 1;
6636
6637
6638
6639
6640
6641
6642 pageblock_order = order;
6643 }
6644 #else
6645
6646
6647
6648
6649
6650
6651
6652 void __init set_pageblock_order(void)
6653 {
6654 }
6655
6656 #endif
6657
6658 static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
6659 unsigned long present_pages)
6660 {
6661 unsigned long pages = spanned_pages;
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671 if (spanned_pages > present_pages + (present_pages >> 4) &&
6672 IS_ENABLED(CONFIG_SPARSEMEM))
6673 pages = present_pages;
6674
6675 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
6676 }
6677
6678 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
6679 static void pgdat_init_split_queue(struct pglist_data *pgdat)
6680 {
6681 struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
6682
6683 spin_lock_init(&ds_queue->split_queue_lock);
6684 INIT_LIST_HEAD(&ds_queue->split_queue);
6685 ds_queue->split_queue_len = 0;
6686 }
6687 #else
6688 static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
6689 #endif
6690
6691 #ifdef CONFIG_COMPACTION
6692 static void pgdat_init_kcompactd(struct pglist_data *pgdat)
6693 {
6694 init_waitqueue_head(&pgdat->kcompactd_wait);
6695 }
6696 #else
6697 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
6698 #endif
6699
6700 static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
6701 {
6702 pgdat_resize_init(pgdat);
6703
6704 pgdat_init_split_queue(pgdat);
6705 pgdat_init_kcompactd(pgdat);
6706
6707 init_waitqueue_head(&pgdat->kswapd_wait);
6708 init_waitqueue_head(&pgdat->pfmemalloc_wait);
6709
6710 pgdat_page_ext_init(pgdat);
6711 spin_lock_init(&pgdat->lru_lock);
6712 lruvec_init(node_lruvec(pgdat));
6713 }
6714
6715 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
6716 unsigned long remaining_pages)
6717 {
6718 atomic_long_set(&zone->managed_pages, remaining_pages);
6719 zone_set_nid(zone, nid);
6720 zone->name = zone_names[idx];
6721 zone->zone_pgdat = NODE_DATA(nid);
6722 spin_lock_init(&zone->lock);
6723 zone_seqlock_init(zone);
6724 zone_pcp_init(zone);
6725 }
6726
6727
6728
6729
6730
6731
6732
6733
6734 #ifdef CONFIG_MEMORY_HOTPLUG
6735 void __ref free_area_init_core_hotplug(int nid)
6736 {
6737 enum zone_type z;
6738 pg_data_t *pgdat = NODE_DATA(nid);
6739
6740 pgdat_init_internals(pgdat);
6741 for (z = 0; z < MAX_NR_ZONES; z++)
6742 zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
6743 }
6744 #endif
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755 static void __init free_area_init_core(struct pglist_data *pgdat)
6756 {
6757 enum zone_type j;
6758 int nid = pgdat->node_id;
6759
6760 pgdat_init_internals(pgdat);
6761 pgdat->per_cpu_nodestats = &boot_nodestats;
6762
6763 for (j = 0; j < MAX_NR_ZONES; j++) {
6764 struct zone *zone = pgdat->node_zones + j;
6765 unsigned long size, freesize, memmap_pages;
6766 unsigned long zone_start_pfn = zone->zone_start_pfn;
6767
6768 size = zone->spanned_pages;
6769 freesize = zone->present_pages;
6770
6771
6772
6773
6774
6775
6776 memmap_pages = calc_memmap_size(size, freesize);
6777 if (!is_highmem_idx(j)) {
6778 if (freesize >= memmap_pages) {
6779 freesize -= memmap_pages;
6780 if (memmap_pages)
6781 printk(KERN_DEBUG
6782 " %s zone: %lu pages used for memmap\n",
6783 zone_names[j], memmap_pages);
6784 } else
6785 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n",
6786 zone_names[j], memmap_pages, freesize);
6787 }
6788
6789
6790 if (j == 0 && freesize > dma_reserve) {
6791 freesize -= dma_reserve;
6792 printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
6793 zone_names[0], dma_reserve);
6794 }
6795
6796 if (!is_highmem_idx(j))
6797 nr_kernel_pages += freesize;
6798
6799 else if (nr_kernel_pages > memmap_pages * 2)
6800 nr_kernel_pages -= memmap_pages;
6801 nr_all_pages += freesize;
6802
6803
6804
6805
6806
6807
6808 zone_init_internals(zone, j, nid, freesize);
6809
6810 if (!size)
6811 continue;
6812
6813 set_pageblock_order();
6814 setup_usemap(pgdat, zone, zone_start_pfn, size);
6815 init_currently_empty_zone(zone, zone_start_pfn, size);
6816 memmap_init(size, nid, j, zone_start_pfn);
6817 }
6818 }
6819
6820 #ifdef CONFIG_FLAT_NODE_MEM_MAP
6821 static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
6822 {
6823 unsigned long __maybe_unused start = 0;
6824 unsigned long __maybe_unused offset = 0;
6825
6826
6827 if (!pgdat->node_spanned_pages)
6828 return;
6829
6830 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
6831 offset = pgdat->node_start_pfn - start;
6832
6833 if (!pgdat->node_mem_map) {
6834 unsigned long size, end;
6835 struct page *map;
6836
6837
6838
6839
6840
6841
6842 end = pgdat_end_pfn(pgdat);
6843 end = ALIGN(end, MAX_ORDER_NR_PAGES);
6844 size = (end - start) * sizeof(struct page);
6845 map = memblock_alloc_node(size, SMP_CACHE_BYTES,
6846 pgdat->node_id);
6847 if (!map)
6848 panic("Failed to allocate %ld bytes for node %d memory map\n",
6849 size, pgdat->node_id);
6850 pgdat->node_mem_map = map + offset;
6851 }
6852 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
6853 __func__, pgdat->node_id, (unsigned long)pgdat,
6854 (unsigned long)pgdat->node_mem_map);
6855 #ifndef CONFIG_NEED_MULTIPLE_NODES
6856
6857
6858
6859 if (pgdat == NODE_DATA(0)) {
6860 mem_map = NODE_DATA(0)->node_mem_map;
6861 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM)
6862 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
6863 mem_map -= offset;
6864 #endif
6865 }
6866 #endif
6867 }
6868 #else
6869 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { }
6870 #endif
6871
6872 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
6873 static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
6874 {
6875 pgdat->first_deferred_pfn = ULONG_MAX;
6876 }
6877 #else
6878 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
6879 #endif
6880
6881 void __init free_area_init_node(int nid, unsigned long *zones_size,
6882 unsigned long node_start_pfn,
6883 unsigned long *zholes_size)
6884 {
6885 pg_data_t *pgdat = NODE_DATA(nid);
6886 unsigned long start_pfn = 0;
6887 unsigned long end_pfn = 0;
6888
6889
6890 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx);
6891
6892 pgdat->node_id = nid;
6893 pgdat->node_start_pfn = node_start_pfn;
6894 pgdat->per_cpu_nodestats = NULL;
6895 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6896 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
6897 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
6898 (u64)start_pfn << PAGE_SHIFT,
6899 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
6900 #else
6901 start_pfn = node_start_pfn;
6902 #endif
6903 calculate_node_totalpages(pgdat, start_pfn, end_pfn,
6904 zones_size, zholes_size);
6905
6906 alloc_node_mem_map(pgdat);
6907 pgdat_set_deferred_range(pgdat);
6908
6909 free_area_init_core(pgdat);
6910 }
6911
6912 #if !defined(CONFIG_FLAT_NODE_MEM_MAP)
6913
6914
6915
6916
6917 static u64 zero_pfn_range(unsigned long spfn, unsigned long epfn)
6918 {
6919 unsigned long pfn;
6920 u64 pgcnt = 0;
6921
6922 for (pfn = spfn; pfn < epfn; pfn++) {
6923 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
6924 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
6925 + pageblock_nr_pages - 1;
6926 continue;
6927 }
6928 mm_zero_struct_page(pfn_to_page(pfn));
6929 pgcnt++;
6930 }
6931
6932 return pgcnt;
6933 }
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948 void __init zero_resv_unavail(void)
6949 {
6950 phys_addr_t start, end;
6951 u64 i, pgcnt;
6952 phys_addr_t next = 0;
6953
6954
6955
6956
6957 pgcnt = 0;
6958 for_each_mem_range(i, &memblock.memory, NULL,
6959 NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) {
6960 if (next < start)
6961 pgcnt += zero_pfn_range(PFN_DOWN(next), PFN_UP(start));
6962 next = end;
6963 }
6964
6965
6966
6967
6968
6969
6970
6971
6972 pgcnt += zero_pfn_range(PFN_DOWN(next),
6973 round_up(max_pfn, PAGES_PER_SECTION));
6974
6975
6976
6977
6978
6979 if (pgcnt)
6980 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt);
6981 }
6982 #endif
6983
6984 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6985
6986 #if MAX_NUMNODES > 1
6987
6988
6989
6990 void __init setup_nr_node_ids(void)
6991 {
6992 unsigned int highest;
6993
6994 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
6995 nr_node_ids = highest + 1;
6996 }
6997 #endif
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018 unsigned long __init node_map_pfn_alignment(void)
7019 {
7020 unsigned long accl_mask = 0, last_end = 0;
7021 unsigned long start, end, mask;
7022 int last_nid = NUMA_NO_NODE;
7023 int i, nid;
7024
7025 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
7026 if (!start || last_nid < 0 || last_nid == nid) {
7027 last_nid = nid;
7028 last_end = end;
7029 continue;
7030 }
7031
7032
7033
7034
7035
7036
7037 mask = ~((1 << __ffs(start)) - 1);
7038 while (mask && last_end <= (start & (mask << 1)))
7039 mask <<= 1;
7040
7041
7042 accl_mask |= mask;
7043 }
7044
7045
7046 return ~accl_mask + 1;
7047 }
7048
7049
7050 static unsigned long __init find_min_pfn_for_node(int nid)
7051 {
7052 unsigned long min_pfn = ULONG_MAX;
7053 unsigned long start_pfn;
7054 int i;
7055
7056 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
7057 min_pfn = min(min_pfn, start_pfn);
7058
7059 if (min_pfn == ULONG_MAX) {
7060 pr_warn("Could not find start_pfn for node %d\n", nid);
7061 return 0;
7062 }
7063
7064 return min_pfn;
7065 }
7066
7067
7068
7069
7070
7071
7072
7073 unsigned long __init find_min_pfn_with_active_regions(void)
7074 {
7075 return find_min_pfn_for_node(MAX_NUMNODES);
7076 }
7077
7078
7079
7080
7081
7082
7083 static unsigned long __init early_calculate_totalpages(void)
7084 {
7085 unsigned long totalpages = 0;
7086 unsigned long start_pfn, end_pfn;
7087 int i, nid;
7088
7089 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
7090 unsigned long pages = end_pfn - start_pfn;
7091
7092 totalpages += pages;
7093 if (pages)
7094 node_set_state(nid, N_MEMORY);
7095 }
7096 return totalpages;
7097 }
7098
7099
7100
7101
7102
7103
7104
7105 static void __init find_zone_movable_pfns_for_nodes(void)
7106 {
7107 int i, nid;
7108 unsigned long usable_startpfn;
7109 unsigned long kernelcore_node, kernelcore_remaining;
7110
7111 nodemask_t saved_node_state = node_states[N_MEMORY];
7112 unsigned long totalpages = early_calculate_totalpages();
7113 int usable_nodes = nodes_weight(node_states[N_MEMORY]);
7114 struct memblock_region *r;
7115
7116
7117 find_usable_zone_for_movable();
7118
7119
7120
7121
7122
7123 if (movable_node_is_enabled()) {
7124 for_each_memblock(memory, r) {
7125 if (!memblock_is_hotpluggable(r))
7126 continue;
7127
7128 nid = r->nid;
7129
7130 usable_startpfn = PFN_DOWN(r->base);
7131 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7132 min(usable_startpfn, zone_movable_pfn[nid]) :
7133 usable_startpfn;
7134 }
7135
7136 goto out2;
7137 }
7138
7139
7140
7141
7142 if (mirrored_kernelcore) {
7143 bool mem_below_4gb_not_mirrored = false;
7144
7145 for_each_memblock(memory, r) {
7146 if (memblock_is_mirror(r))
7147 continue;
7148
7149 nid = r->nid;
7150
7151 usable_startpfn = memblock_region_memory_base_pfn(r);
7152
7153 if (usable_startpfn < 0x100000) {
7154 mem_below_4gb_not_mirrored = true;
7155 continue;
7156 }
7157
7158 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7159 min(usable_startpfn, zone_movable_pfn[nid]) :
7160 usable_startpfn;
7161 }
7162
7163 if (mem_below_4gb_not_mirrored)
7164 pr_warn("This configuration results in unmirrored kernel memory.");
7165
7166 goto out2;
7167 }
7168
7169
7170
7171
7172
7173 if (required_kernelcore_percent)
7174 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
7175 10000UL;
7176 if (required_movablecore_percent)
7177 required_movablecore = (totalpages * 100 * required_movablecore_percent) /
7178 10000UL;
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188 if (required_movablecore) {
7189 unsigned long corepages;
7190
7191
7192
7193
7194
7195 required_movablecore =
7196 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
7197 required_movablecore = min(totalpages, required_movablecore);
7198 corepages = totalpages - required_movablecore;
7199
7200 required_kernelcore = max(required_kernelcore, corepages);
7201 }
7202
7203
7204
7205
7206
7207 if (!required_kernelcore || required_kernelcore >= totalpages)
7208 goto out;
7209
7210
7211 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
7212
7213 restart:
7214
7215 kernelcore_node = required_kernelcore / usable_nodes;
7216 for_each_node_state(nid, N_MEMORY) {
7217 unsigned long start_pfn, end_pfn;
7218
7219
7220
7221
7222
7223
7224 if (required_kernelcore < kernelcore_node)
7225 kernelcore_node = required_kernelcore / usable_nodes;
7226
7227
7228
7229
7230
7231
7232 kernelcore_remaining = kernelcore_node;
7233
7234
7235 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
7236 unsigned long size_pages;
7237
7238 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
7239 if (start_pfn >= end_pfn)
7240 continue;
7241
7242
7243 if (start_pfn < usable_startpfn) {
7244 unsigned long kernel_pages;
7245 kernel_pages = min(end_pfn, usable_startpfn)
7246 - start_pfn;
7247
7248 kernelcore_remaining -= min(kernel_pages,
7249 kernelcore_remaining);
7250 required_kernelcore -= min(kernel_pages,
7251 required_kernelcore);
7252
7253
7254 if (end_pfn <= usable_startpfn) {
7255
7256
7257
7258
7259
7260
7261
7262 zone_movable_pfn[nid] = end_pfn;
7263 continue;
7264 }
7265 start_pfn = usable_startpfn;
7266 }
7267
7268
7269
7270
7271
7272
7273 size_pages = end_pfn - start_pfn;
7274 if (size_pages > kernelcore_remaining)
7275 size_pages = kernelcore_remaining;
7276 zone_movable_pfn[nid] = start_pfn + size_pages;
7277
7278
7279
7280
7281
7282
7283 required_kernelcore -= min(required_kernelcore,
7284 size_pages);
7285 kernelcore_remaining -= size_pages;
7286 if (!kernelcore_remaining)
7287 break;
7288 }
7289 }
7290
7291
7292
7293
7294
7295
7296
7297 usable_nodes--;
7298 if (usable_nodes && required_kernelcore > usable_nodes)
7299 goto restart;
7300
7301 out2:
7302
7303 for (nid = 0; nid < MAX_NUMNODES; nid++)
7304 zone_movable_pfn[nid] =
7305 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
7306
7307 out:
7308
7309 node_states[N_MEMORY] = saved_node_state;
7310 }
7311
7312
7313 static void check_for_memory(pg_data_t *pgdat, int nid)
7314 {
7315 enum zone_type zone_type;
7316
7317 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
7318 struct zone *zone = &pgdat->node_zones[zone_type];
7319 if (populated_zone(zone)) {
7320 if (IS_ENABLED(CONFIG_HIGHMEM))
7321 node_set_state(nid, N_HIGH_MEMORY);
7322 if (zone_type <= ZONE_NORMAL)
7323 node_set_state(nid, N_NORMAL_MEMORY);
7324 break;
7325 }
7326 }
7327 }
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342 void __init free_area_init_nodes(unsigned long *max_zone_pfn)
7343 {
7344 unsigned long start_pfn, end_pfn;
7345 int i, nid;
7346
7347
7348 memset(arch_zone_lowest_possible_pfn, 0,
7349 sizeof(arch_zone_lowest_possible_pfn));
7350 memset(arch_zone_highest_possible_pfn, 0,
7351 sizeof(arch_zone_highest_possible_pfn));
7352
7353 start_pfn = find_min_pfn_with_active_regions();
7354
7355 for (i = 0; i < MAX_NR_ZONES; i++) {
7356 if (i == ZONE_MOVABLE)
7357 continue;
7358
7359 end_pfn = max(max_zone_pfn[i], start_pfn);
7360 arch_zone_lowest_possible_pfn[i] = start_pfn;
7361 arch_zone_highest_possible_pfn[i] = end_pfn;
7362
7363 start_pfn = end_pfn;
7364 }
7365
7366
7367 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
7368 find_zone_movable_pfns_for_nodes();
7369
7370
7371 pr_info("Zone ranges:\n");
7372 for (i = 0; i < MAX_NR_ZONES; i++) {
7373 if (i == ZONE_MOVABLE)
7374 continue;
7375 pr_info(" %-8s ", zone_names[i]);
7376 if (arch_zone_lowest_possible_pfn[i] ==
7377 arch_zone_highest_possible_pfn[i])
7378 pr_cont("empty\n");
7379 else
7380 pr_cont("[mem %#018Lx-%#018Lx]\n",
7381 (u64)arch_zone_lowest_possible_pfn[i]
7382 << PAGE_SHIFT,
7383 ((u64)arch_zone_highest_possible_pfn[i]
7384 << PAGE_SHIFT) - 1);
7385 }
7386
7387
7388 pr_info("Movable zone start for each node\n");
7389 for (i = 0; i < MAX_NUMNODES; i++) {
7390 if (zone_movable_pfn[i])
7391 pr_info(" Node %d: %#018Lx\n", i,
7392 (u64)zone_movable_pfn[i] << PAGE_SHIFT);
7393 }
7394
7395
7396
7397
7398
7399
7400 pr_info("Early memory node ranges\n");
7401 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
7402 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
7403 (u64)start_pfn << PAGE_SHIFT,
7404 ((u64)end_pfn << PAGE_SHIFT) - 1);
7405 subsection_map_init(start_pfn, end_pfn - start_pfn);
7406 }
7407
7408
7409 mminit_verify_pageflags_layout();
7410 setup_nr_node_ids();
7411 zero_resv_unavail();
7412 for_each_online_node(nid) {
7413 pg_data_t *pgdat = NODE_DATA(nid);
7414 free_area_init_node(nid, NULL,
7415 find_min_pfn_for_node(nid), NULL);
7416
7417
7418 if (pgdat->node_present_pages)
7419 node_set_state(nid, N_MEMORY);
7420 check_for_memory(pgdat, nid);
7421 }
7422 }
7423
7424 static int __init cmdline_parse_core(char *p, unsigned long *core,
7425 unsigned long *percent)
7426 {
7427 unsigned long long coremem;
7428 char *endptr;
7429
7430 if (!p)
7431 return -EINVAL;
7432
7433
7434 coremem = simple_strtoull(p, &endptr, 0);
7435 if (*endptr == '%') {
7436
7437 WARN_ON(coremem > 100);
7438
7439 *percent = coremem;
7440 } else {
7441 coremem = memparse(p, &p);
7442
7443 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
7444
7445 *core = coremem >> PAGE_SHIFT;
7446 *percent = 0UL;
7447 }
7448 return 0;
7449 }
7450
7451
7452
7453
7454
7455 static int __init cmdline_parse_kernelcore(char *p)
7456 {
7457
7458 if (parse_option_str(p, "mirror")) {
7459 mirrored_kernelcore = true;
7460 return 0;
7461 }
7462
7463 return cmdline_parse_core(p, &required_kernelcore,
7464 &required_kernelcore_percent);
7465 }
7466
7467
7468
7469
7470
7471 static int __init cmdline_parse_movablecore(char *p)
7472 {
7473 return cmdline_parse_core(p, &required_movablecore,
7474 &required_movablecore_percent);
7475 }
7476
7477 early_param("kernelcore", cmdline_parse_kernelcore);
7478 early_param("movablecore", cmdline_parse_movablecore);
7479
7480 #endif
7481
7482 void adjust_managed_page_count(struct page *page, long count)
7483 {
7484 atomic_long_add(count, &page_zone(page)->managed_pages);
7485 totalram_pages_add(count);
7486 #ifdef CONFIG_HIGHMEM
7487 if (PageHighMem(page))
7488 totalhigh_pages_add(count);
7489 #endif
7490 }
7491 EXPORT_SYMBOL(adjust_managed_page_count);
7492
7493 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s)
7494 {
7495 void *pos;
7496 unsigned long pages = 0;
7497
7498 start = (void *)PAGE_ALIGN((unsigned long)start);
7499 end = (void *)((unsigned long)end & PAGE_MASK);
7500 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
7501 struct page *page = virt_to_page(pos);
7502 void *direct_map_addr;
7503
7504
7505
7506
7507
7508
7509
7510
7511 direct_map_addr = page_address(page);
7512 if ((unsigned int)poison <= 0xFF)
7513 memset(direct_map_addr, poison, PAGE_SIZE);
7514
7515 free_reserved_page(page);
7516 }
7517
7518 if (pages && s)
7519 pr_info("Freeing %s memory: %ldK\n",
7520 s, pages << (PAGE_SHIFT - 10));
7521
7522 return pages;
7523 }
7524
7525 #ifdef CONFIG_HIGHMEM
7526 void free_highmem_page(struct page *page)
7527 {
7528 __free_reserved_page(page);
7529 totalram_pages_inc();
7530 atomic_long_inc(&page_zone(page)->managed_pages);
7531 totalhigh_pages_inc();
7532 }
7533 #endif
7534
7535
7536 void __init mem_init_print_info(const char *str)
7537 {
7538 unsigned long physpages, codesize, datasize, rosize, bss_size;
7539 unsigned long init_code_size, init_data_size;
7540
7541 physpages = get_num_physpages();
7542 codesize = _etext - _stext;
7543 datasize = _edata - _sdata;
7544 rosize = __end_rodata - __start_rodata;
7545 bss_size = __bss_stop - __bss_start;
7546 init_data_size = __init_end - __init_begin;
7547 init_code_size = _einittext - _sinittext;
7548
7549
7550
7551
7552
7553
7554
7555
7556 #define adj_init_size(start, end, size, pos, adj) \
7557 do { \
7558 if (start <= pos && pos < end && size > adj) \
7559 size -= adj; \
7560 } while (0)
7561
7562 adj_init_size(__init_begin, __init_end, init_data_size,
7563 _sinittext, init_code_size);
7564 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
7565 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
7566 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
7567 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
7568
7569 #undef adj_init_size
7570
7571 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
7572 #ifdef CONFIG_HIGHMEM
7573 ", %luK highmem"
7574 #endif
7575 "%s%s)\n",
7576 nr_free_pages() << (PAGE_SHIFT - 10),
7577 physpages << (PAGE_SHIFT - 10),
7578 codesize >> 10, datasize >> 10, rosize >> 10,
7579 (init_data_size + init_code_size) >> 10, bss_size >> 10,
7580 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10),
7581 totalcma_pages << (PAGE_SHIFT - 10),
7582 #ifdef CONFIG_HIGHMEM
7583 totalhigh_pages() << (PAGE_SHIFT - 10),
7584 #endif
7585 str ? ", " : "", str ? str : "");
7586 }
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599 void __init set_dma_reserve(unsigned long new_dma_reserve)
7600 {
7601 dma_reserve = new_dma_reserve;
7602 }
7603
7604 void __init free_area_init(unsigned long *zones_size)
7605 {
7606 zero_resv_unavail();
7607 free_area_init_node(0, zones_size,
7608 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
7609 }
7610
7611 static int page_alloc_cpu_dead(unsigned int cpu)
7612 {
7613
7614 lru_add_drain_cpu(cpu);
7615 drain_pages(cpu);
7616
7617
7618
7619
7620
7621
7622
7623 vm_events_fold_cpu(cpu);
7624
7625
7626
7627
7628
7629
7630
7631
7632 cpu_vm_stats_fold(cpu);
7633 return 0;
7634 }
7635
7636 #ifdef CONFIG_NUMA
7637 int hashdist = HASHDIST_DEFAULT;
7638
7639 static int __init set_hashdist(char *str)
7640 {
7641 if (!str)
7642 return 0;
7643 hashdist = simple_strtoul(str, &str, 0);
7644 return 1;
7645 }
7646 __setup("hashdist=", set_hashdist);
7647 #endif
7648
7649 void __init page_alloc_init(void)
7650 {
7651 int ret;
7652
7653 #ifdef CONFIG_NUMA
7654 if (num_node_state(N_MEMORY) == 1)
7655 hashdist = 0;
7656 #endif
7657
7658 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
7659 "mm/page_alloc:dead", NULL,
7660 page_alloc_cpu_dead);
7661 WARN_ON(ret < 0);
7662 }
7663
7664
7665
7666
7667
7668 static void calculate_totalreserve_pages(void)
7669 {
7670 struct pglist_data *pgdat;
7671 unsigned long reserve_pages = 0;
7672 enum zone_type i, j;
7673
7674 for_each_online_pgdat(pgdat) {
7675
7676 pgdat->totalreserve_pages = 0;
7677
7678 for (i = 0; i < MAX_NR_ZONES; i++) {
7679 struct zone *zone = pgdat->node_zones + i;
7680 long max = 0;
7681 unsigned long managed_pages = zone_managed_pages(zone);
7682
7683
7684 for (j = i; j < MAX_NR_ZONES; j++) {
7685 if (zone->lowmem_reserve[j] > max)
7686 max = zone->lowmem_reserve[j];
7687 }
7688
7689
7690 max += high_wmark_pages(zone);
7691
7692 if (max > managed_pages)
7693 max = managed_pages;
7694
7695 pgdat->totalreserve_pages += max;
7696
7697 reserve_pages += max;
7698 }
7699 }
7700 totalreserve_pages = reserve_pages;
7701 }
7702
7703
7704
7705
7706
7707
7708
7709 static void setup_per_zone_lowmem_reserve(void)
7710 {
7711 struct pglist_data *pgdat;
7712 enum zone_type j, idx;
7713
7714 for_each_online_pgdat(pgdat) {
7715 for (j = 0; j < MAX_NR_ZONES; j++) {
7716 struct zone *zone = pgdat->node_zones + j;
7717 unsigned long managed_pages = zone_managed_pages(zone);
7718
7719 zone->lowmem_reserve[j] = 0;
7720
7721 idx = j;
7722 while (idx) {
7723 struct zone *lower_zone;
7724
7725 idx--;
7726 lower_zone = pgdat->node_zones + idx;
7727
7728 if (sysctl_lowmem_reserve_ratio[idx] < 1) {
7729 sysctl_lowmem_reserve_ratio[idx] = 0;
7730 lower_zone->lowmem_reserve[j] = 0;
7731 } else {
7732 lower_zone->lowmem_reserve[j] =
7733 managed_pages / sysctl_lowmem_reserve_ratio[idx];
7734 }
7735 managed_pages += zone_managed_pages(lower_zone);
7736 }
7737 }
7738 }
7739
7740
7741 calculate_totalreserve_pages();
7742 }
7743
7744 static void __setup_per_zone_wmarks(void)
7745 {
7746 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
7747 unsigned long lowmem_pages = 0;
7748 struct zone *zone;
7749 unsigned long flags;
7750
7751
7752 for_each_zone(zone) {
7753 if (!is_highmem(zone))
7754 lowmem_pages += zone_managed_pages(zone);
7755 }
7756
7757 for_each_zone(zone) {
7758 u64 tmp;
7759
7760 spin_lock_irqsave(&zone->lock, flags);
7761 tmp = (u64)pages_min * zone_managed_pages(zone);
7762 do_div(tmp, lowmem_pages);
7763 if (is_highmem(zone)) {
7764
7765
7766
7767
7768
7769
7770
7771
7772
7773 unsigned long min_pages;
7774
7775 min_pages = zone_managed_pages(zone) / 1024;
7776 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
7777 zone->_watermark[WMARK_MIN] = min_pages;
7778 } else {
7779
7780
7781
7782
7783 zone->_watermark[WMARK_MIN] = tmp;
7784 }
7785
7786
7787
7788
7789
7790
7791 tmp = max_t(u64, tmp >> 2,
7792 mult_frac(zone_managed_pages(zone),
7793 watermark_scale_factor, 10000));
7794
7795 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
7796 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
7797 zone->watermark_boost = 0;
7798
7799 spin_unlock_irqrestore(&zone->lock, flags);
7800 }
7801
7802
7803 calculate_totalreserve_pages();
7804 }
7805
7806
7807
7808
7809
7810
7811
7812
7813 void setup_per_zone_wmarks(void)
7814 {
7815 static DEFINE_SPINLOCK(lock);
7816
7817 spin_lock(&lock);
7818 __setup_per_zone_wmarks();
7819 spin_unlock(&lock);
7820 }
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
7833
7834
7835
7836
7837
7838
7839
7840
7841
7842
7843
7844
7845
7846 int __meminit init_per_zone_wmark_min(void)
7847 {
7848 unsigned long lowmem_kbytes;
7849 int new_min_free_kbytes;
7850
7851 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
7852 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
7853
7854 if (new_min_free_kbytes > user_min_free_kbytes) {
7855 min_free_kbytes = new_min_free_kbytes;
7856 if (min_free_kbytes < 128)
7857 min_free_kbytes = 128;
7858 if (min_free_kbytes > 65536)
7859 min_free_kbytes = 65536;
7860 } else {
7861 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
7862 new_min_free_kbytes, user_min_free_kbytes);
7863 }
7864 setup_per_zone_wmarks();
7865 refresh_zone_stat_thresholds();
7866 setup_per_zone_lowmem_reserve();
7867
7868 #ifdef CONFIG_NUMA
7869 setup_min_unmapped_ratio();
7870 setup_min_slab_ratio();
7871 #endif
7872
7873 return 0;
7874 }
7875 core_initcall(init_per_zone_wmark_min)
7876
7877
7878
7879
7880
7881
7882 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
7883 void __user *buffer, size_t *length, loff_t *ppos)
7884 {
7885 int rc;
7886
7887 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7888 if (rc)
7889 return rc;
7890
7891 if (write) {
7892 user_min_free_kbytes = min_free_kbytes;
7893 setup_per_zone_wmarks();
7894 }
7895 return 0;
7896 }
7897
7898 int watermark_boost_factor_sysctl_handler(struct ctl_table *table, int write,
7899 void __user *buffer, size_t *length, loff_t *ppos)
7900 {
7901 int rc;
7902
7903 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7904 if (rc)
7905 return rc;
7906
7907 return 0;
7908 }
7909
7910 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
7911 void __user *buffer, size_t *length, loff_t *ppos)
7912 {
7913 int rc;
7914
7915 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7916 if (rc)
7917 return rc;
7918
7919 if (write)
7920 setup_per_zone_wmarks();
7921
7922 return 0;
7923 }
7924
7925 #ifdef CONFIG_NUMA
7926 static void setup_min_unmapped_ratio(void)
7927 {
7928 pg_data_t *pgdat;
7929 struct zone *zone;
7930
7931 for_each_online_pgdat(pgdat)
7932 pgdat->min_unmapped_pages = 0;
7933
7934 for_each_zone(zone)
7935 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *
7936 sysctl_min_unmapped_ratio) / 100;
7937 }
7938
7939
7940 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
7941 void __user *buffer, size_t *length, loff_t *ppos)
7942 {
7943 int rc;
7944
7945 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7946 if (rc)
7947 return rc;
7948
7949 setup_min_unmapped_ratio();
7950
7951 return 0;
7952 }
7953
7954 static void setup_min_slab_ratio(void)
7955 {
7956 pg_data_t *pgdat;
7957 struct zone *zone;
7958
7959 for_each_online_pgdat(pgdat)
7960 pgdat->min_slab_pages = 0;
7961
7962 for_each_zone(zone)
7963 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *
7964 sysctl_min_slab_ratio) / 100;
7965 }
7966
7967 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
7968 void __user *buffer, size_t *length, loff_t *ppos)
7969 {
7970 int rc;
7971
7972 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7973 if (rc)
7974 return rc;
7975
7976 setup_min_slab_ratio();
7977
7978 return 0;
7979 }
7980 #endif
7981
7982
7983
7984
7985
7986
7987
7988
7989
7990
7991 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
7992 void __user *buffer, size_t *length, loff_t *ppos)
7993 {
7994 proc_dointvec_minmax(table, write, buffer, length, ppos);
7995 setup_per_zone_lowmem_reserve();
7996 return 0;
7997 }
7998
7999
8000
8001
8002
8003
8004 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
8005 void __user *buffer, size_t *length, loff_t *ppos)
8006 {
8007 struct zone *zone;
8008 int old_percpu_pagelist_fraction;
8009 int ret;
8010
8011 mutex_lock(&pcp_batch_high_lock);
8012 old_percpu_pagelist_fraction = percpu_pagelist_fraction;
8013
8014 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
8015 if (!write || ret < 0)
8016 goto out;
8017
8018
8019 if (percpu_pagelist_fraction &&
8020 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
8021 percpu_pagelist_fraction = old_percpu_pagelist_fraction;
8022 ret = -EINVAL;
8023 goto out;
8024 }
8025
8026
8027 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
8028 goto out;
8029
8030 for_each_populated_zone(zone) {
8031 unsigned int cpu;
8032
8033 for_each_possible_cpu(cpu)
8034 pageset_set_high_and_batch(zone,
8035 per_cpu_ptr(zone->pageset, cpu));
8036 }
8037 out:
8038 mutex_unlock(&pcp_batch_high_lock);
8039 return ret;
8040 }
8041
8042 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
8043
8044
8045
8046
8047 static unsigned long __init arch_reserved_kernel_pages(void)
8048 {
8049 return 0;
8050 }
8051 #endif
8052
8053
8054
8055
8056
8057
8058
8059
8060
8061
8062 #if __BITS_PER_LONG > 32
8063 #define ADAPT_SCALE_BASE (64ul << 30)
8064 #define ADAPT_SCALE_SHIFT 2
8065 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
8066 #endif
8067
8068
8069
8070
8071
8072
8073
8074 void *__init alloc_large_system_hash(const char *tablename,
8075 unsigned long bucketsize,
8076 unsigned long numentries,
8077 int scale,
8078 int flags,
8079 unsigned int *_hash_shift,
8080 unsigned int *_hash_mask,
8081 unsigned long low_limit,
8082 unsigned long high_limit)
8083 {
8084 unsigned long long max = high_limit;
8085 unsigned long log2qty, size;
8086 void *table = NULL;
8087 gfp_t gfp_flags;
8088 bool virt;
8089
8090
8091 if (!numentries) {
8092
8093 numentries = nr_kernel_pages;
8094 numentries -= arch_reserved_kernel_pages();
8095
8096
8097 if (PAGE_SHIFT < 20)
8098 numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
8099
8100 #if __BITS_PER_LONG > 32
8101 if (!high_limit) {
8102 unsigned long adapt;
8103
8104 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
8105 adapt <<= ADAPT_SCALE_SHIFT)
8106 scale++;
8107 }
8108 #endif
8109
8110
8111 if (scale > PAGE_SHIFT)
8112 numentries >>= (scale - PAGE_SHIFT);
8113 else
8114 numentries <<= (PAGE_SHIFT - scale);
8115
8116
8117 if (unlikely(flags & HASH_SMALL)) {
8118
8119 WARN_ON(!(flags & HASH_EARLY));
8120 if (!(numentries >> *_hash_shift)) {
8121 numentries = 1UL << *_hash_shift;
8122 BUG_ON(!numentries);
8123 }
8124 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
8125 numentries = PAGE_SIZE / bucketsize;
8126 }
8127 numentries = roundup_pow_of_two(numentries);
8128
8129
8130 if (max == 0) {
8131 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
8132 do_div(max, bucketsize);
8133 }
8134 max = min(max, 0x80000000ULL);
8135
8136 if (numentries < low_limit)
8137 numentries = low_limit;
8138 if (numentries > max)
8139 numentries = max;
8140
8141 log2qty = ilog2(numentries);
8142
8143 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
8144 do {
8145 virt = false;
8146 size = bucketsize << log2qty;
8147 if (flags & HASH_EARLY) {
8148 if (flags & HASH_ZERO)
8149 table = memblock_alloc(size, SMP_CACHE_BYTES);
8150 else
8151 table = memblock_alloc_raw(size,
8152 SMP_CACHE_BYTES);
8153 } else if (get_order(size) >= MAX_ORDER || hashdist) {
8154 table = __vmalloc(size, gfp_flags, PAGE_KERNEL);
8155 virt = true;
8156 } else {
8157
8158
8159
8160
8161
8162 table = alloc_pages_exact(size, gfp_flags);
8163 kmemleak_alloc(table, size, 1, gfp_flags);
8164 }
8165 } while (!table && size > PAGE_SIZE && --log2qty);
8166
8167 if (!table)
8168 panic("Failed to allocate %s hash table\n", tablename);
8169
8170 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
8171 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
8172 virt ? "vmalloc" : "linear");
8173
8174 if (_hash_shift)
8175 *_hash_shift = log2qty;
8176 if (_hash_mask)
8177 *_hash_mask = (1 << log2qty) - 1;
8178
8179 return table;
8180 }
8181
8182
8183
8184
8185
8186
8187
8188
8189
8190
8191 bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
8192 int migratetype, int flags)
8193 {
8194 unsigned long found;
8195 unsigned long iter = 0;
8196 unsigned long pfn = page_to_pfn(page);
8197 const char *reason = "unmovable page";
8198
8199
8200
8201
8202
8203
8204
8205
8206
8207 if (is_migrate_cma_page(page)) {
8208
8209
8210
8211
8212
8213 if (is_migrate_cma(migratetype))
8214 return false;
8215
8216 reason = "CMA page";
8217 goto unmovable;
8218 }
8219
8220 for (found = 0; iter < pageblock_nr_pages; iter++) {
8221 unsigned long check = pfn + iter;
8222
8223 if (!pfn_valid_within(check))
8224 continue;
8225
8226 page = pfn_to_page(check);
8227
8228 if (PageReserved(page))
8229 goto unmovable;
8230
8231
8232
8233
8234
8235
8236 if (zone_idx(zone) == ZONE_MOVABLE)
8237 continue;
8238
8239
8240
8241
8242
8243
8244 if (PageHuge(page)) {
8245 struct page *head = compound_head(page);
8246 unsigned int skip_pages;
8247
8248 if (!hugepage_migration_supported(page_hstate(head)))
8249 goto unmovable;
8250
8251 skip_pages = compound_nr(head) - (page - head);
8252 iter += skip_pages - 1;
8253 continue;
8254 }
8255
8256
8257
8258
8259
8260
8261
8262 if (!page_ref_count(page)) {
8263 if (PageBuddy(page))
8264 iter += (1 << page_order(page)) - 1;
8265 continue;
8266 }
8267
8268
8269
8270
8271
8272 if ((flags & SKIP_HWPOISON) && PageHWPoison(page))
8273 continue;
8274
8275 if (__PageMovable(page))
8276 continue;
8277
8278 if (!PageLRU(page))
8279 found++;
8280
8281
8282
8283
8284
8285
8286
8287
8288
8289
8290
8291
8292
8293 if (found > count)
8294 goto unmovable;
8295 }
8296 return false;
8297 unmovable:
8298 WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE);
8299 if (flags & REPORT_FAILURE)
8300 dump_page(pfn_to_page(pfn + iter), reason);
8301 return true;
8302 }
8303
8304 #ifdef CONFIG_CONTIG_ALLOC
8305 static unsigned long pfn_max_align_down(unsigned long pfn)
8306 {
8307 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
8308 pageblock_nr_pages) - 1);
8309 }
8310
8311 static unsigned long pfn_max_align_up(unsigned long pfn)
8312 {
8313 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
8314 pageblock_nr_pages));
8315 }
8316
8317
8318 static int __alloc_contig_migrate_range(struct compact_control *cc,
8319 unsigned long start, unsigned long end)
8320 {
8321
8322 unsigned long nr_reclaimed;
8323 unsigned long pfn = start;
8324 unsigned int tries = 0;
8325 int ret = 0;
8326
8327 migrate_prep();
8328
8329 while (pfn < end || !list_empty(&cc->migratepages)) {
8330 if (fatal_signal_pending(current)) {
8331 ret = -EINTR;
8332 break;
8333 }
8334
8335 if (list_empty(&cc->migratepages)) {
8336 cc->nr_migratepages = 0;
8337 pfn = isolate_migratepages_range(cc, pfn, end);
8338 if (!pfn) {
8339 ret = -EINTR;
8340 break;
8341 }
8342 tries = 0;
8343 } else if (++tries == 5) {
8344 ret = ret < 0 ? ret : -EBUSY;
8345 break;
8346 }
8347
8348 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
8349 &cc->migratepages);
8350 cc->nr_migratepages -= nr_reclaimed;
8351
8352 ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
8353 NULL, 0, cc->mode, MR_CONTIG_RANGE);
8354 }
8355 if (ret < 0) {
8356 putback_movable_pages(&cc->migratepages);
8357 return ret;
8358 }
8359 return 0;
8360 }
8361
8362
8363
8364
8365
8366
8367
8368
8369
8370
8371
8372
8373
8374
8375
8376
8377
8378
8379
8380
8381
8382
8383 int alloc_contig_range(unsigned long start, unsigned long end,
8384 unsigned migratetype, gfp_t gfp_mask)
8385 {
8386 unsigned long outer_start, outer_end;
8387 unsigned int order;
8388 int ret = 0;
8389
8390 struct compact_control cc = {
8391 .nr_migratepages = 0,
8392 .order = -1,
8393 .zone = page_zone(pfn_to_page(start)),
8394 .mode = MIGRATE_SYNC,
8395 .ignore_skip_hint = true,
8396 .no_set_skip_hint = true,
8397 .gfp_mask = current_gfp_context(gfp_mask),
8398 };
8399 INIT_LIST_HEAD(&cc.migratepages);
8400
8401
8402
8403
8404
8405
8406
8407
8408
8409
8410
8411
8412
8413
8414
8415
8416
8417
8418
8419
8420
8421
8422
8423
8424
8425 ret = start_isolate_page_range(pfn_max_align_down(start),
8426 pfn_max_align_up(end), migratetype, 0);
8427 if (ret < 0)
8428 return ret;
8429
8430
8431
8432
8433
8434
8435
8436
8437
8438
8439
8440 ret = __alloc_contig_migrate_range(&cc, start, end);
8441 if (ret && ret != -EBUSY)
8442 goto done;
8443 ret =0;
8444
8445
8446
8447
8448
8449
8450
8451
8452
8453
8454
8455
8456
8457
8458
8459
8460
8461
8462 lru_add_drain_all();
8463
8464 order = 0;
8465 outer_start = start;
8466 while (!PageBuddy(pfn_to_page(outer_start))) {
8467 if (++order >= MAX_ORDER) {
8468 outer_start = start;
8469 break;
8470 }
8471 outer_start &= ~0UL << order;
8472 }
8473
8474 if (outer_start != start) {
8475 order = page_order(pfn_to_page(outer_start));
8476
8477
8478
8479
8480
8481
8482
8483 if (outer_start + (1UL << order) <= start)
8484 outer_start = start;
8485 }
8486
8487
8488 if (test_pages_isolated(outer_start, end, false)) {
8489 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
8490 __func__, outer_start, end);
8491 ret = -EBUSY;
8492 goto done;
8493 }
8494
8495
8496 outer_end = isolate_freepages_range(&cc, outer_start, end);
8497 if (!outer_end) {
8498 ret = -EBUSY;
8499 goto done;
8500 }
8501
8502
8503 if (start != outer_start)
8504 free_contig_range(outer_start, start - outer_start);
8505 if (end != outer_end)
8506 free_contig_range(end, outer_end - end);
8507
8508 done:
8509 undo_isolate_page_range(pfn_max_align_down(start),
8510 pfn_max_align_up(end), migratetype);
8511 return ret;
8512 }
8513 #endif
8514
8515 void free_contig_range(unsigned long pfn, unsigned int nr_pages)
8516 {
8517 unsigned int count = 0;
8518
8519 for (; nr_pages--; pfn++) {
8520 struct page *page = pfn_to_page(pfn);
8521
8522 count += page_count(page) != 1;
8523 __free_page(page);
8524 }
8525 WARN(count != 0, "%d pages are still in use!\n", count);
8526 }
8527
8528
8529
8530
8531
8532 void __meminit zone_pcp_update(struct zone *zone)
8533 {
8534 unsigned cpu;
8535 mutex_lock(&pcp_batch_high_lock);
8536 for_each_possible_cpu(cpu)
8537 pageset_set_high_and_batch(zone,
8538 per_cpu_ptr(zone->pageset, cpu));
8539 mutex_unlock(&pcp_batch_high_lock);
8540 }
8541
8542 void zone_pcp_reset(struct zone *zone)
8543 {
8544 unsigned long flags;
8545 int cpu;
8546 struct per_cpu_pageset *pset;
8547
8548
8549 local_irq_save(flags);
8550 if (zone->pageset != &boot_pageset) {
8551 for_each_online_cpu(cpu) {
8552 pset = per_cpu_ptr(zone->pageset, cpu);
8553 drain_zonestat(zone, pset);
8554 }
8555 free_percpu(zone->pageset);
8556 zone->pageset = &boot_pageset;
8557 }
8558 local_irq_restore(flags);
8559 }
8560
8561 #ifdef CONFIG_MEMORY_HOTREMOVE
8562
8563
8564
8565
8566 unsigned long
8567 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
8568 {
8569 struct page *page;
8570 struct zone *zone;
8571 unsigned int order, i;
8572 unsigned long pfn;
8573 unsigned long flags;
8574 unsigned long offlined_pages = 0;
8575
8576
8577 for (pfn = start_pfn; pfn < end_pfn; pfn++)
8578 if (pfn_valid(pfn))
8579 break;
8580 if (pfn == end_pfn)
8581 return offlined_pages;
8582
8583 offline_mem_sections(pfn, end_pfn);
8584 zone = page_zone(pfn_to_page(pfn));
8585 spin_lock_irqsave(&zone->lock, flags);
8586 pfn = start_pfn;
8587 while (pfn < end_pfn) {
8588 if (!pfn_valid(pfn)) {
8589 pfn++;
8590 continue;
8591 }
8592 page = pfn_to_page(pfn);
8593
8594
8595
8596
8597 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
8598 pfn++;
8599 SetPageReserved(page);
8600 offlined_pages++;
8601 continue;
8602 }
8603
8604 BUG_ON(page_count(page));
8605 BUG_ON(!PageBuddy(page));
8606 order = page_order(page);
8607 offlined_pages += 1 << order;
8608 #ifdef CONFIG_DEBUG_VM
8609 pr_info("remove from free list %lx %d %lx\n",
8610 pfn, 1 << order, end_pfn);
8611 #endif
8612 del_page_from_free_area(page, &zone->free_area[order]);
8613 for (i = 0; i < (1 << order); i++)
8614 SetPageReserved((page+i));
8615 pfn += (1 << order);
8616 }
8617 spin_unlock_irqrestore(&zone->lock, flags);
8618
8619 return offlined_pages;
8620 }
8621 #endif
8622
8623 bool is_free_buddy_page(struct page *page)
8624 {
8625 struct zone *zone = page_zone(page);
8626 unsigned long pfn = page_to_pfn(page);
8627 unsigned long flags;
8628 unsigned int order;
8629
8630 spin_lock_irqsave(&zone->lock, flags);
8631 for (order = 0; order < MAX_ORDER; order++) {
8632 struct page *page_head = page - (pfn & ((1 << order) - 1));
8633
8634 if (PageBuddy(page_head) && page_order(page_head) >= order)
8635 break;
8636 }
8637 spin_unlock_irqrestore(&zone->lock, flags);
8638
8639 return order < MAX_ORDER;
8640 }
8641
8642 #ifdef CONFIG_MEMORY_FAILURE
8643
8644
8645
8646
8647
8648 bool set_hwpoison_free_buddy_page(struct page *page)
8649 {
8650 struct zone *zone = page_zone(page);
8651 unsigned long pfn = page_to_pfn(page);
8652 unsigned long flags;
8653 unsigned int order;
8654 bool hwpoisoned = false;
8655
8656 spin_lock_irqsave(&zone->lock, flags);
8657 for (order = 0; order < MAX_ORDER; order++) {
8658 struct page *page_head = page - (pfn & ((1 << order) - 1));
8659
8660 if (PageBuddy(page_head) && page_order(page_head) >= order) {
8661 if (!TestSetPageHWPoison(page))
8662 hwpoisoned = true;
8663 break;
8664 }
8665 }
8666 spin_unlock_irqrestore(&zone->lock, flags);
8667
8668 return hwpoisoned;
8669 }
8670 #endif