1/* 2 * bootmem - A boot-time physical memory allocator and configurator 3 * 4 * Copyright (C) 1999 Ingo Molnar 5 * 1999 Kanoj Sarcar, SGI 6 * 2008 Johannes Weiner 7 * 8 * Access to this subsystem has to be serialized externally (which is true 9 * for the boot process anyway). 10 */ 11#include <linux/init.h> 12#include <linux/pfn.h> 13#include <linux/slab.h> 14#include <linux/bootmem.h> 15#include <linux/export.h> 16#include <linux/kmemleak.h> 17#include <linux/range.h> 18#include <linux/memblock.h> 19 20#include <asm/bug.h> 21#include <asm/io.h> 22#include <asm/processor.h> 23 24#include "internal.h" 25 26#ifndef CONFIG_NEED_MULTIPLE_NODES 27struct pglist_data __refdata contig_page_data; 28EXPORT_SYMBOL(contig_page_data); 29#endif 30 31unsigned long max_low_pfn; 32unsigned long min_low_pfn; 33unsigned long max_pfn; 34 35static void * __init __alloc_memory_core_early(int nid, u64 size, u64 align, 36 u64 goal, u64 limit) 37{ 38 void *ptr; 39 u64 addr; 40 41 if (limit > memblock.current_limit) 42 limit = memblock.current_limit; 43 44 addr = memblock_find_in_range_node(size, align, goal, limit, nid); 45 if (!addr) 46 return NULL; 47 48 if (memblock_reserve(addr, size)) 49 return NULL; 50 51 ptr = phys_to_virt(addr); 52 memset(ptr, 0, size); 53 /* 54 * The min_count is set to 0 so that bootmem allocated blocks 55 * are never reported as leaks. 56 */ 57 kmemleak_alloc(ptr, size, 0, 0); 58 return ptr; 59} 60 61/* 62 * free_bootmem_late - free bootmem pages directly to page allocator 63 * @addr: starting address of the range 64 * @size: size of the range in bytes 65 * 66 * This is only useful when the bootmem allocator has already been torn 67 * down, but we are still initializing the system. Pages are given directly 68 * to the page allocator, no bootmem metadata is updated because it is gone. 69 */ 70void __init free_bootmem_late(unsigned long addr, unsigned long size) 71{ 72 unsigned long cursor, end; 73 74 kmemleak_free_part(__va(addr), size); 75 76 cursor = PFN_UP(addr); 77 end = PFN_DOWN(addr + size); 78 79 for (; cursor < end; cursor++) { 80 __free_pages_bootmem(pfn_to_page(cursor), cursor, 0); 81 totalram_pages++; 82 } 83} 84 85static void __init __free_pages_memory(unsigned long start, unsigned long end) 86{ 87 int order; 88 89 while (start < end) { 90 order = min(MAX_ORDER - 1UL, __ffs(start)); 91 92 while (start + (1UL << order) > end) 93 order--; 94 95 __free_pages_bootmem(pfn_to_page(start), start, order); 96 97 start += (1UL << order); 98 } 99} 100 101static unsigned long __init __free_memory_core(phys_addr_t start, 102 phys_addr_t end) 103{ 104 unsigned long start_pfn = PFN_UP(start); 105 unsigned long end_pfn = min_t(unsigned long, 106 PFN_DOWN(end), max_low_pfn); 107 108 if (start_pfn > end_pfn) 109 return 0; 110 111 __free_pages_memory(start_pfn, end_pfn); 112 113 return end_pfn - start_pfn; 114} 115 116static unsigned long __init free_low_memory_core_early(void) 117{ 118 unsigned long count = 0; 119 phys_addr_t start, end; 120 u64 i; 121 122 memblock_clear_hotplug(0, -1); 123 124 for_each_free_mem_range(i, NUMA_NO_NODE, &start, &end, NULL) 125 count += __free_memory_core(start, end); 126 127#ifdef CONFIG_ARCH_DISCARD_MEMBLOCK 128 { 129 phys_addr_t size; 130 131 /* Free memblock.reserved array if it was allocated */ 132 size = get_allocated_memblock_reserved_regions_info(&start); 133 if (size) 134 count += __free_memory_core(start, start + size); 135 136 /* Free memblock.memory array if it was allocated */ 137 size = get_allocated_memblock_memory_regions_info(&start); 138 if (size) 139 count += __free_memory_core(start, start + size); 140 } 141#endif 142 143 return count; 144} 145 146static int reset_managed_pages_done __initdata; 147 148void reset_node_managed_pages(pg_data_t *pgdat) 149{ 150 struct zone *z; 151 152 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 153 z->managed_pages = 0; 154} 155 156void __init reset_all_zones_managed_pages(void) 157{ 158 struct pglist_data *pgdat; 159 160 if (reset_managed_pages_done) 161 return; 162 163 for_each_online_pgdat(pgdat) 164 reset_node_managed_pages(pgdat); 165 166 reset_managed_pages_done = 1; 167} 168 169/** 170 * free_all_bootmem - release free pages to the buddy allocator 171 * 172 * Returns the number of pages actually released. 173 */ 174unsigned long __init free_all_bootmem(void) 175{ 176 unsigned long pages; 177 178 reset_all_zones_managed_pages(); 179 180 /* 181 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id 182 * because in some case like Node0 doesn't have RAM installed 183 * low ram will be on Node1 184 */ 185 pages = free_low_memory_core_early(); 186 totalram_pages += pages; 187 188 return pages; 189} 190 191/** 192 * free_bootmem_node - mark a page range as usable 193 * @pgdat: node the range resides on 194 * @physaddr: starting address of the range 195 * @size: size of the range in bytes 196 * 197 * Partial pages will be considered reserved and left as they are. 198 * 199 * The range must reside completely on the specified node. 200 */ 201void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 202 unsigned long size) 203{ 204 memblock_free(physaddr, size); 205} 206 207/** 208 * free_bootmem - mark a page range as usable 209 * @addr: starting address of the range 210 * @size: size of the range in bytes 211 * 212 * Partial pages will be considered reserved and left as they are. 213 * 214 * The range must be contiguous but may span node boundaries. 215 */ 216void __init free_bootmem(unsigned long addr, unsigned long size) 217{ 218 memblock_free(addr, size); 219} 220 221static void * __init ___alloc_bootmem_nopanic(unsigned long size, 222 unsigned long align, 223 unsigned long goal, 224 unsigned long limit) 225{ 226 void *ptr; 227 228 if (WARN_ON_ONCE(slab_is_available())) 229 return kzalloc(size, GFP_NOWAIT); 230 231restart: 232 233 ptr = __alloc_memory_core_early(NUMA_NO_NODE, size, align, goal, limit); 234 235 if (ptr) 236 return ptr; 237 238 if (goal != 0) { 239 goal = 0; 240 goto restart; 241 } 242 243 return NULL; 244} 245 246/** 247 * __alloc_bootmem_nopanic - allocate boot memory without panicking 248 * @size: size of the request in bytes 249 * @align: alignment of the region 250 * @goal: preferred starting address of the region 251 * 252 * The goal is dropped if it can not be satisfied and the allocation will 253 * fall back to memory below @goal. 254 * 255 * Allocation may happen on any node in the system. 256 * 257 * Returns NULL on failure. 258 */ 259void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, 260 unsigned long goal) 261{ 262 unsigned long limit = -1UL; 263 264 return ___alloc_bootmem_nopanic(size, align, goal, limit); 265} 266 267static void * __init ___alloc_bootmem(unsigned long size, unsigned long align, 268 unsigned long goal, unsigned long limit) 269{ 270 void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit); 271 272 if (mem) 273 return mem; 274 /* 275 * Whoops, we cannot satisfy the allocation request. 276 */ 277 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size); 278 panic("Out of memory"); 279 return NULL; 280} 281 282/** 283 * __alloc_bootmem - allocate boot memory 284 * @size: size of the request in bytes 285 * @align: alignment of the region 286 * @goal: preferred starting address of the region 287 * 288 * The goal is dropped if it can not be satisfied and the allocation will 289 * fall back to memory below @goal. 290 * 291 * Allocation may happen on any node in the system. 292 * 293 * The function panics if the request can not be satisfied. 294 */ 295void * __init __alloc_bootmem(unsigned long size, unsigned long align, 296 unsigned long goal) 297{ 298 unsigned long limit = -1UL; 299 300 return ___alloc_bootmem(size, align, goal, limit); 301} 302 303void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat, 304 unsigned long size, 305 unsigned long align, 306 unsigned long goal, 307 unsigned long limit) 308{ 309 void *ptr; 310 311again: 312 ptr = __alloc_memory_core_early(pgdat->node_id, size, align, 313 goal, limit); 314 if (ptr) 315 return ptr; 316 317 ptr = __alloc_memory_core_early(NUMA_NO_NODE, size, align, 318 goal, limit); 319 if (ptr) 320 return ptr; 321 322 if (goal) { 323 goal = 0; 324 goto again; 325 } 326 327 return NULL; 328} 329 330void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size, 331 unsigned long align, unsigned long goal) 332{ 333 if (WARN_ON_ONCE(slab_is_available())) 334 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 335 336 return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0); 337} 338 339static void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, 340 unsigned long align, unsigned long goal, 341 unsigned long limit) 342{ 343 void *ptr; 344 345 ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, limit); 346 if (ptr) 347 return ptr; 348 349 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size); 350 panic("Out of memory"); 351 return NULL; 352} 353 354/** 355 * __alloc_bootmem_node - allocate boot memory from a specific node 356 * @pgdat: node to allocate from 357 * @size: size of the request in bytes 358 * @align: alignment of the region 359 * @goal: preferred starting address of the region 360 * 361 * The goal is dropped if it can not be satisfied and the allocation will 362 * fall back to memory below @goal. 363 * 364 * Allocation may fall back to any node in the system if the specified node 365 * can not hold the requested memory. 366 * 367 * The function panics if the request can not be satisfied. 368 */ 369void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, 370 unsigned long align, unsigned long goal) 371{ 372 if (WARN_ON_ONCE(slab_is_available())) 373 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 374 375 return ___alloc_bootmem_node(pgdat, size, align, goal, 0); 376} 377 378void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size, 379 unsigned long align, unsigned long goal) 380{ 381 return __alloc_bootmem_node(pgdat, size, align, goal); 382} 383 384#ifndef ARCH_LOW_ADDRESS_LIMIT 385#define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL 386#endif 387 388/** 389 * __alloc_bootmem_low - allocate low boot memory 390 * @size: size of the request in bytes 391 * @align: alignment of the region 392 * @goal: preferred starting address of the region 393 * 394 * The goal is dropped if it can not be satisfied and the allocation will 395 * fall back to memory below @goal. 396 * 397 * Allocation may happen on any node in the system. 398 * 399 * The function panics if the request can not be satisfied. 400 */ 401void * __init __alloc_bootmem_low(unsigned long size, unsigned long align, 402 unsigned long goal) 403{ 404 return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT); 405} 406 407void * __init __alloc_bootmem_low_nopanic(unsigned long size, 408 unsigned long align, 409 unsigned long goal) 410{ 411 return ___alloc_bootmem_nopanic(size, align, goal, 412 ARCH_LOW_ADDRESS_LIMIT); 413} 414 415/** 416 * __alloc_bootmem_low_node - allocate low boot memory from a specific node 417 * @pgdat: node to allocate from 418 * @size: size of the request in bytes 419 * @align: alignment of the region 420 * @goal: preferred starting address of the region 421 * 422 * The goal is dropped if it can not be satisfied and the allocation will 423 * fall back to memory below @goal. 424 * 425 * Allocation may fall back to any node in the system if the specified node 426 * can not hold the requested memory. 427 * 428 * The function panics if the request can not be satisfied. 429 */ 430void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, 431 unsigned long align, unsigned long goal) 432{ 433 if (WARN_ON_ONCE(slab_is_available())) 434 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 435 436 return ___alloc_bootmem_node(pgdat, size, align, goal, 437 ARCH_LOW_ADDRESS_LIMIT); 438} 439