1/*
2 *  linux/arch/x86_64/mm/init.c
3 *
4 *  Copyright (C) 1995  Linus Torvalds
5 *  Copyright (C) 2000  Pavel Machek <pavel@ucw.cz>
6 *  Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
7 */
8
9#include <linux/signal.h>
10#include <linux/sched.h>
11#include <linux/kernel.h>
12#include <linux/errno.h>
13#include <linux/string.h>
14#include <linux/types.h>
15#include <linux/ptrace.h>
16#include <linux/mman.h>
17#include <linux/mm.h>
18#include <linux/swap.h>
19#include <linux/smp.h>
20#include <linux/init.h>
21#include <linux/initrd.h>
22#include <linux/pagemap.h>
23#include <linux/bootmem.h>
24#include <linux/memblock.h>
25#include <linux/proc_fs.h>
26#include <linux/pci.h>
27#include <linux/pfn.h>
28#include <linux/poison.h>
29#include <linux/dma-mapping.h>
30#include <linux/module.h>
31#include <linux/memory.h>
32#include <linux/memory_hotplug.h>
33#include <linux/nmi.h>
34#include <linux/gfp.h>
35#include <linux/kcore.h>
36
37#include <asm/processor.h>
38#include <asm/bios_ebda.h>
39#include <asm/uaccess.h>
40#include <asm/pgtable.h>
41#include <asm/pgalloc.h>
42#include <asm/dma.h>
43#include <asm/fixmap.h>
44#include <asm/e820.h>
45#include <asm/apic.h>
46#include <asm/tlb.h>
47#include <asm/mmu_context.h>
48#include <asm/proto.h>
49#include <asm/smp.h>
50#include <asm/sections.h>
51#include <asm/kdebug.h>
52#include <asm/numa.h>
53#include <asm/cacheflush.h>
54#include <asm/init.h>
55#include <asm/setup.h>
56
57#include "mm_internal.h"
58
59static void ident_pmd_init(unsigned long pmd_flag, pmd_t *pmd_page,
60			   unsigned long addr, unsigned long end)
61{
62	addr &= PMD_MASK;
63	for (; addr < end; addr += PMD_SIZE) {
64		pmd_t *pmd = pmd_page + pmd_index(addr);
65
66		if (!pmd_present(*pmd))
67			set_pmd(pmd, __pmd(addr | pmd_flag));
68	}
69}
70static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
71			  unsigned long addr, unsigned long end)
72{
73	unsigned long next;
74
75	for (; addr < end; addr = next) {
76		pud_t *pud = pud_page + pud_index(addr);
77		pmd_t *pmd;
78
79		next = (addr & PUD_MASK) + PUD_SIZE;
80		if (next > end)
81			next = end;
82
83		if (pud_present(*pud)) {
84			pmd = pmd_offset(pud, 0);
85			ident_pmd_init(info->pmd_flag, pmd, addr, next);
86			continue;
87		}
88		pmd = (pmd_t *)info->alloc_pgt_page(info->context);
89		if (!pmd)
90			return -ENOMEM;
91		ident_pmd_init(info->pmd_flag, pmd, addr, next);
92		set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
93	}
94
95	return 0;
96}
97
98int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
99			      unsigned long addr, unsigned long end)
100{
101	unsigned long next;
102	int result;
103	int off = info->kernel_mapping ? pgd_index(__PAGE_OFFSET) : 0;
104
105	for (; addr < end; addr = next) {
106		pgd_t *pgd = pgd_page + pgd_index(addr) + off;
107		pud_t *pud;
108
109		next = (addr & PGDIR_MASK) + PGDIR_SIZE;
110		if (next > end)
111			next = end;
112
113		if (pgd_present(*pgd)) {
114			pud = pud_offset(pgd, 0);
115			result = ident_pud_init(info, pud, addr, next);
116			if (result)
117				return result;
118			continue;
119		}
120
121		pud = (pud_t *)info->alloc_pgt_page(info->context);
122		if (!pud)
123			return -ENOMEM;
124		result = ident_pud_init(info, pud, addr, next);
125		if (result)
126			return result;
127		set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
128	}
129
130	return 0;
131}
132
133/*
134 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
135 * physical space so we can cache the place of the first one and move
136 * around without checking the pgd every time.
137 */
138
139pteval_t __supported_pte_mask __read_mostly = ~0;
140EXPORT_SYMBOL_GPL(__supported_pte_mask);
141
142int force_personality32;
143
144/*
145 * noexec32=on|off
146 * Control non executable heap for 32bit processes.
147 * To control the stack too use noexec=off
148 *
149 * on	PROT_READ does not imply PROT_EXEC for 32-bit processes (default)
150 * off	PROT_READ implies PROT_EXEC
151 */
152static int __init nonx32_setup(char *str)
153{
154	if (!strcmp(str, "on"))
155		force_personality32 &= ~READ_IMPLIES_EXEC;
156	else if (!strcmp(str, "off"))
157		force_personality32 |= READ_IMPLIES_EXEC;
158	return 1;
159}
160__setup("noexec32=", nonx32_setup);
161
162/*
163 * When memory was added/removed make sure all the processes MM have
164 * suitable PGD entries in the local PGD level page.
165 */
166void sync_global_pgds(unsigned long start, unsigned long end, int removed)
167{
168	unsigned long address;
169
170	for (address = start; address <= end; address += PGDIR_SIZE) {
171		const pgd_t *pgd_ref = pgd_offset_k(address);
172		struct page *page;
173
174		/*
175		 * When it is called after memory hot remove, pgd_none()
176		 * returns true. In this case (removed == 1), we must clear
177		 * the PGD entries in the local PGD level page.
178		 */
179		if (pgd_none(*pgd_ref) && !removed)
180			continue;
181
182		spin_lock(&pgd_lock);
183		list_for_each_entry(page, &pgd_list, lru) {
184			pgd_t *pgd;
185			spinlock_t *pgt_lock;
186
187			pgd = (pgd_t *)page_address(page) + pgd_index(address);
188			/* the pgt_lock only for Xen */
189			pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
190			spin_lock(pgt_lock);
191
192			if (!pgd_none(*pgd_ref) && !pgd_none(*pgd))
193				BUG_ON(pgd_page_vaddr(*pgd)
194				       != pgd_page_vaddr(*pgd_ref));
195
196			if (removed) {
197				if (pgd_none(*pgd_ref) && !pgd_none(*pgd))
198					pgd_clear(pgd);
199			} else {
200				if (pgd_none(*pgd))
201					set_pgd(pgd, *pgd_ref);
202			}
203
204			spin_unlock(pgt_lock);
205		}
206		spin_unlock(&pgd_lock);
207	}
208}
209
210/*
211 * NOTE: This function is marked __ref because it calls __init function
212 * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0.
213 */
214static __ref void *spp_getpage(void)
215{
216	void *ptr;
217
218	if (after_bootmem)
219		ptr = (void *) get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK);
220	else
221		ptr = alloc_bootmem_pages(PAGE_SIZE);
222
223	if (!ptr || ((unsigned long)ptr & ~PAGE_MASK)) {
224		panic("set_pte_phys: cannot allocate page data %s\n",
225			after_bootmem ? "after bootmem" : "");
226	}
227
228	pr_debug("spp_getpage %p\n", ptr);
229
230	return ptr;
231}
232
233static pud_t *fill_pud(pgd_t *pgd, unsigned long vaddr)
234{
235	if (pgd_none(*pgd)) {
236		pud_t *pud = (pud_t *)spp_getpage();
237		pgd_populate(&init_mm, pgd, pud);
238		if (pud != pud_offset(pgd, 0))
239			printk(KERN_ERR "PAGETABLE BUG #00! %p <-> %p\n",
240			       pud, pud_offset(pgd, 0));
241	}
242	return pud_offset(pgd, vaddr);
243}
244
245static pmd_t *fill_pmd(pud_t *pud, unsigned long vaddr)
246{
247	if (pud_none(*pud)) {
248		pmd_t *pmd = (pmd_t *) spp_getpage();
249		pud_populate(&init_mm, pud, pmd);
250		if (pmd != pmd_offset(pud, 0))
251			printk(KERN_ERR "PAGETABLE BUG #01! %p <-> %p\n",
252			       pmd, pmd_offset(pud, 0));
253	}
254	return pmd_offset(pud, vaddr);
255}
256
257static pte_t *fill_pte(pmd_t *pmd, unsigned long vaddr)
258{
259	if (pmd_none(*pmd)) {
260		pte_t *pte = (pte_t *) spp_getpage();
261		pmd_populate_kernel(&init_mm, pmd, pte);
262		if (pte != pte_offset_kernel(pmd, 0))
263			printk(KERN_ERR "PAGETABLE BUG #02!\n");
264	}
265	return pte_offset_kernel(pmd, vaddr);
266}
267
268void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte)
269{
270	pud_t *pud;
271	pmd_t *pmd;
272	pte_t *pte;
273
274	pud = pud_page + pud_index(vaddr);
275	pmd = fill_pmd(pud, vaddr);
276	pte = fill_pte(pmd, vaddr);
277
278	set_pte(pte, new_pte);
279
280	/*
281	 * It's enough to flush this one mapping.
282	 * (PGE mappings get flushed as well)
283	 */
284	__flush_tlb_one(vaddr);
285}
286
287void set_pte_vaddr(unsigned long vaddr, pte_t pteval)
288{
289	pgd_t *pgd;
290	pud_t *pud_page;
291
292	pr_debug("set_pte_vaddr %lx to %lx\n", vaddr, native_pte_val(pteval));
293
294	pgd = pgd_offset_k(vaddr);
295	if (pgd_none(*pgd)) {
296		printk(KERN_ERR
297			"PGD FIXMAP MISSING, it should be setup in head.S!\n");
298		return;
299	}
300	pud_page = (pud_t*)pgd_page_vaddr(*pgd);
301	set_pte_vaddr_pud(pud_page, vaddr, pteval);
302}
303
304pmd_t * __init populate_extra_pmd(unsigned long vaddr)
305{
306	pgd_t *pgd;
307	pud_t *pud;
308
309	pgd = pgd_offset_k(vaddr);
310	pud = fill_pud(pgd, vaddr);
311	return fill_pmd(pud, vaddr);
312}
313
314pte_t * __init populate_extra_pte(unsigned long vaddr)
315{
316	pmd_t *pmd;
317
318	pmd = populate_extra_pmd(vaddr);
319	return fill_pte(pmd, vaddr);
320}
321
322/*
323 * Create large page table mappings for a range of physical addresses.
324 */
325static void __init __init_extra_mapping(unsigned long phys, unsigned long size,
326					enum page_cache_mode cache)
327{
328	pgd_t *pgd;
329	pud_t *pud;
330	pmd_t *pmd;
331	pgprot_t prot;
332
333	pgprot_val(prot) = pgprot_val(PAGE_KERNEL_LARGE) |
334		pgprot_val(pgprot_4k_2_large(cachemode2pgprot(cache)));
335	BUG_ON((phys & ~PMD_MASK) || (size & ~PMD_MASK));
336	for (; size; phys += PMD_SIZE, size -= PMD_SIZE) {
337		pgd = pgd_offset_k((unsigned long)__va(phys));
338		if (pgd_none(*pgd)) {
339			pud = (pud_t *) spp_getpage();
340			set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE |
341						_PAGE_USER));
342		}
343		pud = pud_offset(pgd, (unsigned long)__va(phys));
344		if (pud_none(*pud)) {
345			pmd = (pmd_t *) spp_getpage();
346			set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE |
347						_PAGE_USER));
348		}
349		pmd = pmd_offset(pud, phys);
350		BUG_ON(!pmd_none(*pmd));
351		set_pmd(pmd, __pmd(phys | pgprot_val(prot)));
352	}
353}
354
355void __init init_extra_mapping_wb(unsigned long phys, unsigned long size)
356{
357	__init_extra_mapping(phys, size, _PAGE_CACHE_MODE_WB);
358}
359
360void __init init_extra_mapping_uc(unsigned long phys, unsigned long size)
361{
362	__init_extra_mapping(phys, size, _PAGE_CACHE_MODE_UC);
363}
364
365/*
366 * The head.S code sets up the kernel high mapping:
367 *
368 *   from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text)
369 *
370 * phys_base holds the negative offset to the kernel, which is added
371 * to the compile time generated pmds. This results in invalid pmds up
372 * to the point where we hit the physaddr 0 mapping.
373 *
374 * We limit the mappings to the region from _text to _brk_end.  _brk_end
375 * is rounded up to the 2MB boundary. This catches the invalid pmds as
376 * well, as they are located before _text:
377 */
378void __init cleanup_highmap(void)
379{
380	unsigned long vaddr = __START_KERNEL_map;
381	unsigned long vaddr_end = __START_KERNEL_map + KERNEL_IMAGE_SIZE;
382	unsigned long end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1;
383	pmd_t *pmd = level2_kernel_pgt;
384
385	/*
386	 * Native path, max_pfn_mapped is not set yet.
387	 * Xen has valid max_pfn_mapped set in
388	 *	arch/x86/xen/mmu.c:xen_setup_kernel_pagetable().
389	 */
390	if (max_pfn_mapped)
391		vaddr_end = __START_KERNEL_map + (max_pfn_mapped << PAGE_SHIFT);
392
393	for (; vaddr + PMD_SIZE - 1 < vaddr_end; pmd++, vaddr += PMD_SIZE) {
394		if (pmd_none(*pmd))
395			continue;
396		if (vaddr < (unsigned long) _text || vaddr > end)
397			set_pmd(pmd, __pmd(0));
398	}
399}
400
401static unsigned long __meminit
402phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end,
403	      pgprot_t prot)
404{
405	unsigned long pages = 0, next;
406	unsigned long last_map_addr = end;
407	int i;
408
409	pte_t *pte = pte_page + pte_index(addr);
410
411	for (i = pte_index(addr); i < PTRS_PER_PTE; i++, addr = next, pte++) {
412		next = (addr & PAGE_MASK) + PAGE_SIZE;
413		if (addr >= end) {
414			if (!after_bootmem &&
415			    !e820_any_mapped(addr & PAGE_MASK, next, E820_RAM) &&
416			    !e820_any_mapped(addr & PAGE_MASK, next, E820_RESERVED_KERN))
417				set_pte(pte, __pte(0));
418			continue;
419		}
420
421		/*
422		 * We will re-use the existing mapping.
423		 * Xen for example has some special requirements, like mapping
424		 * pagetable pages as RO. So assume someone who pre-setup
425		 * these mappings are more intelligent.
426		 */
427		if (pte_val(*pte)) {
428			if (!after_bootmem)
429				pages++;
430			continue;
431		}
432
433		if (0)
434			printk("   pte=%p addr=%lx pte=%016lx\n",
435			       pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte);
436		pages++;
437		set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, prot));
438		last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE;
439	}
440
441	update_page_count(PG_LEVEL_4K, pages);
442
443	return last_map_addr;
444}
445
446static unsigned long __meminit
447phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
448	      unsigned long page_size_mask, pgprot_t prot)
449{
450	unsigned long pages = 0, next;
451	unsigned long last_map_addr = end;
452
453	int i = pmd_index(address);
454
455	for (; i < PTRS_PER_PMD; i++, address = next) {
456		pmd_t *pmd = pmd_page + pmd_index(address);
457		pte_t *pte;
458		pgprot_t new_prot = prot;
459
460		next = (address & PMD_MASK) + PMD_SIZE;
461		if (address >= end) {
462			if (!after_bootmem &&
463			    !e820_any_mapped(address & PMD_MASK, next, E820_RAM) &&
464			    !e820_any_mapped(address & PMD_MASK, next, E820_RESERVED_KERN))
465				set_pmd(pmd, __pmd(0));
466			continue;
467		}
468
469		if (pmd_val(*pmd)) {
470			if (!pmd_large(*pmd)) {
471				spin_lock(&init_mm.page_table_lock);
472				pte = (pte_t *)pmd_page_vaddr(*pmd);
473				last_map_addr = phys_pte_init(pte, address,
474								end, prot);
475				spin_unlock(&init_mm.page_table_lock);
476				continue;
477			}
478			/*
479			 * If we are ok with PG_LEVEL_2M mapping, then we will
480			 * use the existing mapping,
481			 *
482			 * Otherwise, we will split the large page mapping but
483			 * use the same existing protection bits except for
484			 * large page, so that we don't violate Intel's TLB
485			 * Application note (317080) which says, while changing
486			 * the page sizes, new and old translations should
487			 * not differ with respect to page frame and
488			 * attributes.
489			 */
490			if (page_size_mask & (1 << PG_LEVEL_2M)) {
491				if (!after_bootmem)
492					pages++;
493				last_map_addr = next;
494				continue;
495			}
496			new_prot = pte_pgprot(pte_clrhuge(*(pte_t *)pmd));
497		}
498
499		if (page_size_mask & (1<<PG_LEVEL_2M)) {
500			pages++;
501			spin_lock(&init_mm.page_table_lock);
502			set_pte((pte_t *)pmd,
503				pfn_pte((address & PMD_MASK) >> PAGE_SHIFT,
504					__pgprot(pgprot_val(prot) | _PAGE_PSE)));
505			spin_unlock(&init_mm.page_table_lock);
506			last_map_addr = next;
507			continue;
508		}
509
510		pte = alloc_low_page();
511		last_map_addr = phys_pte_init(pte, address, end, new_prot);
512
513		spin_lock(&init_mm.page_table_lock);
514		pmd_populate_kernel(&init_mm, pmd, pte);
515		spin_unlock(&init_mm.page_table_lock);
516	}
517	update_page_count(PG_LEVEL_2M, pages);
518	return last_map_addr;
519}
520
521static unsigned long __meminit
522phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
523			 unsigned long page_size_mask)
524{
525	unsigned long pages = 0, next;
526	unsigned long last_map_addr = end;
527	int i = pud_index(addr);
528
529	for (; i < PTRS_PER_PUD; i++, addr = next) {
530		pud_t *pud = pud_page + pud_index(addr);
531		pmd_t *pmd;
532		pgprot_t prot = PAGE_KERNEL;
533
534		next = (addr & PUD_MASK) + PUD_SIZE;
535		if (addr >= end) {
536			if (!after_bootmem &&
537			    !e820_any_mapped(addr & PUD_MASK, next, E820_RAM) &&
538			    !e820_any_mapped(addr & PUD_MASK, next, E820_RESERVED_KERN))
539				set_pud(pud, __pud(0));
540			continue;
541		}
542
543		if (pud_val(*pud)) {
544			if (!pud_large(*pud)) {
545				pmd = pmd_offset(pud, 0);
546				last_map_addr = phys_pmd_init(pmd, addr, end,
547							 page_size_mask, prot);
548				__flush_tlb_all();
549				continue;
550			}
551			/*
552			 * If we are ok with PG_LEVEL_1G mapping, then we will
553			 * use the existing mapping.
554			 *
555			 * Otherwise, we will split the gbpage mapping but use
556			 * the same existing protection  bits except for large
557			 * page, so that we don't violate Intel's TLB
558			 * Application note (317080) which says, while changing
559			 * the page sizes, new and old translations should
560			 * not differ with respect to page frame and
561			 * attributes.
562			 */
563			if (page_size_mask & (1 << PG_LEVEL_1G)) {
564				if (!after_bootmem)
565					pages++;
566				last_map_addr = next;
567				continue;
568			}
569			prot = pte_pgprot(pte_clrhuge(*(pte_t *)pud));
570		}
571
572		if (page_size_mask & (1<<PG_LEVEL_1G)) {
573			pages++;
574			spin_lock(&init_mm.page_table_lock);
575			set_pte((pte_t *)pud,
576				pfn_pte((addr & PUD_MASK) >> PAGE_SHIFT,
577					PAGE_KERNEL_LARGE));
578			spin_unlock(&init_mm.page_table_lock);
579			last_map_addr = next;
580			continue;
581		}
582
583		pmd = alloc_low_page();
584		last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask,
585					      prot);
586
587		spin_lock(&init_mm.page_table_lock);
588		pud_populate(&init_mm, pud, pmd);
589		spin_unlock(&init_mm.page_table_lock);
590	}
591	__flush_tlb_all();
592
593	update_page_count(PG_LEVEL_1G, pages);
594
595	return last_map_addr;
596}
597
598unsigned long __meminit
599kernel_physical_mapping_init(unsigned long start,
600			     unsigned long end,
601			     unsigned long page_size_mask)
602{
603	bool pgd_changed = false;
604	unsigned long next, last_map_addr = end;
605	unsigned long addr;
606
607	start = (unsigned long)__va(start);
608	end = (unsigned long)__va(end);
609	addr = start;
610
611	for (; start < end; start = next) {
612		pgd_t *pgd = pgd_offset_k(start);
613		pud_t *pud;
614
615		next = (start & PGDIR_MASK) + PGDIR_SIZE;
616
617		if (pgd_val(*pgd)) {
618			pud = (pud_t *)pgd_page_vaddr(*pgd);
619			last_map_addr = phys_pud_init(pud, __pa(start),
620						 __pa(end), page_size_mask);
621			continue;
622		}
623
624		pud = alloc_low_page();
625		last_map_addr = phys_pud_init(pud, __pa(start), __pa(end),
626						 page_size_mask);
627
628		spin_lock(&init_mm.page_table_lock);
629		pgd_populate(&init_mm, pgd, pud);
630		spin_unlock(&init_mm.page_table_lock);
631		pgd_changed = true;
632	}
633
634	if (pgd_changed)
635		sync_global_pgds(addr, end - 1, 0);
636
637	__flush_tlb_all();
638
639	return last_map_addr;
640}
641
642#ifndef CONFIG_NUMA
643void __init initmem_init(void)
644{
645	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
646}
647#endif
648
649void __init paging_init(void)
650{
651	sparse_memory_present_with_active_regions(MAX_NUMNODES);
652	sparse_init();
653
654	/*
655	 * clear the default setting with node 0
656	 * note: don't use nodes_clear here, that is really clearing when
657	 *	 numa support is not compiled in, and later node_set_state
658	 *	 will not set it back.
659	 */
660	node_clear_state(0, N_MEMORY);
661	if (N_MEMORY != N_NORMAL_MEMORY)
662		node_clear_state(0, N_NORMAL_MEMORY);
663
664	zone_sizes_init();
665}
666
667/*
668 * Memory hotplug specific functions
669 */
670#ifdef CONFIG_MEMORY_HOTPLUG
671/*
672 * After memory hotplug the variables max_pfn, max_low_pfn and high_memory need
673 * updating.
674 */
675static void  update_end_of_memory_vars(u64 start, u64 size)
676{
677	unsigned long end_pfn = PFN_UP(start + size);
678
679	if (end_pfn > max_pfn) {
680		max_pfn = end_pfn;
681		max_low_pfn = end_pfn;
682		high_memory = (void *)__va(max_pfn * PAGE_SIZE - 1) + 1;
683	}
684}
685
686/*
687 * Memory is added always to NORMAL zone. This means you will never get
688 * additional DMA/DMA32 memory.
689 */
690int arch_add_memory(int nid, u64 start, u64 size)
691{
692	struct pglist_data *pgdat = NODE_DATA(nid);
693	struct zone *zone = pgdat->node_zones +
694		zone_for_memory(nid, start, size, ZONE_NORMAL);
695	unsigned long start_pfn = start >> PAGE_SHIFT;
696	unsigned long nr_pages = size >> PAGE_SHIFT;
697	int ret;
698
699	init_memory_mapping(start, start + size);
700
701	ret = __add_pages(nid, zone, start_pfn, nr_pages);
702	WARN_ON_ONCE(ret);
703
704	/* update max_pfn, max_low_pfn and high_memory */
705	update_end_of_memory_vars(start, size);
706
707	return ret;
708}
709EXPORT_SYMBOL_GPL(arch_add_memory);
710
711#define PAGE_INUSE 0xFD
712
713static void __meminit free_pagetable(struct page *page, int order)
714{
715	unsigned long magic;
716	unsigned int nr_pages = 1 << order;
717
718	/* bootmem page has reserved flag */
719	if (PageReserved(page)) {
720		__ClearPageReserved(page);
721
722		magic = (unsigned long)page->lru.next;
723		if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) {
724			while (nr_pages--)
725				put_page_bootmem(page++);
726		} else
727			while (nr_pages--)
728				free_reserved_page(page++);
729	} else
730		free_pages((unsigned long)page_address(page), order);
731}
732
733static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
734{
735	pte_t *pte;
736	int i;
737
738	for (i = 0; i < PTRS_PER_PTE; i++) {
739		pte = pte_start + i;
740		if (pte_val(*pte))
741			return;
742	}
743
744	/* free a pte talbe */
745	free_pagetable(pmd_page(*pmd), 0);
746	spin_lock(&init_mm.page_table_lock);
747	pmd_clear(pmd);
748	spin_unlock(&init_mm.page_table_lock);
749}
750
751static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud)
752{
753	pmd_t *pmd;
754	int i;
755
756	for (i = 0; i < PTRS_PER_PMD; i++) {
757		pmd = pmd_start + i;
758		if (pmd_val(*pmd))
759			return;
760	}
761
762	/* free a pmd talbe */
763	free_pagetable(pud_page(*pud), 0);
764	spin_lock(&init_mm.page_table_lock);
765	pud_clear(pud);
766	spin_unlock(&init_mm.page_table_lock);
767}
768
769/* Return true if pgd is changed, otherwise return false. */
770static bool __meminit free_pud_table(pud_t *pud_start, pgd_t *pgd)
771{
772	pud_t *pud;
773	int i;
774
775	for (i = 0; i < PTRS_PER_PUD; i++) {
776		pud = pud_start + i;
777		if (pud_val(*pud))
778			return false;
779	}
780
781	/* free a pud table */
782	free_pagetable(pgd_page(*pgd), 0);
783	spin_lock(&init_mm.page_table_lock);
784	pgd_clear(pgd);
785	spin_unlock(&init_mm.page_table_lock);
786
787	return true;
788}
789
790static void __meminit
791remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
792		 bool direct)
793{
794	unsigned long next, pages = 0;
795	pte_t *pte;
796	void *page_addr;
797	phys_addr_t phys_addr;
798
799	pte = pte_start + pte_index(addr);
800	for (; addr < end; addr = next, pte++) {
801		next = (addr + PAGE_SIZE) & PAGE_MASK;
802		if (next > end)
803			next = end;
804
805		if (!pte_present(*pte))
806			continue;
807
808		/*
809		 * We mapped [0,1G) memory as identity mapping when
810		 * initializing, in arch/x86/kernel/head_64.S. These
811		 * pagetables cannot be removed.
812		 */
813		phys_addr = pte_val(*pte) + (addr & PAGE_MASK);
814		if (phys_addr < (phys_addr_t)0x40000000)
815			return;
816
817		if (IS_ALIGNED(addr, PAGE_SIZE) &&
818		    IS_ALIGNED(next, PAGE_SIZE)) {
819			/*
820			 * Do not free direct mapping pages since they were
821			 * freed when offlining, or simplely not in use.
822			 */
823			if (!direct)
824				free_pagetable(pte_page(*pte), 0);
825
826			spin_lock(&init_mm.page_table_lock);
827			pte_clear(&init_mm, addr, pte);
828			spin_unlock(&init_mm.page_table_lock);
829
830			/* For non-direct mapping, pages means nothing. */
831			pages++;
832		} else {
833			/*
834			 * If we are here, we are freeing vmemmap pages since
835			 * direct mapped memory ranges to be freed are aligned.
836			 *
837			 * If we are not removing the whole page, it means
838			 * other page structs in this page are being used and
839			 * we canot remove them. So fill the unused page_structs
840			 * with 0xFD, and remove the page when it is wholly
841			 * filled with 0xFD.
842			 */
843			memset((void *)addr, PAGE_INUSE, next - addr);
844
845			page_addr = page_address(pte_page(*pte));
846			if (!memchr_inv(page_addr, PAGE_INUSE, PAGE_SIZE)) {
847				free_pagetable(pte_page(*pte), 0);
848
849				spin_lock(&init_mm.page_table_lock);
850				pte_clear(&init_mm, addr, pte);
851				spin_unlock(&init_mm.page_table_lock);
852			}
853		}
854	}
855
856	/* Call free_pte_table() in remove_pmd_table(). */
857	flush_tlb_all();
858	if (direct)
859		update_page_count(PG_LEVEL_4K, -pages);
860}
861
862static void __meminit
863remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
864		 bool direct)
865{
866	unsigned long next, pages = 0;
867	pte_t *pte_base;
868	pmd_t *pmd;
869	void *page_addr;
870
871	pmd = pmd_start + pmd_index(addr);
872	for (; addr < end; addr = next, pmd++) {
873		next = pmd_addr_end(addr, end);
874
875		if (!pmd_present(*pmd))
876			continue;
877
878		if (pmd_large(*pmd)) {
879			if (IS_ALIGNED(addr, PMD_SIZE) &&
880			    IS_ALIGNED(next, PMD_SIZE)) {
881				if (!direct)
882					free_pagetable(pmd_page(*pmd),
883						       get_order(PMD_SIZE));
884
885				spin_lock(&init_mm.page_table_lock);
886				pmd_clear(pmd);
887				spin_unlock(&init_mm.page_table_lock);
888				pages++;
889			} else {
890				/* If here, we are freeing vmemmap pages. */
891				memset((void *)addr, PAGE_INUSE, next - addr);
892
893				page_addr = page_address(pmd_page(*pmd));
894				if (!memchr_inv(page_addr, PAGE_INUSE,
895						PMD_SIZE)) {
896					free_pagetable(pmd_page(*pmd),
897						       get_order(PMD_SIZE));
898
899					spin_lock(&init_mm.page_table_lock);
900					pmd_clear(pmd);
901					spin_unlock(&init_mm.page_table_lock);
902				}
903			}
904
905			continue;
906		}
907
908		pte_base = (pte_t *)pmd_page_vaddr(*pmd);
909		remove_pte_table(pte_base, addr, next, direct);
910		free_pte_table(pte_base, pmd);
911	}
912
913	/* Call free_pmd_table() in remove_pud_table(). */
914	if (direct)
915		update_page_count(PG_LEVEL_2M, -pages);
916}
917
918static void __meminit
919remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
920		 bool direct)
921{
922	unsigned long next, pages = 0;
923	pmd_t *pmd_base;
924	pud_t *pud;
925	void *page_addr;
926
927	pud = pud_start + pud_index(addr);
928	for (; addr < end; addr = next, pud++) {
929		next = pud_addr_end(addr, end);
930
931		if (!pud_present(*pud))
932			continue;
933
934		if (pud_large(*pud)) {
935			if (IS_ALIGNED(addr, PUD_SIZE) &&
936			    IS_ALIGNED(next, PUD_SIZE)) {
937				if (!direct)
938					free_pagetable(pud_page(*pud),
939						       get_order(PUD_SIZE));
940
941				spin_lock(&init_mm.page_table_lock);
942				pud_clear(pud);
943				spin_unlock(&init_mm.page_table_lock);
944				pages++;
945			} else {
946				/* If here, we are freeing vmemmap pages. */
947				memset((void *)addr, PAGE_INUSE, next - addr);
948
949				page_addr = page_address(pud_page(*pud));
950				if (!memchr_inv(page_addr, PAGE_INUSE,
951						PUD_SIZE)) {
952					free_pagetable(pud_page(*pud),
953						       get_order(PUD_SIZE));
954
955					spin_lock(&init_mm.page_table_lock);
956					pud_clear(pud);
957					spin_unlock(&init_mm.page_table_lock);
958				}
959			}
960
961			continue;
962		}
963
964		pmd_base = (pmd_t *)pud_page_vaddr(*pud);
965		remove_pmd_table(pmd_base, addr, next, direct);
966		free_pmd_table(pmd_base, pud);
967	}
968
969	if (direct)
970		update_page_count(PG_LEVEL_1G, -pages);
971}
972
973/* start and end are both virtual address. */
974static void __meminit
975remove_pagetable(unsigned long start, unsigned long end, bool direct)
976{
977	unsigned long next;
978	unsigned long addr;
979	pgd_t *pgd;
980	pud_t *pud;
981	bool pgd_changed = false;
982
983	for (addr = start; addr < end; addr = next) {
984		next = pgd_addr_end(addr, end);
985
986		pgd = pgd_offset_k(addr);
987		if (!pgd_present(*pgd))
988			continue;
989
990		pud = (pud_t *)pgd_page_vaddr(*pgd);
991		remove_pud_table(pud, addr, next, direct);
992		if (free_pud_table(pud, pgd))
993			pgd_changed = true;
994	}
995
996	if (pgd_changed)
997		sync_global_pgds(start, end - 1, 1);
998
999	flush_tlb_all();
1000}
1001
1002void __ref vmemmap_free(unsigned long start, unsigned long end)
1003{
1004	remove_pagetable(start, end, false);
1005}
1006
1007#ifdef CONFIG_MEMORY_HOTREMOVE
1008static void __meminit
1009kernel_physical_mapping_remove(unsigned long start, unsigned long end)
1010{
1011	start = (unsigned long)__va(start);
1012	end = (unsigned long)__va(end);
1013
1014	remove_pagetable(start, end, true);
1015}
1016
1017int __ref arch_remove_memory(u64 start, u64 size)
1018{
1019	unsigned long start_pfn = start >> PAGE_SHIFT;
1020	unsigned long nr_pages = size >> PAGE_SHIFT;
1021	struct zone *zone;
1022	int ret;
1023
1024	zone = page_zone(pfn_to_page(start_pfn));
1025	kernel_physical_mapping_remove(start, start + size);
1026	ret = __remove_pages(zone, start_pfn, nr_pages);
1027	WARN_ON_ONCE(ret);
1028
1029	return ret;
1030}
1031#endif
1032#endif /* CONFIG_MEMORY_HOTPLUG */
1033
1034static struct kcore_list kcore_vsyscall;
1035
1036static void __init register_page_bootmem_info(void)
1037{
1038#ifdef CONFIG_NUMA
1039	int i;
1040
1041	for_each_online_node(i)
1042		register_page_bootmem_info_node(NODE_DATA(i));
1043#endif
1044}
1045
1046void __init mem_init(void)
1047{
1048	pci_iommu_alloc();
1049
1050	/* clear_bss() already clear the empty_zero_page */
1051
1052	register_page_bootmem_info();
1053
1054	/* this will put all memory onto the freelists */
1055	free_all_bootmem();
1056	after_bootmem = 1;
1057
1058	/* Register memory areas for /proc/kcore */
1059	kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR,
1060			 PAGE_SIZE, KCORE_OTHER);
1061
1062	mem_init_print_info(NULL);
1063}
1064
1065#ifdef CONFIG_DEBUG_RODATA
1066const int rodata_test_data = 0xC3;
1067EXPORT_SYMBOL_GPL(rodata_test_data);
1068
1069int kernel_set_to_readonly;
1070
1071void set_kernel_text_rw(void)
1072{
1073	unsigned long start = PFN_ALIGN(_text);
1074	unsigned long end = PFN_ALIGN(__stop___ex_table);
1075
1076	if (!kernel_set_to_readonly)
1077		return;
1078
1079	pr_debug("Set kernel text: %lx - %lx for read write\n",
1080		 start, end);
1081
1082	/*
1083	 * Make the kernel identity mapping for text RW. Kernel text
1084	 * mapping will always be RO. Refer to the comment in
1085	 * static_protections() in pageattr.c
1086	 */
1087	set_memory_rw(start, (end - start) >> PAGE_SHIFT);
1088}
1089
1090void set_kernel_text_ro(void)
1091{
1092	unsigned long start = PFN_ALIGN(_text);
1093	unsigned long end = PFN_ALIGN(__stop___ex_table);
1094
1095	if (!kernel_set_to_readonly)
1096		return;
1097
1098	pr_debug("Set kernel text: %lx - %lx for read only\n",
1099		 start, end);
1100
1101	/*
1102	 * Set the kernel identity mapping for text RO.
1103	 */
1104	set_memory_ro(start, (end - start) >> PAGE_SHIFT);
1105}
1106
1107void mark_rodata_ro(void)
1108{
1109	unsigned long start = PFN_ALIGN(_text);
1110	unsigned long rodata_start = PFN_ALIGN(__start_rodata);
1111	unsigned long end = (unsigned long) &__end_rodata_hpage_align;
1112	unsigned long text_end = PFN_ALIGN(&__stop___ex_table);
1113	unsigned long rodata_end = PFN_ALIGN(&__end_rodata);
1114	unsigned long all_end;
1115
1116	printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
1117	       (end - start) >> 10);
1118	set_memory_ro(start, (end - start) >> PAGE_SHIFT);
1119
1120	kernel_set_to_readonly = 1;
1121
1122	/*
1123	 * The rodata/data/bss/brk section (but not the kernel text!)
1124	 * should also be not-executable.
1125	 *
1126	 * We align all_end to PMD_SIZE because the existing mapping
1127	 * is a full PMD. If we would align _brk_end to PAGE_SIZE we
1128	 * split the PMD and the reminder between _brk_end and the end
1129	 * of the PMD will remain mapped executable.
1130	 *
1131	 * Any PMD which was setup after the one which covers _brk_end
1132	 * has been zapped already via cleanup_highmem().
1133	 */
1134	all_end = roundup((unsigned long)_brk_end, PMD_SIZE);
1135	set_memory_nx(text_end, (all_end - text_end) >> PAGE_SHIFT);
1136
1137	rodata_test();
1138
1139#ifdef CONFIG_CPA_DEBUG
1140	printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, end);
1141	set_memory_rw(start, (end-start) >> PAGE_SHIFT);
1142
1143	printk(KERN_INFO "Testing CPA: again\n");
1144	set_memory_ro(start, (end-start) >> PAGE_SHIFT);
1145#endif
1146
1147	free_init_pages("unused kernel",
1148			(unsigned long) __va(__pa_symbol(text_end)),
1149			(unsigned long) __va(__pa_symbol(rodata_start)));
1150	free_init_pages("unused kernel",
1151			(unsigned long) __va(__pa_symbol(rodata_end)),
1152			(unsigned long) __va(__pa_symbol(_sdata)));
1153}
1154
1155#endif
1156
1157int kern_addr_valid(unsigned long addr)
1158{
1159	unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;
1160	pgd_t *pgd;
1161	pud_t *pud;
1162	pmd_t *pmd;
1163	pte_t *pte;
1164
1165	if (above != 0 && above != -1UL)
1166		return 0;
1167
1168	pgd = pgd_offset_k(addr);
1169	if (pgd_none(*pgd))
1170		return 0;
1171
1172	pud = pud_offset(pgd, addr);
1173	if (pud_none(*pud))
1174		return 0;
1175
1176	if (pud_large(*pud))
1177		return pfn_valid(pud_pfn(*pud));
1178
1179	pmd = pmd_offset(pud, addr);
1180	if (pmd_none(*pmd))
1181		return 0;
1182
1183	if (pmd_large(*pmd))
1184		return pfn_valid(pmd_pfn(*pmd));
1185
1186	pte = pte_offset_kernel(pmd, addr);
1187	if (pte_none(*pte))
1188		return 0;
1189
1190	return pfn_valid(pte_pfn(*pte));
1191}
1192
1193static unsigned long probe_memory_block_size(void)
1194{
1195	/* start from 2g */
1196	unsigned long bz = 1UL<<31;
1197
1198	if (totalram_pages >= (64ULL << (30 - PAGE_SHIFT))) {
1199		pr_info("Using 2GB memory block size for large-memory system\n");
1200		return 2UL * 1024 * 1024 * 1024;
1201	}
1202
1203	/* less than 64g installed */
1204	if ((max_pfn << PAGE_SHIFT) < (16UL << 32))
1205		return MIN_MEMORY_BLOCK_SIZE;
1206
1207	/* get the tail size */
1208	while (bz > MIN_MEMORY_BLOCK_SIZE) {
1209		if (!((max_pfn << PAGE_SHIFT) & (bz - 1)))
1210			break;
1211		bz >>= 1;
1212	}
1213
1214	printk(KERN_DEBUG "memory block size : %ldMB\n", bz >> 20);
1215
1216	return bz;
1217}
1218
1219static unsigned long memory_block_size_probed;
1220unsigned long memory_block_size_bytes(void)
1221{
1222	if (!memory_block_size_probed)
1223		memory_block_size_probed = probe_memory_block_size();
1224
1225	return memory_block_size_probed;
1226}
1227
1228#ifdef CONFIG_SPARSEMEM_VMEMMAP
1229/*
1230 * Initialise the sparsemem vmemmap using huge-pages at the PMD level.
1231 */
1232static long __meminitdata addr_start, addr_end;
1233static void __meminitdata *p_start, *p_end;
1234static int __meminitdata node_start;
1235
1236static int __meminit vmemmap_populate_hugepages(unsigned long start,
1237						unsigned long end, int node)
1238{
1239	unsigned long addr;
1240	unsigned long next;
1241	pgd_t *pgd;
1242	pud_t *pud;
1243	pmd_t *pmd;
1244
1245	for (addr = start; addr < end; addr = next) {
1246		next = pmd_addr_end(addr, end);
1247
1248		pgd = vmemmap_pgd_populate(addr, node);
1249		if (!pgd)
1250			return -ENOMEM;
1251
1252		pud = vmemmap_pud_populate(pgd, addr, node);
1253		if (!pud)
1254			return -ENOMEM;
1255
1256		pmd = pmd_offset(pud, addr);
1257		if (pmd_none(*pmd)) {
1258			void *p;
1259
1260			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
1261			if (p) {
1262				pte_t entry;
1263
1264				entry = pfn_pte(__pa(p) >> PAGE_SHIFT,
1265						PAGE_KERNEL_LARGE);
1266				set_pmd(pmd, __pmd(pte_val(entry)));
1267
1268				/* check to see if we have contiguous blocks */
1269				if (p_end != p || node_start != node) {
1270					if (p_start)
1271						printk(KERN_DEBUG " [%lx-%lx] PMD -> [%p-%p] on node %d\n",
1272						       addr_start, addr_end-1, p_start, p_end-1, node_start);
1273					addr_start = addr;
1274					node_start = node;
1275					p_start = p;
1276				}
1277
1278				addr_end = addr + PMD_SIZE;
1279				p_end = p + PMD_SIZE;
1280				continue;
1281			}
1282		} else if (pmd_large(*pmd)) {
1283			vmemmap_verify((pte_t *)pmd, node, addr, next);
1284			continue;
1285		}
1286		pr_warn_once("vmemmap: falling back to regular page backing\n");
1287		if (vmemmap_populate_basepages(addr, next, node))
1288			return -ENOMEM;
1289	}
1290	return 0;
1291}
1292
1293int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
1294{
1295	int err;
1296
1297	if (cpu_has_pse)
1298		err = vmemmap_populate_hugepages(start, end, node);
1299	else
1300		err = vmemmap_populate_basepages(start, end, node);
1301	if (!err)
1302		sync_global_pgds(start, end - 1, 0);
1303	return err;
1304}
1305
1306#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_HAVE_BOOTMEM_INFO_NODE)
1307void register_page_bootmem_memmap(unsigned long section_nr,
1308				  struct page *start_page, unsigned long size)
1309{
1310	unsigned long addr = (unsigned long)start_page;
1311	unsigned long end = (unsigned long)(start_page + size);
1312	unsigned long next;
1313	pgd_t *pgd;
1314	pud_t *pud;
1315	pmd_t *pmd;
1316	unsigned int nr_pages;
1317	struct page *page;
1318
1319	for (; addr < end; addr = next) {
1320		pte_t *pte = NULL;
1321
1322		pgd = pgd_offset_k(addr);
1323		if (pgd_none(*pgd)) {
1324			next = (addr + PAGE_SIZE) & PAGE_MASK;
1325			continue;
1326		}
1327		get_page_bootmem(section_nr, pgd_page(*pgd), MIX_SECTION_INFO);
1328
1329		pud = pud_offset(pgd, addr);
1330		if (pud_none(*pud)) {
1331			next = (addr + PAGE_SIZE) & PAGE_MASK;
1332			continue;
1333		}
1334		get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO);
1335
1336		if (!cpu_has_pse) {
1337			next = (addr + PAGE_SIZE) & PAGE_MASK;
1338			pmd = pmd_offset(pud, addr);
1339			if (pmd_none(*pmd))
1340				continue;
1341			get_page_bootmem(section_nr, pmd_page(*pmd),
1342					 MIX_SECTION_INFO);
1343
1344			pte = pte_offset_kernel(pmd, addr);
1345			if (pte_none(*pte))
1346				continue;
1347			get_page_bootmem(section_nr, pte_page(*pte),
1348					 SECTION_INFO);
1349		} else {
1350			next = pmd_addr_end(addr, end);
1351
1352			pmd = pmd_offset(pud, addr);
1353			if (pmd_none(*pmd))
1354				continue;
1355
1356			nr_pages = 1 << (get_order(PMD_SIZE));
1357			page = pmd_page(*pmd);
1358			while (nr_pages--)
1359				get_page_bootmem(section_nr, page++,
1360						 SECTION_INFO);
1361		}
1362	}
1363}
1364#endif
1365
1366void __meminit vmemmap_populate_print_last(void)
1367{
1368	if (p_start) {
1369		printk(KERN_DEBUG " [%lx-%lx] PMD -> [%p-%p] on node %d\n",
1370			addr_start, addr_end-1, p_start, p_end-1, node_start);
1371		p_start = NULL;
1372		p_end = NULL;
1373		node_start = 0;
1374	}
1375}
1376#endif
1377