110 static inline unsigned p2m_top_index(unsigned long pfn)  in p2m_top_index()  argument
112 	BUG_ON(pfn >= MAX_P2M_PFN);  in p2m_top_index()
113 	return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE);  in p2m_top_index()
116 static inline unsigned p2m_mid_index(unsigned long pfn)  in p2m_mid_index()  argument
118 	return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE;  in p2m_mid_index()
121 static inline unsigned p2m_index(unsigned long pfn)  in p2m_index()  argument
123 	return pfn % P2M_PER_PAGE;  in p2m_index()
158 static void p2m_init_identity(unsigned long *p2m, unsigned long pfn)  in p2m_init_identity()  argument
163 		p2m[i] = IDENTITY_FRAME(pfn + i);  in p2m_init_identity()
196 	unsigned long pfn, mfn;  in xen_build_mfn_list_list()  local
219 	for (pfn = 0; pfn < xen_max_p2m_pfn && pfn < MAX_P2M_PFN;  in xen_build_mfn_list_list()
220 	     pfn += P2M_PER_PAGE) {  in xen_build_mfn_list_list()
221 		topidx = p2m_top_index(pfn);  in xen_build_mfn_list_list()
222 		mididx = p2m_mid_index(pfn);  in xen_build_mfn_list_list()
225 		ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn),  in xen_build_mfn_list_list()
239 			pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE;  in xen_build_mfn_list_list()
270 	unsigned long pfn;  in xen_build_dynamic_phys_to_machine()  local
278 	for (pfn = xen_start_info->nr_pages; pfn < xen_p2m_size; pfn++)  in xen_build_dynamic_phys_to_machine()
279 		xen_p2m_addr[pfn] = INVALID_P2M_ENTRY;  in xen_build_dynamic_phys_to_machine()
289 static int xen_p2m_elem_type(unsigned long pfn)  in xen_p2m_elem_type()  argument
293 	if (pfn >= xen_p2m_size)  in xen_p2m_elem_type()
296 	mfn = xen_p2m_addr[pfn];  in xen_p2m_elem_type()
310 	unsigned long pfn;  in xen_rebuild_p2m_list()  local
332 	for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += chunk) {  in xen_rebuild_p2m_list()
343 		chunk = (pfn & (P2M_PER_PAGE * P2M_MID_PER_PAGE - 1)) ?  in xen_rebuild_p2m_list()
346 		type = xen_p2m_elem_type(pfn);  in xen_rebuild_p2m_list()
350 				if (xen_p2m_elem_type(pfn + i) != type)  in xen_rebuild_p2m_list()
360 			copy_page(mfns, xen_p2m_addr + pfn);  in xen_rebuild_p2m_list()
362 			mfns = xen_p2m_addr + pfn;  in xen_rebuild_p2m_list()
364 			ptep = populate_extra_pte((unsigned long)(p2m + pfn));  in xen_rebuild_p2m_list()
374 			ptep = populate_extra_pte((unsigned long)(p2m + pfn));  in xen_rebuild_p2m_list()
385 				(unsigned long)(p2m + pfn) + i * PMD_SIZE);  in xen_rebuild_p2m_list()
413 unsigned long get_phys_to_machine(unsigned long pfn)  in get_phys_to_machine()  argument
418 	if (unlikely(pfn >= xen_p2m_size)) {  in get_phys_to_machine()
419 		if (pfn < xen_max_p2m_pfn)  in get_phys_to_machine()
420 			return xen_chk_extra_mem(pfn);  in get_phys_to_machine()
422 		return IDENTITY_FRAME(pfn);  in get_phys_to_machine()
425 	ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), &level);  in get_phys_to_machine()
434 		return IDENTITY_FRAME(pfn);  in get_phys_to_machine()
436 	return xen_p2m_addr[pfn];  in get_phys_to_machine()
505 static bool alloc_p2m(unsigned long pfn)  in alloc_p2m()  argument
512 	unsigned long addr = (unsigned long)(xen_p2m_addr + pfn);  in alloc_p2m()
515 	topidx = p2m_top_index(pfn);  in alloc_p2m()
516 	mididx = p2m_mid_index(pfn);  in alloc_p2m()
574 			p2m_init_identity(p2m, pfn & ~(P2M_PER_PAGE - 1));  in alloc_p2m()
598 	unsigned long pfn;  in set_phys_range_identity()  local
612 	for (pfn = pfn_s; pfn < pfn_e; pfn++)  in set_phys_range_identity()
613 		xen_p2m_addr[pfn] = IDENTITY_FRAME(pfn);  in set_phys_range_identity()
615 	return pfn - pfn_s;  in set_phys_range_identity()
618 bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)  in __set_phys_to_machine()  argument
627 	if (unlikely(pfn >= xen_p2m_size)) {  in __set_phys_to_machine()
632 	if (likely(!xen_safe_write_ulong(xen_p2m_addr + pfn, mfn)))  in __set_phys_to_machine()
635 	ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), &level);  in __set_phys_to_machine()
642 		return mfn == IDENTITY_FRAME(pfn);  in __set_phys_to_machine()
647 bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)  in set_phys_to_machine()  argument
649 	if (unlikely(!__set_phys_to_machine(pfn, mfn))) {  in set_phys_to_machine()
650 		if (!alloc_p2m(pfn))  in set_phys_to_machine()
653 		return __set_phys_to_machine(pfn, mfn);  in set_phys_to_machine()
677 		unsigned long mfn, pfn;  in set_foreign_p2m_mapping()  local
690 		pfn = page_to_pfn(pages[i]);  in set_foreign_p2m_mapping()
692 		WARN(pfn_to_mfn(pfn) != INVALID_P2M_ENTRY, "page must be ballooned");  in set_foreign_p2m_mapping()
694 		if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) {  in set_foreign_p2m_mapping()
716 		unsigned long pfn = page_to_pfn(pages[i]);  in clear_foreign_p2m_mapping()  local
723 		set_phys_to_machine(pfn, INVALID_P2M_ENTRY);  in clear_foreign_p2m_mapping()
743 	unsigned long pfn, first_pfn;  in p2m_dump_show()  local
749 	for (pfn = 0; pfn < xen_p2m_size; pfn++) {  in p2m_dump_show()
750 		type = xen_p2m_elem_type(pfn);  in p2m_dump_show()
752 			seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn,  in p2m_dump_show()
755 			first_pfn = pfn;  in p2m_dump_show()
758 	seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn,  in p2m_dump_show()