124 static inline unsigned p2m_top_index(unsigned long pfn)  in p2m_top_index()  argument
126 	BUG_ON(pfn >= MAX_P2M_PFN);  in p2m_top_index()
127 	return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE);  in p2m_top_index()
130 static inline unsigned p2m_mid_index(unsigned long pfn)  in p2m_mid_index()  argument
132 	return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE;  in p2m_mid_index()
135 static inline unsigned p2m_index(unsigned long pfn)  in p2m_index()  argument
137 	return pfn % P2M_PER_PAGE;  in p2m_index()
172 static void p2m_init_identity(unsigned long *p2m, unsigned long pfn)  in p2m_init_identity()  argument
177 		p2m[i] = IDENTITY_FRAME(pfn + i);  in p2m_init_identity()
210 	unsigned long pfn, mfn;  in xen_build_mfn_list_list()  local
234 	for (pfn = 0; pfn < xen_max_p2m_pfn && pfn < MAX_P2M_PFN;  in xen_build_mfn_list_list()
235 	     pfn += P2M_PER_PAGE) {  in xen_build_mfn_list_list()
236 		topidx = p2m_top_index(pfn);  in xen_build_mfn_list_list()
237 		mididx = p2m_mid_index(pfn);  in xen_build_mfn_list_list()
240 		ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn),  in xen_build_mfn_list_list()
254 			pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE;  in xen_build_mfn_list_list()
292 	unsigned long pfn;  in xen_build_dynamic_phys_to_machine()  local
300 	for (pfn = xen_start_info->nr_pages; pfn < xen_p2m_size; pfn++)  in xen_build_dynamic_phys_to_machine()
301 		xen_p2m_addr[pfn] = INVALID_P2M_ENTRY;  in xen_build_dynamic_phys_to_machine()
311 static int xen_p2m_elem_type(unsigned long pfn)  in xen_p2m_elem_type()  argument
315 	if (pfn >= xen_p2m_size)  in xen_p2m_elem_type()
318 	mfn = xen_p2m_addr[pfn];  in xen_p2m_elem_type()
332 	unsigned long pfn;  in xen_rebuild_p2m_list()  local
354 	for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += chunk) {  in xen_rebuild_p2m_list()
365 		chunk = (pfn & (P2M_PER_PAGE * P2M_MID_PER_PAGE - 1)) ?  in xen_rebuild_p2m_list()
368 		type = xen_p2m_elem_type(pfn);  in xen_rebuild_p2m_list()
372 				if (xen_p2m_elem_type(pfn + i) != type)  in xen_rebuild_p2m_list()
382 			copy_page(mfns, xen_p2m_addr + pfn);  in xen_rebuild_p2m_list()
384 			mfns = xen_p2m_addr + pfn;  in xen_rebuild_p2m_list()
386 			ptep = populate_extra_pte((unsigned long)(p2m + pfn));  in xen_rebuild_p2m_list()
396 			ptep = populate_extra_pte((unsigned long)(p2m + pfn));  in xen_rebuild_p2m_list()
407 				(unsigned long)(p2m + pfn) + i * PMD_SIZE);  in xen_rebuild_p2m_list()
437 unsigned long get_phys_to_machine(unsigned long pfn)  in get_phys_to_machine()  argument
442 	if (unlikely(pfn >= xen_p2m_size)) {  in get_phys_to_machine()
443 		if (pfn < xen_max_p2m_pfn)  in get_phys_to_machine()
444 			return xen_chk_extra_mem(pfn);  in get_phys_to_machine()
446 		return IDENTITY_FRAME(pfn);  in get_phys_to_machine()
449 	ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), &level);  in get_phys_to_machine()
458 		return IDENTITY_FRAME(pfn);  in get_phys_to_machine()
460 	return xen_p2m_addr[pfn];  in get_phys_to_machine()
533 int xen_alloc_p2m_entry(unsigned long pfn)  in xen_alloc_p2m_entry()  argument
540 	unsigned long addr = (unsigned long)(xen_p2m_addr + pfn);  in xen_alloc_p2m_entry()
557 	if (p2m_top_mfn && pfn < MAX_P2M_PFN) {  in xen_alloc_p2m_entry()
558 		topidx = p2m_top_index(pfn);  in xen_alloc_p2m_entry()
603 			p2m_init_identity(p2m, pfn & ~(P2M_PER_PAGE - 1));  in xen_alloc_p2m_entry()
615 				mid_mfn[p2m_mid_index(pfn)] = virt_to_mfn(p2m);  in xen_alloc_p2m_entry()
626 	if (pfn > xen_p2m_last_pfn) {  in xen_alloc_p2m_entry()
627 		xen_p2m_last_pfn = pfn;  in xen_alloc_p2m_entry()
638 	unsigned long pfn;  in set_phys_range_identity()  local
652 	for (pfn = pfn_s; pfn < pfn_e; pfn++)  in set_phys_range_identity()
653 		xen_p2m_addr[pfn] = IDENTITY_FRAME(pfn);  in set_phys_range_identity()
655 	return pfn - pfn_s;  in set_phys_range_identity()
658 bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)  in __set_phys_to_machine()  argument
667 	if (unlikely(pfn >= xen_p2m_size)) {  in __set_phys_to_machine()
677 	if (likely(!xen_safe_write_ulong(xen_p2m_addr + pfn, mfn)))  in __set_phys_to_machine()
680 	ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), &level);  in __set_phys_to_machine()
687 		return mfn == IDENTITY_FRAME(pfn);  in __set_phys_to_machine()
692 bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)  in set_phys_to_machine()  argument
694 	if (unlikely(!__set_phys_to_machine(pfn, mfn))) {  in set_phys_to_machine()
697 		ret = xen_alloc_p2m_entry(pfn);  in set_phys_to_machine()
701 		return __set_phys_to_machine(pfn, mfn);  in set_phys_to_machine()
725 		unsigned long mfn, pfn;  in set_foreign_p2m_mapping()  local
738 		pfn = page_to_pfn(pages[i]);  in set_foreign_p2m_mapping()
740 		WARN(pfn_to_mfn(pfn) != INVALID_P2M_ENTRY, "page must be ballooned");  in set_foreign_p2m_mapping()
742 		if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) {  in set_foreign_p2m_mapping()
764 		unsigned long pfn = page_to_pfn(pages[i]);  in clear_foreign_p2m_mapping()  local
771 		set_phys_to_machine(pfn, INVALID_P2M_ENTRY);  in clear_foreign_p2m_mapping()
791 	unsigned long pfn, first_pfn;  in p2m_dump_show()  local
797 	for (pfn = 0; pfn < xen_p2m_size; pfn++) {  in p2m_dump_show()
798 		type = xen_p2m_elem_type(pfn);  in p2m_dump_show()
800 			seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn,  in p2m_dump_show()
803 			first_pfn = pfn;  in p2m_dump_show()
806 	seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn,  in p2m_dump_show()