335 static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,  in xen_set_pte_at()  argument
338 	trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval);  in xen_set_pte_at()
342 pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,  in xen_ptep_modify_prot_start()  argument
346 	trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep);  in xen_ptep_modify_prot_start()
350 void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,  in xen_ptep_modify_prot_commit()  argument
355 	trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte);  in xen_ptep_modify_prot_commit()
484 static void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)  in xen_pte_clear()  argument
486 	trace_xen_mmu_pte_clear(mm, addr, ptep);  in xen_pte_clear()
488 		native_pte_clear(mm, addr, ptep);  in xen_pte_clear()
609 static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,  in __xen_pgd_walk()  argument
610 			  int (*func)(struct mm_struct *mm, struct page *,  in __xen_pgd_walk()  argument
658 			flush |= (*func)(mm, virt_to_page(pud), PT_PUD);  in __xen_pgd_walk()
673 				flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);  in __xen_pgd_walk()
687 				flush |= (*func)(mm, pte, PT_PTE);  in __xen_pgd_walk()
695 	flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);  in __xen_pgd_walk()
700 static int xen_pgd_walk(struct mm_struct *mm,  in xen_pgd_walk()  argument
701 			int (*func)(struct mm_struct *mm, struct page *,  in xen_pgd_walk()  argument
705 	return __xen_pgd_walk(mm, mm->pgd, func, limit);  in xen_pgd_walk()
710 static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)  in xen_pte_lock()  argument
716 	spin_lock_nest_lock(ptl, &mm->page_table_lock);  in xen_pte_lock()
738 static int xen_pin_page(struct mm_struct *mm, struct page *page,  in xen_pin_page()  argument
780 			ptl = xen_pte_lock(page, mm);  in xen_pin_page()
801 static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)  in __xen_pgd_pin()  argument
803 	trace_xen_mmu_pgd_pin(mm, pgd);  in __xen_pgd_pin()
807 	if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {  in __xen_pgd_pin()
823 			xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);  in __xen_pgd_pin()
831 	xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),  in __xen_pgd_pin()
839 static void xen_pgd_pin(struct mm_struct *mm)  in xen_pgd_pin()  argument
841 	__xen_pgd_pin(mm, mm->pgd);  in xen_pgd_pin()
875 static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page,  in xen_mark_pinned()  argument
887 static int xen_unpin_page(struct mm_struct *mm, struct page *page,  in xen_unpin_page()  argument
906 			ptl = xen_pte_lock(page, mm);  in xen_unpin_page()
928 static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)  in __xen_pgd_unpin()  argument
930 	trace_xen_mmu_pgd_unpin(mm, pgd);  in __xen_pgd_unpin()
943 			xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);  in __xen_pgd_unpin()
950 	xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),  in __xen_pgd_unpin()
954 	__xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);  in __xen_pgd_unpin()
959 static void xen_pgd_unpin(struct mm_struct *mm)  in xen_pgd_unpin()  argument
961 	__xen_pgd_unpin(mm, mm->pgd);  in xen_pgd_unpin()
992 static void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)  in xen_dup_mmap()  argument
994 	spin_lock(&mm->page_table_lock);  in xen_dup_mmap()
995 	xen_pgd_pin(mm);  in xen_dup_mmap()
996 	spin_unlock(&mm->page_table_lock);  in xen_dup_mmap()
1005 	struct mm_struct *mm = info;  in drop_other_mm_ref()  local
1010 	if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)  in drop_other_mm_ref()
1015 	if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))  in drop_other_mm_ref()
1019 static void xen_drop_mm_ref(struct mm_struct *mm)  in xen_drop_mm_ref()  argument
1024 	if (current->active_mm == mm) {  in xen_drop_mm_ref()
1025 		if (current->mm == mm)  in xen_drop_mm_ref()
1034 			if (!cpumask_test_cpu(cpu, mm_cpumask(mm))  in xen_drop_mm_ref()
1035 			    && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))  in xen_drop_mm_ref()
1037 			smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);  in xen_drop_mm_ref()
1041 	cpumask_copy(mask, mm_cpumask(mm));  in xen_drop_mm_ref()
1049 		if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))  in xen_drop_mm_ref()
1054 		smp_call_function_many(mask, drop_other_mm_ref, mm, 1);  in xen_drop_mm_ref()
1058 static void xen_drop_mm_ref(struct mm_struct *mm)  in xen_drop_mm_ref()  argument
1060 	if (current->active_mm == mm)  in xen_drop_mm_ref()
1079 static void xen_exit_mmap(struct mm_struct *mm)  in xen_exit_mmap()  argument
1082 	xen_drop_mm_ref(mm);  in xen_exit_mmap()
1085 	spin_lock(&mm->page_table_lock);  in xen_exit_mmap()
1088 	if (xen_page_pinned(mm->pgd))  in xen_exit_mmap()
1089 		xen_pgd_unpin(mm);  in xen_exit_mmap()
1091 	spin_unlock(&mm->page_table_lock);  in xen_exit_mmap()
1271 				 struct mm_struct *mm, unsigned long start,  in xen_flush_tlb_others()  argument
1284 	trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);  in xen_flush_tlb_others()
1407 static int xen_pgd_alloc(struct mm_struct *mm)  in xen_pgd_alloc()  argument
1409 	pgd_t *pgd = mm->pgd;  in xen_pgd_alloc()
1441 static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)  in xen_pgd_free()  argument
1503 static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)  in xen_alloc_pte_init()  argument
1513 static void __init xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn)  in xen_alloc_pmd_init()  argument
1559 static inline void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn,  in xen_alloc_ptpage()  argument
1562 	bool pinned = PagePinned(virt_to_page(mm->pgd));  in xen_alloc_ptpage()
1564 	trace_xen_mmu_alloc_ptpage(mm, pfn, level, pinned);  in xen_alloc_ptpage()
1588 static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)  in xen_alloc_pte()  argument
1590 	xen_alloc_ptpage(mm, pfn, PT_PTE);  in xen_alloc_pte()
1593 static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)  in xen_alloc_pmd()  argument
1595 	xen_alloc_ptpage(mm, pfn, PT_PMD);  in xen_alloc_pmd()
1632 static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)  in xen_alloc_pud()  argument
1634 	xen_alloc_ptpage(mm, pfn, PT_PUD);  in xen_alloc_pud()
2403 static void xen_hvm_exit_mmap(struct mm_struct *mm)  in xen_hvm_exit_mmap()  argument
2409 	a.gpa = __pa(mm->pgd);  in xen_hvm_exit_mmap()