373 static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,  in _lookup_address_cpa()  argument
376         if (cpa->pgd)  in _lookup_address_cpa()
377 		return lookup_address_in_pgd(cpa->pgd + pgd_index(address),  in _lookup_address_cpa()
475 			struct cpa_data *cpa)  in try_preserve_large_page()  argument
483 	if (cpa->force_split)  in try_preserve_large_page()
491 	tmp = _lookup_address_cpa(cpa, address, &level);  in try_preserve_large_page()
518 	if (numpages < cpa->numpages)  in try_preserve_large_page()
519 		cpa->numpages = numpages;  in try_preserve_large_page()
529 	pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);  in try_preserve_large_page()
530 	pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);  in try_preserve_large_page()
557 	cpa->pfn = pfn;  in try_preserve_large_page()
592 	if (address == (address & pmask) && cpa->numpages == (psize >> PAGE_SHIFT)) {  in try_preserve_large_page()
599 		cpa->flags |= CPA_FLUSHTLB;  in try_preserve_large_page()
610 __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,  in __split_large_page()  argument
624 	tmp = _lookup_address_cpa(cpa, address, &level);  in __split_large_page()
707 static int split_large_page(struct cpa_data *cpa, pte_t *kpte,  in split_large_page()  argument
720 	if (__split_large_page(cpa, kpte, address, base))  in split_large_page()
905 static void populate_pte(struct cpa_data *cpa,  in populate_pte()  argument
917 			cpa->pfn &= ~_PAGE_NX;  in populate_pte()
919 		set_pte(pte, pfn_pte(cpa->pfn >> PAGE_SHIFT, pgprot));  in populate_pte()
922 		cpa->pfn += PAGE_SIZE;  in populate_pte()
927 static int populate_pmd(struct cpa_data *cpa,  in populate_pmd()  argument
954 		populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);  in populate_pmd()
978 		set_pmd(pmd, __pmd(cpa->pfn | _PAGE_PSE |  in populate_pmd()
982 		cpa->pfn  += PMD_SIZE;  in populate_pmd()
995 		populate_pte(cpa, start, end, num_pages - cur_pages,  in populate_pmd()
1001 static int populate_pud(struct cpa_data *cpa, unsigned long start, pgd_t *pgd,  in populate_pud()  argument
1009 	end = start + (cpa->numpages << PAGE_SHIFT);  in populate_pud()
1021 		cur_pages = min_t(int, (int)cpa->numpages, cur_pages);  in populate_pud()
1032 		cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,  in populate_pud()
1041 	if (cpa->numpages == cur_pages)  in populate_pud()
1051 		set_pud(pud, __pud(cpa->pfn | _PAGE_PSE |  in populate_pud()
1055 		cpa->pfn  += PUD_SIZE;  in populate_pud()
1069 		tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,  in populate_pud()
1083 static int populate_pgd(struct cpa_data *cpa, unsigned long addr)  in populate_pgd()  argument
1090 	pgd_entry = cpa->pgd + pgd_index(addr);  in populate_pgd()
1103 	pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);  in populate_pgd()
1104 	pgprot_val(pgprot) |=  pgprot_val(cpa->mask_set);  in populate_pgd()
1106 	ret = populate_pud(cpa, addr, pgd_entry, pgprot);  in populate_pgd()
1108 		unmap_pgd_range(cpa->pgd, addr,  in populate_pgd()
1109 				addr + (cpa->numpages << PAGE_SHIFT));  in populate_pgd()
1113 	cpa->numpages = ret;  in populate_pgd()
1117 static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,  in __cpa_process_fault()  argument
1120 	if (cpa->pgd)  in __cpa_process_fault()
1121 		return populate_pgd(cpa, vaddr);  in __cpa_process_fault()
1138 		cpa->numpages = 1;  in __cpa_process_fault()
1139 		cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;  in __cpa_process_fault()
1144 			*cpa->vaddr);  in __cpa_process_fault()
1150 static int __change_page_attr(struct cpa_data *cpa, int primary)  in __change_page_attr()  argument
1157 	if (cpa->flags & CPA_PAGES_ARRAY) {  in __change_page_attr()
1158 		struct page *page = cpa->pages[cpa->curpage];  in __change_page_attr()
1162 	} else if (cpa->flags & CPA_ARRAY)  in __change_page_attr()
1163 		address = cpa->vaddr[cpa->curpage];  in __change_page_attr()
1165 		address = *cpa->vaddr;  in __change_page_attr()
1167 	kpte = _lookup_address_cpa(cpa, address, &level);  in __change_page_attr()
1169 		return __cpa_process_fault(cpa, address, primary);  in __change_page_attr()
1173 		return __cpa_process_fault(cpa, address, primary);  in __change_page_attr()
1180 		pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);  in __change_page_attr()
1181 		pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);  in __change_page_attr()
1203 		cpa->pfn = pfn;  in __change_page_attr()
1209 			cpa->flags |= CPA_FLUSHTLB;  in __change_page_attr()
1211 		cpa->numpages = 1;  in __change_page_attr()
1219 	do_split = try_preserve_large_page(kpte, address, cpa);  in __change_page_attr()
1231 	err = split_large_page(cpa, kpte, address);  in __change_page_attr()
1258 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
1260 static int cpa_process_alias(struct cpa_data *cpa)  in cpa_process_alias()  argument
1263 	unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);  in cpa_process_alias()
1267 	if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))  in cpa_process_alias()
1274 	if (cpa->flags & CPA_PAGES_ARRAY) {  in cpa_process_alias()
1275 		struct page *page = cpa->pages[cpa->curpage];  in cpa_process_alias()
1279 	} else if (cpa->flags & CPA_ARRAY)  in cpa_process_alias()
1280 		vaddr = cpa->vaddr[cpa->curpage];  in cpa_process_alias()
1282 		vaddr = *cpa->vaddr;  in cpa_process_alias()
1287 		alias_cpa = *cpa;  in cpa_process_alias()
1303 	    within(cpa->pfn, highmap_start_pfn(), highmap_end_pfn())) {  in cpa_process_alias()
1304 		unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +  in cpa_process_alias()
1306 		alias_cpa = *cpa;  in cpa_process_alias()
1321 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)  in __change_page_attr_set_clr()  argument
1323 	int ret, numpages = cpa->numpages;  in __change_page_attr_set_clr()
1330 		cpa->numpages = numpages;  in __change_page_attr_set_clr()
1332 		if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))  in __change_page_attr_set_clr()
1333 			cpa->numpages = 1;  in __change_page_attr_set_clr()
1337 		ret = __change_page_attr(cpa, checkalias);  in __change_page_attr_set_clr()
1344 			ret = cpa_process_alias(cpa);  in __change_page_attr_set_clr()
1354 		BUG_ON(cpa->numpages > numpages || !cpa->numpages);  in __change_page_attr_set_clr()
1355 		numpages -= cpa->numpages;  in __change_page_attr_set_clr()
1356 		if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY))  in __change_page_attr_set_clr()
1357 			cpa->curpage++;  in __change_page_attr_set_clr()
1359 			*cpa->vaddr += cpa->numpages * PAGE_SIZE;  in __change_page_attr_set_clr()
1370 	struct cpa_data cpa;  in change_page_attr_set_clr()  local
1374 	memset(&cpa, 0, sizeof(cpa));  in change_page_attr_set_clr()
1418 	cpa.vaddr = addr;  in change_page_attr_set_clr()
1419 	cpa.pages = pages;  in change_page_attr_set_clr()
1420 	cpa.numpages = numpages;  in change_page_attr_set_clr()
1421 	cpa.mask_set = mask_set;  in change_page_attr_set_clr()
1422 	cpa.mask_clr = mask_clr;  in change_page_attr_set_clr()
1423 	cpa.flags = 0;  in change_page_attr_set_clr()
1424 	cpa.curpage = 0;  in change_page_attr_set_clr()
1425 	cpa.force_split = force_split;  in change_page_attr_set_clr()
1428 		cpa.flags |= in_flag;  in change_page_attr_set_clr()
1433 	ret = __change_page_attr_set_clr(&cpa, checkalias);  in change_page_attr_set_clr()
1438 	if (!(cpa.flags & CPA_FLUSHTLB))  in change_page_attr_set_clr()
1454 		if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {  in change_page_attr_set_clr()
1456 					cpa.flags, pages);  in change_page_attr_set_clr()
1864 	struct cpa_data cpa = { .vaddr = &tempaddr,  in __set_pages_p()  local
1877 	return __change_page_attr_set_clr(&cpa, 0);  in __set_pages_p()
1883 	struct cpa_data cpa = { .vaddr = &tempaddr,  in __set_pages_np()  local
1896 	return __change_page_attr_set_clr(&cpa, 0);  in __set_pages_np()
1950 	struct cpa_data cpa = {  in kernel_map_pages_in_pgd()  local
1964 		cpa.mask_clr = __pgprot(_PAGE_NX);  in kernel_map_pages_in_pgd()
1966 	cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);  in kernel_map_pages_in_pgd()
1968 	retval = __change_page_attr_set_clr(&cpa, 0);  in kernel_map_pages_in_pgd()