Lines Matching refs:cpa
374 static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address, in _lookup_address_cpa() argument
377 if (cpa->pgd) in _lookup_address_cpa()
378 return lookup_address_in_pgd(cpa->pgd + pgd_index(address), in _lookup_address_cpa()
462 struct cpa_data *cpa) in try_preserve_large_page() argument
470 if (cpa->force_split) in try_preserve_large_page()
478 tmp = _lookup_address_cpa(cpa, address, &level); in try_preserve_large_page()
501 if (numpages < cpa->numpages) in try_preserve_large_page()
502 cpa->numpages = numpages; in try_preserve_large_page()
512 pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr); in try_preserve_large_page()
513 pgprot_val(req_prot) |= pgprot_val(cpa->mask_set); in try_preserve_large_page()
540 cpa->pfn = pfn; in try_preserve_large_page()
575 if (address == (address & pmask) && cpa->numpages == (psize >> PAGE_SHIFT)) { in try_preserve_large_page()
582 cpa->flags |= CPA_FLUSHTLB; in try_preserve_large_page()
593 __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, in __split_large_page() argument
607 tmp = _lookup_address_cpa(cpa, address, &level); in __split_large_page()
680 static int split_large_page(struct cpa_data *cpa, pte_t *kpte, in split_large_page() argument
693 if (__split_large_page(cpa, kpte, address, base)) in split_large_page()
878 static void populate_pte(struct cpa_data *cpa, in populate_pte() argument
890 cpa->pfn &= ~_PAGE_NX; in populate_pte()
892 set_pte(pte, pfn_pte(cpa->pfn >> PAGE_SHIFT, pgprot)); in populate_pte()
895 cpa->pfn += PAGE_SIZE; in populate_pte()
900 static int populate_pmd(struct cpa_data *cpa, in populate_pmd() argument
927 populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot); in populate_pmd()
951 set_pmd(pmd, __pmd(cpa->pfn | _PAGE_PSE | in populate_pmd()
955 cpa->pfn += PMD_SIZE; in populate_pmd()
968 populate_pte(cpa, start, end, num_pages - cur_pages, in populate_pmd()
974 static int populate_pud(struct cpa_data *cpa, unsigned long start, pgd_t *pgd, in populate_pud() argument
982 end = start + (cpa->numpages << PAGE_SHIFT); in populate_pud()
994 cur_pages = min_t(int, (int)cpa->numpages, cur_pages); in populate_pud()
1005 cur_pages = populate_pmd(cpa, start, pre_end, cur_pages, in populate_pud()
1014 if (cpa->numpages == cur_pages) in populate_pud()
1024 set_pud(pud, __pud(cpa->pfn | _PAGE_PSE | in populate_pud()
1028 cpa->pfn += PUD_SIZE; in populate_pud()
1042 tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages, in populate_pud()
1056 static int populate_pgd(struct cpa_data *cpa, unsigned long addr) in populate_pgd() argument
1063 pgd_entry = cpa->pgd + pgd_index(addr); in populate_pgd()
1076 pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr); in populate_pgd()
1077 pgprot_val(pgprot) |= pgprot_val(cpa->mask_set); in populate_pgd()
1079 ret = populate_pud(cpa, addr, pgd_entry, pgprot); in populate_pgd()
1081 unmap_pgd_range(cpa->pgd, addr, in populate_pgd()
1082 addr + (cpa->numpages << PAGE_SHIFT)); in populate_pgd()
1086 cpa->numpages = ret; in populate_pgd()
1090 static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr, in __cpa_process_fault() argument
1093 if (cpa->pgd) in __cpa_process_fault()
1094 return populate_pgd(cpa, vaddr); in __cpa_process_fault()
1111 cpa->numpages = 1; in __cpa_process_fault()
1112 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT; in __cpa_process_fault()
1117 *cpa->vaddr); in __cpa_process_fault()
1123 static int __change_page_attr(struct cpa_data *cpa, int primary) in __change_page_attr() argument
1130 if (cpa->flags & CPA_PAGES_ARRAY) { in __change_page_attr()
1131 struct page *page = cpa->pages[cpa->curpage]; in __change_page_attr()
1135 } else if (cpa->flags & CPA_ARRAY) in __change_page_attr()
1136 address = cpa->vaddr[cpa->curpage]; in __change_page_attr()
1138 address = *cpa->vaddr; in __change_page_attr()
1140 kpte = _lookup_address_cpa(cpa, address, &level); in __change_page_attr()
1142 return __cpa_process_fault(cpa, address, primary); in __change_page_attr()
1146 return __cpa_process_fault(cpa, address, primary); in __change_page_attr()
1153 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); in __change_page_attr()
1154 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set); in __change_page_attr()
1176 cpa->pfn = pfn; in __change_page_attr()
1182 cpa->flags |= CPA_FLUSHTLB; in __change_page_attr()
1184 cpa->numpages = 1; in __change_page_attr()
1192 do_split = try_preserve_large_page(kpte, address, cpa); in __change_page_attr()
1204 err = split_large_page(cpa, kpte, address); in __change_page_attr()
1231 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
1233 static int cpa_process_alias(struct cpa_data *cpa) in cpa_process_alias() argument
1236 unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT); in cpa_process_alias()
1240 if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1)) in cpa_process_alias()
1247 if (cpa->flags & CPA_PAGES_ARRAY) { in cpa_process_alias()
1248 struct page *page = cpa->pages[cpa->curpage]; in cpa_process_alias()
1252 } else if (cpa->flags & CPA_ARRAY) in cpa_process_alias()
1253 vaddr = cpa->vaddr[cpa->curpage]; in cpa_process_alias()
1255 vaddr = *cpa->vaddr; in cpa_process_alias()
1260 alias_cpa = *cpa; in cpa_process_alias()
1276 within(cpa->pfn, highmap_start_pfn(), highmap_end_pfn())) { in cpa_process_alias()
1277 unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + in cpa_process_alias()
1279 alias_cpa = *cpa; in cpa_process_alias()
1294 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias) in __change_page_attr_set_clr() argument
1296 int ret, numpages = cpa->numpages; in __change_page_attr_set_clr()
1303 cpa->numpages = numpages; in __change_page_attr_set_clr()
1305 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) in __change_page_attr_set_clr()
1306 cpa->numpages = 1; in __change_page_attr_set_clr()
1310 ret = __change_page_attr(cpa, checkalias); in __change_page_attr_set_clr()
1317 ret = cpa_process_alias(cpa); in __change_page_attr_set_clr()
1327 BUG_ON(cpa->numpages > numpages || !cpa->numpages); in __change_page_attr_set_clr()
1328 numpages -= cpa->numpages; in __change_page_attr_set_clr()
1329 if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) in __change_page_attr_set_clr()
1330 cpa->curpage++; in __change_page_attr_set_clr()
1332 *cpa->vaddr += cpa->numpages * PAGE_SIZE; in __change_page_attr_set_clr()
1343 struct cpa_data cpa; in change_page_attr_set_clr() local
1347 memset(&cpa, 0, sizeof(cpa)); in change_page_attr_set_clr()
1391 cpa.vaddr = addr; in change_page_attr_set_clr()
1392 cpa.pages = pages; in change_page_attr_set_clr()
1393 cpa.numpages = numpages; in change_page_attr_set_clr()
1394 cpa.mask_set = mask_set; in change_page_attr_set_clr()
1395 cpa.mask_clr = mask_clr; in change_page_attr_set_clr()
1396 cpa.flags = 0; in change_page_attr_set_clr()
1397 cpa.curpage = 0; in change_page_attr_set_clr()
1398 cpa.force_split = force_split; in change_page_attr_set_clr()
1401 cpa.flags |= in_flag; in change_page_attr_set_clr()
1406 ret = __change_page_attr_set_clr(&cpa, checkalias); in change_page_attr_set_clr()
1411 if (!(cpa.flags & CPA_FLUSHTLB)) in change_page_attr_set_clr()
1427 if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { in change_page_attr_set_clr()
1429 cpa.flags, pages); in change_page_attr_set_clr()
1801 struct cpa_data cpa = { .vaddr = &tempaddr, in __set_pages_p() local
1814 return __change_page_attr_set_clr(&cpa, 0); in __set_pages_p()
1820 struct cpa_data cpa = { .vaddr = &tempaddr, in __set_pages_np() local
1833 return __change_page_attr_set_clr(&cpa, 0); in __set_pages_np()
1887 struct cpa_data cpa = { in kernel_map_pages_in_pgd() local
1901 cpa.mask_clr = __pgprot(_PAGE_NX); in kernel_map_pages_in_pgd()
1903 cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags); in kernel_map_pages_in_pgd()
1905 retval = __change_page_attr_set_clr(&cpa, 0); in kernel_map_pages_in_pgd()