1196 	u64 *pte;  in increase_address_space()  local
1202 	pte = (void *)get_zeroed_page(gfp);  in increase_address_space()
1203 	if (!pte)  in increase_address_space()
1206 	*pte             = PM_LEVEL_PDE(domain->mode,  in increase_address_space()
1208 	domain->pt_root  = pte;  in increase_address_space()
1222 	u64 *pte, *page;  in alloc_pte()  local
1230 	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1235 		if (!IOMMU_PTE_PRESENT(*pte)) {  in alloc_pte()
1239 			*pte = PM_LEVEL_PDE(level, virt_to_phys(page));  in alloc_pte()
1243 		if (PM_PTE_LEVEL(*pte) != level)  in alloc_pte()
1248 		pte = IOMMU_PTE_PAGE(*pte);  in alloc_pte()
1251 			*pte_page = pte;  in alloc_pte()
1253 		pte = &pte[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1256 	return pte;  in alloc_pte()
1268 	u64 *pte;  in fetch_pte()  local
1274 	pte	   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1280 		if (!IOMMU_PTE_PRESENT(*pte))  in fetch_pte()
1284 		if (PM_PTE_LEVEL(*pte) == 7 ||  in fetch_pte()
1285 		    PM_PTE_LEVEL(*pte) == 0)  in fetch_pte()
1289 		if (PM_PTE_LEVEL(*pte) != level)  in fetch_pte()
1295 		pte	   = IOMMU_PTE_PAGE(*pte);  in fetch_pte()
1296 		pte	   = &pte[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1300 	if (PM_PTE_LEVEL(*pte) == 0x07) {  in fetch_pte()
1307 		*page_size = pte_mask = PTE_PAGE_SIZE(*pte);  in fetch_pte()
1309 		pte        = (u64 *)(((unsigned long)pte) & pte_mask);  in fetch_pte()
1312 	return pte;  in fetch_pte()
1328 	u64 __pte, *pte;  in iommu_map_page()  local
1338 	pte   = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);  in iommu_map_page()
1340 	if (!pte)  in iommu_map_page()
1344 		if (IOMMU_PTE_PRESENT(pte[i]))  in iommu_map_page()
1359 		pte[i] = __pte;  in iommu_map_page()
1372 	u64 *pte;  in iommu_unmap_page()  local
1380 		pte = fetch_pte(dom, bus_addr, &unmap_size);  in iommu_unmap_page()
1382 		if (pte) {  in iommu_unmap_page()
1387 				pte[i] = 0ULL;  in iommu_unmap_page()
1467 		u64 *pte, *pte_page;  in alloc_new_range()  local
1470 			pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,  in alloc_new_range()
1472 			if (!pte)  in alloc_new_range()
1519 		u64 *pte = fetch_pte(&dma_dom->domain, i, &pte_pgsize);  in alloc_new_range()  local
1520 		if (!pte || !IOMMU_PTE_PRESENT(*pte))  in alloc_new_range()
2349 	u64 *pte, *pte_page;  in dma_ops_get_pte()  local
2355 	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];  in dma_ops_get_pte()
2356 	if (!pte) {  in dma_ops_get_pte()
2357 		pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,  in dma_ops_get_pte()
2361 		pte += PM_LEVEL_INDEX(0, address);  in dma_ops_get_pte()
2365 	return pte;  in dma_ops_get_pte()
2377 	u64 *pte, __pte;  in dma_ops_domain_map()  local
2383 	pte  = dma_ops_get_pte(dom, address);  in dma_ops_domain_map()
2384 	if (!pte)  in dma_ops_domain_map()
2396 	WARN_ON(*pte);  in dma_ops_domain_map()
2398 	*pte = __pte;  in dma_ops_domain_map()
2410 	u64 *pte;  in dma_ops_domain_unmap()  local
2419 	pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];  in dma_ops_domain_unmap()
2420 	if (!pte)  in dma_ops_domain_unmap()
2423 	pte += PM_LEVEL_INDEX(0, address);  in dma_ops_domain_unmap()
2425 	WARN_ON(!*pte);  in dma_ops_domain_unmap()
2427 	*pte = 0ULL;  in dma_ops_domain_unmap()
3087 	u64 *pte, __pte;  in amd_iommu_iova_to_phys()  local
3092 	pte = fetch_pte(domain, iova, &pte_pgsize);  in amd_iommu_iova_to_phys()
3094 	if (!pte || !IOMMU_PTE_PRESENT(*pte))  in amd_iommu_iova_to_phys()
3098 	__pte	    = *pte & PM_ADDR_MASK;  in amd_iommu_iova_to_phys()
3376 	u64 *pte;  in __get_gcr3_pte()  local
3381 		pte   = &root[index];  in __get_gcr3_pte()
3386 		if (!(*pte & GCR3_VALID)) {  in __get_gcr3_pte()
3394 			*pte = __pa(root) | GCR3_VALID;  in __get_gcr3_pte()
3397 		root = __va(*pte & PAGE_MASK);  in __get_gcr3_pte()
3402 	return pte;  in __get_gcr3_pte()
3408 	u64 *pte;  in __set_gcr3()  local
3413 	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);  in __set_gcr3()
3414 	if (pte == NULL)  in __set_gcr3()
3417 	*pte = (cr3 & PAGE_MASK) | GCR3_VALID;  in __set_gcr3()
3424 	u64 *pte;  in __clear_gcr3()  local
3429 	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);  in __clear_gcr3()
3430 	if (pte == NULL)  in __clear_gcr3()
3433 	*pte = 0;  in __clear_gcr3()