1263 	u64 *pte;  in increase_address_space()  local
1269 	pte = (void *)get_zeroed_page(gfp);  in increase_address_space()
1270 	if (!pte)  in increase_address_space()
1273 	*pte             = PM_LEVEL_PDE(domain->mode,  in increase_address_space()
1275 	domain->pt_root  = pte;  in increase_address_space()
1289 	u64 *pte, *page;  in alloc_pte()  local
1297 	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1302 		if (!IOMMU_PTE_PRESENT(*pte)) {  in alloc_pte()
1306 			*pte = PM_LEVEL_PDE(level, virt_to_phys(page));  in alloc_pte()
1310 		if (PM_PTE_LEVEL(*pte) != level)  in alloc_pte()
1315 		pte = IOMMU_PTE_PAGE(*pte);  in alloc_pte()
1318 			*pte_page = pte;  in alloc_pte()
1320 		pte = &pte[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1323 	return pte;  in alloc_pte()
1335 	u64 *pte;  in fetch_pte()  local
1341 	pte	   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1347 		if (!IOMMU_PTE_PRESENT(*pte))  in fetch_pte()
1351 		if (PM_PTE_LEVEL(*pte) == 7 ||  in fetch_pte()
1352 		    PM_PTE_LEVEL(*pte) == 0)  in fetch_pte()
1356 		if (PM_PTE_LEVEL(*pte) != level)  in fetch_pte()
1362 		pte	   = IOMMU_PTE_PAGE(*pte);  in fetch_pte()
1363 		pte	   = &pte[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1367 	if (PM_PTE_LEVEL(*pte) == 0x07) {  in fetch_pte()
1374 		*page_size = pte_mask = PTE_PAGE_SIZE(*pte);  in fetch_pte()
1376 		pte        = (u64 *)(((unsigned long)pte) & pte_mask);  in fetch_pte()
1379 	return pte;  in fetch_pte()
1395 	u64 __pte, *pte;  in iommu_map_page()  local
1405 	pte   = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);  in iommu_map_page()
1407 	if (!pte)  in iommu_map_page()
1411 		if (IOMMU_PTE_PRESENT(pte[i]))  in iommu_map_page()
1426 		pte[i] = __pte;  in iommu_map_page()
1439 	u64 *pte;  in iommu_unmap_page()  local
1447 		pte = fetch_pte(dom, bus_addr, &unmap_size);  in iommu_unmap_page()
1449 		if (pte) {  in iommu_unmap_page()
1454 				pte[i] = 0ULL;  in iommu_unmap_page()
1622 		u64 *pte, *pte_page;  in alloc_new_range()  local
1625 			pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,  in alloc_new_range()
1627 			if (!pte)  in alloc_new_range()
1674 		u64 *pte = fetch_pte(&dma_dom->domain, i, &pte_pgsize);  in alloc_new_range()  local
1675 		if (!pte || !IOMMU_PTE_PRESENT(*pte))  in alloc_new_range()
2556 	u64 *pte, *pte_page;  in dma_ops_get_pte()  local
2562 	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];  in dma_ops_get_pte()
2563 	if (!pte) {  in dma_ops_get_pte()
2564 		pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,  in dma_ops_get_pte()
2568 		pte += PM_LEVEL_INDEX(0, address);  in dma_ops_get_pte()
2572 	return pte;  in dma_ops_get_pte()
2584 	u64 *pte, __pte;  in dma_ops_domain_map()  local
2590 	pte  = dma_ops_get_pte(dom, address);  in dma_ops_domain_map()
2591 	if (!pte)  in dma_ops_domain_map()
2603 	WARN_ON(*pte);  in dma_ops_domain_map()
2605 	*pte = __pte;  in dma_ops_domain_map()
2617 	u64 *pte;  in dma_ops_domain_unmap()  local
2626 	pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];  in dma_ops_domain_unmap()
2627 	if (!pte)  in dma_ops_domain_unmap()
2630 	pte += PM_LEVEL_INDEX(0, address);  in dma_ops_domain_unmap()
2632 	WARN_ON(!*pte);  in dma_ops_domain_unmap()
2634 	*pte = 0ULL;  in dma_ops_domain_unmap()
3392 	u64 *pte, __pte;  in amd_iommu_iova_to_phys()  local
3397 	pte = fetch_pte(domain, iova, &pte_pgsize);  in amd_iommu_iova_to_phys()
3399 	if (!pte || !IOMMU_PTE_PRESENT(*pte))  in amd_iommu_iova_to_phys()
3403 	__pte	    = *pte & PM_ADDR_MASK;  in amd_iommu_iova_to_phys()
3657 	u64 *pte;  in __get_gcr3_pte()  local
3662 		pte   = &root[index];  in __get_gcr3_pte()
3667 		if (!(*pte & GCR3_VALID)) {  in __get_gcr3_pte()
3675 			*pte = __pa(root) | GCR3_VALID;  in __get_gcr3_pte()
3678 		root = __va(*pte & PAGE_MASK);  in __get_gcr3_pte()
3683 	return pte;  in __get_gcr3_pte()
3689 	u64 *pte;  in __set_gcr3()  local
3694 	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);  in __set_gcr3()
3695 	if (pte == NULL)  in __set_gcr3()
3698 	*pte = (cr3 & PAGE_MASK) | GCR3_VALID;  in __set_gcr3()
3705 	u64 *pte;  in __clear_gcr3()  local
3710 	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);  in __clear_gcr3()
3711 	if (pte == NULL)  in __clear_gcr3()
3714 	*pte = 0;  in __clear_gcr3()