168 #define iopte_deref(pte,d)					\  argument
169 	(__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)	\
172 #define iopte_type(pte,l)					\  argument
173 	(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
175 #define iopte_prot(pte)	((pte) & ARM_LPAE_PTE_ATTR_MASK)  argument
177 #define iopte_leaf(pte,l)					\  argument
179 		(iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) :	\
180 		(iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK))
182 #define iopte_to_pfn(pte,d)					\  argument
183 	(((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) >> (d)->pg_shift)
212 	arm_lpae_iopte pte = prot;  in arm_lpae_init_pte()  local
232 		pte |= ARM_LPAE_PTE_NS;  in arm_lpae_init_pte()
235 		pte |= ARM_LPAE_PTE_TYPE_PAGE;  in arm_lpae_init_pte()
237 		pte |= ARM_LPAE_PTE_TYPE_BLOCK;  in arm_lpae_init_pte()
239 	pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS;  in arm_lpae_init_pte()
240 	pte |= pfn_to_iopte(paddr >> data->pg_shift, data);  in arm_lpae_init_pte()
242 	*ptep = pte;  in arm_lpae_init_pte()
251 	arm_lpae_iopte *cptep, pte;  in __arm_lpae_map()  local
267 	pte = *ptep;  in __arm_lpae_map()
268 	if (!pte) {  in __arm_lpae_map()
276 		pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE;  in __arm_lpae_map()
278 			pte |= ARM_LPAE_PTE_NSTABLE;  in __arm_lpae_map()
279 		*ptep = pte;  in __arm_lpae_map()
282 		cptep = iopte_deref(pte, data);  in __arm_lpae_map()
292 	arm_lpae_iopte pte;  in arm_lpae_prot_to_pte()  local
296 		pte = ARM_LPAE_PTE_AP_UNPRIV | ARM_LPAE_PTE_nG;  in arm_lpae_prot_to_pte()
299 			pte |= ARM_LPAE_PTE_AP_RDONLY;  in arm_lpae_prot_to_pte()
302 			pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE  in arm_lpae_prot_to_pte()
305 		pte = ARM_LPAE_PTE_HAP_FAULT;  in arm_lpae_prot_to_pte()
307 			pte |= ARM_LPAE_PTE_HAP_READ;  in arm_lpae_prot_to_pte()
309 			pte |= ARM_LPAE_PTE_HAP_WRITE;  in arm_lpae_prot_to_pte()
311 			pte |= ARM_LPAE_PTE_MEMATTR_OIWB;  in arm_lpae_prot_to_pte()
313 			pte |= ARM_LPAE_PTE_MEMATTR_NC;  in arm_lpae_prot_to_pte()
317 		pte |= ARM_LPAE_PTE_XN;  in arm_lpae_prot_to_pte()
319 	return pte;  in arm_lpae_prot_to_pte()
358 		arm_lpae_iopte pte = *ptep++;  in __arm_lpae_free_pgtable()  local
360 		if (!pte || iopte_leaf(pte, lvl))  in __arm_lpae_free_pgtable()
363 		__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));  in __arm_lpae_free_pgtable()
423 	arm_lpae_iopte pte;  in __arm_lpae_unmap()  local
429 	pte = *ptep;  in __arm_lpae_unmap()
432 	if (WARN_ON(!pte || (lvl == ARM_LPAE_MAX_LEVELS)))  in __arm_lpae_unmap()
440 		if (!iopte_leaf(pte, lvl)) {  in __arm_lpae_unmap()
444 			ptep = iopte_deref(pte, data);  in __arm_lpae_unmap()
451 	} else if (iopte_leaf(pte, lvl)) {  in __arm_lpae_unmap()
457 						iopte_prot(pte), lvl, ptep,  in __arm_lpae_unmap()
462 	ptep = iopte_deref(pte, data);  in __arm_lpae_unmap()
486 	arm_lpae_iopte pte, *ptep = data->pgd;  in arm_lpae_iova_to_phys()  local
495 		pte = *(ptep + ARM_LPAE_LVL_IDX(iova, lvl, data));  in arm_lpae_iova_to_phys()
498 		if (!pte)  in arm_lpae_iova_to_phys()
502 		if (iopte_leaf(pte,lvl))  in arm_lpae_iova_to_phys()
506 		ptep = iopte_deref(pte, data);  in arm_lpae_iova_to_phys()
514 	return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova;  in arm_lpae_iova_to_phys()