244 static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)  in iommu_report_fault()  argument
251 		*da = 0;  in iommu_report_fault()
256 	*da = fault_addr;  in iommu_report_fault()
322 	if (e->da & ~(get_cam_va_mask(e->pgsz))) {  in iotlb_alloc_cr()
324 			e->da);  in iotlb_alloc_cr()
332 	cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid;  in iotlb_alloc_cr()
421 static void flush_iotlb_page(struct omap_iommu *obj, u32 da)  in flush_iotlb_page()  argument
438 		if ((start <= da) && (da < start + bytes)) {  in flush_iotlb_page()
440 				__func__, start, da, bytes);  in flush_iotlb_page()
449 		dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);  in flush_iotlb_page()
624 static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)  in iopte_alloc()  argument
653 	iopte = iopte_offset(iopgd, da);  in iopte_alloc()
657 		 __func__, da, iopgd, *iopgd, iopte, *iopte);  in iopte_alloc()
662 static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopgd_alloc_section()  argument
664 	u32 *iopgd = iopgd_offset(obj, da);  in iopgd_alloc_section()
666 	if ((da | pa) & ~IOSECTION_MASK) {  in iopgd_alloc_section()
668 			__func__, da, pa, IOSECTION_SIZE);  in iopgd_alloc_section()
677 static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopgd_alloc_super()  argument
679 	u32 *iopgd = iopgd_offset(obj, da);  in iopgd_alloc_super()
682 	if ((da | pa) & ~IOSUPER_MASK) {  in iopgd_alloc_super()
684 			__func__, da, pa, IOSUPER_SIZE);  in iopgd_alloc_super()
694 static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopte_alloc_page()  argument
696 	u32 *iopgd = iopgd_offset(obj, da);  in iopte_alloc_page()
697 	u32 *iopte = iopte_alloc(obj, iopgd, da);  in iopte_alloc_page()
706 		 __func__, da, pa, iopte, *iopte);  in iopte_alloc_page()
711 static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopte_alloc_large()  argument
713 	u32 *iopgd = iopgd_offset(obj, da);  in iopte_alloc_large()
714 	u32 *iopte = iopte_alloc(obj, iopgd, da);  in iopte_alloc_large()
717 	if ((da | pa) & ~IOLARGE_MASK) {  in iopte_alloc_large()
719 			__func__, da, pa, IOLARGE_SIZE);  in iopte_alloc_large()
764 	err = fn(obj, e->da, e->pa, prot);  in iopgtable_store_entry_core()
780 	flush_iotlb_page(obj, e->da);  in omap_iopgtable_store_entry()
795 iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)  in iopgtable_lookup_entry()  argument
799 	iopgd = iopgd_offset(obj, da);  in iopgtable_lookup_entry()
804 		iopte = iopte_offset(iopgd, da);  in iopgtable_lookup_entry()
810 static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)  in iopgtable_clear_entry_core()  argument
813 	u32 *iopgd = iopgd_offset(obj, da);  in iopgtable_clear_entry_core()
821 		u32 *iopte = iopte_offset(iopgd, da);  in iopgtable_clear_entry_core()
827 			iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));  in iopgtable_clear_entry_core()
848 			iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));  in iopgtable_clear_entry_core()
863 static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)  in iopgtable_clear_entry()  argument
869 	bytes = iopgtable_clear_entry_core(obj, da);  in iopgtable_clear_entry()
870 	flush_iotlb_page(obj, da);  in iopgtable_clear_entry()
884 		u32 da;  in iopgtable_clear_entry_all()  local
887 		da = i << IOPGD_SHIFT;  in iopgtable_clear_entry_all()
888 		iopgd = iopgd_offset(obj, da);  in iopgtable_clear_entry_all()
910 	u32 da, errs;  in iommu_fault_handler()  local
919 	errs = iommu_report_fault(obj, &da);  in iommu_fault_handler()
924 	if (!report_iommu_fault(domain, obj->dev, da, 0))  in iommu_fault_handler()
929 	iopgd = iopgd_offset(obj, da);  in iommu_fault_handler()
933 				obj->name, errs, da, iopgd, *iopgd);  in iommu_fault_handler()
937 	iopte = iopte_offset(iopgd, da);  in iommu_fault_handler()
940 			obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);  in iommu_fault_handler()
1108 static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)  in iotlb_init_entry()  argument
1112 	e->da		= da;  in iotlb_init_entry()
1123 static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,  in omap_iommu_map()  argument
1139 	dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%x\n", da, &pa, bytes);  in omap_iommu_map()
1141 	iotlb_init_entry(&e, da, pa, omap_pgsz);  in omap_iommu_map()
1150 static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,  in omap_iommu_unmap()  argument
1157 	dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size);  in omap_iommu_unmap()
1159 	return iopgtable_clear_entry(oiommu, da);  in omap_iommu_unmap()
1288 					  dma_addr_t da)  in omap_iommu_iova_to_phys()  argument
1296 	iopgtable_lookup_entry(oiommu, da, &pgd, &pte);  in omap_iommu_iova_to_phys()
1300 			ret = omap_iommu_translate(*pte, da, IOPTE_MASK);  in omap_iommu_iova_to_phys()
1302 			ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);  in omap_iommu_iova_to_phys()
1305 							(unsigned long long)da);  in omap_iommu_iova_to_phys()
1308 			ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);  in omap_iommu_iova_to_phys()
1310 			ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);  in omap_iommu_iova_to_phys()
1313 							(unsigned long long)da);  in omap_iommu_iova_to_phys()