513 	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);  in rk_iommu_iova_to_phys()  local
519 	spin_lock_irqsave(&rk_domain->dt_lock, flags);  in rk_iommu_iova_to_phys()
521 	dte = rk_domain->dt[rk_iova_dte_index(iova)];  in rk_iommu_iova_to_phys()
533 	spin_unlock_irqrestore(&rk_domain->dt_lock, flags);  in rk_iommu_iova_to_phys()
538 static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain,  in rk_iommu_zap_iova()  argument
545 	spin_lock_irqsave(&rk_domain->iommus_lock, flags);  in rk_iommu_zap_iova()
546 	list_for_each(pos, &rk_domain->iommus) {  in rk_iommu_zap_iova()
551 	spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);  in rk_iommu_zap_iova()
554 static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain,  in rk_iommu_zap_iova_first_last()  argument
557 	rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE);  in rk_iommu_zap_iova_first_last()
559 		rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE,  in rk_iommu_zap_iova_first_last()
563 static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,  in rk_dte_get_page_table()  argument
570 	assert_spin_locked(&rk_domain->dt_lock);  in rk_dte_get_page_table()
572 	dte_addr = &rk_domain->dt[rk_iova_dte_index(iova)];  in rk_dte_get_page_table()
592 static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain,  in rk_iommu_unmap_iova()  argument
598 	assert_spin_locked(&rk_domain->dt_lock);  in rk_iommu_unmap_iova()
613 static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr,  in rk_iommu_map_iova()  argument
621 	assert_spin_locked(&rk_domain->dt_lock);  in rk_iommu_map_iova()
642 	rk_iommu_zap_iova_first_last(rk_domain, iova, size);  in rk_iommu_map_iova()
647 	rk_iommu_unmap_iova(rk_domain, pte_addr, iova, pte_count * SPAGE_SIZE);  in rk_iommu_map_iova()
660 	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);  in rk_iommu_map()  local
666 	spin_lock_irqsave(&rk_domain->dt_lock, flags);  in rk_iommu_map()
675 	page_table = rk_dte_get_page_table(rk_domain, iova);  in rk_iommu_map()
677 		spin_unlock_irqrestore(&rk_domain->dt_lock, flags);  in rk_iommu_map()
682 	ret = rk_iommu_map_iova(rk_domain, pte_addr, iova, paddr, size, prot);  in rk_iommu_map()
683 	spin_unlock_irqrestore(&rk_domain->dt_lock, flags);  in rk_iommu_map()
691 	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);  in rk_iommu_unmap()  local
699 	spin_lock_irqsave(&rk_domain->dt_lock, flags);  in rk_iommu_unmap()
708 	dte = rk_domain->dt[rk_iova_dte_index(iova)];  in rk_iommu_unmap()
711 		spin_unlock_irqrestore(&rk_domain->dt_lock, flags);  in rk_iommu_unmap()
717 	unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, iova, size);  in rk_iommu_unmap()
719 	spin_unlock_irqrestore(&rk_domain->dt_lock, flags);  in rk_iommu_unmap()
722 	rk_iommu_zap_iova(rk_domain, iova, unmap_size);  in rk_iommu_unmap()
747 	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);  in rk_iommu_attach_device()  local
775 	dte_addr = virt_to_phys(rk_domain->dt);  in rk_iommu_attach_device()
784 	spin_lock_irqsave(&rk_domain->iommus_lock, flags);  in rk_iommu_attach_device()
785 	list_add_tail(&iommu->node, &rk_domain->iommus);  in rk_iommu_attach_device()
786 	spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);  in rk_iommu_attach_device()
799 	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);  in rk_iommu_detach_device()  local
807 	spin_lock_irqsave(&rk_domain->iommus_lock, flags);  in rk_iommu_detach_device()
809 	spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);  in rk_iommu_detach_device()
827 	struct rk_iommu_domain *rk_domain;  in rk_iommu_domain_alloc()  local
832 	rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL);  in rk_iommu_domain_alloc()
833 	if (!rk_domain)  in rk_iommu_domain_alloc()
841 	rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);  in rk_iommu_domain_alloc()
842 	if (!rk_domain->dt)  in rk_iommu_domain_alloc()
845 	rk_table_flush(rk_domain->dt, NUM_DT_ENTRIES);  in rk_iommu_domain_alloc()
847 	spin_lock_init(&rk_domain->iommus_lock);  in rk_iommu_domain_alloc()
848 	spin_lock_init(&rk_domain->dt_lock);  in rk_iommu_domain_alloc()
849 	INIT_LIST_HEAD(&rk_domain->iommus);  in rk_iommu_domain_alloc()
851 	return &rk_domain->domain;  in rk_iommu_domain_alloc()
854 	kfree(rk_domain);  in rk_iommu_domain_alloc()
860 	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);  in rk_iommu_domain_free()  local
863 	WARN_ON(!list_empty(&rk_domain->iommus));  in rk_iommu_domain_free()
866 		u32 dte = rk_domain->dt[i];  in rk_iommu_domain_free()
874 	free_page((unsigned long)rk_domain->dt);  in rk_iommu_domain_free()
875 	kfree(rk_domain);  in rk_iommu_domain_free()