559 	struct arm_smmu_domain *smmu_domain = cookie;  in arm_smmu_tlb_sync()  local
560 	__arm_smmu_tlb_sync(smmu_domain->smmu);  in arm_smmu_tlb_sync()
565 	struct arm_smmu_domain *smmu_domain = cookie;  in arm_smmu_tlb_inv_context()  local
566 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;  in arm_smmu_tlb_inv_context()
567 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_context()
587 	struct arm_smmu_domain *smmu_domain = cookie;  in arm_smmu_tlb_inv_range_nosync()  local
588 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;  in arm_smmu_tlb_inv_range_nosync()
589 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_range_nosync()
633 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_context_fault()  local
634 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;  in arm_smmu_context_fault()
635 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_context_fault()
704 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,  in arm_smmu_init_context_bank()  argument
710 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;  in arm_smmu_init_context_bank()
711 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_init_context_bank()
804 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_init_domain_context()  local
805 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;  in arm_smmu_init_domain_context()
807 	mutex_lock(&smmu_domain->init_mutex);  in arm_smmu_init_domain_context()
808 	if (smmu_domain->smmu)  in arm_smmu_init_domain_context()
830 		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;  in arm_smmu_init_domain_context()
832 		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;  in arm_smmu_init_domain_context()
834 	switch (smmu_domain->stage) {  in arm_smmu_init_domain_context()
886 	smmu_domain->smmu = smmu;  in arm_smmu_init_domain_context()
887 	pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);  in arm_smmu_init_domain_context()
897 	arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);  in arm_smmu_init_domain_context()
912 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_init_domain_context()
915 	smmu_domain->pgtbl_ops = pgtbl_ops;  in arm_smmu_init_domain_context()
919 	smmu_domain->smmu = NULL;  in arm_smmu_init_domain_context()
921 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_init_domain_context()
927 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_destroy_domain_context()  local
928 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_destroy_domain_context()
929 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;  in arm_smmu_destroy_domain_context()
948 	if (smmu_domain->pgtbl_ops)  in arm_smmu_destroy_domain_context()
949 		free_io_pgtable_ops(smmu_domain->pgtbl_ops);  in arm_smmu_destroy_domain_context()
956 	struct arm_smmu_domain *smmu_domain;  in arm_smmu_domain_alloc()  local
965 	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);  in arm_smmu_domain_alloc()
966 	if (!smmu_domain)  in arm_smmu_domain_alloc()
969 	mutex_init(&smmu_domain->init_mutex);  in arm_smmu_domain_alloc()
970 	spin_lock_init(&smmu_domain->pgtbl_lock);  in arm_smmu_domain_alloc()
972 	return &smmu_domain->domain;  in arm_smmu_domain_alloc()
977 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_free()  local
984 	kfree(smmu_domain);  in arm_smmu_domain_free()
1062 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,  in arm_smmu_domain_add_master()  argument
1066 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_add_master()
1079 		       (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT);  in arm_smmu_domain_add_master()
1086 static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,  in arm_smmu_domain_remove_master()  argument
1090 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_remove_master()
1114 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_attach_dev()  local
1138 	if (smmu_domain->smmu != smmu) {  in arm_smmu_attach_dev()
1141 			dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));  in arm_smmu_attach_dev()
1150 	ret = arm_smmu_domain_add_master(smmu_domain, cfg);  in arm_smmu_attach_dev()
1158 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_detach_dev()  local
1166 	arm_smmu_domain_remove_master(smmu_domain, cfg);  in arm_smmu_detach_dev()
1174 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_map()  local
1175 	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;  in arm_smmu_map()
1180 	spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_map()
1182 	spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_map()
1191 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_unmap()  local
1192 	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;  in arm_smmu_unmap()
1197 	spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_unmap()
1199 	spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_unmap()
1206 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_iova_to_phys_hard()  local
1207 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_iova_to_phys_hard()
1208 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;  in arm_smmu_iova_to_phys_hard()
1209 	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;  in arm_smmu_iova_to_phys_hard()
1250 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_iova_to_phys()  local
1251 	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;  in arm_smmu_iova_to_phys()
1256 	spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_iova_to_phys()
1257 	if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&  in arm_smmu_iova_to_phys()
1258 			smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_iova_to_phys()
1264 	spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_iova_to_phys()
1397 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_get_attr()  local
1401 		*(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);  in arm_smmu_domain_get_attr()
1412 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_set_attr()  local
1414 	mutex_lock(&smmu_domain->init_mutex);  in arm_smmu_domain_set_attr()
1418 		if (smmu_domain->smmu) {  in arm_smmu_domain_set_attr()
1424 			smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;  in arm_smmu_domain_set_attr()
1426 			smmu_domain->stage = ARM_SMMU_DOMAIN_S1;  in arm_smmu_domain_set_attr()
1434 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_domain_set_attr()