1314 	struct arm_smmu_domain *smmu_domain = cookie;  in arm_smmu_tlb_sync()  local
1315 	__arm_smmu_tlb_sync(smmu_domain->smmu);  in arm_smmu_tlb_sync()
1320 	struct arm_smmu_domain *smmu_domain = cookie;  in arm_smmu_tlb_inv_context()  local
1321 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_context()
1324 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_tlb_inv_context()
1326 		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;  in arm_smmu_tlb_inv_context()
1330 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;  in arm_smmu_tlb_inv_context()
1340 	struct arm_smmu_domain *smmu_domain = cookie;  in arm_smmu_tlb_inv_range_nosync()  local
1341 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_range_nosync()
1349 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_tlb_inv_range_nosync()
1351 		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;  in arm_smmu_tlb_inv_range_nosync()
1354 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;  in arm_smmu_tlb_inv_range_nosync()
1383 	struct arm_smmu_domain *smmu_domain;  in arm_smmu_domain_alloc()  local
1393 	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);  in arm_smmu_domain_alloc()
1394 	if (!smmu_domain)  in arm_smmu_domain_alloc()
1397 	mutex_init(&smmu_domain->init_mutex);  in arm_smmu_domain_alloc()
1398 	spin_lock_init(&smmu_domain->pgtbl_lock);  in arm_smmu_domain_alloc()
1399 	return &smmu_domain->domain;  in arm_smmu_domain_alloc()
1422 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_free()  local
1423 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_free()
1425 	free_io_pgtable_ops(smmu_domain->pgtbl_ops);  in arm_smmu_domain_free()
1428 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_domain_free()
1429 		struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;  in arm_smmu_domain_free()
1432 			dma_free_coherent(smmu_domain->smmu->dev,  in arm_smmu_domain_free()
1440 		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;  in arm_smmu_domain_free()
1445 	kfree(smmu_domain);  in arm_smmu_domain_free()
1448 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,  in arm_smmu_domain_finalise_s1()  argument
1453 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_finalise_s1()
1454 	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;  in arm_smmu_domain_finalise_s1()
1479 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,  in arm_smmu_domain_finalise_s2()  argument
1483 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_finalise_s2()
1484 	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;  in arm_smmu_domain_finalise_s2()
1507 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_finalise()  local
1508 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_finalise()
1512 		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;  in arm_smmu_domain_finalise()
1514 		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;  in arm_smmu_domain_finalise()
1516 	switch (smmu_domain->stage) {  in arm_smmu_domain_finalise()
1542 	pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);  in arm_smmu_domain_finalise()
1547 	smmu_domain->pgtbl_ops = pgtbl_ops;  in arm_smmu_domain_finalise()
1549 	ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);  in arm_smmu_domain_finalise()
1595 	struct arm_smmu_domain *smmu_domain = smmu_group->domain;  in arm_smmu_install_ste_for_group()  local
1599 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_install_ste_for_group()
1600 		ste->s1_cfg = &smmu_domain->s1_cfg;  in arm_smmu_install_ste_for_group()
1605 		ste->s2_cfg = &smmu_domain->s2_cfg;  in arm_smmu_install_ste_for_group()
1622 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_attach_dev()  local
1629 	if (smmu_group->domain && smmu_group->domain != smmu_domain)  in arm_smmu_attach_dev()
1633 	mutex_lock(&smmu_domain->init_mutex);  in arm_smmu_attach_dev()
1635 	if (!smmu_domain->smmu) {  in arm_smmu_attach_dev()
1636 		smmu_domain->smmu = smmu;  in arm_smmu_attach_dev()
1639 			smmu_domain->smmu = NULL;  in arm_smmu_attach_dev()
1642 	} else if (smmu_domain->smmu != smmu) {  in arm_smmu_attach_dev()
1645 			dev_name(smmu_domain->smmu->dev),  in arm_smmu_attach_dev()
1655 	smmu_group->domain	= smmu_domain;  in arm_smmu_attach_dev()
1663 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_attach_dev()
1669 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_detach_dev()  local
1672 	BUG_ON(!smmu_domain);  in arm_smmu_detach_dev()
1675 	mutex_lock(&smmu_domain->init_mutex);  in arm_smmu_detach_dev()
1676 	BUG_ON(smmu_group->domain != smmu_domain);  in arm_smmu_detach_dev()
1683 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_detach_dev()
1691 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_map()  local
1692 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;  in arm_smmu_map()
1697 	spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_map()
1699 	spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_map()
1708 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_unmap()  local
1709 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;  in arm_smmu_unmap()
1714 	spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_unmap()
1716 	spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_unmap()
1725 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_iova_to_phys()  local
1726 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;  in arm_smmu_iova_to_phys()
1731 	spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_iova_to_phys()
1733 	spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);  in arm_smmu_iova_to_phys()
1873 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_get_attr()  local
1877 		*(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);  in arm_smmu_domain_get_attr()
1888 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);  in arm_smmu_domain_set_attr()  local
1890 	mutex_lock(&smmu_domain->init_mutex);  in arm_smmu_domain_set_attr()
1894 		if (smmu_domain->smmu) {  in arm_smmu_domain_set_attr()
1900 			smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;  in arm_smmu_domain_set_attr()
1902 			smmu_domain->stage = ARM_SMMU_DOMAIN_S1;  in arm_smmu_domain_set_attr()
1910 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_domain_set_attr()