1/* 2 * IOMMU API for ARM architected SMMU implementations. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * along with this program; if not, write to the Free Software 15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * 17 * Copyright (C) 2013 ARM Limited 18 * 19 * Author: Will Deacon <will.deacon@arm.com> 20 * 21 * This driver currently supports: 22 * - SMMUv1 and v2 implementations 23 * - Stream-matching and stream-indexing 24 * - v7/v8 long-descriptor format 25 * - Non-secure access to the SMMU 26 * - Context fault reporting 27 */ 28 29#define pr_fmt(fmt) "arm-smmu: " fmt 30 31#include <linux/delay.h> 32#include <linux/dma-mapping.h> 33#include <linux/err.h> 34#include <linux/interrupt.h> 35#include <linux/io.h> 36#include <linux/iommu.h> 37#include <linux/iopoll.h> 38#include <linux/module.h> 39#include <linux/of.h> 40#include <linux/pci.h> 41#include <linux/platform_device.h> 42#include <linux/slab.h> 43#include <linux/spinlock.h> 44 45#include <linux/amba/bus.h> 46 47#include "io-pgtable.h" 48 49/* Maximum number of stream IDs assigned to a single device */ 50#define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS 51 52/* Maximum number of context banks per SMMU */ 53#define ARM_SMMU_MAX_CBS 128 54 55/* Maximum number of mapping groups per SMMU */ 56#define ARM_SMMU_MAX_SMRS 128 57 58/* SMMU global address space */ 59#define ARM_SMMU_GR0(smmu) ((smmu)->base) 60#define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift)) 61 62/* 63 * SMMU global address space with conditional offset to access secure 64 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448, 65 * nsGFSYNR0: 0x450) 66 */ 67#define ARM_SMMU_GR0_NS(smmu) \ 68 ((smmu)->base + \ 69 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \ 70 ? 0x400 : 0)) 71 72/* Configuration registers */ 73#define ARM_SMMU_GR0_sCR0 0x0 74#define sCR0_CLIENTPD (1 << 0) 75#define sCR0_GFRE (1 << 1) 76#define sCR0_GFIE (1 << 2) 77#define sCR0_GCFGFRE (1 << 4) 78#define sCR0_GCFGFIE (1 << 5) 79#define sCR0_USFCFG (1 << 10) 80#define sCR0_VMIDPNE (1 << 11) 81#define sCR0_PTM (1 << 12) 82#define sCR0_FB (1 << 13) 83#define sCR0_BSU_SHIFT 14 84#define sCR0_BSU_MASK 0x3 85 86/* Identification registers */ 87#define ARM_SMMU_GR0_ID0 0x20 88#define ARM_SMMU_GR0_ID1 0x24 89#define ARM_SMMU_GR0_ID2 0x28 90#define ARM_SMMU_GR0_ID3 0x2c 91#define ARM_SMMU_GR0_ID4 0x30 92#define ARM_SMMU_GR0_ID5 0x34 93#define ARM_SMMU_GR0_ID6 0x38 94#define ARM_SMMU_GR0_ID7 0x3c 95#define ARM_SMMU_GR0_sGFSR 0x48 96#define ARM_SMMU_GR0_sGFSYNR0 0x50 97#define ARM_SMMU_GR0_sGFSYNR1 0x54 98#define ARM_SMMU_GR0_sGFSYNR2 0x58 99 100#define ID0_S1TS (1 << 30) 101#define ID0_S2TS (1 << 29) 102#define ID0_NTS (1 << 28) 103#define ID0_SMS (1 << 27) 104#define ID0_ATOSNS (1 << 26) 105#define ID0_CTTW (1 << 14) 106#define ID0_NUMIRPT_SHIFT 16 107#define ID0_NUMIRPT_MASK 0xff 108#define ID0_NUMSIDB_SHIFT 9 109#define ID0_NUMSIDB_MASK 0xf 110#define ID0_NUMSMRG_SHIFT 0 111#define ID0_NUMSMRG_MASK 0xff 112 113#define ID1_PAGESIZE (1 << 31) 114#define ID1_NUMPAGENDXB_SHIFT 28 115#define ID1_NUMPAGENDXB_MASK 7 116#define ID1_NUMS2CB_SHIFT 16 117#define ID1_NUMS2CB_MASK 0xff 118#define ID1_NUMCB_SHIFT 0 119#define ID1_NUMCB_MASK 0xff 120 121#define ID2_OAS_SHIFT 4 122#define ID2_OAS_MASK 0xf 123#define ID2_IAS_SHIFT 0 124#define ID2_IAS_MASK 0xf 125#define ID2_UBS_SHIFT 8 126#define ID2_UBS_MASK 0xf 127#define ID2_PTFS_4K (1 << 12) 128#define ID2_PTFS_16K (1 << 13) 129#define ID2_PTFS_64K (1 << 14) 130 131/* Global TLB invalidation */ 132#define ARM_SMMU_GR0_TLBIVMID 0x64 133#define ARM_SMMU_GR0_TLBIALLNSNH 0x68 134#define ARM_SMMU_GR0_TLBIALLH 0x6c 135#define ARM_SMMU_GR0_sTLBGSYNC 0x70 136#define ARM_SMMU_GR0_sTLBGSTATUS 0x74 137#define sTLBGSTATUS_GSACTIVE (1 << 0) 138#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ 139 140/* Stream mapping registers */ 141#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2)) 142#define SMR_VALID (1 << 31) 143#define SMR_MASK_SHIFT 16 144#define SMR_MASK_MASK 0x7fff 145#define SMR_ID_SHIFT 0 146#define SMR_ID_MASK 0x7fff 147 148#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2)) 149#define S2CR_CBNDX_SHIFT 0 150#define S2CR_CBNDX_MASK 0xff 151#define S2CR_TYPE_SHIFT 16 152#define S2CR_TYPE_MASK 0x3 153#define S2CR_TYPE_TRANS (0 << S2CR_TYPE_SHIFT) 154#define S2CR_TYPE_BYPASS (1 << S2CR_TYPE_SHIFT) 155#define S2CR_TYPE_FAULT (2 << S2CR_TYPE_SHIFT) 156 157/* Context bank attribute registers */ 158#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) 159#define CBAR_VMID_SHIFT 0 160#define CBAR_VMID_MASK 0xff 161#define CBAR_S1_BPSHCFG_SHIFT 8 162#define CBAR_S1_BPSHCFG_MASK 3 163#define CBAR_S1_BPSHCFG_NSH 3 164#define CBAR_S1_MEMATTR_SHIFT 12 165#define CBAR_S1_MEMATTR_MASK 0xf 166#define CBAR_S1_MEMATTR_WB 0xf 167#define CBAR_TYPE_SHIFT 16 168#define CBAR_TYPE_MASK 0x3 169#define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT) 170#define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT) 171#define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT) 172#define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT) 173#define CBAR_IRPTNDX_SHIFT 24 174#define CBAR_IRPTNDX_MASK 0xff 175 176#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) 177#define CBA2R_RW64_32BIT (0 << 0) 178#define CBA2R_RW64_64BIT (1 << 0) 179 180/* Translation context bank */ 181#define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1)) 182#define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift)) 183 184#define ARM_SMMU_CB_SCTLR 0x0 185#define ARM_SMMU_CB_RESUME 0x8 186#define ARM_SMMU_CB_TTBCR2 0x10 187#define ARM_SMMU_CB_TTBR0_LO 0x20 188#define ARM_SMMU_CB_TTBR0_HI 0x24 189#define ARM_SMMU_CB_TTBR1_LO 0x28 190#define ARM_SMMU_CB_TTBR1_HI 0x2c 191#define ARM_SMMU_CB_TTBCR 0x30 192#define ARM_SMMU_CB_S1_MAIR0 0x38 193#define ARM_SMMU_CB_S1_MAIR1 0x3c 194#define ARM_SMMU_CB_PAR_LO 0x50 195#define ARM_SMMU_CB_PAR_HI 0x54 196#define ARM_SMMU_CB_FSR 0x58 197#define ARM_SMMU_CB_FAR_LO 0x60 198#define ARM_SMMU_CB_FAR_HI 0x64 199#define ARM_SMMU_CB_FSYNR0 0x68 200#define ARM_SMMU_CB_S1_TLBIVA 0x600 201#define ARM_SMMU_CB_S1_TLBIASID 0x610 202#define ARM_SMMU_CB_S1_TLBIVAL 0x620 203#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630 204#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638 205#define ARM_SMMU_CB_ATS1PR_LO 0x800 206#define ARM_SMMU_CB_ATS1PR_HI 0x804 207#define ARM_SMMU_CB_ATSR 0x8f0 208 209#define SCTLR_S1_ASIDPNE (1 << 12) 210#define SCTLR_CFCFG (1 << 7) 211#define SCTLR_CFIE (1 << 6) 212#define SCTLR_CFRE (1 << 5) 213#define SCTLR_E (1 << 4) 214#define SCTLR_AFE (1 << 2) 215#define SCTLR_TRE (1 << 1) 216#define SCTLR_M (1 << 0) 217#define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE) 218 219#define CB_PAR_F (1 << 0) 220 221#define ATSR_ACTIVE (1 << 0) 222 223#define RESUME_RETRY (0 << 0) 224#define RESUME_TERMINATE (1 << 0) 225 226#define TTBCR2_SEP_SHIFT 15 227#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT) 228 229#define TTBRn_HI_ASID_SHIFT 16 230 231#define FSR_MULTI (1 << 31) 232#define FSR_SS (1 << 30) 233#define FSR_UUT (1 << 8) 234#define FSR_ASF (1 << 7) 235#define FSR_TLBLKF (1 << 6) 236#define FSR_TLBMCF (1 << 5) 237#define FSR_EF (1 << 4) 238#define FSR_PF (1 << 3) 239#define FSR_AFF (1 << 2) 240#define FSR_TF (1 << 1) 241 242#define FSR_IGN (FSR_AFF | FSR_ASF | \ 243 FSR_TLBMCF | FSR_TLBLKF) 244#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \ 245 FSR_EF | FSR_PF | FSR_TF | FSR_IGN) 246 247#define FSYNR0_WNR (1 << 4) 248 249static int force_stage; 250module_param_named(force_stage, force_stage, int, S_IRUGO | S_IWUSR); 251MODULE_PARM_DESC(force_stage, 252 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); 253 254enum arm_smmu_arch_version { 255 ARM_SMMU_V1 = 1, 256 ARM_SMMU_V2, 257}; 258 259struct arm_smmu_smr { 260 u8 idx; 261 u16 mask; 262 u16 id; 263}; 264 265struct arm_smmu_master_cfg { 266 int num_streamids; 267 u16 streamids[MAX_MASTER_STREAMIDS]; 268 struct arm_smmu_smr *smrs; 269}; 270 271struct arm_smmu_master { 272 struct device_node *of_node; 273 struct rb_node node; 274 struct arm_smmu_master_cfg cfg; 275}; 276 277struct arm_smmu_device { 278 struct device *dev; 279 280 void __iomem *base; 281 unsigned long size; 282 unsigned long pgshift; 283 284#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0) 285#define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1) 286#define ARM_SMMU_FEAT_TRANS_S1 (1 << 2) 287#define ARM_SMMU_FEAT_TRANS_S2 (1 << 3) 288#define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) 289#define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) 290 u32 features; 291 292#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) 293 u32 options; 294 enum arm_smmu_arch_version version; 295 296 u32 num_context_banks; 297 u32 num_s2_context_banks; 298 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS); 299 atomic_t irptndx; 300 301 u32 num_mapping_groups; 302 DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS); 303 304 unsigned long va_size; 305 unsigned long ipa_size; 306 unsigned long pa_size; 307 308 u32 num_global_irqs; 309 u32 num_context_irqs; 310 unsigned int *irqs; 311 312 struct list_head list; 313 struct rb_root masters; 314}; 315 316struct arm_smmu_cfg { 317 u8 cbndx; 318 u8 irptndx; 319 u32 cbar; 320}; 321#define INVALID_IRPTNDX 0xff 322 323#define ARM_SMMU_CB_ASID(cfg) ((cfg)->cbndx) 324#define ARM_SMMU_CB_VMID(cfg) ((cfg)->cbndx + 1) 325 326enum arm_smmu_domain_stage { 327 ARM_SMMU_DOMAIN_S1 = 0, 328 ARM_SMMU_DOMAIN_S2, 329 ARM_SMMU_DOMAIN_NESTED, 330}; 331 332struct arm_smmu_domain { 333 struct arm_smmu_device *smmu; 334 struct io_pgtable_ops *pgtbl_ops; 335 spinlock_t pgtbl_lock; 336 struct arm_smmu_cfg cfg; 337 enum arm_smmu_domain_stage stage; 338 struct mutex init_mutex; /* Protects smmu pointer */ 339 struct iommu_domain domain; 340}; 341 342static struct iommu_ops arm_smmu_ops; 343 344static DEFINE_SPINLOCK(arm_smmu_devices_lock); 345static LIST_HEAD(arm_smmu_devices); 346 347struct arm_smmu_option_prop { 348 u32 opt; 349 const char *prop; 350}; 351 352static struct arm_smmu_option_prop arm_smmu_options[] = { 353 { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" }, 354 { 0, NULL}, 355}; 356 357static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) 358{ 359 return container_of(dom, struct arm_smmu_domain, domain); 360} 361 362static void parse_driver_options(struct arm_smmu_device *smmu) 363{ 364 int i = 0; 365 366 do { 367 if (of_property_read_bool(smmu->dev->of_node, 368 arm_smmu_options[i].prop)) { 369 smmu->options |= arm_smmu_options[i].opt; 370 dev_notice(smmu->dev, "option %s\n", 371 arm_smmu_options[i].prop); 372 } 373 } while (arm_smmu_options[++i].opt); 374} 375 376static struct device_node *dev_get_dev_node(struct device *dev) 377{ 378 if (dev_is_pci(dev)) { 379 struct pci_bus *bus = to_pci_dev(dev)->bus; 380 381 while (!pci_is_root_bus(bus)) 382 bus = bus->parent; 383 return bus->bridge->parent->of_node; 384 } 385 386 return dev->of_node; 387} 388 389static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu, 390 struct device_node *dev_node) 391{ 392 struct rb_node *node = smmu->masters.rb_node; 393 394 while (node) { 395 struct arm_smmu_master *master; 396 397 master = container_of(node, struct arm_smmu_master, node); 398 399 if (dev_node < master->of_node) 400 node = node->rb_left; 401 else if (dev_node > master->of_node) 402 node = node->rb_right; 403 else 404 return master; 405 } 406 407 return NULL; 408} 409 410static struct arm_smmu_master_cfg * 411find_smmu_master_cfg(struct device *dev) 412{ 413 struct arm_smmu_master_cfg *cfg = NULL; 414 struct iommu_group *group = iommu_group_get(dev); 415 416 if (group) { 417 cfg = iommu_group_get_iommudata(group); 418 iommu_group_put(group); 419 } 420 421 return cfg; 422} 423 424static int insert_smmu_master(struct arm_smmu_device *smmu, 425 struct arm_smmu_master *master) 426{ 427 struct rb_node **new, *parent; 428 429 new = &smmu->masters.rb_node; 430 parent = NULL; 431 while (*new) { 432 struct arm_smmu_master *this 433 = container_of(*new, struct arm_smmu_master, node); 434 435 parent = *new; 436 if (master->of_node < this->of_node) 437 new = &((*new)->rb_left); 438 else if (master->of_node > this->of_node) 439 new = &((*new)->rb_right); 440 else 441 return -EEXIST; 442 } 443 444 rb_link_node(&master->node, parent, new); 445 rb_insert_color(&master->node, &smmu->masters); 446 return 0; 447} 448 449static int register_smmu_master(struct arm_smmu_device *smmu, 450 struct device *dev, 451 struct of_phandle_args *masterspec) 452{ 453 int i; 454 struct arm_smmu_master *master; 455 456 master = find_smmu_master(smmu, masterspec->np); 457 if (master) { 458 dev_err(dev, 459 "rejecting multiple registrations for master device %s\n", 460 masterspec->np->name); 461 return -EBUSY; 462 } 463 464 if (masterspec->args_count > MAX_MASTER_STREAMIDS) { 465 dev_err(dev, 466 "reached maximum number (%d) of stream IDs for master device %s\n", 467 MAX_MASTER_STREAMIDS, masterspec->np->name); 468 return -ENOSPC; 469 } 470 471 master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL); 472 if (!master) 473 return -ENOMEM; 474 475 master->of_node = masterspec->np; 476 master->cfg.num_streamids = masterspec->args_count; 477 478 for (i = 0; i < master->cfg.num_streamids; ++i) { 479 u16 streamid = masterspec->args[i]; 480 481 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && 482 (streamid >= smmu->num_mapping_groups)) { 483 dev_err(dev, 484 "stream ID for master device %s greater than maximum allowed (%d)\n", 485 masterspec->np->name, smmu->num_mapping_groups); 486 return -ERANGE; 487 } 488 master->cfg.streamids[i] = streamid; 489 } 490 return insert_smmu_master(smmu, master); 491} 492 493static struct arm_smmu_device *find_smmu_for_device(struct device *dev) 494{ 495 struct arm_smmu_device *smmu; 496 struct arm_smmu_master *master = NULL; 497 struct device_node *dev_node = dev_get_dev_node(dev); 498 499 spin_lock(&arm_smmu_devices_lock); 500 list_for_each_entry(smmu, &arm_smmu_devices, list) { 501 master = find_smmu_master(smmu, dev_node); 502 if (master) 503 break; 504 } 505 spin_unlock(&arm_smmu_devices_lock); 506 507 return master ? smmu : NULL; 508} 509 510static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end) 511{ 512 int idx; 513 514 do { 515 idx = find_next_zero_bit(map, end, start); 516 if (idx == end) 517 return -ENOSPC; 518 } while (test_and_set_bit(idx, map)); 519 520 return idx; 521} 522 523static void __arm_smmu_free_bitmap(unsigned long *map, int idx) 524{ 525 clear_bit(idx, map); 526} 527 528/* Wait for any pending TLB invalidations to complete */ 529static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu) 530{ 531 int count = 0; 532 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 533 534 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC); 535 while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS) 536 & sTLBGSTATUS_GSACTIVE) { 537 cpu_relax(); 538 if (++count == TLB_LOOP_TIMEOUT) { 539 dev_err_ratelimited(smmu->dev, 540 "TLB sync timed out -- SMMU may be deadlocked\n"); 541 return; 542 } 543 udelay(1); 544 } 545} 546 547static void arm_smmu_tlb_sync(void *cookie) 548{ 549 struct arm_smmu_domain *smmu_domain = cookie; 550 __arm_smmu_tlb_sync(smmu_domain->smmu); 551} 552 553static void arm_smmu_tlb_inv_context(void *cookie) 554{ 555 struct arm_smmu_domain *smmu_domain = cookie; 556 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 557 struct arm_smmu_device *smmu = smmu_domain->smmu; 558 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 559 void __iomem *base; 560 561 if (stage1) { 562 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); 563 writel_relaxed(ARM_SMMU_CB_ASID(cfg), 564 base + ARM_SMMU_CB_S1_TLBIASID); 565 } else { 566 base = ARM_SMMU_GR0(smmu); 567 writel_relaxed(ARM_SMMU_CB_VMID(cfg), 568 base + ARM_SMMU_GR0_TLBIVMID); 569 } 570 571 __arm_smmu_tlb_sync(smmu); 572} 573 574static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, 575 bool leaf, void *cookie) 576{ 577 struct arm_smmu_domain *smmu_domain = cookie; 578 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 579 struct arm_smmu_device *smmu = smmu_domain->smmu; 580 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 581 void __iomem *reg; 582 583 if (stage1) { 584 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); 585 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; 586 587 if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) { 588 iova &= ~12UL; 589 iova |= ARM_SMMU_CB_ASID(cfg); 590 writel_relaxed(iova, reg); 591#ifdef CONFIG_64BIT 592 } else { 593 iova >>= 12; 594 iova |= (u64)ARM_SMMU_CB_ASID(cfg) << 48; 595 writeq_relaxed(iova, reg); 596#endif 597 } 598#ifdef CONFIG_64BIT 599 } else if (smmu->version == ARM_SMMU_V2) { 600 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); 601 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : 602 ARM_SMMU_CB_S2_TLBIIPAS2; 603 writeq_relaxed(iova >> 12, reg); 604#endif 605 } else { 606 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID; 607 writel_relaxed(ARM_SMMU_CB_VMID(cfg), reg); 608 } 609} 610 611static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) 612{ 613 struct arm_smmu_domain *smmu_domain = cookie; 614 struct arm_smmu_device *smmu = smmu_domain->smmu; 615 unsigned long offset = (unsigned long)addr & ~PAGE_MASK; 616 617 618 /* Ensure new page tables are visible to the hardware walker */ 619 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) { 620 dsb(ishst); 621 } else { 622 /* 623 * If the SMMU can't walk tables in the CPU caches, treat them 624 * like non-coherent DMA since we need to flush the new entries 625 * all the way out to memory. There's no possibility of 626 * recursion here as the SMMU table walker will not be wired 627 * through another SMMU. 628 */ 629 dma_map_page(smmu->dev, virt_to_page(addr), offset, size, 630 DMA_TO_DEVICE); 631 } 632} 633 634static struct iommu_gather_ops arm_smmu_gather_ops = { 635 .tlb_flush_all = arm_smmu_tlb_inv_context, 636 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, 637 .tlb_sync = arm_smmu_tlb_sync, 638 .flush_pgtable = arm_smmu_flush_pgtable, 639}; 640 641static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 642{ 643 int flags, ret; 644 u32 fsr, far, fsynr, resume; 645 unsigned long iova; 646 struct iommu_domain *domain = dev; 647 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 648 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 649 struct arm_smmu_device *smmu = smmu_domain->smmu; 650 void __iomem *cb_base; 651 652 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); 653 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); 654 655 if (!(fsr & FSR_FAULT)) 656 return IRQ_NONE; 657 658 if (fsr & FSR_IGN) 659 dev_err_ratelimited(smmu->dev, 660 "Unexpected context fault (fsr 0x%x)\n", 661 fsr); 662 663 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); 664 flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; 665 666 far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_LO); 667 iova = far; 668#ifdef CONFIG_64BIT 669 far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_HI); 670 iova |= ((unsigned long)far << 32); 671#endif 672 673 if (!report_iommu_fault(domain, smmu->dev, iova, flags)) { 674 ret = IRQ_HANDLED; 675 resume = RESUME_RETRY; 676 } else { 677 dev_err_ratelimited(smmu->dev, 678 "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n", 679 iova, fsynr, cfg->cbndx); 680 ret = IRQ_NONE; 681 resume = RESUME_TERMINATE; 682 } 683 684 /* Clear the faulting FSR */ 685 writel(fsr, cb_base + ARM_SMMU_CB_FSR); 686 687 /* Retry or terminate any stalled transactions */ 688 if (fsr & FSR_SS) 689 writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME); 690 691 return ret; 692} 693 694static irqreturn_t arm_smmu_global_fault(int irq, void *dev) 695{ 696 u32 gfsr, gfsynr0, gfsynr1, gfsynr2; 697 struct arm_smmu_device *smmu = dev; 698 void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu); 699 700 gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR); 701 gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0); 702 gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1); 703 gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2); 704 705 if (!gfsr) 706 return IRQ_NONE; 707 708 dev_err_ratelimited(smmu->dev, 709 "Unexpected global fault, this could be serious\n"); 710 dev_err_ratelimited(smmu->dev, 711 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", 712 gfsr, gfsynr0, gfsynr1, gfsynr2); 713 714 writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR); 715 return IRQ_HANDLED; 716} 717 718static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, 719 struct io_pgtable_cfg *pgtbl_cfg) 720{ 721 u32 reg; 722 bool stage1; 723 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 724 struct arm_smmu_device *smmu = smmu_domain->smmu; 725 void __iomem *cb_base, *gr0_base, *gr1_base; 726 727 gr0_base = ARM_SMMU_GR0(smmu); 728 gr1_base = ARM_SMMU_GR1(smmu); 729 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 730 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); 731 732 if (smmu->version > ARM_SMMU_V1) { 733 /* 734 * CBA2R. 735 * *Must* be initialised before CBAR thanks to VMID16 736 * architectural oversight affected some implementations. 737 */ 738#ifdef CONFIG_64BIT 739 reg = CBA2R_RW64_64BIT; 740#else 741 reg = CBA2R_RW64_32BIT; 742#endif 743 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx)); 744 } 745 746 /* CBAR */ 747 reg = cfg->cbar; 748 if (smmu->version == ARM_SMMU_V1) 749 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT; 750 751 /* 752 * Use the weakest shareability/memory types, so they are 753 * overridden by the ttbcr/pte. 754 */ 755 if (stage1) { 756 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) | 757 (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); 758 } else { 759 reg |= ARM_SMMU_CB_VMID(cfg) << CBAR_VMID_SHIFT; 760 } 761 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx)); 762 763 /* TTBRs */ 764 if (stage1) { 765 reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; 766 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO); 767 reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0] >> 32; 768 reg |= ARM_SMMU_CB_ASID(cfg) << TTBRn_HI_ASID_SHIFT; 769 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI); 770 771 reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; 772 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1_LO); 773 reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1] >> 32; 774 reg |= ARM_SMMU_CB_ASID(cfg) << TTBRn_HI_ASID_SHIFT; 775 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1_HI); 776 } else { 777 reg = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; 778 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO); 779 reg = pgtbl_cfg->arm_lpae_s2_cfg.vttbr >> 32; 780 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI); 781 } 782 783 /* TTBCR */ 784 if (stage1) { 785 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr; 786 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); 787 if (smmu->version > ARM_SMMU_V1) { 788 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32; 789 reg |= TTBCR2_SEP_UPSTREAM; 790 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2); 791 } 792 } else { 793 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr; 794 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); 795 } 796 797 /* MAIRs (stage-1 only) */ 798 if (stage1) { 799 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0]; 800 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0); 801 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1]; 802 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1); 803 } 804 805 /* SCTLR */ 806 reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP; 807 if (stage1) 808 reg |= SCTLR_S1_ASIDPNE; 809#ifdef __BIG_ENDIAN 810 reg |= SCTLR_E; 811#endif 812 writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); 813} 814 815static int arm_smmu_init_domain_context(struct iommu_domain *domain, 816 struct arm_smmu_device *smmu) 817{ 818 int irq, start, ret = 0; 819 unsigned long ias, oas; 820 struct io_pgtable_ops *pgtbl_ops; 821 struct io_pgtable_cfg pgtbl_cfg; 822 enum io_pgtable_fmt fmt; 823 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 824 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 825 826 mutex_lock(&smmu_domain->init_mutex); 827 if (smmu_domain->smmu) 828 goto out_unlock; 829 830 /* 831 * Mapping the requested stage onto what we support is surprisingly 832 * complicated, mainly because the spec allows S1+S2 SMMUs without 833 * support for nested translation. That means we end up with the 834 * following table: 835 * 836 * Requested Supported Actual 837 * S1 N S1 838 * S1 S1+S2 S1 839 * S1 S2 S2 840 * S1 S1 S1 841 * N N N 842 * N S1+S2 S2 843 * N S2 S2 844 * N S1 S1 845 * 846 * Note that you can't actually request stage-2 mappings. 847 */ 848 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) 849 smmu_domain->stage = ARM_SMMU_DOMAIN_S2; 850 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) 851 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 852 853 switch (smmu_domain->stage) { 854 case ARM_SMMU_DOMAIN_S1: 855 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; 856 start = smmu->num_s2_context_banks; 857 ias = smmu->va_size; 858 oas = smmu->ipa_size; 859 if (IS_ENABLED(CONFIG_64BIT)) 860 fmt = ARM_64_LPAE_S1; 861 else 862 fmt = ARM_32_LPAE_S1; 863 break; 864 case ARM_SMMU_DOMAIN_NESTED: 865 /* 866 * We will likely want to change this if/when KVM gets 867 * involved. 868 */ 869 case ARM_SMMU_DOMAIN_S2: 870 cfg->cbar = CBAR_TYPE_S2_TRANS; 871 start = 0; 872 ias = smmu->ipa_size; 873 oas = smmu->pa_size; 874 if (IS_ENABLED(CONFIG_64BIT)) 875 fmt = ARM_64_LPAE_S2; 876 else 877 fmt = ARM_32_LPAE_S2; 878 break; 879 default: 880 ret = -EINVAL; 881 goto out_unlock; 882 } 883 884 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, 885 smmu->num_context_banks); 886 if (IS_ERR_VALUE(ret)) 887 goto out_unlock; 888 889 cfg->cbndx = ret; 890 if (smmu->version == ARM_SMMU_V1) { 891 cfg->irptndx = atomic_inc_return(&smmu->irptndx); 892 cfg->irptndx %= smmu->num_context_irqs; 893 } else { 894 cfg->irptndx = cfg->cbndx; 895 } 896 897 pgtbl_cfg = (struct io_pgtable_cfg) { 898 .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap, 899 .ias = ias, 900 .oas = oas, 901 .tlb = &arm_smmu_gather_ops, 902 }; 903 904 smmu_domain->smmu = smmu; 905 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 906 if (!pgtbl_ops) { 907 ret = -ENOMEM; 908 goto out_clear_smmu; 909 } 910 911 /* Update our support page sizes to reflect the page table format */ 912 arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 913 914 /* Initialise the context bank with our page table cfg */ 915 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); 916 917 /* 918 * Request context fault interrupt. Do this last to avoid the 919 * handler seeing a half-initialised domain state. 920 */ 921 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; 922 ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED, 923 "arm-smmu-context-fault", domain); 924 if (IS_ERR_VALUE(ret)) { 925 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", 926 cfg->irptndx, irq); 927 cfg->irptndx = INVALID_IRPTNDX; 928 } 929 930 mutex_unlock(&smmu_domain->init_mutex); 931 932 /* Publish page table ops for map/unmap */ 933 smmu_domain->pgtbl_ops = pgtbl_ops; 934 return 0; 935 936out_clear_smmu: 937 smmu_domain->smmu = NULL; 938out_unlock: 939 mutex_unlock(&smmu_domain->init_mutex); 940 return ret; 941} 942 943static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) 944{ 945 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 946 struct arm_smmu_device *smmu = smmu_domain->smmu; 947 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 948 void __iomem *cb_base; 949 int irq; 950 951 if (!smmu) 952 return; 953 954 /* 955 * Disable the context bank and free the page tables before freeing 956 * it. 957 */ 958 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); 959 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); 960 961 if (cfg->irptndx != INVALID_IRPTNDX) { 962 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; 963 free_irq(irq, domain); 964 } 965 966 if (smmu_domain->pgtbl_ops) 967 free_io_pgtable_ops(smmu_domain->pgtbl_ops); 968 969 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 970} 971 972static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) 973{ 974 struct arm_smmu_domain *smmu_domain; 975 976 if (type != IOMMU_DOMAIN_UNMANAGED) 977 return NULL; 978 /* 979 * Allocate the domain and initialise some of its data structures. 980 * We can't really do anything meaningful until we've added a 981 * master. 982 */ 983 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 984 if (!smmu_domain) 985 return NULL; 986 987 mutex_init(&smmu_domain->init_mutex); 988 spin_lock_init(&smmu_domain->pgtbl_lock); 989 990 return &smmu_domain->domain; 991} 992 993static void arm_smmu_domain_free(struct iommu_domain *domain) 994{ 995 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 996 997 /* 998 * Free the domain resources. We assume that all devices have 999 * already been detached. 1000 */ 1001 arm_smmu_destroy_domain_context(domain); 1002 kfree(smmu_domain); 1003} 1004 1005static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu, 1006 struct arm_smmu_master_cfg *cfg) 1007{ 1008 int i; 1009 struct arm_smmu_smr *smrs; 1010 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 1011 1012 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH)) 1013 return 0; 1014 1015 if (cfg->smrs) 1016 return -EEXIST; 1017 1018 smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL); 1019 if (!smrs) { 1020 dev_err(smmu->dev, "failed to allocate %d SMRs\n", 1021 cfg->num_streamids); 1022 return -ENOMEM; 1023 } 1024 1025 /* Allocate the SMRs on the SMMU */ 1026 for (i = 0; i < cfg->num_streamids; ++i) { 1027 int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0, 1028 smmu->num_mapping_groups); 1029 if (IS_ERR_VALUE(idx)) { 1030 dev_err(smmu->dev, "failed to allocate free SMR\n"); 1031 goto err_free_smrs; 1032 } 1033 1034 smrs[i] = (struct arm_smmu_smr) { 1035 .idx = idx, 1036 .mask = 0, /* We don't currently share SMRs */ 1037 .id = cfg->streamids[i], 1038 }; 1039 } 1040 1041 /* It worked! Now, poke the actual hardware */ 1042 for (i = 0; i < cfg->num_streamids; ++i) { 1043 u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT | 1044 smrs[i].mask << SMR_MASK_SHIFT; 1045 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx)); 1046 } 1047 1048 cfg->smrs = smrs; 1049 return 0; 1050 1051err_free_smrs: 1052 while (--i >= 0) 1053 __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx); 1054 kfree(smrs); 1055 return -ENOSPC; 1056} 1057 1058static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu, 1059 struct arm_smmu_master_cfg *cfg) 1060{ 1061 int i; 1062 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 1063 struct arm_smmu_smr *smrs = cfg->smrs; 1064 1065 if (!smrs) 1066 return; 1067 1068 /* Invalidate the SMRs before freeing back to the allocator */ 1069 for (i = 0; i < cfg->num_streamids; ++i) { 1070 u8 idx = smrs[i].idx; 1071 1072 writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx)); 1073 __arm_smmu_free_bitmap(smmu->smr_map, idx); 1074 } 1075 1076 cfg->smrs = NULL; 1077 kfree(smrs); 1078} 1079 1080static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, 1081 struct arm_smmu_master_cfg *cfg) 1082{ 1083 int i, ret; 1084 struct arm_smmu_device *smmu = smmu_domain->smmu; 1085 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 1086 1087 /* Devices in an IOMMU group may already be configured */ 1088 ret = arm_smmu_master_configure_smrs(smmu, cfg); 1089 if (ret) 1090 return ret == -EEXIST ? 0 : ret; 1091 1092 for (i = 0; i < cfg->num_streamids; ++i) { 1093 u32 idx, s2cr; 1094 1095 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i]; 1096 s2cr = S2CR_TYPE_TRANS | 1097 (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT); 1098 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx)); 1099 } 1100 1101 return 0; 1102} 1103 1104static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, 1105 struct arm_smmu_master_cfg *cfg) 1106{ 1107 int i; 1108 struct arm_smmu_device *smmu = smmu_domain->smmu; 1109 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 1110 1111 /* An IOMMU group is torn down by the first device to be removed */ 1112 if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs) 1113 return; 1114 1115 /* 1116 * We *must* clear the S2CR first, because freeing the SMR means 1117 * that it can be re-allocated immediately. 1118 */ 1119 for (i = 0; i < cfg->num_streamids; ++i) { 1120 u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i]; 1121 1122 writel_relaxed(S2CR_TYPE_BYPASS, 1123 gr0_base + ARM_SMMU_GR0_S2CR(idx)); 1124 } 1125 1126 arm_smmu_master_free_smrs(smmu, cfg); 1127} 1128 1129static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 1130{ 1131 int ret; 1132 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1133 struct arm_smmu_device *smmu; 1134 struct arm_smmu_master_cfg *cfg; 1135 1136 smmu = find_smmu_for_device(dev); 1137 if (!smmu) { 1138 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); 1139 return -ENXIO; 1140 } 1141 1142 if (dev->archdata.iommu) { 1143 dev_err(dev, "already attached to IOMMU domain\n"); 1144 return -EEXIST; 1145 } 1146 1147 /* Ensure that the domain is finalised */ 1148 ret = arm_smmu_init_domain_context(domain, smmu); 1149 if (IS_ERR_VALUE(ret)) 1150 return ret; 1151 1152 /* 1153 * Sanity check the domain. We don't support domains across 1154 * different SMMUs. 1155 */ 1156 if (smmu_domain->smmu != smmu) { 1157 dev_err(dev, 1158 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", 1159 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev)); 1160 return -EINVAL; 1161 } 1162 1163 /* Looks ok, so add the device to the domain */ 1164 cfg = find_smmu_master_cfg(dev); 1165 if (!cfg) 1166 return -ENODEV; 1167 1168 ret = arm_smmu_domain_add_master(smmu_domain, cfg); 1169 if (!ret) 1170 dev->archdata.iommu = domain; 1171 return ret; 1172} 1173 1174static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) 1175{ 1176 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1177 struct arm_smmu_master_cfg *cfg; 1178 1179 cfg = find_smmu_master_cfg(dev); 1180 if (!cfg) 1181 return; 1182 1183 dev->archdata.iommu = NULL; 1184 arm_smmu_domain_remove_master(smmu_domain, cfg); 1185} 1186 1187static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, 1188 phys_addr_t paddr, size_t size, int prot) 1189{ 1190 int ret; 1191 unsigned long flags; 1192 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1193 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; 1194 1195 if (!ops) 1196 return -ENODEV; 1197 1198 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); 1199 ret = ops->map(ops, iova, paddr, size, prot); 1200 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); 1201 return ret; 1202} 1203 1204static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, 1205 size_t size) 1206{ 1207 size_t ret; 1208 unsigned long flags; 1209 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1210 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; 1211 1212 if (!ops) 1213 return 0; 1214 1215 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); 1216 ret = ops->unmap(ops, iova, size); 1217 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); 1218 return ret; 1219} 1220 1221static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, 1222 dma_addr_t iova) 1223{ 1224 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1225 struct arm_smmu_device *smmu = smmu_domain->smmu; 1226 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 1227 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; 1228 struct device *dev = smmu->dev; 1229 void __iomem *cb_base; 1230 u32 tmp; 1231 u64 phys; 1232 1233 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); 1234 1235 if (smmu->version == 1) { 1236 u32 reg = iova & ~0xfff; 1237 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO); 1238 } else { 1239 u32 reg = iova & ~0xfff; 1240 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO); 1241 reg = ((u64)iova & ~0xfff) >> 32; 1242 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_HI); 1243 } 1244 1245 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, 1246 !(tmp & ATSR_ACTIVE), 5, 50)) { 1247 dev_err(dev, 1248 "iova to phys timed out on 0x%pad. Falling back to software table walk.\n", 1249 &iova); 1250 return ops->iova_to_phys(ops, iova); 1251 } 1252 1253 phys = readl_relaxed(cb_base + ARM_SMMU_CB_PAR_LO); 1254 phys |= ((u64)readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32; 1255 1256 if (phys & CB_PAR_F) { 1257 dev_err(dev, "translation fault!\n"); 1258 dev_err(dev, "PAR = 0x%llx\n", phys); 1259 return 0; 1260 } 1261 1262 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); 1263} 1264 1265static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, 1266 dma_addr_t iova) 1267{ 1268 phys_addr_t ret; 1269 unsigned long flags; 1270 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1271 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; 1272 1273 if (!ops) 1274 return 0; 1275 1276 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); 1277 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && 1278 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 1279 ret = arm_smmu_iova_to_phys_hard(domain, iova); 1280 } else { 1281 ret = ops->iova_to_phys(ops, iova); 1282 } 1283 1284 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); 1285 1286 return ret; 1287} 1288 1289static bool arm_smmu_capable(enum iommu_cap cap) 1290{ 1291 switch (cap) { 1292 case IOMMU_CAP_CACHE_COHERENCY: 1293 /* 1294 * Return true here as the SMMU can always send out coherent 1295 * requests. 1296 */ 1297 return true; 1298 case IOMMU_CAP_INTR_REMAP: 1299 return true; /* MSIs are just memory writes */ 1300 case IOMMU_CAP_NOEXEC: 1301 return true; 1302 default: 1303 return false; 1304 } 1305} 1306 1307static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) 1308{ 1309 *((u16 *)data) = alias; 1310 return 0; /* Continue walking */ 1311} 1312 1313static void __arm_smmu_release_pci_iommudata(void *data) 1314{ 1315 kfree(data); 1316} 1317 1318static int arm_smmu_add_pci_device(struct pci_dev *pdev) 1319{ 1320 int i, ret; 1321 u16 sid; 1322 struct iommu_group *group; 1323 struct arm_smmu_master_cfg *cfg; 1324 1325 group = iommu_group_get_for_dev(&pdev->dev); 1326 if (IS_ERR(group)) 1327 return PTR_ERR(group); 1328 1329 cfg = iommu_group_get_iommudata(group); 1330 if (!cfg) { 1331 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 1332 if (!cfg) { 1333 ret = -ENOMEM; 1334 goto out_put_group; 1335 } 1336 1337 iommu_group_set_iommudata(group, cfg, 1338 __arm_smmu_release_pci_iommudata); 1339 } 1340 1341 if (cfg->num_streamids >= MAX_MASTER_STREAMIDS) { 1342 ret = -ENOSPC; 1343 goto out_put_group; 1344 } 1345 1346 /* 1347 * Assume Stream ID == Requester ID for now. 1348 * We need a way to describe the ID mappings in FDT. 1349 */ 1350 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid); 1351 for (i = 0; i < cfg->num_streamids; ++i) 1352 if (cfg->streamids[i] == sid) 1353 break; 1354 1355 /* Avoid duplicate SIDs, as this can lead to SMR conflicts */ 1356 if (i == cfg->num_streamids) 1357 cfg->streamids[cfg->num_streamids++] = sid; 1358 1359 return 0; 1360out_put_group: 1361 iommu_group_put(group); 1362 return ret; 1363} 1364 1365static int arm_smmu_add_platform_device(struct device *dev) 1366{ 1367 struct iommu_group *group; 1368 struct arm_smmu_master *master; 1369 struct arm_smmu_device *smmu = find_smmu_for_device(dev); 1370 1371 if (!smmu) 1372 return -ENODEV; 1373 1374 master = find_smmu_master(smmu, dev->of_node); 1375 if (!master) 1376 return -ENODEV; 1377 1378 /* No automatic group creation for platform devices */ 1379 group = iommu_group_alloc(); 1380 if (IS_ERR(group)) 1381 return PTR_ERR(group); 1382 1383 iommu_group_set_iommudata(group, &master->cfg, NULL); 1384 return iommu_group_add_device(group, dev); 1385} 1386 1387static int arm_smmu_add_device(struct device *dev) 1388{ 1389 if (dev_is_pci(dev)) 1390 return arm_smmu_add_pci_device(to_pci_dev(dev)); 1391 1392 return arm_smmu_add_platform_device(dev); 1393} 1394 1395static void arm_smmu_remove_device(struct device *dev) 1396{ 1397 iommu_group_remove_device(dev); 1398} 1399 1400static int arm_smmu_domain_get_attr(struct iommu_domain *domain, 1401 enum iommu_attr attr, void *data) 1402{ 1403 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1404 1405 switch (attr) { 1406 case DOMAIN_ATTR_NESTING: 1407 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); 1408 return 0; 1409 default: 1410 return -ENODEV; 1411 } 1412} 1413 1414static int arm_smmu_domain_set_attr(struct iommu_domain *domain, 1415 enum iommu_attr attr, void *data) 1416{ 1417 int ret = 0; 1418 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1419 1420 mutex_lock(&smmu_domain->init_mutex); 1421 1422 switch (attr) { 1423 case DOMAIN_ATTR_NESTING: 1424 if (smmu_domain->smmu) { 1425 ret = -EPERM; 1426 goto out_unlock; 1427 } 1428 1429 if (*(int *)data) 1430 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; 1431 else 1432 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 1433 1434 break; 1435 default: 1436 ret = -ENODEV; 1437 } 1438 1439out_unlock: 1440 mutex_unlock(&smmu_domain->init_mutex); 1441 return ret; 1442} 1443 1444static struct iommu_ops arm_smmu_ops = { 1445 .capable = arm_smmu_capable, 1446 .domain_alloc = arm_smmu_domain_alloc, 1447 .domain_free = arm_smmu_domain_free, 1448 .attach_dev = arm_smmu_attach_dev, 1449 .detach_dev = arm_smmu_detach_dev, 1450 .map = arm_smmu_map, 1451 .unmap = arm_smmu_unmap, 1452 .map_sg = default_iommu_map_sg, 1453 .iova_to_phys = arm_smmu_iova_to_phys, 1454 .add_device = arm_smmu_add_device, 1455 .remove_device = arm_smmu_remove_device, 1456 .domain_get_attr = arm_smmu_domain_get_attr, 1457 .domain_set_attr = arm_smmu_domain_set_attr, 1458 .pgsize_bitmap = -1UL, /* Restricted during device attach */ 1459}; 1460 1461static void arm_smmu_device_reset(struct arm_smmu_device *smmu) 1462{ 1463 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 1464 void __iomem *cb_base; 1465 int i = 0; 1466 u32 reg; 1467 1468 /* clear global FSR */ 1469 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); 1470 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); 1471 1472 /* Mark all SMRn as invalid and all S2CRn as bypass */ 1473 for (i = 0; i < smmu->num_mapping_groups; ++i) { 1474 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i)); 1475 writel_relaxed(S2CR_TYPE_BYPASS, 1476 gr0_base + ARM_SMMU_GR0_S2CR(i)); 1477 } 1478 1479 /* Make sure all context banks are disabled and clear CB_FSR */ 1480 for (i = 0; i < smmu->num_context_banks; ++i) { 1481 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i); 1482 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); 1483 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR); 1484 } 1485 1486 /* Invalidate the TLB, just in case */ 1487 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH); 1488 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH); 1489 1490 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); 1491 1492 /* Enable fault reporting */ 1493 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE); 1494 1495 /* Disable TLB broadcasting. */ 1496 reg |= (sCR0_VMIDPNE | sCR0_PTM); 1497 1498 /* Enable client access, but bypass when no mapping is found */ 1499 reg &= ~(sCR0_CLIENTPD | sCR0_USFCFG); 1500 1501 /* Disable forced broadcasting */ 1502 reg &= ~sCR0_FB; 1503 1504 /* Don't upgrade barriers */ 1505 reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT); 1506 1507 /* Push the button */ 1508 __arm_smmu_tlb_sync(smmu); 1509 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); 1510} 1511 1512static int arm_smmu_id_size_to_bits(int size) 1513{ 1514 switch (size) { 1515 case 0: 1516 return 32; 1517 case 1: 1518 return 36; 1519 case 2: 1520 return 40; 1521 case 3: 1522 return 42; 1523 case 4: 1524 return 44; 1525 case 5: 1526 default: 1527 return 48; 1528 } 1529} 1530 1531static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) 1532{ 1533 unsigned long size; 1534 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 1535 u32 id; 1536 1537 dev_notice(smmu->dev, "probing hardware configuration...\n"); 1538 dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version); 1539 1540 /* ID0 */ 1541 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0); 1542 1543 /* Restrict available stages based on module parameter */ 1544 if (force_stage == 1) 1545 id &= ~(ID0_S2TS | ID0_NTS); 1546 else if (force_stage == 2) 1547 id &= ~(ID0_S1TS | ID0_NTS); 1548 1549 if (id & ID0_S1TS) { 1550 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; 1551 dev_notice(smmu->dev, "\tstage 1 translation\n"); 1552 } 1553 1554 if (id & ID0_S2TS) { 1555 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; 1556 dev_notice(smmu->dev, "\tstage 2 translation\n"); 1557 } 1558 1559 if (id & ID0_NTS) { 1560 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; 1561 dev_notice(smmu->dev, "\tnested translation\n"); 1562 } 1563 1564 if (!(smmu->features & 1565 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) { 1566 dev_err(smmu->dev, "\tno translation support!\n"); 1567 return -ENODEV; 1568 } 1569 1570 if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) { 1571 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; 1572 dev_notice(smmu->dev, "\taddress translation ops\n"); 1573 } 1574 1575 if (id & ID0_CTTW) { 1576 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 1577 dev_notice(smmu->dev, "\tcoherent table walk\n"); 1578 } 1579 1580 if (id & ID0_SMS) { 1581 u32 smr, sid, mask; 1582 1583 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; 1584 smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) & 1585 ID0_NUMSMRG_MASK; 1586 if (smmu->num_mapping_groups == 0) { 1587 dev_err(smmu->dev, 1588 "stream-matching supported, but no SMRs present!\n"); 1589 return -ENODEV; 1590 } 1591 1592 smr = SMR_MASK_MASK << SMR_MASK_SHIFT; 1593 smr |= (SMR_ID_MASK << SMR_ID_SHIFT); 1594 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); 1595 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); 1596 1597 mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK; 1598 sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK; 1599 if ((mask & sid) != sid) { 1600 dev_err(smmu->dev, 1601 "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n", 1602 mask, sid); 1603 return -ENODEV; 1604 } 1605 1606 dev_notice(smmu->dev, 1607 "\tstream matching with %u register groups, mask 0x%x", 1608 smmu->num_mapping_groups, mask); 1609 } else { 1610 smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) & 1611 ID0_NUMSIDB_MASK; 1612 } 1613 1614 /* ID1 */ 1615 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1); 1616 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12; 1617 1618 /* Check for size mismatch of SMMU address space from mapped region */ 1619 size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1); 1620 size *= 2 << smmu->pgshift; 1621 if (smmu->size != size) 1622 dev_warn(smmu->dev, 1623 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n", 1624 size, smmu->size); 1625 1626 smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK; 1627 smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK; 1628 if (smmu->num_s2_context_banks > smmu->num_context_banks) { 1629 dev_err(smmu->dev, "impossible number of S2 context banks!\n"); 1630 return -ENODEV; 1631 } 1632 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", 1633 smmu->num_context_banks, smmu->num_s2_context_banks); 1634 1635 /* ID2 */ 1636 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2); 1637 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK); 1638 smmu->ipa_size = size; 1639 1640 /* The output mask is also applied for bypass */ 1641 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); 1642 smmu->pa_size = size; 1643 1644 /* 1645 * What the page table walker can address actually depends on which 1646 * descriptor format is in use, but since a) we don't know that yet, 1647 * and b) it can vary per context bank, this will have to do... 1648 */ 1649 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) 1650 dev_warn(smmu->dev, 1651 "failed to set DMA mask for table walker\n"); 1652 1653 if (smmu->version == ARM_SMMU_V1) { 1654 smmu->va_size = smmu->ipa_size; 1655 size = SZ_4K | SZ_2M | SZ_1G; 1656 } else { 1657 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; 1658 smmu->va_size = arm_smmu_id_size_to_bits(size); 1659#ifndef CONFIG_64BIT 1660 smmu->va_size = min(32UL, smmu->va_size); 1661#endif 1662 size = 0; 1663 if (id & ID2_PTFS_4K) 1664 size |= SZ_4K | SZ_2M | SZ_1G; 1665 if (id & ID2_PTFS_16K) 1666 size |= SZ_16K | SZ_32M; 1667 if (id & ID2_PTFS_64K) 1668 size |= SZ_64K | SZ_512M; 1669 } 1670 1671 arm_smmu_ops.pgsize_bitmap &= size; 1672 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size); 1673 1674 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) 1675 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", 1676 smmu->va_size, smmu->ipa_size); 1677 1678 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) 1679 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", 1680 smmu->ipa_size, smmu->pa_size); 1681 1682 return 0; 1683} 1684 1685static const struct of_device_id arm_smmu_of_match[] = { 1686 { .compatible = "arm,smmu-v1", .data = (void *)ARM_SMMU_V1 }, 1687 { .compatible = "arm,smmu-v2", .data = (void *)ARM_SMMU_V2 }, 1688 { .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 }, 1689 { .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 }, 1690 { .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 }, 1691 { }, 1692}; 1693MODULE_DEVICE_TABLE(of, arm_smmu_of_match); 1694 1695static int arm_smmu_device_dt_probe(struct platform_device *pdev) 1696{ 1697 const struct of_device_id *of_id; 1698 struct resource *res; 1699 struct arm_smmu_device *smmu; 1700 struct device *dev = &pdev->dev; 1701 struct rb_node *node; 1702 struct of_phandle_args masterspec; 1703 int num_irqs, i, err; 1704 1705 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 1706 if (!smmu) { 1707 dev_err(dev, "failed to allocate arm_smmu_device\n"); 1708 return -ENOMEM; 1709 } 1710 smmu->dev = dev; 1711 1712 of_id = of_match_node(arm_smmu_of_match, dev->of_node); 1713 smmu->version = (enum arm_smmu_arch_version)of_id->data; 1714 1715 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1716 smmu->base = devm_ioremap_resource(dev, res); 1717 if (IS_ERR(smmu->base)) 1718 return PTR_ERR(smmu->base); 1719 smmu->size = resource_size(res); 1720 1721 if (of_property_read_u32(dev->of_node, "#global-interrupts", 1722 &smmu->num_global_irqs)) { 1723 dev_err(dev, "missing #global-interrupts property\n"); 1724 return -ENODEV; 1725 } 1726 1727 num_irqs = 0; 1728 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) { 1729 num_irqs++; 1730 if (num_irqs > smmu->num_global_irqs) 1731 smmu->num_context_irqs++; 1732 } 1733 1734 if (!smmu->num_context_irqs) { 1735 dev_err(dev, "found %d interrupts but expected at least %d\n", 1736 num_irqs, smmu->num_global_irqs + 1); 1737 return -ENODEV; 1738 } 1739 1740 smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs, 1741 GFP_KERNEL); 1742 if (!smmu->irqs) { 1743 dev_err(dev, "failed to allocate %d irqs\n", num_irqs); 1744 return -ENOMEM; 1745 } 1746 1747 for (i = 0; i < num_irqs; ++i) { 1748 int irq = platform_get_irq(pdev, i); 1749 1750 if (irq < 0) { 1751 dev_err(dev, "failed to get irq index %d\n", i); 1752 return -ENODEV; 1753 } 1754 smmu->irqs[i] = irq; 1755 } 1756 1757 err = arm_smmu_device_cfg_probe(smmu); 1758 if (err) 1759 return err; 1760 1761 i = 0; 1762 smmu->masters = RB_ROOT; 1763 while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters", 1764 "#stream-id-cells", i, 1765 &masterspec)) { 1766 err = register_smmu_master(smmu, dev, &masterspec); 1767 if (err) { 1768 dev_err(dev, "failed to add master %s\n", 1769 masterspec.np->name); 1770 goto out_put_masters; 1771 } 1772 1773 i++; 1774 } 1775 dev_notice(dev, "registered %d master devices\n", i); 1776 1777 parse_driver_options(smmu); 1778 1779 if (smmu->version > ARM_SMMU_V1 && 1780 smmu->num_context_banks != smmu->num_context_irqs) { 1781 dev_err(dev, 1782 "found only %d context interrupt(s) but %d required\n", 1783 smmu->num_context_irqs, smmu->num_context_banks); 1784 err = -ENODEV; 1785 goto out_put_masters; 1786 } 1787 1788 for (i = 0; i < smmu->num_global_irqs; ++i) { 1789 err = request_irq(smmu->irqs[i], 1790 arm_smmu_global_fault, 1791 IRQF_SHARED, 1792 "arm-smmu global fault", 1793 smmu); 1794 if (err) { 1795 dev_err(dev, "failed to request global IRQ %d (%u)\n", 1796 i, smmu->irqs[i]); 1797 goto out_free_irqs; 1798 } 1799 } 1800 1801 INIT_LIST_HEAD(&smmu->list); 1802 spin_lock(&arm_smmu_devices_lock); 1803 list_add(&smmu->list, &arm_smmu_devices); 1804 spin_unlock(&arm_smmu_devices_lock); 1805 1806 arm_smmu_device_reset(smmu); 1807 return 0; 1808 1809out_free_irqs: 1810 while (i--) 1811 free_irq(smmu->irqs[i], smmu); 1812 1813out_put_masters: 1814 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) { 1815 struct arm_smmu_master *master 1816 = container_of(node, struct arm_smmu_master, node); 1817 of_node_put(master->of_node); 1818 } 1819 1820 return err; 1821} 1822 1823static int arm_smmu_device_remove(struct platform_device *pdev) 1824{ 1825 int i; 1826 struct device *dev = &pdev->dev; 1827 struct arm_smmu_device *curr, *smmu = NULL; 1828 struct rb_node *node; 1829 1830 spin_lock(&arm_smmu_devices_lock); 1831 list_for_each_entry(curr, &arm_smmu_devices, list) { 1832 if (curr->dev == dev) { 1833 smmu = curr; 1834 list_del(&smmu->list); 1835 break; 1836 } 1837 } 1838 spin_unlock(&arm_smmu_devices_lock); 1839 1840 if (!smmu) 1841 return -ENODEV; 1842 1843 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) { 1844 struct arm_smmu_master *master 1845 = container_of(node, struct arm_smmu_master, node); 1846 of_node_put(master->of_node); 1847 } 1848 1849 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) 1850 dev_err(dev, "removing device with active domains!\n"); 1851 1852 for (i = 0; i < smmu->num_global_irqs; ++i) 1853 free_irq(smmu->irqs[i], smmu); 1854 1855 /* Turn the thing off */ 1856 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); 1857 return 0; 1858} 1859 1860static struct platform_driver arm_smmu_driver = { 1861 .driver = { 1862 .name = "arm-smmu", 1863 .of_match_table = of_match_ptr(arm_smmu_of_match), 1864 }, 1865 .probe = arm_smmu_device_dt_probe, 1866 .remove = arm_smmu_device_remove, 1867}; 1868 1869static int __init arm_smmu_init(void) 1870{ 1871 struct device_node *np; 1872 int ret; 1873 1874 /* 1875 * Play nice with systems that don't have an ARM SMMU by checking that 1876 * an ARM SMMU exists in the system before proceeding with the driver 1877 * and IOMMU bus operation registration. 1878 */ 1879 np = of_find_matching_node(NULL, arm_smmu_of_match); 1880 if (!np) 1881 return 0; 1882 1883 of_node_put(np); 1884 1885 ret = platform_driver_register(&arm_smmu_driver); 1886 if (ret) 1887 return ret; 1888 1889 /* Oh, for a proper bus abstraction */ 1890 if (!iommu_present(&platform_bus_type)) 1891 bus_set_iommu(&platform_bus_type, &arm_smmu_ops); 1892 1893#ifdef CONFIG_ARM_AMBA 1894 if (!iommu_present(&amba_bustype)) 1895 bus_set_iommu(&amba_bustype, &arm_smmu_ops); 1896#endif 1897 1898#ifdef CONFIG_PCI 1899 if (!iommu_present(&pci_bus_type)) 1900 bus_set_iommu(&pci_bus_type, &arm_smmu_ops); 1901#endif 1902 1903 return 0; 1904} 1905 1906static void __exit arm_smmu_exit(void) 1907{ 1908 return platform_driver_unregister(&arm_smmu_driver); 1909} 1910 1911subsys_initcall(arm_smmu_init); 1912module_exit(arm_smmu_exit); 1913 1914MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); 1915MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); 1916MODULE_LICENSE("GPL v2"); 1917