drivers/iommu/dmar.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
dmar_register_drhd_unit
dmar_alloc_dev_scope
dmar_free_dev_scope
dmar_alloc_pci_notify_info
dmar_free_pci_notify_info
dmar_match_pci_path
dmar_insert_dev_scope
dmar_remove_dev_scope
dmar_pci_bus_add_dev
dmar_pci_bus_del_dev
dmar_pci_bus_notifier
dmar_find_dmaru
dmar_parse_one_drhd
dmar_free_drhd
dmar_parse_one_andd
dmar_parse_one_rhsa
dmar_table_print_dmar_entry
dmar_table_detect
dmar_walk_remapping_entries
dmar_walk_dmar_table
parse_dmar_table
dmar_pci_device_match
dmar_find_matched_drhd_unit
dmar_acpi_insert_dev_scope
dmar_acpi_dev_scope_init
dmar_dev_scope_init
dmar_register_bus_notifier
dmar_table_init
warn_invalid_dmar
dmar_validate_one_drhd
detect_intel_iommu
unmap_iommu
map_iommu
dmar_alloc_seq_id
dmar_free_seq_id
alloc_iommu
free_iommu
reclaim_free_desc
qi_check_fault
qi_submit_sync
qi_global_iec
qi_flush_context
qi_flush_iotlb
qi_flush_dev_iotlb
dmar_disable_qi
__dmar_enable_qi
dmar_enable_qi
dmar_get_fault_reason
dmar_msi_reg
dmar_msi_unmask
dmar_msi_mask
dmar_msi_write
dmar_msi_read
dmar_fault_do_one
dmar_fault
dmar_set_interrupt
enable_drhd_fault_handling
dmar_reenable_qi
dmar_ir_support
dmar_in_use
dmar_free_unused_resources
dmar_detect_dsm
dmar_walk_dsm_resource
dmar_hp_add_drhd
dmar_hp_remove_drhd
dmar_hp_release_drhd
dmar_hotplug_insert
dmar_hotplug_remove
dmar_get_dsm_handle
dmar_device_hotplug
dmar_device_add
dmar_device_remove
dmar_platform_optin
   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2006, Intel Corporation.
   4  *
   5  * Copyright (C) 2006-2008 Intel Corporation
   6  * Author: Ashok Raj <ashok.raj@intel.com>
   7  * Author: Shaohua Li <shaohua.li@intel.com>
   8  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
   9  *
  10  * This file implements early detection/parsing of Remapping Devices
  11  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  12  * tables.
  13  *
  14  * These routines are used by both DMA-remapping and Interrupt-remapping
  15  */
  16 
  17 #define pr_fmt(fmt)     "DMAR: " fmt
  18 
  19 #include <linux/pci.h>
  20 #include <linux/dmar.h>
  21 #include <linux/iova.h>
  22 #include <linux/intel-iommu.h>
  23 #include <linux/timer.h>
  24 #include <linux/irq.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/tboot.h>
  27 #include <linux/dmi.h>
  28 #include <linux/slab.h>
  29 #include <linux/iommu.h>
  30 #include <linux/numa.h>
  31 #include <linux/limits.h>
  32 #include <asm/irq_remapping.h>
  33 #include <asm/iommu_table.h>
  34 
  35 #include "irq_remapping.h"
  36 
  37 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
  38 struct dmar_res_callback {
  39         dmar_res_handler_t      cb[ACPI_DMAR_TYPE_RESERVED];
  40         void                    *arg[ACPI_DMAR_TYPE_RESERVED];
  41         bool                    ignore_unhandled;
  42         bool                    print_entry;
  43 };
  44 
  45 /*
  46  * Assumptions:
  47  * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  48  *    before IO devices managed by that unit.
  49  * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  50  *    after IO devices managed by that unit.
  51  * 3) Hotplug events are rare.
  52  *
  53  * Locking rules for DMA and interrupt remapping related global data structures:
  54  * 1) Use dmar_global_lock in process context
  55  * 2) Use RCU in interrupt context
  56  */
  57 DECLARE_RWSEM(dmar_global_lock);
  58 LIST_HEAD(dmar_drhd_units);
  59 
  60 struct acpi_table_header * __initdata dmar_tbl;
  61 static int dmar_dev_scope_status = 1;
  62 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
  63 
  64 static int alloc_iommu(struct dmar_drhd_unit *drhd);
  65 static void free_iommu(struct intel_iommu *iommu);
  66 
  67 extern const struct iommu_ops intel_iommu_ops;
  68 
  69 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  70 {
  71         /*
  72          * add INCLUDE_ALL at the tail, so scan the list will find it at
  73          * the very end.
  74          */
  75         if (drhd->include_all)
  76                 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  77         else
  78                 list_add_rcu(&drhd->list, &dmar_drhd_units);
  79 }
  80 
  81 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  82 {
  83         struct acpi_dmar_device_scope *scope;
  84 
  85         *cnt = 0;
  86         while (start < end) {
  87                 scope = start;
  88                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
  89                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  90                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
  91                         (*cnt)++;
  92                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
  93                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
  94                         pr_warn("Unsupported device scope\n");
  95                 }
  96                 start += scope->length;
  97         }
  98         if (*cnt == 0)
  99                 return NULL;
 100 
 101         return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 102 }
 103 
 104 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 105 {
 106         int i;
 107         struct device *tmp_dev;
 108 
 109         if (*devices && *cnt) {
 110                 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 111                         put_device(tmp_dev);
 112                 kfree(*devices);
 113         }
 114 
 115         *devices = NULL;
 116         *cnt = 0;
 117 }
 118 
 119 /* Optimize out kzalloc()/kfree() for normal cases */
 120 static char dmar_pci_notify_info_buf[64];
 121 
 122 static struct dmar_pci_notify_info *
 123 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 124 {
 125         int level = 0;
 126         size_t size;
 127         struct pci_dev *tmp;
 128         struct dmar_pci_notify_info *info;
 129 
 130         BUG_ON(dev->is_virtfn);
 131 
 132         /*
 133          * Ignore devices that have a domain number higher than what can
 134          * be looked up in DMAR, e.g. VMD subdevices with domain 0x10000
 135          */
 136         if (pci_domain_nr(dev->bus) > U16_MAX)
 137                 return NULL;
 138 
 139         /* Only generate path[] for device addition event */
 140         if (event == BUS_NOTIFY_ADD_DEVICE)
 141                 for (tmp = dev; tmp; tmp = tmp->bus->self)
 142                         level++;
 143 
 144         size = struct_size(info, path, level);
 145         if (size <= sizeof(dmar_pci_notify_info_buf)) {
 146                 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 147         } else {
 148                 info = kzalloc(size, GFP_KERNEL);
 149                 if (!info) {
 150                         pr_warn("Out of memory when allocating notify_info "
 151                                 "for %s.\n", pci_name(dev));
 152                         if (dmar_dev_scope_status == 0)
 153                                 dmar_dev_scope_status = -ENOMEM;
 154                         return NULL;
 155                 }
 156         }
 157 
 158         info->event = event;
 159         info->dev = dev;
 160         info->seg = pci_domain_nr(dev->bus);
 161         info->level = level;
 162         if (event == BUS_NOTIFY_ADD_DEVICE) {
 163                 for (tmp = dev; tmp; tmp = tmp->bus->self) {
 164                         level--;
 165                         info->path[level].bus = tmp->bus->number;
 166                         info->path[level].device = PCI_SLOT(tmp->devfn);
 167                         info->path[level].function = PCI_FUNC(tmp->devfn);
 168                         if (pci_is_root_bus(tmp->bus))
 169                                 info->bus = tmp->bus->number;
 170                 }
 171         }
 172 
 173         return info;
 174 }
 175 
 176 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 177 {
 178         if ((void *)info != dmar_pci_notify_info_buf)
 179                 kfree(info);
 180 }
 181 
 182 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 183                                 struct acpi_dmar_pci_path *path, int count)
 184 {
 185         int i;
 186 
 187         if (info->bus != bus)
 188                 goto fallback;
 189         if (info->level != count)
 190                 goto fallback;
 191 
 192         for (i = 0; i < count; i++) {
 193                 if (path[i].device != info->path[i].device ||
 194                     path[i].function != info->path[i].function)
 195                         goto fallback;
 196         }
 197 
 198         return true;
 199 
 200 fallback:
 201 
 202         if (count != 1)
 203                 return false;
 204 
 205         i = info->level - 1;
 206         if (bus              == info->path[i].bus &&
 207             path[0].device   == info->path[i].device &&
 208             path[0].function == info->path[i].function) {
 209                 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
 210                         bus, path[0].device, path[0].function);
 211                 return true;
 212         }
 213 
 214         return false;
 215 }
 216 
 217 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 218 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 219                           void *start, void*end, u16 segment,
 220                           struct dmar_dev_scope *devices,
 221                           int devices_cnt)
 222 {
 223         int i, level;
 224         struct device *tmp, *dev = &info->dev->dev;
 225         struct acpi_dmar_device_scope *scope;
 226         struct acpi_dmar_pci_path *path;
 227 
 228         if (segment != info->seg)
 229                 return 0;
 230 
 231         for (; start < end; start += scope->length) {
 232                 scope = start;
 233                 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 234                     scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 235                         continue;
 236 
 237                 path = (struct acpi_dmar_pci_path *)(scope + 1);
 238                 level = (scope->length - sizeof(*scope)) / sizeof(*path);
 239                 if (!dmar_match_pci_path(info, scope->bus, path, level))
 240                         continue;
 241 
 242                 /*
 243                  * We expect devices with endpoint scope to have normal PCI
 244                  * headers, and devices with bridge scope to have bridge PCI
 245                  * headers.  However PCI NTB devices may be listed in the
 246                  * DMAR table with bridge scope, even though they have a
 247                  * normal PCI header.  NTB devices are identified by class
 248                  * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
 249                  * for this special case.
 250                  */
 251                 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 252                      info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
 253                     (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
 254                      (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
 255                       info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) {
 256                         pr_warn("Device scope type does not match for %s\n",
 257                                 pci_name(info->dev));
 258                         return -EINVAL;
 259                 }
 260 
 261                 for_each_dev_scope(devices, devices_cnt, i, tmp)
 262                         if (tmp == NULL) {
 263                                 devices[i].bus = info->dev->bus->number;
 264                                 devices[i].devfn = info->dev->devfn;
 265                                 rcu_assign_pointer(devices[i].dev,
 266                                                    get_device(dev));
 267                                 return 1;
 268                         }
 269                 BUG_ON(i >= devices_cnt);
 270         }
 271 
 272         return 0;
 273 }
 274 
 275 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 276                           struct dmar_dev_scope *devices, int count)
 277 {
 278         int index;
 279         struct device *tmp;
 280 
 281         if (info->seg != segment)
 282                 return 0;
 283 
 284         for_each_active_dev_scope(devices, count, index, tmp)
 285                 if (tmp == &info->dev->dev) {
 286                         RCU_INIT_POINTER(devices[index].dev, NULL);
 287                         synchronize_rcu();
 288                         put_device(tmp);
 289                         return 1;
 290                 }
 291 
 292         return 0;
 293 }
 294 
 295 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 296 {
 297         int ret = 0;
 298         struct dmar_drhd_unit *dmaru;
 299         struct acpi_dmar_hardware_unit *drhd;
 300 
 301         for_each_drhd_unit(dmaru) {
 302                 if (dmaru->include_all)
 303                         continue;
 304 
 305                 drhd = container_of(dmaru->hdr,
 306                                     struct acpi_dmar_hardware_unit, header);
 307                 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 308                                 ((void *)drhd) + drhd->header.length,
 309                                 dmaru->segment,
 310                                 dmaru->devices, dmaru->devices_cnt);
 311                 if (ret)
 312                         break;
 313         }
 314         if (ret >= 0)
 315                 ret = dmar_iommu_notify_scope_dev(info);
 316         if (ret < 0 && dmar_dev_scope_status == 0)
 317                 dmar_dev_scope_status = ret;
 318 
 319         return ret;
 320 }
 321 
 322 static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 323 {
 324         struct dmar_drhd_unit *dmaru;
 325 
 326         for_each_drhd_unit(dmaru)
 327                 if (dmar_remove_dev_scope(info, dmaru->segment,
 328                         dmaru->devices, dmaru->devices_cnt))
 329                         break;
 330         dmar_iommu_notify_scope_dev(info);
 331 }
 332 
 333 static int dmar_pci_bus_notifier(struct notifier_block *nb,
 334                                  unsigned long action, void *data)
 335 {
 336         struct pci_dev *pdev = to_pci_dev(data);
 337         struct dmar_pci_notify_info *info;
 338 
 339         /* Only care about add/remove events for physical functions.
 340          * For VFs we actually do the lookup based on the corresponding
 341          * PF in device_to_iommu() anyway. */
 342         if (pdev->is_virtfn)
 343                 return NOTIFY_DONE;
 344         if (action != BUS_NOTIFY_ADD_DEVICE &&
 345             action != BUS_NOTIFY_REMOVED_DEVICE)
 346                 return NOTIFY_DONE;
 347 
 348         info = dmar_alloc_pci_notify_info(pdev, action);
 349         if (!info)
 350                 return NOTIFY_DONE;
 351 
 352         down_write(&dmar_global_lock);
 353         if (action == BUS_NOTIFY_ADD_DEVICE)
 354                 dmar_pci_bus_add_dev(info);
 355         else if (action == BUS_NOTIFY_REMOVED_DEVICE)
 356                 dmar_pci_bus_del_dev(info);
 357         up_write(&dmar_global_lock);
 358 
 359         dmar_free_pci_notify_info(info);
 360 
 361         return NOTIFY_OK;
 362 }
 363 
 364 static struct notifier_block dmar_pci_bus_nb = {
 365         .notifier_call = dmar_pci_bus_notifier,
 366         .priority = INT_MIN,
 367 };
 368 
 369 static struct dmar_drhd_unit *
 370 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
 371 {
 372         struct dmar_drhd_unit *dmaru;
 373 
 374         list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list,
 375                                 dmar_rcu_check())
 376                 if (dmaru->segment == drhd->segment &&
 377                     dmaru->reg_base_addr == drhd->address)
 378                         return dmaru;
 379 
 380         return NULL;
 381 }
 382 
 383 /**
 384  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 385  * structure which uniquely represent one DMA remapping hardware unit
 386  * present in the platform
 387  */
 388 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
 389 {
 390         struct acpi_dmar_hardware_unit *drhd;
 391         struct dmar_drhd_unit *dmaru;
 392         int ret;
 393 
 394         drhd = (struct acpi_dmar_hardware_unit *)header;
 395         dmaru = dmar_find_dmaru(drhd);
 396         if (dmaru)
 397                 goto out;
 398 
 399         dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
 400         if (!dmaru)
 401                 return -ENOMEM;
 402 
 403         /*
 404          * If header is allocated from slab by ACPI _DSM method, we need to
 405          * copy the content because the memory buffer will be freed on return.
 406          */
 407         dmaru->hdr = (void *)(dmaru + 1);
 408         memcpy(dmaru->hdr, header, header->length);
 409         dmaru->reg_base_addr = drhd->address;
 410         dmaru->segment = drhd->segment;
 411         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 412         dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 413                                               ((void *)drhd) + drhd->header.length,
 414                                               &dmaru->devices_cnt);
 415         if (dmaru->devices_cnt && dmaru->devices == NULL) {
 416                 kfree(dmaru);
 417                 return -ENOMEM;
 418         }
 419 
 420         ret = alloc_iommu(dmaru);
 421         if (ret) {
 422                 dmar_free_dev_scope(&dmaru->devices,
 423                                     &dmaru->devices_cnt);
 424                 kfree(dmaru);
 425                 return ret;
 426         }
 427         dmar_register_drhd_unit(dmaru);
 428 
 429 out:
 430         if (arg)
 431                 (*(int *)arg)++;
 432 
 433         return 0;
 434 }
 435 
 436 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 437 {
 438         if (dmaru->devices && dmaru->devices_cnt)
 439                 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 440         if (dmaru->iommu)
 441                 free_iommu(dmaru->iommu);
 442         kfree(dmaru);
 443 }
 444 
 445 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
 446                                       void *arg)
 447 {
 448         struct acpi_dmar_andd *andd = (void *)header;
 449 
 450         /* Check for NUL termination within the designated length */
 451         if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
 452                 pr_warn(FW_BUG
 453                            "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 454                            "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 455                            dmi_get_system_info(DMI_BIOS_VENDOR),
 456                            dmi_get_system_info(DMI_BIOS_VERSION),
 457                            dmi_get_system_info(DMI_PRODUCT_VERSION));
 458                 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 459                 return -EINVAL;
 460         }
 461         pr_info("ANDD device: %x name: %s\n", andd->device_number,
 462                 andd->device_name);
 463 
 464         return 0;
 465 }
 466 
 467 #ifdef CONFIG_ACPI_NUMA
 468 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
 469 {
 470         struct acpi_dmar_rhsa *rhsa;
 471         struct dmar_drhd_unit *drhd;
 472 
 473         rhsa = (struct acpi_dmar_rhsa *)header;
 474         for_each_drhd_unit(drhd) {
 475                 if (drhd->reg_base_addr == rhsa->base_address) {
 476                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 477 
 478                         if (!node_online(node))
 479                                 node = NUMA_NO_NODE;
 480                         drhd->iommu->node = node;
 481                         return 0;
 482                 }
 483         }
 484         pr_warn(FW_BUG
 485                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 486                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 487                 rhsa->base_address,
 488                 dmi_get_system_info(DMI_BIOS_VENDOR),
 489                 dmi_get_system_info(DMI_BIOS_VERSION),
 490                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 491         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 492 
 493         return 0;
 494 }
 495 #else
 496 #define dmar_parse_one_rhsa             dmar_res_noop
 497 #endif
 498 
 499 static void
 500 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 501 {
 502         struct acpi_dmar_hardware_unit *drhd;
 503         struct acpi_dmar_reserved_memory *rmrr;
 504         struct acpi_dmar_atsr *atsr;
 505         struct acpi_dmar_rhsa *rhsa;
 506 
 507         switch (header->type) {
 508         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 509                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 510                                     header);
 511                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 512                         (unsigned long long)drhd->address, drhd->flags);
 513                 break;
 514         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 515                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 516                                     header);
 517                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 518                         (unsigned long long)rmrr->base_address,
 519                         (unsigned long long)rmrr->end_address);
 520                 break;
 521         case ACPI_DMAR_TYPE_ROOT_ATS:
 522                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 523                 pr_info("ATSR flags: %#x\n", atsr->flags);
 524                 break;
 525         case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
 526                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 527                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 528                        (unsigned long long)rhsa->base_address,
 529                        rhsa->proximity_domain);
 530                 break;
 531         case ACPI_DMAR_TYPE_NAMESPACE:
 532                 /* We don't print this here because we need to sanity-check
 533                    it first. So print it in dmar_parse_one_andd() instead. */
 534                 break;
 535         }
 536 }
 537 
 538 /**
 539  * dmar_table_detect - checks to see if the platform supports DMAR devices
 540  */
 541 static int __init dmar_table_detect(void)
 542 {
 543         acpi_status status = AE_OK;
 544 
 545         /* if we could find DMAR table, then there are DMAR devices */
 546         status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
 547 
 548         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 549                 pr_warn("Unable to map DMAR\n");
 550                 status = AE_NOT_FOUND;
 551         }
 552 
 553         return ACPI_SUCCESS(status) ? 0 : -ENOENT;
 554 }
 555 
 556 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
 557                                        size_t len, struct dmar_res_callback *cb)
 558 {
 559         struct acpi_dmar_header *iter, *next;
 560         struct acpi_dmar_header *end = ((void *)start) + len;
 561 
 562         for (iter = start; iter < end; iter = next) {
 563                 next = (void *)iter + iter->length;
 564                 if (iter->length == 0) {
 565                         /* Avoid looping forever on bad ACPI tables */
 566                         pr_debug(FW_BUG "Invalid 0-length structure\n");
 567                         break;
 568                 } else if (next > end) {
 569                         /* Avoid passing table end */
 570                         pr_warn(FW_BUG "Record passes table end\n");
 571                         return -EINVAL;
 572                 }
 573 
 574                 if (cb->print_entry)
 575                         dmar_table_print_dmar_entry(iter);
 576 
 577                 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
 578                         /* continue for forward compatibility */
 579                         pr_debug("Unknown DMAR structure type %d\n",
 580                                  iter->type);
 581                 } else if (cb->cb[iter->type]) {
 582                         int ret;
 583 
 584                         ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
 585                         if (ret)
 586                                 return ret;
 587                 } else if (!cb->ignore_unhandled) {
 588                         pr_warn("No handler for DMAR structure type %d\n",
 589                                 iter->type);
 590                         return -EINVAL;
 591                 }
 592         }
 593 
 594         return 0;
 595 }
 596 
 597 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
 598                                        struct dmar_res_callback *cb)
 599 {
 600         return dmar_walk_remapping_entries((void *)(dmar + 1),
 601                         dmar->header.length - sizeof(*dmar), cb);
 602 }
 603 
 604 /**
 605  * parse_dmar_table - parses the DMA reporting table
 606  */
 607 static int __init
 608 parse_dmar_table(void)
 609 {
 610         struct acpi_table_dmar *dmar;
 611         int drhd_count = 0;
 612         int ret;
 613         struct dmar_res_callback cb = {
 614                 .print_entry = true,
 615                 .ignore_unhandled = true,
 616                 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
 617                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
 618                 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
 619                 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
 620                 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
 621                 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
 622         };
 623 
 624         /*
 625          * Do it again, earlier dmar_tbl mapping could be mapped with
 626          * fixed map.
 627          */
 628         dmar_table_detect();
 629 
 630         /*
 631          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 632          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 633          */
 634         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 635 
 636         dmar = (struct acpi_table_dmar *)dmar_tbl;
 637         if (!dmar)
 638                 return -ENODEV;
 639 
 640         if (dmar->width < PAGE_SHIFT - 1) {
 641                 pr_warn("Invalid DMAR haw\n");
 642                 return -EINVAL;
 643         }
 644 
 645         pr_info("Host address width %d\n", dmar->width + 1);
 646         ret = dmar_walk_dmar_table(dmar, &cb);
 647         if (ret == 0 && drhd_count == 0)
 648                 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 649 
 650         return ret;
 651 }
 652 
 653 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 654                                  int cnt, struct pci_dev *dev)
 655 {
 656         int index;
 657         struct device *tmp;
 658 
 659         while (dev) {
 660                 for_each_active_dev_scope(devices, cnt, index, tmp)
 661                         if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 662                                 return 1;
 663 
 664                 /* Check our parent */
 665                 dev = dev->bus->self;
 666         }
 667 
 668         return 0;
 669 }
 670 
 671 struct dmar_drhd_unit *
 672 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 673 {
 674         struct dmar_drhd_unit *dmaru;
 675         struct acpi_dmar_hardware_unit *drhd;
 676 
 677         dev = pci_physfn(dev);
 678 
 679         rcu_read_lock();
 680         for_each_drhd_unit(dmaru) {
 681                 drhd = container_of(dmaru->hdr,
 682                                     struct acpi_dmar_hardware_unit,
 683                                     header);
 684 
 685                 if (dmaru->include_all &&
 686                     drhd->segment == pci_domain_nr(dev->bus))
 687                         goto out;
 688 
 689                 if (dmar_pci_device_match(dmaru->devices,
 690                                           dmaru->devices_cnt, dev))
 691                         goto out;
 692         }
 693         dmaru = NULL;
 694 out:
 695         rcu_read_unlock();
 696 
 697         return dmaru;
 698 }
 699 
 700 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 701                                               struct acpi_device *adev)
 702 {
 703         struct dmar_drhd_unit *dmaru;
 704         struct acpi_dmar_hardware_unit *drhd;
 705         struct acpi_dmar_device_scope *scope;
 706         struct device *tmp;
 707         int i;
 708         struct acpi_dmar_pci_path *path;
 709 
 710         for_each_drhd_unit(dmaru) {
 711                 drhd = container_of(dmaru->hdr,
 712                                     struct acpi_dmar_hardware_unit,
 713                                     header);
 714 
 715                 for (scope = (void *)(drhd + 1);
 716                      (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 717                      scope = ((void *)scope) + scope->length) {
 718                         if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
 719                                 continue;
 720                         if (scope->enumeration_id != device_number)
 721                                 continue;
 722 
 723                         path = (void *)(scope + 1);
 724                         pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 725                                 dev_name(&adev->dev), dmaru->reg_base_addr,
 726                                 scope->bus, path->device, path->function);
 727                         for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 728                                 if (tmp == NULL) {
 729                                         dmaru->devices[i].bus = scope->bus;
 730                                         dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 731                                                                             path->function);
 732                                         rcu_assign_pointer(dmaru->devices[i].dev,
 733                                                            get_device(&adev->dev));
 734                                         return;
 735                                 }
 736                         BUG_ON(i >= dmaru->devices_cnt);
 737                 }
 738         }
 739         pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 740                 device_number, dev_name(&adev->dev));
 741 }
 742 
 743 static int __init dmar_acpi_dev_scope_init(void)
 744 {
 745         struct acpi_dmar_andd *andd;
 746 
 747         if (dmar_tbl == NULL)
 748                 return -ENODEV;
 749 
 750         for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 751              ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 752              andd = ((void *)andd) + andd->header.length) {
 753                 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
 754                         acpi_handle h;
 755                         struct acpi_device *adev;
 756 
 757                         if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 758                                                           andd->device_name,
 759                                                           &h))) {
 760                                 pr_err("Failed to find handle for ACPI object %s\n",
 761                                        andd->device_name);
 762                                 continue;
 763                         }
 764                         if (acpi_bus_get_device(h, &adev)) {
 765                                 pr_err("Failed to get device for ACPI object %s\n",
 766                                        andd->device_name);
 767                                 continue;
 768                         }
 769                         dmar_acpi_insert_dev_scope(andd->device_number, adev);
 770                 }
 771         }
 772         return 0;
 773 }
 774 
 775 int __init dmar_dev_scope_init(void)
 776 {
 777         struct pci_dev *dev = NULL;
 778         struct dmar_pci_notify_info *info;
 779 
 780         if (dmar_dev_scope_status != 1)
 781                 return dmar_dev_scope_status;
 782 
 783         if (list_empty(&dmar_drhd_units)) {
 784                 dmar_dev_scope_status = -ENODEV;
 785         } else {
 786                 dmar_dev_scope_status = 0;
 787 
 788                 dmar_acpi_dev_scope_init();
 789 
 790                 for_each_pci_dev(dev) {
 791                         if (dev->is_virtfn)
 792                                 continue;
 793 
 794                         info = dmar_alloc_pci_notify_info(dev,
 795                                         BUS_NOTIFY_ADD_DEVICE);
 796                         if (!info) {
 797                                 return dmar_dev_scope_status;
 798                         } else {
 799                                 dmar_pci_bus_add_dev(info);
 800                                 dmar_free_pci_notify_info(info);
 801                         }
 802                 }
 803         }
 804 
 805         return dmar_dev_scope_status;
 806 }
 807 
 808 void __init dmar_register_bus_notifier(void)
 809 {
 810         bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 811 }
 812 
 813 
 814 int __init dmar_table_init(void)
 815 {
 816         static int dmar_table_initialized;
 817         int ret;
 818 
 819         if (dmar_table_initialized == 0) {
 820                 ret = parse_dmar_table();
 821                 if (ret < 0) {
 822                         if (ret != -ENODEV)
 823                                 pr_info("Parse DMAR table failure.\n");
 824                 } else  if (list_empty(&dmar_drhd_units)) {
 825                         pr_info("No DMAR devices found\n");
 826                         ret = -ENODEV;
 827                 }
 828 
 829                 if (ret < 0)
 830                         dmar_table_initialized = ret;
 831                 else
 832                         dmar_table_initialized = 1;
 833         }
 834 
 835         return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 836 }
 837 
 838 static void warn_invalid_dmar(u64 addr, const char *message)
 839 {
 840         pr_warn_once(FW_BUG
 841                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 842                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 843                 addr, message,
 844                 dmi_get_system_info(DMI_BIOS_VENDOR),
 845                 dmi_get_system_info(DMI_BIOS_VERSION),
 846                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 847         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 848 }
 849 
 850 static int __ref
 851 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
 852 {
 853         struct acpi_dmar_hardware_unit *drhd;
 854         void __iomem *addr;
 855         u64 cap, ecap;
 856 
 857         drhd = (void *)entry;
 858         if (!drhd->address) {
 859                 warn_invalid_dmar(0, "");
 860                 return -EINVAL;
 861         }
 862 
 863         if (arg)
 864                 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
 865         else
 866                 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 867         if (!addr) {
 868                 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
 869                 return -EINVAL;
 870         }
 871 
 872         cap = dmar_readq(addr + DMAR_CAP_REG);
 873         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 874 
 875         if (arg)
 876                 iounmap(addr);
 877         else
 878                 early_iounmap(addr, VTD_PAGE_SIZE);
 879 
 880         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 881                 warn_invalid_dmar(drhd->address, " returns all ones");
 882                 return -EINVAL;
 883         }
 884 
 885         return 0;
 886 }
 887 
 888 int __init detect_intel_iommu(void)
 889 {
 890         int ret;
 891         struct dmar_res_callback validate_drhd_cb = {
 892                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
 893                 .ignore_unhandled = true,
 894         };
 895 
 896         down_write(&dmar_global_lock);
 897         ret = dmar_table_detect();
 898         if (!ret)
 899                 ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
 900                                            &validate_drhd_cb);
 901         if (!ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 902                 iommu_detected = 1;
 903                 /* Make sure ACS will be enabled */
 904                 pci_request_acs();
 905         }
 906 
 907 #ifdef CONFIG_X86
 908         if (!ret)
 909                 x86_init.iommu.iommu_init = intel_iommu_init;
 910 #endif
 911 
 912         if (dmar_tbl) {
 913                 acpi_put_table(dmar_tbl);
 914                 dmar_tbl = NULL;
 915         }
 916         up_write(&dmar_global_lock);
 917 
 918         return ret ? ret : 1;
 919 }
 920 
 921 static void unmap_iommu(struct intel_iommu *iommu)
 922 {
 923         iounmap(iommu->reg);
 924         release_mem_region(iommu->reg_phys, iommu->reg_size);
 925 }
 926 
 927 /**
 928  * map_iommu: map the iommu's registers
 929  * @iommu: the iommu to map
 930  * @phys_addr: the physical address of the base resgister
 931  *
 932  * Memory map the iommu's registers.  Start w/ a single page, and
 933  * possibly expand if that turns out to be insufficent.
 934  */
 935 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 936 {
 937         int map_size, err=0;
 938 
 939         iommu->reg_phys = phys_addr;
 940         iommu->reg_size = VTD_PAGE_SIZE;
 941 
 942         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 943                 pr_err("Can't reserve memory\n");
 944                 err = -EBUSY;
 945                 goto out;
 946         }
 947 
 948         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 949         if (!iommu->reg) {
 950                 pr_err("Can't map the region\n");
 951                 err = -ENOMEM;
 952                 goto release;
 953         }
 954 
 955         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 956         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 957 
 958         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 959                 err = -EINVAL;
 960                 warn_invalid_dmar(phys_addr, " returns all ones");
 961                 goto unmap;
 962         }
 963 
 964         /* the registers might be more than one page */
 965         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 966                          cap_max_fault_reg_offset(iommu->cap));
 967         map_size = VTD_PAGE_ALIGN(map_size);
 968         if (map_size > iommu->reg_size) {
 969                 iounmap(iommu->reg);
 970                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 971                 iommu->reg_size = map_size;
 972                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 973                                         iommu->name)) {
 974                         pr_err("Can't reserve memory\n");
 975                         err = -EBUSY;
 976                         goto out;
 977                 }
 978                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 979                 if (!iommu->reg) {
 980                         pr_err("Can't map the region\n");
 981                         err = -ENOMEM;
 982                         goto release;
 983                 }
 984         }
 985         err = 0;
 986         goto out;
 987 
 988 unmap:
 989         iounmap(iommu->reg);
 990 release:
 991         release_mem_region(iommu->reg_phys, iommu->reg_size);
 992 out:
 993         return err;
 994 }
 995 
 996 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
 997 {
 998         iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
 999                                             DMAR_UNITS_SUPPORTED);
1000         if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
1001                 iommu->seq_id = -1;
1002         } else {
1003                 set_bit(iommu->seq_id, dmar_seq_ids);
1004                 sprintf(iommu->name, "dmar%d", iommu->seq_id);
1005         }
1006 
1007         return iommu->seq_id;
1008 }
1009 
1010 static void dmar_free_seq_id(struct intel_iommu *iommu)
1011 {
1012         if (iommu->seq_id >= 0) {
1013                 clear_bit(iommu->seq_id, dmar_seq_ids);
1014                 iommu->seq_id = -1;
1015         }
1016 }
1017 
1018 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1019 {
1020         struct intel_iommu *iommu;
1021         u32 ver, sts;
1022         int agaw = 0;
1023         int msagaw = 0;
1024         int err;
1025 
1026         if (!drhd->reg_base_addr) {
1027                 warn_invalid_dmar(0, "");
1028                 return -EINVAL;
1029         }
1030 
1031         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1032         if (!iommu)
1033                 return -ENOMEM;
1034 
1035         if (dmar_alloc_seq_id(iommu) < 0) {
1036                 pr_err("Failed to allocate seq_id\n");
1037                 err = -ENOSPC;
1038                 goto error;
1039         }
1040 
1041         err = map_iommu(iommu, drhd->reg_base_addr);
1042         if (err) {
1043                 pr_err("Failed to map %s\n", iommu->name);
1044                 goto error_free_seq_id;
1045         }
1046 
1047         err = -EINVAL;
1048         agaw = iommu_calculate_agaw(iommu);
1049         if (agaw < 0) {
1050                 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1051                         iommu->seq_id);
1052                 goto err_unmap;
1053         }
1054         msagaw = iommu_calculate_max_sagaw(iommu);
1055         if (msagaw < 0) {
1056                 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1057                         iommu->seq_id);
1058                 goto err_unmap;
1059         }
1060         iommu->agaw = agaw;
1061         iommu->msagaw = msagaw;
1062         iommu->segment = drhd->segment;
1063 
1064         iommu->node = NUMA_NO_NODE;
1065 
1066         ver = readl(iommu->reg + DMAR_VER_REG);
1067         pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1068                 iommu->name,
1069                 (unsigned long long)drhd->reg_base_addr,
1070                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1071                 (unsigned long long)iommu->cap,
1072                 (unsigned long long)iommu->ecap);
1073 
1074         /* Reflect status in gcmd */
1075         sts = readl(iommu->reg + DMAR_GSTS_REG);
1076         if (sts & DMA_GSTS_IRES)
1077                 iommu->gcmd |= DMA_GCMD_IRE;
1078         if (sts & DMA_GSTS_TES)
1079                 iommu->gcmd |= DMA_GCMD_TE;
1080         if (sts & DMA_GSTS_QIES)
1081                 iommu->gcmd |= DMA_GCMD_QIE;
1082 
1083         raw_spin_lock_init(&iommu->register_lock);
1084 
1085         if (intel_iommu_enabled) {
1086                 err = iommu_device_sysfs_add(&iommu->iommu, NULL,
1087                                              intel_iommu_groups,
1088                                              "%s", iommu->name);
1089                 if (err)
1090                         goto err_unmap;
1091 
1092                 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
1093 
1094                 err = iommu_device_register(&iommu->iommu);
1095                 if (err)
1096                         goto err_unmap;
1097         }
1098 
1099         drhd->iommu = iommu;
1100 
1101         return 0;
1102 
1103 err_unmap:
1104         unmap_iommu(iommu);
1105 error_free_seq_id:
1106         dmar_free_seq_id(iommu);
1107 error:
1108         kfree(iommu);
1109         return err;
1110 }
1111 
1112 static void free_iommu(struct intel_iommu *iommu)
1113 {
1114         if (intel_iommu_enabled) {
1115                 iommu_device_unregister(&iommu->iommu);
1116                 iommu_device_sysfs_remove(&iommu->iommu);
1117         }
1118 
1119         if (iommu->irq) {
1120                 if (iommu->pr_irq) {
1121                         free_irq(iommu->pr_irq, iommu);
1122                         dmar_free_hwirq(iommu->pr_irq);
1123                         iommu->pr_irq = 0;
1124                 }
1125                 free_irq(iommu->irq, iommu);
1126                 dmar_free_hwirq(iommu->irq);
1127                 iommu->irq = 0;
1128         }
1129 
1130         if (iommu->qi) {
1131                 free_page((unsigned long)iommu->qi->desc);
1132                 kfree(iommu->qi->desc_status);
1133                 kfree(iommu->qi);
1134         }
1135 
1136         if (iommu->reg)
1137                 unmap_iommu(iommu);
1138 
1139         dmar_free_seq_id(iommu);
1140         kfree(iommu);
1141 }
1142 
1143 /*
1144  * Reclaim all the submitted descriptors which have completed its work.
1145  */
1146 static inline void reclaim_free_desc(struct q_inval *qi)
1147 {
1148         while (qi->desc_status[qi->free_tail] == QI_DONE ||
1149                qi->desc_status[qi->free_tail] == QI_ABORT) {
1150                 qi->desc_status[qi->free_tail] = QI_FREE;
1151                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1152                 qi->free_cnt++;
1153         }
1154 }
1155 
1156 static int qi_check_fault(struct intel_iommu *iommu, int index)
1157 {
1158         u32 fault;
1159         int head, tail;
1160         struct q_inval *qi = iommu->qi;
1161         int wait_index = (index + 1) % QI_LENGTH;
1162         int shift = qi_shift(iommu);
1163 
1164         if (qi->desc_status[wait_index] == QI_ABORT)
1165                 return -EAGAIN;
1166 
1167         fault = readl(iommu->reg + DMAR_FSTS_REG);
1168 
1169         /*
1170          * If IQE happens, the head points to the descriptor associated
1171          * with the error. No new descriptors are fetched until the IQE
1172          * is cleared.
1173          */
1174         if (fault & DMA_FSTS_IQE) {
1175                 head = readl(iommu->reg + DMAR_IQH_REG);
1176                 if ((head >> shift) == index) {
1177                         struct qi_desc *desc = qi->desc + head;
1178 
1179                         /*
1180                          * desc->qw2 and desc->qw3 are either reserved or
1181                          * used by software as private data. We won't print
1182                          * out these two qw's for security consideration.
1183                          */
1184                         pr_err("VT-d detected invalid descriptor: qw0 = %llx, qw1 = %llx\n",
1185                                (unsigned long long)desc->qw0,
1186                                (unsigned long long)desc->qw1);
1187                         memcpy(desc, qi->desc + (wait_index << shift),
1188                                1 << shift);
1189                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1190                         return -EINVAL;
1191                 }
1192         }
1193 
1194         /*
1195          * If ITE happens, all pending wait_desc commands are aborted.
1196          * No new descriptors are fetched until the ITE is cleared.
1197          */
1198         if (fault & DMA_FSTS_ITE) {
1199                 head = readl(iommu->reg + DMAR_IQH_REG);
1200                 head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1201                 head |= 1;
1202                 tail = readl(iommu->reg + DMAR_IQT_REG);
1203                 tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1204 
1205                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1206 
1207                 do {
1208                         if (qi->desc_status[head] == QI_IN_USE)
1209                                 qi->desc_status[head] = QI_ABORT;
1210                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1211                 } while (head != tail);
1212 
1213                 if (qi->desc_status[wait_index] == QI_ABORT)
1214                         return -EAGAIN;
1215         }
1216 
1217         if (fault & DMA_FSTS_ICE)
1218                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1219 
1220         return 0;
1221 }
1222 
1223 /*
1224  * Submit the queued invalidation descriptor to the remapping
1225  * hardware unit and wait for its completion.
1226  */
1227 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1228 {
1229         int rc;
1230         struct q_inval *qi = iommu->qi;
1231         int offset, shift, length;
1232         struct qi_desc wait_desc;
1233         int wait_index, index;
1234         unsigned long flags;
1235 
1236         if (!qi)
1237                 return 0;
1238 
1239 restart:
1240         rc = 0;
1241 
1242         raw_spin_lock_irqsave(&qi->q_lock, flags);
1243         while (qi->free_cnt < 3) {
1244                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1245                 cpu_relax();
1246                 raw_spin_lock_irqsave(&qi->q_lock, flags);
1247         }
1248 
1249         index = qi->free_head;
1250         wait_index = (index + 1) % QI_LENGTH;
1251         shift = qi_shift(iommu);
1252         length = 1 << shift;
1253 
1254         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1255 
1256         offset = index << shift;
1257         memcpy(qi->desc + offset, desc, length);
1258         wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
1259                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1260         wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
1261         wait_desc.qw2 = 0;
1262         wait_desc.qw3 = 0;
1263 
1264         offset = wait_index << shift;
1265         memcpy(qi->desc + offset, &wait_desc, length);
1266 
1267         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1268         qi->free_cnt -= 2;
1269 
1270         /*
1271          * update the HW tail register indicating the presence of
1272          * new descriptors.
1273          */
1274         writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
1275 
1276         while (qi->desc_status[wait_index] != QI_DONE) {
1277                 /*
1278                  * We will leave the interrupts disabled, to prevent interrupt
1279                  * context to queue another cmd while a cmd is already submitted
1280                  * and waiting for completion on this cpu. This is to avoid
1281                  * a deadlock where the interrupt context can wait indefinitely
1282                  * for free slots in the queue.
1283                  */
1284                 rc = qi_check_fault(iommu, index);
1285                 if (rc)
1286                         break;
1287 
1288                 raw_spin_unlock(&qi->q_lock);
1289                 cpu_relax();
1290                 raw_spin_lock(&qi->q_lock);
1291         }
1292 
1293         qi->desc_status[index] = QI_DONE;
1294 
1295         reclaim_free_desc(qi);
1296         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1297 
1298         if (rc == -EAGAIN)
1299                 goto restart;
1300 
1301         return rc;
1302 }
1303 
1304 /*
1305  * Flush the global interrupt entry cache.
1306  */
1307 void qi_global_iec(struct intel_iommu *iommu)
1308 {
1309         struct qi_desc desc;
1310 
1311         desc.qw0 = QI_IEC_TYPE;
1312         desc.qw1 = 0;
1313         desc.qw2 = 0;
1314         desc.qw3 = 0;
1315 
1316         /* should never fail */
1317         qi_submit_sync(&desc, iommu);
1318 }
1319 
1320 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1321                       u64 type)
1322 {
1323         struct qi_desc desc;
1324 
1325         desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1326                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1327         desc.qw1 = 0;
1328         desc.qw2 = 0;
1329         desc.qw3 = 0;
1330 
1331         qi_submit_sync(&desc, iommu);
1332 }
1333 
1334 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1335                     unsigned int size_order, u64 type)
1336 {
1337         u8 dw = 0, dr = 0;
1338 
1339         struct qi_desc desc;
1340         int ih = 0;
1341 
1342         if (cap_write_drain(iommu->cap))
1343                 dw = 1;
1344 
1345         if (cap_read_drain(iommu->cap))
1346                 dr = 1;
1347 
1348         desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1349                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1350         desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1351                 | QI_IOTLB_AM(size_order);
1352         desc.qw2 = 0;
1353         desc.qw3 = 0;
1354 
1355         qi_submit_sync(&desc, iommu);
1356 }
1357 
1358 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1359                         u16 qdep, u64 addr, unsigned mask)
1360 {
1361         struct qi_desc desc;
1362 
1363         if (mask) {
1364                 addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1365                 desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1366         } else
1367                 desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
1368 
1369         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1370                 qdep = 0;
1371 
1372         desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1373                    QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1374         desc.qw2 = 0;
1375         desc.qw3 = 0;
1376 
1377         qi_submit_sync(&desc, iommu);
1378 }
1379 
1380 /*
1381  * Disable Queued Invalidation interface.
1382  */
1383 void dmar_disable_qi(struct intel_iommu *iommu)
1384 {
1385         unsigned long flags;
1386         u32 sts;
1387         cycles_t start_time = get_cycles();
1388 
1389         if (!ecap_qis(iommu->ecap))
1390                 return;
1391 
1392         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1393 
1394         sts =  readl(iommu->reg + DMAR_GSTS_REG);
1395         if (!(sts & DMA_GSTS_QIES))
1396                 goto end;
1397 
1398         /*
1399          * Give a chance to HW to complete the pending invalidation requests.
1400          */
1401         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1402                 readl(iommu->reg + DMAR_IQH_REG)) &&
1403                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1404                 cpu_relax();
1405 
1406         iommu->gcmd &= ~DMA_GCMD_QIE;
1407         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1408 
1409         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1410                       !(sts & DMA_GSTS_QIES), sts);
1411 end:
1412         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1413 }
1414 
1415 /*
1416  * Enable queued invalidation.
1417  */
1418 static void __dmar_enable_qi(struct intel_iommu *iommu)
1419 {
1420         u32 sts;
1421         unsigned long flags;
1422         struct q_inval *qi = iommu->qi;
1423         u64 val = virt_to_phys(qi->desc);
1424 
1425         qi->free_head = qi->free_tail = 0;
1426         qi->free_cnt = QI_LENGTH;
1427 
1428         /*
1429          * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
1430          * is present.
1431          */
1432         if (ecap_smts(iommu->ecap))
1433                 val |= (1 << 11) | 1;
1434 
1435         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1436 
1437         /* write zero to the tail reg */
1438         writel(0, iommu->reg + DMAR_IQT_REG);
1439 
1440         dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
1441 
1442         iommu->gcmd |= DMA_GCMD_QIE;
1443         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1444 
1445         /* Make sure hardware complete it */
1446         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1447 
1448         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1449 }
1450 
1451 /*
1452  * Enable Queued Invalidation interface. This is a must to support
1453  * interrupt-remapping. Also used by DMA-remapping, which replaces
1454  * register based IOTLB invalidation.
1455  */
1456 int dmar_enable_qi(struct intel_iommu *iommu)
1457 {
1458         struct q_inval *qi;
1459         struct page *desc_page;
1460 
1461         if (!ecap_qis(iommu->ecap))
1462                 return -ENOENT;
1463 
1464         /*
1465          * queued invalidation is already setup and enabled.
1466          */
1467         if (iommu->qi)
1468                 return 0;
1469 
1470         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1471         if (!iommu->qi)
1472                 return -ENOMEM;
1473 
1474         qi = iommu->qi;
1475 
1476         /*
1477          * Need two pages to accommodate 256 descriptors of 256 bits each
1478          * if the remapping hardware supports scalable mode translation.
1479          */
1480         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
1481                                      !!ecap_smts(iommu->ecap));
1482         if (!desc_page) {
1483                 kfree(qi);
1484                 iommu->qi = NULL;
1485                 return -ENOMEM;
1486         }
1487 
1488         qi->desc = page_address(desc_page);
1489 
1490         qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC);
1491         if (!qi->desc_status) {
1492                 free_page((unsigned long) qi->desc);
1493                 kfree(qi);
1494                 iommu->qi = NULL;
1495                 return -ENOMEM;
1496         }
1497 
1498         raw_spin_lock_init(&qi->q_lock);
1499 
1500         __dmar_enable_qi(iommu);
1501 
1502         return 0;
1503 }
1504 
1505 /* iommu interrupt handling. Most stuff are MSI-like. */
1506 
1507 enum faulttype {
1508         DMA_REMAP,
1509         INTR_REMAP,
1510         UNKNOWN,
1511 };
1512 
1513 static const char *dma_remap_fault_reasons[] =
1514 {
1515         "Software",
1516         "Present bit in root entry is clear",
1517         "Present bit in context entry is clear",
1518         "Invalid context entry",
1519         "Access beyond MGAW",
1520         "PTE Write access is not set",
1521         "PTE Read access is not set",
1522         "Next page table ptr is invalid",
1523         "Root table address invalid",
1524         "Context table ptr is invalid",
1525         "non-zero reserved fields in RTP",
1526         "non-zero reserved fields in CTP",
1527         "non-zero reserved fields in PTE",
1528         "PCE for translation request specifies blocking",
1529 };
1530 
1531 static const char * const dma_remap_sm_fault_reasons[] = {
1532         "SM: Invalid Root Table Address",
1533         "SM: TTM 0 for request with PASID",
1534         "SM: TTM 0 for page group request",
1535         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x33-0x37 */
1536         "SM: Error attempting to access Root Entry",
1537         "SM: Present bit in Root Entry is clear",
1538         "SM: Non-zero reserved field set in Root Entry",
1539         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x3B-0x3F */
1540         "SM: Error attempting to access Context Entry",
1541         "SM: Present bit in Context Entry is clear",
1542         "SM: Non-zero reserved field set in the Context Entry",
1543         "SM: Invalid Context Entry",
1544         "SM: DTE field in Context Entry is clear",
1545         "SM: PASID Enable field in Context Entry is clear",
1546         "SM: PASID is larger than the max in Context Entry",
1547         "SM: PRE field in Context-Entry is clear",
1548         "SM: RID_PASID field error in Context-Entry",
1549         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x49-0x4F */
1550         "SM: Error attempting to access the PASID Directory Entry",
1551         "SM: Present bit in Directory Entry is clear",
1552         "SM: Non-zero reserved field set in PASID Directory Entry",
1553         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x53-0x57 */
1554         "SM: Error attempting to access PASID Table Entry",
1555         "SM: Present bit in PASID Table Entry is clear",
1556         "SM: Non-zero reserved field set in PASID Table Entry",
1557         "SM: Invalid Scalable-Mode PASID Table Entry",
1558         "SM: ERE field is clear in PASID Table Entry",
1559         "SM: SRE field is clear in PASID Table Entry",
1560         "Unknown", "Unknown",/* 0x5E-0x5F */
1561         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x60-0x67 */
1562         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x68-0x6F */
1563         "SM: Error attempting to access first-level paging entry",
1564         "SM: Present bit in first-level paging entry is clear",
1565         "SM: Non-zero reserved field set in first-level paging entry",
1566         "SM: Error attempting to access FL-PML4 entry",
1567         "SM: First-level entry address beyond MGAW in Nested translation",
1568         "SM: Read permission error in FL-PML4 entry in Nested translation",
1569         "SM: Read permission error in first-level paging entry in Nested translation",
1570         "SM: Write permission error in first-level paging entry in Nested translation",
1571         "SM: Error attempting to access second-level paging entry",
1572         "SM: Read/Write permission error in second-level paging entry",
1573         "SM: Non-zero reserved field set in second-level paging entry",
1574         "SM: Invalid second-level page table pointer",
1575         "SM: A/D bit update needed in second-level entry when set up in no snoop",
1576         "Unknown", "Unknown", "Unknown", /* 0x7D-0x7F */
1577         "SM: Address in first-level translation is not canonical",
1578         "SM: U/S set 0 for first-level translation with user privilege",
1579         "SM: No execute permission for request with PASID and ER=1",
1580         "SM: Address beyond the DMA hardware max",
1581         "SM: Second-level entry address beyond the max",
1582         "SM: No write permission for Write/AtomicOp request",
1583         "SM: No read permission for Read/AtomicOp request",
1584         "SM: Invalid address-interrupt address",
1585         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x88-0x8F */
1586         "SM: A/D bit update needed in first-level entry when set up in no snoop",
1587 };
1588 
1589 static const char *irq_remap_fault_reasons[] =
1590 {
1591         "Detected reserved fields in the decoded interrupt-remapped request",
1592         "Interrupt index exceeded the interrupt-remapping table size",
1593         "Present field in the IRTE entry is clear",
1594         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1595         "Detected reserved fields in the IRTE entry",
1596         "Blocked a compatibility format interrupt request",
1597         "Blocked an interrupt request due to source-id verification failure",
1598 };
1599 
1600 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1601 {
1602         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1603                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1604                 *fault_type = INTR_REMAP;
1605                 return irq_remap_fault_reasons[fault_reason - 0x20];
1606         } else if (fault_reason >= 0x30 && (fault_reason - 0x30 <
1607                         ARRAY_SIZE(dma_remap_sm_fault_reasons))) {
1608                 *fault_type = DMA_REMAP;
1609                 return dma_remap_sm_fault_reasons[fault_reason - 0x30];
1610         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1611                 *fault_type = DMA_REMAP;
1612                 return dma_remap_fault_reasons[fault_reason];
1613         } else {
1614                 *fault_type = UNKNOWN;
1615                 return "Unknown";
1616         }
1617 }
1618 
1619 
1620 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1621 {
1622         if (iommu->irq == irq)
1623                 return DMAR_FECTL_REG;
1624         else if (iommu->pr_irq == irq)
1625                 return DMAR_PECTL_REG;
1626         else
1627                 BUG();
1628 }
1629 
1630 void dmar_msi_unmask(struct irq_data *data)
1631 {
1632         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1633         int reg = dmar_msi_reg(iommu, data->irq);
1634         unsigned long flag;
1635 
1636         /* unmask it */
1637         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1638         writel(0, iommu->reg + reg);
1639         /* Read a reg to force flush the post write */
1640         readl(iommu->reg + reg);
1641         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1642 }
1643 
1644 void dmar_msi_mask(struct irq_data *data)
1645 {
1646         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1647         int reg = dmar_msi_reg(iommu, data->irq);
1648         unsigned long flag;
1649 
1650         /* mask it */
1651         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1652         writel(DMA_FECTL_IM, iommu->reg + reg);
1653         /* Read a reg to force flush the post write */
1654         readl(iommu->reg + reg);
1655         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1656 }
1657 
1658 void dmar_msi_write(int irq, struct msi_msg *msg)
1659 {
1660         struct intel_iommu *iommu = irq_get_handler_data(irq);
1661         int reg = dmar_msi_reg(iommu, irq);
1662         unsigned long flag;
1663 
1664         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1665         writel(msg->data, iommu->reg + reg + 4);
1666         writel(msg->address_lo, iommu->reg + reg + 8);
1667         writel(msg->address_hi, iommu->reg + reg + 12);
1668         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1669 }
1670 
1671 void dmar_msi_read(int irq, struct msi_msg *msg)
1672 {
1673         struct intel_iommu *iommu = irq_get_handler_data(irq);
1674         int reg = dmar_msi_reg(iommu, irq);
1675         unsigned long flag;
1676 
1677         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1678         msg->data = readl(iommu->reg + reg + 4);
1679         msg->address_lo = readl(iommu->reg + reg + 8);
1680         msg->address_hi = readl(iommu->reg + reg + 12);
1681         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1682 }
1683 
1684 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1685                 u8 fault_reason, int pasid, u16 source_id,
1686                 unsigned long long addr)
1687 {
1688         const char *reason;
1689         int fault_type;
1690 
1691         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1692 
1693         if (fault_type == INTR_REMAP)
1694                 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
1695                         source_id >> 8, PCI_SLOT(source_id & 0xFF),
1696                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1697                         fault_reason, reason);
1698         else
1699                 pr_err("[%s] Request device [%02x:%02x.%d] PASID %x fault addr %llx [fault reason %02d] %s\n",
1700                        type ? "DMA Read" : "DMA Write",
1701                        source_id >> 8, PCI_SLOT(source_id & 0xFF),
1702                        PCI_FUNC(source_id & 0xFF), pasid, addr,
1703                        fault_reason, reason);
1704         return 0;
1705 }
1706 
1707 #define PRIMARY_FAULT_REG_LEN (16)
1708 irqreturn_t dmar_fault(int irq, void *dev_id)
1709 {
1710         struct intel_iommu *iommu = dev_id;
1711         int reg, fault_index;
1712         u32 fault_status;
1713         unsigned long flag;
1714         static DEFINE_RATELIMIT_STATE(rs,
1715                                       DEFAULT_RATELIMIT_INTERVAL,
1716                                       DEFAULT_RATELIMIT_BURST);
1717 
1718         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1719         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1720         if (fault_status && __ratelimit(&rs))
1721                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1722 
1723         /* TBD: ignore advanced fault log currently */
1724         if (!(fault_status & DMA_FSTS_PPF))
1725                 goto unlock_exit;
1726 
1727         fault_index = dma_fsts_fault_record_index(fault_status);
1728         reg = cap_fault_reg_offset(iommu->cap);
1729         while (1) {
1730                 /* Disable printing, simply clear the fault when ratelimited */
1731                 bool ratelimited = !__ratelimit(&rs);
1732                 u8 fault_reason;
1733                 u16 source_id;
1734                 u64 guest_addr;
1735                 int type, pasid;
1736                 u32 data;
1737                 bool pasid_present;
1738 
1739                 /* highest 32 bits */
1740                 data = readl(iommu->reg + reg +
1741                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1742                 if (!(data & DMA_FRCD_F))
1743                         break;
1744 
1745                 if (!ratelimited) {
1746                         fault_reason = dma_frcd_fault_reason(data);
1747                         type = dma_frcd_type(data);
1748 
1749                         pasid = dma_frcd_pasid_value(data);
1750                         data = readl(iommu->reg + reg +
1751                                      fault_index * PRIMARY_FAULT_REG_LEN + 8);
1752                         source_id = dma_frcd_source_id(data);
1753 
1754                         pasid_present = dma_frcd_pasid_present(data);
1755                         guest_addr = dmar_readq(iommu->reg + reg +
1756                                         fault_index * PRIMARY_FAULT_REG_LEN);
1757                         guest_addr = dma_frcd_page_addr(guest_addr);
1758                 }
1759 
1760                 /* clear the fault */
1761                 writel(DMA_FRCD_F, iommu->reg + reg +
1762                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1763 
1764                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1765 
1766                 if (!ratelimited)
1767                         /* Using pasid -1 if pasid is not present */
1768                         dmar_fault_do_one(iommu, type, fault_reason,
1769                                           pasid_present ? pasid : -1,
1770                                           source_id, guest_addr);
1771 
1772                 fault_index++;
1773                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1774                         fault_index = 0;
1775                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1776         }
1777 
1778         writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO,
1779                iommu->reg + DMAR_FSTS_REG);
1780 
1781 unlock_exit:
1782         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1783         return IRQ_HANDLED;
1784 }
1785 
1786 int dmar_set_interrupt(struct intel_iommu *iommu)
1787 {
1788         int irq, ret;
1789 
1790         /*
1791          * Check if the fault interrupt is already initialized.
1792          */
1793         if (iommu->irq)
1794                 return 0;
1795 
1796         irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1797         if (irq > 0) {
1798                 iommu->irq = irq;
1799         } else {
1800                 pr_err("No free IRQ vectors\n");
1801                 return -EINVAL;
1802         }
1803 
1804         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1805         if (ret)
1806                 pr_err("Can't request irq\n");
1807         return ret;
1808 }
1809 
1810 int __init enable_drhd_fault_handling(void)
1811 {
1812         struct dmar_drhd_unit *drhd;
1813         struct intel_iommu *iommu;
1814 
1815         /*
1816          * Enable fault control interrupt.
1817          */
1818         for_each_iommu(iommu, drhd) {
1819                 u32 fault_status;
1820                 int ret = dmar_set_interrupt(iommu);
1821 
1822                 if (ret) {
1823                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1824                                (unsigned long long)drhd->reg_base_addr, ret);
1825                         return -1;
1826                 }
1827 
1828                 /*
1829                  * Clear any previous faults.
1830                  */
1831                 dmar_fault(iommu->irq, iommu);
1832                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1833                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1834         }
1835 
1836         return 0;
1837 }
1838 
1839 /*
1840  * Re-enable Queued Invalidation interface.
1841  */
1842 int dmar_reenable_qi(struct intel_iommu *iommu)
1843 {
1844         if (!ecap_qis(iommu->ecap))
1845                 return -ENOENT;
1846 
1847         if (!iommu->qi)
1848                 return -ENOENT;
1849 
1850         /*
1851          * First disable queued invalidation.
1852          */
1853         dmar_disable_qi(iommu);
1854         /*
1855          * Then enable queued invalidation again. Since there is no pending
1856          * invalidation requests now, it's safe to re-enable queued
1857          * invalidation.
1858          */
1859         __dmar_enable_qi(iommu);
1860 
1861         return 0;
1862 }
1863 
1864 /*
1865  * Check interrupt remapping support in DMAR table description.
1866  */
1867 int __init dmar_ir_support(void)
1868 {
1869         struct acpi_table_dmar *dmar;
1870         dmar = (struct acpi_table_dmar *)dmar_tbl;
1871         if (!dmar)
1872                 return 0;
1873         return dmar->flags & 0x1;
1874 }
1875 
1876 /* Check whether DMAR units are in use */
1877 static inline bool dmar_in_use(void)
1878 {
1879         return irq_remapping_enabled || intel_iommu_enabled;
1880 }
1881 
1882 static int __init dmar_free_unused_resources(void)
1883 {
1884         struct dmar_drhd_unit *dmaru, *dmaru_n;
1885 
1886         if (dmar_in_use())
1887                 return 0;
1888 
1889         if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1890                 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1891 
1892         down_write(&dmar_global_lock);
1893         list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1894                 list_del(&dmaru->list);
1895                 dmar_free_drhd(dmaru);
1896         }
1897         up_write(&dmar_global_lock);
1898 
1899         return 0;
1900 }
1901 
1902 late_initcall(dmar_free_unused_resources);
1903 IOMMU_INIT_POST(detect_intel_iommu);
1904 
1905 /*
1906  * DMAR Hotplug Support
1907  * For more details, please refer to Intel(R) Virtualization Technology
1908  * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1909  * "Remapping Hardware Unit Hot Plug".
1910  */
1911 static guid_t dmar_hp_guid =
1912         GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
1913                   0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
1914 
1915 /*
1916  * Currently there's only one revision and BIOS will not check the revision id,
1917  * so use 0 for safety.
1918  */
1919 #define DMAR_DSM_REV_ID                 0
1920 #define DMAR_DSM_FUNC_DRHD              1
1921 #define DMAR_DSM_FUNC_ATSR              2
1922 #define DMAR_DSM_FUNC_RHSA              3
1923 
1924 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1925 {
1926         return acpi_check_dsm(handle, &dmar_hp_guid, DMAR_DSM_REV_ID, 1 << func);
1927 }
1928 
1929 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1930                                   dmar_res_handler_t handler, void *arg)
1931 {
1932         int ret = -ENODEV;
1933         union acpi_object *obj;
1934         struct acpi_dmar_header *start;
1935         struct dmar_res_callback callback;
1936         static int res_type[] = {
1937                 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1938                 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1939                 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1940         };
1941 
1942         if (!dmar_detect_dsm(handle, func))
1943                 return 0;
1944 
1945         obj = acpi_evaluate_dsm_typed(handle, &dmar_hp_guid, DMAR_DSM_REV_ID,
1946                                       func, NULL, ACPI_TYPE_BUFFER);
1947         if (!obj)
1948                 return -ENODEV;
1949 
1950         memset(&callback, 0, sizeof(callback));
1951         callback.cb[res_type[func]] = handler;
1952         callback.arg[res_type[func]] = arg;
1953         start = (struct acpi_dmar_header *)obj->buffer.pointer;
1954         ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1955 
1956         ACPI_FREE(obj);
1957 
1958         return ret;
1959 }
1960 
1961 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1962 {
1963         int ret;
1964         struct dmar_drhd_unit *dmaru;
1965 
1966         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1967         if (!dmaru)
1968                 return -ENODEV;
1969 
1970         ret = dmar_ir_hotplug(dmaru, true);
1971         if (ret == 0)
1972                 ret = dmar_iommu_hotplug(dmaru, true);
1973 
1974         return ret;
1975 }
1976 
1977 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1978 {
1979         int i, ret;
1980         struct device *dev;
1981         struct dmar_drhd_unit *dmaru;
1982 
1983         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1984         if (!dmaru)
1985                 return 0;
1986 
1987         /*
1988          * All PCI devices managed by this unit should have been destroyed.
1989          */
1990         if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
1991                 for_each_active_dev_scope(dmaru->devices,
1992                                           dmaru->devices_cnt, i, dev)
1993                         return -EBUSY;
1994         }
1995 
1996         ret = dmar_ir_hotplug(dmaru, false);
1997         if (ret == 0)
1998                 ret = dmar_iommu_hotplug(dmaru, false);
1999 
2000         return ret;
2001 }
2002 
2003 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
2004 {
2005         struct dmar_drhd_unit *dmaru;
2006 
2007         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2008         if (dmaru) {
2009                 list_del_rcu(&dmaru->list);
2010                 synchronize_rcu();
2011                 dmar_free_drhd(dmaru);
2012         }
2013 
2014         return 0;
2015 }
2016 
2017 static int dmar_hotplug_insert(acpi_handle handle)
2018 {
2019         int ret;
2020         int drhd_count = 0;
2021 
2022         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2023                                      &dmar_validate_one_drhd, (void *)1);
2024         if (ret)
2025                 goto out;
2026 
2027         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2028                                      &dmar_parse_one_drhd, (void *)&drhd_count);
2029         if (ret == 0 && drhd_count == 0) {
2030                 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
2031                 goto out;
2032         } else if (ret) {
2033                 goto release_drhd;
2034         }
2035 
2036         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
2037                                      &dmar_parse_one_rhsa, NULL);
2038         if (ret)
2039                 goto release_drhd;
2040 
2041         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2042                                      &dmar_parse_one_atsr, NULL);
2043         if (ret)
2044                 goto release_atsr;
2045 
2046         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2047                                      &dmar_hp_add_drhd, NULL);
2048         if (!ret)
2049                 return 0;
2050 
2051         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2052                                &dmar_hp_remove_drhd, NULL);
2053 release_atsr:
2054         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2055                                &dmar_release_one_atsr, NULL);
2056 release_drhd:
2057         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2058                                &dmar_hp_release_drhd, NULL);
2059 out:
2060         return ret;
2061 }
2062 
2063 static int dmar_hotplug_remove(acpi_handle handle)
2064 {
2065         int ret;
2066 
2067         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2068                                      &dmar_check_one_atsr, NULL);
2069         if (ret)
2070                 return ret;
2071 
2072         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2073                                      &dmar_hp_remove_drhd, NULL);
2074         if (ret == 0) {
2075                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2076                                                &dmar_release_one_atsr, NULL));
2077                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2078                                                &dmar_hp_release_drhd, NULL));
2079         } else {
2080                 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2081                                        &dmar_hp_add_drhd, NULL);
2082         }
2083 
2084         return ret;
2085 }
2086 
2087 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
2088                                        void *context, void **retval)
2089 {
2090         acpi_handle *phdl = retval;
2091 
2092         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2093                 *phdl = handle;
2094                 return AE_CTRL_TERMINATE;
2095         }
2096 
2097         return AE_OK;
2098 }
2099 
2100 static int dmar_device_hotplug(acpi_handle handle, bool insert)
2101 {
2102         int ret;
2103         acpi_handle tmp = NULL;
2104         acpi_status status;
2105 
2106         if (!dmar_in_use())
2107                 return 0;
2108 
2109         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2110                 tmp = handle;
2111         } else {
2112                 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2113                                              ACPI_UINT32_MAX,
2114                                              dmar_get_dsm_handle,
2115                                              NULL, NULL, &tmp);
2116                 if (ACPI_FAILURE(status)) {
2117                         pr_warn("Failed to locate _DSM method.\n");
2118                         return -ENXIO;
2119                 }
2120         }
2121         if (tmp == NULL)
2122                 return 0;
2123 
2124         down_write(&dmar_global_lock);
2125         if (insert)
2126                 ret = dmar_hotplug_insert(tmp);
2127         else
2128                 ret = dmar_hotplug_remove(tmp);
2129         up_write(&dmar_global_lock);
2130 
2131         return ret;
2132 }
2133 
2134 int dmar_device_add(acpi_handle handle)
2135 {
2136         return dmar_device_hotplug(handle, true);
2137 }
2138 
2139 int dmar_device_remove(acpi_handle handle)
2140 {
2141         return dmar_device_hotplug(handle, false);
2142 }
2143 
2144 /*
2145  * dmar_platform_optin - Is %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in DMAR table
2146  *
2147  * Returns true if the platform has %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in
2148  * the ACPI DMAR table. This means that the platform boot firmware has made
2149  * sure no device can issue DMA outside of RMRR regions.
2150  */
2151 bool dmar_platform_optin(void)
2152 {
2153         struct acpi_table_dmar *dmar;
2154         acpi_status status;
2155         bool ret;
2156 
2157         status = acpi_get_table(ACPI_SIG_DMAR, 0,
2158                                 (struct acpi_table_header **)&dmar);
2159         if (ACPI_FAILURE(status))
2160                 return false;
2161 
2162         ret = !!(dmar->flags & DMAR_PLATFORM_OPT_IN);
2163         acpi_put_table((struct acpi_table_header *)dmar);
2164 
2165         return ret;
2166 }
2167 EXPORT_SYMBOL_GPL(dmar_platform_optin);
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/iommu/dmar.c

DEFINITIONS
