1#include <linux/interrupt.h> 2#include <linux/dmar.h> 3#include <linux/spinlock.h> 4#include <linux/slab.h> 5#include <linux/jiffies.h> 6#include <linux/hpet.h> 7#include <linux/pci.h> 8#include <linux/irq.h> 9#include <linux/intel-iommu.h> 10#include <linux/acpi.h> 11#include <asm/io_apic.h> 12#include <asm/smp.h> 13#include <asm/cpu.h> 14#include <asm/irq_remapping.h> 15#include <asm/pci-direct.h> 16#include <asm/msidef.h> 17 18#include "irq_remapping.h" 19 20struct ioapic_scope { 21 struct intel_iommu *iommu; 22 unsigned int id; 23 unsigned int bus; /* PCI bus number */ 24 unsigned int devfn; /* PCI devfn number */ 25}; 26 27struct hpet_scope { 28 struct intel_iommu *iommu; 29 u8 id; 30 unsigned int bus; 31 unsigned int devfn; 32}; 33 34#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) 35#define IRTE_DEST(dest) ((eim_mode) ? dest : dest << 8) 36 37static int __read_mostly eim_mode; 38static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; 39static struct hpet_scope ir_hpet[MAX_HPET_TBS]; 40 41/* 42 * Lock ordering: 43 * ->dmar_global_lock 44 * ->irq_2_ir_lock 45 * ->qi->q_lock 46 * ->iommu->register_lock 47 * Note: 48 * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called 49 * in single-threaded environment with interrupt disabled, so no need to tabke 50 * the dmar_global_lock. 51 */ 52static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); 53 54static int __init parse_ioapics_under_ir(void); 55 56static struct irq_2_iommu *irq_2_iommu(unsigned int irq) 57{ 58 struct irq_cfg *cfg = irq_cfg(irq); 59 return cfg ? &cfg->irq_2_iommu : NULL; 60} 61 62static int get_irte(int irq, struct irte *entry) 63{ 64 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); 65 unsigned long flags; 66 int index; 67 68 if (!entry || !irq_iommu) 69 return -1; 70 71 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 72 73 if (unlikely(!irq_iommu->iommu)) { 74 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 75 return -1; 76 } 77 78 index = irq_iommu->irte_index + irq_iommu->sub_handle; 79 *entry = *(irq_iommu->iommu->ir_table->base + index); 80 81 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 82 return 0; 83} 84 85static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) 86{ 87 struct ir_table *table = iommu->ir_table; 88 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); 89 struct irq_cfg *cfg = irq_cfg(irq); 90 unsigned int mask = 0; 91 unsigned long flags; 92 int index; 93 94 if (!count || !irq_iommu) 95 return -1; 96 97 if (count > 1) { 98 count = __roundup_pow_of_two(count); 99 mask = ilog2(count); 100 } 101 102 if (mask > ecap_max_handle_mask(iommu->ecap)) { 103 printk(KERN_ERR 104 "Requested mask %x exceeds the max invalidation handle" 105 " mask value %Lx\n", mask, 106 ecap_max_handle_mask(iommu->ecap)); 107 return -1; 108 } 109 110 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 111 index = bitmap_find_free_region(table->bitmap, 112 INTR_REMAP_TABLE_ENTRIES, mask); 113 if (index < 0) { 114 pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id); 115 } else { 116 cfg->remapped = 1; 117 irq_iommu->iommu = iommu; 118 irq_iommu->irte_index = index; 119 irq_iommu->sub_handle = 0; 120 irq_iommu->irte_mask = mask; 121 } 122 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 123 124 return index; 125} 126 127static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) 128{ 129 struct qi_desc desc; 130 131 desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) 132 | QI_IEC_SELECTIVE; 133 desc.high = 0; 134 135 return qi_submit_sync(&desc, iommu); 136} 137 138static int map_irq_to_irte_handle(int irq, u16 *sub_handle) 139{ 140 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); 141 unsigned long flags; 142 int index; 143 144 if (!irq_iommu) 145 return -1; 146 147 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 148 *sub_handle = irq_iommu->sub_handle; 149 index = irq_iommu->irte_index; 150 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 151 return index; 152} 153 154static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) 155{ 156 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); 157 struct irq_cfg *cfg = irq_cfg(irq); 158 unsigned long flags; 159 160 if (!irq_iommu) 161 return -1; 162 163 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 164 165 cfg->remapped = 1; 166 irq_iommu->iommu = iommu; 167 irq_iommu->irte_index = index; 168 irq_iommu->sub_handle = subhandle; 169 irq_iommu->irte_mask = 0; 170 171 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 172 173 return 0; 174} 175 176static int modify_irte(int irq, struct irte *irte_modified) 177{ 178 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); 179 struct intel_iommu *iommu; 180 unsigned long flags; 181 struct irte *irte; 182 int rc, index; 183 184 if (!irq_iommu) 185 return -1; 186 187 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 188 189 iommu = irq_iommu->iommu; 190 191 index = irq_iommu->irte_index + irq_iommu->sub_handle; 192 irte = &iommu->ir_table->base[index]; 193 194 set_64bit(&irte->low, irte_modified->low); 195 set_64bit(&irte->high, irte_modified->high); 196 __iommu_flush_cache(iommu, irte, sizeof(*irte)); 197 198 rc = qi_flush_iec(iommu, index, 0); 199 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 200 201 return rc; 202} 203 204static struct intel_iommu *map_hpet_to_ir(u8 hpet_id) 205{ 206 int i; 207 208 for (i = 0; i < MAX_HPET_TBS; i++) 209 if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu) 210 return ir_hpet[i].iommu; 211 return NULL; 212} 213 214static struct intel_iommu *map_ioapic_to_ir(int apic) 215{ 216 int i; 217 218 for (i = 0; i < MAX_IO_APICS; i++) 219 if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu) 220 return ir_ioapic[i].iommu; 221 return NULL; 222} 223 224static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) 225{ 226 struct dmar_drhd_unit *drhd; 227 228 drhd = dmar_find_matched_drhd_unit(dev); 229 if (!drhd) 230 return NULL; 231 232 return drhd->iommu; 233} 234 235static int clear_entries(struct irq_2_iommu *irq_iommu) 236{ 237 struct irte *start, *entry, *end; 238 struct intel_iommu *iommu; 239 int index; 240 241 if (irq_iommu->sub_handle) 242 return 0; 243 244 iommu = irq_iommu->iommu; 245 index = irq_iommu->irte_index + irq_iommu->sub_handle; 246 247 start = iommu->ir_table->base + index; 248 end = start + (1 << irq_iommu->irte_mask); 249 250 for (entry = start; entry < end; entry++) { 251 set_64bit(&entry->low, 0); 252 set_64bit(&entry->high, 0); 253 } 254 bitmap_release_region(iommu->ir_table->bitmap, index, 255 irq_iommu->irte_mask); 256 257 return qi_flush_iec(iommu, index, irq_iommu->irte_mask); 258} 259 260static int free_irte(int irq) 261{ 262 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); 263 unsigned long flags; 264 int rc; 265 266 if (!irq_iommu) 267 return -1; 268 269 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 270 271 rc = clear_entries(irq_iommu); 272 273 irq_iommu->iommu = NULL; 274 irq_iommu->irte_index = 0; 275 irq_iommu->sub_handle = 0; 276 irq_iommu->irte_mask = 0; 277 278 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 279 280 return rc; 281} 282 283/* 284 * source validation type 285 */ 286#define SVT_NO_VERIFY 0x0 /* no verification is required */ 287#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ 288#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ 289 290/* 291 * source-id qualifier 292 */ 293#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ 294#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore 295 * the third least significant bit 296 */ 297#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore 298 * the second and third least significant bits 299 */ 300#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore 301 * the least three significant bits 302 */ 303 304/* 305 * set SVT, SQ and SID fields of irte to verify 306 * source ids of interrupt requests 307 */ 308static void set_irte_sid(struct irte *irte, unsigned int svt, 309 unsigned int sq, unsigned int sid) 310{ 311 if (disable_sourceid_checking) 312 svt = SVT_NO_VERIFY; 313 irte->svt = svt; 314 irte->sq = sq; 315 irte->sid = sid; 316} 317 318static int set_ioapic_sid(struct irte *irte, int apic) 319{ 320 int i; 321 u16 sid = 0; 322 323 if (!irte) 324 return -1; 325 326 down_read(&dmar_global_lock); 327 for (i = 0; i < MAX_IO_APICS; i++) { 328 if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) { 329 sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; 330 break; 331 } 332 } 333 up_read(&dmar_global_lock); 334 335 if (sid == 0) { 336 pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); 337 return -1; 338 } 339 340 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, sid); 341 342 return 0; 343} 344 345static int set_hpet_sid(struct irte *irte, u8 id) 346{ 347 int i; 348 u16 sid = 0; 349 350 if (!irte) 351 return -1; 352 353 down_read(&dmar_global_lock); 354 for (i = 0; i < MAX_HPET_TBS; i++) { 355 if (ir_hpet[i].iommu && ir_hpet[i].id == id) { 356 sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; 357 break; 358 } 359 } 360 up_read(&dmar_global_lock); 361 362 if (sid == 0) { 363 pr_warning("Failed to set source-id of HPET block (%d)\n", id); 364 return -1; 365 } 366 367 /* 368 * Should really use SQ_ALL_16. Some platforms are broken. 369 * While we figure out the right quirks for these broken platforms, use 370 * SQ_13_IGNORE_3 for now. 371 */ 372 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); 373 374 return 0; 375} 376 377struct set_msi_sid_data { 378 struct pci_dev *pdev; 379 u16 alias; 380}; 381 382static int set_msi_sid_cb(struct pci_dev *pdev, u16 alias, void *opaque) 383{ 384 struct set_msi_sid_data *data = opaque; 385 386 data->pdev = pdev; 387 data->alias = alias; 388 389 return 0; 390} 391 392static int set_msi_sid(struct irte *irte, struct pci_dev *dev) 393{ 394 struct set_msi_sid_data data; 395 396 if (!irte || !dev) 397 return -1; 398 399 pci_for_each_dma_alias(dev, set_msi_sid_cb, &data); 400 401 /* 402 * DMA alias provides us with a PCI device and alias. The only case 403 * where the it will return an alias on a different bus than the 404 * device is the case of a PCIe-to-PCI bridge, where the alias is for 405 * the subordinate bus. In this case we can only verify the bus. 406 * 407 * If the alias device is on a different bus than our source device 408 * then we have a topology based alias, use it. 409 * 410 * Otherwise, the alias is for a device DMA quirk and we cannot 411 * assume that MSI uses the same requester ID. Therefore use the 412 * original device. 413 */ 414 if (PCI_BUS_NUM(data.alias) != data.pdev->bus->number) 415 set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, 416 PCI_DEVID(PCI_BUS_NUM(data.alias), 417 dev->bus->number)); 418 else if (data.pdev->bus->number != dev->bus->number) 419 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, data.alias); 420 else 421 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, 422 PCI_DEVID(dev->bus->number, dev->devfn)); 423 424 return 0; 425} 426 427static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) 428{ 429 u64 addr; 430 u32 sts; 431 unsigned long flags; 432 433 addr = virt_to_phys((void *)iommu->ir_table->base); 434 435 raw_spin_lock_irqsave(&iommu->register_lock, flags); 436 437 dmar_writeq(iommu->reg + DMAR_IRTA_REG, 438 (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); 439 440 /* Set interrupt-remapping table pointer */ 441 writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG); 442 443 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 444 readl, (sts & DMA_GSTS_IRTPS), sts); 445 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 446 447 /* 448 * global invalidation of interrupt entry cache before enabling 449 * interrupt-remapping. 450 */ 451 qi_global_iec(iommu); 452 453 raw_spin_lock_irqsave(&iommu->register_lock, flags); 454 455 /* Enable interrupt-remapping */ 456 iommu->gcmd |= DMA_GCMD_IRE; 457 iommu->gcmd &= ~DMA_GCMD_CFI; /* Block compatibility-format MSIs */ 458 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 459 460 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 461 readl, (sts & DMA_GSTS_IRES), sts); 462 463 /* 464 * With CFI clear in the Global Command register, we should be 465 * protected from dangerous (i.e. compatibility) interrupts 466 * regardless of x2apic status. Check just to be sure. 467 */ 468 if (sts & DMA_GSTS_CFIS) 469 WARN(1, KERN_WARNING 470 "Compatibility-format IRQs enabled despite intr remapping;\n" 471 "you are vulnerable to IRQ injection.\n"); 472 473 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 474} 475 476static int intel_setup_irq_remapping(struct intel_iommu *iommu) 477{ 478 struct ir_table *ir_table; 479 struct page *pages; 480 unsigned long *bitmap; 481 482 if (iommu->ir_table) 483 return 0; 484 485 ir_table = kzalloc(sizeof(struct ir_table), GFP_KERNEL); 486 if (!ir_table) 487 return -ENOMEM; 488 489 pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, 490 INTR_REMAP_PAGE_ORDER); 491 492 if (!pages) { 493 pr_err("IR%d: failed to allocate pages of order %d\n", 494 iommu->seq_id, INTR_REMAP_PAGE_ORDER); 495 goto out_free_table; 496 } 497 498 bitmap = kcalloc(BITS_TO_LONGS(INTR_REMAP_TABLE_ENTRIES), 499 sizeof(long), GFP_ATOMIC); 500 if (bitmap == NULL) { 501 pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id); 502 goto out_free_pages; 503 } 504 505 ir_table->base = page_address(pages); 506 ir_table->bitmap = bitmap; 507 iommu->ir_table = ir_table; 508 return 0; 509 510out_free_pages: 511 __free_pages(pages, INTR_REMAP_PAGE_ORDER); 512out_free_table: 513 kfree(ir_table); 514 return -ENOMEM; 515} 516 517static void intel_teardown_irq_remapping(struct intel_iommu *iommu) 518{ 519 if (iommu && iommu->ir_table) { 520 free_pages((unsigned long)iommu->ir_table->base, 521 INTR_REMAP_PAGE_ORDER); 522 kfree(iommu->ir_table->bitmap); 523 kfree(iommu->ir_table); 524 iommu->ir_table = NULL; 525 } 526} 527 528/* 529 * Disable Interrupt Remapping. 530 */ 531static void iommu_disable_irq_remapping(struct intel_iommu *iommu) 532{ 533 unsigned long flags; 534 u32 sts; 535 536 if (!ecap_ir_support(iommu->ecap)) 537 return; 538 539 /* 540 * global invalidation of interrupt entry cache before disabling 541 * interrupt-remapping. 542 */ 543 qi_global_iec(iommu); 544 545 raw_spin_lock_irqsave(&iommu->register_lock, flags); 546 547 sts = readl(iommu->reg + DMAR_GSTS_REG); 548 if (!(sts & DMA_GSTS_IRES)) 549 goto end; 550 551 iommu->gcmd &= ~DMA_GCMD_IRE; 552 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 553 554 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 555 readl, !(sts & DMA_GSTS_IRES), sts); 556 557end: 558 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 559} 560 561static int __init dmar_x2apic_optout(void) 562{ 563 struct acpi_table_dmar *dmar; 564 dmar = (struct acpi_table_dmar *)dmar_tbl; 565 if (!dmar || no_x2apic_optout) 566 return 0; 567 return dmar->flags & DMAR_X2APIC_OPT_OUT; 568} 569 570static void __init intel_cleanup_irq_remapping(void) 571{ 572 struct dmar_drhd_unit *drhd; 573 struct intel_iommu *iommu; 574 575 for_each_iommu(iommu, drhd) { 576 if (ecap_ir_support(iommu->ecap)) { 577 iommu_disable_irq_remapping(iommu); 578 intel_teardown_irq_remapping(iommu); 579 } 580 } 581 582 if (x2apic_supported()) 583 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); 584} 585 586static int __init intel_prepare_irq_remapping(void) 587{ 588 struct dmar_drhd_unit *drhd; 589 struct intel_iommu *iommu; 590 591 if (irq_remap_broken) { 592 printk(KERN_WARNING 593 "This system BIOS has enabled interrupt remapping\n" 594 "on a chipset that contains an erratum making that\n" 595 "feature unstable. To maintain system stability\n" 596 "interrupt remapping is being disabled. Please\n" 597 "contact your BIOS vendor for an update\n"); 598 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); 599 return -ENODEV; 600 } 601 602 if (dmar_table_init() < 0) 603 return -ENODEV; 604 605 if (!dmar_ir_support()) 606 return -ENODEV; 607 608 if (parse_ioapics_under_ir() != 1) { 609 printk(KERN_INFO "Not enabling interrupt remapping\n"); 610 goto error; 611 } 612 613 /* First make sure all IOMMUs support IRQ remapping */ 614 for_each_iommu(iommu, drhd) 615 if (!ecap_ir_support(iommu->ecap)) 616 goto error; 617 618 /* Do the allocations early */ 619 for_each_iommu(iommu, drhd) 620 if (intel_setup_irq_remapping(iommu)) 621 goto error; 622 623 return 0; 624 625error: 626 intel_cleanup_irq_remapping(); 627 return -ENODEV; 628} 629 630static int __init intel_enable_irq_remapping(void) 631{ 632 struct dmar_drhd_unit *drhd; 633 struct intel_iommu *iommu; 634 bool setup = false; 635 int eim = 0; 636 637 if (x2apic_supported()) { 638 eim = !dmar_x2apic_optout(); 639 if (!eim) 640 pr_info("x2apic is disabled because BIOS sets x2apic opt out bit. You can use 'intremap=no_x2apic_optout' to override the BIOS setting.\n"); 641 } 642 643 for_each_iommu(iommu, drhd) { 644 /* 645 * If the queued invalidation is already initialized, 646 * shouldn't disable it. 647 */ 648 if (iommu->qi) 649 continue; 650 651 /* 652 * Clear previous faults. 653 */ 654 dmar_fault(-1, iommu); 655 656 /* 657 * Disable intr remapping and queued invalidation, if already 658 * enabled prior to OS handover. 659 */ 660 iommu_disable_irq_remapping(iommu); 661 662 dmar_disable_qi(iommu); 663 } 664 665 /* 666 * check for the Interrupt-remapping support 667 */ 668 for_each_iommu(iommu, drhd) 669 if (eim && !ecap_eim_support(iommu->ecap)) { 670 printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " 671 " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); 672 eim = 0; 673 } 674 eim_mode = eim; 675 if (eim) 676 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); 677 678 /* 679 * Enable queued invalidation for all the DRHD's. 680 */ 681 for_each_iommu(iommu, drhd) { 682 int ret = dmar_enable_qi(iommu); 683 684 if (ret) { 685 printk(KERN_ERR "DRHD %Lx: failed to enable queued, " 686 " invalidation, ecap %Lx, ret %d\n", 687 drhd->reg_base_addr, iommu->ecap, ret); 688 goto error; 689 } 690 } 691 692 /* 693 * Setup Interrupt-remapping for all the DRHD's now. 694 */ 695 for_each_iommu(iommu, drhd) { 696 iommu_set_irq_remapping(iommu, eim); 697 setup = true; 698 } 699 700 if (!setup) 701 goto error; 702 703 irq_remapping_enabled = 1; 704 705 /* 706 * VT-d has a different layout for IO-APIC entries when 707 * interrupt remapping is enabled. So it needs a special routine 708 * to print IO-APIC entries for debugging purposes too. 709 */ 710 x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries; 711 712 pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); 713 714 return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; 715 716error: 717 intel_cleanup_irq_remapping(); 718 return -1; 719} 720 721static int ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, 722 struct intel_iommu *iommu, 723 struct acpi_dmar_hardware_unit *drhd) 724{ 725 struct acpi_dmar_pci_path *path; 726 u8 bus; 727 int count, free = -1; 728 729 bus = scope->bus; 730 path = (struct acpi_dmar_pci_path *)(scope + 1); 731 count = (scope->length - sizeof(struct acpi_dmar_device_scope)) 732 / sizeof(struct acpi_dmar_pci_path); 733 734 while (--count > 0) { 735 /* 736 * Access PCI directly due to the PCI 737 * subsystem isn't initialized yet. 738 */ 739 bus = read_pci_config_byte(bus, path->device, path->function, 740 PCI_SECONDARY_BUS); 741 path++; 742 } 743 744 for (count = 0; count < MAX_HPET_TBS; count++) { 745 if (ir_hpet[count].iommu == iommu && 746 ir_hpet[count].id == scope->enumeration_id) 747 return 0; 748 else if (ir_hpet[count].iommu == NULL && free == -1) 749 free = count; 750 } 751 if (free == -1) { 752 pr_warn("Exceeded Max HPET blocks\n"); 753 return -ENOSPC; 754 } 755 756 ir_hpet[free].iommu = iommu; 757 ir_hpet[free].id = scope->enumeration_id; 758 ir_hpet[free].bus = bus; 759 ir_hpet[free].devfn = PCI_DEVFN(path->device, path->function); 760 pr_info("HPET id %d under DRHD base 0x%Lx\n", 761 scope->enumeration_id, drhd->address); 762 763 return 0; 764} 765 766static int ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, 767 struct intel_iommu *iommu, 768 struct acpi_dmar_hardware_unit *drhd) 769{ 770 struct acpi_dmar_pci_path *path; 771 u8 bus; 772 int count, free = -1; 773 774 bus = scope->bus; 775 path = (struct acpi_dmar_pci_path *)(scope + 1); 776 count = (scope->length - sizeof(struct acpi_dmar_device_scope)) 777 / sizeof(struct acpi_dmar_pci_path); 778 779 while (--count > 0) { 780 /* 781 * Access PCI directly due to the PCI 782 * subsystem isn't initialized yet. 783 */ 784 bus = read_pci_config_byte(bus, path->device, path->function, 785 PCI_SECONDARY_BUS); 786 path++; 787 } 788 789 for (count = 0; count < MAX_IO_APICS; count++) { 790 if (ir_ioapic[count].iommu == iommu && 791 ir_ioapic[count].id == scope->enumeration_id) 792 return 0; 793 else if (ir_ioapic[count].iommu == NULL && free == -1) 794 free = count; 795 } 796 if (free == -1) { 797 pr_warn("Exceeded Max IO APICS\n"); 798 return -ENOSPC; 799 } 800 801 ir_ioapic[free].bus = bus; 802 ir_ioapic[free].devfn = PCI_DEVFN(path->device, path->function); 803 ir_ioapic[free].iommu = iommu; 804 ir_ioapic[free].id = scope->enumeration_id; 805 pr_info("IOAPIC id %d under DRHD base 0x%Lx IOMMU %d\n", 806 scope->enumeration_id, drhd->address, iommu->seq_id); 807 808 return 0; 809} 810 811static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, 812 struct intel_iommu *iommu) 813{ 814 int ret = 0; 815 struct acpi_dmar_hardware_unit *drhd; 816 struct acpi_dmar_device_scope *scope; 817 void *start, *end; 818 819 drhd = (struct acpi_dmar_hardware_unit *)header; 820 start = (void *)(drhd + 1); 821 end = ((void *)drhd) + header->length; 822 823 while (start < end && ret == 0) { 824 scope = start; 825 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) 826 ret = ir_parse_one_ioapic_scope(scope, iommu, drhd); 827 else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) 828 ret = ir_parse_one_hpet_scope(scope, iommu, drhd); 829 start += scope->length; 830 } 831 832 return ret; 833} 834 835static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu) 836{ 837 int i; 838 839 for (i = 0; i < MAX_HPET_TBS; i++) 840 if (ir_hpet[i].iommu == iommu) 841 ir_hpet[i].iommu = NULL; 842 843 for (i = 0; i < MAX_IO_APICS; i++) 844 if (ir_ioapic[i].iommu == iommu) 845 ir_ioapic[i].iommu = NULL; 846} 847 848/* 849 * Finds the assocaition between IOAPIC's and its Interrupt-remapping 850 * hardware unit. 851 */ 852static int __init parse_ioapics_under_ir(void) 853{ 854 struct dmar_drhd_unit *drhd; 855 struct intel_iommu *iommu; 856 bool ir_supported = false; 857 int ioapic_idx; 858 859 for_each_iommu(iommu, drhd) 860 if (ecap_ir_support(iommu->ecap)) { 861 if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu)) 862 return -1; 863 864 ir_supported = true; 865 } 866 867 if (!ir_supported) 868 return 0; 869 870 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) { 871 int ioapic_id = mpc_ioapic_id(ioapic_idx); 872 if (!map_ioapic_to_ir(ioapic_id)) { 873 pr_err(FW_BUG "ioapic %d has no mapping iommu, " 874 "interrupt remapping will be disabled\n", 875 ioapic_id); 876 return -1; 877 } 878 } 879 880 return 1; 881} 882 883static int __init ir_dev_scope_init(void) 884{ 885 int ret; 886 887 if (!irq_remapping_enabled) 888 return 0; 889 890 down_write(&dmar_global_lock); 891 ret = dmar_dev_scope_init(); 892 up_write(&dmar_global_lock); 893 894 return ret; 895} 896rootfs_initcall(ir_dev_scope_init); 897 898static void disable_irq_remapping(void) 899{ 900 struct dmar_drhd_unit *drhd; 901 struct intel_iommu *iommu = NULL; 902 903 /* 904 * Disable Interrupt-remapping for all the DRHD's now. 905 */ 906 for_each_iommu(iommu, drhd) { 907 if (!ecap_ir_support(iommu->ecap)) 908 continue; 909 910 iommu_disable_irq_remapping(iommu); 911 } 912} 913 914static int reenable_irq_remapping(int eim) 915{ 916 struct dmar_drhd_unit *drhd; 917 bool setup = false; 918 struct intel_iommu *iommu = NULL; 919 920 for_each_iommu(iommu, drhd) 921 if (iommu->qi) 922 dmar_reenable_qi(iommu); 923 924 /* 925 * Setup Interrupt-remapping for all the DRHD's now. 926 */ 927 for_each_iommu(iommu, drhd) { 928 if (!ecap_ir_support(iommu->ecap)) 929 continue; 930 931 /* Set up interrupt remapping for iommu.*/ 932 iommu_set_irq_remapping(iommu, eim); 933 setup = true; 934 } 935 936 if (!setup) 937 goto error; 938 939 return 0; 940 941error: 942 /* 943 * handle error condition gracefully here! 944 */ 945 return -1; 946} 947 948static void prepare_irte(struct irte *irte, int vector, 949 unsigned int dest) 950{ 951 memset(irte, 0, sizeof(*irte)); 952 953 irte->present = 1; 954 irte->dst_mode = apic->irq_dest_mode; 955 /* 956 * Trigger mode in the IRTE will always be edge, and for IO-APIC, the 957 * actual level or edge trigger will be setup in the IO-APIC 958 * RTE. This will help simplify level triggered irq migration. 959 * For more details, see the comments (in io_apic.c) explainig IO-APIC 960 * irq migration in the presence of interrupt-remapping. 961 */ 962 irte->trigger_mode = 0; 963 irte->dlvry_mode = apic->irq_delivery_mode; 964 irte->vector = vector; 965 irte->dest_id = IRTE_DEST(dest); 966 irte->redir_hint = 1; 967} 968 969static int intel_setup_ioapic_entry(int irq, 970 struct IO_APIC_route_entry *route_entry, 971 unsigned int destination, int vector, 972 struct io_apic_irq_attr *attr) 973{ 974 int ioapic_id = mpc_ioapic_id(attr->ioapic); 975 struct intel_iommu *iommu; 976 struct IR_IO_APIC_route_entry *entry; 977 struct irte irte; 978 int index; 979 980 down_read(&dmar_global_lock); 981 iommu = map_ioapic_to_ir(ioapic_id); 982 if (!iommu) { 983 pr_warn("No mapping iommu for ioapic %d\n", ioapic_id); 984 index = -ENODEV; 985 } else { 986 index = alloc_irte(iommu, irq, 1); 987 if (index < 0) { 988 pr_warn("Failed to allocate IRTE for ioapic %d\n", 989 ioapic_id); 990 index = -ENOMEM; 991 } 992 } 993 up_read(&dmar_global_lock); 994 if (index < 0) 995 return index; 996 997 prepare_irte(&irte, vector, destination); 998 999 /* Set source-id of interrupt request */ 1000 set_ioapic_sid(&irte, ioapic_id); 1001 1002 modify_irte(irq, &irte); 1003 1004 apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: " 1005 "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d " 1006 "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X " 1007 "Avail:%X Vector:%02X Dest:%08X " 1008 "SID:%04X SQ:%X SVT:%X)\n", 1009 attr->ioapic, irte.present, irte.fpd, irte.dst_mode, 1010 irte.redir_hint, irte.trigger_mode, irte.dlvry_mode, 1011 irte.avail, irte.vector, irte.dest_id, 1012 irte.sid, irte.sq, irte.svt); 1013 1014 entry = (struct IR_IO_APIC_route_entry *)route_entry; 1015 memset(entry, 0, sizeof(*entry)); 1016 1017 entry->index2 = (index >> 15) & 0x1; 1018 entry->zero = 0; 1019 entry->format = 1; 1020 entry->index = (index & 0x7fff); 1021 /* 1022 * IO-APIC RTE will be configured with virtual vector. 1023 * irq handler will do the explicit EOI to the io-apic. 1024 */ 1025 entry->vector = attr->ioapic_pin; 1026 entry->mask = 0; /* enable IRQ */ 1027 entry->trigger = attr->trigger; 1028 entry->polarity = attr->polarity; 1029 1030 /* Mask level triggered irqs. 1031 * Use IRQ_DELAYED_DISABLE for edge triggered irqs. 1032 */ 1033 if (attr->trigger) 1034 entry->mask = 1; 1035 1036 return 0; 1037} 1038 1039/* 1040 * Migrate the IO-APIC irq in the presence of intr-remapping. 1041 * 1042 * For both level and edge triggered, irq migration is a simple atomic 1043 * update(of vector and cpu destination) of IRTE and flush the hardware cache. 1044 * 1045 * For level triggered, we eliminate the io-apic RTE modification (with the 1046 * updated vector information), by using a virtual vector (io-apic pin number). 1047 * Real vector that is used for interrupting cpu will be coming from 1048 * the interrupt-remapping table entry. 1049 * 1050 * As the migration is a simple atomic update of IRTE, the same mechanism 1051 * is used to migrate MSI irq's in the presence of interrupt-remapping. 1052 */ 1053static int 1054intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, 1055 bool force) 1056{ 1057 struct irq_cfg *cfg = irqd_cfg(data); 1058 unsigned int dest, irq = data->irq; 1059 struct irte irte; 1060 int err; 1061 1062 if (!config_enabled(CONFIG_SMP)) 1063 return -EINVAL; 1064 1065 if (!cpumask_intersects(mask, cpu_online_mask)) 1066 return -EINVAL; 1067 1068 if (get_irte(irq, &irte)) 1069 return -EBUSY; 1070 1071 err = assign_irq_vector(irq, cfg, mask); 1072 if (err) 1073 return err; 1074 1075 err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest); 1076 if (err) { 1077 if (assign_irq_vector(irq, cfg, data->affinity)) 1078 pr_err("Failed to recover vector for irq %d\n", irq); 1079 return err; 1080 } 1081 1082 irte.vector = cfg->vector; 1083 irte.dest_id = IRTE_DEST(dest); 1084 1085 /* 1086 * Atomically updates the IRTE with the new destination, vector 1087 * and flushes the interrupt entry cache. 1088 */ 1089 modify_irte(irq, &irte); 1090 1091 /* 1092 * After this point, all the interrupts will start arriving 1093 * at the new destination. So, time to cleanup the previous 1094 * vector allocation. 1095 */ 1096 if (cfg->move_in_progress) 1097 send_cleanup_vector(cfg); 1098 1099 cpumask_copy(data->affinity, mask); 1100 return 0; 1101} 1102 1103static void intel_compose_msi_msg(struct pci_dev *pdev, 1104 unsigned int irq, unsigned int dest, 1105 struct msi_msg *msg, u8 hpet_id) 1106{ 1107 struct irq_cfg *cfg; 1108 struct irte irte; 1109 u16 sub_handle = 0; 1110 int ir_index; 1111 1112 cfg = irq_cfg(irq); 1113 1114 ir_index = map_irq_to_irte_handle(irq, &sub_handle); 1115 BUG_ON(ir_index == -1); 1116 1117 prepare_irte(&irte, cfg->vector, dest); 1118 1119 /* Set source-id of interrupt request */ 1120 if (pdev) 1121 set_msi_sid(&irte, pdev); 1122 else 1123 set_hpet_sid(&irte, hpet_id); 1124 1125 modify_irte(irq, &irte); 1126 1127 msg->address_hi = MSI_ADDR_BASE_HI; 1128 msg->data = sub_handle; 1129 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT | 1130 MSI_ADDR_IR_SHV | 1131 MSI_ADDR_IR_INDEX1(ir_index) | 1132 MSI_ADDR_IR_INDEX2(ir_index); 1133} 1134 1135/* 1136 * Map the PCI dev to the corresponding remapping hardware unit 1137 * and allocate 'nvec' consecutive interrupt-remapping table entries 1138 * in it. 1139 */ 1140static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) 1141{ 1142 struct intel_iommu *iommu; 1143 int index; 1144 1145 down_read(&dmar_global_lock); 1146 iommu = map_dev_to_ir(dev); 1147 if (!iommu) { 1148 printk(KERN_ERR 1149 "Unable to map PCI %s to iommu\n", pci_name(dev)); 1150 index = -ENOENT; 1151 } else { 1152 index = alloc_irte(iommu, irq, nvec); 1153 if (index < 0) { 1154 printk(KERN_ERR 1155 "Unable to allocate %d IRTE for PCI %s\n", 1156 nvec, pci_name(dev)); 1157 index = -ENOSPC; 1158 } 1159 } 1160 up_read(&dmar_global_lock); 1161 1162 return index; 1163} 1164 1165static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, 1166 int index, int sub_handle) 1167{ 1168 struct intel_iommu *iommu; 1169 int ret = -ENOENT; 1170 1171 down_read(&dmar_global_lock); 1172 iommu = map_dev_to_ir(pdev); 1173 if (iommu) { 1174 /* 1175 * setup the mapping between the irq and the IRTE 1176 * base index, the sub_handle pointing to the 1177 * appropriate interrupt remap table entry. 1178 */ 1179 set_irte_irq(irq, iommu, index, sub_handle); 1180 ret = 0; 1181 } 1182 up_read(&dmar_global_lock); 1183 1184 return ret; 1185} 1186 1187static int intel_alloc_hpet_msi(unsigned int irq, unsigned int id) 1188{ 1189 int ret = -1; 1190 struct intel_iommu *iommu; 1191 int index; 1192 1193 down_read(&dmar_global_lock); 1194 iommu = map_hpet_to_ir(id); 1195 if (iommu) { 1196 index = alloc_irte(iommu, irq, 1); 1197 if (index >= 0) 1198 ret = 0; 1199 } 1200 up_read(&dmar_global_lock); 1201 1202 return ret; 1203} 1204 1205struct irq_remap_ops intel_irq_remap_ops = { 1206 .prepare = intel_prepare_irq_remapping, 1207 .enable = intel_enable_irq_remapping, 1208 .disable = disable_irq_remapping, 1209 .reenable = reenable_irq_remapping, 1210 .enable_faulting = enable_drhd_fault_handling, 1211 .setup_ioapic_entry = intel_setup_ioapic_entry, 1212 .set_affinity = intel_ioapic_set_affinity, 1213 .free_irq = free_irte, 1214 .compose_msi_msg = intel_compose_msi_msg, 1215 .msi_alloc_irq = intel_msi_alloc_irq, 1216 .msi_setup_irq = intel_msi_setup_irq, 1217 .alloc_hpet_msi = intel_alloc_hpet_msi, 1218}; 1219 1220/* 1221 * Support of Interrupt Remapping Unit Hotplug 1222 */ 1223static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) 1224{ 1225 int ret; 1226 int eim = x2apic_enabled(); 1227 1228 if (eim && !ecap_eim_support(iommu->ecap)) { 1229 pr_info("DRHD %Lx: EIM not supported by DRHD, ecap %Lx\n", 1230 iommu->reg_phys, iommu->ecap); 1231 return -ENODEV; 1232 } 1233 1234 if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) { 1235 pr_warn("DRHD %Lx: failed to parse managed IOAPIC/HPET\n", 1236 iommu->reg_phys); 1237 return -ENODEV; 1238 } 1239 1240 /* TODO: check all IOAPICs are covered by IOMMU */ 1241 1242 /* Setup Interrupt-remapping now. */ 1243 ret = intel_setup_irq_remapping(iommu); 1244 if (ret) { 1245 pr_err("DRHD %Lx: failed to allocate resource\n", 1246 iommu->reg_phys); 1247 ir_remove_ioapic_hpet_scope(iommu); 1248 return ret; 1249 } 1250 1251 if (!iommu->qi) { 1252 /* Clear previous faults. */ 1253 dmar_fault(-1, iommu); 1254 iommu_disable_irq_remapping(iommu); 1255 dmar_disable_qi(iommu); 1256 } 1257 1258 /* Enable queued invalidation */ 1259 ret = dmar_enable_qi(iommu); 1260 if (!ret) { 1261 iommu_set_irq_remapping(iommu, eim); 1262 } else { 1263 pr_err("DRHD %Lx: failed to enable queued invalidation, ecap %Lx, ret %d\n", 1264 iommu->reg_phys, iommu->ecap, ret); 1265 intel_teardown_irq_remapping(iommu); 1266 ir_remove_ioapic_hpet_scope(iommu); 1267 } 1268 1269 return ret; 1270} 1271 1272int dmar_ir_hotplug(struct dmar_drhd_unit *dmaru, bool insert) 1273{ 1274 int ret = 0; 1275 struct intel_iommu *iommu = dmaru->iommu; 1276 1277 if (!irq_remapping_enabled) 1278 return 0; 1279 if (iommu == NULL) 1280 return -EINVAL; 1281 if (!ecap_ir_support(iommu->ecap)) 1282 return 0; 1283 1284 if (insert) { 1285 if (!iommu->ir_table) 1286 ret = dmar_ir_add(dmaru, iommu); 1287 } else { 1288 if (iommu->ir_table) { 1289 if (!bitmap_empty(iommu->ir_table->bitmap, 1290 INTR_REMAP_TABLE_ENTRIES)) { 1291 ret = -EBUSY; 1292 } else { 1293 iommu_disable_irq_remapping(iommu); 1294 intel_teardown_irq_remapping(iommu); 1295 ir_remove_ioapic_hpet_scope(iommu); 1296 } 1297 } 1298 } 1299 1300 return ret; 1301} 1302