1/* 2 * Copyright (C) 2010-2012 Advanced Micro Devices, Inc. 3 * Author: Joerg Roedel <jroedel@suse.de> 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 as published 7 * by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 */ 18 19#include <linux/mmu_notifier.h> 20#include <linux/amd-iommu.h> 21#include <linux/mm_types.h> 22#include <linux/profile.h> 23#include <linux/module.h> 24#include <linux/sched.h> 25#include <linux/iommu.h> 26#include <linux/wait.h> 27#include <linux/pci.h> 28#include <linux/gfp.h> 29 30#include "amd_iommu_types.h" 31#include "amd_iommu_proto.h" 32 33MODULE_LICENSE("GPL v2"); 34MODULE_AUTHOR("Joerg Roedel <jroedel@suse.de>"); 35 36#define MAX_DEVICES 0x10000 37#define PRI_QUEUE_SIZE 512 38 39struct pri_queue { 40 atomic_t inflight; 41 bool finish; 42 int status; 43}; 44 45struct pasid_state { 46 struct list_head list; /* For global state-list */ 47 atomic_t count; /* Reference count */ 48 unsigned mmu_notifier_count; /* Counting nested mmu_notifier 49 calls */ 50 struct mm_struct *mm; /* mm_struct for the faults */ 51 struct mmu_notifier mn; /* mmu_notifier handle */ 52 struct pri_queue pri[PRI_QUEUE_SIZE]; /* PRI tag states */ 53 struct device_state *device_state; /* Link to our device_state */ 54 int pasid; /* PASID index */ 55 bool invalid; /* Used during setup and 56 teardown of the pasid */ 57 spinlock_t lock; /* Protect pri_queues and 58 mmu_notifer_count */ 59 wait_queue_head_t wq; /* To wait for count == 0 */ 60}; 61 62struct device_state { 63 struct list_head list; 64 u16 devid; 65 atomic_t count; 66 struct pci_dev *pdev; 67 struct pasid_state **states; 68 struct iommu_domain *domain; 69 int pasid_levels; 70 int max_pasids; 71 amd_iommu_invalid_ppr_cb inv_ppr_cb; 72 amd_iommu_invalidate_ctx inv_ctx_cb; 73 spinlock_t lock; 74 wait_queue_head_t wq; 75}; 76 77struct fault { 78 struct work_struct work; 79 struct device_state *dev_state; 80 struct pasid_state *state; 81 struct mm_struct *mm; 82 u64 address; 83 u16 devid; 84 u16 pasid; 85 u16 tag; 86 u16 finish; 87 u16 flags; 88}; 89 90static LIST_HEAD(state_list); 91static spinlock_t state_lock; 92 93static struct workqueue_struct *iommu_wq; 94 95static void free_pasid_states(struct device_state *dev_state); 96 97static u16 device_id(struct pci_dev *pdev) 98{ 99 u16 devid; 100 101 devid = pdev->bus->number; 102 devid = (devid << 8) | pdev->devfn; 103 104 return devid; 105} 106 107static struct device_state *__get_device_state(u16 devid) 108{ 109 struct device_state *dev_state; 110 111 list_for_each_entry(dev_state, &state_list, list) { 112 if (dev_state->devid == devid) 113 return dev_state; 114 } 115 116 return NULL; 117} 118 119static struct device_state *get_device_state(u16 devid) 120{ 121 struct device_state *dev_state; 122 unsigned long flags; 123 124 spin_lock_irqsave(&state_lock, flags); 125 dev_state = __get_device_state(devid); 126 if (dev_state != NULL) 127 atomic_inc(&dev_state->count); 128 spin_unlock_irqrestore(&state_lock, flags); 129 130 return dev_state; 131} 132 133static void free_device_state(struct device_state *dev_state) 134{ 135 /* 136 * First detach device from domain - No more PRI requests will arrive 137 * from that device after it is unbound from the IOMMUv2 domain. 138 */ 139 iommu_detach_device(dev_state->domain, &dev_state->pdev->dev); 140 141 /* Everything is down now, free the IOMMUv2 domain */ 142 iommu_domain_free(dev_state->domain); 143 144 /* Finally get rid of the device-state */ 145 kfree(dev_state); 146} 147 148static void put_device_state(struct device_state *dev_state) 149{ 150 if (atomic_dec_and_test(&dev_state->count)) 151 wake_up(&dev_state->wq); 152} 153 154/* Must be called under dev_state->lock */ 155static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state, 156 int pasid, bool alloc) 157{ 158 struct pasid_state **root, **ptr; 159 int level, index; 160 161 level = dev_state->pasid_levels; 162 root = dev_state->states; 163 164 while (true) { 165 166 index = (pasid >> (9 * level)) & 0x1ff; 167 ptr = &root[index]; 168 169 if (level == 0) 170 break; 171 172 if (*ptr == NULL) { 173 if (!alloc) 174 return NULL; 175 176 *ptr = (void *)get_zeroed_page(GFP_ATOMIC); 177 if (*ptr == NULL) 178 return NULL; 179 } 180 181 root = (struct pasid_state **)*ptr; 182 level -= 1; 183 } 184 185 return ptr; 186} 187 188static int set_pasid_state(struct device_state *dev_state, 189 struct pasid_state *pasid_state, 190 int pasid) 191{ 192 struct pasid_state **ptr; 193 unsigned long flags; 194 int ret; 195 196 spin_lock_irqsave(&dev_state->lock, flags); 197 ptr = __get_pasid_state_ptr(dev_state, pasid, true); 198 199 ret = -ENOMEM; 200 if (ptr == NULL) 201 goto out_unlock; 202 203 ret = -ENOMEM; 204 if (*ptr != NULL) 205 goto out_unlock; 206 207 *ptr = pasid_state; 208 209 ret = 0; 210 211out_unlock: 212 spin_unlock_irqrestore(&dev_state->lock, flags); 213 214 return ret; 215} 216 217static void clear_pasid_state(struct device_state *dev_state, int pasid) 218{ 219 struct pasid_state **ptr; 220 unsigned long flags; 221 222 spin_lock_irqsave(&dev_state->lock, flags); 223 ptr = __get_pasid_state_ptr(dev_state, pasid, true); 224 225 if (ptr == NULL) 226 goto out_unlock; 227 228 *ptr = NULL; 229 230out_unlock: 231 spin_unlock_irqrestore(&dev_state->lock, flags); 232} 233 234static struct pasid_state *get_pasid_state(struct device_state *dev_state, 235 int pasid) 236{ 237 struct pasid_state **ptr, *ret = NULL; 238 unsigned long flags; 239 240 spin_lock_irqsave(&dev_state->lock, flags); 241 ptr = __get_pasid_state_ptr(dev_state, pasid, false); 242 243 if (ptr == NULL) 244 goto out_unlock; 245 246 ret = *ptr; 247 if (ret) 248 atomic_inc(&ret->count); 249 250out_unlock: 251 spin_unlock_irqrestore(&dev_state->lock, flags); 252 253 return ret; 254} 255 256static void free_pasid_state(struct pasid_state *pasid_state) 257{ 258 kfree(pasid_state); 259} 260 261static void put_pasid_state(struct pasid_state *pasid_state) 262{ 263 if (atomic_dec_and_test(&pasid_state->count)) 264 wake_up(&pasid_state->wq); 265} 266 267static void put_pasid_state_wait(struct pasid_state *pasid_state) 268{ 269 atomic_dec(&pasid_state->count); 270 wait_event(pasid_state->wq, !atomic_read(&pasid_state->count)); 271 free_pasid_state(pasid_state); 272} 273 274static void unbind_pasid(struct pasid_state *pasid_state) 275{ 276 struct iommu_domain *domain; 277 278 domain = pasid_state->device_state->domain; 279 280 /* 281 * Mark pasid_state as invalid, no more faults will we added to the 282 * work queue after this is visible everywhere. 283 */ 284 pasid_state->invalid = true; 285 286 /* Make sure this is visible */ 287 smp_wmb(); 288 289 /* After this the device/pasid can't access the mm anymore */ 290 amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid); 291 292 /* Make sure no more pending faults are in the queue */ 293 flush_workqueue(iommu_wq); 294} 295 296static void free_pasid_states_level1(struct pasid_state **tbl) 297{ 298 int i; 299 300 for (i = 0; i < 512; ++i) { 301 if (tbl[i] == NULL) 302 continue; 303 304 free_page((unsigned long)tbl[i]); 305 } 306} 307 308static void free_pasid_states_level2(struct pasid_state **tbl) 309{ 310 struct pasid_state **ptr; 311 int i; 312 313 for (i = 0; i < 512; ++i) { 314 if (tbl[i] == NULL) 315 continue; 316 317 ptr = (struct pasid_state **)tbl[i]; 318 free_pasid_states_level1(ptr); 319 } 320} 321 322static void free_pasid_states(struct device_state *dev_state) 323{ 324 struct pasid_state *pasid_state; 325 int i; 326 327 for (i = 0; i < dev_state->max_pasids; ++i) { 328 pasid_state = get_pasid_state(dev_state, i); 329 if (pasid_state == NULL) 330 continue; 331 332 put_pasid_state(pasid_state); 333 334 /* 335 * This will call the mn_release function and 336 * unbind the PASID 337 */ 338 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm); 339 340 put_pasid_state_wait(pasid_state); /* Reference taken in 341 amd_iommu_bind_pasid */ 342 343 /* Drop reference taken in amd_iommu_bind_pasid */ 344 put_device_state(dev_state); 345 } 346 347 if (dev_state->pasid_levels == 2) 348 free_pasid_states_level2(dev_state->states); 349 else if (dev_state->pasid_levels == 1) 350 free_pasid_states_level1(dev_state->states); 351 else if (dev_state->pasid_levels != 0) 352 BUG(); 353 354 free_page((unsigned long)dev_state->states); 355} 356 357static struct pasid_state *mn_to_state(struct mmu_notifier *mn) 358{ 359 return container_of(mn, struct pasid_state, mn); 360} 361 362static void __mn_flush_page(struct mmu_notifier *mn, 363 unsigned long address) 364{ 365 struct pasid_state *pasid_state; 366 struct device_state *dev_state; 367 368 pasid_state = mn_to_state(mn); 369 dev_state = pasid_state->device_state; 370 371 amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, address); 372} 373 374static int mn_clear_flush_young(struct mmu_notifier *mn, 375 struct mm_struct *mm, 376 unsigned long start, 377 unsigned long end) 378{ 379 for (; start < end; start += PAGE_SIZE) 380 __mn_flush_page(mn, start); 381 382 return 0; 383} 384 385static void mn_invalidate_page(struct mmu_notifier *mn, 386 struct mm_struct *mm, 387 unsigned long address) 388{ 389 __mn_flush_page(mn, address); 390} 391 392static void mn_invalidate_range(struct mmu_notifier *mn, 393 struct mm_struct *mm, 394 unsigned long start, unsigned long end) 395{ 396 struct pasid_state *pasid_state; 397 struct device_state *dev_state; 398 399 pasid_state = mn_to_state(mn); 400 dev_state = pasid_state->device_state; 401 402 if ((start ^ (end - 1)) < PAGE_SIZE) 403 amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, 404 start); 405 else 406 amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid); 407} 408 409static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) 410{ 411 struct pasid_state *pasid_state; 412 struct device_state *dev_state; 413 bool run_inv_ctx_cb; 414 415 might_sleep(); 416 417 pasid_state = mn_to_state(mn); 418 dev_state = pasid_state->device_state; 419 run_inv_ctx_cb = !pasid_state->invalid; 420 421 if (run_inv_ctx_cb && dev_state->inv_ctx_cb) 422 dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid); 423 424 unbind_pasid(pasid_state); 425} 426 427static struct mmu_notifier_ops iommu_mn = { 428 .release = mn_release, 429 .clear_flush_young = mn_clear_flush_young, 430 .invalidate_page = mn_invalidate_page, 431 .invalidate_range = mn_invalidate_range, 432}; 433 434static void set_pri_tag_status(struct pasid_state *pasid_state, 435 u16 tag, int status) 436{ 437 unsigned long flags; 438 439 spin_lock_irqsave(&pasid_state->lock, flags); 440 pasid_state->pri[tag].status = status; 441 spin_unlock_irqrestore(&pasid_state->lock, flags); 442} 443 444static void finish_pri_tag(struct device_state *dev_state, 445 struct pasid_state *pasid_state, 446 u16 tag) 447{ 448 unsigned long flags; 449 450 spin_lock_irqsave(&pasid_state->lock, flags); 451 if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) && 452 pasid_state->pri[tag].finish) { 453 amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid, 454 pasid_state->pri[tag].status, tag); 455 pasid_state->pri[tag].finish = false; 456 pasid_state->pri[tag].status = PPR_SUCCESS; 457 } 458 spin_unlock_irqrestore(&pasid_state->lock, flags); 459} 460 461static void handle_fault_error(struct fault *fault) 462{ 463 int status; 464 465 if (!fault->dev_state->inv_ppr_cb) { 466 set_pri_tag_status(fault->state, fault->tag, PPR_INVALID); 467 return; 468 } 469 470 status = fault->dev_state->inv_ppr_cb(fault->dev_state->pdev, 471 fault->pasid, 472 fault->address, 473 fault->flags); 474 switch (status) { 475 case AMD_IOMMU_INV_PRI_RSP_SUCCESS: 476 set_pri_tag_status(fault->state, fault->tag, PPR_SUCCESS); 477 break; 478 case AMD_IOMMU_INV_PRI_RSP_INVALID: 479 set_pri_tag_status(fault->state, fault->tag, PPR_INVALID); 480 break; 481 case AMD_IOMMU_INV_PRI_RSP_FAIL: 482 set_pri_tag_status(fault->state, fault->tag, PPR_FAILURE); 483 break; 484 default: 485 BUG(); 486 } 487} 488 489static void do_fault(struct work_struct *work) 490{ 491 struct fault *fault = container_of(work, struct fault, work); 492 struct mm_struct *mm; 493 struct vm_area_struct *vma; 494 u64 address; 495 int ret, write; 496 497 write = !!(fault->flags & PPR_FAULT_WRITE); 498 499 mm = fault->state->mm; 500 address = fault->address; 501 502 down_read(&mm->mmap_sem); 503 vma = find_extend_vma(mm, address); 504 if (!vma || address < vma->vm_start) { 505 /* failed to get a vma in the right range */ 506 up_read(&mm->mmap_sem); 507 handle_fault_error(fault); 508 goto out; 509 } 510 511 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) { 512 /* handle_mm_fault would BUG_ON() */ 513 up_read(&mm->mmap_sem); 514 handle_fault_error(fault); 515 goto out; 516 } 517 518 ret = handle_mm_fault(mm, vma, address, write); 519 if (ret & VM_FAULT_ERROR) { 520 /* failed to service fault */ 521 up_read(&mm->mmap_sem); 522 handle_fault_error(fault); 523 goto out; 524 } 525 526 up_read(&mm->mmap_sem); 527 528out: 529 finish_pri_tag(fault->dev_state, fault->state, fault->tag); 530 531 put_pasid_state(fault->state); 532 533 kfree(fault); 534} 535 536static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data) 537{ 538 struct amd_iommu_fault *iommu_fault; 539 struct pasid_state *pasid_state; 540 struct device_state *dev_state; 541 unsigned long flags; 542 struct fault *fault; 543 bool finish; 544 u16 tag; 545 int ret; 546 547 iommu_fault = data; 548 tag = iommu_fault->tag & 0x1ff; 549 finish = (iommu_fault->tag >> 9) & 1; 550 551 ret = NOTIFY_DONE; 552 dev_state = get_device_state(iommu_fault->device_id); 553 if (dev_state == NULL) 554 goto out; 555 556 pasid_state = get_pasid_state(dev_state, iommu_fault->pasid); 557 if (pasid_state == NULL || pasid_state->invalid) { 558 /* We know the device but not the PASID -> send INVALID */ 559 amd_iommu_complete_ppr(dev_state->pdev, iommu_fault->pasid, 560 PPR_INVALID, tag); 561 goto out_drop_state; 562 } 563 564 spin_lock_irqsave(&pasid_state->lock, flags); 565 atomic_inc(&pasid_state->pri[tag].inflight); 566 if (finish) 567 pasid_state->pri[tag].finish = true; 568 spin_unlock_irqrestore(&pasid_state->lock, flags); 569 570 fault = kzalloc(sizeof(*fault), GFP_ATOMIC); 571 if (fault == NULL) { 572 /* We are OOM - send success and let the device re-fault */ 573 finish_pri_tag(dev_state, pasid_state, tag); 574 goto out_drop_state; 575 } 576 577 fault->dev_state = dev_state; 578 fault->address = iommu_fault->address; 579 fault->state = pasid_state; 580 fault->tag = tag; 581 fault->finish = finish; 582 fault->pasid = iommu_fault->pasid; 583 fault->flags = iommu_fault->flags; 584 INIT_WORK(&fault->work, do_fault); 585 586 queue_work(iommu_wq, &fault->work); 587 588 ret = NOTIFY_OK; 589 590out_drop_state: 591 592 if (ret != NOTIFY_OK && pasid_state) 593 put_pasid_state(pasid_state); 594 595 put_device_state(dev_state); 596 597out: 598 return ret; 599} 600 601static struct notifier_block ppr_nb = { 602 .notifier_call = ppr_notifier, 603}; 604 605int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid, 606 struct task_struct *task) 607{ 608 struct pasid_state *pasid_state; 609 struct device_state *dev_state; 610 struct mm_struct *mm; 611 u16 devid; 612 int ret; 613 614 might_sleep(); 615 616 if (!amd_iommu_v2_supported()) 617 return -ENODEV; 618 619 devid = device_id(pdev); 620 dev_state = get_device_state(devid); 621 622 if (dev_state == NULL) 623 return -EINVAL; 624 625 ret = -EINVAL; 626 if (pasid < 0 || pasid >= dev_state->max_pasids) 627 goto out; 628 629 ret = -ENOMEM; 630 pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL); 631 if (pasid_state == NULL) 632 goto out; 633 634 635 atomic_set(&pasid_state->count, 1); 636 init_waitqueue_head(&pasid_state->wq); 637 spin_lock_init(&pasid_state->lock); 638 639 mm = get_task_mm(task); 640 pasid_state->mm = mm; 641 pasid_state->device_state = dev_state; 642 pasid_state->pasid = pasid; 643 pasid_state->invalid = true; /* Mark as valid only if we are 644 done with setting up the pasid */ 645 pasid_state->mn.ops = &iommu_mn; 646 647 if (pasid_state->mm == NULL) 648 goto out_free; 649 650 mmu_notifier_register(&pasid_state->mn, mm); 651 652 ret = set_pasid_state(dev_state, pasid_state, pasid); 653 if (ret) 654 goto out_unregister; 655 656 ret = amd_iommu_domain_set_gcr3(dev_state->domain, pasid, 657 __pa(pasid_state->mm->pgd)); 658 if (ret) 659 goto out_clear_state; 660 661 /* Now we are ready to handle faults */ 662 pasid_state->invalid = false; 663 664 /* 665 * Drop the reference to the mm_struct here. We rely on the 666 * mmu_notifier release call-back to inform us when the mm 667 * is going away. 668 */ 669 mmput(mm); 670 671 return 0; 672 673out_clear_state: 674 clear_pasid_state(dev_state, pasid); 675 676out_unregister: 677 mmu_notifier_unregister(&pasid_state->mn, mm); 678 679out_free: 680 mmput(mm); 681 free_pasid_state(pasid_state); 682 683out: 684 put_device_state(dev_state); 685 686 return ret; 687} 688EXPORT_SYMBOL(amd_iommu_bind_pasid); 689 690void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid) 691{ 692 struct pasid_state *pasid_state; 693 struct device_state *dev_state; 694 u16 devid; 695 696 might_sleep(); 697 698 if (!amd_iommu_v2_supported()) 699 return; 700 701 devid = device_id(pdev); 702 dev_state = get_device_state(devid); 703 if (dev_state == NULL) 704 return; 705 706 if (pasid < 0 || pasid >= dev_state->max_pasids) 707 goto out; 708 709 pasid_state = get_pasid_state(dev_state, pasid); 710 if (pasid_state == NULL) 711 goto out; 712 /* 713 * Drop reference taken here. We are safe because we still hold 714 * the reference taken in the amd_iommu_bind_pasid function. 715 */ 716 put_pasid_state(pasid_state); 717 718 /* Clear the pasid state so that the pasid can be re-used */ 719 clear_pasid_state(dev_state, pasid_state->pasid); 720 721 /* 722 * Call mmu_notifier_unregister to drop our reference 723 * to pasid_state->mm 724 */ 725 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm); 726 727 put_pasid_state_wait(pasid_state); /* Reference taken in 728 amd_iommu_bind_pasid */ 729out: 730 /* Drop reference taken in this function */ 731 put_device_state(dev_state); 732 733 /* Drop reference taken in amd_iommu_bind_pasid */ 734 put_device_state(dev_state); 735} 736EXPORT_SYMBOL(amd_iommu_unbind_pasid); 737 738int amd_iommu_init_device(struct pci_dev *pdev, int pasids) 739{ 740 struct device_state *dev_state; 741 unsigned long flags; 742 int ret, tmp; 743 u16 devid; 744 745 might_sleep(); 746 747 if (!amd_iommu_v2_supported()) 748 return -ENODEV; 749 750 if (pasids <= 0 || pasids > (PASID_MASK + 1)) 751 return -EINVAL; 752 753 devid = device_id(pdev); 754 755 dev_state = kzalloc(sizeof(*dev_state), GFP_KERNEL); 756 if (dev_state == NULL) 757 return -ENOMEM; 758 759 spin_lock_init(&dev_state->lock); 760 init_waitqueue_head(&dev_state->wq); 761 dev_state->pdev = pdev; 762 dev_state->devid = devid; 763 764 tmp = pasids; 765 for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9) 766 dev_state->pasid_levels += 1; 767 768 atomic_set(&dev_state->count, 1); 769 dev_state->max_pasids = pasids; 770 771 ret = -ENOMEM; 772 dev_state->states = (void *)get_zeroed_page(GFP_KERNEL); 773 if (dev_state->states == NULL) 774 goto out_free_dev_state; 775 776 dev_state->domain = iommu_domain_alloc(&pci_bus_type); 777 if (dev_state->domain == NULL) 778 goto out_free_states; 779 780 amd_iommu_domain_direct_map(dev_state->domain); 781 782 ret = amd_iommu_domain_enable_v2(dev_state->domain, pasids); 783 if (ret) 784 goto out_free_domain; 785 786 ret = iommu_attach_device(dev_state->domain, &pdev->dev); 787 if (ret != 0) 788 goto out_free_domain; 789 790 spin_lock_irqsave(&state_lock, flags); 791 792 if (__get_device_state(devid) != NULL) { 793 spin_unlock_irqrestore(&state_lock, flags); 794 ret = -EBUSY; 795 goto out_free_domain; 796 } 797 798 list_add_tail(&dev_state->list, &state_list); 799 800 spin_unlock_irqrestore(&state_lock, flags); 801 802 return 0; 803 804out_free_domain: 805 iommu_domain_free(dev_state->domain); 806 807out_free_states: 808 free_page((unsigned long)dev_state->states); 809 810out_free_dev_state: 811 kfree(dev_state); 812 813 return ret; 814} 815EXPORT_SYMBOL(amd_iommu_init_device); 816 817void amd_iommu_free_device(struct pci_dev *pdev) 818{ 819 struct device_state *dev_state; 820 unsigned long flags; 821 u16 devid; 822 823 if (!amd_iommu_v2_supported()) 824 return; 825 826 devid = device_id(pdev); 827 828 spin_lock_irqsave(&state_lock, flags); 829 830 dev_state = __get_device_state(devid); 831 if (dev_state == NULL) { 832 spin_unlock_irqrestore(&state_lock, flags); 833 return; 834 } 835 836 list_del(&dev_state->list); 837 838 spin_unlock_irqrestore(&state_lock, flags); 839 840 /* Get rid of any remaining pasid states */ 841 free_pasid_states(dev_state); 842 843 put_device_state(dev_state); 844 /* 845 * Wait until the last reference is dropped before freeing 846 * the device state. 847 */ 848 wait_event(dev_state->wq, !atomic_read(&dev_state->count)); 849 free_device_state(dev_state); 850} 851EXPORT_SYMBOL(amd_iommu_free_device); 852 853int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev, 854 amd_iommu_invalid_ppr_cb cb) 855{ 856 struct device_state *dev_state; 857 unsigned long flags; 858 u16 devid; 859 int ret; 860 861 if (!amd_iommu_v2_supported()) 862 return -ENODEV; 863 864 devid = device_id(pdev); 865 866 spin_lock_irqsave(&state_lock, flags); 867 868 ret = -EINVAL; 869 dev_state = __get_device_state(devid); 870 if (dev_state == NULL) 871 goto out_unlock; 872 873 dev_state->inv_ppr_cb = cb; 874 875 ret = 0; 876 877out_unlock: 878 spin_unlock_irqrestore(&state_lock, flags); 879 880 return ret; 881} 882EXPORT_SYMBOL(amd_iommu_set_invalid_ppr_cb); 883 884int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev, 885 amd_iommu_invalidate_ctx cb) 886{ 887 struct device_state *dev_state; 888 unsigned long flags; 889 u16 devid; 890 int ret; 891 892 if (!amd_iommu_v2_supported()) 893 return -ENODEV; 894 895 devid = device_id(pdev); 896 897 spin_lock_irqsave(&state_lock, flags); 898 899 ret = -EINVAL; 900 dev_state = __get_device_state(devid); 901 if (dev_state == NULL) 902 goto out_unlock; 903 904 dev_state->inv_ctx_cb = cb; 905 906 ret = 0; 907 908out_unlock: 909 spin_unlock_irqrestore(&state_lock, flags); 910 911 return ret; 912} 913EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb); 914 915static int __init amd_iommu_v2_init(void) 916{ 917 int ret; 918 919 pr_info("AMD IOMMUv2 driver by Joerg Roedel <jroedel@suse.de>\n"); 920 921 if (!amd_iommu_v2_supported()) { 922 pr_info("AMD IOMMUv2 functionality not available on this system\n"); 923 /* 924 * Load anyway to provide the symbols to other modules 925 * which may use AMD IOMMUv2 optionally. 926 */ 927 return 0; 928 } 929 930 spin_lock_init(&state_lock); 931 932 ret = -ENOMEM; 933 iommu_wq = create_workqueue("amd_iommu_v2"); 934 if (iommu_wq == NULL) 935 goto out; 936 937 amd_iommu_register_ppr_notifier(&ppr_nb); 938 939 return 0; 940 941out: 942 return ret; 943} 944 945static void __exit amd_iommu_v2_exit(void) 946{ 947 struct device_state *dev_state; 948 int i; 949 950 if (!amd_iommu_v2_supported()) 951 return; 952 953 amd_iommu_unregister_ppr_notifier(&ppr_nb); 954 955 flush_workqueue(iommu_wq); 956 957 /* 958 * The loop below might call flush_workqueue(), so call 959 * destroy_workqueue() after it 960 */ 961 for (i = 0; i < MAX_DEVICES; ++i) { 962 dev_state = get_device_state(i); 963 964 if (dev_state == NULL) 965 continue; 966 967 WARN_ON_ONCE(1); 968 969 put_device_state(dev_state); 970 amd_iommu_free_device(dev_state->pdev); 971 } 972 973 destroy_workqueue(iommu_wq); 974} 975 976module_init(amd_iommu_v2_init); 977module_exit(amd_iommu_v2_exit); 978