45 struct pasid_state {  struct
67 	struct pasid_state **states;  argument
80 	struct pasid_state *state;
155 static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,  in __get_pasid_state_ptr()
158 	struct pasid_state **root, **ptr;  in __get_pasid_state_ptr()
181 		root   = (struct pasid_state **)*ptr;  in __get_pasid_state_ptr()
189 			   struct pasid_state *pasid_state,  in set_pasid_state()  argument
192 	struct pasid_state **ptr;  in set_pasid_state()
207 	*ptr = pasid_state;  in set_pasid_state()
219 	struct pasid_state **ptr;  in clear_pasid_state()
234 static struct pasid_state *get_pasid_state(struct device_state *dev_state,  in get_pasid_state()
237 	struct pasid_state **ptr, *ret = NULL;  in get_pasid_state()
256 static void free_pasid_state(struct pasid_state *pasid_state)  in free_pasid_state()  argument
258 	kfree(pasid_state);  in free_pasid_state()
261 static void put_pasid_state(struct pasid_state *pasid_state)  in put_pasid_state()  argument
263 	if (atomic_dec_and_test(&pasid_state->count))  in put_pasid_state()
264 		wake_up(&pasid_state->wq);  in put_pasid_state()
267 static void put_pasid_state_wait(struct pasid_state *pasid_state)  in put_pasid_state_wait()  argument
269 	atomic_dec(&pasid_state->count);  in put_pasid_state_wait()
270 	wait_event(pasid_state->wq, !atomic_read(&pasid_state->count));  in put_pasid_state_wait()
271 	free_pasid_state(pasid_state);  in put_pasid_state_wait()
274 static void unbind_pasid(struct pasid_state *pasid_state)  in unbind_pasid()  argument
278 	domain = pasid_state->device_state->domain;  in unbind_pasid()
284 	pasid_state->invalid = true;  in unbind_pasid()
290 	amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);  in unbind_pasid()
296 static void free_pasid_states_level1(struct pasid_state **tbl)  in free_pasid_states_level1()
308 static void free_pasid_states_level2(struct pasid_state **tbl)  in free_pasid_states_level2()
310 	struct pasid_state **ptr;  in free_pasid_states_level2()
317 		ptr = (struct pasid_state **)tbl[i];  in free_pasid_states_level2()
324 	struct pasid_state *pasid_state;  in free_pasid_states()  local
328 		pasid_state = get_pasid_state(dev_state, i);  in free_pasid_states()
329 		if (pasid_state == NULL)  in free_pasid_states()
332 		put_pasid_state(pasid_state);  in free_pasid_states()
338 		mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in free_pasid_states()
340 		put_pasid_state_wait(pasid_state); /* Reference taken in  in free_pasid_states()
357 static struct pasid_state *mn_to_state(struct mmu_notifier *mn)  in mn_to_state()
359 	return container_of(mn, struct pasid_state, mn);  in mn_to_state()
365 	struct pasid_state *pasid_state;  in __mn_flush_page()  local
368 	pasid_state = mn_to_state(mn);  in __mn_flush_page()
369 	dev_state   = pasid_state->device_state;  in __mn_flush_page()
371 	amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, address);  in __mn_flush_page()
396 	struct pasid_state *pasid_state;  in mn_invalidate_range()  local
399 	pasid_state = mn_to_state(mn);  in mn_invalidate_range()
400 	dev_state   = pasid_state->device_state;  in mn_invalidate_range()
403 		amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,  in mn_invalidate_range()
406 		amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);  in mn_invalidate_range()
411 	struct pasid_state *pasid_state;  in mn_release()  local
417 	pasid_state    = mn_to_state(mn);  in mn_release()
418 	dev_state      = pasid_state->device_state;  in mn_release()
419 	run_inv_ctx_cb = !pasid_state->invalid;  in mn_release()
422 		dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);  in mn_release()
424 	unbind_pasid(pasid_state);  in mn_release()
434 static void set_pri_tag_status(struct pasid_state *pasid_state,  in set_pri_tag_status()  argument
439 	spin_lock_irqsave(&pasid_state->lock, flags);  in set_pri_tag_status()
440 	pasid_state->pri[tag].status = status;  in set_pri_tag_status()
441 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in set_pri_tag_status()
445 			   struct pasid_state *pasid_state,  in finish_pri_tag()  argument
450 	spin_lock_irqsave(&pasid_state->lock, flags);  in finish_pri_tag()
451 	if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&  in finish_pri_tag()
452 	    pasid_state->pri[tag].finish) {  in finish_pri_tag()
453 		amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,  in finish_pri_tag()
454 				       pasid_state->pri[tag].status, tag);  in finish_pri_tag()
455 		pasid_state->pri[tag].finish = false;  in finish_pri_tag()
456 		pasid_state->pri[tag].status = PPR_SUCCESS;  in finish_pri_tag()
458 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in finish_pri_tag()
539 	struct pasid_state *pasid_state;  in ppr_notifier()  local
556 	pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);  in ppr_notifier()
557 	if (pasid_state == NULL || pasid_state->invalid) {  in ppr_notifier()
564 	spin_lock_irqsave(&pasid_state->lock, flags);  in ppr_notifier()
565 	atomic_inc(&pasid_state->pri[tag].inflight);  in ppr_notifier()
567 		pasid_state->pri[tag].finish = true;  in ppr_notifier()
568 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in ppr_notifier()
573 		finish_pri_tag(dev_state, pasid_state, tag);  in ppr_notifier()
579 	fault->state     = pasid_state;  in ppr_notifier()
592 	if (ret != NOTIFY_OK && pasid_state)  in ppr_notifier()
593 		put_pasid_state(pasid_state);  in ppr_notifier()
608 	struct pasid_state *pasid_state;  in amd_iommu_bind_pasid()  local
630 	pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);  in amd_iommu_bind_pasid()
631 	if (pasid_state == NULL)  in amd_iommu_bind_pasid()
635 	atomic_set(&pasid_state->count, 1);  in amd_iommu_bind_pasid()
636 	init_waitqueue_head(&pasid_state->wq);  in amd_iommu_bind_pasid()
637 	spin_lock_init(&pasid_state->lock);  in amd_iommu_bind_pasid()
640 	pasid_state->mm           = mm;  in amd_iommu_bind_pasid()
641 	pasid_state->device_state = dev_state;  in amd_iommu_bind_pasid()
642 	pasid_state->pasid        = pasid;  in amd_iommu_bind_pasid()
643 	pasid_state->invalid      = true; /* Mark as valid only if we are  in amd_iommu_bind_pasid()
645 	pasid_state->mn.ops       = &iommu_mn;  in amd_iommu_bind_pasid()
647 	if (pasid_state->mm == NULL)  in amd_iommu_bind_pasid()
650 	mmu_notifier_register(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
652 	ret = set_pasid_state(dev_state, pasid_state, pasid);  in amd_iommu_bind_pasid()
657 					__pa(pasid_state->mm->pgd));  in amd_iommu_bind_pasid()
662 	pasid_state->invalid = false;  in amd_iommu_bind_pasid()
677 	mmu_notifier_unregister(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
681 	free_pasid_state(pasid_state);  in amd_iommu_bind_pasid()
692 	struct pasid_state *pasid_state;  in amd_iommu_unbind_pasid()  local
709 	pasid_state = get_pasid_state(dev_state, pasid);  in amd_iommu_unbind_pasid()
710 	if (pasid_state == NULL)  in amd_iommu_unbind_pasid()
716 	put_pasid_state(pasid_state);  in amd_iommu_unbind_pasid()
719 	clear_pasid_state(dev_state, pasid_state->pasid);  in amd_iommu_unbind_pasid()
725 	mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in amd_iommu_unbind_pasid()
727 	put_pasid_state_wait(pasid_state); /* Reference taken in  in amd_iommu_unbind_pasid()