45 struct pasid_state {  struct
67 	struct pasid_state **states;  argument
80 	struct pasid_state *state;
163 static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,  in __get_pasid_state_ptr()
166 	struct pasid_state **root, **ptr;  in __get_pasid_state_ptr()
189 		root   = (struct pasid_state **)*ptr;  in __get_pasid_state_ptr()
197 			   struct pasid_state *pasid_state,  in set_pasid_state()  argument
200 	struct pasid_state **ptr;  in set_pasid_state()
215 	*ptr = pasid_state;  in set_pasid_state()
227 	struct pasid_state **ptr;  in clear_pasid_state()
242 static struct pasid_state *get_pasid_state(struct device_state *dev_state,  in get_pasid_state()
245 	struct pasid_state **ptr, *ret = NULL;  in get_pasid_state()
264 static void free_pasid_state(struct pasid_state *pasid_state)  in free_pasid_state()  argument
266 	kfree(pasid_state);  in free_pasid_state()
269 static void put_pasid_state(struct pasid_state *pasid_state)  in put_pasid_state()  argument
271 	if (atomic_dec_and_test(&pasid_state->count))  in put_pasid_state()
272 		wake_up(&pasid_state->wq);  in put_pasid_state()
275 static void put_pasid_state_wait(struct pasid_state *pasid_state)  in put_pasid_state_wait()  argument
277 	atomic_dec(&pasid_state->count);  in put_pasid_state_wait()
278 	wait_event(pasid_state->wq, !atomic_read(&pasid_state->count));  in put_pasid_state_wait()
279 	free_pasid_state(pasid_state);  in put_pasid_state_wait()
282 static void unbind_pasid(struct pasid_state *pasid_state)  in unbind_pasid()  argument
286 	domain = pasid_state->device_state->domain;  in unbind_pasid()
292 	pasid_state->invalid = true;  in unbind_pasid()
298 	amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);  in unbind_pasid()
304 static void free_pasid_states_level1(struct pasid_state **tbl)  in free_pasid_states_level1()
316 static void free_pasid_states_level2(struct pasid_state **tbl)  in free_pasid_states_level2()
318 	struct pasid_state **ptr;  in free_pasid_states_level2()
325 		ptr = (struct pasid_state **)tbl[i];  in free_pasid_states_level2()
332 	struct pasid_state *pasid_state;  in free_pasid_states()  local
336 		pasid_state = get_pasid_state(dev_state, i);  in free_pasid_states()
337 		if (pasid_state == NULL)  in free_pasid_states()
340 		put_pasid_state(pasid_state);  in free_pasid_states()
346 		mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in free_pasid_states()
348 		put_pasid_state_wait(pasid_state); /* Reference taken in  in free_pasid_states()
365 static struct pasid_state *mn_to_state(struct mmu_notifier *mn)  in mn_to_state()
367 	return container_of(mn, struct pasid_state, mn);  in mn_to_state()
373 	struct pasid_state *pasid_state;  in __mn_flush_page()  local
376 	pasid_state = mn_to_state(mn);  in __mn_flush_page()
377 	dev_state   = pasid_state->device_state;  in __mn_flush_page()
379 	amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, address);  in __mn_flush_page()
404 	struct pasid_state *pasid_state;  in mn_invalidate_range()  local
407 	pasid_state = mn_to_state(mn);  in mn_invalidate_range()
408 	dev_state   = pasid_state->device_state;  in mn_invalidate_range()
411 		amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,  in mn_invalidate_range()
414 		amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);  in mn_invalidate_range()
419 	struct pasid_state *pasid_state;  in mn_release()  local
425 	pasid_state    = mn_to_state(mn);  in mn_release()
426 	dev_state      = pasid_state->device_state;  in mn_release()
427 	run_inv_ctx_cb = !pasid_state->invalid;  in mn_release()
430 		dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);  in mn_release()
432 	unbind_pasid(pasid_state);  in mn_release()
442 static void set_pri_tag_status(struct pasid_state *pasid_state,  in set_pri_tag_status()  argument
447 	spin_lock_irqsave(&pasid_state->lock, flags);  in set_pri_tag_status()
448 	pasid_state->pri[tag].status = status;  in set_pri_tag_status()
449 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in set_pri_tag_status()
453 			   struct pasid_state *pasid_state,  in finish_pri_tag()  argument
458 	spin_lock_irqsave(&pasid_state->lock, flags);  in finish_pri_tag()
459 	if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&  in finish_pri_tag()
460 	    pasid_state->pri[tag].finish) {  in finish_pri_tag()
461 		amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,  in finish_pri_tag()
462 				       pasid_state->pri[tag].status, tag);  in finish_pri_tag()
463 		pasid_state->pri[tag].finish = false;  in finish_pri_tag()
464 		pasid_state->pri[tag].status = PPR_SUCCESS;  in finish_pri_tag()
466 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in finish_pri_tag()
563 	struct pasid_state *pasid_state;  in ppr_notifier()  local
580 	pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);  in ppr_notifier()
581 	if (pasid_state == NULL || pasid_state->invalid) {  in ppr_notifier()
588 	spin_lock_irqsave(&pasid_state->lock, flags);  in ppr_notifier()
589 	atomic_inc(&pasid_state->pri[tag].inflight);  in ppr_notifier()
591 		pasid_state->pri[tag].finish = true;  in ppr_notifier()
592 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in ppr_notifier()
597 		finish_pri_tag(dev_state, pasid_state, tag);  in ppr_notifier()
603 	fault->state     = pasid_state;  in ppr_notifier()
616 	if (ret != NOTIFY_OK && pasid_state)  in ppr_notifier()
617 		put_pasid_state(pasid_state);  in ppr_notifier()
632 	struct pasid_state *pasid_state;  in amd_iommu_bind_pasid()  local
654 	pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);  in amd_iommu_bind_pasid()
655 	if (pasid_state == NULL)  in amd_iommu_bind_pasid()
659 	atomic_set(&pasid_state->count, 1);  in amd_iommu_bind_pasid()
660 	init_waitqueue_head(&pasid_state->wq);  in amd_iommu_bind_pasid()
661 	spin_lock_init(&pasid_state->lock);  in amd_iommu_bind_pasid()
664 	pasid_state->mm           = mm;  in amd_iommu_bind_pasid()
665 	pasid_state->device_state = dev_state;  in amd_iommu_bind_pasid()
666 	pasid_state->pasid        = pasid;  in amd_iommu_bind_pasid()
667 	pasid_state->invalid      = true; /* Mark as valid only if we are  in amd_iommu_bind_pasid()
669 	pasid_state->mn.ops       = &iommu_mn;  in amd_iommu_bind_pasid()
671 	if (pasid_state->mm == NULL)  in amd_iommu_bind_pasid()
674 	mmu_notifier_register(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
676 	ret = set_pasid_state(dev_state, pasid_state, pasid);  in amd_iommu_bind_pasid()
681 					__pa(pasid_state->mm->pgd));  in amd_iommu_bind_pasid()
686 	pasid_state->invalid = false;  in amd_iommu_bind_pasid()
701 	mmu_notifier_unregister(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
705 	free_pasid_state(pasid_state);  in amd_iommu_bind_pasid()
716 	struct pasid_state *pasid_state;  in amd_iommu_unbind_pasid()  local
733 	pasid_state = get_pasid_state(dev_state, pasid);  in amd_iommu_unbind_pasid()
734 	if (pasid_state == NULL)  in amd_iommu_unbind_pasid()
740 	put_pasid_state(pasid_state);  in amd_iommu_unbind_pasid()
743 	clear_pasid_state(dev_state, pasid_state->pasid);  in amd_iommu_unbind_pasid()
749 	mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in amd_iommu_unbind_pasid()
751 	put_pasid_state_wait(pasid_state); /* Reference taken in  in amd_iommu_unbind_pasid()