58 init_iova_domain(struct iova_domain *iovad, unsigned long granule,  in init_iova_domain()  argument
68 	spin_lock_init(&iovad->iova_rbtree_lock);  in init_iova_domain()
69 	iovad->rbroot = RB_ROOT;  in init_iova_domain()
70 	iovad->cached32_node = NULL;  in init_iova_domain()
71 	iovad->granule = granule;  in init_iova_domain()
72 	iovad->start_pfn = start_pfn;  in init_iova_domain()
73 	iovad->dma_32bit_pfn = pfn_32bit;  in init_iova_domain()
77 __get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn)  in __get_cached_rbnode()  argument
79 	if ((*limit_pfn != iovad->dma_32bit_pfn) ||  in __get_cached_rbnode()
80 		(iovad->cached32_node == NULL))  in __get_cached_rbnode()
81 		return rb_last(&iovad->rbroot);  in __get_cached_rbnode()
83 		struct rb_node *prev_node = rb_prev(iovad->cached32_node);  in __get_cached_rbnode()
85 			container_of(iovad->cached32_node, struct iova, node);  in __get_cached_rbnode()
92 __cached_rbnode_insert_update(struct iova_domain *iovad,  in __cached_rbnode_insert_update()  argument
95 	if (limit_pfn != iovad->dma_32bit_pfn)  in __cached_rbnode_insert_update()
97 	iovad->cached32_node = &new->node;  in __cached_rbnode_insert_update()
101 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)  in __cached_rbnode_delete_update()  argument
106 	if (!iovad->cached32_node)  in __cached_rbnode_delete_update()
108 	curr = iovad->cached32_node;  in __cached_rbnode_delete_update()
116 		if (node && iova->pfn_lo < iovad->dma_32bit_pfn)  in __cached_rbnode_delete_update()
117 			iovad->cached32_node = node;  in __cached_rbnode_delete_update()
119 			iovad->cached32_node = NULL;  in __cached_rbnode_delete_update()
138 static int __alloc_and_insert_iova_range(struct iova_domain *iovad,  in __alloc_and_insert_iova_range()  argument
148 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);  in __alloc_and_insert_iova_range()
150 	curr = __get_cached_rbnode(iovad, &limit_pfn);  in __alloc_and_insert_iova_range()
175 		if ((iovad->start_pfn + size + pad_size) > limit_pfn) {  in __alloc_and_insert_iova_range()
176 			spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);  in __alloc_and_insert_iova_range()
195 			entry = &iovad->rbroot.rb_node;  in __alloc_and_insert_iova_range()
213 		rb_insert_color(&new->node, &iovad->rbroot);  in __alloc_and_insert_iova_range()
215 	__cached_rbnode_insert_update(iovad, saved_pfn, new);  in __alloc_and_insert_iova_range()
217 	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);  in __alloc_and_insert_iova_range()
256 alloc_iova(struct iova_domain *iovad, unsigned long size,  in alloc_iova()  argument
273 	ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn,  in alloc_iova()
291 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn)  in find_iova()  argument
297 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);  in find_iova()
298 	node = iovad->rbroot.rb_node;  in find_iova()
304 			spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);  in find_iova()
320 	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);  in find_iova()
331 __free_iova(struct iova_domain *iovad, struct iova *iova)  in __free_iova()  argument
335 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);  in __free_iova()
336 	__cached_rbnode_delete_update(iovad, iova);  in __free_iova()
337 	rb_erase(&iova->node, &iovad->rbroot);  in __free_iova()
338 	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);  in __free_iova()
350 free_iova(struct iova_domain *iovad, unsigned long pfn)  in free_iova()  argument
352 	struct iova *iova = find_iova(iovad, pfn);  in free_iova()
354 		__free_iova(iovad, iova);  in free_iova()
363 void put_iova_domain(struct iova_domain *iovad)  in put_iova_domain()  argument
368 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);  in put_iova_domain()
369 	node = rb_first(&iovad->rbroot);  in put_iova_domain()
372 		rb_erase(node, &iovad->rbroot);  in put_iova_domain()
374 		node = rb_first(&iovad->rbroot);  in put_iova_domain()
376 	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);  in put_iova_domain()
405 __insert_new_range(struct iova_domain *iovad,  in __insert_new_range()  argument
412 		iova_insert_rbtree(&iovad->rbroot, iova);  in __insert_new_range()
436 reserve_iova(struct iova_domain *iovad,  in reserve_iova()  argument
444 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);  in reserve_iova()
445 	for (node = rb_first(&iovad->rbroot); node; node = rb_next(node)) {  in reserve_iova()
461 	iova = __insert_new_range(iovad, pfn_lo, pfn_hi);  in reserve_iova()
464 	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);  in reserve_iova()
494 split_and_remove_iova(struct iova_domain *iovad, struct iova *iova,  in split_and_remove_iova()  argument
500 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);  in split_and_remove_iova()
512 	__cached_rbnode_delete_update(iovad, iova);  in split_and_remove_iova()
513 	rb_erase(&iova->node, &iovad->rbroot);  in split_and_remove_iova()
516 		iova_insert_rbtree(&iovad->rbroot, prev);  in split_and_remove_iova()
520 		iova_insert_rbtree(&iovad->rbroot, next);  in split_and_remove_iova()
523 	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);  in split_and_remove_iova()
528 	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);  in split_and_remove_iova()