73 static inline struct anon_vma *anon_vma_alloc(void)  in anon_vma_alloc()
75 	struct anon_vma *anon_vma;  in anon_vma_alloc()  local
77 	anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);  in anon_vma_alloc()
78 	if (anon_vma) {  in anon_vma_alloc()
79 		atomic_set(&anon_vma->refcount, 1);  in anon_vma_alloc()
80 		anon_vma->degree = 1;	/* Reference for first vma */  in anon_vma_alloc()
81 		anon_vma->parent = anon_vma;  in anon_vma_alloc()
86 		anon_vma->root = anon_vma;  in anon_vma_alloc()
89 	return anon_vma;  in anon_vma_alloc()
92 static inline void anon_vma_free(struct anon_vma *anon_vma)  in anon_vma_free()  argument
94 	VM_BUG_ON(atomic_read(&anon_vma->refcount));  in anon_vma_free()
114 	if (rwsem_is_locked(&anon_vma->root->rwsem)) {  in anon_vma_free()
115 		anon_vma_lock_write(anon_vma);  in anon_vma_free()
116 		anon_vma_unlock_write(anon_vma);  in anon_vma_free()
119 	kmem_cache_free(anon_vma_cachep, anon_vma);  in anon_vma_free()
134 				struct anon_vma *anon_vma)  in anon_vma_chain_link()  argument
137 	avc->anon_vma = anon_vma;  in anon_vma_chain_link()
139 	anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);  in anon_vma_chain_link()
171 	struct anon_vma *anon_vma = vma->anon_vma;  in anon_vma_prepare()  local
175 	if (unlikely(!anon_vma)) {  in anon_vma_prepare()
177 		struct anon_vma *allocated;  in anon_vma_prepare()
183 		anon_vma = find_mergeable_anon_vma(vma);  in anon_vma_prepare()
185 		if (!anon_vma) {  in anon_vma_prepare()
186 			anon_vma = anon_vma_alloc();  in anon_vma_prepare()
187 			if (unlikely(!anon_vma))  in anon_vma_prepare()
189 			allocated = anon_vma;  in anon_vma_prepare()
192 		anon_vma_lock_write(anon_vma);  in anon_vma_prepare()
195 		if (likely(!vma->anon_vma)) {  in anon_vma_prepare()
196 			vma->anon_vma = anon_vma;  in anon_vma_prepare()
197 			anon_vma_chain_link(vma, avc, anon_vma);  in anon_vma_prepare()
199 			anon_vma->degree++;  in anon_vma_prepare()
204 		anon_vma_unlock_write(anon_vma);  in anon_vma_prepare()
227 static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)  in lock_anon_vma_root()  argument
229 	struct anon_vma *new_root = anon_vma->root;  in lock_anon_vma_root()
239 static inline void unlock_anon_vma_root(struct anon_vma *root)  in unlock_anon_vma_root()
260 	struct anon_vma *root = NULL;  in anon_vma_clone()
263 		struct anon_vma *anon_vma;  in anon_vma_clone()  local
273 		anon_vma = pavc->anon_vma;  in anon_vma_clone()
274 		root = lock_anon_vma_root(root, anon_vma);  in anon_vma_clone()
275 		anon_vma_chain_link(dst, avc, anon_vma);  in anon_vma_clone()
285 		if (!dst->anon_vma && anon_vma != src->anon_vma &&  in anon_vma_clone()
286 				anon_vma->degree < 2)  in anon_vma_clone()
287 			dst->anon_vma = anon_vma;  in anon_vma_clone()
289 	if (dst->anon_vma)  in anon_vma_clone()
290 		dst->anon_vma->degree++;  in anon_vma_clone()
301 	dst->anon_vma = NULL;  in anon_vma_clone()
314 	struct anon_vma *anon_vma;  in anon_vma_fork()  local
318 	if (!pvma->anon_vma)  in anon_vma_fork()
322 	vma->anon_vma = NULL;  in anon_vma_fork()
333 	if (vma->anon_vma)  in anon_vma_fork()
337 	anon_vma = anon_vma_alloc();  in anon_vma_fork()
338 	if (!anon_vma)  in anon_vma_fork()
348 	anon_vma->root = pvma->anon_vma->root;  in anon_vma_fork()
349 	anon_vma->parent = pvma->anon_vma;  in anon_vma_fork()
355 	get_anon_vma(anon_vma->root);  in anon_vma_fork()
357 	vma->anon_vma = anon_vma;  in anon_vma_fork()
358 	anon_vma_lock_write(anon_vma);  in anon_vma_fork()
359 	anon_vma_chain_link(vma, avc, anon_vma);  in anon_vma_fork()
360 	anon_vma->parent->degree++;  in anon_vma_fork()
361 	anon_vma_unlock_write(anon_vma);  in anon_vma_fork()
366 	put_anon_vma(anon_vma);  in anon_vma_fork()
375 	struct anon_vma *root = NULL;  in unlink_anon_vmas()
382 		struct anon_vma *anon_vma = avc->anon_vma;  in unlink_anon_vmas()  local
384 		root = lock_anon_vma_root(root, anon_vma);  in unlink_anon_vmas()
385 		anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);  in unlink_anon_vmas()
391 		if (RB_EMPTY_ROOT(&anon_vma->rb_root)) {  in unlink_anon_vmas()
392 			anon_vma->parent->degree--;  in unlink_anon_vmas()
399 	if (vma->anon_vma)  in unlink_anon_vmas()
400 		vma->anon_vma->degree--;  in unlink_anon_vmas()
409 		struct anon_vma *anon_vma = avc->anon_vma;  in unlink_anon_vmas()  local
411 		BUG_ON(anon_vma->degree);  in unlink_anon_vmas()
412 		put_anon_vma(anon_vma);  in unlink_anon_vmas()
421 	struct anon_vma *anon_vma = data;  in anon_vma_ctor()  local
423 	init_rwsem(&anon_vma->rwsem);  in anon_vma_ctor()
424 	atomic_set(&anon_vma->refcount, 0);  in anon_vma_ctor()
425 	anon_vma->rb_root = RB_ROOT;  in anon_vma_ctor()
430 	anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),  in anon_vma_init()
458 struct anon_vma *page_get_anon_vma(struct page *page)  in page_get_anon_vma()
460 	struct anon_vma *anon_vma = NULL;  in page_get_anon_vma()  local
470 	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);  in page_get_anon_vma()
471 	if (!atomic_inc_not_zero(&anon_vma->refcount)) {  in page_get_anon_vma()
472 		anon_vma = NULL;  in page_get_anon_vma()
485 		put_anon_vma(anon_vma);  in page_get_anon_vma()
491 	return anon_vma;  in page_get_anon_vma()
501 struct anon_vma *page_lock_anon_vma_read(struct page *page)  in page_lock_anon_vma_read()
503 	struct anon_vma *anon_vma = NULL;  in page_lock_anon_vma_read()  local
504 	struct anon_vma *root_anon_vma;  in page_lock_anon_vma_read()
514 	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);  in page_lock_anon_vma_read()
515 	root_anon_vma = READ_ONCE(anon_vma->root);  in page_lock_anon_vma_read()
524 			anon_vma = NULL;  in page_lock_anon_vma_read()
530 	if (!atomic_inc_not_zero(&anon_vma->refcount)) {  in page_lock_anon_vma_read()
531 		anon_vma = NULL;  in page_lock_anon_vma_read()
537 		put_anon_vma(anon_vma);  in page_lock_anon_vma_read()
543 	anon_vma_lock_read(anon_vma);  in page_lock_anon_vma_read()
545 	if (atomic_dec_and_test(&anon_vma->refcount)) {  in page_lock_anon_vma_read()
551 		anon_vma_unlock_read(anon_vma);  in page_lock_anon_vma_read()
552 		__put_anon_vma(anon_vma);  in page_lock_anon_vma_read()
553 		anon_vma = NULL;  in page_lock_anon_vma_read()
556 	return anon_vma;  in page_lock_anon_vma_read()
560 	return anon_vma;  in page_lock_anon_vma_read()
563 void page_unlock_anon_vma_read(struct anon_vma *anon_vma)  in page_unlock_anon_vma_read()  argument
565 	anon_vma_unlock_read(anon_vma);  in page_unlock_anon_vma_read()
698 		struct anon_vma *page__anon_vma = page_anon_vma(page);  in page_address_in_vma()
703 		if (!vma->anon_vma || !page__anon_vma ||  in page_address_in_vma()
704 		    vma->anon_vma->root != page__anon_vma->root)  in page_address_in_vma()
1057 	struct anon_vma *anon_vma = vma->anon_vma;  in page_move_anon_rmap()  local
1060 	VM_BUG_ON_VMA(!anon_vma, vma);  in page_move_anon_rmap()
1063 	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;  in page_move_anon_rmap()
1069 	WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);  in page_move_anon_rmap()
1082 	struct anon_vma *anon_vma = vma->anon_vma;  in __page_set_anon_rmap()  local
1084 	BUG_ON(!anon_vma);  in __page_set_anon_rmap()
1095 		anon_vma = anon_vma->root;  in __page_set_anon_rmap()
1097 	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;  in __page_set_anon_rmap()
1098 	page->mapping = (struct address_space *) anon_vma;  in __page_set_anon_rmap()
1124 	BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);  in __page_check_anon_rmap()
1534 void __put_anon_vma(struct anon_vma *anon_vma)  in __put_anon_vma()  argument
1536 	struct anon_vma *root = anon_vma->root;  in __put_anon_vma()
1538 	anon_vma_free(anon_vma);  in __put_anon_vma()
1539 	if (root != anon_vma && atomic_dec_and_test(&root->refcount))  in __put_anon_vma()
1543 static struct anon_vma *rmap_walk_anon_lock(struct page *page,  in rmap_walk_anon_lock()
1546 	struct anon_vma *anon_vma;  in rmap_walk_anon_lock()  local
1557 	anon_vma = page_anon_vma(page);  in rmap_walk_anon_lock()
1558 	if (!anon_vma)  in rmap_walk_anon_lock()
1561 	anon_vma_lock_read(anon_vma);  in rmap_walk_anon_lock()
1562 	return anon_vma;  in rmap_walk_anon_lock()
1581 	struct anon_vma *anon_vma;  in rmap_walk_anon()  local
1586 	anon_vma = rmap_walk_anon_lock(page, rwc);  in rmap_walk_anon()
1587 	if (!anon_vma)  in rmap_walk_anon()
1591 	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {  in rmap_walk_anon()
1606 	anon_vma_unlock_read(anon_vma);  in rmap_walk_anon()
1682 	struct anon_vma *anon_vma = vma->anon_vma;  in __hugepage_set_anon_rmap()  local
1684 	BUG_ON(!anon_vma);  in __hugepage_set_anon_rmap()
1689 		anon_vma = anon_vma->root;  in __hugepage_set_anon_rmap()
1691 	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;  in __hugepage_set_anon_rmap()
1692 	page->mapping = (struct address_space *) anon_vma;  in __hugepage_set_anon_rmap()
1699 	struct anon_vma *anon_vma = vma->anon_vma;  in hugepage_add_anon_rmap()  local
1703 	BUG_ON(!anon_vma);  in hugepage_add_anon_rmap()