106 	struct mm_struct *mm;  member
169 	struct mm_struct *mm;  member
295 	rmap_item->mm = NULL;	/* debug safety */  in free_rmap_item()
321 static struct mm_slot *get_mm_slot(struct mm_struct *mm)  in get_mm_slot()  argument
325 	hash_for_each_possible(mm_slots_hash, slot, link, (unsigned long)mm)  in get_mm_slot()
326 		if (slot->mm == mm)  in get_mm_slot()
332 static void insert_to_mm_slots_hash(struct mm_struct *mm,  in insert_to_mm_slots_hash()  argument
335 	mm_slot->mm = mm;  in insert_to_mm_slots_hash()
336 	hash_add(mm_slots_hash, &mm_slot->link, (unsigned long)mm);  in insert_to_mm_slots_hash()
347 static inline bool ksm_test_exit(struct mm_struct *mm)  in ksm_test_exit()  argument
349 	return atomic_read(&mm->mm_users) == 0;  in ksm_test_exit()
411 static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm,  in find_mergeable_vma()  argument
415 	if (ksm_test_exit(mm))  in find_mergeable_vma()
417 	vma = find_vma(mm, addr);  in find_mergeable_vma()
427 	struct mm_struct *mm = rmap_item->mm;  in break_cow()  local
437 	down_read(&mm->mmap_sem);  in break_cow()
438 	vma = find_mergeable_vma(mm, addr);  in break_cow()
441 	up_read(&mm->mmap_sem);  in break_cow()
460 	struct mm_struct *mm = rmap_item->mm;  in get_mergeable_page()  local
465 	down_read(&mm->mmap_sem);  in get_mergeable_page()
466 	vma = find_mergeable_vma(mm, addr);  in get_mergeable_page()
480 	up_read(&mm->mmap_sem);  in get_mergeable_page()
770 	struct mm_struct *mm;  in unmerge_and_remove_all_rmap_items()  local
781 		mm = mm_slot->mm;  in unmerge_and_remove_all_rmap_items()
782 		down_read(&mm->mmap_sem);  in unmerge_and_remove_all_rmap_items()
783 		for (vma = mm->mmap; vma; vma = vma->vm_next) {  in unmerge_and_remove_all_rmap_items()
784 			if (ksm_test_exit(mm))  in unmerge_and_remove_all_rmap_items()
799 		if (ksm_test_exit(mm)) {  in unmerge_and_remove_all_rmap_items()
805 			clear_bit(MMF_VM_MERGEABLE, &mm->flags);  in unmerge_and_remove_all_rmap_items()
806 			up_read(&mm->mmap_sem);  in unmerge_and_remove_all_rmap_items()
807 			mmdrop(mm);  in unmerge_and_remove_all_rmap_items()
810 			up_read(&mm->mmap_sem);  in unmerge_and_remove_all_rmap_items()
820 	up_read(&mm->mmap_sem);  in unmerge_and_remove_all_rmap_items()
858 	struct mm_struct *mm = vma->vm_mm;  in write_protect_page()  local
875 	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);  in write_protect_page()
877 	ptep = page_check_address(page, mm, addr, &ptl, 0);  in write_protect_page()
901 			set_pte_at(mm, addr, ptep, entry);  in write_protect_page()
907 		set_pte_at_notify(mm, addr, ptep, entry);  in write_protect_page()
915 	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);  in write_protect_page()
932 	struct mm_struct *mm = vma->vm_mm;  in replace_page()  local
945 	pmd = mm_find_pmd(mm, addr);  in replace_page()
951 	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);  in replace_page()
953 	ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);  in replace_page()
964 	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));  in replace_page()
974 	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);  in replace_page()
1085 	struct mm_struct *mm = rmap_item->mm;  in try_to_merge_with_ksm_page()  local
1089 	down_read(&mm->mmap_sem);  in try_to_merge_with_ksm_page()
1090 	if (ksm_test_exit(mm))  in try_to_merge_with_ksm_page()
1092 	vma = find_vma(mm, rmap_item->address);  in try_to_merge_with_ksm_page()
1107 	up_read(&mm->mmap_sem);  in try_to_merge_with_ksm_page()
1527 		rmap_item->mm = mm_slot->mm;  in get_next_rmap_item()
1537 	struct mm_struct *mm;  in scan_get_next_rmap_item()  local
1599 	mm = slot->mm;  in scan_get_next_rmap_item()
1600 	down_read(&mm->mmap_sem);  in scan_get_next_rmap_item()
1601 	if (ksm_test_exit(mm))  in scan_get_next_rmap_item()
1604 		vma = find_vma(mm, ksm_scan.address);  in scan_get_next_rmap_item()
1615 			if (ksm_test_exit(mm))  in scan_get_next_rmap_item()
1635 				up_read(&mm->mmap_sem);  in scan_get_next_rmap_item()
1644 	if (ksm_test_exit(mm)) {  in scan_get_next_rmap_item()
1672 		clear_bit(MMF_VM_MERGEABLE, &mm->flags);  in scan_get_next_rmap_item()
1673 		up_read(&mm->mmap_sem);  in scan_get_next_rmap_item()
1674 		mmdrop(mm);  in scan_get_next_rmap_item()
1677 		up_read(&mm->mmap_sem);  in scan_get_next_rmap_item()
1741 	struct mm_struct *mm = vma->vm_mm;  in ksm_madvise()  local
1759 		if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) {  in ksm_madvise()
1760 			err = __ksm_enter(mm);  in ksm_madvise()
1785 int __ksm_enter(struct mm_struct *mm)  in __ksm_enter()  argument
1798 	insert_to_mm_slots_hash(mm, mm_slot);  in __ksm_enter()
1815 	set_bit(MMF_VM_MERGEABLE, &mm->flags);  in __ksm_enter()
1816 	atomic_inc(&mm->mm_count);  in __ksm_enter()
1824 void __ksm_exit(struct mm_struct *mm)  in __ksm_exit()  argument
1839 	mm_slot = get_mm_slot(mm);  in __ksm_exit()
1854 		clear_bit(MMF_VM_MERGEABLE, &mm->flags);  in __ksm_exit()
1855 		mmdrop(mm);  in __ksm_exit()
1857 		down_write(&mm->mmap_sem);  in __ksm_exit()
1858 		up_write(&mm->mmap_sem);  in __ksm_exit()
1930 			if ((rmap_item->mm == vma->vm_mm) == search_new_forks)  in rmap_walk_ksm()