106 	struct mm_struct *mm;  member
169 	struct mm_struct *mm;  member
295 	rmap_item->mm = NULL;	/* debug safety */  in free_rmap_item()
321 static struct mm_slot *get_mm_slot(struct mm_struct *mm)  in get_mm_slot()  argument
325 	hash_for_each_possible(mm_slots_hash, slot, link, (unsigned long)mm)  in get_mm_slot()
326 		if (slot->mm == mm)  in get_mm_slot()
332 static void insert_to_mm_slots_hash(struct mm_struct *mm,  in insert_to_mm_slots_hash()  argument
335 	mm_slot->mm = mm;  in insert_to_mm_slots_hash()
336 	hash_add(mm_slots_hash, &mm_slot->link, (unsigned long)mm);  in insert_to_mm_slots_hash()
347 static inline bool ksm_test_exit(struct mm_struct *mm)  in ksm_test_exit()  argument
349 	return atomic_read(&mm->mm_users) == 0;  in ksm_test_exit()
411 static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm,  in find_mergeable_vma()  argument
415 	if (ksm_test_exit(mm))  in find_mergeable_vma()
417 	vma = find_vma(mm, addr);  in find_mergeable_vma()
427 	struct mm_struct *mm = rmap_item->mm;  in break_cow()  local
437 	down_read(&mm->mmap_sem);  in break_cow()
438 	vma = find_mergeable_vma(mm, addr);  in break_cow()
441 	up_read(&mm->mmap_sem);  in break_cow()
460 	struct mm_struct *mm = rmap_item->mm;  in get_mergeable_page()  local
465 	down_read(&mm->mmap_sem);  in get_mergeable_page()
466 	vma = find_mergeable_vma(mm, addr);  in get_mergeable_page()
481 	up_read(&mm->mmap_sem);  in get_mergeable_page()
771 	struct mm_struct *mm;  in unmerge_and_remove_all_rmap_items()  local
782 		mm = mm_slot->mm;  in unmerge_and_remove_all_rmap_items()
783 		down_read(&mm->mmap_sem);  in unmerge_and_remove_all_rmap_items()
784 		for (vma = mm->mmap; vma; vma = vma->vm_next) {  in unmerge_and_remove_all_rmap_items()
785 			if (ksm_test_exit(mm))  in unmerge_and_remove_all_rmap_items()
800 		if (ksm_test_exit(mm)) {  in unmerge_and_remove_all_rmap_items()
806 			clear_bit(MMF_VM_MERGEABLE, &mm->flags);  in unmerge_and_remove_all_rmap_items()
807 			up_read(&mm->mmap_sem);  in unmerge_and_remove_all_rmap_items()
808 			mmdrop(mm);  in unmerge_and_remove_all_rmap_items()
811 			up_read(&mm->mmap_sem);  in unmerge_and_remove_all_rmap_items()
821 	up_read(&mm->mmap_sem);  in unmerge_and_remove_all_rmap_items()
859 	struct mm_struct *mm = vma->vm_mm;  in write_protect_page()  local
876 	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);  in write_protect_page()
878 	ptep = page_check_address(page, mm, addr, &ptl, 0);  in write_protect_page()
902 			set_pte_at(mm, addr, ptep, entry);  in write_protect_page()
908 		set_pte_at_notify(mm, addr, ptep, entry);  in write_protect_page()
916 	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);  in write_protect_page()
933 	struct mm_struct *mm = vma->vm_mm;  in replace_page()  local
946 	pmd = mm_find_pmd(mm, addr);  in replace_page()
952 	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);  in replace_page()
954 	ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);  in replace_page()
965 	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));  in replace_page()
975 	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);  in replace_page()
1084 	struct mm_struct *mm = rmap_item->mm;  in try_to_merge_with_ksm_page()  local
1088 	down_read(&mm->mmap_sem);  in try_to_merge_with_ksm_page()
1089 	vma = find_mergeable_vma(mm, rmap_item->address);  in try_to_merge_with_ksm_page()
1104 	up_read(&mm->mmap_sem);  in try_to_merge_with_ksm_page()
1546 		rmap_item->mm = mm_slot->mm;  in get_next_rmap_item()
1556 	struct mm_struct *mm;  in scan_get_next_rmap_item()  local
1618 	mm = slot->mm;  in scan_get_next_rmap_item()
1619 	down_read(&mm->mmap_sem);  in scan_get_next_rmap_item()
1620 	if (ksm_test_exit(mm))  in scan_get_next_rmap_item()
1623 		vma = find_vma(mm, ksm_scan.address);  in scan_get_next_rmap_item()
1634 			if (ksm_test_exit(mm))  in scan_get_next_rmap_item()
1654 				up_read(&mm->mmap_sem);  in scan_get_next_rmap_item()
1663 	if (ksm_test_exit(mm)) {  in scan_get_next_rmap_item()
1691 		clear_bit(MMF_VM_MERGEABLE, &mm->flags);  in scan_get_next_rmap_item()
1692 		up_read(&mm->mmap_sem);  in scan_get_next_rmap_item()
1693 		mmdrop(mm);  in scan_get_next_rmap_item()
1696 		up_read(&mm->mmap_sem);  in scan_get_next_rmap_item()
1760 	struct mm_struct *mm = vma->vm_mm;  in ksm_madvise()  local
1778 		if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) {  in ksm_madvise()
1779 			err = __ksm_enter(mm);  in ksm_madvise()
1804 int __ksm_enter(struct mm_struct *mm)  in __ksm_enter()  argument
1817 	insert_to_mm_slots_hash(mm, mm_slot);  in __ksm_enter()
1834 	set_bit(MMF_VM_MERGEABLE, &mm->flags);  in __ksm_enter()
1835 	atomic_inc(&mm->mm_count);  in __ksm_enter()
1843 void __ksm_exit(struct mm_struct *mm)  in __ksm_exit()  argument
1858 	mm_slot = get_mm_slot(mm);  in __ksm_exit()
1873 		clear_bit(MMF_VM_MERGEABLE, &mm->flags);  in __ksm_exit()
1874 		mmdrop(mm);  in __ksm_exit()
1876 		down_write(&mm->mmap_sem);  in __ksm_exit()
1877 		up_write(&mm->mmap_sem);  in __ksm_exit()
1951 			if ((rmap_item->mm == vma->vm_mm) == search_new_forks)  in rmap_walk_ksm()