105 	struct rmap_item *rmap_list;
121 	struct rmap_item **rmap_list;
161 struct rmap_item {  struct
162 	struct rmap_item *rmap_list;  argument
252 	rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);  in ksm_slab_init()
282 static inline struct rmap_item *alloc_rmap_item(void)  in alloc_rmap_item()
284 	struct rmap_item *rmap_item;  in alloc_rmap_item()  local
286 	rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL);  in alloc_rmap_item()
287 	if (rmap_item)  in alloc_rmap_item()
289 	return rmap_item;  in alloc_rmap_item()
292 static inline void free_rmap_item(struct rmap_item *rmap_item)  in free_rmap_item()  argument
295 	rmap_item->mm = NULL;	/* debug safety */  in free_rmap_item()
296 	kmem_cache_free(rmap_item_cache, rmap_item);  in free_rmap_item()
425 static void break_cow(struct rmap_item *rmap_item)  in break_cow()  argument
427 	struct mm_struct *mm = rmap_item->mm;  in break_cow()
428 	unsigned long addr = rmap_item->address;  in break_cow()
435 	put_anon_vma(rmap_item->anon_vma);  in break_cow()
458 static struct page *get_mergeable_page(struct rmap_item *rmap_item)  in get_mergeable_page()  argument
460 	struct mm_struct *mm = rmap_item->mm;  in get_mergeable_page()
461 	unsigned long addr = rmap_item->address;  in get_mergeable_page()
498 	struct rmap_item *rmap_item;  in remove_node_from_stable_tree()  local
500 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in remove_node_from_stable_tree()
501 		if (rmap_item->hlist.next)  in remove_node_from_stable_tree()
505 		put_anon_vma(rmap_item->anon_vma);  in remove_node_from_stable_tree()
506 		rmap_item->address &= PAGE_MASK;  in remove_node_from_stable_tree()
614 static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)  in remove_rmap_item_from_tree()  argument
616 	if (rmap_item->address & STABLE_FLAG) {  in remove_rmap_item_from_tree()
620 		stable_node = rmap_item->head;  in remove_rmap_item_from_tree()
625 		hlist_del(&rmap_item->hlist);  in remove_rmap_item_from_tree()
634 		put_anon_vma(rmap_item->anon_vma);  in remove_rmap_item_from_tree()
635 		rmap_item->address &= PAGE_MASK;  in remove_rmap_item_from_tree()
637 	} else if (rmap_item->address & UNSTABLE_FLAG) {  in remove_rmap_item_from_tree()
646 		age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);  in remove_rmap_item_from_tree()
649 			rb_erase(&rmap_item->node,  in remove_rmap_item_from_tree()
650 				 root_unstable_tree + NUMA(rmap_item->nid));  in remove_rmap_item_from_tree()
652 		rmap_item->address &= PAGE_MASK;  in remove_rmap_item_from_tree()
659 				       struct rmap_item **rmap_list)  in remove_trailing_rmap_items()
662 		struct rmap_item *rmap_item = *rmap_list;  in remove_trailing_rmap_items()  local
663 		*rmap_list = rmap_item->rmap_list;  in remove_trailing_rmap_items()
664 		remove_rmap_item_from_tree(rmap_item);  in remove_trailing_rmap_items()
665 		free_rmap_item(rmap_item);  in remove_trailing_rmap_items()
1081 static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,  in try_to_merge_with_ksm_page()  argument
1084 	struct mm_struct *mm = rmap_item->mm;  in try_to_merge_with_ksm_page()
1089 	vma = find_mergeable_vma(mm, rmap_item->address);  in try_to_merge_with_ksm_page()
1098 	remove_rmap_item_from_tree(rmap_item);  in try_to_merge_with_ksm_page()
1101 	rmap_item->anon_vma = vma->anon_vma;  in try_to_merge_with_ksm_page()
1118 static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,  in try_to_merge_two_pages()  argument
1120 					   struct rmap_item *tree_rmap_item,  in try_to_merge_two_pages()
1125 	err = try_to_merge_with_ksm_page(rmap_item, page, NULL);  in try_to_merge_two_pages()
1134 			break_cow(rmap_item);  in try_to_merge_two_pages()
1341 struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,  in unstable_tree_search_insert()  argument
1355 		struct rmap_item *tree_rmap_item;  in unstable_tree_search_insert()
1360 		tree_rmap_item = rb_entry(*new, struct rmap_item, node);  in unstable_tree_search_insert()
1397 	rmap_item->address |= UNSTABLE_FLAG;  in unstable_tree_search_insert()
1398 	rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK);  in unstable_tree_search_insert()
1399 	DO_NUMA(rmap_item->nid = nid);  in unstable_tree_search_insert()
1400 	rb_link_node(&rmap_item->node, parent, new);  in unstable_tree_search_insert()
1401 	rb_insert_color(&rmap_item->node, root);  in unstable_tree_search_insert()
1412 static void stable_tree_append(struct rmap_item *rmap_item,  in stable_tree_append()  argument
1415 	rmap_item->head = stable_node;  in stable_tree_append()
1416 	rmap_item->address |= STABLE_FLAG;  in stable_tree_append()
1417 	hlist_add_head(&rmap_item->hlist, &stable_node->hlist);  in stable_tree_append()
1419 	if (rmap_item->hlist.next)  in stable_tree_append()
1434 static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)  in cmp_and_merge_page()  argument
1436 	struct rmap_item *tree_rmap_item;  in cmp_and_merge_page()
1453 		    rmap_item->head == stable_node)  in cmp_and_merge_page()
1459 	if (kpage == page && rmap_item->head == stable_node) {  in cmp_and_merge_page()
1464 	remove_rmap_item_from_tree(rmap_item);  in cmp_and_merge_page()
1467 		err = try_to_merge_with_ksm_page(rmap_item, page, kpage);  in cmp_and_merge_page()
1474 			stable_tree_append(rmap_item, page_stable_node(kpage));  in cmp_and_merge_page()
1488 	if (rmap_item->oldchecksum != checksum) {  in cmp_and_merge_page()
1489 		rmap_item->oldchecksum = checksum;  in cmp_and_merge_page()
1494 		unstable_tree_search_insert(rmap_item, page, &tree_page);  in cmp_and_merge_page()
1496 		kpage = try_to_merge_two_pages(rmap_item, page,  in cmp_and_merge_page()
1508 				stable_tree_append(rmap_item, stable_node);  in cmp_and_merge_page()
1520 				break_cow(rmap_item);  in cmp_and_merge_page()
1526 static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot,  in get_next_rmap_item()
1527 					    struct rmap_item **rmap_list,  in get_next_rmap_item()
1530 	struct rmap_item *rmap_item;  in get_next_rmap_item()  local
1533 		rmap_item = *rmap_list;  in get_next_rmap_item()
1534 		if ((rmap_item->address & PAGE_MASK) == addr)  in get_next_rmap_item()
1535 			return rmap_item;  in get_next_rmap_item()
1536 		if (rmap_item->address > addr)  in get_next_rmap_item()
1538 		*rmap_list = rmap_item->rmap_list;  in get_next_rmap_item()
1539 		remove_rmap_item_from_tree(rmap_item);  in get_next_rmap_item()
1540 		free_rmap_item(rmap_item);  in get_next_rmap_item()
1543 	rmap_item = alloc_rmap_item();  in get_next_rmap_item()
1544 	if (rmap_item) {  in get_next_rmap_item()
1546 		rmap_item->mm = mm_slot->mm;  in get_next_rmap_item()
1547 		rmap_item->address = addr;  in get_next_rmap_item()
1548 		rmap_item->rmap_list = *rmap_list;  in get_next_rmap_item()
1549 		*rmap_list = rmap_item;  in get_next_rmap_item()
1551 	return rmap_item;  in get_next_rmap_item()
1554 static struct rmap_item *scan_get_next_rmap_item(struct page **page)  in scan_get_next_rmap_item()
1559 	struct rmap_item *rmap_item;  in scan_get_next_rmap_item()  local
1646 				rmap_item = get_next_rmap_item(slot,  in scan_get_next_rmap_item()
1648 				if (rmap_item) {  in scan_get_next_rmap_item()
1650 							&rmap_item->rmap_list;  in scan_get_next_rmap_item()
1655 				return rmap_item;  in scan_get_next_rmap_item()
1714 	struct rmap_item *rmap_item;  in ksm_do_scan()  local
1719 		rmap_item = scan_get_next_rmap_item(&page);  in ksm_do_scan()
1720 		if (!rmap_item)  in ksm_do_scan()
1722 		cmp_and_merge_page(page, rmap_item);  in ksm_do_scan()
1915 	struct rmap_item *rmap_item;  in rmap_walk_ksm()  local
1931 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in rmap_walk_ksm()
1932 		struct anon_vma *anon_vma = rmap_item->anon_vma;  in rmap_walk_ksm()
1942 			if (rmap_item->address < vma->vm_start ||  in rmap_walk_ksm()
1943 			    rmap_item->address >= vma->vm_end)  in rmap_walk_ksm()
1951 			if ((rmap_item->mm == vma->vm_mm) == search_new_forks)  in rmap_walk_ksm()
1958 					rmap_item->address, rwc->arg);  in rmap_walk_ksm()