134 struct stable_node {  struct
175 			struct stable_node *head;
256 	stable_node_cache = KSM_KMEM_CACHE(stable_node, 0);  in ksm_slab_init()
299 static inline struct stable_node *alloc_stable_node(void)  in alloc_stable_node()
304 static inline void free_stable_node(struct stable_node *stable_node)  in free_stable_node()  argument
306 	kmem_cache_free(stable_node_cache, stable_node);  in free_stable_node()
495 static void remove_node_from_stable_tree(struct stable_node *stable_node)  in remove_node_from_stable_tree()  argument
499 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in remove_node_from_stable_tree()
509 	if (stable_node->head == &migrate_nodes)  in remove_node_from_stable_tree()
510 		list_del(&stable_node->list);  in remove_node_from_stable_tree()
512 		rb_erase(&stable_node->node,  in remove_node_from_stable_tree()
513 			 root_stable_tree + NUMA(stable_node->nid));  in remove_node_from_stable_tree()
514 	free_stable_node(stable_node);  in remove_node_from_stable_tree()
536 static struct page *get_ksm_page(struct stable_node *stable_node, bool lock_it)  in get_ksm_page()  argument
542 	expected_mapping = (void *)stable_node +  in get_ksm_page()
545 	kpfn = READ_ONCE(stable_node->kpfn);  in get_ksm_page()
603 	if (READ_ONCE(stable_node->kpfn) != kpfn)  in get_ksm_page()
605 	remove_node_from_stable_tree(stable_node);  in get_ksm_page()
616 		struct stable_node *stable_node;  in remove_rmap_item_from_tree()  local
619 		stable_node = rmap_item->head;  in remove_rmap_item_from_tree()
620 		page = get_ksm_page(stable_node, true);  in remove_rmap_item_from_tree()
628 		if (stable_node->hlist.first)  in remove_rmap_item_from_tree()
702 static int remove_stable_node(struct stable_node *stable_node)  in remove_stable_node()  argument
707 	page = get_ksm_page(stable_node, true);  in remove_stable_node()
731 		remove_node_from_stable_tree(stable_node);  in remove_stable_node()
742 	struct stable_node *stable_node;  in remove_all_stable_nodes()  local
749 			stable_node = rb_entry(root_stable_tree[nid].rb_node,  in remove_all_stable_nodes()
750 						struct stable_node, node);  in remove_all_stable_nodes()
751 			if (remove_stable_node(stable_node)) {  in remove_all_stable_nodes()
759 		stable_node = list_entry(this, struct stable_node, list);  in remove_all_stable_nodes()
760 		if (remove_stable_node(stable_node))  in remove_all_stable_nodes()
1157 	struct stable_node *stable_node;  in stable_tree_search()  local
1158 	struct stable_node *page_node;  in stable_tree_search()
1178 		stable_node = rb_entry(*new, struct stable_node, node);  in stable_tree_search()
1179 		tree_page = get_ksm_page(stable_node, false);  in stable_tree_search()
1199 			tree_page = get_ksm_page(stable_node, true);  in stable_tree_search()
1202 				if (get_kpfn_nid(stable_node->kpfn) !=  in stable_tree_search()
1203 						NUMA(stable_node->nid)) {  in stable_tree_search()
1233 		rb_replace_node(&stable_node->node, &page_node->node, root);  in stable_tree_search()
1236 		rb_erase(&stable_node->node, root);  in stable_tree_search()
1239 	stable_node->head = &migrate_nodes;  in stable_tree_search()
1240 	list_add(&stable_node->list, stable_node->head);  in stable_tree_search()
1251 static struct stable_node *stable_tree_insert(struct page *kpage)  in stable_tree_insert()
1258 	struct stable_node *stable_node;  in stable_tree_insert()  local
1270 		stable_node = rb_entry(*new, struct stable_node, node);  in stable_tree_insert()
1271 		tree_page = get_ksm_page(stable_node, false);  in stable_tree_insert()
1293 	stable_node = alloc_stable_node();  in stable_tree_insert()
1294 	if (!stable_node)  in stable_tree_insert()
1297 	INIT_HLIST_HEAD(&stable_node->hlist);  in stable_tree_insert()
1298 	stable_node->kpfn = kpfn;  in stable_tree_insert()
1299 	set_page_stable_node(kpage, stable_node);  in stable_tree_insert()
1300 	DO_NUMA(stable_node->nid = nid);  in stable_tree_insert()
1301 	rb_link_node(&stable_node->node, parent, new);  in stable_tree_insert()
1302 	rb_insert_color(&stable_node->node, root);  in stable_tree_insert()
1304 	return stable_node;  in stable_tree_insert()
1394 			       struct stable_node *stable_node)  in stable_tree_append()  argument
1396 	rmap_item->head = stable_node;  in stable_tree_append()
1398 	hlist_add_head(&rmap_item->hlist, &stable_node->hlist);  in stable_tree_append()
1419 	struct stable_node *stable_node;  in cmp_and_merge_page()  local
1424 	stable_node = page_stable_node(page);  in cmp_and_merge_page()
1425 	if (stable_node) {  in cmp_and_merge_page()
1426 		if (stable_node->head != &migrate_nodes &&  in cmp_and_merge_page()
1427 		    get_kpfn_nid(stable_node->kpfn) != NUMA(stable_node->nid)) {  in cmp_and_merge_page()
1428 			rb_erase(&stable_node->node,  in cmp_and_merge_page()
1429 				 root_stable_tree + NUMA(stable_node->nid));  in cmp_and_merge_page()
1430 			stable_node->head = &migrate_nodes;  in cmp_and_merge_page()
1431 			list_add(&stable_node->list, stable_node->head);  in cmp_and_merge_page()
1433 		if (stable_node->head != &migrate_nodes &&  in cmp_and_merge_page()
1434 		    rmap_item->head == stable_node)  in cmp_and_merge_page()
1440 	if (kpage == page && rmap_item->head == stable_node) {  in cmp_and_merge_page()
1486 			stable_node = stable_tree_insert(kpage);  in cmp_and_merge_page()
1487 			if (stable_node) {  in cmp_and_merge_page()
1488 				stable_tree_append(tree_rmap_item, stable_node);  in cmp_and_merge_page()
1489 				stable_tree_append(rmap_item, stable_node);  in cmp_and_merge_page()
1499 			if (!stable_node) {  in cmp_and_merge_page()
1567 			struct stable_node *stable_node;  in scan_get_next_rmap_item()  local
1572 				stable_node = list_entry(this,  in scan_get_next_rmap_item()
1573 						struct stable_node, list);  in scan_get_next_rmap_item()
1574 				page = get_ksm_page(stable_node, false);  in scan_get_next_rmap_item()
1895 	struct stable_node *stable_node;  in rmap_walk_ksm()  local
1908 	stable_node = page_stable_node(page);  in rmap_walk_ksm()
1909 	if (!stable_node)  in rmap_walk_ksm()
1912 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in rmap_walk_ksm()
1958 	struct stable_node *stable_node;  in ksm_migrate_page()  local
1964 	stable_node = page_stable_node(newpage);  in ksm_migrate_page()
1965 	if (stable_node) {  in ksm_migrate_page()
1966 		VM_BUG_ON_PAGE(stable_node->kpfn != page_to_pfn(oldpage), oldpage);  in ksm_migrate_page()
1967 		stable_node->kpfn = page_to_pfn(newpage);  in ksm_migrate_page()
1994 	struct stable_node *stable_node;  in ksm_check_stable_tree()  local
2002 			stable_node = rb_entry(node, struct stable_node, node);  in ksm_check_stable_tree()
2003 			if (stable_node->kpfn >= start_pfn &&  in ksm_check_stable_tree()
2004 			    stable_node->kpfn < end_pfn) {  in ksm_check_stable_tree()
2009 				remove_node_from_stable_tree(stable_node);  in ksm_check_stable_tree()
2017 		stable_node = list_entry(this, struct stable_node, list);  in ksm_check_stable_tree()
2018 		if (stable_node->kpfn >= start_pfn &&  in ksm_check_stable_tree()
2019 		    stable_node->kpfn < end_pfn)  in ksm_check_stable_tree()
2020 			remove_node_from_stable_tree(stable_node);  in ksm_check_stable_tree()