134 struct stable_node {  struct
175 			struct stable_node *head;
256 	stable_node_cache = KSM_KMEM_CACHE(stable_node, 0);  in ksm_slab_init()
299 static inline struct stable_node *alloc_stable_node(void)  in alloc_stable_node()
304 static inline void free_stable_node(struct stable_node *stable_node)  in free_stable_node()  argument
306 	kmem_cache_free(stable_node_cache, stable_node);  in free_stable_node()
496 static void remove_node_from_stable_tree(struct stable_node *stable_node)  in remove_node_from_stable_tree()  argument
500 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in remove_node_from_stable_tree()
510 	if (stable_node->head == &migrate_nodes)  in remove_node_from_stable_tree()
511 		list_del(&stable_node->list);  in remove_node_from_stable_tree()
513 		rb_erase(&stable_node->node,  in remove_node_from_stable_tree()
514 			 root_stable_tree + NUMA(stable_node->nid));  in remove_node_from_stable_tree()
515 	free_stable_node(stable_node);  in remove_node_from_stable_tree()
537 static struct page *get_ksm_page(struct stable_node *stable_node, bool lock_it)  in get_ksm_page()  argument
543 	expected_mapping = (void *)stable_node +  in get_ksm_page()
546 	kpfn = READ_ONCE(stable_node->kpfn);  in get_ksm_page()
604 	if (READ_ONCE(stable_node->kpfn) != kpfn)  in get_ksm_page()
606 	remove_node_from_stable_tree(stable_node);  in get_ksm_page()
617 		struct stable_node *stable_node;  in remove_rmap_item_from_tree()  local
620 		stable_node = rmap_item->head;  in remove_rmap_item_from_tree()
621 		page = get_ksm_page(stable_node, true);  in remove_rmap_item_from_tree()
629 		if (!hlist_empty(&stable_node->hlist))  in remove_rmap_item_from_tree()
703 static int remove_stable_node(struct stable_node *stable_node)  in remove_stable_node()  argument
708 	page = get_ksm_page(stable_node, true);  in remove_stable_node()
732 		remove_node_from_stable_tree(stable_node);  in remove_stable_node()
743 	struct stable_node *stable_node;  in remove_all_stable_nodes()  local
750 			stable_node = rb_entry(root_stable_tree[nid].rb_node,  in remove_all_stable_nodes()
751 						struct stable_node, node);  in remove_all_stable_nodes()
752 			if (remove_stable_node(stable_node)) {  in remove_all_stable_nodes()
760 		stable_node = list_entry(this, struct stable_node, list);  in remove_all_stable_nodes()
761 		if (remove_stable_node(stable_node))  in remove_all_stable_nodes()
1154 	struct stable_node *stable_node;  in stable_tree_search()  local
1155 	struct stable_node *page_node;  in stable_tree_search()
1175 		stable_node = rb_entry(*new, struct stable_node, node);  in stable_tree_search()
1176 		tree_page = get_ksm_page(stable_node, false);  in stable_tree_search()
1206 			tree_page = get_ksm_page(stable_node, true);  in stable_tree_search()
1209 				if (get_kpfn_nid(stable_node->kpfn) !=  in stable_tree_search()
1210 						NUMA(stable_node->nid)) {  in stable_tree_search()
1240 		rb_replace_node(&stable_node->node, &page_node->node, root);  in stable_tree_search()
1243 		rb_erase(&stable_node->node, root);  in stable_tree_search()
1246 	stable_node->head = &migrate_nodes;  in stable_tree_search()
1247 	list_add(&stable_node->list, stable_node->head);  in stable_tree_search()
1258 static struct stable_node *stable_tree_insert(struct page *kpage)  in stable_tree_insert()
1265 	struct stable_node *stable_node;  in stable_tree_insert()  local
1279 		stable_node = rb_entry(*new, struct stable_node, node);  in stable_tree_insert()
1280 		tree_page = get_ksm_page(stable_node, false);  in stable_tree_insert()
1312 	stable_node = alloc_stable_node();  in stable_tree_insert()
1313 	if (!stable_node)  in stable_tree_insert()
1316 	INIT_HLIST_HEAD(&stable_node->hlist);  in stable_tree_insert()
1317 	stable_node->kpfn = kpfn;  in stable_tree_insert()
1318 	set_page_stable_node(kpage, stable_node);  in stable_tree_insert()
1319 	DO_NUMA(stable_node->nid = nid);  in stable_tree_insert()
1320 	rb_link_node(&stable_node->node, parent, new);  in stable_tree_insert()
1321 	rb_insert_color(&stable_node->node, root);  in stable_tree_insert()
1323 	return stable_node;  in stable_tree_insert()
1413 			       struct stable_node *stable_node)  in stable_tree_append()  argument
1415 	rmap_item->head = stable_node;  in stable_tree_append()
1417 	hlist_add_head(&rmap_item->hlist, &stable_node->hlist);  in stable_tree_append()
1438 	struct stable_node *stable_node;  in cmp_and_merge_page()  local
1443 	stable_node = page_stable_node(page);  in cmp_and_merge_page()
1444 	if (stable_node) {  in cmp_and_merge_page()
1445 		if (stable_node->head != &migrate_nodes &&  in cmp_and_merge_page()
1446 		    get_kpfn_nid(stable_node->kpfn) != NUMA(stable_node->nid)) {  in cmp_and_merge_page()
1447 			rb_erase(&stable_node->node,  in cmp_and_merge_page()
1448 				 root_stable_tree + NUMA(stable_node->nid));  in cmp_and_merge_page()
1449 			stable_node->head = &migrate_nodes;  in cmp_and_merge_page()
1450 			list_add(&stable_node->list, stable_node->head);  in cmp_and_merge_page()
1452 		if (stable_node->head != &migrate_nodes &&  in cmp_and_merge_page()
1453 		    rmap_item->head == stable_node)  in cmp_and_merge_page()
1459 	if (kpage == page && rmap_item->head == stable_node) {  in cmp_and_merge_page()
1505 			stable_node = stable_tree_insert(kpage);  in cmp_and_merge_page()
1506 			if (stable_node) {  in cmp_and_merge_page()
1507 				stable_tree_append(tree_rmap_item, stable_node);  in cmp_and_merge_page()
1508 				stable_tree_append(rmap_item, stable_node);  in cmp_and_merge_page()
1518 			if (!stable_node) {  in cmp_and_merge_page()
1586 			struct stable_node *stable_node;  in scan_get_next_rmap_item()  local
1591 				stable_node = list_entry(this,  in scan_get_next_rmap_item()
1592 						struct stable_node, list);  in scan_get_next_rmap_item()
1593 				page = get_ksm_page(stable_node, false);  in scan_get_next_rmap_item()
1914 	struct stable_node *stable_node;  in rmap_walk_ksm()  local
1927 	stable_node = page_stable_node(page);  in rmap_walk_ksm()
1928 	if (!stable_node)  in rmap_walk_ksm()
1931 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in rmap_walk_ksm()
1979 	struct stable_node *stable_node;  in ksm_migrate_page()  local
1985 	stable_node = page_stable_node(newpage);  in ksm_migrate_page()
1986 	if (stable_node) {  in ksm_migrate_page()
1987 		VM_BUG_ON_PAGE(stable_node->kpfn != page_to_pfn(oldpage), oldpage);  in ksm_migrate_page()
1988 		stable_node->kpfn = page_to_pfn(newpage);  in ksm_migrate_page()
2015 	struct stable_node *stable_node;  in ksm_check_stable_tree()  local
2023 			stable_node = rb_entry(node, struct stable_node, node);  in ksm_check_stable_tree()
2024 			if (stable_node->kpfn >= start_pfn &&  in ksm_check_stable_tree()
2025 			    stable_node->kpfn < end_pfn) {  in ksm_check_stable_tree()
2030 				remove_node_from_stable_tree(stable_node);  in ksm_check_stable_tree()
2038 		stable_node = list_entry(this, struct stable_node, list);  in ksm_check_stable_tree()
2039 		if (stable_node->kpfn >= start_pfn &&  in ksm_check_stable_tree()
2040 		    stable_node->kpfn < end_pfn)  in ksm_check_stable_tree()
2041 			remove_node_from_stable_tree(stable_node);  in ksm_check_stable_tree()