stable_node 18 include/linux/ksm.h struct stable_node; stable_node 209 mm/ksm.c struct stable_node *head; stable_node 312 mm/ksm.c stable_node_cache = KSM_KMEM_CACHE(stable_node, 0); stable_node 338 mm/ksm.c static __always_inline bool is_stable_node_chain(struct stable_node *chain) stable_node 343 mm/ksm.c static __always_inline bool is_stable_node_dup(struct stable_node *dup) stable_node 348 mm/ksm.c static inline void stable_node_chain_add_dup(struct stable_node *dup, stable_node 349 mm/ksm.c struct stable_node *chain) stable_node 358 mm/ksm.c static inline void __stable_node_dup_del(struct stable_node *dup) stable_node 365 mm/ksm.c static inline void stable_node_dup_del(struct stable_node *dup) stable_node 395 mm/ksm.c static inline struct stable_node *alloc_stable_node(void) stable_node 405 mm/ksm.c static inline void free_stable_node(struct stable_node *stable_node) stable_node 407 mm/ksm.c VM_BUG_ON(stable_node->rmap_hlist_len && stable_node 408 mm/ksm.c !is_stable_node_chain(stable_node)); stable_node 409 mm/ksm.c kmem_cache_free(stable_node_cache, stable_node); stable_node 590 mm/ksm.c static struct stable_node *alloc_stable_node_chain(struct stable_node *dup, stable_node 593 mm/ksm.c struct stable_node *chain = alloc_stable_node(); stable_node 623 mm/ksm.c static inline void free_stable_node_chain(struct stable_node *chain, stable_node 631 mm/ksm.c static void remove_node_from_stable_tree(struct stable_node *stable_node) stable_node 636 mm/ksm.c BUG_ON(stable_node->rmap_hlist_len < 0); stable_node 638 mm/ksm.c hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { stable_node 643 mm/ksm.c VM_BUG_ON(stable_node->rmap_hlist_len <= 0); stable_node 644 mm/ksm.c stable_node->rmap_hlist_len--; stable_node 662 mm/ksm.c if (stable_node->head == &migrate_nodes) stable_node 663 mm/ksm.c list_del(&stable_node->list); stable_node 665 mm/ksm.c stable_node_dup_del(stable_node); stable_node 666 mm/ksm.c free_stable_node(stable_node); stable_node 694 mm/ksm.c static struct page *get_ksm_page(struct stable_node *stable_node, stable_node 701 mm/ksm.c expected_mapping = (void *)((unsigned long)stable_node | stable_node 704 mm/ksm.c kpfn = READ_ONCE(stable_node->kpfn); /* Address dependency. */ stable_node 763 mm/ksm.c if (READ_ONCE(stable_node->kpfn) != kpfn) stable_node 765 mm/ksm.c remove_node_from_stable_tree(stable_node); stable_node 776 mm/ksm.c struct stable_node *stable_node; stable_node 779 mm/ksm.c stable_node = rmap_item->head; stable_node 780 mm/ksm.c page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK); stable_node 788 mm/ksm.c if (!hlist_empty(&stable_node->hlist)) stable_node 792 mm/ksm.c VM_BUG_ON(stable_node->rmap_hlist_len <= 0); stable_node 793 mm/ksm.c stable_node->rmap_hlist_len--; stable_node 860 mm/ksm.c static inline struct stable_node *page_stable_node(struct page *page) stable_node 866 mm/ksm.c struct stable_node *stable_node) stable_node 868 mm/ksm.c page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM); stable_node 875 mm/ksm.c static int remove_stable_node(struct stable_node *stable_node) stable_node 880 mm/ksm.c page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK); stable_node 904 mm/ksm.c remove_node_from_stable_tree(stable_node); stable_node 913 mm/ksm.c static int remove_stable_node_chain(struct stable_node *stable_node, stable_node 916 mm/ksm.c struct stable_node *dup; stable_node 919 mm/ksm.c if (!is_stable_node_chain(stable_node)) { stable_node 920 mm/ksm.c VM_BUG_ON(is_stable_node_dup(stable_node)); stable_node 921 mm/ksm.c if (remove_stable_node(stable_node)) stable_node 928 mm/ksm.c &stable_node->hlist, hlist_dup) { stable_node 933 mm/ksm.c BUG_ON(!hlist_empty(&stable_node->hlist)); stable_node 934 mm/ksm.c free_stable_node_chain(stable_node, root); stable_node 940 mm/ksm.c struct stable_node *stable_node, *next; stable_node 946 mm/ksm.c stable_node = rb_entry(root_stable_tree[nid].rb_node, stable_node 947 mm/ksm.c struct stable_node, node); stable_node 948 mm/ksm.c if (remove_stable_node_chain(stable_node, stable_node 956 mm/ksm.c list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) { stable_node 957 mm/ksm.c if (remove_stable_node(stable_node)) stable_node 1335 mm/ksm.c bool __is_page_sharing_candidate(struct stable_node *stable_node, int offset) stable_node 1337 mm/ksm.c VM_BUG_ON(stable_node->rmap_hlist_len < 0); stable_node 1344 mm/ksm.c return stable_node->rmap_hlist_len && stable_node 1345 mm/ksm.c stable_node->rmap_hlist_len + offset < ksm_max_page_sharing; stable_node 1349 mm/ksm.c bool is_page_sharing_candidate(struct stable_node *stable_node) stable_node 1351 mm/ksm.c return __is_page_sharing_candidate(stable_node, 0); stable_node 1354 mm/ksm.c static struct page *stable_node_dup(struct stable_node **_stable_node_dup, stable_node 1355 mm/ksm.c struct stable_node **_stable_node, stable_node 1359 mm/ksm.c struct stable_node *dup, *found = NULL, *stable_node = *_stable_node; stable_node 1366 mm/ksm.c time_before(jiffies, stable_node->chain_prune_time + stable_node 1371 mm/ksm.c stable_node->chain_prune_time = jiffies; stable_node 1374 mm/ksm.c &stable_node->hlist, hlist_dup) { stable_node 1422 mm/ksm.c BUG_ON(stable_node->hlist.first->next); stable_node 1428 mm/ksm.c rb_replace_node(&stable_node->node, &found->node, stable_node 1430 mm/ksm.c free_stable_node(stable_node); stable_node 1445 mm/ksm.c stable_node = NULL; stable_node 1446 mm/ksm.c } else if (stable_node->hlist.first != &found->hlist_dup && stable_node 1465 mm/ksm.c &stable_node->hlist); stable_node 1473 mm/ksm.c static struct stable_node *stable_node_dup_any(struct stable_node *stable_node, stable_node 1476 mm/ksm.c if (!is_stable_node_chain(stable_node)) stable_node 1477 mm/ksm.c return stable_node; stable_node 1478 mm/ksm.c if (hlist_empty(&stable_node->hlist)) { stable_node 1479 mm/ksm.c free_stable_node_chain(stable_node, root); stable_node 1482 mm/ksm.c return hlist_entry(stable_node->hlist.first, stable_node 1483 mm/ksm.c typeof(*stable_node), hlist_dup); stable_node 1500 mm/ksm.c static struct page *__stable_node_chain(struct stable_node **_stable_node_dup, stable_node 1501 mm/ksm.c struct stable_node **_stable_node, stable_node 1505 mm/ksm.c struct stable_node *stable_node = *_stable_node; stable_node 1506 mm/ksm.c if (!is_stable_node_chain(stable_node)) { stable_node 1507 mm/ksm.c if (is_page_sharing_candidate(stable_node)) { stable_node 1508 mm/ksm.c *_stable_node_dup = stable_node; stable_node 1509 mm/ksm.c return get_ksm_page(stable_node, GET_KSM_PAGE_NOLOCK); stable_node 1522 mm/ksm.c static __always_inline struct page *chain_prune(struct stable_node **s_n_d, stable_node 1523 mm/ksm.c struct stable_node **s_n, stable_node 1529 mm/ksm.c static __always_inline struct page *chain(struct stable_node **s_n_d, stable_node 1530 mm/ksm.c struct stable_node *s_n, stable_node 1533 mm/ksm.c struct stable_node *old_stable_node = s_n; stable_node 1557 mm/ksm.c struct stable_node *stable_node, *stable_node_dup, *stable_node_any; stable_node 1558 mm/ksm.c struct stable_node *page_node; stable_node 1578 mm/ksm.c stable_node = rb_entry(*new, struct stable_node, node); stable_node 1580 mm/ksm.c tree_page = chain_prune(&stable_node_dup, &stable_node, root); stable_node 1599 mm/ksm.c stable_node_any = stable_node_dup_any(stable_node, stable_node 1721 mm/ksm.c if (stable_node_dup == stable_node) { stable_node 1741 mm/ksm.c VM_BUG_ON(!is_stable_node_chain(stable_node)); stable_node 1747 mm/ksm.c stable_node_chain_add_dup(page_node, stable_node); stable_node 1772 mm/ksm.c if (stable_node_dup == stable_node) { stable_node 1776 mm/ksm.c stable_node = alloc_stable_node_chain(stable_node_dup, stable_node 1778 mm/ksm.c if (!stable_node) stable_node 1787 mm/ksm.c VM_BUG_ON(!is_stable_node_chain(stable_node)); stable_node 1792 mm/ksm.c stable_node_chain_add_dup(page_node, stable_node); stable_node 1803 mm/ksm.c static struct stable_node *stable_tree_insert(struct page *kpage) stable_node 1810 mm/ksm.c struct stable_node *stable_node, *stable_node_dup, *stable_node_any; stable_node 1825 mm/ksm.c stable_node = rb_entry(*new, struct stable_node, node); stable_node 1827 mm/ksm.c tree_page = chain(&stable_node_dup, stable_node, root); stable_node 1834 mm/ksm.c stable_node_any = stable_node_dup_any(stable_node, stable_node 1893 mm/ksm.c if (!is_stable_node_chain(stable_node)) { stable_node 1894 mm/ksm.c struct stable_node *orig = stable_node; stable_node 1896 mm/ksm.c stable_node = alloc_stable_node_chain(orig, root); stable_node 1897 mm/ksm.c if (!stable_node) { stable_node 1902 mm/ksm.c stable_node_chain_add_dup(stable_node_dup, stable_node); stable_node 1995 mm/ksm.c struct stable_node *stable_node, stable_node 2008 mm/ksm.c BUG_ON(stable_node->rmap_hlist_len < 0); stable_node 2010 mm/ksm.c stable_node->rmap_hlist_len++; stable_node 2013 mm/ksm.c WARN_ON_ONCE(stable_node->rmap_hlist_len > stable_node 2016 mm/ksm.c rmap_item->head = stable_node; stable_node 2018 mm/ksm.c hlist_add_head(&rmap_item->hlist, &stable_node->hlist); stable_node 2040 mm/ksm.c struct stable_node *stable_node; stable_node 2046 mm/ksm.c stable_node = page_stable_node(page); stable_node 2047 mm/ksm.c if (stable_node) { stable_node 2048 mm/ksm.c if (stable_node->head != &migrate_nodes && stable_node 2049 mm/ksm.c get_kpfn_nid(READ_ONCE(stable_node->kpfn)) != stable_node 2050 mm/ksm.c NUMA(stable_node->nid)) { stable_node 2051 mm/ksm.c stable_node_dup_del(stable_node); stable_node 2052 mm/ksm.c stable_node->head = &migrate_nodes; stable_node 2053 mm/ksm.c list_add(&stable_node->list, stable_node->head); stable_node 2055 mm/ksm.c if (stable_node->head != &migrate_nodes && stable_node 2056 mm/ksm.c rmap_item->head == stable_node) stable_node 2062 mm/ksm.c if (!is_page_sharing_candidate(stable_node)) stable_node 2068 mm/ksm.c if (kpage == page && rmap_item->head == stable_node) { stable_node 2159 mm/ksm.c stable_node = stable_tree_insert(kpage); stable_node 2160 mm/ksm.c if (stable_node) { stable_node 2161 mm/ksm.c stable_tree_append(tree_rmap_item, stable_node, stable_node 2163 mm/ksm.c stable_tree_append(rmap_item, stable_node, stable_node 2174 mm/ksm.c if (!stable_node) { stable_node 2256 mm/ksm.c struct stable_node *stable_node, *next; stable_node 2259 mm/ksm.c list_for_each_entry_safe(stable_node, next, stable_node 2261 mm/ksm.c page = get_ksm_page(stable_node, stable_node 2600 mm/ksm.c struct stable_node *stable_node; stable_node 2612 mm/ksm.c stable_node = page_stable_node(page); stable_node 2613 mm/ksm.c if (!stable_node) stable_node 2616 mm/ksm.c hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { stable_node 2690 mm/ksm.c struct stable_node *stable_node; stable_node 2696 mm/ksm.c stable_node = page_stable_node(newpage); stable_node 2697 mm/ksm.c if (stable_node) { stable_node 2698 mm/ksm.c VM_BUG_ON_PAGE(stable_node->kpfn != page_to_pfn(oldpage), oldpage); stable_node 2699 mm/ksm.c stable_node->kpfn = page_to_pfn(newpage); stable_node 2723 mm/ksm.c static bool stable_node_dup_remove_range(struct stable_node *stable_node, stable_node 2727 mm/ksm.c if (stable_node->kpfn >= start_pfn && stable_node 2728 mm/ksm.c stable_node->kpfn < end_pfn) { stable_node 2733 mm/ksm.c remove_node_from_stable_tree(stable_node); stable_node 2739 mm/ksm.c static bool stable_node_chain_remove_range(struct stable_node *stable_node, stable_node 2744 mm/ksm.c struct stable_node *dup; stable_node 2747 mm/ksm.c if (!is_stable_node_chain(stable_node)) { stable_node 2748 mm/ksm.c VM_BUG_ON(is_stable_node_dup(stable_node)); stable_node 2749 mm/ksm.c return stable_node_dup_remove_range(stable_node, start_pfn, stable_node 2754 mm/ksm.c &stable_node->hlist, hlist_dup) { stable_node 2758 mm/ksm.c if (hlist_empty(&stable_node->hlist)) { stable_node 2759 mm/ksm.c free_stable_node_chain(stable_node, root); stable_node 2768 mm/ksm.c struct stable_node *stable_node, *next; stable_node 2775 mm/ksm.c stable_node = rb_entry(node, struct stable_node, node); stable_node 2776 mm/ksm.c if (stable_node_chain_remove_range(stable_node, stable_node 2786 mm/ksm.c list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) { stable_node 2787 mm/ksm.c if (stable_node->kpfn >= start_pfn && stable_node 2788 mm/ksm.c stable_node->kpfn < end_pfn) stable_node 2789 mm/ksm.c remove_node_from_stable_tree(stable_node);