Lines Matching refs:rmap_item
105 struct rmap_item *rmap_list;
121 struct rmap_item **rmap_list;
161 struct rmap_item { struct
162 struct rmap_item *rmap_list; argument
252 rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0); in ksm_slab_init()
282 static inline struct rmap_item *alloc_rmap_item(void) in alloc_rmap_item()
284 struct rmap_item *rmap_item; in alloc_rmap_item() local
286 rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL); in alloc_rmap_item()
287 if (rmap_item) in alloc_rmap_item()
289 return rmap_item; in alloc_rmap_item()
292 static inline void free_rmap_item(struct rmap_item *rmap_item) in free_rmap_item() argument
295 rmap_item->mm = NULL; /* debug safety */ in free_rmap_item()
296 kmem_cache_free(rmap_item_cache, rmap_item); in free_rmap_item()
425 static void break_cow(struct rmap_item *rmap_item) in break_cow() argument
427 struct mm_struct *mm = rmap_item->mm; in break_cow()
428 unsigned long addr = rmap_item->address; in break_cow()
435 put_anon_vma(rmap_item->anon_vma); in break_cow()
458 static struct page *get_mergeable_page(struct rmap_item *rmap_item) in get_mergeable_page() argument
460 struct mm_struct *mm = rmap_item->mm; in get_mergeable_page()
461 unsigned long addr = rmap_item->address; in get_mergeable_page()
497 struct rmap_item *rmap_item; in remove_node_from_stable_tree() local
499 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { in remove_node_from_stable_tree()
500 if (rmap_item->hlist.next) in remove_node_from_stable_tree()
504 put_anon_vma(rmap_item->anon_vma); in remove_node_from_stable_tree()
505 rmap_item->address &= PAGE_MASK; in remove_node_from_stable_tree()
613 static void remove_rmap_item_from_tree(struct rmap_item *rmap_item) in remove_rmap_item_from_tree() argument
615 if (rmap_item->address & STABLE_FLAG) { in remove_rmap_item_from_tree()
619 stable_node = rmap_item->head; in remove_rmap_item_from_tree()
624 hlist_del(&rmap_item->hlist); in remove_rmap_item_from_tree()
633 put_anon_vma(rmap_item->anon_vma); in remove_rmap_item_from_tree()
634 rmap_item->address &= PAGE_MASK; in remove_rmap_item_from_tree()
636 } else if (rmap_item->address & UNSTABLE_FLAG) { in remove_rmap_item_from_tree()
645 age = (unsigned char)(ksm_scan.seqnr - rmap_item->address); in remove_rmap_item_from_tree()
648 rb_erase(&rmap_item->node, in remove_rmap_item_from_tree()
649 root_unstable_tree + NUMA(rmap_item->nid)); in remove_rmap_item_from_tree()
651 rmap_item->address &= PAGE_MASK; in remove_rmap_item_from_tree()
658 struct rmap_item **rmap_list) in remove_trailing_rmap_items()
661 struct rmap_item *rmap_item = *rmap_list; in remove_trailing_rmap_items() local
662 *rmap_list = rmap_item->rmap_list; in remove_trailing_rmap_items()
663 remove_rmap_item_from_tree(rmap_item); in remove_trailing_rmap_items()
664 free_rmap_item(rmap_item); in remove_trailing_rmap_items()
1082 static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item, in try_to_merge_with_ksm_page() argument
1085 struct mm_struct *mm = rmap_item->mm; in try_to_merge_with_ksm_page()
1092 vma = find_vma(mm, rmap_item->address); in try_to_merge_with_ksm_page()
1093 if (!vma || vma->vm_start > rmap_item->address) in try_to_merge_with_ksm_page()
1101 remove_rmap_item_from_tree(rmap_item); in try_to_merge_with_ksm_page()
1104 rmap_item->anon_vma = vma->anon_vma; in try_to_merge_with_ksm_page()
1121 static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item, in try_to_merge_two_pages() argument
1123 struct rmap_item *tree_rmap_item, in try_to_merge_two_pages()
1128 err = try_to_merge_with_ksm_page(rmap_item, page, NULL); in try_to_merge_two_pages()
1137 break_cow(rmap_item); in try_to_merge_two_pages()
1322 struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item, in unstable_tree_search_insert() argument
1336 struct rmap_item *tree_rmap_item; in unstable_tree_search_insert()
1341 tree_rmap_item = rb_entry(*new, struct rmap_item, node); in unstable_tree_search_insert()
1378 rmap_item->address |= UNSTABLE_FLAG; in unstable_tree_search_insert()
1379 rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK); in unstable_tree_search_insert()
1380 DO_NUMA(rmap_item->nid = nid); in unstable_tree_search_insert()
1381 rb_link_node(&rmap_item->node, parent, new); in unstable_tree_search_insert()
1382 rb_insert_color(&rmap_item->node, root); in unstable_tree_search_insert()
1393 static void stable_tree_append(struct rmap_item *rmap_item, in stable_tree_append() argument
1396 rmap_item->head = stable_node; in stable_tree_append()
1397 rmap_item->address |= STABLE_FLAG; in stable_tree_append()
1398 hlist_add_head(&rmap_item->hlist, &stable_node->hlist); in stable_tree_append()
1400 if (rmap_item->hlist.next) in stable_tree_append()
1415 static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) in cmp_and_merge_page() argument
1417 struct rmap_item *tree_rmap_item; in cmp_and_merge_page()
1434 rmap_item->head == stable_node) in cmp_and_merge_page()
1440 if (kpage == page && rmap_item->head == stable_node) { in cmp_and_merge_page()
1445 remove_rmap_item_from_tree(rmap_item); in cmp_and_merge_page()
1448 err = try_to_merge_with_ksm_page(rmap_item, page, kpage); in cmp_and_merge_page()
1455 stable_tree_append(rmap_item, page_stable_node(kpage)); in cmp_and_merge_page()
1469 if (rmap_item->oldchecksum != checksum) { in cmp_and_merge_page()
1470 rmap_item->oldchecksum = checksum; in cmp_and_merge_page()
1475 unstable_tree_search_insert(rmap_item, page, &tree_page); in cmp_and_merge_page()
1477 kpage = try_to_merge_two_pages(rmap_item, page, in cmp_and_merge_page()
1489 stable_tree_append(rmap_item, stable_node); in cmp_and_merge_page()
1501 break_cow(rmap_item); in cmp_and_merge_page()
1507 static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, in get_next_rmap_item()
1508 struct rmap_item **rmap_list, in get_next_rmap_item()
1511 struct rmap_item *rmap_item; in get_next_rmap_item() local
1514 rmap_item = *rmap_list; in get_next_rmap_item()
1515 if ((rmap_item->address & PAGE_MASK) == addr) in get_next_rmap_item()
1516 return rmap_item; in get_next_rmap_item()
1517 if (rmap_item->address > addr) in get_next_rmap_item()
1519 *rmap_list = rmap_item->rmap_list; in get_next_rmap_item()
1520 remove_rmap_item_from_tree(rmap_item); in get_next_rmap_item()
1521 free_rmap_item(rmap_item); in get_next_rmap_item()
1524 rmap_item = alloc_rmap_item(); in get_next_rmap_item()
1525 if (rmap_item) { in get_next_rmap_item()
1527 rmap_item->mm = mm_slot->mm; in get_next_rmap_item()
1528 rmap_item->address = addr; in get_next_rmap_item()
1529 rmap_item->rmap_list = *rmap_list; in get_next_rmap_item()
1530 *rmap_list = rmap_item; in get_next_rmap_item()
1532 return rmap_item; in get_next_rmap_item()
1535 static struct rmap_item *scan_get_next_rmap_item(struct page **page) in scan_get_next_rmap_item()
1540 struct rmap_item *rmap_item; in scan_get_next_rmap_item() local
1627 rmap_item = get_next_rmap_item(slot, in scan_get_next_rmap_item()
1629 if (rmap_item) { in scan_get_next_rmap_item()
1631 &rmap_item->rmap_list; in scan_get_next_rmap_item()
1636 return rmap_item; in scan_get_next_rmap_item()
1695 struct rmap_item *rmap_item; in ksm_do_scan() local
1700 rmap_item = scan_get_next_rmap_item(&page); in ksm_do_scan()
1701 if (!rmap_item) in ksm_do_scan()
1703 cmp_and_merge_page(page, rmap_item); in ksm_do_scan()
1896 struct rmap_item *rmap_item; in rmap_walk_ksm() local
1912 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { in rmap_walk_ksm()
1913 struct anon_vma *anon_vma = rmap_item->anon_vma; in rmap_walk_ksm()
1921 if (rmap_item->address < vma->vm_start || in rmap_walk_ksm()
1922 rmap_item->address >= vma->vm_end) in rmap_walk_ksm()
1930 if ((rmap_item->mm == vma->vm_mm) == search_new_forks) in rmap_walk_ksm()
1937 rmap_item->address, rwc->arg); in rmap_walk_ksm()