| H A D | ksm.c | 105 struct rmap_item *rmap_list;
121 struct rmap_item **rmap_list;
150 * struct rmap_item - reverse mapping item for virtual addresses
151 * @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
154 * @mm: the memory structure this rmap_item is pointing into
155 * @address: the virtual address this rmap_item tracks (+ flags in low bits)
157 * @node: rb node of this rmap_item in the unstable tree
161 struct rmap_item { struct
162 struct rmap_item *rmap_list;
252 rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0); ksm_slab_init()
282 static inline struct rmap_item *alloc_rmap_item(void) alloc_rmap_item()
284 struct rmap_item *rmap_item; alloc_rmap_item() local
286 rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL); alloc_rmap_item()
287 if (rmap_item) alloc_rmap_item()
289 return rmap_item; alloc_rmap_item()
292 static inline void free_rmap_item(struct rmap_item *rmap_item) free_rmap_item() argument
295 rmap_item->mm = NULL; /* debug safety */ free_rmap_item()
296 kmem_cache_free(rmap_item_cache, rmap_item); free_rmap_item()
425 static void break_cow(struct rmap_item *rmap_item) break_cow() argument
427 struct mm_struct *mm = rmap_item->mm; break_cow()
428 unsigned long addr = rmap_item->address; break_cow()
435 put_anon_vma(rmap_item->anon_vma); break_cow()
458 static struct page *get_mergeable_page(struct rmap_item *rmap_item) get_mergeable_page() argument
460 struct mm_struct *mm = rmap_item->mm; get_mergeable_page()
461 unsigned long addr = rmap_item->address; get_mergeable_page()
498 struct rmap_item *rmap_item; remove_node_from_stable_tree() local
500 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { remove_node_from_stable_tree()
501 if (rmap_item->hlist.next) remove_node_from_stable_tree()
505 put_anon_vma(rmap_item->anon_vma); remove_node_from_stable_tree()
506 rmap_item->address &= PAGE_MASK; remove_node_from_stable_tree()
611 * Removing rmap_item from stable or unstable tree.
614 static void remove_rmap_item_from_tree(struct rmap_item *rmap_item) remove_rmap_item_from_tree() argument
616 if (rmap_item->address & STABLE_FLAG) { remove_rmap_item_from_tree()
620 stable_node = rmap_item->head; remove_rmap_item_from_tree()
625 hlist_del(&rmap_item->hlist); remove_rmap_item_from_tree()
634 put_anon_vma(rmap_item->anon_vma); remove_rmap_item_from_tree()
635 rmap_item->address &= PAGE_MASK; remove_rmap_item_from_tree()
637 } else if (rmap_item->address & UNSTABLE_FLAG) { remove_rmap_item_from_tree()
643 * if this rmap_item was inserted by this scan, rather remove_rmap_item_from_tree()
646 age = (unsigned char)(ksm_scan.seqnr - rmap_item->address); remove_rmap_item_from_tree()
649 rb_erase(&rmap_item->node, remove_rmap_item_from_tree()
650 root_unstable_tree + NUMA(rmap_item->nid)); remove_rmap_item_from_tree()
652 rmap_item->address &= PAGE_MASK; remove_rmap_item_from_tree()
659 struct rmap_item **rmap_list) remove_trailing_rmap_items()
662 struct rmap_item *rmap_item = *rmap_list; remove_trailing_rmap_items() local
663 *rmap_list = rmap_item->rmap_list; remove_trailing_rmap_items()
664 remove_rmap_item_from_tree(rmap_item); remove_trailing_rmap_items()
665 free_rmap_item(rmap_item); remove_trailing_rmap_items()
672 * that - an rmap_item is assigned to the stable tree after inserting ksm
1081 static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item, try_to_merge_with_ksm_page() argument
1084 struct mm_struct *mm = rmap_item->mm; try_to_merge_with_ksm_page()
1089 vma = find_mergeable_vma(mm, rmap_item->address); try_to_merge_with_ksm_page()
1098 remove_rmap_item_from_tree(rmap_item); try_to_merge_with_ksm_page()
1101 rmap_item->anon_vma = vma->anon_vma; try_to_merge_with_ksm_page()
1118 static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item, try_to_merge_two_pages() argument
1120 struct rmap_item *tree_rmap_item, try_to_merge_two_pages()
1125 err = try_to_merge_with_ksm_page(rmap_item, page, NULL); try_to_merge_two_pages()
1134 break_cow(rmap_item); try_to_merge_two_pages()
1328 * else insert rmap_item into the unstable tree.
1332 * tree, we insert rmap_item as a new object into the unstable tree.
1334 * This function returns pointer to rmap_item found to be identical
1341 struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item, unstable_tree_search_insert() argument
1355 struct rmap_item *tree_rmap_item; unstable_tree_search_insert()
1360 tree_rmap_item = rb_entry(*new, struct rmap_item, node); unstable_tree_search_insert()
1397 rmap_item->address |= UNSTABLE_FLAG; unstable_tree_search_insert()
1398 rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK); unstable_tree_search_insert()
1399 DO_NUMA(rmap_item->nid = nid); unstable_tree_search_insert()
1400 rb_link_node(&rmap_item->node, parent, new); unstable_tree_search_insert()
1401 rb_insert_color(&rmap_item->node, root); unstable_tree_search_insert()
1408 * stable_tree_append - add another rmap_item to the linked list of
1412 static void stable_tree_append(struct rmap_item *rmap_item, stable_tree_append() argument
1415 rmap_item->head = stable_node; stable_tree_append()
1416 rmap_item->address |= STABLE_FLAG; stable_tree_append()
1417 hlist_add_head(&rmap_item->hlist, &stable_node->hlist); stable_tree_append()
1419 if (rmap_item->hlist.next) stable_tree_append()
1432 * @rmap_item: the reverse mapping into the virtual address of this page
1434 static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) cmp_and_merge_page() argument
1436 struct rmap_item *tree_rmap_item; cmp_and_merge_page()
1453 rmap_item->head == stable_node) cmp_and_merge_page()
1459 if (kpage == page && rmap_item->head == stable_node) { cmp_and_merge_page()
1464 remove_rmap_item_from_tree(rmap_item); cmp_and_merge_page()
1467 err = try_to_merge_with_ksm_page(rmap_item, page, kpage); cmp_and_merge_page()
1471 * add its rmap_item to the stable tree. cmp_and_merge_page()
1474 stable_tree_append(rmap_item, page_stable_node(kpage)); cmp_and_merge_page()
1488 if (rmap_item->oldchecksum != checksum) { cmp_and_merge_page()
1489 rmap_item->oldchecksum = checksum; cmp_and_merge_page()
1494 unstable_tree_search_insert(rmap_item, page, &tree_page); cmp_and_merge_page()
1496 kpage = try_to_merge_two_pages(rmap_item, page, cmp_and_merge_page()
1508 stable_tree_append(rmap_item, stable_node); cmp_and_merge_page()
1520 break_cow(rmap_item); cmp_and_merge_page()
1526 static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, get_next_rmap_item()
1527 struct rmap_item **rmap_list, get_next_rmap_item()
1530 struct rmap_item *rmap_item; get_next_rmap_item() local
1533 rmap_item = *rmap_list; get_next_rmap_item()
1534 if ((rmap_item->address & PAGE_MASK) == addr) get_next_rmap_item()
1535 return rmap_item; get_next_rmap_item()
1536 if (rmap_item->address > addr) get_next_rmap_item()
1538 *rmap_list = rmap_item->rmap_list; get_next_rmap_item()
1539 remove_rmap_item_from_tree(rmap_item); get_next_rmap_item()
1540 free_rmap_item(rmap_item); get_next_rmap_item()
1543 rmap_item = alloc_rmap_item(); get_next_rmap_item()
1544 if (rmap_item) { get_next_rmap_item()
1546 rmap_item->mm = mm_slot->mm; get_next_rmap_item()
1547 rmap_item->address = addr; get_next_rmap_item()
1548 rmap_item->rmap_list = *rmap_list; get_next_rmap_item()
1549 *rmap_list = rmap_item; get_next_rmap_item()
1551 return rmap_item; get_next_rmap_item()
1554 static struct rmap_item *scan_get_next_rmap_item(struct page **page) scan_get_next_rmap_item()
1559 struct rmap_item *rmap_item; scan_get_next_rmap_item() local
1646 rmap_item = get_next_rmap_item(slot, scan_get_next_rmap_item()
1648 if (rmap_item) { scan_get_next_rmap_item()
1650 &rmap_item->rmap_list; scan_get_next_rmap_item()
1655 return rmap_item; scan_get_next_rmap_item()
1714 struct rmap_item *rmap_item; ksm_do_scan() local
1719 rmap_item = scan_get_next_rmap_item(&page); ksm_do_scan()
1720 if (!rmap_item) ksm_do_scan()
1722 cmp_and_merge_page(page, rmap_item); ksm_do_scan()
1915 struct rmap_item *rmap_item; rmap_walk_ksm() local
1931 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { rmap_walk_ksm()
1932 struct anon_vma *anon_vma = rmap_item->anon_vma; rmap_walk_ksm()
1942 if (rmap_item->address < vma->vm_start || rmap_walk_ksm()
1943 rmap_item->address >= vma->vm_end) rmap_walk_ksm()
1947 * rmap_item; but later, if there is still work to do, rmap_walk_ksm()
1951 if ((rmap_item->mm == vma->vm_mm) == search_new_forks) rmap_walk_ksm()
1958 rmap_item->address, rwc->arg); rmap_walk_ksm()
|