Lines Matching refs:anon_vma
68 static inline struct anon_vma *anon_vma_alloc(void) in anon_vma_alloc()
70 struct anon_vma *anon_vma; in anon_vma_alloc() local
72 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); in anon_vma_alloc()
73 if (anon_vma) { in anon_vma_alloc()
74 atomic_set(&anon_vma->refcount, 1); in anon_vma_alloc()
75 anon_vma->degree = 1; /* Reference for first vma */ in anon_vma_alloc()
76 anon_vma->parent = anon_vma; in anon_vma_alloc()
81 anon_vma->root = anon_vma; in anon_vma_alloc()
84 return anon_vma; in anon_vma_alloc()
87 static inline void anon_vma_free(struct anon_vma *anon_vma) in anon_vma_free() argument
89 VM_BUG_ON(atomic_read(&anon_vma->refcount)); in anon_vma_free()
109 if (rwsem_is_locked(&anon_vma->root->rwsem)) { in anon_vma_free()
110 anon_vma_lock_write(anon_vma); in anon_vma_free()
111 anon_vma_unlock_write(anon_vma); in anon_vma_free()
114 kmem_cache_free(anon_vma_cachep, anon_vma); in anon_vma_free()
129 struct anon_vma *anon_vma) in anon_vma_chain_link() argument
132 avc->anon_vma = anon_vma; in anon_vma_chain_link()
134 anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); in anon_vma_chain_link()
166 struct anon_vma *anon_vma = vma->anon_vma; in anon_vma_prepare() local
170 if (unlikely(!anon_vma)) { in anon_vma_prepare()
172 struct anon_vma *allocated; in anon_vma_prepare()
178 anon_vma = find_mergeable_anon_vma(vma); in anon_vma_prepare()
180 if (!anon_vma) { in anon_vma_prepare()
181 anon_vma = anon_vma_alloc(); in anon_vma_prepare()
182 if (unlikely(!anon_vma)) in anon_vma_prepare()
184 allocated = anon_vma; in anon_vma_prepare()
187 anon_vma_lock_write(anon_vma); in anon_vma_prepare()
190 if (likely(!vma->anon_vma)) { in anon_vma_prepare()
191 vma->anon_vma = anon_vma; in anon_vma_prepare()
192 anon_vma_chain_link(vma, avc, anon_vma); in anon_vma_prepare()
194 anon_vma->degree++; in anon_vma_prepare()
199 anon_vma_unlock_write(anon_vma); in anon_vma_prepare()
222 static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma) in lock_anon_vma_root() argument
224 struct anon_vma *new_root = anon_vma->root; in lock_anon_vma_root()
234 static inline void unlock_anon_vma_root(struct anon_vma *root) in unlock_anon_vma_root()
255 struct anon_vma *root = NULL; in anon_vma_clone()
258 struct anon_vma *anon_vma; in anon_vma_clone() local
268 anon_vma = pavc->anon_vma; in anon_vma_clone()
269 root = lock_anon_vma_root(root, anon_vma); in anon_vma_clone()
270 anon_vma_chain_link(dst, avc, anon_vma); in anon_vma_clone()
280 if (!dst->anon_vma && anon_vma != src->anon_vma && in anon_vma_clone()
281 anon_vma->degree < 2) in anon_vma_clone()
282 dst->anon_vma = anon_vma; in anon_vma_clone()
284 if (dst->anon_vma) in anon_vma_clone()
285 dst->anon_vma->degree++; in anon_vma_clone()
296 dst->anon_vma = NULL; in anon_vma_clone()
309 struct anon_vma *anon_vma; in anon_vma_fork() local
313 if (!pvma->anon_vma) in anon_vma_fork()
317 vma->anon_vma = NULL; in anon_vma_fork()
328 if (vma->anon_vma) in anon_vma_fork()
332 anon_vma = anon_vma_alloc(); in anon_vma_fork()
333 if (!anon_vma) in anon_vma_fork()
343 anon_vma->root = pvma->anon_vma->root; in anon_vma_fork()
344 anon_vma->parent = pvma->anon_vma; in anon_vma_fork()
350 get_anon_vma(anon_vma->root); in anon_vma_fork()
352 vma->anon_vma = anon_vma; in anon_vma_fork()
353 anon_vma_lock_write(anon_vma); in anon_vma_fork()
354 anon_vma_chain_link(vma, avc, anon_vma); in anon_vma_fork()
355 anon_vma->parent->degree++; in anon_vma_fork()
356 anon_vma_unlock_write(anon_vma); in anon_vma_fork()
361 put_anon_vma(anon_vma); in anon_vma_fork()
370 struct anon_vma *root = NULL; in unlink_anon_vmas()
377 struct anon_vma *anon_vma = avc->anon_vma; in unlink_anon_vmas() local
379 root = lock_anon_vma_root(root, anon_vma); in unlink_anon_vmas()
380 anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); in unlink_anon_vmas()
386 if (RB_EMPTY_ROOT(&anon_vma->rb_root)) { in unlink_anon_vmas()
387 anon_vma->parent->degree--; in unlink_anon_vmas()
394 if (vma->anon_vma) in unlink_anon_vmas()
395 vma->anon_vma->degree--; in unlink_anon_vmas()
404 struct anon_vma *anon_vma = avc->anon_vma; in unlink_anon_vmas() local
406 BUG_ON(anon_vma->degree); in unlink_anon_vmas()
407 put_anon_vma(anon_vma); in unlink_anon_vmas()
416 struct anon_vma *anon_vma = data; in anon_vma_ctor() local
418 init_rwsem(&anon_vma->rwsem); in anon_vma_ctor()
419 atomic_set(&anon_vma->refcount, 0); in anon_vma_ctor()
420 anon_vma->rb_root = RB_ROOT; in anon_vma_ctor()
425 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), in anon_vma_init()
453 struct anon_vma *page_get_anon_vma(struct page *page) in page_get_anon_vma()
455 struct anon_vma *anon_vma = NULL; in page_get_anon_vma() local
465 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); in page_get_anon_vma()
466 if (!atomic_inc_not_zero(&anon_vma->refcount)) { in page_get_anon_vma()
467 anon_vma = NULL; in page_get_anon_vma()
480 put_anon_vma(anon_vma); in page_get_anon_vma()
486 return anon_vma; in page_get_anon_vma()
496 struct anon_vma *page_lock_anon_vma_read(struct page *page) in page_lock_anon_vma_read()
498 struct anon_vma *anon_vma = NULL; in page_lock_anon_vma_read() local
499 struct anon_vma *root_anon_vma; in page_lock_anon_vma_read()
509 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); in page_lock_anon_vma_read()
510 root_anon_vma = READ_ONCE(anon_vma->root); in page_lock_anon_vma_read()
519 anon_vma = NULL; in page_lock_anon_vma_read()
525 if (!atomic_inc_not_zero(&anon_vma->refcount)) { in page_lock_anon_vma_read()
526 anon_vma = NULL; in page_lock_anon_vma_read()
532 put_anon_vma(anon_vma); in page_lock_anon_vma_read()
538 anon_vma_lock_read(anon_vma); in page_lock_anon_vma_read()
540 if (atomic_dec_and_test(&anon_vma->refcount)) { in page_lock_anon_vma_read()
546 anon_vma_unlock_read(anon_vma); in page_lock_anon_vma_read()
547 __put_anon_vma(anon_vma); in page_lock_anon_vma_read()
548 anon_vma = NULL; in page_lock_anon_vma_read()
551 return anon_vma; in page_lock_anon_vma_read()
555 return anon_vma; in page_lock_anon_vma_read()
558 void page_unlock_anon_vma_read(struct anon_vma *anon_vma) in page_unlock_anon_vma_read() argument
560 anon_vma_unlock_read(anon_vma); in page_unlock_anon_vma_read()
592 struct anon_vma *page__anon_vma = page_anon_vma(page); in page_address_in_vma()
597 if (!vma->anon_vma || !page__anon_vma || in page_address_in_vma()
598 vma->anon_vma->root != page__anon_vma->root) in page_address_in_vma()
946 struct anon_vma *anon_vma = vma->anon_vma; in page_move_anon_rmap() local
949 VM_BUG_ON_VMA(!anon_vma, vma); in page_move_anon_rmap()
952 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; in page_move_anon_rmap()
953 page->mapping = (struct address_space *) anon_vma; in page_move_anon_rmap()
966 struct anon_vma *anon_vma = vma->anon_vma; in __page_set_anon_rmap() local
968 BUG_ON(!anon_vma); in __page_set_anon_rmap()
979 anon_vma = anon_vma->root; in __page_set_anon_rmap()
981 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; in __page_set_anon_rmap()
982 page->mapping = (struct address_space *) anon_vma; in __page_set_anon_rmap()
1008 BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root); in __page_check_anon_rmap()
1420 void __put_anon_vma(struct anon_vma *anon_vma) in __put_anon_vma() argument
1422 struct anon_vma *root = anon_vma->root; in __put_anon_vma()
1424 anon_vma_free(anon_vma); in __put_anon_vma()
1425 if (root != anon_vma && atomic_dec_and_test(&root->refcount)) in __put_anon_vma()
1429 static struct anon_vma *rmap_walk_anon_lock(struct page *page, in rmap_walk_anon_lock()
1432 struct anon_vma *anon_vma; in rmap_walk_anon_lock() local
1443 anon_vma = page_anon_vma(page); in rmap_walk_anon_lock()
1444 if (!anon_vma) in rmap_walk_anon_lock()
1447 anon_vma_lock_read(anon_vma); in rmap_walk_anon_lock()
1448 return anon_vma; in rmap_walk_anon_lock()
1467 struct anon_vma *anon_vma; in rmap_walk_anon() local
1472 anon_vma = rmap_walk_anon_lock(page, rwc); in rmap_walk_anon()
1473 if (!anon_vma) in rmap_walk_anon()
1477 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) { in rmap_walk_anon()
1490 anon_vma_unlock_read(anon_vma); in rmap_walk_anon()
1564 struct anon_vma *anon_vma = vma->anon_vma; in __hugepage_set_anon_rmap() local
1566 BUG_ON(!anon_vma); in __hugepage_set_anon_rmap()
1571 anon_vma = anon_vma->root; in __hugepage_set_anon_rmap()
1573 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; in __hugepage_set_anon_rmap()
1574 page->mapping = (struct address_space *) anon_vma; in __hugepage_set_anon_rmap()
1581 struct anon_vma *anon_vma = vma->anon_vma; in hugepage_add_anon_rmap() local
1585 BUG_ON(!anon_vma); in hugepage_add_anon_rmap()