Lines Matching refs:vma
445 struct vm_area_struct *vma; in mpol_rebind_mm() local
448 for (vma = mm->mmap; vma; vma = vma->vm_next) in mpol_rebind_mm()
449 mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE); in mpol_rebind_mm()
488 struct vm_area_struct *vma = walk->vma; in queue_pages_pte_range() local
496 split_huge_page_pmd(vma, addr, pmd); in queue_pages_pte_range()
504 page = vm_normal_page(vma, addr, *pte); in queue_pages_pte_range()
537 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); in queue_pages_hugetlb()
567 unsigned long change_prot_numa(struct vm_area_struct *vma, in change_prot_numa() argument
572 nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1); in change_prot_numa()
579 static unsigned long change_prot_numa(struct vm_area_struct *vma, in change_prot_numa() argument
589 struct vm_area_struct *vma = walk->vma; in queue_pages_test_walk() local
591 unsigned long endvma = vma->vm_end; in queue_pages_test_walk()
594 if (vma->vm_flags & VM_PFNMAP) in queue_pages_test_walk()
599 if (vma->vm_start > start) in queue_pages_test_walk()
600 start = vma->vm_start; in queue_pages_test_walk()
603 if (!vma->vm_next && vma->vm_end < end) in queue_pages_test_walk()
605 if (qp->prev && qp->prev->vm_end < vma->vm_start) in queue_pages_test_walk()
609 qp->prev = vma; in queue_pages_test_walk()
613 if (vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)) in queue_pages_test_walk()
614 change_prot_numa(vma, start, endvma); in queue_pages_test_walk()
620 vma_migratable(vma))) in queue_pages_test_walk()
659 static int vma_replace_policy(struct vm_area_struct *vma, in vma_replace_policy() argument
667 vma->vm_start, vma->vm_end, vma->vm_pgoff, in vma_replace_policy()
668 vma->vm_ops, vma->vm_file, in vma_replace_policy()
669 vma->vm_ops ? vma->vm_ops->set_policy : NULL); in vma_replace_policy()
675 if (vma->vm_ops && vma->vm_ops->set_policy) { in vma_replace_policy()
676 err = vma->vm_ops->set_policy(vma, new); in vma_replace_policy()
681 old = vma->vm_policy; in vma_replace_policy()
682 vma->vm_policy = new; /* protected by mmap_sem */ in vma_replace_policy()
697 struct vm_area_struct *vma; in mbind_range() local
703 vma = find_vma(mm, start); in mbind_range()
704 if (!vma || vma->vm_start > start) in mbind_range()
707 prev = vma->vm_prev; in mbind_range()
708 if (start > vma->vm_start) in mbind_range()
709 prev = vma; in mbind_range()
711 for (; vma && vma->vm_start < end; prev = vma, vma = next) { in mbind_range()
712 next = vma->vm_next; in mbind_range()
713 vmstart = max(start, vma->vm_start); in mbind_range()
714 vmend = min(end, vma->vm_end); in mbind_range()
716 if (mpol_equal(vma_policy(vma), new_pol)) in mbind_range()
719 pgoff = vma->vm_pgoff + in mbind_range()
720 ((vmstart - vma->vm_start) >> PAGE_SHIFT); in mbind_range()
721 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags, in mbind_range()
722 vma->anon_vma, vma->vm_file, pgoff, in mbind_range()
723 new_pol, vma->vm_userfaultfd_ctx); in mbind_range()
725 vma = prev; in mbind_range()
726 next = vma->vm_next; in mbind_range()
727 if (mpol_equal(vma_policy(vma), new_pol)) in mbind_range()
732 if (vma->vm_start != vmstart) { in mbind_range()
733 err = split_vma(vma->vm_mm, vma, vmstart, 1); in mbind_range()
737 if (vma->vm_end != vmend) { in mbind_range()
738 err = split_vma(vma->vm_mm, vma, vmend, 0); in mbind_range()
743 err = vma_replace_policy(vma, new_pol); in mbind_range()
835 struct vm_area_struct *vma = NULL; in do_get_mempolicy() local
859 vma = find_vma_intersection(mm, addr, addr+1); in do_get_mempolicy()
860 if (!vma) { in do_get_mempolicy()
864 if (vma->vm_ops && vma->vm_ops->get_policy) in do_get_mempolicy()
865 pol = vma->vm_ops->get_policy(vma, addr); in do_get_mempolicy()
867 pol = vma->vm_policy; in do_get_mempolicy()
897 if (vma) { in do_get_mempolicy()
899 vma = NULL; in do_get_mempolicy()
915 if (vma) in do_get_mempolicy()
1096 struct vm_area_struct *vma; in new_page() local
1099 vma = find_vma(current->mm, start); in new_page()
1100 while (vma) { in new_page()
1101 address = page_address_in_vma(page, vma); in new_page()
1104 vma = vma->vm_next; in new_page()
1108 BUG_ON(!vma); in new_page()
1109 return alloc_huge_page_noerr(vma, address, 1); in new_page()
1114 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); in new_page()
1541 struct mempolicy *__get_vma_policy(struct vm_area_struct *vma, in __get_vma_policy() argument
1546 if (vma) { in __get_vma_policy()
1547 if (vma->vm_ops && vma->vm_ops->get_policy) { in __get_vma_policy()
1548 pol = vma->vm_ops->get_policy(vma, addr); in __get_vma_policy()
1549 } else if (vma->vm_policy) { in __get_vma_policy()
1550 pol = vma->vm_policy; in __get_vma_policy()
1578 static struct mempolicy *get_vma_policy(struct vm_area_struct *vma, in get_vma_policy() argument
1581 struct mempolicy *pol = __get_vma_policy(vma, addr); in get_vma_policy()
1589 bool vma_policy_mof(struct vm_area_struct *vma) in vma_policy_mof() argument
1593 if (vma->vm_ops && vma->vm_ops->get_policy) { in vma_policy_mof()
1596 pol = vma->vm_ops->get_policy(vma, vma->vm_start); in vma_policy_mof()
1604 pol = vma->vm_policy; in vma_policy_mof()
1735 struct vm_area_struct *vma, unsigned long off) in offset_il_node() argument
1755 struct vm_area_struct *vma, unsigned long addr, int shift) in interleave_nid() argument
1757 if (vma) { in interleave_nid()
1768 off = vma->vm_pgoff >> (shift - PAGE_SHIFT); in interleave_nid()
1769 off += (addr - vma->vm_start) >> shift; in interleave_nid()
1770 return offset_il_node(pol, vma, off); in interleave_nid()
1806 struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr, in huge_zonelist() argument
1812 *mpol = get_vma_policy(vma, addr); in huge_zonelist()
1816 zl = node_zonelist(interleave_nid(*mpol, vma, addr, in huge_zonelist()
1817 huge_page_shift(hstate_vma(vma))), gfp_flags); in huge_zonelist()
1959 alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma, in alloc_pages_vma() argument
1969 pol = get_vma_policy(vma, addr); in alloc_pages_vma()
1975 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order); in alloc_pages_vma()
2247 int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr) in mpol_misplaced() argument
2258 BUG_ON(!vma); in mpol_misplaced()
2260 pol = get_vma_policy(vma, addr); in mpol_misplaced()
2266 BUG_ON(addr >= vma->vm_end); in mpol_misplaced()
2267 BUG_ON(addr < vma->vm_start); in mpol_misplaced()
2269 pgoff = vma->vm_pgoff; in mpol_misplaced()
2270 pgoff += (addr - vma->vm_start) >> PAGE_SHIFT; in mpol_misplaced()
2271 polnid = offset_il_node(pol, vma, pgoff); in mpol_misplaced()
2469 struct vm_area_struct *vma, struct mempolicy *npol) in mpol_set_shared_policy() argument
2473 unsigned long sz = vma_pages(vma); in mpol_set_shared_policy()
2476 vma->vm_pgoff, in mpol_set_shared_policy()
2482 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol); in mpol_set_shared_policy()
2486 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new); in mpol_set_shared_policy()