Lines Matching refs:pmd
174 static inline bool is_huge_zero_pmd(pmd_t pmd) in is_huge_zero_pmd() argument
176 return is_huge_zero_page(pmd_page(pmd)); in is_huge_zero_pmd()
702 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) in maybe_pmd_mkwrite() argument
705 pmd = pmd_mkwrite(pmd); in maybe_pmd_mkwrite()
706 return pmd; in maybe_pmd_mkwrite()
719 unsigned long haddr, pmd_t *pmd, in __do_huge_pmd_anonymous_page() argument
745 ptl = pmd_lock(mm, pmd); in __do_huge_pmd_anonymous_page()
746 if (unlikely(!pmd_none(*pmd))) { in __do_huge_pmd_anonymous_page()
758 pgtable_trans_huge_deposit(mm, pmd, pgtable); in __do_huge_pmd_anonymous_page()
759 set_pmd_at(mm, haddr, pmd, entry); in __do_huge_pmd_anonymous_page()
775 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, in set_huge_zero_page() argument
779 if (!pmd_none(*pmd)) in set_huge_zero_page()
783 pgtable_trans_huge_deposit(mm, pmd, pgtable); in set_huge_zero_page()
784 set_pmd_at(mm, haddr, pmd, entry); in set_huge_zero_page()
790 unsigned long address, pmd_t *pmd, in do_huge_pmd_anonymous_page() argument
818 ptl = pmd_lock(mm, pmd); in do_huge_pmd_anonymous_page()
819 set = set_huge_zero_page(pgtable, mm, vma, haddr, pmd, in do_huge_pmd_anonymous_page()
834 if (unlikely(__do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page, gfp))) { in do_huge_pmd_anonymous_page()
850 pmd_t pmd; in copy_huge_pmd() local
864 pmd = *src_pmd; in copy_huge_pmd()
865 if (unlikely(!pmd_trans_huge(pmd))) { in copy_huge_pmd()
874 if (is_huge_zero_pmd(pmd)) { in copy_huge_pmd()
890 if (unlikely(pmd_trans_splitting(pmd))) { in copy_huge_pmd()
899 src_page = pmd_page(pmd); in copy_huge_pmd()
906 pmd = pmd_mkold(pmd_wrprotect(pmd)); in copy_huge_pmd()
908 set_pmd_at(dst_mm, addr, dst_pmd, pmd); in copy_huge_pmd()
922 pmd_t *pmd, pmd_t orig_pmd, in huge_pmd_set_accessed() argument
929 ptl = pmd_lock(mm, pmd); in huge_pmd_set_accessed()
930 if (unlikely(!pmd_same(*pmd, orig_pmd))) in huge_pmd_set_accessed()
935 if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty)) in huge_pmd_set_accessed()
936 update_mmu_cache_pmd(vma, address, pmd); in huge_pmd_set_accessed()
976 pmd_t *pmd, pmd_t orig_pmd, in do_huge_pmd_wp_page_fallback() argument
1029 ptl = pmd_lock(mm, pmd); in do_huge_pmd_wp_page_fallback()
1030 if (unlikely(!pmd_same(*pmd, orig_pmd))) in do_huge_pmd_wp_page_fallback()
1034 pmdp_clear_flush_notify(vma, haddr, pmd); in do_huge_pmd_wp_page_fallback()
1037 pgtable = pgtable_trans_huge_withdraw(mm, pmd); in do_huge_pmd_wp_page_fallback()
1057 pmd_populate(mm, pmd, pgtable); in do_huge_pmd_wp_page_fallback()
1083 unsigned long address, pmd_t *pmd, pmd_t orig_pmd) in do_huge_pmd_wp_page() argument
1094 ptl = pmd_lockptr(mm, pmd); in do_huge_pmd_wp_page()
1100 if (unlikely(!pmd_same(*pmd, orig_pmd))) in do_huge_pmd_wp_page()
1109 if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1)) in do_huge_pmd_wp_page()
1110 update_mmu_cache_pmd(vma, address, pmd); in do_huge_pmd_wp_page()
1126 split_huge_page_pmd(vma, address, pmd); in do_huge_pmd_wp_page()
1130 pmd, orig_pmd, page, haddr); in do_huge_pmd_wp_page()
1147 split_huge_page_pmd(vma, address, pmd); in do_huge_pmd_wp_page()
1168 if (unlikely(!pmd_same(*pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1177 pmdp_clear_flush_notify(vma, haddr, pmd); in do_huge_pmd_wp_page()
1181 set_pmd_at(mm, haddr, pmd, entry); in do_huge_pmd_wp_page()
1182 update_mmu_cache_pmd(vma, address, pmd); in do_huge_pmd_wp_page()
1205 pmd_t *pmd, in follow_trans_huge_pmd() argument
1211 assert_spin_locked(pmd_lockptr(mm, pmd)); in follow_trans_huge_pmd()
1213 if (flags & FOLL_WRITE && !pmd_write(*pmd)) in follow_trans_huge_pmd()
1217 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) in follow_trans_huge_pmd()
1221 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) in follow_trans_huge_pmd()
1224 page = pmd_page(*pmd); in follow_trans_huge_pmd()
1236 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd)); in follow_trans_huge_pmd()
1238 pmd, _pmd, 1)) in follow_trans_huge_pmd()
1239 update_mmu_cache_pmd(vma, addr, pmd); in follow_trans_huge_pmd()
1260 unsigned long addr, pmd_t pmd, pmd_t *pmdp) in do_huge_pmd_numa_page() argument
1277 if (unlikely(!pmd_same(pmd, *pmdp))) in do_huge_pmd_numa_page()
1292 page = pmd_page(pmd); in do_huge_pmd_numa_page()
1336 if (unlikely(!pmd_same(pmd, *pmdp))) { in do_huge_pmd_numa_page()
1356 pmdp, pmd, addr, page, target_nid); in do_huge_pmd_numa_page()
1366 was_writable = pmd_write(pmd); in do_huge_pmd_numa_page()
1367 pmd = pmd_modify(pmd, vma->vm_page_prot); in do_huge_pmd_numa_page()
1368 pmd = pmd_mkyoung(pmd); in do_huge_pmd_numa_page()
1370 pmd = pmd_mkwrite(pmd); in do_huge_pmd_numa_page()
1371 set_pmd_at(mm, haddr, pmdp, pmd); in do_huge_pmd_numa_page()
1388 pmd_t *pmd, unsigned long addr) in zap_huge_pmd() argument
1393 if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) { in zap_huge_pmd()
1403 orig_pmd = pmdp_get_and_clear_full(tlb->mm, addr, pmd, in zap_huge_pmd()
1405 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in zap_huge_pmd()
1406 pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd); in zap_huge_pmd()
1434 pmd_t pmd; in move_huge_pmd() local
1462 pmd = pmdp_get_and_clear(mm, old_addr, old_pmd); in move_huge_pmd()
1470 set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd)); in move_huge_pmd()
1485 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, in change_huge_pmd() argument
1492 if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) { in change_huge_pmd()
1494 bool preserve_write = prot_numa && pmd_write(*pmd); in change_huge_pmd()
1502 if (prot_numa && is_huge_zero_pmd(*pmd)) { in change_huge_pmd()
1507 if (!prot_numa || !pmd_protnone(*pmd)) { in change_huge_pmd()
1508 entry = pmdp_get_and_clear_notify(mm, addr, pmd); in change_huge_pmd()
1513 set_pmd_at(mm, addr, pmd, entry); in change_huge_pmd()
1529 int __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma, in __pmd_trans_huge_lock() argument
1532 *ptl = pmd_lock(vma->vm_mm, pmd); in __pmd_trans_huge_lock()
1533 if (likely(pmd_trans_huge(*pmd))) { in __pmd_trans_huge_lock()
1534 if (unlikely(pmd_trans_splitting(*pmd))) { in __pmd_trans_huge_lock()
1536 wait_split_huge_page(vma->anon_vma, pmd); in __pmd_trans_huge_lock()
1564 pmd_t *pmd; in page_check_address_pmd() local
1575 pmd = pmd_offset(pud, address); in page_check_address_pmd()
1577 *ptl = pmd_lock(mm, pmd); in page_check_address_pmd()
1578 if (!pmd_present(*pmd)) in page_check_address_pmd()
1580 if (pmd_page(*pmd) != page) in page_check_address_pmd()
1590 pmd_trans_splitting(*pmd)) in page_check_address_pmd()
1592 if (pmd_trans_huge(*pmd)) { in page_check_address_pmd()
1594 !pmd_trans_splitting(*pmd)); in page_check_address_pmd()
1595 return pmd; in page_check_address_pmd()
1608 pmd_t *pmd; in __split_huge_page_splitting() local
1615 pmd = page_check_address_pmd(page, mm, address, in __split_huge_page_splitting()
1617 if (pmd) { in __split_huge_page_splitting()
1625 pmdp_splitting_flush(vma, address, pmd); in __split_huge_page_splitting()
1756 pmd_t *pmd, _pmd; in __split_huge_page_map() local
1761 pmd = page_check_address_pmd(page, mm, address, in __split_huge_page_map()
1763 if (pmd) { in __split_huge_page_map()
1764 pgtable = pgtable_trans_huge_withdraw(mm, pmd); in __split_huge_page_map()
1766 if (pmd_write(*pmd)) in __split_huge_page_map()
1780 if (!pmd_write(*pmd)) in __split_huge_page_map()
1782 if (!pmd_young(*pmd)) in __split_huge_page_map()
1817 pmdp_invalidate(vma, address, pmd); in __split_huge_page_map()
1818 pmd_populate(mm, pmd, pgtable); in __split_huge_page_map()
2432 pmd_t *pmd, _pmd; in collapse_huge_page() local
2477 pmd = mm_find_pmd(mm, address); in collapse_huge_page()
2478 if (!pmd) in collapse_huge_page()
2483 pte = pte_offset_map(pmd, address); in collapse_huge_page()
2484 pte_ptl = pte_lockptr(mm, pmd); in collapse_huge_page()
2489 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ in collapse_huge_page()
2496 _pmd = pmdp_clear_flush(vma, address, pmd); in collapse_huge_page()
2507 BUG_ON(!pmd_none(*pmd)); in collapse_huge_page()
2513 pmd_populate(mm, pmd, pmd_pgtable(_pmd)); in collapse_huge_page()
2541 BUG_ON(!pmd_none(*pmd)); in collapse_huge_page()
2545 pgtable_trans_huge_deposit(mm, pmd, pgtable); in collapse_huge_page()
2546 set_pmd_at(mm, address, pmd, _pmd); in collapse_huge_page()
2547 update_mmu_cache_pmd(vma, address, pmd); in collapse_huge_page()
2567 pmd_t *pmd; in khugepaged_scan_pmd() local
2578 pmd = mm_find_pmd(mm, address); in khugepaged_scan_pmd()
2579 if (!pmd) in khugepaged_scan_pmd()
2583 pte = pte_offset_map_lock(mm, pmd, address, &ptl); in khugepaged_scan_pmd()
2855 unsigned long haddr, pmd_t *pmd) in __split_huge_zero_page_pmd() argument
2862 pmdp_clear_flush_notify(vma, haddr, pmd); in __split_huge_zero_page_pmd()
2865 pgtable = pgtable_trans_huge_withdraw(mm, pmd); in __split_huge_zero_page_pmd()
2878 pmd_populate(mm, pmd, pgtable); in __split_huge_zero_page_pmd()
2883 pmd_t *pmd) in __split_huge_page_pmd() argument
2898 ptl = pmd_lock(mm, pmd); in __split_huge_page_pmd()
2899 if (unlikely(!pmd_trans_huge(*pmd))) { in __split_huge_page_pmd()
2904 if (is_huge_zero_pmd(*pmd)) { in __split_huge_page_pmd()
2905 __split_huge_zero_page_pmd(vma, haddr, pmd); in __split_huge_page_pmd()
2910 page = pmd_page(*pmd); in __split_huge_page_pmd()
2925 if (unlikely(pmd_trans_huge(*pmd))) in __split_huge_page_pmd()
2930 pmd_t *pmd) in split_huge_page_pmd_mm() argument
2936 split_huge_page_pmd(vma, address, pmd); in split_huge_page_pmd_mm()
2944 pmd_t *pmd; in split_huge_page_address() local
2956 pmd = pmd_offset(pud, address); in split_huge_page_address()
2957 if (!pmd_present(*pmd)) in split_huge_page_address()
2963 split_huge_page_pmd_mm(mm, address, pmd); in split_huge_page_address()