Lines Matching refs:ptep
42 #define update_mmu_cache(vma, address, ptep) do { } while (0) argument
43 #define update_mmu_cache_pmd(vma, address, ptep) do { } while (0) argument
579 static inline pgste_t pgste_get_lock(pte_t *ptep) in pgste_get_lock() argument
593 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) in pgste_get_lock()
594 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); in pgste_get_lock()
599 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) in pgste_set_unlock() argument
605 : "=Q" (ptep[PTRS_PER_PTE]) in pgste_set_unlock()
606 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) in pgste_set_unlock()
612 static inline pgste_t pgste_get(pte_t *ptep) in pgste_get() argument
616 pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); in pgste_get()
621 static inline void pgste_set(pte_t *ptep, pgste_t pgste) in pgste_set() argument
624 *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; in pgste_set()
628 static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste, in pgste_update_all() argument
634 if (!mm_use_skey(mm) || pte_val(*ptep) & _PAGE_INVALID) in pgste_update_all()
636 address = pte_val(*ptep) & PAGE_MASK; in pgste_update_all()
649 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, in pgste_set_key() argument
658 VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); in pgste_set_key()
671 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) in pgste_set_pte() argument
688 *ptep = entry; in pgste_set_pte()
749 pte_t *ptep, pgste_t pgste) in pgste_ipte_notify() argument
754 gmap_do_ipte_notify(mm, addr, ptep); in pgste_ipte_notify()
766 pte_t *ptep, pte_t entry) in set_pte_at() argument
771 pgste = pgste_get_lock(ptep); in set_pte_at()
773 pgste_set_key(ptep, pgste, entry, mm); in set_pte_at()
774 pgste = pgste_set_pte(ptep, pgste, entry); in set_pte_at()
775 pgste_set_unlock(ptep, pgste); in set_pte_at()
777 *ptep = entry; in set_pte_at()
827 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in pte_clear() argument
829 pte_val(*ptep) = _PAGE_INVALID; in pte_clear()
914 static inline void __ptep_ipte(unsigned long address, pte_t *ptep) in __ptep_ipte() argument
916 unsigned long pto = (unsigned long) ptep; in __ptep_ipte()
921 : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address)); in __ptep_ipte()
924 static inline void __ptep_ipte_local(unsigned long address, pte_t *ptep) in __ptep_ipte_local() argument
926 unsigned long pto = (unsigned long) ptep; in __ptep_ipte_local()
931 : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address)); in __ptep_ipte_local()
934 static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep) in __ptep_ipte_range() argument
936 unsigned long pto = (unsigned long) ptep; in __ptep_ipte_range()
947 unsigned long address, pte_t *ptep) in ptep_flush_direct() argument
951 if (pte_val(*ptep) & _PAGE_INVALID) in ptep_flush_direct()
957 __ptep_ipte_local(address, ptep); in ptep_flush_direct()
959 __ptep_ipte(address, ptep); in ptep_flush_direct()
964 unsigned long address, pte_t *ptep) in ptep_flush_lazy() argument
968 if (pte_val(*ptep) & _PAGE_INVALID) in ptep_flush_lazy()
973 pte_val(*ptep) |= _PAGE_INVALID; in ptep_flush_lazy()
976 __ptep_ipte(address, ptep); in ptep_flush_lazy()
985 pte_t *ptep) in ptep_test_and_clear_user_dirty() argument
993 pgste = pgste_get_lock(ptep); in ptep_test_and_clear_user_dirty()
996 pte = *ptep; in ptep_test_and_clear_user_dirty()
998 pgste = pgste_ipte_notify(mm, addr, ptep, pgste); in ptep_test_and_clear_user_dirty()
999 __ptep_ipte(addr, ptep); in ptep_test_and_clear_user_dirty()
1004 *ptep = pte; in ptep_test_and_clear_user_dirty()
1006 pgste_set_unlock(ptep, pgste); in ptep_test_and_clear_user_dirty()
1012 unsigned long addr, pte_t *ptep) in ptep_test_and_clear_young() argument
1019 pgste = pgste_get_lock(ptep); in ptep_test_and_clear_young()
1020 pgste = pgste_ipte_notify(vma->vm_mm, addr, ptep, pgste); in ptep_test_and_clear_young()
1023 oldpte = pte = *ptep; in ptep_test_and_clear_young()
1024 ptep_flush_direct(vma->vm_mm, addr, ptep); in ptep_test_and_clear_young()
1030 pgste = pgste_set_pte(ptep, pgste, pte); in ptep_test_and_clear_young()
1031 pgste_set_unlock(ptep, pgste); in ptep_test_and_clear_young()
1033 *ptep = pte; in ptep_test_and_clear_young()
1040 unsigned long address, pte_t *ptep) in ptep_clear_flush_young() argument
1042 return ptep_test_and_clear_young(vma, address, ptep); in ptep_clear_flush_young()
1060 unsigned long address, pte_t *ptep) in ptep_get_and_clear() argument
1066 pgste = pgste_get_lock(ptep); in ptep_get_and_clear()
1067 pgste = pgste_ipte_notify(mm, address, ptep, pgste); in ptep_get_and_clear()
1070 pte = *ptep; in ptep_get_and_clear()
1071 ptep_flush_lazy(mm, address, ptep); in ptep_get_and_clear()
1072 pte_val(*ptep) = _PAGE_INVALID; in ptep_get_and_clear()
1076 pgste_set_unlock(ptep, pgste); in ptep_get_and_clear()
1084 pte_t *ptep) in ptep_modify_prot_start() argument
1090 pgste = pgste_get_lock(ptep); in ptep_modify_prot_start()
1091 pgste_ipte_notify(mm, address, ptep, pgste); in ptep_modify_prot_start()
1094 pte = *ptep; in ptep_modify_prot_start()
1095 ptep_flush_lazy(mm, address, ptep); in ptep_modify_prot_start()
1099 pgste_set(ptep, pgste); in ptep_modify_prot_start()
1106 pte_t *ptep, pte_t pte) in ptep_modify_prot_commit() argument
1111 pgste = pgste_get(ptep); in ptep_modify_prot_commit()
1112 pgste_set_key(ptep, pgste, pte, mm); in ptep_modify_prot_commit()
1113 pgste = pgste_set_pte(ptep, pgste, pte); in ptep_modify_prot_commit()
1114 pgste_set_unlock(ptep, pgste); in ptep_modify_prot_commit()
1116 *ptep = pte; in ptep_modify_prot_commit()
1121 unsigned long address, pte_t *ptep) in ptep_clear_flush() argument
1127 pgste = pgste_get_lock(ptep); in ptep_clear_flush()
1128 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste); in ptep_clear_flush()
1131 pte = *ptep; in ptep_clear_flush()
1132 ptep_flush_direct(vma->vm_mm, address, ptep); in ptep_clear_flush()
1133 pte_val(*ptep) = _PAGE_INVALID; in ptep_clear_flush()
1140 pgste_set_unlock(ptep, pgste); in ptep_clear_flush()
1155 pte_t *ptep, int full) in ptep_get_and_clear_full() argument
1161 pgste = pgste_get_lock(ptep); in ptep_get_and_clear_full()
1162 pgste = pgste_ipte_notify(mm, address, ptep, pgste); in ptep_get_and_clear_full()
1165 pte = *ptep; in ptep_get_and_clear_full()
1167 ptep_flush_lazy(mm, address, ptep); in ptep_get_and_clear_full()
1168 pte_val(*ptep) = _PAGE_INVALID; in ptep_get_and_clear_full()
1172 pgste_set_unlock(ptep, pgste); in ptep_get_and_clear_full()
1179 unsigned long address, pte_t *ptep) in ptep_set_wrprotect() argument
1182 pte_t pte = *ptep; in ptep_set_wrprotect()
1186 pgste = pgste_get_lock(ptep); in ptep_set_wrprotect()
1187 pgste = pgste_ipte_notify(mm, address, ptep, pgste); in ptep_set_wrprotect()
1190 ptep_flush_lazy(mm, address, ptep); in ptep_set_wrprotect()
1194 pgste = pgste_set_pte(ptep, pgste, pte); in ptep_set_wrprotect()
1195 pgste_set_unlock(ptep, pgste); in ptep_set_wrprotect()
1197 *ptep = pte; in ptep_set_wrprotect()
1204 unsigned long address, pte_t *ptep, in ptep_set_access_flags() argument
1209 if (pte_same(*ptep, entry)) in ptep_set_access_flags()
1212 pgste = pgste_get_lock(ptep); in ptep_set_access_flags()
1213 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste); in ptep_set_access_flags()
1216 ptep_flush_direct(vma->vm_mm, address, ptep); in ptep_set_access_flags()
1219 pgste_set_key(ptep, pgste, entry, vma->vm_mm); in ptep_set_access_flags()
1220 pgste = pgste_set_pte(ptep, pgste, entry); in ptep_set_access_flags()
1221 pgste_set_unlock(ptep, pgste); in ptep_set_access_flags()
1223 *ptep = entry; in ptep_set_access_flags()