Lines Matching refs:ptep
42 #define update_mmu_cache(vma, address, ptep) do { } while (0) argument
43 #define update_mmu_cache_pmd(vma, address, ptep) do { } while (0) argument
641 static inline pgste_t pgste_get_lock(pte_t *ptep) in pgste_get_lock() argument
655 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) in pgste_get_lock()
656 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); in pgste_get_lock()
661 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) in pgste_set_unlock() argument
667 : "=Q" (ptep[PTRS_PER_PTE]) in pgste_set_unlock()
668 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) in pgste_set_unlock()
674 static inline pgste_t pgste_get(pte_t *ptep) in pgste_get() argument
678 pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); in pgste_get()
683 static inline void pgste_set(pte_t *ptep, pgste_t pgste) in pgste_set() argument
686 *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; in pgste_set()
690 static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste, in pgste_update_all() argument
696 if (!mm_use_skey(mm) || pte_val(*ptep) & _PAGE_INVALID) in pgste_update_all()
698 address = pte_val(*ptep) & PAGE_MASK; in pgste_update_all()
711 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, in pgste_set_key() argument
720 VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); in pgste_set_key()
733 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) in pgste_set_pte() argument
750 *ptep = entry; in pgste_set_pte()
811 pte_t *ptep, pgste_t pgste) in pgste_ipte_notify() argument
816 gmap_do_ipte_notify(mm, addr, ptep); in pgste_ipte_notify()
828 pte_t *ptep, pte_t entry) in set_pte_at() argument
833 pgste = pgste_get_lock(ptep); in set_pte_at()
835 pgste_set_key(ptep, pgste, entry, mm); in set_pte_at()
836 pgste = pgste_set_pte(ptep, pgste, entry); in set_pte_at()
837 pgste_set_unlock(ptep, pgste); in set_pte_at()
839 *ptep = entry; in set_pte_at()
889 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in pte_clear() argument
891 pte_val(*ptep) = _PAGE_INVALID; in pte_clear()
976 static inline void __ptep_ipte(unsigned long address, pte_t *ptep) in __ptep_ipte() argument
978 unsigned long pto = (unsigned long) ptep; in __ptep_ipte()
983 : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address)); in __ptep_ipte()
986 static inline void __ptep_ipte_local(unsigned long address, pte_t *ptep) in __ptep_ipte_local() argument
988 unsigned long pto = (unsigned long) ptep; in __ptep_ipte_local()
993 : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address)); in __ptep_ipte_local()
996 static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep) in __ptep_ipte_range() argument
998 unsigned long pto = (unsigned long) ptep; in __ptep_ipte_range()
1009 unsigned long address, pte_t *ptep) in ptep_flush_direct() argument
1013 if (pte_val(*ptep) & _PAGE_INVALID) in ptep_flush_direct()
1019 __ptep_ipte_local(address, ptep); in ptep_flush_direct()
1021 __ptep_ipte(address, ptep); in ptep_flush_direct()
1026 unsigned long address, pte_t *ptep) in ptep_flush_lazy() argument
1030 if (pte_val(*ptep) & _PAGE_INVALID) in ptep_flush_lazy()
1035 pte_val(*ptep) |= _PAGE_INVALID; in ptep_flush_lazy()
1038 __ptep_ipte(address, ptep); in ptep_flush_lazy()
1047 pte_t *ptep) in ptep_test_and_clear_user_dirty() argument
1055 pgste = pgste_get_lock(ptep); in ptep_test_and_clear_user_dirty()
1058 pte = *ptep; in ptep_test_and_clear_user_dirty()
1060 pgste = pgste_ipte_notify(mm, addr, ptep, pgste); in ptep_test_and_clear_user_dirty()
1061 __ptep_ipte(addr, ptep); in ptep_test_and_clear_user_dirty()
1066 *ptep = pte; in ptep_test_and_clear_user_dirty()
1068 pgste_set_unlock(ptep, pgste); in ptep_test_and_clear_user_dirty()
1074 unsigned long addr, pte_t *ptep) in ptep_test_and_clear_young() argument
1081 pgste = pgste_get_lock(ptep); in ptep_test_and_clear_young()
1082 pgste = pgste_ipte_notify(vma->vm_mm, addr, ptep, pgste); in ptep_test_and_clear_young()
1085 oldpte = pte = *ptep; in ptep_test_and_clear_young()
1086 ptep_flush_direct(vma->vm_mm, addr, ptep); in ptep_test_and_clear_young()
1092 pgste = pgste_set_pte(ptep, pgste, pte); in ptep_test_and_clear_young()
1093 pgste_set_unlock(ptep, pgste); in ptep_test_and_clear_young()
1095 *ptep = pte; in ptep_test_and_clear_young()
1102 unsigned long address, pte_t *ptep) in ptep_clear_flush_young() argument
1104 return ptep_test_and_clear_young(vma, address, ptep); in ptep_clear_flush_young()
1122 unsigned long address, pte_t *ptep) in ptep_get_and_clear() argument
1128 pgste = pgste_get_lock(ptep); in ptep_get_and_clear()
1129 pgste = pgste_ipte_notify(mm, address, ptep, pgste); in ptep_get_and_clear()
1132 pte = *ptep; in ptep_get_and_clear()
1133 ptep_flush_lazy(mm, address, ptep); in ptep_get_and_clear()
1134 pte_val(*ptep) = _PAGE_INVALID; in ptep_get_and_clear()
1138 pgste_set_unlock(ptep, pgste); in ptep_get_and_clear()
1146 pte_t *ptep) in ptep_modify_prot_start() argument
1152 pgste = pgste_get_lock(ptep); in ptep_modify_prot_start()
1153 pgste_ipte_notify(mm, address, ptep, pgste); in ptep_modify_prot_start()
1156 pte = *ptep; in ptep_modify_prot_start()
1157 ptep_flush_lazy(mm, address, ptep); in ptep_modify_prot_start()
1161 pgste_set(ptep, pgste); in ptep_modify_prot_start()
1168 pte_t *ptep, pte_t pte) in ptep_modify_prot_commit() argument
1173 pgste = pgste_get(ptep); in ptep_modify_prot_commit()
1174 pgste_set_key(ptep, pgste, pte, mm); in ptep_modify_prot_commit()
1175 pgste = pgste_set_pte(ptep, pgste, pte); in ptep_modify_prot_commit()
1176 pgste_set_unlock(ptep, pgste); in ptep_modify_prot_commit()
1178 *ptep = pte; in ptep_modify_prot_commit()
1183 unsigned long address, pte_t *ptep) in ptep_clear_flush() argument
1189 pgste = pgste_get_lock(ptep); in ptep_clear_flush()
1190 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste); in ptep_clear_flush()
1193 pte = *ptep; in ptep_clear_flush()
1194 ptep_flush_direct(vma->vm_mm, address, ptep); in ptep_clear_flush()
1195 pte_val(*ptep) = _PAGE_INVALID; in ptep_clear_flush()
1202 pgste_set_unlock(ptep, pgste); in ptep_clear_flush()
1217 pte_t *ptep, int full) in ptep_get_and_clear_full() argument
1223 pgste = pgste_get_lock(ptep); in ptep_get_and_clear_full()
1224 pgste = pgste_ipte_notify(mm, address, ptep, pgste); in ptep_get_and_clear_full()
1227 pte = *ptep; in ptep_get_and_clear_full()
1229 ptep_flush_lazy(mm, address, ptep); in ptep_get_and_clear_full()
1230 pte_val(*ptep) = _PAGE_INVALID; in ptep_get_and_clear_full()
1234 pgste_set_unlock(ptep, pgste); in ptep_get_and_clear_full()
1241 unsigned long address, pte_t *ptep) in ptep_set_wrprotect() argument
1244 pte_t pte = *ptep; in ptep_set_wrprotect()
1248 pgste = pgste_get_lock(ptep); in ptep_set_wrprotect()
1249 pgste = pgste_ipte_notify(mm, address, ptep, pgste); in ptep_set_wrprotect()
1252 ptep_flush_lazy(mm, address, ptep); in ptep_set_wrprotect()
1256 pgste = pgste_set_pte(ptep, pgste, pte); in ptep_set_wrprotect()
1257 pgste_set_unlock(ptep, pgste); in ptep_set_wrprotect()
1259 *ptep = pte; in ptep_set_wrprotect()
1266 unsigned long address, pte_t *ptep, in ptep_set_access_flags() argument
1272 oldpte = *ptep; in ptep_set_access_flags()
1276 pgste = pgste_get_lock(ptep); in ptep_set_access_flags()
1277 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste); in ptep_set_access_flags()
1280 ptep_flush_direct(vma->vm_mm, address, ptep); in ptep_set_access_flags()
1284 pgste_set_key(ptep, pgste, entry, vma->vm_mm); in ptep_set_access_flags()
1285 pgste = pgste_set_pte(ptep, pgste, entry); in ptep_set_access_flags()
1286 pgste_set_unlock(ptep, pgste); in ptep_set_access_flags()
1288 *ptep = entry; in ptep_set_access_flags()