Home
last modified time | relevance | path

Searched refs:pmdp (Results 1 – 99 of 99) sorted by relevance

/linux-4.4.14/mm/
Dpgtable-generic.c103 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
106 int changed = !pmd_same(*pmdp, entry); in pmdp_set_access_flags()
109 set_pmd_at(vma->vm_mm, address, pmdp, entry); in pmdp_set_access_flags()
118 unsigned long address, pmd_t *pmdp) in pmdp_clear_flush_young() argument
122 young = pmdp_test_and_clear_young(vma, address, pmdp); in pmdp_clear_flush_young()
131 pmd_t *pmdp) in pmdp_huge_clear_flush() argument
135 VM_BUG_ON(!pmd_trans_huge(*pmdp)); in pmdp_huge_clear_flush()
136 pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); in pmdp_huge_clear_flush()
144 pmd_t *pmdp) in pmdp_splitting_flush() argument
146 pmd_t pmd = pmd_mksplitting(*pmdp); in pmdp_splitting_flush()
[all …]
Dgup.c1119 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, in gup_huge_pmd() argument
1145 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { in gup_huge_pmd()
1255 pmd_t *pmdp; in gup_pmd_range() local
1257 pmdp = pmd_offset(&pud, addr); in gup_pmd_range()
1259 pmd_t pmd = READ_ONCE(*pmdp); in gup_pmd_range()
1274 if (!gup_huge_pmd(pmd, pmdp, addr, next, write, in gup_pmd_range()
1288 } while (pmdp++, addr = next, addr != end); in gup_pmd_range()
Dhuge_memory.c1329 unsigned long addr, pmd_t pmd, pmd_t *pmdp) in do_huge_pmd_numa_page() argument
1345 ptl = pmd_lock(mm, pmdp); in do_huge_pmd_numa_page()
1346 if (unlikely(!pmd_same(pmd, *pmdp))) in do_huge_pmd_numa_page()
1354 if (unlikely(pmd_trans_migrating(*pmdp))) { in do_huge_pmd_numa_page()
1355 page = pmd_page(*pmdp); in do_huge_pmd_numa_page()
1405 if (unlikely(!pmd_same(pmd, *pmdp))) { in do_huge_pmd_numa_page()
1425 pmdp, pmd, addr, page, target_nid); in do_huge_pmd_numa_page()
1440 set_pmd_at(mm, haddr, pmdp, pmd); in do_huge_pmd_numa_page()
1441 update_mmu_cache_pmd(vma, addr, pmdp); in do_huge_pmd_numa_page()
/linux-4.4.14/arch/powerpc/mm/
Dpgtable_64.c95 pmd_t *pmdp; in map_kernel_page() local
103 pmdp = pmd_alloc(&init_mm, pudp, ea); in map_kernel_page()
104 if (!pmdp) in map_kernel_page()
106 ptep = pte_alloc_kernel(pmdp, ea); in map_kernel_page()
123 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); in map_kernel_page()
124 BUG_ON(pmdp == NULL); in map_kernel_page()
125 pud_populate(&init_mm, pudp, pmdp); in map_kernel_page()
127 pmdp = pmd_offset(pudp, ea); in map_kernel_page()
128 if (!pmd_present(*pmdp)) { in map_kernel_page()
131 pmd_populate_kernel(&init_mm, pmdp, ptep); in map_kernel_page()
[all …]
D40x_mmu.c109 pmd_t *pmdp; in mmu_mapin_ram() local
112 pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v); in mmu_mapin_ram()
113 pmd_val(*pmdp++) = val; in mmu_mapin_ram()
114 pmd_val(*pmdp++) = val; in mmu_mapin_ram()
115 pmd_val(*pmdp++) = val; in mmu_mapin_ram()
116 pmd_val(*pmdp++) = val; in mmu_mapin_ram()
124 pmd_t *pmdp; in mmu_mapin_ram() local
127 pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v); in mmu_mapin_ram()
128 pmd_val(*pmdp) = val; in mmu_mapin_ram()
Dhugepage-hash64.c22 pmd_t *pmdp, unsigned long trap, unsigned long flags, in __hash_page_thp() argument
36 pmd_t pmd = READ_ONCE(*pmdp); in __hash_page_thp()
55 } while (old_pmd != __cmpxchg_u64((unsigned long *)pmdp, in __hash_page_thp()
88 hpte_slot_array = get_hpte_slot_array(pmdp); in __hash_page_thp()
96 flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K, in __hash_page_thp()
170 *pmdp = __pmd(old_pmd); in __hash_page_thp()
194 *pmdp = __pmd(new_pmd & ~_PAGE_BUSY); in __hash_page_thp()
Dpgtable_32.c367 get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) in get_pteptr() argument
385 if (pmdp) in get_pteptr()
386 *pmdp = pmd; in get_pteptr()
Dhugetlbpage.c983 pmd_t pmd, *pmdp; in __find_linux_pte_or_hugepte() local
1028 pmdp = pmd_offset(&pud, ea); in __find_linux_pte_or_hugepte()
1029 pmd = READ_ONCE(*pmdp); in __find_linux_pte_or_hugepte()
1044 ret_pte = (pte_t *) pmdp; in __find_linux_pte_or_hugepte()
1049 ret_pte = (pte_t *) pmdp; in __find_linux_pte_or_hugepte()
Dhash_utils_64.c1333 pmd_t *pmdp, unsigned int psize, int ssize, in flush_hash_hugepage() argument
1343 hpte_slot_array = get_hpte_slot_array(pmdp); in flush_hash_hugepage()
/linux-4.4.14/include/asm-generic/
Dpgtable.h35 unsigned long address, pmd_t *pmdp,
39 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
67 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
69 pmd_t pmd = *pmdp; in pmdp_test_and_clear_young()
74 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); in pmdp_test_and_clear_young()
80 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
96 unsigned long address, pmd_t *pmdp);
103 unsigned long address, pmd_t *pmdp) in pmdp_clear_flush_young() argument
126 pmd_t *pmdp) in pmdp_huge_get_and_clear() argument
128 pmd_t pmd = *pmdp; in pmdp_huge_get_and_clear()
[all …]
Dtlb.h190 #define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0) argument
193 #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \ argument
196 __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
213 #define pmd_free_tlb(tlb, pmdp, address) \ argument
216 __pmd_free_tlb(tlb, pmdp, address); \
/linux-4.4.14/arch/sparc/mm/
Dtlb.c167 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
169 pmd_t orig = *pmdp; in set_pmd_at()
171 *pmdp = pmd; in set_pmd_at()
208 pmd_t *pmdp) in pmdp_invalidate() argument
210 pmd_t entry = *pmdp; in pmdp_invalidate()
214 set_pmd_at(vma->vm_mm, address, pmdp, entry); in pmdp_invalidate()
218 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
226 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit()
229 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); in pgtable_trans_huge_deposit()
230 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
[all …]
Dsrmmu.c112 void pmd_set(pmd_t *pmdp, pte_t *ptep) in pmd_set() argument
119 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); in pmd_set()
124 void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep) in pmd_populate() argument
131 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); in pmd_populate()
486 pmd_t *pmdp; in srmmu_mapioaddr() local
492 pmdp = pmd_offset(pgdp, virt_addr); in srmmu_mapioaddr()
493 ptep = pte_offset_kernel(pmdp, virt_addr); in srmmu_mapioaddr()
521 pmd_t *pmdp; in srmmu_unmapioaddr() local
525 pmdp = pmd_offset(pgdp, virt_addr); in srmmu_unmapioaddr()
526 ptep = pte_offset_kernel(pmdp, virt_addr); in srmmu_unmapioaddr()
[all …]
Dgup.c69 static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, in gup_huge_pmd() argument
99 if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) { in gup_huge_pmd()
122 pmd_t *pmdp; in gup_pmd_range() local
124 pmdp = pmd_offset(&pud, addr); in gup_pmd_range()
126 pmd_t pmd = *pmdp; in gup_pmd_range()
132 if (!gup_huge_pmd(pmdp, pmd, addr, next, in gup_pmd_range()
138 } while (pmdp++, addr = next, addr != end); in gup_pmd_range()
Dfault_64.c96 pmd_t *pmdp; in get_user_insn() local
110 pmdp = pmd_offset(pudp, tpc); in get_user_insn()
111 if (pmd_none(*pmdp) || unlikely(pmd_bad(*pmdp))) in get_user_insn()
115 if (pmd_trans_huge(*pmdp)) { in get_user_insn()
116 if (pmd_trans_splitting(*pmdp)) in get_user_insn()
119 pa = pmd_pfn(*pmdp) << PAGE_SHIFT; in get_user_insn()
129 ptep = pte_offset_map(pmdp, tpc); in get_user_insn()
Dio-unit.c218 pmd_t *pmdp; in iounit_map_dma_area() local
223 pmdp = pmd_offset(pgdp, addr); in iounit_map_dma_area()
224 ptep = pte_offset_map(pmdp, addr); in iounit_map_dma_area()
Diommu.c350 pmd_t *pmdp; in iommu_map_dma_area() local
361 pmdp = pmd_offset(pgdp, addr); in iommu_map_dma_area()
362 ptep = pte_offset_map(pmdp, addr); in iommu_map_dma_area()
Dinit_64.c419 pmd_t *pmdp = pmd_offset(pudp, kaddr); in flush_icache_range() local
420 pte_t *ptep = pte_offset_kernel(pmdp, kaddr); in flush_icache_range()
/linux-4.4.14/arch/powerpc/include/asm/
Dpgtable-ppc64.h151 #define pmd_set(pmdp, pmdval) (pmd_val(*(pmdp)) = (pmdval)) argument
156 #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0) argument
426 static inline char *get_hpte_slot_array(pmd_t *pmdp) in get_hpte_slot_array() argument
435 return *(char **)(pmdp + PTRS_PER_PMD); in get_hpte_slot_array()
442 pmd_t *pmdp, unsigned long old_pmd);
447 pmd_t *pmdp, pmd_t pmd);
484 unsigned long addr, pmd_t *pmdp, in hpte_do_hugepage_flush() argument
553 unsigned long address, pmd_t *pmdp,
558 pmd_t *pmdp,
563 unsigned long addr, pmd_t *pmdp) in __pmdp_test_and_clear_young() argument
[all …]
Dpgtable-ppc32.h133 #define pmd_clear(pmdp) do { pmd_val(*(pmdp)) = 0; } while (0) argument
336 pmd_t **pmdp);
Dtlbflush.h131 pmd_t *pmdp, unsigned int psize, int ssize,
Dmmu-hash64.h342 unsigned long vsid, pmd_t *pmdp, unsigned long trap,
346 unsigned long vsid, pmd_t *pmdp, in __hash_page_thp() argument
/linux-4.4.14/arch/m68k/include/asm/
Dmotorola_pgtable.h108 static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) in pmd_set() argument
111 unsigned long *ptr = pmdp->pmd; in pmd_set()
119 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument
121 pgd_val(*pgdp) = _PAGE_TABLE | _PAGE_ACCESSED | __pa(pmdp); in pgd_set()
140 #define pmd_clear(pmdp) ({ \ argument
141 unsigned long *__ptr = pmdp->pmd; \
217 static inline pte_t *pte_offset_kernel(pmd_t *pmdp, unsigned long address) in pte_offset_kernel() argument
219 return (pte_t *)__pmd_page(*pmdp) + ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); in pte_offset_kernel()
222 #define pte_offset_map(pmdp,address) ((pte_t *)__pmd_page(*pmdp) + (((address) >> PAGE_SHIFT) & (PT… argument
241 pmd_t *pmdp; in nocache_page() local
[all …]
Dmcf_pgtable.h164 #define pmd_set(pmdp, ptep) do {} while (0) argument
166 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument
168 pgd_val(*pgdp) = virt_to_phys(pmdp); in pgd_set()
198 static inline void pmd_clear(pmd_t *pmdp) { pmd_val(*pmdp) = 0; } in pmd_clear() argument
362 pmd_t *pmdp; in nocache_page() local
367 pmdp = pmd_offset(dir, addr); in nocache_page()
368 ptep = pte_offset_kernel(pmdp, addr); in nocache_page()
378 pmd_t *pmdp; in cache_page() local
383 pmdp = pmd_offset(dir, addr); in cache_page()
384 ptep = pte_offset_kernel(pmdp, addr); in cache_page()
[all …]
Dsun3_pgtable.h110 #define pmd_set(pmdp,ptep) do {} while (0) argument
112 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument
114 pgd_val(*pgdp) = virt_to_phys(pmdp); in pgd_set()
145 static inline void pmd_clear (pmd_t *pmdp) { pmd_val (*pmdp) = 0; } in pmd_clear() argument
/linux-4.4.14/arch/s390/include/asm/
Dpgtable.h528 unsigned long addr, pmd_t *pmdp);
532 unsigned long address, pmd_t *pmdp,
537 unsigned long address, pmd_t *pmdp);
884 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
886 pmd_val(*pmdp) = _SEGMENT_ENTRY_INVALID; in pmd_clear()
1450 static inline void __pmdp_csp(pmd_t *pmdp) in __pmdp_csp() argument
1452 register unsigned long reg2 asm("2") = pmd_val(*pmdp); in __pmdp_csp()
1453 register unsigned long reg3 asm("3") = pmd_val(*pmdp) | in __pmdp_csp()
1455 register unsigned long reg4 asm("4") = ((unsigned long) pmdp) + 5; in __pmdp_csp()
1459 : "=m" (*pmdp) in __pmdp_csp()
[all …]
Dtlb.h145 #define tlb_remove_pmd_tlb_entry(tlb, pmdp, addr) do { } while (0) argument
/linux-4.4.14/arch/s390/mm/
Dhugetlbpage.c107 pmd_t *pmdp = (pmd_t *) ptep; in huge_ptep_get_and_clear() local
110 pmdp_flush_direct(mm, addr, pmdp); in huge_ptep_get_and_clear()
111 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; in huge_ptep_get_and_clear()
120 pmd_t *pmdp = NULL; in huge_pte_alloc() local
125 pmdp = pmd_alloc(mm, pudp, addr); in huge_pte_alloc()
126 return (pte_t *) pmdp; in huge_pte_alloc()
133 pmd_t *pmdp = NULL; in huge_pte_offset() local
139 pmdp = pmd_offset(pudp, addr); in huge_pte_offset()
141 return (pte_t *) pmdp; in huge_pte_offset()
Dgup.c20 static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr, in gup_pte_range() argument
54 static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, in gup_huge_pmd() argument
84 if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) { in gup_huge_pmd()
109 pmd_t *pmdp, pmd; in gup_pmd_range() local
111 pmdp = (pmd_t *) pudp; in gup_pmd_range()
113 pmdp = (pmd_t *) pud_deref(pud); in gup_pmd_range()
114 pmdp += pmd_index(addr); in gup_pmd_range()
116 pmd = *pmdp; in gup_pmd_range()
138 if (!gup_huge_pmd(pmdp, pmd, addr, next, in gup_pmd_range()
141 } else if (!gup_pte_range(pmdp, pmd, addr, next, in gup_pmd_range()
[all …]
Dpageattr.c47 pmd_t *pmdp; in walk_page_table() local
56 pmdp = pmd_offset(pudp, addr); in walk_page_table()
57 if (pmd_none(*pmdp) || pmd_large(*pmdp)) in walk_page_table()
59 ptep = pte_offset_kernel(pmdp, addr); in walk_page_table()
Dpgtable.c1258 pmd_t *pmdp) in pmdp_clear_flush_young() argument
1263 return pmdp_test_and_clear_young(vma, address, pmdp); in pmdp_clear_flush_young()
1267 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
1275 if (pmd_same(*pmdp, entry)) in pmdp_set_access_flags()
1277 pmdp_invalidate(vma, address, pmdp); in pmdp_set_access_flags()
1278 set_pmd_at(vma->vm_mm, address, pmdp, entry); in pmdp_set_access_flags()
1288 pmd_t *pmdp) in pmdp_splitting_flush() argument
1292 (unsigned long *) pmdp)) { in pmdp_splitting_flush()
1298 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
1303 assert_spin_locked(pmd_lockptr(mm, pmdp)); in pgtable_trans_huge_deposit()
[all …]
/linux-4.4.14/arch/unicore32/include/asm/
Dpgalloc.h82 static inline void __pmd_populate(pmd_t *pmdp, unsigned long pmdval) in __pmd_populate() argument
84 set_pmd(pmdp, __pmd(pmdval)); in __pmd_populate()
85 flush_pmd_entry(pmdp); in __pmd_populate()
93 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
101 __pmd_populate(pmdp, __pa(pte_ptr) | _PAGE_KERNEL_TABLE); in pmd_populate_kernel()
105 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
107 __pmd_populate(pmdp, in pmd_populate()
Dpgtable.h217 #define pmd_clear(pmdp) \ argument
219 set_pmd(pmdp, __pmd(0));\
220 clean_pmd_entry(pmdp); \
/linux-4.4.14/arch/arm/include/asm/
Dpgalloc.h131 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, in __pmd_populate() argument
135 pmdp[0] = __pmd(pmdval); in __pmd_populate()
137 pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); in __pmd_populate()
139 flush_pmd_entry(pmdp); in __pmd_populate()
149 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
154 __pmd_populate(pmdp, __pa(ptep), _PAGE_KERNEL_TABLE); in pmd_populate_kernel()
158 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
168 __pmd_populate(pmdp, page_to_phys(ptep), prot); in pmd_populate()
Dpgtable-2level.h204 #define pmd_clear(pmdp) \ argument
206 pmdp[0] = __pmd(0); \
207 pmdp[1] = __pmd(0); \
208 clean_pmd_entry(pmdp); \
Dpgtable-3level.h183 #define pmd_clear(pmdp) \ argument
185 *pmdp = __pmd(0); \
186 clean_pmd_entry(pmdp); \
240 pmd_t *pmdp);
275 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
288 *pmdp = __pmd(pmd_val(pmd) | PMD_SECT_nG); in set_pmd_at()
289 flush_pmd_entry(pmdp); in set_pmd_at()
Dtlb.h245 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, in __pmd_free_tlb() argument
250 tlb_remove_entry(tlb, virt_to_page(pmdp)); in __pmd_free_tlb()
255 tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) in tlb_remove_pmd_tlb_entry() argument
261 #define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr) argument
Dkvm_mmu.h161 #define kvm_pmd_table_empty(kvm, pmdp) kvm_page_empty(pmdp) argument
/linux-4.4.14/arch/mips/mm/
Dpgtable-64.c69 pmd_t *pmdp) in pmdp_splitting_flush() argument
71 if (!pmd_trans_splitting(*pmdp)) { in pmdp_splitting_flush()
72 pmd_t pmd = pmd_mksplitting(*pmdp); in pmdp_splitting_flush()
73 set_pmd_at(vma->vm_mm, address, pmdp, pmd); in pmdp_splitting_flush()
89 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
91 *pmdp = pmd; in set_pmd_at()
Dtlb-r8k.c187 pmd_t *pmdp; in __update_tlb() local
204 pmdp = pmd_offset(pgdp, address); in __update_tlb()
205 ptep = pte_offset_map(pmdp, address); in __update_tlb()
Dgup.c105 pmd_t *pmdp; in gup_pmd_range() local
107 pmdp = pmd_offset(&pud, addr); in gup_pmd_range()
109 pmd_t pmd = *pmdp; in gup_pmd_range()
132 } while (pmdp++, addr = next, addr != end); in gup_pmd_range()
Dtlb-r4k.c290 pmd_t *pmdp; in __update_tlb() local
311 pmdp = pmd_offset(pudp, address); in __update_tlb()
315 if (pmd_huge(*pmdp)) { in __update_tlb()
318 ptep = (pte_t *)pmdp; in __update_tlb()
333 ptep = pte_offset_map(pmdp, address); in __update_tlb()
Dc-r3k.c244 pmd_t *pmdp; in r3k_flush_cache_page() local
256 pmdp = pmd_offset(pudp, addr); in r3k_flush_cache_page()
257 ptep = pte_offset(pmdp, addr); in r3k_flush_cache_page()
Dc-tx39.c173 pmd_t *pmdp; in tx39_flush_cache_page() local
186 pmdp = pmd_offset(pudp, page); in tx39_flush_cache_page()
187 ptep = pte_offset(pmdp, page); in tx39_flush_cache_page()
Dc-r4k.c558 pmd_t *pmdp; in local_r4k_flush_cache_page() local
572 pmdp = pmd_offset(pudp, addr); in local_r4k_flush_cache_page()
573 ptep = pte_offset(pmdp, addr); in local_r4k_flush_cache_page()
/linux-4.4.14/arch/arm64/include/asm/
Dpgalloc.h111 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, in __pmd_populate() argument
114 set_pmd(pmdp, __pmd(pte | prot)); in __pmd_populate()
122 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
127 __pmd_populate(pmdp, __pa(ptep), PMD_TYPE_TABLE); in pmd_populate_kernel()
131 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
133 __pmd_populate(pmdp, page_to_phys(ptep), PMD_TYPE_TABLE); in pmd_populate()
Dpgtable.h346 pmd_t *pmdp);
373 #define set_pmd_at(mm, addr, pmdp, pmd) set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd)) argument
416 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) in set_pmd() argument
418 *pmdp = pmd; in set_pmd()
423 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
425 set_pmd(pmdp, __pmd(0)); in pmd_clear()
549 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
552 return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty); in pmdp_set_access_flags()
584 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
586 return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp); in pmdp_test_and_clear_young()
[all …]
Dkvm_mmu.h105 #define kvm_set_pmd(pmdp, pmd) set_pmd(pmdp, pmd)
210 #define kvm_pmd_table_empty(kvm, pmdp) (0)
212 #define kvm_pmd_table_empty(kvm, pmdp) \
213 (kvm_page_empty(pmdp) && (!(kvm) || KVM_PREALLOC_LEVEL < 2))
Dtlb.h66 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, in __pmd_free_tlb() argument
70 tlb_remove_entry(tlb, virt_to_page(pmdp)); in __pmd_free_tlb()
/linux-4.4.14/arch/tile/include/asm/
Dpgalloc.h41 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) in set_pmd() argument
44 set_pte(pmdp, pmd); in set_pmd()
46 set_pte(&pmdp->pud.pgd, pmd.pud.pgd); in set_pmd()
150 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmdp) in pmd_free() argument
152 pgtable_free(mm, virt_to_page(pmdp), L1_USER_PGTABLE_ORDER); in pmd_free()
155 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, in __pmd_free_tlb() argument
158 __pgtable_free_tlb(tlb, virt_to_page(pmdp), address, in __pmd_free_tlb()
Dpgtable.h404 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
406 return ptep_test_and_clear_young(vma, address, pmdp_ptep(pmdp)); in pmdp_test_and_clear_young()
411 unsigned long address, pmd_t *pmdp) in pmdp_set_wrprotect() argument
413 ptep_set_wrprotect(mm, address, pmdp_ptep(pmdp)); in pmdp_set_wrprotect()
420 pmd_t *pmdp) in pmdp_huge_get_and_clear() argument
422 return pte_pmd(ptep_get_and_clear(mm, address, pmdp_ptep(pmdp))); in pmdp_huge_get_and_clear()
425 static inline void __set_pmd(pmd_t *pmdp, pmd_t pmdval) in __set_pmd() argument
427 set_pte(pmdp_ptep(pmdp), pmd_pte(pmdval)); in __set_pmd()
430 #define set_pmd_at(mm, addr, pmdp, pmdval) __set_pmd(pmdp, pmdval) argument
463 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
[all …]
Dpgtable_32.h116 #define pmdp_ptep(pmdp) (&(pmdp)->pud.pgd) argument
Dpgtable_64.h68 #define pmdp_ptep(pmdp) (pmdp) argument
/linux-4.4.14/arch/x86/include/asm/
Dpgtable.h48 #define set_pmd_at(mm, addr, pmdp, pmd) native_set_pmd_at(mm, addr, pmdp, pmd) argument
53 #define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd) argument
706 static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp) in native_local_pmdp_get_and_clear() argument
708 pmd_t res = *pmdp; in native_local_pmdp_get_and_clear()
710 native_pmd_clear(pmdp); in native_local_pmdp_get_and_clear()
721 pmd_t *pmdp , pmd_t pmd) in native_set_pmd_at() argument
723 native_set_pmd(pmdp, pmd); in native_set_pmd_at()
807 unsigned long address, pmd_t *pmdp,
812 unsigned long addr, pmd_t *pmdp);
816 unsigned long address, pmd_t *pmdp);
[all …]
Dpgtable-3level.h70 static inline pmd_t pmd_read_atomic(pmd_t *pmdp) in pmd_read_atomic() argument
73 u32 *tmp = (u32 *)pmdp; in pmd_read_atomic()
93 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) in native_set_pmd() argument
95 set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd)); in native_set_pmd()
164 static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp) in native_pmdp_get_and_clear() argument
166 union split_pmd res, *orig = (union split_pmd *)pmdp; in native_pmdp_get_and_clear()
Dpgtable-2level.h19 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) in native_set_pmd() argument
21 *pmdp = pmd; in native_set_pmd()
29 static inline void native_pmd_clear(pmd_t *pmdp) in native_pmd_clear() argument
31 native_set_pmd(pmdp, __pmd(0)); in native_pmd_clear()
Dparavirt.h385 pmd_t *pmdp) in pmd_update() argument
387 PVOP_VCALL3(pv_mmu_ops.pmd_update, mm, addr, pmdp); in pmd_update()
397 pmd_t *pmdp) in pmd_update_defer() argument
399 PVOP_VCALL3(pv_mmu_ops.pmd_update_defer, mm, addr, pmdp); in pmd_update_defer()
504 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
508 pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd); in set_pmd_at()
510 PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp, in set_pmd_at()
514 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) in set_pmd() argument
519 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32); in set_pmd()
521 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val); in set_pmd()
[all …]
Dpgtable_64.h63 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) in native_set_pmd() argument
65 *pmdp = pmd; in native_set_pmd()
Dparavirt_types.h277 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
279 pmd_t *pmdp, pmd_t pmdval);
285 pmd_t *pmdp);
287 unsigned long addr, pmd_t *pmdp);
305 void (*pmd_clear)(pmd_t *pmdp);
/linux-4.4.14/arch/x86/mm/
Dpgtable.c425 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
428 int changed = !pmd_same(*pmdp, entry); in pmdp_set_access_flags()
433 *pmdp = entry; in pmdp_set_access_flags()
434 pmd_update_defer(vma->vm_mm, address, pmdp); in pmdp_set_access_flags()
464 unsigned long addr, pmd_t *pmdp) in pmdp_test_and_clear_young() argument
468 if (pmd_young(*pmdp)) in pmdp_test_and_clear_young()
470 (unsigned long *)pmdp); in pmdp_test_and_clear_young()
473 pmd_update(vma->vm_mm, addr, pmdp); in pmdp_test_and_clear_young()
500 unsigned long address, pmd_t *pmdp) in pmdp_clear_flush_young() argument
506 young = pmdp_test_and_clear_young(vma, address, pmdp); in pmdp_clear_flush_young()
[all …]
Dgup.c154 pmd_t *pmdp; in gup_pmd_range() local
156 pmdp = pmd_offset(&pud, addr); in gup_pmd_range()
158 pmd_t pmd = *pmdp; in gup_pmd_range()
188 } while (pmdp++, addr = next, addr != end); in gup_pmd_range()
/linux-4.4.14/arch/xtensa/include/asm/
Dpgalloc.h24 #define pmd_populate_kernel(mm, pmdp, ptep) \ argument
25 (pmd_val(*(pmdp)) = ((unsigned long)ptep))
26 #define pmd_populate(mm, pmdp, page) \ argument
27 (pmd_val(*(pmdp)) = ((unsigned long)page_to_virt(page)))
Dpgtable.h258 #define pmd_clear(pmdp) do { set_pmd(pmdp, __pmd(0)); } while (0) argument
326 set_pmd(pmd_t *pmdp, pmd_t pmdval) in set_pmd() argument
328 *pmdp = pmdval; in set_pmd()
/linux-4.4.14/arch/sparc/include/asm/
Dpgalloc_32.h30 static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) in pgd_set() argument
32 unsigned long pa = __nocache_pa((unsigned long)pmdp); in pgd_set()
54 void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep);
57 void pmd_set(pmd_t *pmdp, pte_t *ptep);
Dpgtable_64.h796 pmd_t *pmdp, pmd_t pmd);
799 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
801 *pmdp = pmd; in set_pmd_at()
805 static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_set() argument
809 pmd_val(*pmdp) = val; in pmd_set()
812 #define pud_set(pudp, pmdp) \ argument
813 (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp))))
827 #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) argument
900 pmd_t *pmdp) in pmdp_huge_get_and_clear() argument
902 pmd_t pmd = *pmdp; in pmdp_huge_get_and_clear()
[all …]
Dpgtable_32.h179 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
183 set_pte((pte_t *)&pmdp->pmdv[i], __pte(0)); in pmd_clear()
/linux-4.4.14/arch/arc/include/asm/
Dhugepage.h59 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
61 *pmdp = pmd; in set_pmd_at()
71 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
75 extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
Dpgtable.h260 static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) in pmd_set() argument
262 pmd_val(*pmdp) = (unsigned long)ptep; in pmd_set()
/linux-4.4.14/arch/score/mm/
Dcache.c179 pmd_t *pmdp; in flush_cache_range() local
187 pmdp = pmd_offset(pudp, start); in flush_cache_range()
188 ptep = pte_offset(pmdp, start); in flush_cache_range()
194 pmdp = pmd_offset(pudp, start); in flush_cache_range()
195 ptep = pte_offset(pmdp, start); in flush_cache_range()
/linux-4.4.14/arch/arm64/mm/
Dflush.c106 pmd_t *pmdp) in pmdp_splitting_flush() argument
108 pmd_t pmd = pmd_mksplitting(*pmdp); in pmdp_splitting_flush()
111 set_pmd_at(vma->vm_mm, address, pmdp, pmd); in pmdp_splitting_flush()
/linux-4.4.14/include/trace/events/
Dxen.h203 TP_PROTO(pmd_t *pmdp, pmd_t pmdval),
204 TP_ARGS(pmdp, pmdval),
206 __field(pmd_t *, pmdp)
209 TP_fast_assign(__entry->pmdp = pmdp;
212 __entry->pmdp,
218 TP_PROTO(pmd_t *pmdp),
219 TP_ARGS(pmdp),
221 __field(pmd_t *, pmdp)
223 TP_fast_assign(__entry->pmdp = pmdp),
224 TP_printk("pmdp %p", __entry->pmdp)
/linux-4.4.14/arch/alpha/include/asm/
Dpgtable.h225 extern inline void pmd_set(pmd_t * pmdp, pte_t * ptep) in pmd_set() argument
226 { pmd_val(*pmdp) = _PAGE_TABLE | ((((unsigned long) ptep) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } in pmd_set()
228 extern inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) in pgd_set() argument
229 { pgd_val(*pgdp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } in pgd_set()
256 extern inline void pmd_clear(pmd_t * pmdp) { pmd_val(*pmdp) = 0; } in pmd_clear() argument
/linux-4.4.14/arch/sh/mm/
Dgup.c123 pmd_t *pmdp; in gup_pmd_range() local
125 pmdp = pmd_offset(&pud, addr); in gup_pmd_range()
127 pmd_t pmd = *pmdp; in gup_pmd_range()
134 } while (pmdp++, addr = next, addr != end); in gup_pmd_range()
/linux-4.4.14/arch/x86/lguest/
Dboot.c708 static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) in lguest_set_pmd() argument
710 native_set_pmd(pmdp, pmdval); in lguest_set_pmd()
711 lazy_hcall2(LHCALL_SET_PMD, __pa(pmdp) & PAGE_MASK, in lguest_set_pmd()
712 (__pa(pmdp) & (PAGE_SIZE - 1)) / sizeof(pmd_t)); in lguest_set_pmd()
717 static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) in lguest_set_pmd() argument
719 native_set_pmd(pmdp, pmdval); in lguest_set_pmd()
720 lazy_hcall2(LHCALL_SET_PGD, __pa(pmdp) & PAGE_MASK, in lguest_set_pmd()
721 (__pa(pmdp) & (PAGE_SIZE - 1)) / sizeof(pmd_t)); in lguest_set_pmd()
763 static void lguest_pmd_clear(pmd_t *pmdp) in lguest_pmd_clear() argument
765 lguest_set_pmd(pmdp, __pmd(0)); in lguest_pmd_clear()
/linux-4.4.14/arch/arc/mm/
Dtlb.c662 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
670 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit()
673 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); in pgtable_trans_huge_deposit()
674 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
677 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) in pgtable_trans_huge_withdraw() argument
684 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw()
687 pmd_huge_pte(mm, pmdp) = NULL; in pgtable_trans_huge_withdraw()
689 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; in pgtable_trans_huge_withdraw()
/linux-4.4.14/arch/unicore32/mm/
Dioremap.c74 pmd_t pmd, *pmdp = pmd_offset((pud_t *)pgd, addr); in unmap_area_sections() local
76 pmd = *pmdp; in unmap_area_sections()
85 pmd_clear(pmdp); in unmap_area_sections()
/linux-4.4.14/arch/arm/mm/
Dioremap.c144 pmd_t *pmdp; in unmap_area_sections() local
149 pmdp = pmd_offset(pud, addr); in unmap_area_sections()
151 pmd_t pmd = *pmdp; in unmap_area_sections()
161 pmd_clear(pmdp); in unmap_area_sections()
172 pmdp += 2; in unmap_area_sections()
Dflush.c422 pmd_t *pmdp) in pmdp_splitting_flush() argument
424 pmd_t pmd = pmd_mksplitting(*pmdp); in pmdp_splitting_flush()
426 set_pmd_at(vma->vm_mm, address, pmdp, pmd); in pmdp_splitting_flush()
/linux-4.4.14/arch/mips/include/asm/
Dpgtable.h461 unsigned long address, pmd_t *pmdp) in update_mmu_cache_pmd() argument
463 pte_t pte = *(pte_t *)pmdp; in update_mmu_cache_pmd()
514 pmd_t *pmdp, pmd_t pmd);
520 pmd_t *pmdp);
625 unsigned long address, pmd_t *pmdp) in pmdp_huge_get_and_clear() argument
627 pmd_t old = *pmdp; in pmdp_huge_get_and_clear()
629 pmd_clear(pmdp); in pmdp_huge_get_and_clear()
Dpgtable-32.h101 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
103 pmd_val(*pmdp) = ((unsigned long) invalid_pte_table); in pmd_clear()
Dpgtable-64.h195 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
197 pmd_val(*pmdp) = ((unsigned long) invalid_pte_table); in pmd_clear()
/linux-4.4.14/arch/x86/xen/
Dp2m.c335 pmd_t *pmdp; in xen_rebuild_p2m_list() local
406 pmdp = populate_extra_pmd( in xen_rebuild_p2m_list()
408 set_pmd(pmdp, __pmd(__pa(ptep) | _KERNPG_TABLE)); in xen_rebuild_p2m_list()
474 pmd_t *pmdp; in alloc_p2m_pmd() local
497 pmdp = lookup_pmd_address(vaddr); in alloc_p2m_pmd()
498 BUG_ON(!pmdp); in alloc_p2m_pmd()
506 set_pmd(pmdp, in alloc_p2m_pmd()
Dmmu.c492 static void xen_pmd_clear(pmd_t *pmdp) in xen_pmd_clear() argument
494 trace_xen_mmu_pmd_clear(pmdp); in xen_pmd_clear()
495 set_pmd(pmdp, __pmd(0)); in xen_pmd_clear()
/linux-4.4.14/arch/um/include/asm/
Dtlb.h130 #define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr) argument
/linux-4.4.14/arch/sh/include/asm/
Dtlb.h113 #define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp) argument
/linux-4.4.14/drivers/misc/sgi-gru/
Dgrufault.c223 pmd_t *pmdp; in atomic_pte_lookup() local
235 pmdp = pmd_offset(pudp, vaddr); in atomic_pte_lookup()
236 if (unlikely(pmd_none(*pmdp))) in atomic_pte_lookup()
239 if (unlikely(pmd_large(*pmdp))) in atomic_pte_lookup()
240 pte = *(pte_t *) pmdp; in atomic_pte_lookup()
243 pte = *pte_offset_kernel(pmdp, vaddr); in atomic_pte_lookup()
/linux-4.4.14/arch/microblaze/kernel/
Dsignal.c163 pmd_t *pmdp; in setup_rt_frame() local
198 pmdp = pmd_offset(pud_offset( in setup_rt_frame()
203 ptep = pte_offset_map(pmdp, address); in setup_rt_frame()
/linux-4.4.14/include/linux/
Dhuge_mm.h151 unsigned long addr, pmd_t pmd, pmd_t *pmdp);
211 unsigned long addr, pmd_t pmd, pmd_t *pmdp) in do_huge_pmd_numa_page() argument
/linux-4.4.14/arch/nios2/include/asm/
Dpgtable.h192 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
194 pmd_val(*pmdp) = (unsigned long) invalid_pte_table; in pmd_clear()
/linux-4.4.14/arch/cris/include/asm/
Dpgtable.h225 static inline void pmd_set(pmd_t * pmdp, pte_t * ptep) in pmd_set() argument
226 { pmd_val(*pmdp) = _PAGE_TABLE | (unsigned long) ptep; } in pmd_set()
/linux-4.4.14/fs/proc/
Dtask_mmu.c813 unsigned long addr, pmd_t *pmdp) in clear_soft_dirty_pmd() argument
815 pmd_t pmd = pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp); in clear_soft_dirty_pmd()
823 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); in clear_soft_dirty_pmd()
827 unsigned long addr, pmd_t *pmdp) in clear_soft_dirty_pmd() argument
1105 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, in pagemap_pmd_range() argument
1115 if (pmd_trans_huge_lock(pmdp, vma, &ptl) == 1) { in pagemap_pmd_range()
1117 pmd_t pmd = *pmdp; in pagemap_pmd_range()
1153 if (pmd_trans_unstable(pmdp)) in pagemap_pmd_range()
1161 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl); in pagemap_pmd_range()
/linux-4.4.14/arch/m32r/include/asm/
Dpgtable.h294 static inline void pmd_set(pmd_t * pmdp, pte_t * ptep) in pmd_set() argument
296 pmd_val(*pmdp) = (((unsigned long) ptep) & PAGE_MASK); in pmd_set()
/linux-4.4.14/arch/score/include/asm/
Dpgtable.h60 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
62 pmd_val(*pmdp) = ((unsigned long) invalid_pte_table); in pmd_clear()
/linux-4.4.14/arch/avr32/include/asm/
Dpgtable.h248 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
250 set_pmd(pmdp, __pmd(0)); in pmd_clear()
/linux-4.4.14/arch/openrisc/include/asm/
Dpgtable.h366 static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) in pmd_set() argument
368 pmd_val(*pmdp) = _KERNPG_TABLE | (unsigned long) ptep; in pmd_set()
/linux-4.4.14/arch/sparc/kernel/
Dsignal32.c373 pmd_t *pmdp; in flush_signal_insns() local
394 pmdp = pmd_offset(pudp, address); in flush_signal_insns()
395 if (pmd_none(*pmdp)) in flush_signal_insns()
398 ptep = pte_offset_map(pmdp, address); in flush_signal_insns()
/linux-4.4.14/arch/microblaze/include/asm/
Dpgtable.h304 #define pmd_clear(pmdp) do { pmd_val(*(pmdp)) = 0; } while (0) argument
/linux-4.4.14/arch/ia64/include/asm/
Dpgtable.h273 #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) argument
/linux-4.4.14/drivers/lguest/
Dpage_tables.c1056 void guest_set_pmd(struct lguest *lg, unsigned long pmdp, u32 idx) in guest_set_pmd() argument