This source file includes following definitions.
- native_set_pte
- pmd_read_atomic
- native_set_pte_atomic
- native_set_pmd
- native_set_pud
- native_pte_clear
- native_pmd_clear
- native_pud_clear
- pud_clear
- native_ptep_get_and_clear
- native_pmdp_get_and_clear
- pmdp_establish
- native_pudp_get_and_clear
1
2 #ifndef _ASM_X86_PGTABLE_3LEVEL_H
3 #define _ASM_X86_PGTABLE_3LEVEL_H
4
5 #include <asm/atomic64_32.h>
6
7
8
9
10
11
12
13
14 #define pte_ERROR(e) \
15 pr_err("%s:%d: bad pte %p(%08lx%08lx)\n", \
16 __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low)
17 #define pmd_ERROR(e) \
18 pr_err("%s:%d: bad pmd %p(%016Lx)\n", \
19 __FILE__, __LINE__, &(e), pmd_val(e))
20 #define pgd_ERROR(e) \
21 pr_err("%s:%d: bad pgd %p(%016Lx)\n", \
22 __FILE__, __LINE__, &(e), pgd_val(e))
23
24
25
26
27
28
29
30 static inline void native_set_pte(pte_t *ptep, pte_t pte)
31 {
32 ptep->pte_high = pte.pte_high;
33 smp_wmb();
34 ptep->pte_low = pte.pte_low;
35 }
36
37 #define pmd_read_atomic pmd_read_atomic
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73 static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
74 {
75 pmdval_t ret;
76 u32 *tmp = (u32 *)pmdp;
77
78 ret = (pmdval_t) (*tmp);
79 if (ret) {
80
81
82
83
84 smp_rmb();
85 ret |= ((pmdval_t)*(tmp + 1)) << 32;
86 }
87
88 return (pmd_t) { ret };
89 }
90
91 static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
92 {
93 set_64bit((unsigned long long *)(ptep), native_pte_val(pte));
94 }
95
96 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
97 {
98 set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd));
99 }
100
101 static inline void native_set_pud(pud_t *pudp, pud_t pud)
102 {
103 #ifdef CONFIG_PAGE_TABLE_ISOLATION
104 pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd);
105 #endif
106 set_64bit((unsigned long long *)(pudp), native_pud_val(pud));
107 }
108
109
110
111
112
113
114 static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
115 pte_t *ptep)
116 {
117 ptep->pte_low = 0;
118 smp_wmb();
119 ptep->pte_high = 0;
120 }
121
122 static inline void native_pmd_clear(pmd_t *pmd)
123 {
124 u32 *tmp = (u32 *)pmd;
125 *tmp = 0;
126 smp_wmb();
127 *(tmp + 1) = 0;
128 }
129
130 static inline void native_pud_clear(pud_t *pudp)
131 {
132 }
133
134 static inline void pud_clear(pud_t *pudp)
135 {
136 set_pud(pudp, __pud(0));
137
138
139
140
141
142
143
144
145
146
147
148 }
149
150 #ifdef CONFIG_SMP
151 static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
152 {
153 pte_t res;
154
155 res.pte = (pteval_t)arch_atomic64_xchg((atomic64_t *)ptep, 0);
156
157 return res;
158 }
159 #else
160 #define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
161 #endif
162
163 union split_pmd {
164 struct {
165 u32 pmd_low;
166 u32 pmd_high;
167 };
168 pmd_t pmd;
169 };
170
171 #ifdef CONFIG_SMP
172 static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
173 {
174 union split_pmd res, *orig = (union split_pmd *)pmdp;
175
176
177 res.pmd_low = xchg(&orig->pmd_low, 0);
178 res.pmd_high = orig->pmd_high;
179 orig->pmd_high = 0;
180
181 return res.pmd;
182 }
183 #else
184 #define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
185 #endif
186
187 #ifndef pmdp_establish
188 #define pmdp_establish pmdp_establish
189 static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
190 unsigned long address, pmd_t *pmdp, pmd_t pmd)
191 {
192 pmd_t old;
193
194
195
196
197
198
199 if (!(pmd_val(pmd) & _PAGE_PRESENT)) {
200 union split_pmd old, new, *ptr;
201
202 ptr = (union split_pmd *)pmdp;
203
204 new.pmd = pmd;
205
206
207 old.pmd_low = xchg(&ptr->pmd_low, new.pmd_low);
208 old.pmd_high = ptr->pmd_high;
209 ptr->pmd_high = new.pmd_high;
210 return old.pmd;
211 }
212
213 do {
214 old = *pmdp;
215 } while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);
216
217 return old;
218 }
219 #endif
220
221 #ifdef CONFIG_SMP
222 union split_pud {
223 struct {
224 u32 pud_low;
225 u32 pud_high;
226 };
227 pud_t pud;
228 };
229
230 static inline pud_t native_pudp_get_and_clear(pud_t *pudp)
231 {
232 union split_pud res, *orig = (union split_pud *)pudp;
233
234 #ifdef CONFIG_PAGE_TABLE_ISOLATION
235 pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0));
236 #endif
237
238
239 res.pud_low = xchg(&orig->pud_low, 0);
240 res.pud_high = orig->pud_high;
241 orig->pud_high = 0;
242
243 return res.pud;
244 }
245 #else
246 #define native_pudp_get_and_clear(xp) native_local_pudp_get_and_clear(xp)
247 #endif
248
249
250 #define SWP_TYPE_BITS 5
251
252 #define SWP_OFFSET_FIRST_BIT (_PAGE_BIT_PROTNONE + 1)
253
254
255 #define SWP_OFFSET_SHIFT (SWP_OFFSET_FIRST_BIT + SWP_TYPE_BITS)
256
257 #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > 5)
258 #define __swp_type(x) (((x).val) & 0x1f)
259 #define __swp_offset(x) ((x).val >> 5)
260 #define __swp_entry(type, offset) ((swp_entry_t){(type) | (offset) << 5})
261
262
263
264
265
266
267
268
269
270 #define __swp_pteval_entry(type, offset) ((pteval_t) { \
271 (~(pteval_t)(offset) << SWP_OFFSET_SHIFT >> SWP_TYPE_BITS) \
272 | ((pteval_t)(type) << (64 - SWP_TYPE_BITS)) })
273
274 #define __swp_entry_to_pte(x) ((pte_t){ .pte = \
275 __swp_pteval_entry(__swp_type(x), __swp_offset(x)) })
276
277
278
279
280
281
282 #define __pteval_swp_type(x) ((unsigned long)((x).pte >> (64 - SWP_TYPE_BITS)))
283 #define __pteval_swp_offset(x) ((unsigned long)(~((x).pte) << SWP_TYPE_BITS >> SWP_OFFSET_SHIFT))
284
285 #define __pte_to_swp_entry(pte) (__swp_entry(__pteval_swp_type(pte), \
286 __pteval_swp_offset(pte)))
287
288 #include <asm/pgtable-invert.h>
289
290 #endif