This source file includes following definitions.
- kvm_set_s2pud_readonly
- kvm_s2pud_readonly
- kvm_set_pud
- kvm_s2pud_mkwrite
- kvm_s2pud_mkexec
- kvm_s2pud_exec
- kvm_s2pud_mkyoung
- kvm_s2pud_young
- kvm_s2pte_mkwrite
- kvm_s2pmd_mkwrite
- kvm_s2pte_mkexec
- kvm_s2pmd_mkexec
- kvm_set_s2pte_readonly
- kvm_s2pte_readonly
- kvm_s2pte_exec
- kvm_set_s2pmd_readonly
- kvm_s2pmd_readonly
- kvm_s2pmd_exec
- kvm_page_empty
- vcpu_has_cache_enabled
- __clean_dcache_guest_page
- __invalidate_icache_guest_page
- __kvm_flush_dcache_pte
- __kvm_flush_dcache_pmd
- __kvm_flush_dcache_pud
- __kvm_cpu_uses_extended_idmap
- __kvm_idmap_ptrs_per_pgd
- __kvm_extend_hypmap
- kvm_get_vmid_bits
- kvm_read_guest_lock
- kvm_write_guest_lock
- kvm_get_hyp_vector
- kvm_map_vectors
- hyp_map_aux_data
- kvm_set_ipa_limit
- kvm_get_vttbr
1
2
3
4
5
6
7 #ifndef __ARM_KVM_MMU_H__
8 #define __ARM_KVM_MMU_H__
9
10 #include <asm/memory.h>
11 #include <asm/page.h>
12
13
14
15
16
17 #define kern_hyp_va(kva) (kva)
18
19
20 #define hyp_symbol_addr(s) \
21 ({ \
22 typeof(s) *addr = &(s); \
23 addr; \
24 })
25
26 #ifndef __ASSEMBLY__
27
28 #include <linux/highmem.h>
29 #include <asm/cacheflush.h>
30 #include <asm/cputype.h>
31 #include <asm/kvm_arm.h>
32 #include <asm/kvm_hyp.h>
33 #include <asm/pgalloc.h>
34 #include <asm/stage2_pgtable.h>
35
36
37 #define VA_BITS 32
38
39 #define kvm_phys_shift(kvm) KVM_PHYS_SHIFT
40 #define kvm_phys_size(kvm) (1ULL << kvm_phys_shift(kvm))
41 #define kvm_phys_mask(kvm) (kvm_phys_size(kvm) - 1ULL)
42 #define kvm_vttbr_baddr_mask(kvm) VTTBR_BADDR_MASK
43
44 #define stage2_pgd_size(kvm) (PTRS_PER_S2_PGD * sizeof(pgd_t))
45
46 int create_hyp_mappings(void *from, void *to, pgprot_t prot);
47 int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
48 void __iomem **kaddr,
49 void __iomem **haddr);
50 int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size,
51 void **haddr);
52 void free_hyp_pgds(void);
53
54 void stage2_unmap_vm(struct kvm *kvm);
55 int kvm_alloc_stage2_pgd(struct kvm *kvm);
56 void kvm_free_stage2_pgd(struct kvm *kvm);
57 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
58 phys_addr_t pa, unsigned long size, bool writable);
59
60 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);
61
62 void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
63
64 phys_addr_t kvm_mmu_get_httbr(void);
65 phys_addr_t kvm_get_idmap_vector(void);
66 int kvm_mmu_init(void);
67 void kvm_clear_hyp_idmap(void);
68
69 #define kvm_mk_pmd(ptep) __pmd(__pa(ptep) | PMD_TYPE_TABLE)
70 #define kvm_mk_pud(pmdp) __pud(__pa(pmdp) | PMD_TYPE_TABLE)
71 #define kvm_mk_pgd(pudp) ({ BUILD_BUG(); 0; })
72
73 #define kvm_pfn_pte(pfn, prot) pfn_pte(pfn, prot)
74 #define kvm_pfn_pmd(pfn, prot) pfn_pmd(pfn, prot)
75 #define kvm_pfn_pud(pfn, prot) (__pud(0))
76
77 #define kvm_pud_pfn(pud) ({ WARN_ON(1); 0; })
78
79
80 #define kvm_pmd_mkhuge(pmd) pmd_mkhuge(pmd)
81
82 #define kvm_pud_mkhuge(pud) ( {WARN_ON(1); pud; })
83
84
85
86
87
88 static inline void kvm_set_s2pud_readonly(pud_t *pud)
89 {
90 WARN_ON(1);
91 }
92
93 static inline bool kvm_s2pud_readonly(pud_t *pud)
94 {
95 WARN_ON(1);
96 return false;
97 }
98
99 static inline void kvm_set_pud(pud_t *pud, pud_t new_pud)
100 {
101 WARN_ON(1);
102 }
103
104 static inline pud_t kvm_s2pud_mkwrite(pud_t pud)
105 {
106 WARN_ON(1);
107 return pud;
108 }
109
110 static inline pud_t kvm_s2pud_mkexec(pud_t pud)
111 {
112 WARN_ON(1);
113 return pud;
114 }
115
116 static inline bool kvm_s2pud_exec(pud_t *pud)
117 {
118 WARN_ON(1);
119 return false;
120 }
121
122 static inline pud_t kvm_s2pud_mkyoung(pud_t pud)
123 {
124 BUG();
125 return pud;
126 }
127
128 static inline bool kvm_s2pud_young(pud_t pud)
129 {
130 WARN_ON(1);
131 return false;
132 }
133
134 static inline pte_t kvm_s2pte_mkwrite(pte_t pte)
135 {
136 pte_val(pte) |= L_PTE_S2_RDWR;
137 return pte;
138 }
139
140 static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd)
141 {
142 pmd_val(pmd) |= L_PMD_S2_RDWR;
143 return pmd;
144 }
145
146 static inline pte_t kvm_s2pte_mkexec(pte_t pte)
147 {
148 pte_val(pte) &= ~L_PTE_XN;
149 return pte;
150 }
151
152 static inline pmd_t kvm_s2pmd_mkexec(pmd_t pmd)
153 {
154 pmd_val(pmd) &= ~PMD_SECT_XN;
155 return pmd;
156 }
157
158 static inline void kvm_set_s2pte_readonly(pte_t *pte)
159 {
160 pte_val(*pte) = (pte_val(*pte) & ~L_PTE_S2_RDWR) | L_PTE_S2_RDONLY;
161 }
162
163 static inline bool kvm_s2pte_readonly(pte_t *pte)
164 {
165 return (pte_val(*pte) & L_PTE_S2_RDWR) == L_PTE_S2_RDONLY;
166 }
167
168 static inline bool kvm_s2pte_exec(pte_t *pte)
169 {
170 return !(pte_val(*pte) & L_PTE_XN);
171 }
172
173 static inline void kvm_set_s2pmd_readonly(pmd_t *pmd)
174 {
175 pmd_val(*pmd) = (pmd_val(*pmd) & ~L_PMD_S2_RDWR) | L_PMD_S2_RDONLY;
176 }
177
178 static inline bool kvm_s2pmd_readonly(pmd_t *pmd)
179 {
180 return (pmd_val(*pmd) & L_PMD_S2_RDWR) == L_PMD_S2_RDONLY;
181 }
182
183 static inline bool kvm_s2pmd_exec(pmd_t *pmd)
184 {
185 return !(pmd_val(*pmd) & PMD_SECT_XN);
186 }
187
188 static inline bool kvm_page_empty(void *ptr)
189 {
190 struct page *ptr_page = virt_to_page(ptr);
191 return page_count(ptr_page) == 1;
192 }
193
194 #define kvm_pte_table_empty(kvm, ptep) kvm_page_empty(ptep)
195 #define kvm_pmd_table_empty(kvm, pmdp) kvm_page_empty(pmdp)
196 #define kvm_pud_table_empty(kvm, pudp) false
197
198 #define hyp_pte_table_empty(ptep) kvm_page_empty(ptep)
199 #define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp)
200 #define hyp_pud_table_empty(pudp) false
201
202 struct kvm;
203
204 #define kvm_flush_dcache_to_poc(a,l) __cpuc_flush_dcache_area((a), (l))
205
206 static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
207 {
208 return (vcpu_cp15(vcpu, c1_SCTLR) & 0b101) == 0b101;
209 }
210
211 static inline void __clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size)
212 {
213
214
215
216
217
218
219
220
221
222 VM_BUG_ON(size & ~PAGE_MASK);
223
224 while (size) {
225 void *va = kmap_atomic_pfn(pfn);
226
227 kvm_flush_dcache_to_poc(va, PAGE_SIZE);
228
229 size -= PAGE_SIZE;
230 pfn++;
231
232 kunmap_atomic(va);
233 }
234 }
235
236 static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn,
237 unsigned long size)
238 {
239 u32 iclsz;
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254 VM_BUG_ON(size & ~PAGE_MASK);
255
256 if (icache_is_vivt_asid_tagged())
257 return;
258
259 if (!icache_is_pipt()) {
260
261 __flush_icache_all();
262 return;
263 }
264
265
266
267
268
269
270
271
272 iclsz = 4 << (read_cpuid(CPUID_CACHETYPE) & 0xf);
273
274 while (size) {
275 void *va = kmap_atomic_pfn(pfn);
276 void *end = va + PAGE_SIZE;
277 void *addr = va;
278
279 do {
280 write_sysreg(addr, ICIMVAU);
281 addr += iclsz;
282 } while (addr < end);
283
284 dsb(ishst);
285 isb();
286
287 size -= PAGE_SIZE;
288 pfn++;
289
290 kunmap_atomic(va);
291 }
292
293
294 if ((read_cpuid_ext(CPUID_EXT_MMFR1) >> 28) != 4) {
295 write_sysreg(0, BPIALLIS);
296 dsb(ishst);
297 isb();
298 }
299 }
300
301 static inline void __kvm_flush_dcache_pte(pte_t pte)
302 {
303 void *va = kmap_atomic(pte_page(pte));
304
305 kvm_flush_dcache_to_poc(va, PAGE_SIZE);
306
307 kunmap_atomic(va);
308 }
309
310 static inline void __kvm_flush_dcache_pmd(pmd_t pmd)
311 {
312 unsigned long size = PMD_SIZE;
313 kvm_pfn_t pfn = pmd_pfn(pmd);
314
315 while (size) {
316 void *va = kmap_atomic_pfn(pfn);
317
318 kvm_flush_dcache_to_poc(va, PAGE_SIZE);
319
320 pfn++;
321 size -= PAGE_SIZE;
322
323 kunmap_atomic(va);
324 }
325 }
326
327 static inline void __kvm_flush_dcache_pud(pud_t pud)
328 {
329 }
330
331 #define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x))
332
333 void kvm_set_way_flush(struct kvm_vcpu *vcpu);
334 void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
335
336 static inline bool __kvm_cpu_uses_extended_idmap(void)
337 {
338 return false;
339 }
340
341 static inline unsigned long __kvm_idmap_ptrs_per_pgd(void)
342 {
343 return PTRS_PER_PGD;
344 }
345
346 static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
347 pgd_t *hyp_pgd,
348 pgd_t *merged_hyp_pgd,
349 unsigned long hyp_idmap_start) { }
350
351 static inline unsigned int kvm_get_vmid_bits(void)
352 {
353 return 8;
354 }
355
356
357
358
359
360
361 static inline int kvm_read_guest_lock(struct kvm *kvm,
362 gpa_t gpa, void *data, unsigned long len)
363 {
364 int srcu_idx = srcu_read_lock(&kvm->srcu);
365 int ret = kvm_read_guest(kvm, gpa, data, len);
366
367 srcu_read_unlock(&kvm->srcu, srcu_idx);
368
369 return ret;
370 }
371
372 static inline int kvm_write_guest_lock(struct kvm *kvm, gpa_t gpa,
373 const void *data, unsigned long len)
374 {
375 int srcu_idx = srcu_read_lock(&kvm->srcu);
376 int ret = kvm_write_guest(kvm, gpa, data, len);
377
378 srcu_read_unlock(&kvm->srcu, srcu_idx);
379
380 return ret;
381 }
382
383 static inline void *kvm_get_hyp_vector(void)
384 {
385 switch(read_cpuid_part()) {
386 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
387 case ARM_CPU_PART_CORTEX_A12:
388 case ARM_CPU_PART_CORTEX_A17:
389 {
390 extern char __kvm_hyp_vector_bp_inv[];
391 return kvm_ksym_ref(__kvm_hyp_vector_bp_inv);
392 }
393
394 case ARM_CPU_PART_BRAHMA_B15:
395 case ARM_CPU_PART_CORTEX_A15:
396 {
397 extern char __kvm_hyp_vector_ic_inv[];
398 return kvm_ksym_ref(__kvm_hyp_vector_ic_inv);
399 }
400 #endif
401 default:
402 {
403 extern char __kvm_hyp_vector[];
404 return kvm_ksym_ref(__kvm_hyp_vector);
405 }
406 }
407 }
408
409 static inline int kvm_map_vectors(void)
410 {
411 return 0;
412 }
413
414 static inline int hyp_map_aux_data(void)
415 {
416 return 0;
417 }
418
419 #define kvm_phys_to_vttbr(addr) (addr)
420
421 static inline void kvm_set_ipa_limit(void) {}
422
423 static __always_inline u64 kvm_get_vttbr(struct kvm *kvm)
424 {
425 struct kvm_vmid *vmid = &kvm->arch.vmid;
426 u64 vmid_field, baddr;
427
428 baddr = kvm->arch.pgd_phys;
429 vmid_field = (u64)vmid->vmid << VTTBR_VMID_SHIFT;
430 return kvm_phys_to_vttbr(baddr) | vmid_field;
431 }
432
433 #endif
434
435 #endif