Lines Matching refs:sp
371 struct kvm_mmu_page *sp = page_header(__pa(sptep)); in count_spte_clear() local
378 sp->clear_spte_count++; in count_spte_clear()
455 struct kvm_mmu_page *sp = page_header(__pa(sptep)); in __get_spte_lockless() local
460 count = sp->clear_spte_count; in __get_spte_lockless()
470 count != sp->clear_spte_count)) in __get_spte_lockless()
754 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index) in kvm_mmu_page_get_gfn() argument
756 if (!sp->role.direct) in kvm_mmu_page_get_gfn()
757 return sp->gfns[index]; in kvm_mmu_page_get_gfn()
759 return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS)); in kvm_mmu_page_get_gfn()
762 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn) in kvm_mmu_page_set_gfn() argument
764 if (sp->role.direct) in kvm_mmu_page_set_gfn()
765 BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index)); in kvm_mmu_page_set_gfn()
767 sp->gfns[index] = gfn; in kvm_mmu_page_set_gfn()
1040 struct kvm_mmu_page *sp; in rmap_add() local
1043 sp = page_header(__pa(spte)); in rmap_add()
1044 kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn); in rmap_add()
1045 rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level); in rmap_add()
1051 struct kvm_mmu_page *sp; in rmap_remove() local
1055 sp = page_header(__pa(spte)); in rmap_remove()
1056 gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt); in rmap_remove()
1057 rmapp = gfn_to_rmap(kvm, gfn, sp->role.level); in rmap_remove()
1546 struct kvm_mmu_page *sp; in rmap_recycle() local
1548 sp = page_header(__pa(spte)); in rmap_recycle()
1550 rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level); in rmap_recycle()
1552 kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, gfn, sp->role.level, 0); in rmap_recycle()
1614 static void kvm_mmu_free_page(struct kvm_mmu_page *sp) in kvm_mmu_free_page() argument
1616 MMU_WARN_ON(!is_empty_shadow_page(sp->spt)); in kvm_mmu_free_page()
1617 hlist_del(&sp->hash_link); in kvm_mmu_free_page()
1618 list_del(&sp->link); in kvm_mmu_free_page()
1619 free_page((unsigned long)sp->spt); in kvm_mmu_free_page()
1620 if (!sp->role.direct) in kvm_mmu_free_page()
1621 free_page((unsigned long)sp->gfns); in kvm_mmu_free_page()
1622 kmem_cache_free(mmu_page_header_cache, sp); in kvm_mmu_free_page()
1631 struct kvm_mmu_page *sp, u64 *parent_pte) in mmu_page_add_parent_pte() argument
1636 pte_list_add(vcpu, parent_pte, &sp->parent_ptes); in mmu_page_add_parent_pte()
1639 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp, in mmu_page_remove_parent_pte() argument
1642 pte_list_remove(parent_pte, &sp->parent_ptes); in mmu_page_remove_parent_pte()
1645 static void drop_parent_pte(struct kvm_mmu_page *sp, in drop_parent_pte() argument
1648 mmu_page_remove_parent_pte(sp, parent_pte); in drop_parent_pte()
1655 struct kvm_mmu_page *sp; in kvm_mmu_alloc_page() local
1657 sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache); in kvm_mmu_alloc_page()
1658 sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache); in kvm_mmu_alloc_page()
1660 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache); in kvm_mmu_alloc_page()
1661 set_page_private(virt_to_page(sp->spt), (unsigned long)sp); in kvm_mmu_alloc_page()
1668 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); in kvm_mmu_alloc_page()
1669 sp->parent_ptes = 0; in kvm_mmu_alloc_page()
1670 mmu_page_add_parent_pte(vcpu, sp, parent_pte); in kvm_mmu_alloc_page()
1672 return sp; in kvm_mmu_alloc_page()
1676 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp) in kvm_mmu_mark_parents_unsync() argument
1678 pte_list_walk(&sp->parent_ptes, mark_unsync); in kvm_mmu_mark_parents_unsync()
1683 struct kvm_mmu_page *sp; in mark_unsync() local
1686 sp = page_header(__pa(spte)); in mark_unsync()
1687 index = spte - sp->spt; in mark_unsync()
1688 if (__test_and_set_bit(index, sp->unsync_child_bitmap)) in mark_unsync()
1690 if (sp->unsync_children++) in mark_unsync()
1692 kvm_mmu_mark_parents_unsync(sp); in mark_unsync()
1696 struct kvm_mmu_page *sp) in nonpaging_sync_page() argument
1706 struct kvm_mmu_page *sp, u64 *spte, in nonpaging_update_pte() argument
1716 struct kvm_mmu_page *sp; member
1722 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp, in mmu_pages_add() argument
1727 if (sp->unsync) in mmu_pages_add()
1729 if (pvec->page[i].sp == sp) in mmu_pages_add()
1732 pvec->page[pvec->nr].sp = sp; in mmu_pages_add()
1738 static int __mmu_unsync_walk(struct kvm_mmu_page *sp, in __mmu_unsync_walk() argument
1743 for_each_set_bit(i, sp->unsync_child_bitmap, 512) { in __mmu_unsync_walk()
1745 u64 ent = sp->spt[i]; in __mmu_unsync_walk()
1773 __clear_bit(i, sp->unsync_child_bitmap); in __mmu_unsync_walk()
1774 sp->unsync_children--; in __mmu_unsync_walk()
1775 WARN_ON((int)sp->unsync_children < 0); in __mmu_unsync_walk()
1782 static int mmu_unsync_walk(struct kvm_mmu_page *sp, in mmu_unsync_walk() argument
1785 if (!sp->unsync_children) in mmu_unsync_walk()
1788 mmu_pages_add(pvec, sp, 0); in mmu_unsync_walk()
1789 return __mmu_unsync_walk(sp, pvec); in mmu_unsync_walk()
1792 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp) in kvm_unlink_unsync_page() argument
1794 WARN_ON(!sp->unsync); in kvm_unlink_unsync_page()
1795 trace_kvm_mmu_sync_page(sp); in kvm_unlink_unsync_page()
1796 sp->unsync = 0; in kvm_unlink_unsync_page()
1800 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1825 static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, in __kvm_sync_page() argument
1828 if (sp->role.cr4_pae != !!is_pae(vcpu)) { in __kvm_sync_page()
1829 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); in __kvm_sync_page()
1834 kvm_unlink_unsync_page(vcpu->kvm, sp); in __kvm_sync_page()
1836 if (vcpu->arch.mmu.sync_page(vcpu, sp)) { in __kvm_sync_page()
1837 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); in __kvm_sync_page()
1846 struct kvm_mmu_page *sp) in kvm_sync_page_transient() argument
1851 ret = __kvm_sync_page(vcpu, sp, &invalid_list, false); in kvm_sync_page_transient()
1865 static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, in kvm_sync_page() argument
1868 return __kvm_sync_page(vcpu, sp, invalid_list, true); in kvm_sync_page()
1902 #define for_each_sp(pvec, sp, parents, i) \ argument
1904 sp = pvec.page[i].sp; \
1905 i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
1915 struct kvm_mmu_page *sp = pvec->page[n].sp; in mmu_pages_next() local
1917 if (sp->role.level == PT_PAGE_TABLE_LEVEL) { in mmu_pages_next()
1922 parents->parent[sp->role.level-2] = sp; in mmu_pages_next()
1923 parents->idx[sp->role.level-1] = pvec->page[n].idx; in mmu_pages_next()
1931 struct kvm_mmu_page *sp; in mmu_pages_clear_parents() local
1937 sp = parents->parent[level]; in mmu_pages_clear_parents()
1938 if (!sp) in mmu_pages_clear_parents()
1941 --sp->unsync_children; in mmu_pages_clear_parents()
1942 WARN_ON((int)sp->unsync_children < 0); in mmu_pages_clear_parents()
1943 __clear_bit(idx, sp->unsync_child_bitmap); in mmu_pages_clear_parents()
1945 } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children); in mmu_pages_clear_parents()
1960 struct kvm_mmu_page *sp; in mmu_sync_children() local
1969 for_each_sp(pages, sp, parents, i) in mmu_sync_children()
1970 protected |= rmap_write_protect(vcpu->kvm, sp->gfn); in mmu_sync_children()
1975 for_each_sp(pages, sp, parents, i) { in mmu_sync_children()
1976 kvm_sync_page(vcpu, sp, &invalid_list); in mmu_sync_children()
1985 static void init_shadow_page_table(struct kvm_mmu_page *sp) in init_shadow_page_table() argument
1990 sp->spt[i] = 0ull; in init_shadow_page_table()
1993 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp) in __clear_sp_write_flooding_count() argument
1995 sp->write_flooding_count = 0; in __clear_sp_write_flooding_count()
2000 struct kvm_mmu_page *sp = page_header(__pa(spte)); in clear_sp_write_flooding_count() local
2002 __clear_sp_write_flooding_count(sp); in clear_sp_write_flooding_count()
2005 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp) in is_obsolete_sp() argument
2007 return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen); in is_obsolete_sp()
2020 struct kvm_mmu_page *sp; in kvm_mmu_get_page() local
2035 for_each_gfn_sp(vcpu->kvm, sp, gfn) { in kvm_mmu_get_page()
2036 if (is_obsolete_sp(vcpu->kvm, sp)) in kvm_mmu_get_page()
2039 if (!need_sync && sp->unsync) in kvm_mmu_get_page()
2042 if (sp->role.word != role.word) in kvm_mmu_get_page()
2045 if (sp->unsync && kvm_sync_page_transient(vcpu, sp)) in kvm_mmu_get_page()
2048 mmu_page_add_parent_pte(vcpu, sp, parent_pte); in kvm_mmu_get_page()
2049 if (sp->unsync_children) { in kvm_mmu_get_page()
2051 kvm_mmu_mark_parents_unsync(sp); in kvm_mmu_get_page()
2052 } else if (sp->unsync) in kvm_mmu_get_page()
2053 kvm_mmu_mark_parents_unsync(sp); in kvm_mmu_get_page()
2055 __clear_sp_write_flooding_count(sp); in kvm_mmu_get_page()
2056 trace_kvm_mmu_get_page(sp, false); in kvm_mmu_get_page()
2057 return sp; in kvm_mmu_get_page()
2060 sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct); in kvm_mmu_get_page()
2061 if (!sp) in kvm_mmu_get_page()
2062 return sp; in kvm_mmu_get_page()
2063 sp->gfn = gfn; in kvm_mmu_get_page()
2064 sp->role = role; in kvm_mmu_get_page()
2065 hlist_add_head(&sp->hash_link, in kvm_mmu_get_page()
2075 sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen; in kvm_mmu_get_page()
2076 init_shadow_page_table(sp); in kvm_mmu_get_page()
2077 trace_kvm_mmu_get_page(sp, true); in kvm_mmu_get_page()
2078 return sp; in kvm_mmu_get_page()
2130 static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp, bool accessed) in link_shadow_page() argument
2137 spte = __pa(sp->spt) | PT_PRESENT_MASK | PT_WRITABLE_MASK | in link_shadow_page()
2168 static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp, in mmu_page_zap_pte() argument
2176 if (is_last_spte(pte, sp->role.level)) { in mmu_page_zap_pte()
2194 struct kvm_mmu_page *sp) in kvm_mmu_page_unlink_children() argument
2199 mmu_page_zap_pte(kvm, sp, sp->spt + i); in kvm_mmu_page_unlink_children()
2202 static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte) in kvm_mmu_put_page() argument
2204 mmu_page_remove_parent_pte(sp, parent_pte); in kvm_mmu_put_page()
2207 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp) in kvm_mmu_unlink_parents() argument
2212 while ((sptep = rmap_get_first(sp->parent_ptes, &iter))) in kvm_mmu_unlink_parents()
2213 drop_parent_pte(sp, sptep); in kvm_mmu_unlink_parents()
2229 struct kvm_mmu_page *sp; in mmu_zap_unsync_children() local
2231 for_each_sp(pages, sp, parents, i) { in mmu_zap_unsync_children()
2232 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); in mmu_zap_unsync_children()
2242 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, in kvm_mmu_prepare_zap_page() argument
2247 trace_kvm_mmu_prepare_zap_page(sp); in kvm_mmu_prepare_zap_page()
2249 ret = mmu_zap_unsync_children(kvm, sp, invalid_list); in kvm_mmu_prepare_zap_page()
2250 kvm_mmu_page_unlink_children(kvm, sp); in kvm_mmu_prepare_zap_page()
2251 kvm_mmu_unlink_parents(kvm, sp); in kvm_mmu_prepare_zap_page()
2253 if (!sp->role.invalid && !sp->role.direct) in kvm_mmu_prepare_zap_page()
2254 unaccount_shadowed(kvm, sp->gfn); in kvm_mmu_prepare_zap_page()
2256 if (sp->unsync) in kvm_mmu_prepare_zap_page()
2257 kvm_unlink_unsync_page(kvm, sp); in kvm_mmu_prepare_zap_page()
2258 if (!sp->root_count) { in kvm_mmu_prepare_zap_page()
2261 list_move(&sp->link, invalid_list); in kvm_mmu_prepare_zap_page()
2264 list_move(&sp->link, &kvm->arch.active_mmu_pages); in kvm_mmu_prepare_zap_page()
2270 if (!sp->role.invalid && !is_obsolete_sp(kvm, sp)) in kvm_mmu_prepare_zap_page()
2274 sp->role.invalid = 1; in kvm_mmu_prepare_zap_page()
2281 struct kvm_mmu_page *sp, *nsp; in kvm_mmu_commit_zap_page() local
2298 list_for_each_entry_safe(sp, nsp, invalid_list, link) { in kvm_mmu_commit_zap_page()
2299 WARN_ON(!sp->role.invalid || sp->root_count); in kvm_mmu_commit_zap_page()
2300 kvm_mmu_free_page(sp); in kvm_mmu_commit_zap_page()
2307 struct kvm_mmu_page *sp; in prepare_zap_oldest_mmu_page() local
2312 sp = list_entry(kvm->arch.active_mmu_pages.prev, in prepare_zap_oldest_mmu_page()
2314 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); in prepare_zap_oldest_mmu_page()
2346 struct kvm_mmu_page *sp; in kvm_mmu_unprotect_page() local
2353 for_each_gfn_indirect_valid_sp(kvm, sp, gfn) { in kvm_mmu_unprotect_page()
2355 sp->role.word); in kvm_mmu_unprotect_page()
2357 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); in kvm_mmu_unprotect_page()
2471 static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) in __kvm_unsync_page() argument
2473 trace_kvm_mmu_unsync_page(sp); in __kvm_unsync_page()
2475 sp->unsync = 1; in __kvm_unsync_page()
2477 kvm_mmu_mark_parents_unsync(sp); in __kvm_unsync_page()
2668 struct kvm_mmu_page *sp, in direct_pte_prefetch_many() argument
2672 unsigned access = sp->role.access; in direct_pte_prefetch_many()
2676 gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt); in direct_pte_prefetch_many()
2686 sp->role.level, gfn, page_to_pfn(pages[i]), in direct_pte_prefetch_many()
2693 struct kvm_mmu_page *sp, u64 *sptep) in __direct_pte_prefetch() argument
2698 WARN_ON(!sp->role.direct); in __direct_pte_prefetch()
2700 i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1); in __direct_pte_prefetch()
2701 spte = sp->spt + i; in __direct_pte_prefetch()
2707 if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0) in __direct_pte_prefetch()
2717 struct kvm_mmu_page *sp; in direct_pte_prefetch() local
2728 sp = page_header(__pa(sptep)); in direct_pte_prefetch()
2729 if (sp->role.level > PT_PAGE_TABLE_LEVEL) in direct_pte_prefetch()
2732 __direct_pte_prefetch(vcpu, sp, sptep); in direct_pte_prefetch()
2740 struct kvm_mmu_page *sp; in __direct_map() local
2763 sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr, in __direct_map()
2767 link_shadow_page(iterator.sptep, sp, true); in __direct_map()
2887 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, in fast_pf_fix_direct_spte() argument
2892 WARN_ON(!sp->role.direct); in fast_pf_fix_direct_spte()
2898 gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); in fast_pf_fix_direct_spte()
2927 struct kvm_mmu_page *sp; in fast_page_fault() local
2951 sp = page_header(__pa(iterator.sptep)); in fast_page_fault()
2952 if (!is_last_spte(spte, sp->role.level)) in fast_page_fault()
2983 if (sp->role.level > PT_PAGE_TABLE_LEVEL) in fast_page_fault()
2991 ret = fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte); in fast_page_fault()
3064 struct kvm_mmu_page *sp; in mmu_free_roots() local
3076 sp = page_header(root); in mmu_free_roots()
3077 --sp->root_count; in mmu_free_roots()
3078 if (!sp->root_count && sp->role.invalid) { in mmu_free_roots()
3079 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list); in mmu_free_roots()
3093 sp = page_header(root); in mmu_free_roots()
3094 --sp->root_count; in mmu_free_roots()
3095 if (!sp->root_count && sp->role.invalid) in mmu_free_roots()
3096 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, in mmu_free_roots()
3120 struct kvm_mmu_page *sp; in mmu_alloc_direct_roots() local
3126 sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL, in mmu_alloc_direct_roots()
3128 ++sp->root_count; in mmu_alloc_direct_roots()
3130 vcpu->arch.mmu.root_hpa = __pa(sp->spt); in mmu_alloc_direct_roots()
3138 sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT), in mmu_alloc_direct_roots()
3142 root = __pa(sp->spt); in mmu_alloc_direct_roots()
3143 ++sp->root_count; in mmu_alloc_direct_roots()
3156 struct kvm_mmu_page *sp; in mmu_alloc_shadow_roots() local
3177 sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL, in mmu_alloc_shadow_roots()
3179 root = __pa(sp->spt); in mmu_alloc_shadow_roots()
3180 ++sp->root_count; in mmu_alloc_shadow_roots()
3211 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, in mmu_alloc_shadow_roots()
3214 root = __pa(sp->spt); in mmu_alloc_shadow_roots()
3215 ++sp->root_count; in mmu_alloc_shadow_roots()
3261 struct kvm_mmu_page *sp; in mmu_sync_roots() local
3273 sp = page_header(root); in mmu_sync_roots()
3274 mmu_sync_children(vcpu, sp); in mmu_sync_roots()
3283 sp = page_header(root); in mmu_sync_roots()
3284 mmu_sync_children(vcpu, sp); in mmu_sync_roots()
4016 struct kvm_mmu_page *sp, u64 *spte, in mmu_pte_write_new_pte() argument
4019 if (sp->role.level != PT_PAGE_TABLE_LEVEL) { in mmu_pte_write_new_pte()
4025 vcpu->arch.mmu.update_pte(vcpu, sp, spte, new); in mmu_pte_write_new_pte()
4093 static bool detect_write_flooding(struct kvm_mmu_page *sp) in detect_write_flooding() argument
4099 if (sp->role.level == PT_PAGE_TABLE_LEVEL) in detect_write_flooding()
4102 return ++sp->write_flooding_count >= 3; in detect_write_flooding()
4109 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa, in detect_write_misaligned() argument
4115 gpa, bytes, sp->role.word); in detect_write_misaligned()
4118 pte_size = sp->role.cr4_pae ? 8 : 4; in detect_write_misaligned()
4133 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte) in get_written_sptes() argument
4140 level = sp->role.level; in get_written_sptes()
4142 if (!sp->role.cr4_pae) { in get_written_sptes()
4156 if (quadrant != sp->role.quadrant) in get_written_sptes()
4160 spte = &sp->spt[page_offset / sizeof(*spte)]; in get_written_sptes()
4168 struct kvm_mmu_page *sp; in kvm_mmu_pte_write() local
4205 for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { in kvm_mmu_pte_write()
4206 if (detect_write_misaligned(sp, gpa, bytes) || in kvm_mmu_pte_write()
4207 detect_write_flooding(sp)) { in kvm_mmu_pte_write()
4208 zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp, in kvm_mmu_pte_write()
4214 spte = get_written_sptes(sp, gpa, &npte); in kvm_mmu_pte_write()
4221 mmu_page_zap_pte(vcpu->kvm, sp, spte); in kvm_mmu_pte_write()
4223 !((sp->role.word ^ vcpu->arch.mmu.base_role.word) in kvm_mmu_pte_write()
4225 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry); in kvm_mmu_pte_write()
4438 struct kvm_mmu_page *sp; in kvm_mmu_zap_collapsible_spte() local
4443 sp = page_header(__pa(sptep)); in kvm_mmu_zap_collapsible_spte()
4453 if (sp->role.direct && in kvm_mmu_zap_collapsible_spte()
4616 struct kvm_mmu_page *sp, *node; in kvm_zap_obsolete_pages() local
4620 list_for_each_entry_safe_reverse(sp, node, in kvm_zap_obsolete_pages()
4628 if (!is_obsolete_sp(kvm, sp)) in kvm_zap_obsolete_pages()
4636 if (sp->role.invalid) in kvm_zap_obsolete_pages()
4649 ret = kvm_mmu_prepare_zap_page(kvm, sp, in kvm_zap_obsolete_pages()