151 	u64 *sptep;  member
164 		({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });	\
178 static void mmu_spte_set(u64 *sptep, u64 spte);
231 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,  in mark_mmio_spte()  argument
240 	trace_mark_mmio_spte(sptep, gfn, access, gen);  in mark_mmio_spte()
241 	mmu_spte_set(sptep, mask);  in mark_mmio_spte()
261 static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,  in set_mmio_spte()  argument
265 		mark_mmio_spte(vcpu, sptep, gfn, access);  in set_mmio_spte()
341 static void __set_spte(u64 *sptep, u64 spte)  in __set_spte()  argument
343 	*sptep = spte;  in __set_spte()
346 static void __update_clear_spte_fast(u64 *sptep, u64 spte)  in __update_clear_spte_fast()  argument
348 	*sptep = spte;  in __update_clear_spte_fast()
351 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)  in __update_clear_spte_slow()  argument
353 	return xchg(sptep, spte);  in __update_clear_spte_slow()
356 static u64 __get_spte_lockless(u64 *sptep)  in __get_spte_lockless()  argument
358 	return ACCESS_ONCE(*sptep);  in __get_spte_lockless()
369 static void count_spte_clear(u64 *sptep, u64 spte)  in count_spte_clear()  argument
371 	struct kvm_mmu_page *sp =  page_header(__pa(sptep));  in count_spte_clear()
381 static void __set_spte(u64 *sptep, u64 spte)  in __set_spte()  argument
385 	ssptep = (union split_spte *)sptep;  in __set_spte()
400 static void __update_clear_spte_fast(u64 *sptep, u64 spte)  in __update_clear_spte_fast()  argument
404 	ssptep = (union split_spte *)sptep;  in __update_clear_spte_fast()
416 	count_spte_clear(sptep, spte);  in __update_clear_spte_fast()
419 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)  in __update_clear_spte_slow()  argument
423 	ssptep = (union split_spte *)sptep;  in __update_clear_spte_slow()
430 	count_spte_clear(sptep, spte);  in __update_clear_spte_slow()
453 static u64 __get_spte_lockless(u64 *sptep)  in __get_spte_lockless()  argument
455 	struct kvm_mmu_page *sp =  page_header(__pa(sptep));  in __get_spte_lockless()
456 	union split_spte spte, *orig = (union split_spte *)sptep;  in __get_spte_lockless()
523 static void mmu_spte_set(u64 *sptep, u64 new_spte)  in mmu_spte_set()  argument
525 	WARN_ON(is_shadow_present_pte(*sptep));  in mmu_spte_set()
526 	__set_spte(sptep, new_spte);  in mmu_spte_set()
538 static bool mmu_spte_update(u64 *sptep, u64 new_spte)  in mmu_spte_update()  argument
540 	u64 old_spte = *sptep;  in mmu_spte_update()
546 		mmu_spte_set(sptep, new_spte);  in mmu_spte_update()
551 		__update_clear_spte_fast(sptep, new_spte);  in mmu_spte_update()
553 		old_spte = __update_clear_spte_slow(sptep, new_spte);  in mmu_spte_update()
588 static int mmu_spte_clear_track_bits(u64 *sptep)  in mmu_spte_clear_track_bits()  argument
591 	u64 old_spte = *sptep;  in mmu_spte_clear_track_bits()
594 		__update_clear_spte_fast(sptep, 0ull);  in mmu_spte_clear_track_bits()
596 		old_spte = __update_clear_spte_slow(sptep, 0ull);  in mmu_spte_clear_track_bits()
622 static void mmu_spte_clear_no_track(u64 *sptep)  in mmu_spte_clear_no_track()  argument
624 	__update_clear_spte_fast(sptep, 0ull);  in mmu_spte_clear_no_track()
627 static u64 mmu_spte_get_lockless(u64 *sptep)  in mmu_spte_get_lockless()  argument
629 	return __get_spte_lockless(sptep);  in mmu_spte_get_lockless()
1129 			u64 *sptep;  in rmap_get_next()  local
1132 			sptep = iter->desc->sptes[iter->pos];  in rmap_get_next()
1133 			if (sptep)  in rmap_get_next()
1134 				return sptep;  in rmap_get_next()
1154 static void drop_spte(struct kvm *kvm, u64 *sptep)  in drop_spte()  argument
1156 	if (mmu_spte_clear_track_bits(sptep))  in drop_spte()
1157 		rmap_remove(kvm, sptep);  in drop_spte()
1161 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)  in __drop_large_spte()  argument
1163 	if (is_large_pte(*sptep)) {  in __drop_large_spte()
1164 		WARN_ON(page_header(__pa(sptep))->role.level ==  in __drop_large_spte()
1166 		drop_spte(kvm, sptep);  in __drop_large_spte()
1174 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)  in drop_large_spte()  argument
1176 	if (__drop_large_spte(vcpu->kvm, sptep))  in drop_large_spte()
1193 static bool spte_write_protect(struct kvm *kvm, u64 *sptep, bool pt_protect)  in spte_write_protect()  argument
1195 	u64 spte = *sptep;  in spte_write_protect()
1201 	rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);  in spte_write_protect()
1207 	return mmu_spte_update(sptep, spte);  in spte_write_protect()
1213 	u64 *sptep;  in __rmap_write_protect()  local
1217 	for_each_rmap_spte(rmapp, &iter, sptep)  in __rmap_write_protect()
1218 		flush |= spte_write_protect(kvm, sptep, pt_protect);  in __rmap_write_protect()
1223 static bool spte_clear_dirty(struct kvm *kvm, u64 *sptep)  in spte_clear_dirty()  argument
1225 	u64 spte = *sptep;  in spte_clear_dirty()
1227 	rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);  in spte_clear_dirty()
1231 	return mmu_spte_update(sptep, spte);  in spte_clear_dirty()
1236 	u64 *sptep;  in __rmap_clear_dirty()  local
1240 	for_each_rmap_spte(rmapp, &iter, sptep)  in __rmap_clear_dirty()
1241 		flush |= spte_clear_dirty(kvm, sptep);  in __rmap_clear_dirty()
1246 static bool spte_set_dirty(struct kvm *kvm, u64 *sptep)  in spte_set_dirty()  argument
1248 	u64 spte = *sptep;  in spte_set_dirty()
1250 	rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);  in spte_set_dirty()
1254 	return mmu_spte_update(sptep, spte);  in spte_set_dirty()
1259 	u64 *sptep;  in __rmap_set_dirty()  local
1263 	for_each_rmap_spte(rmapp, &iter, sptep)  in __rmap_set_dirty()
1264 		flush |= spte_set_dirty(kvm, sptep);  in __rmap_set_dirty()
1361 	u64 *sptep;  in kvm_zap_rmapp()  local
1365 	while ((sptep = rmap_get_first(*rmapp, &iter))) {  in kvm_zap_rmapp()
1366 		BUG_ON(!(*sptep & PT_PRESENT_MASK));  in kvm_zap_rmapp()
1367 		rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);  in kvm_zap_rmapp()
1369 		drop_spte(kvm, sptep);  in kvm_zap_rmapp()
1387 	u64 *sptep;  in kvm_set_pte_rmapp()  local
1398 	for_each_rmap_spte(rmapp, &iter, sptep) {  in kvm_set_pte_rmapp()
1400 			     sptep, *sptep, gfn, level);  in kvm_set_pte_rmapp()
1405 			drop_spte(kvm, sptep);  in kvm_set_pte_rmapp()
1408 			new_spte = *sptep & ~PT64_BASE_ADDR_MASK;  in kvm_set_pte_rmapp()
1415 			mmu_spte_clear_track_bits(sptep);  in kvm_set_pte_rmapp()
1416 			mmu_spte_set(sptep, new_spte);  in kvm_set_pte_rmapp()
1570 	u64 *sptep;  in kvm_age_rmapp()  local
1576 	for_each_rmap_spte(rmapp, &iter, sptep)  in kvm_age_rmapp()
1577 		if (*sptep & shadow_accessed_mask) {  in kvm_age_rmapp()
1580 				 (unsigned long *)sptep);  in kvm_age_rmapp()
1591 	u64 *sptep;  in kvm_test_age_rmapp()  local
1603 	for_each_rmap_spte(rmapp, &iter, sptep)  in kvm_test_age_rmapp()
1604 		if (*sptep & shadow_accessed_mask) {  in kvm_test_age_rmapp()
2180 	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;  in shadow_walk_okay()
2198 	return __shadow_walk_next(iterator, *iterator->sptep);  in shadow_walk_next()
2201 static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp, bool accessed)  in link_shadow_page()  argument
2214 	mmu_spte_set(sptep, spte);  in link_shadow_page()
2217 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,  in validate_direct_spte()  argument
2220 	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {  in validate_direct_spte()
2230 		child = page_header(*sptep & PT64_BASE_ADDR_MASK);  in validate_direct_spte()
2234 		drop_parent_pte(child, sptep);  in validate_direct_spte()
2280 	u64 *sptep;  in kvm_mmu_unlink_parents()  local
2283 	while ((sptep = rmap_get_first(sp->parent_ptes, &iter)))  in kvm_mmu_unlink_parents()
2284 		drop_parent_pte(sp, sptep);  in kvm_mmu_unlink_parents()
2487 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,  in set_spte()  argument
2495 	if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access))  in set_spte()
2543 		if (!can_unsync && is_writable_pte(*sptep))  in set_spte()
2561 	if (mmu_spte_update(sptep, spte))  in set_spte()
2567 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,  in mmu_set_spte()  argument
2576 		 *sptep, write_fault, gfn);  in mmu_set_spte()
2578 	if (is_rmap_spte(*sptep)) {  in mmu_set_spte()
2584 		    !is_large_pte(*sptep)) {  in mmu_set_spte()
2586 			u64 pte = *sptep;  in mmu_set_spte()
2589 			drop_parent_pte(child, sptep);  in mmu_set_spte()
2591 		} else if (pfn != spte_to_pfn(*sptep)) {  in mmu_set_spte()
2593 				 spte_to_pfn(*sptep), pfn);  in mmu_set_spte()
2594 			drop_spte(vcpu->kvm, sptep);  in mmu_set_spte()
2600 	if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,  in mmu_set_spte()
2607 	if (unlikely(is_mmio_spte(*sptep) && emulate))  in mmu_set_spte()
2610 	pgprintk("%s: setting spte %llx\n", __func__, *sptep);  in mmu_set_spte()
2612 		 is_large_pte(*sptep)? "2MB" : "4kB",  in mmu_set_spte()
2613 		 *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,  in mmu_set_spte()
2614 		 *sptep, sptep);  in mmu_set_spte()
2615 	if (!was_rmapped && is_large_pte(*sptep))  in mmu_set_spte()
2618 	if (is_shadow_present_pte(*sptep)) {  in mmu_set_spte()
2620 			rmap_count = rmap_add(vcpu, sptep, gfn);  in mmu_set_spte()
2622 				rmap_recycle(vcpu, sptep, gfn);  in mmu_set_spte()
2669 				  struct kvm_mmu_page *sp, u64 *sptep)  in __direct_pte_prefetch()  argument
2676 	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);  in __direct_pte_prefetch()
2680 		if (is_shadow_present_pte(*spte) || spte == sptep) {  in __direct_pte_prefetch()
2691 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)  in direct_pte_prefetch()  argument
2704 	sp = page_header(__pa(sptep));  in direct_pte_prefetch()
2708 	__direct_pte_prefetch(vcpu, sp, sptep);  in direct_pte_prefetch()
2725 			mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,  in __direct_map()
2728 			direct_pte_prefetch(vcpu, iterator.sptep);  in __direct_map()
2733 		drop_large_spte(vcpu, iterator.sptep);  in __direct_map()
2734 		if (!is_shadow_present_pte(*iterator.sptep)) {  in __direct_map()
2741 					      1, ACC_ALL, iterator.sptep);  in __direct_map()
2743 			link_shadow_page(iterator.sptep, sp, true);  in __direct_map()
2864 			u64 *sptep, u64 spte)  in fast_pf_fix_direct_spte()  argument
2874 	gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);  in fast_pf_fix_direct_spte()
2888 	if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)  in fast_pf_fix_direct_spte()
2927 	sp = page_header(__pa(iterator.sptep));  in fast_page_fault()
2967 	ret = fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte);  in fast_page_fault()
2969 	trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,  in fast_page_fault()
3318 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)  in walk_shadow_page_get_mmio_spte()  argument
3334 		spte = mmu_spte_get_lockless(iterator.sptep);  in walk_shadow_page_get_mmio_spte()
3358 	*sptep = spte;  in walk_shadow_page_get_mmio_spte()
3577 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,  in sync_mmio_spte()  argument
3580 	if (unlikely(is_mmio_spte(*sptep))) {  in sync_mmio_spte()
3581 		if (gfn != get_mmio_spte_gfn(*sptep)) {  in sync_mmio_spte()
3582 			mmu_spte_clear_no_track(sptep);  in sync_mmio_spte()
3587 		mark_mmio_spte(vcpu, sptep, gfn, access);  in sync_mmio_spte()
4634 	u64 *sptep;  in kvm_mmu_zap_collapsible_spte()  local
4641 	for_each_rmap_spte(rmapp, &iter, sptep) {  in kvm_mmu_zap_collapsible_spte()
4642 		sp = page_header(__pa(sptep));  in kvm_mmu_zap_collapsible_spte()
4643 		pfn = spte_to_pfn(*sptep);  in kvm_mmu_zap_collapsible_spte()
4655 			drop_spte(kvm, sptep);  in kvm_mmu_zap_collapsible_spte()