65 static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
109 			     struct kvmppc_icp *icp)  in ics_check_resend()  argument
129 		icp_deliver_irq(xics, icp, state->number);  in ics_check_resend()
166 	struct kvmppc_icp *icp;  in kvmppc_xics_set_xive()  local
179 	icp = kvmppc_xics_find_server(kvm, server);  in kvmppc_xics_set_xive()
180 	if (!icp)  in kvmppc_xics_set_xive()
188 		icp_deliver_irq(xics, icp, irq);  in kvmppc_xics_set_xive()
222 	struct kvmppc_icp *icp;  in kvmppc_xics_int_on()  local
235 	icp = kvmppc_xics_find_server(kvm, state->server);  in kvmppc_xics_int_on()
236 	if (!icp)  in kvmppc_xics_int_on()
241 		icp_deliver_irq(xics, icp, irq);  in kvmppc_xics_int_on()
268 static inline bool icp_try_update(struct kvmppc_icp *icp,  in icp_try_update()  argument
279 	success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw;  in icp_try_update()
284 		 icp->server_num,  in icp_try_update()
306 		kvmppc_book3s_queue_irqprio(icp->vcpu,  in icp_try_update()
309 			kvmppc_fast_vcpu_kick(icp->vcpu);  in icp_try_update()
316 			     struct kvmppc_icp *icp)  in icp_check_resend()  argument
322 	for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) {  in icp_check_resend()
325 		if (!test_and_clear_bit(icsid, icp->resend_map))  in icp_check_resend()
329 		ics_check_resend(xics, ics, icp);  in icp_check_resend()
333 static bool icp_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority,  in icp_try_to_deliver()  argument
340 		 icp->server_num);  in icp_try_to_deliver()
343 		old_state = new_state = READ_ONCE(icp->state);  in icp_try_to_deliver()
369 	} while (!icp_try_update(icp, old_state, new_state, false));  in icp_try_to_deliver()
374 static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,  in icp_deliver_irq()  argument
412 	if (!icp || state->server != icp->server_num) {  in icp_deliver_irq()
413 		icp = kvmppc_xics_find_server(xics->kvm, state->server);  in icp_deliver_irq()
414 		if (!icp) {  in icp_deliver_irq()
461 	if (icp_try_to_deliver(icp, new_irq, state->priority, &reject)) {  in icp_deliver_irq()
476 		set_bit(ics->icsid, icp->resend_map);  in icp_deliver_irq()
486 		if (!icp->state.need_resend) {  in icp_deliver_irq()
497 static void icp_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp,  in icp_down_cppr()  argument
533 		old_state = new_state = READ_ONCE(icp->state);  in icp_down_cppr()
559 	} while (!icp_try_update(icp, old_state, new_state, true));  in icp_down_cppr()
567 		icp_check_resend(xics, icp);  in icp_down_cppr()
573 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_h_xirr()  local
577 	kvmppc_book3s_dequeue_irqprio(icp->vcpu,  in kvmppc_h_xirr()
588 		old_state = new_state = READ_ONCE(icp->state);  in kvmppc_h_xirr()
597 	} while (!icp_try_update(icp, old_state, new_state, true));  in kvmppc_h_xirr()
609 	struct kvmppc_icp *icp;  in kvmppc_h_ipi()  local
617 	icp = vcpu->arch.icp;  in kvmppc_h_ipi()
618 	local = icp->server_num == server;  in kvmppc_h_ipi()
620 		icp = kvmppc_xics_find_server(vcpu->kvm, server);  in kvmppc_h_ipi()
621 		if (!icp)  in kvmppc_h_ipi()
655 		old_state = new_state = READ_ONCE(icp->state);  in kvmppc_h_ipi()
676 	} while (!icp_try_update(icp, old_state, new_state, local));  in kvmppc_h_ipi()
680 		icp_deliver_irq(xics, icp, reject);  in kvmppc_h_ipi()
684 		icp_check_resend(xics, icp);  in kvmppc_h_ipi()
692 	struct kvmppc_icp *icp;  in kvmppc_h_ipoll()  local
694 	icp = vcpu->arch.icp;  in kvmppc_h_ipoll()
695 	if (icp->server_num != server) {  in kvmppc_h_ipoll()
696 		icp = kvmppc_xics_find_server(vcpu->kvm, server);  in kvmppc_h_ipoll()
697 		if (!icp)  in kvmppc_h_ipoll()
700 	state = READ_ONCE(icp->state);  in kvmppc_h_ipoll()
710 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_h_cppr()  local
722 	if (cppr > icp->state.cppr)  in kvmppc_h_cppr()
723 		icp_down_cppr(xics, icp, cppr);  in kvmppc_h_cppr()
724 	else if (cppr == icp->state.cppr)  in kvmppc_h_cppr()
738 	kvmppc_book3s_dequeue_irqprio(icp->vcpu,  in kvmppc_h_cppr()
742 		old_state = new_state = READ_ONCE(icp->state);  in kvmppc_h_cppr()
753 	} while (!icp_try_update(icp, old_state, new_state, true));  in kvmppc_h_cppr()
760 		icp_deliver_irq(xics, icp, reject);  in kvmppc_h_cppr()
766 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_h_eoi()  local
788 	icp_down_cppr(xics, icp, xirr >> 24);  in kvmppc_h_eoi()
808 		icp_deliver_irq(xics, icp, irq);  in kvmppc_h_eoi()
818 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_xics_rm_complete()  local
821 		 hcall, icp->rm_action, icp->rm_dbgstate.raw, icp->rm_dbgtgt);  in kvmppc_xics_rm_complete()
823 	if (icp->rm_action & XICS_RM_KICK_VCPU) {  in kvmppc_xics_rm_complete()
824 		icp->n_rm_kick_vcpu++;  in kvmppc_xics_rm_complete()
825 		kvmppc_fast_vcpu_kick(icp->rm_kick_target);  in kvmppc_xics_rm_complete()
827 	if (icp->rm_action & XICS_RM_CHECK_RESEND) {  in kvmppc_xics_rm_complete()
828 		icp->n_rm_check_resend++;  in kvmppc_xics_rm_complete()
829 		icp_check_resend(xics, icp->rm_resend_icp);  in kvmppc_xics_rm_complete()
831 	if (icp->rm_action & XICS_RM_REJECT) {  in kvmppc_xics_rm_complete()
832 		icp->n_rm_reject++;  in kvmppc_xics_rm_complete()
833 		icp_deliver_irq(xics, icp, icp->rm_reject);  in kvmppc_xics_rm_complete()
835 	if (icp->rm_action & XICS_RM_NOTIFY_EOI) {  in kvmppc_xics_rm_complete()
836 		icp->n_rm_notify_eoi++;  in kvmppc_xics_rm_complete()
837 		kvm_notify_acked_irq(vcpu->kvm, 0, icp->rm_eoied_irq);  in kvmppc_xics_rm_complete()
840 	icp->rm_action = 0;  in kvmppc_xics_rm_complete()
852 	if (!xics || !vcpu->arch.icp)  in kvmppc_xics_hcall()
919 		struct kvmppc_icp *icp = vcpu->arch.icp;  in xics_debug_show()  local
922 		if (!icp)  in xics_debug_show()
925 		state.raw = READ_ONCE(icp->state.raw);  in xics_debug_show()
927 			   icp->server_num, state.xisr,  in xics_debug_show()
930 		t_rm_kick_vcpu += icp->n_rm_kick_vcpu;  in xics_debug_show()
931 		t_rm_notify_eoi += icp->n_rm_notify_eoi;  in xics_debug_show()
932 		t_rm_check_resend += icp->n_rm_check_resend;  in xics_debug_show()
933 		t_rm_reject += icp->n_rm_reject;  in xics_debug_show()
934 		t_check_resend += icp->n_check_resend;  in xics_debug_show()
935 		t_reject += icp->n_reject;  in xics_debug_show()
1038 	struct kvmppc_icp *icp;  in kvmppc_xics_create_icp()  local
1046 	icp = kzalloc(sizeof(struct kvmppc_icp), GFP_KERNEL);  in kvmppc_xics_create_icp()
1047 	if (!icp)  in kvmppc_xics_create_icp()
1050 	icp->vcpu = vcpu;  in kvmppc_xics_create_icp()
1051 	icp->server_num = server_num;  in kvmppc_xics_create_icp()
1052 	icp->state.mfrr = MASKED;  in kvmppc_xics_create_icp()
1053 	icp->state.pending_pri = MASKED;  in kvmppc_xics_create_icp()
1054 	vcpu->arch.icp = icp;  in kvmppc_xics_create_icp()
1063 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_xics_get_icp()  local
1066 	if (!icp)  in kvmppc_xics_get_icp()
1068 	state = icp->state;  in kvmppc_xics_get_icp()
1077 	struct kvmppc_icp *icp = vcpu->arch.icp;  in kvmppc_xics_set_icp()  local
1086 	if (!icp || !xics)  in kvmppc_xics_set_icp()
1120 	kvmppc_book3s_dequeue_irqprio(icp->vcpu,  in kvmppc_xics_set_icp()
1134 		old_state = READ_ONCE(icp->state);  in kvmppc_xics_set_icp()
1143 	} while (!icp_try_update(icp, old_state, new_state, false));  in kvmppc_xics_set_icp()
1146 		icp_check_resend(xics, icp);  in kvmppc_xics_set_icp()
1383 	if (!vcpu->arch.icp)  in kvmppc_xics_free_icp()
1385 	kfree(vcpu->arch.icp);  in kvmppc_xics_free_icp()
1386 	vcpu->arch.icp = NULL;  in kvmppc_xics_free_icp()