739 xpc_send_activate_IRQ_ch_uv(struct xpc_channel *ch, unsigned long *irq_flags,  in xpc_send_activate_IRQ_ch_uv()  argument
742 	struct xpc_partition *part = &xpc_partitions[ch->partid];  in xpc_send_activate_IRQ_ch_uv()
748 			spin_unlock_irqrestore(&ch->lock, *irq_flags);  in xpc_send_activate_IRQ_ch_uv()
753 			spin_lock_irqsave(&ch->lock, *irq_flags);  in xpc_send_activate_IRQ_ch_uv()
1082 xpc_allocate_send_msg_slot_uv(struct xpc_channel *ch)  in xpc_allocate_send_msg_slot_uv()  argument
1084 	struct xpc_channel_uv *ch_uv = &ch->sn.uv;  in xpc_allocate_send_msg_slot_uv()
1091 	for (nentries = ch->local_nentries; nentries > 0; nentries--) {  in xpc_allocate_send_msg_slot_uv()
1105 		spin_lock_irqsave(&ch->lock, irq_flags);  in xpc_allocate_send_msg_slot_uv()
1106 		if (nentries < ch->local_nentries)  in xpc_allocate_send_msg_slot_uv()
1107 			ch->local_nentries = nentries;  in xpc_allocate_send_msg_slot_uv()
1108 		spin_unlock_irqrestore(&ch->lock, irq_flags);  in xpc_allocate_send_msg_slot_uv()
1116 xpc_allocate_recv_msg_slot_uv(struct xpc_channel *ch)  in xpc_allocate_recv_msg_slot_uv()  argument
1118 	struct xpc_channel_uv *ch_uv = &ch->sn.uv;  in xpc_allocate_recv_msg_slot_uv()
1125 	for (nentries = ch->remote_nentries; nentries > 0; nentries--) {  in xpc_allocate_recv_msg_slot_uv()
1126 		nbytes = nentries * ch->entry_size;  in xpc_allocate_recv_msg_slot_uv()
1133 			    entry * ch->entry_size;  in xpc_allocate_recv_msg_slot_uv()
1138 		spin_lock_irqsave(&ch->lock, irq_flags);  in xpc_allocate_recv_msg_slot_uv()
1139 		if (nentries < ch->remote_nentries)  in xpc_allocate_recv_msg_slot_uv()
1140 			ch->remote_nentries = nentries;  in xpc_allocate_recv_msg_slot_uv()
1141 		spin_unlock_irqrestore(&ch->lock, irq_flags);  in xpc_allocate_recv_msg_slot_uv()
1152 xpc_setup_msg_structures_uv(struct xpc_channel *ch)  in xpc_setup_msg_structures_uv()  argument
1155 	struct xpc_channel_uv *ch_uv = &ch->sn.uv;  in xpc_setup_msg_structures_uv()
1157 	DBUG_ON(ch->flags & XPC_C_SETUP);  in xpc_setup_msg_structures_uv()
1165 	ret = xpc_allocate_send_msg_slot_uv(ch);  in xpc_setup_msg_structures_uv()
1168 		ret = xpc_allocate_recv_msg_slot_uv(ch);  in xpc_setup_msg_structures_uv()
1182 xpc_teardown_msg_structures_uv(struct xpc_channel *ch)  in xpc_teardown_msg_structures_uv()  argument
1184 	struct xpc_channel_uv *ch_uv = &ch->sn.uv;  in xpc_teardown_msg_structures_uv()
1186 	DBUG_ON(!spin_is_locked(&ch->lock));  in xpc_teardown_msg_structures_uv()
1191 	if (ch->flags & XPC_C_SETUP) {  in xpc_teardown_msg_structures_uv()
1200 xpc_send_chctl_closerequest_uv(struct xpc_channel *ch, unsigned long *irq_flags)  in xpc_send_chctl_closerequest_uv()  argument
1204 	msg.ch_number = ch->number;  in xpc_send_chctl_closerequest_uv()
1205 	msg.reason = ch->reason;  in xpc_send_chctl_closerequest_uv()
1206 	xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),  in xpc_send_chctl_closerequest_uv()
1211 xpc_send_chctl_closereply_uv(struct xpc_channel *ch, unsigned long *irq_flags)  in xpc_send_chctl_closereply_uv()  argument
1215 	msg.ch_number = ch->number;  in xpc_send_chctl_closereply_uv()
1216 	xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),  in xpc_send_chctl_closereply_uv()
1221 xpc_send_chctl_openrequest_uv(struct xpc_channel *ch, unsigned long *irq_flags)  in xpc_send_chctl_openrequest_uv()  argument
1225 	msg.ch_number = ch->number;  in xpc_send_chctl_openrequest_uv()
1226 	msg.entry_size = ch->entry_size;  in xpc_send_chctl_openrequest_uv()
1227 	msg.local_nentries = ch->local_nentries;  in xpc_send_chctl_openrequest_uv()
1228 	xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),  in xpc_send_chctl_openrequest_uv()
1233 xpc_send_chctl_openreply_uv(struct xpc_channel *ch, unsigned long *irq_flags)  in xpc_send_chctl_openreply_uv()  argument
1237 	msg.ch_number = ch->number;  in xpc_send_chctl_openreply_uv()
1238 	msg.local_nentries = ch->local_nentries;  in xpc_send_chctl_openreply_uv()
1239 	msg.remote_nentries = ch->remote_nentries;  in xpc_send_chctl_openreply_uv()
1241 	xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),  in xpc_send_chctl_openreply_uv()
1246 xpc_send_chctl_opencomplete_uv(struct xpc_channel *ch, unsigned long *irq_flags)  in xpc_send_chctl_opencomplete_uv()  argument
1250 	msg.ch_number = ch->number;  in xpc_send_chctl_opencomplete_uv()
1251 	xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),  in xpc_send_chctl_opencomplete_uv()
1268 xpc_save_remote_msgqueue_pa_uv(struct xpc_channel *ch,  in xpc_save_remote_msgqueue_pa_uv()  argument
1271 	struct xpc_channel_uv *ch_uv = &ch->sn.uv;  in xpc_save_remote_msgqueue_pa_uv()
1328 xpc_allocate_msg_slot_uv(struct xpc_channel *ch, u32 flags,  in xpc_allocate_msg_slot_uv()  argument
1336 		entry = xpc_get_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list);  in xpc_allocate_msg_slot_uv()
1343 		ret = xpc_allocate_msg_wait(ch);  in xpc_allocate_msg_slot_uv()
1354 xpc_free_msg_slot_uv(struct xpc_channel *ch,  in xpc_free_msg_slot_uv()  argument
1357 	xpc_put_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list, &msg_slot->next);  in xpc_free_msg_slot_uv()
1360 	if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)  in xpc_free_msg_slot_uv()
1361 		wake_up(&ch->msg_allocate_wq);  in xpc_free_msg_slot_uv()
1365 xpc_notify_sender_uv(struct xpc_channel *ch,  in xpc_notify_sender_uv()  argument
1373 		atomic_dec(&ch->n_to_notify);  in xpc_notify_sender_uv()
1377 			msg_slot->msg_slot_number, ch->partid, ch->number);  in xpc_notify_sender_uv()
1379 		func(reason, ch->partid, ch->number, msg_slot->key);  in xpc_notify_sender_uv()
1383 			msg_slot->msg_slot_number, ch->partid, ch->number);  in xpc_notify_sender_uv()
1388 xpc_handle_notify_mq_ack_uv(struct xpc_channel *ch,  in xpc_handle_notify_mq_ack_uv()  argument
1392 	int entry = msg->hdr.msg_slot_number % ch->local_nentries;  in xpc_handle_notify_mq_ack_uv()
1394 	msg_slot = &ch->sn.uv.send_msg_slots[entry];  in xpc_handle_notify_mq_ack_uv()
1397 	msg_slot->msg_slot_number += ch->local_nentries;  in xpc_handle_notify_mq_ack_uv()
1400 		xpc_notify_sender_uv(ch, msg_slot, xpMsgDelivered);  in xpc_handle_notify_mq_ack_uv()
1402 	xpc_free_msg_slot_uv(ch, msg_slot);  in xpc_handle_notify_mq_ack_uv()
1410 	struct xpc_channel *ch;  in xpc_handle_notify_mq_msg_uv()  local
1433 	ch = &part->channels[ch_number];  in xpc_handle_notify_mq_msg_uv()
1434 	xpc_msgqueue_ref(ch);  in xpc_handle_notify_mq_msg_uv()
1436 	if (!(ch->flags & XPC_C_CONNECTED)) {  in xpc_handle_notify_mq_msg_uv()
1437 		xpc_msgqueue_deref(ch);  in xpc_handle_notify_mq_msg_uv()
1443 		xpc_handle_notify_mq_ack_uv(ch, msg);  in xpc_handle_notify_mq_msg_uv()
1444 		xpc_msgqueue_deref(ch);  in xpc_handle_notify_mq_msg_uv()
1449 	ch_uv = &ch->sn.uv;  in xpc_handle_notify_mq_msg_uv()
1452 	    (msg->hdr.msg_slot_number % ch->remote_nentries) * ch->entry_size;  in xpc_handle_notify_mq_msg_uv()
1460 	if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) {  in xpc_handle_notify_mq_msg_uv()
1466 		if (atomic_read(&ch->kthreads_idle) > 0)  in xpc_handle_notify_mq_msg_uv()
1467 			wake_up_nr(&ch->idle_wq, 1);  in xpc_handle_notify_mq_msg_uv()
1469 			xpc_send_chctl_local_msgrequest_uv(part, ch->number);  in xpc_handle_notify_mq_msg_uv()
1471 	xpc_msgqueue_deref(ch);  in xpc_handle_notify_mq_msg_uv()
1504 xpc_n_of_deliverable_payloads_uv(struct xpc_channel *ch)  in xpc_n_of_deliverable_payloads_uv()  argument
1506 	return xpc_n_of_fifo_entries_uv(&ch->sn.uv.recv_msg_list);  in xpc_n_of_deliverable_payloads_uv()
1512 	struct xpc_channel *ch = &part->channels[ch_number];  in xpc_process_msg_chctl_flags_uv()  local
1515 	xpc_msgqueue_ref(ch);  in xpc_process_msg_chctl_flags_uv()
1517 	ndeliverable_payloads = xpc_n_of_deliverable_payloads_uv(ch);  in xpc_process_msg_chctl_flags_uv()
1520 	    (ch->flags & XPC_C_CONNECTED) &&  in xpc_process_msg_chctl_flags_uv()
1521 	    (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)) {  in xpc_process_msg_chctl_flags_uv()
1523 		xpc_activate_kthreads(ch, ndeliverable_payloads);  in xpc_process_msg_chctl_flags_uv()
1526 	xpc_msgqueue_deref(ch);  in xpc_process_msg_chctl_flags_uv()
1530 xpc_send_payload_uv(struct xpc_channel *ch, u32 flags, void *payload,  in xpc_send_payload_uv()  argument
1543 	if (msg_size > ch->entry_size)  in xpc_send_payload_uv()
1546 	xpc_msgqueue_ref(ch);  in xpc_send_payload_uv()
1548 	if (ch->flags & XPC_C_DISCONNECTING) {  in xpc_send_payload_uv()
1549 		ret = ch->reason;  in xpc_send_payload_uv()
1552 	if (!(ch->flags & XPC_C_CONNECTED)) {  in xpc_send_payload_uv()
1557 	ret = xpc_allocate_msg_slot_uv(ch, flags, &msg_slot);  in xpc_send_payload_uv()
1562 		atomic_inc(&ch->n_to_notify);  in xpc_send_payload_uv()
1568 		if (ch->flags & XPC_C_DISCONNECTING) {  in xpc_send_payload_uv()
1569 			ret = ch->reason;  in xpc_send_payload_uv()
1576 	msg->hdr.ch_number = ch->number;  in xpc_send_payload_uv()
1581 	ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,  in xpc_send_payload_uv()
1586 	XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);  in xpc_send_payload_uv()
1605 		atomic_dec(&ch->n_to_notify);  in xpc_send_payload_uv()
1607 	xpc_free_msg_slot_uv(ch, msg_slot);  in xpc_send_payload_uv()
1609 	xpc_msgqueue_deref(ch);  in xpc_send_payload_uv()
1621 xpc_notify_senders_of_disconnect_uv(struct xpc_channel *ch)  in xpc_notify_senders_of_disconnect_uv()  argument
1626 	DBUG_ON(!(ch->flags & XPC_C_DISCONNECTING));  in xpc_notify_senders_of_disconnect_uv()
1628 	for (entry = 0; entry < ch->local_nentries; entry++) {  in xpc_notify_senders_of_disconnect_uv()
1630 		if (atomic_read(&ch->n_to_notify) == 0)  in xpc_notify_senders_of_disconnect_uv()
1633 		msg_slot = &ch->sn.uv.send_msg_slots[entry];  in xpc_notify_senders_of_disconnect_uv()
1635 			xpc_notify_sender_uv(ch, msg_slot, ch->reason);  in xpc_notify_senders_of_disconnect_uv()
1643 xpc_get_deliverable_payload_uv(struct xpc_channel *ch)  in xpc_get_deliverable_payload_uv()  argument
1649 	if (!(ch->flags & XPC_C_DISCONNECTING)) {  in xpc_get_deliverable_payload_uv()
1650 		entry = xpc_get_fifo_entry_uv(&ch->sn.uv.recv_msg_list);  in xpc_get_deliverable_payload_uv()
1661 xpc_received_payload_uv(struct xpc_channel *ch, void *payload)  in xpc_received_payload_uv()  argument
1673 	ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,  in xpc_received_payload_uv()
1676 		XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);  in xpc_received_payload_uv()