59 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_init()  local
63 	efx->mcdi = kzalloc(sizeof(*efx->mcdi), GFP_KERNEL);  in efx_mcdi_init()
64 	if (!efx->mcdi)  in efx_mcdi_init()
67 	mcdi = efx_mcdi(efx);  in efx_mcdi_init()
68 	mcdi->efx = efx;  in efx_mcdi_init()
69 	init_waitqueue_head(&mcdi->wq);  in efx_mcdi_init()
70 	spin_lock_init(&mcdi->iface_lock);  in efx_mcdi_init()
71 	mcdi->state = MCDI_STATE_QUIESCENT;  in efx_mcdi_init()
72 	mcdi->mode = MCDI_MODE_POLL;  in efx_mcdi_init()
73 	spin_lock_init(&mcdi->async_lock);  in efx_mcdi_init()
74 	INIT_LIST_HEAD(&mcdi->async_list);  in efx_mcdi_init()
75 	setup_timer(&mcdi->async_timer, efx_mcdi_timeout_async,  in efx_mcdi_init()
76 		    (unsigned long)mcdi);  in efx_mcdi_init()
79 	mcdi->new_epoch = true;  in efx_mcdi_init()
100 	if (efx->mcdi->fn_flags &  in efx_mcdi_init()
109 	if (!efx->mcdi)  in efx_mcdi_fini()
112 	BUG_ON(efx->mcdi->iface.state != MCDI_STATE_QUIESCENT);  in efx_mcdi_fini()
117 	kfree(efx->mcdi);  in efx_mcdi_fini()
123 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_send_request()  local
128 	BUG_ON(mcdi->state == MCDI_STATE_QUIESCENT);  in efx_mcdi_send_request()
131 	spin_lock_bh(&mcdi->iface_lock);  in efx_mcdi_send_request()
132 	++mcdi->seqno;  in efx_mcdi_send_request()
133 	spin_unlock_bh(&mcdi->iface_lock);  in efx_mcdi_send_request()
135 	seqno = mcdi->seqno & SEQ_MASK;  in efx_mcdi_send_request()
137 	if (mcdi->mode == MCDI_MODE_EVENTS)  in efx_mcdi_send_request()
149 				     MCDI_HEADER_NOT_EPOCH, !mcdi->new_epoch);  in efx_mcdi_send_request()
161 				     MCDI_HEADER_NOT_EPOCH, !mcdi->new_epoch);  in efx_mcdi_send_request()
170 	mcdi->new_epoch = false;  in efx_mcdi_send_request()
207 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_read_response_header()  local
217 		mcdi->resp_hdr_len = 4;  in efx_mcdi_read_response_header()
218 		mcdi->resp_data_len = EFX_DWORD_FIELD(hdr, MCDI_HEADER_DATALEN);  in efx_mcdi_read_response_header()
221 		mcdi->resp_hdr_len = 8;  in efx_mcdi_read_response_header()
222 		mcdi->resp_data_len =  in efx_mcdi_read_response_header()
226 	if (error && mcdi->resp_data_len == 0) {  in efx_mcdi_read_response_header()
228 		mcdi->resprc = -EIO;  in efx_mcdi_read_response_header()
229 	} else if ((respseq ^ mcdi->seqno) & SEQ_MASK) {  in efx_mcdi_read_response_header()
232 			  respseq, mcdi->seqno);  in efx_mcdi_read_response_header()
233 		mcdi->resprc = -EIO;  in efx_mcdi_read_response_header()
235 		efx->type->mcdi_read_response(efx, &hdr, mcdi->resp_hdr_len, 4);  in efx_mcdi_read_response_header()
236 		mcdi->resprc =  in efx_mcdi_read_response_header()
239 		mcdi->resprc = 0;  in efx_mcdi_read_response_header()
245 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_poll_once()  local
251 	spin_lock_bh(&mcdi->iface_lock);  in efx_mcdi_poll_once()
253 	spin_unlock_bh(&mcdi->iface_lock);  in efx_mcdi_poll_once()
260 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_poll()  local
268 		spin_lock_bh(&mcdi->iface_lock);  in efx_mcdi_poll()
269 		mcdi->resprc = rc;  in efx_mcdi_poll()
270 		mcdi->resp_hdr_len = 0;  in efx_mcdi_poll()
271 		mcdi->resp_data_len = 0;  in efx_mcdi_poll()
272 		spin_unlock_bh(&mcdi->iface_lock);  in efx_mcdi_poll()
309 	if (!efx->mcdi)  in efx_mcdi_poll_reboot()
315 static bool efx_mcdi_acquire_async(struct efx_mcdi_iface *mcdi)  in efx_mcdi_acquire_async()  argument
317 	return cmpxchg(&mcdi->state,  in efx_mcdi_acquire_async()
322 static void efx_mcdi_acquire_sync(struct efx_mcdi_iface *mcdi)  in efx_mcdi_acquire_sync()  argument
327 	wait_event(mcdi->wq,  in efx_mcdi_acquire_sync()
328 		   cmpxchg(&mcdi->state,  in efx_mcdi_acquire_sync()
335 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_await_completion()  local
337 	if (wait_event_timeout(mcdi->wq, mcdi->state == MCDI_STATE_COMPLETED,  in efx_mcdi_await_completion()
349 	if (mcdi->mode == MCDI_MODE_POLL)  in efx_mcdi_await_completion()
358 static bool efx_mcdi_complete_sync(struct efx_mcdi_iface *mcdi)  in efx_mcdi_complete_sync()  argument
360 	if (cmpxchg(&mcdi->state,  in efx_mcdi_complete_sync()
363 		wake_up(&mcdi->wq);  in efx_mcdi_complete_sync()
370 static void efx_mcdi_release(struct efx_mcdi_iface *mcdi)  in efx_mcdi_release()  argument
372 	if (mcdi->mode == MCDI_MODE_EVENTS) {  in efx_mcdi_release()
374 		struct efx_nic *efx = mcdi->efx;  in efx_mcdi_release()
377 		spin_lock_bh(&mcdi->async_lock);  in efx_mcdi_release()
379 			&mcdi->async_list, struct efx_mcdi_async_param, list);  in efx_mcdi_release()
381 			mcdi->state = MCDI_STATE_RUNNING_ASYNC;  in efx_mcdi_release()
385 			mod_timer(&mcdi->async_timer,  in efx_mcdi_release()
388 		spin_unlock_bh(&mcdi->async_lock);  in efx_mcdi_release()
394 	mcdi->state = MCDI_STATE_QUIESCENT;  in efx_mcdi_release()
395 	wake_up(&mcdi->wq);  in efx_mcdi_release()
403 static bool efx_mcdi_complete_async(struct efx_mcdi_iface *mcdi, bool timeout)  in efx_mcdi_complete_async()  argument
405 	struct efx_nic *efx = mcdi->efx;  in efx_mcdi_complete_async()
412 	if (cmpxchg(&mcdi->state,  in efx_mcdi_complete_async()
417 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_complete_async()
422 		++mcdi->seqno;  in efx_mcdi_complete_async()
423 		++mcdi->credits;  in efx_mcdi_complete_async()
428 		rc = mcdi->resprc;  in efx_mcdi_complete_async()
429 		hdr_len = mcdi->resp_hdr_len;  in efx_mcdi_complete_async()
430 		data_len = mcdi->resp_data_len;  in efx_mcdi_complete_async()
432 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_complete_async()
439 		del_timer_sync(&mcdi->async_timer);  in efx_mcdi_complete_async()
441 	spin_lock(&mcdi->async_lock);  in efx_mcdi_complete_async()
442 	async = list_first_entry(&mcdi->async_list,  in efx_mcdi_complete_async()
445 	spin_unlock(&mcdi->async_lock);  in efx_mcdi_complete_async()
460 	efx_mcdi_release(mcdi);  in efx_mcdi_complete_async()
468 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_cpl()  local
471 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_ev_cpl()
473 	if ((seqno ^ mcdi->seqno) & SEQ_MASK) {  in efx_mcdi_ev_cpl()
474 		if (mcdi->credits)  in efx_mcdi_ev_cpl()
476 			--mcdi->credits;  in efx_mcdi_ev_cpl()
480 				  "seq 0x%x\n", seqno, mcdi->seqno);  in efx_mcdi_ev_cpl()
486 			mcdi->resprc = efx_mcdi_errno(mcdi_err);  in efx_mcdi_ev_cpl()
487 			mcdi->resp_hdr_len = 4;  in efx_mcdi_ev_cpl()
488 			mcdi->resp_data_len = datalen;  in efx_mcdi_ev_cpl()
494 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_ev_cpl()
497 		if (!efx_mcdi_complete_async(mcdi, false))  in efx_mcdi_ev_cpl()
498 			(void) efx_mcdi_complete_sync(mcdi);  in efx_mcdi_ev_cpl()
511 	struct efx_mcdi_iface *mcdi = (struct efx_mcdi_iface *)context;  in efx_mcdi_timeout_async()  local
513 	efx_mcdi_complete_async(mcdi, true);  in efx_mcdi_timeout_async()
536 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in _efx_mcdi_rpc_finish()  local
540 	if (mcdi->mode == MCDI_MODE_POLL)  in _efx_mcdi_rpc_finish()
548 			  cmd, (int)inlen, mcdi->mode);  in _efx_mcdi_rpc_finish()
550 		if (mcdi->mode == MCDI_MODE_EVENTS && efx_mcdi_poll_once(efx)) {  in _efx_mcdi_rpc_finish()
562 		spin_lock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
563 		++mcdi->seqno;  in _efx_mcdi_rpc_finish()
564 		++mcdi->credits;  in _efx_mcdi_rpc_finish()
565 		spin_unlock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
578 		spin_lock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
579 		rc = mcdi->resprc;  in _efx_mcdi_rpc_finish()
580 		hdr_len = mcdi->resp_hdr_len;  in _efx_mcdi_rpc_finish()
581 		data_len = mcdi->resp_data_len;  in _efx_mcdi_rpc_finish()
583 		spin_unlock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
608 			mcdi->new_epoch = true;  in _efx_mcdi_rpc_finish()
612 	efx_mcdi_release(mcdi);  in _efx_mcdi_rpc_finish()
662 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_rpc_start()  local
672 	if (mcdi->mode == MCDI_MODE_FAIL)  in efx_mcdi_rpc_start()
675 	efx_mcdi_acquire_sync(mcdi);  in efx_mcdi_rpc_start()
686 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in _efx_mcdi_rpc_async()  local
710 	spin_lock_bh(&mcdi->async_lock);  in _efx_mcdi_rpc_async()
712 	if (mcdi->mode == MCDI_MODE_EVENTS) {  in _efx_mcdi_rpc_async()
713 		list_add_tail(&async->list, &mcdi->async_list);  in _efx_mcdi_rpc_async()
718 		if (mcdi->async_list.next == &async->list &&  in _efx_mcdi_rpc_async()
719 		    efx_mcdi_acquire_async(mcdi)) {  in _efx_mcdi_rpc_async()
721 			mod_timer(&mcdi->async_timer,  in _efx_mcdi_rpc_async()
729 	spin_unlock_bh(&mcdi->async_lock);  in _efx_mcdi_rpc_async()
809 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_mode_poll()  local
811 	if (!efx->mcdi)  in efx_mcdi_mode_poll()
814 	mcdi = efx_mcdi(efx);  in efx_mcdi_mode_poll()
819 	if (mcdi->mode == MCDI_MODE_POLL || mcdi->mode == MCDI_MODE_FAIL)  in efx_mcdi_mode_poll()
830 	mcdi->mode = MCDI_MODE_POLL;  in efx_mcdi_mode_poll()
832 	efx_mcdi_complete_sync(mcdi);  in efx_mcdi_mode_poll()
841 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_flush_async()  local
843 	if (!efx->mcdi)  in efx_mcdi_flush_async()
846 	mcdi = efx_mcdi(efx);  in efx_mcdi_flush_async()
849 	BUG_ON(mcdi->mode == MCDI_MODE_EVENTS);  in efx_mcdi_flush_async()
851 	del_timer_sync(&mcdi->async_timer);  in efx_mcdi_flush_async()
857 	if (mcdi->state == MCDI_STATE_RUNNING_ASYNC) {  in efx_mcdi_flush_async()
859 		mcdi->state = MCDI_STATE_QUIESCENT;  in efx_mcdi_flush_async()
867 	list_for_each_entry_safe(async, next, &mcdi->async_list, list) {  in efx_mcdi_flush_async()
876 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_mode_event()  local
878 	if (!efx->mcdi)  in efx_mcdi_mode_event()
881 	mcdi = efx_mcdi(efx);  in efx_mcdi_mode_event()
886 	if (mcdi->mode == MCDI_MODE_EVENTS || mcdi->mode == MCDI_MODE_FAIL)  in efx_mcdi_mode_event()
896 	efx_mcdi_acquire_sync(mcdi);  in efx_mcdi_mode_event()
897 	mcdi->mode = MCDI_MODE_EVENTS;  in efx_mcdi_mode_event()
898 	efx_mcdi_release(mcdi);  in efx_mcdi_mode_event()
903 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_death()  local
926 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_ev_death()
927 	if (efx_mcdi_complete_sync(mcdi)) {  in efx_mcdi_ev_death()
928 		if (mcdi->mode == MCDI_MODE_EVENTS) {  in efx_mcdi_ev_death()
929 			mcdi->resprc = rc;  in efx_mcdi_ev_death()
930 			mcdi->resp_hdr_len = 0;  in efx_mcdi_ev_death()
931 			mcdi->resp_data_len = 0;  in efx_mcdi_ev_death()
932 			++mcdi->credits;  in efx_mcdi_ev_death()
943 		mcdi->new_epoch = true;  in efx_mcdi_ev_death()
949 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_ev_death()
959 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_bist()  local
961 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_ev_bist()
963 	if (efx_mcdi_complete_sync(mcdi)) {  in efx_mcdi_ev_bist()
964 		if (mcdi->mode == MCDI_MODE_EVENTS) {  in efx_mcdi_ev_bist()
965 			mcdi->resprc = -EIO;  in efx_mcdi_ev_bist()
966 			mcdi->resp_hdr_len = 0;  in efx_mcdi_ev_bist()
967 			mcdi->resp_data_len = 0;  in efx_mcdi_ev_bist()
968 			++mcdi->credits;  in efx_mcdi_ev_bist()
971 	mcdi->new_epoch = true;  in efx_mcdi_ev_bist()
973 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_ev_bist()
981 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_abandon()  local
983 	if (xchg(&mcdi->mode, MCDI_MODE_FAIL) == MCDI_MODE_FAIL)  in efx_mcdi_abandon()
1164 			efx->mcdi->fn_flags =  in efx_mcdi_drv_attach()
1169 			efx->mcdi->fn_flags =  in efx_mcdi_drv_attach()
1182 	    (efx->mcdi->fn_flags &  in efx_mcdi_drv_attach()
1541 		if (efx->mcdi) {  in efx_mcdi_reset()
1542 			struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_reset()  local
1543 			mcdi->mode = MCDI_MODE_POLL;  in efx_mcdi_reset()