67 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_init()  local
71 	efx->mcdi = kzalloc(sizeof(*efx->mcdi), GFP_KERNEL);  in efx_mcdi_init()
72 	if (!efx->mcdi)  in efx_mcdi_init()
75 	mcdi = efx_mcdi(efx);  in efx_mcdi_init()
76 	mcdi->efx = efx;  in efx_mcdi_init()
79 	mcdi->logging_buffer = (char *)__get_free_page(GFP_KERNEL);  in efx_mcdi_init()
80 	if (!mcdi->logging_buffer)  in efx_mcdi_init()
82 	mcdi->logging_enabled = mcdi_logging_default;  in efx_mcdi_init()
84 	init_waitqueue_head(&mcdi->wq);  in efx_mcdi_init()
85 	spin_lock_init(&mcdi->iface_lock);  in efx_mcdi_init()
86 	mcdi->state = MCDI_STATE_QUIESCENT;  in efx_mcdi_init()
87 	mcdi->mode = MCDI_MODE_POLL;  in efx_mcdi_init()
88 	spin_lock_init(&mcdi->async_lock);  in efx_mcdi_init()
89 	INIT_LIST_HEAD(&mcdi->async_list);  in efx_mcdi_init()
90 	setup_timer(&mcdi->async_timer, efx_mcdi_timeout_async,  in efx_mcdi_init()
91 		    (unsigned long)mcdi);  in efx_mcdi_init()
94 	mcdi->new_epoch = true;  in efx_mcdi_init()
115 	if (efx->mcdi->fn_flags &  in efx_mcdi_init()
122 	free_page((unsigned long)mcdi->logging_buffer);  in efx_mcdi_init()
125 	kfree(efx->mcdi);  in efx_mcdi_init()
126 	efx->mcdi = NULL;  in efx_mcdi_init()
133 	if (!efx->mcdi)  in efx_mcdi_fini()
136 	BUG_ON(efx->mcdi->iface.state != MCDI_STATE_QUIESCENT);  in efx_mcdi_fini()
142 	free_page((unsigned long)efx->mcdi->iface.logging_buffer);  in efx_mcdi_fini()
145 	kfree(efx->mcdi);  in efx_mcdi_fini()
151 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_send_request()  local
153 	char *buf = mcdi->logging_buffer; /* page-sized */  in efx_mcdi_send_request()
159 	BUG_ON(mcdi->state == MCDI_STATE_QUIESCENT);  in efx_mcdi_send_request()
162 	spin_lock_bh(&mcdi->iface_lock);  in efx_mcdi_send_request()
163 	++mcdi->seqno;  in efx_mcdi_send_request()
164 	spin_unlock_bh(&mcdi->iface_lock);  in efx_mcdi_send_request()
166 	seqno = mcdi->seqno & SEQ_MASK;  in efx_mcdi_send_request()
168 	if (mcdi->mode == MCDI_MODE_EVENTS)  in efx_mcdi_send_request()
180 				     MCDI_HEADER_NOT_EPOCH, !mcdi->new_epoch);  in efx_mcdi_send_request()
192 				     MCDI_HEADER_NOT_EPOCH, !mcdi->new_epoch);  in efx_mcdi_send_request()
200 	if (mcdi->logging_enabled && !WARN_ON_ONCE(!buf)) {  in efx_mcdi_send_request()
226 	mcdi->new_epoch = false;  in efx_mcdi_send_request()
263 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_read_response_header()  local
266 	char *buf = mcdi->logging_buffer; /* page-sized */  in efx_mcdi_read_response_header()
276 		mcdi->resp_hdr_len = 4;  in efx_mcdi_read_response_header()
277 		mcdi->resp_data_len = EFX_DWORD_FIELD(hdr, MCDI_HEADER_DATALEN);  in efx_mcdi_read_response_header()
280 		mcdi->resp_hdr_len = 8;  in efx_mcdi_read_response_header()
281 		mcdi->resp_data_len =  in efx_mcdi_read_response_header()
286 	if (mcdi->logging_enabled && !WARN_ON_ONCE(!buf)) {  in efx_mcdi_read_response_header()
291 		WARN_ON_ONCE(mcdi->resp_hdr_len % 4);  in efx_mcdi_read_response_header()
292 		hdr_len = mcdi->resp_hdr_len / 4;  in efx_mcdi_read_response_header()
296 		data_len = DIV_ROUND_UP(mcdi->resp_data_len, 4);  in efx_mcdi_read_response_header()
309 					mcdi->resp_hdr_len + (i * 4), 4);  in efx_mcdi_read_response_header()
318 	if (error && mcdi->resp_data_len == 0) {  in efx_mcdi_read_response_header()
320 		mcdi->resprc = -EIO;  in efx_mcdi_read_response_header()
321 	} else if ((respseq ^ mcdi->seqno) & SEQ_MASK) {  in efx_mcdi_read_response_header()
324 			  respseq, mcdi->seqno);  in efx_mcdi_read_response_header()
325 		mcdi->resprc = -EIO;  in efx_mcdi_read_response_header()
327 		efx->type->mcdi_read_response(efx, &hdr, mcdi->resp_hdr_len, 4);  in efx_mcdi_read_response_header()
328 		mcdi->resprc =  in efx_mcdi_read_response_header()
331 		mcdi->resprc = 0;  in efx_mcdi_read_response_header()
337 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_poll_once()  local
343 	spin_lock_bh(&mcdi->iface_lock);  in efx_mcdi_poll_once()
345 	spin_unlock_bh(&mcdi->iface_lock);  in efx_mcdi_poll_once()
352 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_poll()  local
360 		spin_lock_bh(&mcdi->iface_lock);  in efx_mcdi_poll()
361 		mcdi->resprc = rc;  in efx_mcdi_poll()
362 		mcdi->resp_hdr_len = 0;  in efx_mcdi_poll()
363 		mcdi->resp_data_len = 0;  in efx_mcdi_poll()
364 		spin_unlock_bh(&mcdi->iface_lock);  in efx_mcdi_poll()
401 	if (!efx->mcdi)  in efx_mcdi_poll_reboot()
407 static bool efx_mcdi_acquire_async(struct efx_mcdi_iface *mcdi)  in efx_mcdi_acquire_async()  argument
409 	return cmpxchg(&mcdi->state,  in efx_mcdi_acquire_async()
414 static void efx_mcdi_acquire_sync(struct efx_mcdi_iface *mcdi)  in efx_mcdi_acquire_sync()  argument
419 	wait_event(mcdi->wq,  in efx_mcdi_acquire_sync()
420 		   cmpxchg(&mcdi->state,  in efx_mcdi_acquire_sync()
427 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_await_completion()  local
429 	if (wait_event_timeout(mcdi->wq, mcdi->state == MCDI_STATE_COMPLETED,  in efx_mcdi_await_completion()
441 	if (mcdi->mode == MCDI_MODE_POLL)  in efx_mcdi_await_completion()
450 static bool efx_mcdi_complete_sync(struct efx_mcdi_iface *mcdi)  in efx_mcdi_complete_sync()  argument
452 	if (cmpxchg(&mcdi->state,  in efx_mcdi_complete_sync()
455 		wake_up(&mcdi->wq);  in efx_mcdi_complete_sync()
462 static void efx_mcdi_release(struct efx_mcdi_iface *mcdi)  in efx_mcdi_release()  argument
464 	if (mcdi->mode == MCDI_MODE_EVENTS) {  in efx_mcdi_release()
466 		struct efx_nic *efx = mcdi->efx;  in efx_mcdi_release()
469 		spin_lock_bh(&mcdi->async_lock);  in efx_mcdi_release()
471 			&mcdi->async_list, struct efx_mcdi_async_param, list);  in efx_mcdi_release()
473 			mcdi->state = MCDI_STATE_RUNNING_ASYNC;  in efx_mcdi_release()
477 			mod_timer(&mcdi->async_timer,  in efx_mcdi_release()
480 		spin_unlock_bh(&mcdi->async_lock);  in efx_mcdi_release()
486 	mcdi->state = MCDI_STATE_QUIESCENT;  in efx_mcdi_release()
487 	wake_up(&mcdi->wq);  in efx_mcdi_release()
495 static bool efx_mcdi_complete_async(struct efx_mcdi_iface *mcdi, bool timeout)  in efx_mcdi_complete_async()  argument
497 	struct efx_nic *efx = mcdi->efx;  in efx_mcdi_complete_async()
504 	if (cmpxchg(&mcdi->state,  in efx_mcdi_complete_async()
509 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_complete_async()
514 		++mcdi->seqno;  in efx_mcdi_complete_async()
515 		++mcdi->credits;  in efx_mcdi_complete_async()
520 		rc = mcdi->resprc;  in efx_mcdi_complete_async()
521 		hdr_len = mcdi->resp_hdr_len;  in efx_mcdi_complete_async()
522 		data_len = mcdi->resp_data_len;  in efx_mcdi_complete_async()
524 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_complete_async()
531 		del_timer_sync(&mcdi->async_timer);  in efx_mcdi_complete_async()
533 	spin_lock(&mcdi->async_lock);  in efx_mcdi_complete_async()
534 	async = list_first_entry(&mcdi->async_list,  in efx_mcdi_complete_async()
537 	spin_unlock(&mcdi->async_lock);  in efx_mcdi_complete_async()
552 	efx_mcdi_release(mcdi);  in efx_mcdi_complete_async()
560 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_cpl()  local
563 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_ev_cpl()
565 	if ((seqno ^ mcdi->seqno) & SEQ_MASK) {  in efx_mcdi_ev_cpl()
566 		if (mcdi->credits)  in efx_mcdi_ev_cpl()
568 			--mcdi->credits;  in efx_mcdi_ev_cpl()
572 				  "seq 0x%x\n", seqno, mcdi->seqno);  in efx_mcdi_ev_cpl()
578 			mcdi->resprc = efx_mcdi_errno(mcdi_err);  in efx_mcdi_ev_cpl()
579 			mcdi->resp_hdr_len = 4;  in efx_mcdi_ev_cpl()
580 			mcdi->resp_data_len = datalen;  in efx_mcdi_ev_cpl()
586 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_ev_cpl()
589 		if (!efx_mcdi_complete_async(mcdi, false))  in efx_mcdi_ev_cpl()
590 			(void) efx_mcdi_complete_sync(mcdi);  in efx_mcdi_ev_cpl()
603 	struct efx_mcdi_iface *mcdi = (struct efx_mcdi_iface *)context;  in efx_mcdi_timeout_async()  local
605 	efx_mcdi_complete_async(mcdi, true);  in efx_mcdi_timeout_async()
628 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in _efx_mcdi_rpc_finish()  local
632 	if (mcdi->mode == MCDI_MODE_POLL)  in _efx_mcdi_rpc_finish()
640 			  cmd, (int)inlen, mcdi->mode);  in _efx_mcdi_rpc_finish()
642 		if (mcdi->mode == MCDI_MODE_EVENTS && efx_mcdi_poll_once(efx)) {  in _efx_mcdi_rpc_finish()
654 		spin_lock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
655 		++mcdi->seqno;  in _efx_mcdi_rpc_finish()
656 		++mcdi->credits;  in _efx_mcdi_rpc_finish()
657 		spin_unlock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
670 		spin_lock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
671 		rc = mcdi->resprc;  in _efx_mcdi_rpc_finish()
672 		hdr_len = mcdi->resp_hdr_len;  in _efx_mcdi_rpc_finish()
673 		data_len = mcdi->resp_data_len;  in _efx_mcdi_rpc_finish()
675 		spin_unlock_bh(&mcdi->iface_lock);  in _efx_mcdi_rpc_finish()
700 			mcdi->new_epoch = true;  in _efx_mcdi_rpc_finish()
704 	efx_mcdi_release(mcdi);  in _efx_mcdi_rpc_finish()
754 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_rpc_start()  local
764 	if (mcdi->mode == MCDI_MODE_FAIL)  in efx_mcdi_rpc_start()
767 	efx_mcdi_acquire_sync(mcdi);  in efx_mcdi_rpc_start()
778 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in _efx_mcdi_rpc_async()  local
802 	spin_lock_bh(&mcdi->async_lock);  in _efx_mcdi_rpc_async()
804 	if (mcdi->mode == MCDI_MODE_EVENTS) {  in _efx_mcdi_rpc_async()
805 		list_add_tail(&async->list, &mcdi->async_list);  in _efx_mcdi_rpc_async()
810 		if (mcdi->async_list.next == &async->list &&  in _efx_mcdi_rpc_async()
811 		    efx_mcdi_acquire_async(mcdi)) {  in _efx_mcdi_rpc_async()
813 			mod_timer(&mcdi->async_timer,  in _efx_mcdi_rpc_async()
821 	spin_unlock_bh(&mcdi->async_lock);  in _efx_mcdi_rpc_async()
901 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_mode_poll()  local
903 	if (!efx->mcdi)  in efx_mcdi_mode_poll()
906 	mcdi = efx_mcdi(efx);  in efx_mcdi_mode_poll()
911 	if (mcdi->mode == MCDI_MODE_POLL || mcdi->mode == MCDI_MODE_FAIL)  in efx_mcdi_mode_poll()
922 	mcdi->mode = MCDI_MODE_POLL;  in efx_mcdi_mode_poll()
924 	efx_mcdi_complete_sync(mcdi);  in efx_mcdi_mode_poll()
933 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_flush_async()  local
935 	if (!efx->mcdi)  in efx_mcdi_flush_async()
938 	mcdi = efx_mcdi(efx);  in efx_mcdi_flush_async()
941 	BUG_ON(mcdi->mode == MCDI_MODE_EVENTS);  in efx_mcdi_flush_async()
943 	del_timer_sync(&mcdi->async_timer);  in efx_mcdi_flush_async()
949 	if (mcdi->state == MCDI_STATE_RUNNING_ASYNC) {  in efx_mcdi_flush_async()
951 		mcdi->state = MCDI_STATE_QUIESCENT;  in efx_mcdi_flush_async()
959 	list_for_each_entry_safe(async, next, &mcdi->async_list, list) {  in efx_mcdi_flush_async()
968 	struct efx_mcdi_iface *mcdi;  in efx_mcdi_mode_event()  local
970 	if (!efx->mcdi)  in efx_mcdi_mode_event()
973 	mcdi = efx_mcdi(efx);  in efx_mcdi_mode_event()
978 	if (mcdi->mode == MCDI_MODE_EVENTS || mcdi->mode == MCDI_MODE_FAIL)  in efx_mcdi_mode_event()
988 	efx_mcdi_acquire_sync(mcdi);  in efx_mcdi_mode_event()
989 	mcdi->mode = MCDI_MODE_EVENTS;  in efx_mcdi_mode_event()
990 	efx_mcdi_release(mcdi);  in efx_mcdi_mode_event()
995 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_death()  local
1018 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_ev_death()
1019 	if (efx_mcdi_complete_sync(mcdi)) {  in efx_mcdi_ev_death()
1020 		if (mcdi->mode == MCDI_MODE_EVENTS) {  in efx_mcdi_ev_death()
1021 			mcdi->resprc = rc;  in efx_mcdi_ev_death()
1022 			mcdi->resp_hdr_len = 0;  in efx_mcdi_ev_death()
1023 			mcdi->resp_data_len = 0;  in efx_mcdi_ev_death()
1024 			++mcdi->credits;  in efx_mcdi_ev_death()
1046 		mcdi->new_epoch = true;  in efx_mcdi_ev_death()
1052 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_ev_death()
1062 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_ev_bist()  local
1064 	spin_lock(&mcdi->iface_lock);  in efx_mcdi_ev_bist()
1066 	if (efx_mcdi_complete_sync(mcdi)) {  in efx_mcdi_ev_bist()
1067 		if (mcdi->mode == MCDI_MODE_EVENTS) {  in efx_mcdi_ev_bist()
1068 			mcdi->resprc = -EIO;  in efx_mcdi_ev_bist()
1069 			mcdi->resp_hdr_len = 0;  in efx_mcdi_ev_bist()
1070 			mcdi->resp_data_len = 0;  in efx_mcdi_ev_bist()
1071 			++mcdi->credits;  in efx_mcdi_ev_bist()
1074 	mcdi->new_epoch = true;  in efx_mcdi_ev_bist()
1076 	spin_unlock(&mcdi->iface_lock);  in efx_mcdi_ev_bist()
1084 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_abandon()  local
1086 	if (xchg(&mcdi->mode, MCDI_MODE_FAIL) == MCDI_MODE_FAIL)  in efx_mcdi_abandon()
1275 			efx->mcdi->fn_flags =  in efx_mcdi_drv_attach()
1280 			efx->mcdi->fn_flags =  in efx_mcdi_drv_attach()
1652 		if (efx->mcdi) {  in efx_mcdi_reset()
1653 			struct efx_mcdi_iface *mcdi = efx_mcdi(efx);  in efx_mcdi_reset()  local
1654 			mcdi->mode = MCDI_MODE_POLL;  in efx_mcdi_reset()