465 efx_farch_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index)  in efx_farch_build_rx_desc()  argument
470 	rxd = efx_rx_desc(rx_queue, index);  in efx_farch_build_rx_desc()
471 	rx_buf = efx_rx_buffer(rx_queue, index);  in efx_farch_build_rx_desc()
475 			     rx_queue->efx->type->rx_buffer_padding,  in efx_farch_build_rx_desc()
483 void efx_farch_rx_write(struct efx_rx_queue *rx_queue)  in efx_farch_rx_write()  argument
485 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_rx_write()
489 	while (rx_queue->notified_count != rx_queue->added_count) {  in efx_farch_rx_write()
491 			rx_queue,  in efx_farch_rx_write()
492 			rx_queue->notified_count & rx_queue->ptr_mask);  in efx_farch_rx_write()
493 		++rx_queue->notified_count;  in efx_farch_rx_write()
497 	write_ptr = rx_queue->added_count & rx_queue->ptr_mask;  in efx_farch_rx_write()
500 			efx_rx_queue_index(rx_queue));  in efx_farch_rx_write()
503 int efx_farch_rx_probe(struct efx_rx_queue *rx_queue)  in efx_farch_rx_probe()  argument
505 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_rx_probe()
508 	entries = rx_queue->ptr_mask + 1;  in efx_farch_rx_probe()
509 	return efx_alloc_special_buffer(efx, &rx_queue->rxd,  in efx_farch_rx_probe()
513 void efx_farch_rx_init(struct efx_rx_queue *rx_queue)  in efx_farch_rx_init()  argument
516 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_rx_init()
530 		  efx_rx_queue_index(rx_queue), rx_queue->rxd.index,  in efx_farch_rx_init()
531 		  rx_queue->rxd.index + rx_queue->rxd.entries - 1);  in efx_farch_rx_init()
533 	rx_queue->scatter_n = 0;  in efx_farch_rx_init()
536 	efx_init_special_buffer(efx, &rx_queue->rxd);  in efx_farch_rx_init()
542 			      FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,  in efx_farch_rx_init()
544 			      efx_rx_queue_channel(rx_queue)->channel,  in efx_farch_rx_init()
547 			      efx_rx_queue_index(rx_queue),  in efx_farch_rx_init()
549 			      __ffs(rx_queue->rxd.entries),  in efx_farch_rx_init()
554 			 efx_rx_queue_index(rx_queue));  in efx_farch_rx_init()
557 static void efx_farch_flush_rx_queue(struct efx_rx_queue *rx_queue)  in efx_farch_flush_rx_queue()  argument
559 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_flush_rx_queue()
565 			     efx_rx_queue_index(rx_queue));  in efx_farch_flush_rx_queue()
569 void efx_farch_rx_fini(struct efx_rx_queue *rx_queue)  in efx_farch_rx_fini()  argument
572 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_rx_fini()
577 			 efx_rx_queue_index(rx_queue));  in efx_farch_rx_fini()
580 	efx_fini_special_buffer(efx, &rx_queue->rxd);  in efx_farch_rx_fini()
584 void efx_farch_rx_remove(struct efx_rx_queue *rx_queue)  in efx_farch_rx_remove()  argument
586 	efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd);  in efx_farch_rx_remove()
656 	struct efx_rx_queue *rx_queue;  in efx_farch_do_flush()  local
664 		efx_for_each_channel_rx_queue(rx_queue, channel) {  in efx_farch_do_flush()
665 			rx_queue->flush_pending = true;  in efx_farch_do_flush()
686 			efx_for_each_channel_rx_queue(rx_queue, channel) {  in efx_farch_do_flush()
691 				if (rx_queue->flush_pending) {  in efx_farch_do_flush()
692 					rx_queue->flush_pending = false;  in efx_farch_do_flush()
695 					efx_farch_flush_rx_queue(rx_queue);  in efx_farch_do_flush()
726 	struct efx_rx_queue *rx_queue;  in efx_farch_fini_dmaq()  local
739 			efx_for_each_channel_rx_queue(rx_queue, channel)  in efx_farch_fini_dmaq()
740 				efx_farch_rx_fini(rx_queue);  in efx_farch_fini_dmaq()
873 static u16 efx_farch_handle_rx_not_ok(struct efx_rx_queue *rx_queue,  in efx_farch_handle_rx_not_ok()  argument
876 	struct efx_channel *channel = efx_rx_queue_channel(rx_queue);  in efx_farch_handle_rx_not_ok()
877 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_handle_rx_not_ok()
928 			  efx_rx_queue_index(rx_queue), EFX_QWORD_VAL(*event),  in efx_farch_handle_rx_not_ok()
953 efx_farch_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)  in efx_farch_handle_rx_bad_index()  argument
955 	struct efx_channel *channel = efx_rx_queue_channel(rx_queue);  in efx_farch_handle_rx_bad_index()
956 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_handle_rx_bad_index()
959 	if (rx_queue->scatter_n &&  in efx_farch_handle_rx_bad_index()
960 	    index == ((rx_queue->removed_count + rx_queue->scatter_n - 1) &  in efx_farch_handle_rx_bad_index()
961 		      rx_queue->ptr_mask)) {  in efx_farch_handle_rx_bad_index()
966 	expected = rx_queue->removed_count & rx_queue->ptr_mask;  in efx_farch_handle_rx_bad_index()
967 	dropped = (index - expected) & rx_queue->ptr_mask;  in efx_farch_handle_rx_bad_index()
992 	struct efx_rx_queue *rx_queue;  in efx_farch_handle_rx_event()  local
1003 	rx_queue = efx_channel_get_rx_queue(channel);  in efx_farch_handle_rx_event()
1006 	expected_ptr = ((rx_queue->removed_count + rx_queue->scatter_n) &  in efx_farch_handle_rx_event()
1007 			rx_queue->ptr_mask);  in efx_farch_handle_rx_event()
1011 	    unlikely(rx_ev_sop != (rx_queue->scatter_n == 0))) {  in efx_farch_handle_rx_event()
1013 		    !efx_farch_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr))  in efx_farch_handle_rx_event()
1017 		if (rx_queue->scatter_n) {  in efx_farch_handle_rx_event()
1019 				rx_queue,  in efx_farch_handle_rx_event()
1020 				rx_queue->removed_count & rx_queue->ptr_mask,  in efx_farch_handle_rx_event()
1021 				rx_queue->scatter_n, 0, EFX_RX_PKT_DISCARD);  in efx_farch_handle_rx_event()
1022 			rx_queue->removed_count += rx_queue->scatter_n;  in efx_farch_handle_rx_event()
1023 			rx_queue->scatter_n = 0;  in efx_farch_handle_rx_event()
1033 				rx_queue,  in efx_farch_handle_rx_event()
1034 				rx_queue->removed_count & rx_queue->ptr_mask,  in efx_farch_handle_rx_event()
1036 			++rx_queue->removed_count;  in efx_farch_handle_rx_event()
1041 	++rx_queue->scatter_n;  in efx_farch_handle_rx_event()
1066 		flags = efx_farch_handle_rx_not_ok(rx_queue, event);  in efx_farch_handle_rx_event()
1084 	efx_rx_packet(rx_queue,  in efx_farch_handle_rx_event()
1085 		      rx_queue->removed_count & rx_queue->ptr_mask,  in efx_farch_handle_rx_event()
1086 		      rx_queue->scatter_n, rx_ev_byte_cnt, flags);  in efx_farch_handle_rx_event()
1087 	rx_queue->removed_count += rx_queue->scatter_n;  in efx_farch_handle_rx_event()
1088 	rx_queue->scatter_n = 0;  in efx_farch_handle_rx_event()
1120 	struct efx_rx_queue *rx_queue;  in efx_farch_handle_rx_flush_done()  local
1131 	rx_queue = efx_channel_get_rx_queue(channel);  in efx_farch_handle_rx_flush_done()
1136 		rx_queue->flush_pending = true;  in efx_farch_handle_rx_flush_done()
1139 		efx_farch_magic_event(efx_rx_queue_channel(rx_queue),  in efx_farch_handle_rx_flush_done()
1140 				      EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue));  in efx_farch_handle_rx_flush_done()
1162 	struct efx_rx_queue *rx_queue =  in efx_farch_handle_generated_event()  local
1172 	} else if (rx_queue && magic == EFX_CHANNEL_MAGIC_FILL(rx_queue)) {  in efx_farch_handle_generated_event()
1176 		efx_fast_push_rx_descriptors(rx_queue, true);  in efx_farch_handle_generated_event()
1177 	} else if (rx_queue && magic == EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue)) {  in efx_farch_handle_generated_event()
1420 void efx_farch_rx_defer_refill(struct efx_rx_queue *rx_queue)  in efx_farch_rx_defer_refill()  argument
1422 	efx_farch_magic_event(efx_rx_queue_channel(rx_queue),  in efx_farch_rx_defer_refill()
1423 			      EFX_CHANNEL_MAGIC_FILL(rx_queue));  in efx_farch_rx_defer_refill()