204 static void efx_remove_channel(struct efx_channel *channel);
208 static void efx_init_napi_channel(struct efx_channel *channel);
210 static void efx_fini_napi_channel(struct efx_channel *channel);
246 static int efx_process_channel(struct efx_channel *channel, int budget)  in efx_process_channel()  argument
251 	if (unlikely(!channel->enabled))  in efx_process_channel()
254 	efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_process_channel()
259 	spent = efx_nic_process_eventq(channel, budget);  in efx_process_channel()
260 	if (spent && efx_channel_has_rx_queue(channel)) {  in efx_process_channel()
262 			efx_channel_get_rx_queue(channel);  in efx_process_channel()
264 		efx_rx_flush_packet(channel);  in efx_process_channel()
269 	efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_process_channel()
286 	struct efx_channel *channel =  in efx_poll()  local
288 	struct efx_nic *efx = channel->efx;  in efx_poll()
291 	if (!efx_channel_lock_napi(channel))  in efx_poll()
296 		   channel->channel, raw_smp_processor_id());  in efx_poll()
298 	spent = efx_process_channel(channel, budget);  in efx_poll()
301 		if (efx_channel_has_rx_queue(channel) &&  in efx_poll()
303 		    unlikely(++channel->irq_count == 1000)) {  in efx_poll()
304 			if (unlikely(channel->irq_mod_score <  in efx_poll()
306 				if (channel->irq_moderation > 1) {  in efx_poll()
307 					channel->irq_moderation -= 1;  in efx_poll()
308 					efx->type->push_irq_moderation(channel);  in efx_poll()
310 			} else if (unlikely(channel->irq_mod_score >  in efx_poll()
312 				if (channel->irq_moderation <  in efx_poll()
314 					channel->irq_moderation += 1;  in efx_poll()
315 					efx->type->push_irq_moderation(channel);  in efx_poll()
318 			channel->irq_count = 0;  in efx_poll()
319 			channel->irq_mod_score = 0;  in efx_poll()
322 		efx_filter_rfs_expire(channel);  in efx_poll()
330 		efx_nic_eventq_read_ack(channel);  in efx_poll()
333 	efx_channel_unlock_napi(channel);  in efx_poll()
342 static int efx_probe_eventq(struct efx_channel *channel)  in efx_probe_eventq()  argument
344 	struct efx_nic *efx = channel->efx;  in efx_probe_eventq()
348 		  "chan %d create event queue\n", channel->channel);  in efx_probe_eventq()
354 	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;  in efx_probe_eventq()
356 	return efx_nic_probe_eventq(channel);  in efx_probe_eventq()
360 static int efx_init_eventq(struct efx_channel *channel)  in efx_init_eventq()  argument
362 	struct efx_nic *efx = channel->efx;  in efx_init_eventq()
365 	EFX_WARN_ON_PARANOID(channel->eventq_init);  in efx_init_eventq()
368 		  "chan %d init event queue\n", channel->channel);  in efx_init_eventq()
370 	rc = efx_nic_init_eventq(channel);  in efx_init_eventq()
372 		efx->type->push_irq_moderation(channel);  in efx_init_eventq()
373 		channel->eventq_read_ptr = 0;  in efx_init_eventq()
374 		channel->eventq_init = true;  in efx_init_eventq()
380 void efx_start_eventq(struct efx_channel *channel)  in efx_start_eventq()  argument
382 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,  in efx_start_eventq()
383 		  "chan %d start event queue\n", channel->channel);  in efx_start_eventq()
386 	channel->enabled = true;  in efx_start_eventq()
389 	efx_channel_enable(channel);  in efx_start_eventq()
390 	napi_enable(&channel->napi_str);  in efx_start_eventq()
391 	efx_nic_eventq_read_ack(channel);  in efx_start_eventq()
395 void efx_stop_eventq(struct efx_channel *channel)  in efx_stop_eventq()  argument
397 	if (!channel->enabled)  in efx_stop_eventq()
400 	napi_disable(&channel->napi_str);  in efx_stop_eventq()
401 	while (!efx_channel_disable(channel))  in efx_stop_eventq()
403 	channel->enabled = false;  in efx_stop_eventq()
406 static void efx_fini_eventq(struct efx_channel *channel)  in efx_fini_eventq()  argument
408 	if (!channel->eventq_init)  in efx_fini_eventq()
411 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_fini_eventq()
412 		  "chan %d fini event queue\n", channel->channel);  in efx_fini_eventq()
414 	efx_nic_fini_eventq(channel);  in efx_fini_eventq()
415 	channel->eventq_init = false;  in efx_fini_eventq()
418 static void efx_remove_eventq(struct efx_channel *channel)  in efx_remove_eventq()  argument
420 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_remove_eventq()
421 		  "chan %d remove event queue\n", channel->channel);  in efx_remove_eventq()
423 	efx_nic_remove_eventq(channel);  in efx_remove_eventq()
436 	struct efx_channel *channel;  in efx_alloc_channel()  local
441 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);  in efx_alloc_channel()
442 	if (!channel)  in efx_alloc_channel()
445 	channel->efx = efx;  in efx_alloc_channel()
446 	channel->channel = i;  in efx_alloc_channel()
447 	channel->type = &efx_default_channel_type;  in efx_alloc_channel()
450 		tx_queue = &channel->tx_queue[j];  in efx_alloc_channel()
453 		tx_queue->channel = channel;  in efx_alloc_channel()
456 	rx_queue = &channel->rx_queue;  in efx_alloc_channel()
461 	return channel;  in efx_alloc_channel()
470 	struct efx_channel *channel;  in efx_copy_channel()  local
475 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);  in efx_copy_channel()
476 	if (!channel)  in efx_copy_channel()
479 	*channel = *old_channel;  in efx_copy_channel()
481 	channel->napi_dev = NULL;  in efx_copy_channel()
482 	memset(&channel->eventq, 0, sizeof(channel->eventq));  in efx_copy_channel()
485 		tx_queue = &channel->tx_queue[j];  in efx_copy_channel()
486 		if (tx_queue->channel)  in efx_copy_channel()
487 			tx_queue->channel = channel;  in efx_copy_channel()
492 	rx_queue = &channel->rx_queue;  in efx_copy_channel()
498 	return channel;  in efx_copy_channel()
501 static int efx_probe_channel(struct efx_channel *channel)  in efx_probe_channel()  argument
507 	netif_dbg(channel->efx, probe, channel->efx->net_dev,  in efx_probe_channel()
508 		  "creating channel %d\n", channel->channel);  in efx_probe_channel()
510 	rc = channel->type->pre_probe(channel);  in efx_probe_channel()
514 	rc = efx_probe_eventq(channel);  in efx_probe_channel()
518 	efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_probe_channel()
524 	efx_for_each_channel_rx_queue(rx_queue, channel) {  in efx_probe_channel()
533 	efx_remove_channel(channel);  in efx_probe_channel()
538 efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)  in efx_get_channel_name()  argument
540 	struct efx_nic *efx = channel->efx;  in efx_get_channel_name()
544 	number = channel->channel;  in efx_get_channel_name()
547 	} else if (channel->channel < efx->tx_channel_offset) {  in efx_get_channel_name()
558 	struct efx_channel *channel;  in efx_set_channel_names()  local
560 	efx_for_each_channel(channel, efx)  in efx_set_channel_names()
561 		channel->type->get_name(channel,  in efx_set_channel_names()
562 					efx->msi_context[channel->channel].name,  in efx_set_channel_names()
568 	struct efx_channel *channel;  in efx_probe_channels()  local
579 	efx_for_each_channel_rev(channel, efx) {  in efx_probe_channels()
580 		rc = efx_probe_channel(channel);  in efx_probe_channels()
584 				  channel->channel);  in efx_probe_channels()
606 	struct efx_channel *channel;  in efx_start_datapath()  local
662 	efx_for_each_channel(channel, efx) {  in efx_start_datapath()
663 		efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_start_datapath()
668 		efx_for_each_channel_rx_queue(rx_queue, channel) {  in efx_start_datapath()
671 			efx_stop_eventq(channel);  in efx_start_datapath()
673 			efx_start_eventq(channel);  in efx_start_datapath()
676 		WARN_ON(channel->rx_pkt_n_frags);  in efx_start_datapath()
687 	struct efx_channel *channel;  in efx_stop_datapath()  local
698 	efx_for_each_channel(channel, efx) {  in efx_stop_datapath()
699 		efx_for_each_channel_rx_queue(rx_queue, channel)  in efx_stop_datapath()
703 	efx_for_each_channel(channel, efx) {  in efx_stop_datapath()
710 		if (efx_channel_has_rx_queue(channel)) {  in efx_stop_datapath()
711 			efx_stop_eventq(channel);  in efx_stop_datapath()
712 			efx_start_eventq(channel);  in efx_stop_datapath()
733 	efx_for_each_channel(channel, efx) {  in efx_stop_datapath()
734 		efx_for_each_channel_rx_queue(rx_queue, channel)  in efx_stop_datapath()
736 		efx_for_each_possible_channel_tx_queue(tx_queue, channel)  in efx_stop_datapath()
741 static void efx_remove_channel(struct efx_channel *channel)  in efx_remove_channel()  argument
746 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_remove_channel()
747 		  "destroy chan %d\n", channel->channel);  in efx_remove_channel()
749 	efx_for_each_channel_rx_queue(rx_queue, channel)  in efx_remove_channel()
751 	efx_for_each_possible_channel_tx_queue(tx_queue, channel)  in efx_remove_channel()
753 	efx_remove_eventq(channel);  in efx_remove_channel()
754 	channel->type->post_remove(channel);  in efx_remove_channel()
759 	struct efx_channel *channel;  in efx_remove_channels()  local
761 	efx_for_each_channel(channel, efx)  in efx_remove_channels()
762 		efx_remove_channel(channel);  in efx_remove_channels()
768 	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;  in efx_realloc_channels()  local
780 	efx_for_each_channel(channel, efx) {  in efx_realloc_channels()
784 		if (channel->type->copy)  in efx_realloc_channels()
787 					channel->eventq.index +  in efx_realloc_channels()
788 					channel->eventq.entries);  in efx_realloc_channels()
789 		efx_for_each_channel_rx_queue(rx_queue, channel)  in efx_realloc_channels()
793 		efx_for_each_channel_tx_queue(tx_queue, channel)  in efx_realloc_channels()
806 		channel = efx->channel[i];  in efx_realloc_channels()
807 		if (channel->type->copy)  in efx_realloc_channels()
808 			channel = channel->type->copy(channel);  in efx_realloc_channels()
809 		if (!channel) {  in efx_realloc_channels()
813 		other_channel[i] = channel;  in efx_realloc_channels()
822 		channel = efx->channel[i];  in efx_realloc_channels()
823 		efx->channel[i] = other_channel[i];  in efx_realloc_channels()
824 		other_channel[i] = channel;  in efx_realloc_channels()
831 		channel = efx->channel[i];  in efx_realloc_channels()
832 		if (!channel->type->copy)  in efx_realloc_channels()
834 		rc = efx_probe_channel(channel);  in efx_realloc_channels()
837 		efx_init_napi_channel(efx->channel[i]);  in efx_realloc_channels()
843 		channel = other_channel[i];  in efx_realloc_channels()
844 		if (channel && channel->type->copy) {  in efx_realloc_channels()
845 			efx_fini_napi_channel(channel);  in efx_realloc_channels()
846 			efx_remove_channel(channel);  in efx_realloc_channels()
847 			kfree(channel);  in efx_realloc_channels()
868 		channel = efx->channel[i];  in efx_realloc_channels()
869 		efx->channel[i] = other_channel[i];  in efx_realloc_channels()
870 		other_channel[i] = channel;  in efx_realloc_channels()
888 int efx_channel_dummy_op_int(struct efx_channel *channel)  in efx_channel_dummy_op_int()  argument
893 void efx_channel_dummy_op_void(struct efx_channel *channel)  in efx_channel_dummy_op_void()  argument
1491 	struct efx_channel *channel, *end_channel;  in efx_soft_enable_interrupts()  local
1499 	efx_for_each_channel(channel, efx) {  in efx_soft_enable_interrupts()
1500 		if (!channel->type->keep_eventq) {  in efx_soft_enable_interrupts()
1501 			rc = efx_init_eventq(channel);  in efx_soft_enable_interrupts()
1505 		efx_start_eventq(channel);  in efx_soft_enable_interrupts()
1512 	end_channel = channel;  in efx_soft_enable_interrupts()
1513 	efx_for_each_channel(channel, efx) {  in efx_soft_enable_interrupts()
1514 		if (channel == end_channel)  in efx_soft_enable_interrupts()
1516 		efx_stop_eventq(channel);  in efx_soft_enable_interrupts()
1517 		if (!channel->type->keep_eventq)  in efx_soft_enable_interrupts()
1518 			efx_fini_eventq(channel);  in efx_soft_enable_interrupts()
1526 	struct efx_channel *channel;  in efx_soft_disable_interrupts()  local
1539 	efx_for_each_channel(channel, efx) {  in efx_soft_disable_interrupts()
1540 		if (channel->irq)  in efx_soft_disable_interrupts()
1541 			synchronize_irq(channel->irq);  in efx_soft_disable_interrupts()
1543 		efx_stop_eventq(channel);  in efx_soft_disable_interrupts()
1544 		if (!channel->type->keep_eventq)  in efx_soft_disable_interrupts()
1545 			efx_fini_eventq(channel);  in efx_soft_disable_interrupts()
1554 	struct efx_channel *channel, *end_channel;  in efx_enable_interrupts()  local
1566 	efx_for_each_channel(channel, efx) {  in efx_enable_interrupts()
1567 		if (channel->type->keep_eventq) {  in efx_enable_interrupts()
1568 			rc = efx_init_eventq(channel);  in efx_enable_interrupts()
1581 	end_channel = channel;  in efx_enable_interrupts()
1582 	efx_for_each_channel(channel, efx) {  in efx_enable_interrupts()
1583 		if (channel == end_channel)  in efx_enable_interrupts()
1585 		if (channel->type->keep_eventq)  in efx_enable_interrupts()
1586 			efx_fini_eventq(channel);  in efx_enable_interrupts()
1596 	struct efx_channel *channel;  in efx_disable_interrupts()  local
1600 	efx_for_each_channel(channel, efx) {  in efx_disable_interrupts()
1601 		if (channel->type->keep_eventq)  in efx_disable_interrupts()
1602 			efx_fini_eventq(channel);  in efx_disable_interrupts()
1610 	struct efx_channel *channel;  in efx_remove_interrupts()  local
1613 	efx_for_each_channel(channel, efx)  in efx_remove_interrupts()
1614 		channel->irq = 0;  in efx_remove_interrupts()
1624 	struct efx_channel *channel;  in efx_set_channels()  local
1635 	efx_for_each_channel(channel, efx) {  in efx_set_channels()
1636 		if (channel->channel < efx->n_rx_channels)  in efx_set_channels()
1637 			channel->rx_queue.core_index = channel->channel;  in efx_set_channels()
1639 			channel->rx_queue.core_index = -1;  in efx_set_channels()
1641 		efx_for_each_channel_tx_queue(tx_queue, channel)  in efx_set_channels()
1933 	struct efx_channel *channel;  in efx_init_irq_moderation()  local
1957 	efx_for_each_channel(channel, efx) {  in efx_init_irq_moderation()
1958 		if (efx_channel_has_rx_queue(channel))  in efx_init_irq_moderation()
1959 			channel->irq_moderation = rx_ticks;  in efx_init_irq_moderation()
1960 		else if (efx_channel_has_tx_queues(channel))  in efx_init_irq_moderation()
1961 			channel->irq_moderation = tx_ticks;  in efx_init_irq_moderation()
1987 			efx->channel[efx->tx_channel_offset]->irq_moderation *  in efx_get_irq_moderation()
2055 static void efx_init_napi_channel(struct efx_channel *channel)  in efx_init_napi_channel()  argument
2057 	struct efx_nic *efx = channel->efx;  in efx_init_napi_channel()
2059 	channel->napi_dev = efx->net_dev;  in efx_init_napi_channel()
2060 	netif_napi_add(channel->napi_dev, &channel->napi_str,  in efx_init_napi_channel()
2062 	napi_hash_add(&channel->napi_str);  in efx_init_napi_channel()
2063 	efx_channel_busy_poll_init(channel);  in efx_init_napi_channel()
2068 	struct efx_channel *channel;  in efx_init_napi()  local
2070 	efx_for_each_channel(channel, efx)  in efx_init_napi()
2071 		efx_init_napi_channel(channel);  in efx_init_napi()
2074 static void efx_fini_napi_channel(struct efx_channel *channel)  in efx_fini_napi_channel()  argument
2076 	if (channel->napi_dev) {  in efx_fini_napi_channel()
2077 		netif_napi_del(&channel->napi_str);  in efx_fini_napi_channel()
2078 		napi_hash_del(&channel->napi_str);  in efx_fini_napi_channel()
2080 	channel->napi_dev = NULL;  in efx_fini_napi_channel()
2085 	struct efx_channel *channel;  in efx_fini_napi()  local
2087 	efx_for_each_channel(channel, efx)  in efx_fini_napi()
2088 		efx_fini_napi_channel(channel);  in efx_fini_napi()
2106 	struct efx_channel *channel;  in efx_netpoll()  local
2108 	efx_for_each_channel(channel, efx)  in efx_netpoll()
2109 		efx_schedule_channel(channel);  in efx_netpoll()
2117 	struct efx_channel *channel =  in efx_busy_poll()  local
2119 	struct efx_nic *efx = channel->efx;  in efx_busy_poll()
2126 	if (!efx_channel_try_lock_poll(channel))  in efx_busy_poll()
2129 	old_rx_packets = channel->rx_queue.rx_packets;  in efx_busy_poll()
2130 	efx_process_channel(channel, budget);  in efx_busy_poll()
2132 	rx_packets = channel->rx_queue.rx_packets - old_rx_packets;  in efx_busy_poll()
2139 	efx_channel_unlock_poll(channel);  in efx_busy_poll()
2390 	struct efx_channel *channel;  in efx_register_netdev()  local
2428 	efx_for_each_channel(channel, efx) {  in efx_register_netdev()
2430 		efx_for_each_channel_tx_queue(tx_queue, channel)  in efx_register_netdev()
2863 		efx->channel[i] = efx_alloc_channel(efx, i, NULL);  in efx_init_struct()
2864 		if (!efx->channel[i])  in efx_init_struct()
2893 		kfree(efx->channel[i]);  in efx_fini_struct()
2906 	struct efx_channel *channel;  in efx_update_sw_stats()  local
2908 	efx_for_each_channel(channel, efx)  in efx_update_sw_stats()
2909 		n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;  in efx_update_sw_stats()