793 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)  in igb_assign_vector()  argument
795 	struct igb_adapter *adapter = q_vector->adapter;  in igb_assign_vector()
801 	if (q_vector->rx.ring)  in igb_assign_vector()
802 		rx_queue = q_vector->rx.ring->reg_idx;  in igb_assign_vector()
803 	if (q_vector->tx.ring)  in igb_assign_vector()
804 		tx_queue = q_vector->tx.ring->reg_idx;  in igb_assign_vector()
820 		q_vector->eims_value = msixbm;  in igb_assign_vector()
836 		q_vector->eims_value = 1 << msix_vector;  in igb_assign_vector()
857 		q_vector->eims_value = 1 << msix_vector;  in igb_assign_vector()
865 	adapter->eims_enable_mask |= q_vector->eims_value;  in igb_assign_vector()
868 	q_vector->set_itr = 1;  in igb_assign_vector()
932 		igb_assign_vector(adapter->q_vector[i], vector++);  in igb_configure_msix()
956 		struct igb_q_vector *q_vector = adapter->q_vector[i];  in igb_request_msix()  local
960 		q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);  in igb_request_msix()
962 		if (q_vector->rx.ring && q_vector->tx.ring)  in igb_request_msix()
963 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,  in igb_request_msix()
964 				q_vector->rx.ring->queue_index);  in igb_request_msix()
965 		else if (q_vector->tx.ring)  in igb_request_msix()
966 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,  in igb_request_msix()
967 				q_vector->tx.ring->queue_index);  in igb_request_msix()
968 		else if (q_vector->rx.ring)  in igb_request_msix()
969 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,  in igb_request_msix()
970 				q_vector->rx.ring->queue_index);  in igb_request_msix()
972 			sprintf(q_vector->name, "%s-unused", netdev->name);  in igb_request_msix()
975 				  igb_msix_ring, 0, q_vector->name,  in igb_request_msix()
976 				  q_vector);  in igb_request_msix()
991 			 adapter->q_vector[i]);  in igb_request_msix()
1006 	struct igb_q_vector *q_vector = adapter->q_vector[v_idx];  in igb_free_q_vector()  local
1008 	adapter->q_vector[v_idx] = NULL;  in igb_free_q_vector()
1013 	if (q_vector)  in igb_free_q_vector()
1014 		kfree_rcu(q_vector, rcu);  in igb_free_q_vector()
1027 	struct igb_q_vector *q_vector = adapter->q_vector[v_idx];  in igb_reset_q_vector()  local
1032 	if (!q_vector)  in igb_reset_q_vector()
1035 	if (q_vector->tx.ring)  in igb_reset_q_vector()
1036 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;  in igb_reset_q_vector()
1038 	if (q_vector->rx.ring)  in igb_reset_q_vector()
1039 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;  in igb_reset_q_vector()
1041 	netif_napi_del(&q_vector->napi);  in igb_reset_q_vector()
1194 	struct igb_q_vector *q_vector;  in igb_alloc_q_vector()  local
1207 	q_vector = adapter->q_vector[v_idx];  in igb_alloc_q_vector()
1208 	if (!q_vector) {  in igb_alloc_q_vector()
1209 		q_vector = kzalloc(size, GFP_KERNEL);  in igb_alloc_q_vector()
1210 	} else if (size > ksize(q_vector)) {  in igb_alloc_q_vector()
1211 		kfree_rcu(q_vector, rcu);  in igb_alloc_q_vector()
1212 		q_vector = kzalloc(size, GFP_KERNEL);  in igb_alloc_q_vector()
1214 		memset(q_vector, 0, size);  in igb_alloc_q_vector()
1216 	if (!q_vector)  in igb_alloc_q_vector()
1220 	netif_napi_add(adapter->netdev, &q_vector->napi,  in igb_alloc_q_vector()
1224 	adapter->q_vector[v_idx] = q_vector;  in igb_alloc_q_vector()
1225 	q_vector->adapter = adapter;  in igb_alloc_q_vector()
1228 	q_vector->tx.work_limit = adapter->tx_work_limit;  in igb_alloc_q_vector()
1231 	q_vector->itr_register = adapter->hw.hw_addr + E1000_EITR(0);  in igb_alloc_q_vector()
1232 	q_vector->itr_val = IGB_START_ITR;  in igb_alloc_q_vector()
1235 	ring = q_vector->ring;  in igb_alloc_q_vector()
1241 			q_vector->itr_val = adapter->rx_itr_setting;  in igb_alloc_q_vector()
1245 			q_vector->itr_val = adapter->tx_itr_setting;  in igb_alloc_q_vector()
1254 		ring->q_vector = q_vector;  in igb_alloc_q_vector()
1257 		igb_add_ring(ring, &q_vector->tx);  in igb_alloc_q_vector()
1283 		ring->q_vector = q_vector;  in igb_alloc_q_vector()
1286 		igb_add_ring(ring, &q_vector->rx);  in igb_alloc_q_vector()
1431 	igb_assign_vector(adapter->q_vector[0], 0);  in igb_request_irq()
1464 				 adapter->q_vector[i]);  in igb_free_irq()
1736 		napi_enable(&(adapter->q_vector[i]->napi));  in igb_up()
1741 		igb_assign_vector(adapter->q_vector[0], 0);  in igb_up()
1801 		if (adapter->q_vector[i]) {  in igb_down()
1802 			napi_synchronize(&adapter->q_vector[i]->napi);  in igb_down()
1803 			napi_disable(&adapter->q_vector[i]->napi);  in igb_down()
3089 		napi_enable(&(adapter->q_vector[i]->napi));  in __igb_open()
4390 			eics |= adapter->q_vector[i]->eims_value;  in igb_watchdog_task()
4437 static void igb_update_ring_itr(struct igb_q_vector *q_vector)  in igb_update_ring_itr()  argument
4439 	int new_val = q_vector->itr_val;  in igb_update_ring_itr()
4441 	struct igb_adapter *adapter = q_vector->adapter;  in igb_update_ring_itr()
4452 	packets = q_vector->rx.total_packets;  in igb_update_ring_itr()
4454 		avg_wire_size = q_vector->rx.total_bytes / packets;  in igb_update_ring_itr()
4456 	packets = q_vector->tx.total_packets;  in igb_update_ring_itr()
4459 				      q_vector->tx.total_bytes / packets);  in igb_update_ring_itr()
4479 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igb_update_ring_itr()
4480 	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igb_update_ring_itr()
4484 	if (new_val != q_vector->itr_val) {  in igb_update_ring_itr()
4485 		q_vector->itr_val = new_val;  in igb_update_ring_itr()
4486 		q_vector->set_itr = 1;  in igb_update_ring_itr()
4489 	q_vector->rx.total_bytes = 0;  in igb_update_ring_itr()
4490 	q_vector->rx.total_packets = 0;  in igb_update_ring_itr()
4491 	q_vector->tx.total_bytes = 0;  in igb_update_ring_itr()
4492 	q_vector->tx.total_packets = 0;  in igb_update_ring_itr()
4511 static void igb_update_itr(struct igb_q_vector *q_vector,  in igb_update_itr()  argument
4563 static void igb_set_itr(struct igb_q_vector *q_vector)  in igb_set_itr()  argument
4565 	struct igb_adapter *adapter = q_vector->adapter;  in igb_set_itr()
4566 	u32 new_itr = q_vector->itr_val;  in igb_set_itr()
4576 	igb_update_itr(q_vector, &q_vector->tx);  in igb_set_itr()
4577 	igb_update_itr(q_vector, &q_vector->rx);  in igb_set_itr()
4579 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);  in igb_set_itr()
4583 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igb_set_itr()
4584 	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igb_set_itr()
4603 	if (new_itr != q_vector->itr_val) {  in igb_set_itr()
4608 		new_itr = new_itr > q_vector->itr_val ?  in igb_set_itr()
4609 			  max((new_itr * q_vector->itr_val) /  in igb_set_itr()
4610 			  (new_itr + (q_vector->itr_val >> 2)),  in igb_set_itr()
4618 		q_vector->itr_val = new_itr;  in igb_set_itr()
4619 		q_vector->set_itr = 1;  in igb_set_itr()
5513 static void igb_write_itr(struct igb_q_vector *q_vector)  in igb_write_itr()  argument
5515 	struct igb_adapter *adapter = q_vector->adapter;  in igb_write_itr()
5516 	u32 itr_val = q_vector->itr_val & 0x7FFC;  in igb_write_itr()
5518 	if (!q_vector->set_itr)  in igb_write_itr()
5529 	writel(itr_val, q_vector->itr_register);  in igb_write_itr()
5530 	q_vector->set_itr = 0;  in igb_write_itr()
5535 	struct igb_q_vector *q_vector = data;  in igb_msix_ring()  local
5538 	igb_write_itr(q_vector);  in igb_msix_ring()
5540 	napi_schedule(&q_vector->napi);  in igb_msix_ring()
5587 static void igb_update_dca(struct igb_q_vector *q_vector)  in igb_update_dca()  argument
5589 	struct igb_adapter *adapter = q_vector->adapter;  in igb_update_dca()
5592 	if (q_vector->cpu == cpu)  in igb_update_dca()
5595 	if (q_vector->tx.ring)  in igb_update_dca()
5596 		igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);  in igb_update_dca()
5598 	if (q_vector->rx.ring)  in igb_update_dca()
5599 		igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);  in igb_update_dca()
5601 	q_vector->cpu = cpu;  in igb_update_dca()
5618 		adapter->q_vector[i]->cpu = -1;  in igb_setup_dca()
5619 		igb_update_dca(adapter->q_vector[i]);  in igb_setup_dca()
6268 	struct igb_q_vector *q_vector = adapter->q_vector[0];  in igb_intr_msi()  local
6273 	igb_write_itr(q_vector);  in igb_intr_msi()
6292 	napi_schedule(&q_vector->napi);  in igb_intr_msi()
6305 	struct igb_q_vector *q_vector = adapter->q_vector[0];  in igb_intr()  local
6318 	igb_write_itr(q_vector);  in igb_intr()
6338 	napi_schedule(&q_vector->napi);  in igb_intr()
6343 static void igb_ring_irq_enable(struct igb_q_vector *q_vector)  in igb_ring_irq_enable()  argument
6345 	struct igb_adapter *adapter = q_vector->adapter;  in igb_ring_irq_enable()
6348 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||  in igb_ring_irq_enable()
6349 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {  in igb_ring_irq_enable()
6351 			igb_set_itr(q_vector);  in igb_ring_irq_enable()
6353 			igb_update_ring_itr(q_vector);  in igb_ring_irq_enable()
6358 			wr32(E1000_EIMS, q_vector->eims_value);  in igb_ring_irq_enable()
6371 	struct igb_q_vector *q_vector = container_of(napi,  in igb_poll()  local
6377 	if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)  in igb_poll()
6378 		igb_update_dca(q_vector);  in igb_poll()
6380 	if (q_vector->tx.ring)  in igb_poll()
6381 		clean_complete = igb_clean_tx_irq(q_vector);  in igb_poll()
6383 	if (q_vector->rx.ring)  in igb_poll()
6384 		clean_complete &= igb_clean_rx_irq(q_vector, budget);  in igb_poll()
6392 	igb_ring_irq_enable(q_vector);  in igb_poll()
6403 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)  in igb_clean_tx_irq()  argument
6405 	struct igb_adapter *adapter = q_vector->adapter;  in igb_clean_tx_irq()
6406 	struct igb_ring *tx_ring = q_vector->tx.ring;  in igb_clean_tx_irq()
6410 	unsigned int budget = q_vector->tx.work_limit;  in igb_clean_tx_irq()
6500 	q_vector->tx.total_bytes += total_bytes;  in igb_clean_tx_irq()
6501 	q_vector->tx.total_packets += total_packets;  in igb_clean_tx_irq()
6665 			igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);  in igb_add_rx_frag()
6709 		skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);  in igb_fetch_rx_buffer()
6852 		igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);  in igb_pull_tail()
6939 		igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb);  in igb_process_skb_fields()
6959 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)  in igb_clean_rx_irq()  argument
6961 	struct igb_ring *rx_ring = q_vector->rx.ring;  in igb_clean_rx_irq()
7011 		napi_gro_receive(&q_vector->napi, skb);  in igb_clean_rx_irq()
7027 	q_vector->rx.total_packets += total_packets;  in igb_clean_rx_irq()
7028 	q_vector->rx.total_bytes += total_bytes;  in igb_clean_rx_irq()
7629 	struct igb_q_vector *q_vector;  in igb_netpoll()  local
7633 		q_vector = adapter->q_vector[i];  in igb_netpoll()
7635 			wr32(E1000_EIMC, q_vector->eims_value);  in igb_netpoll()
7638 		napi_schedule(&q_vector->napi);  in igb_netpoll()