93 static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx,
96 static void make_tx_response(struct xenvif_queue *queue,
99 static void push_tx_responses(struct xenvif_queue *queue);
101 static inline int tx_work_todo(struct xenvif_queue *queue);
103 static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue,
110 static inline unsigned long idx_to_pfn(struct xenvif_queue *queue,  in idx_to_pfn()  argument
113 	return page_to_pfn(queue->mmap_pages[idx]);  in idx_to_pfn()
116 static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue,  in idx_to_kaddr()  argument
119 	return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue, idx));  in idx_to_kaddr()
152 bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue, int needed)  in xenvif_rx_ring_slots_available()  argument
157 		prod = queue->rx.sring->req_prod;  in xenvif_rx_ring_slots_available()
158 		cons = queue->rx.req_cons;  in xenvif_rx_ring_slots_available()
163 		queue->rx.sring->req_event = prod + 1;  in xenvif_rx_ring_slots_available()
169 	} while (queue->rx.sring->req_prod != prod);  in xenvif_rx_ring_slots_available()
174 void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb)  in xenvif_rx_queue_tail()  argument
178 	spin_lock_irqsave(&queue->rx_queue.lock, flags);  in xenvif_rx_queue_tail()
180 	__skb_queue_tail(&queue->rx_queue, skb);  in xenvif_rx_queue_tail()
182 	queue->rx_queue_len += skb->len;  in xenvif_rx_queue_tail()
183 	if (queue->rx_queue_len > queue->rx_queue_max)  in xenvif_rx_queue_tail()
184 		netif_tx_stop_queue(netdev_get_tx_queue(queue->vif->dev, queue->id));  in xenvif_rx_queue_tail()
186 	spin_unlock_irqrestore(&queue->rx_queue.lock, flags);  in xenvif_rx_queue_tail()
189 static struct sk_buff *xenvif_rx_dequeue(struct xenvif_queue *queue)  in xenvif_rx_dequeue()  argument
193 	spin_lock_irq(&queue->rx_queue.lock);  in xenvif_rx_dequeue()
195 	skb = __skb_dequeue(&queue->rx_queue);  in xenvif_rx_dequeue()
197 		queue->rx_queue_len -= skb->len;  in xenvif_rx_dequeue()
199 	spin_unlock_irq(&queue->rx_queue.lock);  in xenvif_rx_dequeue()
204 static void xenvif_rx_queue_maybe_wake(struct xenvif_queue *queue)  in xenvif_rx_queue_maybe_wake()  argument
206 	spin_lock_irq(&queue->rx_queue.lock);  in xenvif_rx_queue_maybe_wake()
208 	if (queue->rx_queue_len < queue->rx_queue_max)  in xenvif_rx_queue_maybe_wake()
209 		netif_tx_wake_queue(netdev_get_tx_queue(queue->vif->dev, queue->id));  in xenvif_rx_queue_maybe_wake()
211 	spin_unlock_irq(&queue->rx_queue.lock);  in xenvif_rx_queue_maybe_wake()
215 static void xenvif_rx_queue_purge(struct xenvif_queue *queue)  in xenvif_rx_queue_purge()  argument
218 	while ((skb = xenvif_rx_dequeue(queue)) != NULL)  in xenvif_rx_queue_purge()
222 static void xenvif_rx_queue_drop_expired(struct xenvif_queue *queue)  in xenvif_rx_queue_drop_expired()  argument
227 		skb = skb_peek(&queue->rx_queue);  in xenvif_rx_queue_drop_expired()
232 		xenvif_rx_dequeue(queue);  in xenvif_rx_queue_drop_expired()
246 static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif_queue *queue,  in get_next_rx_buffer()  argument
252 	req = RING_GET_REQUEST(&queue->rx, queue->rx.req_cons++);  in get_next_rx_buffer()
270 static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb,  in xenvif_gop_frag_copy()  argument
296 			meta = get_next_rx_buffer(queue, npo);  in xenvif_gop_frag_copy()
321 		copy_gop->dest.domid = queue->vif->domid;  in xenvif_gop_frag_copy()
346 		if (*head && ((1 << gso_type) & queue->vif->gso_mask))  in xenvif_gop_frag_copy()
347 			queue->rx.req_cons++;  in xenvif_gop_frag_copy()
368 			  struct xenvif_queue *queue)  in xenvif_gop_skb()  argument
392 		req = RING_GET_REQUEST(&queue->rx, queue->rx.req_cons++);  in xenvif_gop_skb()
400 	req = RING_GET_REQUEST(&queue->rx, queue->rx.req_cons++);  in xenvif_gop_skb()
424 		xenvif_gop_frag_copy(queue, skb, npo,  in xenvif_gop_skb()
430 		xenvif_gop_frag_copy(queue, skb, npo,  in xenvif_gop_skb()
466 static void xenvif_add_frag_responses(struct xenvif_queue *queue, int status,  in xenvif_add_frag_responses()  argument
487 		make_rx_response(queue, meta[i].id, status, offset,  in xenvif_add_frag_responses()
492 void xenvif_kick_thread(struct xenvif_queue *queue)  in xenvif_kick_thread()  argument
494 	wake_up(&queue->wq);  in xenvif_kick_thread()
497 static void xenvif_rx_action(struct xenvif_queue *queue)  in xenvif_rx_action()  argument
510 		.copy  = queue->grant_copy_op,  in xenvif_rx_action()
511 		.meta  = queue->meta,  in xenvif_rx_action()
516 	while (xenvif_rx_ring_slots_available(queue, XEN_NETBK_RX_SLOTS_MAX)  in xenvif_rx_action()
517 	       && (skb = xenvif_rx_dequeue(queue)) != NULL) {  in xenvif_rx_action()
521 		queue->last_rx_time = jiffies;  in xenvif_rx_action()
523 		old_req_cons = queue->rx.req_cons;  in xenvif_rx_action()
524 		XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo, queue);  in xenvif_rx_action()
525 		ring_slots_used = queue->rx.req_cons - old_req_cons;  in xenvif_rx_action()
530 	BUG_ON(npo.meta_prod > ARRAY_SIZE(queue->meta));  in xenvif_rx_action()
536 	gnttab_batch_copy(queue->grant_copy_op, npo.copy_prod);  in xenvif_rx_action()
540 		if ((1 << queue->meta[npo.meta_cons].gso_type) &  in xenvif_rx_action()
541 		    queue->vif->gso_prefix_mask) {  in xenvif_rx_action()
542 			resp = RING_GET_RESPONSE(&queue->rx,  in xenvif_rx_action()
543 						 queue->rx.rsp_prod_pvt++);  in xenvif_rx_action()
547 			resp->offset = queue->meta[npo.meta_cons].gso_size;  in xenvif_rx_action()
548 			resp->id = queue->meta[npo.meta_cons].id;  in xenvif_rx_action()
556 		queue->stats.tx_bytes += skb->len;  in xenvif_rx_action()
557 		queue->stats.tx_packets++;  in xenvif_rx_action()
559 		status = xenvif_check_gop(queue->vif,  in xenvif_rx_action()
575 		resp = make_rx_response(queue, queue->meta[npo.meta_cons].id,  in xenvif_rx_action()
577 					queue->meta[npo.meta_cons].size,  in xenvif_rx_action()
580 		if ((1 << queue->meta[npo.meta_cons].gso_type) &  in xenvif_rx_action()
581 		    queue->vif->gso_mask) {  in xenvif_rx_action()
584 				RING_GET_RESPONSE(&queue->rx,  in xenvif_rx_action()
585 						  queue->rx.rsp_prod_pvt++);  in xenvif_rx_action()
589 			gso->u.gso.type = queue->meta[npo.meta_cons].gso_type;  in xenvif_rx_action()
590 			gso->u.gso.size = queue->meta[npo.meta_cons].gso_size;  in xenvif_rx_action()
598 		xenvif_add_frag_responses(queue, status,  in xenvif_rx_action()
599 					  queue->meta + npo.meta_cons + 1,  in xenvif_rx_action()
602 		RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->rx, ret);  in xenvif_rx_action()
612 		notify_remote_via_irq(queue->rx_irq);  in xenvif_rx_action()
615 void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue)  in xenvif_napi_schedule_or_enable_events()  argument
619 	RING_FINAL_CHECK_FOR_REQUESTS(&queue->tx, more_to_do);  in xenvif_napi_schedule_or_enable_events()
622 		napi_schedule(&queue->napi);  in xenvif_napi_schedule_or_enable_events()
625 static void tx_add_credit(struct xenvif_queue *queue)  in tx_add_credit()  argument
633 	max_burst = RING_GET_REQUEST(&queue->tx, queue->tx.req_cons)->size;  in tx_add_credit()
635 	max_burst = max(max_burst, queue->credit_bytes);  in tx_add_credit()
638 	max_credit = queue->remaining_credit + queue->credit_bytes;  in tx_add_credit()
639 	if (max_credit < queue->remaining_credit)  in tx_add_credit()
642 	queue->remaining_credit = min(max_credit, max_burst);  in tx_add_credit()
647 	struct xenvif_queue *queue = (struct xenvif_queue *)data;  in xenvif_tx_credit_callback()  local
648 	tx_add_credit(queue);  in xenvif_tx_credit_callback()
649 	xenvif_napi_schedule_or_enable_events(queue);  in xenvif_tx_credit_callback()
652 static void xenvif_tx_err(struct xenvif_queue *queue,  in xenvif_tx_err()  argument
655 	RING_IDX cons = queue->tx.req_cons;  in xenvif_tx_err()
659 		spin_lock_irqsave(&queue->response_lock, flags);  in xenvif_tx_err()
660 		make_tx_response(queue, txp, XEN_NETIF_RSP_ERROR);  in xenvif_tx_err()
661 		push_tx_responses(queue);  in xenvif_tx_err()
662 		spin_unlock_irqrestore(&queue->response_lock, flags);  in xenvif_tx_err()
665 		txp = RING_GET_REQUEST(&queue->tx, cons++);  in xenvif_tx_err()
667 	queue->tx.req_cons = cons;  in xenvif_tx_err()
679 static int xenvif_count_requests(struct xenvif_queue *queue,  in xenvif_count_requests()  argument
684 	RING_IDX cons = queue->tx.req_cons;  in xenvif_count_requests()
696 			netdev_err(queue->vif->dev,  in xenvif_count_requests()
699 			xenvif_fatal_tx_err(queue->vif);  in xenvif_count_requests()
707 			netdev_err(queue->vif->dev,  in xenvif_count_requests()
710 			xenvif_fatal_tx_err(queue->vif);  in xenvif_count_requests()
723 				netdev_dbg(queue->vif->dev,  in xenvif_count_requests()
732 		memcpy(txp, RING_GET_REQUEST(&queue->tx, cons + slots),  in xenvif_count_requests()
746 				netdev_dbg(queue->vif->dev,  in xenvif_count_requests()
756 			netdev_err(queue->vif->dev, "Cross page boundary, txp->offset: %x, size: %u\n",  in xenvif_count_requests()
758 			xenvif_fatal_tx_err(queue->vif);  in xenvif_count_requests()
770 		xenvif_tx_err(queue, first, cons + slots);  in xenvif_count_requests()
784 static inline void xenvif_tx_create_map_op(struct xenvif_queue *queue,  in xenvif_tx_create_map_op()  argument
789 	queue->pages_to_map[mop-queue->tx_map_ops] = queue->mmap_pages[pending_idx];  in xenvif_tx_create_map_op()
790 	gnttab_set_map_op(mop, idx_to_kaddr(queue, pending_idx),  in xenvif_tx_create_map_op()
792 			  txp->gref, queue->vif->domid);  in xenvif_tx_create_map_op()
794 	memcpy(&queue->pending_tx_info[pending_idx].req, txp,  in xenvif_tx_create_map_op()
815 static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif_queue *queue,  in xenvif_get_requests()  argument
842 		index = pending_index(queue->pending_cons++);  in xenvif_get_requests()
843 		pending_idx = queue->pending_ring[index];  in xenvif_get_requests()
844 		xenvif_tx_create_map_op(queue, pending_idx, txp, gop);  in xenvif_get_requests()
852 				netdev_err(queue->vif->dev,  in xenvif_get_requests()
862 			index = pending_index(queue->pending_cons++);  in xenvif_get_requests()
863 			pending_idx = queue->pending_ring[index];  in xenvif_get_requests()
864 			xenvif_tx_create_map_op(queue, pending_idx, txp, gop);  in xenvif_get_requests()
875 static inline void xenvif_grant_handle_set(struct xenvif_queue *queue,  in xenvif_grant_handle_set()  argument
879 	if (unlikely(queue->grant_tx_handle[pending_idx] !=  in xenvif_grant_handle_set()
881 		netdev_err(queue->vif->dev,  in xenvif_grant_handle_set()
886 	queue->grant_tx_handle[pending_idx] = handle;  in xenvif_grant_handle_set()
889 static inline void xenvif_grant_handle_reset(struct xenvif_queue *queue,  in xenvif_grant_handle_reset()  argument
892 	if (unlikely(queue->grant_tx_handle[pending_idx] ==  in xenvif_grant_handle_reset()
894 		netdev_err(queue->vif->dev,  in xenvif_grant_handle_reset()
899 	queue->grant_tx_handle[pending_idx] = NETBACK_INVALID_HANDLE;  in xenvif_grant_handle_reset()
902 static int xenvif_tx_check_gop(struct xenvif_queue *queue,  in xenvif_tx_check_gop()  argument
926 			netdev_dbg(queue->vif->dev,  in xenvif_tx_check_gop()
933 			xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
948 			xenvif_grant_handle_set(queue,  in xenvif_tx_check_gop()
953 				xenvif_idx_unmap(queue, pending_idx);  in xenvif_tx_check_gop()
959 					xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
962 					xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
970 			netdev_dbg(queue->vif->dev,  in xenvif_tx_check_gop()
977 		xenvif_idx_release(queue, pending_idx, XEN_NETIF_RSP_ERROR);  in xenvif_tx_check_gop()
987 			xenvif_idx_release(queue,  in xenvif_tx_check_gop()
994 			xenvif_idx_unmap(queue, pending_idx);  in xenvif_tx_check_gop()
995 			xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
1005 				xenvif_idx_unmap(queue, pending_idx);  in xenvif_tx_check_gop()
1006 				xenvif_idx_release(queue, pending_idx,  in xenvif_tx_check_gop()
1027 static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb)  in xenvif_fill_frags()  argument
1045 				&callback_param(queue, pending_idx);  in xenvif_fill_frags()
1047 			callback_param(queue, prev_pending_idx).ctx =  in xenvif_fill_frags()
1048 				&callback_param(queue, pending_idx);  in xenvif_fill_frags()
1050 		callback_param(queue, pending_idx).ctx = NULL;  in xenvif_fill_frags()
1053 		txp = &queue->pending_tx_info[pending_idx].req;  in xenvif_fill_frags()
1054 		page = virt_to_page(idx_to_kaddr(queue, pending_idx));  in xenvif_fill_frags()
1061 		get_page(queue->mmap_pages[pending_idx]);  in xenvif_fill_frags()
1065 static int xenvif_get_extras(struct xenvif_queue *queue,  in xenvif_get_extras()  argument
1070 	RING_IDX cons = queue->tx.req_cons;  in xenvif_get_extras()
1074 			netdev_err(queue->vif->dev, "Missing extra info\n");  in xenvif_get_extras()
1075 			xenvif_fatal_tx_err(queue->vif);  in xenvif_get_extras()
1079 		memcpy(&extra, RING_GET_REQUEST(&queue->tx, cons),  in xenvif_get_extras()
1083 			queue->tx.req_cons = ++cons;  in xenvif_get_extras()
1084 			netdev_err(queue->vif->dev,  in xenvif_get_extras()
1086 			xenvif_fatal_tx_err(queue->vif);  in xenvif_get_extras()
1091 		queue->tx.req_cons = ++cons;  in xenvif_get_extras()
1126 static int checksum_setup(struct xenvif_queue *queue, struct sk_buff *skb)  in checksum_setup()  argument
1136 		queue->stats.rx_gso_checksum_fixup++;  in checksum_setup()
1148 static bool tx_credit_exceeded(struct xenvif_queue *queue, unsigned size)  in tx_credit_exceeded()  argument
1151 	u64 next_credit = queue->credit_window_start +  in tx_credit_exceeded()
1152 		msecs_to_jiffies(queue->credit_usec / 1000);  in tx_credit_exceeded()
1155 	if (timer_pending(&queue->credit_timeout))  in tx_credit_exceeded()
1160 		queue->credit_window_start = now;  in tx_credit_exceeded()
1161 		tx_add_credit(queue);  in tx_credit_exceeded()
1165 	if (size > queue->remaining_credit) {  in tx_credit_exceeded()
1166 		queue->credit_timeout.data     =  in tx_credit_exceeded()
1167 			(unsigned long)queue;  in tx_credit_exceeded()
1168 		mod_timer(&queue->credit_timeout,  in tx_credit_exceeded()
1170 		queue->credit_window_start = next_credit;  in tx_credit_exceeded()
1178 static void xenvif_tx_build_gops(struct xenvif_queue *queue,  in xenvif_tx_build_gops()  argument
1183 	struct gnttab_map_grant_ref *gop = queue->tx_map_ops, *request_gop;  in xenvif_tx_build_gops()
1187 	while (skb_queue_len(&queue->tx_queue) < budget) {  in xenvif_tx_build_gops()
1197 		if (queue->tx.sring->req_prod - queue->tx.req_cons >  in xenvif_tx_build_gops()
1199 			netdev_err(queue->vif->dev,  in xenvif_tx_build_gops()
1202 				   queue->tx.sring->req_prod, queue->tx.req_cons,  in xenvif_tx_build_gops()
1204 			xenvif_fatal_tx_err(queue->vif);  in xenvif_tx_build_gops()
1208 		work_to_do = RING_HAS_UNCONSUMED_REQUESTS(&queue->tx);  in xenvif_tx_build_gops()
1212 		idx = queue->tx.req_cons;  in xenvif_tx_build_gops()
1214 		memcpy(&txreq, RING_GET_REQUEST(&queue->tx, idx), sizeof(txreq));  in xenvif_tx_build_gops()
1217 		if (txreq.size > queue->remaining_credit &&  in xenvif_tx_build_gops()
1218 		    tx_credit_exceeded(queue, txreq.size))  in xenvif_tx_build_gops()
1221 		queue->remaining_credit -= txreq.size;  in xenvif_tx_build_gops()
1224 		queue->tx.req_cons = ++idx;  in xenvif_tx_build_gops()
1228 			work_to_do = xenvif_get_extras(queue, extras,  in xenvif_tx_build_gops()
1230 			idx = queue->tx.req_cons;  in xenvif_tx_build_gops()
1235 		ret = xenvif_count_requests(queue, &txreq, txfrags, work_to_do);  in xenvif_tx_build_gops()
1242 			netdev_dbg(queue->vif->dev,  in xenvif_tx_build_gops()
1244 			xenvif_tx_err(queue, &txreq, idx);  in xenvif_tx_build_gops()
1250 			netdev_err(queue->vif->dev,  in xenvif_tx_build_gops()
1254 			xenvif_fatal_tx_err(queue->vif);  in xenvif_tx_build_gops()
1258 		index = pending_index(queue->pending_cons);  in xenvif_tx_build_gops()
1259 		pending_idx = queue->pending_ring[index];  in xenvif_tx_build_gops()
1267 			netdev_dbg(queue->vif->dev,  in xenvif_tx_build_gops()
1269 			xenvif_tx_err(queue, &txreq, idx);  in xenvif_tx_build_gops()
1277 			if (xenvif_set_skb_gso(queue->vif, skb, gso)) {  in xenvif_tx_build_gops()
1287 		queue->tx_copy_ops[*copy_ops].source.u.ref = txreq.gref;  in xenvif_tx_build_gops()
1288 		queue->tx_copy_ops[*copy_ops].source.domid = queue->vif->domid;  in xenvif_tx_build_gops()
1289 		queue->tx_copy_ops[*copy_ops].source.offset = txreq.offset;  in xenvif_tx_build_gops()
1291 		queue->tx_copy_ops[*copy_ops].dest.u.gmfn =  in xenvif_tx_build_gops()
1293 		queue->tx_copy_ops[*copy_ops].dest.domid = DOMID_SELF;  in xenvif_tx_build_gops()
1294 		queue->tx_copy_ops[*copy_ops].dest.offset =  in xenvif_tx_build_gops()
1297 		queue->tx_copy_ops[*copy_ops].len = data_len;  in xenvif_tx_build_gops()
1298 		queue->tx_copy_ops[*copy_ops].flags = GNTCOPY_source_gref;  in xenvif_tx_build_gops()
1307 			xenvif_tx_create_map_op(queue, pending_idx, &txreq, gop);  in xenvif_tx_build_gops()
1312 			memcpy(&queue->pending_tx_info[pending_idx].req, &txreq,  in xenvif_tx_build_gops()
1316 		queue->pending_cons++;  in xenvif_tx_build_gops()
1318 		request_gop = xenvif_get_requests(queue, skb, txfrags, gop);  in xenvif_tx_build_gops()
1321 			xenvif_tx_err(queue, &txreq, idx);  in xenvif_tx_build_gops()
1326 		__skb_queue_tail(&queue->tx_queue, skb);  in xenvif_tx_build_gops()
1328 		queue->tx.req_cons = idx;  in xenvif_tx_build_gops()
1330 		if (((gop-queue->tx_map_ops) >= ARRAY_SIZE(queue->tx_map_ops)) ||  in xenvif_tx_build_gops()
1331 		    (*copy_ops >= ARRAY_SIZE(queue->tx_copy_ops)))  in xenvif_tx_build_gops()
1335 	(*map_ops) = gop - queue->tx_map_ops;  in xenvif_tx_build_gops()
1342 static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *skb)  in xenvif_handle_frag_list()  argument
1350 	queue->stats.tx_zerocopy_sent += 2;  in xenvif_handle_frag_list()
1351 	queue->stats.tx_frag_overflow++;  in xenvif_handle_frag_list()
1353 	xenvif_fill_frags(queue, nskb);  in xenvif_handle_frag_list()
1389 	xenvif_skb_zerocopy_prepare(queue, nskb);  in xenvif_handle_frag_list()
1397 	atomic_inc(&queue->inflight_packets);  in xenvif_handle_frag_list()
1409 static int xenvif_tx_submit(struct xenvif_queue *queue)  in xenvif_tx_submit()  argument
1411 	struct gnttab_map_grant_ref *gop_map = queue->tx_map_ops;  in xenvif_tx_submit()
1412 	struct gnttab_copy *gop_copy = queue->tx_copy_ops;  in xenvif_tx_submit()
1416 	while ((skb = __skb_dequeue(&queue->tx_queue)) != NULL) {  in xenvif_tx_submit()
1422 		txp = &queue->pending_tx_info[pending_idx].req;  in xenvif_tx_submit()
1425 		if (unlikely(xenvif_tx_check_gop(queue, skb, &gop_map, &gop_copy))) {  in xenvif_tx_submit()
1441 		callback_param(queue, pending_idx).ctx = NULL;  in xenvif_tx_submit()
1448 			xenvif_idx_release(queue, pending_idx,  in xenvif_tx_submit()
1457 		xenvif_fill_frags(queue, skb);  in xenvif_tx_submit()
1460 			if (xenvif_handle_frag_list(queue, skb)) {  in xenvif_tx_submit()
1462 					netdev_err(queue->vif->dev,  in xenvif_tx_submit()
1464 				xenvif_skb_zerocopy_prepare(queue, skb);  in xenvif_tx_submit()
1470 		skb->dev      = queue->vif->dev;  in xenvif_tx_submit()
1474 		if (checksum_setup(queue, skb)) {  in xenvif_tx_submit()
1475 			netdev_dbg(queue->vif->dev,  in xenvif_tx_submit()
1479 				xenvif_skb_zerocopy_prepare(queue, skb);  in xenvif_tx_submit()
1500 		queue->stats.rx_bytes += skb->len;  in xenvif_tx_submit()
1501 		queue->stats.rx_packets++;  in xenvif_tx_submit()
1511 			xenvif_skb_zerocopy_prepare(queue, skb);  in xenvif_tx_submit()
1512 			queue->stats.tx_zerocopy_sent++;  in xenvif_tx_submit()
1525 	struct xenvif_queue *queue = ubuf_to_queue(ubuf);  in xenvif_zerocopy_callback()  local
1530 	spin_lock_irqsave(&queue->callback_lock, flags);  in xenvif_zerocopy_callback()
1534 		BUG_ON(queue->dealloc_prod - queue->dealloc_cons >=  in xenvif_zerocopy_callback()
1536 		index = pending_index(queue->dealloc_prod);  in xenvif_zerocopy_callback()
1537 		queue->dealloc_ring[index] = pending_idx;  in xenvif_zerocopy_callback()
1542 		queue->dealloc_prod++;  in xenvif_zerocopy_callback()
1544 	wake_up(&queue->dealloc_wq);  in xenvif_zerocopy_callback()
1545 	spin_unlock_irqrestore(&queue->callback_lock, flags);  in xenvif_zerocopy_callback()
1548 		queue->stats.tx_zerocopy_success++;  in xenvif_zerocopy_callback()
1550 		queue->stats.tx_zerocopy_fail++;  in xenvif_zerocopy_callback()
1551 	xenvif_skb_zerocopy_complete(queue);  in xenvif_zerocopy_callback()
1554 static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue)  in xenvif_tx_dealloc_action()  argument
1561 	dc = queue->dealloc_cons;  in xenvif_tx_dealloc_action()
1562 	gop = queue->tx_unmap_ops;  in xenvif_tx_dealloc_action()
1566 		dp = queue->dealloc_prod;  in xenvif_tx_dealloc_action()
1574 			BUG_ON(gop - queue->tx_unmap_ops >= MAX_PENDING_REQS);  in xenvif_tx_dealloc_action()
1576 				queue->dealloc_ring[pending_index(dc++)];  in xenvif_tx_dealloc_action()
1578 			pending_idx_release[gop - queue->tx_unmap_ops] =  in xenvif_tx_dealloc_action()
1580 			queue->pages_to_unmap[gop - queue->tx_unmap_ops] =  in xenvif_tx_dealloc_action()
1581 				queue->mmap_pages[pending_idx];  in xenvif_tx_dealloc_action()
1583 					    idx_to_kaddr(queue, pending_idx),  in xenvif_tx_dealloc_action()
1585 					    queue->grant_tx_handle[pending_idx]);  in xenvif_tx_dealloc_action()
1586 			xenvif_grant_handle_reset(queue, pending_idx);  in xenvif_tx_dealloc_action()
1590 	} while (dp != queue->dealloc_prod);  in xenvif_tx_dealloc_action()
1592 	queue->dealloc_cons = dc;  in xenvif_tx_dealloc_action()
1594 	if (gop - queue->tx_unmap_ops > 0) {  in xenvif_tx_dealloc_action()
1596 		ret = gnttab_unmap_refs(queue->tx_unmap_ops,  in xenvif_tx_dealloc_action()
1598 					queue->pages_to_unmap,  in xenvif_tx_dealloc_action()
1599 					gop - queue->tx_unmap_ops);  in xenvif_tx_dealloc_action()
1601 			netdev_err(queue->vif->dev, "Unmap fail: nr_ops %tx ret %d\n",  in xenvif_tx_dealloc_action()
1602 				   gop - queue->tx_unmap_ops, ret);  in xenvif_tx_dealloc_action()
1603 			for (i = 0; i < gop - queue->tx_unmap_ops; ++i) {  in xenvif_tx_dealloc_action()
1605 					netdev_err(queue->vif->dev,  in xenvif_tx_dealloc_action()
1615 	for (i = 0; i < gop - queue->tx_unmap_ops; ++i)  in xenvif_tx_dealloc_action()
1616 		xenvif_idx_release(queue, pending_idx_release[i],  in xenvif_tx_dealloc_action()
1622 int xenvif_tx_action(struct xenvif_queue *queue, int budget)  in xenvif_tx_action()  argument
1627 	if (unlikely(!tx_work_todo(queue)))  in xenvif_tx_action()
1630 	xenvif_tx_build_gops(queue, budget, &nr_cops, &nr_mops);  in xenvif_tx_action()
1635 	gnttab_batch_copy(queue->tx_copy_ops, nr_cops);  in xenvif_tx_action()
1637 		ret = gnttab_map_refs(queue->tx_map_ops,  in xenvif_tx_action()
1639 				      queue->pages_to_map,  in xenvif_tx_action()
1644 	work_done = xenvif_tx_submit(queue);  in xenvif_tx_action()
1649 static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx,  in xenvif_idx_release()  argument
1656 	pending_tx_info = &queue->pending_tx_info[pending_idx];  in xenvif_idx_release()
1658 	spin_lock_irqsave(&queue->response_lock, flags);  in xenvif_idx_release()
1660 	make_tx_response(queue, &pending_tx_info->req, status);  in xenvif_idx_release()
1666 	index = pending_index(queue->pending_prod++);  in xenvif_idx_release()
1667 	queue->pending_ring[index] = pending_idx;  in xenvif_idx_release()
1669 	push_tx_responses(queue);  in xenvif_idx_release()
1671 	spin_unlock_irqrestore(&queue->response_lock, flags);  in xenvif_idx_release()
1675 static void make_tx_response(struct xenvif_queue *queue,  in make_tx_response()  argument
1679 	RING_IDX i = queue->tx.rsp_prod_pvt;  in make_tx_response()
1682 	resp = RING_GET_RESPONSE(&queue->tx, i);  in make_tx_response()
1687 		RING_GET_RESPONSE(&queue->tx, ++i)->status = XEN_NETIF_RSP_NULL;  in make_tx_response()
1689 	queue->tx.rsp_prod_pvt = ++i;  in make_tx_response()
1692 static void push_tx_responses(struct xenvif_queue *queue)  in push_tx_responses()  argument
1696 	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->tx, notify);  in push_tx_responses()
1698 		notify_remote_via_irq(queue->tx_irq);  in push_tx_responses()
1701 static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue,  in make_rx_response()  argument
1708 	RING_IDX i = queue->rx.rsp_prod_pvt;  in make_rx_response()
1711 	resp = RING_GET_RESPONSE(&queue->rx, i);  in make_rx_response()
1719 	queue->rx.rsp_prod_pvt = ++i;  in make_rx_response()
1724 void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx)  in xenvif_idx_unmap()  argument
1730 			    idx_to_kaddr(queue, pending_idx),  in xenvif_idx_unmap()
1732 			    queue->grant_tx_handle[pending_idx]);  in xenvif_idx_unmap()
1733 	xenvif_grant_handle_reset(queue, pending_idx);  in xenvif_idx_unmap()
1736 				&queue->mmap_pages[pending_idx], 1);  in xenvif_idx_unmap()
1738 		netdev_err(queue->vif->dev,  in xenvif_idx_unmap()
1749 static inline int tx_work_todo(struct xenvif_queue *queue)  in tx_work_todo()  argument
1751 	if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue->tx)))  in tx_work_todo()
1757 static inline bool tx_dealloc_work_todo(struct xenvif_queue *queue)  in tx_dealloc_work_todo()  argument
1759 	return queue->dealloc_cons != queue->dealloc_prod;  in tx_dealloc_work_todo()
1762 void xenvif_unmap_frontend_rings(struct xenvif_queue *queue)  in xenvif_unmap_frontend_rings()  argument
1764 	if (queue->tx.sring)  in xenvif_unmap_frontend_rings()
1765 		xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif),  in xenvif_unmap_frontend_rings()
1766 					queue->tx.sring);  in xenvif_unmap_frontend_rings()
1767 	if (queue->rx.sring)  in xenvif_unmap_frontend_rings()
1768 		xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif),  in xenvif_unmap_frontend_rings()
1769 					queue->rx.sring);  in xenvif_unmap_frontend_rings()
1772 int xenvif_map_frontend_rings(struct xenvif_queue *queue,  in xenvif_map_frontend_rings()  argument
1782 	err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif),  in xenvif_map_frontend_rings()
1788 	BACK_RING_INIT(&queue->tx, txs, PAGE_SIZE);  in xenvif_map_frontend_rings()
1790 	err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif),  in xenvif_map_frontend_rings()
1796 	BACK_RING_INIT(&queue->rx, rxs, PAGE_SIZE);  in xenvif_map_frontend_rings()
1801 	xenvif_unmap_frontend_rings(queue);  in xenvif_map_frontend_rings()
1805 static void xenvif_queue_carrier_off(struct xenvif_queue *queue)  in xenvif_queue_carrier_off()  argument
1807 	struct xenvif *vif = queue->vif;  in xenvif_queue_carrier_off()
1809 	queue->stalled = true;  in xenvif_queue_carrier_off()
1820 static void xenvif_queue_carrier_on(struct xenvif_queue *queue)  in xenvif_queue_carrier_on()  argument
1822 	struct xenvif *vif = queue->vif;  in xenvif_queue_carrier_on()
1824 	queue->last_rx_time = jiffies; /* Reset Rx stall detection. */  in xenvif_queue_carrier_on()
1825 	queue->stalled = false;  in xenvif_queue_carrier_on()
1836 static bool xenvif_rx_queue_stalled(struct xenvif_queue *queue)  in xenvif_rx_queue_stalled()  argument
1840 	prod = queue->rx.sring->req_prod;  in xenvif_rx_queue_stalled()
1841 	cons = queue->rx.req_cons;  in xenvif_rx_queue_stalled()
1843 	return !queue->stalled  in xenvif_rx_queue_stalled()
1846 			      queue->last_rx_time + queue->vif->stall_timeout);  in xenvif_rx_queue_stalled()
1849 static bool xenvif_rx_queue_ready(struct xenvif_queue *queue)  in xenvif_rx_queue_ready()  argument
1853 	prod = queue->rx.sring->req_prod;  in xenvif_rx_queue_ready()
1854 	cons = queue->rx.req_cons;  in xenvif_rx_queue_ready()
1856 	return queue->stalled  in xenvif_rx_queue_ready()
1860 static bool xenvif_have_rx_work(struct xenvif_queue *queue)  in xenvif_have_rx_work()  argument
1862 	return (!skb_queue_empty(&queue->rx_queue)  in xenvif_have_rx_work()
1863 		&& xenvif_rx_ring_slots_available(queue, XEN_NETBK_RX_SLOTS_MAX))  in xenvif_have_rx_work()
1864 		|| (queue->vif->stall_timeout &&  in xenvif_have_rx_work()
1865 		    (xenvif_rx_queue_stalled(queue)  in xenvif_have_rx_work()
1866 		     || xenvif_rx_queue_ready(queue)))  in xenvif_have_rx_work()
1868 		|| queue->vif->disabled;  in xenvif_have_rx_work()
1871 static long xenvif_rx_queue_timeout(struct xenvif_queue *queue)  in xenvif_rx_queue_timeout()  argument
1876 	skb = skb_peek(&queue->rx_queue);  in xenvif_rx_queue_timeout()
1894 static void xenvif_wait_for_rx_work(struct xenvif_queue *queue)  in xenvif_wait_for_rx_work()  argument
1898 	if (xenvif_have_rx_work(queue))  in xenvif_wait_for_rx_work()
1904 		prepare_to_wait(&queue->wq, &wait, TASK_INTERRUPTIBLE);  in xenvif_wait_for_rx_work()
1905 		if (xenvif_have_rx_work(queue))  in xenvif_wait_for_rx_work()
1907 		ret = schedule_timeout(xenvif_rx_queue_timeout(queue));  in xenvif_wait_for_rx_work()
1911 	finish_wait(&queue->wq, &wait);  in xenvif_wait_for_rx_work()
1916 	struct xenvif_queue *queue = data;  in xenvif_kthread_guest_rx()  local
1917 	struct xenvif *vif = queue->vif;  in xenvif_kthread_guest_rx()
1920 		xenvif_queue_carrier_on(queue);  in xenvif_kthread_guest_rx()
1923 		xenvif_wait_for_rx_work(queue);  in xenvif_kthread_guest_rx()
1935 		if (unlikely(vif->disabled && queue->id == 0)) {  in xenvif_kthread_guest_rx()
1940 		if (!skb_queue_empty(&queue->rx_queue))  in xenvif_kthread_guest_rx()
1941 			xenvif_rx_action(queue);  in xenvif_kthread_guest_rx()
1948 			if (xenvif_rx_queue_stalled(queue))  in xenvif_kthread_guest_rx()
1949 				xenvif_queue_carrier_off(queue);  in xenvif_kthread_guest_rx()
1950 			else if (xenvif_rx_queue_ready(queue))  in xenvif_kthread_guest_rx()
1951 				xenvif_queue_carrier_on(queue);  in xenvif_kthread_guest_rx()
1959 		xenvif_rx_queue_drop_expired(queue);  in xenvif_kthread_guest_rx()
1961 		xenvif_rx_queue_maybe_wake(queue);  in xenvif_kthread_guest_rx()
1967 	xenvif_rx_queue_purge(queue);  in xenvif_kthread_guest_rx()
1972 static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue *queue)  in xenvif_dealloc_kthread_should_stop()  argument
1978 		!atomic_read(&queue->inflight_packets);  in xenvif_dealloc_kthread_should_stop()
1983 	struct xenvif_queue *queue = data;  in xenvif_dealloc_kthread()  local
1986 		wait_event_interruptible(queue->dealloc_wq,  in xenvif_dealloc_kthread()
1987 					 tx_dealloc_work_todo(queue) ||  in xenvif_dealloc_kthread()
1988 					 xenvif_dealloc_kthread_should_stop(queue));  in xenvif_dealloc_kthread()
1989 		if (xenvif_dealloc_kthread_should_stop(queue))  in xenvif_dealloc_kthread()
1992 		xenvif_tx_dealloc_action(queue);  in xenvif_dealloc_kthread()
1997 	if (tx_dealloc_work_todo(queue))  in xenvif_dealloc_kthread()
1998 		xenvif_tx_dealloc_action(queue);  in xenvif_dealloc_kthread()