45 	return &tx_queue->buffer[efx_tx_queue_get_insert_index(tx_queue)];  in __efx_tx_queue_get_insert_buffer()
51 	struct efx_tx_buffer *buffer =  in efx_tx_queue_get_insert_buffer()  local
54 	EFX_BUG_ON_PARANOID(buffer->len);  in efx_tx_queue_get_insert_buffer()
55 	EFX_BUG_ON_PARANOID(buffer->flags);  in efx_tx_queue_get_insert_buffer()
56 	EFX_BUG_ON_PARANOID(buffer->unmap_len);  in efx_tx_queue_get_insert_buffer()
58 	return buffer;  in efx_tx_queue_get_insert_buffer()
62 			       struct efx_tx_buffer *buffer,  in efx_dequeue_buffer()  argument
66 	if (buffer->unmap_len) {  in efx_dequeue_buffer()
68 		dma_addr_t unmap_addr = buffer->dma_addr - buffer->dma_offset;  in efx_dequeue_buffer()
69 		if (buffer->flags & EFX_TX_BUF_MAP_SINGLE)  in efx_dequeue_buffer()
70 			dma_unmap_single(dma_dev, unmap_addr, buffer->unmap_len,  in efx_dequeue_buffer()
73 			dma_unmap_page(dma_dev, unmap_addr, buffer->unmap_len,  in efx_dequeue_buffer()
75 		buffer->unmap_len = 0;  in efx_dequeue_buffer()
78 	if (buffer->flags & EFX_TX_BUF_SKB) {  in efx_dequeue_buffer()
80 		(*bytes_compl) += buffer->skb->len;  in efx_dequeue_buffer()
81 		dev_consume_skb_any((struct sk_buff *)buffer->skb);  in efx_dequeue_buffer()
85 	} else if (buffer->flags & EFX_TX_BUF_HEAP) {  in efx_dequeue_buffer()
86 		kfree(buffer->heap_buf);  in efx_dequeue_buffer()
89 	buffer->len = 0;  in efx_dequeue_buffer()
90 	buffer->flags = 0;  in efx_dequeue_buffer()
273 	struct efx_tx_buffer *buffer =  in efx_enqueue_skb_pio()  local
304 	EFX_POPULATE_QWORD_5(buffer->option,  in efx_enqueue_skb_pio()
313 	return buffer;  in efx_enqueue_skb_pio()
337 	struct efx_tx_buffer *buffer;  in efx_enqueue_skb()  local
364 		buffer = efx_enqueue_skb_pio(tx_queue, skb);  in efx_enqueue_skb()
389 			buffer = efx_tx_queue_get_insert_buffer(tx_queue);  in efx_enqueue_skb()
396 			buffer->len = dma_len;  in efx_enqueue_skb()
397 			buffer->dma_addr = dma_addr;  in efx_enqueue_skb()
398 			buffer->flags = EFX_TX_BUF_CONT;  in efx_enqueue_skb()
405 		buffer->flags = EFX_TX_BUF_CONT | dma_flags;  in efx_enqueue_skb()
406 		buffer->unmap_len = unmap_len;  in efx_enqueue_skb()
407 		buffer->dma_offset = buffer->dma_addr - unmap_addr;  in efx_enqueue_skb()
426 	buffer->skb = skb;  in efx_enqueue_skb()
427 	buffer->flags = EFX_TX_BUF_SKB | dma_flags;  in efx_enqueue_skb()
466 		buffer = __efx_tx_queue_get_insert_buffer(tx_queue);  in efx_enqueue_skb()
467 		efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);  in efx_enqueue_skb()
500 		struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];  in efx_dequeue_buffers()  local
502 		if (!(buffer->flags & EFX_TX_BUF_OPTION) &&  in efx_dequeue_buffers()
503 		    unlikely(buffer->len == 0)) {  in efx_dequeue_buffers()
511 		efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl);  in efx_dequeue_buffers()
591 				if (!tx_queue->buffer) {  in efx_setup_tc()
695 	tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer),  in efx_probe_tx_queue()
697 	if (!tx_queue->buffer)  in efx_probe_tx_queue()
721 	kfree(tx_queue->buffer);  in efx_probe_tx_queue()
722 	tx_queue->buffer = NULL;  in efx_probe_tx_queue()
747 	struct efx_tx_buffer *buffer;  in efx_fini_tx_queue()  local
752 	if (!tx_queue->buffer)  in efx_fini_tx_queue()
758 		buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask];  in efx_fini_tx_queue()
759 		efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);  in efx_fini_tx_queue()
771 	if (!tx_queue->buffer)  in efx_remove_tx_queue()
786 	kfree(tx_queue->buffer);  in efx_remove_tx_queue()
787 	tx_queue->buffer = NULL;  in efx_remove_tx_queue()
877 			       struct efx_tx_buffer *buffer, unsigned int len)  in efx_tsoh_get_buffer()  argument
881 	EFX_BUG_ON_PARANOID(buffer->len);  in efx_tsoh_get_buffer()
882 	EFX_BUG_ON_PARANOID(buffer->flags);  in efx_tsoh_get_buffer()
883 	EFX_BUG_ON_PARANOID(buffer->unmap_len);  in efx_tsoh_get_buffer()
899 		buffer->dma_addr = page_buf->dma_addr + offset;  in efx_tsoh_get_buffer()
900 		buffer->flags = EFX_TX_BUF_CONT;  in efx_tsoh_get_buffer()
904 		buffer->heap_buf = kmalloc(NET_IP_ALIGN + len, GFP_ATOMIC);  in efx_tsoh_get_buffer()
905 		if (unlikely(!buffer->heap_buf))  in efx_tsoh_get_buffer()
907 		result = (u8 *)buffer->heap_buf + NET_IP_ALIGN;  in efx_tsoh_get_buffer()
908 		buffer->flags = EFX_TX_BUF_CONT | EFX_TX_BUF_HEAP;  in efx_tsoh_get_buffer()
911 	buffer->len = len;  in efx_tsoh_get_buffer()
929 	struct efx_tx_buffer *buffer;  in efx_tx_queue_insert()  local
936 		buffer = efx_tx_queue_get_insert_buffer(tx_queue);  in efx_tx_queue_insert()
943 		buffer->dma_addr = dma_addr;  in efx_tx_queue_insert()
951 		buffer->len = dma_len;  in efx_tx_queue_insert()
952 		buffer->flags = EFX_TX_BUF_CONT;  in efx_tx_queue_insert()
958 	buffer->len = len;  in efx_tx_queue_insert()
959 	*final_buffer = buffer;  in efx_tx_queue_insert()
971 			      struct efx_tx_buffer *buffer, u8 *header)  in efx_tso_put_header()  argument
973 	if (unlikely(buffer->flags & EFX_TX_BUF_HEAP)) {  in efx_tso_put_header()
974 		buffer->dma_addr = dma_map_single(&tx_queue->efx->pci_dev->dev,  in efx_tso_put_header()
975 						  header, buffer->len,  in efx_tso_put_header()
978 					       buffer->dma_addr))) {  in efx_tso_put_header()
979 			kfree(buffer->heap_buf);  in efx_tso_put_header()
980 			buffer->len = 0;  in efx_tso_put_header()
981 			buffer->flags = 0;  in efx_tso_put_header()
984 		buffer->unmap_len = buffer->len;  in efx_tso_put_header()
985 		buffer->dma_offset = 0;  in efx_tso_put_header()
986 		buffer->flags |= EFX_TX_BUF_MAP_SINGLE;  in efx_tso_put_header()
1000 	struct efx_tx_buffer *buffer;  in efx_enqueue_unwind()  local
1005 		buffer = __efx_tx_queue_get_insert_buffer(tx_queue);  in efx_enqueue_unwind()
1006 		efx_dequeue_buffer(tx_queue, buffer, NULL, NULL);  in efx_enqueue_unwind()
1098 	struct efx_tx_buffer *buffer;  in tso_fill_packet_with_fragment()  local
1115 	efx_tx_queue_insert(tx_queue, st->dma_addr, n, &buffer);  in tso_fill_packet_with_fragment()
1119 		buffer->skb = skb;  in tso_fill_packet_with_fragment()
1120 		buffer->flags = EFX_TX_BUF_SKB;  in tso_fill_packet_with_fragment()
1122 		buffer->flags = EFX_TX_BUF_CONT;  in tso_fill_packet_with_fragment()
1127 		buffer->unmap_len = st->unmap_len;  in tso_fill_packet_with_fragment()
1128 		buffer->dma_offset = buffer->unmap_len - buffer->len;  in tso_fill_packet_with_fragment()
1129 		buffer->flags |= st->dma_flags;  in tso_fill_packet_with_fragment()
1150 	struct efx_tx_buffer *buffer =  in tso_start_new_packet()  local
1170 		header = efx_tsoh_get_buffer(tx_queue, buffer, st->header_len);  in tso_start_new_packet()
1197 		rc = efx_tso_put_header(tx_queue, buffer, header);  in tso_start_new_packet()
1206 		buffer->flags = EFX_TX_BUF_OPTION;  in tso_start_new_packet()
1207 		buffer->len = 0;  in tso_start_new_packet()
1208 		buffer->unmap_len = 0;  in tso_start_new_packet()
1209 		EFX_POPULATE_QWORD_5(buffer->option,  in tso_start_new_packet()
1221 		buffer = efx_tx_queue_get_insert_buffer(tx_queue);  in tso_start_new_packet()
1222 		buffer->dma_addr = st->header_dma_addr;  in tso_start_new_packet()
1223 		buffer->len = st->header_len;  in tso_start_new_packet()
1225 			buffer->flags = EFX_TX_BUF_CONT | EFX_TX_BUF_MAP_SINGLE;  in tso_start_new_packet()
1226 			buffer->unmap_len = st->header_unmap_len;  in tso_start_new_packet()
1227 			buffer->dma_offset = 0;  in tso_start_new_packet()
1233 			buffer->flags = EFX_TX_BUF_CONT;  in tso_start_new_packet()
1234 			buffer->unmap_len = 0;  in tso_start_new_packet()