root/drivers/net/ethernet/google/gve/gve_tx.c

DEFINITIONS

1. gve_tx_put_doorbell
2. gve_tx_fifo_init
3. gve_tx_fifo_release
4. gve_tx_fifo_pad_alloc_one_frag
5. gve_tx_fifo_can_alloc
6. gve_tx_alloc_fifo
7. gve_tx_free_fifo
8. gve_tx_remove_from_block
9. gve_tx_free_ring
10. gve_tx_add_to_block
11. gve_tx_alloc_ring
12. gve_tx_alloc_rings
13. gve_tx_free_rings
14. gve_tx_avail
15. gve_skb_fifo_bytes_required
16. gve_can_tx
17. gve_maybe_stop_tx
18. gve_tx_fill_pkt_desc
19. gve_tx_fill_seg_desc
20. gve_dma_sync_for_device
21. gve_tx_add_skb
22. gve_tx
23. gve_clean_tx_done
24. gve_tx_load_event_counter
25. gve_tx_poll

   1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2 /* Google virtual Ethernet (gve) driver
   3  *
   4  * Copyright (C) 2015-2019 Google, Inc.
   5  */
   6 
   7 #include "gve.h"
   8 #include "gve_adminq.h"
   9 #include <linux/ip.h>
  10 #include <linux/tcp.h>
  11 #include <linux/vmalloc.h>
  12 #include <linux/skbuff.h>
  13 
  14 static inline void gve_tx_put_doorbell(struct gve_priv *priv,
  15                                        struct gve_queue_resources *q_resources,
  16                                        u32 val)
  17 {
  18         iowrite32be(val, &priv->db_bar2[be32_to_cpu(q_resources->db_index)]);
  19 }
  20 
  21 /* gvnic can only transmit from a Registered Segment.
  22  * We copy skb payloads into the registered segment before writing Tx
  23  * descriptors and ringing the Tx doorbell.
  24  *
  25  * gve_tx_fifo_* manages the Registered Segment as a FIFO - clients must
  26  * free allocations in the order they were allocated.
  27  */
  28 
  29 static int gve_tx_fifo_init(struct gve_priv *priv, struct gve_tx_fifo *fifo)
  30 {
  31         fifo->base = vmap(fifo->qpl->pages, fifo->qpl->num_entries, VM_MAP,
  32                           PAGE_KERNEL);
  33         if (unlikely(!fifo->base)) {
  34                 netif_err(priv, drv, priv->dev, "Failed to vmap fifo, qpl_id = %d\n",
  35                           fifo->qpl->id);
  36                 return -ENOMEM;
  37         }
  38 
  39         fifo->size = fifo->qpl->num_entries * PAGE_SIZE;
  40         atomic_set(&fifo->available, fifo->size);
  41         fifo->head = 0;
  42         return 0;
  43 }
  44 
  45 static void gve_tx_fifo_release(struct gve_priv *priv, struct gve_tx_fifo *fifo)
  46 {
  47         WARN(atomic_read(&fifo->available) != fifo->size,
  48              "Releasing non-empty fifo");
  49 
  50         vunmap(fifo->base);
  51 }
  52 
  53 static int gve_tx_fifo_pad_alloc_one_frag(struct gve_tx_fifo *fifo,
  54                                           size_t bytes)
  55 {
  56         return (fifo->head + bytes < fifo->size) ? 0 : fifo->size - fifo->head;
  57 }
  58 
  59 static bool gve_tx_fifo_can_alloc(struct gve_tx_fifo *fifo, size_t bytes)
  60 {
  61         return (atomic_read(&fifo->available) <= bytes) ? false : true;
  62 }
  63 
  64 /* gve_tx_alloc_fifo - Allocate fragment(s) from Tx FIFO
  65  * @fifo: FIFO to allocate from
  66  * @bytes: Allocation size
  67  * @iov: Scatter-gather elements to fill with allocation fragment base/len
  68  *
  69  * Returns number of valid elements in iov[] or negative on error.
  70  *
  71  * Allocations from a given FIFO must be externally synchronized but concurrent
  72  * allocation and frees are allowed.
  73  */
  74 static int gve_tx_alloc_fifo(struct gve_tx_fifo *fifo, size_t bytes,
  75                              struct gve_tx_iovec iov[2])
  76 {
  77         size_t overflow, padding;
  78         u32 aligned_head;
  79         int nfrags = 0;
  80 
  81         if (!bytes)
  82                 return 0;
  83 
  84         /* This check happens before we know how much padding is needed to
  85          * align to a cacheline boundary for the payload, but that is fine,
  86          * because the FIFO head always start aligned, and the FIFO's boundaries
  87          * are aligned, so if there is space for the data, there is space for
  88          * the padding to the next alignment.
  89          */
  90         WARN(!gve_tx_fifo_can_alloc(fifo, bytes),
  91              "Reached %s when there's not enough space in the fifo", __func__);
  92 
  93         nfrags++;
  94 
  95         iov[0].iov_offset = fifo->head;
  96         iov[0].iov_len = bytes;
  97         fifo->head += bytes;
  98 
  99         if (fifo->head > fifo->size) {
 100                 /* If the allocation did not fit in the tail fragment of the
 101                  * FIFO, also use the head fragment.
 102                  */
 103                 nfrags++;
 104                 overflow = fifo->head - fifo->size;
 105                 iov[0].iov_len -= overflow;
 106                 iov[1].iov_offset = 0;  /* Start of fifo*/
 107                 iov[1].iov_len = overflow;
 108 
 109                 fifo->head = overflow;
 110         }
 111 
 112         /* Re-align to a cacheline boundary */
 113         aligned_head = L1_CACHE_ALIGN(fifo->head);
 114         padding = aligned_head - fifo->head;
 115         iov[nfrags - 1].iov_padding = padding;
 116         atomic_sub(bytes + padding, &fifo->available);
 117         fifo->head = aligned_head;
 118 
 119         if (fifo->head == fifo->size)
 120                 fifo->head = 0;
 121 
 122         return nfrags;
 123 }
 124 
 125 /* gve_tx_free_fifo - Return space to Tx FIFO
 126  * @fifo: FIFO to return fragments to
 127  * @bytes: Bytes to free
 128  */
 129 static void gve_tx_free_fifo(struct gve_tx_fifo *fifo, size_t bytes)
 130 {
 131         atomic_add(bytes, &fifo->available);
 132 }
 133 
 134 static void gve_tx_remove_from_block(struct gve_priv *priv, int queue_idx)
 135 {
 136         struct gve_notify_block *block =
 137                         &priv->ntfy_blocks[gve_tx_idx_to_ntfy(priv, queue_idx)];
 138 
 139         block->tx = NULL;
 140 }
 141 
 142 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
 143                              u32 to_do, bool try_to_wake);
 144 
 145 static void gve_tx_free_ring(struct gve_priv *priv, int idx)
 146 {
 147         struct gve_tx_ring *tx = &priv->tx[idx];
 148         struct device *hdev = &priv->pdev->dev;
 149         size_t bytes;
 150         u32 slots;
 151 
 152         gve_tx_remove_from_block(priv, idx);
 153         slots = tx->mask + 1;
 154         gve_clean_tx_done(priv, tx, tx->req, false);
 155         netdev_tx_reset_queue(tx->netdev_txq);
 156 
 157         dma_free_coherent(hdev, sizeof(*tx->q_resources),
 158                           tx->q_resources, tx->q_resources_bus);
 159         tx->q_resources = NULL;
 160 
 161         gve_tx_fifo_release(priv, &tx->tx_fifo);
 162         gve_unassign_qpl(priv, tx->tx_fifo.qpl->id);
 163         tx->tx_fifo.qpl = NULL;
 164 
 165         bytes = sizeof(*tx->desc) * slots;
 166         dma_free_coherent(hdev, bytes, tx->desc, tx->bus);
 167         tx->desc = NULL;
 168 
 169         vfree(tx->info);
 170         tx->info = NULL;
 171 
 172         netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx);
 173 }
 174 
 175 static void gve_tx_add_to_block(struct gve_priv *priv, int queue_idx)
 176 {
 177         int ntfy_idx = gve_tx_idx_to_ntfy(priv, queue_idx);
 178         struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 179         struct gve_tx_ring *tx = &priv->tx[queue_idx];
 180 
 181         block->tx = tx;
 182         tx->ntfy_id = ntfy_idx;
 183 }
 184 
 185 static int gve_tx_alloc_ring(struct gve_priv *priv, int idx)
 186 {
 187         struct gve_tx_ring *tx = &priv->tx[idx];
 188         struct device *hdev = &priv->pdev->dev;
 189         u32 slots = priv->tx_desc_cnt;
 190         size_t bytes;
 191 
 192         /* Make sure everything is zeroed to start */
 193         memset(tx, 0, sizeof(*tx));
 194         tx->q_num = idx;
 195 
 196         tx->mask = slots - 1;
 197 
 198         /* alloc metadata */
 199         tx->info = vzalloc(sizeof(*tx->info) * slots);
 200         if (!tx->info)
 201                 return -ENOMEM;
 202 
 203         /* alloc tx queue */
 204         bytes = sizeof(*tx->desc) * slots;
 205         tx->desc = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL);
 206         if (!tx->desc)
 207                 goto abort_with_info;
 208 
 209         tx->tx_fifo.qpl = gve_assign_tx_qpl(priv);
 210 
 211         /* map Tx FIFO */
 212         if (gve_tx_fifo_init(priv, &tx->tx_fifo))
 213                 goto abort_with_desc;
 214 
 215         tx->q_resources =
 216                 dma_alloc_coherent(hdev,
 217                                    sizeof(*tx->q_resources),
 218                                    &tx->q_resources_bus,
 219                                    GFP_KERNEL);
 220         if (!tx->q_resources)
 221                 goto abort_with_fifo;
 222 
 223         netif_dbg(priv, drv, priv->dev, "tx[%d]->bus=%lx\n", idx,
 224                   (unsigned long)tx->bus);
 225         tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx);
 226         gve_tx_add_to_block(priv, idx);
 227 
 228         return 0;
 229 
 230 abort_with_fifo:
 231         gve_tx_fifo_release(priv, &tx->tx_fifo);
 232 abort_with_desc:
 233         dma_free_coherent(hdev, bytes, tx->desc, tx->bus);
 234         tx->desc = NULL;
 235 abort_with_info:
 236         vfree(tx->info);
 237         tx->info = NULL;
 238         return -ENOMEM;
 239 }
 240 
 241 int gve_tx_alloc_rings(struct gve_priv *priv)
 242 {
 243         int err = 0;
 244         int i;
 245 
 246         for (i = 0; i < priv->tx_cfg.num_queues; i++) {
 247                 err = gve_tx_alloc_ring(priv, i);
 248                 if (err) {
 249                         netif_err(priv, drv, priv->dev,
 250                                   "Failed to alloc tx ring=%d: err=%d\n",
 251                                   i, err);
 252                         break;
 253                 }
 254         }
 255         /* Unallocate if there was an error */
 256         if (err) {
 257                 int j;
 258 
 259                 for (j = 0; j < i; j++)
 260                         gve_tx_free_ring(priv, j);
 261         }
 262         return err;
 263 }
 264 
 265 void gve_tx_free_rings(struct gve_priv *priv)
 266 {
 267         int i;
 268 
 269         for (i = 0; i < priv->tx_cfg.num_queues; i++)
 270                 gve_tx_free_ring(priv, i);
 271 }
 272 
 273 /* gve_tx_avail - Calculates the number of slots available in the ring
 274  * @tx: tx ring to check
 275  *
 276  * Returns the number of slots available
 277  *
 278  * The capacity of the queue is mask + 1. We don't need to reserve an entry.
 279  **/
 280 static inline u32 gve_tx_avail(struct gve_tx_ring *tx)
 281 {
 282         return tx->mask + 1 - (tx->req - tx->done);
 283 }
 284 
 285 static inline int gve_skb_fifo_bytes_required(struct gve_tx_ring *tx,
 286                                               struct sk_buff *skb)
 287 {
 288         int pad_bytes, align_hdr_pad;
 289         int bytes;
 290         int hlen;
 291 
 292         hlen = skb_is_gso(skb) ? skb_checksum_start_offset(skb) +
 293                                  tcp_hdrlen(skb) : skb_headlen(skb);
 294 
 295         pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo,
 296                                                    hlen);
 297         /* We need to take into account the header alignment padding. */
 298         align_hdr_pad = L1_CACHE_ALIGN(hlen) - hlen;
 299         bytes = align_hdr_pad + pad_bytes + skb->len;
 300 
 301         return bytes;
 302 }
 303 
 304 /* The most descriptors we could need are 3 - 1 for the headers, 1 for
 305  * the beginning of the payload at the end of the FIFO, and 1 if the
 306  * payload wraps to the beginning of the FIFO.
 307  */
 308 #define MAX_TX_DESC_NEEDED      3
 309 
 310 /* Check if sufficient resources (descriptor ring space, FIFO space) are
 311  * available to transmit the given number of bytes.
 312  */
 313 static inline bool gve_can_tx(struct gve_tx_ring *tx, int bytes_required)
 314 {
 315         return (gve_tx_avail(tx) >= MAX_TX_DESC_NEEDED &&
 316                 gve_tx_fifo_can_alloc(&tx->tx_fifo, bytes_required));
 317 }
 318 
 319 /* Stops the queue if the skb cannot be transmitted. */
 320 static int gve_maybe_stop_tx(struct gve_tx_ring *tx, struct sk_buff *skb)
 321 {
 322         int bytes_required;
 323 
 324         bytes_required = gve_skb_fifo_bytes_required(tx, skb);
 325         if (likely(gve_can_tx(tx, bytes_required)))
 326                 return 0;
 327 
 328         /* No space, so stop the queue */
 329         tx->stop_queue++;
 330         netif_tx_stop_queue(tx->netdev_txq);
 331         smp_mb();       /* sync with restarting queue in gve_clean_tx_done() */
 332 
 333         /* Now check for resources again, in case gve_clean_tx_done() freed
 334          * resources after we checked and we stopped the queue after
 335          * gve_clean_tx_done() checked.
 336          *
 337          * gve_maybe_stop_tx()                  gve_clean_tx_done()
 338          *   nsegs/can_alloc test failed
 339          *                                        gve_tx_free_fifo()
 340          *                                        if (tx queue stopped)
 341          *                                          netif_tx_queue_wake()
 342          *   netif_tx_stop_queue()
 343          *   Need to check again for space here!
 344          */
 345         if (likely(!gve_can_tx(tx, bytes_required)))
 346                 return -EBUSY;
 347 
 348         netif_tx_start_queue(tx->netdev_txq);
 349         tx->wake_queue++;
 350         return 0;
 351 }
 352 
 353 static void gve_tx_fill_pkt_desc(union gve_tx_desc *pkt_desc,
 354                                  struct sk_buff *skb, bool is_gso,
 355                                  int l4_hdr_offset, u32 desc_cnt,
 356                                  u16 hlen, u64 addr)
 357 {
 358         /* l4_hdr_offset and csum_offset are in units of 16-bit words */
 359         if (is_gso) {
 360                 pkt_desc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM;
 361                 pkt_desc->pkt.l4_csum_offset = skb->csum_offset >> 1;
 362                 pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
 363         } else if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
 364                 pkt_desc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM;
 365                 pkt_desc->pkt.l4_csum_offset = skb->csum_offset >> 1;
 366                 pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
 367         } else {
 368                 pkt_desc->pkt.type_flags = GVE_TXD_STD;
 369                 pkt_desc->pkt.l4_csum_offset = 0;
 370                 pkt_desc->pkt.l4_hdr_offset = 0;
 371         }
 372         pkt_desc->pkt.desc_cnt = desc_cnt;
 373         pkt_desc->pkt.len = cpu_to_be16(skb->len);
 374         pkt_desc->pkt.seg_len = cpu_to_be16(hlen);
 375         pkt_desc->pkt.seg_addr = cpu_to_be64(addr);
 376 }
 377 
 378 static void gve_tx_fill_seg_desc(union gve_tx_desc *seg_desc,
 379                                  struct sk_buff *skb, bool is_gso,
 380                                  u16 len, u64 addr)
 381 {
 382         seg_desc->seg.type_flags = GVE_TXD_SEG;
 383         if (is_gso) {
 384                 if (skb_is_gso_v6(skb))
 385                         seg_desc->seg.type_flags |= GVE_TXSF_IPV6;
 386                 seg_desc->seg.l3_offset = skb_network_offset(skb) >> 1;
 387                 seg_desc->seg.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
 388         }
 389         seg_desc->seg.seg_len = cpu_to_be16(len);
 390         seg_desc->seg.seg_addr = cpu_to_be64(addr);
 391 }
 392 
 393 static void gve_dma_sync_for_device(struct device *dev, dma_addr_t *page_buses,
 394                                     u64 iov_offset, u64 iov_len)
 395 {
 396         u64 last_page = (iov_offset + iov_len - 1) / PAGE_SIZE;
 397         u64 first_page = iov_offset / PAGE_SIZE;
 398         dma_addr_t dma;
 399         u64 page;
 400 
 401         for (page = first_page; page <= last_page; page++) {
 402                 dma = page_buses[page];
 403                 dma_sync_single_for_device(dev, dma, PAGE_SIZE, DMA_TO_DEVICE);
 404         }
 405 }
 406 
 407 static int gve_tx_add_skb(struct gve_tx_ring *tx, struct sk_buff *skb,
 408                           struct device *dev)
 409 {
 410         int pad_bytes, hlen, hdr_nfrags, payload_nfrags, l4_hdr_offset;
 411         union gve_tx_desc *pkt_desc, *seg_desc;
 412         struct gve_tx_buffer_state *info;
 413         bool is_gso = skb_is_gso(skb);
 414         u32 idx = tx->req & tx->mask;
 415         int payload_iov = 2;
 416         int copy_offset;
 417         u32 next_idx;
 418         int i;
 419 
 420         info = &tx->info[idx];
 421         pkt_desc = &tx->desc[idx];
 422 
 423         l4_hdr_offset = skb_checksum_start_offset(skb);
 424         /* If the skb is gso, then we want the tcp header in the first segment
 425          * otherwise we want the linear portion of the skb (which will contain
 426          * the checksum because skb->csum_start and skb->csum_offset are given
 427          * relative to skb->head) in the first segment.
 428          */
 429         hlen = is_gso ? l4_hdr_offset + tcp_hdrlen(skb) :
 430                         skb_headlen(skb);
 431 
 432         info->skb =  skb;
 433         /* We don't want to split the header, so if necessary, pad to the end
 434          * of the fifo and then put the header at the beginning of the fifo.
 435          */
 436         pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, hlen);
 437         hdr_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, hlen + pad_bytes,
 438                                        &info->iov[0]);
 439         WARN(!hdr_nfrags, "hdr_nfrags should never be 0!");
 440         payload_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, skb->len - hlen,
 441                                            &info->iov[payload_iov]);
 442 
 443         gve_tx_fill_pkt_desc(pkt_desc, skb, is_gso, l4_hdr_offset,
 444                              1 + payload_nfrags, hlen,
 445                              info->iov[hdr_nfrags - 1].iov_offset);
 446 
 447         skb_copy_bits(skb, 0,
 448                       tx->tx_fifo.base + info->iov[hdr_nfrags - 1].iov_offset,
 449                       hlen);
 450         gve_dma_sync_for_device(dev, tx->tx_fifo.qpl->page_buses,
 451                                 info->iov[hdr_nfrags - 1].iov_offset,
 452                                 info->iov[hdr_nfrags - 1].iov_len);
 453         copy_offset = hlen;
 454 
 455         for (i = payload_iov; i < payload_nfrags + payload_iov; i++) {
 456                 next_idx = (tx->req + 1 + i - payload_iov) & tx->mask;
 457                 seg_desc = &tx->desc[next_idx];
 458 
 459                 gve_tx_fill_seg_desc(seg_desc, skb, is_gso,
 460                                      info->iov[i].iov_len,
 461                                      info->iov[i].iov_offset);
 462 
 463                 skb_copy_bits(skb, copy_offset,
 464                               tx->tx_fifo.base + info->iov[i].iov_offset,
 465                               info->iov[i].iov_len);
 466                 gve_dma_sync_for_device(dev, tx->tx_fifo.qpl->page_buses,
 467                                         info->iov[i].iov_offset,
 468                                         info->iov[i].iov_len);
 469                 copy_offset += info->iov[i].iov_len;
 470         }
 471 
 472         return 1 + payload_nfrags;
 473 }
 474 
 475 netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev)
 476 {
 477         struct gve_priv *priv = netdev_priv(dev);
 478         struct gve_tx_ring *tx;
 479         int nsegs;
 480 
 481         WARN(skb_get_queue_mapping(skb) > priv->tx_cfg.num_queues,
 482              "skb queue index out of range");
 483         tx = &priv->tx[skb_get_queue_mapping(skb)];
 484         if (unlikely(gve_maybe_stop_tx(tx, skb))) {
 485                 /* We need to ring the txq doorbell -- we have stopped the Tx
 486                  * queue for want of resources, but prior calls to gve_tx()
 487                  * may have added descriptors without ringing the doorbell.
 488                  */
 489 
 490                 gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
 491                 return NETDEV_TX_BUSY;
 492         }
 493         nsegs = gve_tx_add_skb(tx, skb, &priv->pdev->dev);
 494 
 495         netdev_tx_sent_queue(tx->netdev_txq, skb->len);
 496         skb_tx_timestamp(skb);
 497 
 498         /* give packets to NIC */
 499         tx->req += nsegs;
 500 
 501         if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more())
 502                 return NETDEV_TX_OK;
 503 
 504         gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
 505         return NETDEV_TX_OK;
 506 }
 507 
 508 #define GVE_TX_START_THRESH     PAGE_SIZE
 509 
 510 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
 511                              u32 to_do, bool try_to_wake)
 512 {
 513         struct gve_tx_buffer_state *info;
 514         u64 pkts = 0, bytes = 0;
 515         size_t space_freed = 0;
 516         struct sk_buff *skb;
 517         int i, j;
 518         u32 idx;
 519 
 520         for (j = 0; j < to_do; j++) {
 521                 idx = tx->done & tx->mask;
 522                 netif_info(priv, tx_done, priv->dev,
 523                            "[%d] %s: idx=%d (req=%u done=%u)\n",
 524                            tx->q_num, __func__, idx, tx->req, tx->done);
 525                 info = &tx->info[idx];
 526                 skb = info->skb;
 527 
 528                 /* Mark as free */
 529                 if (skb) {
 530                         info->skb = NULL;
 531                         bytes += skb->len;
 532                         pkts++;
 533                         dev_consume_skb_any(skb);
 534                         /* FIFO free */
 535                         for (i = 0; i < ARRAY_SIZE(info->iov); i++) {
 536                                 space_freed += info->iov[i].iov_len +
 537                                                info->iov[i].iov_padding;
 538                                 info->iov[i].iov_len = 0;
 539                                 info->iov[i].iov_padding = 0;
 540                         }
 541                 }
 542                 tx->done++;
 543         }
 544 
 545         gve_tx_free_fifo(&tx->tx_fifo, space_freed);
 546         u64_stats_update_begin(&tx->statss);
 547         tx->bytes_done += bytes;
 548         tx->pkt_done += pkts;
 549         u64_stats_update_end(&tx->statss);
 550         netdev_tx_completed_queue(tx->netdev_txq, pkts, bytes);
 551 
 552         /* start the queue if we've stopped it */
 553 #ifndef CONFIG_BQL
 554         /* Make sure that the doorbells are synced */
 555         smp_mb();
 556 #endif
 557         if (try_to_wake && netif_tx_queue_stopped(tx->netdev_txq) &&
 558             likely(gve_can_tx(tx, GVE_TX_START_THRESH))) {
 559                 tx->wake_queue++;
 560                 netif_tx_wake_queue(tx->netdev_txq);
 561         }
 562 
 563         return pkts;
 564 }
 565 
 566 __be32 gve_tx_load_event_counter(struct gve_priv *priv,
 567                                  struct gve_tx_ring *tx)
 568 {
 569         u32 counter_index = be32_to_cpu((tx->q_resources->counter_index));
 570 
 571         return READ_ONCE(priv->counter_array[counter_index]);
 572 }
 573 
 574 bool gve_tx_poll(struct gve_notify_block *block, int budget)
 575 {
 576         struct gve_priv *priv = block->priv;
 577         struct gve_tx_ring *tx = block->tx;
 578         bool repoll = false;
 579         u32 nic_done;
 580         u32 to_do;
 581 
 582         /* If budget is 0, do all the work */
 583         if (budget == 0)
 584                 budget = INT_MAX;
 585 
 586         /* Find out how much work there is to be done */
 587         tx->last_nic_done = gve_tx_load_event_counter(priv, tx);
 588         nic_done = be32_to_cpu(tx->last_nic_done);
 589         if (budget > 0) {
 590                 /* Do as much work as we have that the budget will
 591                  * allow
 592                  */
 593                 to_do = min_t(u32, (nic_done - tx->done), budget);
 594                 gve_clean_tx_done(priv, tx, to_do, true);
 595         }
 596         /* If we still have work we want to repoll */
 597         repoll |= (nic_done != tx->done);
 598         return repoll;
 599 }