Lines Matching refs:ring
154 struct mlx4_en_rx_ring *ring) in mlx4_en_init_allocator() argument
162 if (mlx4_alloc_pages(priv, &ring->page_alloc[i], in mlx4_en_init_allocator()
167 i, ring->page_alloc[i].page_size, in mlx4_en_init_allocator()
168 atomic_read(&ring->page_alloc[i].page->_count)); in mlx4_en_init_allocator()
176 page_alloc = &ring->page_alloc[i]; in mlx4_en_init_allocator()
188 struct mlx4_en_rx_ring *ring) in mlx4_en_destroy_allocator() argument
196 page_alloc = &ring->page_alloc[i]; in mlx4_en_destroy_allocator()
212 struct mlx4_en_rx_ring *ring, int index) in mlx4_en_init_rx_desc() argument
214 struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index; in mlx4_en_init_rx_desc()
228 possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; in mlx4_en_init_rx_desc()
237 struct mlx4_en_rx_ring *ring, int index, in mlx4_en_prepare_rx_desc() argument
240 struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride); in mlx4_en_prepare_rx_desc()
241 struct mlx4_en_rx_alloc *frags = ring->rx_info + in mlx4_en_prepare_rx_desc()
244 return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc, gfp); in mlx4_en_prepare_rx_desc()
247 static inline bool mlx4_en_is_ring_empty(struct mlx4_en_rx_ring *ring) in mlx4_en_is_ring_empty() argument
249 BUG_ON((u32)(ring->prod - ring->cons) > ring->actual_size); in mlx4_en_is_ring_empty()
250 return ring->prod == ring->cons; in mlx4_en_is_ring_empty()
253 static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) in mlx4_en_update_rx_prod_db() argument
255 *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); in mlx4_en_update_rx_prod_db()
259 struct mlx4_en_rx_ring *ring, in mlx4_en_free_rx_desc() argument
265 frags = ring->rx_info + (index << priv->log_rx_info); in mlx4_en_free_rx_desc()
274 struct mlx4_en_rx_ring *ring; in mlx4_en_fill_rx_buffers() local
281 ring = priv->rx_ring[ring_ind]; in mlx4_en_fill_rx_buffers()
283 if (mlx4_en_prepare_rx_desc(priv, ring, in mlx4_en_fill_rx_buffers()
284 ring->actual_size, in mlx4_en_fill_rx_buffers()
286 if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) { in mlx4_en_fill_rx_buffers()
290 new_size = rounddown_pow_of_two(ring->actual_size); in mlx4_en_fill_rx_buffers()
292 ring->actual_size, new_size); in mlx4_en_fill_rx_buffers()
296 ring->actual_size++; in mlx4_en_fill_rx_buffers()
297 ring->prod++; in mlx4_en_fill_rx_buffers()
304 ring = priv->rx_ring[ring_ind]; in mlx4_en_fill_rx_buffers()
305 while (ring->actual_size > new_size) { in mlx4_en_fill_rx_buffers()
306 ring->actual_size--; in mlx4_en_fill_rx_buffers()
307 ring->prod--; in mlx4_en_fill_rx_buffers()
308 mlx4_en_free_rx_desc(priv, ring, ring->actual_size); in mlx4_en_fill_rx_buffers()
316 struct mlx4_en_rx_ring *ring) in mlx4_en_free_rx_buf() argument
321 ring->cons, ring->prod); in mlx4_en_free_rx_buf()
324 while (!mlx4_en_is_ring_empty(ring)) { in mlx4_en_free_rx_buf()
325 index = ring->cons & ring->size_mask; in mlx4_en_free_rx_buf()
327 mlx4_en_free_rx_desc(priv, ring, index); in mlx4_en_free_rx_buf()
328 ++ring->cons; in mlx4_en_free_rx_buf()
363 struct mlx4_en_rx_ring *ring; in mlx4_en_create_rx_ring() local
367 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node); in mlx4_en_create_rx_ring()
368 if (!ring) { in mlx4_en_create_rx_ring()
369 ring = kzalloc(sizeof(*ring), GFP_KERNEL); in mlx4_en_create_rx_ring()
370 if (!ring) { in mlx4_en_create_rx_ring()
376 ring->prod = 0; in mlx4_en_create_rx_ring()
377 ring->cons = 0; in mlx4_en_create_rx_ring()
378 ring->size = size; in mlx4_en_create_rx_ring()
379 ring->size_mask = size - 1; in mlx4_en_create_rx_ring()
380 ring->stride = stride; in mlx4_en_create_rx_ring()
381 ring->log_stride = ffs(ring->stride) - 1; in mlx4_en_create_rx_ring()
382 ring->buf_size = ring->size * ring->stride + TXBB_SIZE; in mlx4_en_create_rx_ring()
386 ring->rx_info = vmalloc_node(tmp, node); in mlx4_en_create_rx_ring()
387 if (!ring->rx_info) { in mlx4_en_create_rx_ring()
388 ring->rx_info = vmalloc(tmp); in mlx4_en_create_rx_ring()
389 if (!ring->rx_info) { in mlx4_en_create_rx_ring()
396 ring->rx_info, tmp); in mlx4_en_create_rx_ring()
400 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, in mlx4_en_create_rx_ring()
401 ring->buf_size, 2 * PAGE_SIZE); in mlx4_en_create_rx_ring()
406 err = mlx4_en_map_buffer(&ring->wqres.buf); in mlx4_en_create_rx_ring()
411 ring->buf = ring->wqres.buf.direct.buf; in mlx4_en_create_rx_ring()
413 ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter; in mlx4_en_create_rx_ring()
415 *pring = ring; in mlx4_en_create_rx_ring()
419 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); in mlx4_en_create_rx_ring()
421 vfree(ring->rx_info); in mlx4_en_create_rx_ring()
422 ring->rx_info = NULL; in mlx4_en_create_rx_ring()
424 kfree(ring); in mlx4_en_create_rx_ring()
432 struct mlx4_en_rx_ring *ring; in mlx4_en_activate_rx_rings() local
440 ring = priv->rx_ring[ring_ind]; in mlx4_en_activate_rx_rings()
442 ring->prod = 0; in mlx4_en_activate_rx_rings()
443 ring->cons = 0; in mlx4_en_activate_rx_rings()
444 ring->actual_size = 0; in mlx4_en_activate_rx_rings()
445 ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; in mlx4_en_activate_rx_rings()
447 ring->stride = stride; in mlx4_en_activate_rx_rings()
448 if (ring->stride <= TXBB_SIZE) in mlx4_en_activate_rx_rings()
449 ring->buf += TXBB_SIZE; in mlx4_en_activate_rx_rings()
451 ring->log_stride = ffs(ring->stride) - 1; in mlx4_en_activate_rx_rings()
452 ring->buf_size = ring->size * ring->stride; in mlx4_en_activate_rx_rings()
454 memset(ring->buf, 0, ring->buf_size); in mlx4_en_activate_rx_rings()
455 mlx4_en_update_rx_prod_db(ring); in mlx4_en_activate_rx_rings()
458 for (i = 0; i < ring->size; i++) in mlx4_en_activate_rx_rings()
459 mlx4_en_init_rx_desc(priv, ring, i); in mlx4_en_activate_rx_rings()
462 err = mlx4_en_init_allocator(priv, ring); in mlx4_en_activate_rx_rings()
465 if (ring->stride <= TXBB_SIZE) in mlx4_en_activate_rx_rings()
466 ring->buf -= TXBB_SIZE; in mlx4_en_activate_rx_rings()
476 ring = priv->rx_ring[ring_ind]; in mlx4_en_activate_rx_rings()
478 ring->size_mask = ring->actual_size - 1; in mlx4_en_activate_rx_rings()
479 mlx4_en_update_rx_prod_db(ring); in mlx4_en_activate_rx_rings()
505 int ring; in mlx4_en_recover_from_oom() local
510 for (ring = 0; ring < priv->rx_ring_num; ring++) { in mlx4_en_recover_from_oom()
511 if (mlx4_en_is_ring_empty(priv->rx_ring[ring])) in mlx4_en_recover_from_oom()
512 napi_reschedule(&priv->rx_cq[ring]->napi); in mlx4_en_recover_from_oom()
521 struct mlx4_en_rx_ring *ring = *pring; in mlx4_en_destroy_rx_ring() local
523 mlx4_en_unmap_buffer(&ring->wqres.buf); in mlx4_en_destroy_rx_ring()
524 mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); in mlx4_en_destroy_rx_ring()
525 vfree(ring->rx_info); in mlx4_en_destroy_rx_ring()
526 ring->rx_info = NULL; in mlx4_en_destroy_rx_ring()
527 kfree(ring); in mlx4_en_destroy_rx_ring()
535 struct mlx4_en_rx_ring *ring) in mlx4_en_deactivate_rx_ring() argument
537 mlx4_en_free_rx_buf(priv, ring); in mlx4_en_deactivate_rx_ring()
538 if (ring->stride <= TXBB_SIZE) in mlx4_en_deactivate_rx_ring()
539 ring->buf -= TXBB_SIZE; in mlx4_en_deactivate_rx_ring()
540 mlx4_en_destroy_allocator(priv, ring); in mlx4_en_deactivate_rx_ring()
663 struct mlx4_en_rx_ring *ring) in mlx4_en_refill_rx_buffers() argument
665 int index = ring->prod & ring->size_mask; in mlx4_en_refill_rx_buffers()
667 while ((u32) (ring->prod - ring->cons) < ring->actual_size) { in mlx4_en_refill_rx_buffers()
668 if (mlx4_en_prepare_rx_desc(priv, ring, index, in mlx4_en_refill_rx_buffers()
671 ring->prod++; in mlx4_en_refill_rx_buffers()
672 index = ring->prod & ring->size_mask; in mlx4_en_refill_rx_buffers()
755 struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; in mlx4_en_process_rx_cq() local
777 index = cq->mcq.cons_index & ring->size_mask; in mlx4_en_process_rx_cq()
784 frags = ring->rx_info + (index << priv->log_rx_info); in mlx4_en_process_rx_cq()
785 rx_desc = ring->buf + (index << ring->log_stride); in mlx4_en_process_rx_cq()
844 length -= ring->fcs_del; in mlx4_en_process_rx_cq()
845 ring->bytes += length; in mlx4_en_process_rx_cq()
846 ring->packets++; in mlx4_en_process_rx_cq()
856 ring->csum_ok++; in mlx4_en_process_rx_cq()
859 ring->csum_none++; in mlx4_en_process_rx_cq()
866 ring->csum_complete++; in mlx4_en_process_rx_cq()
869 ring->csum_none++; in mlx4_en_process_rx_cq()
874 ring->csum_none++; in mlx4_en_process_rx_cq()
901 ring->csum_none++; in mlx4_en_process_rx_cq()
902 ring->csum_complete--; in mlx4_en_process_rx_cq()
927 skb_record_rx_queue(gro_skb, cq->ring); in mlx4_en_process_rx_cq()
930 if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) { in mlx4_en_process_rx_cq()
956 ring->csum_complete--; in mlx4_en_process_rx_cq()
957 ring->csum_none++; in mlx4_en_process_rx_cq()
963 skb_record_rx_queue(skb, cq->ring); in mlx4_en_process_rx_cq()
978 if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) { in mlx4_en_process_rx_cq()
996 index = (cq->mcq.cons_index) & ring->size_mask; in mlx4_en_process_rx_cq()
1006 ring->cons = cq->mcq.cons_index; in mlx4_en_process_rx_cq()
1007 mlx4_en_refill_rx_buffers(priv, ring); in mlx4_en_process_rx_cq()
1008 mlx4_en_update_rx_prod_db(ring); in mlx4_en_process_rx_cq()
1109 struct mlx4_en_rx_ring *ring, in mlx4_en_config_rss_qp() argument
1129 mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, in mlx4_en_config_rss_qp()
1130 qpn, ring->cqn, -1, context); in mlx4_en_config_rss_qp()
1131 context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); in mlx4_en_config_rss_qp()
1137 ring->fcs_del = 0; in mlx4_en_config_rss_qp()
1139 ring->fcs_del = ETH_FCS_LEN; in mlx4_en_config_rss_qp()
1141 ring->fcs_del = 0; in mlx4_en_config_rss_qp()
1143 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); in mlx4_en_config_rss_qp()
1148 mlx4_en_update_rx_prod_db(ring); in mlx4_en_config_rss_qp()