Lines Matching refs:ring
154 struct mlx4_en_rx_ring *ring) in mlx4_en_init_allocator() argument
162 if (mlx4_alloc_pages(priv, &ring->page_alloc[i], in mlx4_en_init_allocator()
167 i, ring->page_alloc[i].page_size, in mlx4_en_init_allocator()
168 atomic_read(&ring->page_alloc[i].page->_count)); in mlx4_en_init_allocator()
176 page_alloc = &ring->page_alloc[i]; in mlx4_en_init_allocator()
188 struct mlx4_en_rx_ring *ring) in mlx4_en_destroy_allocator() argument
196 page_alloc = &ring->page_alloc[i]; in mlx4_en_destroy_allocator()
212 struct mlx4_en_rx_ring *ring, int index) in mlx4_en_init_rx_desc() argument
214 struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index; in mlx4_en_init_rx_desc()
228 possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; in mlx4_en_init_rx_desc()
237 struct mlx4_en_rx_ring *ring, int index, in mlx4_en_prepare_rx_desc() argument
240 struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride); in mlx4_en_prepare_rx_desc()
241 struct mlx4_en_rx_alloc *frags = ring->rx_info + in mlx4_en_prepare_rx_desc()
244 return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc, gfp); in mlx4_en_prepare_rx_desc()
247 static inline bool mlx4_en_is_ring_empty(struct mlx4_en_rx_ring *ring) in mlx4_en_is_ring_empty() argument
249 return ring->prod == ring->cons; in mlx4_en_is_ring_empty()
252 static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) in mlx4_en_update_rx_prod_db() argument
254 *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); in mlx4_en_update_rx_prod_db()
258 struct mlx4_en_rx_ring *ring, in mlx4_en_free_rx_desc() argument
264 frags = ring->rx_info + (index << priv->log_rx_info); in mlx4_en_free_rx_desc()
273 struct mlx4_en_rx_ring *ring; in mlx4_en_fill_rx_buffers() local
280 ring = priv->rx_ring[ring_ind]; in mlx4_en_fill_rx_buffers()
282 if (mlx4_en_prepare_rx_desc(priv, ring, in mlx4_en_fill_rx_buffers()
283 ring->actual_size, in mlx4_en_fill_rx_buffers()
285 if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) { in mlx4_en_fill_rx_buffers()
289 new_size = rounddown_pow_of_two(ring->actual_size); in mlx4_en_fill_rx_buffers()
291 ring->actual_size, new_size); in mlx4_en_fill_rx_buffers()
295 ring->actual_size++; in mlx4_en_fill_rx_buffers()
296 ring->prod++; in mlx4_en_fill_rx_buffers()
303 ring = priv->rx_ring[ring_ind]; in mlx4_en_fill_rx_buffers()
304 while (ring->actual_size > new_size) { in mlx4_en_fill_rx_buffers()
305 ring->actual_size--; in mlx4_en_fill_rx_buffers()
306 ring->prod--; in mlx4_en_fill_rx_buffers()
307 mlx4_en_free_rx_desc(priv, ring, ring->actual_size); in mlx4_en_fill_rx_buffers()
315 struct mlx4_en_rx_ring *ring) in mlx4_en_free_rx_buf() argument
320 ring->cons, ring->prod); in mlx4_en_free_rx_buf()
323 while (!mlx4_en_is_ring_empty(ring)) { in mlx4_en_free_rx_buf()
324 index = ring->cons & ring->size_mask; in mlx4_en_free_rx_buf()
326 mlx4_en_free_rx_desc(priv, ring, index); in mlx4_en_free_rx_buf()
327 ++ring->cons; in mlx4_en_free_rx_buf()
357 struct mlx4_en_rx_ring *ring; in mlx4_en_create_rx_ring() local
361 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node); in mlx4_en_create_rx_ring()
362 if (!ring) { in mlx4_en_create_rx_ring()
363 ring = kzalloc(sizeof(*ring), GFP_KERNEL); in mlx4_en_create_rx_ring()
364 if (!ring) { in mlx4_en_create_rx_ring()
370 ring->prod = 0; in mlx4_en_create_rx_ring()
371 ring->cons = 0; in mlx4_en_create_rx_ring()
372 ring->size = size; in mlx4_en_create_rx_ring()
373 ring->size_mask = size - 1; in mlx4_en_create_rx_ring()
374 ring->stride = stride; in mlx4_en_create_rx_ring()
375 ring->log_stride = ffs(ring->stride) - 1; in mlx4_en_create_rx_ring()
376 ring->buf_size = ring->size * ring->stride + TXBB_SIZE; in mlx4_en_create_rx_ring()
380 ring->rx_info = vmalloc_node(tmp, node); in mlx4_en_create_rx_ring()
381 if (!ring->rx_info) { in mlx4_en_create_rx_ring()
382 ring->rx_info = vmalloc(tmp); in mlx4_en_create_rx_ring()
383 if (!ring->rx_info) { in mlx4_en_create_rx_ring()
390 ring->rx_info, tmp); in mlx4_en_create_rx_ring()
394 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, in mlx4_en_create_rx_ring()
395 ring->buf_size, 2 * PAGE_SIZE); in mlx4_en_create_rx_ring()
400 err = mlx4_en_map_buffer(&ring->wqres.buf); in mlx4_en_create_rx_ring()
405 ring->buf = ring->wqres.buf.direct.buf; in mlx4_en_create_rx_ring()
407 ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter; in mlx4_en_create_rx_ring()
409 *pring = ring; in mlx4_en_create_rx_ring()
413 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); in mlx4_en_create_rx_ring()
415 vfree(ring->rx_info); in mlx4_en_create_rx_ring()
416 ring->rx_info = NULL; in mlx4_en_create_rx_ring()
418 kfree(ring); in mlx4_en_create_rx_ring()
426 struct mlx4_en_rx_ring *ring; in mlx4_en_activate_rx_rings() local
434 ring = priv->rx_ring[ring_ind]; in mlx4_en_activate_rx_rings()
436 ring->prod = 0; in mlx4_en_activate_rx_rings()
437 ring->cons = 0; in mlx4_en_activate_rx_rings()
438 ring->actual_size = 0; in mlx4_en_activate_rx_rings()
439 ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; in mlx4_en_activate_rx_rings()
441 ring->stride = stride; in mlx4_en_activate_rx_rings()
442 if (ring->stride <= TXBB_SIZE) in mlx4_en_activate_rx_rings()
443 ring->buf += TXBB_SIZE; in mlx4_en_activate_rx_rings()
445 ring->log_stride = ffs(ring->stride) - 1; in mlx4_en_activate_rx_rings()
446 ring->buf_size = ring->size * ring->stride; in mlx4_en_activate_rx_rings()
448 memset(ring->buf, 0, ring->buf_size); in mlx4_en_activate_rx_rings()
449 mlx4_en_update_rx_prod_db(ring); in mlx4_en_activate_rx_rings()
452 for (i = 0; i < ring->size; i++) in mlx4_en_activate_rx_rings()
453 mlx4_en_init_rx_desc(priv, ring, i); in mlx4_en_activate_rx_rings()
456 err = mlx4_en_init_allocator(priv, ring); in mlx4_en_activate_rx_rings()
459 if (ring->stride <= TXBB_SIZE) in mlx4_en_activate_rx_rings()
460 ring->buf -= TXBB_SIZE; in mlx4_en_activate_rx_rings()
470 ring = priv->rx_ring[ring_ind]; in mlx4_en_activate_rx_rings()
472 ring->size_mask = ring->actual_size - 1; in mlx4_en_activate_rx_rings()
473 mlx4_en_update_rx_prod_db(ring); in mlx4_en_activate_rx_rings()
499 int ring; in mlx4_en_recover_from_oom() local
504 for (ring = 0; ring < priv->rx_ring_num; ring++) { in mlx4_en_recover_from_oom()
505 if (mlx4_en_is_ring_empty(priv->rx_ring[ring])) in mlx4_en_recover_from_oom()
506 napi_reschedule(&priv->rx_cq[ring]->napi); in mlx4_en_recover_from_oom()
515 struct mlx4_en_rx_ring *ring = *pring; in mlx4_en_destroy_rx_ring() local
517 mlx4_en_unmap_buffer(&ring->wqres.buf); in mlx4_en_destroy_rx_ring()
518 mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); in mlx4_en_destroy_rx_ring()
519 vfree(ring->rx_info); in mlx4_en_destroy_rx_ring()
520 ring->rx_info = NULL; in mlx4_en_destroy_rx_ring()
521 kfree(ring); in mlx4_en_destroy_rx_ring()
529 struct mlx4_en_rx_ring *ring) in mlx4_en_deactivate_rx_ring() argument
531 mlx4_en_free_rx_buf(priv, ring); in mlx4_en_deactivate_rx_ring()
532 if (ring->stride <= TXBB_SIZE) in mlx4_en_deactivate_rx_ring()
533 ring->buf -= TXBB_SIZE; in mlx4_en_deactivate_rx_ring()
534 mlx4_en_destroy_allocator(priv, ring); in mlx4_en_deactivate_rx_ring()
657 struct mlx4_en_rx_ring *ring) in mlx4_en_refill_rx_buffers() argument
659 int index = ring->prod & ring->size_mask; in mlx4_en_refill_rx_buffers()
661 while ((u32) (ring->prod - ring->cons) < ring->actual_size) { in mlx4_en_refill_rx_buffers()
662 if (mlx4_en_prepare_rx_desc(priv, ring, index, in mlx4_en_refill_rx_buffers()
665 ring->prod++; in mlx4_en_refill_rx_buffers()
666 index = ring->prod & ring->size_mask; in mlx4_en_refill_rx_buffers()
749 struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; in mlx4_en_process_rx_cq() local
771 index = cq->mcq.cons_index & ring->size_mask; in mlx4_en_process_rx_cq()
778 frags = ring->rx_info + (index << priv->log_rx_info); in mlx4_en_process_rx_cq()
779 rx_desc = ring->buf + (index << ring->log_stride); in mlx4_en_process_rx_cq()
838 length -= ring->fcs_del; in mlx4_en_process_rx_cq()
839 ring->bytes += length; in mlx4_en_process_rx_cq()
840 ring->packets++; in mlx4_en_process_rx_cq()
850 ring->csum_ok++; in mlx4_en_process_rx_cq()
853 ring->csum_none++; in mlx4_en_process_rx_cq()
860 ring->csum_complete++; in mlx4_en_process_rx_cq()
863 ring->csum_none++; in mlx4_en_process_rx_cq()
868 ring->csum_none++; in mlx4_en_process_rx_cq()
895 ring->csum_none++; in mlx4_en_process_rx_cq()
896 ring->csum_complete--; in mlx4_en_process_rx_cq()
929 skb_record_rx_queue(gro_skb, cq->ring); in mlx4_en_process_rx_cq()
932 if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) { in mlx4_en_process_rx_cq()
958 ring->csum_complete--; in mlx4_en_process_rx_cq()
959 ring->csum_none++; in mlx4_en_process_rx_cq()
965 skb_record_rx_queue(skb, cq->ring); in mlx4_en_process_rx_cq()
987 if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) { in mlx4_en_process_rx_cq()
1005 index = (cq->mcq.cons_index) & ring->size_mask; in mlx4_en_process_rx_cq()
1015 ring->cons = cq->mcq.cons_index; in mlx4_en_process_rx_cq()
1016 mlx4_en_refill_rx_buffers(priv, ring); in mlx4_en_process_rx_cq()
1017 mlx4_en_update_rx_prod_db(ring); in mlx4_en_process_rx_cq()
1123 struct mlx4_en_rx_ring *ring, in mlx4_en_config_rss_qp() argument
1143 mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, in mlx4_en_config_rss_qp()
1144 qpn, ring->cqn, -1, context); in mlx4_en_config_rss_qp()
1145 context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); in mlx4_en_config_rss_qp()
1151 ring->fcs_del = 0; in mlx4_en_config_rss_qp()
1153 ring->fcs_del = ETH_FCS_LEN; in mlx4_en_config_rss_qp()
1155 ring->fcs_del = 0; in mlx4_en_config_rss_qp()
1157 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); in mlx4_en_config_rss_qp()
1162 mlx4_en_update_rx_prod_db(ring); in mlx4_en_config_rss_qp()