xsk 12831 drivers/net/ethernet/intel/i40e/i40e_main.c return i40e_xsk_umem_setup(vsi, xdp->xsk.umem, xsk 12832 drivers/net/ethernet/intel/i40e/i40e_main.c xdp->xsk.queue_id); xsk 10312 drivers/net/ethernet/intel/ixgbe/ixgbe_main.c return ixgbe_xsk_umem_setup(adapter, xdp->xsk.umem, xsk 10313 drivers/net/ethernet/intel/ixgbe/ixgbe_main.c xdp->xsk.queue_id); xsk 269 drivers/net/ethernet/mellanox/mlx5/core/en.h struct mlx5e_xsk *xsk; xsk 435 drivers/net/ethernet/mellanox/mlx5/core/en.h } xsk; xsk 877 drivers/net/ethernet/mellanox/mlx5/core/en.h struct mlx5e_xsk xsk; xsk 1012 drivers/net/ethernet/mellanox/mlx5/core/en.h struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk, xsk 1183 drivers/net/ethernet/mellanox/mlx5/core/en.h struct mlx5e_xsk *xsk, xsk 7 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 9 drivers/net/ethernet/mellanox/mlx5/core/en/params.c return params->xdp_prog || xsk; xsk 13 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 17 drivers/net/ethernet/mellanox/mlx5/core/en/params.c if (mlx5e_rx_is_xdp(params, xsk)) { xsk 19 drivers/net/ethernet/mellanox/mlx5/core/en/params.c if (xsk) xsk 20 drivers/net/ethernet/mellanox/mlx5/core/en/params.c headroom += xsk->headroom; xsk 29 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 32 drivers/net/ethernet/mellanox/mlx5/core/en/params.c u16 linear_rq_headroom = mlx5e_get_linear_rq_headroom(params, xsk); xsk 38 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 40 drivers/net/ethernet/mellanox/mlx5/core/en/params.c u32 frag_sz = mlx5e_rx_get_min_frag_sz(params, xsk); xsk 43 drivers/net/ethernet/mellanox/mlx5/core/en/params.c if (!xsk) xsk 55 drivers/net/ethernet/mellanox/mlx5/core/en/params.c if (mlx5e_rx_is_xdp(params, xsk)) xsk 61 drivers/net/ethernet/mellanox/mlx5/core/en/params.c if (xsk) xsk 62 drivers/net/ethernet/mellanox/mlx5/core/en/params.c frag_sz = max_t(u32, frag_sz, xsk->chunk_size); xsk 68 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 70 drivers/net/ethernet/mellanox/mlx5/core/en/params.c u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk); xsk 76 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 81 drivers/net/ethernet/mellanox/mlx5/core/en/params.c u32 linear_frag_sz = max(mlx5e_rx_get_linear_frag_sz(params, xsk), xsk 91 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 93 drivers/net/ethernet/mellanox/mlx5/core/en/params.c u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk); xsk 97 drivers/net/ethernet/mellanox/mlx5/core/en/params.c if (!mlx5e_rx_is_linear_skb(params, xsk)) xsk 114 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 116 drivers/net/ethernet/mellanox/mlx5/core/en/params.c u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params, xsk); xsk 128 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 130 drivers/net/ethernet/mellanox/mlx5/core/en/params.c if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) xsk 131 drivers/net/ethernet/mellanox/mlx5/core/en/params.c return order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk)); xsk 138 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 141 drivers/net/ethernet/mellanox/mlx5/core/en/params.c mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); xsk 146 drivers/net/ethernet/mellanox/mlx5/core/en/params.c struct mlx5e_xsk_param *xsk) xsk 149 drivers/net/ethernet/mellanox/mlx5/core/en/params.c mlx5e_rx_is_linear_skb(params, xsk) : xsk 150 drivers/net/ethernet/mellanox/mlx5/core/en/params.c mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk); xsk 152 drivers/net/ethernet/mellanox/mlx5/core/en/params.c return is_linear_skb ? mlx5e_get_linear_rq_headroom(params, xsk) : 0; xsk 78 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 80 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 82 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 84 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 86 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 89 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 91 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 94 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 97 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 100 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk); xsk 106 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk, xsk 112 drivers/net/ethernet/mellanox/mlx5/core/en/params.h struct mlx5e_xsk_param *xsk, xsk 38 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) xsk 40 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c int hr = mlx5e_get_linear_rq_headroom(params, xsk); xsk 122 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c void *va, u16 *rx_headroom, u32 *len, bool xsk) xsk 137 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c if (xsk) xsk 138 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c xdp.handle = di->xsk.handle; xsk 142 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c if (xsk) { xsk 164 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c if (!xsk) xsk 64 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); xsk 66 drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h void *va, u16 *rx_headroom, u32 *len, bool xsk); xsk 27 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c dma_info->xsk.handle = xsk_umem_adjust_offset(umem, handle, xsk 29 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c dma_info->xsk.data = xdp_umem_get_data(umem, dma_info->xsk.handle); xsk 58 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c mlx5e_xsk_recycle_frame(rq, dma_info->xsk.handle); xsk 113 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c va = di->xsk.data; xsk 169 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c va = di->xsk.data; xsk 14 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c struct mlx5e_xsk_param *xsk, xsk 18 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c if (xsk->chunk_size > PAGE_SIZE || xsk 19 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE) xsk 23 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c if (mlx5e_rx_get_min_frag_sz(params, xsk) > xsk->chunk_size) xsk 31 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c return mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk); xsk 33 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c return mlx5e_rx_is_linear_skb(params, xsk); xsk 51 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c struct mlx5e_xsk_param *xsk, xsk 56 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c mlx5e_build_rq_param(priv, params, xsk, &cparam->rq); xsk 59 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c mlx5e_build_rx_cq_param(priv, params, xsk, &cparam->rx_cq); xsk 65 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c struct mlx5e_xsk_param *xsk, struct xdp_umem *umem, xsk 72 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev)) xsk 79 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c mlx5e_build_xsk_cparam(priv, params, xsk, cparam); xsk 85 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c err = mlx5e_open_rq(c, params, &cparam->rq, xsk, umem, &c->xskrq); xsk 207 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c if (!priv->xsk.refcnt) xsk 238 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c if (!priv->xsk.refcnt) xsk 12 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h struct mlx5e_xsk_param *xsk, xsk 15 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h struct mlx5e_xsk_param *xsk, struct xdp_umem *umem, xsk 49 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c static int mlx5e_xsk_get_umems(struct mlx5e_xsk *xsk) xsk 51 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c if (!xsk->umems) { xsk 52 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c xsk->umems = kcalloc(MLX5E_MAX_NUM_CHANNELS, xsk 53 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c sizeof(*xsk->umems), GFP_KERNEL); xsk 54 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c if (unlikely(!xsk->umems)) xsk 58 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c xsk->refcnt++; xsk 59 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c xsk->ever_used = true; xsk 64 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c static void mlx5e_xsk_put_umems(struct mlx5e_xsk *xsk) xsk 66 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c if (!--xsk->refcnt) { xsk 67 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c kfree(xsk->umems); xsk 68 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c xsk->umems = NULL; xsk 72 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c static int mlx5e_xsk_add_umem(struct mlx5e_xsk *xsk, struct xdp_umem *umem, u16 ix) xsk 76 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c err = mlx5e_xsk_get_umems(xsk); xsk 80 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c xsk->umems[ix] = umem; xsk 84 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c static void mlx5e_xsk_remove_umem(struct mlx5e_xsk *xsk, u16 ix) xsk 86 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c xsk->umems[ix] = NULL; xsk 88 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c mlx5e_xsk_put_umems(xsk); xsk 96 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c void mlx5e_build_xsk_param(struct xdp_umem *umem, struct mlx5e_xsk_param *xsk) xsk 98 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c xsk->headroom = umem->headroom; xsk 99 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c xsk->chunk_size = umem->chunk_size_nohr + umem->headroom; xsk 106 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c struct mlx5e_xsk_param xsk; xsk 110 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c if (unlikely(mlx5e_xsk_get_umem(&priv->channels.params, &priv->xsk, ix))) xsk 120 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c err = mlx5e_xsk_add_umem(&priv->xsk, umem, ix); xsk 124 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c mlx5e_build_xsk_param(umem, &xsk); xsk 140 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c err = mlx5e_open_xsk(priv, params, &xsk, umem, c); xsk 161 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c mlx5e_xsk_remove_umem(&priv->xsk, ix); xsk 172 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) { xsk 183 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c &priv->xsk, ix); xsk 202 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c mlx5e_xsk_remove_umem(&priv->xsk, ix); xsk 256 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c u16 mlx5e_xsk_first_unused_channel(struct mlx5e_params *params, struct mlx5e_xsk *xsk) xsk 258 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c u16 res = xsk->refcnt ? params->num_channels : 0; xsk 261 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c if (mlx5e_xsk_get_umem(params, xsk, res - 1)) xsk 10 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h struct mlx5e_xsk *xsk, u16 ix) xsk 12 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h if (!xsk || !xsk->umems) xsk 18 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h return xsk->umems[ix]; xsk 22 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h void mlx5e_build_xsk_param(struct xdp_umem *umem, struct mlx5e_xsk_param *xsk); xsk 29 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h u16 mlx5e_xsk_first_unused_channel(struct mlx5e_params *params, struct mlx5e_xsk *xsk); xsk 396 drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c if (priv->xsk.refcnt) { xsk 436 drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c if (priv->xsk.refcnt) { xsk 375 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_xsk_param *xsk, xsk 419 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (xsk) xsk 426 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); xsk 427 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->buff.umem_headroom = xsk ? xsk->headroom : 0; xsk 441 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (xsk) xsk 443 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); xsk 446 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5e_mpwqe_get_log_rq_size(params, xsk); xsk 465 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->mpwqe.skb_from_cqe_mpwrq = xsk ? xsk 471 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); xsk 473 drivers/net/ethernet/mellanox/mlx5/core/en_main.c BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); xsk 494 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (xsk) xsk 526 drivers/net/ethernet/mellanox/mlx5/core/en_main.c rq->wqe.skb_from_cqe = xsk ? xsk 534 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (xsk) { xsk 883 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk, xsk 888 drivers/net/ethernet/mellanox/mlx5/core/en_main.c err = mlx5e_alloc_rq(c, params, xsk, umem, param, rq); xsk 1970 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_xsk_param xsk; xsk 2009 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5e_build_xsk_param(umem, &xsk); xsk 2010 drivers/net/ethernet/mellanox/mlx5/core/en_main.c err = mlx5e_open_xsk(priv, params, &xsk, umem, c); xsk 2074 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_xsk_param *xsk, xsk 2087 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (mlx5e_rx_is_linear_skb(params, xsk)) { xsk 2090 drivers/net/ethernet/mellanox/mlx5/core/en_main.c frag_stride = mlx5e_rx_get_linear_frag_sz(params, xsk); xsk 2150 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_xsk_param *xsk, xsk 2161 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk) - xsk 2164 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk) - xsk 2166 drivers/net/ethernet/mellanox/mlx5/core/en_main.c MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk)); xsk 2170 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5e_build_rq_frags_info(mdev, params, xsk, ¶m->frags_info); xsk 2240 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_xsk_param *xsk, xsk 2249 drivers/net/ethernet/mellanox/mlx5/core/en_main.c log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) + xsk 2250 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); xsk 2365 drivers/net/ethernet/mellanox/mlx5/core/en_main.c umem = mlx5e_xsk_get_umem(&chs->params, chs->params.xsk, i); xsk 3669 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (enable && priv->xsk.refcnt) { xsk 3671 drivers/net/ethernet/mellanox/mlx5/core/en_main.c priv->xsk.refcnt); xsk 3889 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct xdp_umem *umem = mlx5e_xsk_get_umem(&chs->params, chs->params.xsk, ix); xsk 3890 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_xsk_param xsk; xsk 3895 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5e_build_xsk_param(umem, &xsk); xsk 3897 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev)) { xsk 3898 drivers/net/ethernet/mellanox/mlx5/core/en_main.c u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk); xsk 3905 drivers/net/ethernet/mellanox/mlx5/core/en_main.c max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr); xsk 3906 drivers/net/ethernet/mellanox/mlx5/core/en_main.c max_mtu_page = mlx5e_xdp_max_mtu(new_params, &xsk); xsk 3945 drivers/net/ethernet/mellanox/mlx5/core/en_main.c if (priv->xsk.refcnt && xsk 3960 drivers/net/ethernet/mellanox/mlx5/core/en_main.c is_linear |= priv->xsk.refcnt; xsk 4531 drivers/net/ethernet/mellanox/mlx5/core/en_main.c return mlx5e_xsk_setup_umem(dev, xdp->xsk.umem, xsk 4532 drivers/net/ethernet/mellanox/mlx5/core/en_main.c xdp->xsk.queue_id); xsk 4795 drivers/net/ethernet/mellanox/mlx5/core/en_main.c struct mlx5e_xsk *xsk, xsk 4856 drivers/net/ethernet/mellanox/mlx5/core/en_main.c params->xsk = xsk; xsk 5041 drivers/net/ethernet/mellanox/mlx5/core/en_main.c mlx5e_build_nic_params(mdev, &priv->xsk, rss, &priv->channels.params, xsk 1574 drivers/net/ethernet/mellanox/mlx5/core/en_stats.c (NUM_XSKRQ_STATS * max_nch * priv->xsk.ever_used) + xsk 1575 drivers/net/ethernet/mellanox/mlx5/core/en_stats.c (NUM_XSKSQ_STATS * max_nch * priv->xsk.ever_used); xsk 1581 drivers/net/ethernet/mellanox/mlx5/core/en_stats.c bool is_xsk = priv->xsk.ever_used; xsk 1624 drivers/net/ethernet/mellanox/mlx5/core/en_stats.c bool is_xsk = priv->xsk.ever_used; xsk 903 include/linux/netdevice.h } xsk; xsk 136 net/xdp/xdp_umem.c bpf.xsk.umem = umem; xsk 137 net/xdp/xdp_umem.c bpf.xsk.queue_id = queue_id; xsk 166 net/xdp/xdp_umem.c bpf.xsk.umem = NULL; xsk 167 net/xdp/xdp_umem.c bpf.xsk.queue_id = umem->queue_id; xsk 91 samples/bpf/xdpsock_user.c struct xsk_socket *xsk; xsk 203 samples/bpf/xdpsock_user.c xsk_socket__delete(xsks[0]->xsk); xsk 313 samples/bpf/xdpsock_user.c struct xsk_socket_info *xsk; xsk 318 samples/bpf/xdpsock_user.c xsk = calloc(1, sizeof(*xsk)); xsk 319 samples/bpf/xdpsock_user.c if (!xsk) xsk 322 samples/bpf/xdpsock_user.c xsk->umem = umem; xsk 328 samples/bpf/xdpsock_user.c ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem, xsk 329 samples/bpf/xdpsock_user.c &xsk->rx, &xsk->tx, &cfg); xsk 337 samples/bpf/xdpsock_user.c ret = xsk_ring_prod__reserve(&xsk->umem->fq, xsk 343 samples/bpf/xdpsock_user.c *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx++) = xsk 345 samples/bpf/xdpsock_user.c xsk_ring_prod__submit(&xsk->umem->fq, xsk 348 samples/bpf/xdpsock_user.c return xsk; xsk 476 samples/bpf/xdpsock_user.c static void kick_tx(struct xsk_socket_info *xsk) xsk 480 samples/bpf/xdpsock_user.c ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); xsk 486 samples/bpf/xdpsock_user.c static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, xsk 489 samples/bpf/xdpsock_user.c struct xsk_umem_info *umem = xsk->umem; xsk 494 samples/bpf/xdpsock_user.c if (!xsk->outstanding_tx) xsk 497 samples/bpf/xdpsock_user.c if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) xsk 498 samples/bpf/xdpsock_user.c kick_tx(xsk); xsk 500 samples/bpf/xdpsock_user.c ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE : xsk 501 samples/bpf/xdpsock_user.c xsk->outstanding_tx; xsk 522 samples/bpf/xdpsock_user.c xsk_ring_prod__submit(&xsk->umem->fq, rcvd); xsk 523 samples/bpf/xdpsock_user.c xsk_ring_cons__release(&xsk->umem->cq, rcvd); xsk 524 samples/bpf/xdpsock_user.c xsk->outstanding_tx -= rcvd; xsk 525 samples/bpf/xdpsock_user.c xsk->tx_npkts += rcvd; xsk 529 samples/bpf/xdpsock_user.c static inline void complete_tx_only(struct xsk_socket_info *xsk) xsk 534 samples/bpf/xdpsock_user.c if (!xsk->outstanding_tx) xsk 537 samples/bpf/xdpsock_user.c if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) xsk 538 samples/bpf/xdpsock_user.c kick_tx(xsk); xsk 540 samples/bpf/xdpsock_user.c rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx); xsk 542 samples/bpf/xdpsock_user.c xsk_ring_cons__release(&xsk->umem->cq, rcvd); xsk 543 samples/bpf/xdpsock_user.c xsk->outstanding_tx -= rcvd; xsk 544 samples/bpf/xdpsock_user.c xsk->tx_npkts += rcvd; xsk 548 samples/bpf/xdpsock_user.c static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds) xsk 554 samples/bpf/xdpsock_user.c rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); xsk 556 samples/bpf/xdpsock_user.c if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) xsk 561 samples/bpf/xdpsock_user.c ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); xsk 565 samples/bpf/xdpsock_user.c if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) xsk 567 samples/bpf/xdpsock_user.c ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); xsk 571 samples/bpf/xdpsock_user.c u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; xsk 572 samples/bpf/xdpsock_user.c u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; xsk 576 samples/bpf/xdpsock_user.c char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); xsk 579 samples/bpf/xdpsock_user.c *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig; xsk 582 samples/bpf/xdpsock_user.c xsk_ring_prod__submit(&xsk->umem->fq, rcvd); xsk 583 samples/bpf/xdpsock_user.c xsk_ring_cons__release(&xsk->rx, rcvd); xsk 584 samples/bpf/xdpsock_user.c xsk->rx_npkts += rcvd; xsk 595 samples/bpf/xdpsock_user.c fds[i].fd = xsk_socket__fd(xsks[i]->xsk); xsk 611 samples/bpf/xdpsock_user.c static void tx_only(struct xsk_socket_info *xsk, u32 frame_nb) xsk 615 samples/bpf/xdpsock_user.c if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) == BATCH_SIZE) { xsk 619 samples/bpf/xdpsock_user.c xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr = xsk 621 samples/bpf/xdpsock_user.c xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len = xsk 625 samples/bpf/xdpsock_user.c xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE); xsk 626 samples/bpf/xdpsock_user.c xsk->outstanding_tx += BATCH_SIZE; xsk 631 samples/bpf/xdpsock_user.c complete_tx_only(xsk); xsk 642 samples/bpf/xdpsock_user.c fds[0].fd = xsk_socket__fd(xsks[i]->xsk); xsk 661 samples/bpf/xdpsock_user.c static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds) xsk 667 samples/bpf/xdpsock_user.c complete_tx_l2fwd(xsk, fds); xsk 669 samples/bpf/xdpsock_user.c rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); xsk 671 samples/bpf/xdpsock_user.c if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) xsk 676 samples/bpf/xdpsock_user.c ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); xsk 680 samples/bpf/xdpsock_user.c if (xsk_ring_prod__needs_wakeup(&xsk->tx)) xsk 681 samples/bpf/xdpsock_user.c kick_tx(xsk); xsk 682 samples/bpf/xdpsock_user.c ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); xsk 686 samples/bpf/xdpsock_user.c u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; xsk 687 samples/bpf/xdpsock_user.c u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; xsk 691 samples/bpf/xdpsock_user.c char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); xsk 696 samples/bpf/xdpsock_user.c xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig; xsk 697 samples/bpf/xdpsock_user.c xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len; xsk 700 samples/bpf/xdpsock_user.c xsk_ring_prod__submit(&xsk->tx, rcvd); xsk 701 samples/bpf/xdpsock_user.c xsk_ring_cons__release(&xsk->rx, rcvd); xsk 703 samples/bpf/xdpsock_user.c xsk->rx_npkts += rcvd; xsk 704 samples/bpf/xdpsock_user.c xsk->outstanding_tx += rcvd; xsk 715 samples/bpf/xdpsock_user.c fds[i].fd = xsk_socket__fd(xsks[i]->xsk); xsk 96 tools/lib/bpf/xsk.c int xsk_socket__fd(const struct xsk_socket *xsk) xsk 98 tools/lib/bpf/xsk.c return xsk ? xsk->fd : -EINVAL; xsk 333 tools/lib/bpf/xsk.c static int xsk_load_xdp_prog(struct xsk_socket *xsk) xsk 359 tools/lib/bpf/xsk.c BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd), xsk 366 tools/lib/bpf/xsk.c BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd), xsk 383 tools/lib/bpf/xsk.c err = bpf_set_link_xdp_fd(xsk->ifindex, prog_fd, xsk->config.xdp_flags); xsk 389 tools/lib/bpf/xsk.c xsk->prog_fd = prog_fd; xsk 393 tools/lib/bpf/xsk.c static int xsk_get_max_queues(struct xsk_socket *xsk) xsk 404 tools/lib/bpf/xsk.c memcpy(ifr.ifr_name, xsk->ifname, IFNAMSIZ - 1); xsk 430 tools/lib/bpf/xsk.c static int xsk_create_bpf_maps(struct xsk_socket *xsk) xsk 435 tools/lib/bpf/xsk.c max_queues = xsk_get_max_queues(xsk); xsk 444 tools/lib/bpf/xsk.c xsk->xsks_map_fd = fd; xsk 449 tools/lib/bpf/xsk.c static void xsk_delete_bpf_maps(struct xsk_socket *xsk) xsk 451 tools/lib/bpf/xsk.c bpf_map_delete_elem(xsk->xsks_map_fd, &xsk->queue_id); xsk 452 tools/lib/bpf/xsk.c close(xsk->xsks_map_fd); xsk 455 tools/lib/bpf/xsk.c static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) xsk 463 tools/lib/bpf/xsk.c err = bpf_obj_get_info_by_fd(xsk->prog_fd, &prog_info, &prog_len); xsk 477 tools/lib/bpf/xsk.c err = bpf_obj_get_info_by_fd(xsk->prog_fd, &prog_info, &prog_len); xsk 481 tools/lib/bpf/xsk.c xsk->xsks_map_fd = -1; xsk 495 tools/lib/bpf/xsk.c xsk->xsks_map_fd = fd; xsk 503 tools/lib/bpf/xsk.c if (xsk->xsks_map_fd == -1) xsk 511 tools/lib/bpf/xsk.c static int xsk_set_bpf_maps(struct xsk_socket *xsk) xsk 513 tools/lib/bpf/xsk.c return bpf_map_update_elem(xsk->xsks_map_fd, &xsk->queue_id, xsk 514 tools/lib/bpf/xsk.c &xsk->fd, 0); xsk 517 tools/lib/bpf/xsk.c static int xsk_setup_xdp_prog(struct xsk_socket *xsk) xsk 522 tools/lib/bpf/xsk.c err = bpf_get_link_xdp_id(xsk->ifindex, &prog_id, xsk 523 tools/lib/bpf/xsk.c xsk->config.xdp_flags); xsk 528 tools/lib/bpf/xsk.c err = xsk_create_bpf_maps(xsk); xsk 532 tools/lib/bpf/xsk.c err = xsk_load_xdp_prog(xsk); xsk 534 tools/lib/bpf/xsk.c xsk_delete_bpf_maps(xsk); xsk 538 tools/lib/bpf/xsk.c xsk->prog_fd = bpf_prog_get_fd_by_id(prog_id); xsk 539 tools/lib/bpf/xsk.c if (xsk->prog_fd < 0) xsk 541 tools/lib/bpf/xsk.c err = xsk_lookup_bpf_maps(xsk); xsk 543 tools/lib/bpf/xsk.c close(xsk->prog_fd); xsk 548 tools/lib/bpf/xsk.c err = xsk_set_bpf_maps(xsk); xsk 550 tools/lib/bpf/xsk.c xsk_delete_bpf_maps(xsk); xsk 551 tools/lib/bpf/xsk.c close(xsk->prog_fd); xsk 566 tools/lib/bpf/xsk.c struct xsk_socket *xsk; xsk 577 tools/lib/bpf/xsk.c xsk = calloc(1, sizeof(*xsk)); xsk 578 tools/lib/bpf/xsk.c if (!xsk) xsk 582 tools/lib/bpf/xsk.c xsk->fd = socket(AF_XDP, SOCK_RAW, 0); xsk 583 tools/lib/bpf/xsk.c if (xsk->fd < 0) { xsk 588 tools/lib/bpf/xsk.c xsk->fd = umem->fd; xsk 591 tools/lib/bpf/xsk.c xsk->outstanding_tx = 0; xsk 592 tools/lib/bpf/xsk.c xsk->queue_id = queue_id; xsk 593 tools/lib/bpf/xsk.c xsk->umem = umem; xsk 594 tools/lib/bpf/xsk.c xsk->ifindex = if_nametoindex(ifname); xsk 595 tools/lib/bpf/xsk.c if (!xsk->ifindex) { xsk 599 tools/lib/bpf/xsk.c memcpy(xsk->ifname, ifname, IFNAMSIZ - 1); xsk 600 tools/lib/bpf/xsk.c xsk->ifname[IFNAMSIZ - 1] = '\0'; xsk 602 tools/lib/bpf/xsk.c err = xsk_set_xdp_socket_config(&xsk->config, usr_config); xsk 607 tools/lib/bpf/xsk.c err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING, xsk 608 tools/lib/bpf/xsk.c &xsk->config.rx_size, xsk 609 tools/lib/bpf/xsk.c sizeof(xsk->config.rx_size)); xsk 616 tools/lib/bpf/xsk.c err = setsockopt(xsk->fd, SOL_XDP, XDP_TX_RING, xsk 617 tools/lib/bpf/xsk.c &xsk->config.tx_size, xsk 618 tools/lib/bpf/xsk.c sizeof(xsk->config.tx_size)); xsk 625 tools/lib/bpf/xsk.c err = xsk_get_mmap_offsets(xsk->fd, &off); xsk 633 tools/lib/bpf/xsk.c xsk->config.rx_size * sizeof(struct xdp_desc), xsk 635 tools/lib/bpf/xsk.c xsk->fd, XDP_PGOFF_RX_RING); xsk 641 tools/lib/bpf/xsk.c rx->mask = xsk->config.rx_size - 1; xsk 642 tools/lib/bpf/xsk.c rx->size = xsk->config.rx_size; xsk 648 tools/lib/bpf/xsk.c xsk->rx = rx; xsk 652 tools/lib/bpf/xsk.c xsk->config.tx_size * sizeof(struct xdp_desc), xsk 654 tools/lib/bpf/xsk.c xsk->fd, XDP_PGOFF_TX_RING); xsk 660 tools/lib/bpf/xsk.c tx->mask = xsk->config.tx_size - 1; xsk 661 tools/lib/bpf/xsk.c tx->size = xsk->config.tx_size; xsk 666 tools/lib/bpf/xsk.c tx->cached_cons = xsk->config.tx_size; xsk 668 tools/lib/bpf/xsk.c xsk->tx = tx; xsk 671 tools/lib/bpf/xsk.c sxdp.sxdp_ifindex = xsk->ifindex; xsk 672 tools/lib/bpf/xsk.c sxdp.sxdp_queue_id = xsk->queue_id; xsk 673 tools/lib/bpf/xsk.c sxdp.sxdp_flags = xsk->config.bind_flags; xsk 675 tools/lib/bpf/xsk.c err = bind(xsk->fd, (struct sockaddr *)&sxdp, sizeof(sxdp)); xsk 681 tools/lib/bpf/xsk.c xsk->prog_fd = -1; xsk 683 tools/lib/bpf/xsk.c if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) { xsk 684 tools/lib/bpf/xsk.c err = xsk_setup_xdp_prog(xsk); xsk 689 tools/lib/bpf/xsk.c *xsk_ptr = xsk; xsk 695 tools/lib/bpf/xsk.c xsk->config.tx_size * sizeof(struct xdp_desc)); xsk 699 tools/lib/bpf/xsk.c xsk->config.rx_size * sizeof(struct xdp_desc)); xsk 702 tools/lib/bpf/xsk.c close(xsk->fd); xsk 704 tools/lib/bpf/xsk.c free(xsk); xsk 733 tools/lib/bpf/xsk.c void xsk_socket__delete(struct xsk_socket *xsk) xsk 739 tools/lib/bpf/xsk.c if (!xsk) xsk 742 tools/lib/bpf/xsk.c if (xsk->prog_fd != -1) { xsk 743 tools/lib/bpf/xsk.c xsk_delete_bpf_maps(xsk); xsk 744 tools/lib/bpf/xsk.c close(xsk->prog_fd); xsk 747 tools/lib/bpf/xsk.c err = xsk_get_mmap_offsets(xsk->fd, &off); xsk 749 tools/lib/bpf/xsk.c if (xsk->rx) { xsk 750 tools/lib/bpf/xsk.c munmap(xsk->rx->ring - off.rx.desc, xsk 751 tools/lib/bpf/xsk.c off.rx.desc + xsk->config.rx_size * desc_sz); xsk 753 tools/lib/bpf/xsk.c if (xsk->tx) { xsk 754 tools/lib/bpf/xsk.c munmap(xsk->tx->ring - off.tx.desc, xsk 755 tools/lib/bpf/xsk.c off.tx.desc + xsk->config.tx_size * desc_sz); xsk 760 tools/lib/bpf/xsk.c xsk->umem->refcount--; xsk 764 tools/lib/bpf/xsk.c if (xsk->fd != xsk->umem->fd) xsk 765 tools/lib/bpf/xsk.c close(xsk->fd); xsk 766 tools/lib/bpf/xsk.c free(xsk); xsk 187 tools/lib/bpf/xsk.h LIBBPF_API int xsk_socket__fd(const struct xsk_socket *xsk); xsk 231 tools/lib/bpf/xsk.h LIBBPF_API int xsk_socket__create(struct xsk_socket **xsk, xsk 240 tools/lib/bpf/xsk.h LIBBPF_API void xsk_socket__delete(struct xsk_socket *xsk);