root/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c

DEFINITIONS

1. mlx5e_xsk_pages_enough_umem
2. mlx5e_xsk_page_alloc_umem
3. mlx5e_xsk_recycle_frame
4. mlx5e_xsk_page_release
5. mlx5e_xsk_zca_free
6. mlx5e_xsk_construct_skb
7. mlx5e_xsk_skb_from_cqe_mpwrq_linear
8. mlx5e_xsk_skb_from_cqe_linear

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2019 Mellanox Technologies. */
   3 
   4 #include "rx.h"
   5 #include "en/xdp.h"
   6 #include <net/xdp_sock.h>
   7 
   8 /* RX data path */
   9 
  10 bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count)
  11 {
  12         /* Check in advance that we have enough frames, instead of allocating
  13          * one-by-one, failing and moving frames to the Reuse Ring.
  14          */
  15         return xsk_umem_has_addrs_rq(rq->umem, count);
  16 }
  17 
  18 int mlx5e_xsk_page_alloc_umem(struct mlx5e_rq *rq,
  19                               struct mlx5e_dma_info *dma_info)
  20 {
  21         struct xdp_umem *umem = rq->umem;
  22         u64 handle;
  23 
  24         if (!xsk_umem_peek_addr_rq(umem, &handle))
  25                 return -ENOMEM;
  26 
  27         dma_info->xsk.handle = xsk_umem_adjust_offset(umem, handle,
  28                                                       rq->buff.umem_headroom);
  29         dma_info->xsk.data = xdp_umem_get_data(umem, dma_info->xsk.handle);
  30 
  31         /* No need to add headroom to the DMA address. In striding RQ case, we
  32          * just provide pages for UMR, and headroom is counted at the setup
  33          * stage when creating a WQE. In non-striding RQ case, headroom is
  34          * accounted in mlx5e_alloc_rx_wqe.
  35          */
  36         dma_info->addr = xdp_umem_get_dma(umem, handle);
  37 
  38         xsk_umem_discard_addr_rq(umem);
  39 
  40         dma_sync_single_for_device(rq->pdev, dma_info->addr, PAGE_SIZE,
  41                                    DMA_BIDIRECTIONAL);
  42 
  43         return 0;
  44 }
  45 
  46 static inline void mlx5e_xsk_recycle_frame(struct mlx5e_rq *rq, u64 handle)
  47 {
  48         xsk_umem_fq_reuse(rq->umem, handle & rq->umem->chunk_mask);
  49 }
  50 
  51 /* XSKRQ uses pages from UMEM, they must not be released. They are returned to
  52  * the userspace if possible, and if not, this function is called to reuse them
  53  * in the driver.
  54  */
  55 void mlx5e_xsk_page_release(struct mlx5e_rq *rq,
  56                             struct mlx5e_dma_info *dma_info)
  57 {
  58         mlx5e_xsk_recycle_frame(rq, dma_info->xsk.handle);
  59 }
  60 
  61 /* Return a frame back to the hardware to fill in again. It is used by XDP when
  62  * the XDP program returns XDP_TX or XDP_REDIRECT not to an XSKMAP.
  63  */
  64 void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle)
  65 {
  66         struct mlx5e_rq *rq = container_of(zca, struct mlx5e_rq, zca);
  67 
  68         mlx5e_xsk_recycle_frame(rq, handle);
  69 }
  70 
  71 static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, void *data,
  72                                                u32 cqe_bcnt)
  73 {
  74         struct sk_buff *skb;
  75 
  76         skb = napi_alloc_skb(rq->cq.napi, cqe_bcnt);
  77         if (unlikely(!skb)) {
  78                 rq->stats->buff_alloc_err++;
  79                 return NULL;
  80         }
  81 
  82         skb_put_data(skb, data, cqe_bcnt);
  83 
  84         return skb;
  85 }
  86 
  87 struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
  88                                                     struct mlx5e_mpw_info *wi,
  89                                                     u16 cqe_bcnt,
  90                                                     u32 head_offset,
  91                                                     u32 page_idx)
  92 {
  93         struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
  94         u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
  95         u32 cqe_bcnt32 = cqe_bcnt;
  96         void *va, *data;
  97         u32 frag_size;
  98         bool consumed;
  99 
 100         /* Check packet size. Note LRO doesn't use linear SKB */
 101         if (unlikely(cqe_bcnt > rq->hw_mtu)) {
 102                 rq->stats->oversize_pkts_sw_drop++;
 103                 return NULL;
 104         }
 105 
 106         /* head_offset is not used in this function, because di->xsk.data and
 107          * di->addr point directly to the necessary place. Furthermore, in the
 108          * current implementation, UMR pages are mapped to XSK frames, so
 109          * head_offset should always be 0.
 110          */
 111         WARN_ON_ONCE(head_offset);
 112 
 113         va             = di->xsk.data;
 114         data           = va + rx_headroom;
 115         frag_size      = rq->buff.headroom + cqe_bcnt32;
 116 
 117         dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
 118         prefetch(data);
 119 
 120         rcu_read_lock();
 121         consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32, true);
 122         rcu_read_unlock();
 123 
 124         /* Possible flows:
 125          * - XDP_REDIRECT to XSKMAP:
 126          *   The page is owned by the userspace from now.
 127          * - XDP_TX and other XDP_REDIRECTs:
 128          *   The page was returned by ZCA and recycled.
 129          * - XDP_DROP:
 130          *   Recycle the page.
 131          * - XDP_PASS:
 132          *   Allocate an SKB, copy the data and recycle the page.
 133          *
 134          * Pages to be recycled go to the Reuse Ring on MPWQE deallocation. Its
 135          * size is the same as the Driver RX Ring's size, and pages for WQEs are
 136          * allocated first from the Reuse Ring, so it has enough space.
 137          */
 138 
 139         if (likely(consumed)) {
 140                 if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)))
 141                         __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
 142                 return NULL; /* page/packet was consumed by XDP */
 143         }
 144 
 145         /* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the
 146          * frame. On SKB allocation failure, NULL is returned.
 147          */
 148         return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt32);
 149 }
 150 
 151 struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
 152                                               struct mlx5_cqe64 *cqe,
 153                                               struct mlx5e_wqe_frag_info *wi,
 154                                               u32 cqe_bcnt)
 155 {
 156         struct mlx5e_dma_info *di = wi->di;
 157         u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
 158         void *va, *data;
 159         bool consumed;
 160         u32 frag_size;
 161 
 162         /* wi->offset is not used in this function, because di->xsk.data and
 163          * di->addr point directly to the necessary place. Furthermore, in the
 164          * current implementation, one page = one packet = one frame, so
 165          * wi->offset should always be 0.
 166          */
 167         WARN_ON_ONCE(wi->offset);
 168 
 169         va             = di->xsk.data;
 170         data           = va + rx_headroom;
 171         frag_size      = rq->buff.headroom + cqe_bcnt;
 172 
 173         dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
 174         prefetch(data);
 175 
 176         if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)) {
 177                 rq->stats->wqe_err++;
 178                 return NULL;
 179         }
 180 
 181         rcu_read_lock();
 182         consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt, true);
 183         rcu_read_unlock();
 184 
 185         if (likely(consumed))
 186                 return NULL; /* page/packet was consumed by XDP */
 187 
 188         /* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse
 189          * will be handled by mlx5e_put_rx_frag.
 190          * On SKB allocation failure, NULL is returned.
 191          */
 192         return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt);
 193 }