root/drivers/net/ethernet/google/gve/gve_rx.c

DEFINITIONS

1. gve_rx_remove_from_block
2. gve_rx_free_ring
3. gve_setup_rx_buffer
4. gve_prefill_rx_pages
5. gve_rx_add_to_block
6. gve_rx_alloc_ring
7. gve_rx_alloc_rings
8. gve_rx_free_rings
9. gve_rx_write_doorbell
10. gve_rss_type
11. gve_rx_copy
12. gve_rx_add_frags
13. gve_rx_flip_buff
14. gve_rx
15. gve_rx_work_pending
16. gve_clean_rx_done
17. gve_rx_poll

   1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2 /* Google virtual Ethernet (gve) driver
   3  *
   4  * Copyright (C) 2015-2019 Google, Inc.
   5  */
   6 
   7 #include "gve.h"
   8 #include "gve_adminq.h"
   9 #include <linux/etherdevice.h>
  10 
  11 static void gve_rx_remove_from_block(struct gve_priv *priv, int queue_idx)
  12 {
  13         struct gve_notify_block *block =
  14                         &priv->ntfy_blocks[gve_rx_idx_to_ntfy(priv, queue_idx)];
  15 
  16         block->rx = NULL;
  17 }
  18 
  19 static void gve_rx_free_ring(struct gve_priv *priv, int idx)
  20 {
  21         struct gve_rx_ring *rx = &priv->rx[idx];
  22         struct device *dev = &priv->pdev->dev;
  23         size_t bytes;
  24         u32 slots;
  25 
  26         gve_rx_remove_from_block(priv, idx);
  27 
  28         bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
  29         dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
  30         rx->desc.desc_ring = NULL;
  31 
  32         dma_free_coherent(dev, sizeof(*rx->q_resources),
  33                           rx->q_resources, rx->q_resources_bus);
  34         rx->q_resources = NULL;
  35 
  36         gve_unassign_qpl(priv, rx->data.qpl->id);
  37         rx->data.qpl = NULL;
  38         kvfree(rx->data.page_info);
  39 
  40         slots = rx->mask + 1;
  41         bytes = sizeof(*rx->data.data_ring) * slots;
  42         dma_free_coherent(dev, bytes, rx->data.data_ring,
  43                           rx->data.data_bus);
  44         rx->data.data_ring = NULL;
  45         netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
  46 }
  47 
  48 static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
  49                                 struct gve_rx_data_slot *slot,
  50                                 dma_addr_t addr, struct page *page)
  51 {
  52         page_info->page = page;
  53         page_info->page_offset = 0;
  54         page_info->page_address = page_address(page);
  55         slot->qpl_offset = cpu_to_be64(addr);
  56 }
  57 
  58 static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
  59 {
  60         struct gve_priv *priv = rx->gve;
  61         u32 slots;
  62         int i;
  63 
  64         /* Allocate one page per Rx queue slot. Each page is split into two
  65          * packet buffers, when possible we "page flip" between the two.
  66          */
  67         slots = rx->mask + 1;
  68 
  69         rx->data.page_info = kvzalloc(slots *
  70                                       sizeof(*rx->data.page_info), GFP_KERNEL);
  71         if (!rx->data.page_info)
  72                 return -ENOMEM;
  73 
  74         rx->data.qpl = gve_assign_rx_qpl(priv);
  75 
  76         for (i = 0; i < slots; i++) {
  77                 struct page *page = rx->data.qpl->pages[i];
  78                 dma_addr_t addr = i * PAGE_SIZE;
  79 
  80                 gve_setup_rx_buffer(&rx->data.page_info[i],
  81                                     &rx->data.data_ring[i], addr, page);
  82         }
  83 
  84         return slots;
  85 }
  86 
  87 static void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx)
  88 {
  89         u32 ntfy_idx = gve_rx_idx_to_ntfy(priv, queue_idx);
  90         struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
  91         struct gve_rx_ring *rx = &priv->rx[queue_idx];
  92 
  93         block->rx = rx;
  94         rx->ntfy_id = ntfy_idx;
  95 }
  96 
  97 static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
  98 {
  99         struct gve_rx_ring *rx = &priv->rx[idx];
 100         struct device *hdev = &priv->pdev->dev;
 101         u32 slots, npages;
 102         int filled_pages;
 103         size_t bytes;
 104         int err;
 105 
 106         netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
 107         /* Make sure everything is zeroed to start with */
 108         memset(rx, 0, sizeof(*rx));
 109 
 110         rx->gve = priv;
 111         rx->q_num = idx;
 112 
 113         slots = priv->rx_pages_per_qpl;
 114         rx->mask = slots - 1;
 115 
 116         /* alloc rx data ring */
 117         bytes = sizeof(*rx->data.data_ring) * slots;
 118         rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
 119                                                 &rx->data.data_bus,
 120                                                 GFP_KERNEL);
 121         if (!rx->data.data_ring)
 122                 return -ENOMEM;
 123         filled_pages = gve_prefill_rx_pages(rx);
 124         if (filled_pages < 0) {
 125                 err = -ENOMEM;
 126                 goto abort_with_slots;
 127         }
 128         rx->fill_cnt = filled_pages;
 129         /* Ensure data ring slots (packet buffers) are visible. */
 130         dma_wmb();
 131 
 132         /* Alloc gve_queue_resources */
 133         rx->q_resources =
 134                 dma_alloc_coherent(hdev,
 135                                    sizeof(*rx->q_resources),
 136                                    &rx->q_resources_bus,
 137                                    GFP_KERNEL);
 138         if (!rx->q_resources) {
 139                 err = -ENOMEM;
 140                 goto abort_filled;
 141         }
 142         netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
 143                   (unsigned long)rx->data.data_bus);
 144 
 145         /* alloc rx desc ring */
 146         bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
 147         npages = bytes / PAGE_SIZE;
 148         if (npages * PAGE_SIZE != bytes) {
 149                 err = -EIO;
 150                 goto abort_with_q_resources;
 151         }
 152 
 153         rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
 154                                                 GFP_KERNEL);
 155         if (!rx->desc.desc_ring) {
 156                 err = -ENOMEM;
 157                 goto abort_with_q_resources;
 158         }
 159         rx->mask = slots - 1;
 160         rx->cnt = 0;
 161         rx->desc.seqno = 1;
 162         gve_rx_add_to_block(priv, idx);
 163 
 164         return 0;
 165 
 166 abort_with_q_resources:
 167         dma_free_coherent(hdev, sizeof(*rx->q_resources),
 168                           rx->q_resources, rx->q_resources_bus);
 169         rx->q_resources = NULL;
 170 abort_filled:
 171         kvfree(rx->data.page_info);
 172 abort_with_slots:
 173         bytes = sizeof(*rx->data.data_ring) * slots;
 174         dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
 175         rx->data.data_ring = NULL;
 176 
 177         return err;
 178 }
 179 
 180 int gve_rx_alloc_rings(struct gve_priv *priv)
 181 {
 182         int err = 0;
 183         int i;
 184 
 185         for (i = 0; i < priv->rx_cfg.num_queues; i++) {
 186                 err = gve_rx_alloc_ring(priv, i);
 187                 if (err) {
 188                         netif_err(priv, drv, priv->dev,
 189                                   "Failed to alloc rx ring=%d: err=%d\n",
 190                                   i, err);
 191                         break;
 192                 }
 193         }
 194         /* Unallocate if there was an error */
 195         if (err) {
 196                 int j;
 197 
 198                 for (j = 0; j < i; j++)
 199                         gve_rx_free_ring(priv, j);
 200         }
 201         return err;
 202 }
 203 
 204 void gve_rx_free_rings(struct gve_priv *priv)
 205 {
 206         int i;
 207 
 208         for (i = 0; i < priv->rx_cfg.num_queues; i++)
 209                 gve_rx_free_ring(priv, i);
 210 }
 211 
 212 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx)
 213 {
 214         u32 db_idx = be32_to_cpu(rx->q_resources->db_index);
 215 
 216         iowrite32be(rx->fill_cnt, &priv->db_bar2[db_idx]);
 217 }
 218 
 219 static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
 220 {
 221         if (likely(pkt_flags & (GVE_RXF_TCP | GVE_RXF_UDP)))
 222                 return PKT_HASH_TYPE_L4;
 223         if (pkt_flags & (GVE_RXF_IPV4 | GVE_RXF_IPV6))
 224                 return PKT_HASH_TYPE_L3;
 225         return PKT_HASH_TYPE_L2;
 226 }
 227 
 228 static struct sk_buff *gve_rx_copy(struct net_device *dev,
 229                                    struct napi_struct *napi,
 230                                    struct gve_rx_slot_page_info *page_info,
 231                                    u16 len)
 232 {
 233         struct sk_buff *skb = napi_alloc_skb(napi, len);
 234         void *va = page_info->page_address + GVE_RX_PAD +
 235                    page_info->page_offset;
 236 
 237         if (unlikely(!skb))
 238                 return NULL;
 239 
 240         __skb_put(skb, len);
 241 
 242         skb_copy_to_linear_data(skb, va, len);
 243 
 244         skb->protocol = eth_type_trans(skb, dev);
 245         return skb;
 246 }
 247 
 248 static struct sk_buff *gve_rx_add_frags(struct net_device *dev,
 249                                         struct napi_struct *napi,
 250                                         struct gve_rx_slot_page_info *page_info,
 251                                         u16 len)
 252 {
 253         struct sk_buff *skb = napi_get_frags(napi);
 254 
 255         if (unlikely(!skb))
 256                 return NULL;
 257 
 258         skb_add_rx_frag(skb, 0, page_info->page,
 259                         page_info->page_offset +
 260                         GVE_RX_PAD, len, PAGE_SIZE / 2);
 261 
 262         return skb;
 263 }
 264 
 265 static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info,
 266                              struct gve_rx_data_slot *data_ring)
 267 {
 268         u64 addr = be64_to_cpu(data_ring->qpl_offset);
 269 
 270         page_info->page_offset ^= PAGE_SIZE / 2;
 271         addr ^= PAGE_SIZE / 2;
 272         data_ring->qpl_offset = cpu_to_be64(addr);
 273 }
 274 
 275 static bool gve_rx(struct gve_rx_ring *rx, struct gve_rx_desc *rx_desc,
 276                    netdev_features_t feat, u32 idx)
 277 {
 278         struct gve_rx_slot_page_info *page_info;
 279         struct gve_priv *priv = rx->gve;
 280         struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
 281         struct net_device *dev = priv->dev;
 282         struct sk_buff *skb;
 283         int pagecount;
 284         u16 len;
 285 
 286         /* drop this packet */
 287         if (unlikely(rx_desc->flags_seq & GVE_RXF_ERR))
 288                 return true;
 289 
 290         len = be16_to_cpu(rx_desc->len) - GVE_RX_PAD;
 291         page_info = &rx->data.page_info[idx];
 292         dma_sync_single_for_cpu(&priv->pdev->dev, rx->data.qpl->page_buses[idx],
 293                                 PAGE_SIZE, DMA_FROM_DEVICE);
 294 
 295         /* gvnic can only receive into registered segments. If the buffer
 296          * can't be recycled, our only choice is to copy the data out of
 297          * it so that we can return it to the device.
 298          */
 299 
 300         if (PAGE_SIZE == 4096) {
 301                 if (len <= priv->rx_copybreak) {
 302                         /* Just copy small packets */
 303                         skb = gve_rx_copy(dev, napi, page_info, len);
 304                         goto have_skb;
 305                 }
 306                 if (unlikely(!gve_can_recycle_pages(dev))) {
 307                         skb = gve_rx_copy(dev, napi, page_info, len);
 308                         goto have_skb;
 309                 }
 310                 pagecount = page_count(page_info->page);
 311                 if (pagecount == 1) {
 312                         /* No part of this page is used by any SKBs; we attach
 313                          * the page fragment to a new SKB and pass it up the
 314                          * stack.
 315                          */
 316                         skb = gve_rx_add_frags(dev, napi, page_info, len);
 317                         if (!skb)
 318                                 return true;
 319                         /* Make sure the kernel stack can't release the page */
 320                         get_page(page_info->page);
 321                         /* "flip" to other packet buffer on this page */
 322                         gve_rx_flip_buff(page_info, &rx->data.data_ring[idx]);
 323                 } else if (pagecount >= 2) {
 324                         /* We have previously passed the other half of this
 325                          * page up the stack, but it has not yet been freed.
 326                          */
 327                         skb = gve_rx_copy(dev, napi, page_info, len);
 328                 } else {
 329                         WARN(pagecount < 1, "Pagecount should never be < 1");
 330                         return false;
 331                 }
 332         } else {
 333                 skb = gve_rx_copy(dev, napi, page_info, len);
 334         }
 335 
 336 have_skb:
 337         /* We didn't manage to allocate an skb but we haven't had any
 338          * reset worthy failures.
 339          */
 340         if (!skb)
 341                 return true;
 342 
 343         if (likely(feat & NETIF_F_RXCSUM)) {
 344                 /* NIC passes up the partial sum */
 345                 if (rx_desc->csum)
 346                         skb->ip_summed = CHECKSUM_COMPLETE;
 347                 else
 348                         skb->ip_summed = CHECKSUM_NONE;
 349                 skb->csum = csum_unfold(rx_desc->csum);
 350         }
 351 
 352         /* parse flags & pass relevant info up */
 353         if (likely(feat & NETIF_F_RXHASH) &&
 354             gve_needs_rss(rx_desc->flags_seq))
 355                 skb_set_hash(skb, be32_to_cpu(rx_desc->rss_hash),
 356                              gve_rss_type(rx_desc->flags_seq));
 357 
 358         if (skb_is_nonlinear(skb))
 359                 napi_gro_frags(napi);
 360         else
 361                 napi_gro_receive(napi, skb);
 362         return true;
 363 }
 364 
 365 static bool gve_rx_work_pending(struct gve_rx_ring *rx)
 366 {
 367         struct gve_rx_desc *desc;
 368         __be16 flags_seq;
 369         u32 next_idx;
 370 
 371         next_idx = rx->cnt & rx->mask;
 372         desc = rx->desc.desc_ring + next_idx;
 373 
 374         flags_seq = desc->flags_seq;
 375         /* Make sure we have synchronized the seq no with the device */
 376         smp_rmb();
 377 
 378         return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
 379 }
 380 
 381 bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
 382                        netdev_features_t feat)
 383 {
 384         struct gve_priv *priv = rx->gve;
 385         struct gve_rx_desc *desc;
 386         u32 cnt = rx->cnt;
 387         u32 idx = cnt & rx->mask;
 388         u32 work_done = 0;
 389         u64 bytes = 0;
 390 
 391         desc = rx->desc.desc_ring + idx;
 392         while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
 393                work_done < budget) {
 394                 netif_info(priv, rx_status, priv->dev,
 395                            "[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
 396                            rx->q_num, idx, desc, desc->flags_seq);
 397                 netif_info(priv, rx_status, priv->dev,
 398                            "[%d] seqno=%d rx->desc.seqno=%d\n",
 399                            rx->q_num, GVE_SEQNO(desc->flags_seq),
 400                            rx->desc.seqno);
 401                 bytes += be16_to_cpu(desc->len) - GVE_RX_PAD;
 402                 if (!gve_rx(rx, desc, feat, idx))
 403                         gve_schedule_reset(priv);
 404                 cnt++;
 405                 idx = cnt & rx->mask;
 406                 desc = rx->desc.desc_ring + idx;
 407                 rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
 408                 work_done++;
 409         }
 410 
 411         if (!work_done)
 412                 return false;
 413 
 414         u64_stats_update_begin(&rx->statss);
 415         rx->rpackets += work_done;
 416         rx->rbytes += bytes;
 417         u64_stats_update_end(&rx->statss);
 418         rx->cnt = cnt;
 419         rx->fill_cnt += work_done;
 420 
 421         gve_rx_write_doorbell(priv, rx);
 422         return gve_rx_work_pending(rx);
 423 }
 424 
 425 bool gve_rx_poll(struct gve_notify_block *block, int budget)
 426 {
 427         struct gve_rx_ring *rx = block->rx;
 428         netdev_features_t feat;
 429         bool repoll = false;
 430 
 431         feat = block->napi.dev->features;
 432 
 433         /* If budget is 0, do all the work */
 434         if (budget == 0)
 435                 budget = INT_MAX;
 436 
 437         if (budget > 0)
 438                 repoll |= gve_clean_rx_done(rx, budget, feat);
 439         else
 440                 repoll |= gve_rx_work_pending(rx);
 441         return repoll;
 442 }