drivers/net/virtio/virtio_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <errno.h>
  10
  11 #include <rte_cycles.h>
  12 #include <rte_memory.h>
  13 #include <rte_branch_prediction.h>
  14 #include <rte_mempool.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_ether.h>
  18 #include <rte_ethdev_driver.h>
  19 #include <rte_prefetch.h>
  20 #include <rte_string_fns.h>
  21 #include <rte_errno.h>
  22 #include <rte_byteorder.h>
  23 #include <rte_net.h>
  24 #include <rte_ip.h>
  25 #include <rte_udp.h>
  26 #include <rte_tcp.h>
  27
  28 #include "virtio_logs.h"
  29 #include "virtio_ethdev.h"
  30 #include "virtio_pci.h"
  31 #include "virtqueue.h"
  32 #include "virtio_rxtx.h"
  33 #include "virtio_rxtx_simple.h"
  34 #include "virtio_ring.h"
  35
  36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
  37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
  38 #else
  39 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
  40 #endif
  41
  42 int
  43 virtio_dev_rx_queue_done(void *rxq, uint16_t offset)
  44 {
  45         struct virtnet_rx *rxvq = rxq;
  46         struct virtqueue *vq = rxvq->vq;
  47
  48         return VIRTQUEUE_NUSED(vq) >= offset;
  49 }
  50
  51 void
  52 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num)
  53 {
  54         vq->vq_free_cnt += num;
  55         vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1);
  56 }
  57
  58 void
  59 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
  60 {
  61         struct vring_desc *dp, *dp_tail;
  62         struct vq_desc_extra *dxp;
  63         uint16_t desc_idx_last = desc_idx;
  64
  65         dp  = &vq->vq_split.ring.desc[desc_idx];
  66         dxp = &vq->vq_descx[desc_idx];
  67         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
  68         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
  69                 while (dp->flags & VRING_DESC_F_NEXT) {
  70                         desc_idx_last = dp->next;
  71                         dp = &vq->vq_split.ring.desc[dp->next];
  72                 }
  73         }
  74         dxp->ndescs = 0;
  75
  76         /*
  77          * We must append the existing free chain, if any, to the end of
  78          * newly freed chain. If the virtqueue was completely used, then
  79          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
  80          */
  81         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
  82                 vq->vq_desc_head_idx = desc_idx;
  83         } else {
  84                 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx];
  85                 dp_tail->next = desc_idx;
  86         }
  87
  88         vq->vq_desc_tail_idx = desc_idx_last;
  89         dp->next = VQ_RING_DESC_CHAIN_END;
  90 }
  91
  92 static void
  93 vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id)
  94 {
  95         struct vq_desc_extra *dxp;
  96
  97         dxp = &vq->vq_descx[id];
  98         vq->vq_free_cnt += dxp->ndescs;
  99
 100         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END)
 101                 vq->vq_desc_head_idx = id;
 102         else
 103                 vq->vq_descx[vq->vq_desc_tail_idx].next = id;
 104
 105         vq->vq_desc_tail_idx = id;
 106         dxp->next = VQ_RING_DESC_CHAIN_END;
 107 }
 108
 109 void
 110 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
 111 {
 112         uint32_t s = mbuf->pkt_len;
 113         struct rte_ether_addr *ea;
 114
 115         stats->bytes += s;
 116
 117         if (s == 64) {
 118                 stats->size_bins[1]++;
 119         } else if (s > 64 && s < 1024) {
 120                 uint32_t bin;
 121
 122                 /* count zeros, and offset into correct bin */
 123                 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
 124                 stats->size_bins[bin]++;
 125         } else {
 126                 if (s < 64)
 127                         stats->size_bins[0]++;
 128                 else if (s < 1519)
 129                         stats->size_bins[6]++;
 130                 else
 131                         stats->size_bins[7]++;
 132         }
 133
 134         ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *);
 135         if (rte_is_multicast_ether_addr(ea)) {
 136                 if (rte_is_broadcast_ether_addr(ea))
 137                         stats->broadcast++;
 138                 else
 139                         stats->multicast++;
 140         }
 141 }
 142
 143 static inline void
 144 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m)
 145 {
 146         VIRTIO_DUMP_PACKET(m, m->data_len);
 147
 148         virtio_update_packet_stats(&rxvq->stats, m);
 149 }
 150
 151 static uint16_t
 152 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq,
 153                                   struct rte_mbuf **rx_pkts,
 154                                   uint32_t *len,
 155                                   uint16_t num)
 156 {
 157         struct rte_mbuf *cookie;
 158         uint16_t used_idx;
 159         uint16_t id;
 160         struct vring_packed_desc *desc;
 161         uint16_t i;
 162
 163         desc = vq->vq_packed.ring.desc;
 164
 165         for (i = 0; i < num; i++) {
 166                 used_idx = vq->vq_used_cons_idx;
 167                 /* desc_is_used has a load-acquire or rte_cio_rmb inside
 168                  * and wait for used desc in virtqueue.
 169                  */
 170                 if (!desc_is_used(&desc[used_idx], vq))
 171                         return i;
 172                 len[i] = desc[used_idx].len;
 173                 id = desc[used_idx].id;
 174                 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
 175                 if (unlikely(cookie == NULL)) {
 176                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
 177                                 vq->vq_used_cons_idx);
 178                         break;
 179                 }
 180                 rte_prefetch0(cookie);
 181                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 182                 rx_pkts[i] = cookie;
 183
 184                 vq->vq_free_cnt++;
 185                 vq->vq_used_cons_idx++;
 186                 if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 187                         vq->vq_used_cons_idx -= vq->vq_nentries;
 188                         vq->vq_packed.used_wrap_counter ^= 1;
 189                 }
 190         }
 191
 192         return i;
 193 }
 194
 195 static uint16_t
 196 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 197                            uint32_t *len, uint16_t num)
 198 {
 199         struct vring_used_elem *uep;
 200         struct rte_mbuf *cookie;
 201         uint16_t used_idx, desc_idx;
 202         uint16_t i;
 203
 204         /*  Caller does the check */
 205         for (i = 0; i < num ; i++) {
 206                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 207                 uep = &vq->vq_split.ring.used->ring[used_idx];
 208                 desc_idx = (uint16_t) uep->id;
 209                 len[i] = uep->len;
 210                 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
 211
 212                 if (unlikely(cookie == NULL)) {
 213                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
 214                                 vq->vq_used_cons_idx);
 215                         break;
 216                 }
 217
 218                 rte_prefetch0(cookie);
 219                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 220                 rx_pkts[i]  = cookie;
 221                 vq->vq_used_cons_idx++;
 222                 vq_ring_free_chain(vq, desc_idx);
 223                 vq->vq_descx[desc_idx].cookie = NULL;
 224         }
 225
 226         return i;
 227 }
 228
 229 static uint16_t
 230 virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
 231                         struct rte_mbuf **rx_pkts,
 232                         uint32_t *len,
 233                         uint16_t num)
 234 {
 235         struct vring_used_elem *uep;
 236         struct rte_mbuf *cookie;
 237         uint16_t used_idx = 0;
 238         uint16_t i;
 239
 240         if (unlikely(num == 0))
 241                 return 0;
 242
 243         for (i = 0; i < num; i++) {
 244                 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1);
 245                 /* Desc idx same as used idx */
 246                 uep = &vq->vq_split.ring.used->ring[used_idx];
 247                 len[i] = uep->len;
 248                 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie;
 249
 250                 if (unlikely(cookie == NULL)) {
 251                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
 252                                 vq->vq_used_cons_idx);
 253                         break;
 254                 }
 255
 256                 rte_prefetch0(cookie);
 257                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 258                 rx_pkts[i]  = cookie;
 259                 vq->vq_used_cons_idx++;
 260                 vq->vq_descx[used_idx].cookie = NULL;
 261         }
 262
 263         vq_ring_free_inorder(vq, used_idx, i);
 264         return i;
 265 }
 266
 267 #ifndef DEFAULT_TX_FREE_THRESH
 268 #define DEFAULT_TX_FREE_THRESH 32
 269 #endif
 270
 271 static void
 272 virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
 273 {
 274         uint16_t used_idx, id, curr_id, free_cnt = 0;
 275         uint16_t size = vq->vq_nentries;
 276         struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
 277         struct vq_desc_extra *dxp;
 278
 279         used_idx = vq->vq_used_cons_idx;
 280         /* desc_is_used has a load-acquire or rte_cio_rmb inside
 281          * and wait for used desc in virtqueue.
 282          */
 283         while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
 284                 id = desc[used_idx].id;
 285                 do {
 286                         curr_id = used_idx;
 287                         dxp = &vq->vq_descx[used_idx];
 288                         used_idx += dxp->ndescs;
 289                         free_cnt += dxp->ndescs;
 290                         num -= dxp->ndescs;
 291                         if (used_idx >= size) {
 292                                 used_idx -= size;
 293                                 vq->vq_packed.used_wrap_counter ^= 1;
 294                         }
 295                         if (dxp->cookie != NULL) {
 296                                 rte_pktmbuf_free(dxp->cookie);
 297                                 dxp->cookie = NULL;
 298                         }
 299                 } while (curr_id != id);
 300         }
 301         vq->vq_used_cons_idx = used_idx;
 302         vq->vq_free_cnt += free_cnt;
 303 }
 304
 305 static void
 306 virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
 307 {
 308         uint16_t used_idx, id;
 309         uint16_t size = vq->vq_nentries;
 310         struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
 311         struct vq_desc_extra *dxp;
 312
 313         used_idx = vq->vq_used_cons_idx;
 314         /* desc_is_used has a load-acquire or rte_cio_rmb inside
 315          * and wait for used desc in virtqueue.
 316          */
 317         while (num-- && desc_is_used(&desc[used_idx], vq)) {
 318                 id = desc[used_idx].id;
 319                 dxp = &vq->vq_descx[id];
 320                 vq->vq_used_cons_idx += dxp->ndescs;
 321                 if (vq->vq_used_cons_idx >= size) {
 322                         vq->vq_used_cons_idx -= size;
 323                         vq->vq_packed.used_wrap_counter ^= 1;
 324                 }
 325                 vq_ring_free_id_packed(vq, id);
 326                 if (dxp->cookie != NULL) {
 327                         rte_pktmbuf_free(dxp->cookie);
 328                         dxp->cookie = NULL;
 329                 }
 330                 used_idx = vq->vq_used_cons_idx;
 331         }
 332 }
 333
 334 /* Cleanup from completed transmits. */
 335 static inline void
 336 virtio_xmit_cleanup_packed(struct virtqueue *vq, int num, int in_order)
 337 {
 338         if (in_order)
 339                 virtio_xmit_cleanup_inorder_packed(vq, num);
 340         else
 341                 virtio_xmit_cleanup_normal_packed(vq, num);
 342 }
 343
 344 static void
 345 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
 346 {
 347         uint16_t i, used_idx, desc_idx;
 348         for (i = 0; i < num; i++) {
 349                 struct vring_used_elem *uep;
 350                 struct vq_desc_extra *dxp;
 351
 352                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 353                 uep = &vq->vq_split.ring.used->ring[used_idx];
 354
 355                 desc_idx = (uint16_t) uep->id;
 356                 dxp = &vq->vq_descx[desc_idx];
 357                 vq->vq_used_cons_idx++;
 358                 vq_ring_free_chain(vq, desc_idx);
 359
 360                 if (dxp->cookie != NULL) {
 361                         rte_pktmbuf_free(dxp->cookie);
 362                         dxp->cookie = NULL;
 363                 }
 364         }
 365 }
 366
 367 /* Cleanup from completed inorder transmits. */
 368 static __rte_always_inline void
 369 virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num)
 370 {
 371         uint16_t i, idx = vq->vq_used_cons_idx;
 372         int16_t free_cnt = 0;
 373         struct vq_desc_extra *dxp = NULL;
 374
 375         if (unlikely(num == 0))
 376                 return;
 377
 378         for (i = 0; i < num; i++) {
 379                 dxp = &vq->vq_descx[idx++ & (vq->vq_nentries - 1)];
 380                 free_cnt += dxp->ndescs;
 381                 if (dxp->cookie != NULL) {
 382                         rte_pktmbuf_free(dxp->cookie);
 383                         dxp->cookie = NULL;
 384                 }
 385         }
 386
 387         vq->vq_free_cnt += free_cnt;
 388         vq->vq_used_cons_idx = idx;
 389 }
 390
 391 static inline int
 392 virtqueue_enqueue_refill_inorder(struct virtqueue *vq,
 393                         struct rte_mbuf **cookies,
 394                         uint16_t num)
 395 {
 396         struct vq_desc_extra *dxp;
 397         struct virtio_hw *hw = vq->hw;
 398         struct vring_desc *start_dp;
 399         uint16_t head_idx, idx, i = 0;
 400
 401         if (unlikely(vq->vq_free_cnt == 0))
 402                 return -ENOSPC;
 403         if (unlikely(vq->vq_free_cnt < num))
 404                 return -EMSGSIZE;
 405
 406         head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1);
 407         start_dp = vq->vq_split.ring.desc;
 408
 409         while (i < num) {
 410                 idx = head_idx & (vq->vq_nentries - 1);
 411                 dxp = &vq->vq_descx[idx];
 412                 dxp->cookie = (void *)cookies[i];
 413                 dxp->ndescs = 1;
 414
 415                 start_dp[idx].addr =
 416                                 VIRTIO_MBUF_ADDR(cookies[i], vq) +
 417                                 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
 418                 start_dp[idx].len =
 419                                 cookies[i]->buf_len -
 420                                 RTE_PKTMBUF_HEADROOM +
 421                                 hw->vtnet_hdr_size;
 422                 start_dp[idx].flags =  VRING_DESC_F_WRITE;
 423
 424                 vq_update_avail_ring(vq, idx);
 425                 head_idx++;
 426                 i++;
 427         }
 428
 429         vq->vq_desc_head_idx += num;
 430         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 431         return 0;
 432 }
 433
 434 static inline int
 435 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie,
 436                                 uint16_t num)
 437 {
 438         struct vq_desc_extra *dxp;
 439         struct virtio_hw *hw = vq->hw;
 440         struct vring_desc *start_dp = vq->vq_split.ring.desc;
 441         uint16_t idx, i;
 442
 443         if (unlikely(vq->vq_free_cnt == 0))
 444                 return -ENOSPC;
 445         if (unlikely(vq->vq_free_cnt < num))
 446                 return -EMSGSIZE;
 447
 448         if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries))
 449                 return -EFAULT;
 450
 451         for (i = 0; i < num; i++) {
 452                 idx = vq->vq_desc_head_idx;
 453                 dxp = &vq->vq_descx[idx];
 454                 dxp->cookie = (void *)cookie[i];
 455                 dxp->ndescs = 1;
 456
 457                 start_dp[idx].addr =
 458                         VIRTIO_MBUF_ADDR(cookie[i], vq) +
 459                         RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
 460                 start_dp[idx].len =
 461                         cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM +
 462                         hw->vtnet_hdr_size;
 463                 start_dp[idx].flags = VRING_DESC_F_WRITE;
 464                 vq->vq_desc_head_idx = start_dp[idx].next;
 465                 vq_update_avail_ring(vq, idx);
 466                 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) {
 467                         vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
 468                         break;
 469                 }
 470         }
 471
 472         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 473
 474         return 0;
 475 }
 476
 477 static inline int
 478 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
 479                                      struct rte_mbuf **cookie, uint16_t num)
 480 {
 481         struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
 482         uint16_t flags = vq->vq_packed.cached_flags;
 483         struct virtio_hw *hw = vq->hw;
 484         struct vq_desc_extra *dxp;
 485         uint16_t idx;
 486         int i;
 487
 488         if (unlikely(vq->vq_free_cnt == 0))
 489                 return -ENOSPC;
 490         if (unlikely(vq->vq_free_cnt < num))
 491                 return -EMSGSIZE;
 492
 493         for (i = 0; i < num; i++) {
 494                 idx = vq->vq_avail_idx;
 495                 dxp = &vq->vq_descx[idx];
 496                 dxp->cookie = (void *)cookie[i];
 497                 dxp->ndescs = 1;
 498
 499                 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) +
 500                                 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
 501                 start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM
 502                                         + hw->vtnet_hdr_size;
 503
 504                 vq->vq_desc_head_idx = dxp->next;
 505                 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 506                         vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
 507
 508                 virtqueue_store_flags_packed(&start_dp[idx], flags,
 509                                              hw->weak_barriers);
 510
 511                 if (++vq->vq_avail_idx >= vq->vq_nentries) {
 512                         vq->vq_avail_idx -= vq->vq_nentries;
 513                         vq->vq_packed.cached_flags ^=
 514                                 VRING_PACKED_DESC_F_AVAIL_USED;
 515                         flags = vq->vq_packed.cached_flags;
 516                 }
 517         }
 518         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 519         return 0;
 520 }
 521
 522 /* When doing TSO, the IP length is not included in the pseudo header
 523  * checksum of the packet given to the PMD, but for virtio it is
 524  * expected.
 525  */
 526 static void
 527 virtio_tso_fix_cksum(struct rte_mbuf *m)
 528 {
 529         /* common case: header is not fragmented */
 530         if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len +
 531                         m->l4_len)) {
 532                 struct rte_ipv4_hdr *iph;
 533                 struct rte_ipv6_hdr *ip6h;
 534                 struct rte_tcp_hdr *th;
 535                 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen;
 536                 uint32_t tmp;
 537
 538                 iph = rte_pktmbuf_mtod_offset(m,
 539                                         struct rte_ipv4_hdr *, m->l2_len);
 540                 th = RTE_PTR_ADD(iph, m->l3_len);
 541                 if ((iph->version_ihl >> 4) == 4) {
 542                         iph->hdr_checksum = 0;
 543                         iph->hdr_checksum = rte_ipv4_cksum(iph);
 544                         ip_len = iph->total_length;
 545                         ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) -
 546                                 m->l3_len);
 547                 } else {
 548                         ip6h = (struct rte_ipv6_hdr *)iph;
 549                         ip_paylen = ip6h->payload_len;
 550                 }
 551
 552                 /* calculate the new phdr checksum not including ip_paylen */
 553                 prev_cksum = th->cksum;
 554                 tmp = prev_cksum;
 555                 tmp += ip_paylen;
 556                 tmp = (tmp & 0xffff) + (tmp >> 16);
 557                 new_cksum = tmp;
 558
 559                 /* replace it in the packet */
 560                 th->cksum = new_cksum;
 561         }
 562 }
 563
 564
 565 /* avoid write operation when necessary, to lessen cache issues */
 566 #define ASSIGN_UNLESS_EQUAL(var, val) do {      \
 567         if ((var) != (val))                     \
 568                 (var) = (val);                  \
 569 } while (0)
 570
 571 #define virtqueue_clear_net_hdr(_hdr) do {              \
 572         ASSIGN_UNLESS_EQUAL((_hdr)->csum_start, 0);     \
 573         ASSIGN_UNLESS_EQUAL((_hdr)->csum_offset, 0);    \
 574         ASSIGN_UNLESS_EQUAL((_hdr)->flags, 0);          \
 575         ASSIGN_UNLESS_EQUAL((_hdr)->gso_type, 0);       \
 576         ASSIGN_UNLESS_EQUAL((_hdr)->gso_size, 0);       \
 577         ASSIGN_UNLESS_EQUAL((_hdr)->hdr_len, 0);        \
 578 } while (0)
 579
 580 static inline void
 581 virtqueue_xmit_offload(struct virtio_net_hdr *hdr,
 582                         struct rte_mbuf *cookie,
 583                         bool offload)
 584 {
 585         if (offload) {
 586                 if (cookie->ol_flags & PKT_TX_TCP_SEG)
 587                         cookie->ol_flags |= PKT_TX_TCP_CKSUM;
 588
 589                 switch (cookie->ol_flags & PKT_TX_L4_MASK) {
 590                 case PKT_TX_UDP_CKSUM:
 591                         hdr->csum_start = cookie->l2_len + cookie->l3_len;
 592                         hdr->csum_offset = offsetof(struct rte_udp_hdr,
 593                                 dgram_cksum);
 594                         hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
 595                         break;
 596
 597                 case PKT_TX_TCP_CKSUM:
 598                         hdr->csum_start = cookie->l2_len + cookie->l3_len;
 599                         hdr->csum_offset = offsetof(struct rte_tcp_hdr, cksum);
 600                         hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
 601                         break;
 602
 603                 default:
 604                         ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0);
 605                         ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0);
 606                         ASSIGN_UNLESS_EQUAL(hdr->flags, 0);
 607                         break;
 608                 }
 609
 610                 /* TCP Segmentation Offload */
 611                 if (cookie->ol_flags & PKT_TX_TCP_SEG) {
 612                         hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ?
 613                                 VIRTIO_NET_HDR_GSO_TCPV6 :
 614                                 VIRTIO_NET_HDR_GSO_TCPV4;
 615                         hdr->gso_size = cookie->tso_segsz;
 616                         hdr->hdr_len =
 617                                 cookie->l2_len +
 618                                 cookie->l3_len +
 619                                 cookie->l4_len;
 620                 } else {
 621                         ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0);
 622                         ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0);
 623                         ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0);
 624                 }
 625         }
 626 }
 627
 628 static inline void
 629 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq,
 630                         struct rte_mbuf **cookies,
 631                         uint16_t num)
 632 {
 633         struct vq_desc_extra *dxp;
 634         struct virtqueue *vq = txvq->vq;
 635         struct vring_desc *start_dp;
 636         struct virtio_net_hdr *hdr;
 637         uint16_t idx;
 638         uint16_t head_size = vq->hw->vtnet_hdr_size;
 639         uint16_t i = 0;
 640
 641         idx = vq->vq_desc_head_idx;
 642         start_dp = vq->vq_split.ring.desc;
 643
 644         while (i < num) {
 645                 idx = idx & (vq->vq_nentries - 1);
 646                 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)];
 647                 dxp->cookie = (void *)cookies[i];
 648                 dxp->ndescs = 1;
 649                 virtio_update_packet_stats(&txvq->stats, cookies[i]);
 650
 651                 hdr = (struct virtio_net_hdr *)(char *)cookies[i]->buf_addr +
 652                         cookies[i]->data_off - head_size;
 653
 654                 /* if offload disabled, hdr is not zeroed yet, do it now */
 655                 if (!vq->hw->has_tx_offload)
 656                         virtqueue_clear_net_hdr(hdr);
 657                 else
 658                         virtqueue_xmit_offload(hdr, cookies[i], true);
 659
 660                 start_dp[idx].addr  =
 661                         VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq) - head_size;
 662                 start_dp[idx].len   = cookies[i]->data_len + head_size;
 663                 start_dp[idx].flags = 0;
 664
 665
 666                 vq_update_avail_ring(vq, idx);
 667
 668                 idx++;
 669                 i++;
 670         };
 671
 672         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 673         vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1);
 674 }
 675
 676 static inline void
 677 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq,
 678                                    struct rte_mbuf *cookie,
 679                                    int in_order)
 680 {
 681         struct virtqueue *vq = txvq->vq;
 682         struct vring_packed_desc *dp;
 683         struct vq_desc_extra *dxp;
 684         uint16_t idx, id, flags;
 685         uint16_t head_size = vq->hw->vtnet_hdr_size;
 686         struct virtio_net_hdr *hdr;
 687
 688         id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
 689         idx = vq->vq_avail_idx;
 690         dp = &vq->vq_packed.ring.desc[idx];
 691
 692         dxp = &vq->vq_descx[id];
 693         dxp->ndescs = 1;
 694         dxp->cookie = cookie;
 695
 696         flags = vq->vq_packed.cached_flags;
 697
 698         /* prepend cannot fail, checked by caller */
 699         hdr = (struct virtio_net_hdr *)(char *)cookie->buf_addr +
 700                 cookie->data_off - head_size;
 701
 702         /* if offload disabled, hdr is not zeroed yet, do it now */
 703         if (!vq->hw->has_tx_offload)
 704                 virtqueue_clear_net_hdr(hdr);
 705         else
 706                 virtqueue_xmit_offload(hdr, cookie, true);
 707
 708         dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size;
 709         dp->len  = cookie->data_len + head_size;
 710         dp->id   = id;
 711
 712         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 713                 vq->vq_avail_idx -= vq->vq_nentries;
 714                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 715         }
 716
 717         vq->vq_free_cnt--;
 718
 719         if (!in_order) {
 720                 vq->vq_desc_head_idx = dxp->next;
 721                 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 722                         vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 723         }
 724
 725         virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers);
 726 }
 727
 728 static inline void
 729 virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 730                               uint16_t needed, int can_push, int in_order)
 731 {
 732         struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
 733         struct vq_desc_extra *dxp;
 734         struct virtqueue *vq = txvq->vq;
 735         struct vring_packed_desc *start_dp, *head_dp;
 736         uint16_t idx, id, head_idx, head_flags;
 737         uint16_t head_size = vq->hw->vtnet_hdr_size;
 738         struct virtio_net_hdr *hdr;
 739         uint16_t prev;
 740         bool prepend_header = false;
 741
 742         id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
 743
 744         dxp = &vq->vq_descx[id];
 745         dxp->ndescs = needed;
 746         dxp->cookie = cookie;
 747
 748         head_idx = vq->vq_avail_idx;
 749         idx = head_idx;
 750         prev = head_idx;
 751         start_dp = vq->vq_packed.ring.desc;
 752
 753         head_dp = &vq->vq_packed.ring.desc[idx];
 754         head_flags = cookie->next ? VRING_DESC_F_NEXT : 0;
 755         head_flags |= vq->vq_packed.cached_flags;
 756
 757         if (can_push) {
 758                 /* prepend cannot fail, checked by caller */
 759                 hdr = (struct virtio_net_hdr *)(char *)cookie->buf_addr +
 760                         cookie->data_off - head_size;
 761                 prepend_header = true;
 762
 763                 /* if offload disabled, it is not zeroed below, do it now */
 764                 if (!vq->hw->has_tx_offload)
 765                         virtqueue_clear_net_hdr(hdr);
 766         } else {
 767                 /* setup first tx ring slot to point to header
 768                  * stored in reserved region.
 769                  */
 770                 start_dp[idx].addr  = txvq->virtio_net_hdr_mem +
 771                         RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);
 772                 start_dp[idx].len   = vq->hw->vtnet_hdr_size;
 773                 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
 774                 idx++;
 775                 if (idx >= vq->vq_nentries) {
 776                         idx -= vq->vq_nentries;
 777                         vq->vq_packed.cached_flags ^=
 778                                 VRING_PACKED_DESC_F_AVAIL_USED;
 779                 }
 780         }
 781
 782         virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload);
 783
 784         do {
 785                 uint16_t flags;
 786
 787                 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq);
 788                 start_dp[idx].len  = cookie->data_len;
 789                 if (prepend_header) {
 790                         start_dp[idx].addr -= head_size;
 791                         start_dp[idx].len += head_size;
 792                         prepend_header = false;
 793                 }
 794
 795                 if (likely(idx != head_idx)) {
 796                         flags = cookie->next ? VRING_DESC_F_NEXT : 0;
 797                         flags |= vq->vq_packed.cached_flags;
 798                         start_dp[idx].flags = flags;
 799                 }
 800                 prev = idx;
 801                 idx++;
 802                 if (idx >= vq->vq_nentries) {
 803                         idx -= vq->vq_nentries;
 804                         vq->vq_packed.cached_flags ^=
 805                                 VRING_PACKED_DESC_F_AVAIL_USED;
 806                 }
 807         } while ((cookie = cookie->next) != NULL);
 808
 809         start_dp[prev].id = id;
 810
 811         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
 812         vq->vq_avail_idx = idx;
 813
 814         if (!in_order) {
 815                 vq->vq_desc_head_idx = dxp->next;
 816                 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 817                         vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 818         }
 819
 820         virtqueue_store_flags_packed(head_dp, head_flags,
 821                                      vq->hw->weak_barriers);
 822 }
 823
 824 static inline void
 825 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 826                         uint16_t needed, int use_indirect, int can_push,
 827                         int in_order)
 828 {
 829         struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
 830         struct vq_desc_extra *dxp;
 831         struct virtqueue *vq = txvq->vq;
 832         struct vring_desc *start_dp;
 833         uint16_t seg_num = cookie->nb_segs;
 834         uint16_t head_idx, idx;
 835         uint16_t head_size = vq->hw->vtnet_hdr_size;
 836         bool prepend_header = false;
 837         struct virtio_net_hdr *hdr;
 838
 839         head_idx = vq->vq_desc_head_idx;
 840         idx = head_idx;
 841         if (in_order)
 842                 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)];
 843         else
 844                 dxp = &vq->vq_descx[idx];
 845         dxp->cookie = (void *)cookie;
 846         dxp->ndescs = needed;
 847
 848         start_dp = vq->vq_split.ring.desc;
 849
 850         if (can_push) {
 851                 /* prepend cannot fail, checked by caller */
 852                 hdr = (struct virtio_net_hdr *)(char *)cookie->buf_addr +
 853                         cookie->data_off - head_size;
 854                 prepend_header = true;
 855
 856                 /* if offload disabled, it is not zeroed below, do it now */
 857                 if (!vq->hw->has_tx_offload)
 858                         virtqueue_clear_net_hdr(hdr);
 859         } else if (use_indirect) {
 860                 /* setup tx ring slot to point to indirect
 861                  * descriptor list stored in reserved region.
 862                  *
 863                  * the first slot in indirect ring is already preset
 864                  * to point to the header in reserved region
 865                  */
 866                 start_dp[idx].addr  = txvq->virtio_net_hdr_mem +
 867                         RTE_PTR_DIFF(&txr[idx].tx_indir, txr);
 868                 start_dp[idx].len   = (seg_num + 1) * sizeof(struct vring_desc);
 869                 start_dp[idx].flags = VRING_DESC_F_INDIRECT;
 870                 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
 871
 872                 /* loop below will fill in rest of the indirect elements */
 873                 start_dp = txr[idx].tx_indir;
 874                 idx = 1;
 875         } else {
 876                 /* setup first tx ring slot to point to header
 877                  * stored in reserved region.
 878                  */
 879                 start_dp[idx].addr  = txvq->virtio_net_hdr_mem +
 880                         RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);
 881                 start_dp[idx].len   = vq->hw->vtnet_hdr_size;
 882                 start_dp[idx].flags = VRING_DESC_F_NEXT;
 883                 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
 884
 885                 idx = start_dp[idx].next;
 886         }
 887
 888         virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload);
 889
 890         do {
 891                 start_dp[idx].addr  = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq);
 892                 start_dp[idx].len   = cookie->data_len;
 893                 if (prepend_header) {
 894                         start_dp[idx].addr -= head_size;
 895                         start_dp[idx].len += head_size;
 896                         prepend_header = false;
 897                 }
 898                 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0;
 899                 idx = start_dp[idx].next;
 900         } while ((cookie = cookie->next) != NULL);
 901
 902         if (use_indirect)
 903                 idx = vq->vq_split.ring.desc[head_idx].next;
 904
 905         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
 906
 907         vq->vq_desc_head_idx = idx;
 908         vq_update_avail_ring(vq, head_idx);
 909
 910         if (!in_order) {
 911                 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 912                         vq->vq_desc_tail_idx = idx;
 913         }
 914 }
 915
 916 void
 917 virtio_dev_cq_start(struct rte_eth_dev *dev)
 918 {
 919         struct virtio_hw *hw = dev->data->dev_private;
 920
 921         if (hw->cvq && hw->cvq->vq) {
 922                 rte_spinlock_init(&hw->cvq->lock);
 923                 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq->vq);
 924         }
 925 }
 926
 927 int
 928 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 929                         uint16_t queue_idx,
 930                         uint16_t nb_desc,
 931                         unsigned int socket_id __rte_unused,
 932                         const struct rte_eth_rxconf *rx_conf,
 933                         struct rte_mempool *mp)
 934 {
 935         uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
 936         struct virtio_hw *hw = dev->data->dev_private;
 937         struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
 938         struct virtnet_rx *rxvq;
 939
 940         PMD_INIT_FUNC_TRACE();
 941
 942         if (rx_conf->rx_deferred_start) {
 943                 PMD_INIT_LOG(ERR, "Rx deferred start is not supported");
 944                 return -EINVAL;
 945         }
 946
 947         if (nb_desc == 0 || nb_desc > vq->vq_nentries)
 948                 nb_desc = vq->vq_nentries;
 949         vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
 950
 951         rxvq = &vq->rxq;
 952         rxvq->queue_id = queue_idx;
 953         rxvq->mpool = mp;
 954         dev->data->rx_queues[queue_idx] = rxvq;
 955
 956         return 0;
 957 }
 958
 959 int
 960 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx)
 961 {
 962         uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
 963         struct virtio_hw *hw = dev->data->dev_private;
 964         struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
 965         struct virtnet_rx *rxvq = &vq->rxq;
 966         struct rte_mbuf *m;
 967         uint16_t desc_idx;
 968         int error, nbufs, i;
 969
 970         PMD_INIT_FUNC_TRACE();
 971
 972         /* Allocate blank mbufs for the each rx descriptor */
 973         nbufs = 0;
 974
 975         if (hw->use_simple_rx) {
 976                 for (desc_idx = 0; desc_idx < vq->vq_nentries;
 977                      desc_idx++) {
 978                         vq->vq_split.ring.avail->ring[desc_idx] = desc_idx;
 979                         vq->vq_split.ring.desc[desc_idx].flags =
 980                                 VRING_DESC_F_WRITE;
 981                 }
 982
 983                 virtio_rxq_vec_setup(rxvq);
 984         }
 985
 986         memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf));
 987         for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST;
 988              desc_idx++) {
 989                 vq->sw_ring[vq->vq_nentries + desc_idx] =
 990                         &rxvq->fake_mbuf;
 991         }
 992
 993         if (hw->use_simple_rx) {
 994                 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
 995                         virtio_rxq_rearm_vec(rxvq);
 996                         nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
 997                 }
 998         } else if (hw->use_inorder_rx) {
 999                 if ((!virtqueue_full(vq))) {
1000                         uint16_t free_cnt = vq->vq_free_cnt;
1001                         struct rte_mbuf *pkts[free_cnt];
1002
1003                         if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts,
1004                                 free_cnt)) {
1005                                 error = virtqueue_enqueue_refill_inorder(vq,
1006                                                 pkts,
1007                                                 free_cnt);
1008                                 if (unlikely(error)) {
1009                                         for (i = 0; i < free_cnt; i++)
1010                                                 rte_pktmbuf_free(pkts[i]);
1011                                 }
1012                         }
1013
1014                         nbufs += free_cnt;
1015                         vq_update_avail_idx(vq);
1016                 }
1017         } else {
1018                 while (!virtqueue_full(vq)) {
1019                         m = rte_mbuf_raw_alloc(rxvq->mpool);
1020                         if (m == NULL)
1021                                 break;
1022
1023                         /* Enqueue allocated buffers */
1024                         if (vtpci_packed_queue(vq->hw))
1025                                 error = virtqueue_enqueue_recv_refill_packed(vq,
1026                                                 &m, 1);
1027                         else
1028                                 error = virtqueue_enqueue_recv_refill(vq,
1029                                                 &m, 1);
1030                         if (error) {
1031                                 rte_pktmbuf_free(m);
1032                                 break;
1033                         }
1034                         nbufs++;
1035                 }
1036
1037                 if (!vtpci_packed_queue(vq->hw))
1038                         vq_update_avail_idx(vq);
1039         }
1040
1041         PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
1042
1043         VIRTQUEUE_DUMP(vq);
1044
1045         return 0;
1046 }
1047
1048 /*
1049  * struct rte_eth_dev *dev: Used to update dev
1050  * uint16_t nb_desc: Defaults to values read from config space
1051  * unsigned int socket_id: Used to allocate memzone
1052  * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
1053  * uint16_t queue_idx: Just used as an index in dev txq list
1054  */
1055 int
1056 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
1057                         uint16_t queue_idx,
1058                         uint16_t nb_desc,
1059                         unsigned int socket_id __rte_unused,
1060                         const struct rte_eth_txconf *tx_conf)
1061 {
1062         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
1063         struct virtio_hw *hw = dev->data->dev_private;
1064         struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
1065         struct virtnet_tx *txvq;
1066         uint16_t tx_free_thresh;
1067
1068         PMD_INIT_FUNC_TRACE();
1069
1070         if (nb_desc == 0 || nb_desc > vq->vq_nentries)
1071                 nb_desc = vq->vq_nentries;
1072         vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
1073
1074         txvq = &vq->txq;
1075         txvq->queue_id = queue_idx;
1076
1077         tx_free_thresh = tx_conf->tx_free_thresh;
1078         if (tx_free_thresh == 0)
1079                 tx_free_thresh =
1080                         RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
1081
1082         if (tx_free_thresh >= (vq->vq_nentries - 3)) {
1083                 RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the "
1084                         "number of TX entries minus 3 (%u)."
1085                         " (tx_free_thresh=%u port=%u queue=%u)\n",
1086                         vq->vq_nentries - 3,
1087                         tx_free_thresh, dev->data->port_id, queue_idx);
1088                 return -EINVAL;
1089         }
1090
1091         vq->vq_free_thresh = tx_free_thresh;
1092
1093         dev->data->tx_queues[queue_idx] = txvq;
1094         return 0;
1095 }
1096
1097 int
1098 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
1099                                 uint16_t queue_idx)
1100 {
1101         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
1102         struct virtio_hw *hw = dev->data->dev_private;
1103         struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
1104
1105         PMD_INIT_FUNC_TRACE();
1106
1107         if (!vtpci_packed_queue(hw)) {
1108                 if (hw->use_inorder_tx)
1109                         vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0;
1110         }
1111
1112         VIRTQUEUE_DUMP(vq);
1113
1114         return 0;
1115 }
1116
1117 static inline void
1118 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
1119 {
1120         int error;
1121         /*
1122          * Requeue the discarded mbuf. This should always be
1123          * successful since it was just dequeued.
1124          */
1125         if (vtpci_packed_queue(vq->hw))
1126                 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1);
1127         else
1128                 error = virtqueue_enqueue_recv_refill(vq, &m, 1);
1129
1130         if (unlikely(error)) {
1131                 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
1132                 rte_pktmbuf_free(m);
1133         }
1134 }
1135
1136 static inline void
1137 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m)
1138 {
1139         int error;
1140
1141         error = virtqueue_enqueue_refill_inorder(vq, &m, 1);
1142         if (unlikely(error)) {
1143                 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
1144                 rte_pktmbuf_free(m);
1145         }
1146 }
1147
1148 /* Optionally fill offload information in structure */
1149 static inline int
1150 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
1151 {
1152         struct rte_net_hdr_lens hdr_lens;
1153         uint32_t hdrlen, ptype;
1154         int l4_supported = 0;
1155
1156         /* nothing to do */
1157         if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
1158                 return 0;
1159
1160         m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
1161
1162         ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK);
1163         m->packet_type = ptype;
1164         if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP ||
1165             (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP ||
1166             (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP)
1167                 l4_supported = 1;
1168
1169         if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1170                 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len;
1171                 if (hdr->csum_start <= hdrlen && l4_supported) {
1172                         m->ol_flags |= PKT_RX_L4_CKSUM_NONE;
1173                 } else {
1174                         /* Unknown proto or tunnel, do sw cksum. We can assume
1175                          * the cksum field is in the first segment since the
1176                          * buffers we provided to the host are large enough.
1177                          * In case of SCTP, this will be wrong since it's a CRC
1178                          * but there's nothing we can do.
1179                          */
1180                         uint16_t csum = 0, off;
1181
1182                         rte_raw_cksum_mbuf(m, hdr->csum_start,
1183                                 rte_pktmbuf_pkt_len(m) - hdr->csum_start,
1184                                 &csum);
1185                         if (likely(csum != 0xffff))
1186                                 csum = ~csum;
1187                         off = hdr->csum_offset + hdr->csum_start;
1188                         if (rte_pktmbuf_data_len(m) >= off + 1)
1189                                 *rte_pktmbuf_mtod_offset(m, uint16_t *,
1190                                         off) = csum;
1191                 }
1192         } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) {
1193                 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
1194         }
1195
1196         /* GSO request, save required information in mbuf */
1197         if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
1198                 /* Check unsupported modes */
1199                 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) ||
1200                     (hdr->gso_size == 0)) {
1201                         return -EINVAL;
1202                 }
1203
1204                 /* Update mss lengthes in mbuf */
1205                 m->tso_segsz = hdr->gso_size;
1206                 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
1207                         case VIRTIO_NET_HDR_GSO_TCPV4:
1208                         case VIRTIO_NET_HDR_GSO_TCPV6:
1209                                 m->ol_flags |= PKT_RX_LRO | \
1210                                         PKT_RX_L4_CKSUM_NONE;
1211                                 break;
1212                         default:
1213                                 return -EINVAL;
1214                 }
1215         }
1216
1217         return 0;
1218 }
1219
1220 #define VIRTIO_MBUF_BURST_SZ 64
1221 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
1222 uint16_t
1223 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1224 {
1225         struct virtnet_rx *rxvq = rx_queue;
1226         struct virtqueue *vq = rxvq->vq;
1227         struct virtio_hw *hw = vq->hw;
1228         struct rte_mbuf *rxm;
1229         uint16_t nb_used, num, nb_rx;
1230         uint32_t len[VIRTIO_MBUF_BURST_SZ];
1231         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1232         int error;
1233         uint32_t i, nb_enqueued;
1234         uint32_t hdr_size;
1235         struct virtio_net_hdr *hdr;
1236
1237         nb_rx = 0;
1238         if (unlikely(hw->started == 0))
1239                 return nb_rx;
1240
1241         nb_used = VIRTQUEUE_NUSED(vq);
1242
1243         virtio_rmb(hw->weak_barriers);
1244
1245         num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
1246         if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1247                 num = VIRTIO_MBUF_BURST_SZ;
1248         if (likely(num > DESC_PER_CACHELINE))
1249                 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1250
1251         num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
1252         PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
1253
1254         nb_enqueued = 0;
1255         hdr_size = hw->vtnet_hdr_size;
1256
1257         for (i = 0; i < num ; i++) {
1258                 rxm = rcv_pkts[i];
1259
1260                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1261
1262                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1263                         PMD_RX_LOG(ERR, "Packet drop");
1264                         nb_enqueued++;
1265                         virtio_discard_rxbuf(vq, rxm);
1266                         rxvq->stats.errors++;
1267                         continue;
1268                 }
1269
1270                 rxm->port = rxvq->port_id;
1271                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1272                 rxm->ol_flags = 0;
1273                 rxm->vlan_tci = 0;
1274
1275                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1276                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1277
1278                 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
1279                         RTE_PKTMBUF_HEADROOM - hdr_size);
1280
1281                 if (hw->vlan_strip)
1282                         rte_vlan_strip(rxm);
1283
1284                 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
1285                         virtio_discard_rxbuf(vq, rxm);
1286                         rxvq->stats.errors++;
1287                         continue;
1288                 }
1289
1290                 virtio_rx_stats_updated(rxvq, rxm);
1291
1292                 rx_pkts[nb_rx++] = rxm;
1293         }
1294
1295         rxvq->stats.packets += nb_rx;
1296
1297         /* Allocate new mbuf for the used descriptor */
1298         if (likely(!virtqueue_full(vq))) {
1299                 uint16_t free_cnt = vq->vq_free_cnt;
1300                 struct rte_mbuf *new_pkts[free_cnt];
1301
1302                 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
1303                                                 free_cnt) == 0)) {
1304                         error = virtqueue_enqueue_recv_refill(vq, new_pkts,
1305                                         free_cnt);
1306                         if (unlikely(error)) {
1307                                 for (i = 0; i < free_cnt; i++)
1308                                         rte_pktmbuf_free(new_pkts[i]);
1309                         }
1310                         nb_enqueued += free_cnt;
1311                 } else {
1312                         struct rte_eth_dev *dev =
1313                                 &rte_eth_devices[rxvq->port_id];
1314                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1315                 }
1316         }
1317
1318         if (likely(nb_enqueued)) {
1319                 vq_update_avail_idx(vq);
1320
1321                 if (unlikely(virtqueue_kick_prepare(vq))) {
1322                         virtqueue_notify(vq);
1323                         PMD_RX_LOG(DEBUG, "Notified");
1324                 }
1325         }
1326
1327         return nb_rx;
1328 }
1329
1330 uint16_t
1331 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts,
1332                         uint16_t nb_pkts)
1333 {
1334         struct virtnet_rx *rxvq = rx_queue;
1335         struct virtqueue *vq = rxvq->vq;
1336         struct virtio_hw *hw = vq->hw;
1337         struct rte_mbuf *rxm;
1338         uint16_t num, nb_rx;
1339         uint32_t len[VIRTIO_MBUF_BURST_SZ];
1340         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1341         int error;
1342         uint32_t i, nb_enqueued;
1343         uint32_t hdr_size;
1344         struct virtio_net_hdr *hdr;
1345
1346         nb_rx = 0;
1347         if (unlikely(hw->started == 0))
1348                 return nb_rx;
1349
1350         num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts);
1351         if (likely(num > DESC_PER_CACHELINE))
1352                 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1353
1354         num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
1355         PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1356
1357         nb_enqueued = 0;
1358         hdr_size = hw->vtnet_hdr_size;
1359
1360         for (i = 0; i < num; i++) {
1361                 rxm = rcv_pkts[i];
1362
1363                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1364
1365                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1366                         PMD_RX_LOG(ERR, "Packet drop");
1367                         nb_enqueued++;
1368                         virtio_discard_rxbuf(vq, rxm);
1369                         rxvq->stats.errors++;
1370                         continue;
1371                 }
1372
1373                 rxm->port = rxvq->port_id;
1374                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1375                 rxm->ol_flags = 0;
1376                 rxm->vlan_tci = 0;
1377
1378                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1379                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1380
1381                 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
1382                         RTE_PKTMBUF_HEADROOM - hdr_size);
1383
1384                 if (hw->vlan_strip)
1385                         rte_vlan_strip(rxm);
1386
1387                 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
1388                         virtio_discard_rxbuf(vq, rxm);
1389                         rxvq->stats.errors++;
1390                         continue;
1391                 }
1392
1393                 virtio_rx_stats_updated(rxvq, rxm);
1394
1395                 rx_pkts[nb_rx++] = rxm;
1396         }
1397
1398         rxvq->stats.packets += nb_rx;
1399
1400         /* Allocate new mbuf for the used descriptor */
1401         if (likely(!virtqueue_full(vq))) {
1402                 uint16_t free_cnt = vq->vq_free_cnt;
1403                 struct rte_mbuf *new_pkts[free_cnt];
1404
1405                 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
1406                                                 free_cnt) == 0)) {
1407                         error = virtqueue_enqueue_recv_refill_packed(vq,
1408                                         new_pkts, free_cnt);
1409                         if (unlikely(error)) {
1410                                 for (i = 0; i < free_cnt; i++)
1411                                         rte_pktmbuf_free(new_pkts[i]);
1412                         }
1413                         nb_enqueued += free_cnt;
1414                 } else {
1415                         struct rte_eth_dev *dev =
1416                                 &rte_eth_devices[rxvq->port_id];
1417                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1418                 }
1419         }
1420
1421         if (likely(nb_enqueued)) {
1422                 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1423                         virtqueue_notify(vq);
1424                         PMD_RX_LOG(DEBUG, "Notified");
1425                 }
1426         }
1427
1428         return nb_rx;
1429 }
1430
1431
1432 uint16_t
1433 virtio_recv_pkts_inorder(void *rx_queue,
1434                         struct rte_mbuf **rx_pkts,
1435                         uint16_t nb_pkts)
1436 {
1437         struct virtnet_rx *rxvq = rx_queue;
1438         struct virtqueue *vq = rxvq->vq;
1439         struct virtio_hw *hw = vq->hw;
1440         struct rte_mbuf *rxm;
1441         struct rte_mbuf *prev = NULL;
1442         uint16_t nb_used, num, nb_rx;
1443         uint32_t len[VIRTIO_MBUF_BURST_SZ];
1444         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1445         int error;
1446         uint32_t nb_enqueued;
1447         uint32_t seg_num;
1448         uint32_t seg_res;
1449         uint32_t hdr_size;
1450         int32_t i;
1451
1452         nb_rx = 0;
1453         if (unlikely(hw->started == 0))
1454                 return nb_rx;
1455
1456         nb_used = VIRTQUEUE_NUSED(vq);
1457         nb_used = RTE_MIN(nb_used, nb_pkts);
1458         nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
1459
1460         virtio_rmb(hw->weak_barriers);
1461
1462         PMD_RX_LOG(DEBUG, "used:%d", nb_used);
1463
1464         nb_enqueued = 0;
1465         seg_num = 1;
1466         seg_res = 0;
1467         hdr_size = hw->vtnet_hdr_size;
1468
1469         num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used);
1470
1471         for (i = 0; i < num; i++) {
1472                 struct virtio_net_hdr_mrg_rxbuf *header;
1473
1474                 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1475                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1476
1477                 rxm = rcv_pkts[i];
1478
1479                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1480                         PMD_RX_LOG(ERR, "Packet drop");
1481                         nb_enqueued++;
1482                         virtio_discard_rxbuf_inorder(vq, rxm);
1483                         rxvq->stats.errors++;
1484                         continue;
1485                 }
1486
1487                 header = (struct virtio_net_hdr_mrg_rxbuf *)
1488                          ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
1489                          - hdr_size);
1490
1491                 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1492                         seg_num = header->num_buffers;
1493                         if (seg_num == 0)
1494                                 seg_num = 1;
1495                 } else {
1496                         seg_num = 1;
1497                 }
1498
1499                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1500                 rxm->nb_segs = seg_num;
1501                 rxm->ol_flags = 0;
1502                 rxm->vlan_tci = 0;
1503                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1504                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1505
1506                 rxm->port = rxvq->port_id;
1507
1508                 rx_pkts[nb_rx] = rxm;
1509                 prev = rxm;
1510
1511                 if (vq->hw->has_rx_offload &&
1512                                 virtio_rx_offload(rxm, &header->hdr) < 0) {
1513                         virtio_discard_rxbuf_inorder(vq, rxm);
1514                         rxvq->stats.errors++;
1515                         continue;
1516                 }
1517
1518                 if (hw->vlan_strip)
1519                         rte_vlan_strip(rx_pkts[nb_rx]);
1520
1521                 seg_res = seg_num - 1;
1522
1523                 /* Merge remaining segments */
1524                 while (seg_res != 0 && i < (num - 1)) {
1525                         i++;
1526
1527                         rxm = rcv_pkts[i];
1528                         rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1529                         rxm->pkt_len = (uint32_t)(len[i]);
1530                         rxm->data_len = (uint16_t)(len[i]);
1531
1532                         rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1533
1534                         prev->next = rxm;
1535                         prev = rxm;
1536                         seg_res -= 1;
1537                 }
1538
1539                 if (!seg_res) {
1540                         virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1541                         nb_rx++;
1542                 }
1543         }
1544
1545         /* Last packet still need merge segments */
1546         while (seg_res != 0) {
1547                 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1548                                         VIRTIO_MBUF_BURST_SZ);
1549
1550                 if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) {
1551                         virtio_rmb(hw->weak_barriers);
1552                         num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len,
1553                                                            rcv_cnt);
1554                         uint16_t extra_idx = 0;
1555
1556                         rcv_cnt = num;
1557                         while (extra_idx < rcv_cnt) {
1558                                 rxm = rcv_pkts[extra_idx];
1559                                 rxm->data_off =
1560                                         RTE_PKTMBUF_HEADROOM - hdr_size;
1561                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1562                                 rxm->data_len = (uint16_t)(len[extra_idx]);
1563                                 prev->next = rxm;
1564                                 prev = rxm;
1565                                 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1566                                 extra_idx += 1;
1567                         };
1568                         seg_res -= rcv_cnt;
1569
1570                         if (!seg_res) {
1571                                 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1572                                 nb_rx++;
1573                         }
1574                 } else {
1575                         PMD_RX_LOG(ERR,
1576                                         "No enough segments for packet.");
1577                         rte_pktmbuf_free(rx_pkts[nb_rx]);
1578                         rxvq->stats.errors++;
1579                         break;
1580                 }
1581         }
1582
1583         rxvq->stats.packets += nb_rx;
1584
1585         /* Allocate new mbuf for the used descriptor */
1586
1587         if (likely(!virtqueue_full(vq))) {
1588                 /* free_cnt may include mrg descs */
1589                 uint16_t free_cnt = vq->vq_free_cnt;
1590                 struct rte_mbuf *new_pkts[free_cnt];
1591
1592                 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1593                         error = virtqueue_enqueue_refill_inorder(vq, new_pkts,
1594                                         free_cnt);
1595                         if (unlikely(error)) {
1596                                 for (i = 0; i < free_cnt; i++)
1597                                         rte_pktmbuf_free(new_pkts[i]);
1598                         }
1599                         nb_enqueued += free_cnt;
1600                 } else {
1601                         struct rte_eth_dev *dev =
1602                                 &rte_eth_devices[rxvq->port_id];
1603                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1604                 }
1605         }
1606
1607         if (likely(nb_enqueued)) {
1608                 vq_update_avail_idx(vq);
1609
1610                 if (unlikely(virtqueue_kick_prepare(vq))) {
1611                         virtqueue_notify(vq);
1612                         PMD_RX_LOG(DEBUG, "Notified");
1613                 }
1614         }
1615
1616         return nb_rx;
1617 }
1618
1619 uint16_t
1620 virtio_recv_mergeable_pkts(void *rx_queue,
1621                         struct rte_mbuf **rx_pkts,
1622                         uint16_t nb_pkts)
1623 {
1624         struct virtnet_rx *rxvq = rx_queue;
1625         struct virtqueue *vq = rxvq->vq;
1626         struct virtio_hw *hw = vq->hw;
1627         struct rte_mbuf *rxm;
1628         struct rte_mbuf *prev = NULL;
1629         uint16_t nb_used, num, nb_rx = 0;
1630         uint32_t len[VIRTIO_MBUF_BURST_SZ];
1631         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1632         int error;
1633         uint32_t nb_enqueued = 0;
1634         uint32_t seg_num = 0;
1635         uint32_t seg_res = 0;
1636         uint32_t hdr_size = hw->vtnet_hdr_size;
1637         int32_t i;
1638
1639         if (unlikely(hw->started == 0))
1640                 return nb_rx;
1641
1642         nb_used = VIRTQUEUE_NUSED(vq);
1643
1644         virtio_rmb(hw->weak_barriers);
1645
1646         PMD_RX_LOG(DEBUG, "used:%d", nb_used);
1647
1648         num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
1649         if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1650                 num = VIRTIO_MBUF_BURST_SZ;
1651         if (likely(num > DESC_PER_CACHELINE))
1652                 num = num - ((vq->vq_used_cons_idx + num) %
1653                                 DESC_PER_CACHELINE);
1654
1655
1656         num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
1657
1658         for (i = 0; i < num; i++) {
1659                 struct virtio_net_hdr_mrg_rxbuf *header;
1660
1661                 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1662                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1663
1664                 rxm = rcv_pkts[i];
1665
1666                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1667                         PMD_RX_LOG(ERR, "Packet drop");
1668                         nb_enqueued++;
1669                         virtio_discard_rxbuf(vq, rxm);
1670                         rxvq->stats.errors++;
1671                         continue;
1672                 }
1673
1674                 header = (struct virtio_net_hdr_mrg_rxbuf *)
1675                          ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
1676                          - hdr_size);
1677                 seg_num = header->num_buffers;
1678                 if (seg_num == 0)
1679                         seg_num = 1;
1680
1681                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1682                 rxm->nb_segs = seg_num;
1683                 rxm->ol_flags = 0;
1684                 rxm->vlan_tci = 0;
1685                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1686                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1687
1688                 rxm->port = rxvq->port_id;
1689
1690                 rx_pkts[nb_rx] = rxm;
1691                 prev = rxm;
1692
1693                 if (hw->has_rx_offload &&
1694                                 virtio_rx_offload(rxm, &header->hdr) < 0) {
1695                         virtio_discard_rxbuf(vq, rxm);
1696                         rxvq->stats.errors++;
1697                         continue;
1698                 }
1699
1700                 if (hw->vlan_strip)
1701                         rte_vlan_strip(rx_pkts[nb_rx]);
1702
1703                 seg_res = seg_num - 1;
1704
1705                 /* Merge remaining segments */
1706                 while (seg_res != 0 && i < (num - 1)) {
1707                         i++;
1708
1709                         rxm = rcv_pkts[i];
1710                         rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1711                         rxm->pkt_len = (uint32_t)(len[i]);
1712                         rxm->data_len = (uint16_t)(len[i]);
1713
1714                         rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1715
1716                         prev->next = rxm;
1717                         prev = rxm;
1718                         seg_res -= 1;
1719                 }
1720
1721                 if (!seg_res) {
1722                         virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1723                         nb_rx++;
1724                 }
1725         }
1726
1727         /* Last packet still need merge segments */
1728         while (seg_res != 0) {
1729                 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1730                                         VIRTIO_MBUF_BURST_SZ);
1731
1732                 if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) {
1733                         virtio_rmb(hw->weak_barriers);
1734                         num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len,
1735                                                            rcv_cnt);
1736                         uint16_t extra_idx = 0;
1737
1738                         rcv_cnt = num;
1739                         while (extra_idx < rcv_cnt) {
1740                                 rxm = rcv_pkts[extra_idx];
1741                                 rxm->data_off =
1742                                         RTE_PKTMBUF_HEADROOM - hdr_size;
1743                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1744                                 rxm->data_len = (uint16_t)(len[extra_idx]);
1745                                 prev->next = rxm;
1746                                 prev = rxm;
1747                                 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1748                                 extra_idx += 1;
1749                         };
1750                         seg_res -= rcv_cnt;
1751
1752                         if (!seg_res) {
1753                                 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1754                                 nb_rx++;
1755                         }
1756                 } else {
1757                         PMD_RX_LOG(ERR,
1758                                         "No enough segments for packet.");
1759                         rte_pktmbuf_free(rx_pkts[nb_rx]);
1760                         rxvq->stats.errors++;
1761                         break;
1762                 }
1763         }
1764
1765         rxvq->stats.packets += nb_rx;
1766
1767         /* Allocate new mbuf for the used descriptor */
1768         if (likely(!virtqueue_full(vq))) {
1769                 /* free_cnt may include mrg descs */
1770                 uint16_t free_cnt = vq->vq_free_cnt;
1771                 struct rte_mbuf *new_pkts[free_cnt];
1772
1773                 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1774                         error = virtqueue_enqueue_recv_refill(vq, new_pkts,
1775                                         free_cnt);
1776                         if (unlikely(error)) {
1777                                 for (i = 0; i < free_cnt; i++)
1778                                         rte_pktmbuf_free(new_pkts[i]);
1779                         }
1780                         nb_enqueued += free_cnt;
1781                 } else {
1782                         struct rte_eth_dev *dev =
1783                                 &rte_eth_devices[rxvq->port_id];
1784                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1785                 }
1786         }
1787
1788         if (likely(nb_enqueued)) {
1789                 vq_update_avail_idx(vq);
1790
1791                 if (unlikely(virtqueue_kick_prepare(vq))) {
1792                         virtqueue_notify(vq);
1793                         PMD_RX_LOG(DEBUG, "Notified");
1794                 }
1795         }
1796
1797         return nb_rx;
1798 }
1799
1800 uint16_t
1801 virtio_recv_mergeable_pkts_packed(void *rx_queue,
1802                         struct rte_mbuf **rx_pkts,
1803                         uint16_t nb_pkts)
1804 {
1805         struct virtnet_rx *rxvq = rx_queue;
1806         struct virtqueue *vq = rxvq->vq;
1807         struct virtio_hw *hw = vq->hw;
1808         struct rte_mbuf *rxm;
1809         struct rte_mbuf *prev = NULL;
1810         uint16_t num, nb_rx = 0;
1811         uint32_t len[VIRTIO_MBUF_BURST_SZ];
1812         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1813         uint32_t nb_enqueued = 0;
1814         uint32_t seg_num = 0;
1815         uint32_t seg_res = 0;
1816         uint32_t hdr_size = hw->vtnet_hdr_size;
1817         int32_t i;
1818         int error;
1819
1820         if (unlikely(hw->started == 0))
1821                 return nb_rx;
1822
1823
1824         num = nb_pkts;
1825         if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1826                 num = VIRTIO_MBUF_BURST_SZ;
1827         if (likely(num > DESC_PER_CACHELINE))
1828                 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1829
1830         num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
1831
1832         for (i = 0; i < num; i++) {
1833                 struct virtio_net_hdr_mrg_rxbuf *header;
1834
1835                 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1836                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1837
1838                 rxm = rcv_pkts[i];
1839
1840                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1841                         PMD_RX_LOG(ERR, "Packet drop");
1842                         nb_enqueued++;
1843                         virtio_discard_rxbuf(vq, rxm);
1844                         rxvq->stats.errors++;
1845                         continue;
1846                 }
1847
1848                 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)
1849                           rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size);
1850                 seg_num = header->num_buffers;
1851
1852                 if (seg_num == 0)
1853                         seg_num = 1;
1854
1855                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1856                 rxm->nb_segs = seg_num;
1857                 rxm->ol_flags = 0;
1858                 rxm->vlan_tci = 0;
1859                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1860                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1861
1862                 rxm->port = rxvq->port_id;
1863                 rx_pkts[nb_rx] = rxm;
1864                 prev = rxm;
1865
1866                 if (hw->has_rx_offload &&
1867                                 virtio_rx_offload(rxm, &header->hdr) < 0) {
1868                         virtio_discard_rxbuf(vq, rxm);
1869                         rxvq->stats.errors++;
1870                         continue;
1871                 }
1872
1873                 if (hw->vlan_strip)
1874                         rte_vlan_strip(rx_pkts[nb_rx]);
1875
1876                 seg_res = seg_num - 1;
1877
1878                 /* Merge remaining segments */
1879                 while (seg_res != 0 && i < (num - 1)) {
1880                         i++;
1881
1882                         rxm = rcv_pkts[i];
1883                         rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1884                         rxm->pkt_len = (uint32_t)(len[i]);
1885                         rxm->data_len = (uint16_t)(len[i]);
1886
1887                         rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1888
1889                         prev->next = rxm;
1890                         prev = rxm;
1891                         seg_res -= 1;
1892                 }
1893
1894                 if (!seg_res) {
1895                         virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1896                         nb_rx++;
1897                 }
1898         }
1899
1900         /* Last packet still need merge segments */
1901         while (seg_res != 0) {
1902                 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1903                                         VIRTIO_MBUF_BURST_SZ);
1904                 uint16_t extra_idx = 0;
1905
1906                 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts,
1907                                 len, rcv_cnt);
1908                 if (unlikely(rcv_cnt == 0)) {
1909                         PMD_RX_LOG(ERR, "No enough segments for packet.");
1910                         rte_pktmbuf_free(rx_pkts[nb_rx]);
1911                         rxvq->stats.errors++;
1912                         break;
1913                 }
1914
1915                 while (extra_idx < rcv_cnt) {
1916                         rxm = rcv_pkts[extra_idx];
1917
1918                         rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1919                         rxm->pkt_len = (uint32_t)(len[extra_idx]);
1920                         rxm->data_len = (uint16_t)(len[extra_idx]);
1921
1922                         prev->next = rxm;
1923                         prev = rxm;
1924                         rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1925                         extra_idx += 1;
1926                 }
1927                 seg_res -= rcv_cnt;
1928                 if (!seg_res) {
1929                         virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1930                         nb_rx++;
1931                 }
1932         }
1933
1934         rxvq->stats.packets += nb_rx;
1935
1936         /* Allocate new mbuf for the used descriptor */
1937         if (likely(!virtqueue_full(vq))) {
1938                 /* free_cnt may include mrg descs */
1939                 uint16_t free_cnt = vq->vq_free_cnt;
1940                 struct rte_mbuf *new_pkts[free_cnt];
1941
1942                 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1943                         error = virtqueue_enqueue_recv_refill_packed(vq,
1944                                         new_pkts, free_cnt);
1945                         if (unlikely(error)) {
1946                                 for (i = 0; i < free_cnt; i++)
1947                                         rte_pktmbuf_free(new_pkts[i]);
1948                         }
1949                         nb_enqueued += free_cnt;
1950                 } else {
1951                         struct rte_eth_dev *dev =
1952                                 &rte_eth_devices[rxvq->port_id];
1953                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1954                 }
1955         }
1956
1957         if (likely(nb_enqueued)) {
1958                 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1959                         virtqueue_notify(vq);
1960                         PMD_RX_LOG(DEBUG, "Notified");
1961                 }
1962         }
1963
1964         return nb_rx;
1965 }
1966
1967 uint16_t
1968 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
1969                         uint16_t nb_pkts)
1970 {
1971         uint16_t nb_tx;
1972         int error;
1973
1974         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1975                 struct rte_mbuf *m = tx_pkts[nb_tx];
1976
1977 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
1978                 error = rte_validate_tx_offload(m);
1979                 if (unlikely(error)) {
1980                         rte_errno = -error;
1981                         break;
1982                 }
1983 #endif
1984
1985                 /* Do VLAN tag insertion */
1986                 if (unlikely(m->ol_flags & PKT_TX_VLAN_PKT)) {
1987                         error = rte_vlan_insert(&m);
1988                         /* rte_vlan_insert() may change pointer
1989                          * even in the case of failure
1990                          */
1991                         tx_pkts[nb_tx] = m;
1992
1993                         if (unlikely(error)) {
1994                                 rte_errno = -error;
1995                                 break;
1996                         }
1997                 }
1998
1999                 error = rte_net_intel_cksum_prepare(m);
2000                 if (unlikely(error)) {
2001                         rte_errno = -error;
2002                         break;
2003                 }
2004
2005                 if (m->ol_flags & PKT_TX_TCP_SEG)
2006                         virtio_tso_fix_cksum(m);
2007         }
2008
2009         return nb_tx;
2010 }
2011
2012 uint16_t
2013 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts,
2014                         uint16_t nb_pkts)
2015 {
2016         struct virtnet_tx *txvq = tx_queue;
2017         struct virtqueue *vq = txvq->vq;
2018         struct virtio_hw *hw = vq->hw;
2019         uint16_t hdr_size = hw->vtnet_hdr_size;
2020         uint16_t nb_tx = 0;
2021         bool in_order = hw->use_inorder_tx;
2022
2023         if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
2024                 return nb_tx;
2025
2026         if (unlikely(nb_pkts < 1))
2027                 return nb_pkts;
2028
2029         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
2030
2031         if (nb_pkts > vq->vq_free_cnt)
2032                 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt,
2033                                            in_order);
2034
2035         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
2036                 struct rte_mbuf *txm = tx_pkts[nb_tx];
2037                 int can_push = 0, slots, need;
2038
2039                 /* optimize ring usage */
2040                 if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
2041                       vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) &&
2042                     rte_mbuf_refcnt_read(txm) == 1 &&
2043                     RTE_MBUF_DIRECT(txm) &&
2044                     txm->nb_segs == 1 &&
2045                     rte_pktmbuf_headroom(txm) >= hdr_size &&
2046                     rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
2047                            __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
2048                         can_push = 1;
2049
2050                 /* How many main ring entries are needed to this Tx?
2051                  * any_layout => number of segments
2052                  * default    => number of segments + 1
2053                  */
2054                 slots = txm->nb_segs + !can_push;
2055                 need = slots - vq->vq_free_cnt;
2056
2057                 /* Positive value indicates it need free vring descriptors */
2058                 if (unlikely(need > 0)) {
2059                         virtio_xmit_cleanup_packed(vq, need, in_order);
2060                         need = slots - vq->vq_free_cnt;
2061                         if (unlikely(need > 0)) {
2062                                 PMD_TX_LOG(ERR,
2063                                            "No free tx descriptors to transmit");
2064                                 break;
2065                         }
2066                 }
2067
2068                 /* Enqueue Packet buffers */
2069                 if (can_push)
2070                         virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order);
2071                 else
2072                         virtqueue_enqueue_xmit_packed(txvq, txm, slots, 0,
2073                                                       in_order);
2074
2075                 virtio_update_packet_stats(&txvq->stats, txm);
2076         }
2077
2078         txvq->stats.packets += nb_tx;
2079
2080         if (likely(nb_tx)) {
2081                 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
2082                         virtqueue_notify(vq);
2083                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
2084                 }
2085         }
2086
2087         return nb_tx;
2088 }
2089
2090 uint16_t
2091 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2092 {
2093         struct virtnet_tx *txvq = tx_queue;
2094         struct virtqueue *vq = txvq->vq;
2095         struct virtio_hw *hw = vq->hw;
2096         uint16_t hdr_size = hw->vtnet_hdr_size;
2097         uint16_t nb_used, nb_tx = 0;
2098
2099         if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
2100                 return nb_tx;
2101
2102         if (unlikely(nb_pkts < 1))
2103                 return nb_pkts;
2104
2105         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
2106         nb_used = VIRTQUEUE_NUSED(vq);
2107
2108         virtio_rmb(hw->weak_barriers);
2109         if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
2110                 virtio_xmit_cleanup(vq, nb_used);
2111
2112         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
2113                 struct rte_mbuf *txm = tx_pkts[nb_tx];
2114                 int can_push = 0, use_indirect = 0, slots, need;
2115
2116                 /* optimize ring usage */
2117                 if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
2118                       vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) &&
2119                     rte_mbuf_refcnt_read(txm) == 1 &&
2120                     RTE_MBUF_DIRECT(txm) &&
2121                     txm->nb_segs == 1 &&
2122                     rte_pktmbuf_headroom(txm) >= hdr_size &&
2123                     rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
2124                                    __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
2125                         can_push = 1;
2126                 else if (vtpci_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
2127                          txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
2128                         use_indirect = 1;
2129
2130                 /* How many main ring entries are needed to this Tx?
2131                  * any_layout => number of segments
2132                  * indirect   => 1
2133                  * default    => number of segments + 1
2134                  */
2135                 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
2136                 need = slots - vq->vq_free_cnt;
2137
2138                 /* Positive value indicates it need free vring descriptors */
2139                 if (unlikely(need > 0)) {
2140                         nb_used = VIRTQUEUE_NUSED(vq);
2141                         virtio_rmb(hw->weak_barriers);
2142                         need = RTE_MIN(need, (int)nb_used);
2143
2144                         virtio_xmit_cleanup(vq, need);
2145                         need = slots - vq->vq_free_cnt;
2146                         if (unlikely(need > 0)) {
2147                                 PMD_TX_LOG(ERR,
2148                                            "No free tx descriptors to transmit");
2149                                 break;
2150                         }
2151                 }
2152
2153                 /* Enqueue Packet buffers */
2154                 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect,
2155                         can_push, 0);
2156
2157                 virtio_update_packet_stats(&txvq->stats, txm);
2158         }
2159
2160         txvq->stats.packets += nb_tx;
2161
2162         if (likely(nb_tx)) {
2163                 vq_update_avail_idx(vq);
2164
2165                 if (unlikely(virtqueue_kick_prepare(vq))) {
2166                         virtqueue_notify(vq);
2167                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
2168                 }
2169         }
2170
2171         return nb_tx;
2172 }
2173
2174 static __rte_always_inline int
2175 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need)
2176 {
2177         uint16_t nb_used, nb_clean, nb_descs;
2178         struct virtio_hw *hw = vq->hw;
2179
2180         nb_descs = vq->vq_free_cnt + need;
2181         nb_used = VIRTQUEUE_NUSED(vq);
2182         virtio_rmb(hw->weak_barriers);
2183         nb_clean = RTE_MIN(need, (int)nb_used);
2184
2185         virtio_xmit_cleanup_inorder(vq, nb_clean);
2186
2187         return nb_descs - vq->vq_free_cnt;
2188 }
2189
2190 uint16_t
2191 virtio_xmit_pkts_inorder(void *tx_queue,
2192                         struct rte_mbuf **tx_pkts,
2193                         uint16_t nb_pkts)
2194 {
2195         struct virtnet_tx *txvq = tx_queue;
2196         struct virtqueue *vq = txvq->vq;
2197         struct virtio_hw *hw = vq->hw;
2198         uint16_t hdr_size = hw->vtnet_hdr_size;
2199         uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0;
2200         struct rte_mbuf *inorder_pkts[nb_pkts];
2201         int need;
2202
2203         if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
2204                 return nb_tx;
2205
2206         if (unlikely(nb_pkts < 1))
2207                 return nb_pkts;
2208
2209         VIRTQUEUE_DUMP(vq);
2210         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
2211         nb_used = VIRTQUEUE_NUSED(vq);
2212
2213         virtio_rmb(hw->weak_barriers);
2214         if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
2215                 virtio_xmit_cleanup_inorder(vq, nb_used);
2216
2217         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
2218                 struct rte_mbuf *txm = tx_pkts[nb_tx];
2219                 int slots;
2220
2221                 /* optimize ring usage */
2222                 if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
2223                      vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) &&
2224                      rte_mbuf_refcnt_read(txm) == 1 &&
2225                      RTE_MBUF_DIRECT(txm) &&
2226                      txm->nb_segs == 1 &&
2227                      rte_pktmbuf_headroom(txm) >= hdr_size &&
2228                      rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
2229                                 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) {
2230                         inorder_pkts[nb_inorder_pkts] = txm;
2231                         nb_inorder_pkts++;
2232
2233                         continue;
2234                 }
2235
2236                 if (nb_inorder_pkts) {
2237                         need = nb_inorder_pkts - vq->vq_free_cnt;
2238                         if (unlikely(need > 0)) {
2239                                 need = virtio_xmit_try_cleanup_inorder(vq,
2240                                                                        need);
2241                                 if (unlikely(need > 0)) {
2242                                         PMD_TX_LOG(ERR,
2243                                                 "No free tx descriptors to "
2244                                                 "transmit");
2245                                         break;
2246                                 }
2247                         }
2248                         virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
2249                                                         nb_inorder_pkts);
2250                         nb_inorder_pkts = 0;
2251                 }
2252
2253                 slots = txm->nb_segs + 1;
2254                 need = slots - vq->vq_free_cnt;
2255                 if (unlikely(need > 0)) {
2256                         need = virtio_xmit_try_cleanup_inorder(vq, slots);
2257
2258                         if (unlikely(need > 0)) {
2259                                 PMD_TX_LOG(ERR,
2260                                         "No free tx descriptors to transmit");
2261                                 break;
2262                         }
2263                 }
2264                 /* Enqueue Packet buffers */
2265                 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1);
2266
2267                 virtio_update_packet_stats(&txvq->stats, txm);
2268         }
2269
2270         /* Transmit all inorder packets */
2271         if (nb_inorder_pkts) {
2272                 need = nb_inorder_pkts - vq->vq_free_cnt;
2273                 if (unlikely(need > 0)) {
2274                         need = virtio_xmit_try_cleanup_inorder(vq,
2275                                                                   need);
2276                         if (unlikely(need > 0)) {
2277                                 PMD_TX_LOG(ERR,
2278                                         "No free tx descriptors to transmit");
2279                                 nb_inorder_pkts = vq->vq_free_cnt;
2280                                 nb_tx -= need;
2281                         }
2282                 }
2283
2284                 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
2285                                                 nb_inorder_pkts);
2286         }
2287
2288         txvq->stats.packets += nb_tx;
2289
2290         if (likely(nb_tx)) {
2291                 vq_update_avail_idx(vq);
2292
2293                 if (unlikely(virtqueue_kick_prepare(vq))) {
2294                         virtqueue_notify(vq);
2295                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
2296                 }
2297         }
2298
2299         VIRTQUEUE_DUMP(vq);
2300
2301         return nb_tx;
2302 }