drivers/net/virtio/virtio_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <errno.h>
  10
  11 #include <rte_cycles.h>
  12 #include <rte_memory.h>
  13 #include <rte_branch_prediction.h>
  14 #include <rte_mempool.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_ether.h>
  18 #include <ethdev_driver.h>
  19 #include <rte_prefetch.h>
  20 #include <rte_string_fns.h>
  21 #include <rte_errno.h>
  22 #include <rte_byteorder.h>
  23 #include <rte_net.h>
  24 #include <rte_ip.h>
  25 #include <rte_udp.h>
  26 #include <rte_tcp.h>
  27
  28 #include "virtio_logs.h"
  29 #include "virtio_ethdev.h"
  30 #include "virtio.h"
  31 #include "virtqueue.h"
  32 #include "virtio_rxtx.h"
  33 #include "virtio_rxtx_simple.h"
  34 #include "virtio_ring.h"
  35
  36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
  37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
  38 #else
  39 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
  40 #endif
  41
  42 int
  43 virtio_dev_rx_queue_done(void *rxq, uint16_t offset)
  44 {
  45         struct virtnet_rx *rxvq = rxq;
  46         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
  47
  48         return virtqueue_nused(vq) >= offset;
  49 }
  50
  51 void
  52 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num)
  53 {
  54         vq->vq_free_cnt += num;
  55         vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1);
  56 }
  57
  58 void
  59 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
  60 {
  61         struct vring_desc *dp, *dp_tail;
  62         struct vq_desc_extra *dxp;
  63         uint16_t desc_idx_last = desc_idx;
  64
  65         dp  = &vq->vq_split.ring.desc[desc_idx];
  66         dxp = &vq->vq_descx[desc_idx];
  67         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
  68         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
  69                 while (dp->flags & VRING_DESC_F_NEXT) {
  70                         desc_idx_last = dp->next;
  71                         dp = &vq->vq_split.ring.desc[dp->next];
  72                 }
  73         }
  74         dxp->ndescs = 0;
  75
  76         /*
  77          * We must append the existing free chain, if any, to the end of
  78          * newly freed chain. If the virtqueue was completely used, then
  79          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
  80          */
  81         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
  82                 vq->vq_desc_head_idx = desc_idx;
  83         } else {
  84                 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx];
  85                 dp_tail->next = desc_idx;
  86         }
  87
  88         vq->vq_desc_tail_idx = desc_idx_last;
  89         dp->next = VQ_RING_DESC_CHAIN_END;
  90 }
  91
  92 void
  93 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
  94 {
  95         uint32_t s = mbuf->pkt_len;
  96         struct rte_ether_addr *ea;
  97
  98         stats->bytes += s;
  99
 100         if (s == 64) {
 101                 stats->size_bins[1]++;
 102         } else if (s > 64 && s < 1024) {
 103                 uint32_t bin;
 104
 105                 /* count zeros, and offset into correct bin */
 106                 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
 107                 stats->size_bins[bin]++;
 108         } else {
 109                 if (s < 64)
 110                         stats->size_bins[0]++;
 111                 else if (s < 1519)
 112                         stats->size_bins[6]++;
 113                 else
 114                         stats->size_bins[7]++;
 115         }
 116
 117         ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *);
 118         if (rte_is_multicast_ether_addr(ea)) {
 119                 if (rte_is_broadcast_ether_addr(ea))
 120                         stats->broadcast++;
 121                 else
 122                         stats->multicast++;
 123         }
 124 }
 125
 126 static inline void
 127 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m)
 128 {
 129         VIRTIO_DUMP_PACKET(m, m->data_len);
 130
 131         virtio_update_packet_stats(&rxvq->stats, m);
 132 }
 133
 134 static uint16_t
 135 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq,
 136                                   struct rte_mbuf **rx_pkts,
 137                                   uint32_t *len,
 138                                   uint16_t num)
 139 {
 140         struct rte_mbuf *cookie;
 141         uint16_t used_idx;
 142         uint16_t id;
 143         struct vring_packed_desc *desc;
 144         uint16_t i;
 145
 146         desc = vq->vq_packed.ring.desc;
 147
 148         for (i = 0; i < num; i++) {
 149                 used_idx = vq->vq_used_cons_idx;
 150                 /* desc_is_used has a load-acquire or rte_io_rmb inside
 151                  * and wait for used desc in virtqueue.
 152                  */
 153                 if (!desc_is_used(&desc[used_idx], vq))
 154                         return i;
 155                 len[i] = desc[used_idx].len;
 156                 id = desc[used_idx].id;
 157                 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
 158                 if (unlikely(cookie == NULL)) {
 159                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
 160                                 vq->vq_used_cons_idx);
 161                         break;
 162                 }
 163                 rte_prefetch0(cookie);
 164                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 165                 rx_pkts[i] = cookie;
 166
 167                 vq->vq_free_cnt++;
 168                 vq->vq_used_cons_idx++;
 169                 if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 170                         vq->vq_used_cons_idx -= vq->vq_nentries;
 171                         vq->vq_packed.used_wrap_counter ^= 1;
 172                 }
 173         }
 174
 175         return i;
 176 }
 177
 178 static uint16_t
 179 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 180                            uint32_t *len, uint16_t num)
 181 {
 182         struct vring_used_elem *uep;
 183         struct rte_mbuf *cookie;
 184         uint16_t used_idx, desc_idx;
 185         uint16_t i;
 186
 187         /*  Caller does the check */
 188         for (i = 0; i < num ; i++) {
 189                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 190                 uep = &vq->vq_split.ring.used->ring[used_idx];
 191                 desc_idx = (uint16_t) uep->id;
 192                 len[i] = uep->len;
 193                 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
 194
 195                 if (unlikely(cookie == NULL)) {
 196                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
 197                                 vq->vq_used_cons_idx);
 198                         break;
 199                 }
 200
 201                 rte_prefetch0(cookie);
 202                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 203                 rx_pkts[i]  = cookie;
 204                 vq->vq_used_cons_idx++;
 205                 vq_ring_free_chain(vq, desc_idx);
 206                 vq->vq_descx[desc_idx].cookie = NULL;
 207         }
 208
 209         return i;
 210 }
 211
 212 static uint16_t
 213 virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
 214                         struct rte_mbuf **rx_pkts,
 215                         uint32_t *len,
 216                         uint16_t num)
 217 {
 218         struct vring_used_elem *uep;
 219         struct rte_mbuf *cookie;
 220         uint16_t used_idx = 0;
 221         uint16_t i;
 222
 223         if (unlikely(num == 0))
 224                 return 0;
 225
 226         for (i = 0; i < num; i++) {
 227                 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1);
 228                 /* Desc idx same as used idx */
 229                 uep = &vq->vq_split.ring.used->ring[used_idx];
 230                 len[i] = uep->len;
 231                 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie;
 232
 233                 if (unlikely(cookie == NULL)) {
 234                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
 235                                 vq->vq_used_cons_idx);
 236                         break;
 237                 }
 238
 239                 rte_prefetch0(cookie);
 240                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 241                 rx_pkts[i]  = cookie;
 242                 vq->vq_used_cons_idx++;
 243                 vq->vq_descx[used_idx].cookie = NULL;
 244         }
 245
 246         vq_ring_free_inorder(vq, used_idx, i);
 247         return i;
 248 }
 249
 250 static inline int
 251 virtqueue_enqueue_refill_inorder(struct virtqueue *vq,
 252                         struct rte_mbuf **cookies,
 253                         uint16_t num)
 254 {
 255         struct vq_desc_extra *dxp;
 256         struct virtio_hw *hw = vq->hw;
 257         struct vring_desc *start_dp;
 258         uint16_t head_idx, idx, i = 0;
 259
 260         if (unlikely(vq->vq_free_cnt == 0))
 261                 return -ENOSPC;
 262         if (unlikely(vq->vq_free_cnt < num))
 263                 return -EMSGSIZE;
 264
 265         head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1);
 266         start_dp = vq->vq_split.ring.desc;
 267
 268         while (i < num) {
 269                 idx = head_idx & (vq->vq_nentries - 1);
 270                 dxp = &vq->vq_descx[idx];
 271                 dxp->cookie = (void *)cookies[i];
 272                 dxp->ndescs = 1;
 273
 274                 start_dp[idx].addr = cookies[i]->buf_iova +
 275                         RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
 276                 start_dp[idx].len = cookies[i]->buf_len -
 277                         RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
 278                 start_dp[idx].flags =  VRING_DESC_F_WRITE;
 279
 280                 vq_update_avail_ring(vq, idx);
 281                 head_idx++;
 282                 i++;
 283         }
 284
 285         vq->vq_desc_head_idx += num;
 286         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 287         return 0;
 288 }
 289
 290 static inline int
 291 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie,
 292                                 uint16_t num)
 293 {
 294         struct vq_desc_extra *dxp;
 295         struct virtio_hw *hw = vq->hw;
 296         struct vring_desc *start_dp = vq->vq_split.ring.desc;
 297         uint16_t idx, i;
 298
 299         if (unlikely(vq->vq_free_cnt == 0))
 300                 return -ENOSPC;
 301         if (unlikely(vq->vq_free_cnt < num))
 302                 return -EMSGSIZE;
 303
 304         if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries))
 305                 return -EFAULT;
 306
 307         for (i = 0; i < num; i++) {
 308                 idx = vq->vq_desc_head_idx;
 309                 dxp = &vq->vq_descx[idx];
 310                 dxp->cookie = (void *)cookie[i];
 311                 dxp->ndescs = 1;
 312
 313                 start_dp[idx].addr = cookie[i]->buf_iova +
 314                         RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
 315                 start_dp[idx].len = cookie[i]->buf_len -
 316                         RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
 317                 start_dp[idx].flags = VRING_DESC_F_WRITE;
 318                 vq->vq_desc_head_idx = start_dp[idx].next;
 319                 vq_update_avail_ring(vq, idx);
 320                 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) {
 321                         vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
 322                         break;
 323                 }
 324         }
 325
 326         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 327
 328         return 0;
 329 }
 330
 331 static inline void
 332 virtqueue_refill_single_packed(struct virtqueue *vq,
 333                                struct vring_packed_desc *dp,
 334                                struct rte_mbuf *cookie)
 335 {
 336         uint16_t flags = vq->vq_packed.cached_flags;
 337         struct virtio_hw *hw = vq->hw;
 338
 339         dp->addr = cookie->buf_iova +
 340                         RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
 341         dp->len = cookie->buf_len -
 342                 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
 343
 344         virtqueue_store_flags_packed(dp, flags,
 345                                      hw->weak_barriers);
 346
 347         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 348                 vq->vq_avail_idx -= vq->vq_nentries;
 349                 vq->vq_packed.cached_flags ^=
 350                         VRING_PACKED_DESC_F_AVAIL_USED;
 351                 flags = vq->vq_packed.cached_flags;
 352         }
 353 }
 354
 355 static inline int
 356 virtqueue_enqueue_recv_refill_packed_init(struct virtqueue *vq,
 357                                      struct rte_mbuf **cookie, uint16_t num)
 358 {
 359         struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
 360         struct vq_desc_extra *dxp;
 361         uint16_t idx;
 362         int i;
 363
 364         if (unlikely(vq->vq_free_cnt == 0))
 365                 return -ENOSPC;
 366         if (unlikely(vq->vq_free_cnt < num))
 367                 return -EMSGSIZE;
 368
 369         for (i = 0; i < num; i++) {
 370                 idx = vq->vq_avail_idx;
 371                 dxp = &vq->vq_descx[idx];
 372                 dxp->cookie = (void *)cookie[i];
 373                 dxp->ndescs = 1;
 374
 375                 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]);
 376         }
 377         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 378         return 0;
 379 }
 380
 381 static inline int
 382 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
 383                                      struct rte_mbuf **cookie, uint16_t num)
 384 {
 385         struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
 386         struct vq_desc_extra *dxp;
 387         uint16_t idx, did;
 388         int i;
 389
 390         if (unlikely(vq->vq_free_cnt == 0))
 391                 return -ENOSPC;
 392         if (unlikely(vq->vq_free_cnt < num))
 393                 return -EMSGSIZE;
 394
 395         for (i = 0; i < num; i++) {
 396                 idx = vq->vq_avail_idx;
 397                 did = start_dp[idx].id;
 398                 dxp = &vq->vq_descx[did];
 399                 dxp->cookie = (void *)cookie[i];
 400                 dxp->ndescs = 1;
 401
 402                 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]);
 403         }
 404         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 405         return 0;
 406 }
 407
 408 /* When doing TSO, the IP length is not included in the pseudo header
 409  * checksum of the packet given to the PMD, but for virtio it is
 410  * expected.
 411  */
 412 static void
 413 virtio_tso_fix_cksum(struct rte_mbuf *m)
 414 {
 415         /* common case: header is not fragmented */
 416         if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len +
 417                         m->l4_len)) {
 418                 struct rte_ipv4_hdr *iph;
 419                 struct rte_ipv6_hdr *ip6h;
 420                 struct rte_tcp_hdr *th;
 421                 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen;
 422                 uint32_t tmp;
 423
 424                 iph = rte_pktmbuf_mtod_offset(m,
 425                                         struct rte_ipv4_hdr *, m->l2_len);
 426                 th = RTE_PTR_ADD(iph, m->l3_len);
 427                 if ((iph->version_ihl >> 4) == 4) {
 428                         iph->hdr_checksum = 0;
 429                         iph->hdr_checksum = rte_ipv4_cksum(iph);
 430                         ip_len = iph->total_length;
 431                         ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) -
 432                                 m->l3_len);
 433                 } else {
 434                         ip6h = (struct rte_ipv6_hdr *)iph;
 435                         ip_paylen = ip6h->payload_len;
 436                 }
 437
 438                 /* calculate the new phdr checksum not including ip_paylen */
 439                 prev_cksum = th->cksum;
 440                 tmp = prev_cksum;
 441                 tmp += ip_paylen;
 442                 tmp = (tmp & 0xffff) + (tmp >> 16);
 443                 new_cksum = tmp;
 444
 445                 /* replace it in the packet */
 446                 th->cksum = new_cksum;
 447         }
 448 }
 449
 450
 451
 452
 453 static inline void
 454 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq,
 455                         struct rte_mbuf **cookies,
 456                         uint16_t num)
 457 {
 458         struct vq_desc_extra *dxp;
 459         struct virtqueue *vq = virtnet_txq_to_vq(txvq);
 460         struct vring_desc *start_dp;
 461         struct virtio_net_hdr *hdr;
 462         uint16_t idx;
 463         int16_t head_size = vq->hw->vtnet_hdr_size;
 464         uint16_t i = 0;
 465
 466         idx = vq->vq_desc_head_idx;
 467         start_dp = vq->vq_split.ring.desc;
 468
 469         while (i < num) {
 470                 idx = idx & (vq->vq_nentries - 1);
 471                 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)];
 472                 dxp->cookie = (void *)cookies[i];
 473                 dxp->ndescs = 1;
 474                 virtio_update_packet_stats(&txvq->stats, cookies[i]);
 475
 476                 hdr = rte_pktmbuf_mtod_offset(cookies[i],
 477                                 struct virtio_net_hdr *, -head_size);
 478
 479                 /* if offload disabled, hdr is not zeroed yet, do it now */
 480                 if (!vq->hw->has_tx_offload)
 481                         virtqueue_clear_net_hdr(hdr);
 482                 else
 483                         virtqueue_xmit_offload(hdr, cookies[i]);
 484
 485                 start_dp[idx].addr  = rte_mbuf_data_iova(cookies[i]) - head_size;
 486                 start_dp[idx].len   = cookies[i]->data_len + head_size;
 487                 start_dp[idx].flags = 0;
 488
 489
 490                 vq_update_avail_ring(vq, idx);
 491
 492                 idx++;
 493                 i++;
 494         };
 495
 496         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 497         vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1);
 498 }
 499
 500 static inline void
 501 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq,
 502                                    struct rte_mbuf *cookie,
 503                                    int in_order)
 504 {
 505         struct virtqueue *vq = virtnet_txq_to_vq(txvq);
 506         struct vring_packed_desc *dp;
 507         struct vq_desc_extra *dxp;
 508         uint16_t idx, id, flags;
 509         int16_t head_size = vq->hw->vtnet_hdr_size;
 510         struct virtio_net_hdr *hdr;
 511
 512         id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
 513         idx = vq->vq_avail_idx;
 514         dp = &vq->vq_packed.ring.desc[idx];
 515
 516         dxp = &vq->vq_descx[id];
 517         dxp->ndescs = 1;
 518         dxp->cookie = cookie;
 519
 520         flags = vq->vq_packed.cached_flags;
 521
 522         /* prepend cannot fail, checked by caller */
 523         hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *,
 524                                       -head_size);
 525
 526         /* if offload disabled, hdr is not zeroed yet, do it now */
 527         if (!vq->hw->has_tx_offload)
 528                 virtqueue_clear_net_hdr(hdr);
 529         else
 530                 virtqueue_xmit_offload(hdr, cookie);
 531
 532         dp->addr = rte_mbuf_data_iova(cookie) - head_size;
 533         dp->len  = cookie->data_len + head_size;
 534         dp->id   = id;
 535
 536         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 537                 vq->vq_avail_idx -= vq->vq_nentries;
 538                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 539         }
 540
 541         vq->vq_free_cnt--;
 542
 543         if (!in_order) {
 544                 vq->vq_desc_head_idx = dxp->next;
 545                 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 546                         vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 547         }
 548
 549         virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers);
 550 }
 551
 552 static inline void
 553 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 554                         uint16_t needed, int use_indirect, int can_push,
 555                         int in_order)
 556 {
 557         struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
 558         struct vq_desc_extra *dxp;
 559         struct virtqueue *vq = virtnet_txq_to_vq(txvq);
 560         struct vring_desc *start_dp;
 561         uint16_t seg_num = cookie->nb_segs;
 562         uint16_t head_idx, idx;
 563         int16_t head_size = vq->hw->vtnet_hdr_size;
 564         bool prepend_header = false;
 565         struct virtio_net_hdr *hdr;
 566
 567         head_idx = vq->vq_desc_head_idx;
 568         idx = head_idx;
 569         if (in_order)
 570                 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)];
 571         else
 572                 dxp = &vq->vq_descx[idx];
 573         dxp->cookie = (void *)cookie;
 574         dxp->ndescs = needed;
 575
 576         start_dp = vq->vq_split.ring.desc;
 577
 578         if (can_push) {
 579                 /* prepend cannot fail, checked by caller */
 580                 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *,
 581                                               -head_size);
 582                 prepend_header = true;
 583
 584                 /* if offload disabled, it is not zeroed below, do it now */
 585                 if (!vq->hw->has_tx_offload)
 586                         virtqueue_clear_net_hdr(hdr);
 587         } else if (use_indirect) {
 588                 /* setup tx ring slot to point to indirect
 589                  * descriptor list stored in reserved region.
 590                  *
 591                  * the first slot in indirect ring is already preset
 592                  * to point to the header in reserved region
 593                  */
 594                 start_dp[idx].addr  = txvq->virtio_net_hdr_mem +
 595                         RTE_PTR_DIFF(&txr[idx].tx_indir, txr);
 596                 start_dp[idx].len   = (seg_num + 1) * sizeof(struct vring_desc);
 597                 start_dp[idx].flags = VRING_DESC_F_INDIRECT;
 598                 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
 599
 600                 /* loop below will fill in rest of the indirect elements */
 601                 start_dp = txr[idx].tx_indir;
 602                 idx = 1;
 603         } else {
 604                 /* setup first tx ring slot to point to header
 605                  * stored in reserved region.
 606                  */
 607                 start_dp[idx].addr  = txvq->virtio_net_hdr_mem +
 608                         RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);
 609                 start_dp[idx].len   = vq->hw->vtnet_hdr_size;
 610                 start_dp[idx].flags = VRING_DESC_F_NEXT;
 611                 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
 612
 613                 idx = start_dp[idx].next;
 614         }
 615
 616         if (vq->hw->has_tx_offload)
 617                 virtqueue_xmit_offload(hdr, cookie);
 618
 619         do {
 620                 start_dp[idx].addr  = rte_mbuf_data_iova(cookie);
 621                 start_dp[idx].len   = cookie->data_len;
 622                 if (prepend_header) {
 623                         start_dp[idx].addr -= head_size;
 624                         start_dp[idx].len += head_size;
 625                         prepend_header = false;
 626                 }
 627                 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0;
 628                 idx = start_dp[idx].next;
 629         } while ((cookie = cookie->next) != NULL);
 630
 631         if (use_indirect)
 632                 idx = vq->vq_split.ring.desc[head_idx].next;
 633
 634         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
 635
 636         vq->vq_desc_head_idx = idx;
 637         vq_update_avail_ring(vq, head_idx);
 638
 639         if (!in_order) {
 640                 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 641                         vq->vq_desc_tail_idx = idx;
 642         }
 643 }
 644
 645 void
 646 virtio_dev_cq_start(struct rte_eth_dev *dev)
 647 {
 648         struct virtio_hw *hw = dev->data->dev_private;
 649
 650         if (hw->cvq) {
 651                 rte_spinlock_init(&hw->cvq->lock);
 652                 VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq));
 653         }
 654 }
 655
 656 int
 657 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 658                         uint16_t queue_idx,
 659                         uint16_t nb_desc,
 660                         unsigned int socket_id __rte_unused,
 661                         const struct rte_eth_rxconf *rx_conf,
 662                         struct rte_mempool *mp)
 663 {
 664         uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
 665         struct virtio_hw *hw = dev->data->dev_private;
 666         struct virtqueue *vq = hw->vqs[vq_idx];
 667         struct virtnet_rx *rxvq;
 668         uint16_t rx_free_thresh;
 669         uint16_t buf_size;
 670         const char *error;
 671
 672         PMD_INIT_FUNC_TRACE();
 673
 674         if (rx_conf->rx_deferred_start) {
 675                 PMD_INIT_LOG(ERR, "Rx deferred start is not supported");
 676                 return -EINVAL;
 677         }
 678
 679         buf_size = virtio_rx_mem_pool_buf_size(mp);
 680         if (!virtio_rx_check_scatter(hw->max_rx_pkt_len, buf_size,
 681                                      hw->rx_ol_scatter, &error)) {
 682                 PMD_INIT_LOG(ERR, "RxQ %u Rx scatter check failed: %s",
 683                              queue_idx, error);
 684                 return -EINVAL;
 685         }
 686
 687         rx_free_thresh = rx_conf->rx_free_thresh;
 688         if (rx_free_thresh == 0)
 689                 rx_free_thresh =
 690                         RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH);
 691
 692         if (rx_free_thresh & 0x3) {
 693                 RTE_LOG(ERR, PMD, "rx_free_thresh must be multiples of four."
 694                         " (rx_free_thresh=%u port=%u queue=%u)\n",
 695                         rx_free_thresh, dev->data->port_id, queue_idx);
 696                 return -EINVAL;
 697         }
 698
 699         if (rx_free_thresh >= vq->vq_nentries) {
 700                 RTE_LOG(ERR, PMD, "rx_free_thresh must be less than the "
 701                         "number of RX entries (%u)."
 702                         " (rx_free_thresh=%u port=%u queue=%u)\n",
 703                         vq->vq_nentries,
 704                         rx_free_thresh, dev->data->port_id, queue_idx);
 705                 return -EINVAL;
 706         }
 707         vq->vq_free_thresh = rx_free_thresh;
 708
 709         if (nb_desc > vq->vq_nentries)
 710                 nb_desc = vq->vq_nentries;
 711         vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
 712
 713         rxvq = &vq->rxq;
 714         rxvq->queue_id = queue_idx;
 715         rxvq->mpool = mp;
 716         dev->data->rx_queues[queue_idx] = rxvq;
 717
 718         return 0;
 719 }
 720
 721 int
 722 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx)
 723 {
 724         uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
 725         struct virtio_hw *hw = dev->data->dev_private;
 726         struct virtqueue *vq = hw->vqs[vq_idx];
 727         struct virtnet_rx *rxvq = &vq->rxq;
 728         struct rte_mbuf *m;
 729         uint16_t desc_idx;
 730         int error, nbufs, i;
 731         bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER);
 732
 733         PMD_INIT_FUNC_TRACE();
 734
 735         /* Allocate blank mbufs for the each rx descriptor */
 736         nbufs = 0;
 737
 738         if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) {
 739                 for (desc_idx = 0; desc_idx < vq->vq_nentries;
 740                      desc_idx++) {
 741                         vq->vq_split.ring.avail->ring[desc_idx] = desc_idx;
 742                         vq->vq_split.ring.desc[desc_idx].flags =
 743                                 VRING_DESC_F_WRITE;
 744                 }
 745
 746                 virtio_rxq_vec_setup(rxvq);
 747         }
 748
 749         memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf));
 750         for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++)
 751                 vq->sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf;
 752
 753         if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) {
 754                 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
 755                         virtio_rxq_rearm_vec(rxvq);
 756                         nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
 757                 }
 758         } else if (!virtio_with_packed_queue(vq->hw) && in_order) {
 759                 if ((!virtqueue_full(vq))) {
 760                         uint16_t free_cnt = vq->vq_free_cnt;
 761                         struct rte_mbuf *pkts[free_cnt];
 762
 763                         if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts,
 764                                 free_cnt)) {
 765                                 error = virtqueue_enqueue_refill_inorder(vq,
 766                                                 pkts,
 767                                                 free_cnt);
 768                                 if (unlikely(error)) {
 769                                         for (i = 0; i < free_cnt; i++)
 770                                                 rte_pktmbuf_free(pkts[i]);
 771                                 } else {
 772                                         nbufs += free_cnt;
 773                                 }
 774                         }
 775
 776                         vq_update_avail_idx(vq);
 777                 }
 778         } else {
 779                 while (!virtqueue_full(vq)) {
 780                         m = rte_mbuf_raw_alloc(rxvq->mpool);
 781                         if (m == NULL)
 782                                 break;
 783
 784                         /* Enqueue allocated buffers */
 785                         if (virtio_with_packed_queue(vq->hw))
 786                                 error = virtqueue_enqueue_recv_refill_packed_init(vq,
 787                                                 &m, 1);
 788                         else
 789                                 error = virtqueue_enqueue_recv_refill(vq,
 790                                                 &m, 1);
 791                         if (error) {
 792                                 rte_pktmbuf_free(m);
 793                                 break;
 794                         }
 795                         nbufs++;
 796                 }
 797
 798                 if (!virtio_with_packed_queue(vq->hw))
 799                         vq_update_avail_idx(vq);
 800         }
 801
 802         PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
 803
 804         VIRTQUEUE_DUMP(vq);
 805
 806         return 0;
 807 }
 808
 809 /*
 810  * struct rte_eth_dev *dev: Used to update dev
 811  * uint16_t nb_desc: Defaults to values read from config space
 812  * unsigned int socket_id: Used to allocate memzone
 813  * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
 814  * uint16_t queue_idx: Just used as an index in dev txq list
 815  */
 816 int
 817 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 818                         uint16_t queue_idx,
 819                         uint16_t nb_desc,
 820                         unsigned int socket_id __rte_unused,
 821                         const struct rte_eth_txconf *tx_conf)
 822 {
 823         uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
 824         struct virtio_hw *hw = dev->data->dev_private;
 825         struct virtqueue *vq = hw->vqs[vq_idx];
 826         struct virtnet_tx *txvq;
 827         uint16_t tx_free_thresh;
 828
 829         PMD_INIT_FUNC_TRACE();
 830
 831         if (tx_conf->tx_deferred_start) {
 832                 PMD_INIT_LOG(ERR, "Tx deferred start is not supported");
 833                 return -EINVAL;
 834         }
 835
 836         if (nb_desc == 0 || nb_desc > vq->vq_nentries)
 837                 nb_desc = vq->vq_nentries;
 838         vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
 839
 840         txvq = &vq->txq;
 841         txvq->queue_id = queue_idx;
 842
 843         tx_free_thresh = tx_conf->tx_free_thresh;
 844         if (tx_free_thresh == 0)
 845                 tx_free_thresh =
 846                         RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
 847
 848         if (tx_free_thresh >= (vq->vq_nentries - 3)) {
 849                 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the "
 850                         "number of TX entries minus 3 (%u)."
 851                         " (tx_free_thresh=%u port=%u queue=%u)\n",
 852                         vq->vq_nentries - 3,
 853                         tx_free_thresh, dev->data->port_id, queue_idx);
 854                 return -EINVAL;
 855         }
 856
 857         vq->vq_free_thresh = tx_free_thresh;
 858
 859         dev->data->tx_queues[queue_idx] = txvq;
 860         return 0;
 861 }
 862
 863 int
 864 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
 865                                 uint16_t queue_idx)
 866 {
 867         uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
 868         struct virtio_hw *hw = dev->data->dev_private;
 869         struct virtqueue *vq = hw->vqs[vq_idx];
 870
 871         PMD_INIT_FUNC_TRACE();
 872
 873         if (!virtio_with_packed_queue(hw)) {
 874                 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER))
 875                         vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0;
 876         }
 877
 878         VIRTQUEUE_DUMP(vq);
 879
 880         return 0;
 881 }
 882
 883 static inline void
 884 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
 885 {
 886         int error;
 887         /*
 888          * Requeue the discarded mbuf. This should always be
 889          * successful since it was just dequeued.
 890          */
 891         if (virtio_with_packed_queue(vq->hw))
 892                 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1);
 893         else
 894                 error = virtqueue_enqueue_recv_refill(vq, &m, 1);
 895
 896         if (unlikely(error)) {
 897                 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf");
 898                 rte_pktmbuf_free(m);
 899         }
 900 }
 901
 902 static inline void
 903 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m)
 904 {
 905         int error;
 906
 907         error = virtqueue_enqueue_refill_inorder(vq, &m, 1);
 908         if (unlikely(error)) {
 909                 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf");
 910                 rte_pktmbuf_free(m);
 911         }
 912 }
 913
 914 /* Optionally fill offload information in structure */
 915 static inline int
 916 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
 917 {
 918         struct rte_net_hdr_lens hdr_lens;
 919         uint32_t hdrlen, ptype;
 920         int l4_supported = 0;
 921
 922         /* nothing to do */
 923         if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
 924                 return 0;
 925
 926         m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
 927
 928         ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK);
 929         m->packet_type = ptype;
 930         if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP ||
 931             (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP ||
 932             (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP)
 933                 l4_supported = 1;
 934
 935         if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
 936                 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len;
 937                 if (hdr->csum_start <= hdrlen && l4_supported) {
 938                         m->ol_flags |= PKT_RX_L4_CKSUM_NONE;
 939                 } else {
 940                         /* Unknown proto or tunnel, do sw cksum. We can assume
 941                          * the cksum field is in the first segment since the
 942                          * buffers we provided to the host are large enough.
 943                          * In case of SCTP, this will be wrong since it's a CRC
 944                          * but there's nothing we can do.
 945                          */
 946                         uint16_t csum = 0, off;
 947
 948                         if (rte_raw_cksum_mbuf(m, hdr->csum_start,
 949                                 rte_pktmbuf_pkt_len(m) - hdr->csum_start,
 950                                 &csum) < 0)
 951                                 return -EINVAL;
 952                         if (likely(csum != 0xffff))
 953                                 csum = ~csum;
 954                         off = hdr->csum_offset + hdr->csum_start;
 955                         if (rte_pktmbuf_data_len(m) >= off + 1)
 956                                 *rte_pktmbuf_mtod_offset(m, uint16_t *,
 957                                         off) = csum;
 958                 }
 959         } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) {
 960                 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 961         }
 962
 963         /* GSO request, save required information in mbuf */
 964         if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
 965                 /* Check unsupported modes */
 966                 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) ||
 967                     (hdr->gso_size == 0)) {
 968                         return -EINVAL;
 969                 }
 970
 971                 /* Update mss lengthes in mbuf */
 972                 m->tso_segsz = hdr->gso_size;
 973                 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
 974                         case VIRTIO_NET_HDR_GSO_TCPV4:
 975                         case VIRTIO_NET_HDR_GSO_TCPV6:
 976                                 m->ol_flags |= PKT_RX_LRO | \
 977                                         PKT_RX_L4_CKSUM_NONE;
 978                                 break;
 979                         default:
 980                                 return -EINVAL;
 981                 }
 982         }
 983
 984         return 0;
 985 }
 986
 987 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 988 uint16_t
 989 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 990 {
 991         struct virtnet_rx *rxvq = rx_queue;
 992         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
 993         struct virtio_hw *hw = vq->hw;
 994         struct rte_mbuf *rxm;
 995         uint16_t nb_used, num, nb_rx;
 996         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 997         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 998         int error;
 999         uint32_t i, nb_enqueued;
1000         uint32_t hdr_size;
1001         struct virtio_net_hdr *hdr;
1002
1003         nb_rx = 0;
1004         if (unlikely(hw->started == 0))
1005                 return nb_rx;
1006
1007         nb_used = virtqueue_nused(vq);
1008
1009         num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
1010         if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1011                 num = VIRTIO_MBUF_BURST_SZ;
1012         if (likely(num > DESC_PER_CACHELINE))
1013                 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1014
1015         num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
1016         PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
1017
1018         nb_enqueued = 0;
1019         hdr_size = hw->vtnet_hdr_size;
1020
1021         for (i = 0; i < num ; i++) {
1022                 rxm = rcv_pkts[i];
1023
1024                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1025
1026                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1027                         PMD_RX_LOG(ERR, "Packet drop");
1028                         nb_enqueued++;
1029                         virtio_discard_rxbuf(vq, rxm);
1030                         rxvq->stats.errors++;
1031                         continue;
1032                 }
1033
1034                 rxm->port = rxvq->port_id;
1035                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1036                 rxm->ol_flags = 0;
1037                 rxm->vlan_tci = 0;
1038
1039                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1040                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1041
1042                 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
1043                         RTE_PKTMBUF_HEADROOM - hdr_size);
1044
1045                 if (hw->vlan_strip)
1046                         rte_vlan_strip(rxm);
1047
1048                 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
1049                         virtio_discard_rxbuf(vq, rxm);
1050                         rxvq->stats.errors++;
1051                         continue;
1052                 }
1053
1054                 virtio_rx_stats_updated(rxvq, rxm);
1055
1056                 rx_pkts[nb_rx++] = rxm;
1057         }
1058
1059         rxvq->stats.packets += nb_rx;
1060
1061         /* Allocate new mbuf for the used descriptor */
1062         if (likely(!virtqueue_full(vq))) {
1063                 uint16_t free_cnt = vq->vq_free_cnt;
1064                 struct rte_mbuf *new_pkts[free_cnt];
1065
1066                 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
1067                                                 free_cnt) == 0)) {
1068                         error = virtqueue_enqueue_recv_refill(vq, new_pkts,
1069                                         free_cnt);
1070                         if (unlikely(error)) {
1071                                 for (i = 0; i < free_cnt; i++)
1072                                         rte_pktmbuf_free(new_pkts[i]);
1073                         }
1074                         nb_enqueued += free_cnt;
1075                 } else {
1076                         struct rte_eth_dev *dev =
1077                                 &rte_eth_devices[rxvq->port_id];
1078                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1079                 }
1080         }
1081
1082         if (likely(nb_enqueued)) {
1083                 vq_update_avail_idx(vq);
1084
1085                 if (unlikely(virtqueue_kick_prepare(vq))) {
1086                         virtqueue_notify(vq);
1087                         PMD_RX_LOG(DEBUG, "Notified");
1088                 }
1089         }
1090
1091         return nb_rx;
1092 }
1093
1094 uint16_t
1095 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts,
1096                         uint16_t nb_pkts)
1097 {
1098         struct virtnet_rx *rxvq = rx_queue;
1099         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1100         struct virtio_hw *hw = vq->hw;
1101         struct rte_mbuf *rxm;
1102         uint16_t num, nb_rx;
1103         uint32_t len[VIRTIO_MBUF_BURST_SZ];
1104         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1105         int error;
1106         uint32_t i, nb_enqueued;
1107         uint32_t hdr_size;
1108         struct virtio_net_hdr *hdr;
1109
1110         nb_rx = 0;
1111         if (unlikely(hw->started == 0))
1112                 return nb_rx;
1113
1114         num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts);
1115         if (likely(num > DESC_PER_CACHELINE))
1116                 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1117
1118         num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
1119         PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1120
1121         nb_enqueued = 0;
1122         hdr_size = hw->vtnet_hdr_size;
1123
1124         for (i = 0; i < num; i++) {
1125                 rxm = rcv_pkts[i];
1126
1127                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1128
1129                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1130                         PMD_RX_LOG(ERR, "Packet drop");
1131                         nb_enqueued++;
1132                         virtio_discard_rxbuf(vq, rxm);
1133                         rxvq->stats.errors++;
1134                         continue;
1135                 }
1136
1137                 rxm->port = rxvq->port_id;
1138                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1139                 rxm->ol_flags = 0;
1140                 rxm->vlan_tci = 0;
1141
1142                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1143                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1144
1145                 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
1146                         RTE_PKTMBUF_HEADROOM - hdr_size);
1147
1148                 if (hw->vlan_strip)
1149                         rte_vlan_strip(rxm);
1150
1151                 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
1152                         virtio_discard_rxbuf(vq, rxm);
1153                         rxvq->stats.errors++;
1154                         continue;
1155                 }
1156
1157                 virtio_rx_stats_updated(rxvq, rxm);
1158
1159                 rx_pkts[nb_rx++] = rxm;
1160         }
1161
1162         rxvq->stats.packets += nb_rx;
1163
1164         /* Allocate new mbuf for the used descriptor */
1165         if (likely(!virtqueue_full(vq))) {
1166                 uint16_t free_cnt = vq->vq_free_cnt;
1167                 struct rte_mbuf *new_pkts[free_cnt];
1168
1169                 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
1170                                                 free_cnt) == 0)) {
1171                         error = virtqueue_enqueue_recv_refill_packed(vq,
1172                                         new_pkts, free_cnt);
1173                         if (unlikely(error)) {
1174                                 for (i = 0; i < free_cnt; i++)
1175                                         rte_pktmbuf_free(new_pkts[i]);
1176                         }
1177                         nb_enqueued += free_cnt;
1178                 } else {
1179                         struct rte_eth_dev *dev =
1180                                 &rte_eth_devices[rxvq->port_id];
1181                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1182                 }
1183         }
1184
1185         if (likely(nb_enqueued)) {
1186                 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1187                         virtqueue_notify(vq);
1188                         PMD_RX_LOG(DEBUG, "Notified");
1189                 }
1190         }
1191
1192         return nb_rx;
1193 }
1194
1195
1196 uint16_t
1197 virtio_recv_pkts_inorder(void *rx_queue,
1198                         struct rte_mbuf **rx_pkts,
1199                         uint16_t nb_pkts)
1200 {
1201         struct virtnet_rx *rxvq = rx_queue;
1202         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1203         struct virtio_hw *hw = vq->hw;
1204         struct rte_mbuf *rxm;
1205         struct rte_mbuf *prev = NULL;
1206         uint16_t nb_used, num, nb_rx;
1207         uint32_t len[VIRTIO_MBUF_BURST_SZ];
1208         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1209         int error;
1210         uint32_t nb_enqueued;
1211         uint32_t seg_num;
1212         uint32_t seg_res;
1213         uint32_t hdr_size;
1214         int32_t i;
1215
1216         nb_rx = 0;
1217         if (unlikely(hw->started == 0))
1218                 return nb_rx;
1219
1220         nb_used = virtqueue_nused(vq);
1221         nb_used = RTE_MIN(nb_used, nb_pkts);
1222         nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
1223
1224         PMD_RX_LOG(DEBUG, "used:%d", nb_used);
1225
1226         nb_enqueued = 0;
1227         seg_num = 1;
1228         seg_res = 0;
1229         hdr_size = hw->vtnet_hdr_size;
1230
1231         num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used);
1232
1233         for (i = 0; i < num; i++) {
1234                 struct virtio_net_hdr_mrg_rxbuf *header;
1235
1236                 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1237                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1238
1239                 rxm = rcv_pkts[i];
1240
1241                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1242                         PMD_RX_LOG(ERR, "Packet drop");
1243                         nb_enqueued++;
1244                         virtio_discard_rxbuf_inorder(vq, rxm);
1245                         rxvq->stats.errors++;
1246                         continue;
1247                 }
1248
1249                 header = (struct virtio_net_hdr_mrg_rxbuf *)
1250                          ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
1251                          - hdr_size);
1252
1253                 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1254                         seg_num = header->num_buffers;
1255                         if (seg_num == 0)
1256                                 seg_num = 1;
1257                 } else {
1258                         seg_num = 1;
1259                 }
1260
1261                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1262                 rxm->nb_segs = seg_num;
1263                 rxm->ol_flags = 0;
1264                 rxm->vlan_tci = 0;
1265                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1266                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1267
1268                 rxm->port = rxvq->port_id;
1269
1270                 rx_pkts[nb_rx] = rxm;
1271                 prev = rxm;
1272
1273                 if (vq->hw->has_rx_offload &&
1274                                 virtio_rx_offload(rxm, &header->hdr) < 0) {
1275                         virtio_discard_rxbuf_inorder(vq, rxm);
1276                         rxvq->stats.errors++;
1277                         continue;
1278                 }
1279
1280                 if (hw->vlan_strip)
1281                         rte_vlan_strip(rx_pkts[nb_rx]);
1282
1283                 seg_res = seg_num - 1;
1284
1285                 /* Merge remaining segments */
1286                 while (seg_res != 0 && i < (num - 1)) {
1287                         i++;
1288
1289                         rxm = rcv_pkts[i];
1290                         rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1291                         rxm->pkt_len = (uint32_t)(len[i]);
1292                         rxm->data_len = (uint16_t)(len[i]);
1293
1294                         rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1295
1296                         prev->next = rxm;
1297                         prev = rxm;
1298                         seg_res -= 1;
1299                 }
1300
1301                 if (!seg_res) {
1302                         virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1303                         nb_rx++;
1304                 }
1305         }
1306
1307         /* Last packet still need merge segments */
1308         while (seg_res != 0) {
1309                 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1310                                         VIRTIO_MBUF_BURST_SZ);
1311
1312                 if (likely(virtqueue_nused(vq) >= rcv_cnt)) {
1313                         num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len,
1314                                                            rcv_cnt);
1315                         uint16_t extra_idx = 0;
1316
1317                         rcv_cnt = num;
1318                         while (extra_idx < rcv_cnt) {
1319                                 rxm = rcv_pkts[extra_idx];
1320                                 rxm->data_off =
1321                                         RTE_PKTMBUF_HEADROOM - hdr_size;
1322                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1323                                 rxm->data_len = (uint16_t)(len[extra_idx]);
1324                                 prev->next = rxm;
1325                                 prev = rxm;
1326                                 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1327                                 extra_idx += 1;
1328                         };
1329                         seg_res -= rcv_cnt;
1330
1331                         if (!seg_res) {
1332                                 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1333                                 nb_rx++;
1334                         }
1335                 } else {
1336                         PMD_RX_LOG(ERR,
1337                                         "No enough segments for packet.");
1338                         rte_pktmbuf_free(rx_pkts[nb_rx]);
1339                         rxvq->stats.errors++;
1340                         break;
1341                 }
1342         }
1343
1344         rxvq->stats.packets += nb_rx;
1345
1346         /* Allocate new mbuf for the used descriptor */
1347
1348         if (likely(!virtqueue_full(vq))) {
1349                 /* free_cnt may include mrg descs */
1350                 uint16_t free_cnt = vq->vq_free_cnt;
1351                 struct rte_mbuf *new_pkts[free_cnt];
1352
1353                 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1354                         error = virtqueue_enqueue_refill_inorder(vq, new_pkts,
1355                                         free_cnt);
1356                         if (unlikely(error)) {
1357                                 for (i = 0; i < free_cnt; i++)
1358                                         rte_pktmbuf_free(new_pkts[i]);
1359                         }
1360                         nb_enqueued += free_cnt;
1361                 } else {
1362                         struct rte_eth_dev *dev =
1363                                 &rte_eth_devices[rxvq->port_id];
1364                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1365                 }
1366         }
1367
1368         if (likely(nb_enqueued)) {
1369                 vq_update_avail_idx(vq);
1370
1371                 if (unlikely(virtqueue_kick_prepare(vq))) {
1372                         virtqueue_notify(vq);
1373                         PMD_RX_LOG(DEBUG, "Notified");
1374                 }
1375         }
1376
1377         return nb_rx;
1378 }
1379
1380 uint16_t
1381 virtio_recv_mergeable_pkts(void *rx_queue,
1382                         struct rte_mbuf **rx_pkts,
1383                         uint16_t nb_pkts)
1384 {
1385         struct virtnet_rx *rxvq = rx_queue;
1386         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1387         struct virtio_hw *hw = vq->hw;
1388         struct rte_mbuf *rxm;
1389         struct rte_mbuf *prev = NULL;
1390         uint16_t nb_used, num, nb_rx = 0;
1391         uint32_t len[VIRTIO_MBUF_BURST_SZ];
1392         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1393         int error;
1394         uint32_t nb_enqueued = 0;
1395         uint32_t seg_num = 0;
1396         uint32_t seg_res = 0;
1397         uint32_t hdr_size = hw->vtnet_hdr_size;
1398         int32_t i;
1399
1400         if (unlikely(hw->started == 0))
1401                 return nb_rx;
1402
1403         nb_used = virtqueue_nused(vq);
1404
1405         PMD_RX_LOG(DEBUG, "used:%d", nb_used);
1406
1407         num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
1408         if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1409                 num = VIRTIO_MBUF_BURST_SZ;
1410         if (likely(num > DESC_PER_CACHELINE))
1411                 num = num - ((vq->vq_used_cons_idx + num) %
1412                                 DESC_PER_CACHELINE);
1413
1414
1415         num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
1416
1417         for (i = 0; i < num; i++) {
1418                 struct virtio_net_hdr_mrg_rxbuf *header;
1419
1420                 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1421                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1422
1423                 rxm = rcv_pkts[i];
1424
1425                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1426                         PMD_RX_LOG(ERR, "Packet drop");
1427                         nb_enqueued++;
1428                         virtio_discard_rxbuf(vq, rxm);
1429                         rxvq->stats.errors++;
1430                         continue;
1431                 }
1432
1433                 header = (struct virtio_net_hdr_mrg_rxbuf *)
1434                          ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
1435                          - hdr_size);
1436                 seg_num = header->num_buffers;
1437                 if (seg_num == 0)
1438                         seg_num = 1;
1439
1440                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1441                 rxm->nb_segs = seg_num;
1442                 rxm->ol_flags = 0;
1443                 rxm->vlan_tci = 0;
1444                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1445                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1446
1447                 rxm->port = rxvq->port_id;
1448
1449                 rx_pkts[nb_rx] = rxm;
1450                 prev = rxm;
1451
1452                 if (hw->has_rx_offload &&
1453                                 virtio_rx_offload(rxm, &header->hdr) < 0) {
1454                         virtio_discard_rxbuf(vq, rxm);
1455                         rxvq->stats.errors++;
1456                         continue;
1457                 }
1458
1459                 if (hw->vlan_strip)
1460                         rte_vlan_strip(rx_pkts[nb_rx]);
1461
1462                 seg_res = seg_num - 1;
1463
1464                 /* Merge remaining segments */
1465                 while (seg_res != 0 && i < (num - 1)) {
1466                         i++;
1467
1468                         rxm = rcv_pkts[i];
1469                         rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1470                         rxm->pkt_len = (uint32_t)(len[i]);
1471                         rxm->data_len = (uint16_t)(len[i]);
1472
1473                         rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1474
1475                         prev->next = rxm;
1476                         prev = rxm;
1477                         seg_res -= 1;
1478                 }
1479
1480                 if (!seg_res) {
1481                         virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1482                         nb_rx++;
1483                 }
1484         }
1485
1486         /* Last packet still need merge segments */
1487         while (seg_res != 0) {
1488                 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1489                                         VIRTIO_MBUF_BURST_SZ);
1490
1491                 if (likely(virtqueue_nused(vq) >= rcv_cnt)) {
1492                         num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len,
1493                                                            rcv_cnt);
1494                         uint16_t extra_idx = 0;
1495
1496                         rcv_cnt = num;
1497                         while (extra_idx < rcv_cnt) {
1498                                 rxm = rcv_pkts[extra_idx];
1499                                 rxm->data_off =
1500                                         RTE_PKTMBUF_HEADROOM - hdr_size;
1501                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1502                                 rxm->data_len = (uint16_t)(len[extra_idx]);
1503                                 prev->next = rxm;
1504                                 prev = rxm;
1505                                 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1506                                 extra_idx += 1;
1507                         };
1508                         seg_res -= rcv_cnt;
1509
1510                         if (!seg_res) {
1511                                 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1512                                 nb_rx++;
1513                         }
1514                 } else {
1515                         PMD_RX_LOG(ERR,
1516                                         "No enough segments for packet.");
1517                         rte_pktmbuf_free(rx_pkts[nb_rx]);
1518                         rxvq->stats.errors++;
1519                         break;
1520                 }
1521         }
1522
1523         rxvq->stats.packets += nb_rx;
1524
1525         /* Allocate new mbuf for the used descriptor */
1526         if (likely(!virtqueue_full(vq))) {
1527                 /* free_cnt may include mrg descs */
1528                 uint16_t free_cnt = vq->vq_free_cnt;
1529                 struct rte_mbuf *new_pkts[free_cnt];
1530
1531                 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1532                         error = virtqueue_enqueue_recv_refill(vq, new_pkts,
1533                                         free_cnt);
1534                         if (unlikely(error)) {
1535                                 for (i = 0; i < free_cnt; i++)
1536                                         rte_pktmbuf_free(new_pkts[i]);
1537                         }
1538                         nb_enqueued += free_cnt;
1539                 } else {
1540                         struct rte_eth_dev *dev =
1541                                 &rte_eth_devices[rxvq->port_id];
1542                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1543                 }
1544         }
1545
1546         if (likely(nb_enqueued)) {
1547                 vq_update_avail_idx(vq);
1548
1549                 if (unlikely(virtqueue_kick_prepare(vq))) {
1550                         virtqueue_notify(vq);
1551                         PMD_RX_LOG(DEBUG, "Notified");
1552                 }
1553         }
1554
1555         return nb_rx;
1556 }
1557
1558 uint16_t
1559 virtio_recv_mergeable_pkts_packed(void *rx_queue,
1560                         struct rte_mbuf **rx_pkts,
1561                         uint16_t nb_pkts)
1562 {
1563         struct virtnet_rx *rxvq = rx_queue;
1564         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1565         struct virtio_hw *hw = vq->hw;
1566         struct rte_mbuf *rxm;
1567         struct rte_mbuf *prev = NULL;
1568         uint16_t num, nb_rx = 0;
1569         uint32_t len[VIRTIO_MBUF_BURST_SZ];
1570         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1571         uint32_t nb_enqueued = 0;
1572         uint32_t seg_num = 0;
1573         uint32_t seg_res = 0;
1574         uint32_t hdr_size = hw->vtnet_hdr_size;
1575         int32_t i;
1576         int error;
1577
1578         if (unlikely(hw->started == 0))
1579                 return nb_rx;
1580
1581
1582         num = nb_pkts;
1583         if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1584                 num = VIRTIO_MBUF_BURST_SZ;
1585         if (likely(num > DESC_PER_CACHELINE))
1586                 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1587
1588         num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
1589
1590         for (i = 0; i < num; i++) {
1591                 struct virtio_net_hdr_mrg_rxbuf *header;
1592
1593                 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1594                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1595
1596                 rxm = rcv_pkts[i];
1597
1598                 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1599                         PMD_RX_LOG(ERR, "Packet drop");
1600                         nb_enqueued++;
1601                         virtio_discard_rxbuf(vq, rxm);
1602                         rxvq->stats.errors++;
1603                         continue;
1604                 }
1605
1606                 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)
1607                           rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size);
1608                 seg_num = header->num_buffers;
1609
1610                 if (seg_num == 0)
1611                         seg_num = 1;
1612
1613                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1614                 rxm->nb_segs = seg_num;
1615                 rxm->ol_flags = 0;
1616                 rxm->vlan_tci = 0;
1617                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1618                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1619
1620                 rxm->port = rxvq->port_id;
1621                 rx_pkts[nb_rx] = rxm;
1622                 prev = rxm;
1623
1624                 if (hw->has_rx_offload &&
1625                                 virtio_rx_offload(rxm, &header->hdr) < 0) {
1626                         virtio_discard_rxbuf(vq, rxm);
1627                         rxvq->stats.errors++;
1628                         continue;
1629                 }
1630
1631                 if (hw->vlan_strip)
1632                         rte_vlan_strip(rx_pkts[nb_rx]);
1633
1634                 seg_res = seg_num - 1;
1635
1636                 /* Merge remaining segments */
1637                 while (seg_res != 0 && i < (num - 1)) {
1638                         i++;
1639
1640                         rxm = rcv_pkts[i];
1641                         rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1642                         rxm->pkt_len = (uint32_t)(len[i]);
1643                         rxm->data_len = (uint16_t)(len[i]);
1644
1645                         rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1646
1647                         prev->next = rxm;
1648                         prev = rxm;
1649                         seg_res -= 1;
1650                 }
1651
1652                 if (!seg_res) {
1653                         virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1654                         nb_rx++;
1655                 }
1656         }
1657
1658         /* Last packet still need merge segments */
1659         while (seg_res != 0) {
1660                 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1661                                         VIRTIO_MBUF_BURST_SZ);
1662                 uint16_t extra_idx = 0;
1663
1664                 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts,
1665                                 len, rcv_cnt);
1666                 if (unlikely(rcv_cnt == 0)) {
1667                         PMD_RX_LOG(ERR, "No enough segments for packet.");
1668                         rte_pktmbuf_free(rx_pkts[nb_rx]);
1669                         rxvq->stats.errors++;
1670                         break;
1671                 }
1672
1673                 while (extra_idx < rcv_cnt) {
1674                         rxm = rcv_pkts[extra_idx];
1675
1676                         rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1677                         rxm->pkt_len = (uint32_t)(len[extra_idx]);
1678                         rxm->data_len = (uint16_t)(len[extra_idx]);
1679
1680                         prev->next = rxm;
1681                         prev = rxm;
1682                         rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1683                         extra_idx += 1;
1684                 }
1685                 seg_res -= rcv_cnt;
1686                 if (!seg_res) {
1687                         virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1688                         nb_rx++;
1689                 }
1690         }
1691
1692         rxvq->stats.packets += nb_rx;
1693
1694         /* Allocate new mbuf for the used descriptor */
1695         if (likely(!virtqueue_full(vq))) {
1696                 /* free_cnt may include mrg descs */
1697                 uint16_t free_cnt = vq->vq_free_cnt;
1698                 struct rte_mbuf *new_pkts[free_cnt];
1699
1700                 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1701                         error = virtqueue_enqueue_recv_refill_packed(vq,
1702                                         new_pkts, free_cnt);
1703                         if (unlikely(error)) {
1704                                 for (i = 0; i < free_cnt; i++)
1705                                         rte_pktmbuf_free(new_pkts[i]);
1706                         }
1707                         nb_enqueued += free_cnt;
1708                 } else {
1709                         struct rte_eth_dev *dev =
1710                                 &rte_eth_devices[rxvq->port_id];
1711                         dev->data->rx_mbuf_alloc_failed += free_cnt;
1712                 }
1713         }
1714
1715         if (likely(nb_enqueued)) {
1716                 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1717                         virtqueue_notify(vq);
1718                         PMD_RX_LOG(DEBUG, "Notified");
1719                 }
1720         }
1721
1722         return nb_rx;
1723 }
1724
1725 uint16_t
1726 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
1727                         uint16_t nb_pkts)
1728 {
1729         uint16_t nb_tx;
1730         int error;
1731
1732         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1733                 struct rte_mbuf *m = tx_pkts[nb_tx];
1734
1735 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
1736                 error = rte_validate_tx_offload(m);
1737                 if (unlikely(error)) {
1738                         rte_errno = -error;
1739                         break;
1740                 }
1741 #endif
1742
1743                 /* Do VLAN tag insertion */
1744                 if (unlikely(m->ol_flags & PKT_TX_VLAN_PKT)) {
1745                         error = rte_vlan_insert(&m);
1746                         /* rte_vlan_insert() may change pointer
1747                          * even in the case of failure
1748                          */
1749                         tx_pkts[nb_tx] = m;
1750
1751                         if (unlikely(error)) {
1752                                 rte_errno = -error;
1753                                 break;
1754                         }
1755                 }
1756
1757                 error = rte_net_intel_cksum_prepare(m);
1758                 if (unlikely(error)) {
1759                         rte_errno = -error;
1760                         break;
1761                 }
1762
1763                 if (m->ol_flags & PKT_TX_TCP_SEG)
1764                         virtio_tso_fix_cksum(m);
1765         }
1766
1767         return nb_tx;
1768 }
1769
1770 uint16_t
1771 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts,
1772                         uint16_t nb_pkts)
1773 {
1774         struct virtnet_tx *txvq = tx_queue;
1775         struct virtqueue *vq = virtnet_txq_to_vq(txvq);
1776         struct virtio_hw *hw = vq->hw;
1777         uint16_t hdr_size = hw->vtnet_hdr_size;
1778         uint16_t nb_tx = 0;
1779         bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER);
1780
1781         if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
1782                 return nb_tx;
1783
1784         if (unlikely(nb_pkts < 1))
1785                 return nb_pkts;
1786
1787         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
1788
1789         if (nb_pkts > vq->vq_free_cnt)
1790                 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt,
1791                                            in_order);
1792
1793         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1794                 struct rte_mbuf *txm = tx_pkts[nb_tx];
1795                 int can_push = 0, use_indirect = 0, slots, need;
1796
1797                 /* optimize ring usage */
1798                 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
1799                       virtio_with_feature(hw, VIRTIO_F_VERSION_1)) &&
1800                     rte_mbuf_refcnt_read(txm) == 1 &&
1801                     RTE_MBUF_DIRECT(txm) &&
1802                     txm->nb_segs == 1 &&
1803                     rte_pktmbuf_headroom(txm) >= hdr_size &&
1804                     rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
1805                            __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
1806                         can_push = 1;
1807                 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
1808                          txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
1809                         use_indirect = 1;
1810                 /* How many main ring entries are needed to this Tx?
1811                  * indirect   => 1
1812                  * any_layout => number of segments
1813                  * default    => number of segments + 1
1814                  */
1815                 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
1816                 need = slots - vq->vq_free_cnt;
1817
1818                 /* Positive value indicates it need free vring descriptors */
1819                 if (unlikely(need > 0)) {
1820                         virtio_xmit_cleanup_packed(vq, need, in_order);
1821                         need = slots - vq->vq_free_cnt;
1822                         if (unlikely(need > 0)) {
1823                                 PMD_TX_LOG(ERR,
1824                                            "No free tx descriptors to transmit");
1825                                 break;
1826                         }
1827                 }
1828
1829                 /* Enqueue Packet buffers */
1830                 if (can_push)
1831                         virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order);
1832                 else
1833                         virtqueue_enqueue_xmit_packed(txvq, txm, slots,
1834                                                       use_indirect, 0,
1835                                                       in_order);
1836
1837                 virtio_update_packet_stats(&txvq->stats, txm);
1838         }
1839
1840         txvq->stats.packets += nb_tx;
1841
1842         if (likely(nb_tx)) {
1843                 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1844                         virtqueue_notify(vq);
1845                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
1846                 }
1847         }
1848
1849         return nb_tx;
1850 }
1851
1852 uint16_t
1853 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1854 {
1855         struct virtnet_tx *txvq = tx_queue;
1856         struct virtqueue *vq = virtnet_txq_to_vq(txvq);
1857         struct virtio_hw *hw = vq->hw;
1858         uint16_t hdr_size = hw->vtnet_hdr_size;
1859         uint16_t nb_used, nb_tx = 0;
1860
1861         if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
1862                 return nb_tx;
1863
1864         if (unlikely(nb_pkts < 1))
1865                 return nb_pkts;
1866
1867         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
1868
1869         nb_used = virtqueue_nused(vq);
1870
1871         if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
1872                 virtio_xmit_cleanup(vq, nb_used);
1873
1874         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1875                 struct rte_mbuf *txm = tx_pkts[nb_tx];
1876                 int can_push = 0, use_indirect = 0, slots, need;
1877
1878                 /* optimize ring usage */
1879                 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
1880                       virtio_with_feature(hw, VIRTIO_F_VERSION_1)) &&
1881                     rte_mbuf_refcnt_read(txm) == 1 &&
1882                     RTE_MBUF_DIRECT(txm) &&
1883                     txm->nb_segs == 1 &&
1884                     rte_pktmbuf_headroom(txm) >= hdr_size &&
1885                     rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
1886                                    __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
1887                         can_push = 1;
1888                 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
1889                          txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
1890                         use_indirect = 1;
1891
1892                 /* How many main ring entries are needed to this Tx?
1893                  * any_layout => number of segments
1894                  * indirect   => 1
1895                  * default    => number of segments + 1
1896                  */
1897                 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
1898                 need = slots - vq->vq_free_cnt;
1899
1900                 /* Positive value indicates it need free vring descriptors */
1901                 if (unlikely(need > 0)) {
1902                         nb_used = virtqueue_nused(vq);
1903
1904                         need = RTE_MIN(need, (int)nb_used);
1905
1906                         virtio_xmit_cleanup(vq, need);
1907                         need = slots - vq->vq_free_cnt;
1908                         if (unlikely(need > 0)) {
1909                                 PMD_TX_LOG(ERR,
1910                                            "No free tx descriptors to transmit");
1911                                 break;
1912                         }
1913                 }
1914
1915                 /* Enqueue Packet buffers */
1916                 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect,
1917                         can_push, 0);
1918
1919                 virtio_update_packet_stats(&txvq->stats, txm);
1920         }
1921
1922         txvq->stats.packets += nb_tx;
1923
1924         if (likely(nb_tx)) {
1925                 vq_update_avail_idx(vq);
1926
1927                 if (unlikely(virtqueue_kick_prepare(vq))) {
1928                         virtqueue_notify(vq);
1929                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
1930                 }
1931         }
1932
1933         return nb_tx;
1934 }
1935
1936 static __rte_always_inline int
1937 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need)
1938 {
1939         uint16_t nb_used, nb_clean, nb_descs;
1940
1941         nb_descs = vq->vq_free_cnt + need;
1942         nb_used = virtqueue_nused(vq);
1943         nb_clean = RTE_MIN(need, (int)nb_used);
1944
1945         virtio_xmit_cleanup_inorder(vq, nb_clean);
1946
1947         return nb_descs - vq->vq_free_cnt;
1948 }
1949
1950 uint16_t
1951 virtio_xmit_pkts_inorder(void *tx_queue,
1952                         struct rte_mbuf **tx_pkts,
1953                         uint16_t nb_pkts)
1954 {
1955         struct virtnet_tx *txvq = tx_queue;
1956         struct virtqueue *vq = virtnet_txq_to_vq(txvq);
1957         struct virtio_hw *hw = vq->hw;
1958         uint16_t hdr_size = hw->vtnet_hdr_size;
1959         uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0;
1960         struct rte_mbuf *inorder_pkts[nb_pkts];
1961         int need;
1962
1963         if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
1964                 return nb_tx;
1965
1966         if (unlikely(nb_pkts < 1))
1967                 return nb_pkts;
1968
1969         VIRTQUEUE_DUMP(vq);
1970         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
1971         nb_used = virtqueue_nused(vq);
1972
1973         if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
1974                 virtio_xmit_cleanup_inorder(vq, nb_used);
1975
1976         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1977                 struct rte_mbuf *txm = tx_pkts[nb_tx];
1978                 int slots;
1979
1980                 /* optimize ring usage */
1981                 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
1982                      virtio_with_feature(hw, VIRTIO_F_VERSION_1)) &&
1983                      rte_mbuf_refcnt_read(txm) == 1 &&
1984                      RTE_MBUF_DIRECT(txm) &&
1985                      txm->nb_segs == 1 &&
1986                      rte_pktmbuf_headroom(txm) >= hdr_size &&
1987                      rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
1988                                 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) {
1989                         inorder_pkts[nb_inorder_pkts] = txm;
1990                         nb_inorder_pkts++;
1991
1992                         continue;
1993                 }
1994
1995                 if (nb_inorder_pkts) {
1996                         need = nb_inorder_pkts - vq->vq_free_cnt;
1997                         if (unlikely(need > 0)) {
1998                                 need = virtio_xmit_try_cleanup_inorder(vq,
1999                                                                        need);
2000                                 if (unlikely(need > 0)) {
2001                                         PMD_TX_LOG(ERR,
2002                                                 "No free tx descriptors to "
2003                                                 "transmit");
2004                                         break;
2005                                 }
2006                         }
2007                         virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
2008                                                         nb_inorder_pkts);
2009                         nb_inorder_pkts = 0;
2010                 }
2011
2012                 slots = txm->nb_segs + 1;
2013                 need = slots - vq->vq_free_cnt;
2014                 if (unlikely(need > 0)) {
2015                         need = virtio_xmit_try_cleanup_inorder(vq, slots);
2016
2017                         if (unlikely(need > 0)) {
2018                                 PMD_TX_LOG(ERR,
2019                                         "No free tx descriptors to transmit");
2020                                 break;
2021                         }
2022                 }
2023                 /* Enqueue Packet buffers */
2024                 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1);
2025
2026                 virtio_update_packet_stats(&txvq->stats, txm);
2027         }
2028
2029         /* Transmit all inorder packets */
2030         if (nb_inorder_pkts) {
2031                 need = nb_inorder_pkts - vq->vq_free_cnt;
2032                 if (unlikely(need > 0)) {
2033                         need = virtio_xmit_try_cleanup_inorder(vq,
2034                                                                   need);
2035                         if (unlikely(need > 0)) {
2036                                 PMD_TX_LOG(ERR,
2037                                         "No free tx descriptors to transmit");
2038                                 nb_inorder_pkts = vq->vq_free_cnt;
2039                                 nb_tx -= need;
2040                         }
2041                 }
2042
2043                 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
2044                                                 nb_inorder_pkts);
2045         }
2046
2047         txvq->stats.packets += nb_tx;
2048
2049         if (likely(nb_tx)) {
2050                 vq_update_avail_idx(vq);
2051
2052                 if (unlikely(virtqueue_kick_prepare(vq))) {
2053                         virtqueue_notify(vq);
2054                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
2055                 }
2056         }
2057
2058         VIRTQUEUE_DUMP(vq);
2059
2060         return nb_tx;
2061 }
2062
2063 __rte_weak uint16_t
2064 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
2065                             struct rte_mbuf **rx_pkts __rte_unused,
2066                             uint16_t nb_pkts __rte_unused)
2067 {
2068         return 0;
2069 }
2070
2071 __rte_weak uint16_t
2072 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
2073                             struct rte_mbuf **tx_pkts __rte_unused,
2074                             uint16_t nb_pkts __rte_unused)
2075 {
2076         return 0;
2077 }