drivers/net/virtio/virtio_rxtx_simple_neon.c

   1 /*
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (C) Cavium, Inc. 2016
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Cavium, Inc nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31 */
  32
  33 #include <stdint.h>
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <errno.h>
  38
  39 #include <rte_byteorder.h>
  40 #include <rte_branch_prediction.h>
  41 #include <rte_cycles.h>
  42 #include <rte_ether.h>
  43 #include <rte_ethdev.h>
  44 #include <rte_errno.h>
  45 #include <rte_memory.h>
  46 #include <rte_memzone.h>
  47 #include <rte_mempool.h>
  48 #include <rte_malloc.h>
  49 #include <rte_mbuf.h>
  50 #include <rte_prefetch.h>
  51 #include <rte_string_fns.h>
  52 #include <rte_vect.h>
  53
  54 #include "virtio_rxtx_simple.h"
  55
  56 #define RTE_VIRTIO_VPMD_RX_BURST 32
  57 #define RTE_VIRTIO_DESC_PER_LOOP 8
  58 #define RTE_VIRTIO_VPMD_RX_REARM_THRESH RTE_VIRTIO_VPMD_RX_BURST
  59
  60 /* virtio vPMD receive routine, only accept(nb_pkts >= RTE_VIRTIO_DESC_PER_LOOP)
  61  *
  62  * This routine is for non-mergeable RX, one desc for each guest buffer.
  63  * This routine is based on the RX ring layout optimization. Each entry in the
  64  * avail ring points to the desc with the same index in the desc ring and this
  65  * will never be changed in the driver.
  66  *
  67  * - nb_pkts < RTE_VIRTIO_DESC_PER_LOOP, just return no packet
  68  */
  69 uint16_t
  70 virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
  71         uint16_t nb_pkts)
  72 {
  73         struct virtnet_rx *rxvq = rx_queue;
  74         struct virtqueue *vq = rxvq->vq;
  75         struct virtio_hw *hw = vq->hw;
  76         uint16_t nb_used;
  77         uint16_t desc_idx;
  78         struct vring_used_elem *rused;
  79         struct rte_mbuf **sw_ring;
  80         struct rte_mbuf **sw_ring_end;
  81         uint16_t nb_pkts_received = 0;
  82
  83         uint8x16_t shuf_msk1 = {
  84                 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */
  85                 4, 5, 0xFF, 0xFF,       /* pkt len */
  86                 4, 5,                   /* dat len */
  87                 0xFF, 0xFF,             /* vlan tci */
  88                 0xFF, 0xFF, 0xFF, 0xFF
  89         };
  90
  91         uint8x16_t shuf_msk2 = {
  92                 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */
  93                 12, 13, 0xFF, 0xFF,     /* pkt len */
  94                 12, 13,                 /* dat len */
  95                 0xFF, 0xFF,             /* vlan tci */
  96                 0xFF, 0xFF, 0xFF, 0xFF
  97         };
  98
  99         /* Subtract the header length.
 100          *  In which case do we need the header length in used->len ?
 101          */
 102         uint16x8_t len_adjust = {
 103                 0, 0,
 104                 (uint16_t)vq->hw->vtnet_hdr_size, 0,
 105                 (uint16_t)vq->hw->vtnet_hdr_size,
 106                 0,
 107                 0, 0
 108         };
 109
 110         if (unlikely(hw->started == 0))
 111                 return nb_pkts_received;
 112
 113         if (unlikely(nb_pkts < RTE_VIRTIO_DESC_PER_LOOP))
 114                 return 0;
 115
 116         nb_used = VIRTQUEUE_NUSED(vq);
 117
 118         rte_rmb();
 119
 120         if (unlikely(nb_used == 0))
 121                 return 0;
 122
 123         nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_VIRTIO_DESC_PER_LOOP);
 124         nb_used = RTE_MIN(nb_used, nb_pkts);
 125
 126         desc_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 127         rused = &vq->vq_ring.used->ring[desc_idx];
 128         sw_ring  = &vq->sw_ring[desc_idx];
 129         sw_ring_end = &vq->sw_ring[vq->vq_nentries];
 130
 131         rte_prefetch_non_temporal(rused);
 132
 133         if (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
 134                 virtio_rxq_rearm_vec(rxvq);
 135                 if (unlikely(virtqueue_kick_prepare(vq)))
 136                         virtqueue_notify(vq);
 137         }
 138
 139         for (nb_pkts_received = 0;
 140                 nb_pkts_received < nb_used;) {
 141                 uint64x2_t desc[RTE_VIRTIO_DESC_PER_LOOP / 2];
 142                 uint64x2_t mbp[RTE_VIRTIO_DESC_PER_LOOP / 2];
 143                 uint64x2_t pkt_mb[RTE_VIRTIO_DESC_PER_LOOP];
 144
 145                 mbp[0] = vld1q_u64((uint64_t *)(sw_ring + 0));
 146                 desc[0] = vld1q_u64((uint64_t *)(rused + 0));
 147                 vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0]);
 148
 149                 mbp[1] = vld1q_u64((uint64_t *)(sw_ring + 2));
 150                 desc[1] = vld1q_u64((uint64_t *)(rused + 2));
 151                 vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1]);
 152
 153                 mbp[2] = vld1q_u64((uint64_t *)(sw_ring + 4));
 154                 desc[2] = vld1q_u64((uint64_t *)(rused + 4));
 155                 vst1q_u64((uint64_t *)&rx_pkts[4], mbp[2]);
 156
 157                 mbp[3] = vld1q_u64((uint64_t *)(sw_ring + 6));
 158                 desc[3] = vld1q_u64((uint64_t *)(rused + 6));
 159                 vst1q_u64((uint64_t *)&rx_pkts[6], mbp[3]);
 160
 161                 pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
 162                                 vreinterpretq_u8_u64(desc[0]), shuf_msk2));
 163                 pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
 164                                 vreinterpretq_u8_u64(desc[0]), shuf_msk1));
 165                 pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16(
 166                                 vreinterpretq_u16_u64(pkt_mb[1]), len_adjust));
 167                 pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16(
 168                                 vreinterpretq_u16_u64(pkt_mb[0]), len_adjust));
 169                 vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1,
 170                         pkt_mb[1]);
 171                 vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1,
 172                         pkt_mb[0]);
 173
 174                 pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8(
 175                                 vreinterpretq_u8_u64(desc[1]), shuf_msk2));
 176                 pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
 177                                 vreinterpretq_u8_u64(desc[1]), shuf_msk1));
 178                 pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16(
 179                                 vreinterpretq_u16_u64(pkt_mb[3]), len_adjust));
 180                 pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16(
 181                                 vreinterpretq_u16_u64(pkt_mb[2]), len_adjust));
 182                 vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1,
 183                         pkt_mb[3]);
 184                 vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1,
 185                         pkt_mb[2]);
 186
 187                 pkt_mb[5] = vreinterpretq_u64_u8(vqtbl1q_u8(
 188                                 vreinterpretq_u8_u64(desc[2]), shuf_msk2));
 189                 pkt_mb[4] = vreinterpretq_u64_u8(vqtbl1q_u8(
 190                                 vreinterpretq_u8_u64(desc[2]), shuf_msk1));
 191                 pkt_mb[5] = vreinterpretq_u64_u16(vsubq_u16(
 192                                 vreinterpretq_u16_u64(pkt_mb[5]), len_adjust));
 193                 pkt_mb[4] = vreinterpretq_u64_u16(vsubq_u16(
 194                                 vreinterpretq_u16_u64(pkt_mb[4]), len_adjust));
 195                 vst1q_u64((void *)&rx_pkts[5]->rx_descriptor_fields1,
 196                         pkt_mb[5]);
 197                 vst1q_u64((void *)&rx_pkts[4]->rx_descriptor_fields1,
 198                         pkt_mb[4]);
 199
 200                 pkt_mb[7] = vreinterpretq_u64_u8(vqtbl1q_u8(
 201                                 vreinterpretq_u8_u64(desc[3]), shuf_msk2));
 202                 pkt_mb[6] = vreinterpretq_u64_u8(vqtbl1q_u8(
 203                                 vreinterpretq_u8_u64(desc[3]), shuf_msk1));
 204                 pkt_mb[7] = vreinterpretq_u64_u16(vsubq_u16(
 205                                 vreinterpretq_u16_u64(pkt_mb[7]), len_adjust));
 206                 pkt_mb[6] = vreinterpretq_u64_u16(vsubq_u16(
 207                                 vreinterpretq_u16_u64(pkt_mb[6]), len_adjust));
 208                 vst1q_u64((void *)&rx_pkts[7]->rx_descriptor_fields1,
 209                         pkt_mb[7]);
 210                 vst1q_u64((void *)&rx_pkts[6]->rx_descriptor_fields1,
 211                         pkt_mb[6]);
 212
 213                 if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) {
 214                         if (sw_ring + nb_used <= sw_ring_end)
 215                                 nb_pkts_received += nb_used;
 216                         else
 217                                 nb_pkts_received += sw_ring_end - sw_ring;
 218                         break;
 219                 } else {
 220                         if (unlikely(sw_ring + RTE_VIRTIO_DESC_PER_LOOP >=
 221                                 sw_ring_end)) {
 222                                 nb_pkts_received += sw_ring_end - sw_ring;
 223                                 break;
 224                         } else {
 225                                 nb_pkts_received += RTE_VIRTIO_DESC_PER_LOOP;
 226
 227                                 rx_pkts += RTE_VIRTIO_DESC_PER_LOOP;
 228                                 sw_ring += RTE_VIRTIO_DESC_PER_LOOP;
 229                                 rused   += RTE_VIRTIO_DESC_PER_LOOP;
 230                                 nb_used -= RTE_VIRTIO_DESC_PER_LOOP;
 231                         }
 232                 }
 233         }
 234
 235         vq->vq_used_cons_idx += nb_pkts_received;
 236         vq->vq_free_cnt += nb_pkts_received;
 237         rxvq->stats.packets += nb_pkts_received;
 238         return nb_pkts_received;
 239 }