drivers/net/virtio/virtio_rxtx_packed_avx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2020 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <errno.h>
  10
  11 #include <rte_net.h>
  12
  13 #include "virtio_logs.h"
  14 #include "virtio_ethdev.h"
  15 #include "virtio_pci.h"
  16 #include "virtqueue.h"
  17
  18 #define BYTE_SIZE 8
  19 /* flag bits offset in packed ring desc higher 64bits */
  20 #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
  21         offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
  22
  23 #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
  24         FLAGS_BITS_OFFSET)
  25
  26 #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
  27         sizeof(struct vring_packed_desc))
  28 #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
  29
  30 #ifdef VIRTIO_GCC_UNROLL_PRAGMA
  31 #define virtio_for_each_try_unroll(iter, val, size) _Pragma("GCC unroll 4") \
  32         for (iter = val; iter < size; iter++)
  33 #endif
  34
  35 #ifdef VIRTIO_CLANG_UNROLL_PRAGMA
  36 #define virtio_for_each_try_unroll(iter, val, size) _Pragma("unroll 4") \
  37         for (iter = val; iter < size; iter++)
  38 #endif
  39
  40 #ifdef VIRTIO_ICC_UNROLL_PRAGMA
  41 #define virtio_for_each_try_unroll(iter, val, size) _Pragma("unroll (4)") \
  42         for (iter = val; iter < size; iter++)
  43 #endif
  44
  45 #ifndef virtio_for_each_try_unroll
  46 #define virtio_for_each_try_unroll(iter, val, num) \
  47         for (iter = val; iter < num; iter++)
  48 #endif
  49
  50 static inline void
  51 virtio_update_batch_stats(struct virtnet_stats *stats,
  52                           uint16_t pkt_len1,
  53                           uint16_t pkt_len2,
  54                           uint16_t pkt_len3,
  55                           uint16_t pkt_len4)
  56 {
  57         stats->bytes += pkt_len1;
  58         stats->bytes += pkt_len2;
  59         stats->bytes += pkt_len3;
  60         stats->bytes += pkt_len4;
  61 }
  62
  63 /* Optionally fill offload information in structure */
  64 static inline int
  65 virtio_vec_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
  66 {
  67         struct rte_net_hdr_lens hdr_lens;
  68         uint32_t hdrlen, ptype;
  69         int l4_supported = 0;
  70
  71         /* nothing to do */
  72         if (hdr->flags == 0)
  73                 return 0;
  74
  75         /* GSO not support in vec path, skip check */
  76         m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
  77
  78         ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK);
  79         m->packet_type = ptype;
  80         if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP ||
  81             (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP ||
  82             (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP)
  83                 l4_supported = 1;
  84
  85         if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
  86                 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len;
  87                 if (hdr->csum_start <= hdrlen && l4_supported) {
  88                         m->ol_flags |= PKT_RX_L4_CKSUM_NONE;
  89                 } else {
  90                         /* Unknown proto or tunnel, do sw cksum. We can assume
  91                          * the cksum field is in the first segment since the
  92                          * buffers we provided to the host are large enough.
  93                          * In case of SCTP, this will be wrong since it's a CRC
  94                          * but there's nothing we can do.
  95                          */
  96                         uint16_t csum = 0, off;
  97
  98                         rte_raw_cksum_mbuf(m, hdr->csum_start,
  99                                 rte_pktmbuf_pkt_len(m) - hdr->csum_start,
 100                                 &csum);
 101                         if (likely(csum != 0xffff))
 102                                 csum = ~csum;
 103                         off = hdr->csum_offset + hdr->csum_start;
 104                         if (rte_pktmbuf_data_len(m) >= off + 1)
 105                                 *rte_pktmbuf_mtod_offset(m, uint16_t *,
 106                                         off) = csum;
 107                 }
 108         } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) {
 109                 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 110         }
 111
 112         return 0;
 113 }
 114
 115 static inline uint16_t
 116 virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
 117                                    struct rte_mbuf **rx_pkts)
 118 {
 119         struct virtqueue *vq = rxvq->vq;
 120         struct virtio_hw *hw = vq->hw;
 121         uint16_t hdr_size = hw->vtnet_hdr_size;
 122         uint64_t addrs[PACKED_BATCH_SIZE];
 123         uint16_t id = vq->vq_used_cons_idx;
 124         uint8_t desc_stats;
 125         uint16_t i;
 126         void *desc_addr;
 127
 128         if (id & PACKED_BATCH_MASK)
 129                 return -1;
 130
 131         if (unlikely((id + PACKED_BATCH_SIZE) > vq->vq_nentries))
 132                 return -1;
 133
 134         /* only care avail/used bits */
 135         __m512i v_mask = _mm512_maskz_set1_epi64(0xaa, PACKED_FLAGS_MASK);
 136         desc_addr = &vq->vq_packed.ring.desc[id];
 137
 138         __m512i v_desc = _mm512_loadu_si512(desc_addr);
 139         __m512i v_flag = _mm512_and_epi64(v_desc, v_mask);
 140
 141         __m512i v_used_flag = _mm512_setzero_si512();
 142         if (vq->vq_packed.used_wrap_counter)
 143                 v_used_flag = _mm512_maskz_set1_epi64(0xaa, PACKED_FLAGS_MASK);
 144
 145         /* Check all descs are used */
 146         desc_stats = _mm512_cmpneq_epu64_mask(v_flag, v_used_flag);
 147         if (desc_stats)
 148                 return -1;
 149
 150         virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
 151                 rx_pkts[i] = (struct rte_mbuf *)vq->vq_descx[id + i].cookie;
 152                 rte_packet_prefetch(rte_pktmbuf_mtod(rx_pkts[i], void *));
 153
 154                 addrs[i] = (uintptr_t)rx_pkts[i]->rx_descriptor_fields1;
 155         }
 156
 157         /*
 158          * load len from desc, store into mbuf pkt_len and data_len
 159          * len limiated by l6bit buf_len, pkt_len[16:31] can be ignored
 160          */
 161         const __mmask16 mask = 0x6 | 0x6 << 4 | 0x6 << 8 | 0x6 << 12;
 162         __m512i values = _mm512_maskz_shuffle_epi32(mask, v_desc, 0xAA);
 163
 164         /* reduce hdr_len from pkt_len and data_len */
 165         __m512i mbuf_len_offset = _mm512_maskz_set1_epi32(mask,
 166                         (uint32_t)-hdr_size);
 167
 168         __m512i v_value = _mm512_add_epi32(values, mbuf_len_offset);
 169
 170         /* assert offset of data_len */
 171         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
 172                 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
 173
 174         __m512i v_index = _mm512_set_epi64(addrs[3] + 8, addrs[3],
 175                                            addrs[2] + 8, addrs[2],
 176                                            addrs[1] + 8, addrs[1],
 177                                            addrs[0] + 8, addrs[0]);
 178         /* batch store into mbufs */
 179         _mm512_i64scatter_epi64(0, v_index, v_value, 1);
 180
 181         if (hw->has_rx_offload) {
 182                 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
 183                         char *addr = (char *)rx_pkts[i]->buf_addr +
 184                                 RTE_PKTMBUF_HEADROOM - hdr_size;
 185                         virtio_vec_rx_offload(rx_pkts[i],
 186                                         (struct virtio_net_hdr *)addr);
 187                 }
 188         }
 189
 190         virtio_update_batch_stats(&rxvq->stats, rx_pkts[0]->pkt_len,
 191                         rx_pkts[1]->pkt_len, rx_pkts[2]->pkt_len,
 192                         rx_pkts[3]->pkt_len);
 193
 194         vq->vq_free_cnt += PACKED_BATCH_SIZE;
 195
 196         vq->vq_used_cons_idx += PACKED_BATCH_SIZE;
 197         if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 198                 vq->vq_used_cons_idx -= vq->vq_nentries;
 199                 vq->vq_packed.used_wrap_counter ^= 1;
 200         }
 201
 202         return 0;
 203 }
 204
 205 static uint16_t
 206 virtqueue_dequeue_single_packed_vec(struct virtnet_rx *rxvq,
 207                                     struct rte_mbuf **rx_pkts)
 208 {
 209         uint16_t used_idx, id;
 210         uint32_t len;
 211         struct virtqueue *vq = rxvq->vq;
 212         struct virtio_hw *hw = vq->hw;
 213         uint32_t hdr_size = hw->vtnet_hdr_size;
 214         struct virtio_net_hdr *hdr;
 215         struct vring_packed_desc *desc;
 216         struct rte_mbuf *cookie;
 217
 218         desc = vq->vq_packed.ring.desc;
 219         used_idx = vq->vq_used_cons_idx;
 220         if (!desc_is_used(&desc[used_idx], vq))
 221                 return -1;
 222
 223         len = desc[used_idx].len;
 224         id = desc[used_idx].id;
 225         cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
 226         if (unlikely(cookie == NULL)) {
 227                 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
 228                                 vq->vq_used_cons_idx);
 229                 return -1;
 230         }
 231         rte_prefetch0(cookie);
 232         rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 233
 234         cookie->data_off = RTE_PKTMBUF_HEADROOM;
 235         cookie->ol_flags = 0;
 236         cookie->pkt_len = (uint32_t)(len - hdr_size);
 237         cookie->data_len = (uint32_t)(len - hdr_size);
 238
 239         hdr = (struct virtio_net_hdr *)((char *)cookie->buf_addr +
 240                                         RTE_PKTMBUF_HEADROOM - hdr_size);
 241         if (hw->has_rx_offload)
 242                 virtio_vec_rx_offload(cookie, hdr);
 243
 244         *rx_pkts = cookie;
 245
 246         rxvq->stats.bytes += cookie->pkt_len;
 247
 248         vq->vq_free_cnt++;
 249         vq->vq_used_cons_idx++;
 250         if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 251                 vq->vq_used_cons_idx -= vq->vq_nentries;
 252                 vq->vq_packed.used_wrap_counter ^= 1;
 253         }
 254
 255         return 0;
 256 }
 257
 258 static inline void
 259 virtio_recv_refill_packed_vec(struct virtnet_rx *rxvq,
 260                               struct rte_mbuf **cookie,
 261                               uint16_t num)
 262 {
 263         struct virtqueue *vq = rxvq->vq;
 264         struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
 265         uint16_t flags = vq->vq_packed.cached_flags;
 266         struct virtio_hw *hw = vq->hw;
 267         struct vq_desc_extra *dxp;
 268         uint16_t idx, i;
 269         uint16_t batch_num, total_num = 0;
 270         uint16_t head_idx = vq->vq_avail_idx;
 271         uint16_t head_flag = vq->vq_packed.cached_flags;
 272         uint64_t addr;
 273
 274         do {
 275                 idx = vq->vq_avail_idx;
 276
 277                 batch_num = PACKED_BATCH_SIZE;
 278                 if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
 279                         batch_num = vq->vq_nentries - idx;
 280                 if (unlikely((total_num + batch_num) > num))
 281                         batch_num = num - total_num;
 282
 283                 virtio_for_each_try_unroll(i, 0, batch_num) {
 284                         dxp = &vq->vq_descx[idx + i];
 285                         dxp->cookie = (void *)cookie[total_num + i];
 286
 287                         addr = VIRTIO_MBUF_ADDR(cookie[total_num + i], vq) +
 288                                 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
 289                         start_dp[idx + i].addr = addr;
 290                         start_dp[idx + i].len = cookie[total_num + i]->buf_len
 291                                 - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
 292                         if (total_num || i) {
 293                                 virtqueue_store_flags_packed(&start_dp[idx + i],
 294                                                 flags, hw->weak_barriers);
 295                         }
 296                 }
 297
 298                 vq->vq_avail_idx += batch_num;
 299                 if (vq->vq_avail_idx >= vq->vq_nentries) {
 300                         vq->vq_avail_idx -= vq->vq_nentries;
 301                         vq->vq_packed.cached_flags ^=
 302                                 VRING_PACKED_DESC_F_AVAIL_USED;
 303                         flags = vq->vq_packed.cached_flags;
 304                 }
 305                 total_num += batch_num;
 306         } while (total_num < num);
 307
 308         virtqueue_store_flags_packed(&start_dp[head_idx], head_flag,
 309                                 hw->weak_barriers);
 310         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
 311 }
 312
 313 uint16_t
 314 virtio_recv_pkts_packed_vec(void *rx_queue,
 315                             struct rte_mbuf **rx_pkts,
 316                             uint16_t nb_pkts)
 317 {
 318         struct virtnet_rx *rxvq = rx_queue;
 319         struct virtqueue *vq = rxvq->vq;
 320         struct virtio_hw *hw = vq->hw;
 321         uint16_t num, nb_rx = 0;
 322         uint32_t nb_enqueued = 0;
 323         uint16_t free_cnt = vq->vq_free_thresh;
 324
 325         if (unlikely(hw->started == 0))
 326                 return nb_rx;
 327
 328         num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts);
 329         if (likely(num > PACKED_BATCH_SIZE))
 330                 num = num - ((vq->vq_used_cons_idx + num) % PACKED_BATCH_SIZE);
 331
 332         while (num) {
 333                 if (!virtqueue_dequeue_batch_packed_vec(rxvq,
 334                                         &rx_pkts[nb_rx])) {
 335                         nb_rx += PACKED_BATCH_SIZE;
 336                         num -= PACKED_BATCH_SIZE;
 337                         continue;
 338                 }
 339                 if (!virtqueue_dequeue_single_packed_vec(rxvq,
 340                                         &rx_pkts[nb_rx])) {
 341                         nb_rx++;
 342                         num--;
 343                         continue;
 344                 }
 345                 break;
 346         };
 347
 348         PMD_RX_LOG(DEBUG, "dequeue:%d", num);
 349
 350         rxvq->stats.packets += nb_rx;
 351
 352         if (likely(vq->vq_free_cnt >= free_cnt)) {
 353                 struct rte_mbuf *new_pkts[free_cnt];
 354                 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
 355                                                 free_cnt) == 0)) {
 356                         virtio_recv_refill_packed_vec(rxvq, new_pkts,
 357                                         free_cnt);
 358                         nb_enqueued += free_cnt;
 359                 } else {
 360                         struct rte_eth_dev *dev =
 361                                 &rte_eth_devices[rxvq->port_id];
 362                         dev->data->rx_mbuf_alloc_failed += free_cnt;
 363                 }
 364         }
 365
 366         if (likely(nb_enqueued)) {
 367                 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
 368                         virtqueue_notify(vq);
 369                         PMD_RX_LOG(DEBUG, "Notified");
 370                 }
 371         }
 372
 373         return nb_rx;
 374 }