drivers/net/hns3/hns3_rxtx_vec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020-2021 HiSilicon Limited.
   3  */
   4
   5 #include <rte_io.h>
   6 #include <ethdev_driver.h>
   7
   8 #include "hns3_ethdev.h"
   9 #include "hns3_rxtx.h"
  10 #include "hns3_rxtx_vec.h"
  11
  12 #if defined RTE_ARCH_ARM64
  13 #include "hns3_rxtx_vec_neon.h"
  14 #endif
  15
  16 int
  17 hns3_tx_check_vec_support(struct rte_eth_dev *dev)
  18 {
  19         struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
  20         struct hns3_adapter *hns = dev->data->dev_private;
  21         struct hns3_pf *pf = &hns->pf;
  22
  23         /* Only support RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE */
  24         if (txmode->offloads != RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)
  25                 return -ENOTSUP;
  26
  27         /* Vec is not supported when PTP enabled */
  28         if (pf->ptp_enable)
  29                 return -ENOTSUP;
  30
  31         return 0;
  32 }
  33
  34 uint16_t
  35 hns3_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
  36 {
  37         struct hns3_tx_queue *txq = (struct hns3_tx_queue *)tx_queue;
  38         uint16_t nb_tx = 0;
  39
  40         while (nb_pkts) {
  41                 uint16_t ret, new_burst;
  42
  43                 new_burst = RTE_MIN(nb_pkts, txq->tx_rs_thresh);
  44                 ret = hns3_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
  45                                                 new_burst);
  46                 nb_tx += ret;
  47                 nb_pkts -= ret;
  48                 if (ret < new_burst)
  49                         break;
  50         }
  51
  52         return nb_tx;
  53 }
  54
  55 static inline void
  56 hns3_rxq_rearm_mbuf(struct hns3_rx_queue *rxq)
  57 {
  58 #define REARM_LOOP_STEP_NUM     4
  59         struct hns3_entry *rxep = &rxq->sw_ring[rxq->rx_rearm_start];
  60         struct hns3_desc *rxdp = rxq->rx_ring + rxq->rx_rearm_start;
  61         uint64_t dma_addr;
  62         int i;
  63
  64         if (unlikely(rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
  65                                           HNS3_DEFAULT_RXQ_REARM_THRESH) < 0)) {
  66                 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
  67                 return;
  68         }
  69
  70         for (i = 0; i < HNS3_DEFAULT_RXQ_REARM_THRESH; i += REARM_LOOP_STEP_NUM,
  71                 rxep += REARM_LOOP_STEP_NUM, rxdp += REARM_LOOP_STEP_NUM) {
  72                 if (likely(i <
  73                         HNS3_DEFAULT_RXQ_REARM_THRESH - REARM_LOOP_STEP_NUM)) {
  74                         rte_prefetch_non_temporal(rxep[4].mbuf);
  75                         rte_prefetch_non_temporal(rxep[5].mbuf);
  76                         rte_prefetch_non_temporal(rxep[6].mbuf);
  77                         rte_prefetch_non_temporal(rxep[7].mbuf);
  78                 }
  79
  80                 dma_addr = rte_mbuf_data_iova_default(rxep[0].mbuf);
  81                 rxdp[0].addr = rte_cpu_to_le_64(dma_addr);
  82                 rxdp[0].rx.bd_base_info = 0;
  83
  84                 dma_addr = rte_mbuf_data_iova_default(rxep[1].mbuf);
  85                 rxdp[1].addr = rte_cpu_to_le_64(dma_addr);
  86                 rxdp[1].rx.bd_base_info = 0;
  87
  88                 dma_addr = rte_mbuf_data_iova_default(rxep[2].mbuf);
  89                 rxdp[2].addr = rte_cpu_to_le_64(dma_addr);
  90                 rxdp[2].rx.bd_base_info = 0;
  91
  92                 dma_addr = rte_mbuf_data_iova_default(rxep[3].mbuf);
  93                 rxdp[3].addr = rte_cpu_to_le_64(dma_addr);
  94                 rxdp[3].rx.bd_base_info = 0;
  95         }
  96
  97         rxq->rx_rearm_start += HNS3_DEFAULT_RXQ_REARM_THRESH;
  98         if (rxq->rx_rearm_start >= rxq->nb_rx_desc)
  99                 rxq->rx_rearm_start = 0;
 100
 101         rxq->rx_rearm_nb -= HNS3_DEFAULT_RXQ_REARM_THRESH;
 102
 103         hns3_write_reg_opt(rxq->io_head_reg, HNS3_DEFAULT_RXQ_REARM_THRESH);
 104 }
 105
 106 uint16_t
 107 hns3_recv_pkts_vec(void *__restrict rx_queue,
 108                    struct rte_mbuf **__restrict rx_pkts,
 109                    uint16_t nb_pkts)
 110 {
 111         struct hns3_rx_queue *rxq = rx_queue;
 112         struct hns3_desc *rxdp = &rxq->rx_ring[rxq->next_to_use];
 113         uint64_t pkt_err_mask;  /* bit mask indicate whick pkts is error */
 114         uint16_t nb_rx;
 115
 116         rte_prefetch_non_temporal(rxdp);
 117
 118         nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, HNS3_DEFAULT_DESCS_PER_LOOP);
 119
 120         if (rxq->rx_rearm_nb > HNS3_DEFAULT_RXQ_REARM_THRESH)
 121                 hns3_rxq_rearm_mbuf(rxq);
 122
 123         if (unlikely(!(rxdp->rx.bd_base_info &
 124                         rte_cpu_to_le_32(1u << HNS3_RXD_VLD_B))))
 125                 return 0;
 126
 127         rte_prefetch0(rxq->sw_ring[rxq->next_to_use + 0].mbuf);
 128         rte_prefetch0(rxq->sw_ring[rxq->next_to_use + 1].mbuf);
 129         rte_prefetch0(rxq->sw_ring[rxq->next_to_use + 2].mbuf);
 130         rte_prefetch0(rxq->sw_ring[rxq->next_to_use + 3].mbuf);
 131
 132         if (likely(nb_pkts <= HNS3_DEFAULT_RX_BURST)) {
 133                 pkt_err_mask = 0;
 134                 nb_rx = hns3_recv_burst_vec(rxq, rx_pkts, nb_pkts,
 135                                             &pkt_err_mask);
 136                 nb_rx = hns3_rx_reassemble_pkts(rx_pkts, nb_rx, pkt_err_mask);
 137                 return nb_rx;
 138         }
 139
 140         nb_rx = 0;
 141         while (nb_pkts > 0) {
 142                 uint16_t ret, n;
 143
 144                 n = RTE_MIN(nb_pkts, HNS3_DEFAULT_RX_BURST);
 145                 pkt_err_mask = 0;
 146                 ret = hns3_recv_burst_vec(rxq, &rx_pkts[nb_rx], n,
 147                                           &pkt_err_mask);
 148                 nb_pkts -= ret;
 149                 nb_rx += hns3_rx_reassemble_pkts(&rx_pkts[nb_rx], ret,
 150                                                  pkt_err_mask);
 151                 if (ret < n)
 152                         break;
 153
 154                 if (rxq->rx_rearm_nb > HNS3_DEFAULT_RXQ_REARM_THRESH)
 155                         hns3_rxq_rearm_mbuf(rxq);
 156         }
 157
 158         return nb_rx;
 159 }
 160
 161 static void
 162 hns3_rxq_vec_setup_rearm_data(struct hns3_rx_queue *rxq)
 163 {
 164         uintptr_t p;
 165         struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
 166
 167         mb_def.nb_segs = 1;
 168         mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 169         mb_def.port = rxq->port_id;
 170         rte_mbuf_refcnt_set(&mb_def, 1);
 171
 172         /* compile-time verifies the rearm_data first 8bytes */
 173         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) <
 174                          offsetof(struct rte_mbuf, rearm_data));
 175         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) <
 176                          offsetof(struct rte_mbuf, rearm_data));
 177         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) <
 178                          offsetof(struct rte_mbuf, rearm_data));
 179         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) <
 180                          offsetof(struct rte_mbuf, rearm_data));
 181         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) -
 182                          offsetof(struct rte_mbuf, rearm_data) > 6);
 183         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) -
 184                          offsetof(struct rte_mbuf, rearm_data) > 6);
 185         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) -
 186                          offsetof(struct rte_mbuf, rearm_data) > 6);
 187         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) -
 188                          offsetof(struct rte_mbuf, rearm_data) > 6);
 189
 190         /* prevent compiler reordering: rearm_data covers previous fields */
 191         rte_compiler_barrier();
 192         p = (uintptr_t)&mb_def.rearm_data;
 193         rxq->mbuf_initializer = *(uint64_t *)p;
 194 }
 195
 196 void
 197 hns3_rxq_vec_setup(struct hns3_rx_queue *rxq)
 198 {
 199         struct hns3_entry *sw_ring = &rxq->sw_ring[rxq->nb_rx_desc];
 200         unsigned int i;
 201
 202         memset(&rxq->rx_ring[rxq->nb_rx_desc], 0,
 203                 sizeof(struct hns3_desc) * HNS3_DEFAULT_RX_BURST);
 204
 205         memset(&rxq->fake_mbuf, 0, sizeof(rxq->fake_mbuf));
 206         for (i = 0; i < HNS3_DEFAULT_RX_BURST; i++)
 207                 sw_ring[i].mbuf = &rxq->fake_mbuf;
 208
 209         hns3_rxq_vec_setup_rearm_data(rxq);
 210
 211         memset(rxq->offset_table, 0, sizeof(rxq->offset_table));
 212 }
 213
 214 static int
 215 hns3_rxq_vec_check(struct hns3_rx_queue *rxq, void *arg)
 216 {
 217         uint32_t min_vec_bds = HNS3_DEFAULT_RXQ_REARM_THRESH +
 218                                 HNS3_DEFAULT_RX_BURST;
 219
 220         if (rxq->nb_rx_desc < min_vec_bds)
 221                 return -ENOTSUP;
 222
 223         if (rxq->nb_rx_desc % HNS3_DEFAULT_RXQ_REARM_THRESH)
 224                 return -ENOTSUP;
 225
 226         RTE_SET_USED(arg);
 227         return 0;
 228 }
 229
 230 int
 231 hns3_rx_check_vec_support(struct rte_eth_dev *dev)
 232 {
 233         struct rte_eth_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf;
 234         struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
 235         uint64_t offloads_mask = RTE_ETH_RX_OFFLOAD_TCP_LRO |
 236                                  RTE_ETH_RX_OFFLOAD_VLAN;
 237         struct hns3_adapter *hns = dev->data->dev_private;
 238         struct hns3_pf *pf = &hns->pf;
 239
 240         if (dev->data->scattered_rx)
 241                 return -ENOTSUP;
 242
 243         if (fconf->mode != RTE_FDIR_MODE_NONE)
 244                 return -ENOTSUP;
 245
 246         if (rxmode->offloads & offloads_mask)
 247                 return -ENOTSUP;
 248
 249         if (hns3_rxq_iterate(dev, hns3_rxq_vec_check, NULL) != 0)
 250                 return -ENOTSUP;
 251
 252         /* Vec is not supported when PTP enabled */
 253         if (pf->ptp_enable)
 254                 return -ENOTSUP;
 255
 256         return 0;
 257 }