drivers/net/ice/ice_rxtx_vec_common.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #ifndef _ICE_RXTX_VEC_COMMON_H_
   6 #define _ICE_RXTX_VEC_COMMON_H_
   7
   8 #include "ice_rxtx.h"
   9
  10 #ifndef __INTEL_COMPILER
  11 #pragma GCC diagnostic ignored "-Wcast-qual"
  12 #endif
  13
  14 static inline uint16_t
  15 ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs,
  16                           uint16_t nb_bufs, uint8_t *split_flags)
  17 {
  18         struct rte_mbuf *pkts[ICE_VPMD_RX_BURST] = {0}; /*finished pkts*/
  19         struct rte_mbuf *start = rxq->pkt_first_seg;
  20         struct rte_mbuf *end =  rxq->pkt_last_seg;
  21         unsigned int pkt_idx, buf_idx;
  22
  23         for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
  24                 if (end) {
  25                         /* processing a split packet */
  26                         end->next = rx_bufs[buf_idx];
  27                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  28
  29                         start->nb_segs++;
  30                         start->pkt_len += rx_bufs[buf_idx]->data_len;
  31                         end = end->next;
  32
  33                         if (!split_flags[buf_idx]) {
  34                                 /* it's the last packet of the set */
  35                                 start->hash = end->hash;
  36                                 start->vlan_tci = end->vlan_tci;
  37                                 start->ol_flags = end->ol_flags;
  38                                 /* we need to strip crc for the whole packet */
  39                                 start->pkt_len -= rxq->crc_len;
  40                                 if (end->data_len > rxq->crc_len) {
  41                                         end->data_len -= rxq->crc_len;
  42                                 } else {
  43                                         /* free up last mbuf */
  44                                         struct rte_mbuf *secondlast = start;
  45
  46                                         start->nb_segs--;
  47                                         while (secondlast->next != end)
  48                                                 secondlast = secondlast->next;
  49                                         secondlast->data_len -= (rxq->crc_len -
  50                                                         end->data_len);
  51                                         secondlast->next = NULL;
  52                                         rte_pktmbuf_free_seg(end);
  53                                 }
  54                                 pkts[pkt_idx++] = start;
  55                                 start = NULL;
  56                                 end = NULL;
  57                         }
  58                 } else {
  59                         /* not processing a split packet */
  60                         if (!split_flags[buf_idx]) {
  61                                 /* not a split packet, save and skip */
  62                                 pkts[pkt_idx++] = rx_bufs[buf_idx];
  63                                 continue;
  64                         }
  65                         start = rx_bufs[buf_idx];
  66                         end = start;
  67                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  68                         rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
  69                 }
  70         }
  71
  72         /* save the partial packet for next time */
  73         rxq->pkt_first_seg = start;
  74         rxq->pkt_last_seg = end;
  75         rte_memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
  76         return pkt_idx;
  77 }
  78
  79 static __rte_always_inline int
  80 ice_tx_free_bufs_vec(struct ice_tx_queue *txq)
  81 {
  82         struct ice_tx_entry *txep;
  83         uint32_t n;
  84         uint32_t i;
  85         int nb_free = 0;
  86         struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ];
  87
  88         /* check DD bits on threshold descriptor */
  89         if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
  90                         rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) !=
  91                         rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
  92                 return 0;
  93
  94         n = txq->tx_rs_thresh;
  95
  96          /* first buffer to free from S/W ring is at index
  97           * tx_next_dd - (tx_rs_thresh-1)
  98           */
  99         txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
 100         m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
 101         if (likely(m)) {
 102                 free[0] = m;
 103                 nb_free = 1;
 104                 for (i = 1; i < n; i++) {
 105                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 106                         if (likely(m)) {
 107                                 if (likely(m->pool == free[0]->pool)) {
 108                                         free[nb_free++] = m;
 109                                 } else {
 110                                         rte_mempool_put_bulk(free[0]->pool,
 111                                                              (void *)free,
 112                                                              nb_free);
 113                                         free[0] = m;
 114                                         nb_free = 1;
 115                                 }
 116                         }
 117                 }
 118                 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
 119         } else {
 120                 for (i = 1; i < n; i++) {
 121                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 122                         if (m)
 123                                 rte_mempool_put(m->pool, m);
 124                 }
 125         }
 126
 127         /* buffers were freed, update counters */
 128         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
 129         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
 130         if (txq->tx_next_dd >= txq->nb_tx_desc)
 131                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
 132
 133         return txq->tx_rs_thresh;
 134 }
 135
 136 static __rte_always_inline void
 137 ice_tx_backlog_entry(struct ice_tx_entry *txep,
 138                      struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 139 {
 140         int i;
 141
 142         for (i = 0; i < (int)nb_pkts; ++i)
 143                 txep[i].mbuf = tx_pkts[i];
 144 }
 145
 146 static inline void
 147 _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq)
 148 {
 149         const unsigned int mask = rxq->nb_rx_desc - 1;
 150         unsigned int i;
 151
 152         if (unlikely(!rxq->sw_ring)) {
 153                 PMD_DRV_LOG(DEBUG, "sw_ring is NULL");
 154                 return;
 155         }
 156
 157         if (rxq->rxrearm_nb >= rxq->nb_rx_desc)
 158                 return;
 159
 160         /* free all mbufs that are valid in the ring */
 161         if (rxq->rxrearm_nb == 0) {
 162                 for (i = 0; i < rxq->nb_rx_desc; i++) {
 163                         if (rxq->sw_ring[i].mbuf)
 164                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 165                 }
 166         } else {
 167                 for (i = rxq->rx_tail;
 168                      i != rxq->rxrearm_start;
 169                      i = (i + 1) & mask) {
 170                         if (rxq->sw_ring[i].mbuf)
 171                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 172                 }
 173         }
 174
 175         rxq->rxrearm_nb = rxq->nb_rx_desc;
 176
 177         /* set all entries to NULL */
 178         memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
 179 }
 180
 181 static inline void
 182 _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq)
 183 {
 184         uint16_t i;
 185
 186         if (unlikely(!txq || !txq->sw_ring)) {
 187                 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
 188                 return;
 189         }
 190
 191         /**
 192          *  vPMD tx will not set sw_ring's mbuf to NULL after free,
 193          *  so need to free remains more carefully.
 194          */
 195         i = txq->tx_next_dd - txq->tx_rs_thresh + 1;
 196
 197 #ifdef __AVX512VL__
 198         struct rte_eth_dev *dev = &rte_eth_devices[txq->vsi->adapter->pf.dev_data->port_id];
 199
 200         if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 ||
 201             dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) {
 202                 struct ice_vec_tx_entry *swr = (void *)txq->sw_ring;
 203
 204                 if (txq->tx_tail < i) {
 205                         for (; i < txq->nb_tx_desc; i++) {
 206                                 rte_pktmbuf_free_seg(swr[i].mbuf);
 207                                 swr[i].mbuf = NULL;
 208                         }
 209                         i = 0;
 210                 }
 211                 for (; i < txq->tx_tail; i++) {
 212                         rte_pktmbuf_free_seg(swr[i].mbuf);
 213                         swr[i].mbuf = NULL;
 214                 }
 215         } else
 216 #endif
 217         {
 218                 if (txq->tx_tail < i) {
 219                         for (; i < txq->nb_tx_desc; i++) {
 220                                 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 221                                 txq->sw_ring[i].mbuf = NULL;
 222                         }
 223                         i = 0;
 224                 }
 225                 for (; i < txq->tx_tail; i++) {
 226                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 227                         txq->sw_ring[i].mbuf = NULL;
 228                 }
 229         }
 230 }
 231
 232 static inline int
 233 ice_rxq_vec_setup_default(struct ice_rx_queue *rxq)
 234 {
 235         uintptr_t p;
 236         struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
 237
 238         mb_def.nb_segs = 1;
 239         mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 240         mb_def.port = rxq->port_id;
 241         rte_mbuf_refcnt_set(&mb_def, 1);
 242
 243         /* prevent compiler reordering: rearm_data covers previous fields */
 244         rte_compiler_barrier();
 245         p = (uintptr_t)&mb_def.rearm_data;
 246         rxq->mbuf_initializer = *(uint64_t *)p;
 247         return 0;
 248 }
 249
 250 #define ICE_TX_NO_VECTOR_FLAGS (                        \
 251                 RTE_ETH_TX_OFFLOAD_MULTI_SEGS |         \
 252                 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |   \
 253                 RTE_ETH_TX_OFFLOAD_TCP_TSO)
 254
 255 #define ICE_TX_VECTOR_OFFLOAD (                         \
 256                 RTE_ETH_TX_OFFLOAD_VLAN_INSERT |                \
 257                 RTE_ETH_TX_OFFLOAD_QINQ_INSERT |                \
 258                 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |         \
 259                 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM |         \
 260                 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |          \
 261                 RTE_ETH_TX_OFFLOAD_TCP_CKSUM)
 262
 263 #define ICE_RX_VECTOR_OFFLOAD (                         \
 264                 RTE_ETH_RX_OFFLOAD_CHECKSUM |           \
 265                 RTE_ETH_RX_OFFLOAD_SCTP_CKSUM |         \
 266                 RTE_ETH_RX_OFFLOAD_VLAN |                       \
 267                 RTE_ETH_RX_OFFLOAD_RSS_HASH)
 268
 269 #define ICE_VECTOR_PATH         0
 270 #define ICE_VECTOR_OFFLOAD_PATH 1
 271
 272 static inline int
 273 ice_rx_vec_queue_default(struct ice_rx_queue *rxq)
 274 {
 275         if (!rxq)
 276                 return -1;
 277
 278         if (!rte_is_power_of_2(rxq->nb_rx_desc))
 279                 return -1;
 280
 281         if (rxq->rx_free_thresh < ICE_VPMD_RX_BURST)
 282                 return -1;
 283
 284         if (rxq->nb_rx_desc % rxq->rx_free_thresh)
 285                 return -1;
 286
 287         if (rxq->proto_xtr != PROTO_XTR_NONE)
 288                 return -1;
 289
 290         if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)
 291                 return -1;
 292
 293         if (rxq->offloads & ICE_RX_VECTOR_OFFLOAD)
 294                 return ICE_VECTOR_OFFLOAD_PATH;
 295
 296         return ICE_VECTOR_PATH;
 297 }
 298
 299 static inline int
 300 ice_tx_vec_queue_default(struct ice_tx_queue *txq)
 301 {
 302         if (!txq)
 303                 return -1;
 304
 305         if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST ||
 306             txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ)
 307                 return -1;
 308
 309         if (txq->offloads & ICE_TX_NO_VECTOR_FLAGS)
 310                 return -1;
 311
 312         if (txq->offloads & ICE_TX_VECTOR_OFFLOAD)
 313                 return ICE_VECTOR_OFFLOAD_PATH;
 314
 315         return ICE_VECTOR_PATH;
 316 }
 317
 318 static inline int
 319 ice_rx_vec_dev_check_default(struct rte_eth_dev *dev)
 320 {
 321         int i;
 322         struct ice_rx_queue *rxq;
 323         int ret = 0;
 324         int result = 0;
 325
 326         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 327                 rxq = dev->data->rx_queues[i];
 328                 ret = (ice_rx_vec_queue_default(rxq));
 329                 if (ret < 0)
 330                         return -1;
 331                 if (ret == ICE_VECTOR_OFFLOAD_PATH)
 332                         result = ret;
 333         }
 334
 335         return result;
 336 }
 337
 338 static inline int
 339 ice_tx_vec_dev_check_default(struct rte_eth_dev *dev)
 340 {
 341         int i;
 342         struct ice_tx_queue *txq;
 343         int ret = 0;
 344         int result = 0;
 345
 346         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 347                 txq = dev->data->tx_queues[i];
 348                 ret = ice_tx_vec_queue_default(txq);
 349                 if (ret < 0)
 350                         return -1;
 351                 if (ret == ICE_VECTOR_OFFLOAD_PATH)
 352                         result = ret;
 353         }
 354
 355         return result;
 356 }
 357
 358 static inline void
 359 ice_txd_enable_offload(struct rte_mbuf *tx_pkt,
 360                        uint64_t *txd_hi)
 361 {
 362         uint64_t ol_flags = tx_pkt->ol_flags;
 363         uint32_t td_cmd = 0;
 364         uint32_t td_offset = 0;
 365
 366         /* Tx Checksum Offload */
 367         /* SET MACLEN */
 368         td_offset |= (tx_pkt->l2_len >> 1) <<
 369                         ICE_TX_DESC_LEN_MACLEN_S;
 370
 371         /* Enable L3 checksum offload */
 372         if (ol_flags & PKT_TX_IP_CKSUM) {
 373                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM;
 374                 td_offset |= (tx_pkt->l3_len >> 2) <<
 375                         ICE_TX_DESC_LEN_IPLEN_S;
 376         } else if (ol_flags & PKT_TX_IPV4) {
 377                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4;
 378                 td_offset |= (tx_pkt->l3_len >> 2) <<
 379                         ICE_TX_DESC_LEN_IPLEN_S;
 380         } else if (ol_flags & PKT_TX_IPV6) {
 381                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6;
 382                 td_offset |= (tx_pkt->l3_len >> 2) <<
 383                         ICE_TX_DESC_LEN_IPLEN_S;
 384         }
 385
 386         /* Enable L4 checksum offloads */
 387         switch (ol_flags & PKT_TX_L4_MASK) {
 388         case PKT_TX_TCP_CKSUM:
 389                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
 390                 td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
 391                         ICE_TX_DESC_LEN_L4_LEN_S;
 392                 break;
 393         case PKT_TX_SCTP_CKSUM:
 394                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP;
 395                 td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
 396                         ICE_TX_DESC_LEN_L4_LEN_S;
 397                 break;
 398         case PKT_TX_UDP_CKSUM:
 399                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP;
 400                 td_offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
 401                         ICE_TX_DESC_LEN_L4_LEN_S;
 402                 break;
 403         default:
 404                 break;
 405         }
 406
 407         *txd_hi |= ((uint64_t)td_offset) << ICE_TXD_QW1_OFFSET_S;
 408
 409         /* Tx VLAN/QINQ insertion Offload */
 410         if (ol_flags & (PKT_TX_VLAN | PKT_TX_QINQ)) {
 411                 td_cmd |= ICE_TX_DESC_CMD_IL2TAG1;
 412                 *txd_hi |= ((uint64_t)tx_pkt->vlan_tci <<
 413                                 ICE_TXD_QW1_L2TAG1_S);
 414         }
 415
 416         *txd_hi |= ((uint64_t)td_cmd) << ICE_TXD_QW1_CMD_S;
 417 }
 418 #endif