drivers/net/enetc/enetc_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2020 NXP
   3  */
   4
   5 #include <stdbool.h>
   6 #include <stdint.h>
   7 #include <unistd.h>
   8
   9 #include "rte_ethdev.h"
  10 #include "rte_malloc.h"
  11 #include "rte_memzone.h"
  12
  13 #include "base/enetc_hw.h"
  14 #include "enetc.h"
  15 #include "enetc_logs.h"
  16
  17 #define ENETC_RXBD_BUNDLE 16 /* Number of buffers to allocate at once */
  18
  19 static int
  20 enetc_clean_tx_ring(struct enetc_bdr *tx_ring)
  21 {
  22         int tx_frm_cnt = 0;
  23         struct enetc_swbd *tx_swbd, *tx_swbd_base;
  24         int i, hwci, bd_count;
  25         struct rte_mbuf *m[ENETC_RXBD_BUNDLE];
  26
  27         /* we don't need barriers here, we just want a relatively current value
  28          * from HW.
  29          */
  30         hwci = (int)(rte_read32_relaxed(tx_ring->tcisr) &
  31                      ENETC_TBCISR_IDX_MASK);
  32
  33         tx_swbd_base = tx_ring->q_swbd;
  34         bd_count = tx_ring->bd_count;
  35         i = tx_ring->next_to_clean;
  36         tx_swbd = &tx_swbd_base[i];
  37
  38         /* we're only reading the CI index once here, which means HW may update
  39          * it while we're doing clean-up.  We could read the register in a loop
  40          * but for now I assume it's OK to leave a few Tx frames for next call.
  41          * The issue with reading the register in a loop is that we're stalling
  42          * here trying to catch up with HW which keeps sending traffic as long
  43          * as it has traffic to send, so in effect we could be waiting here for
  44          * the Tx ring to be drained by HW, instead of us doing Rx in that
  45          * meantime.
  46          */
  47         while (i != hwci) {
  48                 /* It seems calling rte_pktmbuf_free is wasting a lot of cycles,
  49                  * make a list and call _free when it's done.
  50                  */
  51                 if (tx_frm_cnt == ENETC_RXBD_BUNDLE) {
  52                         rte_pktmbuf_free_bulk(m, tx_frm_cnt);
  53                         tx_frm_cnt = 0;
  54                 }
  55
  56                 m[tx_frm_cnt] = tx_swbd->buffer_addr;
  57                 tx_swbd->buffer_addr = NULL;
  58
  59                 i++;
  60                 tx_swbd++;
  61                 if (unlikely(i == bd_count)) {
  62                         i = 0;
  63                         tx_swbd = tx_swbd_base;
  64                 }
  65
  66                 tx_frm_cnt++;
  67         }
  68
  69         if (tx_frm_cnt)
  70                 rte_pktmbuf_free_bulk(m, tx_frm_cnt);
  71
  72         tx_ring->next_to_clean = i;
  73
  74         return 0;
  75 }
  76
  77 uint16_t
  78 enetc_xmit_pkts(void *tx_queue,
  79                 struct rte_mbuf **tx_pkts,
  80                 uint16_t nb_pkts)
  81 {
  82         struct enetc_swbd *tx_swbd;
  83         int i, start, bds_to_use;
  84         struct enetc_tx_bd *txbd;
  85         struct enetc_bdr *tx_ring = (struct enetc_bdr *)tx_queue;
  86
  87         i = tx_ring->next_to_use;
  88
  89         bds_to_use = enetc_bd_unused(tx_ring);
  90         if (bds_to_use < nb_pkts)
  91                 nb_pkts = bds_to_use;
  92
  93         start = 0;
  94         while (nb_pkts--) {
  95                 tx_ring->q_swbd[i].buffer_addr = tx_pkts[start];
  96                 txbd = ENETC_TXBD(*tx_ring, i);
  97                 tx_swbd = &tx_ring->q_swbd[i];
  98                 txbd->frm_len = tx_pkts[start]->pkt_len;
  99                 txbd->buf_len = txbd->frm_len;
 100                 txbd->flags = rte_cpu_to_le_16(ENETC_TXBD_FLAGS_F);
 101                 txbd->addr = (uint64_t)(uintptr_t)
 102                 rte_cpu_to_le_64((size_t)tx_swbd->buffer_addr->buf_iova +
 103                                  tx_swbd->buffer_addr->data_off);
 104                 i++;
 105                 start++;
 106                 if (unlikely(i == tx_ring->bd_count))
 107                         i = 0;
 108         }
 109
 110         /* we're only cleaning up the Tx ring here, on the assumption that
 111          * software is slower than hardware and hardware completed sending
 112          * older frames out by now.
 113          * We're also cleaning up the ring before kicking off Tx for the new
 114          * batch to minimize chances of contention on the Tx ring
 115          */
 116         enetc_clean_tx_ring(tx_ring);
 117
 118         tx_ring->next_to_use = i;
 119         enetc_wr_reg(tx_ring->tcir, i);
 120         return start;
 121 }
 122
 123 int
 124 enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
 125 {
 126         struct enetc_swbd *rx_swbd;
 127         union enetc_rx_bd *rxbd;
 128         int i, j, k = ENETC_RXBD_BUNDLE;
 129         struct rte_mbuf *m[ENETC_RXBD_BUNDLE];
 130         struct rte_mempool *mb_pool;
 131
 132         i = rx_ring->next_to_use;
 133         mb_pool = rx_ring->mb_pool;
 134         rx_swbd = &rx_ring->q_swbd[i];
 135         rxbd = ENETC_RXBD(*rx_ring, i);
 136         for (j = 0; j < buff_cnt; j++) {
 137                 /* bulk alloc for the next up to 8 BDs */
 138                 if (k == ENETC_RXBD_BUNDLE) {
 139                         k = 0;
 140                         int m_cnt = RTE_MIN(buff_cnt - j, ENETC_RXBD_BUNDLE);
 141
 142                         if (rte_pktmbuf_alloc_bulk(mb_pool, m, m_cnt))
 143                                 return -1;
 144                 }
 145
 146                 rx_swbd->buffer_addr = m[k];
 147                 rxbd->w.addr = (uint64_t)(uintptr_t)
 148                                rx_swbd->buffer_addr->buf_iova +
 149                                rx_swbd->buffer_addr->data_off;
 150                 /* clear 'R" as well */
 151                 rxbd->r.lstatus = 0;
 152                 rx_swbd++;
 153                 rxbd++;
 154                 i++;
 155                 k++;
 156                 if (unlikely(i == rx_ring->bd_count)) {
 157                         i = 0;
 158                         rxbd = ENETC_RXBD(*rx_ring, 0);
 159                         rx_swbd = &rx_ring->q_swbd[i];
 160                 }
 161         }
 162
 163         if (likely(j)) {
 164                 rx_ring->next_to_alloc = i;
 165                 rx_ring->next_to_use = i;
 166                 enetc_wr_reg(rx_ring->rcir, i);
 167         }
 168
 169         return j;
 170 }
 171
 172 static inline void enetc_slow_parsing(struct rte_mbuf *m,
 173                                      uint64_t parse_results)
 174 {
 175         m->ol_flags &= ~(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 176
 177         switch (parse_results) {
 178         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4:
 179                 m->packet_type = RTE_PTYPE_L2_ETHER |
 180                                  RTE_PTYPE_L3_IPV4;
 181                 m->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 182                 return;
 183         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6:
 184                 m->packet_type = RTE_PTYPE_L2_ETHER |
 185                                  RTE_PTYPE_L3_IPV6;
 186                 m->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 187                 return;
 188         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4_TCP:
 189                 m->packet_type = RTE_PTYPE_L2_ETHER |
 190                                  RTE_PTYPE_L3_IPV4 |
 191                                  RTE_PTYPE_L4_TCP;
 192                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 193                                PKT_RX_L4_CKSUM_BAD;
 194                 return;
 195         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6_TCP:
 196                 m->packet_type = RTE_PTYPE_L2_ETHER |
 197                                  RTE_PTYPE_L3_IPV6 |
 198                                  RTE_PTYPE_L4_TCP;
 199                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 200                                PKT_RX_L4_CKSUM_BAD;
 201                 return;
 202         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4_UDP:
 203                 m->packet_type = RTE_PTYPE_L2_ETHER |
 204                                  RTE_PTYPE_L3_IPV4 |
 205                                  RTE_PTYPE_L4_UDP;
 206                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 207                                PKT_RX_L4_CKSUM_BAD;
 208                 return;
 209         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6_UDP:
 210                 m->packet_type = RTE_PTYPE_L2_ETHER |
 211                                  RTE_PTYPE_L3_IPV6 |
 212                                  RTE_PTYPE_L4_UDP;
 213                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 214                                PKT_RX_L4_CKSUM_BAD;
 215                 return;
 216         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4_SCTP:
 217                 m->packet_type = RTE_PTYPE_L2_ETHER |
 218                                  RTE_PTYPE_L3_IPV4 |
 219                                  RTE_PTYPE_L4_SCTP;
 220                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 221                                PKT_RX_L4_CKSUM_BAD;
 222                 return;
 223         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6_SCTP:
 224                 m->packet_type = RTE_PTYPE_L2_ETHER |
 225                                  RTE_PTYPE_L3_IPV6 |
 226                                  RTE_PTYPE_L4_SCTP;
 227                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 228                                PKT_RX_L4_CKSUM_BAD;
 229                 return;
 230         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4_ICMP:
 231                 m->packet_type = RTE_PTYPE_L2_ETHER |
 232                                  RTE_PTYPE_L3_IPV4 |
 233                                  RTE_PTYPE_L4_ICMP;
 234                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 235                                PKT_RX_L4_CKSUM_BAD;
 236                 return;
 237         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6_ICMP:
 238                 m->packet_type = RTE_PTYPE_L2_ETHER |
 239                                  RTE_PTYPE_L3_IPV6 |
 240                                  RTE_PTYPE_L4_ICMP;
 241                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 242                                PKT_RX_L4_CKSUM_BAD;
 243                 return;
 244         /* More switch cases can be added */
 245         default:
 246                 m->packet_type = RTE_PTYPE_UNKNOWN;
 247                 m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN |
 248                                PKT_RX_L4_CKSUM_UNKNOWN;
 249         }
 250 }
 251
 252
 253 static inline void __attribute__((hot))
 254 enetc_dev_rx_parse(struct rte_mbuf *m, uint16_t parse_results)
 255 {
 256         ENETC_PMD_DP_DEBUG("parse summary = 0x%x   ", parse_results);
 257         m->ol_flags |= PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD;
 258
 259         switch (parse_results) {
 260         case ENETC_PKT_TYPE_ETHER:
 261                 m->packet_type = RTE_PTYPE_L2_ETHER;
 262                 return;
 263         case ENETC_PKT_TYPE_IPV4:
 264                 m->packet_type = RTE_PTYPE_L2_ETHER |
 265                                  RTE_PTYPE_L3_IPV4;
 266                 return;
 267         case ENETC_PKT_TYPE_IPV6:
 268                 m->packet_type = RTE_PTYPE_L2_ETHER |
 269                                  RTE_PTYPE_L3_IPV6;
 270                 return;
 271         case ENETC_PKT_TYPE_IPV4_TCP:
 272                 m->packet_type = RTE_PTYPE_L2_ETHER |
 273                                  RTE_PTYPE_L3_IPV4 |
 274                                  RTE_PTYPE_L4_TCP;
 275                 return;
 276         case ENETC_PKT_TYPE_IPV6_TCP:
 277                 m->packet_type = RTE_PTYPE_L2_ETHER |
 278                                  RTE_PTYPE_L3_IPV6 |
 279                                  RTE_PTYPE_L4_TCP;
 280                 return;
 281         case ENETC_PKT_TYPE_IPV4_UDP:
 282                 m->packet_type = RTE_PTYPE_L2_ETHER |
 283                                  RTE_PTYPE_L3_IPV4 |
 284                                  RTE_PTYPE_L4_UDP;
 285                 return;
 286         case ENETC_PKT_TYPE_IPV6_UDP:
 287                 m->packet_type = RTE_PTYPE_L2_ETHER |
 288                                  RTE_PTYPE_L3_IPV6 |
 289                                  RTE_PTYPE_L4_UDP;
 290                 return;
 291         case ENETC_PKT_TYPE_IPV4_SCTP:
 292                 m->packet_type = RTE_PTYPE_L2_ETHER |
 293                                  RTE_PTYPE_L3_IPV4 |
 294                                  RTE_PTYPE_L4_SCTP;
 295                 return;
 296         case ENETC_PKT_TYPE_IPV6_SCTP:
 297                 m->packet_type = RTE_PTYPE_L2_ETHER |
 298                                  RTE_PTYPE_L3_IPV6 |
 299                                  RTE_PTYPE_L4_SCTP;
 300                 return;
 301         case ENETC_PKT_TYPE_IPV4_ICMP:
 302                 m->packet_type = RTE_PTYPE_L2_ETHER |
 303                                  RTE_PTYPE_L3_IPV4 |
 304                                  RTE_PTYPE_L4_ICMP;
 305                 return;
 306         case ENETC_PKT_TYPE_IPV6_ICMP:
 307                 m->packet_type = RTE_PTYPE_L2_ETHER |
 308                                  RTE_PTYPE_L3_IPV6 |
 309                                  RTE_PTYPE_L4_ICMP;
 310                 return;
 311         /* More switch cases can be added */
 312         default:
 313                 enetc_slow_parsing(m, parse_results);
 314         }
 315
 316 }
 317
 318 static int
 319 enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
 320                     struct rte_mbuf **rx_pkts,
 321                     int work_limit)
 322 {
 323         int rx_frm_cnt = 0;
 324         int cleaned_cnt, i;
 325         struct enetc_swbd *rx_swbd;
 326
 327         cleaned_cnt = enetc_bd_unused(rx_ring);
 328         /* next descriptor to process */
 329         i = rx_ring->next_to_clean;
 330         rx_swbd = &rx_ring->q_swbd[i];
 331         while (likely(rx_frm_cnt < work_limit)) {
 332                 union enetc_rx_bd *rxbd;
 333                 uint32_t bd_status;
 334
 335                 rxbd = ENETC_RXBD(*rx_ring, i);
 336                 bd_status = rte_le_to_cpu_32(rxbd->r.lstatus);
 337                 if (!bd_status)
 338                         break;
 339
 340                 rx_swbd->buffer_addr->pkt_len = rxbd->r.buf_len -
 341                                                 rx_ring->crc_len;
 342                 rx_swbd->buffer_addr->data_len = rxbd->r.buf_len -
 343                                                  rx_ring->crc_len;
 344                 rx_swbd->buffer_addr->hash.rss = rxbd->r.rss_hash;
 345                 rx_swbd->buffer_addr->ol_flags = 0;
 346                 enetc_dev_rx_parse(rx_swbd->buffer_addr,
 347                                    rxbd->r.parse_summary);
 348                 rx_pkts[rx_frm_cnt] = rx_swbd->buffer_addr;
 349                 cleaned_cnt++;
 350                 rx_swbd++;
 351                 i++;
 352                 if (unlikely(i == rx_ring->bd_count)) {
 353                         i = 0;
 354                         rx_swbd = &rx_ring->q_swbd[i];
 355                 }
 356
 357                 rx_ring->next_to_clean = i;
 358                 rx_frm_cnt++;
 359         }
 360
 361         enetc_refill_rx_ring(rx_ring, cleaned_cnt);
 362
 363         return rx_frm_cnt;
 364 }
 365
 366 uint16_t
 367 enetc_recv_pkts(void *rxq, struct rte_mbuf **rx_pkts,
 368                 uint16_t nb_pkts)
 369 {
 370         struct enetc_bdr *rx_ring = (struct enetc_bdr *)rxq;
 371
 372         return enetc_clean_rx_ring(rx_ring, rx_pkts, nb_pkts);
 373 }