drivers/net/enetc/enetc_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2020 NXP
   3  */
   4
   5 #include <stdbool.h>
   6 #include <stdint.h>
   7 #include <unistd.h>
   8
   9 #include "rte_ethdev.h"
  10 #include "rte_malloc.h"
  11 #include "rte_memzone.h"
  12
  13 #include "base/enetc_hw.h"
  14 #include "enetc.h"
  15 #include "enetc_logs.h"
  16
  17 static int
  18 enetc_clean_tx_ring(struct enetc_bdr *tx_ring)
  19 {
  20         int tx_frm_cnt = 0;
  21         struct enetc_swbd *tx_swbd;
  22         int i, hwci;
  23
  24         /* we don't need barriers here, we just want a relatively current value
  25          * from HW.
  26          */
  27         hwci = (int)(rte_read32_relaxed(tx_ring->tcisr) &
  28                      ENETC_TBCISR_IDX_MASK);
  29
  30         i = tx_ring->next_to_clean;
  31         tx_swbd = &tx_ring->q_swbd[i];
  32
  33         /* we're only reading the CI index once here, which means HW may update
  34          * it while we're doing clean-up.  We could read the register in a loop
  35          * but for now I assume it's OK to leave a few Tx frames for next call.
  36          * The issue with reading the register in a loop is that we're stalling
  37          * here trying to catch up with HW which keeps sending traffic as long
  38          * as it has traffic to send, so in effect we could be waiting here for
  39          * the Tx ring to be drained by HW, instead of us doing Rx in that
  40          * meantime.
  41          */
  42         while (i != hwci) {
  43                 rte_pktmbuf_free(tx_swbd->buffer_addr);
  44                 tx_swbd->buffer_addr = NULL;
  45                 tx_swbd++;
  46                 i++;
  47                 if (unlikely(i == tx_ring->bd_count)) {
  48                         i = 0;
  49                         tx_swbd = &tx_ring->q_swbd[0];
  50                 }
  51
  52                 tx_frm_cnt++;
  53         }
  54
  55         tx_ring->next_to_clean = i;
  56         return tx_frm_cnt++;
  57 }
  58
  59 uint16_t
  60 enetc_xmit_pkts(void *tx_queue,
  61                 struct rte_mbuf **tx_pkts,
  62                 uint16_t nb_pkts)
  63 {
  64         struct enetc_swbd *tx_swbd;
  65         int i, start, bds_to_use;
  66         struct enetc_tx_bd *txbd;
  67         struct enetc_bdr *tx_ring = (struct enetc_bdr *)tx_queue;
  68
  69         i = tx_ring->next_to_use;
  70
  71         bds_to_use = enetc_bd_unused(tx_ring);
  72         if (bds_to_use < nb_pkts)
  73                 nb_pkts = bds_to_use;
  74
  75         start = 0;
  76         while (nb_pkts--) {
  77                 tx_ring->q_swbd[i].buffer_addr = tx_pkts[start];
  78                 txbd = ENETC_TXBD(*tx_ring, i);
  79                 tx_swbd = &tx_ring->q_swbd[i];
  80                 txbd->frm_len = tx_pkts[start]->pkt_len;
  81                 txbd->buf_len = txbd->frm_len;
  82                 txbd->flags = rte_cpu_to_le_16(ENETC_TXBD_FLAGS_F);
  83                 txbd->addr = (uint64_t)(uintptr_t)
  84                 rte_cpu_to_le_64((size_t)tx_swbd->buffer_addr->buf_iova +
  85                                  tx_swbd->buffer_addr->data_off);
  86                 i++;
  87                 start++;
  88                 if (unlikely(i == tx_ring->bd_count))
  89                         i = 0;
  90         }
  91
  92         /* we're only cleaning up the Tx ring here, on the assumption that
  93          * software is slower than hardware and hardware completed sending
  94          * older frames out by now.
  95          * We're also cleaning up the ring before kicking off Tx for the new
  96          * batch to minimize chances of contention on the Tx ring
  97          */
  98         enetc_clean_tx_ring(tx_ring);
  99
 100         tx_ring->next_to_use = i;
 101         enetc_wr_reg(tx_ring->tcir, i);
 102         return start;
 103 }
 104
 105 int
 106 enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
 107 {
 108         struct enetc_swbd *rx_swbd;
 109         union enetc_rx_bd *rxbd;
 110         int i, j;
 111
 112         i = rx_ring->next_to_use;
 113         rx_swbd = &rx_ring->q_swbd[i];
 114         rxbd = ENETC_RXBD(*rx_ring, i);
 115         for (j = 0; j < buff_cnt; j++) {
 116                 rx_swbd->buffer_addr = (void *)(uintptr_t)
 117                         rte_cpu_to_le_64((uint64_t)(uintptr_t)
 118                                         rte_pktmbuf_alloc(rx_ring->mb_pool));
 119                 rxbd->w.addr = (uint64_t)(uintptr_t)
 120                                rx_swbd->buffer_addr->buf_iova +
 121                                rx_swbd->buffer_addr->data_off;
 122                 /* clear 'R" as well */
 123                 rxbd->r.lstatus = 0;
 124                 rx_swbd++;
 125                 rxbd++;
 126                 i++;
 127                 if (unlikely(i == rx_ring->bd_count)) {
 128                         i = 0;
 129                         rxbd = ENETC_RXBD(*rx_ring, 0);
 130                         rx_swbd = &rx_ring->q_swbd[i];
 131                 }
 132         }
 133
 134         if (likely(j)) {
 135                 rx_ring->next_to_alloc = i;
 136                 rx_ring->next_to_use = i;
 137                 enetc_wr_reg(rx_ring->rcir, i);
 138         }
 139
 140         return j;
 141 }
 142
 143 static inline void enetc_slow_parsing(struct rte_mbuf *m,
 144                                      uint64_t parse_results)
 145 {
 146         m->ol_flags &= ~(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 147
 148         switch (parse_results) {
 149         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4:
 150                 m->packet_type = RTE_PTYPE_L2_ETHER |
 151                                  RTE_PTYPE_L3_IPV4;
 152                 m->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 153                 return;
 154         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6:
 155                 m->packet_type = RTE_PTYPE_L2_ETHER |
 156                                  RTE_PTYPE_L3_IPV6;
 157                 m->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 158                 return;
 159         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4_TCP:
 160                 m->packet_type = RTE_PTYPE_L2_ETHER |
 161                                  RTE_PTYPE_L3_IPV4 |
 162                                  RTE_PTYPE_L4_TCP;
 163                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 164                                PKT_RX_L4_CKSUM_BAD;
 165                 return;
 166         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6_TCP:
 167                 m->packet_type = RTE_PTYPE_L2_ETHER |
 168                                  RTE_PTYPE_L3_IPV6 |
 169                                  RTE_PTYPE_L4_TCP;
 170                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 171                                PKT_RX_L4_CKSUM_BAD;
 172                 return;
 173         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4_UDP:
 174                 m->packet_type = RTE_PTYPE_L2_ETHER |
 175                                  RTE_PTYPE_L3_IPV4 |
 176                                  RTE_PTYPE_L4_UDP;
 177                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 178                                PKT_RX_L4_CKSUM_BAD;
 179                 return;
 180         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6_UDP:
 181                 m->packet_type = RTE_PTYPE_L2_ETHER |
 182                                  RTE_PTYPE_L3_IPV6 |
 183                                  RTE_PTYPE_L4_UDP;
 184                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 185                                PKT_RX_L4_CKSUM_BAD;
 186                 return;
 187         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4_SCTP:
 188                 m->packet_type = RTE_PTYPE_L2_ETHER |
 189                                  RTE_PTYPE_L3_IPV4 |
 190                                  RTE_PTYPE_L4_SCTP;
 191                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 192                                PKT_RX_L4_CKSUM_BAD;
 193                 return;
 194         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6_SCTP:
 195                 m->packet_type = RTE_PTYPE_L2_ETHER |
 196                                  RTE_PTYPE_L3_IPV6 |
 197                                  RTE_PTYPE_L4_SCTP;
 198                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 199                                PKT_RX_L4_CKSUM_BAD;
 200                 return;
 201         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV4_ICMP:
 202                 m->packet_type = RTE_PTYPE_L2_ETHER |
 203                                  RTE_PTYPE_L3_IPV4 |
 204                                  RTE_PTYPE_L4_ICMP;
 205                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 206                                PKT_RX_L4_CKSUM_BAD;
 207                 return;
 208         case ENETC_PARSE_ERROR | ENETC_PKT_TYPE_IPV6_ICMP:
 209                 m->packet_type = RTE_PTYPE_L2_ETHER |
 210                                  RTE_PTYPE_L3_IPV6 |
 211                                  RTE_PTYPE_L4_ICMP;
 212                 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD |
 213                                PKT_RX_L4_CKSUM_BAD;
 214                 return;
 215         /* More switch cases can be added */
 216         default:
 217                 m->packet_type = RTE_PTYPE_UNKNOWN;
 218                 m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN |
 219                                PKT_RX_L4_CKSUM_UNKNOWN;
 220         }
 221 }
 222
 223
 224 static inline void __attribute__((hot))
 225 enetc_dev_rx_parse(struct rte_mbuf *m, uint16_t parse_results)
 226 {
 227         ENETC_PMD_DP_DEBUG("parse summary = 0x%x   ", parse_results);
 228         m->ol_flags |= PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD;
 229
 230         switch (parse_results) {
 231         case ENETC_PKT_TYPE_ETHER:
 232                 m->packet_type = RTE_PTYPE_L2_ETHER;
 233                 return;
 234         case ENETC_PKT_TYPE_IPV4:
 235                 m->packet_type = RTE_PTYPE_L2_ETHER |
 236                                  RTE_PTYPE_L3_IPV4;
 237                 return;
 238         case ENETC_PKT_TYPE_IPV6:
 239                 m->packet_type = RTE_PTYPE_L2_ETHER |
 240                                  RTE_PTYPE_L3_IPV6;
 241                 return;
 242         case ENETC_PKT_TYPE_IPV4_TCP:
 243                 m->packet_type = RTE_PTYPE_L2_ETHER |
 244                                  RTE_PTYPE_L3_IPV4 |
 245                                  RTE_PTYPE_L4_TCP;
 246                 return;
 247         case ENETC_PKT_TYPE_IPV6_TCP:
 248                 m->packet_type = RTE_PTYPE_L2_ETHER |
 249                                  RTE_PTYPE_L3_IPV6 |
 250                                  RTE_PTYPE_L4_TCP;
 251                 return;
 252         case ENETC_PKT_TYPE_IPV4_UDP:
 253                 m->packet_type = RTE_PTYPE_L2_ETHER |
 254                                  RTE_PTYPE_L3_IPV4 |
 255                                  RTE_PTYPE_L4_UDP;
 256                 return;
 257         case ENETC_PKT_TYPE_IPV6_UDP:
 258                 m->packet_type = RTE_PTYPE_L2_ETHER |
 259                                  RTE_PTYPE_L3_IPV6 |
 260                                  RTE_PTYPE_L4_UDP;
 261                 return;
 262         case ENETC_PKT_TYPE_IPV4_SCTP:
 263                 m->packet_type = RTE_PTYPE_L2_ETHER |
 264                                  RTE_PTYPE_L3_IPV4 |
 265                                  RTE_PTYPE_L4_SCTP;
 266                 return;
 267         case ENETC_PKT_TYPE_IPV6_SCTP:
 268                 m->packet_type = RTE_PTYPE_L2_ETHER |
 269                                  RTE_PTYPE_L3_IPV6 |
 270                                  RTE_PTYPE_L4_SCTP;
 271                 return;
 272         case ENETC_PKT_TYPE_IPV4_ICMP:
 273                 m->packet_type = RTE_PTYPE_L2_ETHER |
 274                                  RTE_PTYPE_L3_IPV4 |
 275                                  RTE_PTYPE_L4_ICMP;
 276                 return;
 277         case ENETC_PKT_TYPE_IPV6_ICMP:
 278                 m->packet_type = RTE_PTYPE_L2_ETHER |
 279                                  RTE_PTYPE_L3_IPV6 |
 280                                  RTE_PTYPE_L4_ICMP;
 281                 return;
 282         /* More switch cases can be added */
 283         default:
 284                 enetc_slow_parsing(m, parse_results);
 285         }
 286
 287 }
 288
 289 static int
 290 enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
 291                     struct rte_mbuf **rx_pkts,
 292                     int work_limit)
 293 {
 294         int rx_frm_cnt = 0;
 295         int cleaned_cnt, i;
 296         struct enetc_swbd *rx_swbd;
 297
 298         cleaned_cnt = enetc_bd_unused(rx_ring);
 299         /* next descriptor to process */
 300         i = rx_ring->next_to_clean;
 301         rx_swbd = &rx_ring->q_swbd[i];
 302         while (likely(rx_frm_cnt < work_limit)) {
 303                 union enetc_rx_bd *rxbd;
 304                 uint32_t bd_status;
 305
 306                 rxbd = ENETC_RXBD(*rx_ring, i);
 307                 bd_status = rte_le_to_cpu_32(rxbd->r.lstatus);
 308                 if (!bd_status)
 309                         break;
 310
 311                 rx_swbd->buffer_addr->pkt_len = rxbd->r.buf_len -
 312                                                 rx_ring->crc_len;
 313                 rx_swbd->buffer_addr->data_len = rxbd->r.buf_len -
 314                                                  rx_ring->crc_len;
 315                 rx_swbd->buffer_addr->hash.rss = rxbd->r.rss_hash;
 316                 rx_swbd->buffer_addr->ol_flags = 0;
 317                 enetc_dev_rx_parse(rx_swbd->buffer_addr,
 318                                    rxbd->r.parse_summary);
 319                 rx_pkts[rx_frm_cnt] = rx_swbd->buffer_addr;
 320                 cleaned_cnt++;
 321                 rx_swbd++;
 322                 i++;
 323                 if (unlikely(i == rx_ring->bd_count)) {
 324                         i = 0;
 325                         rx_swbd = &rx_ring->q_swbd[i];
 326                 }
 327
 328                 rx_ring->next_to_clean = i;
 329                 rx_frm_cnt++;
 330         }
 331
 332         enetc_refill_rx_ring(rx_ring, cleaned_cnt);
 333
 334         return rx_frm_cnt;
 335 }
 336
 337 uint16_t
 338 enetc_recv_pkts(void *rxq, struct rte_mbuf **rx_pkts,
 339                 uint16_t nb_pkts)
 340 {
 341         struct enetc_bdr *rx_ring = (struct enetc_bdr *)rxq;
 342
 343         return enetc_clean_rx_ring(rx_ring, rx_pkts, nb_pkts);
 344 }