drivers/net/ionic/ionic_rxtx.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <errno.h>
  10 #include <stdint.h>
  11 #include <stdarg.h>
  12 #include <unistd.h>
  13 #include <inttypes.h>
  14
  15 #include <rte_byteorder.h>
  16 #include <rte_common.h>
  17 #include <rte_cycles.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <rte_interrupts.h>
  21 #include <rte_pci.h>
  22 #include <rte_memory.h>
  23 #include <rte_memzone.h>
  24 #include <rte_launch.h>
  25 #include <rte_eal.h>
  26 #include <rte_per_lcore.h>
  27 #include <rte_lcore.h>
  28 #include <rte_atomic.h>
  29 #include <rte_branch_prediction.h>
  30 #include <rte_mempool.h>
  31 #include <rte_malloc.h>
  32 #include <rte_mbuf.h>
  33 #include <rte_ether.h>
  34 #include <rte_ethdev_driver.h>
  35 #include <rte_prefetch.h>
  36 #include <rte_udp.h>
  37 #include <rte_tcp.h>
  38 #include <rte_sctp.h>
  39 #include <rte_string_fns.h>
  40 #include <rte_errno.h>
  41 #include <rte_ip.h>
  42 #include <rte_net.h>
  43
  44 #include "ionic_logs.h"
  45 #include "ionic_mac_api.h"
  46 #include "ionic_ethdev.h"
  47 #include "ionic_lif.h"
  48 #include "ionic_rxtx.h"
  49
  50 #define IONIC_RX_RING_DOORBELL_STRIDE           (32 - 1)
  51
  52 /*********************************************************************
  53  *
  54  *  TX functions
  55  *
  56  **********************************************************************/
  57
  58 void
  59 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
  60                 struct rte_eth_txq_info *qinfo)
  61 {
  62         struct ionic_qcq *txq = dev->data->tx_queues[queue_id];
  63         struct ionic_queue *q = &txq->q;
  64
  65         qinfo->nb_desc = q->num_descs;
  66         qinfo->conf.offloads = txq->offloads;
  67         qinfo->conf.tx_deferred_start = txq->deferred_start;
  68 }
  69
  70 static inline void __attribute__((cold))
  71 ionic_tx_flush(struct ionic_cq *cq)
  72 {
  73         struct ionic_queue *q = cq->bound_q;
  74         struct ionic_desc_info *q_desc_info;
  75         struct rte_mbuf *txm, *next;
  76         struct ionic_txq_comp *cq_desc_base = cq->base;
  77         struct ionic_txq_comp *cq_desc;
  78         u_int32_t comp_index = (u_int32_t)-1;
  79
  80         cq_desc = &cq_desc_base[cq->tail_idx];
  81         while (color_match(cq_desc->color, cq->done_color)) {
  82                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
  83
  84                 /* Prefetch the next 4 descriptors (not really useful here) */
  85                 if ((cq->tail_idx & 0x3) == 0)
  86                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
  87
  88                 if (cq->tail_idx == 0)
  89                         cq->done_color = !cq->done_color;
  90
  91                 comp_index = cq_desc->comp_index;
  92
  93                 cq_desc = &cq_desc_base[cq->tail_idx];
  94         }
  95
  96         if (comp_index != (u_int32_t)-1) {
  97                 while (q->tail_idx != comp_index) {
  98                         q_desc_info = &q->info[q->tail_idx];
  99
 100                         q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 101
 102                         /* Prefetch the next 4 descriptors */
 103                         if ((q->tail_idx & 0x3) == 0)
 104                                 /* q desc info */
 105                                 rte_prefetch0(&q->info[q->tail_idx]);
 106
 107                         /*
 108                          * Note: you can just use rte_pktmbuf_free,
 109                          * but this loop is faster
 110                          */
 111                         txm = q_desc_info->cb_arg;
 112                         while (txm != NULL) {
 113                                 next = txm->next;
 114                                 rte_pktmbuf_free_seg(txm);
 115                                 txm = next;
 116                         }
 117                 }
 118         }
 119 }
 120
 121 void __attribute__((cold))
 122 ionic_dev_tx_queue_release(void *tx_queue)
 123 {
 124         struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue;
 125
 126         IONIC_PRINT_CALL();
 127
 128         ionic_qcq_free(txq);
 129 }
 130
 131 int __attribute__((cold))
 132 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 133 {
 134         struct ionic_qcq *txq;
 135
 136         IONIC_PRINT_CALL();
 137
 138         txq = eth_dev->data->tx_queues[tx_queue_id];
 139
 140         /*
 141          * Note: we should better post NOP Tx desc and wait for its completion
 142          * before disabling Tx queue
 143          */
 144
 145         ionic_qcq_disable(txq);
 146
 147         ionic_tx_flush(&txq->cq);
 148
 149         ionic_lif_txq_deinit(txq);
 150
 151         eth_dev->data->tx_queue_state[tx_queue_id] =
 152                 RTE_ETH_QUEUE_STATE_STOPPED;
 153
 154         return 0;
 155 }
 156
 157 int __attribute__((cold))
 158 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
 159                 uint16_t nb_desc, uint32_t socket_id __rte_unused,
 160                 const struct rte_eth_txconf *tx_conf)
 161 {
 162         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 163         struct ionic_qcq *txq;
 164         uint64_t offloads;
 165         int err;
 166
 167         IONIC_PRINT_CALL();
 168
 169         IONIC_PRINT(DEBUG, "Configuring TX queue %u with %u buffers",
 170                 tx_queue_id, nb_desc);
 171
 172         if (tx_queue_id >= lif->ntxqcqs) {
 173                 IONIC_PRINT(DEBUG, "Queue index %u not available "
 174                         "(max %u queues)",
 175                         tx_queue_id, lif->ntxqcqs);
 176                 return -EINVAL;
 177         }
 178
 179         offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads;
 180
 181         /* Validate number of receive descriptors */
 182         if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC)
 183                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 184
 185         /* Free memory prior to re-allocation if needed... */
 186         if (eth_dev->data->tx_queues[tx_queue_id] != NULL) {
 187                 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id];
 188                 ionic_dev_tx_queue_release(tx_queue);
 189                 eth_dev->data->tx_queues[tx_queue_id] = NULL;
 190         }
 191
 192         err = ionic_tx_qcq_alloc(lif, tx_queue_id, nb_desc, &txq);
 193         if (err) {
 194                 IONIC_PRINT(DEBUG, "Queue allocation failure");
 195                 return -EINVAL;
 196         }
 197
 198         /* Do not start queue with rte_eth_dev_start() */
 199         txq->deferred_start = tx_conf->tx_deferred_start;
 200
 201         txq->offloads = offloads;
 202
 203         eth_dev->data->tx_queues[tx_queue_id] = txq;
 204
 205         return 0;
 206 }
 207
 208 /*
 209  * Start Transmit Units for specified queue.
 210  */
 211 int __attribute__((cold))
 212 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 213 {
 214         struct ionic_qcq *txq;
 215         int err;
 216
 217         IONIC_PRINT_CALL();
 218
 219         txq = eth_dev->data->tx_queues[tx_queue_id];
 220
 221         err = ionic_lif_txq_init(txq);
 222         if (err)
 223                 return err;
 224
 225         ionic_qcq_enable(txq);
 226
 227         eth_dev->data->tx_queue_state[tx_queue_id] =
 228                 RTE_ETH_QUEUE_STATE_STARTED;
 229
 230         return 0;
 231 }
 232
 233 static void
 234 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
 235                 struct rte_mbuf *txm,
 236                 rte_iova_t addr, uint8_t nsge, uint16_t len,
 237                 uint32_t hdrlen, uint32_t mss,
 238                 uint16_t vlan_tci, bool has_vlan,
 239                 bool start, bool done)
 240 {
 241         uint8_t flags = 0;
 242         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 243         flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
 244         flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
 245
 246         desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO,
 247                 flags, nsge, addr);
 248         desc->len = len;
 249         desc->vlan_tci = vlan_tci;
 250         desc->hdr_len = hdrlen;
 251         desc->mss = mss;
 252
 253         ionic_q_post(q, done, NULL, done ? txm : NULL);
 254 }
 255
 256 static struct ionic_txq_desc *
 257 ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem)
 258 {
 259         struct ionic_txq_desc *desc_base = q->base;
 260         struct ionic_txq_sg_desc *sg_desc_base = q->sg_base;
 261         struct ionic_txq_desc *desc = &desc_base[q->head_idx];
 262         struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx];
 263
 264         *elem = sg_desc->elems;
 265         return desc;
 266 }
 267
 268 static int
 269 ionic_tx_tso(struct ionic_queue *q, struct rte_mbuf *txm,
 270                 uint64_t offloads __rte_unused, bool not_xmit_more)
 271 {
 272         struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
 273         struct ionic_txq_desc *desc;
 274         struct ionic_txq_sg_elem *elem;
 275         struct rte_mbuf *txm_seg;
 276         uint64_t desc_addr = 0;
 277         uint16_t desc_len = 0;
 278         uint8_t desc_nsge;
 279         uint32_t hdrlen;
 280         uint32_t mss = txm->tso_segsz;
 281         uint32_t frag_left = 0;
 282         uint32_t left;
 283         uint32_t seglen;
 284         uint32_t len;
 285         uint32_t offset = 0;
 286         bool start, done;
 287         bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT);
 288         uint16_t vlan_tci = txm->vlan_tci;
 289
 290         hdrlen = txm->l2_len + txm->l3_len;
 291
 292         seglen = hdrlen + mss;
 293         left = txm->data_len;
 294
 295         desc = ionic_tx_tso_next(q, &elem);
 296         start = true;
 297
 298         /* Chop data up into desc segments */
 299
 300         while (left > 0) {
 301                 len = RTE_MIN(seglen, left);
 302                 frag_left = seglen - len;
 303                 desc_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(txm));
 304                 desc_len = len;
 305                 desc_nsge = 0;
 306                 left -= len;
 307                 offset += len;
 308                 if (txm->nb_segs > 1 && frag_left > 0)
 309                         continue;
 310                 done = (txm->nb_segs == 1 && left == 0);
 311                 ionic_tx_tso_post(q, desc, txm,
 312                         desc_addr, desc_nsge, desc_len,
 313                         hdrlen, mss,
 314                         vlan_tci, has_vlan,
 315                         start, done && not_xmit_more);
 316                 desc = ionic_tx_tso_next(q, &elem);
 317                 start = false;
 318                 seglen = mss;
 319         }
 320
 321         /* Chop frags into desc segments */
 322
 323         txm_seg = txm->next;
 324         while (txm_seg != NULL) {
 325                 offset = 0;
 326                 left = txm_seg->data_len;
 327                 stats->frags++;
 328
 329                 while (left > 0) {
 330                         rte_iova_t data_iova;
 331                         data_iova = rte_mbuf_data_iova(txm_seg);
 332                         elem->addr = rte_cpu_to_le_64(data_iova) + offset;
 333                         if (frag_left > 0) {
 334                                 len = RTE_MIN(frag_left, left);
 335                                 frag_left -= len;
 336                                 elem->len = len;
 337                                 elem++;
 338                                 desc_nsge++;
 339                         } else {
 340                                 len = RTE_MIN(mss, left);
 341                                 frag_left = mss - len;
 342                                 data_iova = rte_mbuf_data_iova(txm_seg);
 343                                 desc_addr = rte_cpu_to_le_64(data_iova);
 344                                 desc_len = len;
 345                                 desc_nsge = 0;
 346                         }
 347                         left -= len;
 348                         offset += len;
 349                         if (txm_seg->next != NULL && frag_left > 0)
 350                                 continue;
 351                         done = (txm_seg->next == NULL && left == 0);
 352                         ionic_tx_tso_post(q, desc, txm_seg,
 353                                 desc_addr, desc_nsge, desc_len,
 354                                 hdrlen, mss,
 355                                 vlan_tci, has_vlan,
 356                                 start, done && not_xmit_more);
 357                         desc = ionic_tx_tso_next(q, &elem);
 358                         start = false;
 359                 }
 360
 361                 txm_seg = txm_seg->next;
 362         }
 363
 364         stats->tso++;
 365
 366         return 0;
 367 }
 368
 369 static int
 370 ionic_tx(struct ionic_queue *q, struct rte_mbuf *txm,
 371                 uint64_t offloads __rte_unused, bool not_xmit_more)
 372 {
 373         struct ionic_txq_desc *desc_base = q->base;
 374         struct ionic_txq_sg_desc *sg_desc_base = q->sg_base;
 375         struct ionic_txq_desc *desc = &desc_base[q->head_idx];
 376         struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx];
 377         struct ionic_txq_sg_elem *elem = sg_desc->elems;
 378         struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
 379         struct rte_mbuf *txm_seg;
 380         bool has_vlan;
 381         uint64_t ol_flags = txm->ol_flags;
 382         uint64_t addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(txm));
 383         uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
 384         uint8_t flags = 0;
 385
 386         has_vlan = (ol_flags & PKT_TX_VLAN_PKT);
 387
 388         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 389
 390         desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr);
 391         desc->len = txm->data_len;
 392         desc->vlan_tci = txm->vlan_tci;
 393
 394         txm_seg = txm->next;
 395         while (txm_seg != NULL) {
 396                 elem->len = txm_seg->data_len;
 397                 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg));
 398                 stats->frags++;
 399                 elem++;
 400                 txm_seg = txm_seg->next;
 401         }
 402
 403         ionic_q_post(q, not_xmit_more, NULL, txm);
 404
 405         return 0;
 406 }
 407
 408 uint16_t
 409 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 410                 uint16_t nb_pkts)
 411 {
 412         struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue;
 413         struct ionic_queue *q = &txq->q;
 414         struct ionic_cq *cq = &txq->cq;
 415         struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
 416         uint32_t next_q_head_idx;
 417         uint32_t bytes_tx = 0;
 418         uint16_t nb_tx = 0;
 419         int err;
 420         bool last;
 421
 422         /* Cleaning old buffers */
 423         ionic_tx_flush(cq);
 424
 425         if (unlikely(ionic_q_space_avail(q) < nb_pkts)) {
 426                 stats->stop += nb_pkts;
 427                 return 0;
 428         }
 429
 430         while (nb_tx < nb_pkts) {
 431                 last = (nb_tx == (nb_pkts - 1));
 432
 433                 next_q_head_idx = (q->head_idx + 1) & (q->num_descs - 1);
 434                 if ((next_q_head_idx & 0x3) == 0) {
 435                         struct ionic_txq_desc *desc_base = q->base;
 436                         rte_prefetch0(&desc_base[next_q_head_idx]);
 437                         rte_prefetch0(&q->info[next_q_head_idx]);
 438                 }
 439
 440                 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG)
 441                         err = ionic_tx_tso(q, tx_pkts[nb_tx], txq->offloads,
 442                                 last);
 443                 else
 444                         err = ionic_tx(q, tx_pkts[nb_tx], txq->offloads, last);
 445                 if (err) {
 446                         stats->drop += nb_pkts - nb_tx;
 447                         if (nb_tx > 0)
 448                                 ionic_q_flush(q);
 449                         break;
 450                 }
 451
 452                 bytes_tx += tx_pkts[nb_tx]->pkt_len;
 453                 nb_tx++;
 454         }
 455
 456         stats->packets += nb_tx;
 457         stats->bytes += bytes_tx;
 458
 459         return nb_tx;
 460 }
 461
 462 /*********************************************************************
 463  *
 464  *  TX prep functions
 465  *
 466  **********************************************************************/
 467
 468 #define IONIC_TX_OFFLOAD_MASK ( \
 469         PKT_TX_IPV4 |           \
 470         PKT_TX_IPV6 |           \
 471         PKT_TX_VLAN |           \
 472         PKT_TX_TCP_SEG |        \
 473         PKT_TX_L4_MASK)
 474
 475 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \
 476         (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK)
 477
 478 uint16_t
 479 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
 480                 uint16_t nb_pkts)
 481 {
 482         struct rte_mbuf *txm;
 483         uint64_t offloads;
 484         int i = 0;
 485
 486         for (i = 0; i < nb_pkts; i++) {
 487                 txm = tx_pkts[i];
 488
 489                 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS) {
 490                         rte_errno = -EINVAL;
 491                         break;
 492                 }
 493
 494                 offloads = txm->ol_flags;
 495
 496                 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) {
 497                         rte_errno = -ENOTSUP;
 498                         break;
 499                 }
 500         }
 501
 502         return i;
 503 }
 504
 505 /*********************************************************************
 506  *
 507  *  RX functions
 508  *
 509  **********************************************************************/
 510
 511 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 512                 struct rte_mbuf *mbuf);
 513
 514 void
 515 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 516                 struct rte_eth_rxq_info *qinfo)
 517 {
 518         struct ionic_qcq *rxq = dev->data->rx_queues[queue_id];
 519         struct ionic_queue *q = &rxq->q;
 520
 521         qinfo->mp = rxq->mb_pool;
 522         qinfo->scattered_rx = dev->data->scattered_rx;
 523         qinfo->nb_desc = q->num_descs;
 524         qinfo->conf.rx_deferred_start = rxq->deferred_start;
 525         qinfo->conf.offloads = rxq->offloads;
 526 }
 527
 528 static void __attribute__((cold))
 529 ionic_rx_empty(struct ionic_queue *q)
 530 {
 531         struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q);
 532         struct ionic_desc_info *cur;
 533         struct rte_mbuf *mbuf;
 534
 535         while (q->tail_idx != q->head_idx) {
 536                 cur = &q->info[q->tail_idx];
 537                 mbuf = cur->cb_arg;
 538                 rte_mempool_put(rxq->mb_pool, mbuf);
 539
 540                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 541         }
 542 }
 543
 544 void __attribute__((cold))
 545 ionic_dev_rx_queue_release(void *rx_queue)
 546 {
 547         struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue;
 548
 549         IONIC_PRINT_CALL();
 550
 551         ionic_rx_empty(&rxq->q);
 552
 553         ionic_qcq_free(rxq);
 554 }
 555
 556 int __attribute__((cold))
 557 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 558                 uint16_t rx_queue_id,
 559                 uint16_t nb_desc,
 560                 uint32_t socket_id __rte_unused,
 561                 const struct rte_eth_rxconf *rx_conf,
 562                 struct rte_mempool *mp)
 563 {
 564         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 565         struct ionic_qcq *rxq;
 566         uint64_t offloads;
 567         int err;
 568
 569         IONIC_PRINT_CALL();
 570
 571         IONIC_PRINT(DEBUG, "Configuring RX queue %u with %u buffers",
 572                 rx_queue_id, nb_desc);
 573
 574         if (rx_queue_id >= lif->nrxqcqs) {
 575                 IONIC_PRINT(ERR,
 576                         "Queue index %u not available (max %u queues)",
 577                         rx_queue_id, lif->nrxqcqs);
 578                 return -EINVAL;
 579         }
 580
 581         offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads;
 582
 583         /* Validate number of receive descriptors */
 584         if (!rte_is_power_of_2(nb_desc) ||
 585                         nb_desc < IONIC_MIN_RING_DESC ||
 586                         nb_desc > IONIC_MAX_RING_DESC) {
 587                 IONIC_PRINT(ERR,
 588                         "Bad number of descriptors (%u) for queue %u (min: %u)",
 589                         nb_desc, rx_queue_id, IONIC_MIN_RING_DESC);
 590                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 591         }
 592
 593         if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
 594                 eth_dev->data->scattered_rx = 1;
 595
 596         /* Free memory prior to re-allocation if needed... */
 597         if (eth_dev->data->rx_queues[rx_queue_id] != NULL) {
 598                 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id];
 599                 ionic_dev_rx_queue_release(rx_queue);
 600                 eth_dev->data->rx_queues[rx_queue_id] = NULL;
 601         }
 602
 603         err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, &rxq);
 604         if (err) {
 605                 IONIC_PRINT(ERR, "Queue allocation failure");
 606                 return -EINVAL;
 607         }
 608
 609         rxq->mb_pool = mp;
 610
 611         /*
 612          * Note: the interface does not currently support
 613          * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN
 614          * when the adapter will be able to keep the CRC and subtract
 615          * it to the length for all received packets:
 616          * if (eth_dev->data->dev_conf.rxmode.offloads &
 617          *     DEV_RX_OFFLOAD_KEEP_CRC)
 618          *   rxq->crc_len = ETHER_CRC_LEN;
 619          */
 620
 621         /* Do not start queue with rte_eth_dev_start() */
 622         rxq->deferred_start = rx_conf->rx_deferred_start;
 623
 624         rxq->offloads = offloads;
 625
 626         eth_dev->data->rx_queues[rx_queue_id] = rxq;
 627
 628         return 0;
 629 }
 630
 631 static void
 632 ionic_rx_clean(struct ionic_queue *q,
 633                 uint32_t q_desc_index, uint32_t cq_desc_index,
 634                 void *cb_arg, void *service_cb_arg)
 635 {
 636         struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base;
 637         struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index];
 638         struct rte_mbuf *rxm = cb_arg;
 639         struct rte_mbuf *rxm_seg;
 640         struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q);
 641         uint32_t max_frame_size =
 642                 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 643         uint64_t pkt_flags = 0;
 644         uint32_t pkt_type;
 645         struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q);
 646         struct ionic_rx_service *recv_args = (struct ionic_rx_service *)
 647                 service_cb_arg;
 648         uint32_t buf_size = (uint16_t)
 649                 (rte_pktmbuf_data_room_size(rxq->mb_pool) -
 650                 RTE_PKTMBUF_HEADROOM);
 651         uint32_t left;
 652
 653         if (!recv_args) {
 654                 stats->no_cb_arg++;
 655                 /* Flush */
 656                 rte_pktmbuf_free(rxm);
 657                 /*
 658                  * Note: rte_mempool_put is faster with no segs
 659                  * rte_mempool_put(rxq->mb_pool, rxm);
 660                  */
 661                 return;
 662         }
 663
 664         if (cq_desc->status) {
 665                 stats->bad_cq_status++;
 666                 ionic_rx_recycle(q, q_desc_index, rxm);
 667                 return;
 668         }
 669
 670         if (recv_args->nb_rx >= recv_args->nb_pkts) {
 671                 stats->no_room++;
 672                 ionic_rx_recycle(q, q_desc_index, rxm);
 673                 return;
 674         }
 675
 676         if (cq_desc->len > max_frame_size ||
 677                         cq_desc->len == 0) {
 678                 stats->bad_len++;
 679                 ionic_rx_recycle(q, q_desc_index, rxm);
 680                 return;
 681         }
 682
 683         rxm->data_off = RTE_PKTMBUF_HEADROOM;
 684         rte_prefetch1((char *)rxm->buf_addr + rxm->data_off);
 685         rxm->nb_segs = 1; /* cq_desc->num_sg_elems */
 686         rxm->pkt_len = cq_desc->len;
 687         rxm->port = rxq->lif->port_id;
 688
 689         left = cq_desc->len;
 690
 691         rxm->data_len = RTE_MIN(buf_size, left);
 692         left -= rxm->data_len;
 693
 694         rxm_seg = rxm->next;
 695         while (rxm_seg && left) {
 696                 rxm_seg->data_len = RTE_MIN(buf_size, left);
 697                 left -= rxm_seg->data_len;
 698
 699                 rxm_seg = rxm_seg->next;
 700                 rxm->nb_segs++;
 701         }
 702
 703         /* Vlan Strip */
 704         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
 705                 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 706                 rxm->vlan_tci = cq_desc->vlan_tci;
 707         }
 708
 709         /* Checksum */
 710         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
 711                 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK)
 712                         pkt_flags |= PKT_RX_IP_CKSUM_GOOD;
 713                 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)
 714                         pkt_flags |= PKT_RX_IP_CKSUM_BAD;
 715
 716                 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) ||
 717                         (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK))
 718                         pkt_flags |= PKT_RX_L4_CKSUM_GOOD;
 719                 else if ((cq_desc->csum_flags &
 720                                 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
 721                                 (cq_desc->csum_flags &
 722                                 IONIC_RXQ_COMP_CSUM_F_UDP_BAD))
 723                         pkt_flags |= PKT_RX_L4_CKSUM_BAD;
 724         }
 725
 726         rxm->ol_flags = pkt_flags;
 727
 728         /* Packet Type */
 729         switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
 730         case IONIC_PKT_TYPE_IPV4:
 731                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 732                 break;
 733         case IONIC_PKT_TYPE_IPV6:
 734                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 735                 break;
 736         case IONIC_PKT_TYPE_IPV4_TCP:
 737                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 738                         RTE_PTYPE_L4_TCP;
 739                 break;
 740         case IONIC_PKT_TYPE_IPV6_TCP:
 741                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 742                         RTE_PTYPE_L4_TCP;
 743                 break;
 744         case IONIC_PKT_TYPE_IPV4_UDP:
 745                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 746                         RTE_PTYPE_L4_UDP;
 747                 break;
 748         case IONIC_PKT_TYPE_IPV6_UDP:
 749                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 750                         RTE_PTYPE_L4_UDP;
 751                 break;
 752         default:
 753                 {
 754                         struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
 755                                 struct rte_ether_hdr *);
 756                         uint16_t ether_type = eth_h->ether_type;
 757                         if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
 758                                 pkt_type = RTE_PTYPE_L2_ETHER_ARP;
 759                         else
 760                                 pkt_type = RTE_PTYPE_UNKNOWN;
 761                         break;
 762                 }
 763         }
 764
 765         rxm->packet_type = pkt_type;
 766
 767         recv_args->rx_pkts[recv_args->nb_rx] = rxm;
 768         recv_args->nb_rx++;
 769
 770         stats->packets++;
 771         stats->bytes += rxm->pkt_len;
 772 }
 773
 774 static void
 775 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 776                  struct rte_mbuf *mbuf)
 777 {
 778         struct ionic_rxq_desc *desc_base = q->base;
 779         struct ionic_rxq_desc *old = &desc_base[q_desc_index];
 780         struct ionic_rxq_desc *new = &desc_base[q->head_idx];
 781
 782         new->addr = old->addr;
 783         new->len = old->len;
 784
 785         ionic_q_post(q, true, ionic_rx_clean, mbuf);
 786 }
 787
 788 static int __attribute__((cold))
 789 ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len)
 790 {
 791         struct ionic_queue *q = &rxq->q;
 792         struct ionic_rxq_desc *desc_base = q->base;
 793         struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base;
 794         struct ionic_rxq_desc *desc;
 795         struct ionic_rxq_sg_desc *sg_desc;
 796         struct ionic_rxq_sg_elem *elem;
 797         rte_iova_t dma_addr;
 798         uint32_t i, j, nsegs, buf_size, size;
 799         bool ring_doorbell;
 800
 801         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 802                 RTE_PKTMBUF_HEADROOM);
 803
 804         /* Initialize software ring entries */
 805         for (i = ionic_q_space_avail(q); i; i--) {
 806                 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool);
 807                 struct rte_mbuf *prev_rxm_seg;
 808
 809                 if (rxm == NULL) {
 810                         IONIC_PRINT(ERR, "RX mbuf alloc failed");
 811                         return -ENOMEM;
 812                 }
 813
 814                 nsegs = (len + buf_size - 1) / buf_size;
 815
 816                 desc = &desc_base[q->head_idx];
 817                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm));
 818                 desc->addr = dma_addr;
 819                 desc->len = buf_size;
 820                 size = buf_size;
 821                 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
 822                         IONIC_RXQ_DESC_OPCODE_SIMPLE;
 823                 rxm->next = NULL;
 824
 825                 prev_rxm_seg = rxm;
 826                 sg_desc = &sg_desc_base[q->head_idx];
 827                 elem = sg_desc->elems;
 828                 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) {
 829                         struct rte_mbuf *rxm_seg;
 830                         rte_iova_t data_iova;
 831
 832                         rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool);
 833                         if (rxm_seg == NULL) {
 834                                 IONIC_PRINT(ERR, "RX mbuf alloc failed");
 835                                 return -ENOMEM;
 836                         }
 837
 838                         data_iova = rte_mbuf_data_iova(rxm_seg);
 839                         dma_addr = rte_cpu_to_le_64(data_iova);
 840                         elem->addr = dma_addr;
 841                         elem->len = buf_size;
 842                         size += buf_size;
 843                         elem++;
 844                         rxm_seg->next = NULL;
 845                         prev_rxm_seg->next = rxm_seg;
 846                         prev_rxm_seg = rxm_seg;
 847                 }
 848
 849                 if (size < len)
 850                         IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)",
 851                                 size, len);
 852
 853                 ring_doorbell = ((q->head_idx + 1) &
 854                         IONIC_RX_RING_DOORBELL_STRIDE) == 0;
 855
 856                 ionic_q_post(q, ring_doorbell, ionic_rx_clean, rxm);
 857         }
 858
 859         return 0;
 860 }
 861
 862 /*
 863  * Start Receive Units for specified queue.
 864  */
 865 int __attribute__((cold))
 866 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 867 {
 868         uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 869         struct ionic_qcq *rxq;
 870         int err;
 871
 872         IONIC_PRINT_CALL();
 873
 874         IONIC_PRINT(DEBUG, "Allocating RX queue buffers (size: %u)",
 875                 frame_size);
 876
 877         rxq = eth_dev->data->rx_queues[rx_queue_id];
 878
 879         err = ionic_lif_rxq_init(rxq);
 880         if (err)
 881                 return err;
 882
 883         /* Allocate buffers for descriptor rings */
 884         if (ionic_rx_fill(rxq, frame_size) != 0) {
 885                 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d",
 886                         rx_queue_id);
 887                 return -1;
 888         }
 889
 890         ionic_qcq_enable(rxq);
 891
 892         eth_dev->data->rx_queue_state[rx_queue_id] =
 893                 RTE_ETH_QUEUE_STATE_STARTED;
 894
 895         return 0;
 896 }
 897
 898 static inline void __attribute__((cold))
 899 ionic_rxq_service(struct ionic_cq *cq, uint32_t work_to_do,
 900                 void *service_cb_arg)
 901 {
 902         struct ionic_queue *q = cq->bound_q;
 903         struct ionic_desc_info *q_desc_info;
 904         struct ionic_rxq_comp *cq_desc_base = cq->base;
 905         struct ionic_rxq_comp *cq_desc;
 906         bool more;
 907         uint32_t curr_q_tail_idx, curr_cq_tail_idx;
 908         uint32_t work_done = 0;
 909
 910         if (work_to_do == 0)
 911                 return;
 912
 913         cq_desc = &cq_desc_base[cq->tail_idx];
 914         while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
 915                 curr_cq_tail_idx = cq->tail_idx;
 916                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
 917
 918                 if (cq->tail_idx == 0)
 919                         cq->done_color = !cq->done_color;
 920
 921                 /* Prefetch the next 4 descriptors */
 922                 if ((cq->tail_idx & 0x3) == 0)
 923                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
 924
 925                 do {
 926                         more = (q->tail_idx != cq_desc->comp_index);
 927
 928                         q_desc_info = &q->info[q->tail_idx];
 929
 930                         curr_q_tail_idx = q->tail_idx;
 931                         q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 932
 933                         /* Prefetch the next 4 descriptors */
 934                         if ((q->tail_idx & 0x3) == 0)
 935                                 /* q desc info */
 936                                 rte_prefetch0(&q->info[q->tail_idx]);
 937
 938                         ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx,
 939                                 q_desc_info->cb_arg, service_cb_arg);
 940
 941                 } while (more);
 942
 943                 if (++work_done == work_to_do)
 944                         break;
 945
 946                 cq_desc = &cq_desc_base[cq->tail_idx];
 947         }
 948 }
 949
 950 /*
 951  * Stop Receive Units for specified queue.
 952  */
 953 int __attribute__((cold))
 954 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 955 {
 956         struct ionic_qcq *rxq;
 957
 958         IONIC_PRINT_CALL();
 959
 960         rxq = eth_dev->data->rx_queues[rx_queue_id];
 961
 962         ionic_qcq_disable(rxq);
 963
 964         /* Flush */
 965         ionic_rxq_service(&rxq->cq, -1, NULL);
 966
 967         ionic_lif_rxq_deinit(rxq);
 968
 969         eth_dev->data->rx_queue_state[rx_queue_id] =
 970                 RTE_ETH_QUEUE_STATE_STOPPED;
 971
 972         return 0;
 973 }
 974
 975 uint16_t
 976 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 977                 uint16_t nb_pkts)
 978 {
 979         struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue;
 980         uint32_t frame_size =
 981                 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 982         struct ionic_cq *cq = &rxq->cq;
 983         struct ionic_rx_service service_cb_arg;
 984
 985         service_cb_arg.rx_pkts = rx_pkts;
 986         service_cb_arg.nb_pkts = nb_pkts;
 987         service_cb_arg.nb_rx = 0;
 988
 989         ionic_rxq_service(cq, nb_pkts, &service_cb_arg);
 990
 991         ionic_rx_fill(rxq, frame_size);
 992
 993         return service_cb_arg.nb_rx;
 994 }