drivers/net/ionic/ionic_rxtx.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <errno.h>
  10 #include <stdint.h>
  11 #include <stdarg.h>
  12 #include <unistd.h>
  13 #include <inttypes.h>
  14
  15 #include <rte_byteorder.h>
  16 #include <rte_common.h>
  17 #include <rte_cycles.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <rte_interrupts.h>
  21 #include <rte_pci.h>
  22 #include <rte_memory.h>
  23 #include <rte_memzone.h>
  24 #include <rte_launch.h>
  25 #include <rte_eal.h>
  26 #include <rte_per_lcore.h>
  27 #include <rte_lcore.h>
  28 #include <rte_atomic.h>
  29 #include <rte_branch_prediction.h>
  30 #include <rte_mempool.h>
  31 #include <rte_malloc.h>
  32 #include <rte_mbuf.h>
  33 #include <rte_ether.h>
  34 #include <ethdev_driver.h>
  35 #include <rte_prefetch.h>
  36 #include <rte_udp.h>
  37 #include <rte_tcp.h>
  38 #include <rte_sctp.h>
  39 #include <rte_string_fns.h>
  40 #include <rte_errno.h>
  41 #include <rte_ip.h>
  42 #include <rte_net.h>
  43
  44 #include "ionic_logs.h"
  45 #include "ionic_mac_api.h"
  46 #include "ionic_ethdev.h"
  47 #include "ionic_lif.h"
  48 #include "ionic_rxtx.h"
  49
  50 #define IONIC_RX_RING_DOORBELL_STRIDE           (32 - 1)
  51
  52 /*********************************************************************
  53  *
  54  *  TX functions
  55  *
  56  **********************************************************************/
  57
  58 void
  59 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
  60                 struct rte_eth_txq_info *qinfo)
  61 {
  62         struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id];
  63         struct ionic_queue *q = &txq->qcq.q;
  64
  65         qinfo->nb_desc = q->num_descs;
  66         qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
  67         qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED;
  68 }
  69
  70 static __rte_always_inline void
  71 ionic_tx_flush(struct ionic_tx_qcq *txq)
  72 {
  73         struct ionic_cq *cq = &txq->qcq.cq;
  74         struct ionic_queue *q = &txq->qcq.q;
  75         struct rte_mbuf *txm, *next;
  76         struct ionic_txq_comp *cq_desc_base = cq->base;
  77         struct ionic_txq_comp *cq_desc;
  78         void **info;
  79         u_int32_t comp_index = (u_int32_t)-1;
  80
  81         cq_desc = &cq_desc_base[cq->tail_idx];
  82         while (color_match(cq_desc->color, cq->done_color)) {
  83                 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
  84
  85                 /* Prefetch the next 4 descriptors (not really useful here) */
  86                 if ((cq->tail_idx & 0x3) == 0)
  87                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
  88
  89                 if (cq->tail_idx == 0)
  90                         cq->done_color = !cq->done_color;
  91
  92                 comp_index = cq_desc->comp_index;
  93
  94                 cq_desc = &cq_desc_base[cq->tail_idx];
  95         }
  96
  97         if (comp_index != (u_int32_t)-1) {
  98                 while (q->tail_idx != comp_index) {
  99                         info = IONIC_INFO_PTR(q, q->tail_idx);
 100
 101                         q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
 102
 103                         /* Prefetch the next 4 descriptors */
 104                         if ((q->tail_idx & 0x3) == 0)
 105                                 /* q desc info */
 106                                 rte_prefetch0(&q->info[q->tail_idx]);
 107
 108                         /*
 109                          * Note: you can just use rte_pktmbuf_free,
 110                          * but this loop is faster
 111                          */
 112                         txm = info[0];
 113                         while (txm != NULL) {
 114                                 next = txm->next;
 115                                 rte_pktmbuf_free_seg(txm);
 116                                 txm = next;
 117                         }
 118                 }
 119         }
 120 }
 121
 122 void __rte_cold
 123 ionic_dev_tx_queue_release(void *tx_queue)
 124 {
 125         struct ionic_tx_qcq *txq = tx_queue;
 126
 127         IONIC_PRINT_CALL();
 128
 129         ionic_lif_txq_deinit(txq);
 130
 131         ionic_qcq_free(&txq->qcq);
 132 }
 133
 134 int __rte_cold
 135 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 136 {
 137         struct ionic_tx_qcq *txq;
 138
 139         IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id);
 140
 141         txq = eth_dev->data->tx_queues[tx_queue_id];
 142
 143         eth_dev->data->tx_queue_state[tx_queue_id] =
 144                 RTE_ETH_QUEUE_STATE_STOPPED;
 145
 146         /*
 147          * Note: we should better post NOP Tx desc and wait for its completion
 148          * before disabling Tx queue
 149          */
 150
 151         ionic_qcq_disable(&txq->qcq);
 152
 153         ionic_tx_flush(txq);
 154
 155         return 0;
 156 }
 157
 158 int __rte_cold
 159 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
 160                 uint16_t nb_desc, uint32_t socket_id,
 161                 const struct rte_eth_txconf *tx_conf)
 162 {
 163         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 164         struct ionic_tx_qcq *txq;
 165         uint64_t offloads;
 166         int err;
 167
 168         if (tx_queue_id >= lif->ntxqcqs) {
 169                 IONIC_PRINT(DEBUG, "Queue index %u not available "
 170                         "(max %u queues)",
 171                         tx_queue_id, lif->ntxqcqs);
 172                 return -EINVAL;
 173         }
 174
 175         offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads;
 176         IONIC_PRINT(DEBUG,
 177                 "Configuring skt %u TX queue %u with %u buffers, offloads %jx",
 178                 socket_id, tx_queue_id, nb_desc, offloads);
 179
 180         /* Validate number of receive descriptors */
 181         if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC)
 182                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 183
 184         /* Free memory prior to re-allocation if needed... */
 185         if (eth_dev->data->tx_queues[tx_queue_id] != NULL) {
 186                 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id];
 187                 ionic_dev_tx_queue_release(tx_queue);
 188                 eth_dev->data->tx_queues[tx_queue_id] = NULL;
 189         }
 190
 191         eth_dev->data->tx_queue_state[tx_queue_id] =
 192                 RTE_ETH_QUEUE_STATE_STOPPED;
 193
 194         err = ionic_tx_qcq_alloc(lif, socket_id, tx_queue_id, nb_desc, &txq);
 195         if (err) {
 196                 IONIC_PRINT(DEBUG, "Queue allocation failure");
 197                 return -EINVAL;
 198         }
 199
 200         /* Do not start queue with rte_eth_dev_start() */
 201         if (tx_conf->tx_deferred_start)
 202                 txq->flags |= IONIC_QCQ_F_DEFERRED;
 203
 204         /* Convert the offload flags into queue flags */
 205         if (offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)
 206                 txq->flags |= IONIC_QCQ_F_CSUM_L3;
 207         if (offloads & DEV_TX_OFFLOAD_TCP_CKSUM)
 208                 txq->flags |= IONIC_QCQ_F_CSUM_TCP;
 209         if (offloads & DEV_TX_OFFLOAD_UDP_CKSUM)
 210                 txq->flags |= IONIC_QCQ_F_CSUM_UDP;
 211
 212         eth_dev->data->tx_queues[tx_queue_id] = txq;
 213
 214         return 0;
 215 }
 216
 217 /*
 218  * Start Transmit Units for specified queue.
 219  */
 220 int __rte_cold
 221 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 222 {
 223         uint8_t *tx_queue_state = eth_dev->data->tx_queue_state;
 224         struct ionic_tx_qcq *txq;
 225         int err;
 226
 227         if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
 228                 IONIC_PRINT(DEBUG, "TX queue %u already started",
 229                         tx_queue_id);
 230                 return 0;
 231         }
 232
 233         txq = eth_dev->data->tx_queues[tx_queue_id];
 234
 235         IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs",
 236                 tx_queue_id, txq->qcq.q.num_descs);
 237
 238         if (!(txq->flags & IONIC_QCQ_F_INITED)) {
 239                 err = ionic_lif_txq_init(txq);
 240                 if (err)
 241                         return err;
 242         } else {
 243                 ionic_qcq_enable(&txq->qcq);
 244         }
 245
 246         tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 247
 248         return 0;
 249 }
 250
 251 static void
 252 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm)
 253 {
 254         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
 255         char *l3_hdr = ((char *)eth_hdr) + txm->l2_len;
 256         struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
 257                 (l3_hdr + txm->l3_len);
 258
 259         if (txm->ol_flags & PKT_TX_IP_CKSUM) {
 260                 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
 261                 ipv4_hdr->hdr_checksum = 0;
 262                 tcp_hdr->cksum = 0;
 263                 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
 264         } else {
 265                 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
 266                 tcp_hdr->cksum = 0;
 267                 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
 268         }
 269 }
 270
 271 static void
 272 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm)
 273 {
 274         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
 275         char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len +
 276                 txm->outer_l3_len + txm->l2_len;
 277         struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
 278                 (l3_hdr + txm->l3_len);
 279
 280         if (txm->ol_flags & PKT_TX_IPV4) {
 281                 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
 282                 ipv4_hdr->hdr_checksum = 0;
 283                 tcp_hdr->cksum = 0;
 284                 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
 285         } else {
 286                 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
 287                 tcp_hdr->cksum = 0;
 288                 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
 289         }
 290 }
 291
 292 static void
 293 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
 294                 struct rte_mbuf *txm,
 295                 rte_iova_t addr, uint8_t nsge, uint16_t len,
 296                 uint32_t hdrlen, uint32_t mss,
 297                 bool encap,
 298                 uint16_t vlan_tci, bool has_vlan,
 299                 bool start, bool done)
 300 {
 301         uint8_t flags = 0;
 302         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 303         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
 304         flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
 305         flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
 306
 307         desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO,
 308                 flags, nsge, addr);
 309         desc->len = len;
 310         desc->vlan_tci = vlan_tci;
 311         desc->hdr_len = hdrlen;
 312         desc->mss = mss;
 313
 314         ionic_q_post(q, done, done ? txm : NULL);
 315 }
 316
 317 static struct ionic_txq_desc *
 318 ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem)
 319 {
 320         struct ionic_queue *q = &txq->qcq.q;
 321         struct ionic_txq_desc *desc_base = q->base;
 322         struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
 323         struct ionic_txq_desc *desc = &desc_base[q->head_idx];
 324         struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx];
 325
 326         *elem = sg_desc->elems;
 327         return desc;
 328 }
 329
 330 static int
 331 ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm,
 332                 bool not_xmit_more)
 333 {
 334         struct ionic_queue *q = &txq->qcq.q;
 335         struct ionic_tx_stats *stats = &txq->stats;
 336         struct ionic_txq_desc *desc;
 337         struct ionic_txq_sg_elem *elem;
 338         struct rte_mbuf *txm_seg;
 339         rte_iova_t data_iova;
 340         uint64_t desc_addr = 0, next_addr;
 341         uint16_t desc_len = 0;
 342         uint8_t desc_nsge;
 343         uint32_t hdrlen;
 344         uint32_t mss = txm->tso_segsz;
 345         uint32_t frag_left = 0;
 346         uint32_t left;
 347         uint32_t seglen;
 348         uint32_t len;
 349         uint32_t offset = 0;
 350         bool start, done;
 351         bool encap;
 352         bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT);
 353         uint16_t vlan_tci = txm->vlan_tci;
 354         uint64_t ol_flags = txm->ol_flags;
 355
 356         encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
 357                 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
 358                 ((ol_flags & PKT_TX_OUTER_IPV4) ||
 359                 (ol_flags & PKT_TX_OUTER_IPV6));
 360
 361         /* Preload inner-most TCP csum field with IP pseudo hdr
 362          * calculated with IP length set to zero.  HW will later
 363          * add in length to each TCP segment resulting from the TSO.
 364          */
 365
 366         if (encap) {
 367                 ionic_tx_tcp_inner_pseudo_csum(txm);
 368                 hdrlen = txm->outer_l2_len + txm->outer_l3_len +
 369                         txm->l2_len + txm->l3_len + txm->l4_len;
 370         } else {
 371                 ionic_tx_tcp_pseudo_csum(txm);
 372                 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len;
 373         }
 374
 375         seglen = hdrlen + mss;
 376         left = txm->data_len;
 377         data_iova = rte_mbuf_data_iova(txm);
 378
 379         desc = ionic_tx_tso_next(txq, &elem);
 380         start = true;
 381
 382         /* Chop data up into desc segments */
 383
 384         while (left > 0) {
 385                 len = RTE_MIN(seglen, left);
 386                 frag_left = seglen - len;
 387                 desc_addr = rte_cpu_to_le_64(data_iova + offset);
 388                 desc_len = len;
 389                 desc_nsge = 0;
 390                 left -= len;
 391                 offset += len;
 392                 if (txm->nb_segs > 1 && frag_left > 0)
 393                         continue;
 394                 done = (txm->nb_segs == 1 && left == 0);
 395                 ionic_tx_tso_post(q, desc, txm,
 396                         desc_addr, desc_nsge, desc_len,
 397                         hdrlen, mss,
 398                         encap,
 399                         vlan_tci, has_vlan,
 400                         start, done && not_xmit_more);
 401                 desc = ionic_tx_tso_next(txq, &elem);
 402                 start = false;
 403                 seglen = mss;
 404         }
 405
 406         /* Chop frags into desc segments */
 407
 408         txm_seg = txm->next;
 409         while (txm_seg != NULL) {
 410                 offset = 0;
 411                 data_iova = rte_mbuf_data_iova(txm_seg);
 412                 left = txm_seg->data_len;
 413                 stats->frags++;
 414
 415                 while (left > 0) {
 416                         next_addr = rte_cpu_to_le_64(data_iova + offset);
 417                         if (frag_left > 0) {
 418                                 len = RTE_MIN(frag_left, left);
 419                                 frag_left -= len;
 420                                 elem->addr = next_addr;
 421                                 elem->len = len;
 422                                 elem++;
 423                                 desc_nsge++;
 424                         } else {
 425                                 len = RTE_MIN(mss, left);
 426                                 frag_left = mss - len;
 427                                 desc_addr = next_addr;
 428                                 desc_len = len;
 429                                 desc_nsge = 0;
 430                         }
 431                         left -= len;
 432                         offset += len;
 433                         if (txm_seg->next != NULL && frag_left > 0)
 434                                 continue;
 435
 436                         done = (txm_seg->next == NULL && left == 0);
 437                         ionic_tx_tso_post(q, desc, txm_seg,
 438                                 desc_addr, desc_nsge, desc_len,
 439                                 hdrlen, mss,
 440                                 encap,
 441                                 vlan_tci, has_vlan,
 442                                 start, done && not_xmit_more);
 443                         desc = ionic_tx_tso_next(txq, &elem);
 444                         start = false;
 445                 }
 446
 447                 txm_seg = txm_seg->next;
 448         }
 449
 450         stats->tso++;
 451
 452         return 0;
 453 }
 454
 455 static __rte_always_inline int
 456 ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm,
 457                 bool not_xmit_more)
 458 {
 459         struct ionic_queue *q = &txq->qcq.q;
 460         struct ionic_txq_desc *desc, *desc_base = q->base;
 461         struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
 462         struct ionic_txq_sg_elem *elem;
 463         struct ionic_tx_stats *stats = &txq->stats;
 464         struct rte_mbuf *txm_seg;
 465         bool encap;
 466         bool has_vlan;
 467         uint64_t ol_flags = txm->ol_flags;
 468         uint64_t addr;
 469         uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
 470         uint8_t flags = 0;
 471
 472         desc = &desc_base[q->head_idx];
 473
 474         if ((ol_flags & PKT_TX_IP_CKSUM) &&
 475             (txq->flags & IONIC_QCQ_F_CSUM_L3)) {
 476                 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
 477                 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
 478         }
 479
 480         if (((ol_flags & PKT_TX_TCP_CKSUM) &&
 481              (txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
 482             ((ol_flags & PKT_TX_UDP_CKSUM) &&
 483              (txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
 484                 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
 485                 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
 486         }
 487
 488         if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
 489                 stats->no_csum++;
 490
 491         has_vlan = (ol_flags & PKT_TX_VLAN_PKT);
 492         encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
 493                         (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
 494                         ((ol_flags & PKT_TX_OUTER_IPV4) ||
 495                         (ol_flags & PKT_TX_OUTER_IPV6));
 496
 497         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 498         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
 499
 500         addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm));
 501
 502         desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr);
 503         desc->len = txm->data_len;
 504         desc->vlan_tci = txm->vlan_tci;
 505
 506         elem = sg_desc_base[q->head_idx].elems;
 507         txm_seg = txm->next;
 508         while (txm_seg != NULL) {
 509                 elem->len = txm_seg->data_len;
 510                 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg));
 511                 stats->frags++;
 512                 elem++;
 513                 txm_seg = txm_seg->next;
 514         }
 515
 516         ionic_q_post(q, not_xmit_more, txm);
 517
 518         return 0;
 519 }
 520
 521 uint16_t
 522 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 523                 uint16_t nb_pkts)
 524 {
 525         struct ionic_tx_qcq *txq = tx_queue;
 526         struct ionic_queue *q = &txq->qcq.q;
 527         struct ionic_tx_stats *stats = &txq->stats;
 528         uint32_t next_q_head_idx;
 529         uint32_t bytes_tx = 0;
 530         uint16_t nb_tx = 0;
 531         int err;
 532         bool last;
 533
 534         /* Cleaning old buffers */
 535         ionic_tx_flush(txq);
 536
 537         if (unlikely(ionic_q_space_avail(q) < nb_pkts)) {
 538                 stats->stop += nb_pkts;
 539                 return 0;
 540         }
 541
 542         while (nb_tx < nb_pkts) {
 543                 last = (nb_tx == (nb_pkts - 1));
 544
 545                 next_q_head_idx = Q_NEXT_TO_POST(q, 1);
 546                 if ((next_q_head_idx & 0x3) == 0) {
 547                         struct ionic_txq_desc *desc_base = q->base;
 548                         rte_prefetch0(&desc_base[next_q_head_idx]);
 549                         rte_prefetch0(&q->info[next_q_head_idx]);
 550                 }
 551
 552                 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG)
 553                         err = ionic_tx_tso(txq, tx_pkts[nb_tx], last);
 554                 else
 555                         err = ionic_tx(txq, tx_pkts[nb_tx], last);
 556                 if (err) {
 557                         stats->drop += nb_pkts - nb_tx;
 558                         if (nb_tx > 0)
 559                                 ionic_q_flush(q);
 560                         break;
 561                 }
 562
 563                 bytes_tx += tx_pkts[nb_tx]->pkt_len;
 564                 nb_tx++;
 565         }
 566
 567         stats->packets += nb_tx;
 568         stats->bytes += bytes_tx;
 569
 570         return nb_tx;
 571 }
 572
 573 /*********************************************************************
 574  *
 575  *  TX prep functions
 576  *
 577  **********************************************************************/
 578
 579 #define IONIC_TX_OFFLOAD_MASK ( \
 580         PKT_TX_IPV4 |           \
 581         PKT_TX_IPV6 |           \
 582         PKT_TX_VLAN |           \
 583         PKT_TX_IP_CKSUM |       \
 584         PKT_TX_TCP_SEG |        \
 585         PKT_TX_L4_MASK)
 586
 587 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \
 588         (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK)
 589
 590 uint16_t
 591 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
 592                 uint16_t nb_pkts)
 593 {
 594         struct rte_mbuf *txm;
 595         uint64_t offloads;
 596         int i = 0;
 597
 598         for (i = 0; i < nb_pkts; i++) {
 599                 txm = tx_pkts[i];
 600
 601                 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS_V1 + 1) {
 602                         rte_errno = -EINVAL;
 603                         break;
 604                 }
 605
 606                 offloads = txm->ol_flags;
 607
 608                 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) {
 609                         rte_errno = -ENOTSUP;
 610                         break;
 611                 }
 612         }
 613
 614         return i;
 615 }
 616
 617 /*********************************************************************
 618  *
 619  *  RX functions
 620  *
 621  **********************************************************************/
 622
 623 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 624                 struct rte_mbuf *mbuf);
 625
 626 void
 627 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 628                 struct rte_eth_rxq_info *qinfo)
 629 {
 630         struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id];
 631         struct ionic_queue *q = &rxq->qcq.q;
 632
 633         qinfo->mp = rxq->mb_pool;
 634         qinfo->scattered_rx = dev->data->scattered_rx;
 635         qinfo->nb_desc = q->num_descs;
 636         qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED;
 637         qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
 638 }
 639
 640 static void __rte_cold
 641 ionic_rx_empty(struct ionic_rx_qcq *rxq)
 642 {
 643         struct ionic_queue *q = &rxq->qcq.q;
 644         struct rte_mbuf *mbuf;
 645         void **info;
 646
 647         while (q->tail_idx != q->head_idx) {
 648                 info = IONIC_INFO_PTR(q, q->tail_idx);
 649                 mbuf = info[0];
 650                 rte_mempool_put(rxq->mb_pool, mbuf);
 651
 652                 q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
 653         }
 654 }
 655
 656 void __rte_cold
 657 ionic_dev_rx_queue_release(void *rx_queue)
 658 {
 659         struct ionic_rx_qcq *rxq = rx_queue;
 660
 661         if (!rxq)
 662                 return;
 663
 664         IONIC_PRINT_CALL();
 665
 666         ionic_rx_empty(rxq);
 667
 668         ionic_lif_rxq_deinit(rxq);
 669
 670         ionic_qcq_free(&rxq->qcq);
 671 }
 672
 673 int __rte_cold
 674 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 675                 uint16_t rx_queue_id,
 676                 uint16_t nb_desc,
 677                 uint32_t socket_id,
 678                 const struct rte_eth_rxconf *rx_conf,
 679                 struct rte_mempool *mp)
 680 {
 681         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 682         struct ionic_rx_qcq *rxq;
 683         uint64_t offloads;
 684         int err;
 685
 686         if (rx_queue_id >= lif->nrxqcqs) {
 687                 IONIC_PRINT(ERR,
 688                         "Queue index %u not available (max %u queues)",
 689                         rx_queue_id, lif->nrxqcqs);
 690                 return -EINVAL;
 691         }
 692
 693         offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads;
 694         IONIC_PRINT(DEBUG,
 695                 "Configuring skt %u RX queue %u with %u buffers, offloads %jx",
 696                 socket_id, rx_queue_id, nb_desc, offloads);
 697
 698         if (!rx_conf->rx_drop_en)
 699                 IONIC_PRINT(WARNING, "No-drop mode is not supported");
 700
 701         /* Validate number of receive descriptors */
 702         if (!rte_is_power_of_2(nb_desc) ||
 703                         nb_desc < IONIC_MIN_RING_DESC ||
 704                         nb_desc > IONIC_MAX_RING_DESC) {
 705                 IONIC_PRINT(ERR,
 706                         "Bad descriptor count (%u) for queue %u (min: %u)",
 707                         nb_desc, rx_queue_id, IONIC_MIN_RING_DESC);
 708                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 709         }
 710
 711         /* Free memory prior to re-allocation if needed... */
 712         if (eth_dev->data->rx_queues[rx_queue_id] != NULL) {
 713                 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id];
 714                 ionic_dev_rx_queue_release(rx_queue);
 715                 eth_dev->data->rx_queues[rx_queue_id] = NULL;
 716         }
 717
 718         eth_dev->data->rx_queue_state[rx_queue_id] =
 719                 RTE_ETH_QUEUE_STATE_STOPPED;
 720
 721         err = ionic_rx_qcq_alloc(lif, socket_id, rx_queue_id, nb_desc,
 722                         &rxq);
 723         if (err) {
 724                 IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id);
 725                 return -EINVAL;
 726         }
 727
 728         rxq->mb_pool = mp;
 729
 730         /*
 731          * Note: the interface does not currently support
 732          * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN
 733          * when the adapter will be able to keep the CRC and subtract
 734          * it to the length for all received packets:
 735          * if (eth_dev->data->dev_conf.rxmode.offloads &
 736          *     DEV_RX_OFFLOAD_KEEP_CRC)
 737          *   rxq->crc_len = ETHER_CRC_LEN;
 738          */
 739
 740         /* Do not start queue with rte_eth_dev_start() */
 741         if (rx_conf->rx_deferred_start)
 742                 rxq->flags |= IONIC_QCQ_F_DEFERRED;
 743
 744         eth_dev->data->rx_queues[rx_queue_id] = rxq;
 745
 746         return 0;
 747 }
 748
 749 static __rte_always_inline void
 750 ionic_rx_clean(struct ionic_rx_qcq *rxq,
 751                 uint32_t q_desc_index, uint32_t cq_desc_index,
 752                 void *service_cb_arg)
 753 {
 754         struct ionic_queue *q = &rxq->qcq.q;
 755         struct ionic_cq *cq = &rxq->qcq.cq;
 756         struct ionic_rxq_comp *cq_desc_base = cq->base;
 757         struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index];
 758         struct rte_mbuf *rxm, *rxm_seg;
 759         uint32_t max_frame_size =
 760                 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 761         uint64_t pkt_flags = 0;
 762         uint32_t pkt_type;
 763         struct ionic_rx_stats *stats = &rxq->stats;
 764         struct ionic_rx_service *recv_args = (struct ionic_rx_service *)
 765                 service_cb_arg;
 766         uint32_t buf_size = (uint16_t)
 767                 (rte_pktmbuf_data_room_size(rxq->mb_pool) -
 768                 RTE_PKTMBUF_HEADROOM);
 769         uint32_t left;
 770         void **info;
 771
 772         assert(q_desc_index == cq_desc->comp_index);
 773
 774         info = IONIC_INFO_PTR(q, cq_desc->comp_index);
 775
 776         rxm = info[0];
 777
 778         if (!recv_args) {
 779                 stats->no_cb_arg++;
 780                 /* Flush */
 781                 rte_pktmbuf_free(rxm);
 782                 /*
 783                  * Note: rte_mempool_put is faster with no segs
 784                  * rte_mempool_put(rxq->mb_pool, rxm);
 785                  */
 786                 return;
 787         }
 788
 789         if (cq_desc->status) {
 790                 stats->bad_cq_status++;
 791                 ionic_rx_recycle(q, q_desc_index, rxm);
 792                 return;
 793         }
 794
 795         if (recv_args->nb_rx >= recv_args->nb_pkts) {
 796                 stats->no_room++;
 797                 ionic_rx_recycle(q, q_desc_index, rxm);
 798                 return;
 799         }
 800
 801         if (cq_desc->len > max_frame_size ||
 802                         cq_desc->len == 0) {
 803                 stats->bad_len++;
 804                 ionic_rx_recycle(q, q_desc_index, rxm);
 805                 return;
 806         }
 807
 808         rxm->data_off = RTE_PKTMBUF_HEADROOM;
 809         rte_prefetch1((char *)rxm->buf_addr + rxm->data_off);
 810         rxm->nb_segs = 1; /* cq_desc->num_sg_elems */
 811         rxm->pkt_len = cq_desc->len;
 812         rxm->port = rxq->qcq.lif->port_id;
 813
 814         left = cq_desc->len;
 815
 816         rxm->data_len = RTE_MIN(buf_size, left);
 817         left -= rxm->data_len;
 818
 819         rxm_seg = rxm->next;
 820         while (rxm_seg && left) {
 821                 rxm_seg->data_len = RTE_MIN(buf_size, left);
 822                 left -= rxm_seg->data_len;
 823
 824                 rxm_seg = rxm_seg->next;
 825                 rxm->nb_segs++;
 826         }
 827
 828         /* RSS */
 829         pkt_flags |= PKT_RX_RSS_HASH;
 830         rxm->hash.rss = cq_desc->rss_hash;
 831
 832         /* Vlan Strip */
 833         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
 834                 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 835                 rxm->vlan_tci = cq_desc->vlan_tci;
 836         }
 837
 838         /* Checksum */
 839         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
 840                 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK)
 841                         pkt_flags |= PKT_RX_IP_CKSUM_GOOD;
 842                 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)
 843                         pkt_flags |= PKT_RX_IP_CKSUM_BAD;
 844
 845                 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) ||
 846                         (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK))
 847                         pkt_flags |= PKT_RX_L4_CKSUM_GOOD;
 848                 else if ((cq_desc->csum_flags &
 849                                 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
 850                                 (cq_desc->csum_flags &
 851                                 IONIC_RXQ_COMP_CSUM_F_UDP_BAD))
 852                         pkt_flags |= PKT_RX_L4_CKSUM_BAD;
 853         }
 854
 855         rxm->ol_flags = pkt_flags;
 856
 857         /* Packet Type */
 858         switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
 859         case IONIC_PKT_TYPE_IPV4:
 860                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 861                 break;
 862         case IONIC_PKT_TYPE_IPV6:
 863                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 864                 break;
 865         case IONIC_PKT_TYPE_IPV4_TCP:
 866                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 867                         RTE_PTYPE_L4_TCP;
 868                 break;
 869         case IONIC_PKT_TYPE_IPV6_TCP:
 870                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 871                         RTE_PTYPE_L4_TCP;
 872                 break;
 873         case IONIC_PKT_TYPE_IPV4_UDP:
 874                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 875                         RTE_PTYPE_L4_UDP;
 876                 break;
 877         case IONIC_PKT_TYPE_IPV6_UDP:
 878                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 879                         RTE_PTYPE_L4_UDP;
 880                 break;
 881         default:
 882                 {
 883                         struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
 884                                 struct rte_ether_hdr *);
 885                         uint16_t ether_type = eth_h->ether_type;
 886                         if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
 887                                 pkt_type = RTE_PTYPE_L2_ETHER_ARP;
 888                         else
 889                                 pkt_type = RTE_PTYPE_UNKNOWN;
 890                         break;
 891                 }
 892         }
 893
 894         rxm->packet_type = pkt_type;
 895
 896         recv_args->rx_pkts[recv_args->nb_rx] = rxm;
 897         recv_args->nb_rx++;
 898
 899         stats->packets++;
 900         stats->bytes += rxm->pkt_len;
 901 }
 902
 903 static void
 904 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 905                  struct rte_mbuf *mbuf)
 906 {
 907         struct ionic_rxq_desc *desc_base = q->base;
 908         struct ionic_rxq_desc *old = &desc_base[q_desc_index];
 909         struct ionic_rxq_desc *new = &desc_base[q->head_idx];
 910
 911         new->addr = old->addr;
 912         new->len = old->len;
 913
 914         ionic_q_post(q, true, mbuf);
 915 }
 916
 917 static __rte_always_inline int
 918 ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len)
 919 {
 920         struct ionic_queue *q = &rxq->qcq.q;
 921         struct ionic_rxq_desc *desc, *desc_base = q->base;
 922         struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
 923         struct ionic_rxq_sg_elem *elem;
 924         rte_iova_t dma_addr;
 925         uint32_t i, j, nsegs, buf_size, size;
 926         bool ring_doorbell;
 927
 928         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 929                 RTE_PKTMBUF_HEADROOM);
 930
 931         /* Initialize software ring entries */
 932         for (i = ionic_q_space_avail(q); i; i--) {
 933                 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool);
 934                 struct rte_mbuf *prev_rxm_seg;
 935
 936                 if (rxm == NULL) {
 937                         IONIC_PRINT(ERR, "RX mbuf alloc failed");
 938                         return -ENOMEM;
 939                 }
 940
 941                 nsegs = (len + buf_size - 1) / buf_size;
 942
 943                 desc = &desc_base[q->head_idx];
 944                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm));
 945                 desc->addr = dma_addr;
 946                 desc->len = buf_size;
 947                 size = buf_size;
 948                 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
 949                         IONIC_RXQ_DESC_OPCODE_SIMPLE;
 950                 rxm->next = NULL;
 951
 952                 prev_rxm_seg = rxm;
 953                 sg_desc = &sg_desc_base[q->head_idx];
 954                 elem = sg_desc->elems;
 955                 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) {
 956                         struct rte_mbuf *rxm_seg;
 957                         rte_iova_t data_iova;
 958
 959                         rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool);
 960                         if (rxm_seg == NULL) {
 961                                 IONIC_PRINT(ERR, "RX mbuf alloc failed");
 962                                 return -ENOMEM;
 963                         }
 964
 965                         data_iova = rte_mbuf_data_iova(rxm_seg);
 966                         dma_addr = rte_cpu_to_le_64(data_iova);
 967                         elem->addr = dma_addr;
 968                         elem->len = buf_size;
 969                         size += buf_size;
 970                         elem++;
 971                         rxm_seg->next = NULL;
 972                         prev_rxm_seg->next = rxm_seg;
 973                         prev_rxm_seg = rxm_seg;
 974                 }
 975
 976                 if (size < len)
 977                         IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)",
 978                                 size, len);
 979
 980                 ring_doorbell = ((q->head_idx + 1) &
 981                         IONIC_RX_RING_DOORBELL_STRIDE) == 0;
 982
 983                 ionic_q_post(q, ring_doorbell, rxm);
 984         }
 985
 986         return 0;
 987 }
 988
 989 /*
 990  * Start Receive Units for specified queue.
 991  */
 992 int __rte_cold
 993 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 994 {
 995         uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 996         uint8_t *rx_queue_state = eth_dev->data->rx_queue_state;
 997         struct ionic_rx_qcq *rxq;
 998         int err;
 999
1000         if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
1001                 IONIC_PRINT(DEBUG, "RX queue %u already started",
1002                         rx_queue_id);
1003                 return 0;
1004         }
1005
1006         rxq = eth_dev->data->rx_queues[rx_queue_id];
1007
1008         IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)",
1009                 rx_queue_id, rxq->qcq.q.num_descs, frame_size);
1010
1011         if (!(rxq->flags & IONIC_QCQ_F_INITED)) {
1012                 err = ionic_lif_rxq_init(rxq);
1013                 if (err)
1014                         return err;
1015         } else {
1016                 ionic_qcq_enable(&rxq->qcq);
1017         }
1018
1019         /* Allocate buffers for descriptor rings */
1020         if (ionic_rx_fill(rxq, frame_size) != 0) {
1021                 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d",
1022                         rx_queue_id);
1023                 return -1;
1024         }
1025
1026         rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1027
1028         return 0;
1029 }
1030
1031 static __rte_always_inline void
1032 ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
1033                 void *service_cb_arg)
1034 {
1035         struct ionic_cq *cq = &rxq->qcq.cq;
1036         struct ionic_queue *q = &rxq->qcq.q;
1037         struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;
1038         bool more;
1039         uint32_t curr_q_tail_idx, curr_cq_tail_idx;
1040         uint32_t work_done = 0;
1041
1042         if (work_to_do == 0)
1043                 return;
1044
1045         cq_desc = &cq_desc_base[cq->tail_idx];
1046         while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
1047                 curr_cq_tail_idx = cq->tail_idx;
1048                 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
1049
1050                 if (cq->tail_idx == 0)
1051                         cq->done_color = !cq->done_color;
1052
1053                 /* Prefetch the next 4 descriptors */
1054                 if ((cq->tail_idx & 0x3) == 0)
1055                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
1056
1057                 do {
1058                         more = (q->tail_idx != cq_desc->comp_index);
1059
1060                         curr_q_tail_idx = q->tail_idx;
1061                         q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
1062
1063                         /* Prefetch the next 4 descriptors */
1064                         if ((q->tail_idx & 0x3) == 0)
1065                                 /* q desc info */
1066                                 rte_prefetch0(&q->info[q->tail_idx]);
1067
1068                         ionic_rx_clean(rxq, curr_q_tail_idx, curr_cq_tail_idx,
1069                                 service_cb_arg);
1070
1071                 } while (more);
1072
1073                 if (++work_done == work_to_do)
1074                         break;
1075
1076                 cq_desc = &cq_desc_base[cq->tail_idx];
1077         }
1078 }
1079
1080 /*
1081  * Stop Receive Units for specified queue.
1082  */
1083 int __rte_cold
1084 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
1085 {
1086         struct ionic_rx_qcq *rxq;
1087
1088         IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id);
1089
1090         rxq = eth_dev->data->rx_queues[rx_queue_id];
1091
1092         eth_dev->data->rx_queue_state[rx_queue_id] =
1093                 RTE_ETH_QUEUE_STATE_STOPPED;
1094
1095         ionic_qcq_disable(&rxq->qcq);
1096
1097         /* Flush */
1098         ionic_rxq_service(rxq, -1, NULL);
1099
1100         return 0;
1101 }
1102
1103 uint16_t
1104 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1105                 uint16_t nb_pkts)
1106 {
1107         struct ionic_rx_qcq *rxq = rx_queue;
1108         uint32_t frame_size =
1109                 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
1110         struct ionic_rx_service service_cb_arg;
1111
1112         service_cb_arg.rx_pkts = rx_pkts;
1113         service_cb_arg.nb_pkts = nb_pkts;
1114         service_cb_arg.nb_rx = 0;
1115
1116         ionic_rxq_service(rxq, nb_pkts, &service_cb_arg);
1117
1118         ionic_rx_fill(rxq, frame_size);
1119
1120         return service_cb_arg.nb_rx;
1121 }