drivers/net/ionic/ionic_rxtx.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <errno.h>
  10 #include <stdint.h>
  11 #include <stdarg.h>
  12 #include <unistd.h>
  13 #include <inttypes.h>
  14
  15 #include <rte_byteorder.h>
  16 #include <rte_common.h>
  17 #include <rte_cycles.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <rte_interrupts.h>
  21 #include <rte_pci.h>
  22 #include <rte_memory.h>
  23 #include <rte_memzone.h>
  24 #include <rte_launch.h>
  25 #include <rte_eal.h>
  26 #include <rte_per_lcore.h>
  27 #include <rte_lcore.h>
  28 #include <rte_atomic.h>
  29 #include <rte_branch_prediction.h>
  30 #include <rte_mempool.h>
  31 #include <rte_malloc.h>
  32 #include <rte_mbuf.h>
  33 #include <rte_ether.h>
  34 #include <ethdev_driver.h>
  35 #include <rte_prefetch.h>
  36 #include <rte_udp.h>
  37 #include <rte_tcp.h>
  38 #include <rte_sctp.h>
  39 #include <rte_string_fns.h>
  40 #include <rte_errno.h>
  41 #include <rte_ip.h>
  42 #include <rte_net.h>
  43
  44 #include "ionic_logs.h"
  45 #include "ionic_mac_api.h"
  46 #include "ionic_ethdev.h"
  47 #include "ionic_lif.h"
  48 #include "ionic_rxtx.h"
  49
  50 #define IONIC_RX_RING_DOORBELL_STRIDE           (32 - 1)
  51
  52 /*********************************************************************
  53  *
  54  *  TX functions
  55  *
  56  **********************************************************************/
  57
  58 void
  59 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
  60                 struct rte_eth_txq_info *qinfo)
  61 {
  62         struct ionic_qcq *txq = dev->data->tx_queues[queue_id];
  63         struct ionic_queue *q = &txq->q;
  64
  65         qinfo->nb_desc = q->num_descs;
  66         qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
  67         qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED;
  68 }
  69
  70 static __rte_always_inline void
  71 ionic_tx_flush(struct ionic_cq *cq)
  72 {
  73         struct ionic_queue *q = cq->bound_q;
  74         struct ionic_desc_info *q_desc_info;
  75         struct rte_mbuf *txm, *next;
  76         struct ionic_txq_comp *cq_desc_base = cq->base;
  77         struct ionic_txq_comp *cq_desc;
  78         u_int32_t comp_index = (u_int32_t)-1;
  79
  80         cq_desc = &cq_desc_base[cq->tail_idx];
  81         while (color_match(cq_desc->color, cq->done_color)) {
  82                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
  83
  84                 /* Prefetch the next 4 descriptors (not really useful here) */
  85                 if ((cq->tail_idx & 0x3) == 0)
  86                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
  87
  88                 if (cq->tail_idx == 0)
  89                         cq->done_color = !cq->done_color;
  90
  91                 comp_index = cq_desc->comp_index;
  92
  93                 cq_desc = &cq_desc_base[cq->tail_idx];
  94         }
  95
  96         if (comp_index != (u_int32_t)-1) {
  97                 while (q->tail_idx != comp_index) {
  98                         q_desc_info = &q->info[q->tail_idx];
  99
 100                         q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 101
 102                         /* Prefetch the next 4 descriptors */
 103                         if ((q->tail_idx & 0x3) == 0)
 104                                 /* q desc info */
 105                                 rte_prefetch0(&q->info[q->tail_idx]);
 106
 107                         /*
 108                          * Note: you can just use rte_pktmbuf_free,
 109                          * but this loop is faster
 110                          */
 111                         txm = q_desc_info->cb_arg;
 112                         while (txm != NULL) {
 113                                 next = txm->next;
 114                                 rte_pktmbuf_free_seg(txm);
 115                                 txm = next;
 116                         }
 117                 }
 118         }
 119 }
 120
 121 void __rte_cold
 122 ionic_dev_tx_queue_release(void *tx_queue)
 123 {
 124         struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue;
 125
 126         IONIC_PRINT_CALL();
 127
 128         ionic_lif_txq_deinit(txq);
 129
 130         ionic_qcq_free(txq);
 131 }
 132
 133 int __rte_cold
 134 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 135 {
 136         struct ionic_qcq *txq;
 137
 138         IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id);
 139
 140         txq = eth_dev->data->tx_queues[tx_queue_id];
 141
 142         eth_dev->data->tx_queue_state[tx_queue_id] =
 143                 RTE_ETH_QUEUE_STATE_STOPPED;
 144
 145         /*
 146          * Note: we should better post NOP Tx desc and wait for its completion
 147          * before disabling Tx queue
 148          */
 149
 150         ionic_qcq_disable(txq);
 151
 152         ionic_tx_flush(&txq->cq);
 153
 154         return 0;
 155 }
 156
 157 int __rte_cold
 158 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
 159                 uint16_t nb_desc, uint32_t socket_id,
 160                 const struct rte_eth_txconf *tx_conf)
 161 {
 162         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 163         struct ionic_qcq *txq;
 164         uint64_t offloads;
 165         int err;
 166
 167         if (tx_queue_id >= lif->ntxqcqs) {
 168                 IONIC_PRINT(DEBUG, "Queue index %u not available "
 169                         "(max %u queues)",
 170                         tx_queue_id, lif->ntxqcqs);
 171                 return -EINVAL;
 172         }
 173
 174         offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads;
 175         IONIC_PRINT(DEBUG,
 176                 "Configuring skt %u TX queue %u with %u buffers, offloads %jx",
 177                 socket_id, tx_queue_id, nb_desc, offloads);
 178
 179         /* Validate number of receive descriptors */
 180         if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC)
 181                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 182
 183         /* Free memory prior to re-allocation if needed... */
 184         if (eth_dev->data->tx_queues[tx_queue_id] != NULL) {
 185                 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id];
 186                 ionic_dev_tx_queue_release(tx_queue);
 187                 eth_dev->data->tx_queues[tx_queue_id] = NULL;
 188         }
 189
 190         eth_dev->data->tx_queue_state[tx_queue_id] =
 191                 RTE_ETH_QUEUE_STATE_STOPPED;
 192
 193         err = ionic_tx_qcq_alloc(lif, tx_queue_id, nb_desc, &txq);
 194         if (err) {
 195                 IONIC_PRINT(DEBUG, "Queue allocation failure");
 196                 return -EINVAL;
 197         }
 198
 199         /* Do not start queue with rte_eth_dev_start() */
 200         if (tx_conf->tx_deferred_start)
 201                 txq->flags |= IONIC_QCQ_F_DEFERRED;
 202
 203         /* Convert the offload flags into queue flags */
 204         if (offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)
 205                 txq->flags |= IONIC_QCQ_F_CSUM_L3;
 206         if (offloads & DEV_TX_OFFLOAD_TCP_CKSUM)
 207                 txq->flags |= IONIC_QCQ_F_CSUM_TCP;
 208         if (offloads & DEV_TX_OFFLOAD_UDP_CKSUM)
 209                 txq->flags |= IONIC_QCQ_F_CSUM_UDP;
 210
 211         eth_dev->data->tx_queues[tx_queue_id] = txq;
 212
 213         return 0;
 214 }
 215
 216 /*
 217  * Start Transmit Units for specified queue.
 218  */
 219 int __rte_cold
 220 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 221 {
 222         uint8_t *tx_queue_state = eth_dev->data->tx_queue_state;
 223         struct ionic_qcq *txq;
 224         int err;
 225
 226         if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
 227                 IONIC_PRINT(DEBUG, "TX queue %u already started",
 228                         tx_queue_id);
 229                 return 0;
 230         }
 231
 232         txq = eth_dev->data->tx_queues[tx_queue_id];
 233
 234         IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs",
 235                 tx_queue_id, txq->q.num_descs);
 236
 237         if (!(txq->flags & IONIC_QCQ_F_INITED)) {
 238                 err = ionic_lif_txq_init(txq);
 239                 if (err)
 240                         return err;
 241         } else {
 242                 ionic_qcq_enable(txq);
 243         }
 244
 245         tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 246
 247         return 0;
 248 }
 249
 250 static void
 251 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm)
 252 {
 253         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
 254         char *l3_hdr = ((char *)eth_hdr) + txm->l2_len;
 255         struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
 256                 (l3_hdr + txm->l3_len);
 257
 258         if (txm->ol_flags & PKT_TX_IP_CKSUM) {
 259                 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
 260                 ipv4_hdr->hdr_checksum = 0;
 261                 tcp_hdr->cksum = 0;
 262                 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
 263         } else {
 264                 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
 265                 tcp_hdr->cksum = 0;
 266                 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
 267         }
 268 }
 269
 270 static void
 271 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm)
 272 {
 273         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
 274         char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len +
 275                 txm->outer_l3_len + txm->l2_len;
 276         struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
 277                 (l3_hdr + txm->l3_len);
 278
 279         if (txm->ol_flags & PKT_TX_IPV4) {
 280                 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
 281                 ipv4_hdr->hdr_checksum = 0;
 282                 tcp_hdr->cksum = 0;
 283                 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
 284         } else {
 285                 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
 286                 tcp_hdr->cksum = 0;
 287                 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
 288         }
 289 }
 290
 291 static void
 292 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
 293                 struct rte_mbuf *txm,
 294                 rte_iova_t addr, uint8_t nsge, uint16_t len,
 295                 uint32_t hdrlen, uint32_t mss,
 296                 bool encap,
 297                 uint16_t vlan_tci, bool has_vlan,
 298                 bool start, bool done)
 299 {
 300         uint8_t flags = 0;
 301         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 302         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
 303         flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
 304         flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
 305
 306         desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO,
 307                 flags, nsge, addr);
 308         desc->len = len;
 309         desc->vlan_tci = vlan_tci;
 310         desc->hdr_len = hdrlen;
 311         desc->mss = mss;
 312
 313         ionic_q_post(q, done, NULL, done ? txm : NULL);
 314 }
 315
 316 static struct ionic_txq_desc *
 317 ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem)
 318 {
 319         struct ionic_txq_desc *desc_base = q->base;
 320         struct ionic_txq_sg_desc *sg_desc_base = q->sg_base;
 321         struct ionic_txq_desc *desc = &desc_base[q->head_idx];
 322         struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx];
 323
 324         *elem = sg_desc->elems;
 325         return desc;
 326 }
 327
 328 static int
 329 ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm,
 330                 bool not_xmit_more)
 331 {
 332         struct ionic_queue *q = &txq->q;
 333         struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
 334         struct ionic_txq_desc *desc;
 335         struct ionic_txq_sg_elem *elem;
 336         struct rte_mbuf *txm_seg;
 337         uint64_t desc_addr = 0;
 338         uint16_t desc_len = 0;
 339         uint8_t desc_nsge;
 340         uint32_t hdrlen;
 341         uint32_t mss = txm->tso_segsz;
 342         uint32_t frag_left = 0;
 343         uint32_t left;
 344         uint32_t seglen;
 345         uint32_t len;
 346         uint32_t offset = 0;
 347         bool start, done;
 348         bool encap;
 349         bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT);
 350         uint16_t vlan_tci = txm->vlan_tci;
 351         uint64_t ol_flags = txm->ol_flags;
 352
 353         encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
 354                 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
 355                 ((ol_flags & PKT_TX_OUTER_IPV4) ||
 356                 (ol_flags & PKT_TX_OUTER_IPV6));
 357
 358         /* Preload inner-most TCP csum field with IP pseudo hdr
 359          * calculated with IP length set to zero.  HW will later
 360          * add in length to each TCP segment resulting from the TSO.
 361          */
 362
 363         if (encap) {
 364                 ionic_tx_tcp_inner_pseudo_csum(txm);
 365                 hdrlen = txm->outer_l2_len + txm->outer_l3_len +
 366                         txm->l2_len + txm->l3_len + txm->l4_len;
 367         } else {
 368                 ionic_tx_tcp_pseudo_csum(txm);
 369                 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len;
 370         }
 371
 372         seglen = hdrlen + mss;
 373         left = txm->data_len;
 374
 375         desc = ionic_tx_tso_next(q, &elem);
 376         start = true;
 377
 378         /* Chop data up into desc segments */
 379
 380         while (left > 0) {
 381                 len = RTE_MIN(seglen, left);
 382                 frag_left = seglen - len;
 383                 desc_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(txm));
 384                 desc_len = len;
 385                 desc_nsge = 0;
 386                 left -= len;
 387                 offset += len;
 388                 if (txm->nb_segs > 1 && frag_left > 0)
 389                         continue;
 390                 done = (txm->nb_segs == 1 && left == 0);
 391                 ionic_tx_tso_post(q, desc, txm,
 392                         desc_addr, desc_nsge, desc_len,
 393                         hdrlen, mss,
 394                         encap,
 395                         vlan_tci, has_vlan,
 396                         start, done && not_xmit_more);
 397                 desc = ionic_tx_tso_next(q, &elem);
 398                 start = false;
 399                 seglen = mss;
 400         }
 401
 402         /* Chop frags into desc segments */
 403
 404         txm_seg = txm->next;
 405         while (txm_seg != NULL) {
 406                 offset = 0;
 407                 left = txm_seg->data_len;
 408                 stats->frags++;
 409
 410                 while (left > 0) {
 411                         rte_iova_t data_iova;
 412                         data_iova = rte_mbuf_data_iova(txm_seg);
 413                         elem->addr = rte_cpu_to_le_64(data_iova) + offset;
 414                         if (frag_left > 0) {
 415                                 len = RTE_MIN(frag_left, left);
 416                                 frag_left -= len;
 417                                 elem->len = len;
 418                                 elem++;
 419                                 desc_nsge++;
 420                         } else {
 421                                 len = RTE_MIN(mss, left);
 422                                 frag_left = mss - len;
 423                                 data_iova = rte_mbuf_data_iova(txm_seg);
 424                                 desc_addr = rte_cpu_to_le_64(data_iova);
 425                                 desc_len = len;
 426                                 desc_nsge = 0;
 427                         }
 428                         left -= len;
 429                         offset += len;
 430                         if (txm_seg->next != NULL && frag_left > 0)
 431                                 continue;
 432                         done = (txm_seg->next == NULL && left == 0);
 433                         ionic_tx_tso_post(q, desc, txm_seg,
 434                                 desc_addr, desc_nsge, desc_len,
 435                                 hdrlen, mss,
 436                                 encap,
 437                                 vlan_tci, has_vlan,
 438                                 start, done && not_xmit_more);
 439                         desc = ionic_tx_tso_next(q, &elem);
 440                         start = false;
 441                 }
 442
 443                 txm_seg = txm_seg->next;
 444         }
 445
 446         stats->tso++;
 447
 448         return 0;
 449 }
 450
 451 static __rte_always_inline int
 452 ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm,
 453                 bool not_xmit_more)
 454 {
 455         struct ionic_queue *q = &txq->q;
 456         struct ionic_txq_desc *desc_base = q->base;
 457         struct ionic_txq_sg_desc *sg_desc_base = q->sg_base;
 458         struct ionic_txq_desc *desc = &desc_base[q->head_idx];
 459         struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx];
 460         struct ionic_txq_sg_elem *elem = sg_desc->elems;
 461         struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
 462         struct rte_mbuf *txm_seg;
 463         bool encap;
 464         bool has_vlan;
 465         uint64_t ol_flags = txm->ol_flags;
 466         uint64_t addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(txm));
 467         uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
 468         uint8_t flags = 0;
 469
 470         if ((ol_flags & PKT_TX_IP_CKSUM) &&
 471             (txq->flags & IONIC_QCQ_F_CSUM_L3)) {
 472                 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
 473                 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
 474         }
 475
 476         if (((ol_flags & PKT_TX_TCP_CKSUM) &&
 477              (txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
 478             ((ol_flags & PKT_TX_UDP_CKSUM) &&
 479              (txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
 480                 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
 481                 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
 482         }
 483
 484         if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
 485                 stats->no_csum++;
 486
 487         has_vlan = (ol_flags & PKT_TX_VLAN_PKT);
 488         encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
 489                         (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
 490                         ((ol_flags & PKT_TX_OUTER_IPV4) ||
 491                         (ol_flags & PKT_TX_OUTER_IPV6));
 492
 493         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 494         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
 495
 496         desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr);
 497         desc->len = txm->data_len;
 498         desc->vlan_tci = txm->vlan_tci;
 499
 500         txm_seg = txm->next;
 501         while (txm_seg != NULL) {
 502                 elem->len = txm_seg->data_len;
 503                 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg));
 504                 stats->frags++;
 505                 elem++;
 506                 txm_seg = txm_seg->next;
 507         }
 508
 509         ionic_q_post(q, not_xmit_more, NULL, txm);
 510
 511         return 0;
 512 }
 513
 514 uint16_t
 515 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 516                 uint16_t nb_pkts)
 517 {
 518         struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue;
 519         struct ionic_queue *q = &txq->q;
 520         struct ionic_cq *cq = &txq->cq;
 521         struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
 522         uint32_t next_q_head_idx;
 523         uint32_t bytes_tx = 0;
 524         uint16_t nb_tx = 0;
 525         int err;
 526         bool last;
 527
 528         /* Cleaning old buffers */
 529         ionic_tx_flush(cq);
 530
 531         if (unlikely(ionic_q_space_avail(q) < nb_pkts)) {
 532                 stats->stop += nb_pkts;
 533                 return 0;
 534         }
 535
 536         while (nb_tx < nb_pkts) {
 537                 last = (nb_tx == (nb_pkts - 1));
 538
 539                 next_q_head_idx = (q->head_idx + 1) & (q->num_descs - 1);
 540                 if ((next_q_head_idx & 0x3) == 0) {
 541                         struct ionic_txq_desc *desc_base = q->base;
 542                         rte_prefetch0(&desc_base[next_q_head_idx]);
 543                         rte_prefetch0(&q->info[next_q_head_idx]);
 544                 }
 545
 546                 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG)
 547                         err = ionic_tx_tso(txq, tx_pkts[nb_tx], last);
 548                 else
 549                         err = ionic_tx(txq, tx_pkts[nb_tx], last);
 550                 if (err) {
 551                         stats->drop += nb_pkts - nb_tx;
 552                         if (nb_tx > 0)
 553                                 ionic_q_flush(q);
 554                         break;
 555                 }
 556
 557                 bytes_tx += tx_pkts[nb_tx]->pkt_len;
 558                 nb_tx++;
 559         }
 560
 561         stats->packets += nb_tx;
 562         stats->bytes += bytes_tx;
 563
 564         return nb_tx;
 565 }
 566
 567 /*********************************************************************
 568  *
 569  *  TX prep functions
 570  *
 571  **********************************************************************/
 572
 573 #define IONIC_TX_OFFLOAD_MASK ( \
 574         PKT_TX_IPV4 |           \
 575         PKT_TX_IPV6 |           \
 576         PKT_TX_VLAN |           \
 577         PKT_TX_IP_CKSUM |       \
 578         PKT_TX_TCP_SEG |        \
 579         PKT_TX_L4_MASK)
 580
 581 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \
 582         (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK)
 583
 584 uint16_t
 585 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
 586                 uint16_t nb_pkts)
 587 {
 588         struct rte_mbuf *txm;
 589         uint64_t offloads;
 590         int i = 0;
 591
 592         for (i = 0; i < nb_pkts; i++) {
 593                 txm = tx_pkts[i];
 594
 595                 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS) {
 596                         rte_errno = -EINVAL;
 597                         break;
 598                 }
 599
 600                 offloads = txm->ol_flags;
 601
 602                 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) {
 603                         rte_errno = -ENOTSUP;
 604                         break;
 605                 }
 606         }
 607
 608         return i;
 609 }
 610
 611 /*********************************************************************
 612  *
 613  *  RX functions
 614  *
 615  **********************************************************************/
 616
 617 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 618                 struct rte_mbuf *mbuf);
 619
 620 void
 621 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 622                 struct rte_eth_rxq_info *qinfo)
 623 {
 624         struct ionic_qcq *rxq = dev->data->rx_queues[queue_id];
 625         struct ionic_queue *q = &rxq->q;
 626
 627         qinfo->mp = rxq->mb_pool;
 628         qinfo->scattered_rx = dev->data->scattered_rx;
 629         qinfo->nb_desc = q->num_descs;
 630         qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED;
 631         qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
 632 }
 633
 634 static void __rte_cold
 635 ionic_rx_empty(struct ionic_queue *q)
 636 {
 637         struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q);
 638         struct ionic_desc_info *cur;
 639         struct rte_mbuf *mbuf;
 640
 641         while (q->tail_idx != q->head_idx) {
 642                 cur = &q->info[q->tail_idx];
 643                 mbuf = cur->cb_arg;
 644                 rte_mempool_put(rxq->mb_pool, mbuf);
 645
 646                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 647         }
 648 }
 649
 650 void __rte_cold
 651 ionic_dev_rx_queue_release(void *rx_queue)
 652 {
 653         struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue;
 654
 655         IONIC_PRINT_CALL();
 656
 657         ionic_rx_empty(&rxq->q);
 658
 659         ionic_lif_rxq_deinit(rxq);
 660
 661         ionic_qcq_free(rxq);
 662 }
 663
 664 int __rte_cold
 665 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 666                 uint16_t rx_queue_id,
 667                 uint16_t nb_desc,
 668                 uint32_t socket_id,
 669                 const struct rte_eth_rxconf *rx_conf,
 670                 struct rte_mempool *mp)
 671 {
 672         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 673         struct ionic_qcq *rxq;
 674         uint64_t offloads;
 675         int err;
 676
 677         if (rx_queue_id >= lif->nrxqcqs) {
 678                 IONIC_PRINT(ERR,
 679                         "Queue index %u not available (max %u queues)",
 680                         rx_queue_id, lif->nrxqcqs);
 681                 return -EINVAL;
 682         }
 683
 684         offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads;
 685         IONIC_PRINT(DEBUG,
 686                 "Configuring skt %u RX queue %u with %u buffers, offloads %jx",
 687                 socket_id, rx_queue_id, nb_desc, offloads);
 688
 689         if (!rx_conf->rx_drop_en)
 690                 IONIC_PRINT(WARNING, "No-drop mode is not supported");
 691
 692         /* Validate number of receive descriptors */
 693         if (!rte_is_power_of_2(nb_desc) ||
 694                         nb_desc < IONIC_MIN_RING_DESC ||
 695                         nb_desc > IONIC_MAX_RING_DESC) {
 696                 IONIC_PRINT(ERR,
 697                         "Bad descriptor count (%u) for queue %u (min: %u)",
 698                         nb_desc, rx_queue_id, IONIC_MIN_RING_DESC);
 699                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 700         }
 701
 702         /* Free memory prior to re-allocation if needed... */
 703         if (eth_dev->data->rx_queues[rx_queue_id] != NULL) {
 704                 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id];
 705                 ionic_dev_rx_queue_release(rx_queue);
 706                 eth_dev->data->rx_queues[rx_queue_id] = NULL;
 707         }
 708
 709         eth_dev->data->rx_queue_state[rx_queue_id] =
 710                 RTE_ETH_QUEUE_STATE_STOPPED;
 711
 712         err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, &rxq);
 713         if (err) {
 714                 IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id);
 715                 return -EINVAL;
 716         }
 717
 718         rxq->mb_pool = mp;
 719
 720         /*
 721          * Note: the interface does not currently support
 722          * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN
 723          * when the adapter will be able to keep the CRC and subtract
 724          * it to the length for all received packets:
 725          * if (eth_dev->data->dev_conf.rxmode.offloads &
 726          *     DEV_RX_OFFLOAD_KEEP_CRC)
 727          *   rxq->crc_len = ETHER_CRC_LEN;
 728          */
 729
 730         /* Do not start queue with rte_eth_dev_start() */
 731         if (rx_conf->rx_deferred_start)
 732                 rxq->flags |= IONIC_QCQ_F_DEFERRED;
 733
 734         eth_dev->data->rx_queues[rx_queue_id] = rxq;
 735
 736         return 0;
 737 }
 738
 739 static __rte_always_inline void
 740 ionic_rx_clean(struct ionic_queue *q,
 741                 uint32_t q_desc_index, uint32_t cq_desc_index,
 742                 void *cb_arg, void *service_cb_arg)
 743 {
 744         struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base;
 745         struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index];
 746         struct rte_mbuf *rxm = cb_arg;
 747         struct rte_mbuf *rxm_seg;
 748         struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q);
 749         uint32_t max_frame_size =
 750                 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 751         uint64_t pkt_flags = 0;
 752         uint32_t pkt_type;
 753         struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q);
 754         struct ionic_rx_service *recv_args = (struct ionic_rx_service *)
 755                 service_cb_arg;
 756         uint32_t buf_size = (uint16_t)
 757                 (rte_pktmbuf_data_room_size(rxq->mb_pool) -
 758                 RTE_PKTMBUF_HEADROOM);
 759         uint32_t left;
 760
 761         if (!recv_args) {
 762                 stats->no_cb_arg++;
 763                 /* Flush */
 764                 rte_pktmbuf_free(rxm);
 765                 /*
 766                  * Note: rte_mempool_put is faster with no segs
 767                  * rte_mempool_put(rxq->mb_pool, rxm);
 768                  */
 769                 return;
 770         }
 771
 772         if (cq_desc->status) {
 773                 stats->bad_cq_status++;
 774                 ionic_rx_recycle(q, q_desc_index, rxm);
 775                 return;
 776         }
 777
 778         if (recv_args->nb_rx >= recv_args->nb_pkts) {
 779                 stats->no_room++;
 780                 ionic_rx_recycle(q, q_desc_index, rxm);
 781                 return;
 782         }
 783
 784         if (cq_desc->len > max_frame_size ||
 785                         cq_desc->len == 0) {
 786                 stats->bad_len++;
 787                 ionic_rx_recycle(q, q_desc_index, rxm);
 788                 return;
 789         }
 790
 791         rxm->data_off = RTE_PKTMBUF_HEADROOM;
 792         rte_prefetch1((char *)rxm->buf_addr + rxm->data_off);
 793         rxm->nb_segs = 1; /* cq_desc->num_sg_elems */
 794         rxm->pkt_len = cq_desc->len;
 795         rxm->port = rxq->lif->port_id;
 796
 797         left = cq_desc->len;
 798
 799         rxm->data_len = RTE_MIN(buf_size, left);
 800         left -= rxm->data_len;
 801
 802         rxm_seg = rxm->next;
 803         while (rxm_seg && left) {
 804                 rxm_seg->data_len = RTE_MIN(buf_size, left);
 805                 left -= rxm_seg->data_len;
 806
 807                 rxm_seg = rxm_seg->next;
 808                 rxm->nb_segs++;
 809         }
 810
 811         /* RSS */
 812         pkt_flags |= PKT_RX_RSS_HASH;
 813         rxm->hash.rss = cq_desc->rss_hash;
 814
 815         /* Vlan Strip */
 816         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
 817                 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 818                 rxm->vlan_tci = cq_desc->vlan_tci;
 819         }
 820
 821         /* Checksum */
 822         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
 823                 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK)
 824                         pkt_flags |= PKT_RX_IP_CKSUM_GOOD;
 825                 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)
 826                         pkt_flags |= PKT_RX_IP_CKSUM_BAD;
 827
 828                 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) ||
 829                         (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK))
 830                         pkt_flags |= PKT_RX_L4_CKSUM_GOOD;
 831                 else if ((cq_desc->csum_flags &
 832                                 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
 833                                 (cq_desc->csum_flags &
 834                                 IONIC_RXQ_COMP_CSUM_F_UDP_BAD))
 835                         pkt_flags |= PKT_RX_L4_CKSUM_BAD;
 836         }
 837
 838         rxm->ol_flags = pkt_flags;
 839
 840         /* Packet Type */
 841         switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
 842         case IONIC_PKT_TYPE_IPV4:
 843                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 844                 break;
 845         case IONIC_PKT_TYPE_IPV6:
 846                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 847                 break;
 848         case IONIC_PKT_TYPE_IPV4_TCP:
 849                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 850                         RTE_PTYPE_L4_TCP;
 851                 break;
 852         case IONIC_PKT_TYPE_IPV6_TCP:
 853                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 854                         RTE_PTYPE_L4_TCP;
 855                 break;
 856         case IONIC_PKT_TYPE_IPV4_UDP:
 857                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 858                         RTE_PTYPE_L4_UDP;
 859                 break;
 860         case IONIC_PKT_TYPE_IPV6_UDP:
 861                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 862                         RTE_PTYPE_L4_UDP;
 863                 break;
 864         default:
 865                 {
 866                         struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
 867                                 struct rte_ether_hdr *);
 868                         uint16_t ether_type = eth_h->ether_type;
 869                         if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
 870                                 pkt_type = RTE_PTYPE_L2_ETHER_ARP;
 871                         else
 872                                 pkt_type = RTE_PTYPE_UNKNOWN;
 873                         break;
 874                 }
 875         }
 876
 877         rxm->packet_type = pkt_type;
 878
 879         recv_args->rx_pkts[recv_args->nb_rx] = rxm;
 880         recv_args->nb_rx++;
 881
 882         stats->packets++;
 883         stats->bytes += rxm->pkt_len;
 884 }
 885
 886 static void
 887 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 888                  struct rte_mbuf *mbuf)
 889 {
 890         struct ionic_rxq_desc *desc_base = q->base;
 891         struct ionic_rxq_desc *old = &desc_base[q_desc_index];
 892         struct ionic_rxq_desc *new = &desc_base[q->head_idx];
 893
 894         new->addr = old->addr;
 895         new->len = old->len;
 896
 897         ionic_q_post(q, true, ionic_rx_clean, mbuf);
 898 }
 899
 900 static __rte_always_inline int
 901 ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len)
 902 {
 903         struct ionic_queue *q = &rxq->q;
 904         struct ionic_rxq_desc *desc_base = q->base;
 905         struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base;
 906         struct ionic_rxq_desc *desc;
 907         struct ionic_rxq_sg_desc *sg_desc;
 908         struct ionic_rxq_sg_elem *elem;
 909         rte_iova_t dma_addr;
 910         uint32_t i, j, nsegs, buf_size, size;
 911         bool ring_doorbell;
 912
 913         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 914                 RTE_PKTMBUF_HEADROOM);
 915
 916         /* Initialize software ring entries */
 917         for (i = ionic_q_space_avail(q); i; i--) {
 918                 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool);
 919                 struct rte_mbuf *prev_rxm_seg;
 920
 921                 if (rxm == NULL) {
 922                         IONIC_PRINT(ERR, "RX mbuf alloc failed");
 923                         return -ENOMEM;
 924                 }
 925
 926                 nsegs = (len + buf_size - 1) / buf_size;
 927
 928                 desc = &desc_base[q->head_idx];
 929                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm));
 930                 desc->addr = dma_addr;
 931                 desc->len = buf_size;
 932                 size = buf_size;
 933                 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
 934                         IONIC_RXQ_DESC_OPCODE_SIMPLE;
 935                 rxm->next = NULL;
 936
 937                 prev_rxm_seg = rxm;
 938                 sg_desc = &sg_desc_base[q->head_idx];
 939                 elem = sg_desc->elems;
 940                 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) {
 941                         struct rte_mbuf *rxm_seg;
 942                         rte_iova_t data_iova;
 943
 944                         rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool);
 945                         if (rxm_seg == NULL) {
 946                                 IONIC_PRINT(ERR, "RX mbuf alloc failed");
 947                                 return -ENOMEM;
 948                         }
 949
 950                         data_iova = rte_mbuf_data_iova(rxm_seg);
 951                         dma_addr = rte_cpu_to_le_64(data_iova);
 952                         elem->addr = dma_addr;
 953                         elem->len = buf_size;
 954                         size += buf_size;
 955                         elem++;
 956                         rxm_seg->next = NULL;
 957                         prev_rxm_seg->next = rxm_seg;
 958                         prev_rxm_seg = rxm_seg;
 959                 }
 960
 961                 if (size < len)
 962                         IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)",
 963                                 size, len);
 964
 965                 ring_doorbell = ((q->head_idx + 1) &
 966                         IONIC_RX_RING_DOORBELL_STRIDE) == 0;
 967
 968                 ionic_q_post(q, ring_doorbell, ionic_rx_clean, rxm);
 969         }
 970
 971         return 0;
 972 }
 973
 974 /*
 975  * Start Receive Units for specified queue.
 976  */
 977 int __rte_cold
 978 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 979 {
 980         uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 981         uint8_t *rx_queue_state = eth_dev->data->rx_queue_state;
 982         struct ionic_qcq *rxq;
 983         int err;
 984
 985         if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
 986                 IONIC_PRINT(DEBUG, "RX queue %u already started",
 987                         rx_queue_id);
 988                 return 0;
 989         }
 990
 991         rxq = eth_dev->data->rx_queues[rx_queue_id];
 992
 993         IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)",
 994                 rx_queue_id, rxq->q.num_descs, frame_size);
 995
 996         if (!(rxq->flags & IONIC_QCQ_F_INITED)) {
 997                 err = ionic_lif_rxq_init(rxq);
 998                 if (err)
 999                         return err;
1000         } else {
1001                 ionic_qcq_enable(rxq);
1002         }
1003
1004         /* Allocate buffers for descriptor rings */
1005         if (ionic_rx_fill(rxq, frame_size) != 0) {
1006                 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d",
1007                         rx_queue_id);
1008                 return -1;
1009         }
1010
1011         rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1012
1013         return 0;
1014 }
1015
1016 static __rte_always_inline void
1017 ionic_rxq_service(struct ionic_cq *cq, uint32_t work_to_do,
1018                 void *service_cb_arg)
1019 {
1020         struct ionic_queue *q = cq->bound_q;
1021         struct ionic_desc_info *q_desc_info;
1022         struct ionic_rxq_comp *cq_desc_base = cq->base;
1023         struct ionic_rxq_comp *cq_desc;
1024         bool more;
1025         uint32_t curr_q_tail_idx, curr_cq_tail_idx;
1026         uint32_t work_done = 0;
1027
1028         if (work_to_do == 0)
1029                 return;
1030
1031         cq_desc = &cq_desc_base[cq->tail_idx];
1032         while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
1033                 curr_cq_tail_idx = cq->tail_idx;
1034                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
1035
1036                 if (cq->tail_idx == 0)
1037                         cq->done_color = !cq->done_color;
1038
1039                 /* Prefetch the next 4 descriptors */
1040                 if ((cq->tail_idx & 0x3) == 0)
1041                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
1042
1043                 do {
1044                         more = (q->tail_idx != cq_desc->comp_index);
1045
1046                         q_desc_info = &q->info[q->tail_idx];
1047
1048                         curr_q_tail_idx = q->tail_idx;
1049                         q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
1050
1051                         /* Prefetch the next 4 descriptors */
1052                         if ((q->tail_idx & 0x3) == 0)
1053                                 /* q desc info */
1054                                 rte_prefetch0(&q->info[q->tail_idx]);
1055
1056                         ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx,
1057                                 q_desc_info->cb_arg, service_cb_arg);
1058
1059                 } while (more);
1060
1061                 if (++work_done == work_to_do)
1062                         break;
1063
1064                 cq_desc = &cq_desc_base[cq->tail_idx];
1065         }
1066 }
1067
1068 /*
1069  * Stop Receive Units for specified queue.
1070  */
1071 int __rte_cold
1072 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
1073 {
1074         struct ionic_qcq *rxq;
1075
1076         IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id);
1077
1078         rxq = eth_dev->data->rx_queues[rx_queue_id];
1079
1080         eth_dev->data->rx_queue_state[rx_queue_id] =
1081                 RTE_ETH_QUEUE_STATE_STOPPED;
1082
1083         ionic_qcq_disable(rxq);
1084
1085         /* Flush */
1086         ionic_rxq_service(&rxq->cq, -1, NULL);
1087
1088         return 0;
1089 }
1090
1091 uint16_t
1092 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1093                 uint16_t nb_pkts)
1094 {
1095         struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue;
1096         uint32_t frame_size =
1097                 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
1098         struct ionic_cq *cq = &rxq->cq;
1099         struct ionic_rx_service service_cb_arg;
1100
1101         service_cb_arg.rx_pkts = rx_pkts;
1102         service_cb_arg.nb_pkts = nb_pkts;
1103         service_cb_arg.nb_rx = 0;
1104
1105         ionic_rxq_service(cq, nb_pkts, &service_cb_arg);
1106
1107         ionic_rx_fill(rxq, frame_size);
1108
1109         return service_cb_arg.nb_rx;
1110 }